diff --git a/.dockerignore b/.dockerignore
index 083905c7439..eb71138c679 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -55,6 +55,8 @@
 !rust/arrow-flight/Cargo.toml
 !rust/parquet/Cargo.toml
 !rust/parquet/build.rs
+!rust/parquet_derive/Cargo.toml
+!rust/parquet_derive_test/Cargo.toml
 !rust/datafusion/Cargo.toml
 !rust/datafusion/benches
 !rust/integration-testing/Cargo.toml
diff --git a/.env b/.env
index 62f5ae52b71..5c6aaa5182a 100644
--- a/.env
+++ b/.env
@@ -24,13 +24,13 @@ ARCH=amd64
 CUDA=9.1
 DEBIAN=10
 UBUNTU=18.04
-FEDORA=32
+FEDORA=33
 PYTHON=3.6
-LLVM=10
+LLVM=11
 CLANG_TOOLS=8
 RUST=nightly-2020-04-22
 GO=1.12
-NODE=11
+NODE=14
 MAVEN=3.5.4
 JDK=8
 PANDAS=latest
diff --git a/.github/workflows/archery.yml b/.github/workflows/archery.yml
index 4eda469f49f..b230bce82e1 100644
--- a/.github/workflows/archery.yml
+++ b/.github/workflows/archery.yml
@@ -42,9 +42,10 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Fetch Submodules and Tags
+      - name: Git Fixup
+        if: ${{ github.event_name == 'pull_request' }}
         shell: bash
-        run: ci/scripts/util_checkout.sh
+        run: git branch master origin/master
       - name: Free Up Disk Space
         run: ci/scripts/util_cleanup.sh
       - name: Setup Python
diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml
index 340d6bc4719..b55da96beb2 100644
--- a/.github/workflows/comment_bot.yml
+++ b/.github/workflows/comment_bot.yml
@@ -69,7 +69,7 @@ jobs:
           git remote add upstream https://github.com/apache/arrow
           git fetch upstream
           changed() {
-            git diff --name-only HEAD^..upstream/master | grep -e "$1" >/dev/null 2>&1
+            git diff --name-only HEAD..upstream/master | grep -e "$1" >/dev/null 2>&1
           }
           if changed '^r/.*\.R$'; then
             echo "::set-env name=R_DOCS::true"
@@ -84,10 +84,11 @@ jobs:
             echo "::set-env name=CLANG_FORMAT_R::true"
           fi
       - name: Run cmake_format
-        if: false
-        # TODO: make this work https://issues.apache.org/jira/browse/ARROW-8489
-        # if: env.CMAKE_FORMAT == 'true' || endsWith(github.event.comment.body, 'everything')
+        if: env.CMAKE_FORMAT == 'true' || endsWith(github.event.comment.body, 'everything')
         run: |
+          set -ex
+          export PATH=/home/runner/.local/bin:$PATH
+          python3 -m pip install --upgrade pip setuptools wheel
           python3 -m pip install -r dev/archery/requirements-lint.txt
           python3 run-cmake-format.py
       - name: Run clang-format on cpp
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index 07cd4a91914..4e3e27016c1 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -102,9 +102,11 @@ jobs:
     #         hosted machines
     name: ${{ matrix.title }}
     runs-on: ${{ matrix.runner }}
-    if: github.event_name == 'push'
+    # TODO(kszucs): re-enable once the self-hosted workers are properly
+    # registered to github
+    if: false && github.event_name == 'push'
     defaults:
-      # to use certain environment variables are set by .bashrc an interactive
+      # To use certain environment variables set by .bashrc, an interactive
       # bash shell must be used
       run:
         shell: bash -i {0}
@@ -119,11 +121,12 @@ jobs:
             debian: 10
             title: ARM32v7 Debian 10 C++
             image: |
-              -e CPP_MAKE_PARALLELISM=4 \
+              -e CPP_MAKE_PARALLELISM=2 \
               -e CXXFLAGS=-Wno-psabi \
               -e ARROW_PARQUET=OFF \
               -e ARROW_FLIGHT=OFF \
               -e ARROW_GANDIVA=OFF \
+              -e ARROW_ORC=OFF \
               -e CMAKE_ARGS=-DARROW_CPU_FLAG=armv7 \
               debian-cpp
             arch: 'arm32v7'
@@ -186,23 +189,24 @@ jobs:
     strategy:
       fail-fast: false
     env:
+      ARROW_BUILD_TESTS: ON
+      ARROW_DATASET: ON
+      ARROW_FLIGHT: ON
+      ARROW_GANDIVA: ON
+      ARROW_HDFS: ON
       ARROW_HOME: /usr/local
       ARROW_JEMALLOC: ON
       # TODO(kszucs): link error in the tests
-      ARROW_DATASET: ON
       ARROW_ORC: OFF
-      ARROW_FLIGHT: ON
-      ARROW_HDFS: ON
-      ARROW_PLASMA: ON
-      ARROW_GANDIVA: ON
       ARROW_PARQUET: ON
-      ARROW_WITH_ZLIB: ON
-      ARROW_WITH_LZ4: ON
+      ARROW_PLASMA: ON
+      ARROW_S3: ON
+      ARROW_WITH_BROTLI: ON
       ARROW_WITH_BZ2: ON
-      ARROW_WITH_ZSTD: ON
+      ARROW_WITH_LZ4: ON
       ARROW_WITH_SNAPPY: ON
-      ARROW_WITH_BROTLI: ON
-      ARROW_BUILD_TESTS: ON
+      ARROW_WITH_ZLIB: ON
+      ARROW_WITH_ZSTD: ON
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v2
@@ -319,6 +323,7 @@ jobs:
       ARROW_JEMALLOC: OFF
       ARROW_PARQUET: ON
       ARROW_PYTHON: ON
+      ARROW_S3: ON
       ARROW_USE_GLOG: OFF
       ARROW_VERBOSE_THIRDPARTY_BUILD: OFF
       ARROW_WITH_BROTLI: ON
@@ -333,7 +338,6 @@ jobs:
       CMAKE_ARGS: >-
         -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }}
         -DBoost_NO_BOOST_CMAKE=ON
-      CMAKE_GENERATOR: MSYS Makefiles
       CMAKE_UNITY_BUILD: ON
     steps:
       - name: Disable Crash Dialogs
@@ -370,6 +374,14 @@ jobs:
         run: |
           export CMAKE_BUILD_PARALLEL_LEVEL=$NUMBER_OF_PROCESSORS
           ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
+      - name: Download MinIO
+        shell: msys2 {0}
+        run: |
+          mkdir -p /usr/local/bin
+          wget \
+            --output-document /usr/local/bin/minio.exe \
+            https://dl.min.io/server/minio/release/windows-amd64/minio.exe
+          chmod +x /usr/local/bin/minio.exe
       - name: Test
         shell: msys2 {0}
         run: |
diff --git a/.github/workflows/cpp_cron.yml b/.github/workflows/cpp_cron.yml
index 95c5f4b7cbb..5f404b22fe5 100644
--- a/.github/workflows/cpp_cron.yml
+++ b/.github/workflows/cpp_cron.yml
@@ -46,7 +46,7 @@ jobs:
       matrix:
         name:
           - amd64-debian-10-cpp
-          - amd64-fedora-32-cpp
+          - amd64-fedora-33-cpp
           - amd64-ubuntu-16.04-cpp
           - amd64-ubuntu-18.04-cpp
           - amd64-ubuntu-18.04-cpp-cmake32
@@ -55,10 +55,10 @@ jobs:
             image: debian-cpp
             title: AMD64 Debian 10 C++
             debian: 10
-          - name: amd64-fedora-32-cpp
+          - name: amd64-fedora-33-cpp
             image: fedora-cpp
-            title: AMD64 Fedora 32 C++
-            fedora: 32
+            title: AMD64 Fedora 33 C++
+            fedora: 33
           - name: amd64-ubuntu-16.04-cpp
             image: ubuntu-cpp
             title: AMD64 Ubuntu 16.04 C++
@@ -75,7 +75,7 @@ jobs:
       # the defaults here should correspond to the values in .env
       ARCH: 'amd64'
       DEBIAN: ${{ matrix.debian || 10 }}
-      FEDORA: ${{ matrix.fedora || 32 }}
+      FEDORA: ${{ matrix.fedora || 33 }}
       UBUNTU: ${{ matrix.ubuntu || 18.04 }}
     steps:
       - name: Checkout Arrow
diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
index 59e82021722..c7f5821394d 100644
--- a/.github/workflows/dev.yml
+++ b/.github/workflows/dev.yml
@@ -59,43 +59,6 @@ jobs:
         continue-on-error: true
         run: archery docker push ubuntu-lint
 
-  docs:
-    name: Sphinx and API documentations
-    runs-on: ubuntu-latest
-    if: github.event_name == 'push'
-    steps:
-      - name: Checkout Arrow
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-      - name: Fetch Submodules and Tags
-        shell: bash
-        run: ci/scripts/util_checkout.sh
-      - name: Free Up Disk Space
-        shell: bash
-        run: ci/scripts/util_cleanup.sh
-      - name: Cache Docker Volumes
-        uses: actions/cache@v1
-        with:
-          path: .docker
-          key: ubuntu-18.04-${{ hashFiles('cpp/**') }}
-          restore-keys: ubuntu-18.04-
-      - name: Setup Python
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.8
-      - name: Setup Archery
-        run: pip install -e dev/archery[docker]
-      - name: Execute Docker Build
-        run: |
-          sudo sysctl -w kernel.core_pattern="core.%e.%p"
-          ulimit -c unlimited
-          archery docker run ubuntu-docs
-      - name: Docker Push
-        if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
-        continue-on-error: true
-        run: archery docker push ubuntu-docs
-
   release:
     name: Source Release and Merge Script
     runs-on: ubuntu-latest
diff --git a/.github/workflows/dev_labeler.yml b/.github/workflows/dev_labeler.yml
new file mode 100644
index 00000000000..f5c3a348567
--- /dev/null
+++ b/.github/workflows/dev_labeler.yml
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: PR labeler
+on:
+  pull_request_target:
+    types: [opened, reopened]
+    paths:
+      - 'rust/**'
+
+jobs:
+  assign-rust-labels:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Assign Github labels
+      uses: actions/labeler@2.2.0
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        configuration-path: .github/workflows/dev_labeler/labeler.yml
+        sync-labels: true
diff --git a/.github/workflows/dev_labeler/labeler.yml b/.github/workflows/dev_labeler/labeler.yml
new file mode 100644
index 00000000000..1753caa733f
--- /dev/null
+++ b/.github/workflows/dev_labeler/labeler.yml
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+lang-rust:
+  - rust/**/*
+
+datafusion:
+  - rust/datafusion/**/*
diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
index a6796225e40..385d0210120 100644
--- a/.github/workflows/java.yml
+++ b/.github/workflows/java.yml
@@ -83,7 +83,7 @@ jobs:
       - name: Docker Push
         if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
         continue-on-error: true
-        run: archery docker push debian-go
+        run: archery docker push debian-java
 
   macos:
     name: AMD64 MacOS 10.15 Java JDK ${{ matrix.jdk }}
diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml
index b12c2290c61..e84a812a078 100644
--- a/.github/workflows/js.yml
+++ b/.github/workflows/js.yml
@@ -40,7 +40,7 @@ env:
 jobs:
 
   docker:
-    name: AMD64 Debian 10 NodeJS 11
+    name: AMD64 Debian 10 NodeJS 14
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     steps:
@@ -75,7 +75,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        node: [11]
+        node: [14]
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v2
@@ -103,7 +103,7 @@ jobs:
   #   strategy:
   #     fail-fast: false
   #     matrix:
-  #       node: [11]
+  #       node: [14]
   #   steps:
   #     - name: Checkout Arrow
   #       uses: actions/checkout@v1
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index 84c5bb601ee..6b84ed2cdb8 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -50,7 +50,7 @@ jobs:
           - ubuntu-16.04-python-3
           - conda-python-3.8-nopandas
           - conda-python-3.6-pandas-0.23
-          - conda-python-3.6-pandas-latest
+          - conda-python-3.7-pandas-latest
           - centos-python-3.6-manylinux1
         include:
           - name: ubuntu-16.04-python-3
@@ -70,11 +70,11 @@ jobs:
             title: AMD64 Conda Python 3.6 Pandas 0.23
             python: 3.6
             pandas: 0.23
-          - name: conda-python-3.6-pandas-latest
-            cache: conda-python-3.6
+          - name: conda-python-3.7-pandas-latest
+            cache: conda-python-3.7
             image: conda-python-pandas
-            title: AMD64 Conda Python 3.6 Pandas latest
-            python: 3.6
+            title: AMD64 Conda Python 3.7 Pandas latest
+            python: 3.7
             pandas: latest
           - name: centos-python-3.6-manylinux1
             cache: manylinux1
@@ -116,7 +116,7 @@ jobs:
         run: archery docker push ${{ matrix.image }}
 
   macos:
-    name: AMD64 MacOS 10.15 Python 3.7
+    name: AMD64 MacOS 10.15 Python 3
     runs-on: macos-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     env:
@@ -150,8 +150,9 @@ jobs:
           brew update --preinstall
           brew bundle --file=cpp/Brewfile
           brew install coreutils python
-          pip3 install -r python/requirements-build.txt \
-                       -r python/requirements-test.txt
+          python3 -mpip install \
+            -r python/requirements-build.txt \
+            -r python/requirements-test.txt
       - name: Build
         shell: bash
         run: |
diff --git a/.github/workflows/python_cron.yml b/.github/workflows/python_cron.yml
index 133a73b59bf..256856bd197 100644
--- a/.github/workflows/python_cron.yml
+++ b/.github/workflows/python_cron.yml
@@ -45,7 +45,7 @@ jobs:
       matrix:
         name:
           - debian-10-python-3
-          - fedora-32-python-3
+          - fedora-33-python-3
           - ubuntu-18.04-python-3
           - conda-python-3.7-dask-latest
           - conda-python-3.7-turbodbc-latest
@@ -59,11 +59,11 @@ jobs:
             image: debian-python
             title: AMD64 Debian 10 Python 3
             debian: 10
-          - name: fedora-32-python-3
-            cache: fedora-32-python-3
+          - name: fedora-33-python-3
+            cache: fedora-33-python-3
             image: fedora-python
-            title: AMD64 Fedora 32 Python 3
-            fedora: 32
+            title: AMD64 Fedora 33 Python 3
+            fedora: 33
           - name: ubuntu-18.04-python-3
             cache: ubuntu-18.04-python-3
             image: ubuntu-python
@@ -102,7 +102,7 @@ jobs:
     env:
       # the defaults here should correspond to the values in .env
       DEBIAN: ${{ matrix.debian || 10 }}
-      FEDORA: ${{ matrix.fedora || 32 }}
+      FEDORA: ${{ matrix.fedora || 33 }}
       UBUNTU: ${{ matrix.ubuntu || 18.04 }}
       PYTHON: ${{ matrix.python || 3.7 }}
       HDFS: ${{ matrix.hdfs || '2.9.2' }}
diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml
index 29ffd444370..37aee196883 100644
--- a/.github/workflows/r.yml
+++ b/.github/workflows/r.yml
@@ -53,7 +53,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        r: ["3.6", "4.0"]
+        r: ["3.6"]
         ubuntu: [18.04]
     env:
       R: ${{ matrix.r }}
@@ -92,21 +92,20 @@ jobs:
         continue-on-error: true
         run: archery docker push ubuntu-r
 
-  rstudio:
-    name: "rstudio/r-base:${{ matrix.r_version }}-${{ matrix.r_image }}"
+  bundled:
+    name: "${{ matrix.config.org }}/${{ matrix.config.image }}:${{ matrix.config.tag }}"
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     strategy:
       fail-fast: false
       matrix:
-        # See https://hub.docker.com/r/rstudio/r-base
-        r_version: ["3.6", "4.0"]
-        r_image:
-          - centos7
+        config:
+          - {org: 'rstudio', image: 'r-base', tag: '4.0-centos7'}
+          - {org: 'rhub', image: 'debian-gcc-devel', tag: 'latest'}
     env:
-      R_ORG: rstudio
-      R_IMAGE: r-base
-      R_TAG: ${{ matrix.r_version }}-${{ matrix.r_image }}
+      R_ORG: ${{ matrix.config.org }}
+      R_IMAGE: ${{ matrix.config.image }}
+      R_TAG: ${{ matrix.config.tag }}
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v2
@@ -120,8 +119,8 @@ jobs:
         uses: actions/cache@v1
         with:
           path: .docker
-          key: ${{ matrix.r_image }}-r-${{ hashFiles('cpp/**') }}
-          restore-keys: ${{ matrix.r_image }}-r-
+          key: ${{ matrix.config.image }}-r-${{ hashFiles('cpp/**') }}
+          restore-keys: ${{ matrix.config.image }}-r-
       - name: Setup Python
         uses: actions/setup-python@v1
         with:
diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml
index 783ca91cb82..0558ac8a2cf 100644
--- a/.github/workflows/ruby.yml
+++ b/.github/workflows/ruby.yml
@@ -174,6 +174,7 @@ jobs:
       ARROW_JEMALLOC: OFF
       ARROW_PARQUET: ON
       ARROW_PYTHON: OFF
+      ARROW_S3: ON
       ARROW_USE_GLOG: OFF
       ARROW_WITH_BROTLI: ON
       ARROW_WITH_BZ2: ON
@@ -187,7 +188,6 @@ jobs:
       CMAKE_ARGS: >-
         -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }}
         -DBoost_NO_BOOST_CMAKE=ON
-      CMAKE_GENERATOR: MSYS Makefiles
       CMAKE_UNITY_BUILD: ON
     steps:
       - name: Disable Crash Dialogs
diff --git a/.github/workflows/rust_cron.yml b/.github/workflows/rust_cron.yml
new file mode 100644
index 00000000000..378f2dd1081
--- /dev/null
+++ b/.github/workflows/rust_cron.yml
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Rust Cron
+
+on:
+  push:
+    paths:
+      - '.github/workflows/rust_cron.yml'
+  pull_request:
+    paths:
+      - '.github/workflows/rust_cron.yml'
+  schedule:
+    - cron: 0 */12 * * *
+
+jobs:
+  coverage:
+    name: AMD64 Debian 10 Rust ${{ matrix.rust }} Coverage
+    runs-on: ubuntu-latest
+    if: ${{ !contains(github.event.pull_request.title, 'WIP') && github.repository == 'apache/arrow' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        rust: [nightly-2020-04-22]
+    env:
+      RUST: ${{ matrix.rust }}
+    steps:
+      - name: Checkout Arrow
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Fetch Submodules and Tags
+        run: ci/scripts/util_checkout.sh
+      - name: Run coverage
+        shell: bash
+        run: |
+          echo ${RUST} > rust/rust-toolchain &&
+          ci/scripts/rust_coverage.sh `pwd` `pwd`/build $RUST
+      - name: Report coverage
+        continue-on-error: true
+        shell: bash
+        run: bash <(curl -s https://codecov.io/bash)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index d72b126ef34..e70eaceaf41 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -55,7 +55,7 @@ repos:
     hooks:
      - id: flake8
        name: Python Format
-       files: ^(python|crossbow|integration)/
+       files: ^(python|dev|integration)/
        types:
          - file
          - python
diff --git a/.travis.yml b/.travis.yml
index fa5d84c82c7..84bbbce23db 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -40,22 +40,81 @@ jobs:
         ARCH: arm64v8
         ARROW_CI_MODULES: "CPP"
         DOCKER_IMAGE_ID: ubuntu-cpp
+        # ARROW_USE_GLOG=OFF is needed to avoid build error caused by
+        # glog and CMAKE_UNITY_BUILD=ON.
+        #
+        # Disable ARROW_S3 because it often causes "No output has
+        # been received in the last 10m0s, this potentially indicates
+        # a stalled build or something wrong with the build itself."
+        # on Travis CI.
+        DOCKER_RUN_ARGS: >-
+          "
+          -e ARROW_BUILD_STATIC=OFF
+          -e ARROW_ORC=OFF
+          -e ARROW_S3=OFF
+          -e ARROW_USE_GLOG=OFF
+          -e CMAKE_UNITY_BUILD=ON
+          "
+        # We need to use smaller build when cache doesn't exist
+        # because Travis CI has "No output has been received in the
+        # last 10m0s" limitation. If we build many modules, we reach
+        # the limitation.
+        DOCKER_RUN_ARGS_NO_CACHE: >-
+          "
+          -e ARROW_BUILD_TESTS=OFF
+          -e ARROW_GANDIVA=OFF
+          -e ARROW_PARQUET=OFF
+          "
+        # The LLVM's APT repository provides only arm64 binaries.
+        # We should use LLVM provided by Ubuntu.
+        LLVM: "10"
         UBUNTU: "20.04"
+
     - name: "C++ on s390x"
       os: linux
       arch: s390x
       env:
         ARCH: s390x
         ARROW_CI_MODULES: "CPP"
-        ARROW_FLIGHT: "ON"
-        ARROW_PARQUET: "OFF"
         DOCKER_IMAGE_ID: ubuntu-cpp
-        PARQUET_BUILD_EXAMPLES: "OFF"
-        PARQUET_BUILD_EXECUTABLES: "OFF"
-        Protobuf_SOURCE: "BUNDLED"
+        # Can't use CMAKE_UNITIFY_BUILD=ON because of compiler crash.
+        # Can't enable ARROW_S3 because compiler is killed while compiling
+        # aws-sdk-cpp.
+        DOCKER_RUN_ARGS: >-
+          "
+          -e ARROW_BUILD_STATIC=OFF
+          -e ARROW_FLIGHT=ON
+          -e ARROW_ORC=OFF
+          -e ARROW_PARQUET=OFF
+          -e ARROW_S3=OFF
+          -e PARQUET_BUILD_EXAMPLES=OFF
+          -e PARQUET_BUILD_EXECUTABLES=OFF
+          -e Protobuf_SOURCE=BUNDLED
+          -e cares_SOURCE=BUNDLED
+          -e gRPC_SOURCE=BUNDLED
+          "
+        # The LLVM's APT repository provides only arm64 binaries.
+        # We should use LLVM provided by Ubuntu.
+        LLVM: "10"
         UBUNTU: "20.04"
-        cares_SOURCE: "BUNDLED"
-        gRPC_SOURCE: "BUNDLED"
+
+    - name: "Go on s390x"
+      os: linux
+      arch: s390x
+      env:
+        ARCH: s390x
+        ARROW_CI_MODULES: "GO"
+        DOCKER_IMAGE_ID: debian-go
+
+    - name: "Java on s390x"
+      os: linux
+      arch: s390x
+      env:
+        ARCH: s390x
+        ARROW_CI_MODULES: "JAVA"
+        DOCKER_IMAGE_ID: debian-java
+        JDK: 11
+
   allow_failures:
     - arch: s390x
 
@@ -87,15 +146,13 @@ script:
   #   /home/travis/.travis/functions: line 109: ulimit: core file size: cannot modify limit: Operation not permitted
   - |
     ulimit -c unlimited || :
+  - |
+    if [ $(ls $TRAVIS_BUILD_DIR/.docker | wc -l) -eq 0 ]; then
+      DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} ${DOCKER_RUN_ARGS_NO_CACHE}"
+    fi
   - |
     archery docker run \
-      -e ARROW_FLIGHT=${ARROW_FLIGHT:-OFF} \
-      -e ARROW_PARQUET=${ARROW_PARQUET:-ON} \
-      -e PARQUET_BUILD_EXAMPLES=${PARQUET_BUILD_EXAMPLES:-ON} \
-      -e PARQUET_BUILD_EXECUTABLES=${PARQUET_BUILD_EXECUTABLES:-ON} \
-      -e Protobuf_SOURCE=${Protobuf_SOURCE:-} \
-      -e cares_SOURCE=${cares_SOURCE:-} \
-      -e gRPC_SOURCE=${gRPC_SOURCE:-} \
+      ${DOCKER_RUN_ARGS} \
       --volume ${PWD}/build:/build \
       ${DOCKER_IMAGE_ID}
 
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3e5619b9a33..686f1d4ef5e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,588 @@
 
+# Apache Arrow 2.0.0 (2020-10-13)
+
+## Bug Fixes
+
+* [ARROW-2367](https://issues.apache.org/jira/browse/ARROW-2367) - [Python] ListArray has trouble with sizes greater than kMaximumCapacity
+* [ARROW-4189](https://issues.apache.org/jira/browse/ARROW-4189) - [CI] [Rust] Fix broken cargo coverage
+* [ARROW-4917](https://issues.apache.org/jira/browse/ARROW-4917) - [C++] orc\_ep fails in cpp-alpine docker
+* [ARROW-5578](https://issues.apache.org/jira/browse/ARROW-5578) - [C++][Flight] Flight does not build out of the box on Alpine Linux
+* [ARROW-7226](https://issues.apache.org/jira/browse/ARROW-7226) - [JSON][Python] Json loader fails on example in documentation.
+* [ARROW-7384](https://issues.apache.org/jira/browse/ARROW-7384) - [Website] Fix search indexing warning reported by Google
+* [ARROW-7517](https://issues.apache.org/jira/browse/ARROW-7517) - [C++] Builder does not honour dictionary type provided during initialization
+* [ARROW-7663](https://issues.apache.org/jira/browse/ARROW-7663) - [Python] from\_pandas gives TypeError instead of ArrowTypeError in some cases
+* [ARROW-7903](https://issues.apache.org/jira/browse/ARROW-7903) - [Rust] [DataFusion] Upgrade SQLParser dependency for DataFusion
+* [ARROW-7957](https://issues.apache.org/jira/browse/ARROW-7957) - [Python] ParquetDataset cannot take HadoopFileSystem as filesystem
+* [ARROW-8265](https://issues.apache.org/jira/browse/ARROW-8265) - [Rust] [DataFusion] Table API collect() should not require context
+* [ARROW-8394](https://issues.apache.org/jira/browse/ARROW-8394) - [JS] Typescript compiler errors for arrow d.ts files, when using es2015-esm package
+* [ARROW-8735](https://issues.apache.org/jira/browse/ARROW-8735) - [Rust] [Parquet] Parquet crate fails to compile on Arm architecture
+* [ARROW-8749](https://issues.apache.org/jira/browse/ARROW-8749) - [C++] IpcFormatWriter writes dictionary batches with wrong ID
+* [ARROW-8773](https://issues.apache.org/jira/browse/ARROW-8773) - [Python] pyarrow schema.empty\_table() does not preserve nullability of fields
+* [ARROW-9028](https://issues.apache.org/jira/browse/ARROW-9028) - [R] Should be able to convert an empty table
+* [ARROW-9096](https://issues.apache.org/jira/browse/ARROW-9096) - [Python] Pandas roundtrip with object-dtype column labels with integer values: data type "integer" not understood
+* [ARROW-9177](https://issues.apache.org/jira/browse/ARROW-9177) - [C++][Parquet] Tracking issue for cross-implementation LZ4 Parquet compression compatibility
+* [ARROW-9414](https://issues.apache.org/jira/browse/ARROW-9414) - [C++] apt package includes headers for S3 interface, but no support
+* [ARROW-9462](https://issues.apache.org/jira/browse/ARROW-9462) - [Go] The Indentation after the first Record arrjson writer is missing
+* [ARROW-9463](https://issues.apache.org/jira/browse/ARROW-9463) - [Go] The writer is double closed in TestReadWrite
+* [ARROW-9490](https://issues.apache.org/jira/browse/ARROW-9490) - [Python] pyarrow array creation for specific set of numpy scalars fails
+* [ARROW-9495](https://issues.apache.org/jira/browse/ARROW-9495) - [C++] Equality assertions don't handle Inf /  -Inf properly
+* [ARROW-9520](https://issues.apache.org/jira/browse/ARROW-9520) - [Rust] [DataFusion] Can't alias an aggregate expression
+* [ARROW-9528](https://issues.apache.org/jira/browse/ARROW-9528) - [Python] Honor tzinfo information when converting from datetime to pyarrow
+* [ARROW-9532](https://issues.apache.org/jira/browse/ARROW-9532) - [Python] Building pyarrow for MacPorts on macOS
+* [ARROW-9535](https://issues.apache.org/jira/browse/ARROW-9535) - [Python] Remove symlink fixes from conda recipe
+* [ARROW-9536](https://issues.apache.org/jira/browse/ARROW-9536) - Missing parameters in PlasmaOutOfMemoryException.java
+* [ARROW-9541](https://issues.apache.org/jira/browse/ARROW-9541) - [C++] CMakeLists requires UTF8PROC\_STATIC when building static library
+* [ARROW-9544](https://issues.apache.org/jira/browse/ARROW-9544) - [R] version argument of write\_parquet not working
+* [ARROW-9546](https://issues.apache.org/jira/browse/ARROW-9546) - [Python] Clean up Pandas Metadata Conversion test
+* [ARROW-9548](https://issues.apache.org/jira/browse/ARROW-9548) - [Go] Test output files in tmp directory are not removed correctly
+* [ARROW-9549](https://issues.apache.org/jira/browse/ARROW-9549) - [Rust] Parquet no longer builds
+* [ARROW-9554](https://issues.apache.org/jira/browse/ARROW-9554) - [Java] FixedWidthInPlaceVectorSorter sometimes produces wrong result
+* [ARROW-9556](https://issues.apache.org/jira/browse/ARROW-9556) - [Python][C++] Segfaults in UnionArray with null values
+* [ARROW-9560](https://issues.apache.org/jira/browse/ARROW-9560) - [Packaging] conda recipes failing due to missing conda-forge.yml
+* [ARROW-9569](https://issues.apache.org/jira/browse/ARROW-9569) - [CI][R] Fix rtools35 builds for msys2 key change
+* [ARROW-9570](https://issues.apache.org/jira/browse/ARROW-9570) - [Doc] Clean up sphinx sidebar
+* [ARROW-9573](https://issues.apache.org/jira/browse/ARROW-9573) - [Python] Parquet doesn't load when partitioned column starts with '\_'
+* [ARROW-9574](https://issues.apache.org/jira/browse/ARROW-9574) - [R] Cleanups for CRAN 1.0.0 release
+* [ARROW-9575](https://issues.apache.org/jira/browse/ARROW-9575) - [R] gcc-UBSAN failure on CRAN
+* [ARROW-9577](https://issues.apache.org/jira/browse/ARROW-9577) - [Python][C++] posix\_madvise error on Debian in pyarrow 1.0.0
+* [ARROW-9583](https://issues.apache.org/jira/browse/ARROW-9583) - [Rust] Offset is mishandled in arithmetic and boolean compute kernels
+* [ARROW-9588](https://issues.apache.org/jira/browse/ARROW-9588) - [C++] clang/win: Copy constructor of ParquetInvalidOrCorruptedFileException not correctly triggered
+* [ARROW-9589](https://issues.apache.org/jira/browse/ARROW-9589) - [C++/R] arrow\_exports.h contains structs declared as class
+* [ARROW-9592](https://issues.apache.org/jira/browse/ARROW-9592) - [CI] Update homebrew before calling brew bundle
+* [ARROW-9596](https://issues.apache.org/jira/browse/ARROW-9596) - [CI][Crossbow] Fix homebrew-cpp again, again
+* [ARROW-9597](https://issues.apache.org/jira/browse/ARROW-9597) - [C++] AddAlias in compute::FunctionRegistry should be synchronized
+* [ARROW-9598](https://issues.apache.org/jira/browse/ARROW-9598) - [C++][Parquet]  Spaced definition levels is not assigned correctly.
+* [ARROW-9599](https://issues.apache.org/jira/browse/ARROW-9599) - [CI] Appveyor toolchain build fails because CMake detects different C and C++ compilers
+* [ARROW-9600](https://issues.apache.org/jira/browse/ARROW-9600) - [Rust] When used as a crate dependency, arrow-flight is rebuilt on every invocation of cargo build
+* [ARROW-9600](https://issues.apache.org/jira/browse/ARROW-9600) - [Rust] When used as a crate dependency, arrow-flight is rebuilt on every invocation of cargo build
+* [ARROW-9602](https://issues.apache.org/jira/browse/ARROW-9602) - [R] Improve cmake detection in Linux build
+* [ARROW-9603](https://issues.apache.org/jira/browse/ARROW-9603) - [C++][Parquet] Write Arrow relies on unspecified behavior for nested types
+* [ARROW-9606](https://issues.apache.org/jira/browse/ARROW-9606) - [C++][Dataset] in expressions don't work with \>1 partition levels
+* [ARROW-9609](https://issues.apache.org/jira/browse/ARROW-9609) - [C++] CSV datasets don't materialize virtual columns
+* [ARROW-9621](https://issues.apache.org/jira/browse/ARROW-9621) - [Python] test\_move\_file() is failed with fsspec 0.8.0
+* [ARROW-9622](https://issues.apache.org/jira/browse/ARROW-9622) - [Java] ComplexCopier fails if a structvector has a child UnionVector with nulls
+* [ARROW-9628](https://issues.apache.org/jira/browse/ARROW-9628) - [Rust] Clippy PR test failing intermittently on Rust / AMD64 MacOS 
+* [ARROW-9629](https://issues.apache.org/jira/browse/ARROW-9629) - [Python] Kartothek integration tests failing due to missing freezegun module
+* [ARROW-9631](https://issues.apache.org/jira/browse/ARROW-9631) - [Rust] Arrow crate should not depend on flight
+* [ARROW-9631](https://issues.apache.org/jira/browse/ARROW-9631) - [Rust] Arrow crate should not depend on flight
+* [ARROW-9642](https://issues.apache.org/jira/browse/ARROW-9642) - [C++] Let MakeBuilder refer DictionaryType's index\_type for deciding the starting bit width of the indices
+* [ARROW-9643](https://issues.apache.org/jira/browse/ARROW-9643) - [C++] Illegal instruction on haswell cpu
+* [ARROW-9644](https://issues.apache.org/jira/browse/ARROW-9644) - [C++][Dataset] Do not check for ignore\_prefixes in the base path
+* [ARROW-9652](https://issues.apache.org/jira/browse/ARROW-9652) - [Rust][DataFusion] Panic trying to select \* from a CSV (panicked at 'index out of bounds: the len is 0 but the index is 0)
+* [ARROW-9653](https://issues.apache.org/jira/browse/ARROW-9653) - [Rust][DataFusion] Multi-column Group by: Invalid Argument Error 
+* [ARROW-9659](https://issues.apache.org/jira/browse/ARROW-9659) - [C++] RecordBatchStreamReader throws on CUDA device buffers
+* [ARROW-9660](https://issues.apache.org/jira/browse/ARROW-9660) - [C++] IPC - dictionaries in maps
+* [ARROW-9666](https://issues.apache.org/jira/browse/ARROW-9666) - [Python][wheel][Windows] library missing failure by ARROW-9412
+* [ARROW-9670](https://issues.apache.org/jira/browse/ARROW-9670) - [C++][FlightRPC] Close()ing a DoPut with an ongoing read locks up the client
+* [ARROW-9684](https://issues.apache.org/jira/browse/ARROW-9684) - [C++] Fix undefined behaviour on invalid IPC / Parquet input (OSS-Fuzz)
+* [ARROW-9692](https://issues.apache.org/jira/browse/ARROW-9692) - [Python] distutils import warning
+* [ARROW-9693](https://issues.apache.org/jira/browse/ARROW-9693) - [CI][Docs] Nightly docs build fails
+* [ARROW-9696](https://issues.apache.org/jira/browse/ARROW-9696) - [Rust] [Datafusion] nested binary expressions broken
+* [ARROW-9698](https://issues.apache.org/jira/browse/ARROW-9698) - [C++] Revert "Add -NDEBUG flag to arrow.pc"
+* [ARROW-9700](https://issues.apache.org/jira/browse/ARROW-9700) - [Python] create\_library\_symlinks doesn't work in macos
+* [ARROW-9712](https://issues.apache.org/jira/browse/ARROW-9712) - [Rust] [DataFusion] ParquetScanExec panics on error
+* [ARROW-9714](https://issues.apache.org/jira/browse/ARROW-9714) - [Rust] [DataFusion] TypeCoercionRule not implemented for Limit or Sort
+* [ARROW-9716](https://issues.apache.org/jira/browse/ARROW-9716) - [Rust] [DataFusion] MergeExec  should have concurrency limit
+* [ARROW-9726](https://issues.apache.org/jira/browse/ARROW-9726) - [Rust] [DataFusion] ParquetScanExec launches threads too early
+* [ARROW-9727](https://issues.apache.org/jira/browse/ARROW-9727) - [C++] Fix crash on invalid IPC input (OSS-Fuzz)
+* [ARROW-9729](https://issues.apache.org/jira/browse/ARROW-9729) - [Java] Error Prone causes other annotation processors to not work with Eclipse
+* [ARROW-9733](https://issues.apache.org/jira/browse/ARROW-9733) - [Rust][DataFusion] Aggregates COUNT/MIN/MAX don't work on VARCHAR columns
+* [ARROW-9734](https://issues.apache.org/jira/browse/ARROW-9734) - [Rust] [DataFusion] TableProvider.scan executing partitions prematurely
+* [ARROW-9741](https://issues.apache.org/jira/browse/ARROW-9741) - [Rust] [DataFusion] Incorrect count in TPC-H query 1 result set
+* [ARROW-9743](https://issues.apache.org/jira/browse/ARROW-9743) - [R] Sanitize paths in open\_dataset
+* [ARROW-9744](https://issues.apache.org/jira/browse/ARROW-9744) - [Python] Failed to install on aarch64
+* [ARROW-9764](https://issues.apache.org/jira/browse/ARROW-9764) - [CI][Java] Push wrong Docker image
+* [ARROW-9768](https://issues.apache.org/jira/browse/ARROW-9768) - [Python] Pyarrow allows for unsafe conversions of datetime objects to timestamp nanoseconds
+* [ARROW-9768](https://issues.apache.org/jira/browse/ARROW-9768) - [Python] Pyarrow allows for unsafe conversions of datetime objects to timestamp nanoseconds
+* [ARROW-9778](https://issues.apache.org/jira/browse/ARROW-9778) - [Rust] [DataFusion] Logical and physical schemas' nullability does not match in 8 out of 20 end-to-end tests
+* [ARROW-9783](https://issues.apache.org/jira/browse/ARROW-9783) - [Rust] [DataFusion] Logical aggregate expressions require explicit data type
+* [ARROW-9785](https://issues.apache.org/jira/browse/ARROW-9785) - [Python] pyarrow/tests/test\_fs.py::test\_s3\_options too slow
+* [ARROW-9789](https://issues.apache.org/jira/browse/ARROW-9789) - [C++] Don't install jemalloc in parallel
+* [ARROW-9790](https://issues.apache.org/jira/browse/ARROW-9790) - [Rust] [Parquet] ParquetFileArrowReader fails to decode all pages if batches fall exactly on row group boundaries
+* [ARROW-9790](https://issues.apache.org/jira/browse/ARROW-9790) - [Rust] [Parquet] ParquetFileArrowReader fails to decode all pages if batches fall exactly on row group boundaries
+* [ARROW-9793](https://issues.apache.org/jira/browse/ARROW-9793) - [Rust] [DataFusion] Tests failing in master
+* [ARROW-9797](https://issues.apache.org/jira/browse/ARROW-9797) - [Rust] AMD64 Conda Integration Tests is failing for the Master branch
+* [ARROW-9799](https://issues.apache.org/jira/browse/ARROW-9799) - [Rust] [DataFusion] Implementation of physical binary expression get\_type method is incorrect
+* [ARROW-9800](https://issues.apache.org/jira/browse/ARROW-9800) - [Rust] [Parquet] "min" and "max" written to standard out when writing columns
+* [ARROW-9809](https://issues.apache.org/jira/browse/ARROW-9809) - [Rust] [DataFusion] logical schema = physical schema is not true
+* [ARROW-9814](https://issues.apache.org/jira/browse/ARROW-9814) - [Python] Crash in test\_parquet.py::test\_read\_partitioned\_directory\_s3fs
+* [ARROW-9815](https://issues.apache.org/jira/browse/ARROW-9815) - [Rust] [DataFusion] Deadlock in creation of physical plan with two udfs
+* [ARROW-9815](https://issues.apache.org/jira/browse/ARROW-9815) - [Rust] [DataFusion] Deadlock in creation of physical plan with two udfs
+* [ARROW-9815](https://issues.apache.org/jira/browse/ARROW-9815) - [Rust] [DataFusion] Deadlock in creation of physical plan with two udfs
+* [ARROW-9816](https://issues.apache.org/jira/browse/ARROW-9816) - [C++] Escape quotes in config.h
+* [ARROW-9827](https://issues.apache.org/jira/browse/ARROW-9827) - [Python] pandas.read\_parquet fails for wide parquet files and pyarrow 1.0.X
+* [ARROW-9831](https://issues.apache.org/jira/browse/ARROW-9831) - [Rust] [DataFusion] Fix compilation error
+* [ARROW-9840](https://issues.apache.org/jira/browse/ARROW-9840) - [Python] Python fs documentation out of date with code
+* [ARROW-9846](https://issues.apache.org/jira/browse/ARROW-9846) - [Rust] Master branch broken build
+* [ARROW-9851](https://issues.apache.org/jira/browse/ARROW-9851) - [C++] Valgrind errors due to unrecognized instructions
+* [ARROW-9852](https://issues.apache.org/jira/browse/ARROW-9852) - [C++] Fix crash on invalid IPC input (OSS-Fuzz)
+* [ARROW-9852](https://issues.apache.org/jira/browse/ARROW-9852) - [C++] Fix crash on invalid IPC input (OSS-Fuzz)
+* [ARROW-9855](https://issues.apache.org/jira/browse/ARROW-9855) - [R] Fix bad merge/Rcpp conflict
+* [ARROW-9859](https://issues.apache.org/jira/browse/ARROW-9859) - [C++] S3 FileSystemFromUri with special char in secret key fails
+* [ARROW-9864](https://issues.apache.org/jira/browse/ARROW-9864) - [Python] pathlib.Path not supported in write\_to\_dataset with partition columns
+* [ARROW-9874](https://issues.apache.org/jira/browse/ARROW-9874) - [C++] NewStreamWriter / NewFileWriter don't own output stream
+* [ARROW-9876](https://issues.apache.org/jira/browse/ARROW-9876) - [CI][C++] Travis ARM jobs timeout
+* [ARROW-9877](https://issues.apache.org/jira/browse/ARROW-9877) - [C++][CI] homebrew-cpp fails due to avx512
+* [ARROW-9879](https://issues.apache.org/jira/browse/ARROW-9879) - [Python] ChunkedArray.\_\_getitem\_\_ doesn't work with numpy scalars
+* [ARROW-9882](https://issues.apache.org/jira/browse/ARROW-9882) - [C++/Python] Update conda-forge-pinning to 3 for OSX conda packages
+* [ARROW-9883](https://issues.apache.org/jira/browse/ARROW-9883) - [R] Fix linuxlibs.R install script for R < 3.6
+* [ARROW-9888](https://issues.apache.org/jira/browse/ARROW-9888) - [Rust] [DataFusion] ExecutionContext can not be shared between threads
+* [ARROW-9889](https://issues.apache.org/jira/browse/ARROW-9889) - [Rust][DataFusion] Datafusion CLI: CREATE EXTERNAL TABLE errors with "Unsupported logical plan variant"
+* [ARROW-9897](https://issues.apache.org/jira/browse/ARROW-9897) - [C++][Gandiva] Add to\_date() function from pattern
+* [ARROW-9906](https://issues.apache.org/jira/browse/ARROW-9906) - [Python] Crash in test\_parquet.py::test\_parquet\_writer\_filesystem\_s3\_uri (closing NativeFile from S3FileSystem)
+* [ARROW-9913](https://issues.apache.org/jira/browse/ARROW-9913) - [C++] Outputs of Decimal128::FromString depend on presence of one another
+* [ARROW-9920](https://issues.apache.org/jira/browse/ARROW-9920) - [Python] pyarrow.concat\_arrays segfaults when passing it a chunked array
+* [ARROW-9922](https://issues.apache.org/jira/browse/ARROW-9922) - [Rust] Add \`try\_from(Vec<Option<(&str, ArrayRef)\>\>)\` to StructArray
+* [ARROW-9924](https://issues.apache.org/jira/browse/ARROW-9924) - [Python] Performance regression reading individual Parquet files using Dataset interface
+* [ARROW-9931](https://issues.apache.org/jira/browse/ARROW-9931) - [C++] Fix undefined behaviour on invalid IPC (OSS-Fuzz)
+* [ARROW-9932](https://issues.apache.org/jira/browse/ARROW-9932) - [R] Arrow 1.0.1 R package fails to install on R3.4 over linux
+* [ARROW-9936](https://issues.apache.org/jira/browse/ARROW-9936) - [Python] Fix / test relative file paths in pyarrow.parquet
+* [ARROW-9937](https://issues.apache.org/jira/browse/ARROW-9937) - [Rust] [DataFusion] Average is not correct
+* [ARROW-9943](https://issues.apache.org/jira/browse/ARROW-9943) - [C++] Arrow metadata not applied recursively when reading Parquet file
+* [ARROW-9946](https://issues.apache.org/jira/browse/ARROW-9946) - [R] ParquetFileWriter segfaults when \`sink\` is a string
+* [ARROW-9953](https://issues.apache.org/jira/browse/ARROW-9953) - [R] Declare minimum version for bit64
+* [ARROW-9962](https://issues.apache.org/jira/browse/ARROW-9962) - [Python] Conversion to pandas with index column using fixed timezone fails
+* [ARROW-9968](https://issues.apache.org/jira/browse/ARROW-9968) - [C++] UBSAN link failure with \_\_int8\_t
+* [ARROW-9969](https://issues.apache.org/jira/browse/ARROW-9969) - [C++] RecordBatchBuilder yields invalid result with dictionary fields
+* [ARROW-9970](https://issues.apache.org/jira/browse/ARROW-9970) - [Go] checkptr failures in sum methods
+* [ARROW-9972](https://issues.apache.org/jira/browse/ARROW-9972) - [CI] Work around grpc-re2 clash on Homebrew
+* [ARROW-9973](https://issues.apache.org/jira/browse/ARROW-9973) - [Java] JDBC DateConsumer does not allow dates before epoch
+* [ARROW-9976](https://issues.apache.org/jira/browse/ARROW-9976) - [Python] ArrowCapacityError when doing Table.from\_pandas with large dataframe
+* [ARROW-9990](https://issues.apache.org/jira/browse/ARROW-9990) - [Rust] [DataFusion] NOT is not plannable
+* [ARROW-9993](https://issues.apache.org/jira/browse/ARROW-9993) - [Python] Tzinfo - string roundtrip fails on pytz.StaticTzInfo objects
+* [ARROW-9994](https://issues.apache.org/jira/browse/ARROW-9994) - [C++][Python] Auto chunking nested array containing binary-like fields result malformed output
+* [ARROW-9996](https://issues.apache.org/jira/browse/ARROW-9996) - [C++] Dictionary is unset when calling DictionaryArray.GetScalar for null values
+* [ARROW-10003](https://issues.apache.org/jira/browse/ARROW-10003) - [C++] Create directories in CopyFiles when copying within the same filesystem
+* [ARROW-10008](https://issues.apache.org/jira/browse/ARROW-10008) - [Python] pyarrow.parquet.read\_table fails with predicate pushdown on categorical data with use\_legacy\_dataset=False
+* [ARROW-10011](https://issues.apache.org/jira/browse/ARROW-10011) - [C++] Make FindRE2.cmake re-entrant
+* [ARROW-10012](https://issues.apache.org/jira/browse/ARROW-10012) - [C++] Sporadic failures in CopyFiles test
+* [ARROW-10013](https://issues.apache.org/jira/browse/ARROW-10013) - [C++][CI] Flight test failure in TestFlightClient.GenericOptions
+* [ARROW-10017](https://issues.apache.org/jira/browse/ARROW-10017) - [Java] LargeMemoryUtil.checkedCastToInt has buggy logic
+* [ARROW-10022](https://issues.apache.org/jira/browse/ARROW-10022) - [C++] [Compute] core dumped on some scalar-arithmetic-benchmark
+* [ARROW-10027](https://issues.apache.org/jira/browse/ARROW-10027) - [Python] Incorrect null column returned when using a dataset filter expression.
+* [ARROW-10034](https://issues.apache.org/jira/browse/ARROW-10034) - [Rust] Master build broken
+* [ARROW-10041](https://issues.apache.org/jira/browse/ARROW-10041) - [Rust] Possible to create LargeStringArray with DataType::Utf8
+* [ARROW-10047](https://issues.apache.org/jira/browse/ARROW-10047) - [CI] Conda integration tests failing with cmake error
+* [ARROW-10048](https://issues.apache.org/jira/browse/ARROW-10048) - [Rust] Error in aggregate of min/max for strings
+* [ARROW-10049](https://issues.apache.org/jira/browse/ARROW-10049) - [C++/Python] Sync conda recipe with conda-forge
+* [ARROW-10060](https://issues.apache.org/jira/browse/ARROW-10060) - [Rust] [DataFusion] MergeExec currently discards partitions with errors
+* [ARROW-10062](https://issues.apache.org/jira/browse/ARROW-10062) - [Rust]: Fix for null elems for DoubleEndedIter for DictArray
+* [ARROW-10073](https://issues.apache.org/jira/browse/ARROW-10073) - [Python] Test test\_parquet\_nested\_storage relies on dict item ordering
+* [ARROW-10081](https://issues.apache.org/jira/browse/ARROW-10081) - [C++/Python] Fix bash syntax in drone.io conda builds
+* [ARROW-10085](https://issues.apache.org/jira/browse/ARROW-10085) - [C++] S3 tests fail on AppVeyor
+* [ARROW-10087](https://issues.apache.org/jira/browse/ARROW-10087) - [CI] Fix nightly docs job
+* [ARROW-10098](https://issues.apache.org/jira/browse/ARROW-10098) - [R][Doc] Fix copy\_files doc mismatch
+* [ARROW-10104](https://issues.apache.org/jira/browse/ARROW-10104) - [Python] Separate tests into its own conda package
+* [ARROW-10114](https://issues.apache.org/jira/browse/ARROW-10114) - [R] Segfault in to\_dataframe\_parallel with deeply nested structs
+* [ARROW-10116](https://issues.apache.org/jira/browse/ARROW-10116) - [Python][Packaging] Fix gRPC linking error in macOS wheels builds
+* [ARROW-10119](https://issues.apache.org/jira/browse/ARROW-10119) - [C++] Fix Parquet crashes on invalid input (OSS-Fuzz)
+* [ARROW-10121](https://issues.apache.org/jira/browse/ARROW-10121) - [C++][Python] Variable dictionaries do not survive roundtrip to IPC stream
+* [ARROW-10124](https://issues.apache.org/jira/browse/ARROW-10124) - [R] Write functions don't follow umask setting
+* [ARROW-10125](https://issues.apache.org/jira/browse/ARROW-10125) - [R] Int64 downcast check doesn't consider all chunks
+* [ARROW-10130](https://issues.apache.org/jira/browse/ARROW-10130) - [C++][Dataset] ParquetFileFragment::SplitByRowGroup does not preserve "complete\_metadata" status
+* [ARROW-10136](https://issues.apache.org/jira/browse/ARROW-10136) - [Rust][Arrow] Nulls are transformed into "" after filtering for StringArray
+* [ARROW-10137](https://issues.apache.org/jira/browse/ARROW-10137) - [R] Fix cpp helper that breaks if libarrow is not present
+* [ARROW-10147](https://issues.apache.org/jira/browse/ARROW-10147) - [Python] Constructing pandas metadata fails if an Index name is not JSON-serializable by default
+* [ARROW-10150](https://issues.apache.org/jira/browse/ARROW-10150) - [C++] Fix crashes on invalid Parquet file (OSS-Fuzz)
+* [ARROW-10169](https://issues.apache.org/jira/browse/ARROW-10169) - [Rust] Nulls should be rendered as "" rather than default value when pretty printing arrays
+* [ARROW-10175](https://issues.apache.org/jira/browse/ARROW-10175) - [CI] Nightly hdfs integration test job fails
+* [ARROW-10176](https://issues.apache.org/jira/browse/ARROW-10176) - [CI] Nightly valgrind job fails
+* [ARROW-10178](https://issues.apache.org/jira/browse/ARROW-10178) - [CI] Fix spark master integration test build setup
+* [ARROW-10179](https://issues.apache.org/jira/browse/ARROW-10179) - [Rust] Labeler is not labeling
+* [ARROW-10181](https://issues.apache.org/jira/browse/ARROW-10181) - [Rust] Arrow tests fail to compile on Raspberry Pi (32 bit)
+* [ARROW-10188](https://issues.apache.org/jira/browse/ARROW-10188) - [Rust] [DataFusion] Some examples are broken
+* [ARROW-10189](https://issues.apache.org/jira/browse/ARROW-10189) - [Doc] C data interface example for i32 uses \`l\`, not \`i\`, in the format
+* [ARROW-10192](https://issues.apache.org/jira/browse/ARROW-10192) - [C++][Python] Segfault when converting nested struct array with dictionary field to pandas series
+* [ARROW-10193](https://issues.apache.org/jira/browse/ARROW-10193) - [Python] Segfault when converting to fixed size binary array
+* [ARROW-10200](https://issues.apache.org/jira/browse/ARROW-10200) - [Java][CI] Fix failure of Java CI on s390x
+* [ARROW-10204](https://issues.apache.org/jira/browse/ARROW-10204) - [RUST] [Datafusion] Test failure in aggregate\_grouped\_empty with simd feature enabled
+* [ARROW-10214](https://issues.apache.org/jira/browse/ARROW-10214) - [Python] UnicodeDecodeError when printing schema with binary metadata
+* [ARROW-10226](https://issues.apache.org/jira/browse/ARROW-10226) - [Rust] [Parquet] Parquet reader reading wrong columns in some batches within a parquet file
+* [ARROW-10230](https://issues.apache.org/jira/browse/ARROW-10230) - [JS][Doc] JavaScript documentation fails to build
+* [ARROW-10232](https://issues.apache.org/jira/browse/ARROW-10232) - FixedSizeListArray is incorrectly written/read to/from parquet
+* [ARROW-10234](https://issues.apache.org/jira/browse/ARROW-10234) - [C++][Gandiva] Fix logic of round() for floats/decimals in Gandiva
+* [ARROW-10237](https://issues.apache.org/jira/browse/ARROW-10237) - [C++] Duplicate values in a dictionary result in corrupted parquet
+* [ARROW-10238](https://issues.apache.org/jira/browse/ARROW-10238) - [C\#] List<Struct\> is broken
+* [ARROW-10239](https://issues.apache.org/jira/browse/ARROW-10239) - [C++] aws-sdk-cpp apparently requires zlib too
+* [ARROW-10244](https://issues.apache.org/jira/browse/ARROW-10244) - [Python][Docs] Add docs on using pyarrow.dataset.parquet\_dataset
+* [ARROW-10248](https://issues.apache.org/jira/browse/ARROW-10248) - [C++][Dataset] Dataset writing does not write schema metadata
+* [ARROW-10262](https://issues.apache.org/jira/browse/ARROW-10262) - [C++] Some TypeClass in Scalar classes seem incorrect
+* [ARROW-10271](https://issues.apache.org/jira/browse/ARROW-10271) - [Rust] packed\_simd is broken and continued under a new project
+* [ARROW-10279](https://issues.apache.org/jira/browse/ARROW-10279) - [Release][Python] Fix verification script to align with the new macos wheel platform tags
+* [ARROW-10280](https://issues.apache.org/jira/browse/ARROW-10280) - [Packaging][Python] Fix macOS wheel artifact patterns
+* [ARROW-10281](https://issues.apache.org/jira/browse/ARROW-10281) - [Python] Fix warnings when running tests
+* [ARROW-10284](https://issues.apache.org/jira/browse/ARROW-10284) - [Python] Pyarrow is raising deprecation warning about filesystems on import
+* [ARROW-10285](https://issues.apache.org/jira/browse/ARROW-10285) - [Python] pyarrow.orc submodule is using deprecated functionality
+* [ARROW-10286](https://issues.apache.org/jira/browse/ARROW-10286) - [C++][Flight] Misleading CMake errors
+* [ARROW-10288](https://issues.apache.org/jira/browse/ARROW-10288) - [C++] Compilation fails on i386
+* [ARROW-10290](https://issues.apache.org/jira/browse/ARROW-10290) - [C++] List POP\_BACK is not available in older CMake versions
+
+
+## New Features and Improvements
+
+* [ARROW-983](https://issues.apache.org/jira/browse/ARROW-983) - [C++] Implement InputStream and OutputStream classes for interacting with socket connections
+* [ARROW-1105](https://issues.apache.org/jira/browse/ARROW-1105) - [C++] SQLite record batch reader
+* [ARROW-1509](https://issues.apache.org/jira/browse/ARROW-1509) - [Python] Write serialized object as a stream of encapsulated IPC messages
+* [ARROW-1669](https://issues.apache.org/jira/browse/ARROW-1669) - [C++] Consider adding Abseil (Google C++11 standard library extensions) to toolchain
+* [ARROW-1797](https://issues.apache.org/jira/browse/ARROW-1797) - [C++] Implement binary arithmetic kernels for numeric arrays
+* [ARROW-2164](https://issues.apache.org/jira/browse/ARROW-2164) - [C++] Clean up unnecessary decimal module refs
+* [ARROW-3080](https://issues.apache.org/jira/browse/ARROW-3080) - [Python] Unify Arrow to Python object conversion paths
+* [ARROW-3757](https://issues.apache.org/jira/browse/ARROW-3757) - [R] R bindings for Flight RPC client
+* [ARROW-3872](https://issues.apache.org/jira/browse/ARROW-3872) - [R] Add ad hoc test of feather compatibility
+* [ARROW-4046](https://issues.apache.org/jira/browse/ARROW-4046) - [Python/CI] Exercise large memory tests
+* [ARROW-4248](https://issues.apache.org/jira/browse/ARROW-4248) - [C++][Plasma] Build on Windows / Visual Studio
+* [ARROW-4685](https://issues.apache.org/jira/browse/ARROW-4685) - [C++] Update Boost to 1.69 in manylinux1 docker image
+* [ARROW-4927](https://issues.apache.org/jira/browse/ARROW-4927) - [Rust] Update top level README to describe current functionality
+* [ARROW-4957](https://issues.apache.org/jira/browse/ARROW-4957) - [Rust] [DataFusion] Implement get\_supertype correctly
+* [ARROW-4965](https://issues.apache.org/jira/browse/ARROW-4965) - [Python] Timestamp array type detection should use tzname of datetime.datetime objects
+* [ARROW-5034](https://issues.apache.org/jira/browse/ARROW-5034) - [C\#] ArrowStreamWriter should expose synchronous Write methods
+* [ARROW-5123](https://issues.apache.org/jira/browse/ARROW-5123) - [Rust] derive RecordWriter from struct definitions
+* [ARROW-6075](https://issues.apache.org/jira/browse/ARROW-6075) - [FlightRPC] Handle uncaught exceptions in middleware
+* [ARROW-6281](https://issues.apache.org/jira/browse/ARROW-6281) - [Python] Produce chunked arrays for nested types in pyarrow.array
+* [ARROW-6282](https://issues.apache.org/jira/browse/ARROW-6282) - [Format] Support lossy compression
+* [ARROW-6437](https://issues.apache.org/jira/browse/ARROW-6437) - [R] Add AWS SDK to system dependencies for macOS and Windows
+* [ARROW-6535](https://issues.apache.org/jira/browse/ARROW-6535) - [C++] Status::WithMessage should accept variadic parameters
+* [ARROW-6537](https://issues.apache.org/jira/browse/ARROW-6537) - [R] Pass column\_types to CSV reader
+* [ARROW-6972](https://issues.apache.org/jira/browse/ARROW-6972) - [C\#] Should support StructField arrays
+* [ARROW-6982](https://issues.apache.org/jira/browse/ARROW-6982) - [R] Add bindings for compare and boolean kernels
+* [ARROW-7136](https://issues.apache.org/jira/browse/ARROW-7136) - [Rust][CI] Pre-install the rust dependencies in the dockerfile
+* [ARROW-7218](https://issues.apache.org/jira/browse/ARROW-7218) - [Python] Conversion from boolean numpy scalars not working
+* [ARROW-7302](https://issues.apache.org/jira/browse/ARROW-7302) - [C++] CSV: allow converting a column to a specific dictionary type
+* [ARROW-7372](https://issues.apache.org/jira/browse/ARROW-7372) - [C++] Allow creating dictionary array from simple JSON
+* [ARROW-7871](https://issues.apache.org/jira/browse/ARROW-7871) - [Python] Expose more compute kernels
+* [ARROW-7960](https://issues.apache.org/jira/browse/ARROW-7960) - [C++][Parquet] Add support for schema translation from parquet nodes back to arrow for missing types
+* [ARROW-8001](https://issues.apache.org/jira/browse/ARROW-8001) - [R][Dataset] Bindings for dataset writing
+* [ARROW-8002](https://issues.apache.org/jira/browse/ARROW-8002) - [C++][Dataset] Dataset writing should let you (re)partition the data
+* [ARROW-8048](https://issues.apache.org/jira/browse/ARROW-8048) - [Python] Run memory leak tests nightly as follow up to ARROW-4120
+* [ARROW-8172](https://issues.apache.org/jira/browse/ARROW-8172) - [C++] ArrayFromJSON for dictionary arrays
+* [ARROW-8205](https://issues.apache.org/jira/browse/ARROW-8205) - [Rust] [DataFusion] DataFusion should enforce unique field names in a schema
+* [ARROW-8253](https://issues.apache.org/jira/browse/ARROW-8253) - [Rust] [DataFusion] Improve ergonomics of registering UDFs
+* [ARROW-8262](https://issues.apache.org/jira/browse/ARROW-8262) - [Rust] [DataFusion] Add example that uses LogicalPlanBuilder
+* [ARROW-8289](https://issues.apache.org/jira/browse/ARROW-8289) - [Rust] [Parquet] Implement minimal Arrow Parquet writer as starting point for full writer
+* [ARROW-8296](https://issues.apache.org/jira/browse/ARROW-8296) - [C++][Dataset] IpcFileFormat should support writing files with compressed buffers
+* [ARROW-8355](https://issues.apache.org/jira/browse/ARROW-8355) - [Python] Reduce the number of pandas dependent test cases in test\_feather
+* [ARROW-8359](https://issues.apache.org/jira/browse/ARROW-8359) - [C++/Python] Enable aarch64/ppc64le build in conda recipes
+* [ARROW-8383](https://issues.apache.org/jira/browse/ARROW-8383) - [Rust] Easier random access to DictionaryArray keys and values
+* [ARROW-8402](https://issues.apache.org/jira/browse/ARROW-8402) - [Java] Support ValidateFull methods in Java
+* [ARROW-8423](https://issues.apache.org/jira/browse/ARROW-8423) - [Rust] [Parquet] Serialize arrow schema into metadata when writing parquet
+* [ARROW-8426](https://issues.apache.org/jira/browse/ARROW-8426) - [Rust] [Parquet] Add support for writing dictionary types
+* [ARROW-8493](https://issues.apache.org/jira/browse/ARROW-8493) - [C++] Create unified schema resolution code for Array reconstruction.
+* [ARROW-8494](https://issues.apache.org/jira/browse/ARROW-8494) - [C++] Implement basic array-by-array  reassembly logic
+* [ARROW-8581](https://issues.apache.org/jira/browse/ARROW-8581) - [C\#] Date32/64Array.Builder should accept DateTime, not DateTimeOffset
+* [ARROW-8601](https://issues.apache.org/jira/browse/ARROW-8601) - [Go][Flight] Implement Flight Writer interface
+* [ARROW-8601](https://issues.apache.org/jira/browse/ARROW-8601) - [Go][Flight] Implement Flight Writer interface
+* [ARROW-8618](https://issues.apache.org/jira/browse/ARROW-8618) - [C++] ASSIGN\_OR\_RAISE should move its argument
+* [ARROW-8678](https://issues.apache.org/jira/browse/ARROW-8678) - [C++][Parquet] Remove legacy arrow to level translation.
+* [ARROW-8712](https://issues.apache.org/jira/browse/ARROW-8712) - [R] Expose strptime timestamp parsing in read\_csv conversion options
+* [ARROW-8774](https://issues.apache.org/jira/browse/ARROW-8774) - [Rust] [DataFusion] Improve threading model
+* [ARROW-8810](https://issues.apache.org/jira/browse/ARROW-8810) - [R] Add documentation about Parquet format, appending to stream format
+* [ARROW-8824](https://issues.apache.org/jira/browse/ARROW-8824) - [Rust] [DataFusion] Implement new SQL parser
+* [ARROW-8828](https://issues.apache.org/jira/browse/ARROW-8828) - [Rust] Implement SQL tokenizer
+* [ARROW-8829](https://issues.apache.org/jira/browse/ARROW-8829) - [Rust] Implement SQL parser
+* [ARROW-9010](https://issues.apache.org/jira/browse/ARROW-9010) - [Java] Framework and interface changes for RecordBatch IPC buffer compression
+* [ARROW-9065](https://issues.apache.org/jira/browse/ARROW-9065) - [C++] Support parsing date32 in dataset partition folders
+* [ARROW-9068](https://issues.apache.org/jira/browse/ARROW-9068) - [C++][Dataset] Simplify Partitioning interface
+* [ARROW-9078](https://issues.apache.org/jira/browse/ARROW-9078) - [C++] Parquet writing of extension type with nested storage type fails
+* [ARROW-9104](https://issues.apache.org/jira/browse/ARROW-9104) - [C++] Parquet encryption tests should write files to a temporary directory instead of the testing submodule's directory
+* [ARROW-9107](https://issues.apache.org/jira/browse/ARROW-9107) - [C++][Dataset] Time-based types support
+* [ARROW-9147](https://issues.apache.org/jira/browse/ARROW-9147) - [C++][Dataset] Support null -\> other type promotion in Dataset scanning
+* [ARROW-9205](https://issues.apache.org/jira/browse/ARROW-9205) - [Documentation] Fix typos in Columnar.rst
+* [ARROW-9266](https://issues.apache.org/jira/browse/ARROW-9266) - [Python][Packaging] Enable S3 support in macOS wheels
+* [ARROW-9271](https://issues.apache.org/jira/browse/ARROW-9271) - [R] Preserve data frame metadata in round trip
+* [ARROW-9286](https://issues.apache.org/jira/browse/ARROW-9286) - [C++] Add function "aliases" to compute::FunctionRegistry
+* [ARROW-9328](https://issues.apache.org/jira/browse/ARROW-9328) - [C++][Gandiva] Add LTRIM, RTRIM, BTRIM functions for string
+* [ARROW-9338](https://issues.apache.org/jira/browse/ARROW-9338) - [Rust] Add instructions for running clippy locally
+* [ARROW-9344](https://issues.apache.org/jira/browse/ARROW-9344) - [C++][Flight] measure latency quantile in flight benchmark
+* [ARROW-9358](https://issues.apache.org/jira/browse/ARROW-9358) - [Integration] Reconsider generated\_large\_batch.json
+* [ARROW-9371](https://issues.apache.org/jira/browse/ARROW-9371) - [Java] Run vector tests for both allocators
+* [ARROW-9377](https://issues.apache.org/jira/browse/ARROW-9377) - [Java] Support unsigned dictionary indices
+* [ARROW-9387](https://issues.apache.org/jira/browse/ARROW-9387) - [R] Use new C++ table select method
+* [ARROW-9388](https://issues.apache.org/jira/browse/ARROW-9388) - [C++] Division kernels
+* [ARROW-9394](https://issues.apache.org/jira/browse/ARROW-9394) - [Python] Support pickling of Scalars
+* [ARROW-9398](https://issues.apache.org/jira/browse/ARROW-9398) - [C++] Register the SIMD sum variants under function instance instead a SIMD function
+* [ARROW-9402](https://issues.apache.org/jira/browse/ARROW-9402) - [C++] Add portable wrappers for \_\_builtin\_add\_overflow and friends
+* [ARROW-9405](https://issues.apache.org/jira/browse/ARROW-9405) - [R] Switch to cpp11
+* [ARROW-9412](https://issues.apache.org/jira/browse/ARROW-9412) - [C++] Add non-BUNDLED dependencies to exported INSTALL\_INTERFACE\_LIBS of arrow\_static and test that it works
+* [ARROW-9429](https://issues.apache.org/jira/browse/ARROW-9429) - [Python] ChunkedArray.to\_numpy
+* [ARROW-9454](https://issues.apache.org/jira/browse/ARROW-9454) - [GLib] Add binding of some dictionary builders
+* [ARROW-9465](https://issues.apache.org/jira/browse/ARROW-9465) - [Python] Improve ergonomics of compute functions
+* [ARROW-9469](https://issues.apache.org/jira/browse/ARROW-9469) - [Python] Make more objects weakrefable
+* [ARROW-9487](https://issues.apache.org/jira/browse/ARROW-9487) - [Developer] Cover the archery release utilities with unittests
+* [ARROW-9488](https://issues.apache.org/jira/browse/ARROW-9488) - [Release] Use the new changelog generation when updating the website
+* [ARROW-9507](https://issues.apache.org/jira/browse/ARROW-9507) - [Rust] [DataFusion] PhysicalExpr should implement Display trait
+* [ARROW-9508](https://issues.apache.org/jira/browse/ARROW-9508) - [Release][APT][Yum] Enable verification for arm64 binaries
+* [ARROW-9516](https://issues.apache.org/jira/browse/ARROW-9516) - [Rust][DataFusion] Refactor physical expressions to not care about their names nor indexes
+* [ARROW-9517](https://issues.apache.org/jira/browse/ARROW-9517) - [C++][Python] Allow session\_token argument when initializing S3FileSystem
+* [ARROW-9518](https://issues.apache.org/jira/browse/ARROW-9518) - [Python] Deprecate pyarrow serialization
+* [ARROW-9521](https://issues.apache.org/jira/browse/ARROW-9521) - [Rust] CsvReadOptions should allow file extension to be specified
+* [ARROW-9523](https://issues.apache.org/jira/browse/ARROW-9523) - [Rust] improve performance of filter kernel
+* [ARROW-9534](https://issues.apache.org/jira/browse/ARROW-9534) - [Rust] [DataFusion] Implement functions for creating literal expressions for all types
+* [ARROW-9550](https://issues.apache.org/jira/browse/ARROW-9550) - [Rust] [DataFusion] Remove Rc<RefCell<\_\>\> from hash aggregate operator
+* [ARROW-9553](https://issues.apache.org/jira/browse/ARROW-9553) - [Rust] Release script doesn't bump parquet crate's arrow dependency version
+* [ARROW-9557](https://issues.apache.org/jira/browse/ARROW-9557) - [R] Iterating over parquet columns is slow in R
+* [ARROW-9559](https://issues.apache.org/jira/browse/ARROW-9559) - [Rust] [DataFusion] Revert privatization of exprlist\_to\_fields
+* [ARROW-9563](https://issues.apache.org/jira/browse/ARROW-9563) - [Dev][Release] Use archery's changelog generator when creating release notes for the website 
+* [ARROW-9568](https://issues.apache.org/jira/browse/ARROW-9568) - [CI] Use official msys action on GHA
+* [ARROW-9576](https://issues.apache.org/jira/browse/ARROW-9576) - [Python][Doc] Fix error in code example for extension types
+* [ARROW-9580](https://issues.apache.org/jira/browse/ARROW-9580) - [JS] Docs have superfluous ()
+* [ARROW-9581](https://issues.apache.org/jira/browse/ARROW-9581) - [Dev][Release] Bump next snapshot versions to 2.0.0
+* [ARROW-9582](https://issues.apache.org/jira/browse/ARROW-9582) - [Rust] Implement Array::memory\_size()
+* [ARROW-9585](https://issues.apache.org/jira/browse/ARROW-9585) - [Rust] Remove duplicated to-do line in DataFusion readme
+* [ARROW-9587](https://issues.apache.org/jira/browse/ARROW-9587) - [FlightRPC][Java] Clean up DoPut/FlightStream memory handling
+* [ARROW-9593](https://issues.apache.org/jira/browse/ARROW-9593) - [Python] Add custom pickle reducers for DictionaryScalar
+* [ARROW-9604](https://issues.apache.org/jira/browse/ARROW-9604) - [C++] Add benchmark for aggregate min/max compute kernels
+* [ARROW-9605](https://issues.apache.org/jira/browse/ARROW-9605) - [C++] Optimize performance for aggregate min/max compute kernels
+* [ARROW-9607](https://issues.apache.org/jira/browse/ARROW-9607) - [C++][Gandiva] Add bitwise\_and(), bitwise\_or() and bitwise\_not() functions for integers
+* [ARROW-9608](https://issues.apache.org/jira/browse/ARROW-9608) - [Rust] Remove arrow flight from parquet's feature gating
+* [ARROW-9615](https://issues.apache.org/jira/browse/ARROW-9615) - [Rust] Add kernel to compute length of string array
+* [ARROW-9617](https://issues.apache.org/jira/browse/ARROW-9617) - [Rust] [DataFusion] Add length of string array
+* [ARROW-9618](https://issues.apache.org/jira/browse/ARROW-9618) - [Rust] [DataFusion] Make it easier to write optimizers
+* [ARROW-9619](https://issues.apache.org/jira/browse/ARROW-9619) - [Rust] [DataFusion] Add predicate push-down
+* [ARROW-9632](https://issues.apache.org/jira/browse/ARROW-9632) - [Rust] Add a "new" method for ExecutionContextSchemaProvider
+* [ARROW-9638](https://issues.apache.org/jira/browse/ARROW-9638) - [C++][Compute] Implement mode(most frequent number) kernel
+* [ARROW-9639](https://issues.apache.org/jira/browse/ARROW-9639) - [Ruby] Add dependency version check
+* [ARROW-9640](https://issues.apache.org/jira/browse/ARROW-9640) - [C++][Gandiva] Implement round() for integers and long integers
+* [ARROW-9641](https://issues.apache.org/jira/browse/ARROW-9641) - [C++][Gandiva] Implement round() for floating point and double floating point numbers
+* [ARROW-9645](https://issues.apache.org/jira/browse/ARROW-9645) - [Python] Deprecate the legacy pyarrow.filesystem interface
+* [ARROW-9646](https://issues.apache.org/jira/browse/ARROW-9646) - [C++][Dataset] Add support for writing parquet datasets
+* [ARROW-9650](https://issues.apache.org/jira/browse/ARROW-9650) - [Packaging][APT] Drop support for Ubuntu 19.10
+* [ARROW-9654](https://issues.apache.org/jira/browse/ARROW-9654) - [Rust][DataFusion] Add an EXPLAIN command to the datafusion CLI
+* [ARROW-9656](https://issues.apache.org/jira/browse/ARROW-9656) - [Rust][DataFusion] Slightly confusing error message when unsupported type is provided to CREATE EXTERNAL TABLE
+* [ARROW-9658](https://issues.apache.org/jira/browse/ARROW-9658) - [Python][Dataset] Bindings for dataset writing
+* [ARROW-9665](https://issues.apache.org/jira/browse/ARROW-9665) - [R] head/tail/take for Datasets
+* [ARROW-9667](https://issues.apache.org/jira/browse/ARROW-9667) - [CI][Crossbow] Segfault in 2 nightly R builds
+* [ARROW-9671](https://issues.apache.org/jira/browse/ARROW-9671) - [C++] BasicDecimal128 constructor interprets uint64\_t integers with highest bit set as negative
+* [ARROW-9673](https://issues.apache.org/jira/browse/ARROW-9673) - [Rust] Add a param "dialect" for DFParser::parse\_sql
+* [ARROW-9678](https://issues.apache.org/jira/browse/ARROW-9678) - [Rust] [DataFusion] Improve projection push down to remove unused columns
+* [ARROW-9679](https://issues.apache.org/jira/browse/ARROW-9679) - [Rust] [DataFusion] HashAggregate walks map many times building final batch
+* [ARROW-9681](https://issues.apache.org/jira/browse/ARROW-9681) - [Java] Failed Arrow Memory - Core on big-endian platform
+* [ARROW-9683](https://issues.apache.org/jira/browse/ARROW-9683) - [Rust][DataFusion] Implement Debug for ExecutionPlan trait
+* [ARROW-9691](https://issues.apache.org/jira/browse/ARROW-9691) - [Rust] [DataFusion] Make sql\_statement\_to\_plan public
+* [ARROW-9695](https://issues.apache.org/jira/browse/ARROW-9695) - [Rust][DataFusion] Improve documentation on LogicalPlan variants
+* [ARROW-9699](https://issues.apache.org/jira/browse/ARROW-9699) - [C++][Compute] Improve mode kernel performance for small integer types
+* [ARROW-9701](https://issues.apache.org/jira/browse/ARROW-9701) - [Java][CI] Add a test job on s390x
+* [ARROW-9702](https://issues.apache.org/jira/browse/ARROW-9702) - [C++] Move bpacking simd to runtime path
+* [ARROW-9703](https://issues.apache.org/jira/browse/ARROW-9703) - [Developer][Archery] Restartable cherry-picking process for creating maintenance branches
+* [ARROW-9706](https://issues.apache.org/jira/browse/ARROW-9706) - [Java] Tests in TestLargeListVector fails on big endian platform
+* [ARROW-9710](https://issues.apache.org/jira/browse/ARROW-9710) - [C++] Generalize Decimal ToString in preparation for Decimal256
+* [ARROW-9711](https://issues.apache.org/jira/browse/ARROW-9711) - [Rust] Add benchmark based on TPC-H
+* [ARROW-9713](https://issues.apache.org/jira/browse/ARROW-9713) - [Rust][DataFusion] Remove explicit panics
+* [ARROW-9715](https://issues.apache.org/jira/browse/ARROW-9715) - [R] changelog/doc updates for 1.0.1
+* [ARROW-9718](https://issues.apache.org/jira/browse/ARROW-9718) - [Python] Make pyarrow.parquet work with the new filesystem interfaces
+* [ARROW-9721](https://issues.apache.org/jira/browse/ARROW-9721) - [Packaging][Python] Update wheel dependency files
+* [ARROW-9722](https://issues.apache.org/jira/browse/ARROW-9722) - [Rust]: Shorten key lifetime for reverse lookup for dictionary arrays
+* [ARROW-9723](https://issues.apache.org/jira/browse/ARROW-9723) - [C++] Expected behaviour of "mode" kernel with NaNs ?
+* [ARROW-9725](https://issues.apache.org/jira/browse/ARROW-9725) - [Rust] [DataFusion] LimitExec and SortExec should use MergeExec
+* [ARROW-9737](https://issues.apache.org/jira/browse/ARROW-9737) - [C++][Gandiva] Add bitwise\_xor() for integers
+* [ARROW-9739](https://issues.apache.org/jira/browse/ARROW-9739) - [CI][Ruby] Don't install gem documents
+* [ARROW-9742](https://issues.apache.org/jira/browse/ARROW-9742) - [Rust] Create one standard DataFrame API
+* [ARROW-9751](https://issues.apache.org/jira/browse/ARROW-9751) - [Rust] [DataFusion] Extend UDFs to accept more than one type per argument
+* [ARROW-9752](https://issues.apache.org/jira/browse/ARROW-9752) - [Rust] [DataFusion] Add support for Aggregate UDFs
+* [ARROW-9753](https://issues.apache.org/jira/browse/ARROW-9753) - [Rust] [DataFusion] Remove the use of Mutex in ExecutionPlan trait
+* [ARROW-9754](https://issues.apache.org/jira/browse/ARROW-9754) - [Rust] [DataFusion] Implement async in DataFusion traits
+* [ARROW-9757](https://issues.apache.org/jira/browse/ARROW-9757) - [Rust] [DataFusion] Use "pub use" to expose a clean public API
+* [ARROW-9758](https://issues.apache.org/jira/browse/ARROW-9758) - [Rust] [DataFusion] Implement extension API for DataFusion
+* [ARROW-9759](https://issues.apache.org/jira/browse/ARROW-9759) - [Rust] [DataFusion] Implement DataFrame::sort
+* [ARROW-9760](https://issues.apache.org/jira/browse/ARROW-9760) - [Rust] [DataFusion] Implement DataFrame::explain
+* [ARROW-9761](https://issues.apache.org/jira/browse/ARROW-9761) - [C++] Add experimental pull-based iterator structures to C interface implementation
+* [ARROW-9762](https://issues.apache.org/jira/browse/ARROW-9762) - [Rust] [DataFusion] ExecutionContext::sql should return DataFrame
+* [ARROW-9769](https://issues.apache.org/jira/browse/ARROW-9769) - [Python] Remove skip for in-memory fsspec in test\_move\_file
+* [ARROW-9775](https://issues.apache.org/jira/browse/ARROW-9775) - [C++] Automatic S3 region selection
+* [ARROW-9781](https://issues.apache.org/jira/browse/ARROW-9781) - [C++] Fix uninitialized value warnings
+* [ARROW-9782](https://issues.apache.org/jira/browse/ARROW-9782) - [C++][Dataset] Ability to write ".feather" files with IpcFileFormat
+* [ARROW-9784](https://issues.apache.org/jira/browse/ARROW-9784) - [Rust] [DataFusion] Improve instructions for running tpch benchmark
+* [ARROW-9786](https://issues.apache.org/jira/browse/ARROW-9786) - [R] Unvendor cpp11 before release
+* [ARROW-9788](https://issues.apache.org/jira/browse/ARROW-9788) - Handle naming inconsistencies between SQL, DataFrame API and struct names
+* [ARROW-9792](https://issues.apache.org/jira/browse/ARROW-9792) - [Rust] [DataFusion] Logical aggregate functions should not return Result
+* [ARROW-9794](https://issues.apache.org/jira/browse/ARROW-9794) - [C++] Add functionality to cpu\_info to discriminate between Intel vs AMD x86
+* [ARROW-9795](https://issues.apache.org/jira/browse/ARROW-9795) - [C++][Gandiva] Implement castTIMESTAMP(int64) in Gandiva
+* [ARROW-9806](https://issues.apache.org/jira/browse/ARROW-9806) - [R] More compute kernel bindings
+* [ARROW-9807](https://issues.apache.org/jira/browse/ARROW-9807) - [R] News update/version bump post-1.0.1
+* [ARROW-9808](https://issues.apache.org/jira/browse/ARROW-9808) - [Python] parquet.read\_table docstring wrong use\_legacy\_dataset explanation
+* [ARROW-9811](https://issues.apache.org/jira/browse/ARROW-9811) - [C++] Unchecked floating point division by 0 should succeed
+* [ARROW-9813](https://issues.apache.org/jira/browse/ARROW-9813) - [C++] Disable semantic interposition
+* [ARROW-9819](https://issues.apache.org/jira/browse/ARROW-9819) - [C++] Bump mimalloc to 1.6.4
+* [ARROW-9821](https://issues.apache.org/jira/browse/ARROW-9821) - [Rust][DataFusion] User Defined PlanNode / Operator API
+* [ARROW-9821](https://issues.apache.org/jira/browse/ARROW-9821) - [Rust][DataFusion] User Defined PlanNode / Operator API
+* [ARROW-9823](https://issues.apache.org/jira/browse/ARROW-9823) - [CI][C++][MinGW] Enable S3
+* [ARROW-9832](https://issues.apache.org/jira/browse/ARROW-9832) - [Rust] [DataFusion] Refactor PhysicalPlan to remove Partition
+* [ARROW-9833](https://issues.apache.org/jira/browse/ARROW-9833) - [Rust] [DataFusion] Refactor TableProvider.scan to return ExecutionPlan
+* [ARROW-9834](https://issues.apache.org/jira/browse/ARROW-9834) - [Rust] [DataFusion] Remove Partition trait
+* [ARROW-9835](https://issues.apache.org/jira/browse/ARROW-9835) - [Rust] [DataFusion] Remove FunctionMeta
+* [ARROW-9836](https://issues.apache.org/jira/browse/ARROW-9836) - [Rust] [DataFusion] Improve API for usage of UDFs
+* [ARROW-9837](https://issues.apache.org/jira/browse/ARROW-9837) - [Rust] Add provider for variable
+* [ARROW-9838](https://issues.apache.org/jira/browse/ARROW-9838) - [Rust] [DataFusion] DefaultPhysicalPlanner should insert explicit MergeExec nodes
+* [ARROW-9839](https://issues.apache.org/jira/browse/ARROW-9839) - [Rust] [DataFusion] Add ability to downcast ExecutionPlan to specific operator
+* [ARROW-9841](https://issues.apache.org/jira/browse/ARROW-9841) - [Rust] Update checked-in flatbuffer files
+* [ARROW-9844](https://issues.apache.org/jira/browse/ARROW-9844) - [Go][CI] Add Travis CI job for Go on s390x
+* [ARROW-9845](https://issues.apache.org/jira/browse/ARROW-9845) - [Rust] [Parquet] serde\_json is only used in tests but isn't in dev-dependencies
+* [ARROW-9848](https://issues.apache.org/jira/browse/ARROW-9848) - [Rust] Implement changes to ensure flatbuffer alignment
+* [ARROW-9849](https://issues.apache.org/jira/browse/ARROW-9849) - [Rust] [DataFusion] Make UDFs not need a Field
+* [ARROW-9850](https://issues.apache.org/jira/browse/ARROW-9850) - [Go] Defer should not be used in the loop
+* [ARROW-9853](https://issues.apache.org/jira/browse/ARROW-9853) - [RUST] Implement "take" kernel for dictionary arrays
+* [ARROW-9854](https://issues.apache.org/jira/browse/ARROW-9854) - [R] Support reading/writing data to/from S3
+* [ARROW-9858](https://issues.apache.org/jira/browse/ARROW-9858) - [C++][Python][Docs] Expand user guide for FileSystem
+* [ARROW-9863](https://issues.apache.org/jira/browse/ARROW-9863) - [C++] [PARQUET] Optimize meta data recovery of ApplicationVersion
+* [ARROW-9867](https://issues.apache.org/jira/browse/ARROW-9867) - [C++][Dataset] FileSystemDataset should expose its filesystem
+* [ARROW-9868](https://issues.apache.org/jira/browse/ARROW-9868) - [C++] Provide utility for copying files between filesystems
+* [ARROW-9869](https://issues.apache.org/jira/browse/ARROW-9869) - [R] Implement full S3FileSystem/S3Options constructor
+* [ARROW-9870](https://issues.apache.org/jira/browse/ARROW-9870) - [R] Friendly interface for filesystems (S3)
+* [ARROW-9871](https://issues.apache.org/jira/browse/ARROW-9871) - [C++] Add uppercase support to ARROW\_USER\_SIMD\_LEVEL.
+* [ARROW-9873](https://issues.apache.org/jira/browse/ARROW-9873) - [C++][Compute] Improve mode kernel for intergers within limited value range
+* [ARROW-9875](https://issues.apache.org/jira/browse/ARROW-9875) - [Python] Let FileSystem.get\_file\_info accept a single path
+* [ARROW-9884](https://issues.apache.org/jira/browse/ARROW-9884) - [R] Bindings for writing datasets to Parquet
+* [ARROW-9885](https://issues.apache.org/jira/browse/ARROW-9885) - [Rust] [DataFusion] Simplify code of type coercion for binary types
+* [ARROW-9886](https://issues.apache.org/jira/browse/ARROW-9886) - [Rust] [DataFusion] Simplify code to test cast
+* [ARROW-9887](https://issues.apache.org/jira/browse/ARROW-9887) - [Rust] [DataFusion] Add support for complex return types of built-in functions
+* [ARROW-9890](https://issues.apache.org/jira/browse/ARROW-9890) - [R] Add zstandard compression codec in macOS build
+* [ARROW-9891](https://issues.apache.org/jira/browse/ARROW-9891) - [Rust] [DataFusion] Make math functions support f32
+* [ARROW-9892](https://issues.apache.org/jira/browse/ARROW-9892) - [Rust] [DataFusion] Add support for concat
+* [ARROW-9893](https://issues.apache.org/jira/browse/ARROW-9893) - [Python] Bindings for writing datasets to Parquet
+* [ARROW-9895](https://issues.apache.org/jira/browse/ARROW-9895) - [RUST] Improve sort kernels
+* [ARROW-9899](https://issues.apache.org/jira/browse/ARROW-9899) - [Rust] [DataFusion] Switch from Box<Schema\> --\> SchemaRef (Arc<Schema\>) to be consistent with the rest of Arrow
+* [ARROW-9900](https://issues.apache.org/jira/browse/ARROW-9900) - [Rust][DataFusion] Use Arc<\> instead of Box<\> in LogicalPlan
+* [ARROW-9901](https://issues.apache.org/jira/browse/ARROW-9901) - [C++] Add hand-crafted Parquet to Arrow reconstruction test for nested reading
+* [ARROW-9902](https://issues.apache.org/jira/browse/ARROW-9902) - [Rust] [DataFusion] Add support for array()
+* [ARROW-9904](https://issues.apache.org/jira/browse/ARROW-9904) - [C++] Unroll the loop manually for CountSetBits
+* [ARROW-9908](https://issues.apache.org/jira/browse/ARROW-9908) - [Rust] Support temporal data types in JSON reader
+* [ARROW-9910](https://issues.apache.org/jira/browse/ARROW-9910) - [Rust] [DataFusion] Type coercion of Variadic is wrong
+* [ARROW-9914](https://issues.apache.org/jira/browse/ARROW-9914) - [Rust][DataFusion] Document the SQL -\> Arrow type mapping
+* [ARROW-9916](https://issues.apache.org/jira/browse/ARROW-9916) - [RUST] Avoid cloning ArrayData in several places
+* [ARROW-9917](https://issues.apache.org/jira/browse/ARROW-9917) - [Python][Compute] Add bindings for mode kernel
+* [ARROW-9919](https://issues.apache.org/jira/browse/ARROW-9919) - [Rust] [DataFusion] Math functions
+* [ARROW-9921](https://issues.apache.org/jira/browse/ARROW-9921) - [Rust] Add \`from(Vec<Option<&str\>\>)\` to [Large]StringArray
+* [ARROW-9925](https://issues.apache.org/jira/browse/ARROW-9925) - [GLib] Add low level value readers for GArrowListArray family
+* [ARROW-9926](https://issues.apache.org/jira/browse/ARROW-9926) - [GLib] Use placement new for GArrowRecordBatchFileReader
+* [ARROW-9928](https://issues.apache.org/jira/browse/ARROW-9928) - [C++] Speed up integer parsing slightly
+* [ARROW-9929](https://issues.apache.org/jira/browse/ARROW-9929) - [Developer] Autotune cmake-format
+* [ARROW-9933](https://issues.apache.org/jira/browse/ARROW-9933) - [Developer] Add drone as a CI provider for crossbow
+* [ARROW-9934](https://issues.apache.org/jira/browse/ARROW-9934) - [Rust] Shape and stride check in tensor
+* [ARROW-9941](https://issues.apache.org/jira/browse/ARROW-9941) - [Python] Better string representation for extension types
+* [ARROW-9944](https://issues.apache.org/jira/browse/ARROW-9944) - [Rust] Implement TO\_TIMESTAMP function
+* [ARROW-9949](https://issues.apache.org/jira/browse/ARROW-9949) - [C++] Generalize Decimal128::FromString for reuse in Decimal256
+* [ARROW-9950](https://issues.apache.org/jira/browse/ARROW-9950) - [Rust] [DataFusion] Allow UDF usage without registry
+* [ARROW-9952](https://issues.apache.org/jira/browse/ARROW-9952) - [Python] Use pyarrow.dataset writing for pq.write\_to\_dataset
+* [ARROW-9954](https://issues.apache.org/jira/browse/ARROW-9954) - [Rust] [DataFusion] Simplify code of aggregate planning
+* [ARROW-9956](https://issues.apache.org/jira/browse/ARROW-9956) - [C++][Gandiva] Implement Binary string function in Gandiva
+* [ARROW-9957](https://issues.apache.org/jira/browse/ARROW-9957) - [Rust] Remove unmaintained tempdir dependency
+* [ARROW-9961](https://issues.apache.org/jira/browse/ARROW-9961) - [Rust][DataFusion] to\_timestamp function parses timestamp without timezone offset as UTC rather than local
+* [ARROW-9964](https://issues.apache.org/jira/browse/ARROW-9964) - [C++] CSV date support
+* [ARROW-9965](https://issues.apache.org/jira/browse/ARROW-9965) - [Java] Buffer capacity calculations are slow for fixed-width vectors
+* [ARROW-9966](https://issues.apache.org/jira/browse/ARROW-9966) - [Rust] Speedup aggregate kernels
+* [ARROW-9967](https://issues.apache.org/jira/browse/ARROW-9967) - [Python] Add compute module docs
+* [ARROW-9971](https://issues.apache.org/jira/browse/ARROW-9971) - [Rust] Speedup take
+* [ARROW-9977](https://issues.apache.org/jira/browse/ARROW-9977) - [Rust] Add min/max for [Large]String
+* [ARROW-9979](https://issues.apache.org/jira/browse/ARROW-9979) - [Rust] Fix arrow crate clippy lints
+* [ARROW-9980](https://issues.apache.org/jira/browse/ARROW-9980) - [Rust] Fix parquet crate clippy lints
+* [ARROW-9981](https://issues.apache.org/jira/browse/ARROW-9981) - [Rust] Allow configuring flight IPC with IpcWriteOptions
+* [ARROW-9983](https://issues.apache.org/jira/browse/ARROW-9983) - [C++][Dataset][Python] Use larger default batch size than 32K for Datasets API
+* [ARROW-9984](https://issues.apache.org/jira/browse/ARROW-9984) - [Rust] [DataFusion] DRY of function to string
+* [ARROW-9986](https://issues.apache.org/jira/browse/ARROW-9986) - [Rust][DataFusion] TO\_TIMESTAMP function erroneously requires fractional seconds when no timezone is present
+* [ARROW-9987](https://issues.apache.org/jira/browse/ARROW-9987) - [Rust] [DataFusion] Improve docs of \`Expr\`.
+* [ARROW-9988](https://issues.apache.org/jira/browse/ARROW-9988) - [Rust] [DataFusion] Added std::ops to logical expressions
+* [ARROW-9992](https://issues.apache.org/jira/browse/ARROW-9992) - [C++][Python] Refactor python to arrow conversions based on a reusable conversion API 
+* [ARROW-9998](https://issues.apache.org/jira/browse/ARROW-9998) - [Python] Support pickling DictionaryScalar
+* [ARROW-9999](https://issues.apache.org/jira/browse/ARROW-9999) - [Python] Support constructing dictionary array directly through pa.array()
+* [ARROW-10000](https://issues.apache.org/jira/browse/ARROW-10000) - [C++][Python] Support constructing StructArray from list of key-value pairs
+* [ARROW-10001](https://issues.apache.org/jira/browse/ARROW-10001) - [Rust] [DataFusion] Add developer guide to README
+* [ARROW-10010](https://issues.apache.org/jira/browse/ARROW-10010) - [Rust] Speedup arithmetic
+* [ARROW-10015](https://issues.apache.org/jira/browse/ARROW-10015) - [Rust] Implement SIMD for aggregate kernel sum
+* [ARROW-10016](https://issues.apache.org/jira/browse/ARROW-10016) - [Rust] [DataFusion] Implement IsNull and IsNotNull
+* [ARROW-10018](https://issues.apache.org/jira/browse/ARROW-10018) - [CI] Disable Sphinx and API documentation build since it takes 6 hours on master
+* [ARROW-10019](https://issues.apache.org/jira/browse/ARROW-10019) - [Rust] Add substring kernel
+* [ARROW-10023](https://issues.apache.org/jira/browse/ARROW-10023) - [Gandiva][C++] Implementing Split part function in gandiva
+* [ARROW-10024](https://issues.apache.org/jira/browse/ARROW-10024) - [C++][Parquet] Create nested reading benchmarks
+* [ARROW-10028](https://issues.apache.org/jira/browse/ARROW-10028) - [Rust] Simplify macro def\_numeric\_from\_vec
+* [ARROW-10030](https://issues.apache.org/jira/browse/ARROW-10030) - [Rust] Support fromIter and toIter
+* [ARROW-10035](https://issues.apache.org/jira/browse/ARROW-10035) - [C++] Bump versions of vendored code
+* [ARROW-10037](https://issues.apache.org/jira/browse/ARROW-10037) - [C++] Workaround to force find AWS SDK to look for shared libraries 
+* [ARROW-10040](https://issues.apache.org/jira/browse/ARROW-10040) - [Rust] Create a way to slice unalligned offset buffers
+* [ARROW-10043](https://issues.apache.org/jira/browse/ARROW-10043) - [Rust] [DataFusion] Introduce support for DISTINCT by partially implementing COUNT(DISTINCT)
+* [ARROW-10044](https://issues.apache.org/jira/browse/ARROW-10044) - [Rust] Improve README
+* [ARROW-10046](https://issues.apache.org/jira/browse/ARROW-10046) - [Rust] [DataFusion] Made \`\*Iterator\` implement Iterator
+* [ARROW-10050](https://issues.apache.org/jira/browse/ARROW-10050) - [C++][Gandiva] Implement concat() in Gandiva for up to 10 arguments
+* [ARROW-10051](https://issues.apache.org/jira/browse/ARROW-10051) - [C++][Compute] Make aggregate kernel merge state mutable
+* [ARROW-10054](https://issues.apache.org/jira/browse/ARROW-10054) - [Python] Slice methods should return empty arrays instead of crashing
+* [ARROW-10055](https://issues.apache.org/jira/browse/ARROW-10055) - [Rust] Implement DoubleEndedIterator for NullableIter
+* [ARROW-10057](https://issues.apache.org/jira/browse/ARROW-10057) - [C++] Add Parquet-Arrow roundtrip tests for nested data
+* [ARROW-10058](https://issues.apache.org/jira/browse/ARROW-10058) - [C++] Investigate performance of LevelsToBitmap without BMI2
+* [ARROW-10059](https://issues.apache.org/jira/browse/ARROW-10059) - [R][Doc] Give more advice on how to set up C++ build
+* [ARROW-10063](https://issues.apache.org/jira/browse/ARROW-10063) - [Archery][CI] Fetch main branch in archery build only when it is a pull request
+* [ARROW-10064](https://issues.apache.org/jira/browse/ARROW-10064) - [C++] Resolve compile warnings on Apple Clang 12
+* [ARROW-10065](https://issues.apache.org/jira/browse/ARROW-10065) - [Rust] DRY downcasted Arrays
+* [ARROW-10066](https://issues.apache.org/jira/browse/ARROW-10066) - [C++] Make sure that default AWS region is respected
+* [ARROW-10068](https://issues.apache.org/jira/browse/ARROW-10068) - [C++] Add bundled external project for aws-sdk-cpp
+* [ARROW-10069](https://issues.apache.org/jira/browse/ARROW-10069) - [Java] Support running Java benchmarks from command line
+* [ARROW-10070](https://issues.apache.org/jira/browse/ARROW-10070) - [C++][Compute] Implement stdev aggregate kernel
+* [ARROW-10071](https://issues.apache.org/jira/browse/ARROW-10071) - [R] segfault with ArrowObject from previous session, or saved
+* [ARROW-10074](https://issues.apache.org/jira/browse/ARROW-10074) - [C++] Don't use string\_view.to\_string()
+* [ARROW-10075](https://issues.apache.org/jira/browse/ARROW-10075) - [C++] Don't use nonstd::nullopt this breaks out vendoring abstraction.
+* [ARROW-10076](https://issues.apache.org/jira/browse/ARROW-10076) - [C++] Use TemporaryDir for all tests that don't already use it.
+* [ARROW-10077](https://issues.apache.org/jira/browse/ARROW-10077) - [C++] Potential overflow in bit\_stream\_utils.h multiplication.
+* [ARROW-10083](https://issues.apache.org/jira/browse/ARROW-10083) - [C++] Improve Parquet fuzz seed corpus
+* [ARROW-10084](https://issues.apache.org/jira/browse/ARROW-10084) - [Rust] [DataFusion] Add length of large string array
+* [ARROW-10086](https://issues.apache.org/jira/browse/ARROW-10086) - [Rust] Migrate min\_large\_string -\> min\_string kernels
+* [ARROW-10090](https://issues.apache.org/jira/browse/ARROW-10090) - [C++][Compute] Improve mode kernel
+* [ARROW-10092](https://issues.apache.org/jira/browse/ARROW-10092) - [Dev][Go] Add grpc generated go files to rat exclusion list
+* [ARROW-10093](https://issues.apache.org/jira/browse/ARROW-10093) - [R] Add ability to opt-out of int64 -\> int demotion
+* [ARROW-10095](https://issues.apache.org/jira/browse/ARROW-10095) - [Rust] [Parquet] Update for IPC changes
+* [ARROW-10096](https://issues.apache.org/jira/browse/ARROW-10096) - [Rust] [DataFusion] Remove unused code
+* [ARROW-10099](https://issues.apache.org/jira/browse/ARROW-10099) - [C++][Dataset] Also allow integer partition fields to be dictionary encoded
+* [ARROW-10100](https://issues.apache.org/jira/browse/ARROW-10100) - [C++][Dataset] Ability to read/subset a ParquetFileFragment with given set of row group ids
+* [ARROW-10102](https://issues.apache.org/jira/browse/ARROW-10102) - [C++] Generalize BasicDecimal128::operator\*= for reuse in Decimal256
+* [ARROW-10103](https://issues.apache.org/jira/browse/ARROW-10103) - [Rust] Add a Contains kernel
+* [ARROW-10105](https://issues.apache.org/jira/browse/ARROW-10105) - [FlightRPC] Add client option to disable certificate validation with TLS
+* [ARROW-10120](https://issues.apache.org/jira/browse/ARROW-10120) - [C++][Parquet] Create reading benchmarks for 2-level nested data
+* [ARROW-10127](https://issues.apache.org/jira/browse/ARROW-10127) - [Format] Update specification to support 256-bit Decimal types
+* [ARROW-10129](https://issues.apache.org/jira/browse/ARROW-10129) - [Rust] Cargo build is rebuilding dependencies on arrow changes
+* [ARROW-10134](https://issues.apache.org/jira/browse/ARROW-10134) - [C++][Dataset] Add ParquetFileFragment::num\_row\_groups property
+* [ARROW-10139](https://issues.apache.org/jira/browse/ARROW-10139) - [C++] Add support for building arrow\_testing without building tests
+* [ARROW-10148](https://issues.apache.org/jira/browse/ARROW-10148) - [Rust] Add documentation to lib.rs
+* [ARROW-10151](https://issues.apache.org/jira/browse/ARROW-10151) - [Python] Add support MapArray to\_pandas conversion
+* [ARROW-10155](https://issues.apache.org/jira/browse/ARROW-10155) - [Rust] [DataFusion] Add documentation to lib.rs
+* [ARROW-10156](https://issues.apache.org/jira/browse/ARROW-10156) - [Rust] Auto-label PRs
+* [ARROW-10157](https://issues.apache.org/jira/browse/ARROW-10157) - [Rust] Add more documentation about take
+* [ARROW-10160](https://issues.apache.org/jira/browse/ARROW-10160) - [Rust] Improve documentation of DictionaryType
+* [ARROW-10161](https://issues.apache.org/jira/browse/ARROW-10161) - [Rust] [DataFusion] Simplify expression tests
+* [ARROW-10162](https://issues.apache.org/jira/browse/ARROW-10162) - [Rust] Support display of DictionaryArrays in pretty printing
+* [ARROW-10164](https://issues.apache.org/jira/browse/ARROW-10164) - [Rust] Add support for DictionaryArray types to cast kernels
+* [ARROW-10167](https://issues.apache.org/jira/browse/ARROW-10167) - [Rust] Support display of DictionaryArrays in sql.rs
+* [ARROW-10168](https://issues.apache.org/jira/browse/ARROW-10168) - [Rust] [Parquet] Extend arrow schema conversion to projected fields
+* [ARROW-10171](https://issues.apache.org/jira/browse/ARROW-10171) - [Rust] [DataFusion] Add \`ExecutionContext::from<ExecutionContextState\>\`
+* [ARROW-10190](https://issues.apache.org/jira/browse/ARROW-10190) - [Website] Add Jorge to list of committers
+* [ARROW-10191](https://issues.apache.org/jira/browse/ARROW-10191) - [Rust] [Parquet] Add roundtrip tests for single column batches
+* [ARROW-10196](https://issues.apache.org/jira/browse/ARROW-10196) - [C++] Add Future::DeferNotOk()
+* [ARROW-10199](https://issues.apache.org/jira/browse/ARROW-10199) - [Rust][Parquet] Release Parquet at crates.io to remove debug prints
+* [ARROW-10201](https://issues.apache.org/jira/browse/ARROW-10201) - [C++][CI] Disable S3 in arm64 job on Travis CI
+* [ARROW-10202](https://issues.apache.org/jira/browse/ARROW-10202) - [CI][Windows] Use sf.net mirror for MSYS2
+* [ARROW-10205](https://issues.apache.org/jira/browse/ARROW-10205) - [Java][FlightRPC] Add client option to disable server verification
+* [ARROW-10206](https://issues.apache.org/jira/browse/ARROW-10206) - [Python][C++][FlightRPC] Add client option to disable server validation
+* [ARROW-10215](https://issues.apache.org/jira/browse/ARROW-10215) - [Rust] [DataFusion] Rename "Source" typedef
+* [ARROW-10217](https://issues.apache.org/jira/browse/ARROW-10217) - [CI] Run fewer GitHub Actions jobs
+* [ARROW-10225](https://issues.apache.org/jira/browse/ARROW-10225) - [Rust] [Parquet] Fix null bitmap comparisons in roundtrip tests
+* [ARROW-10227](https://issues.apache.org/jira/browse/ARROW-10227) - [Ruby] Use a table size as the default for parquet chunk\_size
+* [ARROW-10229](https://issues.apache.org/jira/browse/ARROW-10229) - [C++][Parquet] Remove left over ARROW\_LOG statement.
+* [ARROW-10231](https://issues.apache.org/jira/browse/ARROW-10231) - [CI] Unable to download minio in arm32v7 docker image
+* [ARROW-10233](https://issues.apache.org/jira/browse/ARROW-10233) - [Rust] Make array\_value\_to\_string available in all Arrow builds
+* [ARROW-10235](https://issues.apache.org/jira/browse/ARROW-10235) - [Rust][DataFusion] Improve documentation for type coercion
+* [ARROW-10240](https://issues.apache.org/jira/browse/ARROW-10240) - [Rust] [Datafusion] Optionally load tpch data into memory before running benchmark query
+* [ARROW-10251](https://issues.apache.org/jira/browse/ARROW-10251) - [Rust] [DataFusion] MemTable::load() should load partitions in parallel
+* [ARROW-10252](https://issues.apache.org/jira/browse/ARROW-10252) - [Python] Add option to skip inclusion of Arrow headers in Python installation
+* [ARROW-10256](https://issues.apache.org/jira/browse/ARROW-10256) - [C++][Flight] Disable -Werror carefully
+* [ARROW-10257](https://issues.apache.org/jira/browse/ARROW-10257) - [R] Prepare news/docs for 2.0 release
+* [ARROW-10260](https://issues.apache.org/jira/browse/ARROW-10260) - [Python] Missing MapType to Pandas dtype
+* [ARROW-10265](https://issues.apache.org/jira/browse/ARROW-10265) - [CI] Use smaler build when cache doesn't exit on Travis CI
+* [ARROW-10266](https://issues.apache.org/jira/browse/ARROW-10266) - [CI][macOS] Ensure using Python 3.8 with Homebrew
+* [ARROW-10267](https://issues.apache.org/jira/browse/ARROW-10267) - [Python] Skip flight test if disable\_server\_verification feature is not available
+* [ARROW-10272](https://issues.apache.org/jira/browse/ARROW-10272) - [Packaging][Python] Pin newer multibuild version to avoid updating homebrew
+* [ARROW-10273](https://issues.apache.org/jira/browse/ARROW-10273) - [CI][Homebrew] Fix "brew audit" usage
+* [ARROW-10287](https://issues.apache.org/jira/browse/ARROW-10287) - [C++] Avoid std::random\_device whenever possible
+* [PARQUET-1845](https://issues.apache.org/jira/browse/PARQUET-1845) - [C++] Int96 memory images in test cases assume only little-endian
+* [PARQUET-1878](https://issues.apache.org/jira/browse/PARQUET-1878) - [C++] lz4 codec is not compatible with Hadoop Lz4Codec
+* [PARQUET-1904](https://issues.apache.org/jira/browse/PARQUET-1904) - [C++] Export file\_offset in RowGroupMetaData
+
+
+
 # Apache Arrow 1.0.0 (2020-07-20)
 
 ## Bug Fixes
diff --git a/LICENSE.txt b/LICENSE.txt
index e79841d212d..a37ca36c3f0 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -849,9 +849,9 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 
-The files in cpp/src/arrow/vendored/utf8cpp/ have the following license
+The files in cpp/src/arrow/vendored/utfcpp/ have the following license
 
-Copyright 2006 Nemanja Trifunovic
+Copyright 2006-2018 Nemanja Trifunovic
 
 Permission is hereby granted, free of charge, to any person or organization
 obtaining a copy of the software and accompanying documentation covered by
@@ -2223,3 +2223,11 @@ exception of some code pulled in from other repositories (such as
 public domain, released using the CC0 1.0 Universal dedication (*).
 
 (*) https://creativecommons.org/publicdomain/zero/1.0/legalcode
+
+--------------------------------------------------------------------------------
+
+The files in cpp/src/arrow/vendored/fast_float/ contain code from
+
+https://github.com/lemire/fast_float
+
+which is made available under the Apache License 2.0.
diff --git a/README.md b/README.md
index b1e96dccc6b..133018c72df 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,7 @@ Major components of the project include:
    a standard and efficient in-memory representation of various datatypes, plain or nested
  - [The Arrow IPC Format](https://github.com/apache/arrow/blob/master/docs/source/format/Columnar.rst#serialization-and-interprocess-communication-ipc):
    an efficient serialization of the Arrow format and associated metadata,
-   for communication between processes and heterogenous environments
+   for communication between processes and heterogeneous environments
  - [The Arrow Flight RPC protocol](https://github.com/apache/arrow/tree/master/format/Flight.proto):
    based on the Arrow IPC format, a building block for remote services exchanging
    Arrow data with application-defined semantics (for example a storage server or a database)
@@ -60,7 +60,7 @@ Arrow is an [Apache Software Foundation](https://www.apache.org) project. Learn
 
 ## What's in the Arrow libraries?
 
-The reference Arrow libraries contain a number of distinct software components:
+The reference Arrow libraries contain many distinct software components:
 
 - Columnar vector and table-like containers (similar to data frames) supporting
   flat or nested types
diff --git a/c_glib/arrow-glib/codec.cpp b/c_glib/arrow-glib/codec.cpp
index fdd61e70a17..33b3d1c9149 100644
--- a/c_glib/arrow-glib/codec.cpp
+++ b/c_glib/arrow-glib/codec.cpp
@@ -38,7 +38,7 @@ G_BEGIN_DECLS
  */
 
 typedef struct GArrowCodecPrivate_ {
-  arrow::util::Codec *codec;
+  std::shared_ptr<arrow::util::Codec> codec;
 } GArrowCodecPrivate;
 
 enum {
@@ -57,7 +57,7 @@ garrow_codec_finalize(GObject *object)
 {
   auto priv = GARROW_CODEC_GET_PRIVATE(object);
 
-  delete priv->codec;
+  priv->codec.~shared_ptr();
 
   G_OBJECT_CLASS(garrow_codec_parent_class)->finalize(object);
 }
@@ -72,7 +72,8 @@ garrow_codec_set_property(GObject *object,
 
   switch (prop_id) {
   case PROP_CODEC:
-    priv->codec = static_cast<arrow::util::Codec *>(g_value_get_pointer(value));
+    priv->codec =
+      *static_cast<std::shared_ptr<arrow::util::Codec> *>(g_value_get_pointer(value));
     break;
   default:
     G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
@@ -96,6 +97,8 @@ garrow_codec_get_property(GObject *object,
 static void
 garrow_codec_init(GArrowCodec *object)
 {
+  auto priv = GARROW_CODEC_GET_PRIVATE(object);
+  new(&priv->codec) std::shared_ptr<arrow::util::Codec>;
 }
 
 static void
@@ -111,7 +114,7 @@ garrow_codec_class_init(GArrowCodecClass *klass)
 
   spec = g_param_spec_pointer("codec",
                               "Codec",
-                              "The raw arrow::util::Codec *",
+                              "The raw std::shared_ptr<arrow::util::Codec> *",
                               static_cast<GParamFlags>(G_PARAM_WRITABLE |
                                                        G_PARAM_CONSTRUCT_ONLY));
   g_object_class_install_property(gobject_class, PROP_CODEC, spec);
@@ -133,7 +136,9 @@ garrow_codec_new(GArrowCompressionType type,
   auto arrow_type = garrow_compression_type_to_raw(type);
   auto arrow_codec = arrow::util::Codec::Create(arrow_type);
   if (garrow::check(error, arrow_codec, "[codec][new]")) {
-    return garrow_codec_new_raw(arrow_codec.ValueOrDie().release());
+    std::shared_ptr<arrow::util::Codec> arrow_codec_shared =
+      std::move(*arrow_codec);
+    return garrow_codec_new_raw(&arrow_codec_shared);
   } else {
     return NULL;
   }
@@ -151,7 +156,46 @@ const gchar *
 garrow_codec_get_name(GArrowCodec *codec)
 {
   auto arrow_codec = garrow_codec_get_raw(codec);
-  return arrow_codec->name();
+  if (!arrow_codec) {
+    return NULL;
+  }
+  return arrow_codec->name().c_str();
+}
+
+/**
+ * garrow_codec_get_compression_type:
+ * @codec: A #GArrowCodec.
+ *
+ * Returns: The compression type of the codec.
+ *
+ * Since: 2.0.0
+ */
+GArrowCompressionType
+garrow_codec_get_compression_type(GArrowCodec *codec)
+{
+  auto arrow_codec = garrow_codec_get_raw(codec);
+  if (!arrow_codec) {
+    return GARROW_COMPRESSION_TYPE_UNCOMPRESSED;
+  }
+  return garrow_compression_type_from_raw(arrow_codec->compression_type());
+}
+
+/**
+ * garrow_codec_get_compression_level:
+ * @codec: A #GArrowCodec.
+ *
+ * Returns: The compression level of the codec.
+ *
+ * Since: 2.0.0
+ */
+gint
+garrow_codec_get_compression_level(GArrowCodec *codec)
+{
+  auto arrow_codec = garrow_codec_get_raw(codec);
+  if (!arrow_codec) {
+    return arrow::util::Codec::UseDefaultCompressionLevel();
+  }
+  return arrow_codec->compression_level();
 }
 
 G_END_DECLS
@@ -207,7 +251,7 @@ garrow_compression_type_to_raw(GArrowCompressionType type)
 }
 
 GArrowCodec *
-garrow_codec_new_raw(arrow::util::Codec *arrow_codec)
+garrow_codec_new_raw(std::shared_ptr<arrow::util::Codec> *arrow_codec)
 {
   auto codec = GARROW_CODEC(g_object_new(GARROW_TYPE_CODEC,
                                          "codec", arrow_codec,
@@ -215,7 +259,7 @@ garrow_codec_new_raw(arrow::util::Codec *arrow_codec)
   return codec;
 }
 
-arrow::util::Codec *
+std::shared_ptr<arrow::util::Codec>
 garrow_codec_get_raw(GArrowCodec *codec)
 {
   auto priv = GARROW_CODEC_GET_PRIVATE(codec);
diff --git a/c_glib/arrow-glib/codec.h b/c_glib/arrow-glib/codec.h
index 5feab2b7d4d..6e177af9eed 100644
--- a/c_glib/arrow-glib/codec.h
+++ b/c_glib/arrow-glib/codec.h
@@ -20,6 +20,7 @@
 #pragma once
 
 #include <arrow-glib/gobject-type.h>
+#include <arrow-glib/version.h>
 
 G_BEGIN_DECLS
 
@@ -63,5 +64,11 @@ GArrowCodec *garrow_codec_new(GArrowCompressionType type,
                               GError **error);
 
 const gchar *garrow_codec_get_name(GArrowCodec *codec);
+GARROW_AVAILABLE_IN_2_0
+GArrowCompressionType
+garrow_codec_get_compression_type(GArrowCodec *codec);
+GARROW_AVAILABLE_IN_2_0
+gint
+garrow_codec_get_compression_level(GArrowCodec *codec);
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/codec.hpp b/c_glib/arrow-glib/codec.hpp
index 14c3ad77ccf..f4cfaba18a0 100644
--- a/c_glib/arrow-glib/codec.hpp
+++ b/c_glib/arrow-glib/codec.hpp
@@ -29,6 +29,6 @@ arrow::Compression::type
 garrow_compression_type_to_raw(GArrowCompressionType type);
 
 GArrowCodec *
-garrow_codec_new_raw(arrow::util::Codec *arrow_codec);
-arrow::util::Codec *
+garrow_codec_new_raw(std::shared_ptr<arrow::util::Codec> *arrow_codec);
+std::shared_ptr<arrow::util::Codec>
 garrow_codec_get_raw(GArrowCodec *codec);
diff --git a/c_glib/arrow-glib/composite-array.cpp b/c_glib/arrow-glib/composite-array.cpp
index 14dda373575..688c548bf2f 100644
--- a/c_glib/arrow-glib/composite-array.cpp
+++ b/c_glib/arrow-glib/composite-array.cpp
@@ -140,6 +140,53 @@ garrow_base_list_array_get_value(GArrowArray *array,
                               "parent", array,
                               NULL);
 };
+
+template <typename LIST_ARRAY_CLASS>
+GArrowArray *
+garrow_base_list_array_get_values(GArrowArray *array)
+{
+  auto arrow_array = garrow_array_get_raw(array);
+  auto arrow_list_array =
+    std::static_pointer_cast<LIST_ARRAY_CLASS>(arrow_array);
+  auto arrow_values = arrow_list_array->values();
+  return garrow_array_new_raw(&arrow_values,
+                              "array", &arrow_values,
+                              "parent", array,
+                              NULL);
+};
+
+template <typename LIST_ARRAY_CLASS>
+typename LIST_ARRAY_CLASS::offset_type
+garrow_base_list_array_get_value_offset(GArrowArray *array, gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(array);
+  auto arrow_list_array =
+    std::static_pointer_cast<LIST_ARRAY_CLASS>(arrow_array);
+  return arrow_list_array->value_offset(i);
+};
+
+template <typename LIST_ARRAY_CLASS>
+typename LIST_ARRAY_CLASS::offset_type
+garrow_base_list_array_get_value_length(GArrowArray *array, gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(array);
+  auto arrow_list_array =
+    std::static_pointer_cast<LIST_ARRAY_CLASS>(arrow_array);
+  return arrow_list_array->value_length(i);
+};
+
+template <typename LIST_ARRAY_CLASS>
+const typename LIST_ARRAY_CLASS::offset_type *
+garrow_base_list_array_get_value_offsets(GArrowArray *array, gint64 *n_offsets)
+{
+  auto arrow_array = garrow_array_get_raw(array);
+  *n_offsets = arrow_array->length() + 1;
+  auto arrow_list_array =
+    std::static_pointer_cast<LIST_ARRAY_CLASS>(arrow_array);
+  return arrow_list_array->raw_value_offsets();
+};
+
+
 G_BEGIN_DECLS
 
 static void
@@ -279,6 +326,70 @@ garrow_list_array_get_value(GArrowListArray *array,
     GARROW_ARRAY(array), i);
 }
 
+/**
+ * garrow_list_array_get_values:
+ * @array: A #GArrowListArray.
+ *
+ * Returns: (transfer full): The array containing the list's values.
+ *
+ * Since: 2.0.0
+ */
+GArrowArray *
+garrow_list_array_get_values(GArrowListArray *array)
+{
+  return garrow_base_list_array_get_values<arrow::ListArray>(
+    GARROW_ARRAY(array));
+}
+
+/**
+ * garrow_list_array_get_offset:
+ * @array: A #GArrowListArray.
+ * @i: The index of the offset of the target value.
+ *
+ * Returns: The target offset in the array containing the list's values.
+ *
+ * Since: 2.0.0
+ */
+gint32
+garrow_list_array_get_value_offset(GArrowListArray *array, gint64 i)
+{
+  return garrow_base_list_array_get_value_offset<arrow::ListArray>(
+    GARROW_ARRAY(array), i);
+}
+
+/**
+ * garrow_list_array_get_value_length:
+ * @array: A #GArrowListArray.
+ * @i: The index of the length of the target value.
+ *
+ * Returns: The target length in the array containing the list's values.
+ *
+ * Since: 2.0.0
+ */
+gint32
+garrow_list_array_get_value_length(GArrowListArray *array, gint64 i)
+{
+  return garrow_base_list_array_get_value_length<arrow::ListArray>(
+    GARROW_ARRAY(array), i);
+}
+
+/**
+ * garrow_list_array_get_value_offsets:
+ * @array: A #GArrowListArray.
+ * @n_offsets: The number of offsets to be returned.
+ *
+ * Returns: (array length=n_offsets): The target offsets in the array
+ * containing the list's values.
+ *
+ * Since: 2.0.0
+ */
+const gint32 *
+garrow_list_array_get_value_offsets(GArrowListArray *array, gint64 *n_offsets)
+{
+  return garrow_base_list_array_get_value_offsets<arrow::ListArray>(
+    GARROW_ARRAY(array), n_offsets);
+}
+
 
 typedef struct GArrowLargeListArrayPrivate_ {
   GArrowArray *raw_values;
@@ -434,6 +545,71 @@ garrow_large_list_array_get_value(GArrowLargeListArray *array,
     i);
 }
 
+/**
+ * garrow_large_list_array_get_values:
+ * @array: A #GArrowLargeListArray.
+ *
+ * Returns: (transfer full): The array containing the list's values.
+ *
+ * Since: 2.0.0
+ */
+GArrowArray *
+garrow_large_list_array_get_values(GArrowLargeListArray *array)
+{
+  return garrow_base_list_array_get_values<arrow::LargeListArray>(
+    GARROW_ARRAY(array));
+}
+
+/**
+ * garrow_large_list_array_get_value_offset:
+ * @array: A #GArrowLargeListArray.
+ * @i: The index of the offset of the target value.
+ *
+ * Returns: The target offset in the array containing the list's values.
+ *
+ * Since: 2.0.0
+ */
+gint64
+garrow_large_list_array_get_value_offset(GArrowLargeListArray *array, gint64 i)
+{
+  return garrow_base_list_array_get_value_offset<arrow::LargeListArray>(
+    GARROW_ARRAY(array), i);
+}
+
+/**
+ * garrow_large_list_array_get_length:
+ * @array: A #GArrowLargeListArray.
+ * @i: The index of the length of the target value.
+ *
+ * Returns: The target length in the array containing the list's values.
+ *
+ * Since: 2.0.0
+ */
+gint64
+garrow_large_list_array_get_value_length(GArrowLargeListArray *array, gint64 i)
+{
+  return garrow_base_list_array_get_value_length<arrow::LargeListArray>(
+    GARROW_ARRAY(array), i);
+}
+
+/**
+ * garrow_large_list_array_get_value_offsets:
+ * @array: A #GArrowLargeListArray.
+ * @n_offsets: The number of offsets to be returned.
+ *
+ * Returns: (array length=n_offsets): The target offsets in the array
+ * containing the list's values.
+ *
+ * Since: 2.0.0
+ */
+const gint64 *
+garrow_large_list_array_get_value_offsets(GArrowLargeListArray *array,
+                                          gint64 *n_offsets)
+{
+  return garrow_base_list_array_get_value_offsets<arrow::LargeListArray>(
+    GARROW_ARRAY(array), n_offsets);
+}
+
 
 typedef struct GArrowStructArrayPrivate_ {
   GPtrArray *fields;
diff --git a/c_glib/arrow-glib/composite-array.h b/c_glib/arrow-glib/composite-array.h
index dd0c72668f9..cfaeb4c768c 100644
--- a/c_glib/arrow-glib/composite-array.h
+++ b/c_glib/arrow-glib/composite-array.h
@@ -47,6 +47,18 @@ GArrowListArray *garrow_list_array_new(GArrowDataType *data_type,
 GArrowDataType *garrow_list_array_get_value_type(GArrowListArray *array);
 GArrowArray *garrow_list_array_get_value(GArrowListArray *array,
                                          gint64 i);
+GARROW_AVAILABLE_IN_2_0
+GArrowArray *garrow_list_array_get_values(GArrowListArray *array);
+GARROW_AVAILABLE_IN_2_0
+gint32 garrow_list_array_get_value_offset(GArrowListArray *array,
+                                          gint64 i);
+GARROW_AVAILABLE_IN_2_0
+gint32 garrow_list_array_get_value_length(GArrowListArray *array,
+                                          gint64 i);
+GARROW_AVAILABLE_IN_2_0
+const gint32 *
+garrow_list_array_get_value_offsets(GArrowListArray *array,
+                                    gint64 *n_offsets);
 
 
 #define GARROW_TYPE_LARGE_LIST_ARRAY (garrow_large_list_array_get_type())
@@ -73,6 +85,18 @@ GArrowDataType *garrow_large_list_array_get_value_type(GArrowLargeListArray *arr
 GARROW_AVAILABLE_IN_0_16
 GArrowArray *garrow_large_list_array_get_value(GArrowLargeListArray *array,
                                                gint64 i);
+GARROW_AVAILABLE_IN_2_0
+GArrowArray *garrow_large_list_array_get_values(GArrowLargeListArray *array);
+GARROW_AVAILABLE_IN_2_0
+gint64 garrow_large_list_array_get_value_offset(GArrowLargeListArray *array,
+                                                gint64 i);
+GARROW_AVAILABLE_IN_2_0
+gint64 garrow_large_list_array_get_value_length(GArrowLargeListArray *array,
+                                                gint64 i);
+GARROW_AVAILABLE_IN_2_0
+const gint64 *
+garrow_large_list_array_get_value_offsets(GArrowLargeListArray *array,
+                                          gint64 *n_offsets);
 
 
 #define GARROW_TYPE_STRUCT_ARRAY (garrow_struct_array_get_type())
diff --git a/c_glib/arrow-glib/compute.cpp b/c_glib/arrow-glib/compute.cpp
index 20d910e3250..777adee41a5 100644
--- a/c_glib/arrow-glib/compute.cpp
+++ b/c_glib/arrow-glib/compute.cpp
@@ -300,7 +300,7 @@ garrow_function_find(const gchar *name)
  * @error: (nullable): Return location for a #GError or %NULL.
  *
  * Returns: (nullable) (transfer full):
- *   A return value of the execution as #GArrowData on success, %NULL on error.
+ *   A return value of the execution as #GArrowDatum on success, %NULL on error.
  *
  * Since: 1.0.0
  */
diff --git a/c_glib/arrow-glib/input-stream.cpp b/c_glib/arrow-glib/input-stream.cpp
index 3751d41ad3a..84904b74265 100644
--- a/c_glib/arrow-glib/input-stream.cpp
+++ b/c_glib/arrow-glib/input-stream.cpp
@@ -1132,7 +1132,7 @@ garrow_compressed_input_stream_new(GArrowCodec *codec,
                                    GArrowInputStream *raw,
                                    GError **error)
 {
-  auto arrow_codec = garrow_codec_get_raw(codec);
+  auto arrow_codec = garrow_codec_get_raw(codec).get();
   auto arrow_raw = garrow_input_stream_get_raw(raw);
   auto arrow_stream =
     arrow::io::CompressedInputStream::Make(arrow_codec, arrow_raw);
diff --git a/c_glib/arrow-glib/ipc-options.cpp b/c_glib/arrow-glib/ipc-options.cpp
index 1cddd25bb6d..b9b2c414348 100644
--- a/c_glib/arrow-glib/ipc-options.cpp
+++ b/c_glib/arrow-glib/ipc-options.cpp
@@ -21,6 +21,7 @@
 #  include <config.h>
 #endif
 
+#include <arrow-glib/codec.hpp>
 #include <arrow-glib/enums.h>
 #include <arrow-glib/ipc-options.hpp>
 
@@ -242,6 +243,7 @@ garrow_read_options_set_included_fields(GArrowReadOptions *options,
 
 typedef struct GArrowWriteOptionsPrivate_ {
   arrow::ipc::IpcWriteOptions options;
+  GArrowCodec *codec;
 } GArrowWriteOptionsPrivate;
 
 enum {
@@ -249,8 +251,7 @@ enum {
   PROP_WRITE_OPTIONS_MAX_RECURSION_DEPTH,
   PROP_WRITE_OPTIONS_ALIGNMENT,
   PROP_WRITE_OPTIONS_WRITE_LEGACY_IPC_FORMAT,
-  PROP_WRITE_OPTIONS_COMPRESSION,
-  PROP_WRITE_OPTIONS_COMPRESSION_LEVEL,
+  PROP_WRITE_OPTIONS_CODEC,
   PROP_WRITE_OPTIONS_USE_THREADS,
 };
 
@@ -263,6 +264,19 @@ G_DEFINE_TYPE_WITH_PRIVATE(GArrowWriteOptions,
     garrow_write_options_get_instance_private(               \
       GARROW_WRITE_OPTIONS(obj)))
 
+static void
+garrow_write_options_dispose(GObject *object)
+{
+  auto priv = GARROW_WRITE_OPTIONS_GET_PRIVATE(object);
+
+  if (priv->codec) {
+    g_object_unref(priv->codec);
+    priv->codec = NULL;
+  }
+
+  G_OBJECT_CLASS(garrow_write_options_parent_class)->dispose(object);
+}
+
 static void
 garrow_write_options_finalize(GObject *object)
 {
@@ -294,12 +308,12 @@ garrow_write_options_set_property(GObject *object,
   case PROP_WRITE_OPTIONS_WRITE_LEGACY_IPC_FORMAT:
     priv->options.write_legacy_ipc_format = g_value_get_boolean(value);
     break;
-  case PROP_WRITE_OPTIONS_COMPRESSION:
-    priv->options.compression =
-      static_cast<arrow::Compression::type>(g_value_get_enum(value));
-    break;
-  case PROP_WRITE_OPTIONS_COMPRESSION_LEVEL:
-    priv->options.compression_level = g_value_get_int(value);
+  case PROP_WRITE_OPTIONS_CODEC:
+    if (priv->codec) {
+      g_object_unref(priv->codec);
+    }
+    priv->codec = GARROW_CODEC(g_value_dup_object(value));
+    priv->options.codec = garrow_codec_get_raw(priv->codec);
     break;
   case PROP_WRITE_OPTIONS_USE_THREADS:
     priv->options.use_threads = g_value_get_boolean(value);
@@ -331,11 +345,8 @@ garrow_write_options_get_property(GObject *object,
   case PROP_WRITE_OPTIONS_WRITE_LEGACY_IPC_FORMAT:
     g_value_set_boolean(value, priv->options.write_legacy_ipc_format);
     break;
-  case PROP_WRITE_OPTIONS_COMPRESSION:
-    g_value_set_enum(value, priv->options.compression);
-    break;
-  case PROP_WRITE_OPTIONS_COMPRESSION_LEVEL:
-    g_value_set_int(value, priv->options.compression_level);
+  case PROP_WRITE_OPTIONS_CODEC:
+    g_value_set_object(value, priv->codec);
     break;
   case PROP_WRITE_OPTIONS_USE_THREADS:
     g_value_set_boolean(value, priv->options.use_threads);
@@ -352,6 +363,11 @@ garrow_write_options_init(GArrowWriteOptions *object)
   auto priv = GARROW_WRITE_OPTIONS_GET_PRIVATE(object);
   new(&priv->options) arrow::ipc::IpcWriteOptions;
   priv->options = arrow::ipc::IpcWriteOptions::Defaults();
+  if (priv->options.codec) {
+    priv->codec = garrow_codec_new_raw(&(priv->options.codec));
+  } else {
+    priv->codec = NULL;
+  }
 }
 
 static void
@@ -359,6 +375,7 @@ garrow_write_options_class_init(GArrowWriteOptionsClass *klass)
 {
   auto gobject_class = G_OBJECT_CLASS(klass);
 
+  gobject_class->dispose      = garrow_write_options_dispose;
   gobject_class->finalize     = garrow_write_options_finalize;
   gobject_class->set_property = garrow_write_options_set_property;
   gobject_class->get_property = garrow_write_options_get_property;
@@ -441,42 +458,24 @@ garrow_write_options_class_init(GArrowWriteOptionsClass *klass)
                                   spec);
 
   /**
-   * GArrowWriteOptions:compression:
+   * GArrowWriteOptions:codec:
    *
    * Codec to use for compressing and decompressing record batch body
    * buffers. This is not part of the Arrow IPC protocol and only for
-   * internal use (e.g. Feather files). May only be LZ4_FRAME and
-   * ZSTD.
+   * internal use (e.g. Feather files).
    *
-   * Since: 1.0.0
-   */
-  spec = g_param_spec_enum("compression",
-                           "Compression",
-                           "Codec to use for "
-                           "compressing record batch body buffers.",
-                           GARROW_TYPE_COMPRESSION_TYPE,
-                           options.compression,
-                           static_cast<GParamFlags>(G_PARAM_READWRITE));
-  g_object_class_install_property(gobject_class,
-                                  PROP_WRITE_OPTIONS_COMPRESSION,
-                                  spec);
-
-  /**
-   * GArrowWriteOptions:compression-level:
-   *
-   * The level for compression.
+   * May only be UNCOMPRESSED, LZ4_FRAME and ZSTD.
    *
-   * Since: 1.0.0
+   * Since: 2.0.0
    */
-  spec = g_param_spec_int("compression-level",
-                          "Compression level",
-                          "The level for compression",
-                          G_MININT,
-                          G_MAXINT,
-                          options.compression_level,
-                          static_cast<GParamFlags>(G_PARAM_READWRITE));
+  spec = g_param_spec_object("codec",
+                             "Codec",
+                             "Codec to use for "
+                             "compressing record batch body buffers.",
+                             GARROW_TYPE_CODEC,
+                             static_cast<GParamFlags>(G_PARAM_READWRITE));
   g_object_class_install_property(gobject_class,
-                                  PROP_WRITE_OPTIONS_COMPRESSION_LEVEL,
+                                  PROP_WRITE_OPTIONS_CODEC,
                                   spec);
 
   /**
diff --git a/c_glib/arrow-glib/output-stream.cpp b/c_glib/arrow-glib/output-stream.cpp
index 2c3ccafdb13..1619bac45d4 100644
--- a/c_glib/arrow-glib/output-stream.cpp
+++ b/c_glib/arrow-glib/output-stream.cpp
@@ -688,7 +688,7 @@ garrow_compressed_output_stream_new(GArrowCodec *codec,
                                     GArrowOutputStream *raw,
                                     GError **error)
 {
-  auto arrow_codec = garrow_codec_get_raw(codec);
+  auto arrow_codec = garrow_codec_get_raw(codec).get();
   auto arrow_raw = garrow_output_stream_get_raw(raw);
   auto arrow_stream = arrow::io::CompressedOutputStream::Make(arrow_codec,
                                                               arrow_raw);
diff --git a/c_glib/arrow-glib/reader.cpp b/c_glib/arrow-glib/reader.cpp
index 51ad3ac444d..c3082271ca5 100644
--- a/c_glib/arrow-glib/reader.cpp
+++ b/c_glib/arrow-glib/reader.cpp
@@ -332,7 +332,7 @@ garrow_record_batch_file_reader_finalize(GObject *object)
 {
   auto priv = GARROW_RECORD_BATCH_FILE_READER_GET_PRIVATE(object);
 
-  priv->record_batch_file_reader = nullptr;
+  priv->record_batch_file_reader.~shared_ptr();
 
   G_OBJECT_CLASS(garrow_record_batch_file_reader_parent_class)->finalize(object);
 }
@@ -372,6 +372,9 @@ garrow_record_batch_file_reader_get_property(GObject *object,
 static void
 garrow_record_batch_file_reader_init(GArrowRecordBatchFileReader *object)
 {
+  auto priv = GARROW_RECORD_BATCH_FILE_READER_GET_PRIVATE(object);
+  new(&priv->record_batch_file_reader)
+    std::shared_ptr<arrow::ipc::RecordBatchFileReader>;
 }
 
 static void
@@ -1181,7 +1184,7 @@ garrow_csv_read_options_add_schema(GArrowCSVReadOptions *options,
 {
   auto priv = GARROW_CSV_READ_OPTIONS_GET_PRIVATE(options);
   auto arrow_schema = garrow_schema_get_raw(schema);
-  for (const auto field : arrow_schema->fields()) {
+  for (const auto &field : arrow_schema->fields()) {
     priv->convert_options.column_types[field->name()] = field->type();
   }
 }
@@ -1203,7 +1206,7 @@ garrow_csv_read_options_get_column_types(GArrowCSVReadOptions *options)
                                             g_str_equal,
                                             g_free,
                                             g_object_unref);
-  for (const auto iter : priv->convert_options.column_types) {
+  for (const auto &iter : priv->convert_options.column_types) {
     auto arrow_name = iter.first;
     auto arrow_data_type = iter.second;
     g_hash_table_insert(types,
diff --git a/c_glib/arrow-glib/writer.cpp b/c_glib/arrow-glib/writer.cpp
index 074c83af120..82d18e58dbd 100644
--- a/c_glib/arrow-glib/writer.cpp
+++ b/c_glib/arrow-glib/writer.cpp
@@ -235,10 +235,10 @@ garrow_record_batch_stream_writer_new(GArrowOutputStream *sink,
                                       GArrowSchema *schema,
                                       GError **error)
 {
-  auto arrow_sink = garrow_output_stream_get_raw(sink).get();
+  auto arrow_sink = garrow_output_stream_get_raw(sink);
   auto arrow_schema = garrow_schema_get_raw(schema);
   auto arrow_writer_result =
-    arrow::ipc::NewStreamWriter(arrow_sink, arrow_schema);
+    arrow::ipc::MakeStreamWriter(arrow_sink, arrow_schema);
   if (garrow::check(error,
                     arrow_writer_result,
                     "[record-batch-stream-writer][open]")) {
@@ -280,11 +280,11 @@ garrow_record_batch_file_writer_new(GArrowOutputStream *sink,
                        GArrowSchema *schema,
                        GError **error)
 {
-  auto arrow_sink = garrow_output_stream_get_raw(sink).get();
+  auto arrow_sink = garrow_output_stream_get_raw(sink);
   auto arrow_schema = garrow_schema_get_raw(schema);
   std::shared_ptr<arrow::ipc::RecordBatchWriter> arrow_writer;
   auto arrow_writer_result =
-    arrow::ipc::NewFileWriter(arrow_sink, arrow_schema);
+    arrow::ipc::MakeFileWriter(arrow_sink, arrow_schema);
   if (garrow::check(error,
                     arrow_writer_result,
                     "[record-batch-file-writer][open]")) {
diff --git a/c_glib/configure.ac b/c_glib/configure.ac
index 5919ec95d7c..dc0624e1630 100644
--- a/c_glib/configure.ac
+++ b/c_glib/configure.ac
@@ -17,7 +17,7 @@
 
 AC_PREREQ(2.65)
 
-m4_define([arrow_glib_version], 2.0.0-SNAPSHOT)
+m4_define([arrow_glib_version], 3.0.0-SNAPSHOT)
 AC_INIT([arrow-glib],
         arrow_glib_version,
         [https://issues.apache.org/jira/browse/ARROW],
diff --git a/c_glib/doc/arrow-glib/arrow-glib-docs.xml b/c_glib/doc/arrow-glib/arrow-glib-docs.xml
index 57a53c2eec7..72a01f50e4f 100644
--- a/c_glib/doc/arrow-glib/arrow-glib-docs.xml
+++ b/c_glib/doc/arrow-glib/arrow-glib-docs.xml
@@ -179,6 +179,10 @@
     <title>Index of deprecated API</title>
     <xi:include href="xml/api-index-deprecated.xml"><xi:fallback /></xi:include>
   </index>
+  <index id="api-index-2-0-0" role="2.0.0">
+    <title>Index of new symbols in 2.0.0</title>
+    <xi:include href="xml/api-index-2.0.0.xml"><xi:fallback /></xi:include>
+  </index>
   <index id="api-index-1-0-0" role="1.0.0">
     <title>Index of new symbols in 1.0.0</title>
     <xi:include href="xml/api-index-1.0.0.xml"><xi:fallback /></xi:include>
diff --git a/c_glib/meson.build b/c_glib/meson.build
index 7b8b21c7b20..e6a4a9b6671 100644
--- a/c_glib/meson.build
+++ b/c_glib/meson.build
@@ -23,7 +23,7 @@ project('arrow-glib', 'c', 'cpp',
           'cpp_std=c++11',
         ])
 
-version = '2.0.0-SNAPSHOT'
+version = '3.0.0-SNAPSHOT'
 if version.endswith('-SNAPSHOT')
   version_numbers = version.split('-')[0].split('.')
   version_tag = version.split('-')[1]
diff --git a/c_glib/test/dataset/test-scan-options.rb b/c_glib/test/dataset/test-scan-options.rb
index c5657fe7e17..1f5b77f2e9f 100644
--- a/c_glib/test/dataset/test-scan-options.rb
+++ b/c_glib/test/dataset/test-scan-options.rb
@@ -28,7 +28,7 @@ def test_schema
   end
 
   def test_batch_size
-    assert_equal(1<<15,
+    assert_equal(1<<20,
                  @scan_options.batch_size)
     @scan_options.batch_size = 42
     assert_equal(42,
diff --git a/c_glib/test/test-codec.rb b/c_glib/test/test-codec.rb
index 6617815df9b..a32ec4dc757 100644
--- a/c_glib/test/test-codec.rb
+++ b/c_glib/test/test-codec.rb
@@ -20,4 +20,14 @@ def test_name
     codec = Arrow::Codec.new(:gzip)
     assert_equal("gzip", codec.name)
   end
+
+  def test_compression_type
+    codec = Arrow::Codec.new(:gzip)
+    assert_equal(Arrow::CompressionType::GZIP, codec.compression_type)
+  end
+
+  def test_compression_level
+    codec = Arrow::Codec.new(:gzip)
+    assert_equal(9, codec.compression_level)
+  end
 end
diff --git a/c_glib/test/test-decimal128.rb b/c_glib/test/test-decimal128.rb
index 0e4bc8264d5..98789d3812e 100644
--- a/c_glib/test/test-decimal128.rb
+++ b/c_glib/test/test-decimal128.rb
@@ -214,7 +214,7 @@ def test_rescale_fail
     decimal = Arrow::Decimal128.new(10)
     message =
       "[decimal128][rescale]: Invalid: " +
-      "Rescaling decimal value would cause data loss"
+      "Rescaling Decimal128 value would cause data loss"
     assert_raise(Arrow::Error::Invalid.new(message)) do
       decimal.rescale(1, -1)
     end
diff --git a/c_glib/test/test-large-list-array.rb b/c_glib/test/test-large-list-array.rb
index 9840989ab89..2f7efab5a07 100644
--- a/c_glib/test/test-large-list-array.rb
+++ b/c_glib/test/test-large-list-array.rb
@@ -36,21 +36,11 @@ def test_new
   end
 
   def test_value
-    field = Arrow::Field.new("value", Arrow::Int64DataType.new)
-    data_type = Arrow::LargeListDataType.new(field)
-    builder = Arrow::LargeListArrayBuilder.new(data_type)
-    value_builder = builder.value_builder
-
-    builder.append_value
-    value_builder.append_value(-29)
-    value_builder.append_value(29)
-
-    builder.append_value
-    value_builder.append_value(-1)
-    value_builder.append_value(0)
-    value_builder.append_value(1)
-
-    array = builder.finish
+    array = build_large_list_array(Arrow::Int8DataType.new,
+                                   [
+                                     [-29, 29],
+                                     [-1, 0, 1],
+                                   ])
     value = array.get_value(1)
     assert_equal([-1, 0, 1],
                  value.length.times.collect {|i| value.get_value(i)})
@@ -63,4 +53,46 @@ def test_value_type
     array = builder.finish
     assert_equal(Arrow::Int64DataType.new, array.value_type)
   end
+
+
+  def test_values
+    array = build_large_list_array(Arrow::Int8DataType.new,
+                                   [
+                                     [-29, 29],
+                                     [-1, 0, 1],
+                                   ])
+    values = array.values
+    assert_equal([-29, 29, -1, 0, 1],
+                 values.length.times.collect {|i| values.get_value(i)})
+  end
+
+  def test_value_offset
+    array = build_large_list_array(Arrow::Int8DataType.new,
+                                   [
+                                     [-29, 29],
+                                     [-1, 0, 1],
+                                   ])
+    assert_equal([0, 2],
+                 array.length.times.collect {|i| array.get_value_offset(i)})
+  end
+
+  def test_value_length
+    array = build_large_list_array(Arrow::Int8DataType.new,
+                                   [
+                                     [-29, 29],
+                                     [-1, 0, 1],
+                                   ])
+    assert_equal([2, 3],
+                 array.length.times.collect {|i| array.get_value_length(i)})
+  end
+
+  def test_value_offsets
+    array = build_large_list_array(Arrow::Int8DataType.new,
+                             [
+                               [-29, 29],
+                               [-1, 0, 1],
+                             ])
+    assert_equal([0, 2, 5],
+                 array.value_offsets)
+  end
 end
diff --git a/c_glib/test/test-list-array.rb b/c_glib/test/test-list-array.rb
index eea16ccbbb3..f94b28dd1cd 100644
--- a/c_glib/test/test-list-array.rb
+++ b/c_glib/test/test-list-array.rb
@@ -36,21 +36,11 @@ def test_new
   end
 
   def test_value
-    field = Arrow::Field.new("value", Arrow::Int8DataType.new)
-    data_type = Arrow::ListDataType.new(field)
-    builder = Arrow::ListArrayBuilder.new(data_type)
-    value_builder = builder.value_builder
-
-    builder.append_value
-    value_builder.append_value(-29)
-    value_builder.append_value(29)
-
-    builder.append_value
-    value_builder.append_value(-1)
-    value_builder.append_value(0)
-    value_builder.append_value(1)
-
-    array = builder.finish
+    array = build_list_array(Arrow::Int8DataType.new,
+                             [
+                               [-29, 29],
+                               [-1, 0, 1],
+                             ])
     value = array.get_value(1)
     assert_equal([-1, 0, 1],
                  value.length.times.collect {|i| value.get_value(i)})
@@ -63,4 +53,45 @@ def test_value_type
     array = builder.finish
     assert_equal(Arrow::Int8DataType.new, array.value_type)
   end
+
+  def test_values
+    array = build_list_array(Arrow::Int8DataType.new,
+                             [
+                               [-29, 29],
+                               [-1, 0, 1],
+                             ])
+    values = array.values
+    assert_equal([-29, 29, -1, 0, 1],
+                 values.length.times.collect {|i| values.get_value(i)})
+  end
+
+  def test_value_offset
+    array = build_list_array(Arrow::Int8DataType.new,
+                             [
+                               [-29, 29],
+                               [-1, 0, 1],
+                             ])
+    assert_equal([0, 2],
+                 array.length.times.collect {|i| array.get_value_offset(i)})
+  end
+
+  def test_value_length
+    array = build_list_array(Arrow::Int8DataType.new,
+                             [
+                               [-29, 29],
+                               [-1, 0, 1],
+                             ])
+    assert_equal([2, 3],
+                 array.length.times.collect {|i| array.get_value_length(i)})
+  end
+
+  def test_value_offsets
+    array = build_list_array(Arrow::Int8DataType.new,
+                             [
+                               [-29, 29],
+                               [-1, 0, 1],
+                             ])
+    assert_equal([0, 2, 5],
+                 array.value_offsets)
+  end
 end
diff --git a/c_glib/test/test-write-options.rb b/c_glib/test/test-write-options.rb
index d30b78b9cdb..c528ce673d4 100644
--- a/c_glib/test/test-write-options.rb
+++ b/c_glib/test/test-write-options.rb
@@ -73,27 +73,15 @@ def test_accessor
     end
   end
 
-  sub_test_case("compression") do
+  sub_test_case("codec") do
     def test_default
-      assert_equal(Arrow::CompressionType::UNCOMPRESSED,
-                   @options.compression)
+      assert_nil(@options.codec)
     end
 
     def test_accessor
-      @options.compression = :zstd
-      assert_equal(Arrow::CompressionType::ZSTD,
-                   @options.compression)
-    end
-  end
-
-  sub_test_case("compression-level") do
-    def test_default
-      assert_equal(-(2 ** 31), @options.compression_level)
-    end
-
-    def test_accessor
-      @options.compression_level = 8
-      assert_equal(8, @options.compression_level)
+      @options.codec = Arrow::Codec.new(:zstd)
+      assert_equal("zstd",
+                   @options.codec.name)
     end
   end
 
diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat
index a2ce0765617..6b930939660 100644
--- a/ci/appveyor-cpp-build.bat
+++ b/ci/appveyor-cpp-build.bat
@@ -89,7 +89,7 @@ pushd cpp\build
 @rem and enable runtime assertions.
 
 cmake -G "%GENERATOR%" %CMAKE_ARGS% ^
-      -DARROW_BOOST_USE_SHARED=OFF ^
+      -DARROW_BOOST_USE_SHARED=ON ^
       -DARROW_BUILD_EXAMPLES=ON ^
       -DARROW_BUILD_STATIC=OFF ^
       -DARROW_BUILD_TESTS=ON ^
diff --git a/ci/appveyor-cpp-setup.bat b/ci/appveyor-cpp-setup.bat
index 14bc0fd7a21..616232d202c 100644
--- a/ci/appveyor-cpp-setup.bat
+++ b/ci/appveyor-cpp-setup.bat
@@ -62,10 +62,10 @@ if "%JOB%" NEQ "Build_Debug" (
     --file=ci\conda_env_python.yml ^
     %CONDA_PACKAGES%  ^
     "cmake=3.17" ^
-    "boost-cpp<1.70" ^
     "ninja" ^
     "nomkl" ^
     "pandas" ^
+    "fsspec" ^
     "python=%PYTHON%" ^
     || exit /B
 )
diff --git a/ci/conda_env_archery.yml b/ci/conda_env_archery.yml
index 3eb8003e274..81c314eb4b2 100644
--- a/ci/conda_env_archery.yml
+++ b/ci/conda_env_archery.yml
@@ -16,5 +16,7 @@
 # under the License.
 
 click
+gitpython
 pygithub
 ruamel.yaml
+semver
diff --git a/ci/conda_env_cpp.yml b/ci/conda_env_cpp.yml
index a0c2e99aca7..90cef3ea2d1 100644
--- a/ci/conda_env_cpp.yml
+++ b/ci/conda_env_cpp.yml
@@ -25,7 +25,7 @@ cmake
 gflags
 glog
 gmock>=1.8.1
-grpc-cpp>=1.21.4
+grpc-cpp>=1.27.3
 gtest=1.8.1
 libprotobuf
 libutf8proc
diff --git a/ci/conda_env_gandiva.yml b/ci/conda_env_gandiva.yml
index fa15d77df83..5056456fc66 100644
--- a/ci/conda_env_gandiva.yml
+++ b/ci/conda_env_gandiva.yml
@@ -15,6 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-clangdev=10
-llvmdev=10
+clangdev=11
+llvmdev=11
 re2
diff --git a/ci/conda_env_python.yml b/ci/conda_env_python.yml
index f6c89923870..f2f46c84436 100644
--- a/ci/conda_env_python.yml
+++ b/ci/conda_env_python.yml
@@ -16,6 +16,7 @@
 # under the License.
 
 # don't add pandas here, because it is not a mandatory test dependency
+boto3  # not a direct dependency of s3fs, but needed for our s3fs fixture
 cffi
 cython
 cloudpickle
@@ -26,5 +27,6 @@ pytest
 pytest-faulthandler
 pytest-lazy-fixture
 pytz
+s3fs>=0.4
 setuptools
 setuptools_scm
diff --git a/ci/conda_env_r.yml b/ci/conda_env_r.yml
index b9a13fdbe33..03d5f3b625c 100644
--- a/ci/conda_env_r.yml
+++ b/ci/conda_env_r.yml
@@ -21,7 +21,7 @@ r-bit64
 r-dplyr
 r-purrr
 r-r6
-r-rcpp >=1.0.1
+r-cpp11
 r-rlang
 r-tidyselect
 r-vctrs
diff --git a/ci/conda_env_sphinx.yml b/ci/conda_env_sphinx.yml
index 318ef75e46c..8654d231065 100644
--- a/ci/conda_env_sphinx.yml
+++ b/ci/conda_env_sphinx.yml
@@ -19,6 +19,6 @@
 breathe
 doxygen
 ipython
-# Pinned per ARROW-8340
-sphinx=2.4.4
+# Pinned per ARROW-9693
+sphinx=3.1.2
 sphinx_rtd_theme
diff --git a/ci/docker/conda-integration.dockerfile b/ci/docker/conda-integration.dockerfile
index 088d93e3921..e6e5ac859dd 100644
--- a/ci/docker/conda-integration.dockerfile
+++ b/ci/docker/conda-integration.dockerfile
@@ -21,7 +21,7 @@ FROM ${repo}:${arch}-conda-cpp
 
 ARG arch=amd64
 ARG maven=3.5
-ARG node=11
+ARG node=14
 ARG jdk=8
 ARG go=1.12
 
diff --git a/ci/docker/conda-python-kartothek.dockerfile b/ci/docker/conda-python-kartothek.dockerfile
index aa013fe5f3d..d523161822c 100644
--- a/ci/docker/conda-python-kartothek.dockerfile
+++ b/ci/docker/conda-python-kartothek.dockerfile
@@ -22,11 +22,17 @@ FROM ${repo}:${arch}-conda-python-${python}
 
 # install kartothek dependencies from conda-forge
 RUN conda install -c conda-forge -q \
+        attrs \
+        click \
+        cloudpickle \
         dask \
         decorator \
+        freezegun \
         msgpack-python \
+        prompt-toolkit \
         pytest-mock \
         pytest-xdist \
+        pyyaml \
         simplejson \
         simplekv \
         storefact \
diff --git a/ci/docker/conda-python-spark.dockerfile b/ci/docker/conda-python-spark.dockerfile
index a20f1ff3521..d3f0a224582 100644
--- a/ci/docker/conda-python-spark.dockerfile
+++ b/ci/docker/conda-python-spark.dockerfile
@@ -36,10 +36,6 @@ ARG spark=master
 COPY ci/scripts/install_spark.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_spark.sh ${spark} /spark
 
-# patch spark to build with current Arrow Java
-COPY ci/etc/integration_spark_ARROW-9438.patch /arrow/ci/etc/
-RUN patch -d /spark -p1 -i /arrow/ci/etc/integration_spark_ARROW-9438.patch
-
 # build cpp with tests
 ENV CC=gcc \
     CXX=g++ \
diff --git a/ci/docker/conda-r.dockerfile b/ci/docker/conda-r.dockerfile
deleted file mode 100644
index 79b6ebc3b2d..00000000000
--- a/ci/docker/conda-r.dockerfile
+++ /dev/null
@@ -1,53 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-ARG repo
-ARG arch
-FROM ${repo}:${arch}-conda-cpp
-
-# Need locales so we can set UTF-8
-RUN apt-get update -y && \
-    apt-get install -y locales && \
-    locale-gen en_US.UTF-8 && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-# install R specific packages
-ARG r=3.6.1
-COPY ci/conda_env_r.yml /arrow/ci/
-RUN conda install -q \
-        --file arrow/ci/conda_env_r.yml \
-        r-base=$r \
-        nomkl && \
-    conda clean --all
-
-# Ensure parallel compilation of of C/C++ code
-RUN echo "MAKEFLAGS=-j$(R -s -e 'cat(parallel::detectCores())')" >> $CONDA_PREFIX/lib/R/etc/Makeconf
-
-ENV ARROW_BUILD_STATIC=OFF \
-    ARROW_BUILD_TESTS=OFF \
-    ARROW_BUILD_UTILITIES=OFF \
-    ARROW_DEPENDENCY_SOURCE=SYSTEM \
-    ARROW_FLIGHT=OFF \
-    ARROW_GANDIVA=OFF \
-    ARROW_NO_DEPRECATED_API=ON \
-    ARROW_ORC=OFF \
-    ARROW_PARQUET=ON \
-    ARROW_PLASMA=OFF \
-    ARROW_USE_CCACHE=ON \
-    ARROW_USE_GLOG=OFF \
-    LC_ALL=en_US.UTF-8
diff --git a/ci/docker/debian-10-cpp.dockerfile b/ci/docker/debian-10-cpp.dockerfile
index f86c009b57b..74143dcbfa4 100644
--- a/ci/docker/debian-10-cpp.dockerfile
+++ b/ci/docker/debian-10-cpp.dockerfile
@@ -17,6 +17,7 @@
 
 ARG arch=amd64
 FROM ${arch}/debian:10
+ARG arch
 
 ENV DEBIAN_FRONTEND noninteractive
 
@@ -26,7 +27,7 @@ RUN \
 
 ARG llvm
 RUN apt-get update -y -q && \
-     apt-get install -y -q --no-install-recommends \
+    apt-get install -y -q --no-install-recommends \
         apt-transport-https \
         ca-certificates \
         gnupg \
@@ -49,6 +50,7 @@ RUN apt-get update -y -q && \
         libbrotli-dev \
         libbz2-dev \
         libc-ares-dev \
+        libcurl4-openssl-dev \
         libgflags-dev \
         libgmock-dev \
         libgoogle-glog-dev \
@@ -71,6 +73,10 @@ RUN apt-get update -y -q && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+COPY ci/scripts/install_minio.sh \
+     /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
+
 ENV ARROW_BUILD_TESTS=ON \
     ARROW_DEPENDENCY_SOURCE=SYSTEM \
     ARROW_DATASET=ON \
@@ -80,6 +86,7 @@ ENV ARROW_BUILD_TESTS=ON \
     ARROW_ORC=ON \
     ARROW_PARQUET=ON \
     ARROW_PLASMA=ON \
+    ARROW_S3=ON \
     ARROW_USE_CCACHE=ON \
     ARROW_WITH_BROTLI=ON \
     ARROW_WITH_BZ2=ON \
@@ -87,6 +94,7 @@ ENV ARROW_BUILD_TESTS=ON \
     ARROW_WITH_SNAPPY=ON \
     ARROW_WITH_ZLIB=ON \
     ARROW_WITH_ZSTD=ON \
+    AWSSDK_SOURCE=BUNDLED \
     cares_SOURCE=BUNDLED \
     CC=gcc \
     CXX=g++ \
diff --git a/ci/docker/debian-10-js.dockerfile b/ci/docker/debian-10-js.dockerfile
index b43c45abdf4..5bb31f2e32e 100644
--- a/ci/docker/debian-10-js.dockerfile
+++ b/ci/docker/debian-10-js.dockerfile
@@ -16,7 +16,7 @@
 # under the License.
 
 ARG arch=amd64
-ARG node=11
+ARG node=14
 FROM ${arch}/node:${node}
 
 ENV NODE_NO_WARNINGS=1
diff --git a/ci/docker/debian-10-rust.dockerfile b/ci/docker/debian-10-rust.dockerfile
index b23b03c9a1c..9c9c9b51048 100644
--- a/ci/docker/debian-10-rust.dockerfile
+++ b/ci/docker/debian-10-rust.dockerfile
@@ -58,14 +58,18 @@ RUN mkdir \
         /arrow/rust/benchmarks/src \
         /arrow/rust/datafusion/src \
         /arrow/rust/integration-testing/src  \
-        /arrow/rust/parquet/src && \
+        /arrow/rust/parquet/src \
+        /arrow/rust/parquet_derive/src \
+        /arrow/rust/parquet_derive_test/src && \
     touch \
         /arrow/rust/arrow-flight/src/lib.rs \
         /arrow/rust/arrow/src/lib.rs \
         /arrow/rust/benchmarks/src/lib.rs \
         /arrow/rust/datafusion/src/lib.rs \
         /arrow/rust/integration-testing/src/lib.rs  \
-        /arrow/rust/parquet/src/lib.rs
+        /arrow/rust/parquet/src/lib.rs  \
+        /arrow/rust/parquet_derive/src/lib.rs  \
+        /arrow/rust/parquet_derive_test/src/lib.rs
 
 # Compile dependencies for the whole workspace
 RUN cd /arrow/rust && cargo build --workspace --lib --all-features
diff --git a/ci/docker/fedora-32-cpp.dockerfile b/ci/docker/fedora-33-cpp.dockerfile
similarity index 84%
rename from ci/docker/fedora-32-cpp.dockerfile
rename to ci/docker/fedora-33-cpp.dockerfile
index 535f8b4b761..9dde6999510 100644
--- a/ci/docker/fedora-32-cpp.dockerfile
+++ b/ci/docker/fedora-33-cpp.dockerfile
@@ -16,11 +16,12 @@
 # under the License.
 
 ARG arch
-FROM ${arch}/fedora:32
+FROM ${arch}/fedora:33
+ARG arch
 
 # install dependencies
 RUN dnf update -y && \
-	dnf install -y \
+    dnf install -y \
         autoconf \
         boost-devel \
         brotli-devel \
@@ -29,18 +30,20 @@ RUN dnf update -y && \
         ccache \
         clang-devel \
         cmake \
+        curl-devel \
         flatbuffers-devel \
-        java-1.8.0-openjdk-devel \
-        java-1.8.0-openjdk-headless \
         gcc \
         gcc-c++ \
-        glog-devel \
         gflags-devel \
+        git \
+        glog-devel \
         gmock-devel \
         google-benchmark-devel \
-        protobuf-devel \
+        grpc-devel \
+        grpc-plugins \
         gtest-devel \
-        git \
+        java-latest-openjdk-devel \
+        java-latest-openjdk-headless \
         libzstd-devel \
         llvm-devel \
         llvm-static \
@@ -48,16 +51,21 @@ RUN dnf update -y && \
         make \
         ninja-build \
         openssl-devel \
+        protobuf-devel \
         python \
         rapidjson-devel \
         re2-devel \
         snappy-devel \
         thrift-devel \
         utf8proc-devel \
+        wget \
         which \
         zlib-devel
 
-# * gRPC 1.26 in Fedora 32 may have a problem. arrow-flight-test is stuck.
+COPY ci/scripts/install_minio.sh \
+     /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
+
 ENV ARROW_BUILD_TESTS=ON \
     ARROW_DEPENDENCY_SOURCE=SYSTEM \
     ARROW_DATASET=ON \
@@ -67,6 +75,7 @@ ENV ARROW_BUILD_TESTS=ON \
     ARROW_HOME=/usr/local \
     ARROW_ORC=ON \
     ARROW_PARQUET=ON \
+    ARROW_S3=ON \
     ARROW_USE_CCACHE=ON \
     ARROW_WITH_BROTLI=ON \
     ARROW_WITH_BZ2=ON \
@@ -74,9 +83,9 @@ ENV ARROW_BUILD_TESTS=ON \
     ARROW_WITH_SNAPPY=ON \
     ARROW_WITH_ZLIB=ON \
     ARROW_WITH_ZSTD=ON \
+    AWSSDK_SOURCE=BUNDLED \
     CC=gcc \
     CXX=g++ \
-    gRPC_SOURCE=BUNDLED \
     ORC_SOURCE=BUNDLED \
     PARQUET_BUILD_EXECUTABLES=ON \
     PARQUET_BUILD_EXAMPLES=ON \
diff --git a/ci/docker/linux-apt-c-glib.dockerfile b/ci/docker/linux-apt-c-glib.dockerfile
index 3d1658ff7b8..12c6e23a00d 100644
--- a/ci/docker/linux-apt-c-glib.dockerfile
+++ b/ci/docker/linux-apt-c-glib.dockerfile
@@ -25,6 +25,7 @@ RUN apt-get update -y -q && \
         gtk-doc-tools \
         libgirepository1.0-dev \
         libglib2.0-doc \
+        lsb-release \
         luarocks \
         pkg-config \
         ruby-dev && \
@@ -54,7 +55,7 @@ RUN luarocks install lgi
 # ERROR: Command errored out with exit status 1: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay Check the logs for full command output.
 RUN (python3 -m pip install meson || \
          python3 -m pip install --no-use-pep517 meson) && \
-    gem install bundler
+    gem install --no-document bundler
 
 COPY c_glib/Gemfile /arrow/c_glib/
 RUN bundle install --gemfile /arrow/c_glib/Gemfile
diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile
index d0d98d5cd30..ec474f99861 100644
--- a/ci/docker/linux-apt-docs.dockerfile
+++ b/ci/docker/linux-apt-docs.dockerfile
@@ -38,8 +38,11 @@ RUN apt-get update -y && \
         gobject-introspection \
         gtk-doc-tools \
         libcurl4-openssl-dev \
+        libfontconfig1-dev \
+        libfribidi-dev \
         libgirepository1.0-dev \
         libglib2.0-doc \
+        libharfbuzz-dev \
         libtool \
         libxml2-dev \
         ninja-build \
@@ -62,23 +65,31 @@ RUN /arrow/ci/scripts/util_download_apache.sh \
 ENV PATH=/opt/apache-maven-${maven}/bin:$PATH
 RUN mvn -version
 
-ARG node=11
+ARG node=14
 RUN wget -q -O - https://deb.nodesource.com/setup_${node}.x | bash - && \
     apt-get install -y nodejs && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+# Sphinx is pinned because of ARROW-9693
 RUN pip install \
         meson \
         breathe \
         ipython \
-        sphinx \
+        sphinx==3.1.2 \
         sphinx_rtd_theme
 
 COPY c_glib/Gemfile /arrow/c_glib/
-RUN gem install bundler && \
+RUN gem install --no-document bundler && \
     bundle install --gemfile /arrow/c_glib/Gemfile
 
+# Ensure parallel R package installation, set CRAN repo mirror,
+# and use pre-built binaries where possible
+COPY ci/etc/rprofile /arrow/ci/etc/
+RUN cat /arrow/ci/etc/rprofile >> $(R RHOME)/etc/Rprofile.site
+# Also ensure parallel compilation of C/C++ code
+RUN echo "MAKEFLAGS=-j$(R -s -e 'cat(parallel::detectCores())')" >> $(R RHOME)/etc/Makeconf
+
 COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
 COPY r/DESCRIPTION /arrow/r/
 RUN /arrow/ci/scripts/r_deps.sh /arrow && \
diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile
index 85827358dfb..f47044e334b 100644
--- a/ci/docker/linux-apt-r.dockerfile
+++ b/ci/docker/linux-apt-r.dockerfile
@@ -17,6 +17,7 @@
 
 ARG base
 FROM ${base}
+ARG arch
 
 # Build R
 # [1] https://www.digitalocean.com/community/tutorials/how-to-install-r-on-ubuntu-18-04
@@ -70,6 +71,10 @@ COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
 COPY r/DESCRIPTION /arrow/r/
 RUN /arrow/ci/scripts/r_deps.sh /arrow
 
+COPY ci/scripts/install_minio.sh \
+     /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
+
 # Set up Python 3 and its dependencies
 RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
     ln -s /usr/bin/pip3 /usr/local/bin/pip
@@ -89,6 +94,7 @@ ENV \
     ARROW_PARQUET=ON \
     ARROW_PLASMA=OFF \
     ARROW_PYTHON=ON \
+    ARROW_S3=ON \
     ARROW_USE_CCACHE=ON \
     ARROW_USE_GLOG=OFF \
     LC_ALL=en_US.UTF-8
diff --git a/ci/docker/linux-r.dockerfile b/ci/docker/linux-r.dockerfile
index 1d963a20d14..5223d7aafa5 100644
--- a/ci/docker/linux-r.dockerfile
+++ b/ci/docker/linux-r.dockerfile
@@ -24,12 +24,16 @@ FROM ${base}
 ARG r_bin=R
 ENV R_BIN=${r_bin}
 
+ARG r_dev=FALSE
+ENV ARROW_R_DEV=${r_dev}
+
 # Make sure R is on the path for the R-hub devel versions (where RPREFIX is set in its dockerfile)
 ENV PATH "${RPREFIX}/bin:${PATH}"
 
 # Patch up some of the docker images
 COPY ci/scripts/r_docker_configure.sh /arrow/ci/scripts/
 COPY ci/etc/rprofile /arrow/ci/etc/
+COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/r_docker_configure.sh
 
 COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
diff --git a/ci/docker/ubuntu-18.04-cpp.dockerfile b/ci/docker/ubuntu-18.04-cpp.dockerfile
index a0fe1b3f6be..bfff20b441c 100644
--- a/ci/docker/ubuntu-18.04-cpp.dockerfile
+++ b/ci/docker/ubuntu-18.04-cpp.dockerfile
@@ -70,6 +70,7 @@ RUN apt-get update -y -q && \
         libboost-system-dev \
         libbrotli-dev \
         libbz2-dev \
+        libcurl4-openssl-dev \
         libgflags-dev \
         libgoogle-glog-dev \
         liblz4-dev \
@@ -96,6 +97,7 @@ RUN apt-get update -y -q && \
 # - libgtest-dev only provide sources
 # - libprotobuf-dev only provide sources
 # - thrift is too old
+# - s3 tests would require boost-asio that is included since Boost 1.66.0
 ENV ARROW_BUILD_TESTS=ON \
     ARROW_DEPENDENCY_SOURCE=SYSTEM \
     ARROW_DATASET=ON \
@@ -117,6 +119,7 @@ ENV ARROW_BUILD_TESTS=ON \
     ARROW_WITH_SNAPPY=ON \
     ARROW_WITH_ZLIB=ON \
     ARROW_WITH_ZSTD=ON \
+    AWSSDK_SOURCE=BUNDLED \
     GTest_SOURCE=BUNDLED \
     ORC_SOURCE=BUNDLED \
     PARQUET_BUILD_EXECUTABLES=ON \
diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile
index ce738f5e554..fbcda444915 100644
--- a/ci/docker/ubuntu-20.04-cpp.dockerfile
+++ b/ci/docker/ubuntu-20.04-cpp.dockerfile
@@ -17,6 +17,7 @@
 
 ARG base=amd64/ubuntu:20.04
 FROM ${base}
+ARG arch
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
@@ -29,7 +30,22 @@ RUN echo "debconf debconf/frontend select Noninteractive" | \
 # while debugging package list with docker build.
 ARG clang_tools
 ARG llvm
-RUN apt-get update -y -q && \
+RUN if [ "${llvm}" -gt "10" ]; then \
+      apt-get update -y -q && \
+      apt-get install -y -q --no-install-recommends \
+          apt-transport-https \
+          ca-certificates \
+          gnupg \
+          wget && \
+      wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+      echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${llvm} main" > \
+         /etc/apt/sources.list.d/llvm.list && \
+      if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -gt 10 ]; then \
+        echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${clang_tools} main" > \
+           /etc/apt/sources.list.d/clang-tools.list; \
+      fi \
+    fi && \
+    apt-get update -y -q && \
     apt-get install -y -q --no-install-recommends \
         clang-${clang_tools} \
         clang-${llvm} \
@@ -57,6 +73,7 @@ RUN apt-get update -y -q && \
         libbrotli-dev \
         libbz2-dev \
         libgflags-dev \
+        libcurl4-openssl-dev \
         libgoogle-glog-dev \
         liblz4-dev \
         libprotobuf-dev \
@@ -72,10 +89,15 @@ RUN apt-get update -y -q && \
         pkg-config \
         protobuf-compiler \
         rapidjson-dev \
-        tzdata && \
+        tzdata \
+        wget && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists*
 
+COPY ci/scripts/install_minio.sh \
+     /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
+
 # Prioritize system packages and local installation
 # The following dependencies will be downloaded due to missing/invalid packages
 # provided by the distribution:
@@ -95,6 +117,7 @@ ENV ARROW_BUILD_TESTS=ON \
     ARROW_ORC=ON \
     ARROW_PARQUET=ON \
     ARROW_PLASMA=ON \
+    ARROW_S3=ON \
     ARROW_USE_ASAN=OFF \
     ARROW_USE_CCACHE=ON \
     ARROW_USE_UBSAN=OFF \
@@ -104,6 +127,7 @@ ENV ARROW_BUILD_TESTS=ON \
     ARROW_WITH_SNAPPY=ON \
     ARROW_WITH_ZLIB=ON \
     ARROW_WITH_ZSTD=ON \
+    AWSSDK_SOURCE=BUNDLED \
     GTest_SOURCE=BUNDLED \
     ORC_SOURCE=BUNDLED \
     PARQUET_BUILD_EXAMPLES=ON \
diff --git a/ci/etc/integration_spark_ARROW-9438.patch b/ci/etc/integration_spark_ARROW-9438.patch
deleted file mode 100644
index 2baed303717..00000000000
--- a/ci/etc/integration_spark_ARROW-9438.patch
+++ /dev/null
@@ -1,72 +0,0 @@
-From 0b5388a945a7e5c5706cf00d0754540a6c68254d Mon Sep 17 00:00:00 2001
-From: Bryan Cutler <cutlerb@gmail.com>
-Date: Mon, 13 Jul 2020 23:12:25 -0700
-Subject: [PATCH] Update Arrow Java for 1.0.0
-
----
- pom.xml              | 17 ++++++++++++++---
- sql/catalyst/pom.xml |  4 ++++
- 2 files changed, 18 insertions(+), 3 deletions(-)
-
-diff --git a/pom.xml b/pom.xml
-index 08ca13bfe9..6619fca200 100644
---- a/pom.xml
-+++ b/pom.xml
-@@ -199,7 +199,7 @@
-     If you are changing Arrow version specification, please check ./python/pyspark/sql/utils.py,
-     and ./python/setup.py too.
-     -->
--    <arrow.version>0.15.1</arrow.version>
-+    <arrow.version>1.0.0-SNAPSHOT</arrow.version>
-     <!-- org.fusesource.leveldbjni will be used except on arm64 platform. -->
-     <leveldbjni.group>org.fusesource.leveldbjni</leveldbjni.group>
- 
-@@ -2288,7 +2288,7 @@
-           </exclusion>
-           <exclusion>
-             <groupId>com.fasterxml.jackson.core</groupId>
--            <artifactId>jackson-databind</artifactId>
-+            <artifactId>jackson-core</artifactId>
-           </exclusion>
-           <exclusion>
-             <groupId>io.netty</groupId>
-@@ -2298,9 +2298,20 @@
-             <groupId>io.netty</groupId>
-             <artifactId>netty-common</artifactId>
-           </exclusion>
-+        </exclusions>
-+      </dependency>
-+      <dependency>
-+        <groupId>org.apache.arrow</groupId>
-+        <artifactId>arrow-memory-netty</artifactId>
-+        <version>${arrow.version}</version>
-+        <exclusions>
-           <exclusion>
-             <groupId>io.netty</groupId>
--            <artifactId>netty-handler</artifactId>
-+            <artifactId>netty-buffer</artifactId>
-+          </exclusion>
-+          <exclusion>
-+            <groupId>io.netty</groupId>
-+            <artifactId>netty-common</artifactId>
-           </exclusion>
-         </exclusions>
-       </dependency>
-diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
-index 9edbb7fec9..6b79eb722f 100644
---- a/sql/catalyst/pom.xml
-+++ b/sql/catalyst/pom.xml
-@@ -117,6 +117,10 @@
-       <groupId>org.apache.arrow</groupId>
-       <artifactId>arrow-vector</artifactId>
-     </dependency>
-+    <dependency>
-+      <groupId>org.apache.arrow</groupId>
-+      <artifactId>arrow-memory-netty</artifactId>
-+    </dependency>
-   </dependencies>
-   <build>
-     <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
--- 
-2.17.1
-
diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD
index df5f5860445..2bfbcafbaf7 100644
--- a/ci/scripts/PKGBUILD
+++ b/ci/scripts/PKGBUILD
@@ -18,13 +18,14 @@
 _realname=arrow
 pkgbase=mingw-w64-${_realname}
 pkgname="${MINGW_PACKAGE_PREFIX}-${_realname}"
-pkgver=1.0.0.9000
+pkgver=2.0.0.9000
 pkgrel=8000
 pkgdesc="Apache Arrow is a cross-language development platform for in-memory data (mingw-w64)"
 arch=("any")
 url="https://arrow.apache.org/"
 license=("Apache-2.0")
-depends=("${MINGW_PACKAGE_PREFIX}-thrift"
+depends=("${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp"
+         "${MINGW_PACKAGE_PREFIX}-thrift"
          "${MINGW_PACKAGE_PREFIX}-snappy"
          "${MINGW_PACKAGE_PREFIX}-zlib"
          "${MINGW_PACKAGE_PREFIX}-lz4"
@@ -74,6 +75,9 @@ build() {
     export PATH="/C/Rtools${MINGW_PREFIX/mingw/mingw_}/bin:$PATH"
     export CPPFLAGS="${CPPFLAGS} -I${MINGW_PREFIX}/include"
     export LIBS="-L${MINGW_PREFIX}/libs"
+    export ARROW_S3=OFF
+  else
+    export ARROW_S3=ON
   fi
 
   MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \
@@ -94,6 +98,7 @@ build() {
     -DARROW_MIMALLOC=ON \
     -DARROW_PACKAGE_PREFIX="${MINGW_PREFIX}" \
     -DARROW_PARQUET=ON \
+    -DARROW_S3="${ARROW_S3}" \
     -DARROW_SNAPPY_USE_SHARED=OFF \
     -DARROW_USE_GLOG=OFF \
     -DARROW_WITH_LZ4=ON \
diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh
index aebc02ed2cf..fe109b77b09 100755
--- a/ci/scripts/cpp_build.sh
+++ b/ci/scripts/cpp_build.sh
@@ -82,6 +82,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
       -DARROW_PLASMA_JAVA_CLIENT=${ARROW_PLASMA_JAVA_CLIENT:-OFF} \
       -DARROW_PLASMA=${ARROW_PLASMA:-OFF} \
       -DARROW_PYTHON=${ARROW_PYTHON:-OFF} \
+      -DARROW_RUNTIME_SIMD_LEVEL=${ARROW_RUNTIME_SIMD_LEVEL:-MAX} \
       -DARROW_S3=${ARROW_S3:-OFF} \
       -DARROW_TEST_LINKAGE=${ARROW_TEST_LINKAGE:-shared} \
       -DARROW_TEST_MEMCHECK=${ARROW_TEST_MEMCHECK:-OFF} \
@@ -100,6 +101,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
       -DARROW_WITH_UTF8PROC=${ARROW_WITH_UTF8PROC:-ON} \
       -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB:-OFF} \
       -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD:-OFF} \
+      -DAWSSDK_SOURCE=${AWSSDK_SOURCE:-} \
       -Dbenchmark_SOURCE=${benchmark_SOURCE:-} \
       -DBOOST_SOURCE=${BOOST_SOURCE:-} \
       -DBrotli_SOURCE=${Brotli_SOURCE:-} \
diff --git a/ci/scripts/install_minio.sh b/ci/scripts/install_minio.sh
index 9ed70afc03b..42f7ce040e0 100755
--- a/ci/scripts/install_minio.sh
+++ b/ci/scripts/install_minio.sh
@@ -20,11 +20,14 @@
 set -e
 
 declare -A archs
-archs=([amd64]=amd64)
+archs=([amd64]=amd64
+       [arm64v8]=arm64
+       [arm32v7]=arm
+       [s390x]=s390x)
 
 declare -A platforms
-platforms=([macos]=darwin
-           [linux]=linux)
+platforms=([linux]=linux
+           [macos]=darwin)
 
 arch=${archs[$1]}
 platform=${platforms[$2]}
@@ -34,10 +37,10 @@ prefix=$4
 if [ "$#" -ne 4 ]; then
   echo "Usage: $0 <architecture> <platform> <version> <prefix>"
   exit 1
-elif [[ -z ${archs[$1]} ]]; then
+elif [[ -z ${arch} ]]; then
   echo "Unexpected architecture: ${1}"
   exit 1
-elif [[ -z ${platforms[$2]} ]]; then
+elif [[ -z ${platform} ]]; then
   echo "Unexpected platform: ${2}"
   exit 1
 elif [[ ${version} != "latest" ]]; then
@@ -45,5 +48,5 @@ elif [[ ${version} != "latest" ]]; then
   exit 1
 fi
 
-wget -nv -P ${prefix}/bin https://dl.min.io/server/minio/release/linux-${arch}/minio
+wget -nv -P ${prefix}/bin https://dl.min.io/server/minio/release/${platform}-${arch}/minio
 chmod +x ${prefix}/bin/minio
diff --git a/ci/scripts/integration_kartothek.sh b/ci/scripts/integration_kartothek.sh
index f1465ba40e6..6e89f726339 100755
--- a/ci/scripts/integration_kartothek.sh
+++ b/ci/scripts/integration_kartothek.sh
@@ -27,4 +27,4 @@ python -c "import pyarrow.parquet"
 python -c "import kartothek"
 
 pushd /kartothek
-pytest -n0
+pytest -n0 --ignore tests/cli/test_query.py
diff --git a/ci/scripts/integration_spark.sh b/ci/scripts/integration_spark.sh
index 9828a28a1ec..a45ed7a7125 100755
--- a/ci/scripts/integration_spark.sh
+++ b/ci/scripts/integration_spark.sh
@@ -22,6 +22,9 @@ source_dir=${1}
 spark_dir=${2}
 spark_version=${SPARK_VERSION:-master}
 
+# Use old behavior that always dropped tiemzones.
+export PYARROW_IGNORE_TIMEZONE=1
+
 if [ "${SPARK_VERSION:0:2}" == "2." ]; then
   # https://github.com/apache/spark/blob/master/docs/sql-pyspark-pandas-with-arrow.md#compatibility-setting-for-pyarrow--0150-and-spark-23x-24x
   export ARROW_PRE_0_15_IPC_FORMAT=1
diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh
index b5643f77840..a2deafa17ba 100755
--- a/ci/scripts/java_build.sh
+++ b/ci/scripts/java_build.sh
@@ -23,6 +23,62 @@ source_dir=${1}/java
 cpp_build_dir=${2}/cpp/${ARROW_BUILD_TYPE:-debug}
 with_docs=${3:-false}
 
+if [[ "$(uname -s)" == "Linux" ]] && [[ "$(uname -m)" == "s390x" ]]; then
+  # Since some files for s390_64 are not available at maven central,
+  # download pre-build files from bintray and install them explicitly
+  mvn_install="mvn install:install-file"
+  wget="wget"
+  bintray_base_url="https://dl.bintray.com/apache/arrow"
+
+  bintray_dir="flatc-binary"
+  group="com.github.icexelloss"
+  artifact="flatc-linux-s390_64"
+  ver="1.9.0"
+  extension="exe"
+  target=${artifact}-${ver}.${extension}
+  ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/${target}
+  ${mvn_install} -DgroupId=${group} -DartifactId=${artifact} -Dversion=${ver} -Dpackaging=${extension} -Dfile=$(pwd)/${target}
+
+  bintray_dir="protoc-binary"
+  group="com.google.protobuf"
+  artifact="protoc"
+  ver="3.7.1"
+  classifier="linux-s390_64"
+  extension="exe"
+  target=${artifact}-${ver}-${classifier}.${extension}
+  ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/${target}
+  ${mvn_install} -DgroupId=${group} -DartifactId=${artifact} -Dversion=${ver} -Dclassifier=${classifier} -Dpackaging=${extension} -Dfile=$(pwd)/${target}
+  # protoc requires libprotoc.so.18 libprotobuf.so.18
+  ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/libprotoc.so.18
+  ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/libprotobuf.so.18
+  export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$(pwd)
+
+  bintray_dir="protoc-gen-grpc-java-binary"
+  group="io.grpc"
+  artifact="protoc-gen-grpc-java"
+  ver="1.30.2"
+  classifier="linux-s390_64"
+  extension="exe"
+  target=${artifact}-${ver}-${classifier}.${extension}
+  ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/${target}
+  ${mvn_install} -DgroupId=${group} -DartifactId=${artifact} -Dversion=${ver} -Dclassifier=${classifier} -Dpackaging=${extension} -Dfile=$(pwd)/${target}
+
+  bintray_dir="netty-binary"
+  group="io.netty"
+  artifact="netty-transport-native-unix-common"
+  ver="4.1.48.Final"
+  classifier="linux-s390_64"
+  extension="jar"
+  target=${artifact}-${ver}-${classifier}.${extension}
+  ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/${target}
+  ${mvn_install} -DgroupId=${group} -DartifactId=${artifact} -Dversion=${ver} -Dclassifier=${classifier} -Dpackaging=${extension} -Dfile=$(pwd)/${target}
+  artifact="netty-transport-native-epoll"
+  extension="jar"
+  target=${artifact}-${ver}-${classifier}.${extension}
+  ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/${target}
+  ${mvn_install} -DgroupId=${group} -DartifactId=${artifact} -Dversion=${ver} -Dclassifier=${classifier} -Dpackaging=${extension} -Dfile=$(pwd)/${target}
+fi
+
 mvn="mvn -B -DskipTests -Drat.skip=true -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
 # Use `2 * ncores` threads
 mvn="${mvn} -T 2C"
diff --git a/ci/scripts/msys2_setup.sh b/ci/scripts/msys2_setup.sh
index a2845d1473b..51dc45fc3a2 100755
--- a/ci/scripts/msys2_setup.sh
+++ b/ci/scripts/msys2_setup.sh
@@ -24,25 +24,26 @@ target=$1
 packages=()
 case "${target}" in
   cpp|c_glib|ruby)
-    packages+=(make)
-    packages+=(${MINGW_PACKAGE_PREFIX}-ccache)
+    packages+=(${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp)
     packages+=(${MINGW_PACKAGE_PREFIX}-boost)
     packages+=(${MINGW_PACKAGE_PREFIX}-brotli)
+    packages+=(${MINGW_PACKAGE_PREFIX}-ccache)
     packages+=(${MINGW_PACKAGE_PREFIX}-clang)
     packages+=(${MINGW_PACKAGE_PREFIX}-cmake)
     packages+=(${MINGW_PACKAGE_PREFIX}-gcc)
     packages+=(${MINGW_PACKAGE_PREFIX}-gflags)
     packages+=(${MINGW_PACKAGE_PREFIX}-grpc)
     packages+=(${MINGW_PACKAGE_PREFIX}-gtest)
+    packages+=(${MINGW_PACKAGE_PREFIX}-libutf8proc)
     packages+=(${MINGW_PACKAGE_PREFIX}-llvm)
     packages+=(${MINGW_PACKAGE_PREFIX}-lz4)
+    packages+=(${MINGW_PACKAGE_PREFIX}-ninja)
     packages+=(${MINGW_PACKAGE_PREFIX}-polly)
     packages+=(${MINGW_PACKAGE_PREFIX}-protobuf)
     packages+=(${MINGW_PACKAGE_PREFIX}-python3-numpy)
     packages+=(${MINGW_PACKAGE_PREFIX}-rapidjson)
     packages+=(${MINGW_PACKAGE_PREFIX}-snappy)
     packages+=(${MINGW_PACKAGE_PREFIX}-thrift)
-    packages+=(${MINGW_PACKAGE_PREFIX}-libutf8proc)
     packages+=(${MINGW_PACKAGE_PREFIX}-zlib)
     packages+=(${MINGW_PACKAGE_PREFIX}-zstd)
   ;;
diff --git a/ci/scripts/msys2_system_upgrade_phase1.sh b/ci/scripts/msys2_system_upgrade_phase1.sh
index 0839228f419..aecd3089332 100755
--- a/ci/scripts/msys2_system_upgrade_phase1.sh
+++ b/ci/scripts/msys2_system_upgrade_phase1.sh
@@ -20,6 +20,9 @@
 set -eux
 
 # https://www.msys2.org/news/#2020-06-29-new-packagers
+msys2_repo_base_url=https://repo.msys2.org/msys
+# Mirror
+msys2_repo_base_url=https://sourceforge.net/projects/msys2/files/REPOS/MSYS2
 msys2_keyring_pkg=msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz
 for suffix in "" ".sig"; do
   curl \
@@ -27,7 +30,7 @@ for suffix in "" ".sig"; do
     --remote-name \
     --show-error \
     --silent \
-    https://repo.msys2.org/msys/x86_64/${msys2_keyring_pkg}${suffix}
+    ${msys2_repo_base_url}/x86_64/${msys2_keyring_pkg}${suffix}
 done
 pacman-key --verify ${msys2_keyring_pkg}.sig
 pacman \
diff --git a/ci/scripts/python_test.sh b/ci/scripts/python_test.sh
index 6f961d2f8e0..80a9cdef4a3 100755
--- a/ci/scripts/python_test.sh
+++ b/ci/scripts/python_test.sh
@@ -29,4 +29,4 @@ export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
 # Enable some checks inside Python itself
 export PYTHONDEVMODE=1
 
-pytest -r s --pyargs pyarrow
+pytest -r s ${PYTEST_ARGS} --pyargs pyarrow
diff --git a/ci/scripts/r_deps.sh b/ci/scripts/r_deps.sh
index a2dc58fd97b..7e9d2eac7a9 100755
--- a/ci/scripts/r_deps.sh
+++ b/ci/scripts/r_deps.sh
@@ -25,7 +25,7 @@ source_dir=${1}/r
 pushd ${source_dir}
 
 # Install R package dependencies
-${R_BIN} -e "install.packages('remotes'); remotes::install_cran(c('glue', 'rcmdcheck'))"
+${R_BIN} -e "install.packages('remotes'); remotes::install_cran(c('glue', 'rcmdcheck', 'sys'))"
 ${R_BIN} -e "remotes::install_deps(dependencies = TRUE)"
 
 popd
diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh
index 1d7e8de8bf5..e6594e03a88 100755
--- a/ci/scripts/r_docker_configure.sh
+++ b/ci/scripts/r_docker_configure.sh
@@ -39,6 +39,25 @@ if [ "$RHUB_PLATFORM" = "linux-x86_64-fedora-clang" ]; then
   rm -rf $(${R_BIN} RHOME)/etc/Makeconf.bak
 fi
 
+# Install openssl for S3 support
+if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then
+  if [ "`which dnf`" ]; then
+    dnf install -y libcurl-devel openssl-devel
+  elif [ "`which yum`" ]; then
+    yum install -y libcurl-devel openssl-devel
+  elif [ "`which zypper`" ]; then
+    zypper install -y libcurl-devel libopenssl-devel
+  else
+    apt-get update
+    apt-get install -y libcurl4-openssl-dev libssl-dev
+  fi
+
+  # The Dockerfile should have put this file here
+  if [ -f "/arrow/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then
+    /arrow/ci/scripts/install_minio.sh amd64 linux latest /usr/local
+  fi
+fi
+
 # Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786
 Rscript -e 'x <- file.path(R.home("doc"), "html"); if (!file.exists(x)) {dir.create(x, recursive=TRUE); file.copy(system.file("html/R.css", package="stats"), x)}'
 
diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh
index 05c70d8a560..a2428e912be 100755
--- a/ci/scripts/r_test.sh
+++ b/ci/scripts/r_test.sh
@@ -59,6 +59,13 @@ ${R_BIN} -e "as_cran <- !identical(tolower(Sys.getenv('NOT_CRAN')), 'true')
   if (as_cran) {
     rcmdcheck::rcmdcheck(args = c('--as-cran', '--run-donttest'), error_on = 'warning', check_dir = 'check')
   } else {
+    if (nzchar(Sys.which('minio'))) {
+      message('Running minio for S3 tests (if build supports them)')
+      minio_dir <- tempfile()
+      dir.create(minio_dir)
+      pid <- sys::exec_background('minio', c('server', minio_dir))
+      on.exit(tools::pskill(pid))
+    }
     rcmdcheck::rcmdcheck(build_args = '--no-build-vignettes', args = c('--no-manual', '--ignore-vignettes', '--run-donttest'), error_on = 'warning', check_dir = 'check')
   }"
 
diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh
index ed9e211f9ac..cb33e676a7d 100755
--- a/ci/scripts/r_windows_build.sh
+++ b/ci/scripts/r_windows_build.sh
@@ -27,13 +27,25 @@ if [ "$RTOOLS_VERSION" = "35" ]; then
   # Use rtools-backports if building with rtools35
   curl https://raw.githubusercontent.com/r-windows/rtools-backports/master/pacman.conf > /etc/pacman.conf
   # Update keys: https://www.msys2.org/news/#2020-06-29-new-packagers
-  curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz"
+  msys2_repo_base_url=https://repo.msys2.org/msys
+  # Mirror
+  msys2_repo_base_url=https://sourceforge.net/projects/msys2/files/REPOS/MSYS2
+  curl -OSsL "${msys2_repo_base_url}/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz"
   pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz
+  # Use sf.net instead of http://repo.msys2.org/ temporary.
+  sed -i -e "s,^Server = http://repo\.msys2\.org/msys,Server = ${msys2_repo_base_url},g" \
+    /etc/pacman.conf
   pacman --noconfirm -Scc
   pacman --noconfirm -Syy
   # lib-4.9.3 is for libraries compiled with gcc 4.9 (Rtools 3.5)
   RWINLIB_LIB_DIR="lib-4.9.3"
 else
+  # Uncomment L38-41 if you're testing a new rtools dependency that hasn't yet sync'd to CRAN
+  # curl https://raw.githubusercontent.com/r-windows/rtools-packages/master/pacman.conf > /etc/pacman.conf
+  # curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz"
+  # pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz
+  # pacman --noconfirm -Scc
+
   pacman --noconfirm -Syy
   RWINLIB_LIB_DIR="lib"
 fi
@@ -84,8 +96,8 @@ cp $MSYS_LIB_DIR/mingw64/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/x6
 cp $MSYS_LIB_DIR/mingw32/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/i386
 
 # These are from https://dl.bintray.com/rtools/mingw{32,64}/
-cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,crypto}.a $DST_DIR/lib/x64
-cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,crypto}.a $DST_DIR/lib/i386
+cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,crypto,aws*}.a $DST_DIR/lib/x64
+cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,crypto,aws*}.a $DST_DIR/lib/i386
 
 # Create build artifact
 zip -r ${DST_DIR}.zip $DST_DIR
diff --git a/ci/scripts/rust_coverage.sh b/ci/scripts/rust_coverage.sh
new file mode 100755
index 00000000000..fbe5b0d853a
--- /dev/null
+++ b/ci/scripts/rust_coverage.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+
+arrow_dir=${1}
+source_dir=${1}/rust
+build_dir=${2}/rust
+rust=${3}
+
+export ARROW_TEST_DATA=${arrow_dir}/testing/data
+export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data
+export CARGO_TARGET_DIR=${build_dir}
+
+pushd ${source_dir}
+
+rustup default ${rust}
+rustup component add rustfmt --toolchain ${rust}-x86_64-unknown-linux-gnu
+cargo install cargo-tarpaulin
+
+cargo tarpaulin --out Xml
+
+popd
diff --git a/cpp/Brewfile b/cpp/Brewfile
index 6887956f273..7de6c7deabe 100644
--- a/cpp/Brewfile
+++ b/cpp/Brewfile
@@ -28,13 +28,16 @@ brew "grpc"
 brew "llvm"
 brew "llvm@8"
 brew "lz4"
+brew "minio"
 brew "ninja"
 brew "numpy"
 brew "openssl@1.1"
 brew "protobuf"
 brew "python"
 brew "rapidjson"
-brew "re2"
+# grpc bundles re2 and causes a conflict when Homebrew tries to install it,
+# so temporarily skip installing re2. See ARROW-9972.
+# brew "re2"
 brew "snappy"
 brew "thrift"
 brew "wget"
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 721ed22811f..f9ab1548fbd 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -54,7 +54,7 @@ if(POLICY CMP0063)
   cmake_policy(SET CMP0063 NEW)
 endif()
 
-set(ARROW_VERSION "2.0.0-SNAPSHOT")
+set(ARROW_VERSION "3.0.0-SNAPSHOT")
 
 string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ARROW_BASE_VERSION "${ARROW_VERSION}")
 
@@ -109,7 +109,12 @@ set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
 set(ARROW_CMAKE_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}")
 set(ARROW_DOC_DIR "share/doc/${PROJECT_NAME}")
 
-set(ARROW_LLVM_VERSIONS "10" "9" "8" "7")
+set(ARROW_LLVM_VERSIONS
+    "11"
+    "10"
+    "9"
+    "8"
+    "7")
 list(GET ARROW_LLVM_VERSIONS 0 ARROW_LLVM_VERSION_PRIMARY)
 string(REGEX
        REPLACE "^([0-9]+)(\\..+)?" "\\1" ARROW_LLVM_VERSION_PRIMARY_MAJOR
@@ -148,6 +153,13 @@ if(APPLE)
   endif()
 endif()
 
+if(WIN32 AND NOT MINGW)
+  # This is used to handle builds using e.g. clang in an MSVC setting.
+  set(MSVC_TOOLCHAIN TRUE)
+else()
+  set(MSVC_TOOLCHAIN FALSE)
+endif()
+
 find_package(ClangTools)
 find_package(InferTools)
 if("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR CLANG_TIDY_FOUND OR INFER_FOUND)
@@ -315,6 +327,7 @@ if(ARROW_BUILD_BENCHMARKS
    OR ARROW_BUILD_INTEGRATION
    OR ARROW_FUZZING)
   set(ARROW_JSON ON)
+  set(ARROW_TESTING ON)
 endif()
 
 if(ARROW_CUDA
@@ -343,7 +356,7 @@ if(ARROW_PYTHON)
   set(ARROW_JSON ON)
 endif()
 
-if(MSVC)
+if(MSVC_TOOLCHAIN)
   # ORC doesn't build on windows
   set(ARROW_ORC OFF)
   # Plasma using glog is not fully tested on windows.
@@ -410,6 +423,10 @@ if(ARROW_TEST_MEMCHECK)
   add_definitions(-DARROW_VALGRIND)
 endif()
 
+if(ARROW_USE_UBSAN)
+  add_definitions(-DARROW_UBSAN)
+endif()
+
 #
 # Compiler flags
 #
@@ -718,6 +735,7 @@ endif()
 
 if(ARROW_S3)
   list(APPEND ARROW_LINK_LIBS ${AWSSDK_LINK_LIBRARIES})
+  list(APPEND ARROW_STATIC_LINK_LIBS ${AWSSDK_LINK_LIBRARIES})
 endif()
 
 if(ARROW_WITH_UTF8PROC)
@@ -741,7 +759,7 @@ add_dependencies(arrow_test_dependencies toolchain-tests)
 if(ARROW_STATIC_LINK_LIBS)
   add_dependencies(arrow_dependencies ${ARROW_STATIC_LINK_LIBS})
   if(ARROW_ORC)
-    if(NOT MSVC)
+    if(NOT MSVC_TOOLCHAIN)
       list(APPEND ARROW_STATIC_LINK_LIBS ${CMAKE_DL_LIBS})
       list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${CMAKE_DL_LIBS})
     endif()
@@ -755,7 +773,7 @@ if(((ARROW_FLIGHT OR ARROW_S3) AND (ARROW_BUILD_TESTS OR ARROW_BUILD_INTEGRATION
   list(APPEND ARROW_TEST_LINK_LIBS ${BOOST_FILESYSTEM_LIBRARY} ${BOOST_SYSTEM_LIBRARY})
 endif()
 
-if(NOT MSVC)
+if(NOT MSVC_TOOLCHAIN)
   list(APPEND ARROW_LINK_LIBS ${CMAKE_DL_LIBS})
   list(APPEND ARROW_SHARED_INSTALL_INTERFACE_LIBS ${CMAKE_DL_LIBS})
 endif()
diff --git a/cpp/build-support/run-test.sh b/cpp/build-support/run-test.sh
index 7e2a22f069a..1cda600d154 100755
--- a/cpp/build-support/run-test.sh
+++ b/cpp/build-support/run-test.sh
@@ -96,12 +96,14 @@ function run_test() {
   # even when retries are successful.
   rm -f $XMLFILE
 
-  $TEST_EXECUTABLE "$@" 2>&1 \
+  $TEST_EXECUTABLE "$@" > $LOGFILE.raw 2>&1
+  STATUS=$?
+  cat $LOGFILE.raw \
     | ${PYTHON:-python} $ROOT/build-support/asan_symbolize.py \
     | ${CXXFILT:-c++filt} \
     | $ROOT/build-support/stacktrace_addr2line.pl $TEST_EXECUTABLE \
     | $pipe_cmd 2>&1 | tee $LOGFILE
-  STATUS=$?
+  rm -f $LOGFILE.raw
 
   # TSAN doesn't always exit with a non-zero exit code due to a bug:
   # mutex errors don't get reported through the normal error reporting infrastructure.
diff --git a/cpp/build-support/sanitizer-disallowed-entries.txt b/cpp/build-support/sanitizer-disallowed-entries.txt
index f6900c643db..636cfda233a 100644
--- a/cpp/build-support/sanitizer-disallowed-entries.txt
+++ b/cpp/build-support/sanitizer-disallowed-entries.txt
@@ -20,3 +20,6 @@
 # Seen error:
 # thirdparty/gmock-1.7.0/include/gmock/gmock-spec-builders.h:1529:12: runtime error: member call on null pointer of type 'testing::internal::ActionResultHolder<void>'
 fun:*testing*internal*InvokeWith*
+
+# Workaround for RapidJSON https://github.com/Tencent/rapidjson/issues/1724
+src:*/rapidjson/internal/*
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index f92966f78eb..2fd897b5d1d 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -30,6 +30,22 @@ set(ARROW_LIBRARY_PATH_SUFFIXES
     "Library/bin")
 set(ARROW_INCLUDE_PATH_SUFFIXES "include" "Library" "Library/include")
 
+set(ARROW_BOOST_PROCESS_COMPILE_DEFINITIONS)
+if(WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+  # boost/process/detail/windows/handle_workaround.hpp doesn't work
+  # without BOOST_USE_WINDOWS_H with MinGW because MinGW doesn't
+  # provide __kernel_entry without winternl.h.
+  #
+  # See also:
+  # https://github.com/boostorg/process/blob/develop/include/boost/process/detail/windows/handle_workaround.hpp
+  #
+  # You can use this like the following:
+  #
+  #   target_compile_definitions(target PRIVATE
+  #                              ${ARROW_BOOST_PROCESS_COMPILE_DEFINITIONS})
+  list(APPEND ARROW_BOOST_PROCESS_COMPILE_DEFINITIONS "BOOST_USE_WINDOWS_H=1")
+endif()
+
 function(ADD_THIRDPARTY_LIB LIB_NAME)
   set(options)
   set(one_value_args SHARED_LIB STATIC_LIB)
@@ -161,7 +177,7 @@ function(create_merged_static_lib output_target)
         "-o"
         ${output_lib_path}
         ${all_library_paths})
-  elseif(CMAKE_CXX_COMPILER_ID MATCHES "^(Clang|GNU)$")
+  elseif(CMAKE_CXX_COMPILER_ID MATCHES "^(Clang|GNU|Intel)$")
     set(ar_script_path ${CMAKE_BINARY_DIR}/${ARG_NAME}.ar)
 
     file(WRITE ${ar_script_path}.in "CREATE ${output_lib_path}\n")
@@ -424,7 +440,7 @@ function(ADD_ARROW_LIB LIB_NAME)
       target_include_directories(${LIB_NAME}_static PRIVATE ${ARG_PRIVATE_INCLUDES})
     endif()
 
-    if(MSVC)
+    if(MSVC_TOOLCHAIN)
       set(LIB_NAME_STATIC ${LIB_NAME}_static)
     else()
       set(LIB_NAME_STATIC ${LIB_NAME})
diff --git a/cpp/cmake_modules/DefineOptions.cmake b/cpp/cmake_modules/DefineOptions.cmake
index cf860cb4189..a68c3a92cc7 100644
--- a/cpp/cmake_modules/DefineOptions.cmake
+++ b/cpp/cmake_modules/DefineOptions.cmake
@@ -105,15 +105,27 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
                 OFF)
 
   define_option_string(ARROW_SIMD_LEVEL
-                       "SIMD compiler optimization level"
+                       "Compile-time SIMD optimization level"
                        "SSE4_2" # default to SSE4.2
                        "NONE"
                        "SSE4_2"
                        "AVX2"
                        "AVX512")
 
+  define_option_string(ARROW_RUNTIME_SIMD_LEVEL
+                       "Max runtime SIMD optimization level"
+                       "MAX" # default to max supported by compiler
+                       "NONE"
+                       "SSE4_2"
+                       "AVX2"
+                       "AVX512"
+                       "MAX")
+
   # Arm64 architectures and extensions can lead to exploding combinations.
   # So set it directly through cmake command line.
+  #
+  # If you change this, you need to change the definition in
+  # python/CMakeLists.txt too.
   define_option_string(ARROW_ARMV8_ARCH
                        "Arm64 arch and extensions"
                        "armv8-a" # Default
@@ -244,6 +256,8 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
 
   define_option(ARROW_TENSORFLOW "Build Arrow with TensorFlow support enabled" OFF)
 
+  define_option(ARROW_TESTING "Build the Arrow testing libraries" OFF)
+
   #----------------------------------------------------------------------
   set_option_category("Thirdparty toolchain")
 
@@ -304,6 +318,9 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   define_option(ARROW_LZ4_USE_SHARED "Rely on lz4 shared libraries where relevant"
                 ${ARROW_DEPENDENCY_USE_SHARED})
 
+  define_option(ARROW_OPENSSL_USE_SHARED "Rely on OpenSSL shared libraries where relevant"
+                ${ARROW_DEPENDENCY_USE_SHARED})
+
   define_option(ARROW_PROTOBUF_USE_SHARED
                 "Rely on Protocol Buffers shared libraries where relevant"
                 ${ARROW_DEPENDENCY_USE_SHARED})
@@ -348,7 +365,7 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
                 "Build with support for Unicode properties using the utf8proc library" ON)
 
   #----------------------------------------------------------------------
-  if(MSVC)
+  if(MSVC_TOOLCHAIN)
     set_option_category("MSVC")
 
     define_option(MSVC_LINK_VERBOSE
diff --git a/cpp/cmake_modules/FindArrow.cmake b/cpp/cmake_modules/FindArrow.cmake
index 02fd9a15801..9c987665896 100644
--- a/cpp/cmake_modules/FindArrow.cmake
+++ b/cpp/cmake_modules/FindArrow.cmake
@@ -39,6 +39,13 @@ endif()
 include(FindPkgConfig)
 include(FindPackageHandleStandardArgs)
 
+if(WIN32 AND NOT MINGW)
+  # This is used to handle builds using e.g. clang in an MSVC setting.
+  set(MSVC_TOOLCHAIN TRUE)
+else()
+  set(MSVC_TOOLCHAIN FALSE)
+endif()
+
 set(ARROW_SEARCH_LIB_PATH_SUFFIXES)
 if(CMAKE_LIBRARY_ARCHITECTURE)
   list(APPEND ARROW_SEARCH_LIB_PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}")
@@ -61,7 +68,7 @@ if(CMAKE_BUILD_TYPE)
 endif()
 
 if(NOT DEFINED ARROW_MSVC_STATIC_LIB_SUFFIX)
-  if(MSVC)
+  if(MSVC_TOOLCHAIN)
     set(ARROW_MSVC_STATIC_LIB_SUFFIX "_static")
   else()
     set(ARROW_MSVC_STATIC_LIB_SUFFIX "")
@@ -147,7 +154,7 @@ macro(arrow_find_package_home)
   set(include_dir "${${prefix}_include_dir}")
   set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE)
 
-  if(MSVC)
+  if(MSVC_TOOLCHAIN)
     set(CMAKE_SHARED_LIBRARY_SUFFIXES_ORIGINAL ${CMAKE_FIND_LIBRARY_SUFFIXES})
     # .dll isn't found by find_library with MSVC because .dll isn't included in
     # CMAKE_FIND_LIBRARY_SUFFIXES.
@@ -158,7 +165,7 @@ macro(arrow_find_package_home)
                PATHS "${home}"
                PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES}
                NO_DEFAULT_PATH)
-  if(MSVC)
+  if(MSVC_TOOLCHAIN)
     set(CMAKE_SHARED_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_ORIGINAL})
   endif()
   set(shared_lib "${${prefix}_shared_lib}")
diff --git a/cpp/cmake_modules/FindBoostAlt.cmake b/cpp/cmake_modules/FindBoostAlt.cmake
index 8f16439bf86..300080d4fb1 100644
--- a/cpp/cmake_modules/FindBoostAlt.cmake
+++ b/cpp/cmake_modules/FindBoostAlt.cmake
@@ -52,7 +52,7 @@ endif()
 
 if(Boost_FOUND)
   set(BoostAlt_FOUND ON)
-  if(MSVC)
+  if(MSVC_TOOLCHAIN)
     # disable autolinking in boost
     add_definitions(-DBOOST_ALL_NO_LIB)
     if(ARROW_BOOST_USE_SHARED)
diff --git a/cpp/cmake_modules/FindGTest.cmake b/cpp/cmake_modules/FindGTest.cmake
index 4be2cf57046..8581d921b1c 100644
--- a/cpp/cmake_modules/FindGTest.cmake
+++ b/cpp/cmake_modules/FindGTest.cmake
@@ -171,7 +171,7 @@ if(NOT DEFINED GTEST_MSVC_SEARCH)
 endif()
 
 set(_gtest_libpath_suffixes lib)
-if(MSVC)
+if(MSVC_TOOLCHAIN)
     if(GTEST_MSVC_SEARCH STREQUAL "MD")
         list(APPEND _gtest_libpath_suffixes
             msvc/gtest-md/Debug
@@ -198,7 +198,7 @@ find_path(GTEST_INCLUDE_DIR gtest/gtest.h
 )
 mark_as_advanced(GTEST_INCLUDE_DIR)
 
-if(MSVC AND GTEST_MSVC_SEARCH STREQUAL "MD")
+if(MSVC_TOOLCHAIN AND GTEST_MSVC_SEARCH STREQUAL "MD")
     # The provided /MD project files for Google Test add -md suffixes to the
     # library names.
     __gtest_find_library(GTEST_LIBRARY            gtest-md  gtest)
diff --git a/cpp/cmake_modules/FindLz4.cmake b/cpp/cmake_modules/FindLz4.cmake
index dbc6c5f8d3b..7159f96f70f 100644
--- a/cpp/cmake_modules/FindLz4.cmake
+++ b/cpp/cmake_modules/FindLz4.cmake
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-if(MSVC AND NOT DEFINED LZ4_MSVC_LIB_PREFIX)
+if(MSVC_TOOLCHAIN AND NOT DEFINED LZ4_MSVC_LIB_PREFIX)
   set(LZ4_MSVC_LIB_PREFIX "lib")
 endif()
 set(LZ4_LIB_NAME_BASE "${LZ4_MSVC_LIB_PREFIX}lz4")
diff --git a/cpp/cmake_modules/FindPython3Alt.cmake b/cpp/cmake_modules/FindPython3Alt.cmake
index 11f26f5a1e4..131a0d395fc 100644
--- a/cpp/cmake_modules/FindPython3Alt.cmake
+++ b/cpp/cmake_modules/FindPython3Alt.cmake
@@ -36,16 +36,26 @@ if(${CMAKE_VERSION} VERSION_LESS "3.15.0")
   find_package_handle_standard_args(Python3Alt
                                     REQUIRED_VARS
                                     PYTHON_EXECUTABLE
-                                    PYTHON_LIBRARIES
                                     PYTHON_INCLUDE_DIRS
                                     NUMPY_INCLUDE_DIRS)
   return()
 endif()
 
-if(Python3Alt_FIND_REQUIRED)
-  find_package(Python3 COMPONENTS Interpreter Development NumPy REQUIRED)
+if(${CMAKE_VERSION} VERSION_LESS "3.18.0" OR ARROW_BUILD_TESTS)
+  # When building arrow-python-test, we need libpython to be present, so ask for
+  # the full "Development" component.  Also ask for it on CMake < 3.18,
+  # where "Development.Module" is not available.
+  if(Python3Alt_FIND_REQUIRED)
+    find_package(Python3 COMPONENTS Interpreter Development NumPy REQUIRED)
+  else()
+    find_package(Python3 COMPONENTS Interpreter Development NumPy)
+  endif()
 else()
-  find_package(Python3 COMPONENTS Interpreter Development NumPy)
+  if(Python3Alt_FIND_REQUIRED)
+    find_package(Python3 COMPONENTS Interpreter Development.Module NumPy REQUIRED)
+  else()
+    find_package(Python3 COMPONENTS Interpreter Development.Module NumPy)
+  endif()
 endif()
 
 if(NOT Python3_FOUND)
@@ -85,6 +95,5 @@ endfunction()
 find_package_handle_standard_args(Python3Alt
                                   REQUIRED_VARS
                                   PYTHON_EXECUTABLE
-                                  PYTHON_LIBRARIES
                                   PYTHON_INCLUDE_DIRS
                                   NUMPY_INCLUDE_DIRS)
diff --git a/cpp/cmake_modules/FindRE2.cmake b/cpp/cmake_modules/FindRE2.cmake
index 4562e3ae89b..645a20f7c09 100644
--- a/cpp/cmake_modules/FindRE2.cmake
+++ b/cpp/cmake_modules/FindRE2.cmake
@@ -63,8 +63,17 @@ endif()
 find_package_handle_standard_args(RE2 REQUIRED_VARS RE2_LIB RE2_INCLUDE_DIR)
 
 if(RE2_FOUND)
-  add_library(RE2::re2 UNKNOWN IMPORTED)
-  set_target_properties(RE2::re2
-                        PROPERTIES IMPORTED_LOCATION "${RE2_LIB}"
-                                   INTERFACE_INCLUDE_DIRECTORIES "${RE2_INCLUDE_DIR}")
+  if(NOT TARGET RE2::re2)
+    add_library(RE2::re2 UNKNOWN IMPORTED)
+    set_target_properties(RE2::re2
+                          PROPERTIES IMPORTED_LOCATION "${RE2_LIB}"
+                                     INTERFACE_INCLUDE_DIRECTORIES "${RE2_INCLUDE_DIR}")
+  endif()
+  # Some third-party dependencies (namely gRPC) are on the look-out for a lower-case re2 Target.
+  if(NOT TARGET re2::re2)
+    add_library(re2::re2 UNKNOWN IMPORTED)
+    set_target_properties(re2::re2
+                          PROPERTIES IMPORTED_LOCATION "${RE2_LIB}"
+                                     INTERFACE_INCLUDE_DIRECTORIES "${RE2_INCLUDE_DIR}")
+  endif()
 endif()
diff --git a/cpp/cmake_modules/FindThrift.cmake b/cpp/cmake_modules/FindThrift.cmake
index d266f02e29b..273d907ed07 100644
--- a/cpp/cmake_modules/FindThrift.cmake
+++ b/cpp/cmake_modules/FindThrift.cmake
@@ -39,7 +39,7 @@ function(EXTRACT_THRIFT_VERSION)
   endif()
 endfunction(EXTRACT_THRIFT_VERSION)
 
-if(MSVC AND NOT DEFINED THRIFT_MSVC_LIB_SUFFIX)
+if(MSVC_TOOLCHAIN AND NOT DEFINED THRIFT_MSVC_LIB_SUFFIX)
   if(NOT ARROW_THRIFT_USE_SHARED)
     if(ARROW_USE_STATIC_CRT)
       set(THRIFT_MSVC_LIB_SUFFIX "mt")
@@ -133,7 +133,7 @@ if(Thrift_FOUND OR THRIFT_FOUND)
   set_target_properties(thrift::thrift
                         PROPERTIES IMPORTED_LOCATION "${THRIFT_LIB}"
                                    INTERFACE_INCLUDE_DIRECTORIES "${THRIFT_INCLUDE_DIR}")
-  if(WIN32 AND NOT MSVC)
+  if(WIN32 AND NOT MSVC_TOOLCHAIN)
     # We don't need this for Visual C++ because Thrift uses
     # "#pragma comment(lib, "Ws2_32.lib")" in
     # thrift/windows/config.h for Visual C++.
diff --git a/cpp/cmake_modules/Findzstd.cmake b/cpp/cmake_modules/Findzstd.cmake
index f7c68134e9d..6659a682da7 100644
--- a/cpp/cmake_modules/Findzstd.cmake
+++ b/cpp/cmake_modules/Findzstd.cmake
@@ -23,18 +23,24 @@ set(ZSTD_LIB_NAME_BASE "${ZSTD_MSVC_LIB_PREFIX}zstd")
 if(ARROW_ZSTD_USE_SHARED)
   set(ZSTD_LIB_NAMES)
   if(CMAKE_IMPORT_LIBRARY_SUFFIX)
-    list(APPEND ZSTD_LIB_NAMES
-                "${CMAKE_IMPORT_LIBRARY_PREFIX}${ZSTD_LIB_NAME_BASE}${CMAKE_IMPORT_LIBRARY_SUFFIX}")
+    list(
+      APPEND
+        ZSTD_LIB_NAMES
+        "${CMAKE_IMPORT_LIBRARY_PREFIX}${ZSTD_LIB_NAME_BASE}${CMAKE_IMPORT_LIBRARY_SUFFIX}"
+      )
   endif()
-  list(APPEND ZSTD_LIB_NAMES
-              "${CMAKE_SHARED_LIBRARY_PREFIX}${ZSTD_LIB_NAME_BASE}${CMAKE_SHARED_LIBRARY_SUFFIX}")
+  list(
+    APPEND
+      ZSTD_LIB_NAMES
+      "${CMAKE_SHARED_LIBRARY_PREFIX}${ZSTD_LIB_NAME_BASE}${CMAKE_SHARED_LIBRARY_SUFFIX}")
 else()
   if(MSVC AND NOT DEFINED ZSTD_MSVC_STATIC_LIB_SUFFIX)
     set(ZSTD_MSVC_STATIC_LIB_SUFFIX "_static")
   endif()
   set(ZSTD_STATIC_LIB_SUFFIX
       "${ZSTD_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}")
-  set(ZSTD_LIB_NAMES "${CMAKE_STATIC_LIBRARY_PREFIX}${ZSTD_LIB_NAME_BASE}${ZSTD_STATIC_LIB_SUFFIX}")
+  set(ZSTD_LIB_NAMES
+      "${CMAKE_STATIC_LIBRARY_PREFIX}${ZSTD_LIB_NAME_BASE}${ZSTD_STATIC_LIB_SUFFIX}")
 endif()
 
 # First, find via if specified ZTD_ROOT
@@ -66,7 +72,9 @@ else()
                  PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES})
   else()
     # Third, check all other CMake paths
-    find_library(ZSTD_LIB NAMES ${ZSTD_LIB_NAMES} PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES})
+    find_library(ZSTD_LIB
+                 NAMES ${ZSTD_LIB_NAMES}
+                 PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES})
     find_path(ZSTD_INCLUDE_DIR NAMES zstd.h PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES})
   endif()
 endif()
diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index bf1767260db..a5cd95bd7ab 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -18,6 +18,7 @@
 # Check if the target architecture and compiler supports some special
 # instruction sets that would boost performance.
 include(CheckCXXCompilerFlag)
+include(CheckCXXSourceCompiles)
 # Get cpu architecture
 
 message(STATUS "System processor: ${CMAKE_SYSTEM_PROCESSOR}")
@@ -46,9 +47,13 @@ if(ARROW_CPU_FLAG STREQUAL "x86")
     set(CXX_SUPPORTS_SSE4_2 TRUE)
   else()
     set(ARROW_SSE4_2_FLAG "-msse4.2")
-    set(ARROW_AVX2_FLAG "-mavx2")
+    set(ARROW_AVX2_FLAG "-march=haswell")
     # skylake-avx512 consists of AVX512F,AVX512BW,AVX512VL,AVX512CD,AVX512DQ
     set(ARROW_AVX512_FLAG "-march=skylake-avx512 -mbmi2")
+    # Append the avx2/avx512 subset option also, fix issue ARROW-9877 for homebrew-cpp
+    set(ARROW_AVX2_FLAG "${ARROW_AVX2_FLAG} -mavx2")
+    set(ARROW_AVX512_FLAG
+        "${ARROW_AVX512_FLAG} -mavx512f -mavx512cd -mavx512vl -mavx512dq -mavx512bw")
     check_cxx_compiler_flag(${ARROW_SSE4_2_FLAG} CXX_SUPPORTS_SSE4_2)
   endif()
   check_cxx_compiler_flag(${ARROW_AVX2_FLAG} CXX_SUPPORTS_AVX2)
@@ -56,17 +61,37 @@ if(ARROW_CPU_FLAG STREQUAL "x86")
     # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65782
     message(STATUS "Disable AVX512 support on MINGW for now")
   else()
-    check_cxx_compiler_flag(${ARROW_AVX512_FLAG} CXX_SUPPORTS_AVX512)
+    # Check for AVX512 support in the compiler.
+    set(OLD_CMAKE_REQURED_FLAGS ${CMAKE_REQUIRED_FLAGS})
+    set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${ARROW_AVX512_FLAG}")
+    check_cxx_source_compiles("
+      #ifdef _MSC_VER
+      #include <intrin.h>
+      #else
+      #include <immintrin.h>
+      #endif
+
+      int main() {
+        __m512i mask = _mm512_set1_epi32(0x1);
+        char out[32];
+        _mm512_storeu_si512(out, mask);
+        return 0;
+      }" CXX_SUPPORTS_AVX512)
+    set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQURED_FLAGS})
   endif()
-  # Runtime SIMD level it can get from compiler
-  if(CXX_SUPPORTS_SSE4_2)
+  # Runtime SIMD level it can get from compiler and ARROW_RUNTIME_SIMD_LEVEL
+  if(CXX_SUPPORTS_SSE4_2
+     AND ARROW_RUNTIME_SIMD_LEVEL MATCHES "^(SSE4_2|AVX2|AVX512|MAX)$")
+    set(ARROW_HAVE_RUNTIME_SSE4_2 ON)
     add_definitions(-DARROW_HAVE_RUNTIME_SSE4_2)
   endif()
-  if(CXX_SUPPORTS_AVX2)
-    add_definitions(-DARROW_HAVE_RUNTIME_AVX2)
+  if(CXX_SUPPORTS_AVX2 AND ARROW_RUNTIME_SIMD_LEVEL MATCHES "^(AVX2|AVX512|MAX)$")
+    set(ARROW_HAVE_RUNTIME_AVX2 ON)
+    add_definitions(-DARROW_HAVE_RUNTIME_AVX2 -DARROW_HAVE_RUNTIME_BMI2)
   endif()
-  if(CXX_SUPPORTS_AVX512)
-    add_definitions(-DARROW_HAVE_RUNTIME_AVX512)
+  if(CXX_SUPPORTS_AVX512 AND ARROW_RUNTIME_SIMD_LEVEL MATCHES "^(AVX512|MAX)$")
+    set(ARROW_HAVE_RUNTIME_AVX512 ON)
+    add_definitions(-DARROW_HAVE_RUNTIME_AVX512 -DARROW_HAVE_RUNTIME_BMI2)
   endif()
 elseif(ARROW_CPU_FLAG STREQUAL "ppc")
   # power compiler flags, gcc/clang only
@@ -101,7 +126,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE)
 
 set(UNKNOWN_COMPILER_MESSAGE
-    "Unknown compiler: ${CMAKE_CXX_COMPILER_VERSION} ${CMAKE_CXX_COMPILER_VERSION}")
+    "Unknown compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}")
 
 # compiler flags that are common across debug/release builds
 if(WIN32)
@@ -258,7 +283,8 @@ else()
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /W3")
   elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang"
          OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang"
-         OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+         OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU"
+         OR CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall")
   else()
     message(FATAL_ERROR "${UNKNOWN_COMPILER_MESSAGE}")
@@ -283,6 +309,13 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
     set(CXX_ONLY_FLAGS "${CXX_ONLY_FLAGS} -Wno-noexcept-type")
   endif()
 
+  if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "5.2")
+    # Disabling semantic interposition allows faster calling conventions
+    # when calling global functions internally, and can also help inlining.
+    # See https://stackoverflow.com/questions/35745543/new-option-in-gcc-5-3-fno-semantic-interposition
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -fno-semantic-interposition")
+  endif()
+
   if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "4.9")
     # Add colors when paired with ninja
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always")
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index f983d58f9b9..d084f346803 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -249,6 +249,10 @@ if(ARROW_ORC OR ARROW_FLIGHT OR ARROW_GANDIVA)
   set(ARROW_WITH_PROTOBUF ON)
 endif()
 
+if(ARROW_S3)
+  set(ARROW_WITH_ZLIB ON)
+endif()
+
 if(NOT ARROW_COMPUTE)
   # utf8proc is only potentially used in kernels for now
   set(ARROW_WITH_UTF8PROC OFF)
@@ -300,6 +304,33 @@ else()
     "https://github.com/abseil/abseil-cpp/archive/${ARROW_ABSL_BUILD_VERSION}.tar.gz")
 endif()
 
+if(DEFINED ENV{ARROW_AWS_C_COMMON_URL})
+  set(AWS_C_COMMON_SOURCE_URL "$ENV{ARROW_AWS_C_COMMON_URL}")
+else()
+  set_urls(
+    AWS_C_COMMON_SOURCE_URL
+    "https://github.com/awslabs/aws-c-common/archive/${ARROW_AWS_C_COMMON_BUILD_VERSION}.tar.gz"
+    )
+endif()
+
+if(DEFINED ENV{ARROW_AWS_CHECKSUMS_URL})
+  set(AWS_CHECKSUMS_SOURCE_URL "$ENV{ARROW_AWS_CHECKSUMS_URL}")
+else()
+  set_urls(
+    AWS_CHECKSUMS_SOURCE_URL
+    "https://github.com/awslabs/aws-checksums/archive/${ARROW_AWS_CHECKSUMS_BUILD_VERSION}.tar.gz"
+    )
+endif()
+
+if(DEFINED ENV{ARROW_AWS_C_EVENT_STREAM_URL})
+  set(AWS_C_EVENT_STREAM_SOURCE_URL "$ENV{ARROW_AWS_C_EVENT_STREAM_URL}")
+else()
+  set_urls(
+    AWS_C_EVENT_STREAM_SOURCE_URL
+    "https://github.com/awslabs/aws-c-event-stream/archive/${ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION}.tar.gz"
+    )
+endif()
+
 if(DEFINED ENV{ARROW_AWSSDK_URL})
   set(AWSSDK_SOURCE_URL "$ENV{ARROW_AWSSDK_URL}")
 else()
@@ -552,7 +583,7 @@ endif()
 set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}")
 set(EP_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}}")
 
-if(NOT MSVC)
+if(NOT MSVC_TOOLCHAIN)
   # Set -fPIC on all external projects
   set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -fPIC")
   set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC")
@@ -971,8 +1002,24 @@ endif()
 
 set(ARROW_USE_OPENSSL OFF)
 if(PARQUET_REQUIRE_ENCRYPTION OR ARROW_FLIGHT OR ARROW_S3)
-  # This must work
-  find_package(OpenSSL ${ARROW_OPENSSL_REQUIRED_VERSION} REQUIRED)
+  # OpenSSL is required
+  if(ARROW_OPENSSL_USE_SHARED)
+    # Find shared OpenSSL libraries.
+    set(OpenSSL_USE_STATIC_LIBS OFF)
+    # Seems that different envs capitalize this differently?
+    set(OPENSSL_USE_STATIC_LIBS OFF)
+    set(BUILD_SHARED_LIBS_KEEP ${BUILD_SHARED_LIBS})
+    set(BUILD_SHARED_LIBS ON)
+
+    find_package(OpenSSL ${ARROW_OPENSSL_REQUIRED_VERSION} REQUIRED)
+    set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_KEEP})
+    unset(BUILD_SHARED_LIBS_KEEP)
+  else()
+    # Find static OpenSSL headers and libs
+    set(OpenSSL_USE_STATIC_LIBS ON)
+    set(OPENSSL_USE_STATIC_LIBS ON)
+    find_package(OpenSSL ${ARROW_OPENSSL_REQUIRED_VERSION} REQUIRED)
+  endif()
   set(ARROW_USE_OPENSSL ON)
 endif()
 
@@ -1348,7 +1395,7 @@ if(ARROW_WITH_PROTOBUF)
   endif()
   resolve_dependency(Protobuf REQUIRED_VERSION ${ARROW_PROTOBUF_REQUIRED_VERSION})
 
-  if(ARROW_PROTOBUF_USE_SHARED AND MSVC)
+  if(ARROW_PROTOBUF_USE_SHARED AND MSVC_TOOLCHAIN)
     add_definitions(-DPROTOBUF_USE_DLLS)
   endif()
 
@@ -1454,7 +1501,7 @@ if(ARROW_JEMALLOC)
     BUILD_IN_SOURCE 1
     BUILD_COMMAND ${JEMALLOC_BUILD_COMMAND}
     BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}"
-    INSTALL_COMMAND ${MAKE} install)
+    INSTALL_COMMAND ${MAKE} -j1 install)
 
   # Don't use the include directory directly so that we can point to a path
   # that is unique to our codebase.
@@ -1647,10 +1694,7 @@ macro(build_gtest)
   add_dependencies(GTest::gmock googletest_ep)
 endmacro()
 
-if(ARROW_BUILD_TESTS
-   OR ARROW_BUILD_BENCHMARKS
-   OR ARROW_BUILD_INTEGRATION
-   OR ARROW_FUZZING)
+if(ARROW_TESTING)
   resolve_dependency(GTest)
 
   if(NOT GTEST_VENDORED)
@@ -1780,6 +1824,7 @@ macro(build_rapidjson)
   set(RAPIDJSON_INCLUDE_DIR "${RAPIDJSON_PREFIX}/include")
 
   add_dependencies(toolchain rapidjson_ep)
+  add_dependencies(toolchain-tests rapidjson_ep)
   add_dependencies(rapidjson rapidjson_ep)
 
   set(RAPIDJSON_VENDORED TRUE)
@@ -1850,9 +1895,11 @@ macro(build_zlib)
   file(MAKE_DIRECTORY "${ZLIB_PREFIX}/include")
 
   add_library(ZLIB::ZLIB STATIC IMPORTED)
+  set(ZLIB_LIBRARIES ${ZLIB_STATIC_LIB})
+  set(ZLIB_INCLUDE_DIRS "${ZLIB_PREFIX}/include")
   set_target_properties(ZLIB::ZLIB
-                        PROPERTIES IMPORTED_LOCATION "${ZLIB_STATIC_LIB}"
-                                   INTERFACE_INCLUDE_DIRECTORIES "${ZLIB_PREFIX}/include")
+                        PROPERTIES IMPORTED_LOCATION ${ZLIB_LIBRARIES}
+                                   INTERFACE_INCLUDE_DIRECTORIES ${ZLIB_INCLUDE_DIRS})
 
   add_dependencies(toolchain zlib_ep)
   add_dependencies(ZLIB::ZLIB zlib_ep)
@@ -2195,6 +2242,13 @@ macro(build_cares)
                                    INTERFACE_INCLUDE_DIRECTORIES "${CARES_INCLUDE_DIR}")
   add_dependencies(c-ares::cares cares_ep)
 
+  if(APPLE)
+    # libresolv must be linked from c-ares version 1.16.1
+    find_library(LIBRESOLV_LIBRARY NAMES resolv libresolv REQUIRED)
+    set_target_properties(c-ares::cares
+                          PROPERTIES INTERFACE_LINK_LIBRARIES "${LIBRESOLV_LIBRARY}")
+  endif()
+
   set(CARES_VENDORED TRUE)
 
   list(APPEND ARROW_BUNDLED_STATIC_LIBS c-ares::cares)
@@ -2612,12 +2666,14 @@ endif()
 # AWS SDK for C++
 
 macro(build_awssdk)
-  message(
-    FATAL_ERROR "FIXME: Building AWS C++ SDK from source will link with wrong libcrypto")
   message("Building AWS C++ SDK from source")
-
+  if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU"
+     AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.9")
+    message(FATAL_ERROR "AWS C++ SDK requires gcc >= 4.9")
+  endif()
   set(AWSSDK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/awssdk_ep-install")
   set(AWSSDK_INCLUDE_DIR "${AWSSDK_PREFIX}/include")
+  set(AWSSDK_LIB_DIR "lib")
 
   if(WIN32)
     # On Windows, need to match build types
@@ -2626,56 +2682,178 @@ macro(build_awssdk)
     # Otherwise, always build in release mode.
     # Especially with gcc, debug builds can fail with "asm constraint" errors:
     # https://github.com/TileDB-Inc/TileDB/issues/1351
-    set(AWSSDK_BUILD_TYPE Release)
+    set(AWSSDK_BUILD_TYPE release)
   endif()
 
-  set(AWSSDK_CMAKE_ARGS
-      -DCMAKE_BUILD_TYPE=Release
-      -DCMAKE_INSTALL_LIBDIR=lib
-      -DBUILD_ONLY=s3;core;config
-      -DENABLE_UNITY_BUILD=on
-      -DENABLE_TESTING=off
-      "-DCMAKE_C_FLAGS=${EP_C_FLAGS}"
-      "-DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX}")
+  set(AWSSDK_COMMON_CMAKE_ARGS
+      ${EP_COMMON_CMAKE_ARGS}
+      -DBUILD_SHARED_LIBS=OFF
+      -DCMAKE_BUILD_TYPE=${AWSSDK_BUILD_TYPE}
+      -DCMAKE_INSTALL_LIBDIR=${AWSSDK_LIB_DIR}
+      -DENABLE_TESTING=OFF
+      -DENABLE_UNITY_BUILD=ON
+      "-DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX}"
+      "-DCMAKE_PREFIX_PATH=${AWSSDK_PREFIX}")
 
   set(
-    AWSSDK_CORE_SHARED_LIB
-    "${AWSSDK_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}aws-cpp-sdk-core${CMAKE_SHARED_LIBRARY_SUFFIX}"
-    )
-  set(
-    AWSSDK_S3_SHARED_LIB
-    "${AWSSDK_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}aws-cpp-sdk-s3${CMAKE_SHARED_LIBRARY_SUFFIX}"
-    )
-  set(AWSSDK_SHARED_LIBS "${AWSSDK_CORE_SHARED_LIB}" "${AWSSDK_S3_SHARED_LIB}")
+    AWSSDK_CMAKE_ARGS
+    ${AWSSDK_COMMON_CMAKE_ARGS} -DBUILD_DEPS=OFF
+    -DBUILD_ONLY=config\\$<SEMICOLON>s3\\$<SEMICOLON>transfer\\$<SEMICOLON>identity-management\\$<SEMICOLON>sts
+    -DMINIMIZE_SIZE=ON)
+  if(UNIX AND TARGET zlib_ep)
+    list(APPEND AWSSDK_CMAKE_ARGS -DZLIB_INCLUDE_DIR=${ZLIB_INCLUDE_DIRS}
+                -DZLIB_LIBRARY=${ZLIB_LIBRARIES})
+  endif()
+
+  file(MAKE_DIRECTORY ${AWSSDK_INCLUDE_DIR})
+
+  # AWS C++ SDK related libraries to link statically
+  set(_AWSSDK_LIBS
+      aws-cpp-sdk-identity-management
+      aws-cpp-sdk-sts
+      aws-cpp-sdk-cognito-identity
+      aws-cpp-sdk-s3
+      aws-cpp-sdk-core
+      aws-c-event-stream
+      aws-checksums
+      aws-c-common)
+  set(AWSSDK_LIBRARIES)
+  foreach(_AWSSDK_LIB ${_AWSSDK_LIBS})
+    # aws-c-common -> AWS-C-COMMON
+    string(TOUPPER ${_AWSSDK_LIB} _AWSSDK_LIB_UPPER)
+    # AWS-C-COMMON -> AWS_C_COMMON
+    string(REPLACE "-" "_" _AWSSDK_LIB_NAME_PREFIX ${_AWSSDK_LIB_UPPER})
+    set(
+      _AWSSDK_STATIC_LIBRARY
+      "${AWSSDK_PREFIX}/${AWSSDK_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${_AWSSDK_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}"
+      )
+    if(${_AWSSDK_LIB} MATCHES "^aws-cpp-sdk-")
+      set(_AWSSDK_TARGET_NAME ${_AWSSDK_LIB})
+    else()
+      set(_AWSSDK_TARGET_NAME AWS::${_AWSSDK_LIB})
+    endif()
+    add_library(${_AWSSDK_TARGET_NAME} STATIC IMPORTED)
+    set_target_properties(
+      ${_AWSSDK_TARGET_NAME}
+      PROPERTIES IMPORTED_LOCATION ${_AWSSDK_STATIC_LIBRARY} INTERFACE_INCLUDE_DIRECTORIES
+                 "${AWSSDK_INCLUDE_DIR}")
+    set("${_AWSSDK_LIB_NAME_PREFIX}_STATIC_LIBRARY" ${_AWSSDK_STATIC_LIBRARY})
+    list(APPEND AWSSDK_LIBRARIES ${_AWSSDK_TARGET_NAME})
+  endforeach()
+
+  externalproject_add(aws_c_common_ep
+                      ${EP_LOG_OPTIONS}
+                      URL ${AWS_C_COMMON_SOURCE_URL}
+                      CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
+                      BUILD_BYPRODUCTS ${AWS_C_COMMON_STATIC_LIBRARY})
+  add_dependencies(AWS::aws-c-common aws_c_common_ep)
+
+  externalproject_add(aws_checksums_ep
+                      ${EP_LOG_OPTIONS}
+                      URL ${AWS_CHECKSUMS_SOURCE_URL}
+                      CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
+                      BUILD_BYPRODUCTS ${AWS_CHECKSUMS_STATIC_LIBRARY})
+  add_dependencies(AWS::aws-checksums aws_checksums_ep)
+
+  externalproject_add(aws_c_event_stream_ep
+                      ${EP_LOG_OPTIONS}
+                      URL ${AWS_C_EVENT_STREAM_SOURCE_URL}
+                      CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
+                      BUILD_BYPRODUCTS ${AWS_C_EVENT_STREAM_STATIC_LIBRARY}
+                      DEPENDS aws_c_common_ep aws_checksums_ep)
+  add_dependencies(AWS::aws-c-event-stream aws_c_event_stream_ep)
 
   externalproject_add(awssdk_ep
                       ${EP_LOG_OPTIONS}
                       URL ${AWSSDK_SOURCE_URL}
                       CMAKE_ARGS ${AWSSDK_CMAKE_ARGS}
-                      BUILD_BYPRODUCTS ${AWSSDK_SHARED_LIBS})
-
-  file(MAKE_DIRECTORY ${AWSSDK_INCLUDE_DIR})
-
+                      BUILD_BYPRODUCTS ${AWS_CPP_SDK_COGNITO_IDENTITY_STATIC_LIBRARY}
+                                       ${AWS_CPP_SDK_CORE_STATIC_LIBRARY}
+                                       ${AWS_CPP_SDK_IDENTITY_MANAGEMENT_STATIC_LIBRARY}
+                                       ${AWS_CPP_SDK_S3_STATIC_LIBRARY}
+                                       ${AWS_CPP_SDK_STS_STATIC_LIBRARY}
+                      DEPENDS aws_c_event_stream_ep)
   add_dependencies(toolchain awssdk_ep)
-  set(AWSSDK_LINK_LIBRARIES ${AWSSDK_SHARED_LIBS})
+  foreach(_AWSSDK_LIB ${_AWSSDK_LIBS})
+    if(${_AWSSDK_LIB} MATCHES "^aws-cpp-sdk-")
+      add_dependencies(${_AWSSDK_LIB} awssdk_ep)
+    endif()
+  endforeach()
+
   set(AWSSDK_VENDORED TRUE)
+  list(APPEND ARROW_BUNDLED_STATIC_LIBS ${AWSSDK_LIBRARIES})
+  set(AWSSDK_LINK_LIBRARIES ${AWSSDK_LIBRARIES})
+  if(UNIX)
+    # on linux and macos curl seems to be required
+    find_package(CURL REQUIRED)
+    if(NOT TARGET CURL::libcurl)
+      # For old FindCURL.cmake
+      add_library(CURL::libcurl UNKNOWN IMPORTED)
+      set_target_properties(CURL::libcurl
+                            PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
+                                       "${CURL_INCLUDE_DIRS}" IMPORTED_LOCATION
+                                       "${CURL_LIBRARIES}")
+    endif()
+    set_property(TARGET aws-cpp-sdk-core
+                 APPEND
+                 PROPERTY INTERFACE_LINK_LIBRARIES CURL::libcurl)
+    set_property(TARGET CURL::libcurl
+                 APPEND
+                 PROPERTY INTERFACE_LINK_LIBRARIES OpenSSL::SSL)
+    if(TARGET zlib_ep)
+      set_property(TARGET aws-cpp-sdk-core
+                   APPEND
+                   PROPERTY INTERFACE_LINK_LIBRARIES ZLIB::ZLIB)
+      add_dependencies(awssdk_ep zlib_ep)
+    endif()
+  endif()
 
-  # AWSSDK is shared-only build
+  # AWSSDK is static-only build
 endmacro()
 
 if(ARROW_S3)
   # See https://aws.amazon.com/blogs/developer/developer-experience-of-the-aws-sdk-for-c-now-simplified-by-cmake/
 
+  # Workaround to force AWS cmake configuration to look for shared libraries
+  if(DEFINED ENV{CONDA_PREFIX})
+    if(DEFINED BUILD_SHARED_LIBS)
+      set(BUILD_SHARED_LIBS_WAS_SET TRUE)
+      set(BUILD_SHARED_LIBS_VALUE ${BUILD_SHARED_LIBS})
+    else()
+      set(BUILD_SHARED_LIBS_WAS_SET FALSE)
+    endif()
+    set(BUILD_SHARED_LIBS "ON")
+  endif()
+
   # Need to customize the find_package() call, so cannot call resolve_dependency()
   if(AWSSDK_SOURCE STREQUAL "AUTO")
-    find_package(AWSSDK COMPONENTS config s3 transfer)
+    find_package(AWSSDK
+                 COMPONENTS config
+                            s3
+                            transfer
+                            identity-management
+                            sts)
     if(NOT AWSSDK_FOUND)
       build_awssdk()
     endif()
   elseif(AWSSDK_SOURCE STREQUAL "BUNDLED")
     build_awssdk()
   elseif(AWSSDK_SOURCE STREQUAL "SYSTEM")
-    find_package(AWSSDK REQUIRED COMPONENTS config s3 transfer)
+    find_package(AWSSDK REQUIRED
+                 COMPONENTS config
+                            s3
+                            transfer
+                            identity-management
+                            sts)
+  endif()
+
+  # Restore previous value of BUILD_SHARED_LIBS
+  if(DEFINED ENV{CONDA_PREFIX})
+    if(BUILD_SHARED_LIBS_WAS_SET)
+      set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_VALUE})
+    else()
+      unset(BUILD_SHARED_LIBS)
+    endif()
   endif()
 
   include_directories(SYSTEM ${AWSSDK_INCLUDE_DIR})
@@ -2697,6 +2875,6 @@ message(STATUS "All bundled static libraries: ${ARROW_BUNDLED_STATIC_LIBS}")
 
 # Write out the package configurations.
 
-configure_file("src/arrow/util/config.h.cmake" "src/arrow/util/config.h")
+configure_file("src/arrow/util/config.h.cmake" "src/arrow/util/config.h" ESCAPE_QUOTES)
 install(FILES "${ARROW_BINARY_DIR}/src/arrow/util/config.h"
         DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/util")
diff --git a/cpp/cmake_modules/UseCython.cmake b/cpp/cmake_modules/UseCython.cmake
index ccdeb4f3f03..0d4b17d3e57 100644
--- a/cpp/cmake_modules/UseCython.cmake
+++ b/cpp/cmake_modules/UseCython.cmake
@@ -22,7 +22,7 @@
 # (this is an inherent limitation of Cython).
 #
 # The sample paths set with the CMake include_directories() command will be used
-# for include directories to search for *.pxd when running the Cython complire.
+# for include directories to search for *.pxd when running the Cython compiler.
 #
 # Cache variables that effect the behavior include:
 #
diff --git a/cpp/cmake_modules/san-config.cmake b/cpp/cmake_modules/san-config.cmake
index 2e2807801fb..5eee6278009 100644
--- a/cpp/cmake_modules/san-config.cmake
+++ b/cpp/cmake_modules/san-config.cmake
@@ -35,14 +35,17 @@ endif()
 # - disable 'vptr' because of RTTI issues across shared libraries (?)
 # - disable 'alignment' because unaligned access is really OK on Nehalem and we do it
 #   all over the place.
-# - disable 'function' because it appears to give a false positive https://github.com/google/sanitizers/issues/911
+# - disable 'function' because it appears to give a false positive
+#   (https://github.com/google/sanitizers/issues/911)
+# - disable 'float-divide-by-zero' on clang, which considers it UB
+#   (https://bugs.llvm.org/show_bug.cgi?id=17000#c1)
 #   Note: GCC does not support the 'function' flag.
 if(${ARROW_USE_UBSAN})
   if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang"
      OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
     set(
       CMAKE_CXX_FLAGS
-      "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-sanitize=alignment,vptr,function -fno-sanitize-recover=all"
+      "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-sanitize=alignment,vptr,function,float-divide-by-zero -fno-sanitize-recover=all"
       )
   elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU"
          AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "5.1")
diff --git a/cpp/examples/arrow/row-wise-conversion-example.cc b/cpp/examples/arrow/row-wise-conversion-example.cc
index 42cab6cc76e..fb54b040f44 100644
--- a/cpp/examples/arrow/row-wise-conversion-example.cc
+++ b/cpp/examples/arrow/row-wise-conversion-example.cc
@@ -78,7 +78,7 @@ arrow::Status VectorToColumnarTable(const std::vector<struct data_row>& rows,
     // Indicate the start of a new list row. This will memorise the current
     // offset in the values builder.
     ARROW_RETURN_NOT_OK(components_builder.Append());
-    // Store the actual values. The final nullptr argument tells the underyling
+    // Store the actual values. The final nullptr argument tells the underlying
     // builder that all added values are valid, i.e. non-null.
     ARROW_RETURN_NOT_OK(cost_components_builder.AppendValues(row.cost_components.data(),
                                                              row.cost_components.size()));
diff --git a/cpp/examples/minimal_build/example.cc b/cpp/examples/minimal_build/example.cc
index 7c60093e95f..4b6acd2a0dd 100644
--- a/cpp/examples/minimal_build/example.cc
+++ b/cpp/examples/minimal_build/example.cc
@@ -52,8 +52,8 @@ Status RunMain(int argc, char** argv) {
   ARROW_ASSIGN_OR_RAISE(auto output_file,
                         arrow::io::FileOutputStream::Open(arrow_filename));
   ARROW_ASSIGN_OR_RAISE(auto batch_writer,
-                        arrow::ipc::NewFileWriter(output_file.get(),
-                                                  table->schema()));
+                        arrow::ipc::MakeFileWriter(output_file,
+                                                   table->schema()));
   ARROW_RETURN_NOT_OK(batch_writer->WriteTable(*table));
   ARROW_RETURN_NOT_OK(batch_writer->Close());
 
diff --git a/cpp/examples/parquet/parquet-stream-api/stream-reader-writer.cc b/cpp/examples/parquet/parquet-stream-api/stream-reader-writer.cc
index 5a255bff275..64e44803ccf 100644
--- a/cpp/examples/parquet/parquet-stream-api/stream-reader-writer.cc
+++ b/cpp/examples/parquet/parquet-stream-api/stream-reader-writer.cc
@@ -48,7 +48,8 @@ class UserTimestamp {
   bool operator==(const UserTimestamp& x) const { return ts_ == x.ts_; }
 
   void dump(std::ostream& os) const {
-    std::time_t t{std::chrono::duration_cast<std::chrono::seconds>(ts_).count()};
+    const auto t = static_cast<std::time_t>(
+        std::chrono::duration_cast<std::chrono::seconds>(ts_).count());
     os << std::put_time(std::gmtime(&t), "%Y%m%d-%H%M%S");
   }
 
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 328553c93c3..dd17720595a 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -180,6 +180,7 @@ set(ARROW_SRCS
     util/bitmap.cc
     util/bitmap_builders.cc
     util/bitmap_ops.cc
+    util/bpacking.cc
     util/compression.cc
     util/cpu_info.cc
     util/decimal.cc
@@ -213,6 +214,20 @@ set(ARROW_SRCS
     vendored/double-conversion/diy-fp.cc
     vendored/double-conversion/strtod.cc)
 
+if(ARROW_HAVE_RUNTIME_AVX2)
+  list(APPEND ARROW_SRCS util/bpacking_avx2.cc)
+  set_source_files_properties(util/bpacking_avx2.cc PROPERTIES SKIP_PRECOMPILE_HEADERS ON)
+  set_source_files_properties(util/bpacking_avx2.cc PROPERTIES COMPILE_FLAGS
+                              ${ARROW_AVX2_FLAG})
+endif()
+if(ARROW_HAVE_RUNTIME_AVX512)
+  list(APPEND ARROW_SRCS util/bpacking_avx512.cc)
+  set_source_files_properties(util/bpacking_avx512.cc PROPERTIES SKIP_PRECOMPILE_HEADERS
+                              ON)
+  set_source_files_properties(util/bpacking_avx512.cc PROPERTIES COMPILE_FLAGS
+                              ${ARROW_AVX512_FLAG})
+endif()
+
 if(APPLE)
   list(APPEND ARROW_SRCS vendored/datetime/ios.mm)
 endif()
@@ -350,6 +365,7 @@ if(ARROW_COMPUTE)
               compute/registry.cc
               compute/kernels/aggregate_basic.cc
               compute/kernels/aggregate_mode.cc
+              compute/kernels/aggregate_var_std.cc
               compute/kernels/codegen_internal.cc
               compute/kernels/scalar_arithmetic.cc
               compute/kernels/scalar_boolean.cc
@@ -371,18 +387,18 @@ if(ARROW_COMPUTE)
               compute/kernels/vector_selection.cc
               compute/kernels/vector_sort.cc)
 
-  if(CXX_SUPPORTS_AVX2)
-    list(APPEND ARROW_SRCS compute/kernels/aggregate_sum_avx2.cc)
-    set_source_files_properties(compute/kernels/aggregate_sum_avx2.cc PROPERTIES
+  if(ARROW_HAVE_RUNTIME_AVX2)
+    list(APPEND ARROW_SRCS compute/kernels/aggregate_basic_avx2.cc)
+    set_source_files_properties(compute/kernels/aggregate_basic_avx2.cc PROPERTIES
                                 SKIP_PRECOMPILE_HEADERS ON)
-    set_source_files_properties(compute/kernels/aggregate_sum_avx2.cc PROPERTIES
+    set_source_files_properties(compute/kernels/aggregate_basic_avx2.cc PROPERTIES
                                 COMPILE_FLAGS ${ARROW_AVX2_FLAG})
   endif()
-  if(CXX_SUPPORTS_AVX512)
-    list(APPEND ARROW_SRCS compute/kernels/aggregate_sum_avx512.cc)
-    set_source_files_properties(compute/kernels/aggregate_sum_avx512.cc PROPERTIES
+  if(ARROW_HAVE_RUNTIME_AVX512)
+    list(APPEND ARROW_SRCS compute/kernels/aggregate_basic_avx512.cc)
+    set_source_files_properties(compute/kernels/aggregate_basic_avx512.cc PROPERTIES
                                 SKIP_PRECOMPILE_HEADERS ON)
-    set_source_files_properties(compute/kernels/aggregate_sum_avx512.cc PROPERTIES
+    set_source_files_properties(compute/kernels/aggregate_basic_avx512.cc PROPERTIES
                                 COMPILE_FLAGS ${ARROW_AVX512_FLAG})
   endif()
 endif()
@@ -391,9 +407,6 @@ if(ARROW_FILESYSTEM)
   if(ARROW_HDFS)
     add_definitions(-DARROW_HDFS)
   endif()
-  if(ARROW_S3)
-    add_definitions(-DARROW_S3)
-  endif()
 
   list(APPEND ARROW_SRCS
               filesystem/filesystem.cc
@@ -448,7 +461,7 @@ if(ARROW_ORC)
   list(APPEND ARROW_SRCS adapters/orc/adapter.cc adapters/orc/adapter_util.cc)
 endif()
 
-if(NOT APPLE AND NOT MSVC)
+if(NOT APPLE AND NOT MSVC_TOOLCHAIN)
   # Localize thirdparty symbols using a linker version script. This hides them
   # from the client application. The OS X linker does not support the
   # version-script option.
@@ -514,11 +527,7 @@ if(ARROW_BUILD_STATIC AND ARROW_BUNDLED_STATIC_LIBS)
                            ${_OTHER_LIBS})
 endif()
 
-if(ARROW_BUILD_TESTS
-   OR ARROW_BUILD_BENCHMARKS
-   OR ARROW_BUILD_INTEGRATION
-   OR ARROW_FUZZING)
-
+if(ARROW_TESTING)
   # that depend on gtest
   add_arrow_lib(arrow_testing
                 CMAKE_PACKAGE_NAME
diff --git a/cpp/src/arrow/array/array_base.cc b/cpp/src/arrow/array/array_base.cc
index 0781dd4a2df..b2524afe4f8 100644
--- a/cpp/src/arrow/array/array_base.cc
+++ b/cpp/src/arrow/array/array_base.cc
@@ -73,6 +73,10 @@ struct ScalarFromArraySlotImpl {
     return Finish(Decimal128(a.GetValue(index_)));
   }
 
+  Status Visit(const Decimal256Array& a) {
+    return Finish(Decimal256(a.GetValue(index_)));
+  }
+
   template <typename T>
   Status Visit(const BaseBinaryArray<T>& a) {
     return Finish(a.GetString(index_));
@@ -161,7 +165,13 @@ struct ScalarFromArraySlotImpl {
     }
 
     if (array_.IsNull(index_)) {
-      return MakeNullScalar(array_.type());
+      auto null = MakeNullScalar(array_.type());
+      if (is_dictionary(array_.type()->id())) {
+        auto& dict_null = checked_cast<DictionaryScalar&>(*null);
+        const auto& dict_array = checked_cast<const DictionaryArray&>(array_);
+        dict_null.value.dictionary = dict_array.dictionary();
+      }
+      return null;
     }
 
     RETURN_NOT_OK(VisitArrayInline(array_, this));
diff --git a/cpp/src/arrow/array/array_binary_test.cc b/cpp/src/arrow/array/array_binary_test.cc
index 9c2cd888692..5c247a6dc66 100644
--- a/cpp/src/arrow/array/array_binary_test.cc
+++ b/cpp/src/arrow/array/array_binary_test.cc
@@ -25,8 +25,8 @@
 #include <gtest/gtest.h>
 
 #include "arrow/array.h"
+#include "arrow/array/builder_binary.h"
 #include "arrow/buffer.h"
-#include "arrow/builder.h"
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/testing/gtest_common.h"
@@ -570,6 +570,26 @@ class TestStringBuilder : public TestBuilder {
     ASSERT_EQ(reps * 40, result_->value_data()->size());
   }
 
+  void TestOverflowCheck() {
+    auto max_size = builder_->memory_limit();
+
+    ASSERT_OK(builder_->ValidateOverflow(1));
+    ASSERT_OK(builder_->ValidateOverflow(max_size));
+    ASSERT_RAISES(CapacityError, builder_->ValidateOverflow(max_size + 1));
+
+    ASSERT_OK(builder_->Append("bb"));
+    ASSERT_OK(builder_->ValidateOverflow(max_size - 2));
+    ASSERT_RAISES(CapacityError, builder_->ValidateOverflow(max_size - 1));
+
+    ASSERT_OK(builder_->AppendNull());
+    ASSERT_OK(builder_->ValidateOverflow(max_size - 2));
+    ASSERT_RAISES(CapacityError, builder_->ValidateOverflow(max_size - 1));
+
+    ASSERT_OK(builder_->Append("ccc"));
+    ASSERT_OK(builder_->ValidateOverflow(max_size - 5));
+    ASSERT_RAISES(CapacityError, builder_->ValidateOverflow(max_size - 4));
+  }
+
   void TestZeroLength() {
     // All buffers are null
     Done();
@@ -602,6 +622,8 @@ TYPED_TEST(TestStringBuilder, TestCapacityReserve) { this->TestCapacityReserve()
 
 TYPED_TEST(TestStringBuilder, TestZeroLength) { this->TestZeroLength(); }
 
+TYPED_TEST(TestStringBuilder, TestOverflowCheck) { this->TestOverflowCheck(); }
+
 // ----------------------------------------------------------------------
 // ChunkedBinaryBuilder tests
 
diff --git a/cpp/src/arrow/array/array_decimal.cc b/cpp/src/arrow/array/array_decimal.cc
index 1e813f2e515..d65f6ee5356 100644
--- a/cpp/src/arrow/array/array_decimal.cc
+++ b/cpp/src/arrow/array/array_decimal.cc
@@ -33,11 +33,11 @@ namespace arrow {
 using internal::checked_cast;
 
 // ----------------------------------------------------------------------
-// Decimal
+// Decimal128
 
 Decimal128Array::Decimal128Array(const std::shared_ptr<ArrayData>& data)
     : FixedSizeBinaryArray(data) {
-  ARROW_CHECK_EQ(data->type->id(), Type::DECIMAL);
+  ARROW_CHECK_EQ(data->type->id(), Type::DECIMAL128);
 }
 
 std::string Decimal128Array::FormatValue(int64_t i) const {
@@ -46,4 +46,18 @@ std::string Decimal128Array::FormatValue(int64_t i) const {
   return value.ToString(type_.scale());
 }
 
+// ----------------------------------------------------------------------
+// Decimal256
+
+Decimal256Array::Decimal256Array(const std::shared_ptr<ArrayData>& data)
+    : FixedSizeBinaryArray(data) {
+  ARROW_CHECK_EQ(data->type->id(), Type::DECIMAL256);
+}
+
+std::string Decimal256Array::FormatValue(int64_t i) const {
+  const auto& type_ = checked_cast<const Decimal256Type&>(*type());
+  const Decimal256 value(GetValue(i));
+  return value.ToString(type_.scale());
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/array/array_decimal.h b/cpp/src/arrow/array/array_decimal.h
index 6d5e884118b..8d7d1c59cd0 100644
--- a/cpp/src/arrow/array/array_decimal.h
+++ b/cpp/src/arrow/array/array_decimal.h
@@ -47,4 +47,20 @@ class ARROW_EXPORT Decimal128Array : public FixedSizeBinaryArray {
 // Backward compatibility
 using DecimalArray = Decimal128Array;
 
+// ----------------------------------------------------------------------
+// Decimal256Array
+
+/// Concrete Array class for 256-bit decimal data
+class ARROW_EXPORT Decimal256Array : public FixedSizeBinaryArray {
+ public:
+  using TypeClass = Decimal256Type;
+
+  using FixedSizeBinaryArray::FixedSizeBinaryArray;
+
+  /// \brief Construct Decimal256Array from ArrayData instance
+  explicit Decimal256Array(const std::shared_ptr<ArrayData>& data);
+
+  std::string FormatValue(int64_t i) const;
+};
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/array/array_dict.h b/cpp/src/arrow/array/array_dict.h
index c87606f7caf..ce1f49ce5fa 100644
--- a/cpp/src/arrow/array/array_dict.h
+++ b/cpp/src/arrow/array/array_dict.h
@@ -75,6 +75,12 @@ class ARROW_EXPORT DictionaryArray : public Array {
       const std::shared_ptr<DataType>& type, const std::shared_ptr<Array>& indices,
       const std::shared_ptr<Array>& dictionary);
 
+  static Result<std::shared_ptr<Array>> FromArrays(
+      const std::shared_ptr<Array>& indices, const std::shared_ptr<Array>& dictionary) {
+    return FromArrays(::arrow::dictionary(indices->type(), dictionary->type()), indices,
+                      dictionary);
+  }
+
   /// \brief Transpose this DictionaryArray
   ///
   /// This method constructs a new dictionary array with the given dictionary
diff --git a/cpp/src/arrow/array/array_dict_test.cc b/cpp/src/arrow/array/array_dict_test.cc
index 2ad31121d2a..fca442b2567 100644
--- a/cpp/src/arrow/array/array_dict_test.cc
+++ b/cpp/src/arrow/array/array_dict_test.cc
@@ -25,7 +25,9 @@
 #include <gtest/gtest.h>
 
 #include "arrow/array.h"
-#include "arrow/builder.h"
+#include "arrow/array/builder_decimal.h"
+#include "arrow/array/builder_dict.h"
+#include "arrow/array/builder_nested.h"
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/testing/gtest_common.h"
@@ -448,13 +450,14 @@ TEST(TestStringDictionaryBuilder, ArrayInit) {
   AssertArraysEqual(expected, *result);
 }
 
-TEST(TestStringDictionaryBuilder, MakeBuilder) {
-  auto dict_array = ArrayFromJSON(utf8(), R"(["test", "test2"])");
-  auto dict_type = dictionary(int8(), utf8());
+template <typename BuilderType>
+void TestStringDictionaryMakeBuilder(const std::shared_ptr<DataType>& value_type) {
+  auto dict_array = ArrayFromJSON(value_type, R"(["test", "test2"])");
+  auto dict_type = dictionary(int8(), value_type);
   auto int_array = ArrayFromJSON(int8(), "[0, 1, 0]");
   std::unique_ptr<ArrayBuilder> boxed_builder;
   ASSERT_OK(MakeBuilder(default_memory_pool(), dict_type, &boxed_builder));
-  auto& builder = checked_cast<StringDictionaryBuilder&>(*boxed_builder);
+  auto& builder = checked_cast<BuilderType&>(*boxed_builder);
 
   // Build the dictionary Array
   ASSERT_OK(builder.Append("test"));
@@ -470,6 +473,14 @@ TEST(TestStringDictionaryBuilder, MakeBuilder) {
   AssertArraysEqual(expected, *result);
 }
 
+TEST(TestStringDictionaryBuilder, MakeBuilder) {
+  TestStringDictionaryMakeBuilder<DictionaryBuilder<StringType>>(utf8());
+}
+
+TEST(TestLargeStringDictionaryBuilder, MakeBuilder) {
+  TestStringDictionaryMakeBuilder<DictionaryBuilder<LargeStringType>>(large_utf8());
+}
+
 // ARROW-4367
 TEST(TestStringDictionaryBuilder, OnlyNull) {
   // Build the dictionary Array
@@ -814,27 +825,25 @@ TEST(TestFixedSizeBinaryDictionaryBuilder, DoubleTableSize) {
   ASSERT_TRUE(expected.Equals(result));
 }
 
-TEST(TestFixedSizeBinaryDictionaryBuilder, InvalidTypeAppend) {
+#ifndef NDEBUG
+TEST(TestFixedSizeBinaryDictionaryBuilder, AppendArrayInvalidType) {
   // Build the dictionary Array
-  auto value_type = arrow::fixed_size_binary(4);
+  auto value_type = fixed_size_binary(4);
   DictionaryBuilder<FixedSizeBinaryType> builder(value_type);
   // Build an array with different byte width
-  FixedSizeBinaryBuilder fsb_builder(arrow::fixed_size_binary(5));
-  std::vector<uint8_t> value{100, 1, 1, 1, 1};
-  ASSERT_OK(fsb_builder.Append(value.data()));
-  std::shared_ptr<Array> fsb_array;
-  ASSERT_OK(fsb_builder.Finish(&fsb_array));
+  auto fsb_array = ArrayFromJSON(fixed_size_binary(3), R"(["foo", "bar"])");
 
-  ASSERT_RAISES(Invalid, builder.AppendArray(*fsb_array));
+  ASSERT_RAISES(TypeError, builder.AppendArray(*fsb_array));
 }
+#endif
 
-TEST(TestDecimalDictionaryBuilder, Basic) {
+template <typename DecimalValue>
+void TestDecimalDictionaryBuilderBasic(std::shared_ptr<DataType> decimal_type) {
   // Build the dictionary Array
-  auto decimal_type = arrow::decimal(2, 0);
   DictionaryBuilder<FixedSizeBinaryType> builder(decimal_type);
 
   // Test data
-  std::vector<Decimal128> test{12, 12, 11, 12};
+  std::vector<DecimalValue> test{12, 12, 11, 12};
   for (const auto& value : test) {
     ASSERT_OK(builder.Append(value.ToBytes().data()));
   }
@@ -850,40 +859,48 @@ TEST(TestDecimalDictionaryBuilder, Basic) {
   ASSERT_TRUE(expected.Equals(result));
 }
 
-TEST(TestDecimalDictionaryBuilder, DoubleTableSize) {
-  const auto& decimal_type = arrow::decimal(21, 0);
+TEST(TestDecimal128DictionaryBuilder, Basic) {
+  TestDecimalDictionaryBuilderBasic<Decimal128>(arrow::decimal128(2, 0));
+}
+
+TEST(TestDecimal256DictionaryBuilder, Basic) {
+  TestDecimalDictionaryBuilderBasic<Decimal256>(arrow::decimal256(76, 0));
+}
 
+void TestDecimalDictionaryBuilderDoubleTableSize(
+    std::shared_ptr<DataType> decimal_type, FixedSizeBinaryBuilder& decimal_builder) {
   // Build the dictionary Array
   DictionaryBuilder<FixedSizeBinaryType> dict_builder(decimal_type);
 
   // Build expected data
-  Decimal128Builder decimal_builder(decimal_type);
   Int16Builder int_builder;
 
   // Fill with 1024 different values
   for (int64_t i = 0; i < 1024; i++) {
-    const uint8_t bytes[] = {0,
-                             0,
-                             0,
-                             0,
-                             0,
-                             0,
-                             0,
-                             0,
-                             0,
-                             0,
-                             0,
-                             0,
-                             12,
-                             12,
-                             static_cast<uint8_t>(i / 128),
-                             static_cast<uint8_t>(i % 128)};
+    // Decimal256Builder takes 32 bytes, while Decimal128Builder takes only the first 16
+    // bytes.
+    const uint8_t bytes[32] = {0,
+                               0,
+                               0,
+                               0,
+                               0,
+                               0,
+                               0,
+                               0,
+                               0,
+                               0,
+                               0,
+                               0,
+                               12,
+                               12,
+                               static_cast<uint8_t>(i / 128),
+                               static_cast<uint8_t>(i % 128)};
     ASSERT_OK(dict_builder.Append(bytes));
     ASSERT_OK(decimal_builder.Append(bytes));
     ASSERT_OK(int_builder.Append(static_cast<uint16_t>(i)));
   }
   // Fill with an already existing value
-  const uint8_t known_value[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 0, 1};
+  const uint8_t known_value[32] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 0, 1};
   for (int64_t i = 0; i < 1024; i++) {
     ASSERT_OK(dict_builder.Append(known_value));
     ASSERT_OK(int_builder.Append(1));
@@ -904,6 +921,90 @@ TEST(TestDecimalDictionaryBuilder, DoubleTableSize) {
   ASSERT_TRUE(expected.Equals(result));
 }
 
+TEST(TestDecimal128DictionaryBuilder, DoubleTableSize) {
+  const auto& decimal_type = arrow::decimal128(21, 0);
+  Decimal128Builder decimal_builder(decimal_type);
+  TestDecimalDictionaryBuilderDoubleTableSize(decimal_type, decimal_builder);
+}
+
+TEST(TestDecimal256DictionaryBuilder, DoubleTableSize) {
+  const auto& decimal_type = arrow::decimal256(21, 0);
+  Decimal256Builder decimal_builder(decimal_type);
+  TestDecimalDictionaryBuilderDoubleTableSize(decimal_type, decimal_builder);
+}
+
+TEST(TestNullDictionaryBuilder, Basic) {
+  // MakeBuilder
+  auto dict_type = dictionary(int8(), null());
+  std::unique_ptr<ArrayBuilder> boxed_builder;
+  ASSERT_OK(MakeBuilder(default_memory_pool(), dict_type, &boxed_builder));
+  auto& builder = checked_cast<DictionaryBuilder<NullType>&>(*boxed_builder);
+
+  ASSERT_OK(builder.AppendNull());
+  ASSERT_OK(builder.AppendNull());
+  ASSERT_OK(builder.AppendNull());
+  ASSERT_EQ(3, builder.length());
+  ASSERT_EQ(3, builder.null_count());
+
+  ASSERT_OK(builder.AppendNulls(4));
+  ASSERT_EQ(7, builder.length());
+  ASSERT_EQ(7, builder.null_count());
+
+  auto null_array = ArrayFromJSON(null(), "[null, null, null, null]");
+  ASSERT_OK(builder.AppendArray(*null_array));
+  ASSERT_EQ(11, builder.length());
+  ASSERT_EQ(11, builder.null_count());
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+  AssertTypeEqual(*dict_type, *result->type());
+  ASSERT_EQ(11, result->length());
+  ASSERT_EQ(11, result->null_count());
+}
+
+#ifndef NDEBUG
+TEST(TestNullDictionaryBuilder, AppendArrayInvalidType) {
+  // MakeBuilder
+  auto dict_type = dictionary(int8(), null());
+  std::unique_ptr<ArrayBuilder> boxed_builder;
+  ASSERT_OK(MakeBuilder(default_memory_pool(), dict_type, &boxed_builder));
+  auto& builder = checked_cast<DictionaryBuilder<NullType>&>(*boxed_builder);
+
+  auto int8_array = ArrayFromJSON(int8(), "[0, 1, 0, null]");
+  ASSERT_RAISES(TypeError, builder.AppendArray(*int8_array));
+}
+#endif
+
+// ----------------------------------------------------------------------
+// Index byte width tests
+
+template <typename IndexType, typename ValueType>
+void AssertIndexByteWidth(const std::shared_ptr<DataType>& value_type =
+                              TypeTraits<ValueType>::type_singleton()) {
+  auto index_type = TypeTraits<IndexType>::type_singleton();
+  auto dict_type =
+      checked_pointer_cast<DictionaryType>(dictionary(index_type, value_type));
+  std::unique_ptr<ArrayBuilder> builder;
+  ASSERT_OK(MakeBuilder(default_memory_pool(), dict_type, &builder));
+  auto builder_dict_type = checked_pointer_cast<DictionaryType>(builder->type());
+  AssertTypeEqual(dict_type->index_type(), builder_dict_type->index_type());
+}
+
+typedef ::testing::Types<Int8Type, Int16Type, Int32Type, Int64Type> IndexTypes;
+
+template <typename Type>
+class TestDictionaryBuilderIndexByteWidth : public TestBuilder {};
+
+TYPED_TEST_SUITE(TestDictionaryBuilderIndexByteWidth, IndexTypes);
+
+TYPED_TEST(TestDictionaryBuilderIndexByteWidth, MakeBuilder) {
+  AssertIndexByteWidth<TypeParam, FloatType>();
+  AssertIndexByteWidth<TypeParam, BinaryType>();
+  AssertIndexByteWidth<TypeParam, StringType>();
+  AssertIndexByteWidth<TypeParam, FixedSizeBinaryType>(fixed_size_binary(4));
+  AssertIndexByteWidth<TypeParam, NullType>();
+}
+
 // ----------------------------------------------------------------------
 // DictionaryArray tests
 
diff --git a/cpp/src/arrow/array/array_list_test.cc b/cpp/src/arrow/array/array_list_test.cc
index df0eb522cf4..1696653850b 100644
--- a/cpp/src/arrow/array/array_list_test.cc
+++ b/cpp/src/arrow/array/array_list_test.cc
@@ -23,8 +23,8 @@
 #include <gtest/gtest.h>
 
 #include "arrow/array.h"
+#include "arrow/array/builder_nested.h"
 #include "arrow/buffer.h"
-#include "arrow/builder.h"
 #include "arrow/status.h"
 #include "arrow/testing/gtest_common.h"
 #include "arrow/testing/gtest_util.h"
@@ -467,6 +467,32 @@ class TestListArray : public TestBuilder {
     AssertArraysEqual(*result_, *expected);
   }
 
+  void TestOverflowCheck() {
+    Int16Builder* vb = checked_cast<Int16Builder*>(builder_->value_builder());
+    auto max_elements = builder_->maximum_elements();
+
+    ASSERT_OK(builder_->ValidateOverflow(1));
+    ASSERT_OK(builder_->ValidateOverflow(max_elements));
+    ASSERT_RAISES(CapacityError, builder_->ValidateOverflow(max_elements + 1));
+
+    ASSERT_OK(builder_->Append());
+    ASSERT_OK(vb->Append(1));
+    ASSERT_OK(vb->Append(2));
+    ASSERT_OK(builder_->ValidateOverflow(max_elements - 2));
+    ASSERT_RAISES(CapacityError, builder_->ValidateOverflow(max_elements - 1));
+
+    ASSERT_OK(builder_->AppendNull());
+    ASSERT_OK(builder_->ValidateOverflow(max_elements - 2));
+    ASSERT_RAISES(CapacityError, builder_->ValidateOverflow(max_elements - 1));
+
+    ASSERT_OK(builder_->Append());
+    ASSERT_OK(vb->Append(1));
+    ASSERT_OK(vb->Append(2));
+    ASSERT_OK(vb->Append(3));
+    ASSERT_OK(builder_->ValidateOverflow(max_elements - 5));
+    ASSERT_RAISES(CapacityError, builder_->ValidateOverflow(max_elements - 4));
+  }
+
  protected:
   std::shared_ptr<DataType> value_type_;
 
@@ -508,6 +534,12 @@ TYPED_TEST(TestListArray, ValidateOffsets) { this->TestValidateOffsets(); }
 
 TYPED_TEST(TestListArray, CornerCases) { this->TestCornerCases(); }
 
+#ifndef ARROW_LARGE_MEMORY_TESTS
+TYPED_TEST(TestListArray, DISABLED_TestOverflowCheck) { this->TestOverflowCheck(); }
+#else
+TYPED_TEST(TestListArray, TestOverflowCheck) { this->TestOverflowCheck(); }
+#endif
+
 // ----------------------------------------------------------------------
 // Map tests
 
diff --git a/cpp/src/arrow/array/array_struct_test.cc b/cpp/src/arrow/array/array_struct_test.cc
index 0afadcf9285..f54b43465e9 100644
--- a/cpp/src/arrow/array/array_struct_test.cc
+++ b/cpp/src/arrow/array/array_struct_test.cc
@@ -15,15 +15,15 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <gtest/gtest.h>
+
 #include <cstdint>
 #include <cstring>
 #include <memory>
 #include <vector>
 
-#include <gtest/gtest.h>
-
 #include "arrow/array.h"
-#include "arrow/builder.h"
+#include "arrow/array/builder_nested.h"
 #include "arrow/status.h"
 #include "arrow/testing/gtest_common.h"
 #include "arrow/testing/gtest_util.h"
@@ -266,10 +266,8 @@ TEST_F(TestStructBuilder, TestAppendNull) {
   ASSERT_EQ(2, result_->field(1)->length());
   ASSERT_TRUE(result_->IsNull(0));
   ASSERT_TRUE(result_->IsNull(1));
-  ASSERT_TRUE(result_->field(0)->IsNull(0));
-  ASSERT_TRUE(result_->field(0)->IsNull(1));
-  ASSERT_TRUE(result_->field(1)->IsNull(0));
-  ASSERT_TRUE(result_->field(1)->IsNull(1));
+  ASSERT_EQ(0, result_->field(0)->null_count());
+  ASSERT_EQ(0, result_->field(1)->null_count());
 
   ASSERT_EQ(Type::LIST, result_->field(0)->type_id());
   ASSERT_EQ(Type::INT32, result_->field(1)->type_id());
diff --git a/cpp/src/arrow/array/array_test.cc b/cpp/src/arrow/array/array_test.cc
index 2702c355c01..89087ee318c 100644
--- a/cpp/src/arrow/array/array_test.cc
+++ b/cpp/src/arrow/array/array_test.cc
@@ -16,11 +16,9 @@
 // under the License.
 
 #include <algorithm>
-#include <array>
 #include <cmath>
 #include <cstdint>
 #include <cstring>
-#include <iterator>
 #include <limits>
 #include <memory>
 #include <numeric>
@@ -43,9 +41,7 @@
 #include "arrow/array/util.h"
 #include "arrow/buffer.h"
 #include "arrow/buffer_builder.h"
-#include "arrow/builder.h"
 #include "arrow/compare.h"
-#include "arrow/memory_pool.h"
 #include "arrow/result.h"
 #include "arrow/scalar.h"
 #include "arrow/status.h"
@@ -70,6 +66,7 @@
 namespace arrow {
 
 using internal::checked_cast;
+using internal::checked_pointer_cast;
 
 class TestArray : public ::testing::Test {
  public:
@@ -234,6 +231,9 @@ TEST_F(TestArray, SliceRecomputeNullCount) {
   slice = array->Slice(4);
   ASSERT_EQ(4, slice->null_count());
 
+  auto slice2 = slice->Slice(0);
+  ASSERT_EQ(4, slice2->null_count());
+
   slice = array->Slice(0);
   ASSERT_EQ(5, slice->null_count());
 
@@ -425,6 +425,7 @@ TEST_F(TestArray, TestMakeArrayFromScalar) {
       std::make_shared<FixedSizeBinaryScalar>(
           hello, fixed_size_binary(static_cast<int32_t>(hello->size()))),
       std::make_shared<Decimal128Scalar>(Decimal128(10), decimal(16, 4)),
+      std::make_shared<Decimal256Scalar>(Decimal256(10), decimal(76, 38)),
       std::make_shared<StringScalar>(hello),
       std::make_shared<LargeStringScalar>(hello),
       std::make_shared<ListScalar>(ArrayFromJSON(int8(), "[1, 2, 3]")),
@@ -638,7 +639,7 @@ class TestPrimitiveBuilder : public TestBuilder {
     std::shared_ptr<Array> out;
     FinishAndCheckPadding(builder.get(), &out);
 
-    std::shared_ptr<ArrayType> result = std::dynamic_pointer_cast<ArrayType>(out);
+    std::shared_ptr<ArrayType> result = checked_pointer_cast<ArrayType>(out);
 
     // Builder is now reset
     ASSERT_EQ(0, builder->length());
@@ -763,7 +764,7 @@ void TestPrimitiveBuilder<PBoolean>::Check(const std::unique_ptr<BooleanBuilder>
   std::shared_ptr<Array> out;
   FinishAndCheckPadding(builder.get(), &out);
 
-  std::shared_ptr<BooleanArray> result = std::dynamic_pointer_cast<BooleanArray>(out);
+  std::shared_ptr<BooleanArray> result = checked_pointer_cast<BooleanArray>(out);
 
   ASSERT_EQ(ex_null_count, result->null_count());
   ASSERT_EQ(size, result->length());
@@ -880,7 +881,7 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendNull) {
 
   std::shared_ptr<Array> out;
   FinishAndCheckPadding(this->builder_.get(), &out);
-  auto result = std::dynamic_pointer_cast<typename TypeParam::ArrayType>(out);
+  auto result = checked_pointer_cast<typename TypeParam::ArrayType>(out);
 
   for (int64_t i = 0; i < size; ++i) {
     ASSERT_TRUE(result->IsNull(i)) << i;
@@ -914,6 +915,33 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendNulls) {
   }
 }
 
+TYPED_TEST(TestPrimitiveBuilder, TestAppendEmptyValue) {
+  ASSERT_OK(this->builder_->AppendNull());
+  ASSERT_OK(this->builder_->AppendEmptyValue());
+  ASSERT_OK(this->builder_->AppendNulls(2));
+  ASSERT_OK(this->builder_->AppendEmptyValues(2));
+
+  std::shared_ptr<Array> out;
+  FinishAndCheckPadding(this->builder_.get(), &out);
+  ASSERT_OK(out->ValidateFull());
+
+  auto result = checked_pointer_cast<typename TypeParam::ArrayType>(out);
+  ASSERT_EQ(result->length(), 6);
+  ASSERT_EQ(result->null_count(), 3);
+
+  ASSERT_TRUE(result->IsNull(0));
+  ASSERT_FALSE(result->IsNull(1));
+  ASSERT_TRUE(result->IsNull(2));
+  ASSERT_TRUE(result->IsNull(3));
+  ASSERT_FALSE(result->IsNull(4));
+  ASSERT_FALSE(result->IsNull(5));
+
+  // implementation detail: the value slots are 0-initialized
+  for (int64_t i = 0; i < result->length(); ++i) {
+    ASSERT_EQ(result->Value(i), 0);
+  }
+}
+
 TYPED_TEST(TestPrimitiveBuilder, TestArrayDtorDealloc) {
   DECL_T();
 
@@ -1501,7 +1529,7 @@ void CheckFloatingNanEquality() {
 
   // NaN != non-NaN
   ArrayFromVector<TYPE>(type, {false, true}, {0.5, nan_value}, &a);
-  ArrayFromVector<TYPE>(type, {false, true}, {0.5, 0.0}, &a);
+  ArrayFromVector<TYPE>(type, {false, true}, {0.5, 0.0}, &b);
   ASSERT_FALSE(a->Equals(b));
   ASSERT_FALSE(b->Equals(a));
   ASSERT_FALSE(a->Equals(b, EqualOptions().nans_equal(true)));
@@ -1520,6 +1548,73 @@ void CheckFloatingNanEquality() {
   ASSERT_TRUE(b->RangeEquals(a, 0, 1, 0));
 }
 
+template <typename TYPE>
+void CheckFloatingInfinityEquality() {
+  std::shared_ptr<Array> a, b;
+  std::shared_ptr<DataType> type = TypeTraits<TYPE>::type_singleton();
+
+  const auto infinity = std::numeric_limits<typename TYPE::c_type>::infinity();
+
+  for (auto nans_equal : {false, true}) {
+    // Infinity in a null entry
+    ArrayFromVector<TYPE>(type, {true, false}, {0.5, infinity}, &a);
+    ArrayFromVector<TYPE>(type, {true, false}, {0.5, -infinity}, &b);
+    ASSERT_TRUE(a->Equals(b));
+    ASSERT_TRUE(b->Equals(a));
+    ASSERT_TRUE(a->ApproxEquals(b, EqualOptions().atol(1e-5).nans_equal(nans_equal)));
+    ASSERT_TRUE(b->ApproxEquals(a, EqualOptions().atol(1e-5).nans_equal(nans_equal)));
+    ASSERT_TRUE(a->RangeEquals(b, 0, 2, 0));
+    ASSERT_TRUE(b->RangeEquals(a, 0, 2, 0));
+    ASSERT_TRUE(a->RangeEquals(b, 1, 2, 1));
+    ASSERT_TRUE(b->RangeEquals(a, 1, 2, 1));
+
+    // Infinity in a valid entry
+    ArrayFromVector<TYPE>(type, {false, true}, {0.5, infinity}, &a);
+    ArrayFromVector<TYPE>(type, {false, true}, {0.5, infinity}, &b);
+    ASSERT_TRUE(a->Equals(b));
+    ASSERT_TRUE(b->Equals(a));
+    ASSERT_TRUE(a->ApproxEquals(b, EqualOptions().atol(1e-5).nans_equal(nans_equal)));
+    ASSERT_TRUE(b->ApproxEquals(a, EqualOptions().atol(1e-5).nans_equal(nans_equal)));
+    ASSERT_TRUE(a->ApproxEquals(b, EqualOptions().atol(1e-5).nans_equal(nans_equal)));
+    ASSERT_TRUE(b->ApproxEquals(a, EqualOptions().atol(1e-5).nans_equal(nans_equal)));
+    // Infinity in tested range
+    ASSERT_TRUE(a->RangeEquals(b, 0, 2, 0));
+    ASSERT_TRUE(b->RangeEquals(a, 0, 2, 0));
+    ASSERT_TRUE(a->RangeEquals(b, 1, 2, 1));
+    ASSERT_TRUE(b->RangeEquals(a, 1, 2, 1));
+    // Infinity not in tested range
+    ASSERT_TRUE(a->RangeEquals(b, 0, 1, 0));
+    ASSERT_TRUE(b->RangeEquals(a, 0, 1, 0));
+
+    // Infinity != non-infinity
+    ArrayFromVector<TYPE>(type, {false, true}, {0.5, -infinity}, &a);
+    ArrayFromVector<TYPE>(type, {false, true}, {0.5, 0.0}, &b);
+    ASSERT_FALSE(a->Equals(b));
+    ASSERT_FALSE(b->Equals(a));
+    ASSERT_FALSE(a->ApproxEquals(b, EqualOptions().atol(1e-5).nans_equal(nans_equal)));
+    ASSERT_FALSE(b->ApproxEquals(a));
+    ASSERT_FALSE(a->ApproxEquals(b, EqualOptions().atol(1e-5).nans_equal(nans_equal)));
+    ASSERT_FALSE(b->ApproxEquals(a, EqualOptions().atol(1e-5).nans_equal(nans_equal)));
+    // Infinity != Negative infinity
+    ArrayFromVector<TYPE>(type, {true, true}, {0.5, -infinity}, &a);
+    ArrayFromVector<TYPE>(type, {true, true}, {0.5, infinity}, &b);
+    ASSERT_FALSE(a->Equals(b));
+    ASSERT_FALSE(b->Equals(a));
+    ASSERT_FALSE(a->ApproxEquals(b));
+    ASSERT_FALSE(b->ApproxEquals(a));
+    ASSERT_FALSE(a->ApproxEquals(b, EqualOptions().atol(1e-5).nans_equal(nans_equal)));
+    ASSERT_FALSE(b->ApproxEquals(a, EqualOptions().atol(1e-5).nans_equal(nans_equal)));
+    // Infinity in tested range
+    ASSERT_FALSE(a->RangeEquals(b, 0, 2, 0));
+    ASSERT_FALSE(b->RangeEquals(a, 0, 2, 0));
+    ASSERT_FALSE(a->RangeEquals(b, 1, 2, 1));
+    ASSERT_FALSE(b->RangeEquals(a, 1, 2, 1));
+    // Infinity not in tested range
+    ASSERT_TRUE(a->RangeEquals(b, 0, 1, 0));
+    ASSERT_TRUE(b->RangeEquals(a, 0, 1, 0));
+  }
+}
+
 TEST(TestPrimitiveAdHoc, FloatingApproxEquals) {
   CheckApproxEquals<FloatType>();
   CheckApproxEquals<DoubleType>();
@@ -1535,6 +1630,11 @@ TEST(TestPrimitiveAdHoc, FloatingNanEquality) {
   CheckFloatingNanEquality<DoubleType>();
 }
 
+TEST(TestPrimitiveAdHoc, FloatingInfinityEquality) {
+  CheckFloatingInfinityEquality<FloatType>();
+  CheckFloatingInfinityEquality<DoubleType>();
+}
+
 // ----------------------------------------------------------------------
 // FixedSizeBinary tests
 
@@ -2029,6 +2129,31 @@ TEST_F(TestAdaptiveIntBuilder, TestAppendNulls) {
   }
 }
 
+TEST_F(TestAdaptiveIntBuilder, TestAppendEmptyValue) {
+  ASSERT_OK(builder_->AppendNulls(2));
+  ASSERT_OK(builder_->AppendEmptyValue());
+  ASSERT_OK(builder_->Append(42));
+  ASSERT_OK(builder_->AppendEmptyValues(2));
+  Done();
+
+  ASSERT_OK(result_->ValidateFull());
+  // NOTE: The fact that we get 0 is really an implementation detail
+  AssertArraysEqual(*result_, *ArrayFromJSON(int8(), "[null, null, 0, 42, 0, 0]"));
+}
+
+TEST(TestAdaptiveIntBuilderWithStartIntSize, TestReset) {
+  auto builder = std::make_shared<AdaptiveIntBuilder>(
+      static_cast<uint8_t>(sizeof(int16_t)), default_memory_pool());
+  AssertTypeEqual(*int16(), *builder->type());
+
+  ASSERT_OK(
+      builder->Append(static_cast<int64_t>(std::numeric_limits<int16_t>::max()) + 1));
+  AssertTypeEqual(*int32(), *builder->type());
+
+  builder->Reset();
+  AssertTypeEqual(*int16(), *builder->type());
+}
+
 class TestAdaptiveUIntBuilder : public TestBuilder {
  public:
   void SetUp() {
@@ -2234,13 +2359,42 @@ TEST_F(TestAdaptiveUIntBuilder, TestAppendNulls) {
   }
 }
 
+TEST_F(TestAdaptiveUIntBuilder, TestAppendEmptyValue) {
+  ASSERT_OK(builder_->AppendNulls(2));
+  ASSERT_OK(builder_->AppendEmptyValue());
+  ASSERT_OK(builder_->Append(42));
+  ASSERT_OK(builder_->AppendEmptyValues(2));
+  Done();
+
+  ASSERT_OK(result_->ValidateFull());
+  // NOTE: The fact that we get 0 is really an implementation detail
+  AssertArraysEqual(*result_, *ArrayFromJSON(uint8(), "[null, null, 0, 42, 0, 0]"));
+}
+
+TEST(TestAdaptiveUIntBuilderWithStartIntSize, TestReset) {
+  auto builder = std::make_shared<AdaptiveUIntBuilder>(
+      static_cast<uint8_t>(sizeof(uint16_t)), default_memory_pool());
+  AssertTypeEqual(uint16(), builder->type());
+
+  ASSERT_OK(
+      builder->Append(static_cast<uint64_t>(std::numeric_limits<uint16_t>::max()) + 1));
+  AssertTypeEqual(uint32(), builder->type());
+
+  builder->Reset();
+  AssertTypeEqual(uint16(), builder->type());
+}
+
 // ----------------------------------------------------------------------
 // Test Decimal arrays
 
-using DecimalVector = std::vector<Decimal128>;
-
+template <typename TYPE>
 class DecimalTest : public ::testing::TestWithParam<int> {
  public:
+  using DecimalBuilder = typename TypeTraits<TYPE>::BuilderType;
+  using DecimalValue = typename TypeTraits<TYPE>::ScalarType::ValueType;
+  using DecimalArray = typename TypeTraits<TYPE>::ArrayType;
+  using DecimalVector = std::vector<DecimalValue>;
+
   DecimalTest() {}
 
   template <size_t BYTE_WIDTH = 16>
@@ -2256,8 +2410,8 @@ class DecimalTest : public ::testing::TestWithParam<int> {
   template <size_t BYTE_WIDTH = 16>
   void TestCreate(int32_t precision, const DecimalVector& draw,
                   const std::vector<uint8_t>& valid_bytes, int64_t offset) const {
-    auto type = std::make_shared<Decimal128Type>(precision, 4);
-    auto builder = std::make_shared<Decimal128Builder>(type);
+    auto type = std::make_shared<TYPE>(precision, 4);
+    auto builder = std::make_shared<DecimalBuilder>(type);
 
     size_t null_count = 0;
 
@@ -2288,7 +2442,7 @@ class DecimalTest : public ::testing::TestWithParam<int> {
     ASSERT_OK_AND_ASSIGN(expected_null_bitmap, internal::BytesToBits(valid_bytes));
 
     int64_t expected_null_count = CountNulls(valid_bytes);
-    auto expected = std::make_shared<Decimal128Array>(
+    auto expected = std::make_shared<DecimalArray>(
         type, size, expected_data, expected_null_bitmap, expected_null_count);
 
     std::shared_ptr<Array> lhs = out->Slice(offset);
@@ -2297,7 +2451,9 @@ class DecimalTest : public ::testing::TestWithParam<int> {
   }
 };
 
-TEST_P(DecimalTest, NoNulls) {
+using Decimal128Test = DecimalTest<Decimal128Type>;
+
+TEST_P(Decimal128Test, NoNulls) {
   int32_t precision = GetParam();
   std::vector<Decimal128> draw = {Decimal128(1), Decimal128(-2), Decimal128(2389),
                                   Decimal128(4), Decimal128(-12348)};
@@ -2306,7 +2462,7 @@ TEST_P(DecimalTest, NoNulls) {
   this->TestCreate(precision, draw, valid_bytes, 2);
 }
 
-TEST_P(DecimalTest, WithNulls) {
+TEST_P(Decimal128Test, WithNulls) {
   int32_t precision = GetParam();
   std::vector<Decimal128> draw = {Decimal128(1), Decimal128(2),  Decimal128(-1),
                                   Decimal128(4), Decimal128(-1), Decimal128(1),
@@ -2325,7 +2481,44 @@ TEST_P(DecimalTest, WithNulls) {
   this->TestCreate(precision, draw, valid_bytes, 2);
 }
 
-INSTANTIATE_TEST_SUITE_P(DecimalTest, DecimalTest, ::testing::Range(1, 38));
+INSTANTIATE_TEST_SUITE_P(Decimal128Test, Decimal128Test, ::testing::Range(1, 38));
+
+using Decimal256Test = DecimalTest<Decimal256Type>;
+
+TEST_P(Decimal256Test, NoNulls) {
+  int32_t precision = GetParam();
+  std::vector<Decimal256> draw = {Decimal256(1), Decimal256(-2), Decimal256(2389),
+                                  Decimal256(4), Decimal256(-12348)};
+  std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
+  this->TestCreate(precision, draw, valid_bytes, 0);
+  this->TestCreate(precision, draw, valid_bytes, 2);
+}
+
+TEST_P(Decimal256Test, WithNulls) {
+  int32_t precision = GetParam();
+  std::vector<Decimal256> draw = {Decimal256(1), Decimal256(2),  Decimal256(-1),
+                                  Decimal256(4), Decimal256(-1), Decimal256(1),
+                                  Decimal256(2)};
+  Decimal256 big;  // (pow(2, 255) - 1) / pow(10, 38)
+  ASSERT_OK_AND_ASSIGN(big,
+                       Decimal256::FromString("578960446186580977117854925043439539266."
+                                              "34992332820282019728792003956564819967"));
+  draw.push_back(big);
+
+  Decimal256 big_negative;  // -pow(2, 255) / pow(10, 38)
+  ASSERT_OK_AND_ASSIGN(big_negative,
+                       Decimal256::FromString("-578960446186580977117854925043439539266."
+                                              "34992332820282019728792003956564819968"));
+  draw.push_back(big_negative);
+
+  std::vector<uint8_t> valid_bytes = {true, true, false, true, false,
+                                      true, true, true,  true};
+  this->TestCreate(precision, draw, valid_bytes, 0);
+  this->TestCreate(precision, draw, valid_bytes, 2);
+}
+
+INSTANTIATE_TEST_SUITE_P(Decimal256Test, Decimal256Test,
+                         ::testing::Values(1, 2, 5, 10, 38, 39, 40, 75, 76));
 
 // ----------------------------------------------------------------------
 // Test rechunking
diff --git a/cpp/src/arrow/array/array_union_test.cc b/cpp/src/arrow/array/array_union_test.cc
index a32b8b868de..1eb722b13c5 100644
--- a/cpp/src/arrow/array/array_union_test.cc
+++ b/cpp/src/arrow/array/array_union_test.cc
@@ -20,6 +20,8 @@
 #include <gtest/gtest.h>
 
 #include "arrow/array.h"
+#include "arrow/array/builder_nested.h"
+#include "arrow/array/builder_union.h"
 // TODO ipc shouldn't be included here
 #include "arrow/ipc/test_common.h"
 #include "arrow/testing/gtest_util.h"
@@ -307,7 +309,24 @@ class UnionBuilderTest : public ::testing::Test {
     AppendString("def");
     AppendInt(-10);
     AppendDouble(0.5);
+
+    ASSERT_OK(union_builder->Finish(&actual));
+    ASSERT_OK(actual->ValidateFull());
+    ArrayFromVector<Int8Type, uint8_t>(expected_types_vector, &expected_types);
+  }
+
+  void AppendNullsAndEmptyValues() {
+    AppendString("abc");
+    ASSERT_OK(union_builder->AppendNull());
+    ASSERT_OK(union_builder->AppendEmptyValue());
+    expected_types_vector.insert(expected_types_vector.end(), 3, I8);
+    AppendInt(42);
+    ASSERT_OK(union_builder->AppendNulls(2));
+    ASSERT_OK(union_builder->AppendEmptyValues(2));
+    expected_types_vector.insert(expected_types_vector.end(), 3, I8);
+
     ASSERT_OK(union_builder->Finish(&actual));
+    ASSERT_OK(actual->ValidateFull());
     ArrayFromVector<Int8Type, uint8_t>(expected_types_vector, &expected_types);
   }
 
@@ -329,7 +348,9 @@ class UnionBuilderTest : public ::testing::Test {
     AppendDouble(1.0);
     AppendDouble(-1.0);
     AppendDouble(0.5);
+
     ASSERT_OK(union_builder->Finish(&actual));
+    ASSERT_OK(actual->ValidateFull());
     ArrayFromVector<Int8Type, uint8_t>(expected_types_vector, &expected_types);
 
     ASSERT_EQ(I8, 0);
@@ -357,6 +378,7 @@ class UnionBuilderTest : public ::testing::Test {
     AppendDouble(0.5);
 
     ASSERT_OK(list_builder.Finish(actual));
+    ASSERT_OK((*actual)->ValidateFull());
     ArrayFromVector<Int8Type, uint8_t>(expected_types_vector, &expected_types);
   }
 
@@ -376,20 +398,20 @@ class SparseUnionBuilderTest : public UnionBuilderTest<SparseUnionBuilder> {
 
   void AppendInt(int8_t i) override {
     Base::AppendInt(i);
-    ASSERT_OK(str_builder->AppendNull());
-    ASSERT_OK(dbl_builder->AppendNull());
+    ASSERT_OK(str_builder->AppendEmptyValue());
+    ASSERT_OK(dbl_builder->AppendEmptyValue());
   }
 
   void AppendString(const std::string& str) override {
     Base::AppendString(str);
-    ASSERT_OK(i8_builder->AppendNull());
-    ASSERT_OK(dbl_builder->AppendNull());
+    ASSERT_OK(i8_builder->AppendEmptyValue());
+    ASSERT_OK(dbl_builder->AppendEmptyValue());
   }
 
   void AppendDouble(double dbl) override {
     Base::AppendDouble(dbl);
-    ASSERT_OK(i8_builder->AppendNull());
-    ASSERT_OK(str_builder->AppendNull());
+    ASSERT_OK(i8_builder->AppendEmptyValue());
+    ASSERT_OK(str_builder->AppendEmptyValue());
   }
 };
 
@@ -415,6 +437,34 @@ TEST_F(DenseUnionBuilderTest, Basics) {
   ASSERT_ARRAYS_EQUAL(*expected, *actual);
 }
 
+TEST_F(DenseUnionBuilderTest, NullsAndEmptyValues) {
+  union_builder.reset(new DenseUnionBuilder(
+      default_memory_pool(), {i8_builder, str_builder, dbl_builder},
+      dense_union({field("i8", int8()), field("str", utf8()), field("dbl", float64())},
+                  {I8, STR, DBL})));
+  AppendNullsAndEmptyValues();
+
+  // Four null / empty values (the latter implementation-defined) were appended to I8
+  auto expected_i8 = ArrayFromJSON(int8(), "[null, 0, 42, null, 0]");
+  auto expected_str = ArrayFromJSON(utf8(), R"(["abc"])");
+  auto expected_dbl = ArrayFromJSON(float64(), "[]");
+
+  // "abc", null, 0, 42, null, null, 0, 0
+  auto expected_offsets = ArrayFromJSON(int32(), "[0, 0, 1, 2, 3, 3, 4, 4]");
+
+  ASSERT_OK_AND_ASSIGN(auto expected,
+                       DenseUnionArray::Make(*expected_types, *expected_offsets,
+                                             {expected_i8, expected_str, expected_dbl},
+                                             {"i8", "str", "dbl"}, {I8, STR, DBL}));
+
+  ASSERT_EQ(expected->type()->ToString(), actual->type()->ToString());
+  ASSERT_ARRAYS_EQUAL(*expected, *actual);
+  // Physical arrays must be as expected
+  ASSERT_ARRAYS_EQUAL(*expected_i8, *actual->field(0));
+  ASSERT_ARRAYS_EQUAL(*expected_str, *actual->field(1));
+  ASSERT_ARRAYS_EQUAL(*expected_dbl, *actual->field(2));
+}
+
 TEST_F(DenseUnionBuilderTest, InferredType) {
   AppendInferred();
 
@@ -467,6 +517,32 @@ TEST_F(SparseUnionBuilderTest, Basics) {
   ASSERT_ARRAYS_EQUAL(*expected, *actual);
 }
 
+TEST_F(SparseUnionBuilderTest, NullsAndEmptyValues) {
+  union_builder.reset(new SparseUnionBuilder(
+      default_memory_pool(), {i8_builder, str_builder, dbl_builder},
+      sparse_union({field("i8", int8()), field("str", utf8()), field("dbl", float64())},
+                   {I8, STR, DBL})));
+  AppendNullsAndEmptyValues();
+
+  // "abc", null, 0, 42, null, null, 0, 0
+  // (note that getting 0 for empty values is implementation-defined)
+  auto expected_i8 = ArrayFromJSON(int8(), "[0, null, 0, 42, null, null, 0, 0]");
+  auto expected_str = ArrayFromJSON(utf8(), R"(["abc", "", "", "", "", "", "", ""])");
+  auto expected_dbl = ArrayFromJSON(float64(), "[0, 0, 0, 0, 0, 0, 0, 0]");
+
+  ASSERT_OK_AND_ASSIGN(
+      auto expected,
+      SparseUnionArray::Make(*expected_types, {expected_i8, expected_str, expected_dbl},
+                             {"i8", "str", "dbl"}, {I8, STR, DBL}));
+
+  ASSERT_EQ(expected->type()->ToString(), actual->type()->ToString());
+  ASSERT_ARRAYS_EQUAL(*expected, *actual);
+  // Physical arrays must be as expected
+  ASSERT_ARRAYS_EQUAL(*expected_i8, *actual->field(0));
+  ASSERT_ARRAYS_EQUAL(*expected_str, *actual->field(1));
+  ASSERT_ARRAYS_EQUAL(*expected_dbl, *actual->field(2));
+}
+
 TEST_F(SparseUnionBuilderTest, InferredType) {
   AppendInferred();
 
diff --git a/cpp/src/arrow/array/builder_adaptive.cc b/cpp/src/arrow/array/builder_adaptive.cc
index 47880e91663..36e5546a749 100644
--- a/cpp/src/arrow/array/builder_adaptive.cc
+++ b/cpp/src/arrow/array/builder_adaptive.cc
@@ -33,7 +33,8 @@ namespace arrow {
 
 using internal::AdaptiveIntBuilderBase;
 
-AdaptiveIntBuilderBase::AdaptiveIntBuilderBase(MemoryPool* pool) : ArrayBuilder(pool) {}
+AdaptiveIntBuilderBase::AdaptiveIntBuilderBase(uint8_t start_int_size, MemoryPool* pool)
+    : ArrayBuilder(pool), start_int_size_(start_int_size), int_size_(start_int_size) {}
 
 void AdaptiveIntBuilderBase::Reset() {
   ArrayBuilder::Reset();
@@ -41,7 +42,7 @@ void AdaptiveIntBuilderBase::Reset() {
   raw_data_ = nullptr;
   pending_pos_ = 0;
   pending_has_nulls_ = false;
-  int_size_ = sizeof(uint8_t);
+  int_size_ = start_int_size_;
 }
 
 Status AdaptiveIntBuilderBase::Resize(int64_t capacity) {
@@ -124,7 +125,8 @@ std::shared_ptr<DataType> AdaptiveIntBuilder::type() const {
   return nullptr;
 }
 
-AdaptiveIntBuilder::AdaptiveIntBuilder(MemoryPool* pool) : AdaptiveIntBuilderBase(pool) {}
+AdaptiveIntBuilder::AdaptiveIntBuilder(uint8_t start_int_size, MemoryPool* pool)
+    : AdaptiveIntBuilderBase(start_int_size, pool) {}
 
 Status AdaptiveIntBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
   RETURN_NOT_OK(CommitPendingData());
@@ -264,8 +266,8 @@ Status AdaptiveIntBuilder::ExpandIntSize(uint8_t new_int_size) {
   return Status::OK();
 }
 
-AdaptiveUIntBuilder::AdaptiveUIntBuilder(MemoryPool* pool)
-    : AdaptiveIntBuilderBase(pool) {}
+AdaptiveUIntBuilder::AdaptiveUIntBuilder(uint8_t start_int_size, MemoryPool* pool)
+    : AdaptiveIntBuilderBase(start_int_size, pool) {}
 
 Status AdaptiveUIntBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
   RETURN_NOT_OK(CommitPendingData());
diff --git a/cpp/src/arrow/array/builder_adaptive.h b/cpp/src/arrow/array/builder_adaptive.h
index 1bce339433d..c0df797256d 100644
--- a/cpp/src/arrow/array/builder_adaptive.h
+++ b/cpp/src/arrow/array/builder_adaptive.h
@@ -35,7 +35,10 @@ namespace internal {
 
 class ARROW_EXPORT AdaptiveIntBuilderBase : public ArrayBuilder {
  public:
-  explicit AdaptiveIntBuilderBase(MemoryPool* pool);
+  AdaptiveIntBuilderBase(uint8_t start_int_size, MemoryPool* pool);
+
+  explicit AdaptiveIntBuilderBase(MemoryPool* pool)
+      : AdaptiveIntBuilderBase(sizeof(uint8_t), pool) {}
 
   /// \brief Append multiple nulls
   /// \param[in] length the number of nulls to append
@@ -61,6 +64,26 @@ class ARROW_EXPORT AdaptiveIntBuilderBase : public ArrayBuilder {
     return Status::OK();
   }
 
+  Status AppendEmptyValues(int64_t length) final {
+    ARROW_RETURN_NOT_OK(CommitPendingData());
+    ARROW_RETURN_NOT_OK(Reserve(length));
+    memset(data_->mutable_data() + length_ * int_size_, 0, int_size_ * length);
+    UnsafeSetNotNull(length);
+    return Status::OK();
+  }
+
+  Status AppendEmptyValue() final {
+    pending_data_[pending_pos_] = 0;
+    pending_valid_[pending_pos_] = 1;
+    ++pending_pos_;
+    ++length_;
+
+    if (ARROW_PREDICT_FALSE(pending_pos_ >= pending_size_)) {
+      return CommitPendingData();
+    }
+    return Status::OK();
+  }
+
   void Reset() override;
   Status Resize(int64_t capacity) override;
 
@@ -88,7 +111,9 @@ class ARROW_EXPORT AdaptiveIntBuilderBase : public ArrayBuilder {
 
   std::shared_ptr<ResizableBuffer> data_;
   uint8_t* raw_data_ = NULLPTR;
-  uint8_t int_size_ = sizeof(uint8_t);
+
+  const uint8_t start_int_size_;
+  uint8_t int_size_;
 
   static constexpr int32_t pending_size_ = 1024;
   uint8_t pending_valid_[pending_size_];
@@ -101,7 +126,11 @@ class ARROW_EXPORT AdaptiveIntBuilderBase : public ArrayBuilder {
 
 class ARROW_EXPORT AdaptiveUIntBuilder : public internal::AdaptiveIntBuilderBase {
  public:
-  explicit AdaptiveUIntBuilder(MemoryPool* pool = default_memory_pool());
+  explicit AdaptiveUIntBuilder(uint8_t start_int_size,
+                               MemoryPool* pool = default_memory_pool());
+
+  explicit AdaptiveUIntBuilder(MemoryPool* pool = default_memory_pool())
+      : AdaptiveUIntBuilder(sizeof(uint8_t), pool) {}
 
   using ArrayBuilder::Advance;
   using internal::AdaptiveIntBuilderBase::Reset;
@@ -135,7 +164,11 @@ class ARROW_EXPORT AdaptiveUIntBuilder : public internal::AdaptiveIntBuilderBase
 
 class ARROW_EXPORT AdaptiveIntBuilder : public internal::AdaptiveIntBuilderBase {
  public:
-  explicit AdaptiveIntBuilder(MemoryPool* pool = default_memory_pool());
+  explicit AdaptiveIntBuilder(uint8_t start_int_size,
+                              MemoryPool* pool = default_memory_pool());
+
+  explicit AdaptiveIntBuilder(MemoryPool* pool = default_memory_pool())
+      : AdaptiveIntBuilder(sizeof(uint8_t), pool) {}
 
   using ArrayBuilder::Advance;
   using internal::AdaptiveIntBuilderBase::Reset;
diff --git a/cpp/src/arrow/array/builder_base.cc b/cpp/src/arrow/array/builder_base.cc
index 4c21859fae3..b92cc285894 100644
--- a/cpp/src/arrow/array/builder_base.cc
+++ b/cpp/src/arrow/array/builder_base.cc
@@ -29,6 +29,22 @@
 
 namespace arrow {
 
+Status ArrayBuilder::CheckArrayType(const std::shared_ptr<DataType>& expected_type,
+                                    const Array& array, const char* message) {
+  if (!expected_type->Equals(*array.type())) {
+    return Status::TypeError(message);
+  }
+  return Status::OK();
+}
+
+Status ArrayBuilder::CheckArrayType(Type::type expected_type, const Array& array,
+                                    const char* message) {
+  if (array.type_id() != expected_type) {
+    return Status::TypeError(message);
+  }
+  return Status::OK();
+}
+
 Status ArrayBuilder::TrimBuffer(const int64_t bytes_filled, ResizableBuffer* buffer) {
   if (buffer) {
     if (bytes_filled < buffer->size()) {
@@ -83,6 +99,12 @@ Status ArrayBuilder::Finish(std::shared_ptr<Array>* out) {
   return Status::OK();
 }
 
+Result<std::shared_ptr<Array>> ArrayBuilder::Finish() {
+  std::shared_ptr<Array> out;
+  RETURN_NOT_OK(Finish(&out));
+  return out;
+}
+
 void ArrayBuilder::Reset() {
   capacity_ = length_ = null_count_ = 0;
   null_bitmap_builder_.Reset();
diff --git a/cpp/src/arrow/array/builder_base.h b/cpp/src/arrow/array/builder_base.h
index 8d327b713b6..15c726241b5 100644
--- a/cpp/src/arrow/array/builder_base.h
+++ b/cpp/src/arrow/array/builder_base.h
@@ -29,7 +29,7 @@
 #include "arrow/buffer.h"
 #include "arrow/buffer_builder.h"
 #include "arrow/status.h"
-#include "arrow/type.h"
+#include "arrow/type_fwd.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
@@ -56,6 +56,8 @@ class ARROW_EXPORT ArrayBuilder {
   /// skip shared pointers and just return a raw pointer
   ArrayBuilder* child(int i) { return children_[i].get(); }
 
+  const std::shared_ptr<ArrayBuilder>& child_builder(int i) const { return children_[i]; }
+
   int num_children() const { return static_cast<int>(children_.size()); }
 
   virtual int64_t length() const { return length_; }
@@ -95,9 +97,25 @@ class ARROW_EXPORT ArrayBuilder {
   /// Reset the builder.
   virtual void Reset();
 
+  /// \brief Append a null value to builder
   virtual Status AppendNull() = 0;
+  /// \brief Append a number of null values to builder
   virtual Status AppendNulls(int64_t length) = 0;
 
+  /// \brief Append a non-null value to builder
+  ///
+  /// The appended value is an implementation detail, but the corresponding
+  /// memory slot is guaranteed to be initialized.
+  /// This method is useful when appending a null value to a parent nested type.
+  virtual Status AppendEmptyValue() = 0;
+
+  /// \brief Append a number of non-null values to builder
+  ///
+  /// The appended values are an implementation detail, but the corresponding
+  /// memory slot is guaranteed to be initialized.
+  /// This method is useful when appending null values to a parent nested type.
+  virtual Status AppendEmptyValues(int64_t length) = 0;
+
   /// For cases where raw data was memcpy'd into the internal buffers, allows us
   /// to advance the length of the builder. It is your responsibility to use
   /// this function responsibly.
@@ -118,6 +136,13 @@ class ARROW_EXPORT ArrayBuilder {
   /// \return Status
   Status Finish(std::shared_ptr<Array>* out);
 
+  /// \brief Return result of builder as an Array object.
+  ///
+  /// The builder is reset except for DictionaryBuilder.
+  ///
+  /// \return The finalized Array object
+  Result<std::shared_ptr<Array>> Finish();
+
   /// \brief Return the type of the built Array
   virtual std::shared_ptr<DataType> type() const = 0;
 
@@ -200,6 +225,12 @@ class ARROW_EXPORT ArrayBuilder {
     return Status::OK();
   }
 
+  // Check for array type
+  Status CheckArrayType(const std::shared_ptr<DataType>& expected_type,
+                        const Array& array, const char* message);
+  Status CheckArrayType(Type::type expected_type, const Array& array,
+                        const char* message);
+
   MemoryPool* pool_;
 
   TypedBufferBuilder<bool> null_bitmap_builder_;
@@ -216,4 +247,24 @@ class ARROW_EXPORT ArrayBuilder {
   ARROW_DISALLOW_COPY_AND_ASSIGN(ArrayBuilder);
 };
 
+/// \brief Construct an empty ArrayBuilder corresponding to the data
+/// type
+/// \param[in] pool the MemoryPool to use for allocations
+/// \param[in] type the data type to create the builder for
+/// \param[out] out the created ArrayBuilder
+ARROW_EXPORT
+Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
+                   std::unique_ptr<ArrayBuilder>* out);
+
+/// \brief Construct an empty DictionaryBuilder initialized optionally
+/// with a pre-existing dictionary
+/// \param[in] pool the MemoryPool to use for allocations
+/// \param[in] type the dictionary type to create the builder for
+/// \param[in] dictionary the initial dictionary, if any. May be nullptr
+/// \param[out] out the created ArrayBuilder
+ARROW_EXPORT
+Status MakeDictionaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
+                             const std::shared_ptr<Array>& dictionary,
+                             std::unique_ptr<ArrayBuilder>* out);
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/array/builder_binary.cc b/cpp/src/arrow/array/builder_binary.cc
index ecb0e95fb44..6822dc89903 100644
--- a/cpp/src/arrow/array/builder_binary.cc
+++ b/cpp/src/arrow/array/builder_binary.cc
@@ -73,6 +73,20 @@ Status FixedSizeBinaryBuilder::AppendNulls(int64_t length) {
   return Status::OK();
 }
 
+Status FixedSizeBinaryBuilder::AppendEmptyValue() {
+  RETURN_NOT_OK(Reserve(1));
+  UnsafeAppendToBitmap(true);
+  byte_builder_.UnsafeAppend(/*num_copies=*/byte_width_, 0);
+  return Status::OK();
+}
+
+Status FixedSizeBinaryBuilder::AppendEmptyValues(int64_t length) {
+  RETURN_NOT_OK(Reserve(length));
+  UnsafeAppendToBitmap(length, true);
+  byte_builder_.UnsafeAppend(/*num_copies=*/length * byte_width_, 0);
+  return Status::OK();
+}
+
 void FixedSizeBinaryBuilder::Reset() {
   ArrayBuilder::Reset();
   byte_builder_.Reset();
diff --git a/cpp/src/arrow/array/builder_binary.h b/cpp/src/arrow/array/builder_binary.h
index 593b533a19c..bc49c7d6787 100644
--- a/cpp/src/arrow/array/builder_binary.h
+++ b/cpp/src/arrow/array/builder_binary.h
@@ -61,6 +61,7 @@ class BaseBinaryBuilder : public ArrayBuilder {
     ARROW_RETURN_NOT_OK(AppendNextOffset());
     // Safety check for UBSAN.
     if (ARROW_PREDICT_TRUE(length > 0)) {
+      ARROW_RETURN_NOT_OK(ValidateOverflow(length));
       ARROW_RETURN_NOT_OK(value_data_builder_.Append(value, length));
     }
 
@@ -78,9 +79,6 @@ class BaseBinaryBuilder : public ArrayBuilder {
 
   Status AppendNulls(int64_t length) final {
     const int64_t num_bytes = value_data_builder_.length();
-    if (ARROW_PREDICT_FALSE(num_bytes > memory_limit())) {
-      return AppendOverflow(num_bytes);
-    }
     ARROW_RETURN_NOT_OK(Reserve(length));
     for (int64_t i = 0; i < length; ++i) {
       offsets_builder_.UnsafeAppend(static_cast<offset_type>(num_bytes));
@@ -96,6 +94,23 @@ class BaseBinaryBuilder : public ArrayBuilder {
     return Status::OK();
   }
 
+  Status AppendEmptyValue() final {
+    ARROW_RETURN_NOT_OK(AppendNextOffset());
+    ARROW_RETURN_NOT_OK(Reserve(1));
+    UnsafeAppendToBitmap(true);
+    return Status::OK();
+  }
+
+  Status AppendEmptyValues(int64_t length) final {
+    const int64_t num_bytes = value_data_builder_.length();
+    ARROW_RETURN_NOT_OK(Reserve(length));
+    for (int64_t i = 0; i < length; ++i) {
+      offsets_builder_.UnsafeAppend(static_cast<offset_type>(num_bytes));
+    }
+    UnsafeAppendToBitmap(length, true);
+    return Status::OK();
+  }
+
   /// \brief Append without checking capacity
   ///
   /// Offsets and data should have been presized using Reserve() and
@@ -232,6 +247,16 @@ class BaseBinaryBuilder : public ArrayBuilder {
     value_data_builder_.Reset();
   }
 
+  Status ValidateOverflow(int64_t new_bytes) {
+    auto new_size = value_data_builder_.length() + new_bytes;
+    if (ARROW_PREDICT_FALSE(new_size > memory_limit())) {
+      return Status::CapacityError("array cannot contain more than ", memory_limit(),
+                                   " bytes, have ", new_size);
+    } else {
+      return Status::OK();
+    }
+  }
+
   Status Resize(int64_t capacity) override {
     // XXX Why is this check necessary?  There is no reason to disallow, say,
     // binary arrays with more than 2**31 empty or null values.
@@ -249,12 +274,8 @@ class BaseBinaryBuilder : public ArrayBuilder {
   /// \brief Ensures there is enough allocated capacity to append the indicated
   /// number of bytes to the value data buffer without additional allocations
   Status ReserveData(int64_t elements) {
-    const int64_t size = value_data_length() + elements;
-    ARROW_RETURN_IF(size > memory_limit(),
-                    Status::CapacityError("Cannot reserve capacity larger than ",
-                                          memory_limit(), " bytes"));
-    return (size > value_data_capacity()) ? value_data_builder_.Reserve(elements)
-                                          : Status::OK();
+    ARROW_RETURN_NOT_OK(ValidateOverflow(elements));
+    return value_data_builder_.Reserve(elements);
   }
 
   Status FinishInternal(std::shared_ptr<ArrayData>* out) override {
@@ -317,16 +338,8 @@ class BaseBinaryBuilder : public ArrayBuilder {
   TypedBufferBuilder<offset_type> offsets_builder_;
   TypedBufferBuilder<uint8_t> value_data_builder_;
 
-  Status AppendOverflow(int64_t num_bytes) {
-    return Status::CapacityError("array cannot contain more than ", memory_limit(),
-                                 " bytes, have ", num_bytes);
-  }
-
   Status AppendNextOffset() {
     const int64_t num_bytes = value_data_builder_.length();
-    if (ARROW_PREDICT_FALSE(num_bytes > memory_limit())) {
-      return AppendOverflow(num_bytes);
-    }
     return offsets_builder_.Append(static_cast<offset_type>(num_bytes));
   }
 
@@ -440,9 +453,11 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
                       const uint8_t* valid_bytes = NULLPTR);
 
   Status AppendNull() final;
-
   Status AppendNulls(int64_t length) final;
 
+  Status AppendEmptyValue() final;
+  Status AppendEmptyValues(int64_t length) final;
+
   void UnsafeAppend(const uint8_t* value) {
     UnsafeAppendToBitmap(true);
     if (ARROW_PREDICT_TRUE(byte_width_ > 0)) {
@@ -450,6 +465,10 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
     }
   }
 
+  void UnsafeAppend(const char* value) {
+    UnsafeAppend(reinterpret_cast<const uint8_t*>(value));
+  }
+
   void UnsafeAppend(util::string_view value) {
 #ifndef NDEBUG
     CheckValueSize(static_cast<size_t>(value.size()));
@@ -462,6 +481,23 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
     byte_builder_.UnsafeAppend(/*num_copies=*/byte_width_, 0);
   }
 
+  Status ValidateOverflow(int64_t new_bytes) const {
+    auto new_size = byte_builder_.length() + new_bytes;
+    if (ARROW_PREDICT_FALSE(new_size > memory_limit())) {
+      return Status::CapacityError("array cannot contain more than ", memory_limit(),
+                                   " bytes, have ", new_size);
+    } else {
+      return Status::OK();
+    }
+  }
+
+  /// \brief Ensures there is enough allocated capacity to append the indicated
+  /// number of bytes to the value data buffer without additional allocations
+  Status ReserveData(int64_t elements) {
+    ARROW_RETURN_NOT_OK(ValidateOverflow(elements));
+    return byte_builder_.Reserve(elements);
+  }
+
   void Reset() override;
   Status Resize(int64_t capacity) override;
   Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
diff --git a/cpp/src/arrow/array/builder_decimal.cc b/cpp/src/arrow/array/builder_decimal.cc
index ea5c9ebd0c3..bd7615a7309 100644
--- a/cpp/src/arrow/array/builder_decimal.cc
+++ b/cpp/src/arrow/array/builder_decimal.cc
@@ -67,4 +67,39 @@ Status Decimal128Builder::FinishInternal(std::shared_ptr<ArrayData>* out) {
   return Status::OK();
 }
 
+// ----------------------------------------------------------------------
+// Decimal256Builder
+
+Decimal256Builder::Decimal256Builder(const std::shared_ptr<DataType>& type,
+                                     MemoryPool* pool)
+    : FixedSizeBinaryBuilder(type, pool),
+      decimal_type_(internal::checked_pointer_cast<Decimal256Type>(type)) {}
+
+Status Decimal256Builder::Append(const Decimal256& value) {
+  RETURN_NOT_OK(FixedSizeBinaryBuilder::Reserve(1));
+  UnsafeAppend(value);
+  return Status::OK();
+}
+
+void Decimal256Builder::UnsafeAppend(const Decimal256& value) {
+  value.ToBytes(GetMutableValue(length()));
+  byte_builder_.UnsafeAdvance(32);
+  UnsafeAppendToBitmap(true);
+}
+
+void Decimal256Builder::UnsafeAppend(util::string_view value) {
+  FixedSizeBinaryBuilder::UnsafeAppend(value);
+}
+
+Status Decimal256Builder::FinishInternal(std::shared_ptr<ArrayData>* out) {
+  std::shared_ptr<Buffer> data;
+  RETURN_NOT_OK(byte_builder_.Finish(&data));
+  std::shared_ptr<Buffer> null_bitmap;
+  RETURN_NOT_OK(null_bitmap_builder_.Finish(&null_bitmap));
+
+  *out = ArrayData::Make(type(), length_, {null_bitmap, data}, null_count_);
+  capacity_ = length_ = null_count_ = 0;
+  return Status::OK();
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/array/builder_decimal.h b/cpp/src/arrow/array/builder_decimal.h
index 8f0ff83288c..8c75e7dd674 100644
--- a/cpp/src/arrow/array/builder_decimal.h
+++ b/cpp/src/arrow/array/builder_decimal.h
@@ -58,6 +58,35 @@ class ARROW_EXPORT Decimal128Builder : public FixedSizeBinaryBuilder {
   std::shared_ptr<Decimal128Type> decimal_type_;
 };
 
+class ARROW_EXPORT Decimal256Builder : public FixedSizeBinaryBuilder {
+ public:
+  using TypeClass = Decimal256Type;
+
+  explicit Decimal256Builder(const std::shared_ptr<DataType>& type,
+                             MemoryPool* pool = default_memory_pool());
+
+  using FixedSizeBinaryBuilder::Append;
+  using FixedSizeBinaryBuilder::AppendValues;
+  using FixedSizeBinaryBuilder::Reset;
+
+  Status Append(const Decimal256& val);
+  void UnsafeAppend(const Decimal256& val);
+  void UnsafeAppend(util::string_view val);
+
+  Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
+
+  /// \cond FALSE
+  using ArrayBuilder::Finish;
+  /// \endcond
+
+  Status Finish(std::shared_ptr<Decimal256Array>* out) { return FinishTyped(out); }
+
+  std::shared_ptr<DataType> type() const override { return decimal_type_; }
+
+ protected:
+  std::shared_ptr<Decimal256Type> decimal_type_;
+};
+
 using DecimalBuilder = Decimal128Builder;
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/array/builder_dict.cc b/cpp/src/arrow/array/builder_dict.cc
index 54fd94856ea..b13f6a2db34 100644
--- a/cpp/src/arrow/array/builder_dict.cc
+++ b/cpp/src/arrow/array/builder_dict.cc
@@ -18,6 +18,7 @@
 #include "arrow/array/builder_dict.h"
 
 #include <cstdint>
+#include <utility>
 
 #include "arrow/array/dict_internal.h"
 #include "arrow/status.h"
@@ -44,7 +45,7 @@ class DictionaryMemoTable::DictionaryMemoTableImpl {
 
     template <typename T>
     enable_if_no_memoize<T, Status> Visit(const T&) {
-      return Status::NotImplemented("Initialization of ", value_type_,
+      return Status::NotImplemented("Initialization of ", value_type_->ToString(),
                                     " memo table is not implemented");
     }
 
@@ -68,21 +69,20 @@ class DictionaryMemoTable::DictionaryMemoTableImpl {
     }
 
    private:
-    template <typename DType, typename ArrayType>
-    enable_if_no_memoize<DType, Status> InsertValues(const DType& type,
-                                                     const ArrayType&) {
+    template <typename T, typename ArrayType>
+    enable_if_no_memoize<T, Status> InsertValues(const T& type, const ArrayType&) {
       return Status::NotImplemented("Inserting array values of ", type,
                                     " is not implemented");
     }
 
-    template <typename DType, typename ArrayType>
-    enable_if_memoize<DType, Status> InsertValues(const DType&, const ArrayType& array) {
+    template <typename T, typename ArrayType>
+    enable_if_memoize<T, Status> InsertValues(const T&, const ArrayType& array) {
       if (array.null_count() > 0) {
         return Status::Invalid("Cannot insert dictionary values containing nulls");
       }
       for (int64_t i = 0; i < array.length(); ++i) {
         int32_t unused_memo_index;
-        RETURN_NOT_OK(impl_->GetOrInsert(array.GetView(i), &unused_memo_index));
+        RETURN_NOT_OK(impl_->GetOrInsert<T>(array.GetView(i), &unused_memo_index));
       }
       return Status::OK();
     }
@@ -112,8 +112,8 @@ class DictionaryMemoTable::DictionaryMemoTableImpl {
   };
 
  public:
-  DictionaryMemoTableImpl(MemoryPool* pool, const std::shared_ptr<DataType>& type)
-      : pool_(pool), type_(type), memo_table_(nullptr) {
+  DictionaryMemoTableImpl(MemoryPool* pool, std::shared_ptr<DataType> type)
+      : pool_(pool), type_(std::move(type)), memo_table_(nullptr) {
     MemoTableInitializer visitor{type_, pool_, &memo_table_};
     ARROW_CHECK_OK(VisitTypeInline(*type_, &visitor));
   }
@@ -127,9 +127,10 @@ class DictionaryMemoTable::DictionaryMemoTableImpl {
     return VisitTypeInline(*array.type(), &visitor);
   }
 
-  template <typename T>
-  Status GetOrInsert(const T& value, int32_t* out) {
-    using ConcreteMemoTable = typename DictionaryCTraits<T>::MemoTableType;
+  template <typename PhysicalType,
+            typename CType = typename DictionaryValue<PhysicalType>::type>
+  Status GetOrInsert(CType value, int32_t* out) {
+    using ConcreteMemoTable = typename DictionaryTraits<PhysicalType>::MemoTableType;
     return checked_cast<ConcreteMemoTable*>(memo_table_.get())->GetOrInsert(value, out);
   }
 
@@ -158,9 +159,10 @@ DictionaryMemoTable::DictionaryMemoTable(MemoryPool* pool,
 
 DictionaryMemoTable::~DictionaryMemoTable() = default;
 
-#define GET_OR_INSERT(C_TYPE)                                           \
-  Status DictionaryMemoTable::GetOrInsert(C_TYPE value, int32_t* out) { \
-    return impl_->GetOrInsert(value, out);                              \
+#define GET_OR_INSERT(C_TYPE)                                                       \
+  Status DictionaryMemoTable::GetOrInsert(                                          \
+      const typename CTypeTraits<C_TYPE>::ArrowType*, C_TYPE value, int32_t* out) { \
+    return impl_->GetOrInsert<typename CTypeTraits<C_TYPE>::ArrowType>(value, out); \
   }
 
 GET_OR_INSERT(bool)
@@ -174,10 +176,19 @@ GET_OR_INSERT(uint32_t)
 GET_OR_INSERT(uint64_t)
 GET_OR_INSERT(float)
 GET_OR_INSERT(double)
-GET_OR_INSERT(util::string_view)
 
 #undef GET_OR_INSERT
 
+Status DictionaryMemoTable::GetOrInsert(const BinaryType*, util::string_view value,
+                                        int32_t* out) {
+  return impl_->GetOrInsert<BinaryType>(value, out);
+}
+
+Status DictionaryMemoTable::GetOrInsert(const LargeBinaryType*, util::string_view value,
+                                        int32_t* out) {
+  return impl_->GetOrInsert<LargeBinaryType>(value, out);
+}
+
 Status DictionaryMemoTable::GetArrayData(int64_t start_offset,
                                          std::shared_ptr<ArrayData>* out) {
   return impl_->GetArrayData(start_offset, out);
diff --git a/cpp/src/arrow/array/builder_dict.h b/cpp/src/arrow/array/builder_dict.h
index 9a0f268c4d0..40d6ce1ba9a 100644
--- a/cpp/src/arrow/array/builder_dict.h
+++ b/cpp/src/arrow/array/builder_dict.h
@@ -32,6 +32,8 @@
 #include "arrow/status.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
+#include "arrow/util/checked_cast.h"
+#include "arrow/util/decimal.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
@@ -42,24 +44,24 @@ namespace arrow {
 
 namespace internal {
 
-template <typename T>
-struct DictionaryScalar {
+template <typename T, typename Enable = void>
+struct DictionaryValue {
   using type = typename T::c_type;
+  using PhysicalType = T;
 };
 
-template <>
-struct DictionaryScalar<BinaryType> {
-  using type = util::string_view;
-};
-
-template <>
-struct DictionaryScalar<StringType> {
+template <typename T>
+struct DictionaryValue<T, enable_if_base_binary<T>> {
   using type = util::string_view;
+  using PhysicalType =
+      typename std::conditional<std::is_same<typename T::offset_type, int32_t>::value,
+                                BinaryType, LargeBinaryType>::type;
 };
 
-template <>
-struct DictionaryScalar<FixedSizeBinaryType> {
+template <typename T>
+struct DictionaryValue<T, enable_if_fixed_size_binary<T>> {
   using type = util::string_view;
+  using PhysicalType = BinaryType;
 };
 
 class ARROW_EXPORT DictionaryMemoTable {
@@ -68,19 +70,6 @@ class ARROW_EXPORT DictionaryMemoTable {
   DictionaryMemoTable(MemoryPool* pool, const std::shared_ptr<Array>& dictionary);
   ~DictionaryMemoTable();
 
-  Status GetOrInsert(bool value, int32_t* out);
-  Status GetOrInsert(int8_t value, int32_t* out);
-  Status GetOrInsert(int16_t value, int32_t* out);
-  Status GetOrInsert(int32_t value, int32_t* out);
-  Status GetOrInsert(int64_t value, int32_t* out);
-  Status GetOrInsert(uint8_t value, int32_t* out);
-  Status GetOrInsert(uint16_t value, int32_t* out);
-  Status GetOrInsert(uint32_t value, int32_t* out);
-  Status GetOrInsert(uint64_t value, int32_t* out);
-  Status GetOrInsert(float value, int32_t* out);
-  Status GetOrInsert(double value, int32_t* out);
-  Status GetOrInsert(util::string_view value, int32_t* out);
-
   Status GetArrayData(int64_t start_offset, std::shared_ptr<ArrayData>* out);
 
   /// \brief Insert new memo values
@@ -88,7 +77,31 @@ class ARROW_EXPORT DictionaryMemoTable {
 
   int32_t size() const;
 
+  template <typename T>
+  Status GetOrInsert(typename DictionaryValue<T>::type value, int32_t* out) {
+    // We want to keep the DictionaryMemoTable implementation private, also we can't
+    // use extern template classes because of compiler issues (MinGW?).  Instead,
+    // we expose explicit function overrides for each supported physical type.
+    const typename DictionaryValue<T>::PhysicalType* physical_type = NULLPTR;
+    return GetOrInsert(physical_type, value, out);
+  }
+
  private:
+  Status GetOrInsert(const BooleanType*, bool value, int32_t* out);
+  Status GetOrInsert(const Int8Type*, int8_t value, int32_t* out);
+  Status GetOrInsert(const Int16Type*, int16_t value, int32_t* out);
+  Status GetOrInsert(const Int32Type*, int32_t value, int32_t* out);
+  Status GetOrInsert(const Int64Type*, int64_t value, int32_t* out);
+  Status GetOrInsert(const UInt8Type*, uint8_t value, int32_t* out);
+  Status GetOrInsert(const UInt16Type*, uint16_t value, int32_t* out);
+  Status GetOrInsert(const UInt32Type*, uint32_t value, int32_t* out);
+  Status GetOrInsert(const UInt64Type*, uint64_t value, int32_t* out);
+  Status GetOrInsert(const FloatType*, float value, int32_t* out);
+  Status GetOrInsert(const DoubleType*, double value, int32_t* out);
+
+  Status GetOrInsert(const BinaryType*, util::string_view value, int32_t* out);
+  Status GetOrInsert(const LargeBinaryType*, util::string_view value, int32_t* out);
+
   class DictionaryMemoTableImpl;
   std::unique_ptr<DictionaryMemoTableImpl> impl_;
 };
@@ -101,15 +114,30 @@ class ARROW_EXPORT DictionaryMemoTable {
 template <typename BuilderType, typename T>
 class DictionaryBuilderBase : public ArrayBuilder {
  public:
-  using Scalar = typename DictionaryScalar<T>::type;
+  using TypeClass = DictionaryType;
+  using Value = typename DictionaryValue<T>::type;
 
   // WARNING: the type given below is the value type, not the DictionaryType.
   // The DictionaryType is instantiated on the Finish() call.
-  template <typename T1 = T>
-  DictionaryBuilderBase(enable_if_t<!std::is_base_of<FixedSizeBinaryType, T1>::value,
+  template <typename B = BuilderType, typename T1 = T>
+  DictionaryBuilderBase(uint8_t start_int_size,
+                        enable_if_t<std::is_base_of<AdaptiveIntBuilderBase, B>::value &&
+                                        !is_fixed_size_binary_type<T1>::value,
                                     const std::shared_ptr<DataType>&>
                             value_type,
                         MemoryPool* pool = default_memory_pool())
+      : ArrayBuilder(pool),
+        memo_table_(new internal::DictionaryMemoTable(pool, value_type)),
+        delta_offset_(0),
+        byte_width_(-1),
+        indices_builder_(start_int_size, pool),
+        value_type_(value_type) {}
+
+  template <typename T1 = T>
+  explicit DictionaryBuilderBase(
+      enable_if_t<!is_fixed_size_binary_type<T1>::value, const std::shared_ptr<DataType>&>
+          value_type,
+      MemoryPool* pool = default_memory_pool())
       : ArrayBuilder(pool),
         memo_table_(new internal::DictionaryMemoTable(pool, value_type)),
         delta_offset_(0),
@@ -117,6 +145,20 @@ class DictionaryBuilderBase : public ArrayBuilder {
         indices_builder_(pool),
         value_type_(value_type) {}
 
+  template <typename B = BuilderType, typename T1 = T>
+  DictionaryBuilderBase(uint8_t start_int_size,
+                        enable_if_t<std::is_base_of<AdaptiveIntBuilderBase, B>::value &&
+                                        is_fixed_size_binary_type<T1>::value,
+                                    const std::shared_ptr<DataType>&>
+                            value_type,
+                        MemoryPool* pool = default_memory_pool())
+      : ArrayBuilder(pool),
+        memo_table_(new internal::DictionaryMemoTable(pool, value_type)),
+        delta_offset_(0),
+        byte_width_(static_cast<const T1&>(*value_type).byte_width()),
+        indices_builder_(start_int_size, pool),
+        value_type_(value_type) {}
+
   template <typename T1 = T>
   explicit DictionaryBuilderBase(
       enable_if_fixed_size_binary<T1, const std::shared_ptr<DataType>&> value_type,
@@ -134,8 +176,8 @@ class DictionaryBuilderBase : public ArrayBuilder {
       : DictionaryBuilderBase<BuilderType, T1>(TypeTraits<T1>::type_singleton(), pool) {}
 
   // This constructor doesn't check for errors. Use InsertMemoValues instead.
-  DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
-                        MemoryPool* pool = default_memory_pool())
+  explicit DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
+                                 MemoryPool* pool = default_memory_pool())
       : ArrayBuilder(pool),
         memo_table_(new internal::DictionaryMemoTable(pool, dictionary)),
         delta_offset_(0),
@@ -148,12 +190,18 @@ class DictionaryBuilderBase : public ArrayBuilder {
   /// \brief The current number of entries in the dictionary
   int64_t dictionary_length() const { return memo_table_->size(); }
 
+  /// \brief The value byte width (for FixedSizeBinaryType)
+  template <typename T1 = T>
+  enable_if_fixed_size_binary<T1, int32_t> byte_width() const {
+    return byte_width_;
+  }
+
   /// \brief Append a scalar value
-  Status Append(const Scalar& value) {
+  Status Append(Value value) {
     ARROW_RETURN_NOT_OK(Reserve(1));
 
     int32_t memo_index;
-    ARROW_RETURN_NOT_OK(memo_table_->GetOrInsert(value, &memo_index));
+    ARROW_RETURN_NOT_OK(memo_table_->GetOrInsert<T>(value, &memo_index));
     ARROW_RETURN_NOT_OK(indices_builder_.Append(memo_index));
     length_ += 1;
 
@@ -190,6 +238,22 @@ class DictionaryBuilderBase : public ArrayBuilder {
     return Append(util::string_view(value, length));
   }
 
+  /// \brief Append a decimal (only for Decimal128Type)
+  template <typename T1 = T>
+  enable_if_decimal128<T1, Status> Append(const Decimal128& value) {
+    uint8_t data[16];
+    value.ToBytes(data);
+    return Append(data, 16);
+  }
+
+  /// \brief Append a decimal (only for Decimal128Type)
+  template <typename T1 = T>
+  enable_if_decimal256<T1, Status> Append(const Decimal256& value) {
+    uint8_t data[32];
+    value.ToBytes(data);
+    return Append(data, 32);
+  }
+
   /// \brief Append a scalar null value
   Status AppendNull() final {
     length_ += 1;
@@ -205,6 +269,18 @@ class DictionaryBuilderBase : public ArrayBuilder {
     return indices_builder_.AppendNulls(length);
   }
 
+  Status AppendEmptyValue() final {
+    length_ += 1;
+
+    return indices_builder_.AppendEmptyValue();
+  }
+
+  Status AppendEmptyValues(int64_t length) final {
+    length_ += length;
+
+    return indices_builder_.AppendEmptyValues(length);
+  }
+
   /// \brief Insert values into the dictionary's memo, but do not append any
   /// indices. Can be used to initialize a new builder with known dictionary
   /// values
@@ -220,6 +296,11 @@ class DictionaryBuilderBase : public ArrayBuilder {
       const Array& array) {
     using ArrayType = typename TypeTraits<T>::ArrayType;
 
+#ifndef NDEBUG
+    ARROW_RETURN_NOT_OK(ArrayBuilder::CheckArrayType(
+        value_type_, array, "Wrong value type of array to be appended"));
+#endif
+
     const auto& concrete_array = static_cast<const ArrayType&>(array);
     for (int64_t i = 0; i < array.length(); i++) {
       if (array.IsNull(i)) {
@@ -233,10 +314,10 @@ class DictionaryBuilderBase : public ArrayBuilder {
 
   template <typename T1 = T>
   enable_if_fixed_size_binary<T1, Status> AppendArray(const Array& array) {
-    if (!value_type_->Equals(*array.type())) {
-      return Status::Invalid(
-          "Cannot append FixedSizeBinary array with non-matching type");
-    }
+#ifndef NDEBUG
+    ARROW_RETURN_NOT_OK(ArrayBuilder::CheckArrayType(
+        value_type_, array, "Wrong value type of array to be appended"));
+#endif
 
     const auto& concrete_array = static_cast<const FixedSizeBinaryArray&>(array);
     for (int64_t i = 0; i < array.length(); i++) {
@@ -335,15 +416,30 @@ class DictionaryBuilderBase : public ArrayBuilder {
 template <typename BuilderType>
 class DictionaryBuilderBase<BuilderType, NullType> : public ArrayBuilder {
  public:
-  DictionaryBuilderBase(const std::shared_ptr<DataType>& value_type,
-                        MemoryPool* pool = default_memory_pool())
+  template <typename B = BuilderType>
+  DictionaryBuilderBase(
+      enable_if_t<std::is_base_of<AdaptiveIntBuilderBase, B>::value, uint8_t>
+          start_int_size,
+      const std::shared_ptr<DataType>& value_type,
+      MemoryPool* pool = default_memory_pool())
+      : ArrayBuilder(pool), indices_builder_(start_int_size, pool) {}
+
+  explicit DictionaryBuilderBase(const std::shared_ptr<DataType>& value_type,
+                                 MemoryPool* pool = default_memory_pool())
       : ArrayBuilder(pool), indices_builder_(pool) {}
 
+  template <typename B = BuilderType>
+  explicit DictionaryBuilderBase(
+      enable_if_t<std::is_base_of<AdaptiveIntBuilderBase, B>::value, uint8_t>
+          start_int_size,
+      MemoryPool* pool = default_memory_pool())
+      : ArrayBuilder(pool), indices_builder_(start_int_size, pool) {}
+
   explicit DictionaryBuilderBase(MemoryPool* pool = default_memory_pool())
       : ArrayBuilder(pool), indices_builder_(pool) {}
 
-  DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
-                        MemoryPool* pool = default_memory_pool())
+  explicit DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
+                                 MemoryPool* pool = default_memory_pool())
       : ArrayBuilder(pool), indices_builder_(pool) {}
 
   /// \brief Append a scalar null value
@@ -361,8 +457,24 @@ class DictionaryBuilderBase<BuilderType, NullType> : public ArrayBuilder {
     return indices_builder_.AppendNulls(length);
   }
 
+  Status AppendEmptyValue() final {
+    length_ += 1;
+
+    return indices_builder_.AppendEmptyValue();
+  }
+
+  Status AppendEmptyValues(int64_t length) final {
+    length_ += length;
+
+    return indices_builder_.AppendEmptyValues(length);
+  }
+
   /// \brief Append a whole dense array to the builder
   Status AppendArray(const Array& array) {
+#ifndef NDEBUG
+    ARROW_RETURN_NOT_OK(ArrayBuilder::CheckArrayType(
+        Type::NA, array, "Wrong value type of array to be appended"));
+#endif
     for (int64_t i = 0; i < array.length(); i++) {
       ARROW_RETURN_NOT_OK(AppendNull());
     }
diff --git a/cpp/src/arrow/array/builder_nested.cc b/cpp/src/arrow/array/builder_nested.cc
index b8af62fab14..a3bcde0381a 100644
--- a/cpp/src/arrow/array/builder_nested.cc
+++ b/cpp/src/arrow/array/builder_nested.cc
@@ -54,6 +54,18 @@ MapBuilder::MapBuilder(MemoryPool* pool, const std::shared_ptr<ArrayBuilder>& ke
     : MapBuilder(pool, key_builder, item_builder,
                  map(key_builder->type(), item_builder->type(), keys_sorted)) {}
 
+MapBuilder::MapBuilder(MemoryPool* pool,
+                       const std::shared_ptr<ArrayBuilder>& struct_builder,
+                       const std::shared_ptr<DataType>& type)
+    : ArrayBuilder(pool) {
+  auto map_type = internal::checked_cast<const MapType*>(type.get());
+  keys_sorted_ = map_type->keys_sorted();
+  key_builder_ = struct_builder->child_builder(0);
+  item_builder_ = struct_builder->child_builder(1);
+  list_builder_ =
+      std::make_shared<ListBuilder>(pool, struct_builder, struct_builder->type());
+}
+
 Status MapBuilder::Resize(int64_t capacity) {
   RETURN_NOT_OK(list_builder_->Resize(capacity));
   capacity_ = list_builder_->capacity();
@@ -111,6 +123,24 @@ Status MapBuilder::AppendNulls(int64_t length) {
   return Status::OK();
 }
 
+Status MapBuilder::AppendEmptyValue() {
+  DCHECK_EQ(item_builder_->length(), key_builder_->length());
+  RETURN_NOT_OK(AdjustStructBuilderLength());
+  RETURN_NOT_OK(list_builder_->AppendEmptyValue());
+  length_ = list_builder_->length();
+  null_count_ = list_builder_->null_count();
+  return Status::OK();
+}
+
+Status MapBuilder::AppendEmptyValues(int64_t length) {
+  DCHECK_EQ(item_builder_->length(), key_builder_->length());
+  RETURN_NOT_OK(AdjustStructBuilderLength());
+  RETURN_NOT_OK(list_builder_->AppendEmptyValues(length));
+  length_ = list_builder_->length();
+  null_count_ = list_builder_->null_count();
+  return Status::OK();
+}
+
 Status MapBuilder::AdjustStructBuilderLength() {
   // If key/item builders have been appended, adjust struct builder length
   // to match. Struct and key are non-nullable, append all valid values.
@@ -170,6 +200,31 @@ Status FixedSizeListBuilder::AppendNulls(int64_t length) {
   return value_builder_->AppendNulls(list_size_ * length);
 }
 
+Status FixedSizeListBuilder::ValidateOverflow(int64_t new_elements) {
+  auto new_length = value_builder_->length() + new_elements;
+  if (new_elements != list_size_) {
+    return Status::Invalid("Length of item not correct: expected ", list_size_,
+                           " but got array of size ", new_elements);
+  }
+  if (new_length > maximum_elements()) {
+    return Status::CapacityError("array cannot contain more than ", maximum_elements(),
+                                 " elements, have ", new_elements);
+  }
+  return Status::OK();
+}
+
+Status FixedSizeListBuilder::AppendEmptyValue() {
+  RETURN_NOT_OK(Reserve(1));
+  UnsafeAppendToBitmap(true);
+  return value_builder_->AppendEmptyValues(list_size_);
+}
+
+Status FixedSizeListBuilder::AppendEmptyValues(int64_t length) {
+  RETURN_NOT_OK(Reserve(length));
+  UnsafeAppendToBitmap(length, true);
+  return value_builder_->AppendEmptyValues(list_size_ * length);
+}
+
 Status FixedSizeListBuilder::Resize(int64_t capacity) {
   RETURN_NOT_OK(CheckCapacity(capacity));
   return ArrayBuilder::Resize(capacity);
@@ -207,15 +262,6 @@ void StructBuilder::Reset() {
   }
 }
 
-Status StructBuilder::AppendNulls(int64_t length) {
-  for (const auto& field : children_) {
-    RETURN_NOT_OK(field->AppendNulls(length));
-  }
-  ARROW_RETURN_NOT_OK(Reserve(length));
-  UnsafeAppendToBitmap(length, false);
-  return Status::OK();
-}
-
 Status StructBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
   std::shared_ptr<Buffer> null_bitmap;
   RETURN_NOT_OK(null_bitmap_builder_.Finish(&null_bitmap));
diff --git a/cpp/src/arrow/array/builder_nested.h b/cpp/src/arrow/array/builder_nested.h
index cd6fadfcc2f..12b999b786e 100644
--- a/cpp/src/arrow/array/builder_nested.h
+++ b/cpp/src/arrow/array/builder_nested.h
@@ -100,7 +100,7 @@ class BaseListBuilder : public ArrayBuilder {
 
   Status AppendNulls(int64_t length) final {
     ARROW_RETURN_NOT_OK(Reserve(length));
-    ARROW_RETURN_NOT_OK(CheckNextOffset());
+    ARROW_RETURN_NOT_OK(ValidateOverflow(0));
     UnsafeAppendToBitmap(length, false);
     const int64_t num_values = value_builder_->length();
     for (int64_t i = 0; i < length; ++i) {
@@ -109,6 +109,19 @@ class BaseListBuilder : public ArrayBuilder {
     return Status::OK();
   }
 
+  Status AppendEmptyValue() final { return Append(true); }
+
+  Status AppendEmptyValues(int64_t length) final {
+    ARROW_RETURN_NOT_OK(Reserve(length));
+    ARROW_RETURN_NOT_OK(ValidateOverflow(0));
+    UnsafeAppendToBitmap(length, true);
+    const int64_t num_values = value_builder_->length();
+    for (int64_t i = 0; i < length; ++i) {
+      offsets_builder_.UnsafeAppend(static_cast<offset_type>(num_values));
+    }
+    return Status::OK();
+  }
+
   Status FinishInternal(std::shared_ptr<ArrayData>* out) override {
     ARROW_RETURN_NOT_OK(AppendNextOffset());
 
@@ -131,6 +144,16 @@ class BaseListBuilder : public ArrayBuilder {
     return Status::OK();
   }
 
+  Status ValidateOverflow(int64_t new_elements) const {
+    auto new_length = value_builder_->length() + new_elements;
+    if (ARROW_PREDICT_FALSE(new_length > maximum_elements())) {
+      return Status::CapacityError("List array cannot contain more than ",
+                                   maximum_elements(), " elements, have ", new_elements);
+    } else {
+      return Status::OK();
+    }
+  }
+
   ArrayBuilder* value_builder() const { return value_builder_.get(); }
 
   // Cannot make this a static attribute because of linking issues
@@ -147,17 +170,8 @@ class BaseListBuilder : public ArrayBuilder {
   std::shared_ptr<ArrayBuilder> value_builder_;
   std::shared_ptr<Field> value_field_;
 
-  Status CheckNextOffset() const {
-    const int64_t num_values = value_builder_->length();
-    ARROW_RETURN_IF(
-        num_values > maximum_elements(),
-        Status::CapacityError("List array cannot contain more than ", maximum_elements(),
-                              " child elements,", " have ", num_values));
-    return Status::OK();
-  }
-
   Status AppendNextOffset() {
-    ARROW_RETURN_NOT_OK(CheckNextOffset());
+    ARROW_RETURN_NOT_OK(ValidateOverflow(0));
     const int64_t num_values = value_builder_->length();
     return offsets_builder_.Append(static_cast<offset_type>(num_values));
   }
@@ -227,6 +241,9 @@ class ARROW_EXPORT MapBuilder : public ArrayBuilder {
   MapBuilder(MemoryPool* pool, const std::shared_ptr<ArrayBuilder>& key_builder,
              const std::shared_ptr<ArrayBuilder>& item_builder, bool keys_sorted = false);
 
+  MapBuilder(MemoryPool* pool, const std::shared_ptr<ArrayBuilder>& item_builder,
+             const std::shared_ptr<DataType>& type);
+
   Status Resize(int64_t capacity) override;
   void Reset() override;
   Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
@@ -254,6 +271,10 @@ class ARROW_EXPORT MapBuilder : public ArrayBuilder {
 
   Status AppendNulls(int64_t length) final;
 
+  Status AppendEmptyValue() final;
+
+  Status AppendEmptyValues(int64_t length) final;
+
   /// \brief Get builder to append keys.
   ///
   /// Append a key with this builder should be followed by appending
@@ -276,6 +297,10 @@ class ARROW_EXPORT MapBuilder : public ArrayBuilder {
     return map(key_builder_->type(), item_builder_->type(), keys_sorted_);
   }
 
+  Status ValidateOverflow(int64_t new_elements) {
+    return list_builder_->ValidateOverflow(new_elements);
+  }
+
  protected:
   inline Status AdjustStructBuilderLength();
 
@@ -343,12 +368,23 @@ class ARROW_EXPORT FixedSizeListBuilder : public ArrayBuilder {
   /// automatically.
   Status AppendNulls(int64_t length) final;
 
+  Status ValidateOverflow(int64_t new_elements);
+
+  Status AppendEmptyValue() final;
+
+  Status AppendEmptyValues(int64_t length) final;
+
   ArrayBuilder* value_builder() const { return value_builder_.get(); }
 
   std::shared_ptr<DataType> type() const override {
     return fixed_size_list(value_field_->WithType(value_builder_->type()), list_size_);
   }
 
+  // Cannot make this a static attribute because of linking issues
+  static constexpr int64_t maximum_elements() {
+    return std::numeric_limits<FixedSizeListType::offset_type>::max() - 1;
+  }
+
  protected:
   std::shared_ptr<Field> value_field_;
   const int32_t list_size_;
@@ -395,18 +431,41 @@ class ARROW_EXPORT StructBuilder : public ArrayBuilder {
     return Status::OK();
   }
 
-  /// \brief Append a null value. Automatically appends a null to each child
+  /// \brief Append a null value. Automatically appends an empty value to each child
   /// builder.
   Status AppendNull() final {
     for (const auto& field : children_) {
-      ARROW_RETURN_NOT_OK(field->AppendNull());
+      ARROW_RETURN_NOT_OK(field->AppendEmptyValue());
     }
     return Append(false);
   }
 
-  /// \brief Append multiple null values. Automatically appends nulls to each
+  /// \brief Append multiple null values. Automatically appends empty values to each
   /// child builder.
-  Status AppendNulls(int64_t length) final;
+  Status AppendNulls(int64_t length) final {
+    for (const auto& field : children_) {
+      ARROW_RETURN_NOT_OK(field->AppendEmptyValues(length));
+    }
+    ARROW_RETURN_NOT_OK(Reserve(length));
+    UnsafeAppendToBitmap(length, false);
+    return Status::OK();
+  }
+
+  Status AppendEmptyValue() final {
+    for (const auto& field : children_) {
+      ARROW_RETURN_NOT_OK(field->AppendEmptyValue());
+    }
+    return Append(true);
+  }
+
+  Status AppendEmptyValues(int64_t length) final {
+    for (const auto& field : children_) {
+      ARROW_RETURN_NOT_OK(field->AppendEmptyValues(length));
+    }
+    ARROW_RETURN_NOT_OK(Reserve(length));
+    UnsafeAppendToBitmap(length, true);
+    return Status::OK();
+  }
 
   void Reset() override;
 
diff --git a/cpp/src/arrow/array/builder_primitive.h b/cpp/src/arrow/array/builder_primitive.h
index e6b1baa5879..e10f11fdd6c 100644
--- a/cpp/src/arrow/array/builder_primitive.h
+++ b/cpp/src/arrow/array/builder_primitive.h
@@ -31,6 +31,9 @@ namespace arrow {
 class ARROW_EXPORT NullBuilder : public ArrayBuilder {
  public:
   explicit NullBuilder(MemoryPool* pool = default_memory_pool()) : ArrayBuilder(pool) {}
+  explicit NullBuilder(const std::shared_ptr<DataType>& type,
+                       MemoryPool* pool = default_memory_pool())
+      : NullBuilder(pool) {}
 
   /// \brief Append the specified number of null elements
   Status AppendNulls(int64_t length) final {
@@ -43,6 +46,10 @@ class ARROW_EXPORT NullBuilder : public ArrayBuilder {
   /// \brief Append a single null element
   Status AppendNull() final { return AppendNulls(1); }
 
+  Status AppendEmptyValues(int64_t length) final { return AppendNulls(length); }
+
+  Status AppendEmptyValue() final { return AppendEmptyValues(1); }
+
   Status Append(std::nullptr_t) { return AppendNull(); }
 
   Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
@@ -97,6 +104,22 @@ class NumericBuilder : public ArrayBuilder {
     return Status::OK();
   }
 
+  /// \brief Append a empty element
+  Status AppendEmptyValue() final {
+    ARROW_RETURN_NOT_OK(Reserve(1));
+    data_builder_.UnsafeAppend(value_type{});  // zero
+    UnsafeAppendToBitmap(true);
+    return Status::OK();
+  }
+
+  /// \brief Append several empty elements
+  Status AppendEmptyValues(int64_t length) final {
+    ARROW_RETURN_NOT_OK(Reserve(length));
+    data_builder_.UnsafeAppend(length, value_type{});  // zero
+    UnsafeSetNotNull(length);
+    return Status::OK();
+  }
+
   value_type GetValue(int64_t index) const { return data_builder_.data()[index]; }
 
   void Reset() override { data_builder_.Reset(); }
@@ -294,6 +317,20 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
     return Status::OK();
   }
 
+  Status AppendEmptyValue() final {
+    ARROW_RETURN_NOT_OK(Reserve(1));
+    data_builder_.UnsafeAppend(false);
+    UnsafeSetNotNull(1);
+    return Status::OK();
+  }
+
+  Status AppendEmptyValues(int64_t length) final {
+    ARROW_RETURN_NOT_OK(Reserve(length));
+    data_builder_.UnsafeAppend(length, false);
+    UnsafeSetNotNull(length);
+    return Status::OK();
+  }
+
   /// Scalar append
   Status Append(const bool val) {
     ARROW_RETURN_NOT_OK(Reserve(1));
diff --git a/cpp/src/arrow/array/builder_union.h b/cpp/src/arrow/array/builder_union.h
index 1ccc7ef159f..060be474fb8 100644
--- a/cpp/src/arrow/array/builder_union.h
+++ b/cpp/src/arrow/array/builder_union.h
@@ -117,6 +117,26 @@ class ARROW_EXPORT DenseUnionBuilder : public BasicUnionBuilder {
     return child_builder->AppendNull();
   }
 
+  Status AppendEmptyValue() final {
+    const int8_t first_child_code = type_codes_[0];
+    ArrayBuilder* child_builder = type_id_to_children_[first_child_code];
+    ARROW_RETURN_NOT_OK(types_builder_.Append(first_child_code));
+    ARROW_RETURN_NOT_OK(
+        offsets_builder_.Append(static_cast<int32_t>(child_builder->length())));
+    // Append an empty value arbitrarily to the first child
+    return child_builder->AppendEmptyValue();
+  }
+
+  Status AppendEmptyValues(int64_t length) final {
+    const int8_t first_child_code = type_codes_[0];
+    ArrayBuilder* child_builder = type_id_to_children_[first_child_code];
+    ARROW_RETURN_NOT_OK(types_builder_.Append(length, first_child_code));
+    ARROW_RETURN_NOT_OK(
+        offsets_builder_.Append(length, static_cast<int32_t>(child_builder->length())));
+    // Append just a single empty value to the first child
+    return child_builder->AppendEmptyValue();
+  }
+
   /// \brief Append an element to the UnionArray. This must be followed
   ///        by an append to the appropriate child builder.
   ///
@@ -159,23 +179,45 @@ class ARROW_EXPORT SparseUnionBuilder : public BasicUnionBuilder {
                      const std::shared_ptr<DataType>& type)
       : BasicUnionBuilder(pool, children, type) {}
 
-  /// \brief Append a null value. A null is added automatically to all the
-  /// children but the type id in the slot will be 0
+  /// \brief Append a null value.
+  ///
+  /// A null is appended to the first child, empty values to the other children.
   Status AppendNull() final {
+    const auto first_child_code = type_codes_[0];
+    ARROW_RETURN_NOT_OK(types_builder_.Append(first_child_code));
+    ARROW_RETURN_NOT_OK(type_id_to_children_[first_child_code]->AppendNull());
+    for (int i = 1; i < static_cast<int>(type_codes_.size()); ++i) {
+      ARROW_RETURN_NOT_OK(type_id_to_children_[type_codes_[i]]->AppendEmptyValue());
+    }
+    return Status::OK();
+  }
+
+  /// \brief Append multiple null values.
+  ///
+  /// Nulls are appended to the first child, empty values to the other children.
+  Status AppendNulls(int64_t length) final {
+    const auto first_child_code = type_codes_[0];
+    ARROW_RETURN_NOT_OK(types_builder_.Append(length, first_child_code));
+    ARROW_RETURN_NOT_OK(type_id_to_children_[first_child_code]->AppendNulls(length));
+    for (int i = 1; i < static_cast<int>(type_codes_.size()); ++i) {
+      ARROW_RETURN_NOT_OK(
+          type_id_to_children_[type_codes_[i]]->AppendEmptyValues(length));
+    }
+    return Status::OK();
+  }
+
+  Status AppendEmptyValue() final {
     ARROW_RETURN_NOT_OK(types_builder_.Append(type_codes_[0]));
     for (int8_t code : type_codes_) {
-      ARROW_RETURN_NOT_OK(type_id_to_children_[code]->AppendNull());
+      ARROW_RETURN_NOT_OK(type_id_to_children_[code]->AppendEmptyValue());
     }
     return Status::OK();
   }
 
-  /// \brief Append multiple null values. Nulls will be automatically appended
-  /// to all the children but the type ids will be all 0.
-  Status AppendNulls(int64_t length) final {
+  Status AppendEmptyValues(int64_t length) final {
     ARROW_RETURN_NOT_OK(types_builder_.Append(length, type_codes_[0]));
-    // Append nulls to children
     for (int8_t code : type_codes_) {
-      ARROW_RETURN_NOT_OK(type_id_to_children_[code]->AppendNulls(length));
+      ARROW_RETURN_NOT_OK(type_id_to_children_[code]->AppendEmptyValues(length));
     }
     return Status::OK();
   }
@@ -186,7 +228,7 @@ class ARROW_EXPORT SparseUnionBuilder : public BasicUnionBuilder {
   /// \param[in] next_type type_id of the child to which the next value will be appended.
   ///
   /// The corresponding child builder must be appended to independently after this method
-  /// is called, and all other child builders must have null appended
+  /// is called, and all other child builders must have null or empty value appended.
   Status Append(int8_t next_type) { return types_builder_.Append(next_type); }
 };
 
diff --git a/cpp/src/arrow/array/concatenate.cc b/cpp/src/arrow/array/concatenate.cc
index 6989d7a1a37..30eeeee2a2d 100644
--- a/cpp/src/arrow/array/concatenate.cc
+++ b/cpp/src/arrow/array/concatenate.cc
@@ -70,7 +70,9 @@ static Status ConcatenateBitmaps(const std::vector<Bitmap>& bitmaps, MemoryPool*
                                  std::shared_ptr<Buffer>* out) {
   int64_t out_length = 0;
   for (const auto& bitmap : bitmaps) {
-    out_length += bitmap.range.length;
+    if (internal::AddWithOverflow(out_length, bitmap.range.length, &out_length)) {
+      return Status::Invalid("Length overflow when concatenating arrays");
+    }
   }
   ARROW_ASSIGN_OR_RAISE(*out, AllocateBitmap(out_length, pool));
   uint8_t* dst = (*out)->mutable_data();
@@ -86,10 +88,6 @@ static Status ConcatenateBitmaps(const std::vector<Bitmap>& bitmaps, MemoryPool*
     bitmap_offset += bitmap.range.length;
   }
 
-  // finally (if applicable) zero out any trailing bits
-  if (auto preceding_bits = BitUtil::kPrecedingBitmask[out_length % 8]) {
-    dst[out_length / 8] &= preceding_bits;
-  }
   return Status::OK();
 }
 
@@ -203,7 +201,7 @@ class ConcatenateImpl {
   }
 
   Status Visit(const FixedWidthType& fixed) {
-    // Handles numbers, decimal128, fixed_size_binary
+    // Handles numbers, decimal128, decimal256, fixed_size_binary
     ARROW_ASSIGN_OR_RAISE(auto buffers, Buffers(1, fixed));
     return ConcatenateBuffers(buffers, pool_).Value(&out_->buffers[1]);
   }
diff --git a/cpp/src/arrow/array/concatenate_test.cc b/cpp/src/arrow/array/concatenate_test.cc
index fb35f43ec6d..f85a730815b 100644
--- a/cpp/src/arrow/array/concatenate_test.cc
+++ b/cpp/src/arrow/array/concatenate_test.cc
@@ -31,9 +31,9 @@
 #include <gtest/gtest.h>
 
 #include "arrow/array.h"
+#include "arrow/array/builder_binary.h"
 #include "arrow/array/concatenate.h"
 #include "arrow/buffer.h"
-#include "arrow/builder.h"
 #include "arrow/status.h"
 #include "arrow/testing/gtest_common.h"
 #include "arrow/testing/random.h"
diff --git a/cpp/src/arrow/array/data.cc b/cpp/src/arrow/array/data.cc
index 7bfb39532dd..9c5e630bb2b 100644
--- a/cpp/src/arrow/array/data.cc
+++ b/cpp/src/arrow/array/data.cc
@@ -100,6 +100,8 @@ std::shared_ptr<ArrayData> ArrayData::Slice(int64_t off, int64_t len) const {
   copy->offset = off;
   if (null_count == length) {
     copy->null_count = len;
+  } else if (off == offset && len == length) {  // A copy of current.
+    copy->null_count = null_count.load();
   } else {
     copy->null_count = null_count != 0 ? kUnknownNullCount : 0;
   }
diff --git a/cpp/src/arrow/array/dict_internal.h b/cpp/src/arrow/array/dict_internal.h
index 5bf584c8216..aa027ac22de 100644
--- a/cpp/src/arrow/array/dict_internal.h
+++ b/cpp/src/arrow/array/dict_internal.h
@@ -189,16 +189,5 @@ struct DictionaryTraits<T, enable_if_fixed_size_binary<T>> {
   }
 };
 
-template <typename T>
-struct DictionaryCTraits {
-  using ArrowType = typename CTypeTraits<T>::ArrowType;
-  using MemoTableType = typename DictionaryTraits<ArrowType>::MemoTableType;
-};
-
-template <>
-struct DictionaryCTraits<util::string_view> {
-  using MemoTableType = DictionaryTraits<BinaryType>::MemoTableType;
-};
-
 }  // namespace internal
 }  // namespace arrow
diff --git a/cpp/src/arrow/array/diff_test.cc b/cpp/src/arrow/array/diff_test.cc
index bfe46d4762c..b80ed2fd955 100644
--- a/cpp/src/arrow/array/diff_test.cc
+++ b/cpp/src/arrow/array/diff_test.cc
@@ -16,10 +16,8 @@
 // under the License.
 
 #include <algorithm>
-#include <array>
 #include <cstdint>
 #include <cstring>
-#include <iterator>
 #include <limits>
 #include <memory>
 #include <numeric>
@@ -31,8 +29,6 @@
 
 #include "arrow/array.h"
 #include "arrow/array/diff.h"
-#include "arrow/buffer.h"
-#include "arrow/builder.h"
 #include "arrow/compute/api.h"
 #include "arrow/status.h"
 #include "arrow/testing/gtest_common.h"
diff --git a/cpp/src/arrow/array/validate.cc b/cpp/src/arrow/array/validate.cc
index 3063f5580cd..5bc0bf31d07 100644
--- a/cpp/src/arrow/array/validate.cc
+++ b/cpp/src/arrow/array/validate.cc
@@ -64,6 +64,13 @@ struct ValidateArrayVisitor {
     return Status::OK();
   }
 
+  Status Visit(const Decimal256Array& array) {
+    if (array.length() > 0 && array.values() == nullptr) {
+      return Status::Invalid("values is null");
+    }
+    return Status::OK();
+  }
+
   Status Visit(const StringArray& array) { return ValidateBinaryArray(array); }
 
   Status Visit(const BinaryArray& array) { return ValidateBinaryArray(array); }
diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index 6aac70eebfd..9215d9ab544 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -206,7 +206,7 @@ class PoolBuffer : public ResizableBuffer {
   }
 
   Status Resize(const int64_t new_size, bool shrink_to_fit = true) override {
-    if (new_size < 0) {
+    if (ARROW_PREDICT_FALSE(new_size < 0)) {
       return Status::Invalid("Negative buffer resize: ", new_size);
     }
     if (mutable_data_ && shrink_to_fit && new_size <= size_) {
@@ -277,13 +277,18 @@ Result<std::unique_ptr<ResizableBuffer>> AllocateResizableBuffer(const int64_t s
 }
 
 Result<std::shared_ptr<Buffer>> AllocateBitmap(int64_t length, MemoryPool* pool) {
-  return AllocateBuffer(BitUtil::BytesForBits(length), pool);
+  ARROW_ASSIGN_OR_RAISE(auto buf, AllocateBuffer(BitUtil::BytesForBits(length), pool));
+  // Zero out any trailing bits
+  if (buf->size() > 0) {
+    buf->mutable_data()[buf->size() - 1] = 0;
+  }
+  return std::move(buf);
 }
 
 Result<std::shared_ptr<Buffer>> AllocateEmptyBitmap(int64_t length, MemoryPool* pool) {
-  ARROW_ASSIGN_OR_RAISE(auto buf, AllocateBitmap(length, pool));
+  ARROW_ASSIGN_OR_RAISE(auto buf, AllocateBuffer(BitUtil::BytesForBits(length), pool));
   memset(buf->mutable_data(), 0, static_cast<size_t>(buf->size()));
-  return buf;
+  return std::move(buf);
 }
 
 Status AllocateEmptyBitmap(int64_t length, std::shared_ptr<Buffer>* out) {
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 6be07d6ca75..f22228a4588 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -40,9 +40,18 @@ struct DictionaryBuilderCase {
     return CreateFor<ValueType>();
   }
 
+  Status Visit(const NullType&) { return CreateFor<NullType>(); }
   Status Visit(const BinaryType&) { return Create<BinaryDictionaryBuilder>(); }
   Status Visit(const StringType&) { return Create<StringDictionaryBuilder>(); }
+  Status Visit(const LargeBinaryType&) {
+    return Create<DictionaryBuilder<LargeBinaryType>>();
+  }
+  Status Visit(const LargeStringType&) {
+    return Create<DictionaryBuilder<LargeStringType>>();
+  }
   Status Visit(const FixedSizeBinaryType&) { return CreateFor<FixedSizeBinaryType>(); }
+  Status Visit(const Decimal128Type&) { return CreateFor<Decimal128Type>(); }
+  Status Visit(const Decimal256Type&) { return CreateFor<Decimal256Type>(); }
 
   Status Visit(const DataType& value_type) { return NotImplemented(value_type); }
   Status Visit(const HalfFloatType& value_type) { return NotImplemented(value_type); }
@@ -59,17 +68,21 @@ struct DictionaryBuilderCase {
 
   template <typename BuilderType>
   Status Create() {
+    BuilderType* builder;
     if (dictionary != nullptr) {
-      out->reset(new BuilderType(dictionary, pool));
+      builder = new BuilderType(dictionary, pool);
     } else {
-      out->reset(new BuilderType(value_type, pool));
+      auto start_int_size = internal::GetByteWidth(*index_type);
+      builder = new BuilderType(start_int_size, value_type, pool);
     }
+    out->reset(builder);
     return Status::OK();
   }
 
   Status Make() { return VisitTypeInline(*value_type, this); }
 
   MemoryPool* pool;
+  const std::shared_ptr<DataType>& index_type;
   const std::shared_ptr<DataType>& value_type;
   const std::shared_ptr<Array>& dictionary;
   std::unique_ptr<ArrayBuilder>* out;
@@ -126,10 +139,12 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
       BUILDER_CASE(LargeBinary);
       BUILDER_CASE(FixedSizeBinary);
       BUILDER_CASE(Decimal128);
+      BUILDER_CASE(Decimal256);
 
     case Type::DICTIONARY: {
       const auto& dict_type = static_cast<const DictionaryType&>(*type);
-      DictionaryBuilderCase visitor = {pool, dict_type.value_type(), nullptr, out};
+      DictionaryBuilderCase visitor = {pool, dict_type.index_type(),
+                                       dict_type.value_type(), nullptr, out};
       return visitor.Make();
     }
 
@@ -199,7 +214,8 @@ Status MakeDictionaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>&
                              const std::shared_ptr<Array>& dictionary,
                              std::unique_ptr<ArrayBuilder>* out) {
   const auto& dict_type = static_cast<const DictionaryType&>(*type);
-  DictionaryBuilderCase visitor = {pool, dict_type.value_type(), dictionary, out};
+  DictionaryBuilderCase visitor = {pool, dict_type.index_type(), dict_type.value_type(),
+                                   dictionary, out};
   return visitor.Make();
 }
 
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 3202312c47e..4b80e558004 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -30,30 +30,3 @@
 #include "arrow/array/builder_union.h"      // IWYU pragma: keep
 #include "arrow/status.h"
 #include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class DataType;
-class MemoryPool;
-
-/// \brief Construct an empty ArrayBuilder corresponding to the data
-/// type
-/// \param[in] pool the MemoryPool to use for allocations
-/// \param[in] type an instance of DictionaryType
-/// \param[out] out the created ArrayBuilder
-ARROW_EXPORT
-Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
-                   std::unique_ptr<ArrayBuilder>* out);
-
-/// \brief Construct an empty DictionaryBuilder initialized optionally
-/// with a pre-existing dictionary
-/// \param[in] pool the MemoryPool to use for allocations
-/// \param[in] type an instance of DictionaryType
-/// \param[in] dictionary the initial dictionary, if any. May be nullptr
-/// \param[out] out the created ArrayBuilder
-ARROW_EXPORT
-Status MakeDictionaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
-                             const std::shared_ptr<Array>& dictionary,
-                             std::unique_ptr<ArrayBuilder>* out);
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/c/abi.h b/cpp/src/arrow/c/abi.h
index 821bc961281..a78170dbdbc 100644
--- a/cpp/src/arrow/c/abi.h
+++ b/cpp/src/arrow/c/abi.h
@@ -60,6 +60,44 @@ struct ArrowArray {
   void* private_data;
 };
 
+// EXPERIMENTAL: C stream interface
+
+struct ArrowArrayStream {
+  // Callback to get the stream type
+  // (will be the same for all arrays in the stream).
+  //
+  // Return value: 0 if successful, an `errno`-compatible error code otherwise.
+  //
+  // If successful, the ArrowSchema must be released independently from the stream.
+  int (*get_schema)(struct ArrowArrayStream*, struct ArrowSchema* out);
+
+  // Callback to get the next array
+  // (if no error and the array is released, the stream has ended)
+  //
+  // Return value: 0 if successful, an `errno`-compatible error code otherwise.
+  //
+  // If successful, the ArrowArray must be released independently from the stream.
+  int (*get_next)(struct ArrowArrayStream*, struct ArrowArray* out);
+
+  // Callback to get optional detailed error information.
+  // This must only be called if the last stream operation failed
+  // with a non-0 return code.
+  //
+  // Return value: pointer to a null-terminated character array describing
+  // the last error, or NULL if no description is available.
+  //
+  // The returned pointer is only valid until the next operation on this stream
+  // (including release).
+  const char* (*get_last_error)(struct ArrowArrayStream*);
+
+  // Release callback: release the stream's own resources.
+  // Note that arrays returned by `get_next` must be individually released.
+  void (*release)(struct ArrowArrayStream*);
+
+  // Opaque producer-specific data
+  void* private_data;
+};
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/cpp/src/arrow/c/bridge.cc b/cpp/src/arrow/c/bridge.cc
index 1e602a6a310..5cb3e577235 100644
--- a/cpp/src/arrow/c/bridge.cc
+++ b/cpp/src/arrow/c/bridge.cc
@@ -18,6 +18,7 @@
 #include "arrow/c/bridge.h"
 
 #include <algorithm>
+#include <cerrno>
 #include <cstring>
 #include <string>
 #include <utility>
@@ -303,9 +304,16 @@ struct SchemaExporter {
     return SetFormat("w:" + std::to_string(type.byte_width()));
   }
 
-  Status Visit(const Decimal128Type& type) {
-    return SetFormat("d:" + std::to_string(type.precision()) + "," +
-                     std::to_string(type.scale()));
+  Status Visit(const DecimalType& type) {
+    if (type.bit_width() == 128) {
+      // 128 is the default bit-width
+      return SetFormat("d:" + std::to_string(type.precision()) + "," +
+                       std::to_string(type.scale()));
+    } else {
+      return SetFormat("d:" + std::to_string(type.precision()) + "," +
+                       std::to_string(type.scale()) + "," +
+                       std::to_string(type.bit_width()));
+    }
   }
 
   Status Visit(const BinaryType& type) { return SetFormat("z"); }
@@ -972,13 +980,20 @@ struct SchemaImporter {
   Status ProcessDecimal() {
     RETURN_NOT_OK(f_parser_.CheckNext(':'));
     ARROW_ASSIGN_OR_RAISE(auto prec_scale, f_parser_.ParseInts(f_parser_.Rest()));
-    if (prec_scale.size() != 2) {
+    // 3 elements indicates bit width was communicated as well.
+    if (prec_scale.size() != 2 && prec_scale.size() != 3) {
       return f_parser_.Invalid();
     }
     if (prec_scale[0] <= 0 || prec_scale[1] <= 0) {
       return f_parser_.Invalid();
     }
-    type_ = decimal(prec_scale[0], prec_scale[1]);
+    if (prec_scale.size() == 2 || prec_scale[2] == 128) {
+      type_ = decimal(prec_scale[0], prec_scale[1]);
+    } else if (prec_scale[2] == 256) {
+      type_ = decimal256(prec_scale[0], prec_scale[1]);
+    } else {
+      return f_parser_.Invalid();
+    }
     return Status::OK();
   }
 
@@ -1501,4 +1516,197 @@ Result<std::shared_ptr<RecordBatch>> ImportRecordBatch(struct ArrowArray* array,
   return ImportRecordBatch(array, *maybe_schema);
 }
 
+//////////////////////////////////////////////////////////////////////////
+// C stream export
+
+namespace {
+
+class ExportedArrayStream {
+ public:
+  struct PrivateData {
+    explicit PrivateData(std::shared_ptr<RecordBatchReader> reader)
+        : reader_(std::move(reader)) {}
+
+    std::shared_ptr<RecordBatchReader> reader_;
+    std::string last_error_;
+
+    PrivateData() = default;
+    ARROW_DISALLOW_COPY_AND_ASSIGN(PrivateData);
+  };
+
+  explicit ExportedArrayStream(struct ArrowArrayStream* stream) : stream_(stream) {}
+
+  Status GetSchema(struct ArrowSchema* out_schema) {
+    return ExportSchema(*reader()->schema(), out_schema);
+  }
+
+  Status GetNext(struct ArrowArray* out_array) {
+    std::shared_ptr<RecordBatch> batch;
+    RETURN_NOT_OK(reader()->ReadNext(&batch));
+    if (batch == nullptr) {
+      // End of stream
+      ArrowArrayMarkReleased(out_array);
+      return Status::OK();
+    } else {
+      return ExportRecordBatch(*batch, out_array);
+    }
+  }
+
+  const char* GetLastError() {
+    const auto& last_error = private_data()->last_error_;
+    return last_error.empty() ? nullptr : last_error.c_str();
+  }
+
+  void Release() {
+    if (ArrowArrayStreamIsReleased(stream_)) {
+      return;
+    }
+    DCHECK_NE(private_data(), nullptr);
+    delete private_data();
+
+    ArrowArrayStreamMarkReleased(stream_);
+  }
+
+  // C-compatible callbacks
+
+  static int StaticGetSchema(struct ArrowArrayStream* stream,
+                             struct ArrowSchema* out_schema) {
+    ExportedArrayStream self{stream};
+    return self.ToCError(self.GetSchema(out_schema));
+  }
+
+  static int StaticGetNext(struct ArrowArrayStream* stream,
+                           struct ArrowArray* out_array) {
+    ExportedArrayStream self{stream};
+    return self.ToCError(self.GetNext(out_array));
+  }
+
+  static void StaticRelease(struct ArrowArrayStream* stream) {
+    ExportedArrayStream{stream}.Release();
+  }
+
+  static const char* StaticGetLastError(struct ArrowArrayStream* stream) {
+    return ExportedArrayStream{stream}.GetLastError();
+  }
+
+ private:
+  int ToCError(const Status& status) {
+    if (ARROW_PREDICT_TRUE(status.ok())) {
+      private_data()->last_error_.clear();
+      return 0;
+    }
+    private_data()->last_error_ = status.ToString();
+    switch (status.code()) {
+      case StatusCode::IOError:
+        return EIO;
+      case StatusCode::NotImplemented:
+        return ENOSYS;
+      case StatusCode::OutOfMemory:
+        return ENOMEM;
+      default:
+        return EINVAL;  // Fallback for Invalid, TypeError, etc.
+    }
+  }
+
+  PrivateData* private_data() {
+    return reinterpret_cast<PrivateData*>(stream_->private_data);
+  }
+
+  const std::shared_ptr<RecordBatchReader>& reader() { return private_data()->reader_; }
+
+  struct ArrowArrayStream* stream_;
+};
+
+}  // namespace
+
+Status ExportRecordBatchReader(std::shared_ptr<RecordBatchReader> reader,
+                               struct ArrowArrayStream* out) {
+  out->get_schema = ExportedArrayStream::StaticGetSchema;
+  out->get_next = ExportedArrayStream::StaticGetNext;
+  out->get_last_error = ExportedArrayStream::StaticGetLastError;
+  out->release = ExportedArrayStream::StaticRelease;
+  out->private_data = new ExportedArrayStream::PrivateData{std::move(reader)};
+  return Status::OK();
+}
+
+//////////////////////////////////////////////////////////////////////////
+// C stream import
+
+namespace {
+
+class ArrayStreamBatchReader : public RecordBatchReader {
+ public:
+  explicit ArrayStreamBatchReader(struct ArrowArrayStream* stream) {
+    ArrowArrayStreamMove(stream, &stream_);
+    DCHECK(!ArrowArrayStreamIsReleased(&stream_));
+  }
+
+  ~ArrayStreamBatchReader() {
+    ArrowArrayStreamRelease(&stream_);
+    DCHECK(ArrowArrayStreamIsReleased(&stream_));
+  }
+
+  std::shared_ptr<Schema> schema() const override { return CacheSchema(); }
+
+  Status ReadNext(std::shared_ptr<RecordBatch>* batch) override {
+    struct ArrowArray c_array;
+    RETURN_NOT_OK(StatusFromCError(stream_.get_next(&stream_, &c_array)));
+    if (ArrowArrayIsReleased(&c_array)) {
+      // End of stream
+      batch->reset();
+      return Status::OK();
+    } else {
+      return ImportRecordBatch(&c_array, CacheSchema()).Value(batch);
+    }
+  }
+
+ private:
+  std::shared_ptr<Schema> CacheSchema() const {
+    if (!schema_) {
+      struct ArrowSchema c_schema;
+      ARROW_CHECK_OK(StatusFromCError(stream_.get_schema(&stream_, &c_schema)));
+      schema_ = ImportSchema(&c_schema).ValueOrDie();
+    }
+    return schema_;
+  }
+
+  Status StatusFromCError(int errno_like) const {
+    if (ARROW_PREDICT_TRUE(errno_like == 0)) {
+      return Status::OK();
+    }
+    StatusCode code;
+    switch (errno_like) {
+      case EDOM:
+      case EINVAL:
+      case ERANGE:
+        code = StatusCode::Invalid;
+        break;
+      case ENOMEM:
+        code = StatusCode::OutOfMemory;
+        break;
+      case ENOSYS:
+        code = StatusCode::NotImplemented;
+      default:
+        code = StatusCode::IOError;
+        break;
+    }
+    const char* last_error = stream_.get_last_error(&stream_);
+    return Status(code, last_error ? std::string(last_error) : "");
+  }
+
+  mutable struct ArrowArrayStream stream_;
+  mutable std::shared_ptr<Schema> schema_;
+};
+
+}  // namespace
+
+Result<std::shared_ptr<RecordBatchReader>> ImportRecordBatchReader(
+    struct ArrowArrayStream* stream) {
+  if (ArrowArrayStreamIsReleased(stream)) {
+    return Status::Invalid("Cannot import released ArrowArrayStream");
+  }
+  // XXX should we call get_schema() here to avoid crashing on error?
+  return std::make_shared<ArrayStreamBatchReader>(stream);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/c/bridge.h b/cpp/src/arrow/c/bridge.h
index 8efb5d98bed..294f53e49fb 100644
--- a/cpp/src/arrow/c/bridge.h
+++ b/cpp/src/arrow/c/bridge.h
@@ -29,6 +29,10 @@
 
 namespace arrow {
 
+/// \defgroup c-data-interface Functions for working with the C data interface.
+///
+/// @{
+
 /// \brief Export C++ DataType using the C data interface format.
 ///
 /// The root type is considered to have empty name and metadata.
@@ -160,4 +164,34 @@ ARROW_EXPORT
 Result<std::shared_ptr<RecordBatch>> ImportRecordBatch(struct ArrowArray* array,
                                                        struct ArrowSchema* schema);
 
+/// @}
+
+/// \defgroup c-stream-interface Functions for working with the C data interface.
+///
+/// @{
+
+/// \brief EXPERIMENTAL: Export C++ RecordBatchReader using the C stream interface.
+///
+/// The resulting ArrowArrayStream struct keeps the record batch reader alive
+/// until its release callback is called by the consumer.
+///
+/// \param[in] reader RecordBatchReader object to export
+/// \param[out] out C struct where to export the stream
+ARROW_EXPORT
+Status ExportRecordBatchReader(std::shared_ptr<RecordBatchReader> reader,
+                               struct ArrowArrayStream* out);
+
+/// \brief EXPERIMENTAL: Import C++ RecordBatchReader from the C stream interface.
+///
+/// The ArrowArrayStream struct has its contents moved to a private object
+/// held alive by the resulting record batch reader.
+///
+/// \param[in,out] stream C stream interface struct
+/// \return Imported RecordBatchReader object
+ARROW_EXPORT
+Result<std::shared_ptr<RecordBatchReader>> ImportRecordBatchReader(
+    struct ArrowArrayStream* stream);
+
+/// @}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/c/bridge_test.cc b/cpp/src/arrow/c/bridge_test.cc
index 6695d6ed5db..fc11f126e72 100644
--- a/cpp/src/arrow/c/bridge_test.cc
+++ b/cpp/src/arrow/c/bridge_test.cc
@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <cerrno>
 #include <deque>
 #include <functional>
 #include <string>
@@ -22,6 +23,7 @@
 #include <utility>
 #include <vector>
 
+#include <gmock/gmock-matchers.h>
 #include <gtest/gtest.h>
 
 #include "arrow/c/bridge.h"
@@ -40,6 +42,8 @@ namespace arrow {
 
 using internal::ArrayExportGuard;
 using internal::ArrayExportTraits;
+using internal::ArrayStreamExportGuard;
+using internal::ArrayStreamExportTraits;
 using internal::SchemaExportGuard;
 using internal::SchemaExportTraits;
 
@@ -78,11 +82,11 @@ class ReleaseCallback {
   explicit ReleaseCallback(CType* c_struct) : called_(false) {
     orig_release_ = c_struct->release;
     orig_private_data_ = c_struct->private_data;
-    c_struct->release = ReleaseUnbound;
+    c_struct->release = StaticRelease;
     c_struct->private_data = this;
   }
 
-  static void ReleaseUnbound(CType* c_struct) {
+  static void StaticRelease(CType* c_struct) {
     reinterpret_cast<ReleaseCallback*>(c_struct->private_data)->Release(c_struct);
   }
 
@@ -277,6 +281,7 @@ TEST_F(TestSchemaExport, Primitive) {
   TestPrimitive(large_utf8(), "U");
 
   TestPrimitive(decimal(16, 4), "d:16,4");
+  TestPrimitive(decimal256(16, 4), "d:16,4,256");
 }
 
 TEST_F(TestSchemaExport, Temporal) {
@@ -736,6 +741,7 @@ TEST_F(TestArrayExport, Primitive) {
   TestPrimitive(large_utf8(), R"(["foo", "bar", null])");
 
   TestPrimitive(decimal(16, 4), R"(["1234.5670", null])");
+  TestPrimitive(decimal256(16, 4), R"(["1234.5670", null])");
 }
 
 TEST_F(TestArrayExport, PrimitiveSliced) {
@@ -1182,6 +1188,13 @@ TEST_F(TestSchemaImport, Primitive) {
   CheckImport(field("", float32()));
   FillPrimitive("g");
   CheckImport(field("", float64()));
+
+  FillPrimitive("d:16,4");
+  CheckImport(field("", decimal128(16, 4)));
+  FillPrimitive("d:16,4,128");
+  CheckImport(field("", decimal128(16, 4)));
+  FillPrimitive("d:16,4,256");
+  CheckImport(field("", decimal256(16, 4)));
 }
 
 TEST_F(TestSchemaImport, Temporal) {
@@ -2369,6 +2382,8 @@ TEST_F(TestSchemaRoundtrip, Primitive) {
   TestWithTypeFactory(float16);
 
   TestWithTypeFactory(std::bind(decimal, 19, 4));
+  TestWithTypeFactory(std::bind(decimal128, 19, 4));
+  TestWithTypeFactory(std::bind(decimal256, 19, 4));
   TestWithTypeFactory(std::bind(fixed_size_binary, 3));
   TestWithTypeFactory(binary);
   TestWithTypeFactory(large_utf8);
@@ -2426,7 +2441,7 @@ TEST_F(TestSchemaRoundtrip, Map) {
 
 TEST_F(TestSchemaRoundtrip, Schema) {
   auto f1 = field("f1", utf8(), /*nullable=*/false);
-  auto f2 = field("f2", list(decimal(19, 4)));
+  auto f2 = field("f2", list(decimal256(19, 4)));
   auto md1 = key_value_metadata(kMetadataKeys1, kMetadataValues1);
   auto md2 = key_value_metadata(kMetadataKeys2, kMetadataValues2);
 
@@ -2570,8 +2585,13 @@ TEST_F(TestArrayRoundtrip, Primitive) {
   TestWithJSON(int32(), "[]");
   TestWithJSON(int32(), "[4, 5, null]");
 
+  TestWithJSON(decimal128(16, 4), R"(["0.4759", "1234.5670", null])");
+  TestWithJSON(decimal256(16, 4), R"(["0.4759", "1234.5670", null])");
+
   TestWithJSONSliced(int32(), "[4, 5]");
   TestWithJSONSliced(int32(), "[4, 5, 6, null]");
+  TestWithJSONSliced(decimal128(16, 4), R"(["0.4759", "1234.5670", null])");
+  TestWithJSONSliced(decimal256(16, 4), R"(["0.4759", "1234.5670", null])");
 }
 
 TEST_F(TestArrayRoundtrip, UnknownNullCount) {
@@ -2678,4 +2698,248 @@ TEST_F(TestArrayRoundtrip, RecordBatch) {
 
 // TODO C -> C++ -> C roundtripping tests?
 
+////////////////////////////////////////////////////////////////////////////
+// Array stream export tests
+
+class FailingRecordBatchReader : public RecordBatchReader {
+ public:
+  explicit FailingRecordBatchReader(Status error) : error_(std::move(error)) {}
+
+  static std::shared_ptr<Schema> expected_schema() { return arrow::schema({}); }
+
+  std::shared_ptr<Schema> schema() const override { return expected_schema(); }
+
+  Status ReadNext(std::shared_ptr<RecordBatch>* batch) override { return error_; }
+
+ protected:
+  Status error_;
+};
+
+class BaseArrayStreamTest : public ::testing::Test {
+ public:
+  void SetUp() override {
+    pool_ = default_memory_pool();
+    orig_allocated_ = pool_->bytes_allocated();
+  }
+
+  void TearDown() override { ASSERT_EQ(pool_->bytes_allocated(), orig_allocated_); }
+
+  RecordBatchVector MakeBatches(std::shared_ptr<Schema> schema, ArrayVector arrays) {
+    DCHECK_EQ(schema->num_fields(), 1);
+    RecordBatchVector batches;
+    for (const auto& array : arrays) {
+      batches.push_back(RecordBatch::Make(schema, array->length(), {array}));
+    }
+    return batches;
+  }
+
+ protected:
+  MemoryPool* pool_;
+  int64_t orig_allocated_;
+};
+
+class TestArrayStreamExport : public BaseArrayStreamTest {
+ public:
+  void AssertStreamSchema(struct ArrowArrayStream* c_stream, const Schema& expected) {
+    struct ArrowSchema c_schema;
+    ASSERT_EQ(0, c_stream->get_schema(c_stream, &c_schema));
+
+    SchemaExportGuard schema_guard(&c_schema);
+    ASSERT_FALSE(ArrowSchemaIsReleased(&c_schema));
+    ASSERT_OK_AND_ASSIGN(auto schema, ImportSchema(&c_schema));
+    AssertSchemaEqual(expected, *schema);
+  }
+
+  void AssertStreamEnd(struct ArrowArrayStream* c_stream) {
+    struct ArrowArray c_array;
+    ASSERT_EQ(0, c_stream->get_next(c_stream, &c_array));
+
+    ArrayExportGuard guard(&c_array);
+    ASSERT_TRUE(ArrowArrayIsReleased(&c_array));
+  }
+
+  void AssertStreamNext(struct ArrowArrayStream* c_stream, const RecordBatch& expected) {
+    struct ArrowArray c_array;
+    ASSERT_EQ(0, c_stream->get_next(c_stream, &c_array));
+
+    ArrayExportGuard guard(&c_array);
+    ASSERT_FALSE(ArrowArrayIsReleased(&c_array));
+
+    ASSERT_OK_AND_ASSIGN(auto batch, ImportRecordBatch(&c_array, expected.schema()));
+    AssertBatchesEqual(expected, *batch);
+  }
+};
+
+TEST_F(TestArrayStreamExport, Empty) {
+  auto schema = arrow::schema({field("ints", int32())});
+  auto batches = MakeBatches(schema, {});
+  ASSERT_OK_AND_ASSIGN(auto reader, RecordBatchReader::Make(batches, schema));
+
+  struct ArrowArrayStream c_stream;
+
+  ASSERT_OK(ExportRecordBatchReader(reader, &c_stream));
+  ArrayStreamExportGuard guard(&c_stream);
+
+  ASSERT_FALSE(ArrowArrayStreamIsReleased(&c_stream));
+  AssertStreamSchema(&c_stream, *schema);
+  AssertStreamEnd(&c_stream);
+  AssertStreamEnd(&c_stream);
+}
+
+TEST_F(TestArrayStreamExport, Simple) {
+  auto schema = arrow::schema({field("ints", int32())});
+  auto batches = MakeBatches(
+      schema, {ArrayFromJSON(int32(), "[1, 2]"), ArrayFromJSON(int32(), "[4, 5, null]")});
+  ASSERT_OK_AND_ASSIGN(auto reader, RecordBatchReader::Make(batches, schema));
+
+  struct ArrowArrayStream c_stream;
+
+  ASSERT_OK(ExportRecordBatchReader(reader, &c_stream));
+  ArrayStreamExportGuard guard(&c_stream);
+
+  ASSERT_FALSE(ArrowArrayStreamIsReleased(&c_stream));
+  AssertStreamSchema(&c_stream, *schema);
+  AssertStreamNext(&c_stream, *batches[0]);
+  AssertStreamNext(&c_stream, *batches[1]);
+  AssertStreamEnd(&c_stream);
+  AssertStreamEnd(&c_stream);
+}
+
+TEST_F(TestArrayStreamExport, ArrayLifetime) {
+  auto schema = arrow::schema({field("ints", int32())});
+  auto batches = MakeBatches(
+      schema, {ArrayFromJSON(int32(), "[1, 2]"), ArrayFromJSON(int32(), "[4, 5, null]")});
+  ASSERT_OK_AND_ASSIGN(auto reader, RecordBatchReader::Make(batches, schema));
+
+  struct ArrowArrayStream c_stream;
+  struct ArrowSchema c_schema;
+  struct ArrowArray c_array0, c_array1;
+
+  ASSERT_OK(ExportRecordBatchReader(reader, &c_stream));
+  {
+    ArrayStreamExportGuard guard(&c_stream);
+    ASSERT_FALSE(ArrowArrayStreamIsReleased(&c_stream));
+
+    ASSERT_EQ(0, c_stream.get_schema(&c_stream, &c_schema));
+    ASSERT_EQ(0, c_stream.get_next(&c_stream, &c_array0));
+    ASSERT_EQ(0, c_stream.get_next(&c_stream, &c_array1));
+    AssertStreamEnd(&c_stream);
+  }
+
+  ArrayExportGuard guard0(&c_array0), guard1(&c_array1);
+
+  {
+    SchemaExportGuard schema_guard(&c_schema);
+    ASSERT_OK_AND_ASSIGN(auto got_schema, ImportSchema(&c_schema));
+    AssertSchemaEqual(*schema, *got_schema);
+  }
+
+  ASSERT_GT(pool_->bytes_allocated(), orig_allocated_);
+  ASSERT_OK_AND_ASSIGN(auto batch, ImportRecordBatch(&c_array1, schema));
+  AssertBatchesEqual(*batches[1], *batch);
+  ASSERT_OK_AND_ASSIGN(batch, ImportRecordBatch(&c_array0, schema));
+  AssertBatchesEqual(*batches[0], *batch);
+}
+
+TEST_F(TestArrayStreamExport, Errors) {
+  auto reader =
+      std::make_shared<FailingRecordBatchReader>(Status::Invalid("some example error"));
+
+  struct ArrowArrayStream c_stream;
+
+  ASSERT_OK(ExportRecordBatchReader(reader, &c_stream));
+  ArrayStreamExportGuard guard(&c_stream);
+
+  struct ArrowSchema c_schema;
+  ASSERT_EQ(0, c_stream.get_schema(&c_stream, &c_schema));
+  ASSERT_FALSE(ArrowSchemaIsReleased(&c_schema));
+  {
+    SchemaExportGuard schema_guard(&c_schema);
+    ASSERT_OK_AND_ASSIGN(auto schema, ImportSchema(&c_schema));
+    AssertSchemaEqual(schema, arrow::schema({}));
+  }
+
+  struct ArrowArray c_array;
+  ASSERT_EQ(EINVAL, c_stream.get_next(&c_stream, &c_array));
+}
+
+////////////////////////////////////////////////////////////////////////////
+// Array stream roundtrip tests
+
+class TestArrayStreamRoundtrip : public BaseArrayStreamTest {
+ public:
+  void Roundtrip(std::shared_ptr<RecordBatchReader>* reader,
+                 struct ArrowArrayStream* c_stream) {
+    ASSERT_OK(ExportRecordBatchReader(*reader, c_stream));
+    ASSERT_FALSE(ArrowArrayStreamIsReleased(c_stream));
+
+    ASSERT_OK_AND_ASSIGN(auto got_reader, ImportRecordBatchReader(c_stream));
+    *reader = std::move(got_reader);
+  }
+
+  void Roundtrip(
+      std::shared_ptr<RecordBatchReader> reader,
+      std::function<void(const std::shared_ptr<RecordBatchReader>&)> check_func) {
+    ArrowArrayStream c_stream;
+
+    // NOTE: ReleaseCallback<> is not immediately usable with ArrowArrayStream,
+    // because get_next and get_schema need the original private_data.
+    std::weak_ptr<RecordBatchReader> weak_reader(reader);
+    ASSERT_EQ(weak_reader.use_count(), 1);  // Expiration check will fail otherwise
+
+    ASSERT_OK(ExportRecordBatchReader(std::move(reader), &c_stream));
+    ASSERT_FALSE(ArrowArrayStreamIsReleased(&c_stream));
+
+    {
+      ASSERT_OK_AND_ASSIGN(auto new_reader, ImportRecordBatchReader(&c_stream));
+      // Stream was moved
+      ASSERT_TRUE(ArrowArrayStreamIsReleased(&c_stream));
+      ASSERT_FALSE(weak_reader.expired());
+
+      check_func(new_reader);
+    }
+    // Stream was released when `new_reader` was destroyed
+    ASSERT_TRUE(weak_reader.expired());
+  }
+
+  void AssertReaderNext(const std::shared_ptr<RecordBatchReader>& reader,
+                        const RecordBatch& expected) {
+    ASSERT_OK_AND_ASSIGN(auto batch, reader->Next());
+    ASSERT_NE(batch, nullptr);
+    AssertBatchesEqual(expected, *batch);
+  }
+
+  void AssertReaderEnd(const std::shared_ptr<RecordBatchReader>& reader) {
+    ASSERT_OK_AND_ASSIGN(auto batch, reader->Next());
+    ASSERT_EQ(batch, nullptr);
+  }
+};
+
+TEST_F(TestArrayStreamRoundtrip, Simple) {
+  auto orig_schema = arrow::schema({field("ints", int32())});
+  auto batches = MakeBatches(orig_schema, {ArrayFromJSON(int32(), "[1, 2]"),
+                                           ArrayFromJSON(int32(), "[4, 5, null]")});
+
+  ASSERT_OK_AND_ASSIGN(auto reader, RecordBatchReader::Make(batches, orig_schema));
+
+  Roundtrip(std::move(reader), [&](const std::shared_ptr<RecordBatchReader>& reader) {
+    AssertSchemaEqual(*orig_schema, *reader->schema());
+    AssertReaderNext(reader, *batches[0]);
+    AssertReaderNext(reader, *batches[1]);
+    AssertReaderEnd(reader);
+    AssertReaderEnd(reader);
+  });
+}
+
+TEST_F(TestArrayStreamRoundtrip, Errors) {
+  auto reader = std::make_shared<FailingRecordBatchReader>(
+      Status::Invalid("roundtrip error example"));
+
+  Roundtrip(std::move(reader), [&](const std::shared_ptr<RecordBatchReader>& reader) {
+    auto status = reader->Next().status();
+    ASSERT_RAISES(Invalid, status);
+    ASSERT_THAT(status.message(), ::testing::HasSubstr("roundtrip error example"));
+  });
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/c/helpers.h b/cpp/src/arrow/c/helpers.h
index a1a1240dd75..a5c1f6fe4ba 100644
--- a/cpp/src/arrow/c/helpers.h
+++ b/cpp/src/arrow/c/helpers.h
@@ -82,6 +82,36 @@ inline void ArrowArrayRelease(struct ArrowArray* array) {
   }
 }
 
+/// Query whether the C array stream is released
+inline int ArrowArrayStreamIsReleased(const struct ArrowArrayStream* stream) {
+  return stream->release == NULL;
+}
+
+/// Mark the C array stream released (for use in release callbacks)
+inline void ArrowArrayStreamMarkReleased(struct ArrowArrayStream* stream) {
+  stream->release = NULL;
+}
+
+/// Move the C array stream from `src` to `dest`
+///
+/// Note `dest` must *not* point to a valid stream already, otherwise there
+/// will be a memory leak.
+inline void ArrowArrayStreamMove(struct ArrowArrayStream* src,
+                                 struct ArrowArrayStream* dest) {
+  assert(dest != src);
+  assert(!ArrowArrayStreamIsReleased(src));
+  memcpy(dest, src, sizeof(struct ArrowArrayStream));
+  ArrowArrayStreamMarkReleased(src);
+}
+
+/// Release the C array stream, if necessary, by calling its release callback
+inline void ArrowArrayStreamRelease(struct ArrowArrayStream* stream) {
+  if (!ArrowArrayStreamIsReleased(stream)) {
+    stream->release(stream);
+    assert(ArrowArrayStreamIsReleased(stream));
+  }
+}
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/cpp/src/arrow/c/util_internal.h b/cpp/src/arrow/c/util_internal.h
index 3ece5245205..6a33be9b0da 100644
--- a/cpp/src/arrow/c/util_internal.h
+++ b/cpp/src/arrow/c/util_internal.h
@@ -34,6 +34,12 @@ struct ArrayExportTraits {
   static constexpr auto ReleaseFunc = &ArrowArrayRelease;
 };
 
+struct ArrayStreamExportTraits {
+  typedef struct ArrowArrayStream CType;
+  static constexpr auto IsReleasedFunc = &ArrowArrayStreamIsReleased;
+  static constexpr auto ReleaseFunc = &ArrowArrayStreamRelease;
+};
+
 // A RAII-style object to release a C Array / Schema struct at block scope exit.
 template <typename Traits>
 class ExportGuard {
@@ -73,6 +79,7 @@ class ExportGuard {
 
 using SchemaExportGuard = ExportGuard<SchemaExportTraits>;
 using ArrayExportGuard = ExportGuard<ArrayExportTraits>;
+using ArrayStreamExportGuard = ExportGuard<ArrayStreamExportTraits>;
 
 }  // namespace internal
 }  // namespace arrow
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index a4cea6f4da4..622f5cb5c5f 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -110,11 +110,12 @@ inline bool FloatingApproxEquals(const NumericArray<ArrowType>& left,
 
   if (opts.nans_equal()) {
     return BaseFloatingEquals<ArrowType>(left, right, [epsilon](T x, T y) -> bool {
-      return (fabs(x - y) <= epsilon) || (std::isnan(x) && std::isnan(y));
+      return (fabs(x - y) <= epsilon) || (x == y) || (std::isnan(x) && std::isnan(y));
     });
   } else {
-    return BaseFloatingEquals<ArrowType>(
-        left, right, [epsilon](T x, T y) -> bool { return fabs(x - y) <= epsilon; });
+    return BaseFloatingEquals<ArrowType>(left, right, [epsilon](T x, T y) -> bool {
+      return (fabs(x - y) <= epsilon) || (x == y);
+    });
   }
 }
 
@@ -352,6 +353,10 @@ class RangeEqualsVisitor {
     return Visit(checked_cast<const FixedSizeBinaryArray&>(left));
   }
 
+  Status Visit(const Decimal256Array& left) {
+    return Visit(checked_cast<const FixedSizeBinaryArray&>(left));
+  }
+
   Status Visit(const NullArray& left) {
     ARROW_UNUSED(left);
     result_ = true;
@@ -805,6 +810,12 @@ class TypeEqualsVisitor {
     return Status::OK();
   }
 
+  Status Visit(const Decimal256Type& left) {
+    const auto& right = checked_cast<const Decimal256Type&>(right_);
+    result_ = left.precision() == right.precision() && left.scale() == right.scale();
+    return Status::OK();
+  }
+
   template <typename T>
   enable_if_t<is_list_like_type<T>::value || is_struct_type<T>::value, Status> Visit(
       const T& left) {
@@ -861,7 +872,9 @@ class TypeEqualsVisitor {
 
 class ScalarEqualsVisitor {
  public:
-  explicit ScalarEqualsVisitor(const Scalar& right) : right_(right), result_(false) {}
+  explicit ScalarEqualsVisitor(const Scalar& right,
+                               const EqualOptions& opts = EqualOptions::Defaults())
+      : right_(right), result_(false), options_(opts) {}
 
   Status Visit(const NullScalar& left) {
     result_ = true;
@@ -874,9 +887,26 @@ class ScalarEqualsVisitor {
     return Status::OK();
   }
 
+  template <typename T>
+  typename std::enable_if<std::is_base_of<FloatScalar, T>::value ||
+                              std::is_base_of<DoubleScalar, T>::value,
+                          Status>::type
+  Visit(const T& left_) {
+    const auto& right = checked_cast<const T&>(right_);
+    if (options_.nans_equal()) {
+      result_ = right.value == left_.value ||
+                (std::isnan(right.value) && std::isnan(left_.value));
+    } else {
+      result_ = right.value == left_.value;
+    }
+    return Status::OK();
+  }
+
   template <typename T>
   typename std::enable_if<
-      std::is_base_of<internal::PrimitiveScalar<typename T::TypeClass>, T>::value ||
+      (std::is_base_of<internal::PrimitiveScalar<typename T::TypeClass>, T>::value &&
+       !std::is_base_of<FloatScalar, T>::value &&
+       !std::is_base_of<DoubleScalar, T>::value) ||
           std::is_base_of<TemporalScalar<typename T::TypeClass>, T>::value,
       Status>::type
   Visit(const T& left_) {
@@ -899,6 +929,12 @@ class ScalarEqualsVisitor {
     return Status::OK();
   }
 
+  Status Visit(const Decimal256Scalar& left) {
+    const auto& right = checked_cast<const Decimal256Scalar&>(right_);
+    result_ = left.value == right.value;
+    return Status::OK();
+  }
+
   Status Visit(const ListScalar& left) {
     const auto& right = checked_cast<const ListScalar&>(right_);
     result_ = internal::SharedPtrEquals(left.value, right.value);
@@ -967,6 +1003,7 @@ class ScalarEqualsVisitor {
  protected:
   const Scalar& right_;
   bool result_;
+  const EqualOptions options_;
 };
 
 Status PrintDiff(const Array& left, const Array& right, std::ostream* os) {
@@ -1385,7 +1422,7 @@ bool TypeEquals(const DataType& left, const DataType& right, bool check_metadata
   }
 }
 
-bool ScalarEquals(const Scalar& left, const Scalar& right) {
+bool ScalarEquals(const Scalar& left, const Scalar& right, const EqualOptions& options) {
   bool are_equal = false;
   if (&left == &right) {
     are_equal = true;
@@ -1394,7 +1431,7 @@ bool ScalarEquals(const Scalar& left, const Scalar& right) {
   } else if (left.is_valid != right.is_valid) {
     are_equal = false;
   } else {
-    ScalarEqualsVisitor visitor(right);
+    ScalarEqualsVisitor visitor(right, options);
     auto error = VisitScalarInline(left, &visitor);
     DCHECK_OK(error);
     are_equal = visitor.result();
diff --git a/cpp/src/arrow/compare.h b/cpp/src/arrow/compare.h
index abcf39a62e5..f7899b7c5c6 100644
--- a/cpp/src/arrow/compare.h
+++ b/cpp/src/arrow/compare.h
@@ -111,6 +111,8 @@ bool ARROW_EXPORT TypeEquals(const DataType& left, const DataType& right,
 /// Returns true if scalars are equal
 /// \param[in] left a Scalar
 /// \param[in] right a Scalar
-bool ARROW_EXPORT ScalarEquals(const Scalar& left, const Scalar& right);
+/// \param[in] options comparison options
+bool ARROW_EXPORT ScalarEquals(const Scalar& left, const Scalar& right,
+                               const EqualOptions& options = EqualOptions::Defaults());
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/compute/CMakeLists.txt b/cpp/src/arrow/compute/CMakeLists.txt
index 97fbd17f07d..e781dff90e2 100644
--- a/cpp/src/arrow/compute/CMakeLists.txt
+++ b/cpp/src/arrow/compute/CMakeLists.txt
@@ -65,4 +65,6 @@ add_arrow_compute_test(internals_test
                        kernel_test.cc
                        registry_test.cc)
 
+add_arrow_benchmark(function_benchmark PREFIX "arrow-compute")
+
 add_subdirectory(kernels)
diff --git a/cpp/src/arrow/compute/api_aggregate.cc b/cpp/src/arrow/compute/api_aggregate.cc
index 2802b02105d..53ee5b9a2b2 100644
--- a/cpp/src/arrow/compute/api_aggregate.cc
+++ b/cpp/src/arrow/compute/api_aggregate.cc
@@ -45,5 +45,15 @@ Result<Datum> Mode(const Datum& value, ExecContext* ctx) {
   return CallFunction("mode", {value}, ctx);
 }
 
+Result<Datum> Stddev(const Datum& value, const VarianceOptions& options,
+                     ExecContext* ctx) {
+  return CallFunction("stddev", {value}, &options, ctx);
+}
+
+Result<Datum> Variance(const Datum& value, const VarianceOptions& options,
+                       ExecContext* ctx) {
+  return CallFunction("variance", {value}, &options, ctx);
+}
+
 }  // namespace compute
 }  // namespace arrow
diff --git a/cpp/src/arrow/compute/api_aggregate.h b/cpp/src/arrow/compute/api_aggregate.h
index 5ae3cf9b5fe..710153740fc 100644
--- a/cpp/src/arrow/compute/api_aggregate.h
+++ b/cpp/src/arrow/compute/api_aggregate.h
@@ -51,11 +51,11 @@ struct ARROW_EXPORT CountOptions : public FunctionOptions {
     COUNT_NULL,
   };
 
-  explicit CountOptions(enum Mode count_mode) : count_mode(count_mode) {}
+  explicit CountOptions(enum Mode count_mode = COUNT_NON_NULL) : count_mode(count_mode) {}
 
   static CountOptions Defaults() { return CountOptions(COUNT_NON_NULL); }
 
-  enum Mode count_mode = COUNT_NON_NULL;
+  enum Mode count_mode;
 };
 
 /// \brief Control MinMax kernel behavior
@@ -66,14 +66,26 @@ struct ARROW_EXPORT MinMaxOptions : public FunctionOptions {
     /// Skip null values
     SKIP = 0,
     /// Any nulls will result in null output
-    OUTPUT_NULL
+    EMIT_NULL
   };
 
   explicit MinMaxOptions(enum Mode null_handling = SKIP) : null_handling(null_handling) {}
 
   static MinMaxOptions Defaults() { return MinMaxOptions{}; }
 
-  enum Mode null_handling = SKIP;
+  enum Mode null_handling;
+};
+
+/// \brief Control Delta Degrees of Freedom (ddof) of Variance and Stddev kernel
+///
+/// The divisor used in calculations is N - ddof, where N is the number of elements.
+/// By default, ddof is zero, and population variance or stddev is returned.
+struct ARROW_EXPORT VarianceOptions : public FunctionOptions {
+  explicit VarianceOptions(int ddof = 0) : ddof(ddof) {}
+
+  static VarianceOptions Defaults() { return VarianceOptions{}; }
+
+  int ddof = 0;
 };
 
 /// @}
@@ -130,37 +142,48 @@ Result<Datum> MinMax(const Datum& value,
                      const MinMaxOptions& options = MinMaxOptions::Defaults(),
                      ExecContext* ctx = NULLPTR);
 
-/// \brief Calculate the min / max of a numeric array.
+/// \brief Calculate the modal (most common) value of a numeric array
 ///
-/// This function returns both the min and max as a collection. The resulting
-/// datum thus consists of two scalar datums: {Datum(min), Datum(max)}
+/// This function returns both mode and count as a struct scalar, with type
+/// struct<mode: T, count: int64>, where T is the input type.
+/// If there is more than one such value, the smallest one is returned.
 ///
-/// \param[in] array input array
-/// \param[in] options see MinMaxOptions for more information
+/// \param[in] value input datum, expecting Array or ChunkedArray
 /// \param[in] ctx the function execution context, optional
-/// \return resulting datum containing a {min, max} collection
+/// \return resulting datum as a struct<mode: T, count: int64> scalar
 ///
-/// \since 1.0.0
+/// \since 2.0.0
 /// \note API not yet finalized
 ARROW_EXPORT
-Result<Datum> MinMax(const Array& array,
-                     const MinMaxOptions& options = MinMaxOptions::Defaults(),
-                     ExecContext* ctx = NULLPTR);
+Result<Datum> Mode(const Datum& value, ExecContext* ctx = NULLPTR);
 
-/// \brief Calculate the modal (most common) value of a numeric array
+/// \brief Calculate the standard deviation of a numeric array
 ///
-/// This function returns both mode and count as a struct scalar, with type
-/// struct<mode: T, count: int64>, where T is the input type.
-/// If there is more than one such value, the smallest one is returned.
+/// \param[in] value input datum, expecting Array or ChunkedArray
+/// \param[in] options see VarianceOptions for more information
+/// \param[in] ctx the function execution context, optional
+/// \return datum of the computed standard deviation as a DoubleScalar
+///
+/// \since 2.0.0
+/// \note API not yet finalized
+ARROW_EXPORT
+Result<Datum> Stddev(const Datum& value,
+                     const VarianceOptions& options = VarianceOptions::Defaults(),
+                     ExecContext* ctx = NULLPTR);
+
+/// \brief Calculate the variance of a numeric array
 ///
-/// \param[in] value input datum, expecting Array
+/// \param[in] value input datum, expecting Array or ChunkedArray
+/// \param[in] options see VarianceOptions for more information
 /// \param[in] ctx the function execution context, optional
-/// \return resulting datum as a struct<mode: T, count: int64> scalar
+/// \return datum of the computed variance as a DoubleScalar
 ///
 /// \since 2.0.0
 /// \note API not yet finalized
 ARROW_EXPORT
-Result<Datum> Mode(const Datum& value, ExecContext* ctx = NULLPTR);
+Result<Datum> Variance(const Datum& value,
+                       const VarianceOptions& options = VarianceOptions::Defaults(),
+                       ExecContext* ctx = NULLPTR);
 
 }  // namespace compute
 }  // namespace arrow
diff --git a/cpp/src/arrow/compute/api_scalar.cc b/cpp/src/arrow/compute/api_scalar.cc
index 9a911030999..353151eade2 100644
--- a/cpp/src/arrow/compute/api_scalar.cc
+++ b/cpp/src/arrow/compute/api_scalar.cc
@@ -51,6 +51,7 @@ namespace compute {
 SCALAR_ARITHMETIC_BINARY(Add, "add", "add_checked")
 SCALAR_ARITHMETIC_BINARY(Subtract, "subtract", "subtract_checked")
 SCALAR_ARITHMETIC_BINARY(Multiply, "multiply", "multiply_checked")
+SCALAR_ARITHMETIC_BINARY(Divide, "divide", "divide_checked")
 
 // ----------------------------------------------------------------------
 // Set-related operations
diff --git a/cpp/src/arrow/compute/api_scalar.h b/cpp/src/arrow/compute/api_scalar.h
index 80e3ebb98b3..62d52d245fb 100644
--- a/cpp/src/arrow/compute/api_scalar.h
+++ b/cpp/src/arrow/compute/api_scalar.h
@@ -49,6 +49,25 @@ struct ARROW_EXPORT MatchSubstringOptions : public FunctionOptions {
   std::string pattern;
 };
 
+struct ARROW_EXPORT SplitOptions : public FunctionOptions {
+  explicit SplitOptions(int64_t max_splits = -1, bool reverse = false)
+      : max_splits(max_splits), reverse(reverse) {}
+
+  /// Maximum number of splits allowed, or unlimited when -1
+  int64_t max_splits;
+  /// Start splitting from the end of the string (only relevant when max_splits != -1)
+  bool reverse;
+};
+
+struct ARROW_EXPORT SplitPatternOptions : public SplitOptions {
+  explicit SplitPatternOptions(std::string pattern, int64_t max_splits = -1,
+                               bool reverse = false)
+      : SplitOptions(max_splits, reverse), pattern(std::move(pattern)) {}
+
+  /// The exact substring to look for inside input values.
+  std::string pattern;
+};
+
 /// Options for IsIn and IndexIn functions
 struct ARROW_EXPORT SetLookupOptions : public FunctionOptions {
   explicit SetLookupOptions(Datum value_set, bool skip_nulls)
@@ -129,6 +148,20 @@ Result<Datum> Multiply(const Datum& left, const Datum& right,
                        ArithmeticOptions options = ArithmeticOptions(),
                        ExecContext* ctx = NULLPTR);
 
+/// \brief Divide two values. Array values must be the same length. If either
+/// argument is null the result will be null. For integer types, if there is
+/// a zero divisor, an error will be raised.
+///
+/// \param[in] left the dividend
+/// \param[in] right the divisor
+/// \param[in] options arithmetic options (enable/disable overflow checking), optional
+/// \param[in] ctx the function execution context, optional
+/// \return the elementwise quotient
+ARROW_EXPORT
+Result<Datum> Divide(const Datum& left, const Datum& right,
+                     ArithmeticOptions options = ArithmeticOptions(),
+                     ExecContext* ctx = NULLPTR);
+
 /// \brief Compare a numeric array with a scalar.
 ///
 /// \param[in] left datum to compare, must be an Array
diff --git a/cpp/src/arrow/compute/api_vector.h b/cpp/src/arrow/compute/api_vector.h
index de36202f019..2c77e8ee155 100644
--- a/cpp/src/arrow/compute/api_vector.h
+++ b/cpp/src/arrow/compute/api_vector.h
@@ -59,7 +59,7 @@ struct ARROW_EXPORT TakeOptions : public FunctionOptions {
 };
 
 /// \brief Partitioning options for NthToIndices
-struct PartitionNthOptions : public FunctionOptions {
+struct ARROW_EXPORT PartitionNthOptions : public FunctionOptions {
   explicit PartitionNthOptions(int64_t pivot) : pivot(pivot) {}
 
   /// The index into the equivalent sorted array of the partition pivot element.
diff --git a/cpp/src/arrow/compute/cast.cc b/cpp/src/arrow/compute/cast.cc
index a9700f3159d..29a80f73241 100644
--- a/cpp/src/arrow/compute/cast.cc
+++ b/cpp/src/arrow/compute/cast.cc
@@ -38,8 +38,10 @@ using internal::ToTypeName;
 namespace compute {
 namespace internal {
 
+namespace {
+
 std::unordered_map<int, std::shared_ptr<CastFunction>> g_cast_table;
-static std::once_flag cast_table_initialized;
+std::once_flag cast_table_initialized;
 
 void AddCastFunctions(const std::vector<std::shared_ptr<CastFunction>>& funcs) {
   for (const auto& func : funcs) {
@@ -57,8 +59,6 @@ void InitCastTable() {
 
 void EnsureInitCastTable() { std::call_once(cast_table_initialized, InitCastTable); }
 
-namespace {
-
 // Private version of GetCastFunction with better error reporting
 // if the input type is known.
 Result<std::shared_ptr<CastFunction>> GetCastFunctionInternal(
@@ -78,13 +78,17 @@ Result<std::shared_ptr<CastFunction>> GetCastFunctionInternal(
   return it->second;
 }
 
-}  // namespace
+const FunctionDoc cast_doc{"Cast values to another data type",
+                           ("Behavior when values wouldn't fit in the target type\n"
+                            "can be controlled through CastOptions."),
+                           {"input"},
+                           "CastOptions"};
 
-// Metafunction for dispatching to appropraite CastFunction. This corresponds
+// Metafunction for dispatching to appropriate CastFunction. This corresponds
 // to the standard SQL CAST(expr AS target_type)
 class CastMetaFunction : public MetaFunction {
  public:
-  CastMetaFunction() : MetaFunction("cast", Arity::Unary()) {}
+  CastMetaFunction() : MetaFunction("cast", Arity::Unary(), &cast_doc) {}
 
   Result<const CastOptions*> ValidateOptions(const FunctionOptions* options) const {
     auto cast_options = static_cast<const CastOptions*>(options);
@@ -112,6 +116,8 @@ class CastMetaFunction : public MetaFunction {
   }
 };
 
+}  // namespace
+
 void RegisterScalarCast(FunctionRegistry* registry) {
   DCHECK_OK(registry->AddFunction(std::make_shared<CastMetaFunction>()));
 }
@@ -124,7 +130,7 @@ struct CastFunction::CastFunctionImpl {
 };
 
 CastFunction::CastFunction(std::string name, Type::type out_type)
-    : ScalarFunction(std::move(name), Arity::Unary()) {
+    : ScalarFunction(std::move(name), Arity::Unary(), /*doc=*/nullptr) {
   impl_.reset(new CastFunctionImpl());
   impl_->out_type = out_type;
 }
@@ -157,7 +163,7 @@ bool CastFunction::CanCastTo(const DataType& out_type) const {
   return impl_->in_types.find(static_cast<int>(out_type.id())) != impl_->in_types.end();
 }
 
-Result<const ScalarKernel*> CastFunction::DispatchExact(
+Result<const Kernel*> CastFunction::DispatchExact(
     const std::vector<ValueDescr>& values) const {
   const int passed_num_args = static_cast<int>(values.size());
 
diff --git a/cpp/src/arrow/compute/cast.h b/cpp/src/arrow/compute/cast.h
index 82dd357e9dd..43392ce99bf 100644
--- a/cpp/src/arrow/compute/cast.h
+++ b/cpp/src/arrow/compute/cast.h
@@ -98,7 +98,7 @@ class CastFunction : public ScalarFunction {
 
   bool CanCastTo(const DataType& out_type) const;
 
-  Result<const ScalarKernel*> DispatchExact(
+  Result<const Kernel*> DispatchExact(
       const std::vector<ValueDescr>& values) const override;
 
  private:
diff --git a/cpp/src/arrow/compute/exec.cc b/cpp/src/arrow/compute/exec.cc
index fbc3693e5be..dd97119151e 100644
--- a/cpp/src/arrow/compute/exec.cc
+++ b/cpp/src/arrow/compute/exec.cc
@@ -45,6 +45,7 @@
 #include "arrow/util/checked_cast.h"
 #include "arrow/util/cpu_info.h"
 #include "arrow/util/logging.h"
+#include "arrow/util/make_unique.h"
 
 namespace arrow {
 
@@ -62,32 +63,58 @@ Result<std::shared_ptr<Buffer>> AllocateDataBuffer(KernelContext* ctx, int64_t l
   if (bit_width == 1) {
     return ctx->AllocateBitmap(length);
   } else {
-    ARROW_CHECK_EQ(bit_width % 8, 0)
-        << "Only bit widths with multiple of 8 are currently supported";
-    int64_t buffer_size = length * bit_width / 8;
+    int64_t buffer_size = BitUtil::BytesForBits(length * bit_width);
     return ctx->Allocate(buffer_size);
   }
   return Status::OK();
 }
 
-bool CanPreallocate(const DataType& type) {
-  // There are currently cases where NullType is the output type, so we disable
-  // any preallocation logic when this occurs
-  return is_fixed_width(type.id()) && type.id() != Type::NA;
-}
+struct BufferPreallocation {
+  explicit BufferPreallocation(int bit_width = -1, int added_length = 0)
+      : bit_width(bit_width), added_length(added_length) {}
 
-Status GetValueDescriptors(const std::vector<Datum>& args,
-                           std::vector<ValueDescr>* descrs) {
-  for (const auto& arg : args) {
-    descrs->emplace_back(arg.descr());
+  int bit_width;
+  int added_length;
+};
+
+void ComputeDataPreallocate(const DataType& type,
+                            std::vector<BufferPreallocation>* widths) {
+  if (is_fixed_width(type.id()) && type.id() != Type::NA) {
+    widths->emplace_back(checked_cast<const FixedWidthType&>(type).bit_width());
+    return;
+  }
+  // Preallocate binary and list offsets
+  switch (type.id()) {
+    case Type::BINARY:
+    case Type::STRING:
+    case Type::LIST:
+    case Type::MAP:
+      widths->emplace_back(32, /*added_length=*/1);
+      return;
+    case Type::LARGE_BINARY:
+    case Type::LARGE_STRING:
+    case Type::LARGE_LIST:
+      widths->emplace_back(64, /*added_length=*/1);
+      return;
+    default:
+      break;
   }
-  return Status::OK();
 }
 
 }  // namespace
 
 namespace detail {
 
+Status CheckAllValues(const std::vector<Datum>& values) {
+  for (const auto& value : values) {
+    if (!value.is_value()) {
+      return Status::Invalid("Tried executing function with non-value type: ",
+                             value.ToString());
+    }
+  }
+  return Status::OK();
+}
+
 ExecBatchIterator::ExecBatchIterator(std::vector<Datum> args, int64_t length,
                                      int64_t max_chunksize)
     : args_(std::move(args)),
@@ -184,6 +211,8 @@ bool ExecBatchIterator::Next(ExecBatch* batch) {
   return true;
 }
 
+namespace {
+
 bool ArrayHasNulls(const ArrayData& data) {
   // As discovered in ARROW-8863 (and not only for that reason)
   // ArrayData::null_count can -1 even when buffers[0] is nullptr. So we check
@@ -393,40 +422,18 @@ class NullPropagator {
   bool bitmap_preallocated_ = false;
 };
 
-Status PropagateNulls(KernelContext* ctx, const ExecBatch& batch, ArrayData* output) {
-  DCHECK_NE(nullptr, output);
-  DCHECK_GT(output->buffers.size(), 0);
-
-  if (output->type->id() == Type::NA) {
-    // Null output type is a no-op (rare when this would happen but we at least
-    // will test for it)
-    return Status::OK();
-  }
-
-  // This function is ONLY able to write into output with non-zero offset
-  // when the bitmap is preallocated. This could be a DCHECK but returning
-  // error Status for now for emphasis
-  if (output->offset != 0 && output->buffers[0] == nullptr) {
-    return Status::Invalid(
-        "Can only propagate nulls into pre-allocated memory "
-        "when the output offset is non-zero");
-  }
-  NullPropagator propagator(ctx, batch, output);
-  return propagator.Execute();
-}
-
 std::shared_ptr<ChunkedArray> ToChunkedArray(const std::vector<Datum>& values,
                                              const std::shared_ptr<DataType>& type) {
   std::vector<std::shared_ptr<Array>> arrays;
-  for (const auto& val : values) {
-    auto boxed = val.make_array();
-    if (boxed->length() == 0) {
+  arrays.reserve(values.size());
+  for (const Datum& val : values) {
+    if (val.length() == 0) {
       // Skip empty chunks
       continue;
     }
-    arrays.emplace_back(std::move(boxed));
+    arrays.emplace_back(val.make_array());
   }
-  return std::make_shared<ChunkedArray>(arrays, type);
+  return std::make_shared<ChunkedArray>(std::move(arrays), type);
 }
 
 bool HaveChunkedArray(const std::vector<Datum>& values) {
@@ -438,106 +445,70 @@ bool HaveChunkedArray(const std::vector<Datum>& values) {
   return false;
 }
 
-Status CheckAllValues(const std::vector<Datum>& values) {
-  for (const auto& value : values) {
-    if (!value.is_value()) {
-      return Status::Invalid("Tried executing function with non-value type: ",
-                             value.ToString());
-    }
-  }
-  return Status::OK();
-}
-
-template <typename FunctionType>
-class FunctionExecutorImpl : public FunctionExecutor {
+template <typename KernelType>
+class KernelExecutorImpl : public KernelExecutor {
  public:
-  FunctionExecutorImpl(ExecContext* exec_ctx, const FunctionType* func,
-                       const FunctionOptions* options)
-      : exec_ctx_(exec_ctx), kernel_ctx_(exec_ctx), func_(func), options_(options) {}
-
- protected:
-  using KernelType = typename FunctionType::KernelType;
+  Status Init(KernelContext* kernel_ctx, KernelInitArgs args) override {
+    kernel_ctx_ = kernel_ctx;
+    kernel_ = static_cast<const KernelType*>(args.kernel);
 
-  void Reset() {}
+    // Resolve the output descriptor for this kernel
+    ARROW_ASSIGN_OR_RAISE(
+        output_descr_, kernel_->signature->out_type().Resolve(kernel_ctx_, args.inputs));
 
-  Status InitState() {
-    // Some kernels require initialization of an opaque state object
-    if (kernel_->init) {
-      KernelInitArgs init_args{kernel_, input_descrs_, options_};
-      state_ = kernel_->init(&kernel_ctx_, init_args);
-      ARROW_CTX_RETURN_IF_ERROR(&kernel_ctx_);
-      kernel_ctx_.SetState(state_.get());
-    }
     return Status::OK();
   }
 
+ protected:
   // This is overridden by the VectorExecutor
   virtual Status SetupArgIteration(const std::vector<Datum>& args) {
-    ARROW_ASSIGN_OR_RAISE(batch_iterator_,
-                          ExecBatchIterator::Make(args, exec_ctx_->exec_chunksize()));
+    ARROW_ASSIGN_OR_RAISE(
+        batch_iterator_, ExecBatchIterator::Make(args, exec_context()->exec_chunksize()));
     return Status::OK();
   }
 
-  Status BindArgs(const std::vector<Datum>& args) {
-    RETURN_NOT_OK(GetValueDescriptors(args, &input_descrs_));
-    ARROW_ASSIGN_OR_RAISE(kernel_, func_->DispatchExact(input_descrs_));
-
-    // Initialize kernel state, since type resolution may depend on this state
-    RETURN_NOT_OK(this->InitState());
-
-    // Resolve the output descriptor for this kernel
-    ARROW_ASSIGN_OR_RAISE(output_descr_, kernel_->signature->out_type().Resolve(
-                                             &kernel_ctx_, input_descrs_));
-
-    return SetupArgIteration(args);
-  }
-
   Result<std::shared_ptr<ArrayData>> PrepareOutput(int64_t length) {
     auto out = std::make_shared<ArrayData>(output_descr_.type, length);
     out->buffers.resize(output_num_buffers_);
 
     if (validity_preallocated_) {
-      ARROW_ASSIGN_OR_RAISE(out->buffers[0], kernel_ctx_.AllocateBitmap(length));
-    }
-    if (data_preallocated_) {
-      const auto& fw_type = checked_cast<const FixedWidthType&>(*out->type);
-      ARROW_ASSIGN_OR_RAISE(
-          out->buffers[1], AllocateDataBuffer(&kernel_ctx_, length, fw_type.bit_width()));
+      ARROW_ASSIGN_OR_RAISE(out->buffers[0], kernel_ctx_->AllocateBitmap(length));
+    }
+    for (size_t i = 0; i < data_preallocated_.size(); ++i) {
+      const auto& prealloc = data_preallocated_[i];
+      if (prealloc.bit_width >= 0) {
+        ARROW_ASSIGN_OR_RAISE(
+            out->buffers[i + 1],
+            AllocateDataBuffer(kernel_ctx_, length + prealloc.added_length,
+                               prealloc.bit_width));
+      }
     }
     return out;
   }
 
-  ValueDescr output_descr() const override { return output_descr_; }
+  ExecContext* exec_context() { return kernel_ctx_->exec_context(); }
+  KernelState* state() { return kernel_ctx_->state(); }
 
   // Not all of these members are used for every executor type
 
-  ExecContext* exec_ctx_;
-  KernelContext kernel_ctx_;
-  const FunctionType* func_;
+  KernelContext* kernel_ctx_;
   const KernelType* kernel_;
   std::unique_ptr<ExecBatchIterator> batch_iterator_;
-  std::unique_ptr<KernelState> state_;
-  std::vector<ValueDescr> input_descrs_;
   ValueDescr output_descr_;
-  const FunctionOptions* options_;
 
   int output_num_buffers_;
 
-  // If true, then the kernel writes into a preallocated data buffer
-  bool data_preallocated_ = false;
-
   // If true, then memory is preallocated for the validity bitmap with the same
   // strategy as the data buffer(s).
   bool validity_preallocated_ = false;
+
+  // The kernel writes into data buffers preallocated for these bit widths
+  // (0 indicates no preallocation);
+  std::vector<BufferPreallocation> data_preallocated_;
 };
 
-class ScalarExecutor : public FunctionExecutorImpl<ScalarFunction> {
+class ScalarExecutor : public KernelExecutorImpl<ScalarKernel> {
  public:
-  using FunctionType = ScalarFunction;
-  static constexpr Function::Kind function_kind = Function::SCALAR;
-  using BASE = FunctionExecutorImpl<ScalarFunction>;
-  using BASE::BASE;
-
   Status Execute(const std::vector<Datum>& args, ExecListener* listener) override {
     RETURN_NOT_OK(PrepareExecute(args));
     ExecBatch batch;
@@ -574,7 +545,9 @@ class ScalarExecutor : public FunctionExecutorImpl<ScalarFunction> {
       } else {
         // XXX: In the case where no outputs are omitted, is returning a 0-length
         // array always the correct move?
-        return MakeArrayOfNull(output_descr_.type, /*length=*/0).ValueOrDie();
+        return MakeArrayOfNull(output_descr_.type, /*length=*/0,
+                               exec_context()->memory_pool())
+            .ValueOrDie();
       }
     }
   }
@@ -587,7 +560,7 @@ class ScalarExecutor : public FunctionExecutorImpl<ScalarFunction> {
     if (output_descr_.shape == ValueDescr::ARRAY) {
       ArrayData* out_arr = out.mutable_array();
       if (kernel_->null_handling == NullHandling::INTERSECTION) {
-        RETURN_NOT_OK(PropagateNulls(&kernel_ctx_, batch, out_arr));
+        RETURN_NOT_OK(PropagateNulls(kernel_ctx_, batch, out_arr));
       } else if (kernel_->null_handling == NullHandling::OUTPUT_NOT_NULL) {
         out_arr->null_count = 0;
       }
@@ -602,8 +575,8 @@ class ScalarExecutor : public FunctionExecutorImpl<ScalarFunction> {
       }
     }
 
-    kernel_->exec(&kernel_ctx_, batch, &out);
-    ARROW_CTX_RETURN_IF_ERROR(&kernel_ctx_);
+    kernel_->exec(kernel_ctx_, batch, &out);
+    ARROW_CTX_RETURN_IF_ERROR(kernel_ctx_);
     if (!preallocate_contiguous_) {
       // If we are producing chunked output rather than one big array, then
       // emit each chunk as soon as it's available
@@ -613,8 +586,7 @@ class ScalarExecutor : public FunctionExecutorImpl<ScalarFunction> {
   }
 
   Status PrepareExecute(const std::vector<Datum>& args) {
-    this->Reset();
-    RETURN_NOT_OK(this->BindArgs(args));
+    RETURN_NOT_OK(this->SetupArgIteration(args));
 
     if (output_descr_.shape == ValueDescr::ARRAY) {
       // If the executor is configured to produce a single large Array output for
@@ -675,24 +647,27 @@ class ScalarExecutor : public FunctionExecutorImpl<ScalarFunction> {
     output_num_buffers_ = static_cast<int>(output_descr_.type->layout().buffers.size());
 
     // Decide if we need to preallocate memory for this kernel
-    data_preallocated_ = ((kernel_->mem_allocation == MemAllocation::PREALLOCATE) &&
-                          CanPreallocate(*output_descr_.type));
     validity_preallocated_ =
         (kernel_->null_handling != NullHandling::COMPUTED_NO_PREALLOCATE &&
          kernel_->null_handling != NullHandling::OUTPUT_NOT_NULL);
+    if (kernel_->mem_allocation == MemAllocation::PREALLOCATE) {
+      ComputeDataPreallocate(*output_descr_.type, &data_preallocated_);
+    }
 
-    // Contiguous preallocation only possible if both the VALIDITY and DATA can
-    // be preallocated. Otherwise, we must go chunk-by-chunk. Note that when
-    // the DATA cannot be preallocated, the VALIDITY may still be preallocated
-    // depending on the NullHandling of the kernel
+    // Contiguous preallocation only possible on non-nested types if all
+    // buffers are preallocated.  Otherwise, we must go chunk-by-chunk.
     //
-    // Some kernels are unable to write into sliced outputs, so we respect the
-    // kernel's attributes
+    // Some kernels are also unable to write into sliced outputs, so we respect the
+    // kernel's attributes.
     preallocate_contiguous_ =
-        (exec_ctx_->preallocate_contiguous() && kernel_->can_write_into_slices &&
-         data_preallocated_ && validity_preallocated_);
+        (exec_context()->preallocate_contiguous() && kernel_->can_write_into_slices &&
+         validity_preallocated_ && !is_nested(output_descr_.type->id()) &&
+         data_preallocated_.size() == static_cast<size_t>(output_num_buffers_ - 1) &&
+         std::all_of(data_preallocated_.begin(), data_preallocated_.end(),
+                     [](const BufferPreallocation& prealloc) {
+                       return prealloc.bit_width >= 0;
+                     }));
     if (preallocate_contiguous_) {
-      DCHECK_EQ(2, output_num_buffers_);
       ARROW_ASSIGN_OR_RAISE(preallocated_, PrepareOutput(total_length));
     }
     return Status::OK();
@@ -727,13 +702,8 @@ Status PackBatchNoChunks(const std::vector<Datum>& args, ExecBatch* out) {
   return Status::OK();
 }
 
-class VectorExecutor : public FunctionExecutorImpl<VectorFunction> {
+class VectorExecutor : public KernelExecutorImpl<VectorKernel> {
  public:
-  using FunctionType = VectorFunction;
-  static constexpr Function::Kind function_kind = Function::VECTOR;
-  using BASE = FunctionExecutorImpl<VectorFunction>;
-  using BASE::BASE;
-
   Status Execute(const std::vector<Datum>& args, ExecListener* listener) override {
     RETURN_NOT_OK(PrepareExecute(args));
     ExecBatch batch;
@@ -784,10 +754,10 @@ class VectorExecutor : public FunctionExecutorImpl<VectorFunction> {
 
     if (kernel_->null_handling == NullHandling::INTERSECTION &&
         output_descr_.shape == ValueDescr::ARRAY) {
-      RETURN_NOT_OK(PropagateNulls(&kernel_ctx_, batch, out.mutable_array()));
+      RETURN_NOT_OK(PropagateNulls(kernel_ctx_, batch, out.mutable_array()));
     }
-    kernel_->exec(&kernel_ctx_, batch, &out);
-    ARROW_CTX_RETURN_IF_ERROR(&kernel_ctx_);
+    kernel_->exec(kernel_ctx_, batch, &out);
+    ARROW_CTX_RETURN_IF_ERROR(kernel_ctx_);
     if (!kernel_->finalize) {
       // If there is no result finalizer (e.g. for hash-based functions, we can
       // emit the processed batch right away rather than waiting
@@ -802,8 +772,8 @@ class VectorExecutor : public FunctionExecutorImpl<VectorFunction> {
     if (kernel_->finalize) {
       // Intermediate results require post-processing after the execution is
       // completed (possibly involving some accumulated state)
-      kernel_->finalize(&kernel_ctx_, &results_);
-      ARROW_CTX_RETURN_IF_ERROR(&kernel_ctx_);
+      kernel_->finalize(kernel_ctx_, &results_);
+      ARROW_CTX_RETURN_IF_ERROR(kernel_ctx_);
       for (const auto& result : results_) {
         RETURN_NOT_OK(listener->OnResult(result));
       }
@@ -813,38 +783,39 @@ class VectorExecutor : public FunctionExecutorImpl<VectorFunction> {
 
   Status SetupArgIteration(const std::vector<Datum>& args) override {
     if (kernel_->can_execute_chunkwise) {
-      ARROW_ASSIGN_OR_RAISE(batch_iterator_,
-                            ExecBatchIterator::Make(args, exec_ctx_->exec_chunksize()));
+      ARROW_ASSIGN_OR_RAISE(batch_iterator_, ExecBatchIterator::Make(
+                                                 args, exec_context()->exec_chunksize()));
     }
     return Status::OK();
   }
 
   Status PrepareExecute(const std::vector<Datum>& args) {
-    this->Reset();
-    RETURN_NOT_OK(this->BindArgs(args));
+    RETURN_NOT_OK(this->SetupArgIteration(args));
     output_num_buffers_ = static_cast<int>(output_descr_.type->layout().buffers.size());
 
     // Decide if we need to preallocate memory for this kernel
-    data_preallocated_ = ((kernel_->mem_allocation == MemAllocation::PREALLOCATE) &&
-                          CanPreallocate(*output_descr_.type));
     validity_preallocated_ =
         (kernel_->null_handling != NullHandling::COMPUTED_NO_PREALLOCATE &&
          kernel_->null_handling != NullHandling::OUTPUT_NOT_NULL);
+    if (kernel_->mem_allocation == MemAllocation::PREALLOCATE) {
+      ComputeDataPreallocate(*output_descr_.type, &data_preallocated_);
+    }
     return Status::OK();
   }
 
   std::vector<Datum> results_;
 };
 
-class ScalarAggExecutor : public FunctionExecutorImpl<ScalarAggregateFunction> {
+class ScalarAggExecutor : public KernelExecutorImpl<ScalarAggregateKernel> {
  public:
-  using FunctionType = ScalarAggregateFunction;
-  static constexpr Function::Kind function_kind = Function::SCALAR_AGGREGATE;
-  using BASE = FunctionExecutorImpl<ScalarAggregateFunction>;
-  using BASE::BASE;
+  Status Init(KernelContext* ctx, KernelInitArgs args) override {
+    input_descrs_ = &args.inputs;
+    options_ = args.options;
+    return KernelExecutorImpl<ScalarAggregateKernel>::Init(ctx, args);
+  }
 
   Status Execute(const std::vector<Datum>& args, ExecListener* listener) override {
-    RETURN_NOT_OK(BindArgs(args));
+    RETURN_NOT_OK(this->SetupArgIteration(args));
 
     ExecBatch batch;
     while (batch_iterator_->Next(&batch)) {
@@ -855,8 +826,8 @@ class ScalarAggExecutor : public FunctionExecutorImpl<ScalarAggregateFunction> {
     }
 
     Datum out;
-    kernel_->finalize(&kernel_ctx_, &out);
-    ARROW_CTX_RETURN_IF_ERROR(&kernel_ctx_);
+    kernel_->finalize(kernel_ctx_, &out);
+    ARROW_CTX_RETURN_IF_ERROR(kernel_ctx_);
     RETURN_NOT_OK(listener->OnResult(std::move(out)));
     return Status::OK();
   }
@@ -869,51 +840,74 @@ class ScalarAggExecutor : public FunctionExecutorImpl<ScalarAggregateFunction> {
 
  private:
   Status Consume(const ExecBatch& batch) {
-    KernelInitArgs init_args{kernel_, input_descrs_, options_};
-    auto batch_state = kernel_->init(&kernel_ctx_, init_args);
-    ARROW_CTX_RETURN_IF_ERROR(&kernel_ctx_);
+    auto batch_state = kernel_->init(kernel_ctx_, {kernel_, *input_descrs_, options_});
+    ARROW_CTX_RETURN_IF_ERROR(kernel_ctx_);
 
     if (batch_state == nullptr) {
-      kernel_ctx_.SetStatus(
+      kernel_ctx_->SetStatus(
           Status::Invalid("ScalarAggregation requires non-null kernel state"));
-      return kernel_ctx_.status();
+      return kernel_ctx_->status();
     }
 
-    KernelContext batch_ctx(exec_ctx_);
+    KernelContext batch_ctx(exec_context());
     batch_ctx.SetState(batch_state.get());
 
     kernel_->consume(&batch_ctx, batch);
     ARROW_CTX_RETURN_IF_ERROR(&batch_ctx);
 
-    kernel_->merge(&kernel_ctx_, *batch_state, state_.get());
-    ARROW_CTX_RETURN_IF_ERROR(&kernel_ctx_);
+    kernel_->merge(kernel_ctx_, std::move(*batch_state), state());
+    ARROW_CTX_RETURN_IF_ERROR(kernel_ctx_);
     return Status::OK();
   }
+
+  const std::vector<ValueDescr>* input_descrs_;
+  const FunctionOptions* options_;
 };
 
 template <typename ExecutorType,
           typename FunctionType = typename ExecutorType::FunctionType>
-Result<std::unique_ptr<FunctionExecutor>> MakeExecutor(ExecContext* ctx,
-                                                       const Function* func,
-                                                       const FunctionOptions* options) {
+Result<std::unique_ptr<KernelExecutor>> MakeExecutor(ExecContext* ctx,
+                                                     const Function* func,
+                                                     const FunctionOptions* options) {
   DCHECK_EQ(ExecutorType::function_kind, func->kind());
   auto typed_func = checked_cast<const FunctionType*>(func);
-  return std::unique_ptr<FunctionExecutor>(new ExecutorType(ctx, typed_func, options));
+  return std::unique_ptr<KernelExecutor>(new ExecutorType(ctx, typed_func, options));
 }
 
-Result<std::unique_ptr<FunctionExecutor>> FunctionExecutor::Make(
-    ExecContext* ctx, const Function* func, const FunctionOptions* options) {
-  switch (func->kind()) {
-    case Function::SCALAR:
-      return MakeExecutor<detail::ScalarExecutor>(ctx, func, options);
-    case Function::VECTOR:
-      return MakeExecutor<detail::VectorExecutor>(ctx, func, options);
-    case Function::SCALAR_AGGREGATE:
-      return MakeExecutor<detail::ScalarAggExecutor>(ctx, func, options);
-    default:
-      DCHECK(false);
-      return nullptr;
+}  // namespace
+
+Status PropagateNulls(KernelContext* ctx, const ExecBatch& batch, ArrayData* output) {
+  DCHECK_NE(nullptr, output);
+  DCHECK_GT(output->buffers.size(), 0);
+
+  if (output->type->id() == Type::NA) {
+    // Null output type is a no-op (rare when this would happen but we at least
+    // will test for it)
+    return Status::OK();
+  }
+
+  // This function is ONLY able to write into output with non-zero offset
+  // when the bitmap is preallocated. This could be a DCHECK but returning
+  // error Status for now for emphasis
+  if (output->offset != 0 && output->buffers[0] == nullptr) {
+    return Status::Invalid(
+        "Can only propagate nulls into pre-allocated memory "
+        "when the output offset is non-zero");
   }
+  NullPropagator propagator(ctx, batch, output);
+  return propagator.Execute();
+}
+
+std::unique_ptr<KernelExecutor> KernelExecutor::MakeScalar() {
+  return ::arrow::internal::make_unique<detail::ScalarExecutor>();
+}
+
+std::unique_ptr<KernelExecutor> KernelExecutor::MakeVector() {
+  return ::arrow::internal::make_unique<detail::VectorExecutor>();
+}
+
+std::unique_ptr<KernelExecutor> KernelExecutor::MakeScalarAggregate() {
+  return ::arrow::internal::make_unique<detail::ScalarAggExecutor>();
 }
 
 }  // namespace detail
@@ -952,9 +946,6 @@ Result<Datum> CallFunction(const std::string& func_name, const std::vector<Datum
   }
   ARROW_ASSIGN_OR_RAISE(std::shared_ptr<const Function> func,
                         ctx->func_registry()->GetFunction(func_name));
-  if (options == nullptr) {
-    options = func->default_options();
-  }
   return func->Execute(args, options, ctx);
 }
 
diff --git a/cpp/src/arrow/compute/exec_internal.h b/cpp/src/arrow/compute/exec_internal.h
index 507cd1703a8..8bad135e40d 100644
--- a/cpp/src/arrow/compute/exec_internal.h
+++ b/cpp/src/arrow/compute/exec_internal.h
@@ -102,22 +102,22 @@ class DatumAccumulator : public ExecListener {
 /// inputs will be split into non-chunked ExecBatch values for execution
 Status CheckAllValues(const std::vector<Datum>& values);
 
-class ARROW_EXPORT FunctionExecutor {
+class ARROW_EXPORT KernelExecutor {
  public:
-  virtual ~FunctionExecutor() = default;
+  virtual ~KernelExecutor() = default;
+
+  virtual Status Init(KernelContext*, KernelInitArgs) = 0;
 
   /// XXX: Better configurability for listener
   /// Not thread-safe
   virtual Status Execute(const std::vector<Datum>& args, ExecListener* listener) = 0;
 
-  virtual ValueDescr output_descr() const = 0;
-
   virtual Datum WrapResults(const std::vector<Datum>& args,
                             const std::vector<Datum>& outputs) = 0;
 
-  static Result<std::unique_ptr<FunctionExecutor>> Make(ExecContext* ctx,
-                                                        const Function* func,
-                                                        const FunctionOptions* options);
+  static std::unique_ptr<KernelExecutor> MakeScalar();
+  static std::unique_ptr<KernelExecutor> MakeVector();
+  static std::unique_ptr<KernelExecutor> MakeScalarAggregate();
 };
 
 /// \brief Populate validity bitmap with the intersection of the nullity of the
diff --git a/cpp/src/arrow/compute/exec_test.cc b/cpp/src/arrow/compute/exec_test.cc
index 75a2089b3dd..e9bd57596b5 100644
--- a/cpp/src/arrow/compute/exec_test.cc
+++ b/cpp/src/arrow/compute/exec_test.cc
@@ -648,7 +648,8 @@ class TestCallScalarFunction : public TestComputeInternals {
 
     // This function simply copies memory from the input argument into the
     // (preallocated) output
-    auto func = std::make_shared<ScalarFunction>("test_copy", Arity::Unary());
+    auto func =
+        std::make_shared<ScalarFunction>("test_copy", Arity::Unary(), /*doc=*/nullptr);
 
     // Add a few kernels. Our implementation only accepts arrays
     ASSERT_OK(func->AddKernel({InputType::Array(uint8())}, uint8(), ExecCopy));
@@ -657,8 +658,8 @@ class TestCallScalarFunction : public TestComputeInternals {
     ASSERT_OK(registry->AddFunction(func));
 
     // A version which doesn't want the executor to call PropagateNulls
-    auto func2 =
-        std::make_shared<ScalarFunction>("test_copy_computed_bitmap", Arity::Unary());
+    auto func2 = std::make_shared<ScalarFunction>("test_copy_computed_bitmap",
+                                                  Arity::Unary(), /*doc=*/nullptr);
     ScalarKernel kernel({InputType::Array(uint8())}, uint8(), ExecComputedBitmap);
     kernel.null_handling = NullHandling::COMPUTED_PREALLOCATE;
     ASSERT_OK(func2->AddKernel(kernel));
@@ -670,9 +671,10 @@ class TestCallScalarFunction : public TestComputeInternals {
 
     // A function that allocates its own output memory. We have cases for both
     // non-preallocated data and non-preallocated validity bitmap
-    auto f1 = std::make_shared<ScalarFunction>("test_nopre_data", Arity::Unary());
-    auto f2 =
-        std::make_shared<ScalarFunction>("test_nopre_validity_or_data", Arity::Unary());
+    auto f1 = std::make_shared<ScalarFunction>("test_nopre_data", Arity::Unary(),
+                                               /*doc=*/nullptr);
+    auto f2 = std::make_shared<ScalarFunction>("test_nopre_validity_or_data",
+                                               Arity::Unary(), /*doc=*/nullptr);
 
     ScalarKernel kernel({InputType::Array(uint8())}, uint8(), ExecNoPreallocatedData);
     kernel.mem_allocation = MemAllocation::NO_PREALLOCATE;
@@ -691,7 +693,8 @@ class TestCallScalarFunction : public TestComputeInternals {
 
     // This function's behavior depends on a static parameter that is made
     // available to the kernel's execution function through its Options object
-    auto func = std::make_shared<ScalarFunction>("test_stateful", Arity::Unary());
+    auto func = std::make_shared<ScalarFunction>("test_stateful", Arity::Unary(),
+                                                 /*doc=*/nullptr);
 
     ScalarKernel kernel({InputType::Array(int32())}, int32(), ExecStateful, InitStateful);
     ASSERT_OK(func->AddKernel(kernel));
@@ -701,8 +704,8 @@ class TestCallScalarFunction : public TestComputeInternals {
   void AddScalarFunction() {
     auto registry = GetFunctionRegistry();
 
-    auto func =
-        std::make_shared<ScalarFunction>("test_scalar_add_int32", Arity::Binary());
+    auto func = std::make_shared<ScalarFunction>("test_scalar_add_int32", Arity::Binary(),
+                                                 /*doc=*/nullptr);
     ASSERT_OK(func->AddKernel({InputType::Scalar(int32()), InputType::Scalar(int32())},
                               int32(), ExecAddInt32));
     ASSERT_OK(registry->AddFunction(func));
diff --git a/cpp/src/arrow/compute/function.cc b/cpp/src/arrow/compute/function.cc
index 41c3e360a07..2d3e06e2fb2 100644
--- a/cpp/src/arrow/compute/function.cc
+++ b/cpp/src/arrow/compute/function.cc
@@ -29,6 +29,10 @@
 namespace arrow {
 namespace compute {
 
+static const FunctionDoc kEmptyFunctionDoc{};
+
+const FunctionDoc& FunctionDoc::Empty() { return kEmptyFunctionDoc; }
+
 Status Function::CheckArity(int passed_num_args) const {
   if (arity_.is_varargs && passed_num_args < arity_.num_args) {
     return Status::Invalid("VarArgs function needs at least ", arity_.num_args,
@@ -103,6 +107,9 @@ Result<const KernelType*> DispatchExactImpl(const Function& func,
 
 Result<Datum> Function::Execute(const std::vector<Datum>& args,
                                 const FunctionOptions* options, ExecContext* ctx) const {
+  if (options == nullptr) {
+    options = default_options();
+  }
   if (ctx == nullptr) {
     ExecContext default_ctx;
     return Execute(args, options, &default_ctx);
@@ -110,13 +117,47 @@ Result<Datum> Function::Execute(const std::vector<Datum>& args,
   // type-check Datum arguments here. Really we'd like to avoid this as much as
   // possible
   RETURN_NOT_OK(detail::CheckAllValues(args));
-  ARROW_ASSIGN_OR_RAISE(auto executor,
-                        detail::FunctionExecutor::Make(ctx, this, options));
+  std::vector<ValueDescr> inputs(args.size());
+  for (size_t i = 0; i != args.size(); ++i) {
+    inputs[i] = args[i].descr();
+  }
+
+  ARROW_ASSIGN_OR_RAISE(auto kernel, DispatchExact(inputs));
+  std::unique_ptr<KernelState> state;
+
+  KernelContext kernel_ctx{ctx};
+  if (kernel->init) {
+    state = kernel->init(&kernel_ctx, {kernel, inputs, options});
+    RETURN_NOT_OK(kernel_ctx.status());
+    kernel_ctx.SetState(state.get());
+  }
+
+  std::unique_ptr<detail::KernelExecutor> executor;
+  if (kind() == Function::SCALAR) {
+    executor = detail::KernelExecutor::MakeScalar();
+  } else if (kind() == Function::VECTOR) {
+    executor = detail::KernelExecutor::MakeVector();
+  } else {
+    executor = detail::KernelExecutor::MakeScalarAggregate();
+  }
+  RETURN_NOT_OK(executor->Init(&kernel_ctx, {kernel, inputs, options}));
+
   auto listener = std::make_shared<detail::DatumAccumulator>();
   RETURN_NOT_OK(executor->Execute(args, listener.get()));
   return executor->WrapResults(args, listener->values());
 }
 
+Status Function::Validate() const {
+  if (!doc_->summary.empty()) {
+    // Documentation given, check its contents
+    if (static_cast<int>(doc_->arg_names.size()) != arity_.num_args) {
+      return Status::Invalid("In function '", name_,
+                             "': ", "number of argument names != function arity");
+    }
+  }
+  return Status::OK();
+}
+
 Status ScalarFunction::AddKernel(std::vector<InputType> in_types, OutputType out_type,
                                  ArrayKernelExec exec, KernelInit init) {
   RETURN_NOT_OK(CheckArity(static_cast<int>(in_types.size())));
@@ -139,7 +180,7 @@ Status ScalarFunction::AddKernel(ScalarKernel kernel) {
   return Status::OK();
 }
 
-Result<const ScalarKernel*> ScalarFunction::DispatchExact(
+Result<const Kernel*> ScalarFunction::DispatchExact(
     const std::vector<ValueDescr>& values) const {
   return DispatchExactImpl(*this, kernels_, values);
 }
@@ -166,7 +207,7 @@ Status VectorFunction::AddKernel(VectorKernel kernel) {
   return Status::OK();
 }
 
-Result<const VectorKernel*> VectorFunction::DispatchExact(
+Result<const Kernel*> VectorFunction::DispatchExact(
     const std::vector<ValueDescr>& values) const {
   return DispatchExactImpl(*this, kernels_, values);
 }
@@ -180,7 +221,7 @@ Status ScalarAggregateFunction::AddKernel(ScalarAggregateKernel kernel) {
   return Status::OK();
 }
 
-Result<const ScalarAggregateKernel*> ScalarAggregateFunction::DispatchExact(
+Result<const Kernel*> ScalarAggregateFunction::DispatchExact(
     const std::vector<ValueDescr>& values) const {
   return DispatchExactImpl(*this, kernels_, values);
 }
@@ -189,6 +230,9 @@ Result<Datum> MetaFunction::Execute(const std::vector<Datum>& args,
                                     const FunctionOptions* options,
                                     ExecContext* ctx) const {
   RETURN_NOT_OK(CheckArity(static_cast<int>(args.size())));
+  if (options == nullptr) {
+    options = default_options();
+  }
   return ExecuteImpl(args, options, ctx);
 }
 
diff --git a/cpp/src/arrow/compute/function.h b/cpp/src/arrow/compute/function.h
index 93a200ee212..a71dbe40292 100644
--- a/cpp/src/arrow/compute/function.h
+++ b/cpp/src/arrow/compute/function.h
@@ -65,7 +65,8 @@ struct ARROW_EXPORT Arity {
   /// invoking the function
   static Arity VarArgs(int min_args = 0) { return Arity(min_args, true); }
 
-  explicit Arity(int num_args, bool is_varargs = false)
+  // NOTE: the 0-argument form (default constructor) is required for Cython
+  explicit Arity(int num_args = 0, bool is_varargs = false)
       : num_args(num_args), is_varargs(is_varargs) {}
 
   /// The number of required arguments (or the minimum number for varargs
@@ -76,6 +77,37 @@ struct ARROW_EXPORT Arity {
   bool is_varargs = false;
 };
 
+struct ARROW_EXPORT FunctionDoc {
+  /// \brief A one-line summary of the function, using a verb.
+  ///
+  /// For example, "Add two numeric arrays or scalars".
+  std::string summary;
+
+  /// \brief A detailed description of the function, meant to follow the summary.
+  std::string description;
+
+  /// \brief Symbolic names (identifiers) for the function arguments.
+  ///
+  /// Some bindings may use this to generate nicer function signatures.
+  std::vector<std::string> arg_names;
+
+  // TODO add argument descriptions?
+
+  /// \brief Name of the options class, if any.
+  std::string options_class;
+
+  FunctionDoc() {}
+
+  FunctionDoc(std::string summary, std::string description,
+              std::vector<std::string> arg_names, std::string options_class = "")
+      : summary(std::move(summary)),
+        description(std::move(description)),
+        arg_names(std::move(arg_names)),
+        options_class(std::move(options_class)) {}
+
+  static const FunctionDoc& Empty();
+};
+
 /// \brief Base class for compute functions. Function implementations contain a
 /// collection of "kernels" which are implementations of the function for
 /// specific argument types. Selecting a viable kernel for executing a function
@@ -117,15 +149,24 @@ class ARROW_EXPORT Function {
   /// function accepts variable numbers of arguments.
   const Arity& arity() const { return arity_; }
 
+  /// \brief Return the function documentation
+  const FunctionDoc& doc() const { return *doc_; }
+
   /// \brief Returns the number of registered kernels for this function.
   virtual int num_kernels() const = 0;
 
+  /// \brief Return a kernel that can execute the function given the exact
+  /// argument types (without implicit type casts or scalar->array promotions).
+  ///
+  /// NB: This function is overridden in CastFunction.
+  virtual Result<const Kernel*> DispatchExact(
+      const std::vector<ValueDescr>& values) const = 0;
+
   /// \brief Execute the function eagerly with the passed input arguments with
   /// kernel dispatch, batch iteration, and memory allocation details taken
   /// care of.
   ///
-  /// Function implementations may assume that options is non-null and valid
-  /// or to forgo options and accept only nullptr for that argument.
+  /// If the `options` pointer is null, then `default_options()` will be used.
   ///
   /// This function can be overridden in subclasses.
   virtual Result<Datum> Execute(const std::vector<Datum>& args,
@@ -137,12 +178,15 @@ class ARROW_EXPORT Function {
   /// that default_options() is valid to pass to Execute as options.
   const FunctionOptions* default_options() const { return default_options_; }
 
+  virtual Status Validate() const;
+
  protected:
   Function(std::string name, Function::Kind kind, const Arity& arity,
-           const FunctionOptions* default_options)
+           const FunctionDoc* doc, const FunctionOptions* default_options)
       : name_(std::move(name)),
         kind_(kind),
         arity_(arity),
+        doc_(doc ? doc : &FunctionDoc::Empty()),
         default_options_(default_options) {}
 
   Status CheckArity(int passed_num_args) const;
@@ -150,6 +194,7 @@ class ARROW_EXPORT Function {
   std::string name_;
   Function::Kind kind_;
   Arity arity_;
+  const FunctionDoc* doc_;
   const FunctionOptions* default_options_ = NULLPTR;
 };
 
@@ -171,8 +216,8 @@ class FunctionImpl : public Function {
 
  protected:
   FunctionImpl(std::string name, Function::Kind kind, const Arity& arity,
-               const FunctionOptions* default_options)
-      : Function(std::move(name), kind, arity, default_options) {}
+               const FunctionDoc* doc, const FunctionOptions* default_options)
+      : Function(std::move(name), kind, arity, doc, default_options) {}
 
   std::vector<KernelType> kernels_;
 };
@@ -188,9 +233,9 @@ class ARROW_EXPORT ScalarFunction : public detail::FunctionImpl<ScalarKernel> {
  public:
   using KernelType = ScalarKernel;
 
-  ScalarFunction(std::string name, const Arity& arity,
+  ScalarFunction(std::string name, const Arity& arity, const FunctionDoc* doc,
                  const FunctionOptions* default_options = NULLPTR)
-      : detail::FunctionImpl<ScalarKernel>(std::move(name), Function::SCALAR, arity,
+      : detail::FunctionImpl<ScalarKernel>(std::move(name), Function::SCALAR, arity, doc,
                                            default_options) {}
 
   /// \brief Add a kernel with given input/output types, no required state
@@ -203,12 +248,8 @@ class ARROW_EXPORT ScalarFunction : public detail::FunctionImpl<ScalarKernel> {
   /// kernel's signature does not match the function's arity.
   Status AddKernel(ScalarKernel kernel);
 
-  /// \brief Return a kernel that can execute the function given the exact
-  /// argument types (without implicit type casts or scalar->array promotions).
-  ///
-  /// NB: This function is overridden in CastFunction.
-  virtual Result<const ScalarKernel*> DispatchExact(
-      const std::vector<ValueDescr>& values) const;
+  Result<const Kernel*> DispatchExact(
+      const std::vector<ValueDescr>& values) const override;
 };
 
 /// \brief A function that executes general array operations that may yield
@@ -219,9 +260,9 @@ class ARROW_EXPORT VectorFunction : public detail::FunctionImpl<VectorKernel> {
  public:
   using KernelType = VectorKernel;
 
-  VectorFunction(std::string name, const Arity& arity,
+  VectorFunction(std::string name, const Arity& arity, const FunctionDoc* doc,
                  const FunctionOptions* default_options = NULLPTR)
-      : detail::FunctionImpl<VectorKernel>(std::move(name), Function::VECTOR, arity,
+      : detail::FunctionImpl<VectorKernel>(std::move(name), Function::VECTOR, arity, doc,
                                            default_options) {}
 
   /// \brief Add a simple kernel with given input/output types, no required
@@ -234,9 +275,8 @@ class ARROW_EXPORT VectorFunction : public detail::FunctionImpl<VectorKernel> {
   /// kernel's signature does not match the function's arity.
   Status AddKernel(VectorKernel kernel);
 
-  /// \brief Return a kernel that can execute the function given the exact
-  /// argument types (without implicit type casts or scalar->array promotions)
-  Result<const VectorKernel*> DispatchExact(const std::vector<ValueDescr>& values) const;
+  Result<const Kernel*> DispatchExact(
+      const std::vector<ValueDescr>& values) const override;
 };
 
 class ARROW_EXPORT ScalarAggregateFunction
@@ -244,19 +284,17 @@ class ARROW_EXPORT ScalarAggregateFunction
  public:
   using KernelType = ScalarAggregateKernel;
 
-  ScalarAggregateFunction(std::string name, const Arity& arity,
+  ScalarAggregateFunction(std::string name, const Arity& arity, const FunctionDoc* doc,
                           const FunctionOptions* default_options = NULLPTR)
       : detail::FunctionImpl<ScalarAggregateKernel>(
-            std::move(name), Function::SCALAR_AGGREGATE, arity, default_options) {}
+            std::move(name), Function::SCALAR_AGGREGATE, arity, doc, default_options) {}
 
   /// \brief Add a kernel (function implementation). Returns error if the
   /// kernel's signature does not match the function's arity.
   Status AddKernel(ScalarAggregateKernel kernel);
 
-  /// \brief Return a kernel that can execute the function given the exact
-  /// argument types (without implicit type casts or scalar->array promotions)
-  Result<const ScalarAggregateKernel*> DispatchExact(
-      const std::vector<ValueDescr>& values) const;
+  Result<const Kernel*> DispatchExact(
+      const std::vector<ValueDescr>& values) const override;
 };
 
 /// \brief A function that dispatches to other functions. Must implement
@@ -271,14 +309,18 @@ class ARROW_EXPORT MetaFunction : public Function {
   Result<Datum> Execute(const std::vector<Datum>& args, const FunctionOptions* options,
                         ExecContext* ctx) const override;
 
+  Result<const Kernel*> DispatchExact(const std::vector<ValueDescr>&) const override {
+    return Status::NotImplemented("DispatchExact for a MetaFunction's Kernels");
+  }
+
  protected:
   virtual Result<Datum> ExecuteImpl(const std::vector<Datum>& args,
                                     const FunctionOptions* options,
                                     ExecContext* ctx) const = 0;
 
-  MetaFunction(std::string name, const Arity& arity,
+  MetaFunction(std::string name, const Arity& arity, const FunctionDoc* doc,
                const FunctionOptions* default_options = NULLPTR)
-      : Function(std::move(name), Function::META, arity, default_options) {}
+      : Function(std::move(name), Function::META, arity, doc, default_options) {}
 };
 
 /// @}
diff --git a/cpp/src/arrow/compute/function_benchmark.cc b/cpp/src/arrow/compute/function_benchmark.cc
new file mode 100644
index 00000000000..e2214f85174
--- /dev/null
+++ b/cpp/src/arrow/compute/function_benchmark.cc
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "benchmark/benchmark.h"
+
+#include "arrow/array/array_base.h"
+#include "arrow/compute/api.h"
+#include "arrow/memory_pool.h"
+#include "arrow/scalar.h"
+#include "arrow/testing/gtest_util.h"
+#include "arrow/testing/random.h"
+#include "arrow/util/benchmark_util.h"
+
+namespace arrow {
+
+using internal::checked_cast;
+
+namespace compute {
+
+constexpr int32_t kSeed = 0xfede4a7e;
+constexpr int64_t kScalarCount = 1 << 10;
+
+inline ScalarVector ToScalars(std::shared_ptr<Array> arr) {
+  ScalarVector scalars{static_cast<size_t>(arr->length())};
+  int64_t i = 0;
+  for (auto& scalar : scalars) {
+    scalar = arr->GetScalar(i++).ValueOrDie();
+  }
+  return scalars;
+}
+
+void BM_CastDispatch(benchmark::State& state) {  // NOLINT non-const reference
+  // Repeatedly invoke a trivial Cast: the main cost should be dispatch
+  random::RandomArrayGenerator rag(kSeed);
+
+  auto int_scalars = ToScalars(rag.Int64(kScalarCount, 0, 1 << 20));
+
+  auto double_type = float64();
+  for (auto _ : state) {
+    Datum timestamp_scalar;
+    for (Datum int_scalar : int_scalars) {
+      ASSERT_OK_AND_ASSIGN(timestamp_scalar, Cast(int_scalar, double_type));
+    }
+    benchmark::DoNotOptimize(timestamp_scalar);
+  }
+
+  state.SetItemsProcessed(state.iterations() * kScalarCount);
+}
+
+void BM_CastDispatchBaseline(benchmark::State& state) {  // NOLINT non-const reference
+  // Repeatedly invoke a trivial Cast with all dispatch outside the hot loop
+  random::RandomArrayGenerator rag(kSeed);
+
+  auto int_scalars = ToScalars(rag.Int64(kScalarCount, 0, 1 << 20));
+
+  auto double_type = float64();
+  CastOptions cast_options;
+  cast_options.to_type = double_type;
+  ASSERT_OK_AND_ASSIGN(auto cast_function, GetCastFunction(double_type));
+  ASSERT_OK_AND_ASSIGN(auto cast_kernel,
+                       cast_function->DispatchExact({int_scalars[0]->type}));
+  const auto& exec = static_cast<const ScalarKernel*>(cast_kernel)->exec;
+
+  ExecContext exec_context;
+  KernelContext kernel_context(&exec_context);
+  auto cast_state =
+      cast_kernel->init(&kernel_context, {cast_kernel, {double_type}, &cast_options});
+  ABORT_NOT_OK(kernel_context.status());
+  kernel_context.SetState(cast_state.get());
+
+  for (auto _ : state) {
+    Datum timestamp_scalar = MakeNullScalar(double_type);
+    for (Datum int_scalar : int_scalars) {
+      exec(&kernel_context, {{std::move(int_scalar)}, 1}, &timestamp_scalar);
+      ABORT_NOT_OK(kernel_context.status());
+    }
+    benchmark::DoNotOptimize(timestamp_scalar);
+  }
+
+  state.SetItemsProcessed(state.iterations() * kScalarCount);
+}
+
+void BM_AddDispatch(benchmark::State& state) {  // NOLINT non-const reference
+  ExecContext exec_context;
+  KernelContext kernel_context(&exec_context);
+
+  for (auto _ : state) {
+    ASSERT_OK_AND_ASSIGN(auto add_function, GetFunctionRegistry()->GetFunction("add"));
+    ASSERT_OK_AND_ASSIGN(auto add_kernel,
+                         checked_cast<const ScalarFunction&>(*add_function)
+                             .DispatchExact({int64(), int64()}));
+    benchmark::DoNotOptimize(add_kernel);
+  }
+
+  state.SetItemsProcessed(state.iterations());
+}
+
+BENCHMARK(BM_CastDispatch)->MinTime(1.0);
+BENCHMARK(BM_CastDispatchBaseline)->MinTime(1.0);
+BENCHMARK(BM_AddDispatch)->MinTime(1.0);
+
+}  // namespace compute
+}  // namespace arrow
diff --git a/cpp/src/arrow/compute/function_test.cc b/cpp/src/arrow/compute/function_test.cc
index 576659d9331..b6f1815b89e 100644
--- a/cpp/src/arrow/compute/function_test.cc
+++ b/cpp/src/arrow/compute/function_test.cc
@@ -57,8 +57,8 @@ TEST(Arity, Basics) {
 }
 
 TEST(ScalarFunction, Basics) {
-  ScalarFunction func("scalar_test", Arity::Binary());
-  ScalarFunction varargs_func("varargs_test", Arity::VarArgs(1));
+  ScalarFunction func("scalar_test", Arity::Binary(), /*doc=*/nullptr);
+  ScalarFunction varargs_func("varargs_test", Arity::VarArgs(1), /*doc=*/nullptr);
 
   ASSERT_EQ("scalar_test", func.name());
   ASSERT_EQ(2, func.arity().num_args);
@@ -72,8 +72,8 @@ TEST(ScalarFunction, Basics) {
 }
 
 TEST(VectorFunction, Basics) {
-  VectorFunction func("vector_test", Arity::Binary());
-  VectorFunction varargs_func("varargs_test", Arity::VarArgs(1));
+  VectorFunction func("vector_test", Arity::Binary(), /*doc=*/nullptr);
+  VectorFunction varargs_func("varargs_test", Arity::VarArgs(1), /*doc=*/nullptr);
 
   ASSERT_EQ("vector_test", func.name());
   ASSERT_EQ(2, func.arity().num_args);
@@ -126,7 +126,7 @@ void CheckAddDispatch(FunctionType* func) {
   KernelType invalid_kernel({boolean()}, boolean(), ExecNYI);
   ASSERT_RAISES(Invalid, func->AddKernel(invalid_kernel));
 
-  ASSERT_OK_AND_ASSIGN(const KernelType* kernel, func->DispatchExact({int32(), int32()}));
+  ASSERT_OK_AND_ASSIGN(const Kernel* kernel, func->DispatchExact({int32(), int32()}));
   KernelSignature expected_sig(in_types1, out_type1);
   ASSERT_TRUE(kernel->signature->Equals(expected_sig));
 
@@ -139,15 +139,15 @@ void CheckAddDispatch(FunctionType* func) {
 }
 
 TEST(ScalarVectorFunction, DispatchExact) {
-  ScalarFunction func1("scalar_test", Arity::Binary());
-  VectorFunction func2("vector_test", Arity::Binary());
+  ScalarFunction func1("scalar_test", Arity::Binary(), /*doc=*/nullptr);
+  VectorFunction func2("vector_test", Arity::Binary(), /*doc=*/nullptr);
 
   CheckAddDispatch(&func1);
   CheckAddDispatch(&func2);
 }
 
 TEST(ArrayFunction, VarArgs) {
-  ScalarFunction va_func("va_test", Arity::VarArgs(1));
+  ScalarFunction va_func("va_test", Arity::VarArgs(1), /*doc=*/nullptr);
 
   std::vector<InputType> va_args = {int8()};
 
@@ -164,7 +164,7 @@ TEST(ArrayFunction, VarArgs) {
   ASSERT_RAISES(Invalid, va_func.AddKernel(non_va_kernel));
 
   std::vector<ValueDescr> args = {ValueDescr::Scalar(int8()), int8(), int8()};
-  ASSERT_OK_AND_ASSIGN(const ScalarKernel* kernel, va_func.DispatchExact(args));
+  ASSERT_OK_AND_ASSIGN(const Kernel* kernel, va_func.DispatchExact(args));
   ASSERT_TRUE(kernel->signature->MatchesInputs(args));
 
   // No dispatch possible because args incompatible
@@ -173,7 +173,7 @@ TEST(ArrayFunction, VarArgs) {
 }
 
 TEST(ScalarAggregateFunction, Basics) {
-  ScalarAggregateFunction func("agg_test", Arity::Unary());
+  ScalarAggregateFunction func("agg_test", Arity::Unary(), /*doc=*/nullptr);
 
   ASSERT_EQ("agg_test", func.name());
   ASSERT_EQ(1, func.arity().num_args);
@@ -190,7 +190,7 @@ void NoopMerge(KernelContext*, const KernelState&, KernelState*) {}
 void NoopFinalize(KernelContext*, Datum*) {}
 
 TEST(ScalarAggregateFunction, DispatchExact) {
-  ScalarAggregateFunction func("agg_test", Arity::Unary());
+  ScalarAggregateFunction func("agg_test", Arity::Unary(), /*doc=*/nullptr);
 
   std::vector<InputType> in_args = {ValueDescr::Array(int8())};
   ScalarAggregateKernel kernel(std::move(in_args), int64(), NoopInit, NoopConsume,
@@ -215,8 +215,7 @@ TEST(ScalarAggregateFunction, DispatchExact) {
   ASSERT_RAISES(Invalid, func.AddKernel(kernel));
 
   std::vector<ValueDescr> dispatch_args = {ValueDescr::Array(int8())};
-  ASSERT_OK_AND_ASSIGN(const ScalarAggregateKernel* selected_kernel,
-                       func.DispatchExact(dispatch_args));
+  ASSERT_OK_AND_ASSIGN(const Kernel* selected_kernel, func.DispatchExact(dispatch_args));
   ASSERT_EQ(func.kernels()[0], selected_kernel);
   ASSERT_TRUE(selected_kernel->signature->MatchesInputs(dispatch_args));
 
diff --git a/cpp/src/arrow/compute/kernel.cc b/cpp/src/arrow/compute/kernel.cc
index 1788eb72963..88b42716fa2 100644
--- a/cpp/src/arrow/compute/kernel.cc
+++ b/cpp/src/arrow/compute/kernel.cc
@@ -281,6 +281,9 @@ std::string InputType::ToString() const {
   }
   ss << "[";
   switch (kind_) {
+    case InputType::ANY_TYPE:
+      ss << "any";
+      break;
     case InputType::EXACT_TYPE:
       ss << type_->ToString();
       break;
@@ -303,6 +306,8 @@ bool InputType::Equals(const InputType& other) const {
     return false;
   }
   switch (kind_) {
+    case InputType::ANY_TYPE:
+      return true;
     case InputType::EXACT_TYPE:
       return type_->Equals(*other.type_);
     case InputType::USE_TYPE_MATCHER:
diff --git a/cpp/src/arrow/compute/kernel.h b/cpp/src/arrow/compute/kernel.h
index 3fb6947107e..67cb5df7908 100644
--- a/cpp/src/arrow/compute/kernel.h
+++ b/cpp/src/arrow/compute/kernel.h
@@ -664,7 +664,7 @@ struct VectorKernel : public ArrayKernel {
 using ScalarAggregateConsume = std::function<void(KernelContext*, const ExecBatch&)>;
 
 using ScalarAggregateMerge =
-    std::function<void(KernelContext*, const KernelState&, KernelState*)>;
+    std::function<void(KernelContext*, KernelState&&, KernelState*)>;
 
 // Finalize returns Datum to permit multiple return values
 using ScalarAggregateFinalize = std::function<void(KernelContext*, Datum*)>;
diff --git a/cpp/src/arrow/compute/kernel_test.cc b/cpp/src/arrow/compute/kernel_test.cc
index 2eb7fd11449..a5ef9d44e18 100644
--- a/cpp/src/arrow/compute/kernel_test.cc
+++ b/cpp/src/arrow/compute/kernel_test.cc
@@ -38,7 +38,7 @@ TEST(TypeMatcher, SameTypeId) {
   ASSERT_TRUE(matcher->Matches(*decimal(12, 2)));
   ASSERT_FALSE(matcher->Matches(*int8()));
 
-  ASSERT_EQ("Type::DECIMAL", matcher->ToString());
+  ASSERT_EQ("Type::DECIMAL128", matcher->ToString());
 
   ASSERT_TRUE(matcher->Equals(*matcher));
   ASSERT_TRUE(matcher->Equals(*match::SameTypeId(Type::DECIMAL)));
@@ -103,7 +103,7 @@ TEST(InputType, Constructors) {
   // Same type id constructor
   InputType ty2(Type::DECIMAL);
   ASSERT_EQ(InputType::USE_TYPE_MATCHER, ty2.kind());
-  ASSERT_EQ("any[Type::DECIMAL]", ty2.ToString());
+  ASSERT_EQ("any[Type::DECIMAL128]", ty2.ToString());
   ASSERT_TRUE(ty2.type_matcher().Matches(*decimal(12, 2)));
   ASSERT_FALSE(ty2.type_matcher().Matches(*int16()));
 
@@ -135,12 +135,21 @@ TEST(InputType, Constructors) {
   ASSERT_EQ("array[int8]", ty1_array.ToString());
   ASSERT_EQ("scalar[int8]", ty1_scalar.ToString());
 
-  ASSERT_EQ("any[Type::DECIMAL]", ty2.ToString());
-  ASSERT_EQ("array[Type::DECIMAL]", ty2_array.ToString());
-  ASSERT_EQ("scalar[Type::DECIMAL]", ty2_scalar.ToString());
+  ASSERT_EQ("any[Type::DECIMAL128]", ty2.ToString());
+  ASSERT_EQ("array[Type::DECIMAL128]", ty2_array.ToString());
+  ASSERT_EQ("scalar[Type::DECIMAL128]", ty2_scalar.ToString());
 
   InputType ty7(match::TimestampTypeUnit(TimeUnit::MICRO));
   ASSERT_EQ("any[timestamp(us)]", ty7.ToString());
+
+  InputType ty8;
+  InputType ty9(ValueDescr::ANY);
+  InputType ty10(ValueDescr::ARRAY);
+  InputType ty11(ValueDescr::SCALAR);
+  ASSERT_EQ("any[any]", ty8.ToString());
+  ASSERT_EQ("any[any]", ty9.ToString());
+  ASSERT_EQ("array[any]", ty10.ToString());
+  ASSERT_EQ("scalar[any]", ty11.ToString());
 }
 
 TEST(InputType, Equals) {
@@ -475,14 +484,14 @@ TEST(KernelSignature, ToString) {
                                      InputType(Type::DECIMAL, ValueDescr::ARRAY),
                                      InputType(utf8())};
   KernelSignature sig(in_types, utf8());
-  ASSERT_EQ("(scalar[int8], array[Type::DECIMAL], any[string]) -> string",
+  ASSERT_EQ("(scalar[int8], array[Type::DECIMAL128], any[string]) -> string",
             sig.ToString());
 
   OutputType out_type([](KernelContext*, const std::vector<ValueDescr>& args) {
     return Status::Invalid("NYI");
   });
   KernelSignature sig2({int8(), InputType(Type::DECIMAL)}, out_type);
-  ASSERT_EQ("(any[int8], any[Type::DECIMAL]) -> computed", sig2.ToString());
+  ASSERT_EQ("(any[int8], any[Type::DECIMAL128]) -> computed", sig2.ToString());
 }
 
 TEST(KernelSignature, VarArgsToString) {
diff --git a/cpp/src/arrow/compute/kernels/CMakeLists.txt b/cpp/src/arrow/compute/kernels/CMakeLists.txt
index fc147e3a69b..0bca453ab28 100644
--- a/cpp/src/arrow/compute/kernels/CMakeLists.txt
+++ b/cpp/src/arrow/compute/kernels/CMakeLists.txt
@@ -34,6 +34,7 @@ add_arrow_compute_test(scalar_test
 add_arrow_benchmark(scalar_arithmetic_benchmark PREFIX "arrow-compute")
 add_arrow_benchmark(scalar_cast_benchmark PREFIX "arrow-compute")
 add_arrow_benchmark(scalar_compare_benchmark PREFIX "arrow-compute")
+add_arrow_benchmark(scalar_set_lookup_benchmark PREFIX "arrow-compute")
 add_arrow_benchmark(scalar_string_benchmark PREFIX "arrow-compute")
 
 # ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/compute/kernels/aggregate_basic.cc b/cpp/src/arrow/compute/kernels/aggregate_basic.cc
index e2cc2a334d8..11c1e2b1730 100644
--- a/cpp/src/arrow/compute/kernels/aggregate_basic.cc
+++ b/cpp/src/arrow/compute/kernels/aggregate_basic.cc
@@ -15,8 +15,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <cmath>
-
 #include "arrow/compute/api_aggregate.h"
 #include "arrow/compute/kernels/aggregate_basic_internal.h"
 #include "arrow/compute/kernels/aggregate_internal.h"
@@ -26,20 +24,34 @@
 
 namespace arrow {
 namespace compute {
-namespace aggregate {
+
+namespace {
 
 void AggregateConsume(KernelContext* ctx, const ExecBatch& batch) {
   checked_cast<ScalarAggregator*>(ctx->state())->Consume(ctx, batch);
 }
 
-void AggregateMerge(KernelContext* ctx, const KernelState& src, KernelState* dst) {
-  checked_cast<ScalarAggregator*>(dst)->MergeFrom(ctx, src);
+void AggregateMerge(KernelContext* ctx, KernelState&& src, KernelState* dst) {
+  checked_cast<ScalarAggregator*>(dst)->MergeFrom(ctx, std::move(src));
 }
 
 void AggregateFinalize(KernelContext* ctx, Datum* out) {
   checked_cast<ScalarAggregator*>(ctx->state())->Finalize(ctx, out);
 }
 
+}  // namespace
+
+void AddAggKernel(std::shared_ptr<KernelSignature> sig, KernelInit init,
+                  ScalarAggregateFunction* func, SimdLevel::type simd_level) {
+  ScalarAggregateKernel kernel(std::move(sig), init, AggregateConsume, AggregateMerge,
+                               AggregateFinalize);
+  // Set the simd level
+  kernel.simd_level = simd_level;
+  DCHECK_OK(func->AddKernel(kernel));
+}
+
+namespace aggregate {
+
 // ----------------------------------------------------------------------
 // Count implementation
 
@@ -53,7 +65,7 @@ struct CountImpl : public ScalarAggregator {
     this->non_nulls += input.length - nulls;
   }
 
-  void MergeFrom(KernelContext*, const KernelState& src) override {
+  void MergeFrom(KernelContext*, KernelState&& src) override {
     const auto& other_state = checked_cast<const CountImpl&>(src);
     this->non_nulls += other_state.non_nulls;
     this->nulls += other_state.nulls;
@@ -132,224 +144,13 @@ std::unique_ptr<KernelState> MeanInit(KernelContext* ctx, const KernelInitArgs&
 // ----------------------------------------------------------------------
 // MinMax implementation
 
-template <typename ArrowType, typename Enable = void>
-struct MinMaxState {};
-
-template <typename ArrowType>
-struct MinMaxState<ArrowType, enable_if_boolean<ArrowType>> {
-  using ThisType = MinMaxState<ArrowType>;
-  using T = typename ArrowType::c_type;
-
-  ThisType& operator+=(const ThisType& rhs) {
-    this->has_nulls |= rhs.has_nulls;
-    this->has_values |= rhs.has_values;
-    this->min = this->min && rhs.min;
-    this->max = this->max || rhs.max;
-    return *this;
-  }
-
-  void MergeOne(T value) {
-    this->min = this->min && value;
-    this->max = this->max || value;
-  }
-
-  T min = true;
-  T max = false;
-  bool has_nulls = false;
-  bool has_values = false;
-};
-
-template <typename ArrowType>
-struct MinMaxState<ArrowType, enable_if_integer<ArrowType>> {
-  using ThisType = MinMaxState<ArrowType>;
-  using T = typename ArrowType::c_type;
-
-  ThisType& operator+=(const ThisType& rhs) {
-    this->has_nulls |= rhs.has_nulls;
-    this->has_values |= rhs.has_values;
-    this->min = std::min(this->min, rhs.min);
-    this->max = std::max(this->max, rhs.max);
-    return *this;
-  }
-
-  void MergeOne(T value) {
-    this->min = std::min(this->min, value);
-    this->max = std::max(this->max, value);
-  }
-
-  T min = std::numeric_limits<T>::max();
-  T max = std::numeric_limits<T>::min();
-  bool has_nulls = false;
-  bool has_values = false;
-};
-
-template <typename ArrowType>
-struct MinMaxState<ArrowType, enable_if_floating_point<ArrowType>> {
-  using ThisType = MinMaxState<ArrowType>;
-  using T = typename ArrowType::c_type;
-
-  ThisType& operator+=(const ThisType& rhs) {
-    this->has_nulls |= rhs.has_nulls;
-    this->has_values |= rhs.has_values;
-    this->min = std::fmin(this->min, rhs.min);
-    this->max = std::fmax(this->max, rhs.max);
-    return *this;
-  }
-
-  void MergeOne(T value) {
-    this->min = std::fmin(this->min, value);
-    this->max = std::fmax(this->max, value);
-  }
-
-  T min = std::numeric_limits<T>::infinity();
-  T max = -std::numeric_limits<T>::infinity();
-  bool has_nulls = false;
-  bool has_values = false;
-};
-
-template <typename ArrowType>
-struct MinMaxImpl : public ScalarAggregator {
-  using ArrayType = typename TypeTraits<ArrowType>::ArrayType;
-  using ThisType = MinMaxImpl<ArrowType>;
-  using StateType = MinMaxState<ArrowType>;
-
-  MinMaxImpl(const std::shared_ptr<DataType>& out_type, const MinMaxOptions& options)
-      : out_type(out_type), options(options) {}
-
-  void Consume(KernelContext*, const ExecBatch& batch) override {
-    StateType local;
-
-    ArrayType arr(batch[0].array());
-
-    const auto null_count = arr.null_count();
-    local.has_nulls = null_count > 0;
-    local.has_values = (arr.length() - null_count) > 0;
-
-    if (local.has_nulls && options.null_handling == MinMaxOptions::OUTPUT_NULL) {
-      this->state = local;
-      return;
-    }
-
-    if (local.has_nulls) {
-      BitmapReader reader(arr.null_bitmap_data(), arr.offset(), arr.length());
-      for (int64_t i = 0; i < arr.length(); i++) {
-        if (reader.IsSet()) {
-          local.MergeOne(arr.Value(i));
-        }
-        reader.Next();
-      }
-    } else {
-      for (int64_t i = 0; i < arr.length(); i++) {
-        local.MergeOne(arr.Value(i));
-      }
-    }
-    this->state = local;
-  }
-
-  void MergeFrom(KernelContext*, const KernelState& src) override {
-    const auto& other = checked_cast<const ThisType&>(src);
-    this->state += other.state;
-  }
-
-  void Finalize(KernelContext*, Datum* out) override {
-    using ScalarType = typename TypeTraits<ArrowType>::ScalarType;
-
-    std::vector<std::shared_ptr<Scalar>> values;
-    if (!state.has_values ||
-        (state.has_nulls && options.null_handling == MinMaxOptions::OUTPUT_NULL)) {
-      // (null, null)
-      values = {std::make_shared<ScalarType>(), std::make_shared<ScalarType>()};
-    } else {
-      values = {std::make_shared<ScalarType>(state.min),
-                std::make_shared<ScalarType>(state.max)};
-    }
-    out->value = std::make_shared<StructScalar>(std::move(values), this->out_type);
-  }
-
-  std::shared_ptr<DataType> out_type;
-  MinMaxOptions options;
-  MinMaxState<ArrowType> state;
-};
-
-struct BooleanMinMaxImpl : public MinMaxImpl<BooleanType> {
-  using MinMaxImpl::MinMaxImpl;
-
-  void Consume(KernelContext*, const ExecBatch& batch) override {
-    StateType local;
-    ArrayType arr(batch[0].array());
-
-    const auto arr_length = arr.length();
-    const auto null_count = arr.null_count();
-    const auto valid_count = arr_length - null_count;
-
-    local.has_nulls = null_count > 0;
-    local.has_values = valid_count > 0;
-    if (local.has_nulls && options.null_handling == MinMaxOptions::OUTPUT_NULL) {
-      this->state = local;
-      return;
-    }
-
-    const auto true_count = arr.true_count();
-    const auto false_count = valid_count - true_count;
-    local.max = true_count > 0;
-    local.min = false_count == 0;
-
-    this->state = local;
-  }
-};
-
-struct MinMaxInitState {
-  std::unique_ptr<KernelState> state;
-  KernelContext* ctx;
-  const DataType& in_type;
-  const std::shared_ptr<DataType>& out_type;
-  const MinMaxOptions& options;
-
-  MinMaxInitState(KernelContext* ctx, const DataType& in_type,
-                  const std::shared_ptr<DataType>& out_type, const MinMaxOptions& options)
-      : ctx(ctx), in_type(in_type), out_type(out_type), options(options) {}
-
-  Status Visit(const DataType&) {
-    return Status::NotImplemented("No min/max implemented");
-  }
-
-  Status Visit(const HalfFloatType&) {
-    return Status::NotImplemented("No sum implemented");
-  }
-
-  Status Visit(const BooleanType&) {
-    state.reset(new BooleanMinMaxImpl(out_type, options));
-    return Status::OK();
-  }
-
-  template <typename Type>
-  enable_if_number<Type, Status> Visit(const Type&) {
-    state.reset(new MinMaxImpl<Type>(out_type, options));
-    return Status::OK();
-  }
-
-  std::unique_ptr<KernelState> Create() {
-    ctx->SetStatus(VisitTypeInline(in_type, this));
-    return std::move(state);
-  }
-};
-
 std::unique_ptr<KernelState> MinMaxInit(KernelContext* ctx, const KernelInitArgs& args) {
-  MinMaxInitState visitor(ctx, *args.inputs[0].type,
-                          args.kernel->signature->out_type().type(),
-                          static_cast<const MinMaxOptions&>(*args.options));
+  MinMaxInitState<SimdLevel::NONE> visitor(
+      ctx, *args.inputs[0].type, args.kernel->signature->out_type().type(),
+      static_cast<const MinMaxOptions&>(*args.options));
   return visitor.Create();
 }
 
-void AddAggKernel(std::shared_ptr<KernelSignature> sig, KernelInit init,
-                  ScalarAggregateFunction* func, SimdLevel::type simd_level) {
-  ScalarAggregateKernel kernel(std::move(sig), init, AggregateConsume, AggregateMerge,
-                               AggregateFinalize);
-  // Set the simd level
-  kernel.simd_level = simd_level;
-  DCHECK_OK(func->AddKernel(kernel));
-}
-
 void AddBasicAggKernels(KernelInit init,
                         const std::vector<std::shared_ptr<DataType>>& types,
                         std::shared_ptr<DataType> out_ty, ScalarAggregateFunction* func,
@@ -363,8 +164,7 @@ void AddBasicAggKernels(KernelInit init,
 
 void AddMinMaxKernels(KernelInit init,
                       const std::vector<std::shared_ptr<DataType>>& types,
-                      ScalarAggregateFunction* func,
-                      SimdLevel::type simd_level = SimdLevel::NONE) {
+                      ScalarAggregateFunction* func, SimdLevel::type simd_level) {
   for (const auto& ty : types) {
     // array[T] -> scalar[struct<min: T, max: T>]
     auto out_ty = struct_({field("min", ty), field("max", ty)});
@@ -376,18 +176,42 @@ void AddMinMaxKernels(KernelInit init,
 }  // namespace aggregate
 
 namespace internal {
+namespace {
+
+const FunctionDoc count_doc{"Count the number of null / non-null values",
+                            ("By default, non-null values are counted.\n"
+                             "This can be changed through CountOptions."),
+                            {"array"},
+                            "CountOptions"};
+
+const FunctionDoc sum_doc{
+    "Sum values of a numeric array", ("Null values are ignored."), {"array"}};
+
+const FunctionDoc mean_doc{"Compute the mean of a numeric array",
+                           ("Null values are ignored. The result is always computed\n"
+                            "as a double, regardless of the input types"),
+                           {"array"}};
+
+const FunctionDoc min_max_doc{"Compute the minimum and maximum values of a numeric array",
+                              ("Null values are ignored by default.\n"
+                               "This can be changed through MinMaxOptions."),
+                              {"array"},
+                              "MinMaxOptions"};
+
+}  // namespace
+
 void RegisterScalarAggregateBasic(FunctionRegistry* registry) {
   static auto default_count_options = CountOptions::Defaults();
-  auto func = std::make_shared<ScalarAggregateFunction>("count", Arity::Unary(),
-                                                        &default_count_options);
+  auto func = std::make_shared<ScalarAggregateFunction>(
+      "count", Arity::Unary(), &count_doc, &default_count_options);
 
   // Takes any array input, outputs int64 scalar
   InputType any_array(ValueDescr::ARRAY);
-  aggregate::AddAggKernel(KernelSignature::Make({any_array}, ValueDescr::Scalar(int64())),
-                          aggregate::CountInit, func.get());
+  AddAggKernel(KernelSignature::Make({any_array}, ValueDescr::Scalar(int64())),
+               aggregate::CountInit, func.get());
   DCHECK_OK(registry->AddFunction(std::move(func)));
 
-  func = std::make_shared<ScalarAggregateFunction>("sum", Arity::Unary());
+  func = std::make_shared<ScalarAggregateFunction>("sum", Arity::Unary(), &sum_doc);
   aggregate::AddBasicAggKernels(aggregate::SumInit, {boolean()}, int64(), func.get());
   aggregate::AddBasicAggKernels(aggregate::SumInit, SignedIntTypes(), int64(),
                                 func.get());
@@ -409,7 +233,7 @@ void RegisterScalarAggregateBasic(FunctionRegistry* registry) {
 #endif
   DCHECK_OK(registry->AddFunction(std::move(func)));
 
-  func = std::make_shared<ScalarAggregateFunction>("mean", Arity::Unary());
+  func = std::make_shared<ScalarAggregateFunction>("mean", Arity::Unary(), &mean_doc);
   aggregate::AddBasicAggKernels(aggregate::MeanInit, {boolean()}, float64(), func.get());
   aggregate::AddBasicAggKernels(aggregate::MeanInit, NumericTypes(), float64(),
                                 func.get());
@@ -428,12 +252,22 @@ void RegisterScalarAggregateBasic(FunctionRegistry* registry) {
 
   static auto default_minmax_options = MinMaxOptions::Defaults();
   func = std::make_shared<ScalarAggregateFunction>("min_max", Arity::Unary(),
-                                                   &default_minmax_options);
+                                                   &min_max_doc, &default_minmax_options);
   aggregate::AddMinMaxKernels(aggregate::MinMaxInit, {boolean()}, func.get());
   aggregate::AddMinMaxKernels(aggregate::MinMaxInit, NumericTypes(), func.get());
-  DCHECK_OK(registry->AddFunction(std::move(func)));
+  // Add the SIMD variants for min max
+#if defined(ARROW_HAVE_RUNTIME_AVX2)
+  if (cpu_info->IsSupported(arrow::internal::CpuInfo::AVX2)) {
+    aggregate::AddMinMaxAvx2AggKernels(func.get());
+  }
+#endif
+#if defined(ARROW_HAVE_RUNTIME_AVX512)
+  if (cpu_info->IsSupported(arrow::internal::CpuInfo::AVX512)) {
+    aggregate::AddMinMaxAvx512AggKernels(func.get());
+  }
+#endif
 
-  DCHECK_OK(registry->AddFunction(aggregate::AddModeAggKernels()));
+  DCHECK_OK(registry->AddFunction(std::move(func)));
 }
 
 }  // namespace internal
diff --git a/cpp/src/arrow/compute/kernels/aggregate_sum_avx2.cc b/cpp/src/arrow/compute/kernels/aggregate_basic_avx2.cc
similarity index 80%
rename from cpp/src/arrow/compute/kernels/aggregate_sum_avx2.cc
rename to cpp/src/arrow/compute/kernels/aggregate_basic_avx2.cc
index 2811c4cd865..e0c1118c714 100644
--- a/cpp/src/arrow/compute/kernels/aggregate_sum_avx2.cc
+++ b/cpp/src/arrow/compute/kernels/aggregate_basic_avx2.cc
@@ -67,6 +67,17 @@ std::unique_ptr<KernelState> MeanInitAvx2(KernelContext* ctx,
   return visitor.Create();
 }
 
+// ----------------------------------------------------------------------
+// MinMax implementation
+
+std::unique_ptr<KernelState> MinMaxInitAvx2(KernelContext* ctx,
+                                            const KernelInitArgs& args) {
+  MinMaxInitState<SimdLevel::AVX2> visitor(
+      ctx, *args.inputs[0].type, args.kernel->signature->out_type().type(),
+      static_cast<const MinMaxOptions&>(*args.options));
+  return visitor.Create();
+}
+
 void AddSumAvx2AggKernels(ScalarAggregateFunction* func) {
   AddBasicAggKernels(SumInitAvx2, internal::SignedIntTypes(), int64(), func,
                      SimdLevel::AVX2);
@@ -81,6 +92,12 @@ void AddMeanAvx2AggKernels(ScalarAggregateFunction* func) {
                      SimdLevel::AVX2);
 }
 
+void AddMinMaxAvx2AggKernels(ScalarAggregateFunction* func) {
+  // Enable int types for AVX2 variants.
+  // No auto vectorize for float/double as it use fmin/fmax which has NaN handling.
+  AddMinMaxKernels(MinMaxInitAvx2, internal::IntTypes(), func, SimdLevel::AVX2);
+}
+
 }  // namespace aggregate
 }  // namespace compute
 }  // namespace arrow
diff --git a/cpp/src/arrow/compute/kernels/aggregate_sum_avx512.cc b/cpp/src/arrow/compute/kernels/aggregate_basic_avx512.cc
similarity index 80%
rename from cpp/src/arrow/compute/kernels/aggregate_sum_avx512.cc
rename to cpp/src/arrow/compute/kernels/aggregate_basic_avx512.cc
index 00408027e1f..c2c748d3af7 100644
--- a/cpp/src/arrow/compute/kernels/aggregate_sum_avx512.cc
+++ b/cpp/src/arrow/compute/kernels/aggregate_basic_avx512.cc
@@ -68,6 +68,17 @@ std::unique_ptr<KernelState> MeanInitAvx512(KernelContext* ctx,
   return visitor.Create();
 }
 
+// ----------------------------------------------------------------------
+// MinMax implementation
+
+std::unique_ptr<KernelState> MinMaxInitAvx512(KernelContext* ctx,
+                                              const KernelInitArgs& args) {
+  MinMaxInitState<SimdLevel::AVX512> visitor(
+      ctx, *args.inputs[0].type, args.kernel->signature->out_type().type(),
+      static_cast<const MinMaxOptions&>(*args.options));
+  return visitor.Create();
+}
+
 void AddSumAvx512AggKernels(ScalarAggregateFunction* func) {
   AddBasicAggKernels(SumInitAvx512, internal::SignedIntTypes(), int64(), func,
                      SimdLevel::AVX512);
@@ -82,6 +93,12 @@ void AddMeanAvx512AggKernels(ScalarAggregateFunction* func) {
                                 SimdLevel::AVX512);
 }
 
+void AddMinMaxAvx512AggKernels(ScalarAggregateFunction* func) {
+  // Enable 32/64 int types for avx512 variants, no advantage on 8/16 int.
+  AddMinMaxKernels(MinMaxInitAvx512, {int32(), uint32(), int64(), uint64()}, func,
+                   SimdLevel::AVX512);
+}
+
 }  // namespace aggregate
 }  // namespace compute
 }  // namespace arrow
diff --git a/cpp/src/arrow/compute/kernels/aggregate_basic_internal.h b/cpp/src/arrow/compute/kernels/aggregate_basic_internal.h
index 29db97381d6..733e6d1d0a6 100644
--- a/cpp/src/arrow/compute/kernels/aggregate_basic_internal.h
+++ b/cpp/src/arrow/compute/kernels/aggregate_basic_internal.h
@@ -17,6 +17,9 @@
 
 #pragma once
 
+#include <cmath>
+
+#include "arrow/compute/api_aggregate.h"
 #include "arrow/compute/kernels/aggregate_internal.h"
 #include "arrow/compute/kernels/common.h"
 #include "arrow/util/align_util.h"
@@ -26,37 +29,32 @@ namespace arrow {
 namespace compute {
 namespace aggregate {
 
-struct ScalarAggregator : public KernelState {
-  virtual void Consume(KernelContext* ctx, const ExecBatch& batch) = 0;
-  virtual void MergeFrom(KernelContext* ctx, const KernelState& src) = 0;
-  virtual void Finalize(KernelContext* ctx, Datum* out) = 0;
-};
-
-void AddAggKernel(std::shared_ptr<KernelSignature> sig, KernelInit init,
-                  ScalarAggregateFunction* func,
-                  SimdLevel::type simd_level = SimdLevel::NONE);
-
 void AddBasicAggKernels(KernelInit init,
                         const std::vector<std::shared_ptr<DataType>>& types,
                         std::shared_ptr<DataType> out_ty, ScalarAggregateFunction* func,
                         SimdLevel::type simd_level = SimdLevel::NONE);
 
+void AddMinMaxKernels(KernelInit init,
+                      const std::vector<std::shared_ptr<DataType>>& types,
+                      ScalarAggregateFunction* func,
+                      SimdLevel::type simd_level = SimdLevel::NONE);
+
 // SIMD variants for kernels
 void AddSumAvx2AggKernels(ScalarAggregateFunction* func);
 void AddMeanAvx2AggKernels(ScalarAggregateFunction* func);
+void AddMinMaxAvx2AggKernels(ScalarAggregateFunction* func);
 
 void AddSumAvx512AggKernels(ScalarAggregateFunction* func);
 void AddMeanAvx512AggKernels(ScalarAggregateFunction* func);
-
-std::shared_ptr<ScalarAggregateFunction> AddModeAggKernels();
+void AddMinMaxAvx512AggKernels(ScalarAggregateFunction* func);
 
 // ----------------------------------------------------------------------
 // Sum implementation
 
-template <int64_t kRoundSize, typename ArrowType, SimdLevel::type simd_level>
+template <int64_t kRoundSize, typename ArrowType, SimdLevel::type SimdLevel>
 struct SumState {
   using SumType = typename FindAccumulatorType<ArrowType>::Type;
-  using ThisType = SumState<kRoundSize, ArrowType, simd_level>;
+  using ThisType = SumState<kRoundSize, ArrowType, SimdLevel>;
   using T = typename TypeTraits<ArrowType>::CType;
   using ArrayType = typename TypeTraits<ArrowType>::ArrayType;
 
@@ -217,10 +215,10 @@ struct SumState {
   }
 };
 
-template <int64_t kRoundSize, SimdLevel::type simd_level>
-struct SumState<kRoundSize, BooleanType, simd_level> {
+template <int64_t kRoundSize, SimdLevel::type SimdLevel>
+struct SumState<kRoundSize, BooleanType, SimdLevel> {
   using SumType = typename FindAccumulatorType<BooleanType>::Type;
-  using ThisType = SumState<kRoundSize, BooleanType, simd_level>;
+  using ThisType = SumState<kRoundSize, BooleanType, SimdLevel>;
 
   ThisType& operator+=(const ThisType& rhs) {
     this->count += rhs.count;
@@ -239,10 +237,10 @@ struct SumState<kRoundSize, BooleanType, simd_level> {
   typename SumType::c_type sum = 0;
 };
 
-template <uint64_t kRoundSize, typename ArrowType, SimdLevel::type simd_level>
+template <uint64_t kRoundSize, typename ArrowType, SimdLevel::type SimdLevel>
 struct SumImpl : public ScalarAggregator {
   using ArrayType = typename TypeTraits<ArrowType>::ArrayType;
-  using ThisType = SumImpl<kRoundSize, ArrowType, simd_level>;
+  using ThisType = SumImpl<kRoundSize, ArrowType, SimdLevel>;
   using SumType = typename FindAccumulatorType<ArrowType>::Type;
   using OutputType = typename TypeTraits<SumType>::ScalarType;
 
@@ -250,7 +248,7 @@ struct SumImpl : public ScalarAggregator {
     this->state.Consume(ArrayType(batch[0].array()));
   }
 
-  void MergeFrom(KernelContext*, const KernelState& src) override {
+  void MergeFrom(KernelContext*, KernelState&& src) override {
     const auto& other = checked_cast<const ThisType&>(src);
     this->state += other.state;
   }
@@ -263,11 +261,11 @@ struct SumImpl : public ScalarAggregator {
     }
   }
 
-  SumState<kRoundSize, ArrowType, simd_level> state;
+  SumState<kRoundSize, ArrowType, SimdLevel> state;
 };
 
-template <int64_t kRoundSize, typename ArrowType, SimdLevel::type simd_level>
-struct MeanImpl : public SumImpl<kRoundSize, ArrowType, simd_level> {
+template <int64_t kRoundSize, typename ArrowType, SimdLevel::type SimdLevel>
+struct MeanImpl : public SumImpl<kRoundSize, ArrowType, SimdLevel> {
   void Finalize(KernelContext*, Datum* out) override {
     const bool is_valid = this->state.count > 0;
     const double divisor = static_cast<double>(is_valid ? this->state.count : 1UL);
@@ -312,6 +310,268 @@ struct SumLikeInit {
   }
 };
 
+// ----------------------------------------------------------------------
+// MinMax implementation
+
+template <typename ArrowType, SimdLevel::type SimdLevel, typename Enable = void>
+struct MinMaxState {};
+
+template <typename ArrowType, SimdLevel::type SimdLevel>
+struct MinMaxState<ArrowType, SimdLevel, enable_if_boolean<ArrowType>> {
+  using ThisType = MinMaxState<ArrowType, SimdLevel>;
+  using T = typename ArrowType::c_type;
+
+  ThisType& operator+=(const ThisType& rhs) {
+    this->has_nulls |= rhs.has_nulls;
+    this->has_values |= rhs.has_values;
+    this->min = this->min && rhs.min;
+    this->max = this->max || rhs.max;
+    return *this;
+  }
+
+  void MergeOne(T value) {
+    this->min = this->min && value;
+    this->max = this->max || value;
+  }
+
+  T min = true;
+  T max = false;
+  bool has_nulls = false;
+  bool has_values = false;
+};
+
+template <typename ArrowType, SimdLevel::type SimdLevel>
+struct MinMaxState<ArrowType, SimdLevel, enable_if_integer<ArrowType>> {
+  using ThisType = MinMaxState<ArrowType, SimdLevel>;
+  using T = typename ArrowType::c_type;
+
+  ThisType& operator+=(const ThisType& rhs) {
+    this->has_nulls |= rhs.has_nulls;
+    this->has_values |= rhs.has_values;
+    this->min = std::min(this->min, rhs.min);
+    this->max = std::max(this->max, rhs.max);
+    return *this;
+  }
+
+  void MergeOne(T value) {
+    this->min = std::min(this->min, value);
+    this->max = std::max(this->max, value);
+  }
+
+  T min = std::numeric_limits<T>::max();
+  T max = std::numeric_limits<T>::min();
+  bool has_nulls = false;
+  bool has_values = false;
+};
+
+template <typename ArrowType, SimdLevel::type SimdLevel>
+struct MinMaxState<ArrowType, SimdLevel, enable_if_floating_point<ArrowType>> {
+  using ThisType = MinMaxState<ArrowType, SimdLevel>;
+  using T = typename ArrowType::c_type;
+
+  ThisType& operator+=(const ThisType& rhs) {
+    this->has_nulls |= rhs.has_nulls;
+    this->has_values |= rhs.has_values;
+    this->min = std::fmin(this->min, rhs.min);
+    this->max = std::fmax(this->max, rhs.max);
+    return *this;
+  }
+
+  void MergeOne(T value) {
+    this->min = std::fmin(this->min, value);
+    this->max = std::fmax(this->max, value);
+  }
+
+  T min = std::numeric_limits<T>::infinity();
+  T max = -std::numeric_limits<T>::infinity();
+  bool has_nulls = false;
+  bool has_values = false;
+};
+
+template <typename ArrowType, SimdLevel::type SimdLevel>
+struct MinMaxImpl : public ScalarAggregator {
+  using ArrayType = typename TypeTraits<ArrowType>::ArrayType;
+  using ThisType = MinMaxImpl<ArrowType, SimdLevel>;
+  using StateType = MinMaxState<ArrowType, SimdLevel>;
+
+  MinMaxImpl(const std::shared_ptr<DataType>& out_type, const MinMaxOptions& options)
+      : out_type(out_type), options(options) {}
+
+  void Consume(KernelContext*, const ExecBatch& batch) override {
+    StateType local;
+
+    ArrayType arr(batch[0].array());
+
+    const auto null_count = arr.null_count();
+    local.has_nulls = null_count > 0;
+    local.has_values = (arr.length() - null_count) > 0;
+
+    if (local.has_nulls && options.null_handling == MinMaxOptions::EMIT_NULL) {
+      this->state = local;
+      return;
+    }
+
+    if (local.has_nulls) {
+      local += ConsumeWithNulls(arr);
+    } else {  // All true values
+      for (int64_t i = 0; i < arr.length(); i++) {
+        local.MergeOne(arr.Value(i));
+      }
+    }
+    this->state = local;
+  }
+
+  void MergeFrom(KernelContext*, KernelState&& src) override {
+    const auto& other = checked_cast<const ThisType&>(src);
+    this->state += other.state;
+  }
+
+  void Finalize(KernelContext*, Datum* out) override {
+    using ScalarType = typename TypeTraits<ArrowType>::ScalarType;
+
+    std::vector<std::shared_ptr<Scalar>> values;
+    if (!state.has_values ||
+        (state.has_nulls && options.null_handling == MinMaxOptions::EMIT_NULL)) {
+      // (null, null)
+      values = {std::make_shared<ScalarType>(), std::make_shared<ScalarType>()};
+    } else {
+      values = {std::make_shared<ScalarType>(state.min),
+                std::make_shared<ScalarType>(state.max)};
+    }
+    out->value = std::make_shared<StructScalar>(std::move(values), this->out_type);
+  }
+
+  std::shared_ptr<DataType> out_type;
+  MinMaxOptions options;
+  MinMaxState<ArrowType, SimdLevel> state;
+
+ private:
+  StateType ConsumeWithNulls(const ArrayType& arr) const {
+    StateType local;
+    const int64_t length = arr.length();
+    int64_t offset = arr.offset();
+    const uint8_t* bitmap = arr.null_bitmap_data();
+    int64_t idx = 0;
+
+    const auto p = arrow::internal::BitmapWordAlign<1>(bitmap, offset, length);
+    // First handle the leading bits
+    const int64_t leading_bits = p.leading_bits;
+    while (idx < leading_bits) {
+      if (BitUtil::GetBit(bitmap, offset)) {
+        local.MergeOne(arr.Value(idx));
+      }
+      idx++;
+      offset++;
+    }
+
+    // The aligned parts scanned with BitBlockCounter
+    arrow::internal::BitBlockCounter data_counter(bitmap, offset, length - leading_bits);
+    auto current_block = data_counter.NextWord();
+    while (idx < length) {
+      if (current_block.AllSet()) {  // All true values
+        int run_length = 0;
+        // Scan forward until a block that has some false values (or the end)
+        while (current_block.length > 0 && current_block.AllSet()) {
+          run_length += current_block.length;
+          current_block = data_counter.NextWord();
+        }
+        for (int64_t i = 0; i < run_length; i++) {
+          local.MergeOne(arr.Value(idx + i));
+        }
+        idx += run_length;
+        offset += run_length;
+        // The current_block already computed, advance to next loop
+        continue;
+      } else if (!current_block.NoneSet()) {  // Some values are null
+        BitmapReader reader(arr.null_bitmap_data(), offset, current_block.length);
+        for (int64_t i = 0; i < current_block.length; i++) {
+          if (reader.IsSet()) {
+            local.MergeOne(arr.Value(idx + i));
+          }
+          reader.Next();
+        }
+
+        idx += current_block.length;
+        offset += current_block.length;
+      } else {  // All null values
+        idx += current_block.length;
+        offset += current_block.length;
+      }
+      current_block = data_counter.NextWord();
+    }
+
+    return local;
+  }
+};
+
+template <SimdLevel::type SimdLevel>
+struct BooleanMinMaxImpl : public MinMaxImpl<BooleanType, SimdLevel> {
+  using StateType = MinMaxState<BooleanType, SimdLevel>;
+  using ArrayType = typename TypeTraits<BooleanType>::ArrayType;
+  using MinMaxImpl<BooleanType, SimdLevel>::MinMaxImpl;
+  using MinMaxImpl<BooleanType, SimdLevel>::options;
+
+  void Consume(KernelContext*, const ExecBatch& batch) override {
+    StateType local;
+    ArrayType arr(batch[0].array());
+
+    const auto arr_length = arr.length();
+    const auto null_count = arr.null_count();
+    const auto valid_count = arr_length - null_count;
+
+    local.has_nulls = null_count > 0;
+    local.has_values = valid_count > 0;
+    if (local.has_nulls && options.null_handling == MinMaxOptions::EMIT_NULL) {
+      this->state = local;
+      return;
+    }
+
+    const auto true_count = arr.true_count();
+    const auto false_count = valid_count - true_count;
+    local.max = true_count > 0;
+    local.min = false_count == 0;
+
+    this->state = local;
+  }
+};
+
+template <SimdLevel::type SimdLevel>
+struct MinMaxInitState {
+  std::unique_ptr<KernelState> state;
+  KernelContext* ctx;
+  const DataType& in_type;
+  const std::shared_ptr<DataType>& out_type;
+  const MinMaxOptions& options;
+
+  MinMaxInitState(KernelContext* ctx, const DataType& in_type,
+                  const std::shared_ptr<DataType>& out_type, const MinMaxOptions& options)
+      : ctx(ctx), in_type(in_type), out_type(out_type), options(options) {}
+
+  Status Visit(const DataType&) {
+    return Status::NotImplemented("No min/max implemented");
+  }
+
+  Status Visit(const HalfFloatType&) {
+    return Status::NotImplemented("No min/max implemented");
+  }
+
+  Status Visit(const BooleanType&) {
+    state.reset(new BooleanMinMaxImpl<SimdLevel>(out_type, options));
+    return Status::OK();
+  }
+
+  template <typename Type>
+  enable_if_number<Type, Status> Visit(const Type&) {
+    state.reset(new MinMaxImpl<Type, SimdLevel>(out_type, options));
+    return Status::OK();
+  }
+
+  std::unique_ptr<KernelState> Create() {
+    ctx->SetStatus(VisitTypeInline(in_type, this));
+    return std::move(state);
+  }
+};
+
 }  // namespace aggregate
 }  // namespace compute
 }  // namespace arrow
diff --git a/cpp/src/arrow/compute/kernels/aggregate_benchmark.cc b/cpp/src/arrow/compute/kernels/aggregate_benchmark.cc
index 882037f2d5d..5b95d7b526a 100644
--- a/cpp/src/arrow/compute/kernels/aggregate_benchmark.cc
+++ b/cpp/src/arrow/compute/kernels/aggregate_benchmark.cc
@@ -19,7 +19,6 @@
 
 #include <vector>
 
-#include "arrow/builder.h"
 #include "arrow/compute/api.h"
 #include "arrow/memory_pool.h"
 #include "arrow/testing/gtest_util.h"
@@ -301,6 +300,10 @@ BENCHMARK_TEMPLATE(ReferenceSum, SumBitmapVectorizeUnroll<int64_t>)
     ->Apply(BenchmarkSetArgs);
 #endif  // ARROW_WITH_BENCHMARKS_REFERENCE
 
+//
+// Sum
+//
+
 template <typename ArrowType>
 static void SumKernel(benchmark::State& state) {
   using CType = typename TypeTraits<ArrowType>::CType;
@@ -330,6 +333,10 @@ SUM_KERNEL_BENCHMARK(SumKernelInt16, Int16Type);
 SUM_KERNEL_BENCHMARK(SumKernelInt32, Int32Type);
 SUM_KERNEL_BENCHMARK(SumKernelInt64, Int64Type);
 
+//
+// Mode
+//
+
 template <typename ArrowType>
 void ModeKernelBench(benchmark::State& state) {
   using CType = typename TypeTraits<ArrowType>::CType;
@@ -369,6 +376,10 @@ MODE_KERNEL_BENCHMARK(ModeKernelInt16, Int16Type);
 MODE_KERNEL_BENCHMARK(ModeKernelInt32, Int32Type);
 MODE_KERNEL_BENCHMARK(ModeKernelInt64, Int64Type);
 
+//
+// MinMax
+//
+
 template <typename ArrowType>
 static void MinMaxKernelBench(benchmark::State& state) {
   using CType = typename TypeTraits<ArrowType>::CType;
@@ -398,5 +409,53 @@ MINMAX_KERNEL_BENCHMARK(MinMaxKernelInt16, Int16Type);
 MINMAX_KERNEL_BENCHMARK(MinMaxKernelInt32, Int32Type);
 MINMAX_KERNEL_BENCHMARK(MinMaxKernelInt64, Int64Type);
 
+//
+// Count
+//
+
+static void CountKernelBenchInt64(benchmark::State& state) {
+  RegressionArgs args(state);
+  const int64_t array_size = args.size / sizeof(int64_t);
+  auto rand = random::RandomArrayGenerator(1923);
+  auto array = rand.Numeric<Int64Type>(array_size, -100, 100, args.null_proportion);
+
+  for (auto _ : state) {
+    ABORT_NOT_OK(Count(array->Slice(1, array_size)).status());
+  }
+}
+BENCHMARK(CountKernelBenchInt64)->Args({1 * 1024 * 1024, 2});  // 1M with 50% null.
+
+//
+// Variance
+//
+
+template <typename ArrowType>
+void VarianceKernelBench(benchmark::State& state) {
+  using CType = typename TypeTraits<ArrowType>::CType;
+
+  VarianceOptions options;
+  RegressionArgs args(state);
+  const int64_t array_size = args.size / sizeof(CType);
+  auto rand = random::RandomArrayGenerator(1925);
+  auto array = rand.Numeric<ArrowType>(array_size, -100000, 100000, args.null_proportion);
+
+  for (auto _ : state) {
+    ABORT_NOT_OK(Variance(array, options).status());
+  }
+}
+
+static void VarianceKernelBenchArgs(benchmark::internal::Benchmark* bench) {
+  BenchmarkSetArgsWithSizes(bench, {1 * 1024 * 1024});
+}
+
+#define VARIANCE_KERNEL_BENCHMARK(FuncName, Type)                                     \
+  static void FuncName(benchmark::State& state) { VarianceKernelBench<Type>(state); } \
+  BENCHMARK(FuncName)->Apply(VarianceKernelBenchArgs)
+
+VARIANCE_KERNEL_BENCHMARK(VarianceKernelInt32, Int32Type);
+VARIANCE_KERNEL_BENCHMARK(VarianceKernelInt64, Int64Type);
+VARIANCE_KERNEL_BENCHMARK(VarianceKernelFloat, FloatType);
+VARIANCE_KERNEL_BENCHMARK(VarianceKernelDouble, DoubleType);
+
 }  // namespace compute
 }  // namespace arrow
diff --git a/cpp/src/arrow/compute/kernels/aggregate_internal.h b/cpp/src/arrow/compute/kernels/aggregate_internal.h
index 5f2f50c0b06..cb67794d942 100644
--- a/cpp/src/arrow/compute/kernels/aggregate_internal.h
+++ b/cpp/src/arrow/compute/kernels/aggregate_internal.h
@@ -47,5 +47,15 @@ struct FindAccumulatorType<I, enable_if_floating_point<I>> {
   using Type = DoubleType;
 };
 
+struct ScalarAggregator : public KernelState {
+  virtual void Consume(KernelContext* ctx, const ExecBatch& batch) = 0;
+  virtual void MergeFrom(KernelContext* ctx, KernelState&& src) = 0;
+  virtual void Finalize(KernelContext* ctx, Datum* out) = 0;
+};
+
+void AddAggKernel(std::shared_ptr<KernelSignature> sig, KernelInit init,
+                  ScalarAggregateFunction* func,
+                  SimdLevel::type simd_level = SimdLevel::NONE);
+
 }  // namespace compute
 }  // namespace arrow
diff --git a/cpp/src/arrow/compute/kernels/aggregate_mode.cc b/cpp/src/arrow/compute/kernels/aggregate_mode.cc
index 7905c89f83f..6544df549e6 100644
--- a/cpp/src/arrow/compute/kernels/aggregate_mode.cc
+++ b/cpp/src/arrow/compute/kernels/aggregate_mode.cc
@@ -18,41 +18,155 @@
 #include <cmath>
 #include <unordered_map>
 
-#include "arrow/compute/kernels/aggregate_basic_internal.h"
+#include "arrow/compute/api_aggregate.h"
+#include "arrow/compute/kernels/aggregate_internal.h"
+#include "arrow/compute/kernels/common.h"
 
 namespace arrow {
 namespace compute {
-namespace aggregate {
+namespace internal {
 
 namespace {
 
-template <typename ArrowType>
-struct ModeState {
-  using ThisType = ModeState<ArrowType>;
-  using T = typename ArrowType::c_type;
+// {value:count} map
+template <typename CType>
+using CounterMap = std::unordered_map<CType, int64_t>;
+
+// map based counter for floating points
+template <typename ArrayType, typename CType = typename ArrayType::TypeClass::c_type>
+enable_if_t<std::is_floating_point<CType>::value, CounterMap<CType>> CountValuesByMap(
+    const ArrayType& array, int64_t& nan_count) {
+  CounterMap<CType> value_counts_map;
+
+  nan_count = 0;
+  if (array.length() > array.null_count()) {
+    VisitArrayDataInline<typename ArrayType::TypeClass>(
+        *array.data(),
+        [&](CType value) {
+          if (std::isnan(value)) {
+            ++nan_count;
+          } else {
+            ++value_counts_map[value];
+          }
+        },
+        []() {});
+  }
 
-  void MergeFrom(const ThisType& state) {
-    for (const auto& value_count : state.value_counts) {
-      auto value = value_count.first;
-      auto count = value_count.second;
-      this->value_counts[value] += count;
+  return value_counts_map;
+}
+
+// map base counter for non floating points
+template <typename ArrayType, typename CType = typename ArrayType::TypeClass::c_type>
+enable_if_t<!std::is_floating_point<CType>::value, CounterMap<CType>> CountValuesByMap(
+    const ArrayType& array) {
+  CounterMap<CType> value_counts_map;
+
+  if (array.length() > array.null_count()) {
+    VisitArrayDataInline<typename ArrayType::TypeClass>(
+        *array.data(), [&](CType value) { ++value_counts_map[value]; }, []() {});
+  }
+
+  return value_counts_map;
+}
+
+// vector based counter for bool/int8 or integers with small value range
+template <typename ArrayType, typename CType = typename ArrayType::TypeClass::c_type>
+CounterMap<CType> CountValuesByVector(const ArrayType& array, CType min, CType max) {
+  const int range = static_cast<int>(max - min);
+  DCHECK(range >= 0 && range < 64 * 1024 * 1024);
+
+  std::vector<int64_t> value_counts_vector(range + 1);
+  if (array.length() > array.null_count()) {
+    VisitArrayDataInline<typename ArrayType::TypeClass>(
+        *array.data(), [&](CType value) { ++value_counts_vector[value - min]; }, []() {});
+  }
+
+  // Transfer value counts to a map to be consistent with other chunks
+  CounterMap<CType> value_counts_map(range + 1);
+  for (int i = 0; i <= range; ++i) {
+    CType value = static_cast<CType>(i + min);
+    int64_t count = value_counts_vector[i];
+    if (count) {
+      value_counts_map[value] = count;
     }
   }
 
-  template <typename ArrowType_ = ArrowType>
-  enable_if_t<!is_floating_type<ArrowType_>::value> MergeOne(T value) {
-    ++this->value_counts[value];
+  return value_counts_map;
+}
+
+// map or vector based counter for int16/32/64 per value range
+template <typename ArrayType, typename CType = typename ArrayType::TypeClass::c_type>
+CounterMap<CType> CountValuesByMapOrVector(const ArrayType& array) {
+  // see https://issues.apache.org/jira/browse/ARROW-9873
+  static constexpr int kMinArraySize = 8192 / sizeof(CType);
+  static constexpr int kMaxValueRange = 16384;
+
+  if ((array.length() - array.null_count()) >= kMinArraySize) {
+    CType min = std::numeric_limits<CType>::max();
+    CType max = std::numeric_limits<CType>::min();
+
+    VisitArrayDataInline<typename ArrayType::TypeClass>(
+        *array.data(),
+        [&](CType value) {
+          min = std::min(min, value);
+          max = std::max(max, value);
+        },
+        []() {});
+
+    if (static_cast<uint64_t>(max) - static_cast<uint64_t>(min) <= kMaxValueRange) {
+      return CountValuesByVector(array, min, max);
+    }
   }
+  return CountValuesByMap(array);
+}
 
-  template <typename ArrowType_ = ArrowType>
-  enable_if_t<is_floating_type<ArrowType_>::value> MergeOne(T value) {
-    if (!std::isnan(value)) {
-      ++this->value_counts[value];
+// bool, int8
+template <typename ArrayType, typename CType = typename ArrayType::TypeClass::c_type>
+enable_if_t<std::is_integral<CType>::value && sizeof(CType) == 1, CounterMap<CType>>
+CountValues(const ArrayType& array, int64_t& nan_count) {
+  using Limits = std::numeric_limits<CType>;
+  nan_count = 0;
+  return CountValuesByVector(array, Limits::min(), Limits::max());
+}
+
+// int16/32/64
+template <typename ArrayType, typename CType = typename ArrayType::TypeClass::c_type>
+enable_if_t<std::is_integral<CType>::value && (sizeof(CType) > 1), CounterMap<CType>>
+CountValues(const ArrayType& array, int64_t& nan_count) {
+  nan_count = 0;
+  return CountValuesByMapOrVector(array);
+}
+
+// float/double
+template <typename ArrayType, typename CType = typename ArrayType::TypeClass::c_type>
+enable_if_t<(std::is_floating_point<CType>::value), CounterMap<CType>>  // NOLINT format
+CountValues(const ArrayType& array, int64_t& nan_count) {
+  nan_count = 0;
+  return CountValuesByMap(array, nan_count);
+}
+
+template <typename ArrowType>
+struct ModeState {
+  using ThisType = ModeState<ArrowType>;
+  using CType = typename ArrowType::c_type;
+
+  void MergeFrom(ThisType&& state) {
+    if (this->value_counts.empty()) {
+      this->value_counts = std::move(state.value_counts);
+    } else {
+      for (const auto& value_count : state.value_counts) {
+        auto value = value_count.first;
+        auto count = value_count.second;
+        this->value_counts[value] += count;
+      }
+    }
+    if (is_floating_type<ArrowType>::value) {
+      this->nan_count += state.nan_count;
     }
   }
 
-  std::pair<T, int64_t> Finalize() {
-    T mode = std::numeric_limits<T>::min();
+  std::pair<CType, int64_t> Finalize() {
+    CType mode = std::numeric_limits<CType>::min();
     int64_t count = 0;
 
     for (const auto& value_count : this->value_counts) {
@@ -63,10 +177,15 @@ struct ModeState {
         mode = this_value;
       }
     }
+    if (is_floating_type<ArrowType>::value && this->nan_count > count) {
+      count = this->nan_count;
+      mode = static_cast<CType>(NAN);
+    }
     return std::make_pair(mode, count);
   }
 
-  std::unordered_map<T, int64_t> value_counts{};
+  int64_t nan_count = 0;  // only make sense to floating types
+  CounterMap<CType> value_counts;
 };
 
 template <typename ArrowType>
@@ -77,28 +196,13 @@ struct ModeImpl : public ScalarAggregator {
   explicit ModeImpl(const std::shared_ptr<DataType>& out_type) : out_type(out_type) {}
 
   void Consume(KernelContext*, const ExecBatch& batch) override {
-    ModeState<ArrowType> local_state;
-    ArrayType arr(batch[0].array());
-
-    if (arr.null_count() > 0) {
-      BitmapReader reader(arr.null_bitmap_data(), arr.offset(), arr.length());
-      for (int64_t i = 0; i < arr.length(); i++) {
-        if (reader.IsSet()) {
-          local_state.MergeOne(arr.Value(i));
-        }
-        reader.Next();
-      }
-    } else {
-      for (int64_t i = 0; i < arr.length(); i++) {
-        local_state.MergeOne(arr.Value(i));
-      }
-    }
-    this->state = std::move(local_state);
+    ArrayType array(batch[0].array());
+    this->state.value_counts = CountValues(array, this->state.nan_count);
   }
 
-  void MergeFrom(KernelContext*, const KernelState& src) override {
-    const auto& other = checked_cast<const ThisType&>(src);
-    this->state.MergeFrom(other.state);
+  void MergeFrom(KernelContext*, KernelState&& src) override {
+    auto& other = checked_cast<ThisType&>(src);
+    this->state.MergeFrom(std::move(other.state));
   }
 
   void Finalize(KernelContext*, Datum* out) override {
@@ -106,12 +210,13 @@ struct ModeImpl : public ScalarAggregator {
     using CountType = typename TypeTraits<Int64Type>::ScalarType;
 
     std::vector<std::shared_ptr<Scalar>> values;
-    if (this->state.value_counts.empty()) {
+    auto mode_count = this->state.Finalize();
+    auto mode = mode_count.first;
+    auto count = mode_count.second;
+    if (count == 0) {
       values = {std::make_shared<ModeType>(), std::make_shared<CountType>()};
     } else {
-      auto mode_count = state.Finalize();
-      values = {std::make_shared<ModeType>(mode_count.first),
-                std::make_shared<CountType>(mode_count.second)};
+      values = {std::make_shared<ModeType>(mode), std::make_shared<CountType>(count)};
     }
     out->value = std::make_shared<StructScalar>(std::move(values), this->out_type);
   }
@@ -165,15 +270,29 @@ void AddModeKernels(KernelInit init, const std::vector<std::shared_ptr<DataType>
   }
 }
 
-}  // namespace
+const FunctionDoc mode_doc{
+    "Calculate the modal (most common) value of a numeric array",
+    ("This function returns both mode and count as a struct scalar,\n"
+     "with type `struct<mode: T, count: int64>`, where T is the input type.\n"
+     "If there is more than one such value, the smallest one is returned.\n"
+     "Nulls are ignored.  If there are no non-null values in the array,\n"
+     "null is returned."),
+    {"array"}};
 
 std::shared_ptr<ScalarAggregateFunction> AddModeAggKernels() {
-  auto func = std::make_shared<ScalarAggregateFunction>("mode", Arity::Unary());
+  auto func =
+      std::make_shared<ScalarAggregateFunction>("mode", Arity::Unary(), &mode_doc);
   AddModeKernels(ModeInit, {boolean()}, func.get());
-  AddModeKernels(ModeInit, internal::NumericTypes(), func.get());
+  AddModeKernels(ModeInit, NumericTypes(), func.get());
   return func;
 }
 
-}  // namespace aggregate
+}  // namespace
+
+void RegisterScalarAggregateMode(FunctionRegistry* registry) {
+  DCHECK_OK(registry->AddFunction(AddModeAggKernels()));
+}
+
+}  // namespace internal
 }  // namespace compute
 }  // namespace arrow
diff --git a/cpp/src/arrow/compute/kernels/aggregate_test.cc b/cpp/src/arrow/compute/kernels/aggregate_test.cc
index e51579b7710..bcaa842fa7f 100644
--- a/cpp/src/arrow/compute/kernels/aggregate_test.cc
+++ b/cpp/src/arrow/compute/kernels/aggregate_test.cc
@@ -19,6 +19,7 @@
 #include <limits>
 #include <memory>
 #include <type_traits>
+#include <unordered_map>
 #include <utility>
 
 #include <gtest/gtest.h>
@@ -512,7 +513,7 @@ TEST_F(TestBooleanMinMaxKernel, Basics) {
   this->AssertMinMaxIs(chunked_input2, false, false, options);
   this->AssertMinMaxIs(chunked_input3, false, true, options);
 
-  options = MinMaxOptions(MinMaxOptions::OUTPUT_NULL);
+  options = MinMaxOptions(MinMaxOptions::EMIT_NULL);
   this->AssertMinMaxIsNull("[]", options);
   this->AssertMinMaxIsNull("[null, null, null]", options);
   this->AssertMinMaxIsNull("[false, null, false]", options);
@@ -542,7 +543,7 @@ TYPED_TEST(TestIntegerMinMaxKernel, Basics) {
   this->AssertMinMaxIs(chunked_input2, 1, 9, options);
   this->AssertMinMaxIs(chunked_input3, 1, 9, options);
 
-  options = MinMaxOptions(MinMaxOptions::OUTPUT_NULL);
+  options = MinMaxOptions(MinMaxOptions::EMIT_NULL);
   this->AssertMinMaxIs("[5, 1, 2, 3, 4]", 1, 5, options);
   // output null
   this->AssertMinMaxIsNull("[5, null, 2, 3, 4]", options);
@@ -569,7 +570,7 @@ TYPED_TEST(TestFloatingMinMaxKernel, Floats) {
   this->AssertMinMaxIs(chunked_input2, 1, 9, options);
   this->AssertMinMaxIs(chunked_input3, 1, 9, options);
 
-  options = MinMaxOptions(MinMaxOptions::OUTPUT_NULL);
+  options = MinMaxOptions(MinMaxOptions::EMIT_NULL);
   this->AssertMinMaxIs("[5, 1, 2, 3, 4]", 1, 5, options);
   this->AssertMinMaxIs("[5, -Inf, 2, 3, 4]", -INFINITY, 5, options);
   // output null
@@ -594,18 +595,150 @@ TYPED_TEST(TestFloatingMinMaxKernel, DefaultOptions) {
   AssertDatumsEqual(explicit_defaults, no_options_provided);
 }
 
+template <typename ArrowType>
+struct MinMaxResult {
+  using T = typename ArrowType::c_type;
+
+  T min = 0;
+  T max = 0;
+  bool is_valid = false;
+};
+
+template <typename ArrowType>
+static enable_if_integer<ArrowType, MinMaxResult<ArrowType>> NaiveMinMax(
+    const Array& array) {
+  using T = typename ArrowType::c_type;
+  using ArrayType = typename TypeTraits<ArrowType>::ArrayType;
+
+  MinMaxResult<ArrowType> result;
+
+  const auto& array_numeric = reinterpret_cast<const ArrayType&>(array);
+  const auto values = array_numeric.raw_values();
+
+  if (array.length() <= array.null_count()) {  // All null values
+    return result;
+  }
+
+  T min = std::numeric_limits<T>::max();
+  T max = std::numeric_limits<T>::min();
+  if (array.null_count() != 0) {  // Some values are null
+    internal::BitmapReader reader(array.null_bitmap_data(), array.offset(),
+                                  array.length());
+    for (int64_t i = 0; i < array.length(); i++) {
+      if (reader.IsSet()) {
+        min = std::min(min, values[i]);
+        max = std::max(max, values[i]);
+      }
+      reader.Next();
+    }
+  } else {  // All true values
+    for (int64_t i = 0; i < array.length(); i++) {
+      min = std::min(min, values[i]);
+      max = std::max(max, values[i]);
+    }
+  }
+
+  result.min = min;
+  result.max = max;
+  result.is_valid = true;
+  return result;
+}
+
+template <typename ArrowType>
+static enable_if_floating_point<ArrowType, MinMaxResult<ArrowType>> NaiveMinMax(
+    const Array& array) {
+  using T = typename ArrowType::c_type;
+  using ArrayType = typename TypeTraits<ArrowType>::ArrayType;
+
+  MinMaxResult<ArrowType> result;
+
+  const auto& array_numeric = reinterpret_cast<const ArrayType&>(array);
+  const auto values = array_numeric.raw_values();
+
+  if (array.length() <= array.null_count()) {  // All null values
+    return result;
+  }
+
+  T min = std::numeric_limits<T>::infinity();
+  T max = -std::numeric_limits<T>::infinity();
+  if (array.null_count() != 0) {  // Some values are null
+    internal::BitmapReader reader(array.null_bitmap_data(), array.offset(),
+                                  array.length());
+    for (int64_t i = 0; i < array.length(); i++) {
+      if (reader.IsSet()) {
+        min = std::fmin(min, values[i]);
+        max = std::fmax(max, values[i]);
+      }
+      reader.Next();
+    }
+  } else {  // All true values
+    for (int64_t i = 0; i < array.length(); i++) {
+      min = std::fmin(min, values[i]);
+      max = std::fmax(max, values[i]);
+    }
+  }
+
+  result.min = min;
+  result.max = max;
+  result.is_valid = true;
+  return result;
+}
+
+template <typename ArrowType>
+void ValidateMinMax(const Array& array) {
+  using Traits = TypeTraits<ArrowType>;
+  using ScalarType = typename Traits::ScalarType;
+
+  ASSERT_OK_AND_ASSIGN(Datum out, MinMax(array));
+  const StructScalar& value = out.scalar_as<StructScalar>();
+
+  auto expected = NaiveMinMax<ArrowType>(array);
+  const auto& out_min = checked_cast<const ScalarType&>(*value.value[0]);
+  const auto& out_max = checked_cast<const ScalarType&>(*value.value[1]);
+
+  if (expected.is_valid) {
+    ASSERT_TRUE(out_min.is_valid);
+    ASSERT_TRUE(out_max.is_valid);
+    ASSERT_EQ(expected.min, out_min.value);
+    ASSERT_EQ(expected.max, out_max.value);
+  } else {  // All null values
+    ASSERT_FALSE(out_min.is_valid);
+    ASSERT_FALSE(out_max.is_valid);
+  }
+}
+
+template <typename ArrowType>
+class TestRandomNumericMinMaxKernel : public ::testing::Test {};
+
+TYPED_TEST_SUITE(TestRandomNumericMinMaxKernel, NumericArrowTypes);
+TYPED_TEST(TestRandomNumericMinMaxKernel, RandomArrayMinMax) {
+  auto rand = random::RandomArrayGenerator(0x8afc055);
+  // Test size up to 1<<11 (2048).
+  for (size_t i = 3; i < 12; i += 2) {
+    for (auto null_probability : {0.0, 0.01, 0.1, 0.5, 0.99, 1.0}) {
+      int64_t base_length = (1UL << i) + 2;
+      auto array = rand.Numeric<TypeParam>(base_length, 0, 100, null_probability);
+      for (auto length_adjust : {-2, -1, 0, 1, 2}) {
+        int64_t length = (1UL << i) + length_adjust;
+        ValidateMinMax<TypeParam>(*array->Slice(0, length));
+      }
+    }
+  }
+}
+
 //
 // Mode
 //
 
-template <typename ArrowType>
+template <typename T>
 class TestPrimitiveModeKernel : public ::testing::Test {
+ public:
+  using ArrowType = T;
   using Traits = TypeTraits<ArrowType>;
   using c_type = typename ArrowType::c_type;
   using ModeType = typename Traits::ScalarType;
   using CountType = typename TypeTraits<Int64Type>::ScalarType;
 
- public:
   void AssertModeIs(const Datum& array, c_type expected_mode, int64_t expected_count) {
     ASSERT_OK_AND_ASSIGN(Datum out, Mode(array));
     const StructScalar& value = out.scalar_as<StructScalar>();
@@ -623,6 +756,12 @@ class TestPrimitiveModeKernel : public ::testing::Test {
     AssertModeIs(array, expected_mode, expected_count);
   }
 
+  void AssertModeIs(const std::vector<std::string>& json, c_type expected_mode,
+                    int64_t expected_count) {
+    auto chunked = ChunkedArrayFromJSON(type_singleton(), json);
+    AssertModeIs(chunked, expected_mode, expected_count);
+  }
+
   void AssertModeIsNull(const Datum& array) {
     ASSERT_OK_AND_ASSIGN(Datum out, Mode(array));
     const StructScalar& value = out.scalar_as<StructScalar>();
@@ -637,6 +776,32 @@ class TestPrimitiveModeKernel : public ::testing::Test {
     AssertModeIsNull(array);
   }
 
+  void AssertModeIsNull(const std::vector<std::string>& json) {
+    auto chunked = ChunkedArrayFromJSON(type_singleton(), json);
+    AssertModeIsNull(chunked);
+  }
+
+  void AssertModeIsNaN(const Datum& array, int64_t expected_count) {
+    ASSERT_OK_AND_ASSIGN(Datum out, Mode(array));
+    const StructScalar& value = out.scalar_as<StructScalar>();
+
+    const auto& out_mode = checked_cast<const ModeType&>(*value.value[0]);
+    ASSERT_NE(out_mode.value, out_mode.value);  // NaN != NaN
+
+    const auto& out_count = checked_cast<const CountType&>(*value.value[1]);
+    ASSERT_EQ(expected_count, out_count.value);
+  }
+
+  void AssertModeIsNaN(const std::string& json, int64_t expected_count) {
+    auto array = ArrayFromJSON(type_singleton(), json);
+    AssertModeIsNaN(array, expected_count);
+  }
+
+  void AssertModeIsNaN(const std::vector<std::string>& json, int64_t expected_count) {
+    auto chunked = ChunkedArrayFromJSON(type_singleton(), json);
+    AssertModeIsNaN(chunked, expected_count);
+  }
+
   std::shared_ptr<DataType> type_singleton() { return Traits::type_singleton(); }
 };
 
@@ -648,6 +813,10 @@ class TestFloatingModeKernel : public TestPrimitiveModeKernel<ArrowType> {};
 
 class TestBooleanModeKernel : public TestPrimitiveModeKernel<BooleanType> {};
 
+class TestInt8ModeKernelValueRange : public TestPrimitiveModeKernel<Int8Type> {};
+
+class TestInt32ModeKernel : public TestPrimitiveModeKernel<Int32Type> {};
+
 TEST_F(TestBooleanModeKernel, Basics) {
   this->AssertModeIs("[false, false]", false, 2);
   this->AssertModeIs("[false, false, true, true, true]", true, 3);
@@ -657,6 +826,10 @@ TEST_F(TestBooleanModeKernel, Basics) {
   this->AssertModeIs("[true, null, false, false, null, true, null, null, true]", true, 3);
   this->AssertModeIsNull("[null, null, null]");
   this->AssertModeIsNull("[]");
+
+  this->AssertModeIs({"[true, false]", "[true, true]", "[false, false]"}, false, 3);
+  this->AssertModeIs({"[true, null]", "[]", "[null, false]"}, false, 1);
+  this->AssertModeIsNull({"[null, null]", "[]", "[null]"});
 }
 
 TYPED_TEST_SUITE(TestIntegerModeKernel, IntegralArrowTypes);
@@ -668,6 +841,10 @@ TYPED_TEST(TestIntegerModeKernel, Basics) {
   this->AssertModeIs("[null, null, 2, null, 1]", 1, 1);
   this->AssertModeIsNull("[null, null, null]");
   this->AssertModeIsNull("[]");
+
+  this->AssertModeIs({"[5]", "[1, 1, 5]", "[5]"}, 5, 3);
+  this->AssertModeIs({"[5]", "[1, 1, 5]", "[5, 1]"}, 1, 3);
+  this->AssertModeIsNull({"[null, null]", "[]", "[null]"});
 }
 
 TYPED_TEST_SUITE(TestFloatingModeKernel, RealArrowTypes);
@@ -678,12 +855,356 @@ TYPED_TEST(TestFloatingModeKernel, Floats) {
   this->AssertModeIs("[Inf, -Inf, Inf, -Inf]", -INFINITY, 2);
 
   this->AssertModeIs("[null, null, 2, null, 1]", 1, 1);
-  this->AssertModeIs("[NaN, NaN, 1]", 1, 1);
+  this->AssertModeIs("[NaN, NaN, 1, null, 1]", 1, 2);
+
   this->AssertModeIsNull("[null, null, null]");
-  this->AssertModeIsNull("[NaN, NaN, null]");
-  this->AssertModeIsNull("[NaN, NaN, NaN]");
   this->AssertModeIsNull("[]");
+
+  this->AssertModeIsNaN("[NaN, NaN, 1]", 2);
+  this->AssertModeIsNaN("[NaN, NaN, null]", 2);
+  this->AssertModeIsNaN("[NaN, NaN, NaN]", 3);
+
+  this->AssertModeIs({"[Inf, 100]", "[Inf, 100]", "[Inf]"}, INFINITY, 3);
+  this->AssertModeIsNull({"[null, null]", "[]", "[null]"});
+  this->AssertModeIsNaN({"[NaN, 1]", "[NaN, 1]", "[NaN]"}, 3);
+}
+
+TEST_F(TestInt8ModeKernelValueRange, Basics) {
+  this->AssertModeIs("[0, 127, -128, -128]", -128, 2);
+  this->AssertModeIs("[127, 127, 127]", 127, 3);
+}
+
+template <typename ArrowType>
+struct ModeResult {
+  using T = typename ArrowType::c_type;
+
+  T mode = std::numeric_limits<T>::min();
+  int64_t count = 0;
+};
+
+template <typename ArrowType>
+ModeResult<ArrowType> NaiveMode(const Array& array) {
+  using ArrayType = typename TypeTraits<ArrowType>::ArrayType;
+  using CTYPE = typename ArrowType::c_type;
+
+  std::unordered_map<CTYPE, int64_t> value_counts;
+
+  const auto& array_numeric = reinterpret_cast<const ArrayType&>(array);
+  const auto values = array_numeric.raw_values();
+  internal::BitmapReader reader(array.null_bitmap_data(), array.offset(), array.length());
+  for (int64_t i = 0; i < array.length(); ++i) {
+    if (reader.IsSet()) {
+      ++value_counts[values[i]];
+    }
+    reader.Next();
+  }
+
+  ModeResult<ArrowType> result;
+  for (const auto& value_count : value_counts) {
+    auto value = value_count.first;
+    auto count = value_count.second;
+    if (count > result.count || (count == result.count && value < result.mode)) {
+      result.count = count;
+      result.mode = value;
+    }
+  }
+
+  return result;
+}
+
+template <typename ArrowType, typename CTYPE = typename ArrowType::c_type>
+void CheckModeWithRange(CTYPE range_min, CTYPE range_max) {
+  using ModeScalar = typename TypeTraits<ArrowType>::ScalarType;
+  using CountScalar = typename TypeTraits<Int64Type>::ScalarType;
+
+  auto rand = random::RandomArrayGenerator(0x5487655);
+  // 32K items (>= counting mode cutoff) within range, 10% null
+  auto array = rand.Numeric<ArrowType>(32 * 1024, range_min, range_max, 0.1);
+
+  auto expected = NaiveMode<ArrowType>(*array);
+  ASSERT_OK_AND_ASSIGN(Datum out, Mode(array));
+  const StructScalar& value = out.scalar_as<StructScalar>();
+
+  ASSERT_TRUE(value.is_valid);
+  const auto& out_mode = checked_cast<const ModeScalar&>(*value.value[0]);
+  const auto& out_count = checked_cast<const CountScalar&>(*value.value[1]);
+  ASSERT_EQ(out_mode.value, expected.mode);
+  ASSERT_EQ(out_count.value, expected.count);
+}
+
+TEST_F(TestInt32ModeKernel, SmallValueRange) {
+  // Small value range => should exercise counter-based Mode implementation
+  CheckModeWithRange<ArrowType>(-100, 100);
+}
+
+TEST_F(TestInt32ModeKernel, LargeValueRange) {
+  // Large value range => should exercise hashmap-based Mode implementation
+  CheckModeWithRange<ArrowType>(-10000000, 10000000);
+}
+
+//
+// Variance/Stddev
+//
+
+template <typename ArrowType>
+class TestPrimitiveVarStdKernel : public ::testing::Test {
+ public:
+  using Traits = TypeTraits<ArrowType>;
+  using ScalarType = typename TypeTraits<DoubleType>::ScalarType;
+
+  void AssertVarStdIs(const Array& array, const VarianceOptions& options,
+                      double expected_var) {
+    AssertVarStdIsInternal(array, options, expected_var);
+  }
+
+  void AssertVarStdIs(const std::shared_ptr<ChunkedArray>& array,
+                      const VarianceOptions& options, double expected_var) {
+    AssertVarStdIsInternal(array, options, expected_var);
+  }
+
+  void AssertVarStdIs(const std::string& json, const VarianceOptions& options,
+                      double expected_var) {
+    auto array = ArrayFromJSON(type_singleton(), json);
+    AssertVarStdIs(*array, options, expected_var);
+  }
+
+  void AssertVarStdIs(const std::vector<std::string>& json,
+                      const VarianceOptions& options, double expected_var) {
+    auto chunked = ChunkedArrayFromJSON(type_singleton(), json);
+    AssertVarStdIs(chunked, options, expected_var);
+  }
+
+  void AssertVarStdIsInvalid(const Array& array, const VarianceOptions& options) {
+    AssertVarStdIsInvalidInternal(array, options);
+  }
+
+  void AssertVarStdIsInvalid(const std::shared_ptr<ChunkedArray>& array,
+                             const VarianceOptions& options) {
+    AssertVarStdIsInvalidInternal(array, options);
+  }
+
+  void AssertVarStdIsInvalid(const std::string& json, const VarianceOptions& options) {
+    auto array = ArrayFromJSON(type_singleton(), json);
+    AssertVarStdIsInvalid(*array, options);
+  }
+
+  void AssertVarStdIsInvalid(const std::vector<std::string>& json,
+                             const VarianceOptions& options) {
+    auto array = ChunkedArrayFromJSON(type_singleton(), json);
+    AssertVarStdIsInvalid(array, options);
+  }
+
+  std::shared_ptr<DataType> type_singleton() { return Traits::type_singleton(); }
+
+ private:
+  void AssertVarStdIsInternal(const Datum& array, const VarianceOptions& options,
+                              double expected_var) {
+    ASSERT_OK_AND_ASSIGN(Datum out_var, Variance(array, options));
+    ASSERT_OK_AND_ASSIGN(Datum out_std, Stddev(array, options));
+    auto var = checked_cast<const ScalarType*>(out_var.scalar().get());
+    auto std = checked_cast<const ScalarType*>(out_std.scalar().get());
+    ASSERT_TRUE(var->is_valid && std->is_valid);
+    ASSERT_DOUBLE_EQ(std->value * std->value, var->value);
+    ASSERT_DOUBLE_EQ(var->value, expected_var);  // < 4ULP
+  }
+
+  void AssertVarStdIsInvalidInternal(const Datum& array, const VarianceOptions& options) {
+    ASSERT_OK_AND_ASSIGN(Datum out_var, Variance(array, options));
+    ASSERT_OK_AND_ASSIGN(Datum out_std, Stddev(array, options));
+    auto var = checked_cast<const ScalarType*>(out_var.scalar().get());
+    auto std = checked_cast<const ScalarType*>(out_std.scalar().get());
+    ASSERT_FALSE(var->is_valid || std->is_valid);
+  }
+};
+
+template <typename ArrowType>
+class TestNumericVarStdKernel : public TestPrimitiveVarStdKernel<ArrowType> {};
+
+// Reference value from numpy.var
+TYPED_TEST_SUITE(TestNumericVarStdKernel, NumericArrowTypes);
+TYPED_TEST(TestNumericVarStdKernel, Basics) {
+  VarianceOptions options;  // ddof = 0, population variance/stddev
+
+  this->AssertVarStdIs("[100]", options, 0);
+  this->AssertVarStdIs("[1, 2, 3]", options, 0.6666666666666666);
+  this->AssertVarStdIs("[null, 1, 2, null, 3]", options, 0.6666666666666666);
+
+  std::vector<std::string> chunks;
+  chunks = {"[]", "[1]", "[2]", "[null]", "[3]"};
+  this->AssertVarStdIs(chunks, options, 0.6666666666666666);
+  chunks = {"[1, 2, 3]", "[4, 5, 6]", "[7, 8]"};
+  this->AssertVarStdIs(chunks, options, 5.25);
+  chunks = {"[1, 2, 3, 4, 5, 6, 7]", "[8]"};
+  this->AssertVarStdIs(chunks, options, 5.25);
+
+  this->AssertVarStdIsInvalid("[null, null, null]", options);
+  this->AssertVarStdIsInvalid("[]", options);
+  this->AssertVarStdIsInvalid("[]", options);
+
+  options.ddof = 1;  // sample variance/stddev
+
+  this->AssertVarStdIs("[1, 2]", options, 0.5);
+
+  chunks = {"[1]", "[2]"};
+  this->AssertVarStdIs(chunks, options, 0.5);
+  chunks = {"[1, 2, 3]", "[4, 5, 6]", "[7, 8]"};
+  this->AssertVarStdIs(chunks, options, 6.0);
+  chunks = {"[1, 2, 3, 4, 5, 6, 7]", "[8]"};
+  this->AssertVarStdIs(chunks, options, 6.0);
+
+  this->AssertVarStdIsInvalid("[100]", options);
+  this->AssertVarStdIsInvalid("[100, null, null]", options);
+  chunks = {"[100]", "[null]", "[]"};
+  this->AssertVarStdIsInvalid(chunks, options);
+}
+
+// Test numerical stability
+template <typename ArrowType>
+class TestVarStdKernelStability : public TestPrimitiveVarStdKernel<ArrowType> {};
+
+typedef ::testing::Types<Int32Type, UInt32Type, Int64Type, UInt64Type, DoubleType>
+    VarStdStabilityTypes;
+
+TYPED_TEST_SUITE(TestVarStdKernelStability, VarStdStabilityTypes);
+TYPED_TEST(TestVarStdKernelStability, Basics) {
+  VarianceOptions options{1};  // ddof = 1
+  this->AssertVarStdIs("[100000004, 100000007, 100000013, 100000016]", options, 30.0);
+  this->AssertVarStdIs("[1000000004, 1000000007, 1000000013, 1000000016]", options, 30.0);
+  if (!is_unsigned_integer_type<TypeParam>::value) {
+    this->AssertVarStdIs("[-1000000016, -1000000013, -1000000007, -1000000004]", options,
+                         30.0);
+  }
+}
+
+// Test numerical stability of variance merging code
+class TestVarStdKernelMergeStability : public TestPrimitiveVarStdKernel<DoubleType> {};
+
+TEST_F(TestVarStdKernelMergeStability, Basics) {
+  VarianceOptions options{1};  // ddof = 1
+
+#ifndef __MINGW32__  // MinGW has precision issues
+  // XXX: The reference value from numpy is actually wrong due to floating
+  // point limits. The correct result should equals variance(90, 0) = 4050.
+  std::vector<std::string> chunks = {"[40000008000000490]", "[40000008000000400]"};
+  this->AssertVarStdIs(chunks, options, 3904.0);
+#endif
+}
+
+// Test integer arithmetic code
+class TestVarStdKernelInt32 : public TestPrimitiveVarStdKernel<Int32Type> {};
+
+TEST_F(TestVarStdKernelInt32, Basics) {
+  VarianceOptions options{1};
+  this->AssertVarStdIs("[-2147483648, -2147483647, -2147483646]", options, 1.0);
+  this->AssertVarStdIs("[2147483645, 2147483646, 2147483647]", options, 1.0);
+  this->AssertVarStdIs("[-2147483648, -2147483648, 2147483647]", options,
+                       6.148914688373205e+18);
+}
+
+class TestVarStdKernelUInt32 : public TestPrimitiveVarStdKernel<UInt32Type> {};
+
+TEST_F(TestVarStdKernelUInt32, Basics) {
+  VarianceOptions options{1};
+  this->AssertVarStdIs("[4294967293, 4294967294, 4294967295]", options, 1.0);
+  this->AssertVarStdIs("[0, 0, 4294967295]", options, 6.148914688373205e+18);
+}
+
+// https://en.wikipedia.org/wiki/Kahan_summation_algorithm
+void KahanSum(double& sum, double& adjust, double addend) {
+  double y = addend - adjust;
+  double t = sum + y;
+  adjust = (t - sum) - y;
+  sum = t;
+}
+
+// Calculate reference variance with Welford's online algorithm + Kahan summation
+// https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm
+// XXX: not stable for long array with very small `stddev / average`
+template <typename ArrayType>
+std::pair<double, double> WelfordVar(const ArrayType& array) {
+  const auto values = array.raw_values();
+  internal::BitmapReader reader(array.null_bitmap_data(), array.offset(), array.length());
+  double count = 0, mean = 0, m2 = 0;
+  double mean_adjust = 0, m2_adjust = 0;
+  for (int64_t i = 0; i < array.length(); ++i) {
+    if (reader.IsSet()) {
+      ++count;
+      double delta = static_cast<double>(values[i]) - mean;
+      KahanSum(mean, mean_adjust, delta / count);
+      double delta2 = static_cast<double>(values[i]) - mean;
+      KahanSum(m2, m2_adjust, delta * delta2);
+    }
+    reader.Next();
+  }
+  return std::make_pair(m2 / count, m2 / (count - 1));
+}
+
+// Test random chunked array
+template <typename ArrowType>
+class TestVarStdKernelRandom : public TestPrimitiveVarStdKernel<ArrowType> {};
+
+typedef ::testing::Types<Int32Type, UInt32Type, Int64Type, UInt64Type, FloatType,
+                         DoubleType>
+    VarStdRandomTypes;
+
+TYPED_TEST_SUITE(TestVarStdKernelRandom, VarStdRandomTypes);
+TYPED_TEST(TestVarStdKernelRandom, Basics) {
+  // Cut array into small chunks
+  constexpr int array_size = 5000;
+  constexpr int chunk_size_max = 50;
+  constexpr int chunk_count = array_size / chunk_size_max;
+
+  std::shared_ptr<Array> array;
+  auto rand = random::RandomArrayGenerator(0x5487656);
+  if (is_floating_type<TypeParam>::value) {
+    array = rand.Numeric<TypeParam>(array_size, -10000.0, 100000.0, 0.1);
+  } else {
+    using CType = typename TypeParam::c_type;
+    constexpr CType min = std::numeric_limits<CType>::min();
+    constexpr CType max = std::numeric_limits<CType>::max();
+    array = rand.Numeric<TypeParam>(array_size, min, max, 0.1);
+  }
+  auto chunk_size_array = rand.Numeric<Int32Type>(chunk_count, 0, chunk_size_max);
+  const int* chunk_size = chunk_size_array->data()->GetValues<int>(1);
+  int total_size = 0;
+
+  ArrayVector array_vector;
+  for (int i = 0; i < chunk_count; ++i) {
+    array_vector.emplace_back(array->Slice(total_size, chunk_size[i]));
+    total_size += chunk_size[i];
+  }
+  auto chunked = *ChunkedArray::Make(array_vector);
+
+  double var_population, var_sample;
+  using ArrayType = typename TypeTraits<TypeParam>::ArrayType;
+  auto typed_array = std::static_pointer_cast<ArrayType>(array->Slice(0, total_size));
+  std::tie(var_population, var_sample) = WelfordVar(*typed_array);
+
+  this->AssertVarStdIs(chunked, VarianceOptions{0}, var_population);
+  this->AssertVarStdIs(chunked, VarianceOptions{1}, var_sample);
+}
+
+// This test is too heavy to run in CI, should be checked manually
+#if 0
+class TestVarStdKernelIntegerLength : public TestPrimitiveVarStdKernel<Int32Type> {};
+
+TEST_F(TestVarStdKernelIntegerLength, Basics) {
+  constexpr int32_t min = std::numeric_limits<int32_t>::min();
+  constexpr int32_t max = std::numeric_limits<int32_t>::max();
+  auto rand = random::RandomArrayGenerator(0x5487657);
+  // large data volume
+  auto array = rand.Numeric<Int32Type>(4000000000, min, max, 0.1);
+  // biased distribution
+  // auto array = rand.Numeric<Int32Type>(4000000000, min, min + 100000, 0.1);
+
+  double var_population, var_sample;
+  auto int32_array = std::static_pointer_cast<Int32Array>(array);
+  std::tie(var_population, var_sample) = WelfordVar(*int32_array);
+
+  this->AssertVarStdIs(*array, VarianceOptions{0}, var_population);
+  this->AssertVarStdIs(*array, VarianceOptions{1}, var_sample);
 }
+#endif
 
 }  // namespace compute
 }  // namespace arrow
diff --git a/cpp/src/arrow/compute/kernels/aggregate_var_std.cc b/cpp/src/arrow/compute/kernels/aggregate_var_std.cc
new file mode 100644
index 00000000000..4dac0a37734
--- /dev/null
+++ b/cpp/src/arrow/compute/kernels/aggregate_var_std.cc
@@ -0,0 +1,284 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cmath>
+
+#include "arrow/compute/api_aggregate.h"
+#include "arrow/compute/kernels/aggregate_internal.h"
+#include "arrow/compute/kernels/common.h"
+#include "arrow/util/int128_internal.h"
+
+namespace arrow {
+namespace compute {
+namespace internal {
+
+namespace {
+
+using arrow::internal::int128_t;
+
+template <typename ArrowType>
+struct VarStdState {
+  using ArrayType = typename TypeTraits<ArrowType>::ArrayType;
+  using CType = typename ArrowType::c_type;
+  using ThisType = VarStdState<ArrowType>;
+
+  // float/double/int64: calculate `m2` (sum((X-mean)^2)) with `two pass algorithm`
+  // https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Two-pass_algorithm
+  template <typename T = ArrowType>
+  enable_if_t<is_floating_type<T>::value || (sizeof(CType) > 4)> Consume(
+      const ArrayType& array) {
+    int64_t count = array.length() - array.null_count();
+    if (count == 0) {
+      return;
+    }
+
+    using SumType =
+        typename std::conditional<is_floating_type<T>::value, double, int128_t>::type;
+    SumType sum = 0;
+    VisitArrayDataInline<ArrowType>(
+        *array.data(), [&sum](CType value) { sum += static_cast<SumType>(value); },
+        []() {});
+
+    double mean = static_cast<double>(sum) / count, m2 = 0;
+    VisitArrayDataInline<ArrowType>(
+        *array.data(),
+        [mean, &m2](CType value) {
+          double v = static_cast<double>(value);
+          m2 += (v - mean) * (v - mean);
+        },
+        []() {});
+
+    this->count = count;
+    this->mean = mean;
+    this->m2 = m2;
+  }
+
+  // int32/16/8: textbook one pass algorithm with integer arithmetic
+  template <typename T = ArrowType>
+  enable_if_t<is_integer_type<T>::value && (sizeof(CType) <= 4)> Consume(
+      const ArrayType& array) {
+    // max number of elements that sum will not overflow int64 (2Gi int32 elements)
+    // for uint32:    0 <= sum < 2^63 (int64 >= 0)
+    // for int32: -2^62 <= sum < 2^62
+    constexpr int64_t max_length = 1ULL << (63 - sizeof(CType) * 8);
+
+    int64_t start_index = 0;
+    int64_t valid_count = array.length() - array.null_count();
+
+    while (valid_count > 0) {
+      // process in chunks that overflow will never happen
+      const auto slice = array.Slice(start_index, max_length);
+      const int64_t count = slice->length() - slice->null_count();
+      start_index += max_length;
+      valid_count -= count;
+
+      if (count > 0) {
+        int64_t sum = 0;
+        int128_t square_sum = 0;
+        VisitArrayDataInline<ArrowType>(
+            *slice->data(),
+            [&sum, &square_sum](CType value) {
+              sum += value;
+              square_sum += static_cast<uint64_t>(value) * value;
+            },
+            []() {});
+
+        const double mean = static_cast<double>(sum) / count;
+        // calculate m2 = square_sum - sum * sum / count
+        // decompose `sum * sum / count` into integers and fractions
+        const int128_t sum_square = static_cast<int128_t>(sum) * sum;
+        const int128_t integers = sum_square / count;
+        const double fractions = static_cast<double>(sum_square % count) / count;
+        const double m2 = static_cast<double>(square_sum - integers) - fractions;
+
+        // merge variance
+        ThisType state;
+        state.count = count;
+        state.mean = mean;
+        state.m2 = m2;
+        this->MergeFrom(state);
+      }
+    }
+  }
+
+  // Combine `m2` from two chunks (m2 = n*s2)
+  // https://www.emathzone.com/tutorials/basic-statistics/combined-variance.html
+  void MergeFrom(const ThisType& state) {
+    if (state.count == 0) {
+      return;
+    }
+    if (this->count == 0) {
+      this->count = state.count;
+      this->mean = state.mean;
+      this->m2 = state.m2;
+      return;
+    }
+    double mean = (this->mean * this->count + state.mean * state.count) /
+                  (this->count + state.count);
+    this->m2 += state.m2 + this->count * (this->mean - mean) * (this->mean - mean) +
+                state.count * (state.mean - mean) * (state.mean - mean);
+    this->count += state.count;
+    this->mean = mean;
+  }
+
+  int64_t count = 0;
+  double mean = 0;
+  double m2 = 0;  // m2 = count*s2 = sum((X-mean)^2)
+};
+
+enum class VarOrStd : bool { Var, Std };
+
+template <typename ArrowType>
+struct VarStdImpl : public ScalarAggregator {
+  using ThisType = VarStdImpl<ArrowType>;
+  using ArrayType = typename TypeTraits<ArrowType>::ArrayType;
+
+  explicit VarStdImpl(const std::shared_ptr<DataType>& out_type,
+                      const VarianceOptions& options, VarOrStd return_type)
+      : out_type(out_type), options(options), return_type(return_type) {}
+
+  void Consume(KernelContext*, const ExecBatch& batch) override {
+    ArrayType array(batch[0].array());
+    this->state.Consume(array);
+  }
+
+  void MergeFrom(KernelContext*, KernelState&& src) override {
+    const auto& other = checked_cast<const ThisType&>(src);
+    this->state.MergeFrom(other.state);
+  }
+
+  void Finalize(KernelContext*, Datum* out) override {
+    if (this->state.count <= options.ddof) {
+      out->value = std::make_shared<DoubleScalar>();
+    } else {
+      double var = this->state.m2 / (this->state.count - options.ddof);
+      out->value =
+          std::make_shared<DoubleScalar>(return_type == VarOrStd::Var ? var : sqrt(var));
+    }
+  }
+
+  std::shared_ptr<DataType> out_type;
+  VarStdState<ArrowType> state;
+  VarianceOptions options;
+  VarOrStd return_type;
+};
+
+struct VarStdInitState {
+  std::unique_ptr<KernelState> state;
+  KernelContext* ctx;
+  const DataType& in_type;
+  const std::shared_ptr<DataType>& out_type;
+  const VarianceOptions& options;
+  VarOrStd return_type;
+
+  VarStdInitState(KernelContext* ctx, const DataType& in_type,
+                  const std::shared_ptr<DataType>& out_type,
+                  const VarianceOptions& options, VarOrStd return_type)
+      : ctx(ctx),
+        in_type(in_type),
+        out_type(out_type),
+        options(options),
+        return_type(return_type) {}
+
+  Status Visit(const DataType&) {
+    return Status::NotImplemented("No variance/stddev implemented");
+  }
+
+  Status Visit(const HalfFloatType&) {
+    return Status::NotImplemented("No variance/stddev implemented");
+  }
+
+  template <typename Type>
+  enable_if_t<is_number_type<Type>::value, Status> Visit(const Type&) {
+    state.reset(new VarStdImpl<Type>(out_type, options, return_type));
+    return Status::OK();
+  }
+
+  std::unique_ptr<KernelState> Create() {
+    ctx->SetStatus(VisitTypeInline(in_type, this));
+    return std::move(state);
+  }
+};
+
+std::unique_ptr<KernelState> StddevInit(KernelContext* ctx, const KernelInitArgs& args) {
+  VarStdInitState visitor(
+      ctx, *args.inputs[0].type, args.kernel->signature->out_type().type(),
+      static_cast<const VarianceOptions&>(*args.options), VarOrStd::Std);
+  return visitor.Create();
+}
+
+std::unique_ptr<KernelState> VarianceInit(KernelContext* ctx,
+                                          const KernelInitArgs& args) {
+  VarStdInitState visitor(
+      ctx, *args.inputs[0].type, args.kernel->signature->out_type().type(),
+      static_cast<const VarianceOptions&>(*args.options), VarOrStd::Var);
+  return visitor.Create();
+}
+
+void AddVarStdKernels(KernelInit init,
+                      const std::vector<std::shared_ptr<DataType>>& types,
+                      ScalarAggregateFunction* func) {
+  for (const auto& ty : types) {
+    auto sig = KernelSignature::Make({InputType::Array(ty)}, float64());
+    AddAggKernel(std::move(sig), init, func);
+  }
+}
+
+const FunctionDoc stddev_doc{
+    "Calculate the standard deviation of a numeric array",
+    ("The number of degrees of freedom can be controlled using VarianceOptions.\n"
+     "By default (`ddof` = 0), the population standard deviation is calculated.\n"
+     "Nulls are ignored.  If there are not enough non-null values in the array\n"
+     "to satisfy `ddof`, null is returned."),
+    {"array"},
+    "VarianceOptions"};
+
+const FunctionDoc variance_doc{
+    "Calculate the variance of a numeric array",
+    ("The number of degrees of freedom can be controlled using VarianceOptions.\n"
+     "By default (`ddof` = 0), the population variance is calculated.\n"
+     "Nulls are ignored.  If there are not enough non-null values in the array\n"
+     "to satisfy `ddof`, null is returned."),
+    {"array"},
+    "VarianceOptions"};
+
+std::shared_ptr<ScalarAggregateFunction> AddStddevAggKernels() {
+  static auto default_std_options = VarianceOptions::Defaults();
+  auto func = std::make_shared<ScalarAggregateFunction>(
+      "stddev", Arity::Unary(), &stddev_doc, &default_std_options);
+  AddVarStdKernels(StddevInit, NumericTypes(), func.get());
+  return func;
+}
+
+std::shared_ptr<ScalarAggregateFunction> AddVarianceAggKernels() {
+  static auto default_var_options = VarianceOptions::Defaults();
+  auto func = std::make_shared<ScalarAggregateFunction>(
+      "variance", Arity::Unary(), &variance_doc, &default_var_options);
+  AddVarStdKernels(VarianceInit, NumericTypes(), func.get());
+  return func;
+}
+
+}  // namespace
+
+void RegisterScalarAggregateVariance(FunctionRegistry* registry) {
+  DCHECK_OK(registry->AddFunction(AddVarianceAggKernels()));
+  DCHECK_OK(registry->AddFunction(AddStddevAggKernels()));
+}
+
+}  // namespace internal
+}  // namespace compute
+}  // namespace arrow
diff --git a/cpp/src/arrow/compute/kernels/codegen_internal.h b/cpp/src/arrow/compute/kernels/codegen_internal.h
index a4e11fe3894..1f940249857 100644
--- a/cpp/src/arrow/compute/kernels/codegen_internal.h
+++ b/cpp/src/arrow/compute/kernels/codegen_internal.h
@@ -728,7 +728,6 @@ struct ScalarBinary {
       }
     } else {
       if (batch[1].kind() == Datum::ARRAY) {
-        // e.g. if we were doing scalar < array, we flip and do array >= scalar
         return ScalarArray(ctx, *batch[0].scalar(), *batch[1].array(), out);
       } else {
         return ScalarScalar(ctx, *batch[0].scalar(), *batch[1].scalar(), out);
@@ -842,6 +841,8 @@ struct ScalarBinaryNotNull {
 template <typename OutType, typename ArgType, typename Op>
 using ScalarBinaryEqualTypes = ScalarBinary<OutType, ArgType, ArgType, Op>;
 
+// A kernel exec generator for non-null binary kernels where both input types are the
+// same
 template <typename OutType, typename ArgType, typename Op>
 using ScalarBinaryNotNullEqualTypes = ScalarBinaryNotNull<OutType, ArgType, ArgType, Op>;
 
diff --git a/cpp/src/arrow/compute/kernels/scalar_arithmetic.cc b/cpp/src/arrow/compute/kernels/scalar_arithmetic.cc
index 299f652dc3d..fc18da7cf13 100644
--- a/cpp/src/arrow/compute/kernels/scalar_arithmetic.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_arithmetic.cc
@@ -22,6 +22,7 @@
 namespace arrow {
 
 using internal::AddWithOverflow;
+using internal::DivideWithOverflow;
 using internal::MultiplyWithOverflow;
 using internal::SubtractWithOverflow;
 
@@ -80,7 +81,7 @@ struct AddChecked {
   template <typename T, typename Arg0, typename Arg1>
   enable_if_integer<T> Call(KernelContext* ctx, Arg0 left, Arg1 right) {
     static_assert(std::is_same<T, Arg0>::value && std::is_same<T, Arg1>::value, "");
-    T result;
+    T result = 0;
     if (ARROW_PREDICT_FALSE(AddWithOverflow(left, right, &result))) {
       ctx->SetStatus(Status::Invalid("overflow"));
     }
@@ -115,7 +116,7 @@ struct SubtractChecked {
   template <typename T, typename Arg0, typename Arg1>
   enable_if_integer<T> Call(KernelContext* ctx, Arg0 left, Arg1 right) {
     static_assert(std::is_same<T, Arg0>::value && std::is_same<T, Arg1>::value, "");
-    T result;
+    T result = 0;
     if (ARROW_PREDICT_FALSE(SubtractWithOverflow(left, right, &result))) {
       ctx->SetStatus(Status::Invalid("overflow"));
     }
@@ -172,7 +173,7 @@ struct MultiplyChecked {
   template <typename T, typename Arg0, typename Arg1>
   enable_if_integer<T> Call(KernelContext* ctx, Arg0 left, Arg1 right) {
     static_assert(std::is_same<T, Arg0>::value && std::is_same<T, Arg1>::value, "");
-    T result;
+    T result = 0;
     if (ARROW_PREDICT_FALSE(MultiplyWithOverflow(left, right, &result))) {
       ctx->SetStatus(Status::Invalid("overflow"));
     }
@@ -186,6 +187,52 @@ struct MultiplyChecked {
   }
 };
 
+struct Divide {
+  template <typename T, typename Arg0, typename Arg1>
+  static enable_if_floating_point<T> Call(KernelContext* ctx, Arg0 left, Arg1 right) {
+    return left / right;
+  }
+
+  template <typename T, typename Arg0, typename Arg1>
+  static enable_if_integer<T> Call(KernelContext* ctx, Arg0 left, Arg1 right) {
+    T result;
+    if (ARROW_PREDICT_FALSE(DivideWithOverflow(left, right, &result))) {
+      if (right == 0) {
+        ctx->SetStatus(Status::Invalid("divide by zero"));
+      } else {
+        result = 0;
+      }
+    }
+    return result;
+  }
+};
+
+struct DivideChecked {
+  template <typename T, typename Arg0, typename Arg1>
+  static enable_if_integer<T> Call(KernelContext* ctx, Arg0 left, Arg1 right) {
+    static_assert(std::is_same<T, Arg0>::value && std::is_same<T, Arg1>::value, "");
+    T result;
+    if (ARROW_PREDICT_FALSE(DivideWithOverflow(left, right, &result))) {
+      if (right == 0) {
+        ctx->SetStatus(Status::Invalid("divide by zero"));
+      } else {
+        ctx->SetStatus(Status::Invalid("overflow"));
+      }
+    }
+    return result;
+  }
+
+  template <typename T, typename Arg0, typename Arg1>
+  static enable_if_floating_point<T> Call(KernelContext* ctx, Arg0 left, Arg1 right) {
+    static_assert(std::is_same<T, Arg0>::value && std::is_same<T, Arg1>::value, "");
+    if (ARROW_PREDICT_FALSE(right == 0)) {
+      ctx->SetStatus(Status::Invalid("divide by zero"));
+      return 0;
+    }
+    return left / right;
+  }
+};
+
 // Generate a kernel given an arithmetic functor
 template <template <typename... Args> class KernelGenerator, typename Op>
 ArrayKernelExec NumericEqualTypesBinary(detail::GetTypeId get_id) {
@@ -218,8 +265,9 @@ ArrayKernelExec NumericEqualTypesBinary(detail::GetTypeId get_id) {
 }
 
 template <typename Op>
-std::shared_ptr<ScalarFunction> MakeArithmeticFunction(std::string name) {
-  auto func = std::make_shared<ScalarFunction>(name, Arity::Binary());
+std::shared_ptr<ScalarFunction> MakeArithmeticFunction(std::string name,
+                                                       const FunctionDoc* doc) {
+  auto func = std::make_shared<ScalarFunction>(name, Arity::Binary(), doc);
   for (const auto& ty : NumericTypes()) {
     auto exec = NumericEqualTypesBinary<ScalarBinaryEqualTypes, Op>(ty);
     DCHECK_OK(func->AddKernel({ty, ty}, ty, exec));
@@ -230,8 +278,9 @@ std::shared_ptr<ScalarFunction> MakeArithmeticFunction(std::string name) {
 // Like MakeArithmeticFunction, but for arithmetic ops that need to run
 // only on non-null output.
 template <typename Op>
-std::shared_ptr<ScalarFunction> MakeArithmeticFunctionNotNull(std::string name) {
-  auto func = std::make_shared<ScalarFunction>(name, Arity::Binary());
+std::shared_ptr<ScalarFunction> MakeArithmeticFunctionNotNull(std::string name,
+                                                              const FunctionDoc* doc) {
+  auto func = std::make_shared<ScalarFunction>(name, Arity::Binary(), doc);
   for (const auto& ty : NumericTypes()) {
     auto exec = NumericEqualTypesBinary<ScalarBinaryNotNullEqualTypes, Op>(ty);
     DCHECK_OK(func->AddKernel({ty, ty}, ty, exec));
@@ -239,20 +288,71 @@ std::shared_ptr<ScalarFunction> MakeArithmeticFunctionNotNull(std::string name)
   return func;
 }
 
+const FunctionDoc add_doc{"Add the arguments element-wise",
+                          ("Results will wrap around on integer overflow.\n"
+                           "Use function \"add_checked\" if you want overflow\n"
+                           "to return an error."),
+                          {"x", "y"}};
+
+const FunctionDoc add_checked_doc{
+    "Add the arguments element-wise",
+    ("This function returns an error on overflow.  For a variant that\n"
+     "doesn't fail on overflow, use function \"add\"."),
+    {"x", "y"}};
+
+const FunctionDoc sub_doc{"Substract the arguments element-wise",
+                          ("Results will wrap around on integer overflow.\n"
+                           "Use function \"subtract_checked\" if you want overflow\n"
+                           "to return an error."),
+                          {"x", "y"}};
+
+const FunctionDoc sub_checked_doc{
+    "Substract the arguments element-wise",
+    ("This function returns an error on overflow.  For a variant that\n"
+     "doesn't fail on overflow, use function \"subtract\"."),
+    {"x", "y"}};
+
+const FunctionDoc mul_doc{"Multiply the arguments element-wise",
+                          ("Results will wrap around on integer overflow.\n"
+                           "Use function \"multiply_checked\" if you want overflow\n"
+                           "to return an error."),
+                          {"x", "y"}};
+
+const FunctionDoc mul_checked_doc{
+    "Multiply the arguments element-wise",
+    ("This function returns an error on overflow.  For a variant that\n"
+     "doesn't fail on overflow, use function \"multiply\"."),
+    {"x", "y"}};
+
+const FunctionDoc div_doc{
+    "Divide the arguments element-wise",
+    ("Integer division by zero returns an error. However, integer overflow\n"
+     "wraps around, and floating-point division by zero returns an infinite.\n"
+     "Use function \"divide_checked\" if you want to get an error\n"
+     "in all the aforementioned cases."),
+    {"dividend", "divisor"}};
+
+const FunctionDoc div_checked_doc{
+    "Divide the arguments element-wise",
+    ("An error is returned when trying to divide by zero, or when\n"
+     "integer overflow is encountered."),
+    {"dividend", "divisor"}};
+
 }  // namespace
 
 void RegisterScalarArithmetic(FunctionRegistry* registry) {
   // ----------------------------------------------------------------------
-  auto add = MakeArithmeticFunction<Add>("add");
+  auto add = MakeArithmeticFunction<Add>("add", &add_doc);
   DCHECK_OK(registry->AddFunction(std::move(add)));
 
   // ----------------------------------------------------------------------
-  auto add_checked = MakeArithmeticFunctionNotNull<AddChecked>("add_checked");
+  auto add_checked =
+      MakeArithmeticFunctionNotNull<AddChecked>("add_checked", &add_checked_doc);
   DCHECK_OK(registry->AddFunction(std::move(add_checked)));
 
   // ----------------------------------------------------------------------
   // subtract
-  auto subtract = MakeArithmeticFunction<Subtract>("subtract");
+  auto subtract = MakeArithmeticFunction<Subtract>("subtract", &sub_doc);
 
   // Add subtract(timestamp, timestamp) -> duration
   for (auto unit : AllTimeUnits()) {
@@ -265,18 +365,27 @@ void RegisterScalarArithmetic(FunctionRegistry* registry) {
   DCHECK_OK(registry->AddFunction(std::move(subtract)));
 
   // ----------------------------------------------------------------------
-  auto subtract_checked =
-      MakeArithmeticFunctionNotNull<SubtractChecked>("subtract_checked");
+  auto subtract_checked = MakeArithmeticFunctionNotNull<SubtractChecked>(
+      "subtract_checked", &sub_checked_doc);
   DCHECK_OK(registry->AddFunction(std::move(subtract_checked)));
 
   // ----------------------------------------------------------------------
-  auto multiply = MakeArithmeticFunction<Multiply>("multiply");
+  auto multiply = MakeArithmeticFunction<Multiply>("multiply", &mul_doc);
   DCHECK_OK(registry->AddFunction(std::move(multiply)));
 
   // ----------------------------------------------------------------------
-  auto multiply_checked =
-      MakeArithmeticFunctionNotNull<MultiplyChecked>("multiply_checked");
+  auto multiply_checked = MakeArithmeticFunctionNotNull<MultiplyChecked>(
+      "multiply_checked", &mul_checked_doc);
   DCHECK_OK(registry->AddFunction(std::move(multiply_checked)));
+
+  // ----------------------------------------------------------------------
+  auto divide = MakeArithmeticFunctionNotNull<Divide>("divide", &div_doc);
+  DCHECK_OK(registry->AddFunction(std::move(divide)));
+
+  // ----------------------------------------------------------------------
+  auto divide_checked =
+      MakeArithmeticFunctionNotNull<DivideChecked>("divide_checked", &div_checked_doc);
+  DCHECK_OK(registry->AddFunction(std::move(divide_checked)));
 }
 
 }  // namespace internal
diff --git a/cpp/src/arrow/compute/kernels/scalar_arithmetic_benchmark.cc b/cpp/src/arrow/compute/kernels/scalar_arithmetic_benchmark.cc
index 8dec7940f08..ab0f07bd441 100644
--- a/cpp/src/arrow/compute/kernels/scalar_arithmetic_benchmark.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_arithmetic_benchmark.cc
@@ -55,6 +55,13 @@ static Result<Datum> MultiplyChecked(const Datum& left, const Datum& right,
   return Multiply(left, right, std::move(options), ctx);
 }
 
+static Result<Datum> DivideChecked(const Datum& left, const Datum& right,
+                                   ArithmeticOptions options = ArithmeticOptions(),
+                                   ExecContext* ctx = NULLPTR) {
+  options.check_overflow = true;
+  return Divide(left, right, std::move(options), ctx);
+}
+
 template <BinaryOp& Op, typename ArrowType, typename CType = typename ArrowType::c_type>
 static void ArrayScalarKernel(benchmark::State& state) {
   RegressionArgs args(state);
@@ -80,16 +87,18 @@ template <BinaryOp& Op, typename ArrowType, typename CType = typename ArrowType:
 static void ArrayArrayKernel(benchmark::State& state) {
   RegressionArgs args(state);
 
-  // Choose values so as to avoid overflow on all ops
+  // Choose values so as to avoid overflow on all ops and types
   const int64_t array_size = args.size / sizeof(CType);
-  auto min = static_cast<CType>(0);
-  auto max = static_cast<CType>(11);
+  auto rmin = static_cast<CType>(1);
+  auto rmax = static_cast<CType>(rmin + 6);  // 7
+  auto lmin = static_cast<CType>(rmax + 1);  // 8
+  auto lmax = static_cast<CType>(lmin + 6);  // 14
 
   auto rand = random::RandomArrayGenerator(kSeed);
   auto lhs = std::static_pointer_cast<NumericArray<ArrowType>>(
-      rand.Numeric<ArrowType>(array_size, min, max, args.null_proportion));
+      rand.Numeric<ArrowType>(array_size, lmin, lmax, args.null_proportion));
   auto rhs = std::static_pointer_cast<NumericArray<ArrowType>>(
-      rand.Numeric<ArrowType>(array_size, min, max, args.null_proportion));
+      rand.Numeric<ArrowType>(array_size, rmin, rmax, args.null_proportion));
 
   for (auto _ : state) {
     ABORT_NOT_OK(Op(lhs, rhs, ArithmeticOptions(), nullptr).status());
@@ -136,6 +145,8 @@ DECLARE_ARITHMETIC_BENCHMARKS(ArrayArrayKernel, Subtract);
 DECLARE_ARITHMETIC_BENCHMARKS(ArrayScalarKernel, Subtract);
 DECLARE_ARITHMETIC_BENCHMARKS(ArrayArrayKernel, Multiply);
 DECLARE_ARITHMETIC_BENCHMARKS(ArrayScalarKernel, Multiply);
+DECLARE_ARITHMETIC_BENCHMARKS(ArrayArrayKernel, Divide);
+DECLARE_ARITHMETIC_BENCHMARKS(ArrayScalarKernel, Divide);
 
 DECLARE_ARITHMETIC_CHECKED_BENCHMARKS(ArrayArrayKernel, AddChecked);
 DECLARE_ARITHMETIC_CHECKED_BENCHMARKS(ArrayScalarKernel, AddChecked);
@@ -143,6 +154,8 @@ DECLARE_ARITHMETIC_CHECKED_BENCHMARKS(ArrayArrayKernel, SubtractChecked);
 DECLARE_ARITHMETIC_CHECKED_BENCHMARKS(ArrayScalarKernel, SubtractChecked);
 DECLARE_ARITHMETIC_CHECKED_BENCHMARKS(ArrayArrayKernel, MultiplyChecked);
 DECLARE_ARITHMETIC_CHECKED_BENCHMARKS(ArrayScalarKernel, MultiplyChecked);
+DECLARE_ARITHMETIC_CHECKED_BENCHMARKS(ArrayArrayKernel, DivideChecked);
+DECLARE_ARITHMETIC_CHECKED_BENCHMARKS(ArrayScalarKernel, DivideChecked);
 
 }  // namespace compute
 }  // namespace arrow
diff --git a/cpp/src/arrow/compute/kernels/scalar_arithmetic_test.cc b/cpp/src/arrow/compute/kernels/scalar_arithmetic_test.cc
index 4593b79013b..9b3ed2a57e3 100644
--- a/cpp/src/arrow/compute/kernels/scalar_arithmetic_test.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_arithmetic_test.cc
@@ -144,7 +144,8 @@ class TestBinaryArithmetic : public TestBase {
       const auto expected_scalar = *expected->GetScalar(i);
       ASSERT_OK_AND_ASSIGN(
           actual, func(*left->GetScalar(i), *right->GetScalar(i), options_, nullptr));
-      AssertScalarsEqual(*expected_scalar, *actual.scalar(), /*verbose=*/true);
+      AssertScalarsEqual(*expected_scalar, *actual.scalar(), /*verbose=*/true,
+                         equal_options_);
     }
   }
 
@@ -165,12 +166,17 @@ class TestBinaryArithmetic : public TestBase {
   void ValidateAndAssertApproxEqual(const std::shared_ptr<Array>& actual,
                                     const std::shared_ptr<Array>& expected) {
     ASSERT_OK(actual->ValidateFull());
-    AssertArraysApproxEqual(*expected, *actual, /*verbose=*/true);
+    AssertArraysApproxEqual(*expected, *actual, /*verbose=*/true, equal_options_);
   }
 
   void SetOverflowCheck(bool value = true) { options_.check_overflow = value; }
 
+  void SetNansEqual(bool value = true) {
+    this->equal_options_ = equal_options_.nans_equal(value);
+  }
+
   ArithmeticOptions options_ = ArithmeticOptions();
+  EqualOptions equal_options_ = EqualOptions::Defaults();
 };
 
 template <typename... Elements>
@@ -492,6 +498,98 @@ TYPED_TEST(TestBinaryArithmeticFloating, Add) {
   this->AssertBinop(Add, "[null, 2.0]", this->MakeNullScalar(), "[null, null]");
 }
 
+TYPED_TEST(TestBinaryArithmeticFloating, Div) {
+  for (auto check_overflow : {false, true}) {
+    this->SetOverflowCheck(check_overflow);
+    // Empty arrays
+    this->AssertBinop(Divide, "[]", "[]", "[]");
+    // Ordinary arrays
+    this->AssertBinop(Divide, "[3.4, 0.64, 1.28]", "[1, 2, 4]", "[3.4, 0.32, 0.32]");
+    // Array with nulls
+    this->AssertBinop(Divide, "[null, 1, 3.3, null, 2]", "[1, 4, 2, 5, 0.1]",
+                      "[null, 0.25, 1.65, null, 20]");
+    // Scalar divides by array
+    this->AssertBinop(Divide, 10.0F, "[null, 1, 2.5, null, 2, 5]",
+                      "[null, 10, 4, null, 5, 2]");
+    // Array divides by scalar
+    this->AssertBinop(Divide, "[null, 1, 2.5, null, 2, 5]", 10.0F,
+                      "[null, 0.1, 0.25, null, 0.2, 0.5]");
+    // Array with infinity
+    this->AssertBinop(Divide, "[3.4, Inf, -Inf]", "[1, 2, 3]", "[3.4, Inf, -Inf]");
+    // Array with NaN
+    this->SetNansEqual(true);
+    this->AssertBinop(Divide, "[3.4, NaN, 2.0]", "[1, 2, 2.0]", "[3.4, NaN, 1.0]");
+    // Scalar divides by scalar
+    this->AssertBinop(Divide, 21.0F, 3.0F, 7.0F);
+  }
+}
+
+TYPED_TEST(TestBinaryArithmeticIntegral, Div) {
+  for (auto check_overflow : {false, true}) {
+    this->SetOverflowCheck(check_overflow);
+
+    // Empty arrays
+    this->AssertBinop(Divide, "[]", "[]", "[]");
+    // Ordinary arrays
+    this->AssertBinop(Divide, "[3, 2, 6]", "[1, 1, 2]", "[3, 2, 3]");
+    // Array with nulls
+    this->AssertBinop(Divide, "[null, 10, 30, null, 20]", "[1, 4, 2, 5, 10]",
+                      "[null, 2, 15, null, 2]");
+    // Scalar divides by array
+    this->AssertBinop(Divide, 33, "[null, 1, 3, null, 2]", "[null, 33, 11, null, 16]");
+    // Array divides by scalar
+    this->AssertBinop(Divide, "[null, 10, 30, null, 2]", 3, "[null, 3, 10, null, 0]");
+    // Scalar divides by scalar
+    this->AssertBinop(Divide, 16, 7, 2);
+  }
+}
+
+TYPED_TEST(TestBinaryArithmeticSigned, Div) {
+  // Ordinary arrays
+  this->AssertBinop(Divide, "[-3, 2, -6]", "[1, 1, 2]", "[-3, 2, -3]");
+  // Array with nulls
+  this->AssertBinop(Divide, "[null, 10, 30, null, -20]", "[1, 4, 2, 5, 10]",
+                    "[null, 2, 15, null, -2]");
+  // Scalar divides by array
+  this->AssertBinop(Divide, 33, "[null, -1, -3, null, 2]", "[null, -33, -11, null, 16]");
+  // Array divides by scalar
+  this->AssertBinop(Divide, "[null, 10, 30, null, 2]", 3, "[null, 3, 10, null, 0]");
+  // Scalar divides by scalar
+  this->AssertBinop(Divide, -16, -8, 2);
+}
+
+TYPED_TEST(TestBinaryArithmeticIntegral, DivideByZero) {
+  for (auto check_overflow : {false, true}) {
+    this->SetOverflowCheck(check_overflow);
+    this->AssertBinopRaises(Divide, "[3, 2, 6]", "[1, 1, 0]", "divide by zero");
+  }
+}
+
+TYPED_TEST(TestBinaryArithmeticFloating, DivideByZero) {
+  this->SetOverflowCheck(true);
+  this->AssertBinopRaises(Divide, "[3.0, 2.0, 6.0]", "[1.0, 1.0, 0.0]", "divide by zero");
+  this->AssertBinopRaises(Divide, "[3.0, 2.0, 0.0]", "[1.0, 1.0, 0.0]", "divide by zero");
+  this->AssertBinopRaises(Divide, "[3.0, 2.0, -6.0]", "[1.0, 1.0, 0.0]",
+                          "divide by zero");
+
+  this->SetOverflowCheck(false);
+  this->SetNansEqual(true);
+  this->AssertBinop(Divide, "[3.0, 2.0, 6.0]", "[1.0, 1.0, 0.0]", "[3.0, 2.0, Inf]");
+  this->AssertBinop(Divide, "[3.0, 2.0, 0.0]", "[1.0, 1.0, 0.0]", "[3.0, 2.0, NaN]");
+  this->AssertBinop(Divide, "[3.0, 2.0, -6.0]", "[1.0, 1.0, 0.0]", "[3.0, 2.0, -Inf]");
+}
+
+TYPED_TEST(TestBinaryArithmeticSigned, DivideOverflowRaises) {
+  using CType = typename TestFixture::CType;
+  auto min = std::numeric_limits<CType>::lowest();
+
+  this->SetOverflowCheck(true);
+  this->AssertBinopRaises(Divide, MakeArray(min), MakeArray(-1), "overflow");
+
+  this->SetOverflowCheck(false);
+  this->AssertBinop(Divide, MakeArray(min), MakeArray(-1), "[0]");
+}
+
 TYPED_TEST(TestBinaryArithmeticFloating, Sub) {
   this->AssertBinop(Subtract, "[]", "[]", "[]");
 
diff --git a/cpp/src/arrow/compute/kernels/scalar_boolean.cc b/cpp/src/arrow/compute/kernels/scalar_boolean.cc
index 2f223dfd245..48acf319402 100644
--- a/cpp/src/arrow/compute/kernels/scalar_boolean.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_boolean.cc
@@ -157,9 +157,10 @@ struct Xor {
 };
 
 void MakeFunction(std::string name, int arity, ArrayKernelExec exec,
-                  FunctionRegistry* registry, bool can_write_into_slices = true,
+                  const FunctionDoc* doc, FunctionRegistry* registry,
+                  bool can_write_into_slices = true,
                   NullHandling::type null_handling = NullHandling::INTERSECTION) {
-  auto func = std::make_shared<ScalarFunction>(name, Arity(arity));
+  auto func = std::make_shared<ScalarFunction>(name, Arity(arity), doc);
 
   // Scalar arguments not yet supported
   std::vector<InputType> in_types(arity, InputType::Array(boolean()));
@@ -171,21 +172,70 @@ void MakeFunction(std::string name, int arity, ArrayKernelExec exec,
   DCHECK_OK(registry->AddFunction(std::move(func)));
 }
 
+const FunctionDoc invert_doc{"Invert boolean values", "", {"values"}};
+
+const FunctionDoc and_doc{
+    "Logical 'and' boolean values",
+    ("When a null is encountered in either input, a null is output.\n"
+     "For a different null behavior, see function \"and_kleene\"."),
+    {"x", "y"}};
+
+const FunctionDoc or_doc{
+    "Logical 'or' boolean values",
+    ("When a null is encountered in either input, a null is output.\n"
+     "For a different null behavior, see function \"or_kleene\"."),
+    {"x", "y"}};
+
+const FunctionDoc xor_doc{
+    "Logical 'xor' boolean values",
+    ("When a null is encountered in either input, a null is output."),
+    {"x", "y"}};
+
+const FunctionDoc and_kleene_doc{
+    "Logical 'and' boolean values (Kleene logic)",
+    ("This function behaves as follows with nulls:\n\n"
+     "- true and null = null\n"
+     "- null and true = null\n"
+     "- false and null = false\n"
+     "- null and false = false\n"
+     "- null and null = null\n"
+     "\n"
+     "In other words, in this context a null value really means \"unknown\",\n"
+     "and an unknown value 'and' false is always false.\n"
+     "For a different null behavior, see function \"and\"."),
+    {"x", "y"}};
+
+const FunctionDoc or_kleene_doc{
+    "Logical 'or' boolean values (Kleene logic)",
+    ("This function behaves as follows with nulls:\n\n"
+     "- true or null = true\n"
+     "- null and true = true\n"
+     "- false and null = null\n"
+     "- null and false = null\n"
+     "- null and null = null\n"
+     "\n"
+     "In other words, in this context a null value really means \"unknown\",\n"
+     "and an unknown value 'or' true is always true.\n"
+     "For a different null behavior, see function \"and\"."),
+    {"x", "y"}};
+
 }  // namespace
 
 namespace internal {
 
 void RegisterScalarBoolean(FunctionRegistry* registry) {
   // These functions can write into sliced output bitmaps
-  MakeFunction("invert", 1, applicator::SimpleUnary<Invert>, registry);
-  MakeFunction("and", 2, applicator::SimpleBinary<And>, registry);
-  MakeFunction("or", 2, applicator::SimpleBinary<Or>, registry);
-  MakeFunction("xor", 2, applicator::SimpleBinary<Xor>, registry);
+  MakeFunction("invert", 1, applicator::SimpleUnary<Invert>, &invert_doc, registry);
+  MakeFunction("and", 2, applicator::SimpleBinary<And>, &and_doc, registry);
+  MakeFunction("or", 2, applicator::SimpleBinary<Or>, &or_doc, registry);
+  MakeFunction("xor", 2, applicator::SimpleBinary<Xor>, &xor_doc, registry);
 
   // The Kleene logic kernels cannot write into sliced output bitmaps
-  MakeFunction("and_kleene", 2, applicator::SimpleBinary<KleeneAnd>, registry,
+  MakeFunction("and_kleene", 2, applicator::SimpleBinary<KleeneAnd>, &and_kleene_doc,
+               registry,
                /*can_write_into_slices=*/false, NullHandling::COMPUTED_PREALLOCATE);
-  MakeFunction("or_kleene", 2, applicator::SimpleBinary<KleeneOr>, registry,
+  MakeFunction("or_kleene", 2, applicator::SimpleBinary<KleeneOr>, &or_kleene_doc,
+               registry,
                /*can_write_into_slices=*/false, NullHandling::COMPUTED_PREALLOCATE);
 }
 
diff --git a/cpp/src/arrow/compute/kernels/scalar_cast_boolean.cc b/cpp/src/arrow/compute/kernels/scalar_cast_boolean.cc
index 207dd30cea4..07026db83be 100644
--- a/cpp/src/arrow/compute/kernels/scalar_cast_boolean.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_cast_boolean.cc
@@ -17,6 +17,7 @@
 
 // Cast types to boolean
 
+#include "arrow/array/builder_primitive.h"
 #include "arrow/compute/kernels/common.h"
 #include "arrow/compute/kernels/scalar_cast_internal.h"
 #include "arrow/util/value_parsing.h"
diff --git a/cpp/src/arrow/compute/kernels/scalar_cast_internal.h b/cpp/src/arrow/compute/kernels/scalar_cast_internal.h
index 914b670ca47..333601cf216 100644
--- a/cpp/src/arrow/compute/kernels/scalar_cast_internal.h
+++ b/cpp/src/arrow/compute/kernels/scalar_cast_internal.h
@@ -17,7 +17,6 @@
 
 #pragma once
 
-#include "arrow/builder.h"
 #include "arrow/compute/api_vector.h"
 #include "arrow/compute/cast.h"           // IWYU pragma: export
 #include "arrow/compute/cast_internal.h"  // IWYU pragma: export
diff --git a/cpp/src/arrow/compute/kernels/scalar_cast_nested.cc b/cpp/src/arrow/compute/kernels/scalar_cast_nested.cc
index 85d84efd302..3cef4026fb6 100644
--- a/cpp/src/arrow/compute/kernels/scalar_cast_nested.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_cast_nested.cc
@@ -20,6 +20,7 @@
 #include <utility>
 #include <vector>
 
+#include "arrow/array/builder_nested.h"
 #include "arrow/compute/cast.h"
 #include "arrow/compute/kernels/common.h"
 #include "arrow/compute/kernels/scalar_cast_internal.h"
diff --git a/cpp/src/arrow/compute/kernels/scalar_cast_numeric.cc b/cpp/src/arrow/compute/kernels/scalar_cast_numeric.cc
index 078902a2f7a..51b5609590f 100644
--- a/cpp/src/arrow/compute/kernels/scalar_cast_numeric.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_cast_numeric.cc
@@ -17,6 +17,7 @@
 
 // Implementation of casting to integer, floating point, or decimal types
 
+#include "arrow/array/builder_primitive.h"
 #include "arrow/compute/kernels/common.h"
 #include "arrow/compute/kernels/scalar_cast_internal.h"
 #include "arrow/util/bit_block_counter.h"
diff --git a/cpp/src/arrow/compute/kernels/scalar_cast_string.cc b/cpp/src/arrow/compute/kernels/scalar_cast_string.cc
index 9b6d404ea79..713ec5b73c0 100644
--- a/cpp/src/arrow/compute/kernels/scalar_cast_string.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_cast_string.cc
@@ -18,6 +18,7 @@
 // Implementation of casting to integer or floating point types
 
 #include "arrow/array/array_base.h"
+#include "arrow/array/builder_binary.h"
 #include "arrow/compute/kernels/common.h"
 #include "arrow/compute/kernels/scalar_cast_internal.h"
 #include "arrow/result.h"
diff --git a/cpp/src/arrow/compute/kernels/scalar_cast_temporal.cc b/cpp/src/arrow/compute/kernels/scalar_cast_temporal.cc
index 96f0b874ef6..10b38d75095 100644
--- a/cpp/src/arrow/compute/kernels/scalar_cast_temporal.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_cast_temporal.cc
@@ -19,6 +19,7 @@
 
 #include <limits>
 
+#include "arrow/array/builder_time.h"
 #include "arrow/compute/kernels/common.h"
 #include "arrow/compute/kernels/scalar_cast_internal.h"
 #include "arrow/util/bitmap_reader.h"
@@ -41,7 +42,7 @@ template <typename in_type, typename out_type>
 void ShiftTime(KernelContext* ctx, const util::DivideOrMultiply factor_op,
                const int64_t factor, const ArrayData& input, ArrayData* output) {
   const CastOptions& options = checked_cast<const CastState&>(*ctx->state()).options;
-  const in_type* in_data = input.GetValues<in_type>(1);
+  auto in_data = input.GetValues<in_type>(1);
   auto out_data = output->GetMutableValues<out_type>(1);
 
   if (factor == 1) {
diff --git a/cpp/src/arrow/compute/kernels/scalar_cast_test.cc b/cpp/src/arrow/compute/kernels/scalar_cast_test.cc
index bea9a0ef8dc..c340a8a50be 100644
--- a/cpp/src/arrow/compute/kernels/scalar_cast_test.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_cast_test.cc
@@ -26,10 +26,10 @@
 #include <gtest/gtest.h>
 
 #include "arrow/array.h"
+#include "arrow/array/builder_decimal.h"
 #include "arrow/buffer.h"
 #include "arrow/chunked_array.h"
 #include "arrow/extension_type.h"
-#include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/testing/extension_type.h"
 #include "arrow/testing/gtest_common.h"
@@ -65,7 +65,7 @@ struct TestCType<T, enable_if_base_binary<T>> {
 };
 
 template <typename T>
-struct TestCType<T, enable_if_decimal<T>> {
+struct TestCType<T, enable_if_decimal128<T>> {
   using type = Decimal128;
 };
 
diff --git a/cpp/src/arrow/compute/kernels/scalar_compare.cc b/cpp/src/arrow/compute/kernels/scalar_compare.cc
index 1e44a35a7b8..cf32c888e8e 100644
--- a/cpp/src/arrow/compute/kernels/scalar_compare.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_compare.cc
@@ -73,8 +73,9 @@ void AddGenericCompare(const std::shared_ptr<DataType>& ty, ScalarFunction* func
 }
 
 template <typename Op>
-std::shared_ptr<ScalarFunction> MakeCompareFunction(std::string name) {
-  auto func = std::make_shared<ScalarFunction>(name, Arity::Binary());
+std::shared_ptr<ScalarFunction> MakeCompareFunction(std::string name,
+                                                    const FunctionDoc* doc) {
+  auto func = std::make_shared<ScalarFunction>(name, Arity::Binary(), doc);
 
   DCHECK_OK(func->AddKernel(
       {boolean(), boolean()}, boolean(),
@@ -133,8 +134,9 @@ std::shared_ptr<ScalarFunction> MakeCompareFunction(std::string name) {
 }
 
 std::shared_ptr<ScalarFunction> MakeFlippedFunction(std::string name,
-                                                    const ScalarFunction& func) {
-  auto flipped_func = std::make_shared<ScalarFunction>(name, Arity::Binary());
+                                                    const ScalarFunction& func,
+                                                    const FunctionDoc* doc) {
+  auto flipped_func = std::make_shared<ScalarFunction>(name, Arity::Binary(), doc);
   for (const ScalarKernel* kernel : func.kernels()) {
     ScalarKernel flipped_kernel = *kernel;
     flipped_kernel.exec = MakeFlippedBinaryExec(kernel->exec);
@@ -143,17 +145,45 @@ std::shared_ptr<ScalarFunction> MakeFlippedFunction(std::string name,
   return flipped_func;
 }
 
+const FunctionDoc equal_doc{"Compare values for equality (x == y)",
+                            ("A null on either side emits a null comparison result."),
+                            {"x", "y"}};
+
+const FunctionDoc not_equal_doc{"Compare values for inequality (x != y)",
+                                ("A null on either side emits a null comparison result."),
+                                {"x", "y"}};
+
+const FunctionDoc greater_doc{"Compare values for ordered inequality (x > y)",
+                              ("A null on either side emits a null comparison result."),
+                              {"x", "y"}};
+
+const FunctionDoc greater_equal_doc{
+    "Compare values for ordered inequality (x >= y)",
+    ("A null on either side emits a null comparison result."),
+    {"x", "y"}};
+
+const FunctionDoc less_doc{"Compare values for ordered inequality (x < y)",
+                           ("A null on either side emits a null comparison result."),
+                           {"x", "y"}};
+
+const FunctionDoc less_equal_doc{
+    "Compare values for ordered inequality (x <= y)",
+    ("A null on either side emits a null comparison result."),
+    {"x", "y"}};
+
 }  // namespace
 
 void RegisterScalarComparison(FunctionRegistry* registry) {
-  DCHECK_OK(registry->AddFunction(MakeCompareFunction<Equal>("equal")));
-  DCHECK_OK(registry->AddFunction(MakeCompareFunction<NotEqual>("not_equal")));
+  DCHECK_OK(registry->AddFunction(MakeCompareFunction<Equal>("equal", &equal_doc)));
+  DCHECK_OK(
+      registry->AddFunction(MakeCompareFunction<NotEqual>("not_equal", &not_equal_doc)));
 
-  auto greater = MakeCompareFunction<Greater>("greater");
-  auto greater_equal = MakeCompareFunction<GreaterEqual>("greater_equal");
+  auto greater = MakeCompareFunction<Greater>("greater", &greater_doc);
+  auto greater_equal =
+      MakeCompareFunction<GreaterEqual>("greater_equal", &greater_equal_doc);
 
-  auto less = MakeFlippedFunction("less", *greater);
-  auto less_equal = MakeFlippedFunction("less_equal", *greater_equal);
+  auto less = MakeFlippedFunction("less", *greater, &less_doc);
+  auto less_equal = MakeFlippedFunction("less_equal", *greater_equal, &less_equal_doc);
   DCHECK_OK(registry->AddFunction(std::move(less)));
   DCHECK_OK(registry->AddFunction(std::move(less_equal)));
   DCHECK_OK(registry->AddFunction(std::move(greater)));
diff --git a/cpp/src/arrow/compute/kernels/scalar_fill_null.cc b/cpp/src/arrow/compute/kernels/scalar_fill_null.cc
index e6b454c664e..e856c85f610 100644
--- a/cpp/src/arrow/compute/kernels/scalar_fill_null.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_fill_null.cc
@@ -153,6 +153,13 @@ void AddBasicFillNullKernels(ScalarKernel kernel, ScalarFunction* func) {
   AddKernels({boolean(), null()});
 }
 
+const FunctionDoc fill_null_doc{
+    "Replace null elements",
+    ("`fill_value` must be a scalar of the same type as `values`.\n"
+     "Each non-null value in `values` is emitted as-is.\n"
+     "Each null value in `values` is replaced with `fill_value`."),
+    {"values", "fill_value"}};
+
 }  // namespace
 
 void RegisterScalarFillNull(FunctionRegistry* registry) {
@@ -160,7 +167,8 @@ void RegisterScalarFillNull(FunctionRegistry* registry) {
     ScalarKernel fill_null_base;
     fill_null_base.null_handling = NullHandling::COMPUTED_NO_PREALLOCATE;
     fill_null_base.mem_allocation = MemAllocation::NO_PREALLOCATE;
-    auto fill_null = std::make_shared<ScalarFunction>("fill_null", Arity::Binary());
+    auto fill_null =
+        std::make_shared<ScalarFunction>("fill_null", Arity::Binary(), &fill_null_doc);
     AddBasicFillNullKernels(fill_null_base, fill_null.get());
     DCHECK_OK(registry->AddFunction(fill_null));
   }
diff --git a/cpp/src/arrow/compute/kernels/scalar_nested.cc b/cpp/src/arrow/compute/kernels/scalar_nested.cc
index 201d501b3f9..a0b35738c70 100644
--- a/cpp/src/arrow/compute/kernels/scalar_nested.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_nested.cc
@@ -52,11 +52,18 @@ void ListValueLength(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
   }
 }
 
+const FunctionDoc list_value_length_doc{
+    "Compute list lengths",
+    ("`lists` must have a list-like type.\n"
+     "For each non-null value in `lists`, its length is emitted.\n"
+     "Null values emit a null in the output."),
+    {"lists"}};
+
 }  // namespace
 
 void RegisterScalarNested(FunctionRegistry* registry) {
-  auto list_value_length =
-      std::make_shared<ScalarFunction>("list_value_length", Arity::Unary());
+  auto list_value_length = std::make_shared<ScalarFunction>(
+      "list_value_length", Arity::Unary(), &list_value_length_doc);
   DCHECK_OK(list_value_length->AddKernel({InputType(Type::LIST)}, int32(),
                                          ListValueLength<ListType>));
   DCHECK_OK(list_value_length->AddKernel({InputType(Type::LARGE_LIST)}, int64(),
diff --git a/cpp/src/arrow/compute/kernels/scalar_set_lookup.cc b/cpp/src/arrow/compute/kernels/scalar_set_lookup.cc
index 726b01fb477..d43083c7538 100644
--- a/cpp/src/arrow/compute/kernels/scalar_set_lookup.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_set_lookup.cc
@@ -399,12 +399,15 @@ void AddBasicSetLookupKernels(ScalarKernel kernel,
 // Enables calling is_in with CallFunction as though it were binary.
 class IsInMetaBinary : public MetaFunction {
  public:
-  IsInMetaBinary() : MetaFunction("is_in_meta_binary", Arity::Binary()) {}
+  IsInMetaBinary()
+      : MetaFunction("is_in_meta_binary", Arity::Binary(), /*doc=*/nullptr) {}
 
   Result<Datum> ExecuteImpl(const std::vector<Datum>& args,
                             const FunctionOptions* options,
                             ExecContext* ctx) const override {
-    DCHECK_EQ(options, nullptr);
+    if (options != nullptr) {
+      return Status::Invalid("Unexpected options for 'is_in_meta_binary' function");
+    }
     return IsIn(args[0], args[1], ctx);
   }
 };
@@ -412,16 +415,35 @@ class IsInMetaBinary : public MetaFunction {
 // Enables calling index_in with CallFunction as though it were binary.
 class IndexInMetaBinary : public MetaFunction {
  public:
-  IndexInMetaBinary() : MetaFunction("index_in_meta_binary", Arity::Binary()) {}
+  IndexInMetaBinary()
+      : MetaFunction("index_in_meta_binary", Arity::Binary(), /*doc=*/nullptr) {}
 
   Result<Datum> ExecuteImpl(const std::vector<Datum>& args,
                             const FunctionOptions* options,
                             ExecContext* ctx) const override {
-    DCHECK_EQ(options, nullptr);
+    if (options != nullptr) {
+      return Status::Invalid("Unexpected options for 'index_in_meta_binary' function");
+    }
     return IndexIn(args[0], args[1], ctx);
   }
 };
 
+const FunctionDoc is_in_doc{
+    "Find each element in a set of values",
+    ("For each element in `values`, return true if it is found in a given\n"
+     "set of values.  The set of values to look for must be given in\n"
+     "SetLookupOptions."),
+    {"values"},
+    "SetLookupOptions"};
+
+const FunctionDoc index_in_doc{
+    "Return index of each element in a set of values",
+    ("For each element in `values`, return its index in a given set of\n"
+     "values, or null if it is not found there.\n"
+     "The set of values to look for must be given in SetLookupOptions."),
+    {"values"},
+    "SetLookupOptions"};
+
 }  // namespace
 
 void RegisterScalarSetLookup(FunctionRegistry* registry) {
@@ -430,7 +452,7 @@ void RegisterScalarSetLookup(FunctionRegistry* registry) {
     ScalarKernel isin_base;
     isin_base.init = InitSetLookup;
     isin_base.exec = ExecIsIn;
-    auto is_in = std::make_shared<ScalarFunction>("is_in", Arity::Unary());
+    auto is_in = std::make_shared<ScalarFunction>("is_in", Arity::Unary(), &is_in_doc);
 
     AddBasicSetLookupKernels(isin_base, /*output_type=*/boolean(), is_in.get());
 
@@ -444,17 +466,19 @@ void RegisterScalarSetLookup(FunctionRegistry* registry) {
 
   // IndexIn uses Int32Builder and so is responsible for all its own allocation
   {
-    ScalarKernel match_base;
-    match_base.init = InitSetLookup;
-    match_base.exec = ExecIndexIn;
-    match_base.null_handling = NullHandling::COMPUTED_NO_PREALLOCATE;
-    match_base.mem_allocation = MemAllocation::NO_PREALLOCATE;
-    auto match = std::make_shared<ScalarFunction>("index_in", Arity::Unary());
-    AddBasicSetLookupKernels(match_base, /*output_type=*/int32(), match.get());
-
-    match_base.signature = KernelSignature::Make({null()}, int32());
-    DCHECK_OK(match->AddKernel(match_base));
-    DCHECK_OK(registry->AddFunction(match));
+    ScalarKernel index_in_base;
+    index_in_base.init = InitSetLookup;
+    index_in_base.exec = ExecIndexIn;
+    index_in_base.null_handling = NullHandling::COMPUTED_NO_PREALLOCATE;
+    index_in_base.mem_allocation = MemAllocation::NO_PREALLOCATE;
+    auto index_in =
+        std::make_shared<ScalarFunction>("index_in", Arity::Unary(), &index_in_doc);
+
+    AddBasicSetLookupKernels(index_in_base, /*output_type=*/int32(), index_in.get());
+
+    index_in_base.signature = KernelSignature::Make({null()}, int32());
+    DCHECK_OK(index_in->AddKernel(index_in_base));
+    DCHECK_OK(registry->AddFunction(index_in));
 
     DCHECK_OK(registry->AddFunction(std::make_shared<IndexInMetaBinary>()));
   }
diff --git a/cpp/src/arrow/compute/kernels/scalar_set_lookup_benchmark.cc b/cpp/src/arrow/compute/kernels/scalar_set_lookup_benchmark.cc
new file mode 100644
index 00000000000..636be8ad504
--- /dev/null
+++ b/cpp/src/arrow/compute/kernels/scalar_set_lookup_benchmark.cc
@@ -0,0 +1,138 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "benchmark/benchmark.h"
+
+#include "arrow/compute/api_scalar.h"
+#include "arrow/compute/kernels/common.h"
+#include "arrow/compute/kernels/test_util.h"
+#include "arrow/testing/gtest_util.h"
+#include "arrow/testing/random.h"
+#include "arrow/util/benchmark_util.h"
+
+namespace arrow {
+namespace compute {
+
+constexpr auto kSeed = 0x94378165;
+
+static void SetLookupBenchmarkString(benchmark::State& state,
+                                     const std::string& func_name,
+                                     const int64_t value_set_length) {
+  const int64_t array_length = 1 << 20;
+  const int64_t value_min_size = 0;
+  const int64_t value_max_size = 32;
+  const double null_probability = 0.01;
+  random::RandomArrayGenerator rng(kSeed);
+
+  auto values =
+      rng.String(array_length, value_min_size, value_max_size, null_probability);
+  auto value_set =
+      rng.String(value_set_length, value_min_size, value_max_size, null_probability);
+  ABORT_NOT_OK(CallFunction(func_name, {values, value_set}));
+  for (auto _ : state) {
+    ABORT_NOT_OK(CallFunction(func_name, {values, value_set}));
+  }
+  state.SetItemsProcessed(state.iterations() * array_length);
+  state.SetBytesProcessed(state.iterations() * values->data()->buffers[2]->size());
+}
+
+template <typename Type>
+static void SetLookupBenchmarkNumeric(benchmark::State& state,
+                                      const std::string& func_name,
+                                      const int64_t value_set_length) {
+  const int64_t array_length = 1 << 20;
+  const int64_t value_min = 0;
+  const int64_t value_max = std::numeric_limits<typename Type::c_type>::max();
+  const double null_probability = 0.01;
+  random::RandomArrayGenerator rng(kSeed);
+
+  auto values = rng.Numeric<Type>(array_length, value_min, value_max, null_probability);
+  auto value_set =
+      rng.Numeric<Type>(value_set_length, value_min, value_max, null_probability);
+  ABORT_NOT_OK(CallFunction(func_name, {values, value_set}));
+  for (auto _ : state) {
+    ABORT_NOT_OK(CallFunction(func_name, {values, value_set}));
+  }
+  state.SetItemsProcessed(state.iterations() * array_length);
+  state.SetBytesProcessed(state.iterations() * values->data()->buffers[1]->size());
+}
+
+static void IndexInStringSmallSet(benchmark::State& state) {
+  SetLookupBenchmarkString(state, "index_in_meta_binary", state.range(0));
+}
+
+static void IsInStringSmallSet(benchmark::State& state) {
+  SetLookupBenchmarkString(state, "is_in_meta_binary", state.range(0));
+}
+
+static void IndexInStringLargeSet(benchmark::State& state) {
+  SetLookupBenchmarkString(state, "index_in_meta_binary", 1 << 10);
+}
+
+static void IsInStringLargeSet(benchmark::State& state) {
+  SetLookupBenchmarkString(state, "is_in_meta_binary", 1 << 10);
+}
+
+static void IndexInInt8SmallSet(benchmark::State& state) {
+  SetLookupBenchmarkNumeric<Int8Type>(state, "index_in_meta_binary", state.range(0));
+}
+
+static void IndexInInt16SmallSet(benchmark::State& state) {
+  SetLookupBenchmarkNumeric<Int16Type>(state, "index_in_meta_binary", state.range(0));
+}
+
+static void IndexInInt32SmallSet(benchmark::State& state) {
+  SetLookupBenchmarkNumeric<Int32Type>(state, "index_in_meta_binary", state.range(0));
+}
+
+static void IndexInInt64SmallSet(benchmark::State& state) {
+  SetLookupBenchmarkNumeric<Int64Type>(state, "index_in_meta_binary", state.range(0));
+}
+
+static void IsInInt8SmallSet(benchmark::State& state) {
+  SetLookupBenchmarkNumeric<Int8Type>(state, "is_in_meta_binary", state.range(0));
+}
+
+static void IsInInt16SmallSet(benchmark::State& state) {
+  SetLookupBenchmarkNumeric<Int16Type>(state, "is_in_meta_binary", state.range(0));
+}
+
+static void IsInInt32SmallSet(benchmark::State& state) {
+  SetLookupBenchmarkNumeric<Int32Type>(state, "is_in_meta_binary", state.range(0));
+}
+
+static void IsInInt64SmallSet(benchmark::State& state) {
+  SetLookupBenchmarkNumeric<Int64Type>(state, "is_in_meta_binary", state.range(0));
+}
+
+BENCHMARK(IndexInStringSmallSet)->RangeMultiplier(4)->Range(2, 64);
+BENCHMARK(IsInStringSmallSet)->RangeMultiplier(4)->Range(2, 64);
+
+BENCHMARK(IndexInStringLargeSet);
+BENCHMARK(IsInStringLargeSet);
+
+BENCHMARK(IndexInInt8SmallSet)->RangeMultiplier(4)->Range(2, 64);
+BENCHMARK(IndexInInt16SmallSet)->RangeMultiplier(4)->Range(2, 64);
+BENCHMARK(IndexInInt32SmallSet)->RangeMultiplier(4)->Range(2, 64);
+BENCHMARK(IndexInInt64SmallSet)->RangeMultiplier(4)->Range(2, 64);
+BENCHMARK(IsInInt8SmallSet)->RangeMultiplier(4)->Range(2, 64);
+BENCHMARK(IsInInt16SmallSet)->RangeMultiplier(4)->Range(2, 64);
+BENCHMARK(IsInInt32SmallSet)->RangeMultiplier(4)->Range(2, 64);
+BENCHMARK(IsInInt64SmallSet)->RangeMultiplier(4)->Range(2, 64);
+
+}  // namespace compute
+}  // namespace arrow
diff --git a/cpp/src/arrow/compute/kernels/scalar_set_lookup_test.cc b/cpp/src/arrow/compute/kernels/scalar_set_lookup_test.cc
index bd1a481eefe..9fae29b8fa6 100644
--- a/cpp/src/arrow/compute/kernels/scalar_set_lookup_test.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_set_lookup_test.cc
@@ -30,6 +30,7 @@
 
 #include "arrow/array/array_base.h"
 #include "arrow/array/builder_binary.h"
+#include "arrow/array/builder_decimal.h"
 #include "arrow/array/builder_primitive.h"
 #include "arrow/chunked_array.h"
 #include "arrow/compute/api.h"
diff --git a/cpp/src/arrow/compute/kernels/scalar_string.cc b/cpp/src/arrow/compute/kernels/scalar_string.cc
index 7e616176444..3f83a9f7a41 100644
--- a/cpp/src/arrow/compute/kernels/scalar_string.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_string.cc
@@ -23,9 +23,11 @@
 #include <utf8proc.h>
 #endif
 
+#include "arrow/array/builder_binary.h"
+#include "arrow/array/builder_nested.h"
+#include "arrow/buffer_builder.h"
 #include "arrow/compute/api_scalar.h"
 #include "arrow/compute/kernels/common.h"
-#include "arrow/compute/kernels/scalar_string_internal.h"
 #include "arrow/util/utf8.h"
 #include "arrow/util/value_parsing.h"
 
@@ -129,13 +131,9 @@ struct UTF8Transform {
                              ctx->Allocate(output_ncodeunits_max));
       output->buffers[2] = values_buffer;
 
-      // We could reuse the indices if the data is all ascii, benchmarking showed this
-      // not to matter.
-      //   output->buffers[1] = input.buffers[1];
-      KERNEL_ASSIGN_OR_RAISE(output->buffers[1], ctx,
-                             ctx->Allocate((input_nstrings + 1) * sizeof(offset_type)));
-      uint8_t* output_str = output->buffers[2]->mutable_data();
+      // String offsets are preallocated
       offset_type* output_string_offsets = output->GetMutableValues<offset_type>(1);
+      uint8_t* output_str = output->buffers[2]->mutable_data();
       offset_type output_ncodeunits = 0;
 
       output_string_offsets[0] = 0;
@@ -198,7 +196,7 @@ struct UTF8Upper : UTF8Transform<Type, UTF8Upper<Type>> {
 
 template <typename Type>
 struct UTF8Lower : UTF8Transform<Type, UTF8Lower<Type>> {
-  static uint32_t TransformCodepoint(uint32_t codepoint) {
+  inline static uint32_t TransformCodepoint(uint32_t codepoint) {
     return codepoint <= kMaxCodepointLookup ? lut_lower_codepoint[codepoint]
                                             : utf8proc_tolower(codepoint);
   }
@@ -392,8 +390,15 @@ struct MatchSubstring {
   }
 };
 
+const FunctionDoc match_substring_doc(
+    "Match strings against literal pattern",
+    ("For each string in `strings`, emit true iff it contains a given pattern.\n"
+     "Null inputs emit null.  The pattern must be given in MatchSubstringOptions."),
+    {"strings"}, "MatchSubstringOptions");
+
 void AddMatchSubstring(FunctionRegistry* registry) {
-  auto func = std::make_shared<ScalarFunction>("match_substring", Arity::Unary());
+  auto func = std::make_shared<ScalarFunction>("match_substring", Arity::Unary(),
+                                               &match_substring_doc);
   auto exec_32 = MatchSubstring<StringType>::Exec;
   auto exec_64 = MatchSubstring<LargeStringType>::Exec;
   DCHECK_OK(func->AddKernel({utf8()}, boolean(), exec_32, MatchSubstringState::Init));
@@ -809,6 +814,386 @@ struct IsUpperAscii : CharacterPredicateAscii<IsUpperAscii> {
   }
 };
 
+// splitting
+
+template <typename Type, typename ListType, typename Options, typename Derived>
+struct SplitBaseTransform {
+  using string_offset_type = typename Type::offset_type;
+  using list_offset_type = typename ListType::offset_type;
+  using ArrayType = typename TypeTraits<Type>::ArrayType;
+  using ArrayListType = typename TypeTraits<ListType>::ArrayType;
+  using ListScalarType = typename TypeTraits<ListType>::ScalarType;
+  using ScalarType = typename TypeTraits<Type>::ScalarType;
+  using BuilderType = typename TypeTraits<Type>::BuilderType;
+  using ListOffsetsBuilderType = TypedBufferBuilder<list_offset_type>;
+  using State = OptionsWrapper<Options>;
+
+  std::vector<util::string_view> parts;
+  Options options;
+
+  explicit SplitBaseTransform(Options options) : options(options) {}
+
+  Status Split(const util::string_view& s, BuilderType* builder) {
+    const uint8_t* begin = reinterpret_cast<const uint8_t*>(s.data());
+    const uint8_t* end = begin + s.length();
+
+    int64_t max_splits = options.max_splits;
+    // if there is no max splits, reversing does not make sense (and is probably less
+    // efficient), but is useful for testing
+    if (options.reverse) {
+      // note that i points 1 further than the 'current'
+      const uint8_t* i = end;
+      // we will record the parts in reverse order
+      parts.clear();
+      if (max_splits > -1) {
+        parts.reserve(max_splits + 1);
+      }
+      while (max_splits != 0) {
+        const uint8_t *separator_begin, *separator_end;
+        // find with whatever algo the part we will 'cut out'
+        if (static_cast<Derived&>(*this).FindReverse(begin, i, &separator_begin,
+                                                     &separator_end, options)) {
+          parts.emplace_back(reinterpret_cast<const char*>(separator_end),
+                             i - separator_end);
+          i = separator_begin;
+          max_splits--;
+        } else {
+          // if we cannot find a separator, we're done
+          break;
+        }
+      }
+      parts.emplace_back(reinterpret_cast<const char*>(begin), i - begin);
+      // now we do the copying
+      for (auto it = parts.rbegin(); it != parts.rend(); ++it) {
+        RETURN_NOT_OK(builder->Append(*it));
+      }
+    } else {
+      const uint8_t* i = begin;
+      while (max_splits != 0) {
+        const uint8_t *separator_begin, *separator_end;
+        // find with whatever algo the part we will 'cut out'
+        if (static_cast<Derived&>(*this).Find(i, end, &separator_begin, &separator_end,
+                                              options)) {
+          // the part till the beginning of the 'cut'
+          RETURN_NOT_OK(
+              builder->Append(i, static_cast<string_offset_type>(separator_begin - i)));
+          i = separator_end;
+          max_splits--;
+        } else {
+          // if we cannot find a separator, we're done
+          break;
+        }
+      }
+      // trailing part
+      RETURN_NOT_OK(builder->Append(i, static_cast<string_offset_type>(end - i)));
+    }
+    return Status::OK();
+  }
+
+  static Status CheckOptions(const Options& options) { return Status::OK(); }
+
+  static void Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+    Options options = State::Get(ctx);
+    Derived splitter(options);  // we make an instance to reuse the parts vectors
+    splitter.Split(ctx, batch, out);
+  }
+
+  void Split(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+    EnsureLookupTablesFilled();  // only needed for unicode
+    KERNEL_RETURN_IF_ERROR(ctx, Derived::CheckOptions(options));
+
+    if (batch[0].kind() == Datum::ARRAY) {
+      const ArrayData& input = *batch[0].array();
+      ArrayType input_boxed(batch[0].array());
+
+      BuilderType builder(input.type, ctx->memory_pool());
+      // a slight overestimate of the data needed
+      KERNEL_RETURN_IF_ERROR(ctx, builder.ReserveData(input_boxed.total_values_length()));
+      // the minimum amount of strings needed
+      KERNEL_RETURN_IF_ERROR(ctx, builder.Resize(input.length));
+
+      ArrayData* output_list = out->mutable_array();
+      // list offsets were preallocated
+      auto* list_offsets = output_list->GetMutableValues<list_offset_type>(1);
+      DCHECK_NE(list_offsets, nullptr);
+      // initial value
+      *list_offsets++ = 0;
+      KERNEL_RETURN_IF_ERROR(
+          ctx,
+          VisitArrayDataInline<Type>(
+              input,
+              [&](util::string_view s) {
+                RETURN_NOT_OK(Split(s, &builder));
+                if (ARROW_PREDICT_FALSE(builder.length() >
+                                        std::numeric_limits<list_offset_type>::max())) {
+                  return Status::CapacityError("List offset does not fit into 32 bit");
+                }
+                *list_offsets++ = static_cast<list_offset_type>(builder.length());
+                return Status::OK();
+              },
+              [&]() {
+                // null value is already taken from input
+                *list_offsets++ = static_cast<list_offset_type>(builder.length());
+                return Status::OK();
+              }));
+      // assign list child data
+      std::shared_ptr<Array> string_array;
+      KERNEL_RETURN_IF_ERROR(ctx, builder.Finish(&string_array));
+      output_list->child_data.push_back(string_array->data());
+
+    } else {
+      const auto& input = checked_cast<const ScalarType&>(*batch[0].scalar());
+      auto result = checked_pointer_cast<ListScalarType>(MakeNullScalar(out->type()));
+      if (input.is_valid) {
+        result->is_valid = true;
+        BuilderType builder(input.type, ctx->memory_pool());
+        util::string_view s = static_cast<util::string_view>(*input.value);
+        KERNEL_RETURN_IF_ERROR(ctx, Split(s, &builder));
+        KERNEL_RETURN_IF_ERROR(ctx, builder.Finish(&result->value));
+      }
+      out->value = result;
+    }
+  }
+};
+
+template <typename Type, typename ListType>
+struct SplitPatternTransform : SplitBaseTransform<Type, ListType, SplitPatternOptions,
+                                                  SplitPatternTransform<Type, ListType>> {
+  using Base = SplitBaseTransform<Type, ListType, SplitPatternOptions,
+                                  SplitPatternTransform<Type, ListType>>;
+  using ArrayType = typename TypeTraits<Type>::ArrayType;
+  using ScalarType = typename TypeTraits<Type>::ScalarType;
+  using string_offset_type = typename Type::offset_type;
+  using Base::Base;
+
+  static Status CheckOptions(const SplitPatternOptions& options) {
+    if (options.pattern.length() == 0) {
+      return Status::Invalid("Empty separator");
+    }
+    return Status::OK();
+  }
+
+  static bool Find(const uint8_t* begin, const uint8_t* end,
+                   const uint8_t** separator_begin, const uint8_t** separator_end,
+                   const SplitPatternOptions& options) {
+    const uint8_t* pattern = reinterpret_cast<const uint8_t*>(options.pattern.c_str());
+    const int64_t pattern_length = options.pattern.length();
+    const uint8_t* i = begin;
+    // this is O(n*m) complexity, we could use the Knuth-Morris-Pratt algorithm used in
+    // the match kernel
+    while ((i + pattern_length <= end)) {
+      i = std::search(i, end, pattern, pattern + pattern_length);
+      if (i != end) {
+        *separator_begin = i;
+        *separator_end = i + pattern_length;
+        return true;
+      }
+    }
+    return false;
+  }
+
+  static bool FindReverse(const uint8_t* begin, const uint8_t* end,
+                          const uint8_t** separator_begin, const uint8_t** separator_end,
+                          const SplitPatternOptions& options) {
+    const uint8_t* pattern = reinterpret_cast<const uint8_t*>(options.pattern.c_str());
+    const int64_t pattern_length = options.pattern.length();
+    // this is O(n*m) complexity, we could use the Knuth-Morris-Pratt algorithm used in
+    // the match kernel
+    std::reverse_iterator<const uint8_t*> ri(end);
+    std::reverse_iterator<const uint8_t*> rend(begin);
+    std::reverse_iterator<const uint8_t*> pattern_rbegin(pattern + pattern_length);
+    std::reverse_iterator<const uint8_t*> pattern_rend(pattern);
+    while (begin <= ri.base() - pattern_length) {
+      ri = std::search(ri, rend, pattern_rbegin, pattern_rend);
+      if (ri != rend) {
+        *separator_begin = ri.base() - pattern_length;
+        *separator_end = ri.base();
+        return true;
+      }
+    }
+    return false;
+  }
+};
+
+const FunctionDoc split_pattern_doc(
+    "Split string according to separator",
+    ("Split each string according to the exact `pattern` defined in\n"
+     "SplitPatternOptions.  The output for each string input is a list\n"
+     "of strings.\n"
+     "\n"
+     "The maximum number of splits and direction of splitting\n"
+     "(forward, reverse) can optionally be defined in SplitPatternOptions."),
+    {"strings"}, "SplitPatternOptions");
+
+const FunctionDoc ascii_split_whitespace_doc(
+    "Split string according to any ASCII whitespace",
+    ("Split each string according any non-zero length sequence of ASCII\n"
+     "whitespace characters.  The output for each string input is a list\n"
+     "of strings.\n"
+     "\n"
+     "The maximum number of splits and direction of splitting\n"
+     "(forward, reverse) can optionally be defined in SplitOptions."),
+    {"strings"}, "SplitOptions");
+
+const FunctionDoc utf8_split_whitespace_doc(
+    "Split string according to any Unicode whitespace",
+    ("Split each string according any non-zero length sequence of Unicode\n"
+     "whitespace characters.  The output for each string input is a list\n"
+     "of strings.\n"
+     "\n"
+     "The maximum number of splits and direction of splitting\n"
+     "(forward, reverse) can optionally be defined in SplitOptions."),
+    {"strings"}, "SplitOptions");
+
+void AddSplitPattern(FunctionRegistry* registry) {
+  auto func = std::make_shared<ScalarFunction>("split_pattern", Arity::Unary(),
+                                               &split_pattern_doc);
+  using t32 = SplitPatternTransform<StringType, ListType>;
+  using t64 = SplitPatternTransform<LargeStringType, ListType>;
+  DCHECK_OK(func->AddKernel({utf8()}, {list(utf8())}, t32::Exec, t32::State::Init));
+  DCHECK_OK(
+      func->AddKernel({large_utf8()}, {list(large_utf8())}, t64::Exec, t64::State::Init));
+  DCHECK_OK(registry->AddFunction(std::move(func)));
+}
+
+template <typename Type, typename ListType>
+struct SplitWhitespaceAsciiTransform
+    : SplitBaseTransform<Type, ListType, SplitOptions,
+                         SplitWhitespaceAsciiTransform<Type, ListType>> {
+  using Base = SplitBaseTransform<Type, ListType, SplitOptions,
+                                  SplitWhitespaceAsciiTransform<Type, ListType>>;
+  using ArrayType = typename TypeTraits<Type>::ArrayType;
+  using ScalarType = typename TypeTraits<Type>::ScalarType;
+  using string_offset_type = typename Type::offset_type;
+  using Base::Base;
+  static bool Find(const uint8_t* begin, const uint8_t* end,
+                   const uint8_t** separator_begin, const uint8_t** separator_end,
+                   const SplitOptions& options) {
+    const uint8_t* i = begin;
+    while ((i < end)) {
+      if (IsSpaceCharacterAscii(*i)) {
+        *separator_begin = i;
+        do {
+          i++;
+        } while (IsSpaceCharacterAscii(*i) && i < end);
+        *separator_end = i;
+        return true;
+      }
+      i++;
+    }
+    return false;
+  }
+  static bool FindReverse(const uint8_t* begin, const uint8_t* end,
+                          const uint8_t** separator_begin, const uint8_t** separator_end,
+                          const SplitOptions& options) {
+    const uint8_t* i = end - 1;
+    while ((i >= begin)) {
+      if (IsSpaceCharacterAscii(*i)) {
+        *separator_end = i + 1;
+        do {
+          i--;
+        } while (IsSpaceCharacterAscii(*i) && i >= begin);
+        *separator_begin = i + 1;
+        return true;
+      }
+      i--;
+    }
+    return false;
+  }
+};
+
+void AddSplitWhitespaceAscii(FunctionRegistry* registry) {
+  static const SplitOptions default_options{};
+  auto func =
+      std::make_shared<ScalarFunction>("ascii_split_whitespace", Arity::Unary(),
+                                       &ascii_split_whitespace_doc, &default_options);
+  using t32 = SplitWhitespaceAsciiTransform<StringType, ListType>;
+  using t64 = SplitWhitespaceAsciiTransform<LargeStringType, ListType>;
+  DCHECK_OK(func->AddKernel({utf8()}, {list(utf8())}, t32::Exec, t32::State::Init));
+  DCHECK_OK(
+      func->AddKernel({large_utf8()}, {list(large_utf8())}, t64::Exec, t64::State::Init));
+  DCHECK_OK(registry->AddFunction(std::move(func)));
+}
+
+#ifdef ARROW_WITH_UTF8PROC
+template <typename Type, typename ListType>
+struct SplitWhitespaceUtf8Transform
+    : SplitBaseTransform<Type, ListType, SplitOptions,
+                         SplitWhitespaceUtf8Transform<Type, ListType>> {
+  using Base = SplitBaseTransform<Type, ListType, SplitOptions,
+                                  SplitWhitespaceUtf8Transform<Type, ListType>>;
+  using ArrayType = typename TypeTraits<Type>::ArrayType;
+  using string_offset_type = typename Type::offset_type;
+  using ScalarType = typename TypeTraits<Type>::ScalarType;
+  using Base::Base;
+  static bool Find(const uint8_t* begin, const uint8_t* end,
+                   const uint8_t** separator_begin, const uint8_t** separator_end,
+                   const SplitOptions& options) {
+    const uint8_t* i = begin;
+    while ((i < end)) {
+      uint32_t codepoint = 0;
+      *separator_begin = i;
+      if (ARROW_PREDICT_FALSE(!arrow::util::UTF8Decode(&i, &codepoint))) {
+        return false;
+      }
+      if (IsSpaceCharacterUnicode(codepoint)) {
+        do {
+          *separator_end = i;
+          if (ARROW_PREDICT_FALSE(!arrow::util::UTF8Decode(&i, &codepoint))) {
+            return false;
+          }
+        } while (IsSpaceCharacterUnicode(codepoint) && i < end);
+        return true;
+      }
+    }
+    return false;
+  }
+  static bool FindReverse(const uint8_t* begin, const uint8_t* end,
+                          const uint8_t** separator_begin, const uint8_t** separator_end,
+                          const SplitOptions& options) {
+    const uint8_t* i = end - 1;
+    while ((i >= begin)) {
+      uint32_t codepoint = 0;
+      *separator_end = i + 1;
+      if (ARROW_PREDICT_FALSE(!arrow::util::UTF8DecodeReverse(&i, &codepoint))) {
+        return false;
+      }
+      if (IsSpaceCharacterUnicode(codepoint)) {
+        do {
+          *separator_begin = i + 1;
+          if (ARROW_PREDICT_FALSE(!arrow::util::UTF8DecodeReverse(&i, &codepoint))) {
+            return false;
+          }
+        } while (IsSpaceCharacterUnicode(codepoint) && i >= begin);
+        return true;
+      }
+    }
+    return false;
+  }
+};
+
+void AddSplitWhitespaceUTF8(FunctionRegistry* registry) {
+  static const SplitOptions default_options{};
+  auto func =
+      std::make_shared<ScalarFunction>("utf8_split_whitespace", Arity::Unary(),
+                                       &utf8_split_whitespace_doc, &default_options);
+  using t32 = SplitWhitespaceUtf8Transform<StringType, ListType>;
+  using t64 = SplitWhitespaceUtf8Transform<LargeStringType, ListType>;
+  DCHECK_OK(func->AddKernel({utf8()}, {list(utf8())}, t32::Exec, t32::State::Init));
+  DCHECK_OK(
+      func->AddKernel({large_utf8()}, {list(large_utf8())}, t64::Exec, t64::State::Init));
+  DCHECK_OK(registry->AddFunction(std::move(func)));
+}
+#endif
+
+void AddSplit(FunctionRegistry* registry) {
+  AddSplitPattern(registry);
+  AddSplitWhitespaceAscii(registry);
+#ifdef ARROW_WITH_UTF8PROC
+  AddSplitWhitespaceUTF8(registry);
+#endif
+}
+
 // ----------------------------------------------------------------------
 // strptime string parsing
 
@@ -846,8 +1231,21 @@ Result<ValueDescr> StrptimeResolve(KernelContext* ctx, const std::vector<ValueDe
   return Status::Invalid("strptime does not provide default StrptimeOptions");
 }
 
+const FunctionDoc strptime_doc(
+    "Parse timestamps",
+    ("For each string in `strings`, parse it as a timestamp.\n"
+     "The timestamp unit and the expected string pattern must be given\n"
+     "in StrptimeOptions.  Null inputs emit null.  If a non-null string\n"
+     "fails parsing, an error is returned."),
+    {"strings"}, "StrptimeOptions");
+
+const FunctionDoc binary_length_doc(
+    "Compute string lengths",
+    ("For each string in `strings`, emit its length.  Null values emit null."),
+    {"strings"});
+
 void AddStrptime(FunctionRegistry* registry) {
-  auto func = std::make_shared<ScalarFunction>("strptime", Arity::Unary());
+  auto func = std::make_shared<ScalarFunction>("strptime", Arity::Unary(), &strptime_doc);
   DCHECK_OK(func->AddKernel({utf8()}, OutputType(StrptimeResolve),
                             StrptimeExec<StringType>, StrptimeState::Init));
   DCHECK_OK(func->AddKernel({large_utf8()}, OutputType(StrptimeResolve),
@@ -856,7 +1254,8 @@ void AddStrptime(FunctionRegistry* registry) {
 }
 
 void AddBinaryLength(FunctionRegistry* registry) {
-  auto func = std::make_shared<ScalarFunction>("binary_length", Arity::Unary());
+  auto func = std::make_shared<ScalarFunction>("binary_length", Arity::Unary(),
+                                               &binary_length_doc);
   ArrayKernelExec exec_offset_32 =
       applicator::ScalarUnaryNotNull<Int32Type, StringType, BinaryLength>::Exec;
   ArrayKernelExec exec_offset_64 =
@@ -871,20 +1270,31 @@ void AddBinaryLength(FunctionRegistry* registry) {
 }
 
 template <template <typename> class ExecFunctor>
-void MakeUnaryStringBatchKernel(std::string name, FunctionRegistry* registry) {
-  auto func = std::make_shared<ScalarFunction>(name, Arity::Unary());
-  auto exec_32 = ExecFunctor<StringType>::Exec;
-  auto exec_64 = ExecFunctor<LargeStringType>::Exec;
-  DCHECK_OK(func->AddKernel({utf8()}, utf8(), exec_32));
-  DCHECK_OK(func->AddKernel({large_utf8()}, large_utf8(), exec_64));
+void MakeUnaryStringBatchKernel(
+    std::string name, FunctionRegistry* registry, const FunctionDoc* doc,
+    MemAllocation::type mem_allocation = MemAllocation::PREALLOCATE) {
+  auto func = std::make_shared<ScalarFunction>(name, Arity::Unary(), doc);
+  {
+    auto exec_32 = ExecFunctor<StringType>::Exec;
+    ScalarKernel kernel{{utf8()}, utf8(), exec_32};
+    kernel.mem_allocation = mem_allocation;
+    DCHECK_OK(func->AddKernel(std::move(kernel)));
+  }
+  {
+    auto exec_64 = ExecFunctor<LargeStringType>::Exec;
+    ScalarKernel kernel{{large_utf8()}, large_utf8(), exec_64};
+    kernel.mem_allocation = mem_allocation;
+    DCHECK_OK(func->AddKernel(std::move(kernel)));
+  }
   DCHECK_OK(registry->AddFunction(std::move(func)));
 }
 
 #ifdef ARROW_WITH_UTF8PROC
 
 template <template <typename> class Transformer>
-void MakeUnaryStringUTF8TransformKernel(std::string name, FunctionRegistry* registry) {
-  auto func = std::make_shared<ScalarFunction>(name, Arity::Unary());
+void MakeUnaryStringUTF8TransformKernel(std::string name, FunctionRegistry* registry,
+                                        const FunctionDoc* doc) {
+  auto func = std::make_shared<ScalarFunction>(name, Arity::Unary(), doc);
   ArrayKernelExec exec_32 = Transformer<StringType>::Exec;
   ArrayKernelExec exec_64 = Transformer<LargeStringType>::Exec;
   DCHECK_OK(func->AddKernel({utf8()}, utf8(), exec_32));
@@ -925,8 +1335,9 @@ void ApplyPredicate(KernelContext* ctx, const ExecBatch& batch, StringPredicate
 }
 
 template <typename Predicate>
-void AddUnaryStringPredicate(std::string name, FunctionRegistry* registry) {
-  auto func = std::make_shared<ScalarFunction>(name, Arity::Unary());
+void AddUnaryStringPredicate(std::string name, FunctionRegistry* registry,
+                             const FunctionDoc* doc) {
+  auto func = std::make_shared<ScalarFunction>(name, Arity::Unary(), doc);
   auto exec_32 = [](KernelContext* ctx, const ExecBatch& batch, Datum* out) {
     ApplyPredicate<StringType>(ctx, batch, Predicate::Call, out);
   };
@@ -938,41 +1349,153 @@ void AddUnaryStringPredicate(std::string name, FunctionRegistry* registry) {
   DCHECK_OK(registry->AddFunction(std::move(func)));
 }
 
-}  // namespace
+FunctionDoc StringPredicateDoc(std::string summary, std::string description) {
+  return FunctionDoc{std::move(summary), std::move(description), {"strings"}};
+}
 
-void RegisterScalarStringAscii(FunctionRegistry* registry) {
-  MakeUnaryStringBatchKernel<AsciiUpper>("ascii_upper", registry);
-  MakeUnaryStringBatchKernel<AsciiLower>("ascii_lower", registry);
+FunctionDoc StringClassifyDoc(std::string class_summary, std::string class_desc,
+                              bool non_empty) {
+  std::string summary, description;
+  {
+    std::stringstream ss;
+    ss << "Classify strings as " << class_summary;
+    summary = ss.str();
+  }
+  {
+    std::stringstream ss;
+    if (non_empty) {
+      ss
+          << ("For each string in `strings`, emit true iff the string is non-empty\n"
+              "and consists only of ");
+    } else {
+      ss
+          << ("For each string in `strings`, emit true iff the string consists only\n"
+              "of ");
+    }
+    ss << class_desc << ".  Null strings emit null.";
+    description = ss.str();
+  }
+  return StringPredicateDoc(std::move(summary), std::move(description));
+}
 
-  AddUnaryStringPredicate<IsAscii>("string_is_ascii", registry);
+const auto string_is_ascii_doc = StringClassifyDoc("ASCII", "ASCII characters", false);
+
+const auto ascii_is_alnum_doc =
+    StringClassifyDoc("ASCII alphanumeric", "alphanumeric ASCII characters", true);
+const auto ascii_is_alpha_doc =
+    StringClassifyDoc("ASCII alphabetic", "alphabetic ASCII characters", true);
+const auto ascii_is_decimal_doc =
+    StringClassifyDoc("ASCII decimal", "decimal ASCII characters", true);
+const auto ascii_is_lower_doc =
+    StringClassifyDoc("ASCII lowercase", "lowercase ASCII characters", true);
+const auto ascii_is_printable_doc =
+    StringClassifyDoc("ASCII printable", "printable ASCII characters", true);
+const auto ascii_is_space_doc =
+    StringClassifyDoc("ASCII whitespace", "whitespace ASCII characters", true);
+const auto ascii_is_upper_doc =
+    StringClassifyDoc("ASCII uppercase", "uppercase ASCII characters", true);
+
+const auto ascii_is_title_doc = StringPredicateDoc(
+    "Classify strings as ASCII titlecase",
+    ("For each string in `strings`, emit true iff the string is title-cased,\n"
+     "i.e. it has at least one cased character, each uppercase character\n"
+     "follows a non-cased character, and each lowercase character follows\n"
+     "an uppercase character.\n"));
+
+const auto utf8_is_alnum_doc =
+    StringClassifyDoc("alphanumeric", "alphanumeric Unicode characters", true);
+const auto utf8_is_alpha_doc =
+    StringClassifyDoc("alphabetic", "alphabetic Unicode characters", true);
+const auto utf8_is_decimal_doc =
+    StringClassifyDoc("decimal", "decimal Unicode characters", true);
+const auto utf8_is_digit_doc = StringClassifyDoc("digits", "Unicode digits", true);
+const auto utf8_is_lower_doc =
+    StringClassifyDoc("lowercase", "lowercase Unicode characters", true);
+const auto utf8_is_numeric_doc =
+    StringClassifyDoc("numeric", "numeric Unicode characters", true);
+const auto utf8_is_printable_doc =
+    StringClassifyDoc("printable", "printable Unicode characters", true);
+const auto utf8_is_space_doc =
+    StringClassifyDoc("whitespace", "whitespace Unicode characters", true);
+const auto utf8_is_upper_doc =
+    StringClassifyDoc("uppercase", "uppercase Unicode characters", true);
+
+const auto utf8_is_title_doc = StringPredicateDoc(
+    "Classify strings as titlecase",
+    ("For each string in `strings`, emit true iff the string is title-cased,\n"
+     "i.e. it has at least one cased character, each uppercase character\n"
+     "follows a non-cased character, and each lowercase character follows\n"
+     "an uppercase character.\n"));
+
+const FunctionDoc ascii_upper_doc(
+    "Transform ASCII input to uppercase",
+    ("For each string in `strings`, return an uppercase version.\n\n"
+     "This function assumes the input is fully ASCII.  It it may contain\n"
+     "non-ASCII characters, use \"utf8_upper\" instead."),
+    {"strings"});
+
+const FunctionDoc ascii_lower_doc(
+    "Transform ASCII input to lowercase",
+    ("For each string in `strings`, return a lowercase version.\n\n"
+     "This function assumes the input is fully ASCII.  It it may contain\n"
+     "non-ASCII characters, use \"utf8_lower\" instead."),
+    {"strings"});
+
+const FunctionDoc utf8_upper_doc(
+    "Transform input to uppercase",
+    ("For each string in `strings`, return an uppercase version."), {"strings"});
+
+const FunctionDoc utf8_lower_doc(
+    "Transform input to lowercase",
+    ("For each string in `strings`, return a lowercase version."), {"strings"});
 
-  AddUnaryStringPredicate<IsAlphaNumericAscii>("ascii_is_alnum", registry);
-  AddUnaryStringPredicate<IsAlphaAscii>("ascii_is_alpha", registry);
-  AddUnaryStringPredicate<IsDecimalAscii>("ascii_is_decimal", registry);
+}  // namespace
+
+void RegisterScalarStringAscii(FunctionRegistry* registry) {
+  // ascii_upper and ascii_lower are able to reuse the original offsets buffer,
+  // so don't preallocate them in the output.
+  MakeUnaryStringBatchKernel<AsciiUpper>("ascii_upper", registry, &ascii_upper_doc,
+                                         MemAllocation::NO_PREALLOCATE);
+  MakeUnaryStringBatchKernel<AsciiLower>("ascii_lower", registry, &ascii_lower_doc,
+                                         MemAllocation::NO_PREALLOCATE);
+
+  AddUnaryStringPredicate<IsAscii>("string_is_ascii", registry, &string_is_ascii_doc);
+
+  AddUnaryStringPredicate<IsAlphaNumericAscii>("ascii_is_alnum", registry,
+                                               &ascii_is_alnum_doc);
+  AddUnaryStringPredicate<IsAlphaAscii>("ascii_is_alpha", registry, &ascii_is_alpha_doc);
+  AddUnaryStringPredicate<IsDecimalAscii>("ascii_is_decimal", registry,
+                                          &ascii_is_decimal_doc);
   // no is_digit for ascii, since it is the same as is_decimal
-  AddUnaryStringPredicate<IsLowerAscii>("ascii_is_lower", registry);
+  AddUnaryStringPredicate<IsLowerAscii>("ascii_is_lower", registry, &ascii_is_lower_doc);
   // no is_numeric for ascii, since it is the same as is_decimal
-  AddUnaryStringPredicate<IsPrintableAscii>("ascii_is_printable", registry);
-  AddUnaryStringPredicate<IsSpaceAscii>("ascii_is_space", registry);
-  AddUnaryStringPredicate<IsTitleAscii>("ascii_is_title", registry);
-  AddUnaryStringPredicate<IsUpperAscii>("ascii_is_upper", registry);
+  AddUnaryStringPredicate<IsPrintableAscii>("ascii_is_printable", registry,
+                                            &ascii_is_printable_doc);
+  AddUnaryStringPredicate<IsSpaceAscii>("ascii_is_space", registry, &ascii_is_space_doc);
+  AddUnaryStringPredicate<IsTitleAscii>("ascii_is_title", registry, &ascii_is_title_doc);
+  AddUnaryStringPredicate<IsUpperAscii>("ascii_is_upper", registry, &ascii_is_upper_doc);
 
 #ifdef ARROW_WITH_UTF8PROC
-  MakeUnaryStringUTF8TransformKernel<UTF8Upper>("utf8_upper", registry);
-  MakeUnaryStringUTF8TransformKernel<UTF8Lower>("utf8_lower", registry);
-
-  AddUnaryStringPredicate<IsAlphaNumericUnicode>("utf8_is_alnum", registry);
-  AddUnaryStringPredicate<IsAlphaUnicode>("utf8_is_alpha", registry);
-  AddUnaryStringPredicate<IsDecimalUnicode>("utf8_is_decimal", registry);
-  AddUnaryStringPredicate<IsDigitUnicode>("utf8_is_digit", registry);
-  AddUnaryStringPredicate<IsLowerUnicode>("utf8_is_lower", registry);
-  AddUnaryStringPredicate<IsNumericUnicode>("utf8_is_numeric", registry);
-  AddUnaryStringPredicate<IsPrintableUnicode>("utf8_is_printable", registry);
-  AddUnaryStringPredicate<IsSpaceUnicode>("utf8_is_space", registry);
-  AddUnaryStringPredicate<IsTitleUnicode>("utf8_is_title", registry);
-  AddUnaryStringPredicate<IsUpperUnicode>("utf8_is_upper", registry);
+  MakeUnaryStringUTF8TransformKernel<UTF8Upper>("utf8_upper", registry, &utf8_upper_doc);
+  MakeUnaryStringUTF8TransformKernel<UTF8Lower>("utf8_lower", registry, &utf8_lower_doc);
+
+  AddUnaryStringPredicate<IsAlphaNumericUnicode>("utf8_is_alnum", registry,
+                                                 &utf8_is_alnum_doc);
+  AddUnaryStringPredicate<IsAlphaUnicode>("utf8_is_alpha", registry, &utf8_is_alpha_doc);
+  AddUnaryStringPredicate<IsDecimalUnicode>("utf8_is_decimal", registry,
+                                            &utf8_is_decimal_doc);
+  AddUnaryStringPredicate<IsDigitUnicode>("utf8_is_digit", registry, &utf8_is_digit_doc);
+  AddUnaryStringPredicate<IsLowerUnicode>("utf8_is_lower", registry, &utf8_is_lower_doc);
+  AddUnaryStringPredicate<IsNumericUnicode>("utf8_is_numeric", registry,
+                                            &utf8_is_numeric_doc);
+  AddUnaryStringPredicate<IsPrintableUnicode>("utf8_is_printable", registry,
+                                              &utf8_is_printable_doc);
+  AddUnaryStringPredicate<IsSpaceUnicode>("utf8_is_space", registry, &utf8_is_space_doc);
+  AddUnaryStringPredicate<IsTitleUnicode>("utf8_is_title", registry, &utf8_is_title_doc);
+  AddUnaryStringPredicate<IsUpperUnicode>("utf8_is_upper", registry, &utf8_is_upper_doc);
 #endif
 
+  AddSplit(registry);
   AddBinaryLength(registry);
   AddMatchSubstring(registry);
   AddStrptime(registry);
diff --git a/cpp/src/arrow/compute/kernels/scalar_string_benchmark.cc b/cpp/src/arrow/compute/kernels/scalar_string_benchmark.cc
index 0ad7f724c5f..4b77cf07bcf 100644
--- a/cpp/src/arrow/compute/kernels/scalar_string_benchmark.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_string_benchmark.cc
@@ -66,6 +66,11 @@ static void MatchSubstring(benchmark::State& state) {
   UnaryStringBenchmark(state, "match_substring", &options);
 }
 
+static void SplitPattern(benchmark::State& state) {
+  SplitPatternOptions options("a");
+  UnaryStringBenchmark(state, "split_pattern", &options);
+}
+
 #ifdef ARROW_WITH_UTF8PROC
 static void Utf8Upper(benchmark::State& state) {
   UnaryStringBenchmark(state, "utf8_upper");
@@ -84,6 +89,7 @@ BENCHMARK(AsciiLower);
 BENCHMARK(AsciiUpper);
 BENCHMARK(IsAlphaNumericAscii);
 BENCHMARK(MatchSubstring);
+BENCHMARK(SplitPattern);
 #ifdef ARROW_WITH_UTF8PROC
 BENCHMARK(Utf8Lower);
 BENCHMARK(Utf8Upper);
diff --git a/cpp/src/arrow/compute/kernels/scalar_string_test.cc b/cpp/src/arrow/compute/kernels/scalar_string_test.cc
index a96716ad39c..c76b50ee176 100644
--- a/cpp/src/arrow/compute/kernels/scalar_string_test.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_string_test.cc
@@ -334,6 +334,88 @@ TYPED_TEST(TestStringKernels, MatchSubstring) {
                    &options_double_char_2);
 }
 
+TYPED_TEST(TestStringKernels, SplitBasics) {
+  SplitPatternOptions options{" "};
+  // basics
+  this->CheckUnary("split_pattern", R"(["foo bar", "foo"])", list(this->type()),
+                   R"([["foo", "bar"], ["foo"]])", &options);
+  this->CheckUnary("split_pattern", R"(["foo bar", "foo", null])", list(this->type()),
+                   R"([["foo", "bar"], ["foo"], null])", &options);
+  // edgy cases
+  this->CheckUnary("split_pattern", R"(["f  o o "])", list(this->type()),
+                   R"([["f", "", "o", "o", ""]])", &options);
+  this->CheckUnary("split_pattern", "[]", list(this->type()), "[]", &options);
+  // longer patterns
+  SplitPatternOptions options_long{"---"};
+  this->CheckUnary("split_pattern", R"(["-foo---bar--", "---foo---b"])",
+                   list(this->type()), R"([["-foo", "bar--"], ["", "foo", "b"]])",
+                   &options_long);
+  SplitPatternOptions options_long_reverse{"---", -1, /*reverse=*/true};
+  this->CheckUnary("split_pattern", R"(["-foo---bar--", "---foo---b"])",
+                   list(this->type()), R"([["-foo", "bar--"], ["", "foo", "b"]])",
+                   &options_long_reverse);
+}
+
+TYPED_TEST(TestStringKernels, SplitMax) {
+  SplitPatternOptions options{"---", 2};
+  SplitPatternOptions options_reverse{"---", 2, /*reverse=*/true};
+  this->CheckUnary("split_pattern", R"(["foo---bar", "foo", "foo---bar------ar"])",
+                   list(this->type()),
+                   R"([["foo", "bar"], ["foo"], ["foo", "bar", "---ar"]])", &options);
+  this->CheckUnary(
+      "split_pattern", R"(["foo---bar", "foo", "foo---bar------ar"])", list(this->type()),
+      R"([["foo", "bar"], ["foo"], ["foo---bar", "", "ar"]])", &options_reverse);
+}
+
+TYPED_TEST(TestStringKernels, SplitWhitespaceAscii) {
+  SplitOptions options;
+  SplitOptions options_max{1};
+  // basics
+  this->CheckUnary("ascii_split_whitespace", R"(["foo bar", "foo  bar \tba"])",
+                   list(this->type()), R"([["foo", "bar"], ["foo", "bar", "ba"]])",
+                   &options);
+  this->CheckUnary("ascii_split_whitespace", R"(["foo bar", "foo  bar \tba"])",
+                   list(this->type()), R"([["foo", "bar"], ["foo", "bar \tba"]])",
+                   &options_max);
+}
+
+TYPED_TEST(TestStringKernels, SplitWhitespaceAsciiReverse) {
+  SplitOptions options{-1, /*reverse=*/true};
+  SplitOptions options_max{1, /*reverse=*/true};
+  // basics
+  this->CheckUnary("ascii_split_whitespace", R"(["foo bar", "foo  bar \tba"])",
+                   list(this->type()), R"([["foo", "bar"], ["foo", "bar", "ba"]])",
+                   &options);
+  this->CheckUnary("ascii_split_whitespace", R"(["foo bar", "foo  bar \tba"])",
+                   list(this->type()), R"([["foo", "bar"], ["foo  bar", "ba"]])",
+                   &options_max);
+}
+
+TYPED_TEST(TestStringKernels, SplitWhitespaceUTF8) {
+  SplitOptions options;
+  SplitOptions options_max{1};
+  // \xe2\x80\x88 is punctuation space
+  this->CheckUnary("utf8_split_whitespace",
+                   "[\"foo bar\", \"foo\xe2\x80\x88  bar \\tba\"]", list(this->type()),
+                   R"([["foo", "bar"], ["foo", "bar", "ba"]])", &options);
+  this->CheckUnary("utf8_split_whitespace",
+                   "[\"foo bar\", \"foo\xe2\x80\x88  bar \\tba\"]", list(this->type()),
+                   R"([["foo", "bar"], ["foo", "bar \tba"]])", &options_max);
+}
+
+TYPED_TEST(TestStringKernels, SplitWhitespaceUTF8Reverse) {
+  SplitOptions options{-1, /*reverse=*/true};
+  SplitOptions options_max{1, /*reverse=*/true};
+  // \xe2\x80\x88 is punctuation space
+  this->CheckUnary("utf8_split_whitespace",
+                   "[\"foo bar\", \"foo\xe2\x80\x88  bar \\tba\"]", list(this->type()),
+                   R"([["foo", "bar"], ["foo", "bar", "ba"]])", &options);
+  this->CheckUnary("utf8_split_whitespace",
+                   "[\"foo bar\", \"foo\xe2\x80\x88  bar \\tba\"]", list(this->type()),
+                   "[[\"foo\", \"bar\"], [\"foo\xe2\x80\x88  bar\", \"ba\"]]",
+                   &options_max);
+}
+
 TYPED_TEST(TestStringKernels, Strptime) {
   std::string input1 = R"(["5/1/2020", null, "12/11/1900"])";
   std::string output1 = R"(["2020-05-01", null, "1900-12-11"])";
diff --git a/cpp/src/arrow/compute/kernels/scalar_validity.cc b/cpp/src/arrow/compute/kernels/scalar_validity.cc
index 5d98d1a3d5a..03702316ac5 100644
--- a/cpp/src/arrow/compute/kernels/scalar_validity.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_validity.cc
@@ -74,11 +74,12 @@ struct IsNullOperator {
   }
 };
 
-void MakeFunction(std::string name, std::vector<InputType> in_types, OutputType out_type,
+void MakeFunction(std::string name, const FunctionDoc* doc,
+                  std::vector<InputType> in_types, OutputType out_type,
                   ArrayKernelExec exec, FunctionRegistry* registry,
                   MemAllocation::type mem_allocation, bool can_write_into_slices) {
   Arity arity{static_cast<int>(in_types.size())};
-  auto func = std::make_shared<ScalarFunction>(name, arity);
+  auto func = std::make_shared<ScalarFunction>(name, arity, doc);
 
   ScalarKernel kernel(std::move(in_types), out_type, exec);
   kernel.null_handling = NullHandling::OUTPUT_NOT_NULL;
@@ -123,14 +124,22 @@ void IsNullExec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
   }
 }
 
+const FunctionDoc is_valid_doc(
+    "Return true if non-null",
+    ("For each input value, emit true iff the value is valid (non-null)."), {"values"});
+
+const FunctionDoc is_null_doc("Return true if null",
+                              ("For each input value, emit true iff the value is null."),
+                              {"values"});
+
 }  // namespace
 
 void RegisterScalarValidity(FunctionRegistry* registry) {
-  MakeFunction("is_valid", {ValueDescr::ANY}, boolean(), IsValidExec, registry,
-               MemAllocation::NO_PREALLOCATE, /*can_write_into_slices=*/false);
+  MakeFunction("is_valid", &is_valid_doc, {ValueDescr::ANY}, boolean(), IsValidExec,
+               registry, MemAllocation::NO_PREALLOCATE, /*can_write_into_slices=*/false);
 
-  MakeFunction("is_null", {ValueDescr::ANY}, boolean(), IsNullExec, registry,
-               MemAllocation::PREALLOCATE,
+  MakeFunction("is_null", &is_null_doc, {ValueDescr::ANY}, boolean(), IsNullExec,
+               registry, MemAllocation::PREALLOCATE,
                /*can_write_into_slices=*/true);
 }
 
diff --git a/cpp/src/arrow/compute/kernels/vector_hash.cc b/cpp/src/arrow/compute/kernels/vector_hash.cc
index f8d1cff8767..0009fe53346 100644
--- a/cpp/src/arrow/compute/kernels/vector_hash.cc
+++ b/cpp/src/arrow/compute/kernels/vector_hash.cc
@@ -498,7 +498,8 @@ KernelInit GetHashInit(Type::type type_id) {
     case Type::LARGE_STRING:
       return HashInit<LargeBinaryType, Action>;
     case Type::FIXED_SIZE_BINARY:
-    case Type::DECIMAL:
+    case Type::DECIMAL128:
+    case Type::DECIMAL256:
       return HashInit<FixedSizeBinaryType, Action>;
     default:
       DCHECK(false);
@@ -624,6 +625,22 @@ void AddHashKernels(VectorFunction* func, VectorKernel base, OutputType out_ty)
   DCHECK_OK(func->AddKernel(base));
 }
 
+const FunctionDoc unique_doc(
+    "Compute unique elements",
+    ("Return an array with distinct values.  Nulls in the input are ignored."),
+    {"array"});
+
+const FunctionDoc value_counts_doc(
+    "Compute counts of unique elements",
+    ("For each distinct value, compute the number of times it occurs in the array.\n"
+     "The result is returned as an array of `struct<input type, int64>`.\n"
+     "Nulls in the input are ignored."),
+    {"array"});
+
+const FunctionDoc dictionary_encode_doc(
+    "Dictionary-encode array",
+    ("Return a dictionary-encoded version of the input array."), {"array"});
+
 }  // namespace
 
 void RegisterVectorHash(FunctionRegistry* registry) {
@@ -635,7 +652,7 @@ void RegisterVectorHash(FunctionRegistry* registry) {
 
   base.finalize = UniqueFinalize;
   base.output_chunked = false;
-  auto unique = std::make_shared<VectorFunction>("unique", Arity::Unary());
+  auto unique = std::make_shared<VectorFunction>("unique", Arity::Unary(), &unique_doc);
   AddHashKernels<UniqueAction>(unique.get(), base, OutputType(FirstType));
 
   // Dictionary unique
@@ -651,7 +668,8 @@ void RegisterVectorHash(FunctionRegistry* registry) {
   // value_counts
 
   base.finalize = ValueCountsFinalize;
-  auto value_counts = std::make_shared<VectorFunction>("value_counts", Arity::Unary());
+  auto value_counts =
+      std::make_shared<VectorFunction>("value_counts", Arity::Unary(), &value_counts_doc);
   AddHashKernels<ValueCountsAction>(value_counts.get(), base,
                                     OutputType(ValueCountsOutput));
 
@@ -670,8 +688,8 @@ void RegisterVectorHash(FunctionRegistry* registry) {
   base.finalize = DictEncodeFinalize;
   // Unique and ValueCounts output unchunked arrays
   base.output_chunked = true;
-  auto dict_encode =
-      std::make_shared<VectorFunction>("dictionary_encode", Arity::Unary());
+  auto dict_encode = std::make_shared<VectorFunction>("dictionary_encode", Arity::Unary(),
+                                                      &dictionary_encode_doc);
   AddHashKernels<DictEncodeAction>(dict_encode.get(), base, OutputType(DictEncodeOutput));
 
   // Calling dictionary_encode on dictionary input not supported, but if it
diff --git a/cpp/src/arrow/compute/kernels/vector_hash_benchmark.cc b/cpp/src/arrow/compute/kernels/vector_hash_benchmark.cc
index 764895e6b62..3be549d05ce 100644
--- a/cpp/src/arrow/compute/kernels/vector_hash_benchmark.cc
+++ b/cpp/src/arrow/compute/kernels/vector_hash_benchmark.cc
@@ -19,7 +19,7 @@
 
 #include <vector>
 
-#include "arrow/builder.h"
+#include "arrow/array/builder_binary.h"
 #include "arrow/memory_pool.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/testing/random.h"
diff --git a/cpp/src/arrow/compute/kernels/vector_hash_test.cc b/cpp/src/arrow/compute/kernels/vector_hash_test.cc
index 10562e96951..e9ae4a64d97 100644
--- a/cpp/src/arrow/compute/kernels/vector_hash_test.cc
+++ b/cpp/src/arrow/compute/kernels/vector_hash_test.cc
@@ -29,13 +29,14 @@
 #include <gtest/gtest.h>
 
 #include "arrow/array.h"
+#include "arrow/array/builder_decimal.h"
 #include "arrow/buffer.h"
 #include "arrow/chunked_array.h"
-#include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/testing/gtest_common.h"
 #include "arrow/testing/util.h"
 #include "arrow/type.h"
+#include "arrow/type_fwd.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/checked_cast.h"
 #include "arrow/util/decimal.h"
diff --git a/cpp/src/arrow/compute/kernels/vector_nested.cc b/cpp/src/arrow/compute/kernels/vector_nested.cc
index eeb58d3824e..b7317e5bea0 100644
--- a/cpp/src/arrow/compute/kernels/vector_nested.cc
+++ b/cpp/src/arrow/compute/kernels/vector_nested.cc
@@ -65,18 +65,33 @@ Result<ValueDescr> ValuesType(KernelContext*, const std::vector<ValueDescr>& arg
   return ValueDescr::Array(list_type.value_type());
 }
 
+const FunctionDoc list_flatten_doc(
+    "Flatten list values",
+    ("`lists` must have a list-like type.\n"
+     "Return an array with the top list level flattened.\n"
+     "Top-level null values in `lists` do not emit anything in the input."),
+    {"lists"});
+
+const FunctionDoc list_parent_indices_doc(
+    "Compute parent indices of nested list values",
+    ("`lists` must have a list-like type.\n"
+     "For each value in each list of `lists`, the top-level list index\n"
+     "is emitted."),
+    {"lists"});
+
 }  // namespace
 
 void RegisterVectorNested(FunctionRegistry* registry) {
-  auto flatten = std::make_shared<VectorFunction>("list_flatten", Arity::Unary());
+  auto flatten =
+      std::make_shared<VectorFunction>("list_flatten", Arity::Unary(), &list_flatten_doc);
   DCHECK_OK(flatten->AddKernel({InputType::Array(Type::LIST)}, OutputType(ValuesType),
                                ListFlatten<ListType>));
   DCHECK_OK(flatten->AddKernel({InputType::Array(Type::LARGE_LIST)},
                                OutputType(ValuesType), ListFlatten<LargeListType>));
   DCHECK_OK(registry->AddFunction(std::move(flatten)));
 
-  auto list_parent_indices =
-      std::make_shared<VectorFunction>("list_parent_indices", Arity::Unary());
+  auto list_parent_indices = std::make_shared<VectorFunction>(
+      "list_parent_indices", Arity::Unary(), &list_parent_indices_doc);
   DCHECK_OK(list_parent_indices->AddKernel({InputType::Array(Type::LIST)}, int32(),
                                            ListParentIndices<ListType>));
   DCHECK_OK(list_parent_indices->AddKernel({InputType::Array(Type::LARGE_LIST)}, int64(),
diff --git a/cpp/src/arrow/compute/kernels/vector_selection.cc b/cpp/src/arrow/compute/kernels/vector_selection.cc
index 1967ce72795..1062b53c8c3 100644
--- a/cpp/src/arrow/compute/kernels/vector_selection.cc
+++ b/cpp/src/arrow/compute/kernels/vector_selection.cc
@@ -87,6 +87,8 @@ int64_t GetFilterOutputSize(const ArrayData& filter,
   return output_size;
 }
 
+namespace {
+
 template <typename IndexType>
 Result<std::shared_ptr<ArrayData>> GetTakeIndicesImpl(
     const ArrayData& filter, FilterOptions::NullSelectionBehavior null_selection,
@@ -214,6 +216,8 @@ Result<std::shared_ptr<ArrayData>> GetTakeIndicesImpl(
   return result;
 }
 
+}  // namespace
+
 Result<std::shared_ptr<ArrayData>> GetTakeIndices(
     const ArrayData& filter, FilterOptions::NullSelectionBehavior null_selection,
     MemoryPool* memory_pool) {
@@ -1171,7 +1175,9 @@ void NullTake(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
   if (TakeState::Get(ctx).boundscheck) {
     KERNEL_RETURN_IF_ERROR(ctx, CheckIndexBounds(*batch[1].array(), batch[0].length()));
   }
-  out->value = std::make_shared<NullArray>(batch.length)->data();
+  // batch.length doesn't take into account the take indices
+  auto new_length = batch[1].array()->length;
+  out->value = std::make_shared<NullArray>(new_length)->data();
 }
 
 void NullFilter(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
@@ -1787,7 +1793,8 @@ void StructFilter(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
   std::shared_ptr<ArrayData> indices;
   KERNEL_RETURN_IF_ERROR(
       ctx,
-      GetTakeIndices(*batch[1].array(), FilterState::Get(ctx).null_selection_behavior)
+      GetTakeIndices(*batch[1].array(), FilterState::Get(ctx).null_selection_behavior,
+                     ctx->memory_pool())
           .Value(&indices));
 
   Datum result;
@@ -1814,7 +1821,8 @@ Result<std::shared_ptr<RecordBatch>> FilterRecordBatch(const RecordBatch& batch,
   const auto& filter_opts = *static_cast<const FilterOptions*>(options);
   ARROW_ASSIGN_OR_RAISE(
       std::shared_ptr<ArrayData> indices,
-      GetTakeIndices(*filter.array(), filter_opts.null_selection_behavior));
+      GetTakeIndices(*filter.array(), filter_opts.null_selection_behavior,
+                     ctx->memory_pool()));
   std::vector<std::shared_ptr<Array>> columns(batch.num_columns());
   for (int i = 0; i < batch.num_columns(); ++i) {
     ARROW_ASSIGN_OR_RAISE(Datum out, Take(batch.column(i)->data(), Datum(indices),
@@ -1847,10 +1855,17 @@ Result<std::shared_ptr<Table>> FilterTable(const Table& table, const Datum& filt
 
 static auto kDefaultFilterOptions = FilterOptions::Defaults();
 
+const FunctionDoc filter_doc(
+    "Filter with a boolean selection filter",
+    ("The output is populated with values from the input at positions\n"
+     "where the selection filter is non-zero.  Nulls in the selection filter\n"
+     "are handled based on FilterOptions."),
+    {"input", "selection_filter"}, "FilterOptions");
+
 class FilterMetaFunction : public MetaFunction {
  public:
   FilterMetaFunction()
-      : MetaFunction("filter", Arity::Binary(), &kDefaultFilterOptions) {}
+      : MetaFunction("filter", Arity::Binary(), &filter_doc, &kDefaultFilterOptions) {}
 
   Result<Datum> ExecuteImpl(const std::vector<Datum>& args,
                             const FunctionOptions* options,
@@ -1985,6 +2000,12 @@ Result<std::shared_ptr<Table>> TakeTC(const Table& table, const ChunkedArray& in
 
 static auto kDefaultTakeOptions = TakeOptions::Defaults();
 
+const FunctionDoc take_doc(
+    "Select values from an input based on indices from another array",
+    ("The output is populated with values from the input at positions\n"
+     "given by `indices`.  Nulls in `indices` emit null in the output."),
+    {"input", "indices"}, "TakeOptions");
+
 // Metafunction for dispatching to different Take implementations other than
 // Array-Array.
 //
@@ -1992,7 +2013,8 @@ static auto kDefaultTakeOptions = TakeOptions::Defaults();
 // overly complex dispatching, there is no parallelization.
 class TakeMetaFunction : public MetaFunction {
  public:
-  TakeMetaFunction() : MetaFunction("take", Arity::Binary(), &kDefaultTakeOptions) {}
+  TakeMetaFunction()
+      : MetaFunction("take", Arity::Binary(), &take_doc, &kDefaultTakeOptions) {}
 
   Result<Datum> ExecuteImpl(const std::vector<Datum>& args,
                             const FunctionOptions* options,
@@ -2062,12 +2084,13 @@ struct SelectionKernelDescr {
   ArrayKernelExec exec;
 };
 
-void RegisterSelectionFunction(const std::string& name, VectorKernel base_kernel,
-                               InputType selection_type,
+void RegisterSelectionFunction(const std::string& name, const FunctionDoc* doc,
+                               VectorKernel base_kernel, InputType selection_type,
                                const std::vector<SelectionKernelDescr>& descrs,
                                const FunctionOptions* default_options,
                                FunctionRegistry* registry) {
-  auto func = std::make_shared<VectorFunction>(name, Arity::Binary(), default_options);
+  auto func =
+      std::make_shared<VectorFunction>(name, Arity::Binary(), doc, default_options);
   for (auto& descr : descrs) {
     base_kernel.signature = KernelSignature::Make(
         {std::move(descr.input), selection_type}, OutputType(FirstType));
@@ -2077,6 +2100,19 @@ void RegisterSelectionFunction(const std::string& name, VectorKernel base_kernel
   DCHECK_OK(registry->AddFunction(std::move(func)));
 }
 
+const FunctionDoc array_filter_doc(
+    "Filter with a boolean selection filter",
+    ("The output is populated with values from the input `array` at positions\n"
+     "where the selection filter is non-zero.  Nulls in the selection filter\n"
+     "are handled based on FilterOptions."),
+    {"array", "selection_filter"}, "FilterOptions");
+
+const FunctionDoc array_take_doc(
+    "Select values from an array based on indices from another array",
+    ("The output is populated with values from the input array at positions\n"
+     "given by `indices`.  Nulls in `indices` emit null in the output."),
+    {"array", "indices"}, "TakeOptions");
+
 }  // namespace
 
 void RegisterVectorSelection(FunctionRegistry* registry) {
@@ -2100,7 +2136,7 @@ void RegisterVectorSelection(FunctionRegistry* registry) {
 
   VectorKernel filter_base;
   filter_base.init = FilterState::Init;
-  RegisterSelectionFunction("array_filter", filter_base,
+  RegisterSelectionFunction("array_filter", &array_filter_doc, filter_base,
                             /*selection_type=*/InputType::Array(boolean()),
                             filter_kernel_descrs, &kDefaultFilterOptions, registry);
 
@@ -2130,7 +2166,7 @@ void RegisterVectorSelection(FunctionRegistry* registry) {
   take_base.init = TakeState::Init;
   take_base.can_execute_chunkwise = false;
   RegisterSelectionFunction(
-      "array_take", take_base,
+      "array_take", &array_take_doc, take_base,
       /*selection_type=*/InputType(match::Integer(), ValueDescr::ARRAY),
       take_kernel_descrs, &kDefaultTakeOptions, registry);
 
diff --git a/cpp/src/arrow/compute/kernels/vector_selection_test.cc b/cpp/src/arrow/compute/kernels/vector_selection_test.cc
index 59c311f0115..f71c867a96a 100644
--- a/cpp/src/arrow/compute/kernels/vector_selection_test.cc
+++ b/cpp/src/arrow/compute/kernels/vector_selection_test.cc
@@ -928,6 +928,7 @@ class TestTakeKernel : public ::testing::Test {};
 
 TEST(TestTakeKernel, TakeNull) {
   AssertTakeNull("[null, null, null]", "[0, 1, 0]", "[null, null, null]");
+  AssertTakeNull("[null, null, null]", "[0, 2]", "[null, null]");
 
   std::shared_ptr<Array> arr;
   ASSERT_RAISES(IndexError,
diff --git a/cpp/src/arrow/compute/kernels/vector_sort.cc b/cpp/src/arrow/compute/kernels/vector_sort.cc
index c8c1b109230..c028ab879be 100644
--- a/cpp/src/arrow/compute/kernels/vector_sort.cc
+++ b/cpp/src/arrow/compute/kernels/vector_sort.cc
@@ -26,6 +26,7 @@
 
 namespace arrow {
 namespace compute {
+namespace internal {
 
 namespace {
 
@@ -110,8 +111,6 @@ inline void VisitRawValuesInline(const ArrayType& values,
   }
 }
 
-}  // namespace
-
 template <typename ArrowType>
 class CompareSorter {
   using ArrayType = typename TypeTraits<ArrowType>::ArrayType;
@@ -281,8 +280,6 @@ struct SortIndices {
   }
 };
 
-namespace internal {
-
 // Sort indices kernels implemented for
 //
 // * Number types
@@ -302,19 +299,44 @@ void AddSortingKernels(VectorKernel base, VectorFunction* func) {
   }
 }
 
+const FunctionDoc sort_indices_doc(
+    "Return the indices that would sort an array",
+    ("This function computes an array of indices that define a non-stable sort\n"
+     "of the input array.  Null values are considered greater than any\n"
+     "other value and are therefore sorted at the end of the array."),
+    {"array"});
+
+const FunctionDoc partition_nth_indices_doc(
+    "Return the indices that would partition an array around a pivot",
+    ("This functions computes an array of indices that define a non-stable\n"
+     "partial sort of the input array.\n"
+     "\n"
+     "The output is such that the `N`'th index points to the `N`'th element\n"
+     "of the input in sorted order, and all indices before the `N`'th point\n"
+     "to elements in the input less or equal to elements at or after the `N`'th.\n"
+     "\n"
+     "Null values are considered greater than any other value and are\n"
+     "therefore partitioned towards the end of the array.\n"
+     "\n"
+     "The pivot index `N` must be given in PartitionNthOptions."),
+    {"array"}, "PartitionNthOptions");
+
+}  // namespace
+
 void RegisterVectorSort(FunctionRegistry* registry) {
   // The kernel outputs into preallocated memory and is never null
   VectorKernel base;
   base.mem_allocation = MemAllocation::PREALLOCATE;
   base.null_handling = NullHandling::OUTPUT_NOT_NULL;
 
-  auto sort_indices = std::make_shared<VectorFunction>("sort_indices", Arity::Unary());
+  auto sort_indices =
+      std::make_shared<VectorFunction>("sort_indices", Arity::Unary(), &sort_indices_doc);
   AddSortingKernels<SortIndices>(base, sort_indices.get());
   DCHECK_OK(registry->AddFunction(std::move(sort_indices)));
 
   // partition_nth_indices has a parameter so needs its init function
-  auto part_indices =
-      std::make_shared<VectorFunction>("partition_nth_indices", Arity::Unary());
+  auto part_indices = std::make_shared<VectorFunction>(
+      "partition_nth_indices", Arity::Unary(), &partition_nth_indices_doc);
   base.init = PartitionNthToIndicesState::Init;
   AddSortingKernels<PartitionNthToIndices>(base, part_indices.get());
   DCHECK_OK(registry->AddFunction(std::move(part_indices)));
diff --git a/cpp/src/arrow/compute/registry.cc b/cpp/src/arrow/compute/registry.cc
index ae4f717743d..7ef1e26d59b 100644
--- a/cpp/src/arrow/compute/registry.cc
+++ b/cpp/src/arrow/compute/registry.cc
@@ -33,6 +33,8 @@ namespace compute {
 class FunctionRegistry::FunctionRegistryImpl {
  public:
   Status AddFunction(std::shared_ptr<Function> function, bool allow_overwrite) {
+    RETURN_NOT_OK(function->Validate());
+
     std::lock_guard<std::mutex> mutation_guard(lock_);
 
     const std::string& name = function->name();
@@ -126,6 +128,8 @@ static std::unique_ptr<FunctionRegistry> CreateBuiltInRegistry() {
 
   // Aggregate functions
   RegisterScalarAggregateBasic(registry.get());
+  RegisterScalarAggregateMode(registry.get());
+  RegisterScalarAggregateVariance(registry.get());
 
   // Vector functions
   RegisterVectorHash(registry.get());
diff --git a/cpp/src/arrow/compute/registry_internal.h b/cpp/src/arrow/compute/registry_internal.h
index d84f85cd153..78e134eb41f 100644
--- a/cpp/src/arrow/compute/registry_internal.h
+++ b/cpp/src/arrow/compute/registry_internal.h
@@ -43,6 +43,8 @@ void RegisterVectorSort(FunctionRegistry* registry);
 
 // Aggregate functions
 void RegisterScalarAggregateBasic(FunctionRegistry* registry);
+void RegisterScalarAggregateMode(FunctionRegistry* registry);
+void RegisterScalarAggregateVariance(FunctionRegistry* registry);
 
 }  // namespace internal
 }  // namespace compute
diff --git a/cpp/src/arrow/compute/registry_test.cc b/cpp/src/arrow/compute/registry_test.cc
index 67a72240253..e1e0d523187 100644
--- a/cpp/src/arrow/compute/registry_test.cc
+++ b/cpp/src/arrow/compute/registry_test.cc
@@ -49,11 +49,12 @@ TEST_F(TestRegistry, CreateBuiltInRegistry) {
 TEST_F(TestRegistry, Basics) {
   ASSERT_EQ(0, registry_->num_functions());
 
-  std::shared_ptr<Function> func = std::make_shared<ScalarFunction>("f1", Arity::Unary());
+  std::shared_ptr<Function> func =
+      std::make_shared<ScalarFunction>("f1", Arity::Unary(), /*doc=*/nullptr);
   ASSERT_OK(registry_->AddFunction(func));
   ASSERT_EQ(1, registry_->num_functions());
 
-  func = std::make_shared<VectorFunction>("f0", Arity::Binary());
+  func = std::make_shared<VectorFunction>("f0", Arity::Binary(), /*doc=*/nullptr);
   ASSERT_OK(registry_->AddFunction(func));
   ASSERT_EQ(2, registry_->num_functions());
 
@@ -64,7 +65,7 @@ TEST_F(TestRegistry, Basics) {
   ASSERT_RAISES(KeyError, registry_->GetFunction("f2"));
 
   // Try adding a function with name collision
-  func = std::make_shared<ScalarAggregateFunction>("f1", Arity::Unary());
+  func = std::make_shared<ScalarAggregateFunction>("f1", Arity::Unary(), /*doc=*/nullptr);
   ASSERT_RAISES(KeyError, registry_->AddFunction(func));
 
   // Allow overwriting by flag
diff --git a/cpp/src/arrow/csv/column_builder.cc b/cpp/src/arrow/csv/column_builder.cc
index 4d0e21313a8..8178b260b4a 100644
--- a/cpp/src/arrow/csv/column_builder.cc
+++ b/cpp/src/arrow/csv/column_builder.cc
@@ -25,16 +25,16 @@
 #include <vector>
 
 #include "arrow/array.h"
-#include "arrow/builder.h"
+#include "arrow/array/builder_base.h"
 #include "arrow/chunked_array.h"
 #include "arrow/csv/column_builder.h"
 #include "arrow/csv/converter.h"
 #include "arrow/csv/inference_internal.h"
 #include "arrow/csv/options.h"
 #include "arrow/csv/parser.h"
-#include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/type.h"
+#include "arrow/type_fwd.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/task_group.h"
 
diff --git a/cpp/src/arrow/csv/column_decoder.cc b/cpp/src/arrow/csv/column_decoder.cc
index 08889a79cb5..c57477ef59d 100644
--- a/cpp/src/arrow/csv/column_decoder.cc
+++ b/cpp/src/arrow/csv/column_decoder.cc
@@ -25,15 +25,15 @@
 #include <vector>
 
 #include "arrow/array.h"
-#include "arrow/builder.h"
+#include "arrow/array/builder_base.h"
 #include "arrow/csv/column_decoder.h"
 #include "arrow/csv/converter.h"
 #include "arrow/csv/inference_internal.h"
 #include "arrow/csv/options.h"
 #include "arrow/csv/parser.h"
-#include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/type.h"
+#include "arrow/type_fwd.h"
 #include "arrow/util/future.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/task_group.h"
diff --git a/cpp/src/arrow/csv/converter.cc b/cpp/src/arrow/csv/converter.cc
index 8bca65a2a65..6d2decbc903 100644
--- a/cpp/src/arrow/csv/converter.cc
+++ b/cpp/src/arrow/csv/converter.cc
@@ -24,11 +24,14 @@
 #include <type_traits>
 #include <vector>
 
-#include "arrow/builder.h"
+#include "arrow/array/builder_binary.h"
+#include "arrow/array/builder_decimal.h"
+#include "arrow/array/builder_dict.h"
+#include "arrow/array/builder_primitive.h"
 #include "arrow/csv/parser.h"
-#include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/type.h"
+#include "arrow/type_fwd.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/checked_cast.h"
 #include "arrow/util/decimal.h"
@@ -90,9 +93,32 @@ Status InitializeTrie(const std::vector<std::string>& inputs, Trie* trie) {
   return Status::OK();
 }
 
-class ConcreteConverterMixin {
- protected:
-  Status InitializeNullTrie(const ConvertOptions& options);
+// Presize a builder based on parser contents
+template <typename BuilderType>
+enable_if_t<!is_base_binary_type<typename BuilderType::TypeClass>::value, Status>
+PresizeBuilder(const BlockParser& parser, BuilderType* builder) {
+  return builder->Resize(parser.num_rows());
+}
+
+// Same, for variable-sized binary builders
+template <typename T>
+Status PresizeBuilder(const BlockParser& parser, BaseBinaryBuilder<T>* builder) {
+  RETURN_NOT_OK(builder->Resize(parser.num_rows()));
+  return builder->ReserveData(parser.num_bytes());
+}
+
+/////////////////////////////////////////////////////////////////////////
+// Per-type value decoders
+
+struct ValueDecoder {
+  explicit ValueDecoder(const std::shared_ptr<DataType>& type,
+                        const ConvertOptions& options)
+      : type_(type), options_(options) {}
+
+  Status Initialize() {
+    // TODO no need to build a separate Trie for each instance
+    return InitializeTrie(options_.null_values, &null_trie_);
+  }
 
   bool IsNull(const uint8_t* data, uint32_t size, bool quoted) {
     if (quoted) {
@@ -102,427 +128,405 @@ class ConcreteConverterMixin {
                util::string_view(reinterpret_cast<const char*>(data), size)) >= 0;
   }
 
+ protected:
   Trie null_trie_;
+  std::shared_ptr<DataType> type_;
+  const ConvertOptions& options_;
 };
 
-Status ConcreteConverterMixin::InitializeNullTrie(const ConvertOptions& options) {
-  // TODO no need to build a separate Trie for each Converter instance
-  return InitializeTrie(options.null_values, &null_trie_);
-}
+//
+// Value decoder for fixed-size binary
+//
 
-class ConcreteConverter : public Converter, public ConcreteConverterMixin {
- public:
-  using Converter::Converter;
+struct FixedSizeBinaryValueDecoder : public ValueDecoder {
+  using value_type = const uint8_t*;
+
+  explicit FixedSizeBinaryValueDecoder(const std::shared_ptr<DataType>& type,
+                                       const ConvertOptions& options)
+      : ValueDecoder(type, options),
+        byte_width_(checked_cast<const FixedSizeBinaryType&>(*type).byte_width()) {}
+
+  Status Decode(const uint8_t* data, uint32_t size, bool quoted, value_type* out) {
+    if (ARROW_PREDICT_FALSE(size != byte_width_)) {
+      return Status::Invalid("CSV conversion error to ", type_->ToString(), ": got a ",
+                             size, "-byte long string");
+    }
+    *out = data;
+    return Status::OK();
+  }
+
+  bool IsNull(const uint8_t* data, uint32_t size, bool quoted) { return false; }
 
  protected:
-  Status Initialize() override { return InitializeNullTrie(options_); }
+  const uint32_t byte_width_;
 };
 
-class ConcreteDictionaryConverter : public DictionaryConverter,
-                                    public ConcreteConverterMixin {
- public:
-  using DictionaryConverter::DictionaryConverter;
+//
+// Value decoder for variable-size binary
+//
 
- protected:
-  Status Initialize() override { return InitializeNullTrie(options_); }
+template <bool CheckUTF8>
+struct BinaryValueDecoder : public ValueDecoder {
+  using value_type = util::string_view;
+
+  using ValueDecoder::ValueDecoder;
+
+  Status Initialize() {
+    util::InitializeUTF8();
+    return ValueDecoder::Initialize();
+  }
+
+  Status Decode(const uint8_t* data, uint32_t size, bool quoted, value_type* out) {
+    if (CheckUTF8 && ARROW_PREDICT_FALSE(!util::ValidateUTF8(data, size))) {
+      return Status::Invalid("CSV conversion error to ", type_->ToString(),
+                             ": invalid UTF8 data");
+    }
+    *out = {reinterpret_cast<const char*>(data), size};
+    return Status::OK();
+  }
+
+  bool IsNull(const uint8_t* data, uint32_t size, bool quoted) {
+    return options_.strings_can_be_null &&
+           ValueDecoder::IsNull(data, size, false /* quoted */);
+  }
 };
 
-/////////////////////////////////////////////////////////////////////////
-// Concrete Converter for null values
+//
+// Value decoder for integers and floats
+//
 
-class NullConverter : public ConcreteConverter {
- public:
-  using ConcreteConverter::ConcreteConverter;
+template <typename T>
+struct NumericValueDecoder : public ValueDecoder {
+  using value_type = typename T::c_type;
 
-  Result<std::shared_ptr<Array>> Convert(const BlockParser& parser,
-                                         int32_t col_index) override {
-    NullBuilder builder(pool_);
+  using ValueDecoder::ValueDecoder;
 
-    auto visit = [&](const uint8_t* data, uint32_t size, bool quoted) -> Status {
-      if (ARROW_PREDICT_TRUE(IsNull(data, size, quoted))) {
-        return builder.AppendNull();
-      } else {
-        return GenericConversionError(type_, data, size);
-      }
-    };
-    RETURN_NOT_OK(parser.VisitColumn(col_index, visit));
-    std::shared_ptr<Array> res;
-    RETURN_NOT_OK(builder.Finish(&res));
-    return res;
+  Status Decode(const uint8_t* data, uint32_t size, bool quoted, value_type* out) {
+    // XXX should quoted values be allowed at all?
+    TrimWhiteSpace(&data, &size);
+    if (ARROW_PREDICT_FALSE(
+            !internal::ParseValue<T>(reinterpret_cast<const char*>(data), size, out))) {
+      return GenericConversionError(type_, data, size);
+    }
+    return Status::OK();
   }
 };
 
-/////////////////////////////////////////////////////////////////////////
-// Concrete Converter for var-sized binary strings
+//
+// Value decoder for booleans
+//
 
-template <typename T, bool CheckUTF8>
-class BinaryConverter : public ConcreteConverter {
- public:
-  using ConcreteConverter::ConcreteConverter;
+struct BooleanValueDecoder : public ValueDecoder {
+  using value_type = bool;
 
-  Result<std::shared_ptr<Array>> Convert(const BlockParser& parser,
-                                         int32_t col_index) override {
-    using BuilderType = typename TypeTraits<T>::BuilderType;
-    BuilderType builder(pool_);
+  using ValueDecoder::ValueDecoder;
 
-    auto visit_non_null = [&](const uint8_t* data, uint32_t size, bool quoted) -> Status {
-      if (CheckUTF8 && ARROW_PREDICT_FALSE(!util::ValidateUTF8(data, size))) {
-        return Status::Invalid("CSV conversion error to ", type_->ToString(),
-                               ": invalid UTF8 data");
-      }
-      builder.UnsafeAppend(data, size);
+  Status Initialize() {
+    // TODO no need to build separate Tries for each instance
+    RETURN_NOT_OK(InitializeTrie(options_.true_values, &true_trie_));
+    RETURN_NOT_OK(InitializeTrie(options_.false_values, &false_trie_));
+    return ValueDecoder::Initialize();
+  }
+
+  Status Decode(const uint8_t* data, uint32_t size, bool quoted, value_type* out) {
+    // XXX should quoted values be allowed at all?
+    if (false_trie_.Find(util::string_view(reinterpret_cast<const char*>(data), size)) >=
+        0) {
+      *out = false;
       return Status::OK();
-    };
+    }
+    if (ARROW_PREDICT_TRUE(true_trie_.Find(util::string_view(
+                               reinterpret_cast<const char*>(data), size)) >= 0)) {
+      *out = true;
+      return Status::OK();
+    }
+    return GenericConversionError(type_, data, size);
+  }
 
-    RETURN_NOT_OK(builder.Resize(parser.num_rows()));
-    RETURN_NOT_OK(builder.ReserveData(parser.num_bytes()));
-
-    if (options_.strings_can_be_null) {
-      auto visit = [&](const uint8_t* data, uint32_t size, bool quoted) -> Status {
-        if (IsNull(data, size, false /* quoted */)) {
-          builder.UnsafeAppendNull();
-          return Status::OK();
-        } else {
-          return visit_non_null(data, size, quoted);
-        }
-      };
-      RETURN_NOT_OK(parser.VisitColumn(col_index, visit));
+ protected:
+  Trie true_trie_;
+  Trie false_trie_;
+};
+
+//
+// Value decoder for decimals
+//
+
+struct DecimalValueDecoder : public ValueDecoder {
+  using value_type = Decimal128;
+
+  explicit DecimalValueDecoder(const std::shared_ptr<DataType>& type,
+                               const ConvertOptions& options)
+      : ValueDecoder(type, options),
+        decimal_type_(internal::checked_cast<const DecimalType&>(*type_)),
+        type_precision_(decimal_type_.precision()),
+        type_scale_(decimal_type_.scale()) {}
+
+  Status Decode(const uint8_t* data, uint32_t size, bool quoted, value_type* out) {
+    TrimWhiteSpace(&data, &size);
+    Decimal128 decimal;
+    int32_t precision, scale;
+    util::string_view view(reinterpret_cast<const char*>(data), size);
+    RETURN_NOT_OK(Decimal128::FromString(view, &decimal, &precision, &scale));
+    if (precision > type_precision_) {
+      return Status::Invalid("Error converting '", view, "' to ", type_->ToString(),
+                             ": precision not supported by type.");
+    }
+    if (scale != type_scale_) {
+      ARROW_ASSIGN_OR_RAISE(*out, decimal.Rescale(scale, type_scale_));
     } else {
-      RETURN_NOT_OK(parser.VisitColumn(col_index, visit_non_null));
+      *out = std::move(decimal);
     }
-
-    std::shared_ptr<Array> res;
-    RETURN_NOT_OK(builder.Finish(&res));
-    return res;
+    return Status::OK();
   }
 
  protected:
-  Status Initialize() override {
-    util::InitializeUTF8();
-    return ConcreteConverter::Initialize();
-  }
+  const DecimalType& decimal_type_;
+  const int32_t type_precision_;
+  const int32_t type_scale_;
 };
 
-template <typename T, bool CheckUTF8>
-class DictionaryBinaryConverter : public ConcreteDictionaryConverter {
- public:
-  using ConcreteDictionaryConverter::ConcreteDictionaryConverter;
+//
+// Value decoders for timestamps
+//
 
-  Result<std::shared_ptr<Array>> Convert(const BlockParser& parser,
-                                         int32_t col_index) override {
-    // We use a fixed index width so that all column chunks get the same index type
-    using BuilderType = Dictionary32Builder<T>;
-    BuilderType builder(value_type_, pool_);
+struct InlineISO8601ValueDecoder : public ValueDecoder {
+  using value_type = int64_t;
 
-    auto visit_non_null = [&](const uint8_t* data, uint32_t size, bool quoted) -> Status {
-      if (CheckUTF8 && ARROW_PREDICT_FALSE(!util::ValidateUTF8(data, size))) {
-        return Status::Invalid("CSV conversion error to ", value_type_->ToString(),
-                               ": invalid UTF8 data");
-      }
-      RETURN_NOT_OK(
-          builder.Append(util::string_view(reinterpret_cast<const char*>(data), size)));
-      if (ARROW_PREDICT_FALSE(builder.dictionary_length() > max_cardinality_)) {
-        return Status::IndexError("Dictionary length exceeded max cardinality");
-      }
-      return Status::OK();
-    };
+  explicit InlineISO8601ValueDecoder(const std::shared_ptr<DataType>& type,
+                                     const ConvertOptions& options)
+      : ValueDecoder(type, options),
+        unit_(checked_cast<const TimestampType&>(*type_).unit()) {}
 
-    RETURN_NOT_OK(builder.Resize(parser.num_rows()));
-
-    if (options_.strings_can_be_null) {
-      auto visit = [&](const uint8_t* data, uint32_t size, bool quoted) -> Status {
-        if (IsNull(data, size, false /* quoted */)) {
-          return builder.AppendNull();
-        } else {
-          return visit_non_null(data, size, quoted);
-        }
-      };
-      RETURN_NOT_OK(parser.VisitColumn(col_index, visit));
-    } else {
-      RETURN_NOT_OK(parser.VisitColumn(col_index, visit_non_null));
+  Status Decode(const uint8_t* data, uint32_t size, bool quoted, value_type* out) {
+    if (ARROW_PREDICT_FALSE(!internal::ParseTimestampISO8601(
+            reinterpret_cast<const char*>(data), size, unit_, out))) {
+      return GenericConversionError(type_, data, size);
     }
-
-    std::shared_ptr<Array> res;
-    RETURN_NOT_OK(builder.Finish(&res));
-    return res;
+    return Status::OK();
   }
 
-  void SetMaxCardinality(int32_t max_length) override { max_cardinality_ = max_length; }
-
  protected:
-  Status Initialize() override {
-    util::InitializeUTF8();
-    return ConcreteDictionaryConverter::Initialize();
-  }
-
-  int32_t max_cardinality_ = std::numeric_limits<int32_t>::max();
+  TimeUnit::type unit_;
 };
 
-/////////////////////////////////////////////////////////////////////////
-// Concrete Converter for fixed-sized binary strings
+struct SingleParserTimestampValueDecoder : public ValueDecoder {
+  using value_type = int64_t;
 
-class FixedSizeBinaryConverter : public ConcreteConverter {
- public:
-  using ConcreteConverter::ConcreteConverter;
+  explicit SingleParserTimestampValueDecoder(const std::shared_ptr<DataType>& type,
+                                             const ConvertOptions& options)
+      : ValueDecoder(type, options),
+        unit_(checked_cast<const TimestampType&>(*type_).unit()),
+        parser_(*options_.timestamp_parsers[0]) {}
 
-  Result<std::shared_ptr<Array>> Convert(const BlockParser& parser,
-                                         int32_t col_index) override {
-    FixedSizeBinaryBuilder builder(type_, pool_);
-    const uint32_t byte_width = static_cast<uint32_t>(builder.byte_width());
+  Status Decode(const uint8_t* data, uint32_t size, bool quoted, value_type* out) {
+    if (ARROW_PREDICT_FALSE(
+            !parser_(reinterpret_cast<const char*>(data), size, unit_, out))) {
+      return GenericConversionError(type_, data, size);
+    }
+    return Status::OK();
+  }
 
-    // TODO do we accept nulls here?
+ protected:
+  TimeUnit::type unit_;
+  const TimestampParser& parser_;
+};
 
-    auto visit = [&](const uint8_t* data, uint32_t size, bool quoted) -> Status {
-      if (ARROW_PREDICT_FALSE(size != byte_width)) {
-        return Status::Invalid("CSV conversion error to ", type_->ToString(), ": got a ",
-                               size, "-byte long string");
+struct MultipleParsersTimestampValueDecoder : public ValueDecoder {
+  using value_type = int64_t;
+
+  explicit MultipleParsersTimestampValueDecoder(const std::shared_ptr<DataType>& type,
+                                                const ConvertOptions& options)
+      : ValueDecoder(type, options),
+        unit_(checked_cast<const TimestampType&>(*type_).unit()),
+        parsers_(GetParsers(options_)) {}
+
+  Status Decode(const uint8_t* data, uint32_t size, bool quoted, value_type* out) {
+    for (const auto& parser : parsers_) {
+      if (parser->operator()(reinterpret_cast<const char*>(data), size, unit_, out)) {
+        return Status::OK();
       }
-      return builder.Append(data);
-    };
-    RETURN_NOT_OK(builder.Resize(parser.num_rows()));
-    RETURN_NOT_OK(parser.VisitColumn(col_index, visit));
+    }
+    return GenericConversionError(type_, data, size);
+  }
 
-    std::shared_ptr<Array> res;
-    RETURN_NOT_OK(builder.Finish(&res));
-    return res;
+ protected:
+  using ParserVector = std::vector<const TimestampParser*>;
+
+  static ParserVector GetParsers(const ConvertOptions& options) {
+    ParserVector parsers(options.timestamp_parsers.size());
+    for (size_t i = 0; i < options.timestamp_parsers.size(); ++i) {
+      parsers[i] = options.timestamp_parsers[i].get();
+    }
+    return parsers;
   }
+
+  TimeUnit::type unit_;
+  std::vector<const TimestampParser*> parsers_;
 };
 
 /////////////////////////////////////////////////////////////////////////
-// Concrete Converter for booleans
+// Concrete Converter hierarchy
 
-class BooleanConverter : public ConcreteConverter {
+class ConcreteConverter : public Converter {
  public:
-  using ConcreteConverter::ConcreteConverter;
+  using Converter::Converter;
+};
+
+class ConcreteDictionaryConverter : public DictionaryConverter {
+ public:
+  using DictionaryConverter::DictionaryConverter;
+};
+
+//
+// Concrete Converter for nulls
+//
+
+class NullConverter : public ConcreteConverter {
+ public:
+  NullConverter(const std::shared_ptr<DataType>& type, const ConvertOptions& options,
+                MemoryPool* pool)
+      : ConcreteConverter(type, options, pool), decoder_(type_, options_) {}
 
   Result<std::shared_ptr<Array>> Convert(const BlockParser& parser,
                                          int32_t col_index) override {
-    BooleanBuilder builder(type_, pool_);
+    NullBuilder builder(pool_);
 
     auto visit = [&](const uint8_t* data, uint32_t size, bool quoted) -> Status {
-      // XXX should quoted values be allowed at all?
-      if (IsNull(data, size, quoted)) {
-        builder.UnsafeAppendNull();
-        return Status::OK();
-      }
-      if (false_trie_.Find(
-              util::string_view(reinterpret_cast<const char*>(data), size)) >= 0) {
-        builder.UnsafeAppend(false);
-        return Status::OK();
-      }
-      if (true_trie_.Find(util::string_view(reinterpret_cast<const char*>(data), size)) >=
-          0) {
-        builder.UnsafeAppend(true);
-        return Status::OK();
+      if (ARROW_PREDICT_TRUE(decoder_.IsNull(data, size, quoted))) {
+        return builder.AppendNull();
+      } else {
+        return GenericConversionError(type_, data, size);
       }
-      return GenericConversionError(type_, data, size);
     };
-    RETURN_NOT_OK(builder.Resize(parser.num_rows()));
     RETURN_NOT_OK(parser.VisitColumn(col_index, visit));
-
     std::shared_ptr<Array> res;
     RETURN_NOT_OK(builder.Finish(&res));
     return res;
   }
 
  protected:
-  Status Initialize() override {
-    // TODO no need to build separate Tries for each BooleanConverter instance
-    RETURN_NOT_OK(InitializeTrie(options_.true_values, &true_trie_));
-    RETURN_NOT_OK(InitializeTrie(options_.false_values, &false_trie_));
-    return ConcreteConverter::Initialize();
-  }
+  Status Initialize() override { return decoder_.Initialize(); }
 
-  Trie true_trie_;
-  Trie false_trie_;
+  ValueDecoder decoder_;
 };
 
-/////////////////////////////////////////////////////////////////////////
-// Concrete Converter for numbers
+//
+// Concrete Converter for primitives
+//
 
-template <typename T>
-class NumericConverter : public ConcreteConverter {
+template <typename T, typename ValueDecoderType>
+class PrimitiveConverter : public ConcreteConverter {
  public:
-  using ConcreteConverter::ConcreteConverter;
+  PrimitiveConverter(const std::shared_ptr<DataType>& type, const ConvertOptions& options,
+                     MemoryPool* pool)
+      : ConcreteConverter(type, options, pool), decoder_(type_, options_) {}
 
   Result<std::shared_ptr<Array>> Convert(const BlockParser& parser,
                                          int32_t col_index) override {
     using BuilderType = typename TypeTraits<T>::BuilderType;
-    using value_type = typename T::c_type;
+    using value_type = typename ValueDecoderType::value_type;
 
     BuilderType builder(type_, pool_);
+    RETURN_NOT_OK(PresizeBuilder(parser, &builder));
 
     auto visit = [&](const uint8_t* data, uint32_t size, bool quoted) -> Status {
-      // XXX should quoted values be allowed at all?
-      value_type value = 0;
-      if (IsNull(data, size, quoted)) {
-        builder.UnsafeAppendNull();
-        return Status::OK();
-      }
-      if (!std::is_same<BooleanType, T>::value) {
-        TrimWhiteSpace(&data, &size);
-      }
-      if (ARROW_PREDICT_FALSE(!internal::ParseValue<T>(
-              reinterpret_cast<const char*>(data), size, &value))) {
-        return GenericConversionError(type_, data, size);
+      if (decoder_.IsNull(data, size, quoted /* quoted */)) {
+        return builder.AppendNull();
       }
+      value_type value;
+      RETURN_NOT_OK(decoder_.Decode(data, size, quoted, &value));
       builder.UnsafeAppend(value);
       return Status::OK();
     };
-    RETURN_NOT_OK(builder.Resize(parser.num_rows()));
     RETURN_NOT_OK(parser.VisitColumn(col_index, visit));
 
     std::shared_ptr<Array> res;
     RETURN_NOT_OK(builder.Finish(&res));
     return res;
   }
-};
 
-/////////////////////////////////////////////////////////////////////////
-// Concrete Converter for timestamps
-
-namespace {
-
-struct InlineISO8601 {
-  TimeUnit::type unit;
-
-  explicit InlineISO8601(TimeUnit::type unit) : unit(unit) {}
-
-  bool operator()(const char* s, size_t length, int64_t* out) const {
-    return internal::ParseTimestampISO8601(s, length, unit, out);
-  }
-};
-
-struct SingleTimestampParser {
-  const TimestampParser& parser;
-  TimeUnit::type unit;
-
-  SingleTimestampParser(const TimestampParser& parser, TimeUnit::type unit)
-      : parser(parser), unit(unit) {}
+ protected:
+  Status Initialize() override { return decoder_.Initialize(); }
 
-  bool operator()(const char* s, size_t length, int64_t* out) const {
-    return this->parser(s, length, this->unit, out);
-  }
+  ValueDecoderType decoder_;
 };
 
-struct MultipleTimestampParsers {
-  std::vector<const TimestampParser*> parsers;
-  TimeUnit::type unit;
-
-  MultipleTimestampParsers(const std::vector<std::shared_ptr<TimestampParser>>& parsers,
-                           TimeUnit::type unit)
-      : unit(unit) {
-    for (const auto& parser : parsers) {
-      this->parsers.push_back(parser.get());
-    }
-  }
+//
+// Concrete Converter for dictionaries
+//
 
-  bool operator()(const char* s, size_t length, int64_t* out) const {
-    for (const auto& parser : this->parsers) {
-      if (parser->operator()(s, length, this->unit, out)) {
-        return true;
-      }
-    }
-    return false;
-  }
-};
+template <typename T, typename ValueDecoderType>
+class TypedDictionaryConverter : public ConcreteDictionaryConverter {
+ public:
+  TypedDictionaryConverter(const std::shared_ptr<DataType>& value_type,
+                           const ConvertOptions& options, MemoryPool* pool)
+      : ConcreteDictionaryConverter(value_type, options, pool),
+        decoder_(value_type, options_) {}
 
-}  // namespace
+  Result<std::shared_ptr<Array>> Convert(const BlockParser& parser,
+                                         int32_t col_index) override {
+    // We use a fixed index width so that all column chunks get the same index type
+    using BuilderType = Dictionary32Builder<T>;
+    using value_type = typename ValueDecoderType::value_type;
 
-class TimestampConverter : public ConcreteConverter {
- public:
-  using ConcreteConverter::ConcreteConverter;
+    BuilderType builder(value_type_, pool_);
+    RETURN_NOT_OK(PresizeBuilder(parser, &builder));
 
-  template <typename ConvertValue>
-  Status ConvertValuesWith(const BlockParser& parser, int32_t col_index,
-                           const ConvertValue& converter, TimestampBuilder* builder) {
-    using value_type = TimestampType::c_type;
     auto visit = [&](const uint8_t* data, uint32_t size, bool quoted) -> Status {
-      value_type value = 0;
-      if (IsNull(data, size, quoted)) {
-        builder->UnsafeAppendNull();
-        return Status::OK();
+      if (decoder_.IsNull(data, size, quoted /* quoted */)) {
+        return builder.AppendNull();
       }
-
-      if (ARROW_PREDICT_FALSE(
-              !converter(reinterpret_cast<const char*>(data), size, &value))) {
-        return GenericConversionError(type_, data, size);
+      if (ARROW_PREDICT_FALSE(builder.dictionary_length() > max_cardinality_)) {
+        return Status::IndexError("Dictionary length exceeded max cardinality");
       }
-      builder->UnsafeAppend(value);
-      return Status::OK();
+      value_type value;
+      RETURN_NOT_OK(decoder_.Decode(data, size, quoted, &value));
+      return builder.Append(value);
     };
-    return parser.VisitColumn(col_index, visit);
-  }
-
-  Result<std::shared_ptr<Array>> Convert(const BlockParser& parser,
-                                         int32_t col_index) override {
-    TimestampBuilder builder(type_, pool_);
-    RETURN_NOT_OK(builder.Resize(parser.num_rows()));
-
-    TimeUnit::type unit = checked_cast<const TimestampType&>(*type_).unit();
-    if (options_.timestamp_parsers.size() == 0) {
-      // Default to ISO-8601
-      InlineISO8601 converter(unit);
-      RETURN_NOT_OK(ConvertValuesWith(parser, col_index, converter, &builder));
-    } else if (options_.timestamp_parsers.size() == 1) {
-      // Single user-supplied converter
-      SingleTimestampParser converter(*options_.timestamp_parsers[0], unit);
-      RETURN_NOT_OK(ConvertValuesWith(parser, col_index, converter, &builder));
-    } else {
-      // Multiple converters, must iterate for each value
-      MultipleTimestampParsers converter(options_.timestamp_parsers, unit);
-      RETURN_NOT_OK(ConvertValuesWith(parser, col_index, converter, &builder));
-    }
+    RETURN_NOT_OK(parser.VisitColumn(col_index, visit));
 
     std::shared_ptr<Array> res;
     RETURN_NOT_OK(builder.Finish(&res));
     return res;
   }
-};
 
-/////////////////////////////////////////////////////////////////////////
-// Concrete Converter for Decimals
+  void SetMaxCardinality(int32_t max_length) override { max_cardinality_ = max_length; }
 
-class DecimalConverter : public ConcreteConverter {
- public:
-  using ConcreteConverter::ConcreteConverter;
+ protected:
+  Status Initialize() override {
+    util::InitializeUTF8();
+    return decoder_.Initialize();
+  }
 
-  Result<std::shared_ptr<Array>> Convert(const BlockParser& parser,
-                                         int32_t col_index) override {
-    Decimal128Builder builder(type_, pool_);
+  ValueDecoderType decoder_;
+  int32_t max_cardinality_ = std::numeric_limits<int32_t>::max();
+};
 
-    auto visit = [&](const uint8_t* data, uint32_t size, bool quoted) -> Status {
-      if (IsNull(data, size, quoted)) {
-        builder.UnsafeAppendNull();
-        return Status::OK();
-      }
-      TrimWhiteSpace(&data, &size);
-      Decimal128 decimal;
-      int32_t precision, scale;
-      util::string_view view(reinterpret_cast<const char*>(data), size);
-      RETURN_NOT_OK(Decimal128::FromString(view, &decimal, &precision, &scale));
-      DecimalType& type = *internal::checked_cast<DecimalType*>(type_.get());
-      if (precision > type.precision()) {
-        return Status::Invalid("Error converting ", view, " to ", type_->ToString(),
-                               " precision not supported by type.");
-      }
-      if (scale != type.scale()) {
-        Decimal128 scaled;
-        ARROW_ASSIGN_OR_RAISE(scaled, decimal.Rescale(scale, type.scale()));
-        builder.UnsafeAppend(scaled);
-      } else {
-        builder.UnsafeAppend(decimal);
-      }
-      return Status::OK();
-    };
-    RETURN_NOT_OK(builder.Resize(parser.num_rows()));
-    RETURN_NOT_OK(parser.VisitColumn(col_index, visit));
+//
+// Concrete Converter factory for timestamps
+//
 
-    std::shared_ptr<Array> res;
-    RETURN_NOT_OK(builder.Finish(&res));
-    return res;
+template <template <typename, typename> class ConverterType>
+std::shared_ptr<Converter> MakeTimestampConverter(const std::shared_ptr<DataType>& type,
+                                                  const ConvertOptions& options,
+                                                  MemoryPool* pool) {
+  if (options.timestamp_parsers.size() == 0) {
+    // Default to ISO-8601
+    return std::make_shared<ConverterType<TimestampType, InlineISO8601ValueDecoder>>(
+        type, options, pool);
+  } else if (options.timestamp_parsers.size() == 1) {
+    // Single user-supplied converter
+    return std::make_shared<
+        ConverterType<TimestampType, SingleParserTimestampValueDecoder>>(type, options,
+                                                                         pool);
+  } else {
+    // Multiple converters, must iterate for each value
+    return std::make_shared<
+        ConverterType<TimestampType, MultipleParsersTimestampValueDecoder>>(type, options,
+                                                                            pool);
   }
-};
+}
 
 }  // namespace
 
@@ -541,78 +545,136 @@ DictionaryConverter::DictionaryConverter(const std::shared_ptr<DataType>& value_
 Result<std::shared_ptr<Converter>> Converter::Make(const std::shared_ptr<DataType>& type,
                                                    const ConvertOptions& options,
                                                    MemoryPool* pool) {
-  Converter* ptr;
+  std::shared_ptr<Converter> ptr;
 
   switch (type->id()) {
-#define CONVERTER_CASE(TYPE_ID, CONVERTER_TYPE)    \
-  case TYPE_ID:                                    \
-    ptr = new CONVERTER_TYPE(type, options, pool); \
+#define CONVERTER_CASE(TYPE_ID, CONVERTER_TYPE)         \
+  case TYPE_ID:                                         \
+    ptr.reset(new CONVERTER_TYPE(type, options, pool)); \
     break;
 
+#define NUMERIC_CONVERTER_CASE(TYPE_ID, TYPE_CLASS) \
+  CONVERTER_CASE(TYPE_ID,                           \
+                 (PrimitiveConverter<TYPE_CLASS, NumericValueDecoder<TYPE_CLASS>>))
+
     CONVERTER_CASE(Type::NA, NullConverter)
-    CONVERTER_CASE(Type::INT8, NumericConverter<Int8Type>)
-    CONVERTER_CASE(Type::INT16, NumericConverter<Int16Type>)
-    CONVERTER_CASE(Type::INT32, NumericConverter<Int32Type>)
-    CONVERTER_CASE(Type::INT64, NumericConverter<Int64Type>)
-    CONVERTER_CASE(Type::UINT8, NumericConverter<UInt8Type>)
-    CONVERTER_CASE(Type::UINT16, NumericConverter<UInt16Type>)
-    CONVERTER_CASE(Type::UINT32, NumericConverter<UInt32Type>)
-    CONVERTER_CASE(Type::UINT64, NumericConverter<UInt64Type>)
-    CONVERTER_CASE(Type::FLOAT, NumericConverter<FloatType>)
-    CONVERTER_CASE(Type::DOUBLE, NumericConverter<DoubleType>)
-    CONVERTER_CASE(Type::BOOL, BooleanConverter)
-    CONVERTER_CASE(Type::TIMESTAMP, TimestampConverter)
-    CONVERTER_CASE(Type::BINARY, (BinaryConverter<BinaryType, false>))
-    CONVERTER_CASE(Type::LARGE_BINARY, (BinaryConverter<LargeBinaryType, false>))
-    CONVERTER_CASE(Type::FIXED_SIZE_BINARY, FixedSizeBinaryConverter)
-    CONVERTER_CASE(Type::DECIMAL, DecimalConverter)
+    NUMERIC_CONVERTER_CASE(Type::INT8, Int8Type)
+    NUMERIC_CONVERTER_CASE(Type::INT16, Int16Type)
+    NUMERIC_CONVERTER_CASE(Type::INT32, Int32Type)
+    NUMERIC_CONVERTER_CASE(Type::INT64, Int64Type)
+    NUMERIC_CONVERTER_CASE(Type::UINT8, UInt8Type)
+    NUMERIC_CONVERTER_CASE(Type::UINT16, UInt16Type)
+    NUMERIC_CONVERTER_CASE(Type::UINT32, UInt32Type)
+    NUMERIC_CONVERTER_CASE(Type::UINT64, UInt64Type)
+    NUMERIC_CONVERTER_CASE(Type::FLOAT, FloatType)
+    NUMERIC_CONVERTER_CASE(Type::DOUBLE, DoubleType)
+    NUMERIC_CONVERTER_CASE(Type::DATE32, Date32Type)
+    NUMERIC_CONVERTER_CASE(Type::DATE64, Date64Type)
+    CONVERTER_CASE(Type::BOOL, (PrimitiveConverter<BooleanType, BooleanValueDecoder>))
+    CONVERTER_CASE(Type::BINARY,
+                   (PrimitiveConverter<BinaryType, BinaryValueDecoder<false>>))
+    CONVERTER_CASE(Type::LARGE_BINARY,
+                   (PrimitiveConverter<LargeBinaryType, BinaryValueDecoder<false>>))
+    CONVERTER_CASE(Type::FIXED_SIZE_BINARY,
+                   (PrimitiveConverter<FixedSizeBinaryType, FixedSizeBinaryValueDecoder>))
+    CONVERTER_CASE(Type::DECIMAL,
+                   (PrimitiveConverter<Decimal128Type, DecimalValueDecoder>))
+
+    case Type::TIMESTAMP:
+      ptr = MakeTimestampConverter<PrimitiveConverter>(type, options, pool);
+      break;
 
     case Type::STRING:
       if (options.check_utf8) {
-        ptr = new BinaryConverter<StringType, true>(type, options, pool);
+        ptr = std::make_shared<PrimitiveConverter<StringType, BinaryValueDecoder<true>>>(
+            type, options, pool);
       } else {
-        ptr = new BinaryConverter<StringType, false>(type, options, pool);
+        ptr = std::make_shared<PrimitiveConverter<StringType, BinaryValueDecoder<false>>>(
+            type, options, pool);
       }
       break;
 
     case Type::LARGE_STRING:
       if (options.check_utf8) {
-        ptr = new BinaryConverter<LargeStringType, true>(type, options, pool);
+        ptr = std::make_shared<
+            PrimitiveConverter<LargeStringType, BinaryValueDecoder<true>>>(type, options,
+                                                                           pool);
       } else {
-        ptr = new BinaryConverter<LargeStringType, false>(type, options, pool);
+        ptr = std::make_shared<
+            PrimitiveConverter<LargeStringType, BinaryValueDecoder<false>>>(type, options,
+                                                                            pool);
       }
       break;
 
+    case Type::DICTIONARY: {
+      const auto& dict_type = checked_cast<const DictionaryType&>(*type);
+      if (dict_type.index_type()->id() != Type::INT32) {
+        return Status::NotImplemented(
+            "CSV conversion to dictionary only supported for int32 indices, "
+            "got ",
+            type->ToString());
+      }
+      return DictionaryConverter::Make(dict_type.value_type(), options, pool);
+    }
+
     default: {
       return Status::NotImplemented("CSV conversion to ", type->ToString(),
                                     " is not supported");
     }
 
 #undef CONVERTER_CASE
+#undef NUMERIC_CONVERTER_CASE
   }
-  std::shared_ptr<Converter> result(ptr);
-  RETURN_NOT_OK(result->Initialize());
-  return result;
+  RETURN_NOT_OK(ptr->Initialize());
+  return ptr;
 }
 
 Result<std::shared_ptr<DictionaryConverter>> DictionaryConverter::Make(
     const std::shared_ptr<DataType>& type, const ConvertOptions& options,
     MemoryPool* pool) {
-  DictionaryConverter* ptr;
+  std::shared_ptr<DictionaryConverter> ptr;
 
   switch (type->id()) {
-#define CONVERTER_CASE(TYPE_ID, CONVERTER_TYPE)    \
-  case TYPE_ID:                                    \
-    ptr = new CONVERTER_TYPE(type, options, pool); \
+#define CONVERTER_CASE(TYPE_ID, TYPE, VALUE_DECODER_TYPE)                             \
+  case TYPE_ID:                                                                       \
+    ptr.reset(                                                                        \
+        new TypedDictionaryConverter<TYPE, VALUE_DECODER_TYPE>(type, options, pool)); \
     break;
 
-    CONVERTER_CASE(Type::BINARY, (DictionaryBinaryConverter<BinaryType, false>))
+    // XXX Are 32-bit types useful?
+    CONVERTER_CASE(Type::INT32, Int32Type, NumericValueDecoder<Int32Type>)
+    CONVERTER_CASE(Type::INT64, Int64Type, NumericValueDecoder<Int64Type>)
+    CONVERTER_CASE(Type::UINT32, UInt32Type, NumericValueDecoder<UInt32Type>)
+    CONVERTER_CASE(Type::UINT64, UInt64Type, NumericValueDecoder<UInt64Type>)
+    CONVERTER_CASE(Type::FLOAT, FloatType, NumericValueDecoder<FloatType>)
+    CONVERTER_CASE(Type::DOUBLE, DoubleType, NumericValueDecoder<DoubleType>)
+    CONVERTER_CASE(Type::DECIMAL, Decimal128Type, DecimalValueDecoder)
+    CONVERTER_CASE(Type::FIXED_SIZE_BINARY, FixedSizeBinaryType,
+                   FixedSizeBinaryValueDecoder)
+    CONVERTER_CASE(Type::BINARY, BinaryType, BinaryValueDecoder<false>)
+    CONVERTER_CASE(Type::LARGE_BINARY, LargeBinaryType, BinaryValueDecoder<false>)
 
     case Type::STRING:
       if (options.check_utf8) {
-        ptr = new DictionaryBinaryConverter<StringType, true>(type, options, pool);
+        ptr = std::make_shared<
+            TypedDictionaryConverter<StringType, BinaryValueDecoder<true>>>(type, options,
+                                                                            pool);
+      } else {
+        ptr = std::make_shared<
+            TypedDictionaryConverter<StringType, BinaryValueDecoder<false>>>(
+            type, options, pool);
+      }
+      break;
+
+    case Type::LARGE_STRING:
+      if (options.check_utf8) {
+        ptr = std::make_shared<
+            TypedDictionaryConverter<LargeStringType, BinaryValueDecoder<true>>>(
+            type, options, pool);
       } else {
-        ptr = new DictionaryBinaryConverter<StringType, false>(type, options, pool);
+        ptr = std::make_shared<
+            TypedDictionaryConverter<LargeStringType, BinaryValueDecoder<false>>>(
+            type, options, pool);
       }
       break;
 
@@ -623,9 +685,8 @@ Result<std::shared_ptr<DictionaryConverter>> DictionaryConverter::Make(
 
 #undef CONVERTER_CASE
   }
-  std::shared_ptr<DictionaryConverter> result(ptr);
-  RETURN_NOT_OK(result->Initialize());
-  return result;
+  RETURN_NOT_OK(ptr->Initialize());
+  return ptr;
 }
 
 }  // namespace csv
diff --git a/cpp/src/arrow/csv/converter.h b/cpp/src/arrow/csv/converter.h
index fcfcd38065e..639f692f26a 100644
--- a/cpp/src/arrow/csv/converter.h
+++ b/cpp/src/arrow/csv/converter.h
@@ -42,6 +42,7 @@ class ARROW_EXPORT Converter {
 
   std::shared_ptr<DataType> type() const { return type_; }
 
+  // Create a Converter for the given data type
   static Result<std::shared_ptr<Converter>> Make(
       const std::shared_ptr<DataType>& type, const ConvertOptions& options,
       MemoryPool* pool = default_memory_pool());
@@ -67,6 +68,8 @@ class ARROW_EXPORT DictionaryConverter : public Converter {
   // with Status::IndexError.
   virtual void SetMaxCardinality(int32_t max_length) = 0;
 
+  // Create a Converter for the given dictionary value type.
+  // The dictionary index type will always be Int32.
   static Result<std::shared_ptr<DictionaryConverter>> Make(
       const std::shared_ptr<DataType>& value_type, const ConvertOptions& options,
       MemoryPool* pool = default_memory_pool());
diff --git a/cpp/src/arrow/csv/converter_test.cc b/cpp/src/arrow/csv/converter_test.cc
index 2e5b773a9df..869a0c07569 100644
--- a/cpp/src/arrow/csv/converter_test.cc
+++ b/cpp/src/arrow/csv/converter_test.cc
@@ -24,6 +24,7 @@
 #include <gtest/gtest.h>
 
 #include "arrow/array.h"
+#include "arrow/array/builder_decimal.h"
 #include "arrow/csv/converter.h"
 #include "arrow/csv/options.h"
 #include "arrow/csv/test_common.h"
@@ -250,7 +251,8 @@ TEST(NullConversion, Basics) {
   std::shared_ptr<Array> array;
   std::shared_ptr<DataType> type = null();
 
-  ASSERT_OK_AND_ASSIGN(converter, Converter::Make(type, ConvertOptions::Defaults()));
+  auto options = ConvertOptions::Defaults();
+  ASSERT_OK_AND_ASSIGN(converter, Converter::Make(type, options));
 
   MakeCSVParser({"NA,z\n", ",0\n"}, &parser);
   ASSERT_OK_AND_ASSIGN(array, converter->Convert(*parser, 0));
@@ -353,6 +355,26 @@ TEST(BooleanConversion, CustomNulls) {
                                       {{true, false}, {false, true}}, options);
 }
 
+TEST(Date32Conversion, Basics) {
+  AssertConversion<Date32Type, int32_t>(date32(), {"1945-05-08\n", "2020-03-15\n"},
+                                        {{-9004, 18336}});
+}
+
+TEST(Date32Conversion, Nulls) {
+  AssertConversion<Date32Type, int32_t>(date32(), {"N/A\n", "2020-03-15\n"}, {{0, 18336}},
+                                        {{false, true}});
+}
+
+TEST(Date64Conversion, Basics) {
+  AssertConversion<Date64Type, int64_t>(date64(), {"1945-05-08\n", "2020-03-15\n"},
+                                        {{-777945600000LL, 1584230400000LL}});
+}
+
+TEST(Date64Conversion, Nulls) {
+  AssertConversion<Date64Type, int64_t>(date64(), {"N/A\n", "2020-03-15\n"},
+                                        {{0, 1584230400000LL}}, {{false, true}});
+}
+
 TEST(TimestampConversion, Basics) {
   auto type = timestamp(TimeUnit::SECOND);
 
@@ -452,7 +474,45 @@ TEST(DecimalConversion, OverflowFails) {
 // DictionaryConverter tests
 
 template <typename T>
-class TestDictConverter : public ::testing::Test {
+class TestNumericDictConverter : public ::testing::Test {
+ public:
+  std::shared_ptr<DataType> type() const { return TypeTraits<T>::type_singleton(); }
+};
+
+using NumericDictConversionTypes =
+    ::testing::Types<Int32Type, UInt32Type, Int64Type, UInt64Type, FloatType, DoubleType>;
+
+TYPED_TEST_SUITE(TestNumericDictConverter, NumericDictConversionTypes);
+
+TYPED_TEST(TestNumericDictConverter, Basics) {
+  auto expected_dict = ArrayFromJSON(this->type(), "[4, 5]");
+  auto expected_indices = ArrayFromJSON(int32(), "[0, 1, 0, 0]");
+
+  AssertDictConversion("4\n5\n4\n4\n", expected_indices, expected_dict);
+}
+
+TYPED_TEST(TestNumericDictConverter, Nulls) {
+  auto expected_dict = ArrayFromJSON(this->type(), "[4, 5]");
+  auto expected_indices = ArrayFromJSON(int32(), "[0, 1, null, 0]");
+
+  AssertDictConversion("4\n5\nN/A\n4\n", expected_indices, expected_dict);
+}
+
+TYPED_TEST(TestNumericDictConverter, Errors) {
+  auto value_type = this->type();
+  ASSERT_RAISES(Invalid, DictConversion(value_type, "xxx\n"));
+
+  // Overflow
+  if (is_integer(value_type->id())) {
+    ASSERT_RAISES(Invalid, DictConversion(value_type, "99999999999999999999999\n"));
+  }
+  if (is_unsigned_integer(value_type->id())) {
+    ASSERT_RAISES(Invalid, DictConversion(value_type, "-1\n"));
+  }
+}
+
+template <typename T>
+class TestStringDictConverter : public ::testing::Test {
  public:
   std::shared_ptr<DataType> type() const { return TypeTraits<T>::type_singleton(); }
 
@@ -461,18 +521,19 @@ class TestDictConverter : public ::testing::Test {
   }
 };
 
-using DictConversionTypes = ::testing::Types<BinaryType, StringType>;
+using StringDictConversionTypes =
+    ::testing::Types<BinaryType, LargeBinaryType, StringType, LargeStringType>;
 
-TYPED_TEST_SUITE(TestDictConverter, DictConversionTypes);
+TYPED_TEST_SUITE(TestStringDictConverter, StringDictConversionTypes);
 
-TYPED_TEST(TestDictConverter, Basics) {
+TYPED_TEST(TestStringDictConverter, Basics) {
   auto expected_dict = ArrayFromJSON(this->type(), R"(["ab", "cdé", ""])");
   auto expected_indices = ArrayFromJSON(int32(), "[0, 1, 2, 0]");
 
   AssertDictConversion("ab\ncdé\n\nab\n", expected_indices, expected_dict);
 }
 
-TYPED_TEST(TestDictConverter, Nulls) {
+TYPED_TEST(TestStringDictConverter, Nulls) {
   auto expected_dict = ArrayFromJSON(this->type(), R"(["ab", "N/A", ""])");
   auto expected_indices = ArrayFromJSON(int32(), "[0, 1, 2, 0]");
 
@@ -485,7 +546,7 @@ TYPED_TEST(TestDictConverter, Nulls) {
   AssertDictConversion("ab\nN/A\n\nab\n", expected_indices, expected_dict, -1, options);
 }
 
-TYPED_TEST(TestDictConverter, NonUTF8) {
+TYPED_TEST(TestStringDictConverter, NonUTF8) {
   auto expected_indices = ArrayFromJSON(int32(), "[0, 1, 2, 0]");
   std::shared_ptr<Array> expected_dict;
   ArrayFromVector<TypeParam, std::string>({"ab", "cd\xff", ""}, &expected_dict);
@@ -503,7 +564,7 @@ TYPED_TEST(TestDictConverter, NonUTF8) {
   }
 }
 
-TYPED_TEST(TestDictConverter, MaxCardinality) {
+TYPED_TEST(TestStringDictConverter, MaxCardinality) {
   auto expected_dict = ArrayFromJSON(this->type(), R"(["ab", "cd", "ef"])");
   auto expected_indices = ArrayFromJSON(int32(), "[0, 1, 2, 1]");
   std::string csv_string = "ab\ncd\nef\ncd\n";
@@ -512,5 +573,38 @@ TYPED_TEST(TestDictConverter, MaxCardinality) {
   ASSERT_RAISES(IndexError, DictConversion(this->type(), csv_string, 2));
 }
 
+TEST(TestFixedSizeBinaryDictConverter, Basics) {
+  auto value_type = fixed_size_binary(3);
+
+  auto expected_dict = ArrayFromJSON(value_type, R"(["abc", "def"])");
+  auto expected_indices = ArrayFromJSON(int32(), "[0, 1, 0, 1]");
+
+  AssertDictConversion("abc\ndef\nabc\ndef\n", expected_indices, expected_dict);
+}
+
+TEST(TestFixedSizeBinaryDictConverter, Errors) {
+  auto value_type = fixed_size_binary(3);
+
+  // Invalid string size
+  ASSERT_RAISES(Invalid, DictConversion(value_type, "abc\nde\n"));
+}
+
+TEST(TestDecimalDictConverter, Basics) {
+  auto value_type = decimal(9, 3);
+
+  auto expected_dict = ArrayFromJSON(value_type, R"(["1.234", "456.789"])");
+  auto expected_indices = ArrayFromJSON(int32(), "[0, 1, null, 1]");
+
+  AssertDictConversion("1.234\n456.789\nN/A\n4.56789e2\n", expected_indices,
+                       expected_dict);
+}
+
+TEST(TestDecimalDictConverter, Errors) {
+  auto value_type = decimal(9, 3);
+
+  // Overflow
+  ASSERT_RAISES(Invalid, DictConversion(value_type, "1e10\n"));
+}
+
 }  // namespace csv
 }  // namespace arrow
diff --git a/cpp/src/arrow/csv/parser.cc b/cpp/src/arrow/csv/parser.cc
index 77ad402da1f..07e561276fa 100644
--- a/cpp/src/arrow/csv/parser.cc
+++ b/cpp/src/arrow/csv/parser.cc
@@ -30,43 +30,22 @@
 namespace arrow {
 namespace csv {
 
-static Status ParseError(const char* message) {
+using detail::DataBatch;
+using detail::ParsedValueDesc;
+
+namespace {
+
+Status ParseError(const char* message) {
   return Status::Invalid("CSV parse error: ", message);
 }
 
-static Status MismatchingColumns(int32_t expected, int32_t actual) {
+Status MismatchingColumns(int32_t expected, int32_t actual) {
   char s[50];
   snprintf(s, sizeof(s), "Expected %d columns, got %d", expected, actual);
   return ParseError(s);
 }
 
-static inline bool IsControlChar(uint8_t c) { return c < ' '; }
-
-int32_t SkipRows(const uint8_t* data, uint32_t size, int32_t num_rows,
-                 const uint8_t** out_data) {
-  const auto end = data + size;
-  int32_t skipped_rows = 0;
-  *out_data = data;
-
-  for (; skipped_rows < num_rows; ++skipped_rows) {
-    uint8_t c;
-    do {
-      while (ARROW_PREDICT_FALSE(data < end && !IsControlChar(*data))) {
-        ++data;
-      }
-      if (ARROW_PREDICT_FALSE(data == end)) {
-        return skipped_rows;
-      }
-      c = *data++;
-    } while (c != '\r' && c != '\n');
-    if (c == '\r' && data < end && *data == '\n') {
-      ++data;
-    }
-    *out_data = data;
-  }
-
-  return skipped_rows;
-}
+inline bool IsControlChar(uint8_t c) { return c < ' '; }
 
 template <bool Quoting, bool Escaping>
 class SpecializedOptions {
@@ -77,9 +56,9 @@ class SpecializedOptions {
 
 // A helper class allocating the buffer for parsed values and writing into it
 // without any further resizes, except at the end.
-class BlockParser::PresizedParsedWriter {
+class PresizedDataWriter {
  public:
-  PresizedParsedWriter(MemoryPool* pool, uint32_t size)
+  PresizedDataWriter(MemoryPool* pool, uint32_t size)
       : parsed_size_(0), parsed_capacity_(size) {
     parsed_buffer_ = *AllocateResizableBuffer(parsed_capacity_, pool);
     parsed_ = parsed_buffer_->mutable_data();
@@ -111,51 +90,44 @@ class BlockParser::PresizedParsedWriter {
   int64_t saved_parsed_size_;
 };
 
-// A helper class handling a growable buffer for values offsets.  This class is
-// used when the number of columns is not yet known and we therefore cannot
-// efficiently presize the target area for a given number of rows.
-class BlockParser::ResizableValuesWriter {
+template <typename Derived>
+class ValueDescWriter {
  public:
-  explicit ResizableValuesWriter(MemoryPool* pool)
-      : values_size_(0), values_capacity_(256) {
-    values_buffer_ = *AllocateResizableBuffer(values_capacity_ * sizeof(*values_), pool);
-    values_ = reinterpret_cast<ValueDesc*>(values_buffer_->mutable_data());
-  }
-
-  template <typename ParsedWriter>
-  void Start(ParsedWriter& parsed_writer) {
-    PushValue({static_cast<uint32_t>(parsed_writer.size()) & 0x7fffffffU, false});
-  }
+  Derived* derived() { return static_cast<Derived*>(this); }
 
-  void Finish(std::shared_ptr<Buffer>* out_values) {
-    ARROW_CHECK_OK(values_buffer_->Resize(values_size_ * sizeof(*values_)));
-    *out_values = values_buffer_;
+  template <typename DataWriter>
+  void Start(DataWriter& parsed_writer) {
+    derived()->PushValue(
+        {static_cast<uint32_t>(parsed_writer.size()) & 0x7fffffffU, false});
   }
 
   void BeginLine() { saved_values_size_ = values_size_; }
 
+  // Rollback the state that was saved in BeginLine()
+  void RollbackLine() { values_size_ = saved_values_size_; }
+
   void StartField(bool quoted) { quoted_ = quoted; }
 
-  template <typename ParsedWriter>
-  void FinishField(ParsedWriter* parsed_writer) {
-    PushValue({static_cast<uint32_t>(parsed_writer->size()) & 0x7fffffffU, quoted_});
+  template <typename DataWriter>
+  void FinishField(DataWriter* parsed_writer) {
+    derived()->PushValue(
+        {static_cast<uint32_t>(parsed_writer->size()) & 0x7fffffffU, quoted_});
   }
 
-  // Rollback the state that was saved in BeginLine()
-  void RollbackLine() { values_size_ = saved_values_size_; }
+  void Finish(std::shared_ptr<Buffer>* out_values) {
+    ARROW_CHECK_OK(values_buffer_->Resize(values_size_ * sizeof(*values_)));
+    *out_values = values_buffer_;
+  }
 
  protected:
-  void PushValue(ValueDesc v) {
-    if (ARROW_PREDICT_FALSE(values_size_ == values_capacity_)) {
-      values_capacity_ = values_capacity_ * 2;
-      ARROW_CHECK_OK(values_buffer_->Resize(values_capacity_ * sizeof(*values_)));
-      values_ = reinterpret_cast<ValueDesc*>(values_buffer_->mutable_data());
-    }
-    values_[values_size_++] = v;
+  ValueDescWriter(MemoryPool* pool, int64_t values_capacity)
+      : values_size_(0), values_capacity_(values_capacity) {
+    values_buffer_ = *AllocateResizableBuffer(values_capacity_ * sizeof(*values_), pool);
+    values_ = reinterpret_cast<ParsedValueDesc*>(values_buffer_->mutable_data());
   }
 
   std::shared_ptr<ResizableBuffer> values_buffer_;
-  ValueDesc* values_;
+  ParsedValueDesc* values_;
   int64_t values_size_;
   int64_t values_capacity_;
   bool quoted_;
@@ -163,392 +135,415 @@ class BlockParser::ResizableValuesWriter {
   int64_t saved_values_size_;
 };
 
+// A helper class handling a growable buffer for values offsets.  This class is
+// used when the number of columns is not yet known and we therefore cannot
+// efficiently presize the target area for a given number of rows.
+class ResizableValueDescWriter : public ValueDescWriter<ResizableValueDescWriter> {
+ public:
+  explicit ResizableValueDescWriter(MemoryPool* pool)
+      : ValueDescWriter(pool, /*values_capacity=*/256) {}
+
+  void PushValue(ParsedValueDesc v) {
+    if (ARROW_PREDICT_FALSE(values_size_ == values_capacity_)) {
+      values_capacity_ = values_capacity_ * 2;
+      ARROW_CHECK_OK(values_buffer_->Resize(values_capacity_ * sizeof(*values_)));
+      values_ = reinterpret_cast<ParsedValueDesc*>(values_buffer_->mutable_data());
+    }
+    values_[values_size_++] = v;
+  }
+};
+
 // A helper class allocating the buffer for values offsets and writing into it
 // without any further resizes, except at the end.  This class is used once the
 // number of columns is known, as it eliminates resizes and generates simpler,
 // faster CSV parsing code.
-class BlockParser::PresizedValuesWriter {
+class PresizedValueDescWriter : public ValueDescWriter<PresizedValueDescWriter> {
  public:
-  PresizedValuesWriter(MemoryPool* pool, int32_t num_rows, int32_t num_cols)
-      : values_size_(0), values_capacity_(1 + num_rows * num_cols) {
-    values_buffer_ = *AllocateResizableBuffer(values_capacity_ * sizeof(*values_), pool);
-    values_ = reinterpret_cast<ValueDesc*>(values_buffer_->mutable_data());
-  }
-
-  template <typename ParsedWriter>
-  void Start(ParsedWriter& parsed_writer) {
-    PushValue({static_cast<uint32_t>(parsed_writer.size()) & 0x7fffffffU, false});
-  }
-
-  void Finish(std::shared_ptr<Buffer>* out_values) {
-    ARROW_CHECK_OK(values_buffer_->Resize(values_size_ * sizeof(*values_)));
-    *out_values = values_buffer_;
-  }
+  PresizedValueDescWriter(MemoryPool* pool, int32_t num_rows, int32_t num_cols)
+      : ValueDescWriter(pool, /*values_capacity=*/1 + num_rows * num_cols) {}
 
-  void BeginLine() { saved_values_size_ = values_size_; }
-
-  void StartField(bool quoted) { quoted_ = quoted; }
-
-  template <typename ParsedWriter>
-  void FinishField(ParsedWriter* parsed_writer) {
-    PushValue({static_cast<uint32_t>(parsed_writer->size()) & 0x7fffffffU, quoted_});
-  }
-
-  // Rollback the state that was saved in BeginLine()
-  void RollbackLine() { values_size_ = saved_values_size_; }
-
- protected:
-  void PushValue(ValueDesc v) {
+  void PushValue(ParsedValueDesc v) {
     DCHECK_LT(values_size_, values_capacity_);
     values_[values_size_++] = v;
   }
-
-  std::shared_ptr<ResizableBuffer> values_buffer_;
-  ValueDesc* values_;
-  int64_t values_size_;
-  const int64_t values_capacity_;
-  bool quoted_;
-  // Checkpointing, for when an incomplete line is encountered at end of block
-  int64_t saved_values_size_;
 };
 
-template <typename SpecializedOptions, typename ValuesWriter, typename ParsedWriter>
-Status BlockParser::ParseLine(ValuesWriter* values_writer, ParsedWriter* parsed_writer,
-                              const char* data, const char* data_end, bool is_final,
-                              const char** out_data) {
-  int32_t num_cols = 0;
-  char c;
+}  // namespace
+
+class BlockParserImpl {
+ public:
+  BlockParserImpl(MemoryPool* pool, ParseOptions options, int32_t num_cols,
+                  int32_t max_num_rows)
+      : pool_(pool), options_(options), max_num_rows_(max_num_rows), batch_(num_cols) {}
+
+  const DataBatch& parsed_batch() const { return batch_; }
 
-  DCHECK_GT(data_end, data);
+  template <typename SpecializedOptions, typename ValueDescWriter, typename DataWriter>
+  Status ParseLine(ValueDescWriter* values_writer, DataWriter* parsed_writer,
+                   const char* data, const char* data_end, bool is_final,
+                   const char** out_data) {
+    int32_t num_cols = 0;
+    char c;
 
-  auto FinishField = [&]() { values_writer->FinishField(parsed_writer); };
+    DCHECK_GT(data_end, data);
 
-  values_writer->BeginLine();
-  parsed_writer->BeginLine();
+    auto FinishField = [&]() { values_writer->FinishField(parsed_writer); };
 
-  // The parsing state machine
+    values_writer->BeginLine();
+    parsed_writer->BeginLine();
 
-  // Special case empty lines: do we start with a newline separator?
-  c = *data;
-  if (ARROW_PREDICT_FALSE(IsControlChar(c))) {
-    if (c == '\r') {
-      data++;
-      if (data < data_end && *data == '\n') {
+    // The parsing state machine
+
+    // Special case empty lines: do we start with a newline separator?
+    c = *data;
+    if (ARROW_PREDICT_FALSE(IsControlChar(c))) {
+      if (c == '\r') {
         data++;
+        if (data < data_end && *data == '\n') {
+          data++;
+        }
+        goto EmptyLine;
+      }
+      if (c == '\n') {
+        data++;
+        goto EmptyLine;
       }
-      goto EmptyLine;
-    }
-    if (c == '\n') {
-      data++;
-      goto EmptyLine;
     }
-  }
 
-FieldStart:
-  // At the start of a field
-  // Quoting is only recognized at start of field
-  if (SpecializedOptions::quoting && ARROW_PREDICT_FALSE(*data == options_.quote_char)) {
-    ++data;
-    values_writer->StartField(true /* quoted */);
-    goto InQuotedField;
-  } else {
-    values_writer->StartField(false /* quoted */);
-    goto InField;
-  }
+  FieldStart:
+    // At the start of a field
+    // Quoting is only recognized at start of field
+    if (SpecializedOptions::quoting &&
+        ARROW_PREDICT_FALSE(*data == options_.quote_char)) {
+      ++data;
+      values_writer->StartField(true /* quoted */);
+      goto InQuotedField;
+    } else {
+      values_writer->StartField(false /* quoted */);
+      goto InField;
+    }
 
-InField:
-  // Inside a non-quoted part of a field
-  if (ARROW_PREDICT_FALSE(data == data_end)) {
-    goto AbortLine;
-  }
-  c = *data++;
-  if (SpecializedOptions::escaping && ARROW_PREDICT_FALSE(c == options_.escape_char)) {
+  InField:
+    // Inside a non-quoted part of a field
     if (ARROW_PREDICT_FALSE(data == data_end)) {
       goto AbortLine;
     }
     c = *data++;
-    parsed_writer->PushFieldChar(c);
-    goto InField;
-  }
-  if (ARROW_PREDICT_FALSE(c == options_.delimiter)) {
-    goto FieldEnd;
-  }
-  if (ARROW_PREDICT_FALSE(IsControlChar(c))) {
-    if (c == '\r') {
-      // In the middle of a newline separator?
-      if (ARROW_PREDICT_TRUE(data < data_end) && *data == '\n') {
-        data++;
+    if (SpecializedOptions::escaping && ARROW_PREDICT_FALSE(c == options_.escape_char)) {
+      if (ARROW_PREDICT_FALSE(data == data_end)) {
+        goto AbortLine;
       }
-      goto LineEnd;
+      c = *data++;
+      parsed_writer->PushFieldChar(c);
+      goto InField;
     }
-    if (c == '\n') {
-      goto LineEnd;
+    if (ARROW_PREDICT_FALSE(c == options_.delimiter)) {
+      goto FieldEnd;
     }
-  }
-  parsed_writer->PushFieldChar(c);
-  goto InField;
+    if (ARROW_PREDICT_FALSE(IsControlChar(c))) {
+      if (c == '\r') {
+        // In the middle of a newline separator?
+        if (ARROW_PREDICT_TRUE(data < data_end) && *data == '\n') {
+          data++;
+        }
+        goto LineEnd;
+      }
+      if (c == '\n') {
+        goto LineEnd;
+      }
+    }
+    parsed_writer->PushFieldChar(c);
+    goto InField;
 
-InQuotedField:
-  // Inside a quoted part of a field
-  if (ARROW_PREDICT_FALSE(data == data_end)) {
-    goto AbortLine;
-  }
-  c = *data++;
-  if (SpecializedOptions::escaping && ARROW_PREDICT_FALSE(c == options_.escape_char)) {
+  InQuotedField:
+    // Inside a quoted part of a field
     if (ARROW_PREDICT_FALSE(data == data_end)) {
       goto AbortLine;
     }
     c = *data++;
+    if (SpecializedOptions::escaping && ARROW_PREDICT_FALSE(c == options_.escape_char)) {
+      if (ARROW_PREDICT_FALSE(data == data_end)) {
+        goto AbortLine;
+      }
+      c = *data++;
+      parsed_writer->PushFieldChar(c);
+      goto InQuotedField;
+    }
+    if (ARROW_PREDICT_FALSE(c == options_.quote_char)) {
+      if (options_.double_quote && ARROW_PREDICT_TRUE(data < data_end) &&
+          ARROW_PREDICT_FALSE(*data == options_.quote_char)) {
+        // Double-quoting
+        ++data;
+      } else {
+        // End of single-quoting
+        goto InField;
+      }
+    }
     parsed_writer->PushFieldChar(c);
     goto InQuotedField;
-  }
-  if (ARROW_PREDICT_FALSE(c == options_.quote_char)) {
-    if (options_.double_quote && ARROW_PREDICT_TRUE(data < data_end) &&
-        ARROW_PREDICT_FALSE(*data == options_.quote_char)) {
-      // Double-quoting
-      ++data;
-    } else {
-      // End of single-quoting
-      goto InField;
-    }
-  }
-  parsed_writer->PushFieldChar(c);
-  goto InQuotedField;
-
-FieldEnd:
-  // At the end of a field
-  FinishField();
-  ++num_cols;
-  if (ARROW_PREDICT_FALSE(data == data_end)) {
-    goto AbortLine;
-  }
-  goto FieldStart;
-
-LineEnd:
-  // At the end of line
-  FinishField();
-  ++num_cols;
-  if (ARROW_PREDICT_FALSE(num_cols != num_cols_)) {
-    if (num_cols_ == -1) {
-      num_cols_ = num_cols;
-    } else {
-      return MismatchingColumns(num_cols_, num_cols);
+
+  FieldEnd:
+    // At the end of a field
+    FinishField();
+    ++num_cols;
+    if (ARROW_PREDICT_FALSE(data == data_end)) {
+      goto AbortLine;
     }
-  }
-  ++num_rows_;
-  *out_data = data;
-  return Status::OK();
+    goto FieldStart;
 
-AbortLine:
-  // Not a full line except perhaps if in final block
-  if (is_final) {
+  LineEnd:
+    // At the end of line
     FinishField();
     ++num_cols;
-    if (num_cols_ == -1) {
-      num_cols_ = num_cols;
-    } else if (num_cols != num_cols_) {
-      return MismatchingColumns(num_cols_, num_cols);
+    if (ARROW_PREDICT_FALSE(num_cols != batch_.num_cols_)) {
+      if (batch_.num_cols_ == -1) {
+        batch_.num_cols_ = num_cols;
+      } else {
+        return MismatchingColumns(batch_.num_cols_, num_cols);
+      }
     }
-    ++num_rows_;
+    ++batch_.num_rows_;
     *out_data = data;
     return Status::OK();
-  }
-  // Truncated line at end of block, rewind parsed state
-  values_writer->RollbackLine();
-  parsed_writer->RollbackLine();
-  return Status::OK();
-
-EmptyLine:
-  if (!options_.ignore_empty_lines) {
-    if (num_cols_ == -1) {
-      // Consider as single value
-      num_cols_ = 1;
+
+  AbortLine:
+    // Not a full line except perhaps if in final block
+    if (is_final) {
+      goto LineEnd;
     }
-    // Record as row of empty (null?) values
-    while (num_cols++ < num_cols_) {
-      values_writer->StartField(false /* quoted */);
-      FinishField();
+    // Truncated line at end of block, rewind parsed state
+    values_writer->RollbackLine();
+    parsed_writer->RollbackLine();
+    return Status::OK();
+
+  EmptyLine:
+    if (!options_.ignore_empty_lines) {
+      if (batch_.num_cols_ == -1) {
+        // Consider as single value
+        batch_.num_cols_ = 1;
+      }
+      // Record as row of empty (null?) values
+      while (num_cols++ < batch_.num_cols_) {
+        values_writer->StartField(false /* quoted */);
+        FinishField();
+      }
+      ++batch_.num_rows_;
     }
-    ++num_rows_;
+    *out_data = data;
+    return Status::OK();
   }
-  *out_data = data;
-  return Status::OK();
-}
 
-template <typename SpecializedOptions, typename ValuesWriter, typename ParsedWriter>
-Status BlockParser::ParseChunk(ValuesWriter* values_writer, ParsedWriter* parsed_writer,
-                               const char* data, const char* data_end, bool is_final,
-                               int32_t rows_in_chunk, const char** out_data,
-                               bool* finished_parsing) {
-  int32_t num_rows_deadline = num_rows_ + rows_in_chunk;
-
-  while (data < data_end && num_rows_ < num_rows_deadline) {
-    const char* line_end = data;
-    RETURN_NOT_OK(ParseLine<SpecializedOptions>(values_writer, parsed_writer, data,
-                                                data_end, is_final, &line_end));
-    if (line_end == data) {
-      // Cannot parse any further
-      *finished_parsing = true;
-      break;
+  template <typename SpecializedOptions, typename ValueDescWriter, typename DataWriter>
+  Status ParseChunk(ValueDescWriter* values_writer, DataWriter* parsed_writer,
+                    const char* data, const char* data_end, bool is_final,
+                    int32_t rows_in_chunk, const char** out_data,
+                    bool* finished_parsing) {
+    int32_t num_rows_deadline = batch_.num_rows_ + rows_in_chunk;
+
+    while (data < data_end && batch_.num_rows_ < num_rows_deadline) {
+      const char* line_end = data;
+      RETURN_NOT_OK(ParseLine<SpecializedOptions>(values_writer, parsed_writer, data,
+                                                  data_end, is_final, &line_end));
+      if (line_end == data) {
+        // Cannot parse any further
+        *finished_parsing = true;
+        break;
+      }
+      data = line_end;
     }
-    data = line_end;
-  }
-  // Append new buffers and update size
-  std::shared_ptr<Buffer> values_buffer;
-  values_writer->Finish(&values_buffer);
-  if (values_buffer->size() > 0) {
-    values_size_ += static_cast<int32_t>(values_buffer->size() / sizeof(ValueDesc) - 1);
-    values_buffers_.push_back(std::move(values_buffer));
+    // Append new buffers and update size
+    std::shared_ptr<Buffer> values_buffer;
+    values_writer->Finish(&values_buffer);
+    if (values_buffer->size() > 0) {
+      values_size_ +=
+          static_cast<int32_t>(values_buffer->size() / sizeof(ParsedValueDesc) - 1);
+      batch_.values_buffers_.push_back(std::move(values_buffer));
+    }
+    *out_data = data;
+    return Status::OK();
   }
-  *out_data = data;
-  return Status::OK();
-}
 
-template <typename SpecializedOptions>
-Status BlockParser::DoParseSpecialized(const std::vector<util::string_view>& views,
-                                       bool is_final, uint32_t* out_size) {
-  num_rows_ = 0;
-  values_size_ = 0;
-  parsed_size_ = 0;
-  values_buffers_.clear();
-  parsed_buffer_.reset();
-  parsed_ = nullptr;
-
-  size_t total_view_length = 0;
-  for (const auto& view : views) {
-    total_view_length += view.length();
-  }
-  if (total_view_length > std::numeric_limits<uint32_t>::max()) {
-    return Status::Invalid("CSV block too large");
-  }
+  template <typename SpecializedOptions>
+  Status ParseSpecialized(const std::vector<util::string_view>& views, bool is_final,
+                          uint32_t* out_size) {
+    batch_ = DataBatch{batch_.num_cols_};
+    values_size_ = 0;
 
-  PresizedParsedWriter parsed_writer(pool_, static_cast<uint32_t>(total_view_length));
-  uint32_t total_parsed_length = 0;
-
-  for (const auto& view : views) {
-    const char* data = view.data();
-    const char* data_end = view.data() + view.length();
-    bool finished_parsing = false;
-
-    if (num_cols_ == -1) {
-      // Can't presize values when the number of columns is not known, first parse
-      // a single line
-      const int32_t rows_in_chunk = 1;
-      ResizableValuesWriter values_writer(pool_);
-      values_writer.Start(parsed_writer);
-
-      RETURN_NOT_OK(ParseChunk<SpecializedOptions>(&values_writer, &parsed_writer, data,
-                                                   data_end, is_final, rows_in_chunk,
-                                                   &data, &finished_parsing));
-      if (num_cols_ == -1) {
-        return ParseError("Empty CSV file or block: cannot infer number of columns");
-      }
+    size_t total_view_length = 0;
+    for (const auto& view : views) {
+      total_view_length += view.length();
+    }
+    if (total_view_length > std::numeric_limits<uint32_t>::max()) {
+      return Status::Invalid("CSV block too large");
     }
 
-    while (!finished_parsing && data < data_end && num_rows_ < max_num_rows_) {
-      // We know the number of columns, so can presize a values array for
-      // a given number of rows
-      DCHECK_GE(num_cols_, 0);
-
-      int32_t rows_in_chunk;
-      constexpr int32_t kTargetChunkSize = 32768;
-      if (num_cols_ > 0) {
-        rows_in_chunk = std::min(std::max(kTargetChunkSize / num_cols_, 512),
-                                 max_num_rows_ - num_rows_);
-      } else {
-        rows_in_chunk = std::min(kTargetChunkSize, max_num_rows_ - num_rows_);
+    PresizedDataWriter parsed_writer(pool_, static_cast<uint32_t>(total_view_length));
+    uint32_t total_parsed_length = 0;
+
+    for (const auto& view : views) {
+      const char* data = view.data();
+      const char* data_end = view.data() + view.length();
+      bool finished_parsing = false;
+
+      if (batch_.num_cols_ == -1) {
+        // Can't presize values when the number of columns is not known, first parse
+        // a single line
+        const int32_t rows_in_chunk = 1;
+        ResizableValueDescWriter values_writer(pool_);
+        values_writer.Start(parsed_writer);
+
+        RETURN_NOT_OK(ParseChunk<SpecializedOptions>(&values_writer, &parsed_writer, data,
+                                                     data_end, is_final, rows_in_chunk,
+                                                     &data, &finished_parsing));
+        if (batch_.num_cols_ == -1) {
+          return ParseError("Empty CSV file or block: cannot infer number of columns");
+        }
       }
 
-      PresizedValuesWriter values_writer(pool_, rows_in_chunk, num_cols_);
-      values_writer.Start(parsed_writer);
-
-      RETURN_NOT_OK(ParseChunk<SpecializedOptions>(&values_writer, &parsed_writer, data,
-                                                   data_end, is_final, rows_in_chunk,
-                                                   &data, &finished_parsing));
-    }
-    DCHECK_GE(data, view.data());
-    DCHECK_LE(data, data_end);
-    total_parsed_length += static_cast<uint32_t>(data - view.data());
+      while (!finished_parsing && data < data_end && batch_.num_rows_ < max_num_rows_) {
+        // We know the number of columns, so can presize a values array for
+        // a given number of rows
+        DCHECK_GE(batch_.num_cols_, 0);
+
+        int32_t rows_in_chunk;
+        constexpr int32_t kTargetChunkSize = 32768;  // in number of values
+        if (batch_.num_cols_ > 0) {
+          rows_in_chunk = std::min(std::max(kTargetChunkSize / batch_.num_cols_, 512),
+                                   max_num_rows_ - batch_.num_rows_);
+        } else {
+          rows_in_chunk = std::min(kTargetChunkSize, max_num_rows_ - batch_.num_rows_);
+        }
+
+        PresizedValueDescWriter values_writer(pool_, rows_in_chunk, batch_.num_cols_);
+        values_writer.Start(parsed_writer);
+
+        RETURN_NOT_OK(ParseChunk<SpecializedOptions>(&values_writer, &parsed_writer, data,
+                                                     data_end, is_final, rows_in_chunk,
+                                                     &data, &finished_parsing));
+      }
+      DCHECK_GE(data, view.data());
+      DCHECK_LE(data, data_end);
+      total_parsed_length += static_cast<uint32_t>(data - view.data());
 
-    if (data < data_end) {
-      // Stopped early, for some reason
-      break;
+      if (data < data_end) {
+        // Stopped early, for some reason
+        break;
+      }
     }
-  }
 
-  parsed_writer.Finish(&parsed_buffer_);
-  parsed_size_ = static_cast<int32_t>(parsed_buffer_->size());
-  parsed_ = parsed_buffer_->data();
+    parsed_writer.Finish(&batch_.parsed_buffer_);
+    batch_.parsed_size_ = static_cast<int32_t>(batch_.parsed_buffer_->size());
+    batch_.parsed_ = batch_.parsed_buffer_->data();
 
-  DCHECK_EQ(values_size_, num_rows_ * num_cols_);
-  if (num_cols_ == -1) {
-    DCHECK_EQ(num_rows_, 0);
-  }
+    if (batch_.num_cols_ == -1) {
+      DCHECK_EQ(batch_.num_rows_, 0);
+    }
+    DCHECK_EQ(values_size_, batch_.num_rows_ * batch_.num_cols_);
 #ifndef NDEBUG
-  if (num_rows_ > 0) {
-    DCHECK_GT(values_buffers_.size(), 0);
-    auto& last_values_buffer = values_buffers_.back();
-    auto last_values = reinterpret_cast<const ValueDesc*>(last_values_buffer->data());
-    auto last_values_size = last_values_buffer->size() / sizeof(ValueDesc);
-    auto check_parsed_size =
-        static_cast<int32_t>(last_values[last_values_size - 1].offset);
-    DCHECK_EQ(parsed_size_, check_parsed_size);
-  } else {
-    DCHECK_EQ(parsed_size_, 0);
-  }
-#endif
-  *out_size = static_cast<uint32_t>(total_parsed_length);
-  return Status::OK();
-}
-
-Status BlockParser::DoParse(const std::vector<util::string_view>& data, bool is_final,
-                            uint32_t* out_size) {
-  if (options_.quoting) {
-    if (options_.escaping) {
-      return DoParseSpecialized<SpecializedOptions<true, true>>(data, is_final, out_size);
+    if (batch_.num_rows_ > 0) {
+      // Ending parsed offset should be equal to number of parsed bytes
+      DCHECK_GT(batch_.values_buffers_.size(), 0);
+      const auto& last_values_buffer = batch_.values_buffers_.back();
+      const auto last_values =
+          reinterpret_cast<const ParsedValueDesc*>(last_values_buffer->data());
+      const auto last_values_size = last_values_buffer->size() / sizeof(ParsedValueDesc);
+      const auto check_parsed_size =
+          static_cast<int32_t>(last_values[last_values_size - 1].offset);
+      DCHECK_EQ(batch_.parsed_size_, check_parsed_size);
     } else {
-      return DoParseSpecialized<SpecializedOptions<true, false>>(data, is_final,
-                                                                 out_size);
+      DCHECK_EQ(batch_.parsed_size_, 0);
     }
-  } else {
-    if (options_.escaping) {
-      return DoParseSpecialized<SpecializedOptions<false, true>>(data, is_final,
+#endif
+    *out_size = static_cast<uint32_t>(total_parsed_length);
+    return Status::OK();
+  }
+
+  Status Parse(const std::vector<util::string_view>& data, bool is_final,
+               uint32_t* out_size) {
+    if (options_.quoting) {
+      if (options_.escaping) {
+        return ParseSpecialized<SpecializedOptions<true, true>>(data, is_final, out_size);
+      } else {
+        return ParseSpecialized<SpecializedOptions<true, false>>(data, is_final,
                                                                  out_size);
+      }
     } else {
-      return DoParseSpecialized<SpecializedOptions<false, false>>(data, is_final,
+      if (options_.escaping) {
+        return ParseSpecialized<SpecializedOptions<false, true>>(data, is_final,
+                                                                 out_size);
+      } else {
+        return ParseSpecialized<SpecializedOptions<false, false>>(data, is_final,
                                                                   out_size);
+      }
     }
   }
-}
+
+ protected:
+  MemoryPool* pool_;
+  const ParseOptions options_;
+  // The maximum number of rows to parse from a block
+  int32_t max_num_rows_;
+
+  // Unparsed data size
+  int32_t values_size_;
+  // Parsed data batch
+  DataBatch batch_;
+};
+
+BlockParser::BlockParser(ParseOptions options, int32_t num_cols, int32_t max_num_rows)
+    : BlockParser(default_memory_pool(), options, num_cols, max_num_rows) {}
+
+BlockParser::BlockParser(MemoryPool* pool, ParseOptions options, int32_t num_cols,
+                         int32_t max_num_rows)
+    : impl_(new BlockParserImpl(pool, std::move(options), num_cols, max_num_rows)) {}
+
+BlockParser::~BlockParser() {}
 
 Status BlockParser::Parse(const std::vector<util::string_view>& data,
                           uint32_t* out_size) {
-  return DoParse(data, false /* is_final */, out_size);
+  return impl_->Parse(data, false /* is_final */, out_size);
 }
 
 Status BlockParser::ParseFinal(const std::vector<util::string_view>& data,
                                uint32_t* out_size) {
-  return DoParse(data, true /* is_final */, out_size);
+  return impl_->Parse(data, true /* is_final */, out_size);
 }
 
 Status BlockParser::Parse(util::string_view data, uint32_t* out_size) {
-  return DoParse({data}, false /* is_final */, out_size);
+  return impl_->Parse({data}, false /* is_final */, out_size);
 }
 
 Status BlockParser::ParseFinal(util::string_view data, uint32_t* out_size) {
-  return DoParse({data}, true /* is_final */, out_size);
+  return impl_->Parse({data}, true /* is_final */, out_size);
 }
 
-BlockParser::BlockParser(MemoryPool* pool, ParseOptions options, int32_t num_cols,
-                         int32_t max_num_rows)
-    : pool_(pool),
-      options_(options),
-      num_rows_(-1),
-      num_cols_(num_cols),
-      max_num_rows_(max_num_rows) {}
+const DataBatch& BlockParser::parsed_batch() const { return impl_->parsed_batch(); }
 
-BlockParser::BlockParser(ParseOptions options, int32_t num_cols, int32_t max_num_rows)
-    : BlockParser(default_memory_pool(), options, num_cols, max_num_rows) {}
+int32_t SkipRows(const uint8_t* data, uint32_t size, int32_t num_rows,
+                 const uint8_t** out_data) {
+  const auto end = data + size;
+  int32_t skipped_rows = 0;
+  *out_data = data;
+
+  for (; skipped_rows < num_rows; ++skipped_rows) {
+    uint8_t c;
+    do {
+      while (ARROW_PREDICT_FALSE(data < end && !IsControlChar(*data))) {
+        ++data;
+      }
+      if (ARROW_PREDICT_FALSE(data == end)) {
+        return skipped_rows;
+      }
+      c = *data++;
+    } while (c != '\r' && c != '\n');
+    if (c == '\r' && data < end && *data == '\n') {
+      ++data;
+    }
+    *out_data = data;
+  }
+
+  return skipped_rows;
+}
 
 }  // namespace csv
 }  // namespace arrow
diff --git a/cpp/src/arrow/csv/parser.h b/cpp/src/arrow/csv/parser.h
index ad44ef28f2e..4fcc52fb3a6 100644
--- a/cpp/src/arrow/csv/parser.h
+++ b/cpp/src/arrow/csv/parser.h
@@ -35,8 +35,6 @@ class MemoryPool;
 
 namespace csv {
 
-constexpr int32_t kMaxParserNumRows = 100000;
-
 /// Skip at most num_rows from the given input.  The input pointer is updated
 /// and the number of actually skipped rows is returns (may be less than
 /// requested if the input is too short).
@@ -44,6 +42,83 @@ ARROW_EXPORT
 int32_t SkipRows(const uint8_t* data, uint32_t size, int32_t num_rows,
                  const uint8_t** out_data);
 
+class BlockParserImpl;
+
+namespace detail {
+
+struct ParsedValueDesc {
+  uint32_t offset : 31;
+  bool quoted : 1;
+};
+
+class ARROW_EXPORT DataBatch {
+ public:
+  explicit DataBatch(int32_t num_cols) : num_cols_(num_cols) {}
+
+  /// \brief Return the number of parsed rows
+  int32_t num_rows() const { return num_rows_; }
+  /// \brief Return the number of parsed columns
+  int32_t num_cols() const { return num_cols_; }
+  /// \brief Return the total size in bytes of parsed data
+  uint32_t num_bytes() const { return parsed_size_; }
+
+  template <typename Visitor>
+  Status VisitColumn(int32_t col_index, Visitor&& visit) const {
+    using detail::ParsedValueDesc;
+
+    for (size_t buf_index = 0; buf_index < values_buffers_.size(); ++buf_index) {
+      const auto& values_buffer = values_buffers_[buf_index];
+      const auto values = reinterpret_cast<const ParsedValueDesc*>(values_buffer->data());
+      const auto max_pos =
+          static_cast<int32_t>(values_buffer->size() / sizeof(ParsedValueDesc)) - 1;
+      for (int32_t pos = col_index; pos < max_pos; pos += num_cols_) {
+        auto start = values[pos].offset;
+        auto stop = values[pos + 1].offset;
+        auto quoted = values[pos + 1].quoted;
+        ARROW_RETURN_NOT_OK(visit(parsed_ + start, stop - start, quoted));
+      }
+    }
+    return Status::OK();
+  }
+
+  template <typename Visitor>
+  Status VisitLastRow(Visitor&& visit) const {
+    using detail::ParsedValueDesc;
+
+    const auto& values_buffer = values_buffers_.back();
+    const auto values = reinterpret_cast<const ParsedValueDesc*>(values_buffer->data());
+    const auto start_pos =
+        static_cast<int32_t>(values_buffer->size() / sizeof(ParsedValueDesc)) -
+        num_cols_ - 1;
+    for (int32_t col_index = 0; col_index < num_cols_; ++col_index) {
+      auto start = values[start_pos + col_index].offset;
+      auto stop = values[start_pos + col_index + 1].offset;
+      auto quoted = values[start_pos + col_index + 1].quoted;
+      ARROW_RETURN_NOT_OK(visit(parsed_ + start, stop - start, quoted));
+    }
+    return Status::OK();
+  }
+
+ protected:
+  // The number of rows in this batch
+  int32_t num_rows_ = 0;
+  // The number of columns
+  int32_t num_cols_ = 0;
+
+  // XXX should we ensure the parsed buffer is padded with 8 or 16 excess zero bytes?
+  // It may help with null parsing...
+  std::vector<std::shared_ptr<Buffer>> values_buffers_;
+  std::shared_ptr<Buffer> parsed_buffer_;
+  const uint8_t* parsed_ = NULLPTR;
+  int32_t parsed_size_ = 0;
+
+  friend class ::arrow::csv::BlockParserImpl;
+};
+
+}  // namespace detail
+
+constexpr int32_t kMaxParserNumRows = 100000;
+
 /// \class BlockParser
 /// \brief A reusable block-based parser for CSV data
 ///
@@ -62,6 +137,7 @@ class ARROW_EXPORT BlockParser {
                        int32_t max_num_rows = kMaxParserNumRows);
   explicit BlockParser(MemoryPool* pool, ParseOptions options, int32_t num_cols = -1,
                        int32_t max_num_rows = kMaxParserNumRows);
+  ~BlockParser();
 
   /// \brief Parse a block of data
   ///
@@ -86,11 +162,11 @@ class ARROW_EXPORT BlockParser {
   Status ParseFinal(const std::vector<util::string_view>& data, uint32_t* out_size);
 
   /// \brief Return the number of parsed rows
-  int32_t num_rows() const { return num_rows_; }
+  int32_t num_rows() const { return parsed_batch().num_rows(); }
   /// \brief Return the number of parsed columns
-  int32_t num_cols() const { return num_cols_; }
+  int32_t num_cols() const { return parsed_batch().num_cols(); }
   /// \brief Return the total size in bytes of parsed data
-  uint32_t num_bytes() const { return parsed_size_; }
+  uint32_t num_bytes() const { return parsed_batch().num_bytes(); }
 
   /// \brief Visit parsed values in a column
   ///
@@ -98,82 +174,18 @@ class ARROW_EXPORT BlockParser {
   /// Status(const uint8_t* data, uint32_t size, bool quoted)
   template <typename Visitor>
   Status VisitColumn(int32_t col_index, Visitor&& visit) const {
-    for (size_t buf_index = 0; buf_index < values_buffers_.size(); ++buf_index) {
-      const auto& values_buffer = values_buffers_[buf_index];
-      const auto values = reinterpret_cast<const ValueDesc*>(values_buffer->data());
-      const auto max_pos =
-          static_cast<int32_t>(values_buffer->size() / sizeof(ValueDesc)) - 1;
-      for (int32_t pos = col_index; pos < max_pos; pos += num_cols_) {
-        auto start = values[pos].offset;
-        auto stop = values[pos + 1].offset;
-        auto quoted = values[pos + 1].quoted;
-        ARROW_RETURN_NOT_OK(visit(parsed_ + start, stop - start, quoted));
-      }
-    }
-    return Status::OK();
+    return parsed_batch().VisitColumn(col_index, std::forward<Visitor>(visit));
   }
 
   template <typename Visitor>
   Status VisitLastRow(Visitor&& visit) const {
-    const auto& values_buffer = values_buffers_.back();
-    const auto values = reinterpret_cast<const ValueDesc*>(values_buffer->data());
-    const auto start_pos =
-        static_cast<int32_t>(values_buffer->size() / sizeof(ValueDesc)) - num_cols_ - 1;
-    for (int32_t col_index = 0; col_index < num_cols_; ++col_index) {
-      auto start = values[start_pos + col_index].offset;
-      auto stop = values[start_pos + col_index + 1].offset;
-      auto quoted = values[start_pos + col_index + 1].quoted;
-      ARROW_RETURN_NOT_OK(visit(parsed_ + start, stop - start, quoted));
-    }
-    return Status::OK();
+    return parsed_batch().VisitLastRow(std::forward<Visitor>(visit));
   }
 
  protected:
-  ARROW_DISALLOW_COPY_AND_ASSIGN(BlockParser);
-
-  Status DoParse(const std::vector<util::string_view>& data, bool is_final,
-                 uint32_t* out_size);
-  template <typename SpecializedOptions>
-  Status DoParseSpecialized(const std::vector<util::string_view>& data, bool is_final,
-                            uint32_t* out_size);
-
-  template <typename SpecializedOptions, typename ValuesWriter, typename ParsedWriter>
-  Status ParseChunk(ValuesWriter* values_writer, ParsedWriter* parsed_writer,
-                    const char* data, const char* data_end, bool is_final,
-                    int32_t rows_in_chunk, const char** out_data, bool* finished_parsing);
-
-  // Parse a single line from the data pointer
-  template <typename SpecializedOptions, typename ValuesWriter, typename ParsedWriter>
-  Status ParseLine(ValuesWriter* values_writer, ParsedWriter* parsed_writer,
-                   const char* data, const char* data_end, bool is_final,
-                   const char** out_data);
-
-  MemoryPool* pool_;
-  const ParseOptions options_;
-  // The number of rows parsed from the block
-  int32_t num_rows_;
-  // The number of columns (can be -1 at start)
-  int32_t num_cols_;
-  // The maximum number of rows to parse from this block
-  int32_t max_num_rows_;
-
-  // Linear scratchpad for parsed values
-  struct ValueDesc {
-    uint32_t offset : 31;
-    bool quoted : 1;
-  };
-
-  // XXX should we ensure the parsed buffer is padded with 8 or 16 excess zero bytes?
-  // It may help with null parsing...
-  std::vector<std::shared_ptr<Buffer>> values_buffers_;
-  std::shared_ptr<Buffer> parsed_buffer_;
-  const uint8_t* parsed_;
-  int32_t values_size_;
-  int32_t parsed_size_;
+  std::unique_ptr<BlockParserImpl> impl_;
 
-  class ResizableValuesWriter;
-  class PresizedValuesWriter;
-  class PresizedParsedWriter;
+  const detail::DataBatch& parsed_batch() const;
 };
 
 }  // namespace csv
diff --git a/cpp/src/arrow/csv/parser_benchmark.cc b/cpp/src/arrow/csv/parser_benchmark.cc
index 3012754ee0b..b279a3c0ce3 100644
--- a/cpp/src/arrow/csv/parser_benchmark.cc
+++ b/cpp/src/arrow/csv/parser_benchmark.cc
@@ -30,17 +30,46 @@
 namespace arrow {
 namespace csv {
 
-// Linter stipulates:
-// >> For a static/global string constant, use a C style string instead
-const char* one_row = "abc,\"d,f\",12.34,\n";
-const char* one_row_escaped = "abc,d\\,f,12.34,\n";
-
-const auto num_rows = static_cast<int32_t>((1024 * 64) / strlen(one_row));
-
-static std::string BuildCSVData(const std::string& row, int32_t repeat) {
+struct Example {
+  int32_t num_rows;
+  const char* csv_rows;
+};
+
+const Example quoted_example{1, "abc,\"d,f\",12.34,\n"};
+const Example escaped_example{1, "abc,d\\,f,12.34,\n"};
+
+const Example flights_example{
+    8,
+    R"(2015,1,1,4,AA,2336,N3KUAA,LAX,PBI,0010,0002,-8,12,0014,280,279,263,2330,0737,4,0750,0741,-9,0,0,,,,,,
+2015,1,1,4,US,840,N171US,SFO,CLT,0020,0018,-2,16,0034,286,293,266,2296,0800,11,0806,0811,5,0,0,,,,,,
+2015,1,1,4,AA,258,N3HYAA,LAX,MIA,0020,0015,-5,15,0030,285,281,258,2342,0748,8,0805,0756,-9,0,0,,,,,,
+2015,1,1,4,AS,135,N527AS,SEA,ANC,0025,0024,-1,11,0035,235,215,199,1448,0254,5,0320,0259,-21,0,0,,,,,,
+2015,1,1,4,DL,806,N3730B,SFO,MSP,0025,0020,-5,18,0038,217,230,206,1589,0604,6,0602,0610,8,0,0,,,,,,
+2015,1,1,4,NK,612,N635NK,LAS,MSP,0025,0019,-6,11,0030,181,170,154,1299,0504,5,0526,0509,-17,0,0,,,,,,
+2015,1,1,4,US,2013,N584UW,LAX,CLT,0030,0044,14,13,0057,273,249,228,2125,0745,8,0803,0753,-10,0,0,,,,,,
+2015,1,1,4,AA,1112,N3LAAA,SFO,DFW,0030,0019,-11,17,0036,195,193,173,1464,0529,3,0545,0532,-13,0,0,,,,,,
+)"};
+
+// NOTE: quoted
+const Example vehicles_example{
+    2,
+    R"(7088743681,https://greensboro.craigslist.org/ctd/d/cary-2004-honda-element-lx-4dr-suv/7088743681.html,greensboro,https://greensboro.craigslist.org,3995,2004,honda,element,,,gas,212526,clean,automatic,5J6YH18314L006498,fwd,,SUV,orange,https://images.craigslist.org/00E0E_eAUnhFF86M4_600x450.jpg,"2004 Honda Element LX 4dr SUV     Offered by: Best Import Auto Sales Inc — (919) 800-0650 — $3,995     EXCELLENT SHAPE INSIDE AND OUT FULLY SERVICED AND READY TO GO ,RUNS AND DRIVES PERFECT ,PLEASE CALL OR TEXT 919 454 4848 OR CALL 919 380 0380 IF INTERESTED.   Best Import Auto Sales Inc    Year: 2004 Make: Honda Model: Element Series: LX 4dr SUV VIN: 5J6YH18314L006498 Stock #: 4L006498 Condition: Used Mileage: 212,526  Exterior: Orange Interior: Black Body: SUV Transmission: Automatic 4-Speed Engine: 2.4L I4      **** Best Import Auto Sales Inc. 🚘 Raleigh Auto Dealer *****  ⚡️⚡️⚡️ Call Or Text (919) 800-0650 ⚡️⚡️⚡️  ✅ - We can arrange Financing Options with most banks and credit unions!!!!     ✅ Extended Warranties Available on most vehicles!! ""Call To Inquire""  ✅ Full Service ASE-Certified Shop Onsite!       More vehicle details: best-import-auto-sales-inc.hammerwebsites.net/v/3kE08kSD     Address: 1501 Buck Jones Rd Raleigh, NC 27606   Phone: (919) 800-0650     Website: www.bestimportsonline.com      📲 ☎️ Call or text (919) 800-0650 for quick answers to your questions about this Honda Element Your message will always be answered by a real human — never an automated system.     Disclaimer: Best Import Auto Sales Inc will never sell, share, or spam your mobile number. Standard text messaging rates may apply.       2004 Honda Element LX 4dr SUV   6fbc204ebd7e4a32a30dcf2c8c3bcdea",,nc,35.7636,-78.7443
+  7088744126,https://greensboro.craigslist.org/cto/d/greensboro-2011-jaguar-xf-premier/7088744126.html,greensboro,https://greensboro.craigslist.org,9500,2011,jaguar,xf,excellent,,gas,85000,clean,automatic,,,,,blue,https://images.craigslist.org/00505_f22HGItCRpc_600x450.jpg,"2011 jaguar XF premium - estate sale. Retired lady executive. Like new, garaged and maintained. Very nice leather, heated seats, electric sunroof, metallic blue paint. 85K miles bumper-to-bumper warranty. Premium radio sound system. Built-in phone connection. Please call  show contact info  cell or  show contact info .  Asking Price $9500",,nc,36.1032,-79.8794
+)"};
+
+const Example stocks_example{
+    3,
+    R"(2,2010-01-27 00:00:00,002204,华锐铸钢,536498.0,135378.0,2652784.2001924426,14160629.45,5.382023337513902,5.288274712474071,5.382023337513902,5.341540976701248,,5.338025403262254,1.01364599,0.21306505690870553
+3,2010-02-05 00:00:00,600266,北京城建,1122615.0,1122615.0,8102476.086666377,57695471.0,7.236029036381633,7.025270909108382,7.170459841229955,7.095523618199466,,7.120720923193468,2.3025570905818964,0.4683513939405588
+4,2010-01-04 00:00:00,600289,亿阳信通,602926.359,602926.359,16393247.138998777,167754890.0,10.381817699665978,9.960037526145015,10.092597009251604,10.321563389162982,,10.233170315655089,4.436963485334562,0.6025431050299465
+)"};
+
+static constexpr int32_t kNumRows = 10000;
+
+static std::string BuildCSVData(const Example& example) {
   std::stringstream ss;
-  for (int32_t i = 0; i < repeat; ++i) {
-    ss << row;
+  for (int32_t i = 0; i < kNumRows; i += example.num_rows) {
+    ss << example.csv_rows;
   }
   return ss.str();
 }
@@ -60,7 +89,7 @@ static void BenchmarkCSVChunking(benchmark::State& state,  // NOLINT non-const r
 }
 
 static void ChunkCSVQuotedBlock(benchmark::State& state) {  // NOLINT non-const reference
-  auto csv = BuildCSVData(one_row, num_rows);
+  auto csv = BuildCSVData(quoted_example);
   auto options = ParseOptions::Defaults();
   options.quoting = true;
   options.escaping = false;
@@ -70,7 +99,7 @@ static void ChunkCSVQuotedBlock(benchmark::State& state) {  // NOLINT non-const
 }
 
 static void ChunkCSVEscapedBlock(benchmark::State& state) {  // NOLINT non-const reference
-  auto csv = BuildCSVData(one_row_escaped, num_rows);
+  auto csv = BuildCSVData(escaped_example);
   auto options = ParseOptions::Defaults();
   options.quoting = false;
   options.escaping = true;
@@ -81,7 +110,7 @@ static void ChunkCSVEscapedBlock(benchmark::State& state) {  // NOLINT non-const
 
 static void ChunkCSVNoNewlinesBlock(
     benchmark::State& state) {  // NOLINT non-const reference
-  auto csv = BuildCSVData(one_row_escaped, num_rows);
+  auto csv = BuildCSVData(escaped_example);
   auto options = ParseOptions::Defaults();
   options.quoting = true;
   options.escaping = false;
@@ -94,9 +123,9 @@ static void ChunkCSVNoNewlinesBlock(
 }
 
 static void BenchmarkCSVParsing(benchmark::State& state,  // NOLINT non-const reference
-                                const std::string& csv, int32_t rows,
+                                const std::string& csv, int32_t num_rows,
                                 ParseOptions options) {
-  BlockParser parser(options, -1, rows + 1);
+  BlockParser parser(options, -1, num_rows + 1);
 
   while (state.KeepRunning()) {
     uint32_t parsed_size = 0;
@@ -121,29 +150,56 @@ static void BenchmarkCSVParsing(benchmark::State& state,  // NOLINT non-const re
   state.SetBytesProcessed(state.iterations() * csv.size());
 }
 
+static void BenchmarkCSVParsing(benchmark::State& state,  // NOLINT non-const reference
+                                const Example& example, ParseOptions options) {
+  auto csv = BuildCSVData(example);
+  BenchmarkCSVParsing(state, csv, kNumRows, options);
+}
+
 static void ParseCSVQuotedBlock(benchmark::State& state) {  // NOLINT non-const reference
-  auto csv = BuildCSVData(one_row, num_rows);
   auto options = ParseOptions::Defaults();
   options.quoting = true;
   options.escaping = false;
 
-  BenchmarkCSVParsing(state, csv, num_rows, options);
+  BenchmarkCSVParsing(state, quoted_example, options);
 }
 
 static void ParseCSVEscapedBlock(benchmark::State& state) {  // NOLINT non-const reference
-  auto csv = BuildCSVData(one_row_escaped, num_rows);
   auto options = ParseOptions::Defaults();
   options.quoting = false;
   options.escaping = true;
 
-  BenchmarkCSVParsing(state, csv, num_rows, options);
+  BenchmarkCSVParsing(state, escaped_example, options);
+}
+
+static void ParseCSVFlightsExample(
+    benchmark::State& state) {  // NOLINT non-const reference
+  BenchmarkCSVParsing(state, flights_example, ParseOptions::Defaults());
+}
+
+static void ParseCSVVehiclesExample(
+    benchmark::State& state) {  // NOLINT non-const reference
+  auto options = ParseOptions::Defaults();
+  options.quoting = true;
+  options.escaping = false;
+
+  BenchmarkCSVParsing(state, vehicles_example, options);
+}
+
+static void ParseCSVStocksExample(
+    benchmark::State& state) {  // NOLINT non-const reference
+  BenchmarkCSVParsing(state, stocks_example, ParseOptions::Defaults());
 }
 
 BENCHMARK(ChunkCSVQuotedBlock);
 BENCHMARK(ChunkCSVEscapedBlock);
 BENCHMARK(ChunkCSVNoNewlinesBlock);
+
 BENCHMARK(ParseCSVQuotedBlock);
 BENCHMARK(ParseCSVEscapedBlock);
+BENCHMARK(ParseCSVFlightsExample);
+BENCHMARK(ParseCSVVehiclesExample);
+BENCHMARK(ParseCSVStocksExample);
 
 }  // namespace csv
 }  // namespace arrow
diff --git a/cpp/src/arrow/dataset/dataset_test.cc b/cpp/src/arrow/dataset/dataset_test.cc
index 7a378cd9210..0b138caafe7 100644
--- a/cpp/src/arrow/dataset/dataset_test.cc
+++ b/cpp/src/arrow/dataset/dataset_test.cc
@@ -211,6 +211,7 @@ TEST(TestProjector, CheckProjectable) {
   auto i8_req = field("i8", int8(), false);
   auto u16_req = field("u16", uint16(), false);
   auto str_req = field("str", utf8(), false);
+  auto str_nil = field("str", null());
 
   // trivial
   Assert({}).ProjectableTo({});
@@ -235,6 +236,8 @@ TEST(TestProjector, CheckProjectable) {
   Assert({i8}).NotProjectableTo({i8_req},
                                 "not nullable but is not required in origin schema");
   Assert({i8_req}).ProjectableTo({i8});
+  Assert({str_nil}).ProjectableTo({str});
+  Assert({str_nil}).NotProjectableTo({str_req});
 
   // change field type
   Assert({i8}).NotProjectableTo({field("i8", utf8())},
@@ -257,15 +260,18 @@ TEST(TestProjector, MismatchedType) {
 TEST(TestProjector, AugmentWithNull) {
   constexpr int64_t kBatchSize = 1024;
 
-  auto from_schema = schema({field("f64", float64()), field("b", boolean())});
+  auto from_schema =
+      schema({field("f64", float64()), field("b", boolean()), field("str", null())});
   auto batch = ConstantArrayGenerator::Zeroes(kBatchSize, from_schema);
-  auto to_schema = schema({field("i32", int32()), field("f64", float64())});
+  auto to_schema =
+      schema({field("i32", int32()), field("f64", float64()), field("str", utf8())});
 
   RecordBatchProjector projector(to_schema);
 
   ASSERT_OK_AND_ASSIGN(auto null_i32, MakeArrayOfNull(int32(), batch->num_rows()));
-  auto expected_batch =
-      RecordBatch::Make(to_schema, batch->num_rows(), {null_i32, batch->column(0)});
+  ASSERT_OK_AND_ASSIGN(auto null_str, MakeArrayOfNull(utf8(), batch->num_rows()));
+  auto expected_batch = RecordBatch::Make(to_schema, batch->num_rows(),
+                                          {null_i32, batch->column(0), null_str});
 
   ASSERT_OK_AND_ASSIGN(auto reconciled_batch, projector.Project(*batch));
   AssertBatchesEqual(*expected_batch, *reconciled_batch);
@@ -506,13 +512,10 @@ TEST_F(TestEndToEnd, EndToEndSingleDataset) {
   // In the simplest case, consumption is simply conversion to a Table.
   ASSERT_OK_AND_ASSIGN(auto table, scanner->ToTable());
 
-  using row_type = std::tuple<double, std::string, util::optional<std::string>>;
-  std::vector<row_type> rows{
-      row_type{152.25, "3", "CA"},
-      row_type{273.5, "3", "US"},
-  };
-  std::shared_ptr<Table> expected;
-  ASSERT_OK(stl::TableFromTupleRange(default_memory_pool(), rows, columns, &expected));
+  auto expected = TableFromJSON(scanner->schema(), {R"([
+    {"sales": 152.25, "model": "3", "country": "CA"},
+    {"sales": 273.5, "model": "3", "country": "US"}
+  ])"});
   AssertTablesEqual(*expected, *table, false, true);
 }
 
@@ -741,7 +744,7 @@ TEST_F(TestSchemaUnification, SelectPartitionColumnsFilterPhysicalColumn) {
 }
 
 TEST_F(TestSchemaUnification, SelectMixedColumnsAndFilter) {
-  // Selects mix of phyical/virtual with a different order and uses a filter on
+  // Selects mix of physical/virtual with a different order and uses a filter on
   // a physical column not selected.
   ASSERT_OK_AND_ASSIGN(auto scan_builder, dataset_->NewScan());
   ASSERT_OK(scan_builder->Filter("phy_2"_ >= 212));
diff --git a/cpp/src/arrow/dataset/discovery.cc b/cpp/src/arrow/dataset/discovery.cc
index 080703d389c..b7af6dcbf0f 100644
--- a/cpp/src/arrow/dataset/discovery.cc
+++ b/cpp/src/arrow/dataset/discovery.cc
@@ -180,20 +180,25 @@ Result<std::shared_ptr<DatasetFactory>> FileSystemDatasetFactory::Make(
   ARROW_ASSIGN_OR_RAISE(auto files, filesystem->GetFileInfo(selector));
 
   // Filter out anything that's not a file or that's explicitly ignored
+  Status st;
   auto files_end =
       std::remove_if(files.begin(), files.end(), [&](const fs::FileInfo& info) {
         if (!info.IsFile()) return true;
 
         auto relative = fs::internal::RemoveAncestor(selector.base_dir, info.path());
-        DCHECK(relative.has_value())
-            << "GetFileInfo() yielded path outside selector.base_dir";
+        if (!relative.has_value()) {
+          st = Status::Invalid("GetFileInfo() yielded path '", info.path(),
+                               "', which is outside base dir '", selector.base_dir, "'");
+          return false;
+        }
 
-        if (StartsWithAnyOf(relative->to_string(), options.selector_ignore_prefixes)) {
+        if (StartsWithAnyOf(std::string(*relative), options.selector_ignore_prefixes)) {
           return true;
         }
 
         return false;
       });
+  RETURN_NOT_OK(st);
   files.erase(files_end, files.end());
 
   // Sorting by path guarantees a stability sometimes needed by unit tests.
@@ -253,7 +258,7 @@ Result<std::shared_ptr<Dataset>> FileSystemDatasetFactory::Finish(FinishOptions
     fragments.push_back(fragment);
   }
 
-  return FileSystemDataset::Make(schema, root_partition_, format_, fragments);
+  return FileSystemDataset::Make(schema, root_partition_, format_, fs_, fragments);
 }
 
 }  // namespace dataset
diff --git a/cpp/src/arrow/dataset/file_base.cc b/cpp/src/arrow/dataset/file_base.cc
index caa44c6edf7..ef044ea3a03 100644
--- a/cpp/src/arrow/dataset/file_base.cc
+++ b/cpp/src/arrow/dataset/file_base.cc
@@ -18,6 +18,9 @@
 #include "arrow/dataset/file_base.h"
 
 #include <algorithm>
+#include <deque>
+#include <unordered_map>
+#include <unordered_set>
 #include <vector>
 
 #include "arrow/dataset/dataset_internal.h"
@@ -31,6 +34,10 @@
 #include "arrow/io/memory.h"
 #include "arrow/util/iterator.h"
 #include "arrow/util/logging.h"
+#include "arrow/util/make_unique.h"
+#include "arrow/util/map.h"
+#include "arrow/util/mutex.h"
+#include "arrow/util/string.h"
 #include "arrow/util/task_group.h"
 
 namespace arrow {
@@ -65,10 +72,6 @@ Result<std::shared_ptr<FileFragment>> FileFormat::MakeFragment(
       new FileFragment(std::move(source), shared_from_this(),
                        std::move(partition_expression), std::move(physical_schema)));
 }
-Status FileFormat::WriteFragment(RecordBatchReader* batches,
-                                 io::OutputStream* destination) {
-  return Status::NotImplemented("writing fragment of format ", type_name());
-}
 
 Result<std::shared_ptr<Schema>> FileFragment::ReadPhysicalSchemaImpl() {
   return format_->Inspect(source_);
@@ -82,25 +85,27 @@ Result<ScanTaskIterator> FileFragment::Scan(std::shared_ptr<ScanOptions> options
 FileSystemDataset::FileSystemDataset(std::shared_ptr<Schema> schema,
                                      std::shared_ptr<Expression> root_partition,
                                      std::shared_ptr<FileFormat> format,
+                                     std::shared_ptr<fs::FileSystem> filesystem,
                                      std::vector<std::shared_ptr<FileFragment>> fragments)
     : Dataset(std::move(schema), std::move(root_partition)),
       format_(std::move(format)),
+      filesystem_(std::move(filesystem)),
       fragments_(std::move(fragments)) {}
 
 Result<std::shared_ptr<FileSystemDataset>> FileSystemDataset::Make(
     std::shared_ptr<Schema> schema, std::shared_ptr<Expression> root_partition,
-    std::shared_ptr<FileFormat> format,
+    std::shared_ptr<FileFormat> format, std::shared_ptr<fs::FileSystem> filesystem,
     std::vector<std::shared_ptr<FileFragment>> fragments) {
-  return std::shared_ptr<FileSystemDataset>(
-      new FileSystemDataset(std::move(schema), std::move(root_partition),
-                            std::move(format), std::move(fragments)));
+  return std::shared_ptr<FileSystemDataset>(new FileSystemDataset(
+      std::move(schema), std::move(root_partition), std::move(format),
+      std::move(filesystem), std::move(fragments)));
 }
 
 Result<std::shared_ptr<Dataset>> FileSystemDataset::ReplaceSchema(
     std::shared_ptr<Schema> schema) const {
   RETURN_NOT_OK(CheckProjectable(*schema_, *schema));
   return std::shared_ptr<Dataset>(new FileSystemDataset(
-      std::move(schema), partition_expression_, format_, fragments_));
+      std::move(schema), partition_expression_, format_, filesystem_, fragments_));
 }
 
 std::vector<std::string> FileSystemDataset::files() const {
@@ -145,97 +150,213 @@ FragmentIterator FileSystemDataset::GetFragmentsImpl(
   return MakeVectorIterator(std::move(fragments));
 }
 
-struct WriteTask {
-  Status Execute();
+Status FileWriter::Write(RecordBatchReader* batches) {
+  while (true) {
+    ARROW_ASSIGN_OR_RAISE(auto batch, batches->Next());
+    if (batch == nullptr) break;
+    RETURN_NOT_OK(Write(batch));
+  }
+  return Status::OK();
+}
 
-  /// The basename of files written by this WriteTask. Extensions
-  /// are derived from format
-  std::string basename;
+constexpr util::string_view kIntegerToken = "{i}";
 
-  /// The partitioning with which paths will be generated
-  std::shared_ptr<Partitioning> partitioning;
+Status ValidateBasenameTemplate(util::string_view basename_template) {
+  if (basename_template.find(fs::internal::kSep) != util::string_view::npos) {
+    return Status::Invalid("basename_template contained '/'");
+  }
+  size_t token_start = basename_template.find(kIntegerToken);
+  if (token_start == util::string_view::npos) {
+    return Status::Invalid("basename_template did not contain '", kIntegerToken, "'");
+  }
+  return Status::OK();
+}
 
-  /// The format in which fragments will be written
-  std::shared_ptr<FileFormat> format;
+/// WriteQueue allows batches to be pushed from multiple threads while another thread
+/// flushes some to disk.
+class WriteQueue {
+ public:
+  WriteQueue(std::string partition_expression, size_t index,
+             std::shared_ptr<Schema> schema)
+      : partition_expression_(std::move(partition_expression)),
+        index_(index),
+        schema_(std::move(schema)) {}
+
+  // Push a batch into the writer's queue of pending writes.
+  void Push(std::shared_ptr<RecordBatch> batch) {
+    auto push_lock = push_mutex_.Lock();
+    pending_.push_back(std::move(batch));
+  }
 
-  /// The FileSystem and base directory into which fragments will be written
-  std::shared_ptr<fs::FileSystem> filesystem;
-  std::string base_dir;
+  // Flush all pending batches, or return immediately if another thread is already
+  // flushing this queue.
+  Status Flush(const FileSystemDatasetWriteOptions& write_options) {
+    if (auto writer_lock = writer_mutex_.TryLock()) {
+      if (writer_ == nullptr) {
+        // FileWriters are opened lazily to avoid blocking access to a scan-wide queue set
+        RETURN_NOT_OK(OpenWriter(write_options));
+      }
+
+      while (true) {
+        std::shared_ptr<RecordBatch> batch;
+        {
+          auto push_lock = push_mutex_.Lock();
+          if (pending_.empty()) {
+            // Ensure the writer_lock is released before the push_lock. Otherwise another
+            // thread might successfully Push() a batch but then fail to Flush() it since
+            // the writer_lock is still held, leaving an unflushed batch in pending_.
+            writer_lock.Unlock();
+            break;
+          }
+          batch = std::move(pending_.front());
+          pending_.pop_front();
+        }
+        RETURN_NOT_OK(writer_->Write(batch));
+      }
+    }
+    return Status::OK();
+  }
 
-  /// Batches to be written
-  std::shared_ptr<RecordBatchReader> batches;
+  const std::shared_ptr<FileWriter>& writer() const { return writer_; }
 
-  /// An Expression already satisfied by every batch to be written
-  std::shared_ptr<Expression> partition_expression;
-};
+ private:
+  Status OpenWriter(const FileSystemDatasetWriteOptions& write_options) {
+    auto dir =
+        fs::internal::EnsureTrailingSlash(write_options.base_dir) + partition_expression_;
 
-Status WriteTask::Execute() {
-  std::unordered_map<std::string, RecordBatchVector> path_to_batches;
-
-  // TODO(bkietz) these calls to Partition() should be scattered across a TaskGroup
-  for (auto maybe_batch : IteratorFromReader(batches)) {
-    ARROW_ASSIGN_OR_RAISE(auto batch, std::move(maybe_batch));
-    ARROW_ASSIGN_OR_RAISE(auto partitioned_batches, partitioning->Partition(batch));
-    for (auto&& partitioned_batch : partitioned_batches) {
-      AndExpression expr(std::move(partitioned_batch.partition_expression),
-                         partition_expression);
-      ARROW_ASSIGN_OR_RAISE(std::string path, partitioning->Format(expr));
-      path = fs::internal::EnsureLeadingSlash(path);
-      path_to_batches[path].push_back(std::move(partitioned_batch.batch));
+    auto basename = internal::Replace(write_options.basename_template, kIntegerToken,
+                                      std::to_string(index_));
+    if (!basename) {
+      return Status::Invalid("string interpolation of basename template failed");
     }
-  }
 
-  for (auto&& path_batches : path_to_batches) {
-    auto dir = base_dir + path_batches.first;
-    RETURN_NOT_OK(filesystem->CreateDir(dir, /*recursive=*/true));
+    auto path = fs::internal::ConcatAbstractPath(dir, *basename);
 
-    auto path = fs::internal::ConcatAbstractPath(dir, basename);
-    ARROW_ASSIGN_OR_RAISE(auto destination, filesystem->OpenOutputStream(path));
+    RETURN_NOT_OK(write_options.filesystem->CreateDir(dir));
+    ARROW_ASSIGN_OR_RAISE(auto destination,
+                          write_options.filesystem->OpenOutputStream(path));
 
-    DCHECK(!path_batches.second.empty());
-    ARROW_ASSIGN_OR_RAISE(auto reader,
-                          RecordBatchReader::Make(std::move(path_batches.second)));
-    RETURN_NOT_OK(format->WriteFragment(reader.get(), destination.get()));
+    ARROW_ASSIGN_OR_RAISE(
+        writer_, write_options.format()->MakeWriter(std::move(destination), schema_,
+                                                    write_options.file_write_options));
+    return Status::OK();
   }
 
-  return Status::OK();
-}
+  util::Mutex writer_mutex_;
+  std::shared_ptr<FileWriter> writer_;
 
-Status FileSystemDataset::Write(std::shared_ptr<Schema> schema,
-                                std::shared_ptr<FileFormat> format,
-                                std::shared_ptr<fs::FileSystem> filesystem,
-                                std::string base_dir,
-                                std::shared_ptr<Partitioning> partitioning,
-                                std::shared_ptr<ScanContext> scan_context,
-                                FragmentIterator fragment_it) {
-  auto task_group = scan_context->TaskGroup();
+  util::Mutex push_mutex_;
+  std::deque<std::shared_ptr<RecordBatch>> pending_;
 
-  base_dir = fs::internal::RemoveTrailingSlash(base_dir).to_string();
+  // The (formatted) partition expression to which this queue corresponds
+  std::string partition_expression_;
 
-  for (const auto& f : partitioning->schema()->fields()) {
-    if (f->type()->id() == Type::DICTIONARY) {
-      return Status::NotImplemented("writing with dictionary partitions");
+  size_t index_;
+
+  std::shared_ptr<Schema> schema_;
+};
+
+Status FileSystemDataset::Write(const FileSystemDatasetWriteOptions& write_options,
+                                std::shared_ptr<Scanner> scanner) {
+  RETURN_NOT_OK(ValidateBasenameTemplate(write_options.basename_template));
+
+  auto task_group = scanner->context()->TaskGroup();
+
+  // Things we'll un-lazy for the sake of simplicity, with the tradeoff they represent:
+  //
+  // - Fragment iteration. Keeping this lazy would allow us to start partitioning/writing
+  //   any fragments we have before waiting for discovery to complete. This isn't
+  //   currently implemented for FileSystemDataset anyway: ARROW-8613
+  //
+  // - ScanTask iteration. Keeping this lazy would save some unnecessary blocking when
+  //   writing Fragments which produce scan tasks slowly. No Fragments do this.
+  //
+  // NB: neither of these will have any impact whatsoever on the common case of writing
+  //     an in-memory table to disk.
+  ARROW_ASSIGN_OR_RAISE(FragmentVector fragments, scanner->GetFragments().ToVector());
+  ScanTaskVector scan_tasks;
+  std::vector<const Fragment*> fragment_for_task;
+
+  // Avoid contention with multithreaded readers
+  auto context = std::make_shared<ScanContext>(*scanner->context());
+  context->use_threads = false;
+
+  for (const auto& fragment : fragments) {
+    auto options = std::make_shared<ScanOptions>(*scanner->options());
+    ARROW_ASSIGN_OR_RAISE(auto scan_task_it,
+                          Scanner(fragment, std::move(options), context).Scan());
+    for (auto maybe_scan_task : scan_task_it) {
+      ARROW_ASSIGN_OR_RAISE(auto scan_task, maybe_scan_task);
+      scan_tasks.push_back(std::move(scan_task));
+      fragment_for_task.push_back(fragment.get());
     }
   }
 
-  int i = 0;
-  for (auto maybe_fragment : fragment_it) {
-    ARROW_ASSIGN_OR_RAISE(auto fragment, std::move(maybe_fragment));
-    auto task = std::make_shared<WriteTask>();
-
-    task->basename = "dat_" + std::to_string(i++) + "." + format->type_name();
-    task->partition_expression = fragment->partition_expression();
-    task->format = format;
-    task->filesystem = filesystem;
-    task->base_dir = base_dir;
-    task->partitioning = partitioning;
-
-    // make a record batch reader which yields from a fragment
-    ARROW_ASSIGN_OR_RAISE(task->batches, FragmentRecordBatchReader::Make(
-                                             std::move(fragment), schema, scan_context));
-    task_group->Append([task] { return task->Execute(); });
+  // Store a mapping from partitions (represened by their formatted partition expressions)
+  // to a WriteQueue which flushes batches into that partition's output file. In principle
+  // any thread could produce a batch for any partition, so each task alternates between
+  // pushing batches and flushing them to disk.
+  util::Mutex queues_mutex;
+  std::unordered_map<std::string, std::unique_ptr<WriteQueue>> queues;
+
+  auto fragment_for_task_it = fragment_for_task.begin();
+  for (const auto& scan_task : scan_tasks) {
+    const Fragment* fragment = *fragment_for_task_it++;
+
+    task_group->Append([&, scan_task, fragment] {
+      ARROW_ASSIGN_OR_RAISE(auto batches, scan_task->Execute());
+
+      for (auto maybe_batch : batches) {
+        ARROW_ASSIGN_OR_RAISE(auto batch, maybe_batch);
+        ARROW_ASSIGN_OR_RAISE(auto groups, write_options.partitioning->Partition(batch));
+        batch.reset();  // drop to hopefully conserve memory
+
+        std::unordered_set<WriteQueue*> need_flushed;
+        for (size_t i = 0; i < groups.batches.size(); ++i) {
+          AndExpression partition_expression(std::move(groups.expressions[i]),
+                                             fragment->partition_expression());
+          auto batch = std::move(groups.batches[i]);
+
+          ARROW_ASSIGN_OR_RAISE(auto part,
+                                write_options.partitioning->Format(partition_expression));
+
+          WriteQueue* queue;
+          {
+            // lookup the queue to which batch should be appended
+            auto queues_lock = queues_mutex.Lock();
+
+            queue = internal::GetOrInsertGenerated(
+                        &queues, std::move(part),
+                        [&](const std::string& emplaced_part) {
+                          // lookup in `queues` also failed,
+                          // generate a new WriteQueue
+                          size_t queue_index = queues.size() - 1;
+
+                          return internal::make_unique<WriteQueue>(
+                              emplaced_part, queue_index, batch->schema());
+                        })
+                        ->second.get();
+          }
+
+          queue->Push(std::move(batch));
+          need_flushed.insert(queue);
+        }
+
+        // flush all touched WriteQueues
+        for (auto queue : need_flushed) {
+          RETURN_NOT_OK(queue->Flush(write_options));
+        }
+      }
+
+      return Status::OK();
+    });
   }
+  RETURN_NOT_OK(task_group->Finish());
 
+  task_group = scanner->context()->TaskGroup();
+  for (const auto& part_queue : queues) {
+    task_group->Append([&] { return part_queue.second->writer()->Finish(); });
+  }
   return task_group->Finish();
 }
 
diff --git a/cpp/src/arrow/dataset/file_base.h b/cpp/src/arrow/dataset/file_base.h
index c64714a364c..192921e7cf0 100644
--- a/cpp/src/arrow/dataset/file_base.h
+++ b/cpp/src/arrow/dataset/file_base.h
@@ -22,7 +22,6 @@
 #include <functional>
 #include <memory>
 #include <string>
-#include <unordered_map>
 #include <utility>
 #include <vector>
 
@@ -128,6 +127,8 @@ class ARROW_DS_EXPORT FileFormat : public std::enable_shared_from_this<FileForma
   /// \brief Return true if fragments of this format can benefit from parallel scanning.
   virtual bool splittable() const { return false; }
 
+  virtual bool Equals(const FileFormat& other) const = 0;
+
   /// \brief Indicate if the FileSource is supported/readable by this format.
   virtual Result<bool> IsSupported(const FileSource& source) const = 0;
 
@@ -151,9 +152,11 @@ class ARROW_DS_EXPORT FileFormat : public std::enable_shared_from_this<FileForma
   Result<std::shared_ptr<FileFragment>> MakeFragment(
       FileSource source, std::shared_ptr<Schema> physical_schema = NULLPTR);
 
-  /// \brief Write a fragment.
-  /// FIXME(bkietz) make this pure virtual
-  virtual Status WriteFragment(RecordBatchReader* batches, io::OutputStream* destination);
+  virtual Result<std::shared_ptr<FileWriter>> MakeWriter(
+      std::shared_ptr<io::OutputStream> destination, std::shared_ptr<Schema> schema,
+      std::shared_ptr<FileWriteOptions> options) const = 0;
+
+  virtual std::shared_ptr<FileWriteOptions> DefaultWriteOptions() = 0;
 };
 
 /// \brief A Fragment that is stored in a file with a known format
@@ -195,31 +198,22 @@ class ARROW_DS_EXPORT FileSystemDataset : public Dataset {
   /// \param[in] schema the schema of the dataset
   /// \param[in] root_partition the partition expression of the dataset
   /// \param[in] format the format of each FileFragment.
-  /// \param[in] fragments list of fragments to create the dataset from
+  /// \param[in] filesystem the filesystem of each FileFragment, or nullptr if the
+  ///            fragments wrap buffers.
+  /// \param[in] fragments list of fragments to create the dataset from.
   ///
-  /// Note that all fragment must be of `FileFragment` type. The type are
-  /// erased to simplify callers.
+  /// Note that fragments wrapping files resident in differing filesystems are not
+  /// permitted; to work with multiple filesystems use a UnionDataset.
   ///
   /// \return A constructed dataset.
   static Result<std::shared_ptr<FileSystemDataset>> Make(
       std::shared_ptr<Schema> schema, std::shared_ptr<Expression> root_partition,
-      std::shared_ptr<FileFormat> format,
+      std::shared_ptr<FileFormat> format, std::shared_ptr<fs::FileSystem> filesystem,
       std::vector<std::shared_ptr<FileFragment>> fragments);
 
   /// \brief Write a dataset.
-  ///
-  /// \param[in] schema Schema of written dataset.
-  /// \param[in] format FileFormat with which fragments will be written.
-  /// \param[in] filesystem FileSystem into which the dataset will be written.
-  /// \param[in] base_dir Root directory into which the dataset will be written.
-  /// \param[in] partitioning Partitioning used to generate fragment paths.
-  /// \param[in] scan_context Resource pool used to scan and write fragments.
-  /// \param[in] fragments Fragments to be written to disk.
-  static Status Write(std::shared_ptr<Schema> schema, std::shared_ptr<FileFormat> format,
-                      std::shared_ptr<fs::FileSystem> filesystem, std::string base_dir,
-                      std::shared_ptr<Partitioning> partitioning,
-                      std::shared_ptr<ScanContext> scan_context,
-                      FragmentIterator fragments);
+  static Status Write(const FileSystemDatasetWriteOptions& write_options,
+                      std::shared_ptr<Scanner> scanner);
 
   /// \brief Return the type name of the dataset.
   std::string type_name() const override { return "filesystem"; }
@@ -234,6 +228,9 @@ class ARROW_DS_EXPORT FileSystemDataset : public Dataset {
   /// \brief Return the format.
   const std::shared_ptr<FileFormat>& format() const { return format_; }
 
+  /// \brief Return the filesystem. May be nullptr if the fragments wrap buffers.
+  const std::shared_ptr<fs::FileSystem>& filesystem() const { return filesystem_; }
+
   std::string ToString() const;
 
  protected:
@@ -242,11 +239,72 @@ class ARROW_DS_EXPORT FileSystemDataset : public Dataset {
   FileSystemDataset(std::shared_ptr<Schema> schema,
                     std::shared_ptr<Expression> root_partition,
                     std::shared_ptr<FileFormat> format,
+                    std::shared_ptr<fs::FileSystem> filesystem,
                     std::vector<std::shared_ptr<FileFragment>> fragments);
 
   std::shared_ptr<FileFormat> format_;
+  std::shared_ptr<fs::FileSystem> filesystem_;
   std::vector<std::shared_ptr<FileFragment>> fragments_;
 };
 
+class ARROW_DS_EXPORT FileWriteOptions {
+ public:
+  virtual ~FileWriteOptions() = default;
+
+  const std::shared_ptr<FileFormat>& format() const { return format_; }
+
+  std::string type_name() const { return format_->type_name(); }
+
+ protected:
+  explicit FileWriteOptions(std::shared_ptr<FileFormat> format)
+      : format_(std::move(format)) {}
+
+  std::shared_ptr<FileFormat> format_;
+};
+
+class ARROW_DS_EXPORT FileWriter {
+ public:
+  virtual ~FileWriter() = default;
+
+  virtual Status Write(const std::shared_ptr<RecordBatch>& batch) = 0;
+
+  Status Write(RecordBatchReader* batches);
+
+  virtual Status Finish() = 0;
+
+  const std::shared_ptr<FileFormat>& format() const { return options_->format(); }
+  const std::shared_ptr<Schema>& schema() const { return schema_; }
+  const std::shared_ptr<FileWriteOptions>& options() const { return options_; }
+
+ protected:
+  FileWriter(std::shared_ptr<Schema> schema, std::shared_ptr<FileWriteOptions> options)
+      : schema_(std::move(schema)), options_(std::move(options)) {}
+
+  std::shared_ptr<Schema> schema_;
+  std::shared_ptr<FileWriteOptions> options_;
+};
+
+struct ARROW_DS_EXPORT FileSystemDatasetWriteOptions {
+  /// Options for individual fragment writing.
+  std::shared_ptr<FileWriteOptions> file_write_options;
+
+  /// FileSystem into which a dataset will be written.
+  std::shared_ptr<fs::FileSystem> filesystem;
+
+  /// Root directory into which the dataset will be written.
+  std::string base_dir;
+
+  /// Partitioning used to generate fragment paths.
+  std::shared_ptr<Partitioning> partitioning;
+
+  /// Template string used to generate fragment basenames.
+  /// {i} will be replaced by an auto incremented integer.
+  std::string basename_template;
+
+  const std::shared_ptr<FileFormat>& format() const {
+    return file_write_options->format();
+  }
+};
+
 }  // namespace dataset
 }  // namespace arrow
diff --git a/cpp/src/arrow/dataset/file_csv.cc b/cpp/src/arrow/dataset/file_csv.cc
index 3df9fa8d257..f889b12fddd 100644
--- a/cpp/src/arrow/dataset/file_csv.cc
+++ b/cpp/src/arrow/dataset/file_csv.cc
@@ -65,7 +65,7 @@ Result<std::unordered_set<std::string>> GetColumnNames(
   RETURN_NOT_OK(
       parser.VisitLastRow([&](const uint8_t* data, uint32_t size, bool quoted) -> Status {
         util::string_view view{reinterpret_cast<const char*>(data), size};
-        if (column_names.emplace(view.to_string()).second) {
+        if (column_names.emplace(std::string(view)).second) {
           return Status::OK();
         }
         return Status::Invalid("CSV file contained multiple columns named ", view);
@@ -157,6 +157,22 @@ class CsvScanTask : public ScanTask {
   FileSource source_;
 };
 
+bool CsvFileFormat::Equals(const FileFormat& format) const {
+  if (type_name() != format.type_name()) return false;
+
+  const auto& other_parse_options =
+      checked_cast<const CsvFileFormat&>(format).parse_options;
+
+  return parse_options.delimiter == other_parse_options.delimiter &&
+         parse_options.quoting == other_parse_options.quoting &&
+         parse_options.quote_char == other_parse_options.quote_char &&
+         parse_options.double_quote == other_parse_options.double_quote &&
+         parse_options.escaping == other_parse_options.escaping &&
+         parse_options.escape_char == other_parse_options.escape_char &&
+         parse_options.newlines_in_values == other_parse_options.newlines_in_values &&
+         parse_options.ignore_empty_lines == other_parse_options.ignore_empty_lines;
+}
+
 Result<bool> CsvFileFormat::IsSupported(const FileSource& source) const {
   RETURN_NOT_OK(source.Open().status());
   return OpenReader(source, *this).ok();
diff --git a/cpp/src/arrow/dataset/file_csv.h b/cpp/src/arrow/dataset/file_csv.h
index 6777d8aec25..df5593459e3 100644
--- a/cpp/src/arrow/dataset/file_csv.h
+++ b/cpp/src/arrow/dataset/file_csv.h
@@ -19,14 +19,12 @@
 
 #include <memory>
 #include <string>
-#include <unordered_map>
-#include <vector>
 
 #include "arrow/csv/options.h"
 #include "arrow/dataset/file_base.h"
 #include "arrow/dataset/type_fwd.h"
 #include "arrow/dataset/visibility.h"
-#include "arrow/result.h"
+#include "arrow/status.h"
 
 namespace arrow {
 namespace dataset {
@@ -39,6 +37,8 @@ class ARROW_DS_EXPORT CsvFileFormat : public FileFormat {
 
   std::string type_name() const override { return "csv"; }
 
+  bool Equals(const FileFormat& other) const override;
+
   Result<bool> IsSupported(const FileSource& source) const override;
 
   /// \brief Return the schema of the file if possible.
@@ -48,6 +48,14 @@ class ARROW_DS_EXPORT CsvFileFormat : public FileFormat {
   Result<ScanTaskIterator> ScanFile(std::shared_ptr<ScanOptions> options,
                                     std::shared_ptr<ScanContext> context,
                                     FileFragment* fragment) const override;
+
+  Result<std::shared_ptr<FileWriter>> MakeWriter(
+      std::shared_ptr<io::OutputStream> destination, std::shared_ptr<Schema> schema,
+      std::shared_ptr<FileWriteOptions> options) const override {
+    return Status::NotImplemented("writing fragment of CsvFileFormat");
+  }
+
+  std::shared_ptr<FileWriteOptions> DefaultWriteOptions() override { return NULLPTR; }
 };
 
 }  // namespace dataset
diff --git a/cpp/src/arrow/dataset/file_csv_test.cc b/cpp/src/arrow/dataset/file_csv_test.cc
index 5c189f040b4..8e73be0ee8f 100644
--- a/cpp/src/arrow/dataset/file_csv_test.cc
+++ b/cpp/src/arrow/dataset/file_csv_test.cc
@@ -76,7 +76,7 @@ TEST_F(TestCsvFileFormat, ScanRecordBatchReader) {
   int64_t row_count = 0;
 
   for (auto maybe_batch : Batches(fragment.get())) {
-    ASSERT_OK_AND_ASSIGN(auto batch, std::move(maybe_batch));
+    ASSERT_OK_AND_ASSIGN(auto batch, maybe_batch);
     row_count += batch->num_rows();
   }
 
@@ -92,7 +92,7 @@ TEST_F(TestCsvFileFormat, ScanRecordBatchReaderWithVirtualColumn) {
   int64_t row_count = 0;
 
   for (auto maybe_batch : Batches(fragment.get())) {
-    ASSERT_OK_AND_ASSIGN(auto batch, std::move(maybe_batch));
+    ASSERT_OK_AND_ASSIGN(auto batch, maybe_batch);
     AssertSchemaEqual(*batch->schema(), *schema_);
     row_count += batch->num_rows();
   }
@@ -155,12 +155,12 @@ N/A,bar
 
   ASSERT_OK_AND_ASSIGN(auto scan_task_it, scanner->Scan());
   for (auto maybe_scan_task : scan_task_it) {
-    ASSERT_OK_AND_ASSIGN(auto scan_task, std::move(maybe_scan_task));
+    ASSERT_OK_AND_ASSIGN(auto scan_task, maybe_scan_task);
     ASSERT_OK_AND_ASSIGN(auto batch_it, scan_task->Execute());
     for (auto maybe_batch : batch_it) {
       // ERROR: "f64" is not projected and reverts to inferred type,
       // breaking the comparison expression
-      ASSERT_OK_AND_ASSIGN(auto batch, std::move(maybe_batch));
+      ASSERT_OK_AND_ASSIGN(auto batch, maybe_batch);
     }
   }
 }
diff --git a/cpp/src/arrow/dataset/file_ipc.cc b/cpp/src/arrow/dataset/file_ipc.cc
index de25875b59a..8bd01218344 100644
--- a/cpp/src/arrow/dataset/file_ipc.cc
+++ b/cpp/src/arrow/dataset/file_ipc.cc
@@ -27,9 +27,13 @@
 #include "arrow/dataset/scanner.h"
 #include "arrow/ipc/reader.h"
 #include "arrow/ipc/writer.h"
+#include "arrow/util/checked_cast.h"
 #include "arrow/util/iterator.h"
 
 namespace arrow {
+
+using internal::checked_pointer_cast;
+
 namespace dataset {
 
 static inline ipc::IpcReadOptions default_read_options() {
@@ -159,18 +163,50 @@ Result<ScanTaskIterator> IpcFileFormat::ScanFile(std::shared_ptr<ScanOptions> op
                                    fragment->source());
 }
 
-Status IpcFileFormat::WriteFragment(RecordBatchReader* batches,
-                                    io::OutputStream* destination) {
-  ARROW_ASSIGN_OR_RAISE(auto writer, ipc::NewFileWriter(destination, batches->schema()));
+//
+// IpcFileWriter, IpcFileWriteOptions
+//
+
+std::shared_ptr<FileWriteOptions> IpcFileFormat::DefaultWriteOptions() {
+  std::shared_ptr<IpcFileWriteOptions> ipc_options(
+      new IpcFileWriteOptions(shared_from_this()));
+
+  ipc_options->options =
+      std::make_shared<ipc::IpcWriteOptions>(ipc::IpcWriteOptions::Defaults());
+  return ipc_options;
+}
 
-  for (;;) {
-    ARROW_ASSIGN_OR_RAISE(auto batch, batches->Next());
-    if (batch == nullptr) break;
-    RETURN_NOT_OK(writer->WriteRecordBatch(*batch));
+Result<std::shared_ptr<FileWriter>> IpcFileFormat::MakeWriter(
+    std::shared_ptr<io::OutputStream> destination, std::shared_ptr<Schema> schema,
+    std::shared_ptr<FileWriteOptions> options) const {
+  if (!Equals(*options->format())) {
+    return Status::TypeError("Mismatching format/write options.");
   }
 
-  return writer->Close();
+  auto ipc_options = checked_pointer_cast<IpcFileWriteOptions>(options);
+
+  // override use_threads to avoid nested parallelism
+  ipc_options->options->use_threads = false;
+
+  ARROW_ASSIGN_OR_RAISE(auto writer,
+                        ipc::MakeFileWriter(destination, schema, *ipc_options->options,
+                                            ipc_options->metadata));
+
+  return std::shared_ptr<FileWriter>(
+      new IpcFileWriter(std::move(writer), std::move(schema), std::move(ipc_options)));
 }
 
+IpcFileWriter::IpcFileWriter(std::shared_ptr<ipc::RecordBatchWriter> writer,
+                             std::shared_ptr<Schema> schema,
+                             std::shared_ptr<IpcFileWriteOptions> options)
+    : FileWriter(std::move(schema), std::move(options)),
+      batch_writer_(std::move(writer)) {}
+
+Status IpcFileWriter::Write(const std::shared_ptr<RecordBatch>& batch) {
+  return batch_writer_->WriteRecordBatch(*batch);
+}
+
+Status IpcFileWriter::Finish() { return batch_writer_->Close(); }
+
 }  // namespace dataset
 }  // namespace arrow
diff --git a/cpp/src/arrow/dataset/file_ipc.h b/cpp/src/arrow/dataset/file_ipc.h
index 2b8b94de984..2cdd837430e 100644
--- a/cpp/src/arrow/dataset/file_ipc.h
+++ b/cpp/src/arrow/dataset/file_ipc.h
@@ -28,6 +28,12 @@
 #include "arrow/result.h"
 
 namespace arrow {
+namespace ipc {
+
+class RecordBatchWriter;
+struct IpcWriteOptions;
+
+}  // namespace ipc
 namespace dataset {
 
 /// \brief A FileFormat implementation that reads from and writes to Ipc files
@@ -35,6 +41,10 @@ class ARROW_DS_EXPORT IpcFileFormat : public FileFormat {
  public:
   std::string type_name() const override { return "ipc"; }
 
+  bool Equals(const FileFormat& other) const override {
+    return type_name() == other.type_name();
+  }
+
   bool splittable() const override { return true; }
 
   Result<bool> IsSupported(const FileSource& source) const override;
@@ -47,8 +57,41 @@ class ARROW_DS_EXPORT IpcFileFormat : public FileFormat {
                                     std::shared_ptr<ScanContext> context,
                                     FileFragment* fragment) const override;
 
-  Status WriteFragment(RecordBatchReader* batches,
-                       io::OutputStream* destination) override;
+  Result<std::shared_ptr<FileWriter>> MakeWriter(
+      std::shared_ptr<io::OutputStream> destination, std::shared_ptr<Schema> schema,
+      std::shared_ptr<FileWriteOptions> options) const override;
+
+  std::shared_ptr<FileWriteOptions> DefaultWriteOptions() override;
+};
+
+class ARROW_DS_EXPORT IpcFileWriteOptions : public FileWriteOptions {
+ public:
+  /// Options passed to ipc::MakeFileWriter. use_threads is ignored
+  std::shared_ptr<ipc::IpcWriteOptions> options;
+
+  /// custom_metadata written to the file's footer
+  std::shared_ptr<const KeyValueMetadata> metadata;
+
+ protected:
+  using FileWriteOptions::FileWriteOptions;
+
+  friend class IpcFileFormat;
+};
+
+class ARROW_DS_EXPORT IpcFileWriter : public FileWriter {
+ public:
+  Status Write(const std::shared_ptr<RecordBatch>& batch) override;
+
+  Status Finish() override;
+
+ private:
+  IpcFileWriter(std::shared_ptr<ipc::RecordBatchWriter> writer,
+                std::shared_ptr<Schema> schema,
+                std::shared_ptr<IpcFileWriteOptions> options);
+
+  std::shared_ptr<ipc::RecordBatchWriter> batch_writer_;
+
+  friend class IpcFileFormat;
 };
 
 }  // namespace dataset
diff --git a/cpp/src/arrow/dataset/file_ipc_test.cc b/cpp/src/arrow/dataset/file_ipc_test.cc
index e1b574095bb..25bbe559cf4 100644
--- a/cpp/src/arrow/dataset/file_ipc_test.cc
+++ b/cpp/src/arrow/dataset/file_ipc_test.cc
@@ -28,11 +28,13 @@
 #include "arrow/dataset/partition.h"
 #include "arrow/dataset/test_util.h"
 #include "arrow/io/memory.h"
+#include "arrow/ipc/reader.h"
 #include "arrow/ipc/writer.h"
 #include "arrow/record_batch.h"
 #include "arrow/table.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/testing/util.h"
+#include "arrow/util/key_value_metadata.h"
 
 namespace arrow {
 namespace dataset {
@@ -48,7 +50,7 @@ class ArrowIpcWriterMixin : public ::testing::Test {
   std::shared_ptr<Buffer> Write(RecordBatchReader* reader) {
     EXPECT_OK_AND_ASSIGN(auto sink, io::BufferOutputStream::Create());
 
-    EXPECT_OK_AND_ASSIGN(auto writer, ipc::NewFileWriter(sink.get(), reader->schema()));
+    EXPECT_OK_AND_ASSIGN(auto writer, ipc::MakeFileWriter(sink, reader->schema()));
 
     std::vector<std::shared_ptr<RecordBatch>> batches;
     ARROW_EXPECT_OK(reader->ReadAll(&batches));
@@ -64,7 +66,7 @@ class ArrowIpcWriterMixin : public ::testing::Test {
 
   std::shared_ptr<Buffer> Write(const Table& table) {
     EXPECT_OK_AND_ASSIGN(auto sink, io::BufferOutputStream::Create());
-    EXPECT_OK_AND_ASSIGN(auto writer, ipc::NewFileWriter(sink.get(), table.schema()));
+    EXPECT_OK_AND_ASSIGN(auto writer, ipc::MakeFileWriter(sink, table.schema()));
 
     ARROW_EXPECT_OK(writer->WriteTable(table));
 
@@ -122,7 +124,7 @@ TEST_F(TestIpcFileFormat, ScanRecordBatchReader) {
   int64_t row_count = 0;
 
   for (auto maybe_batch : Batches(fragment.get())) {
-    ASSERT_OK_AND_ASSIGN(auto batch, std::move(maybe_batch));
+    ASSERT_OK_AND_ASSIGN(auto batch, maybe_batch);
     row_count += batch->num_rows();
   }
 
@@ -139,7 +141,7 @@ TEST_F(TestIpcFileFormat, ScanRecordBatchReaderWithVirtualColumn) {
   int64_t row_count = 0;
 
   for (auto maybe_batch : Batches(fragment.get())) {
-    ASSERT_OK_AND_ASSIGN(auto batch, std::move(maybe_batch));
+    ASSERT_OK_AND_ASSIGN(auto batch, maybe_batch);
     AssertSchemaEqual(*batch->schema(), *schema_);
     row_count += batch->num_rows();
   }
@@ -156,321 +158,70 @@ TEST_F(TestIpcFileFormat, WriteRecordBatchReader) {
 
   EXPECT_OK_AND_ASSIGN(auto sink, GetFileSink());
 
-  ASSERT_OK(format_->WriteFragment(reader.get(), sink.get()));
+  auto options = format_->DefaultWriteOptions();
+  EXPECT_OK_AND_ASSIGN(auto writer, format_->MakeWriter(sink, reader->schema(), options));
+  ASSERT_OK(writer->Write(reader.get()));
+  ASSERT_OK(writer->Finish());
 
   EXPECT_OK_AND_ASSIGN(auto written, sink->Finish());
 
   AssertBufferEqual(*written, *source->buffer());
 }
 
-class TestIpcFileSystemDataset : public TestIpcFileFormat,
-                                 public MakeFileSystemDatasetMixin {
- public:
-  using PathAndContent = std::unordered_map<std::string, std::string>;
-
-  void SetUp() override {
-    PathAndContent source_files;
-
-    source_files["/dataset/year=2018/month=01/dat0.json"] = R"([
-        {"region": "NY", "model": "3", "sales": 742.0, "country": "US"},
-        {"region": "NY", "model": "S", "sales": 304.125, "country": "US"},
-        {"region": "NY", "model": "Y", "sales": 27.5, "country": "US"}
-      ])";
-    source_files["/dataset/year=2018/month=01/dat1.json"] = R"([
-        {"region": "QC", "model": "3", "sales": 512, "country": "CA"},
-        {"region": "QC", "model": "S", "sales": 978, "country": "CA"},
-        {"region": "NY", "model": "X", "sales": 136.25, "country": "US"},
-        {"region": "QC", "model": "X", "sales": 1.0, "country": "CA"},
-        {"region": "QC", "model": "Y", "sales": 69, "country": "CA"}
-      ])";
-    source_files["/dataset/year=2019/month=01/dat0.json"] = R"([
-        {"region": "CA", "model": "3", "sales": 273.5, "country": "US"},
-        {"region": "CA", "model": "S", "sales": 13, "country": "US"},
-        {"region": "CA", "model": "X", "sales": 54, "country": "US"},
-        {"region": "QC", "model": "S", "sales": 10, "country": "CA"},
-        {"region": "CA", "model": "Y", "sales": 21, "country": "US"}
-      ])";
-    source_files["/dataset/year=2019/month=01/dat1.json"] = R"([
-        {"region": "QC", "model": "3", "sales": 152.25, "country": "CA"},
-        {"region": "QC", "model": "X", "sales": 42, "country": "CA"},
-        {"region": "QC", "model": "Y", "sales": 37, "country": "CA"}
-      ])";
-    source_files["/dataset/.pesky"] = "garbage content";
-
-    auto mock_fs = std::make_shared<fs::internal::MockFileSystem>(fs::kNoTime);
-    for (const auto& f : source_files) {
-      ARROW_EXPECT_OK(mock_fs->CreateFile(f.first, f.second, /* recursive */ true));
-    }
-    fs_ = mock_fs;
-
-    /// schema for the whole dataset (both source and destination)
-    schema_ = schema({
-        field("region", utf8()),
-        field("model", utf8()),
-        field("sales", float64()),
-        field("year", int32()),
-        field("month", int32()),
-        field("country", utf8()),
-    });
-
-    /// Dummy file format for source dataset. Note that it isn't partitioned on country
-    auto source_format = std::make_shared<JSONRecordBatchFileFormat>(
-        SchemaFromColumnNames(schema_, {"region", "model", "sales", "country"}));
-
-    fs::FileSelector s;
-    s.base_dir = "/dataset";
-    s.recursive = true;
-
-    FileSystemFactoryOptions options;
-    options.selector_ignore_prefixes = {"."};
-    options.partitioning = HivePartitioning::MakeFactory();
-    ASSERT_OK_AND_ASSIGN(auto factory,
-                         FileSystemDatasetFactory::Make(fs_, s, source_format, options));
-    ASSERT_OK_AND_ASSIGN(dataset_, factory->Finish());
-  }
-
-  void AssertWrittenAsExpected() {
-    std::vector<std::string> files;
-    for (const auto& file_contents : expected_files_) {
-      files.push_back(file_contents.first);
-    }
-    EXPECT_THAT(checked_pointer_cast<FileSystemDataset>(written_)->files(),
-                testing::UnorderedElementsAreArray(files));
-
-    for (auto maybe_fragment : written_->GetFragments()) {
-      ASSERT_OK_AND_ASSIGN(auto fragment, std::move(maybe_fragment));
+TEST_F(TestIpcFileFormat, WriteRecordBatchReaderCustomOptions) {
+  std::shared_ptr<RecordBatchReader> reader = GetRecordBatchReader();
+  auto source = GetFileSource(reader.get());
+  reader = GetRecordBatchReader();
 
-      ASSERT_OK_AND_ASSIGN(auto actual_physical_schema, fragment->ReadPhysicalSchema());
-      AssertSchemaEqual(*expected_physical_schema_, *actual_physical_schema,
-                        /*verbose=*/true);
+  opts_ = ScanOptions::Make(reader->schema());
 
-      const auto& path = checked_pointer_cast<FileFragment>(fragment)->source().path();
+  EXPECT_OK_AND_ASSIGN(auto sink, GetFileSink());
 
-      auto expected_struct = ArrayFromJSON(struct_(expected_physical_schema_->fields()),
-                                           {expected_files_[path]});
+  auto ipc_options =
+      checked_pointer_cast<IpcFileWriteOptions>(format_->DefaultWriteOptions());
+  if (util::Codec::IsAvailable(Compression::ZSTD)) {
+    EXPECT_OK_AND_ASSIGN(ipc_options->options->codec,
+                         util::Codec::Create(Compression::ZSTD));
+  }
+  ipc_options->metadata = key_value_metadata({{"hello", "world"}});
+  EXPECT_OK_AND_ASSIGN(auto writer,
+                       format_->MakeWriter(sink, reader->schema(), ipc_options));
+  ASSERT_OK(writer->Write(reader.get()));
+  ASSERT_OK(writer->Finish());
 
-      ASSERT_OK_AND_ASSIGN(auto scanner, ScannerBuilder(actual_physical_schema, fragment,
-                                                        std::make_shared<ScanContext>())
-                                             .Finish());
-      ASSERT_OK_AND_ASSIGN(auto actual_table, scanner->ToTable());
-      ASSERT_OK_AND_ASSIGN(actual_table, actual_table->CombineChunks());
-      std::shared_ptr<Array> actual_struct;
+  EXPECT_OK_AND_ASSIGN(auto written, sink->Finish());
+  EXPECT_OK_AND_ASSIGN(auto ipc_reader, ipc::RecordBatchFileReader::Open(
+                                            std::make_shared<io::BufferReader>(written)));
 
-      for (auto maybe_batch :
-           IteratorFromReader(std::make_shared<TableBatchReader>(*actual_table))) {
-        ASSERT_OK_AND_ASSIGN(auto batch, std::move(maybe_batch));
-        ASSERT_OK_AND_ASSIGN(actual_struct, batch->ToStructArray());
-      }
+  EXPECT_EQ(ipc_reader->metadata()->sorted_pairs(),
+            ipc_options->metadata->sorted_pairs());
+}
 
-      AssertArraysEqual(*expected_struct, *actual_struct, /*verbose=*/true);
-    }
+class TestIpcFileSystemDataset : public testing::Test,
+                                 public WriteFileSystemDatasetMixin {
+ public:
+  void SetUp() override {
+    MakeSourceDataset();
+    auto ipc_format = std::make_shared<IpcFileFormat>();
+    format_ = ipc_format;
+    SetWriteOptions(ipc_format->DefaultWriteOptions());
   }
-
-  PathAndContent expected_files_;
-  std::shared_ptr<Schema> expected_physical_schema_;
-  std::shared_ptr<Dataset> written_;
 };
 
 TEST_F(TestIpcFileSystemDataset, WriteWithIdenticalPartitioningSchema) {
-  auto desired_partitioning = std::make_shared<DirectoryPartitioning>(
-      SchemaFromColumnNames(schema_, {"year", "month"}));
-
-  ASSERT_OK(FileSystemDataset::Write(
-      schema_, format_, fs_, "new_root/", desired_partitioning,
-      std::make_shared<ScanContext>(), dataset_->GetFragments()));
-
-  fs::FileSelector s;
-  s.recursive = true;
-  s.base_dir = "/new_root";
-
-  FileSystemFactoryOptions options;
-  options.partitioning = desired_partitioning;
-  ASSERT_OK_AND_ASSIGN(auto factory,
-                       FileSystemDatasetFactory::Make(fs_, s, format_, options));
-  ASSERT_OK_AND_ASSIGN(written_, factory->Finish());
-
-  expected_files_["/new_root/2018/1/dat_0.ipc"] = R"([
-        {"region": "NY", "model": "3", "sales": 742.0, "country": "US"},
-        {"region": "NY", "model": "S", "sales": 304.125, "country": "US"},
-        {"region": "NY", "model": "Y", "sales": 27.5, "country": "US"}
-      ])";
-  expected_files_["/new_root/2018/1/dat_1.ipc"] = R"([
-        {"region": "QC", "model": "3", "sales": 512, "country": "CA"},
-        {"region": "QC", "model": "S", "sales": 978, "country": "CA"},
-        {"region": "NY", "model": "X", "sales": 136.25, "country": "US"},
-        {"region": "QC", "model": "X", "sales": 1.0, "country": "CA"},
-        {"region": "QC", "model": "Y", "sales": 69, "country": "CA"}
-      ])";
-  expected_files_["/new_root/2019/1/dat_2.ipc"] = R"([
-        {"region": "CA", "model": "3", "sales": 273.5, "country": "US"},
-        {"region": "CA", "model": "S", "sales": 13, "country": "US"},
-        {"region": "CA", "model": "X", "sales": 54, "country": "US"},
-        {"region": "QC", "model": "S", "sales": 10, "country": "CA"},
-        {"region": "CA", "model": "Y", "sales": 21, "country": "US"}
-      ])";
-  expected_files_["/new_root/2019/1/dat_3.ipc"] = R"([
-        {"region": "QC", "model": "3", "sales": 152.25, "country": "CA"},
-        {"region": "QC", "model": "X", "sales": 42, "country": "CA"},
-        {"region": "QC", "model": "Y", "sales": 37, "country": "CA"}
-      ])";
-  expected_physical_schema_ =
-      SchemaFromColumnNames(schema_, {"region", "model", "sales", "country"});
-
-  AssertWrittenAsExpected();
+  TestWriteWithIdenticalPartitioningSchema();
 }
 
 TEST_F(TestIpcFileSystemDataset, WriteWithUnrelatedPartitioningSchema) {
-  auto desired_partitioning = std::make_shared<DirectoryPartitioning>(
-      SchemaFromColumnNames(schema_, {"country", "region"}));
-
-  ASSERT_OK(FileSystemDataset::Write(
-      schema_, format_, fs_, "new_root/", desired_partitioning,
-      std::make_shared<ScanContext>(), dataset_->GetFragments()));
-
-  fs::FileSelector s;
-  s.recursive = true;
-  s.base_dir = "/new_root";
-
-  FileSystemFactoryOptions options;
-  options.partitioning = desired_partitioning;
-  ASSERT_OK_AND_ASSIGN(auto factory,
-                       FileSystemDatasetFactory::Make(fs_, s, format_, options));
-  ASSERT_OK_AND_ASSIGN(written_, factory->Finish());
-
-  // XXX first thing a user will be annoyed by: we don't support left
-  // padding the month field with 0.
-  expected_files_["/new_root/US/NY/dat_0.ipc"] = R"([
-        {"year": 2018, "month": 1, "model": "3", "sales": 742.0},
-        {"year": 2018, "month": 1, "model": "S", "sales": 304.125},
-        {"year": 2018, "month": 1, "model": "Y", "sales": 27.5}
-  ])";
-  expected_files_["/new_root/US/NY/dat_1.ipc"] = R"([
-        {"year": 2018, "month": 1, "model": "X", "sales": 136.25}
-  ])";
-  expected_files_["/new_root/CA/QC/dat_1.ipc"] = R"([
-        {"year": 2018, "month": 1, "model": "3", "sales": 512},
-        {"year": 2018, "month": 1, "model": "S", "sales": 978},
-        {"year": 2018, "month": 1, "model": "X", "sales": 1.0},
-        {"year": 2018, "month": 1, "model": "Y", "sales": 69}
-  ])";
-  expected_files_["/new_root/US/CA/dat_2.ipc"] = R"([
-        {"year": 2019, "month": 1, "model": "3", "sales": 273.5},
-        {"year": 2019, "month": 1, "model": "S", "sales": 13},
-        {"year": 2019, "month": 1, "model": "X", "sales": 54},
-        {"year": 2019, "month": 1, "model": "Y", "sales": 21}
-  ])";
-  expected_files_["/new_root/CA/QC/dat_2.ipc"] = R"([
-        {"year": 2019, "month": 1, "model": "S", "sales": 10}
-  ])";
-  expected_files_["/new_root/CA/QC/dat_3.ipc"] = R"([
-        {"year": 2019, "month": 1, "model": "3", "sales": 152.25},
-        {"year": 2019, "month": 1, "model": "X", "sales": 42},
-        {"year": 2019, "month": 1, "model": "Y", "sales": 37}
-  ])";
-  expected_physical_schema_ =
-      SchemaFromColumnNames(schema_, {"model", "sales", "year", "month"});
-
-  AssertWrittenAsExpected();
+  TestWriteWithUnrelatedPartitioningSchema();
 }
 
 TEST_F(TestIpcFileSystemDataset, WriteWithSupersetPartitioningSchema) {
-  auto desired_partitioning = std::make_shared<DirectoryPartitioning>(
-      SchemaFromColumnNames(schema_, {"year", "month", "country", "region"}));
-
-  ASSERT_OK(FileSystemDataset::Write(
-      schema_, format_, fs_, "new_root/", desired_partitioning,
-      std::make_shared<ScanContext>(), dataset_->GetFragments()));
-
-  fs::FileSelector s;
-  s.recursive = true;
-  s.base_dir = "/new_root";
-
-  FileSystemFactoryOptions options;
-  options.partitioning = desired_partitioning;
-  ASSERT_OK_AND_ASSIGN(auto factory,
-                       FileSystemDatasetFactory::Make(fs_, s, format_, options));
-  ASSERT_OK_AND_ASSIGN(written_, factory->Finish());
-
-  // XXX first thing a user will be annoyed by: we don't support left
-  // padding the month field with 0.
-  expected_files_["/new_root/2018/1/US/NY/dat_0.ipc"] = R"([
-        {"model": "3", "sales": 742.0},
-        {"model": "S", "sales": 304.125},
-        {"model": "Y", "sales": 27.5}
-  ])";
-  expected_files_["/new_root/2018/1/US/NY/dat_1.ipc"] = R"([
-        {"model": "X", "sales": 136.25}
-  ])";
-  expected_files_["/new_root/2018/1/CA/QC/dat_1.ipc"] = R"([
-        {"model": "3", "sales": 512},
-        {"model": "S", "sales": 978},
-        {"model": "X", "sales": 1.0},
-        {"model": "Y", "sales": 69}
-  ])";
-  expected_files_["/new_root/2019/1/US/CA/dat_2.ipc"] = R"([
-        {"model": "3", "sales": 273.5},
-        {"model": "S", "sales": 13},
-        {"model": "X", "sales": 54},
-        {"model": "Y", "sales": 21}
-  ])";
-  expected_files_["/new_root/2019/1/CA/QC/dat_2.ipc"] = R"([
-        {"model": "S", "sales": 10}
-  ])";
-  expected_files_["/new_root/2019/1/CA/QC/dat_3.ipc"] = R"([
-        {"model": "3", "sales": 152.25},
-        {"model": "X", "sales": 42},
-        {"model": "Y", "sales": 37}
-  ])";
-  expected_physical_schema_ = SchemaFromColumnNames(schema_, {"model", "sales"});
-
-  AssertWrittenAsExpected();
+  TestWriteWithSupersetPartitioningSchema();
 }
 
 TEST_F(TestIpcFileSystemDataset, WriteWithEmptyPartitioningSchema) {
-  auto desired_partitioning =
-      std::make_shared<DirectoryPartitioning>(SchemaFromColumnNames(schema_, {}));
-
-  ASSERT_OK(FileSystemDataset::Write(
-      schema_, format_, fs_, "new_root/", desired_partitioning,
-      std::make_shared<ScanContext>(), dataset_->GetFragments()));
-
-  fs::FileSelector s;
-  s.recursive = true;
-  s.base_dir = "/new_root";
-
-  FileSystemFactoryOptions options;
-  options.partitioning = desired_partitioning;
-  ASSERT_OK_AND_ASSIGN(auto factory,
-                       FileSystemDatasetFactory::Make(fs_, s, format_, options));
-  ASSERT_OK_AND_ASSIGN(written_, factory->Finish());
-
-  expected_files_["/new_root/dat_0.ipc"] = R"([
-        {"country": "US", "region": "NY", "year": 2018, "month": 1, "model": "3", "sales": 742.0},
-        {"country": "US", "region": "NY", "year": 2018, "month": 1, "model": "S", "sales": 304.125},
-        {"country": "US", "region": "NY", "year": 2018, "month": 1, "model": "Y", "sales": 27.5}
-  ])";
-  expected_files_["/new_root/dat_1.ipc"] = R"([
-        {"country": "CA", "region": "QC", "year": 2018, "month": 1, "model": "3", "sales": 512},
-        {"country": "CA", "region": "QC", "year": 2018, "month": 1, "model": "S", "sales": 978},
-        {"country": "US", "region": "NY", "year": 2018, "month": 1, "model": "X", "sales": 136.25},
-        {"country": "CA", "region": "QC", "year": 2018, "month": 1, "model": "X", "sales": 1.0},
-        {"country": "CA", "region": "QC", "year": 2018, "month": 1, "model": "Y", "sales": 69}
-  ])";
-  expected_files_["/new_root/dat_2.ipc"] = R"([
-        {"country": "US", "region": "CA", "year": 2019, "month": 1, "model": "3", "sales": 273.5},
-        {"country": "US", "region": "CA", "year": 2019, "month": 1, "model": "S", "sales": 13},
-        {"country": "US", "region": "CA", "year": 2019, "month": 1, "model": "X", "sales": 54},
-        {"country": "CA", "region": "QC", "year": 2019, "month": 1, "model": "S", "sales": 10},
-        {"country": "US", "region": "CA", "year": 2019, "month": 1, "model": "Y", "sales": 21}
-  ])";
-  expected_files_["/new_root/dat_3.ipc"] = R"([
-        {"country": "CA", "region": "QC", "year": 2019, "month": 1, "model": "3", "sales": 152.25},
-        {"country": "CA", "region": "QC", "year": 2019, "month": 1, "model": "X", "sales": 42},
-        {"country": "CA", "region": "QC", "year": 2019, "month": 1, "model": "Y", "sales": 37}
-  ])";
-  expected_physical_schema_ = schema_;
-
-  AssertWrittenAsExpected();
+  TestWriteWithEmptyPartitioningSchema();
 }
 
 TEST_F(TestIpcFileFormat, OpenFailureWithRelevantError) {
@@ -506,7 +257,7 @@ TEST_F(TestIpcFileFormat, ScanRecordBatchReaderProjected) {
   int64_t row_count = 0;
 
   for (auto maybe_batch : Batches(fragment.get())) {
-    ASSERT_OK_AND_ASSIGN(auto batch, std::move(maybe_batch));
+    ASSERT_OK_AND_ASSIGN(auto batch, maybe_batch);
     row_count += batch->num_rows();
     AssertSchemaEqual(*batch->schema(), *expected_schema,
                       /*check_metadata=*/false);
@@ -553,7 +304,7 @@ TEST_F(TestIpcFileFormat, ScanRecordBatchReaderProjectedMissingCols) {
     int64_t row_count = 0;
 
     for (auto maybe_batch : Batches(fragment.get())) {
-      ASSERT_OK_AND_ASSIGN(auto batch, std::move(maybe_batch));
+      ASSERT_OK_AND_ASSIGN(auto batch, maybe_batch);
       row_count += batch->num_rows();
       AssertSchemaEqual(*batch->schema(), *expected_schema,
                         /*check_metadata=*/false);
diff --git a/cpp/src/arrow/dataset/file_parquet.cc b/cpp/src/arrow/dataset/file_parquet.cc
index 4581faa29a4..e06cee649ef 100644
--- a/cpp/src/arrow/dataset/file_parquet.cc
+++ b/cpp/src/arrow/dataset/file_parquet.cc
@@ -34,6 +34,7 @@
 #include "arrow/util/range.h"
 #include "parquet/arrow/reader.h"
 #include "parquet/arrow/schema.h"
+#include "parquet/arrow/writer.h"
 #include "parquet/file_reader.h"
 #include "parquet/properties.h"
 #include "parquet/statistics.h"
@@ -41,6 +42,7 @@
 namespace arrow {
 
 using internal::checked_cast;
+using internal::checked_pointer_cast;
 
 namespace dataset {
 
@@ -51,12 +53,12 @@ using parquet::arrow::StatisticsAsScalars;
 /// \brief A ScanTask backed by a parquet file and a RowGroup within a parquet file.
 class ParquetScanTask : public ScanTask {
  public:
-  ParquetScanTask(RowGroupInfo row_group, std::vector<int> column_projection,
+  ParquetScanTask(int row_group, std::vector<int> column_projection,
                   std::shared_ptr<parquet::arrow::FileReader> reader,
                   std::shared_ptr<ScanOptions> options,
                   std::shared_ptr<ScanContext> context)
       : ScanTask(std::move(options), std::move(context)),
-        row_group_(std::move(row_group)),
+        row_group_(row_group),
         column_projection_(std::move(column_projection)),
         reader_(std::move(reader)) {}
 
@@ -65,36 +67,31 @@ class ParquetScanTask : public ScanTask {
     // control the memory usage of consumers who materialize all ScanTasks
     // before dispatching them, e.g. for scheduling purposes.
     //
-    // Thus the memory incurred by the RecordBatchReader is allocated when
-    // Scan is called.
-    std::unique_ptr<RecordBatchReader> record_batch_reader;
-    RETURN_NOT_OK(reader_->GetRecordBatchReader({row_group_.id()}, column_projection_,
-                                                &record_batch_reader));
-    return IteratorFromReader(std::move(record_batch_reader));
+    // The memory and IO incurred by the RecordBatchReader is allocated only
+    // when Execute is called.
+    struct {
+      Result<std::shared_ptr<RecordBatch>> operator()() const {
+        return record_batch_reader->Next();
+      }
+
+      // The RecordBatchIterator must hold a reference to the FileReader;
+      // since it must outlive the wrapped RecordBatchReader
+      std::shared_ptr<parquet::arrow::FileReader> file_reader;
+      std::unique_ptr<RecordBatchReader> record_batch_reader;
+    } NextBatch;
+
+    NextBatch.file_reader = reader_;
+    RETURN_NOT_OK(reader_->GetRecordBatchReader({row_group_}, column_projection_,
+                                                &NextBatch.record_batch_reader));
+    return MakeFunctionIterator(std::move(NextBatch));
   }
 
  private:
-  RowGroupInfo row_group_;
+  int row_group_;
   std::vector<int> column_projection_;
-  // The ScanTask _must_ hold a reference to reader_ because there's no
-  // guarantee the producing ParquetScanTaskIterator is still alive. This is a
-  // contract required by record_batch_reader_
   std::shared_ptr<parquet::arrow::FileReader> reader_;
 };
 
-static Result<std::unique_ptr<parquet::ParquetFileReader>> OpenReader(
-    const FileSource& source, parquet::ReaderProperties properties) {
-  ARROW_ASSIGN_OR_RAISE(auto input, source.Open());
-  try {
-    return parquet::ParquetFileReader::Open(std::move(input), std::move(properties));
-  } catch (const ::parquet::ParquetException& e) {
-    return Status::IOError("Could not open parquet input source '", source.path(),
-                           "': ", e.what());
-  }
-
-  return Status::UnknownError("unknown exception caught");
-}
-
 static parquet::ReaderProperties MakeReaderProperties(
     const ParquetFileFormat& format, MemoryPool* pool = default_memory_pool()) {
   parquet::ReaderProperties properties(pool);
@@ -119,26 +116,16 @@ static parquet::ArrowReaderProperties MakeArrowReaderProperties(
 }
 
 template <typename M>
-static Result<SchemaManifest> GetSchemaManifest(
+static Result<std::shared_ptr<SchemaManifest>> GetSchemaManifest(
     const M& metadata, const parquet::ArrowReaderProperties& properties) {
-  SchemaManifest manifest;
+  auto manifest = std::make_shared<SchemaManifest>();
   const std::shared_ptr<const ::arrow::KeyValueMetadata>& key_value_metadata = nullptr;
-  RETURN_NOT_OK(
-      SchemaManifest::Make(metadata.schema(), key_value_metadata, properties, &manifest));
+  RETURN_NOT_OK(SchemaManifest::Make(metadata.schema(), key_value_metadata, properties,
+                                     manifest.get()));
   return manifest;
 }
 
-static std::shared_ptr<StructScalar> MakeMinMaxScalar(std::shared_ptr<Scalar> min,
-                                                      std::shared_ptr<Scalar> max) {
-  DCHECK(min->type->Equals(max->type));
-  return std::make_shared<StructScalar>(ScalarVector{min, max},
-                                        struct_({
-                                            field("min", min->type),
-                                            field("max", max->type),
-                                        }));
-}
-
-static std::shared_ptr<StructScalar> ColumnChunkStatisticsAsStructScalar(
+static std::shared_ptr<Expression> ColumnChunkStatisticsAsExpression(
     const SchemaField& schema_field, const parquet::RowGroupMetaData& metadata) {
   // For the remaining of this function, failure to extract/parse statistics
   // are ignored by returning nullptr. The goal is two fold. First
@@ -161,8 +148,7 @@ static std::shared_ptr<StructScalar> ColumnChunkStatisticsAsStructScalar(
 
   // Optimize for corner case where all values are nulls
   if (statistics->num_values() == statistics->null_count()) {
-    auto null = MakeNullScalar(field->type());
-    return MakeMinMaxScalar(null, null);
+    return equal(std::move(field_expr), scalar(MakeNullScalar(field->type())));
   }
 
   std::shared_ptr<Scalar> min, max;
@@ -170,111 +156,69 @@ static std::shared_ptr<StructScalar> ColumnChunkStatisticsAsStructScalar(
     return nullptr;
   }
 
-  return MakeMinMaxScalar(std::move(min), std::move(max));
-}
-
-static std::shared_ptr<StructScalar> RowGroupStatisticsAsStructScalar(
-    const parquet::RowGroupMetaData& metadata, const SchemaManifest& manifest) {
-  FieldVector fields;
-  ScalarVector statistics;
-  for (const auto& schema_field : manifest.schema_fields) {
-    if (auto min_max = ColumnChunkStatisticsAsStructScalar(schema_field, metadata)) {
-      fields.push_back(field(schema_field.field->name(), min_max->type));
-      statistics.push_back(std::move(min_max));
-    }
+  auto maybe_min = min->CastTo(field->type());
+  auto maybe_max = max->CastTo(field->type());
+  if (maybe_min.ok() && maybe_max.ok()) {
+    min = maybe_min.MoveValueUnsafe();
+    max = maybe_max.MoveValueUnsafe();
+    return and_(greater_equal(field_expr, scalar(min)),
+                less_equal(field_expr, scalar(max)));
   }
 
-  return std::make_shared<StructScalar>(std::move(statistics),
-                                        struct_(std::move(fields)));
+  return nullptr;
 }
 
-class ParquetScanTaskIterator {
- public:
-  static Result<ScanTaskIterator> Make(std::shared_ptr<ScanOptions> options,
-                                       std::shared_ptr<ScanContext> context,
-                                       FileSource source,
-                                       std::unique_ptr<parquet::arrow::FileReader> reader,
-                                       std::vector<RowGroupInfo> row_groups) {
-    auto column_projection = InferColumnProjection(*reader, *options);
-    return static_cast<ScanTaskIterator>(ParquetScanTaskIterator(
-        std::move(options), std::move(context), std::move(source), std::move(reader),
-        std::move(column_projection), std::move(row_groups)));
-  }
-
-  Result<std::shared_ptr<ScanTask>> Next() {
-    if (idx_ >= row_groups_.size()) {
-      return nullptr;
-    }
-
-    auto row_group = row_groups_[idx_++];
-    return std::shared_ptr<ScanTask>(
-        new ParquetScanTask(row_group, column_projection_, reader_, options_, context_));
-  }
-
- private:
-  // Compute the column projection out of an optional arrow::Schema
-  static std::vector<int> InferColumnProjection(const parquet::arrow::FileReader& reader,
-                                                const ScanOptions& options) {
-    auto manifest = reader.manifest();
-    // Checks if the field is needed in either the projection or the filter.
-    auto field_names = options.MaterializedFields();
-    std::unordered_set<std::string> materialized_fields{field_names.cbegin(),
-                                                        field_names.cend()};
-    auto should_materialize_column = [&materialized_fields](const std::string& f) {
-      return materialized_fields.find(f) != materialized_fields.end();
-    };
-
-    std::vector<int> columns_selection;
-    // Note that the loop is using the file's schema to iterate instead of the
-    // materialized fields of the ScanOptions. This ensures that missing
-    // fields in the file (but present in the ScanOptions) will be ignored. The
-    // scanner's projector will take care of padding the column with the proper
-    // values.
-    for (const auto& schema_field : manifest.schema_fields) {
-      if (should_materialize_column(schema_field.field->name())) {
-        AddColumnIndices(schema_field, &columns_selection);
-      }
+static void AddColumnIndices(const SchemaField& schema_field,
+                             std::vector<int>* column_projection) {
+  if (schema_field.is_leaf()) {
+    column_projection->push_back(schema_field.column_index);
+  } else {
+    // The following ensure that complex types, e.g. struct,  are materialized.
+    for (const auto& child : schema_field.children) {
+      AddColumnIndices(child, column_projection);
     }
-
-    return columns_selection;
   }
+}
 
-  static void AddColumnIndices(const SchemaField& schema_field,
-                               std::vector<int>* column_projection) {
-    if (schema_field.is_leaf()) {
-      column_projection->push_back(schema_field.column_index);
-    } else {
-      // The following ensure that complex types, e.g. struct,  are materialized.
-      for (const auto& child : schema_field.children) {
-        AddColumnIndices(child, column_projection);
-      }
+// Compute the column projection out of an optional arrow::Schema
+static std::vector<int> InferColumnProjection(const parquet::arrow::FileReader& reader,
+                                              const ScanOptions& options) {
+  auto manifest = reader.manifest();
+  // Checks if the field is needed in either the projection or the filter.
+  auto field_names = options.MaterializedFields();
+  std::unordered_set<std::string> materialized_fields{field_names.cbegin(),
+                                                      field_names.cend()};
+  auto should_materialize_column = [&materialized_fields](const std::string& f) {
+    return materialized_fields.find(f) != materialized_fields.end();
+  };
+
+  std::vector<int> columns_selection;
+  // Note that the loop is using the file's schema to iterate instead of the
+  // materialized fields of the ScanOptions. This ensures that missing
+  // fields in the file (but present in the ScanOptions) will be ignored. The
+  // scanner's projector will take care of padding the column with the proper
+  // values.
+  for (const auto& schema_field : manifest.schema_fields) {
+    if (should_materialize_column(schema_field.field->name())) {
+      AddColumnIndices(schema_field, &columns_selection);
     }
   }
 
-  ParquetScanTaskIterator(std::shared_ptr<ScanOptions> options,
-                          std::shared_ptr<ScanContext> context, FileSource source,
-                          std::unique_ptr<parquet::arrow::FileReader> reader,
-                          std::vector<int> column_projection,
-                          std::vector<RowGroupInfo> row_groups)
-      : options_(std::move(options)),
-        context_(std::move(context)),
-        source_(std::move(source)),
-        reader_(std::move(reader)),
-        column_projection_(std::move(column_projection)),
-        row_groups_(std::move(row_groups)) {}
-
-  std::shared_ptr<ScanOptions> options_;
-  std::shared_ptr<ScanContext> context_;
+  return columns_selection;
+}
 
-  FileSource source_;
-  std::shared_ptr<parquet::arrow::FileReader> reader_;
+bool ParquetFileFormat::Equals(const FileFormat& other) const {
+  if (other.type_name() != type_name()) return false;
 
-  std::vector<int> column_projection_;
-  std::vector<RowGroupInfo> row_groups_;
+  const auto& other_reader_options =
+      checked_cast<const ParquetFileFormat&>(other).reader_options;
 
-  // row group index.
-  size_t idx_ = 0;
-};
+  // FIXME implement comparison for decryption options
+  // FIXME extract these to scan time options so comparison is unnecessary
+  return reader_options.use_buffered_stream == other_reader_options.use_buffered_stream &&
+         reader_options.buffer_size == other_reader_options.buffer_size &&
+         reader_options.dict_columns == other_reader_options.dict_columns;
+}
 
 ParquetFileFormat::ParquetFileFormat(const parquet::ReaderProperties& reader_properties) {
   reader_options.use_buffered_stream = reader_properties.is_buffered_stream_enabled();
@@ -313,7 +257,15 @@ Result<std::unique_ptr<parquet::arrow::FileReader>> ParquetFileFormat::GetReader
     const FileSource& source, ScanOptions* options, ScanContext* context) const {
   MemoryPool* pool = context ? context->pool : default_memory_pool();
   auto properties = MakeReaderProperties(*this, pool);
-  ARROW_ASSIGN_OR_RAISE(auto reader, OpenReader(source, std::move(properties)));
+
+  ARROW_ASSIGN_OR_RAISE(auto input, source.Open());
+  std::unique_ptr<parquet::ParquetFileReader> reader;
+  try {
+    reader = parquet::ParquetFileReader::Open(std::move(input), std::move(properties));
+  } catch (const ::parquet::ParquetException& e) {
+    return Status::IOError("Could not open parquet input source '", source.path(),
+                           "': ", e.what());
+  }
 
   std::shared_ptr<parquet::FileMetaData> metadata = reader->metadata();
   auto arrow_properties = MakeArrowReaderProperties(*this, *metadata);
@@ -322,59 +274,67 @@ Result<std::unique_ptr<parquet::arrow::FileReader>> ParquetFileFormat::GetReader
     arrow_properties.set_batch_size(options->batch_size);
   }
 
+  if (context && !context->use_threads) {
+    arrow_properties.set_use_threads(reader_options.enable_parallel_column_conversion);
+  }
+
   std::unique_ptr<parquet::arrow::FileReader> arrow_reader;
   RETURN_NOT_OK(parquet::arrow::FileReader::Make(
       pool, std::move(reader), std::move(arrow_properties), &arrow_reader));
   return std::move(arrow_reader);
 }
 
-static inline bool RowGroupInfosAreComplete(const std::vector<RowGroupInfo>& infos) {
-  return !infos.empty() &&
-         std::all_of(infos.cbegin(), infos.cend(),
-                     [](const RowGroupInfo& i) { return i.HasStatistics(); });
-}
-
 Result<ScanTaskIterator> ParquetFileFormat::ScanFile(std::shared_ptr<ScanOptions> options,
                                                      std::shared_ptr<ScanContext> context,
                                                      FileFragment* fragment) const {
   auto* parquet_fragment = checked_cast<ParquetFileFragment*>(fragment);
-  std::vector<RowGroupInfo> row_groups;
+  std::vector<int> row_groups;
+
+  bool pre_filtered = false;
+  auto empty = [] { return MakeEmptyIterator<std::shared_ptr<ScanTask>>(); };
 
   // If RowGroup metadata is cached completely we can pre-filter RowGroups before opening
   // a FileReader, potentially avoiding IO altogether if all RowGroups are excluded due to
   // prior statistics knowledge. In the case where a RowGroup doesn't have statistics
   // metdata, it will not be excluded.
-  if (parquet_fragment->HasCompleteMetadata()) {
+  if (parquet_fragment->metadata() != nullptr) {
     ARROW_ASSIGN_OR_RAISE(row_groups,
                           parquet_fragment->FilterRowGroups(*options->filter));
-    if (row_groups.empty()) {
-      return MakeEmptyIterator<std::shared_ptr<ScanTask>>();
-    }
+
+    pre_filtered = true;
+    if (row_groups.empty()) empty();
   }
 
   // Open the reader and pay the real IO cost.
-  ARROW_ASSIGN_OR_RAISE(auto reader,
+  ARROW_ASSIGN_OR_RAISE(std::shared_ptr<parquet::arrow::FileReader> reader,
                         GetReader(fragment->source(), options.get(), context.get()));
 
-  if (!parquet_fragment->HasCompleteMetadata()) {
+  // Ensure that parquet_fragment has FileMetaData
+  RETURN_NOT_OK(parquet_fragment->EnsureCompleteMetadata(reader.get()));
+
+  if (!pre_filtered) {
     // row groups were not already filtered; do this now
-    RETURN_NOT_OK(parquet_fragment->EnsureCompleteMetadata(reader.get()));
     ARROW_ASSIGN_OR_RAISE(row_groups,
                           parquet_fragment->FilterRowGroups(*options->filter));
-    if (row_groups.empty()) {
-      return MakeEmptyIterator<std::shared_ptr<ScanTask>>();
-    }
+
+    if (row_groups.empty()) empty();
+  }
+
+  auto column_projection = InferColumnProjection(*reader, *options);
+  ScanTaskVector tasks(row_groups.size());
+
+  for (size_t i = 0; i < row_groups.size(); ++i) {
+    tasks[i] = std::make_shared<ParquetScanTask>(row_groups[i], column_projection, reader,
+                                                 options, context);
   }
 
-  return ParquetScanTaskIterator::Make(std::move(options), std::move(context),
-                                       fragment->source(), std::move(reader),
-                                       std::move(row_groups));
+  return MakeVectorIterator(std::move(tasks));
 }
 
-Result<std::shared_ptr<FileFragment>> ParquetFileFormat::MakeFragment(
+Result<std::shared_ptr<ParquetFileFragment>> ParquetFileFormat::MakeFragment(
     FileSource source, std::shared_ptr<Expression> partition_expression,
-    std::vector<RowGroupInfo> row_groups, std::shared_ptr<Schema> physical_schema) {
-  return std::shared_ptr<FileFragment>(new ParquetFileFragment(
+    std::shared_ptr<Schema> physical_schema, std::vector<int> row_groups) {
+  return std::shared_ptr<ParquetFileFragment>(new ParquetFileFragment(
       std::move(source), shared_from_this(), std::move(partition_expression),
       std::move(physical_schema), std::move(row_groups)));
 }
@@ -384,97 +344,77 @@ Result<std::shared_ptr<FileFragment>> ParquetFileFormat::MakeFragment(
     std::shared_ptr<Schema> physical_schema) {
   return std::shared_ptr<FileFragment>(new ParquetFileFragment(
       std::move(source), shared_from_this(), std::move(partition_expression),
-      std::move(physical_schema), {}));
+      std::move(physical_schema), util::nullopt));
 }
 
-///
-/// RowGroupInfo
-///
+//
+// ParquetFileWriter, ParquetFileWriteOptions
+//
 
-std::vector<RowGroupInfo> RowGroupInfo::FromIdentifiers(const std::vector<int> ids) {
-  std::vector<RowGroupInfo> results;
-  results.reserve(ids.size());
-  for (auto i : ids) {
-    results.emplace_back(i);
-  }
-  return results;
+std::shared_ptr<FileWriteOptions> ParquetFileFormat::DefaultWriteOptions() {
+  std::shared_ptr<ParquetFileWriteOptions> options(
+      new ParquetFileWriteOptions(shared_from_this()));
+  options->writer_properties = parquet::default_writer_properties();
+  options->arrow_writer_properties = parquet::default_arrow_writer_properties();
+  return options;
 }
 
-std::vector<RowGroupInfo> RowGroupInfo::FromCount(int count) {
-  std::vector<RowGroupInfo> result;
-  result.reserve(count);
-  for (int i = 0; i < count; i++) {
-    result.emplace_back(i);
+Result<std::shared_ptr<FileWriter>> ParquetFileFormat::MakeWriter(
+    std::shared_ptr<io::OutputStream> destination, std::shared_ptr<Schema> schema,
+    std::shared_ptr<FileWriteOptions> options) const {
+  if (!Equals(*options->format())) {
+    return Status::TypeError("Mismatching format/write options");
   }
-  return result;
-}
 
-void RowGroupInfo::SetStatisticsExpression() {
-  if (!HasStatistics()) {
-    statistics_expression_ = nullptr;
-    return;
-  }
+  auto parquet_options = checked_pointer_cast<ParquetFileWriteOptions>(options);
 
-  if (statistics_->value.empty()) {
-    statistics_expression_ = scalar(true);
-    return;
-  }
-
-  ExpressionVector expressions{statistics_->value.size()};
-
-  for (size_t i = 0; i < expressions.size(); ++i) {
-    const auto& col_stats =
-        internal::checked_cast<const StructScalar&>(*statistics_->value[i]);
-    auto field_expr = field_ref(statistics_->type->field(static_cast<int>(i))->name());
+  std::unique_ptr<parquet::arrow::FileWriter> parquet_writer;
+  RETURN_NOT_OK(parquet::arrow::FileWriter::Open(
+      *schema, default_memory_pool(), destination, parquet_options->writer_properties,
+      parquet_options->arrow_writer_properties, &parquet_writer));
 
-    DCHECK_EQ(col_stats.value.size(), 2);
-    const auto& min = col_stats.value[0];
-    const auto& max = col_stats.value[1];
+  return std::shared_ptr<FileWriter>(
+      new ParquetFileWriter(std::move(parquet_writer), std::move(parquet_options)));
+}
 
-    DCHECK_EQ(min->is_valid, max->is_valid);
-    expressions[i] = min->is_valid ? and_(greater_equal(field_expr, scalar(min)),
-                                          less_equal(field_expr, scalar(max)))
-                                   : equal(std::move(field_expr), scalar(min));
-  }
+ParquetFileWriter::ParquetFileWriter(std::shared_ptr<parquet::arrow::FileWriter> writer,
+                                     std::shared_ptr<ParquetFileWriteOptions> options)
+    : FileWriter(writer->schema(), std::move(options)),
+      parquet_writer_(std::move(writer)) {}
 
-  statistics_expression_ = and_(std::move(expressions));
+Status ParquetFileWriter::Write(const std::shared_ptr<RecordBatch>& batch) {
+  ARROW_ASSIGN_OR_RAISE(auto table, Table::FromRecordBatches(batch->schema(), {batch}));
+  return parquet_writer_->WriteTable(*table, batch->num_rows());
 }
 
-bool RowGroupInfo::Satisfy(const Expression& predicate) const {
-  return !HasStatistics() || predicate.IsSatisfiableWith(statistics_expression_);
-}
+Status ParquetFileWriter::Finish() { return parquet_writer_->Close(); }
 
-///
-/// ParquetFileFragment
-///
+//
+// ParquetFileFragment
+//
 
 ParquetFileFragment::ParquetFileFragment(FileSource source,
                                          std::shared_ptr<FileFormat> format,
                                          std::shared_ptr<Expression> partition_expression,
                                          std::shared_ptr<Schema> physical_schema,
-                                         std::vector<RowGroupInfo> row_groups)
+                                         util::optional<std::vector<int>> row_groups)
     : FileFragment(std::move(source), std::move(format), std::move(partition_expression),
                    std::move(physical_schema)),
-      row_groups_(std::move(row_groups)),
       parquet_format_(checked_cast<ParquetFileFormat&>(*format_)),
-      has_complete_metadata_(RowGroupInfosAreComplete(row_groups_) &&
-                             physical_schema_ != nullptr) {}
+      row_groups_(std::move(row_groups)) {}
 
 Status ParquetFileFragment::EnsureCompleteMetadata(parquet::arrow::FileReader* reader) {
-  if (HasCompleteMetadata()) {
+  auto lock = physical_schema_mutex_.Lock();
+  if (metadata_ != nullptr) {
     return Status::OK();
   }
 
   if (reader == nullptr) {
+    lock.Unlock();
     ARROW_ASSIGN_OR_RAISE(auto reader, parquet_format_.GetReader(source_));
     return EnsureCompleteMetadata(reader.get());
   }
 
-  auto lock = physical_schema_mutex_.Lock();
-  if (HasCompleteMetadata()) {
-    return Status::OK();
-  }
-
   std::shared_ptr<Schema> schema;
   RETURN_NOT_OK(reader->GetSchema(&schema));
   if (physical_schema_ && !physical_schema_->Equals(*schema)) {
@@ -484,33 +424,36 @@ Status ParquetFileFragment::EnsureCompleteMetadata(parquet::arrow::FileReader* r
   }
   physical_schema_ = std::move(schema);
 
-  std::shared_ptr<parquet::FileMetaData> metadata = reader->parquet_reader()->metadata();
-  int num_row_groups = metadata->num_row_groups();
-
-  if (row_groups_.empty()) {
-    row_groups_ = RowGroupInfo::FromCount(num_row_groups);
+  if (!row_groups_) {
+    row_groups_ = internal::Iota(reader->num_row_groups());
   }
 
-  for (const RowGroupInfo& info : row_groups_) {
-    // Ensure RowGroups are indexing valid RowGroups before augmenting.
-    if (info.id() >= num_row_groups) {
-      return Status::IndexError("Trying to scan row group ", info.id(), " but ",
-                                source_.path(), " only has ", num_row_groups,
-                                " row groups");
-    }
-  }
+  ARROW_ASSIGN_OR_RAISE(
+      auto manifest,
+      GetSchemaManifest(*reader->parquet_reader()->metadata(), reader->properties()));
+  return SetMetadata(reader->parquet_reader()->metadata(), std::move(manifest));
+}
+
+Status ParquetFileFragment::SetMetadata(
+    std::shared_ptr<parquet::FileMetaData> metadata,
+    std::shared_ptr<parquet::arrow::SchemaManifest> manifest) {
+  DCHECK(row_groups_.has_value());
 
-  for (RowGroupInfo& info : row_groups_) {
-    // Augment a RowGroup with statistics if missing.
-    if (info.HasStatistics()) continue;
+  metadata_ = std::move(metadata);
+  manifest_ = std::move(manifest);
+
+  statistics_expressions_.resize(row_groups_->size(), scalar(true));
+  statistics_expressions_complete_.resize(physical_schema_->num_fields(), false);
+
+  for (int row_group : *row_groups_) {
+    // Ensure RowGroups are indexing valid RowGroups before augmenting.
+    if (row_group < metadata_->num_row_groups()) continue;
 
-    auto row_group = metadata->RowGroup(info.id());
-    auto statistics = RowGroupStatisticsAsStructScalar(*row_group, reader->manifest());
-    info = RowGroupInfo(info.id(), row_group->num_rows(), row_group->total_byte_size(),
-                        std::move(statistics));
+    return Status::IndexError("ParquetFileFragment references row group ", row_group,
+                              " but ", source_.path(), " only has ",
+                              metadata_->num_row_groups(), " row groups");
   }
 
-  has_complete_metadata_ = true;
   return Status::OK();
 }
 
@@ -520,48 +463,134 @@ Result<FragmentVector> ParquetFileFragment::SplitByRowGroup(
   ARROW_ASSIGN_OR_RAISE(auto row_groups, FilterRowGroups(*predicate));
 
   FragmentVector fragments(row_groups.size());
-  auto fragment = fragments.begin();
-  for (auto&& row_group : row_groups) {
-    ARROW_ASSIGN_OR_RAISE(*fragment++,
+  int i = 0;
+  for (int row_group : row_groups) {
+    ARROW_ASSIGN_OR_RAISE(auto fragment,
                           parquet_format_.MakeFragment(source_, partition_expression(),
-                                                       {std::move(row_group)}));
+                                                       physical_schema_, {row_group}));
+
+    RETURN_NOT_OK(fragment->SetMetadata(metadata_, manifest_));
+    fragments[i++] = std::move(fragment);
   }
 
   return fragments;
 }
 
-Result<std::vector<RowGroupInfo>> ParquetFileFragment::FilterRowGroups(
+Result<std::shared_ptr<Fragment>> ParquetFileFragment::Subset(
+    const std::shared_ptr<Expression>& predicate) {
+  RETURN_NOT_OK(EnsureCompleteMetadata());
+  ARROW_ASSIGN_OR_RAISE(auto row_groups, FilterRowGroups(*predicate));
+  return Subset(std::move(row_groups));
+}
+
+Result<std::shared_ptr<Fragment>> ParquetFileFragment::Subset(
+    std::vector<int> row_groups) {
+  RETURN_NOT_OK(EnsureCompleteMetadata());
+  ARROW_ASSIGN_OR_RAISE(auto new_fragment, parquet_format_.MakeFragment(
+                                               source_, partition_expression(),
+                                               physical_schema_, std::move(row_groups)));
+
+  RETURN_NOT_OK(new_fragment->SetMetadata(metadata_, manifest_));
+  return new_fragment;
+}
+
+inline void FoldingAnd(std::shared_ptr<Expression>* l, std::shared_ptr<Expression> r) {
+  if ((*l)->Equals(true)) {
+    *l = std::move(r);
+  } else {
+    *l = and_(std::move(*l), std::move(r));
+  }
+}
+
+Result<std::vector<int>> ParquetFileFragment::FilterRowGroups(
     const Expression& predicate) {
-  DCHECK(has_complete_metadata_);
+  auto lock = physical_schema_mutex_.Lock();
+
+  DCHECK_NE(metadata_, nullptr);
   RETURN_NOT_OK(predicate.Validate(*physical_schema_));
 
+  for (FieldRef ref : FieldsInExpression(predicate)) {
+    ARROW_ASSIGN_OR_RAISE(auto path, ref.FindOneOrNone(*physical_schema_));
+
+    if (!path) continue;
+    if (statistics_expressions_complete_[path[0]]) continue;
+    statistics_expressions_complete_[path[0]] = true;
+
+    const SchemaField& schema_field = manifest_->schema_fields[path[0]];
+    int i = 0;
+    for (int row_group : *row_groups_) {
+      auto row_group_metadata = metadata_->RowGroup(row_group);
+
+      if (auto minmax =
+              ColumnChunkStatisticsAsExpression(schema_field, *row_group_metadata)) {
+        FoldingAnd(&statistics_expressions_[i], std::move(minmax));
+      }
+
+      ++i;
+    }
+  }
+
   auto simplified_predicate = predicate.Assume(partition_expression_);
   if (!simplified_predicate->IsSatisfiable()) {
-    return std::vector<RowGroupInfo>{};
+    return std::vector<int>{};
+  }
+
+  std::vector<int> row_groups;
+  for (size_t i = 0; i < row_groups_->size(); ++i) {
+    if (simplified_predicate->IsSatisfiableWith(statistics_expressions_[i])) {
+      row_groups.push_back(row_groups_->at(i));
+    }
   }
 
-  auto row_groups = row_groups_;
-  auto end = std::remove_if(row_groups.begin(), row_groups.end(),
-                            [&simplified_predicate](const RowGroupInfo& info) {
-                              return !info.Satisfy(*simplified_predicate);
-                            });
-  row_groups.erase(end, row_groups.end());
   return row_groups;
 }
 
-///
-/// ParquetDatasetFactory
-///
+//
+// ParquetDatasetFactory
+//
+
+static inline Result<std::string> FileFromRowGroup(
+    fs::FileSystem* filesystem, const std::string& base_path,
+    const parquet::RowGroupMetaData& row_group, bool validate_column_chunk_paths) {
+  constexpr auto prefix = "Extracting file path from RowGroup failed. ";
 
-ParquetDatasetFactory::ParquetDatasetFactory(
-    std::shared_ptr<fs::FileSystem> filesystem, std::shared_ptr<ParquetFileFormat> format,
-    std::shared_ptr<parquet::FileMetaData> metadata, std::string base_path,
-    ParquetFactoryOptions options)
-    : filesystem_(std::move(filesystem)),
-      format_(std::move(format)),
-      metadata_(std::move(metadata)),
-      base_path_(std::move(base_path)),
-      options_(std::move(options)) {}
+  if (row_group.num_columns() == 0) {
+    return Status::Invalid(prefix,
+                           "RowGroup must have a least one column to extract path.");
+  }
+
+  auto path = row_group.ColumnChunk(0)->file_path();
+  if (path == "") {
+    return Status::Invalid(
+        prefix,
+        "The column chunks' file paths should be set, but got an empty file path.");
+  }
+
+  if (validate_column_chunk_paths) {
+    for (int i = 1; i < row_group.num_columns(); ++i) {
+      const auto& column_path = row_group.ColumnChunk(i)->file_path();
+      if (column_path != path) {
+        return Status::Invalid(prefix, "Path '", column_path, "' not equal to path '",
+                               path, ", for ColumnChunk at index ", i,
+                               "; ColumnChunks in a RowGroup must have the same path.");
+      }
+    }
+  }
+
+  path = fs::internal::JoinAbstractPath(
+      std::vector<std::string>{base_path, std::move(path)});
+  // Normalizing path is required for Windows.
+  return filesystem->NormalizePath(std::move(path));
+}
+
+Result<std::shared_ptr<Schema>> GetSchema(
+    const parquet::FileMetaData& metadata,
+    const parquet::ArrowReaderProperties& properties) {
+  std::shared_ptr<Schema> schema;
+  RETURN_NOT_OK(parquet::arrow::FromParquetSchema(
+      metadata.schema(), properties, metadata.key_value_metadata(), &schema));
+  return schema;
+}
 
 Result<std::shared_ptr<DatasetFactory>> ParquetDatasetFactory::Make(
     const std::string& metadata_path, std::shared_ptr<fs::FileSystem> filesystem,
@@ -590,155 +619,80 @@ Result<std::shared_ptr<DatasetFactory>> ParquetDatasetFactory::Make(
   ARROW_ASSIGN_OR_RAISE(auto reader, format->GetReader(metadata_source));
   std::shared_ptr<parquet::FileMetaData> metadata = reader->parquet_reader()->metadata();
 
-  return std::shared_ptr<DatasetFactory>(
-      new ParquetDatasetFactory(std::move(filesystem), std::move(format),
-                                std::move(metadata), base_path, std::move(options)));
-}
+  if (metadata->num_columns() == 0) {
+    return Status::Invalid(
+        "ParquetDatasetFactory must contain a schema with at least one column");
+  }
 
-static inline Result<std::string> FileFromRowGroup(
-    fs::FileSystem* filesystem, const std::string& base_path,
-    const parquet::RowGroupMetaData& row_group) {
-  try {
-    auto n_columns = row_group.num_columns();
-    if (n_columns == 0) {
-      return Status::Invalid(
-          "Extracting file path from RowGroup failed. RowGroup must have a least one "
-          "columns to extract path");
-    }
+  auto properties = MakeArrowReaderProperties(*format, *metadata);
+  ARROW_ASSIGN_OR_RAISE(auto physical_schema, GetSchema(*metadata, properties));
+  ARROW_ASSIGN_OR_RAISE(auto manifest, GetSchemaManifest(*metadata, properties));
 
-    auto first_column = row_group.ColumnChunk(0);
-    auto path = first_column->file_path();
-    if (path == "") {
-      return Status::Invalid(
-          "Extracting file path from RowGroup failed. The column chunks "
-          "file path should be set, but got an empty file path.");
-    }
+  std::unordered_map<std::string, std::vector<int>> path_to_row_group_ids;
 
-    for (int i = 1; i < n_columns; i++) {
-      auto column = row_group.ColumnChunk(i);
-      auto column_path = column->file_path();
-      if (column_path != path) {
-        return Status::Invalid("Extracting file path from RowGroup failed. Path '",
-                               column_path, "' not equal to path '", path,
-                               ", for ColumnChunk at index ", i,
-                               "; ColumnChunks in a RowGroup must have the same path.");
-      }
-    }
-
-    // TODO Is it possible to infer the file size and return a populated FileInfo?
-    // This could avoid some spurious HEAD requests on S3 (ARROW-8950)
-    path = fs::internal::JoinAbstractPath(std::vector<std::string>{base_path, path});
-    // Normalizing path is required for Windows.
-    return filesystem->NormalizePath(std::move(path));
-  } catch (const ::parquet::ParquetException& e) {
-    return Status::Invalid("Extracting file path from RowGroup failed. Parquet threw:",
-                           e.what());
-  }
-}
+  for (int i = 0; i < metadata->num_row_groups(); i++) {
+    auto row_group = metadata->RowGroup(i);
+    ARROW_ASSIGN_OR_RAISE(auto path,
+                          FileFromRowGroup(filesystem.get(), base_path, *row_group,
+                                           options.validate_column_chunk_paths));
 
-Result<std::vector<std::string>> ParquetDatasetFactory::CollectPaths(
-    const parquet::FileMetaData& metadata,
-    const parquet::ArrowReaderProperties& properties) {
-  try {
-    std::unordered_set<std::string> unique_paths;
-    ARROW_ASSIGN_OR_RAISE(auto manifest, GetSchemaManifest(metadata, properties));
-
-    for (int i = 0; i < metadata.num_row_groups(); i++) {
-      std::shared_ptr<parquet::RowGroupMetaData> row_group = metadata.RowGroup(i);
-      ARROW_ASSIGN_OR_RAISE(auto path,
-                            FileFromRowGroup(filesystem_.get(), base_path_, *row_group));
-      unique_paths.emplace(std::move(path));
-    }
-
-    std::vector<std::string> paths;
-    for (const auto& path : unique_paths) {
-      paths.emplace_back(path);
-    }
-    return paths;
-  } catch (const ::parquet::ParquetException& e) {
-    return Status::Invalid("Could not infer file paths from FileMetaData:", e.what());
+    // Insert the path, or increase the count of row groups. It will be assumed that the
+    // RowGroup of a file are ordered exactly as in the metadata file.
+    auto row_groups = &path_to_row_group_ids.insert({std::move(path), {}}).first->second;
+    row_groups->emplace_back(i);
   }
-}
 
-Result<std::shared_ptr<Schema>> GetSchema(
-    const parquet::FileMetaData& metadata,
-    const parquet::ArrowReaderProperties& properties) {
-  std::shared_ptr<Schema> schema;
-  RETURN_NOT_OK(parquet::arrow::FromParquetSchema(
-      metadata.schema(), properties, metadata.key_value_metadata(), &schema));
-  return schema;
+  return std::shared_ptr<DatasetFactory>(new ParquetDatasetFactory(
+      std::move(filesystem), std::move(format), std::move(metadata), std::move(manifest),
+      std::move(physical_schema), base_path, std::move(options),
+      std::move(path_to_row_group_ids)));
 }
 
 Result<std::vector<std::shared_ptr<FileFragment>>>
-ParquetDatasetFactory::CollectParquetFragments(
-    const parquet::FileMetaData& metadata,
-    const parquet::ArrowReaderProperties& properties, const Partitioning& partitioning) {
-  try {
-    auto n_columns = metadata.num_columns();
-    if (n_columns == 0) {
-      return Status::Invalid(
-          "ParquetDatasetFactory must contain a schema with at least one column");
-    }
+ParquetDatasetFactory::CollectParquetFragments(const Partitioning& partitioning) {
+  std::vector<std::shared_ptr<FileFragment>> fragments(path_to_row_group_ids_.size());
 
-    std::unordered_map<std::string, std::vector<RowGroupInfo>> path_to_row_group_infos;
-    ARROW_ASSIGN_OR_RAISE(auto manifest, GetSchemaManifest(metadata, properties));
-
-    for (int i = 0; i < metadata.num_row_groups(); i++) {
-      std::shared_ptr<parquet::RowGroupMetaData> row_group = metadata.RowGroup(i);
-      ARROW_ASSIGN_OR_RAISE(auto path,
-                            FileFromRowGroup(filesystem_.get(), base_path_, *row_group));
-      std::shared_ptr<StructScalar> stats =
-          RowGroupStatisticsAsStructScalar(*row_group, manifest);
-
-      int64_t num_rows = row_group->num_rows();
-      int64_t total_byte_size = row_group->total_byte_size();
-
-      // Insert the path, or increase the count of row groups. It will be assumed that the
-      // RowGroup of a file are ordered exactly as in the metadata file.
-      auto path_and_row_groups =
-          path_to_row_group_infos.emplace(path, std::vector<RowGroupInfo>{}).first;
-      auto row_group_id = static_cast<int>(path_and_row_groups->second.size());
-      path_and_row_groups->second.emplace_back(row_group_id, num_rows, total_byte_size,
-                                               stats);
-    }
+  size_t i = 0;
+  for (const auto& e : path_to_row_group_ids_) {
+    const auto& path = e.first;
+    auto metadata_subset = metadata_->Subset(e.second);
 
-    ARROW_ASSIGN_OR_RAISE(auto physical_schema, GetSchema(metadata, properties));
-    std::vector<std::shared_ptr<FileFragment>> fragments;
-    fragments.reserve(path_to_row_group_infos.size());
-    for (auto&& elem : path_to_row_group_infos) {
-      const auto& path = elem.first;
-      auto partition =
-          partitioning.Parse(StripPrefixAndFilename(path, options_.partition_base_dir))
-              .ValueOr(scalar(true));
-      ARROW_ASSIGN_OR_RAISE(
-          auto fragment, format_->MakeFragment({path, filesystem_}, std::move(partition),
-                                               std::move(elem.second), physical_schema));
-      fragments.push_back(std::move(fragment));
-    }
+    auto row_groups = internal::Iota(metadata_subset->num_row_groups());
 
-    return fragments;
-  } catch (const ::parquet::ParquetException& e) {
-    return Status::Invalid("Could not infer file paths from FileMetaData:", e.what());
+    auto partition_expression =
+        partitioning.Parse(StripPrefixAndFilename(path, options_.partition_base_dir))
+            .ValueOr(scalar(true));
+
+    ARROW_ASSIGN_OR_RAISE(
+        auto fragment,
+        format_->MakeFragment({path, filesystem_}, std::move(partition_expression),
+                              physical_schema_, std::move(row_groups)));
+
+    RETURN_NOT_OK(fragment->SetMetadata(metadata_subset, manifest_));
+    fragments[i++] = std::move(fragment);
   }
+
+  return fragments;
 }
 
 Result<std::vector<std::shared_ptr<Schema>>> ParquetDatasetFactory::InspectSchemas(
     InspectOptions options) {
-  std::vector<std::shared_ptr<Schema>> schemas;
-  auto properties = MakeArrowReaderProperties(*format_, *metadata_);
-
   // The physical_schema from the _metadata file is always yielded
-  ARROW_ASSIGN_OR_RAISE(auto physical_schema, GetSchema(*metadata_, properties));
-  schemas.push_back(std::move(physical_schema));
+  std::vector<std::shared_ptr<Schema>> schemas = {physical_schema_};
 
-  if (options_.partitioning.factory() != nullptr) {
+  if (auto factory = options_.partitioning.factory()) {
     // Gather paths found in RowGroups' ColumnChunks.
-    ARROW_ASSIGN_OR_RAISE(auto paths, CollectPaths(*metadata_, properties));
+    std::vector<std::string> stripped(path_to_row_group_ids_.size());
+
+    size_t i = 0;
+    for (const auto& e : path_to_row_group_ids_) {
+      stripped[i++] = StripPrefixAndFilename(e.first, options_.partition_base_dir);
+    }
+    ARROW_ASSIGN_OR_RAISE(auto partition_schema, factory->Inspect(stripped));
 
-    ARROW_ASSIGN_OR_RAISE(auto partition_schema,
-                          options_.partitioning.GetOrInferSchema(StripPrefixAndFilename(
-                              paths, options_.partition_base_dir)));
     schemas.push_back(std::move(partition_schema));
+  } else {
+    schemas.push_back(options_.partitioning.partitioning()->schema());
   }
 
   return schemas;
@@ -757,10 +711,8 @@ Result<std::shared_ptr<Dataset>> ParquetDatasetFactory::Finish(FinishOptions opt
     ARROW_ASSIGN_OR_RAISE(partitioning, factory->Finish(schema));
   }
 
-  auto properties = MakeArrowReaderProperties(*format_, *metadata_);
-  ARROW_ASSIGN_OR_RAISE(auto fragments,
-                        CollectParquetFragments(*metadata_, properties, *partitioning));
-  return FileSystemDataset::Make(std::move(schema), scalar(true), format_,
+  ARROW_ASSIGN_OR_RAISE(auto fragments, CollectParquetFragments(*partitioning));
+  return FileSystemDataset::Make(std::move(schema), scalar(true), format_, filesystem_,
                                  std::move(fragments));
 }
 
diff --git a/cpp/src/arrow/dataset/file_parquet.h b/cpp/src/arrow/dataset/file_parquet.h
index 13f10906da1..b82241a574b 100644
--- a/cpp/src/arrow/dataset/file_parquet.h
+++ b/cpp/src/arrow/dataset/file_parquet.h
@@ -29,24 +29,33 @@
 #include "arrow/dataset/file_base.h"
 #include "arrow/dataset/type_fwd.h"
 #include "arrow/dataset/visibility.h"
+#include "arrow/util/optional.h"
 
 namespace parquet {
 class ParquetFileReader;
+class Statistics;
+class ColumnChunkMetaData;
 class RowGroupMetaData;
 class FileMetaData;
 class FileDecryptionProperties;
+class FileEncryptionProperties;
+
 class ReaderProperties;
 class ArrowReaderProperties;
+
+class WriterProperties;
+class ArrowWriterProperties;
+
 namespace arrow {
 class FileReader;
+class FileWriter;
+struct SchemaManifest;
 }  // namespace arrow
 }  // namespace parquet
 
 namespace arrow {
 namespace dataset {
 
-class RowGroupInfo;
-
 /// \brief A FileFormat implementation that reads from Parquet files
 class ARROW_DS_EXPORT ParquetFileFormat : public FileFormat {
  public:
@@ -60,9 +69,8 @@ class ARROW_DS_EXPORT ParquetFileFormat : public FileFormat {
 
   bool splittable() const override { return true; }
 
-  // Note: the default values are exposed in the python bindings and documented
-  //       in the docstrings, if any of the default values gets changed please
-  //       update there as well.
+  bool Equals(const FileFormat& other) const override;
+
   struct ReaderOptions {
     /// \defgroup parquet-file-format-reader-properties properties which correspond to
     /// members of parquet::ReaderProperties.
@@ -87,6 +95,12 @@ class ARROW_DS_EXPORT ParquetFileFormat : public FileFormat {
     /// @{
     std::unordered_set<std::string> dict_columns;
     /// @}
+
+    /// EXPERIMENTAL: Parallelize conversion across columns. This option is ignored if a
+    /// scan is already parallelized across input files to avoid thread contention. This
+    /// option will be removed after support is added for simultaneous parallelization
+    /// across files and columns.
+    bool enable_parallel_column_conversion = false;
   } reader_options;
 
   Result<bool> IsSupported(const FileSource& source) const override;
@@ -101,82 +115,25 @@ class ARROW_DS_EXPORT ParquetFileFormat : public FileFormat {
 
   using FileFormat::MakeFragment;
 
-  /// \brief Create a Fragment, restricted to the specified row groups.
-  Result<std::shared_ptr<FileFragment>> MakeFragment(
-      FileSource source, std::shared_ptr<Expression> partition_expression,
-      std::vector<RowGroupInfo> row_groups,
-      std::shared_ptr<Schema> physical_schema = NULLPTR);
-
   /// \brief Create a Fragment targeting all RowGroups.
   Result<std::shared_ptr<FileFragment>> MakeFragment(
       FileSource source, std::shared_ptr<Expression> partition_expression,
       std::shared_ptr<Schema> physical_schema) override;
 
+  /// \brief Create a Fragment, restricted to the specified row groups.
+  Result<std::shared_ptr<ParquetFileFragment>> MakeFragment(
+      FileSource source, std::shared_ptr<Expression> partition_expression,
+      std::shared_ptr<Schema> physical_schema, std::vector<int> row_groups);
+
   /// \brief Return a FileReader on the given source.
   Result<std::unique_ptr<parquet::arrow::FileReader>> GetReader(
       const FileSource& source, ScanOptions* = NULLPTR, ScanContext* = NULLPTR) const;
-};
 
-/// \brief Represents a parquet's RowGroup with extra information.
-class ARROW_DS_EXPORT RowGroupInfo : public util::EqualityComparable<RowGroupInfo> {
- public:
-  RowGroupInfo() : RowGroupInfo(-1) {}
-
-  /// \brief Construct a RowGroup from an identifier.
-  explicit RowGroupInfo(int id) : RowGroupInfo(id, -1, -1, NULLPTR) {}
-
-  /// \brief Construct a RowGroup from an identifier with statistics.
-  RowGroupInfo(int id, int64_t num_rows, int64_t total_byte_size,
-               std::shared_ptr<StructScalar> statistics)
-      : id_(id),
-        num_rows_(num_rows),
-        total_byte_size_(total_byte_size),
-        statistics_(std::move(statistics)) {
-    SetStatisticsExpression();
-  }
-
-  /// \brief Transform a vector of identifiers into a vector of RowGroupInfos
-  static std::vector<RowGroupInfo> FromIdentifiers(const std::vector<int> ids);
-  static std::vector<RowGroupInfo> FromCount(int count);
-
-  /// \brief Return the RowGroup's identifier (index in the file).
-  int id() const { return id_; }
-
-  /// \brief Return the RowGroup's number of rows.
-  ///
-  /// If statistics are not provided, return -1.
-  int64_t num_rows() const { return num_rows_; }
-
-  /// \brief Return the RowGroup's total size in bytes.
-  ///
-  /// If statistics are not provided, return -1.
-  int64_t total_byte_size() const { return total_byte_size_; }
-
-  /// \brief Return the RowGroup's statistics as a StructScalar with a field for
-  /// each column with statistics.
-  /// Each field will also be a StructScalar with "min" and "max" fields.
-  const std::shared_ptr<StructScalar>& statistics() const { return statistics_; }
-
-  /// \brief Indicate if statistics are set.
-  bool HasStatistics() const { return statistics_ != NULLPTR; }
-
-  /// \brief Indicate if the RowGroup's statistics satisfy the predicate.
-  ///
-  /// This will return true if the RowGroup was not initialized with statistics
-  /// (rather than silently reading metadata for a complete check).
-  bool Satisfy(const Expression& predicate) const;
-
-  /// \brief Indicate if the other RowGroup points to the same RowGroup.
-  bool Equals(const RowGroupInfo& other) const { return id() == other.id(); }
-
- private:
-  void SetStatisticsExpression();
+  Result<std::shared_ptr<FileWriter>> MakeWriter(
+      std::shared_ptr<io::OutputStream> destination, std::shared_ptr<Schema> schema,
+      std::shared_ptr<FileWriteOptions> options) const override;
 
-  int id_;
-  int64_t num_rows_;
-  int64_t total_byte_size_;
-  std::shared_ptr<Expression> statistics_expression_;
-  std::shared_ptr<StructScalar> statistics_;
+  std::shared_ptr<FileWriteOptions> DefaultWriteOptions() override;
 };
 
 /// \brief A FileFragment with parquet logic.
@@ -188,32 +145,38 @@ class ARROW_DS_EXPORT RowGroupInfo : public util::EqualityComparable<RowGroupInf
 /// number of scanned RowGroups, or to partition the scans across multiple
 /// threads.
 ///
-/// It can also attach optional statistics with each RowGroups, providing
-/// pushdown predicate benefits before invoking any heavy IO. This can induce
+/// Metadata can be explicitly provided, enabling pushdown predicate benefits without
+/// the potentially heavy IO of loading Metadata from the file system. This can induce
 /// significant performance boost when scanning high latency file systems.
 class ARROW_DS_EXPORT ParquetFileFragment : public FileFragment {
  public:
   Result<FragmentVector> SplitByRowGroup(const std::shared_ptr<Expression>& predicate);
 
-  /// \brief Return the RowGroups selected by this fragment. An empty list
-  /// represents all RowGroups in the parquet file.
-  const std::vector<RowGroupInfo>& row_groups() const { return row_groups_; }
+  /// \brief Return the RowGroups selected by this fragment.
+  const std::vector<int>& row_groups() const {
+    if (row_groups_) return *row_groups_;
+    static std::vector<int> empty;
+    return empty;
+  }
 
-  /// \brief Indicate if the attached statistics are complete and the physical schema
-  /// is cached.
-  ///
-  /// The statistics are complete if the provided RowGroups (see `row_groups()`)
-  /// is not empty / and all RowGroup return true on `RowGroup::HasStatistics()`.
-  bool HasCompleteMetadata() const { return has_complete_metadata_; }
+  /// \brief Return the FileMetaData associated with this fragment.
+  const std::shared_ptr<parquet::FileMetaData>& metadata() const { return metadata_; }
 
-  /// \brief Ensure attached statistics are complete and the physical schema is cached.
+  /// \brief Ensure this fragment's FileMetaData is in memory.
   Status EnsureCompleteMetadata(parquet::arrow::FileReader* reader = NULLPTR);
 
+  /// \brief Return fragment which selects a filtered subset of this fragment's RowGroups.
+  Result<std::shared_ptr<Fragment>> Subset(const std::shared_ptr<Expression>& predicate);
+  Result<std::shared_ptr<Fragment>> Subset(std::vector<int> row_group_ids);
+
  private:
   ParquetFileFragment(FileSource source, std::shared_ptr<FileFormat> format,
                       std::shared_ptr<Expression> partition_expression,
                       std::shared_ptr<Schema> physical_schema,
-                      std::vector<RowGroupInfo> row_groups);
+                      util::optional<std::vector<int>> row_groups);
+
+  Status SetMetadata(std::shared_ptr<parquet::FileMetaData> metadata,
+                     std::shared_ptr<parquet::arrow::SchemaManifest> manifest);
 
   // Overridden to opportunistically set metadata since a reader must be opened anyway.
   Result<std::shared_ptr<Schema>> ReadPhysicalSchemaImpl() override {
@@ -221,12 +184,51 @@ class ARROW_DS_EXPORT ParquetFileFragment : public FileFragment {
     return physical_schema_;
   }
 
-  // Return a filtered subset of RowGroupInfos.
-  Result<std::vector<RowGroupInfo>> FilterRowGroups(const Expression& predicate);
+  // Return a filtered subset of row group indices.
+  Result<std::vector<int>> FilterRowGroups(const Expression& predicate);
 
-  std::vector<RowGroupInfo> row_groups_;
   ParquetFileFormat& parquet_format_;
-  bool has_complete_metadata_;
+
+  // Indices of row groups selected by this fragment,
+  // or util::nullopt if all row groups are selected.
+  util::optional<std::vector<int>> row_groups_;
+
+  ExpressionVector statistics_expressions_;
+  std::vector<bool> statistics_expressions_complete_;
+  std::shared_ptr<parquet::FileMetaData> metadata_;
+  std::shared_ptr<parquet::arrow::SchemaManifest> manifest_;
+
+  friend class ParquetFileFormat;
+  friend class ParquetDatasetFactory;
+};
+
+class ARROW_DS_EXPORT ParquetFileWriteOptions : public FileWriteOptions {
+ public:
+  std::shared_ptr<parquet::WriterProperties> writer_properties;
+
+  std::shared_ptr<parquet::ArrowWriterProperties> arrow_writer_properties;
+
+ protected:
+  using FileWriteOptions::FileWriteOptions;
+
+  friend class ParquetFileFormat;
+};
+
+class ARROW_DS_EXPORT ParquetFileWriter : public FileWriter {
+ public:
+  const std::shared_ptr<parquet::arrow::FileWriter>& parquet_writer() const {
+    return parquet_writer_;
+  }
+
+  Status Write(const std::shared_ptr<RecordBatch>& batch) override;
+
+  Status Finish() override;
+
+ private:
+  ParquetFileWriter(std::shared_ptr<parquet::arrow::FileWriter> writer,
+                    std::shared_ptr<ParquetFileWriteOptions> options);
+
+  std::shared_ptr<parquet::arrow::FileWriter> parquet_writer_;
 
   friend class ParquetFileFormat;
 };
@@ -257,6 +259,13 @@ struct ParquetFactoryOptions {
   // This is useful for partitioning which parses directory when ordering
   // is important, e.g. DirectoryPartitioning.
   std::string partition_base_dir;
+
+  // Assert that all ColumnChunk paths are consistent. The parquet spec allows for
+  // ColumnChunk data to be stored in multiple files, but ParquetDatasetFactory
+  // supports only a single file with all ColumnChunk data. If this flag is set
+  // construction of a ParquetDatasetFactory will raise an error if ColumnChunk
+  // data is not resident in a single file.
+  bool validate_column_chunk_paths = false;
 };
 
 /// \brief Create FileSystemDataset from custom `_metadata` cache file.
@@ -305,26 +314,35 @@ class ARROW_DS_EXPORT ParquetDatasetFactory : public DatasetFactory {
   Result<std::shared_ptr<Dataset>> Finish(FinishOptions options) override;
 
  protected:
-  ParquetDatasetFactory(std::shared_ptr<fs::FileSystem> fs,
-                        std::shared_ptr<ParquetFileFormat> format,
-                        std::shared_ptr<parquet::FileMetaData> metadata,
-                        std::string base_path, ParquetFactoryOptions options);
+  ParquetDatasetFactory(
+      std::shared_ptr<fs::FileSystem> filesystem,
+      std::shared_ptr<ParquetFileFormat> format,
+      std::shared_ptr<parquet::FileMetaData> metadata,
+      std::shared_ptr<parquet::arrow::SchemaManifest> manifest,
+      std::shared_ptr<Schema> physical_schema, std::string base_path,
+      ParquetFactoryOptions options,
+      std::unordered_map<std::string, std::vector<int>> path_to_row_group_ids)
+      : filesystem_(std::move(filesystem)),
+        format_(std::move(format)),
+        metadata_(std::move(metadata)),
+        manifest_(std::move(manifest)),
+        physical_schema_(std::move(physical_schema)),
+        base_path_(std::move(base_path)),
+        options_(std::move(options)),
+        path_to_row_group_ids_(std::move(path_to_row_group_ids)) {}
 
   std::shared_ptr<fs::FileSystem> filesystem_;
   std::shared_ptr<ParquetFileFormat> format_;
   std::shared_ptr<parquet::FileMetaData> metadata_;
+  std::shared_ptr<parquet::arrow::SchemaManifest> manifest_;
+  std::shared_ptr<Schema> physical_schema_;
   std::string base_path_;
   ParquetFactoryOptions options_;
-  FragmentVector fragments_;
+  std::unordered_map<std::string, std::vector<int>> path_to_row_group_ids_;
 
  private:
-  Result<std::vector<std::string>> CollectPaths(
-      const parquet::FileMetaData& metadata,
-      const parquet::ArrowReaderProperties& properties);
-
   Result<std::vector<std::shared_ptr<FileFragment>>> CollectParquetFragments(
-      const parquet::FileMetaData& metadata,
-      const parquet::ArrowReaderProperties& properties, const Partitioning& partitioning);
+      const Partitioning& partitioning);
 
   Result<std::shared_ptr<Schema>> PartitionSchema();
 };
diff --git a/cpp/src/arrow/dataset/file_parquet_test.cc b/cpp/src/arrow/dataset/file_parquet_test.cc
index 2a0b9197110..c2e0d6b632d 100644
--- a/cpp/src/arrow/dataset/file_parquet_test.cc
+++ b/cpp/src/arrow/dataset/file_parquet_test.cc
@@ -48,7 +48,6 @@ using parquet::default_writer_properties;
 using parquet::WriterProperties;
 
 using parquet::CreateOutputStream;
-using parquet::arrow::FileWriter;
 using parquet::arrow::WriteTable;
 
 using testing::Pointee;
@@ -57,7 +56,7 @@ using internal::checked_pointer_cast;
 
 class ArrowParquetWriterMixin : public ::testing::Test {
  public:
-  Status WriteRecordBatch(const RecordBatch& batch, FileWriter* writer) {
+  Status WriteRecordBatch(const RecordBatch& batch, parquet::arrow::FileWriter* writer) {
     auto schema = batch.schema();
     auto size = batch.num_rows();
 
@@ -75,7 +74,8 @@ class ArrowParquetWriterMixin : public ::testing::Test {
     return Status::OK();
   }
 
-  Status WriteRecordBatchReader(RecordBatchReader* reader, FileWriter* writer) {
+  Status WriteRecordBatchReader(RecordBatchReader* reader,
+                                parquet::arrow::FileWriter* writer) {
     auto schema = reader->schema();
 
     if (!schema->Equals(*writer->schema(), false)) {
@@ -96,9 +96,9 @@ class ArrowParquetWriterMixin : public ::testing::Test {
       const std::shared_ptr<WriterProperties>& properties = default_writer_properties(),
       const std::shared_ptr<ArrowWriterProperties>& arrow_properties =
           default_arrow_writer_properties()) {
-    std::unique_ptr<FileWriter> writer;
-    RETURN_NOT_OK(FileWriter::Open(*reader->schema(), pool, sink, properties,
-                                   arrow_properties, &writer));
+    std::unique_ptr<parquet::arrow::FileWriter> writer;
+    RETURN_NOT_OK(parquet::arrow::FileWriter::Open(
+        *reader->schema(), pool, sink, properties, arrow_properties, &writer));
     RETURN_NOT_OK(WriteRecordBatchReader(reader, writer.get()));
     return writer->Close();
   }
@@ -143,6 +143,12 @@ class TestParquetFileFormat : public ArrowParquetWriterMixin {
                                     kBatchRepetitions);
   }
 
+  Result<std::shared_ptr<io::BufferOutputStream>> GetFileSink() {
+    ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ResizableBuffer> buffer,
+                          AllocateResizableBuffer(0));
+    return std::make_shared<io::BufferOutputStream>(buffer);
+  }
+
   RecordBatchIterator Batches(ScanTaskIterator scan_task_it) {
     return MakeFlattenIterator(MakeMaybeMapIterator(
         [](std::shared_ptr<ScanTask> scan_task) { return scan_task->Execute(); },
@@ -166,7 +172,7 @@ class TestParquetFileFormat : public ArrowParquetWriterMixin {
     int64_t actual_batches = 0;
 
     for (auto maybe_batch : Batches(fragment)) {
-      ASSERT_OK_AND_ASSIGN(auto batch, std::move(maybe_batch));
+      ASSERT_OK_AND_ASSIGN(auto batch, maybe_batch);
       actual_rows += batch->num_rows();
       ++actual_batches;
     }
@@ -191,8 +197,7 @@ class TestParquetFileFormat : public ArrowParquetWriterMixin {
       auto expected = expected_row_groups[i];
       auto parquet_fragment = checked_pointer_cast<ParquetFileFragment>(fragments[i]);
 
-      EXPECT_EQ(parquet_fragment->row_groups(),
-                RowGroupInfo::FromIdentifiers({expected}));
+      EXPECT_EQ(parquet_fragment->row_groups(), std::vector<int>{expected});
       EXPECT_EQ(SingleBatch(parquet_fragment.get())->num_rows(), expected + 1);
     }
   }
@@ -214,7 +219,7 @@ TEST_F(TestParquetFileFormat, ScanRecordBatchReader) {
   int64_t row_count = 0;
 
   for (auto maybe_batch : Batches(fragment.get())) {
-    ASSERT_OK_AND_ASSIGN(auto batch, std::move(maybe_batch));
+    ASSERT_OK_AND_ASSIGN(auto batch, maybe_batch);
     row_count += batch->num_rows();
   }
 
@@ -238,10 +243,10 @@ TEST_F(TestParquetFileFormat, ScanRecordBatchReaderDictEncoded) {
   Schema expected_schema({field("utf8", dictionary(int32(), utf8()))});
 
   for (auto maybe_task : scan_task_it) {
-    ASSERT_OK_AND_ASSIGN(auto task, std::move(maybe_task));
+    ASSERT_OK_AND_ASSIGN(auto task, maybe_task);
     ASSERT_OK_AND_ASSIGN(auto rb_it, task->Execute());
     for (auto maybe_batch : rb_it) {
-      ASSERT_OK_AND_ASSIGN(auto batch, std::move(maybe_batch));
+      ASSERT_OK_AND_ASSIGN(auto batch, maybe_batch);
       row_count += batch->num_rows();
       AssertSchemaEqual(*batch->schema(), expected_schema, /* check_metadata = */ false);
     }
@@ -283,7 +288,7 @@ TEST_F(TestParquetFileFormat, ScanRecordBatchReaderProjected) {
   int64_t row_count = 0;
 
   for (auto maybe_batch : Batches(fragment.get())) {
-    ASSERT_OK_AND_ASSIGN(auto batch, std::move(maybe_batch));
+    ASSERT_OK_AND_ASSIGN(auto batch, maybe_batch);
     row_count += batch->num_rows();
     AssertSchemaEqual(*batch->schema(), *expected_schema,
                       /*check_metadata=*/false);
@@ -330,7 +335,7 @@ TEST_F(TestParquetFileFormat, ScanRecordBatchReaderProjectedMissingCols) {
     int64_t row_count = 0;
 
     for (auto maybe_batch : Batches(fragment.get())) {
-      ASSERT_OK_AND_ASSIGN(auto batch, std::move(maybe_batch));
+      ASSERT_OK_AND_ASSIGN(auto batch, maybe_batch);
       row_count += batch->num_rows();
       AssertSchemaEqual(*batch->schema(), *expected_schema,
                         /*check_metadata=*/false);
@@ -474,21 +479,20 @@ TEST_F(TestParquetFileFormat, PredicatePushdownRowGroupFragments) {
 }
 
 TEST_F(TestParquetFileFormat, PredicatePushdownRowGroupFragmentsUsingStringColumn) {
-  auto table =
-      TableFromJSON(schema({field("x", utf8())}), {
-                                                      R"([{"x": "a"}, {"x": "a"}])",
-                                                      R"([{"x": "b"}, {"x": "b"}])",
-                                                      R"([{"x": "c"}, {"x": "c"}])",
-                                                      R"([{"x": "a"}, {"x": "b"}])",
-                                                  });
+  auto table = TableFromJSON(schema({field("x", utf8())}),
+                             {
+                                 R"([{"x": "a"}])",
+                                 R"([{"x": "b"}, {"x": "b"}])",
+                                 R"([{"x": "c"}, {"x": "c"}, {"x": "c"}])",
+                                 R"([{"x": "a"}, {"x": "b"}, {"x": "c"}, {"x": "d"}])",
+                             });
   TableBatchReader reader(*table);
   auto source = GetFileSource(&reader);
 
   opts_ = ScanOptions::Make(reader.schema());
   ASSERT_OK_AND_ASSIGN(auto fragment, format_->MakeFragment(*source));
 
-  // TODO(bkietz): support strings in StatisticsAsScalars
-  // CountRowGroupsInFragment(fragment, {0, 3}, "x"_ == "a");
+  CountRowGroupsInFragment(fragment, {0, 3}, "x"_ == "a");
 }
 
 TEST_F(TestParquetFileFormat, ExplicitRowGroupSelection) {
@@ -501,22 +505,29 @@ TEST_F(TestParquetFileFormat, ExplicitRowGroupSelection) {
   opts_ = ScanOptions::Make(reader->schema());
 
   auto row_groups_fragment = [&](std::vector<int> row_groups) {
-    std::shared_ptr<Schema> physical_schema = nullptr;
     EXPECT_OK_AND_ASSIGN(auto fragment,
                          format_->MakeFragment(*source, scalar(true),
-                                               RowGroupInfo::FromIdentifiers(row_groups),
-                                               physical_schema));
-    return internal::checked_pointer_cast<ParquetFileFragment>(fragment);
+                                               /*physical_schema=*/nullptr, row_groups));
+    return fragment;
   };
 
-  // empty selection is identical to selecting all row groups
-  EXPECT_TRUE(row_groups_fragment({})->row_groups().empty());
-  CountRowsAndBatchesInScan(row_groups_fragment({}), kTotalNumRows, kNumRowGroups);
+  // select all row groups
+  EXPECT_OK_AND_ASSIGN(
+      auto all_row_groups_fragment,
+      format_->MakeFragment(*source, scalar(true))
+          .Map([](std::shared_ptr<FileFragment> f) {
+            return internal::checked_pointer_cast<ParquetFileFragment>(f);
+          }));
+
+  EXPECT_EQ(all_row_groups_fragment->row_groups(), std::vector<int>{});
+
+  ARROW_EXPECT_OK(all_row_groups_fragment->EnsureCompleteMetadata());
+  CountRowsAndBatchesInScan(all_row_groups_fragment, kTotalNumRows, kNumRowGroups);
 
   // individual selection selects a single row group
   for (int i = 0; i < kNumRowGroups; ++i) {
     CountRowsAndBatchesInScan(row_groups_fragment({i}), i + 1, 1);
-    EXPECT_EQ(row_groups_fragment({i})->row_groups(), RowGroupInfo::FromIdentifiers({i}));
+    EXPECT_EQ(row_groups_fragment({i})->row_groups(), std::vector<int>{i});
   }
 
   for (int i = 0; i < kNumRowGroups; ++i) {
@@ -540,5 +551,86 @@ TEST_F(TestParquetFileFormat, ExplicitRowGroupSelection) {
       row_groups_fragment({kNumRowGroups + 1})->Scan(opts_, ctx_));
 }
 
+TEST_F(TestParquetFileFormat, WriteRecordBatchReader) {
+  std::shared_ptr<RecordBatchReader> reader = GetRecordBatchReader();
+  auto source = GetFileSource(reader.get());
+  reader = GetRecordBatchReader();
+
+  opts_ = ScanOptions::Make(reader->schema());
+
+  EXPECT_OK_AND_ASSIGN(auto sink, GetFileSink());
+
+  auto options = format_->DefaultWriteOptions();
+  EXPECT_OK_AND_ASSIGN(auto writer, format_->MakeWriter(sink, reader->schema(), options));
+  ASSERT_OK(writer->Write(reader.get()));
+  ASSERT_OK(writer->Finish());
+
+  EXPECT_OK_AND_ASSIGN(auto written, sink->Finish());
+
+  AssertBufferEqual(*written, *source->buffer());
+}
+
+TEST_F(TestParquetFileFormat, WriteRecordBatchReaderCustomOptions) {
+  TimeUnit::type coerce_timestamps_to = TimeUnit::MICRO,
+                 coerce_timestamps_from = TimeUnit::NANO;
+
+  std::shared_ptr<RecordBatchReader> reader =
+      GetRecordBatchReader(schema({field("ts", timestamp(coerce_timestamps_from))}));
+
+  opts_ = ScanOptions::Make(reader->schema());
+
+  EXPECT_OK_AND_ASSIGN(auto sink, GetFileSink());
+
+  auto options =
+      checked_pointer_cast<ParquetFileWriteOptions>(format_->DefaultWriteOptions());
+  options->writer_properties = parquet::WriterProperties::Builder()
+                                   .created_by("TestParquetFileFormat")
+                                   ->disable_statistics()
+                                   ->build();
+
+  options->arrow_writer_properties = parquet::ArrowWriterProperties::Builder()
+                                         .coerce_timestamps(coerce_timestamps_to)
+                                         ->build();
+
+  EXPECT_OK_AND_ASSIGN(auto writer, format_->MakeWriter(sink, reader->schema(), options));
+  ASSERT_OK(writer->Write(reader.get()));
+  ASSERT_OK(writer->Finish());
+
+  EXPECT_OK_AND_ASSIGN(auto written, sink->Finish());
+  EXPECT_OK_AND_ASSIGN(auto fragment, format_->MakeFragment(FileSource{written}));
+
+  EXPECT_OK_AND_ASSIGN(auto actual_schema, fragment->ReadPhysicalSchema());
+  AssertSchemaEqual(Schema({field("ts", timestamp(coerce_timestamps_to))}),
+                    *actual_schema);
+}
+
+class TestParquetFileSystemDataset : public WriteFileSystemDatasetMixin,
+                                     public testing::Test {
+ public:
+  void SetUp() override {
+    MakeSourceDataset();
+    check_metadata_ = false;
+    auto parquet_format = std::make_shared<ParquetFileFormat>();
+    format_ = parquet_format;
+    SetWriteOptions(parquet_format->DefaultWriteOptions());
+  }
+};
+
+TEST_F(TestParquetFileSystemDataset, WriteWithIdenticalPartitioningSchema) {
+  TestWriteWithIdenticalPartitioningSchema();
+}
+
+TEST_F(TestParquetFileSystemDataset, WriteWithUnrelatedPartitioningSchema) {
+  TestWriteWithUnrelatedPartitioningSchema();
+}
+
+TEST_F(TestParquetFileSystemDataset, WriteWithSupersetPartitioningSchema) {
+  TestWriteWithSupersetPartitioningSchema();
+}
+
+TEST_F(TestParquetFileSystemDataset, WriteWithEmptyPartitioningSchema) {
+  TestWriteWithEmptyPartitioningSchema();
+}
+
 }  // namespace dataset
 }  // namespace arrow
diff --git a/cpp/src/arrow/dataset/file_test.cc b/cpp/src/arrow/dataset/file_test.cc
index a2d3adddb6c..8ee2613576f 100644
--- a/cpp/src/arrow/dataset/file_test.cc
+++ b/cpp/src/arrow/dataset/file_test.cc
@@ -98,7 +98,7 @@ TEST_F(TestFileSystemDataset, ReplaceSchema) {
   auto schm = schema({field("i32", int32()), field("f64", float64())});
   auto format = std::make_shared<DummyFileFormat>(schm);
   ASSERT_OK_AND_ASSIGN(auto dataset,
-                       FileSystemDataset::Make(schm, scalar(true), format, {}));
+                       FileSystemDataset::Make(schm, scalar(true), format, nullptr, {}));
 
   // drop field
   ASSERT_OK(dataset->ReplaceSchema(schema({field("i32", int32())})).status());
diff --git a/cpp/src/arrow/dataset/filter.cc b/cpp/src/arrow/dataset/filter.cc
index 25a3a0a0996..bcfc8d745dc 100644
--- a/cpp/src/arrow/dataset/filter.cc
+++ b/cpp/src/arrow/dataset/filter.cc
@@ -22,13 +22,11 @@
 #include <memory>
 #include <numeric>
 #include <string>
-#include <unordered_map>
 #include <utility>
 #include <vector>
 
+#include "arrow/array/builder_primitive.h"
 #include "arrow/buffer.h"
-#include "arrow/buffer_builder.h"
-#include "arrow/builder.h"
 #include "arrow/compute/api.h"
 #include "arrow/dataset/dataset.h"
 #include "arrow/io/memory.h"
@@ -80,6 +78,14 @@ struct Comparison {
   };
 };
 
+Result<std::shared_ptr<Scalar>> EnsureNotDictionary(
+    const std::shared_ptr<Scalar>& scalar) {
+  if (scalar->type->id() == Type::DICTIONARY) {
+    return checked_cast<const DictionaryScalar&>(*scalar).GetEncodedValue();
+  }
+  return scalar;
+}
+
 Result<Comparison::type> Compare(const Scalar& lhs, const Scalar& rhs);
 
 struct CompareVisitor {
@@ -136,6 +142,7 @@ struct CompareVisitor {
   }
 
   Status Visit(const Decimal128Type&) { return CompareValues<Decimal128Type>(); }
+  Status Visit(const Decimal256Type&) { return CompareValues<Decimal256Type>(); }
 
   // Explicit because it falls under `physical_unsigned_integer`.
   // TODO(bkietz) whenever we vendor a float16, this can be implemented
@@ -148,11 +155,7 @@ struct CompareVisitor {
   }
 
   Status Visit(const DictionaryType&) {
-    ARROW_ASSIGN_OR_RAISE(auto lhs,
-                          checked_cast<const DictionaryScalar&>(lhs_).GetEncodedValue());
-    ARROW_ASSIGN_OR_RAISE(auto rhs,
-                          checked_cast<const DictionaryScalar&>(rhs_).GetEncodedValue());
-    return Compare(*lhs, *rhs).Value(&result_);
+    return Status::NotImplemented("comparison of scalars of type ", *lhs_.type);
   }
 
   // defer comparison to ScalarType<T>::value
@@ -357,9 +360,17 @@ std::shared_ptr<Expression> ComparisonExpression::AssumeGivenComparison(
     }
   }
 
-  const auto& this_rhs = checked_cast<const ScalarExpression&>(*right_operand_).value();
-  const auto& given_rhs =
-      checked_cast<const ScalarExpression&>(*given.right_operand_).value();
+  auto this_rhs =
+      EnsureNotDictionary(checked_cast<const ScalarExpression&>(*right_operand_).value())
+          .ValueOr(nullptr);
+  auto given_rhs =
+      EnsureNotDictionary(
+          checked_cast<const ScalarExpression&>(*given.right_operand_).value())
+          .ValueOr(nullptr);
+
+  if (!this_rhs || !given_rhs) {
+    return Copy();
+  }
 
   auto cmp = Compare(*this_rhs, *given_rhs).ValueOrDie();
 
@@ -805,6 +816,7 @@ std::shared_ptr<Expression> and_(std::shared_ptr<Expression> lhs,
 std::shared_ptr<Expression> and_(const ExpressionVector& subexpressions) {
   auto acc = scalar(true);
   for (const auto& next : subexpressions) {
+    if (next->Equals(false)) return next;
     acc = acc->Equals(true) ? next : and_(std::move(acc), next);
   }
   return acc;
@@ -818,6 +830,7 @@ std::shared_ptr<Expression> or_(std::shared_ptr<Expression> lhs,
 std::shared_ptr<Expression> or_(const ExpressionVector& subexpressions) {
   auto acc = scalar(false);
   for (const auto& next : subexpressions) {
+    if (next->Equals(true)) return next;
     acc = acc->Equals(false) ? next : or_(std::move(acc), next);
   }
   return acc;
@@ -1179,8 +1192,8 @@ struct TreeEvaluator::Impl {
                                 Result<Datum> kernel(const Datum& left,
                                                      const Datum& right,
                                                      ExecContext* ctx)) const {
-    ARROW_ASSIGN_OR_RAISE(auto lhs, Evaluate(*expr.left_operand()));
-    ARROW_ASSIGN_OR_RAISE(auto rhs, Evaluate(*expr.right_operand()));
+    ARROW_ASSIGN_OR_RAISE(Datum lhs, Evaluate(*expr.left_operand()));
+    ARROW_ASSIGN_OR_RAISE(Datum rhs, Evaluate(*expr.right_operand()));
 
     if (lhs.is_scalar()) {
       ARROW_ASSIGN_OR_RAISE(
@@ -1200,7 +1213,7 @@ struct TreeEvaluator::Impl {
   }
 
   Result<Datum> operator()(const NotExpression& expr) const {
-    ARROW_ASSIGN_OR_RAISE(auto to_invert, Evaluate(*expr.operand()));
+    ARROW_ASSIGN_OR_RAISE(Datum to_invert, Evaluate(*expr.operand()));
     if (IsNullDatum(to_invert)) {
       return NullDatum();
     }
@@ -1214,7 +1227,7 @@ struct TreeEvaluator::Impl {
   }
 
   Result<Datum> operator()(const InExpression& expr) const {
-    ARROW_ASSIGN_OR_RAISE(auto operand_values, Evaluate(*expr.operand()));
+    ARROW_ASSIGN_OR_RAISE(Datum operand_values, Evaluate(*expr.operand()));
     if (IsNullDatum(operand_values)) {
       return Datum(expr.set()->null_count() != 0);
     }
@@ -1224,7 +1237,7 @@ struct TreeEvaluator::Impl {
   }
 
   Result<Datum> operator()(const IsValidExpression& expr) const {
-    ARROW_ASSIGN_OR_RAISE(auto operand_values, Evaluate(*expr.operand()));
+    ARROW_ASSIGN_OR_RAISE(Datum operand_values, Evaluate(*expr.operand()));
     if (IsNullDatum(operand_values)) {
       return Datum(false);
     }
@@ -1255,14 +1268,42 @@ struct TreeEvaluator::Impl {
   }
 
   Result<Datum> operator()(const ComparisonExpression& expr) const {
-    ARROW_ASSIGN_OR_RAISE(auto lhs, Evaluate(*expr.left_operand()));
-    ARROW_ASSIGN_OR_RAISE(auto rhs, Evaluate(*expr.right_operand()));
+    ARROW_ASSIGN_OR_RAISE(Datum lhs, Evaluate(*expr.left_operand()));
+    ARROW_ASSIGN_OR_RAISE(Datum rhs, Evaluate(*expr.right_operand()));
 
     if (IsNullDatum(lhs) || IsNullDatum(rhs)) {
       return Datum(std::make_shared<BooleanScalar>());
     }
 
-    DCHECK(lhs.is_array());
+    if (lhs.type()->id() == Type::DICTIONARY && rhs.type()->id() == Type::DICTIONARY) {
+      if (lhs.is_array() && rhs.is_array()) {
+        // decode dictionary arrays
+        for (Datum* arg : {&lhs, &rhs}) {
+          auto dict = checked_pointer_cast<DictionaryArray>(arg->make_array());
+          ARROW_ASSIGN_OR_RAISE(*arg, compute::Take(dict->dictionary(), dict->indices(),
+                                                    compute::TakeOptions::Defaults()));
+        }
+      } else if (lhs.is_array() || rhs.is_array()) {
+        auto dict = checked_pointer_cast<DictionaryArray>(
+            (lhs.is_array() ? lhs : rhs).make_array());
+
+        ARROW_ASSIGN_OR_RAISE(auto scalar, checked_cast<const DictionaryScalar&>(
+                                               *(lhs.is_scalar() ? lhs : rhs).scalar())
+                                               .GetEncodedValue());
+        if (lhs.is_array()) {
+          lhs = dict->dictionary();
+          rhs = std::move(scalar);
+        } else {
+          lhs = std::move(scalar);
+          rhs = dict->dictionary();
+        }
+        ARROW_ASSIGN_OR_RAISE(
+            Datum out_dict,
+            compute::Compare(lhs, rhs, compute::CompareOptions(expr.op()), &ctx_));
+
+        return compute::Take(out_dict, dict->indices(), compute::TakeOptions::Defaults());
+      }
+    }
 
     return compute::Compare(lhs, rhs, compute::CompareOptions(expr.op()), &ctx_);
   }
@@ -1309,7 +1350,11 @@ Result<std::shared_ptr<RecordBatch>> TreeEvaluator::Filter(
   return batch->Slice(0, 0);
 }
 
-std::shared_ptr<Expression> scalar(bool value) { return scalar(MakeScalar(value)); }
+const std::shared_ptr<Expression>& scalar(bool value) {
+  static auto true_ = scalar(MakeScalar(true));
+  static auto false_ = scalar(MakeScalar(false));
+  return value ? true_ : false_;
+}
 
 // Serialization is accomplished by converting expressions to single element StructArrays
 // then writing that to an IPC file. The last field is always an int32 column containing
@@ -1408,7 +1453,7 @@ struct SerializeImpl {
     ARROW_ASSIGN_OR_RAISE(auto array, SerializeImpl{}.ToArray(expr));
     ARROW_ASSIGN_OR_RAISE(auto batch, RecordBatch::FromStructArray(array));
     ARROW_ASSIGN_OR_RAISE(auto stream, io::BufferOutputStream::Create());
-    ARROW_ASSIGN_OR_RAISE(auto writer, ipc::NewFileWriter(stream.get(), batch->schema()));
+    ARROW_ASSIGN_OR_RAISE(auto writer, ipc::MakeFileWriter(stream, batch->schema()));
     RETURN_NOT_OK(writer->WriteRecordBatch(*batch));
     RETURN_NOT_OK(writer->Close());
     return stream->Finish();
@@ -1431,7 +1476,7 @@ struct DeserializeImpl {
     switch (expression_type) {
       case ExpressionType::FIELD: {
         ARROW_ASSIGN_OR_RAISE(auto name, GetView<StringType>(struct_array, 0));
-        return field_ref(name.to_string());
+        return field_ref(std::string(name));
       }
 
       case ExpressionType::SCALAR: {
@@ -1617,15 +1662,17 @@ class StructDictionary {
   }
 
  private:
-  Status AddOne(const std::shared_ptr<Array>& column,
-                std::shared_ptr<Int32Array>* fused_indices) {
-    ARROW_ASSIGN_OR_RAISE(Datum encoded, compute::DictionaryEncode(column));
-    ArrayData* encoded_array = encoded.mutable_array();
+  Status AddOne(Datum column, std::shared_ptr<Int32Array>* fused_indices) {
+    ArrayData* encoded;
+    if (column.type()->id() != Type::DICTIONARY) {
+      ARROW_ASSIGN_OR_RAISE(column, compute::DictionaryEncode(column));
+    }
+    encoded = column.mutable_array();
 
-    auto indices = std::make_shared<Int32Array>(encoded_array->length,
-                                                std::move(encoded_array->buffers[1]));
+    auto indices =
+        std::make_shared<Int32Array>(encoded->length, std::move(encoded->buffers[1]));
 
-    dictionaries_.push_back(MakeArray(std::move(encoded_array->dictionary)));
+    dictionaries_.push_back(MakeArray(std::move(encoded->dictionary)));
     auto dictionary_size = static_cast<int32_t>(dictionaries_.back()->length());
 
     if (*fused_indices == nullptr) {
diff --git a/cpp/src/arrow/dataset/partition.cc b/cpp/src/arrow/dataset/partition.cc
index d43cf7ad57e..159e0ac0331 100644
--- a/cpp/src/arrow/dataset/partition.cc
+++ b/cpp/src/arrow/dataset/partition.cc
@@ -28,6 +28,7 @@
 #include "arrow/array/array_base.h"
 #include "arrow/array/array_nested.h"
 #include "arrow/array/builder_binary.h"
+#include "arrow/array/builder_dict.h"
 #include "arrow/compute/api_scalar.h"
 #include "arrow/dataset/dataset_internal.h"
 #include "arrow/dataset/file_base.h"
@@ -39,6 +40,7 @@
 #include "arrow/scalar.h"
 #include "arrow/util/iterator.h"
 #include "arrow/util/logging.h"
+#include "arrow/util/make_unique.h"
 #include "arrow/util/range.h"
 #include "arrow/util/sort.h"
 #include "arrow/util/string_view.h"
@@ -67,9 +69,9 @@ std::shared_ptr<Partitioning> Partitioning::Default() {
                                     " Partitioning");
     }
 
-    Result<std::vector<PartitionedBatch>> Partition(
+    Result<PartitionedBatches> Partition(
         const std::shared_ptr<RecordBatch>& batch) const override {
-      return std::vector<PartitionedBatch>{{batch, scalar(true)}};
+      return PartitionedBatches{{batch}, {scalar(true)}};
     }
   };
 
@@ -137,7 +139,7 @@ inline std::shared_ptr<Expression> ConjunctionFromGroupingRow(Scalar* row) {
   return and_(std::move(equality_expressions));
 }
 
-Result<std::vector<Partitioning::PartitionedBatch>> KeyValuePartitioning::Partition(
+Result<Partitioning::PartitionedBatches> KeyValuePartitioning::Partition(
     const std::shared_ptr<RecordBatch>& batch) const {
   FieldVector by_fields;
   ArrayVector by_columns;
@@ -156,7 +158,7 @@ Result<std::vector<Partitioning::PartitionedBatch>> KeyValuePartitioning::Partit
 
   if (by_fields.empty()) {
     // no fields to group by; return the whole batch
-    return std::vector<PartitionedBatch>{{batch, scalar(true)}};
+    return PartitionedBatches{{batch}, {scalar(true)}};
   }
 
   ARROW_ASSIGN_OR_RAISE(auto by,
@@ -166,13 +168,13 @@ Result<std::vector<Partitioning::PartitionedBatch>> KeyValuePartitioning::Partit
       checked_pointer_cast<ListArray>(groupings_and_values->GetFieldByName("groupings"));
   auto unique_rows = groupings_and_values->GetFieldByName("values");
 
-  ARROW_ASSIGN_OR_RAISE(auto grouped_batches, ApplyGroupings(*groupings, rest));
+  PartitionedBatches out;
+  ARROW_ASSIGN_OR_RAISE(out.batches, ApplyGroupings(*groupings, rest));
+  out.expressions.resize(out.batches.size());
 
-  std::vector<PartitionedBatch> out(grouped_batches.size());
-  for (size_t i = 0; i < out.size(); ++i) {
+  for (size_t i = 0; i < out.batches.size(); ++i) {
     ARROW_ASSIGN_OR_RAISE(auto row, unique_rows->GetScalar(i));
-    out[i].partition_expression = ConjunctionFromGroupingRow(row.get());
-    out[i].batch = std::move(grouped_batches[i]);
+    out.expressions[i] = ConjunctionFromGroupingRow(row.get());
   }
   return out;
 }
@@ -306,75 +308,61 @@ Result<std::string> DirectoryPartitioning::FormatValues(
   return fs::internal::JoinAbstractPath(std::move(segments));
 }
 
-class KeyValuePartitioningInspectImpl {
- public:
-  explicit KeyValuePartitioningInspectImpl(const PartitioningFactoryOptions& options)
+class KeyValuePartitioningFactory : public PartitioningFactory {
+ protected:
+  explicit KeyValuePartitioningFactory(PartitioningFactoryOptions options)
       : options_(options) {}
 
-  static Result<std::shared_ptr<DataType>> InferType(const std::string& name,
-                                                     const std::set<std::string>& reprs,
-                                                     int max_partition_dictionary_size) {
-    if (reprs.empty()) {
-      return Status::Invalid("No segments were available for field '", name,
-                             "'; couldn't infer type");
-    }
-
-    bool all_integral = std::all_of(reprs.begin(), reprs.end(), [](string_view repr) {
-      // TODO(bkietz) use ParseUnsigned or so
-      return repr.find_first_not_of("0123456789") == string_view::npos;
-    });
-
-    if (all_integral) {
-      return int32();
-    }
+  int GetOrInsertField(const std::string& name) {
+    auto it_inserted =
+        name_to_index_.emplace(name, static_cast<int>(name_to_index_.size()));
 
-    if (reprs.size() > static_cast<size_t>(max_partition_dictionary_size)) {
-      return utf8();
+    if (it_inserted.second) {
+      repr_memos_.push_back(MakeMemo());
     }
 
-    return dictionary(int32(), utf8());
+    return it_inserted.first->second;
   }
 
-  int GetOrInsertField(const std::string& name) {
-    auto name_index =
-        name_to_index_.emplace(name, static_cast<int>(name_to_index_.size())).first;
-
-    if (static_cast<size_t>(name_index->second) >= values_.size()) {
-      values_.resize(name_index->second + 1);
-    }
-    return name_index->second;
+  Status InsertRepr(const std::string& name, util::string_view repr) {
+    return InsertRepr(GetOrInsertField(name), repr);
   }
 
-  void InsertRepr(const std::string& name, std::string repr) {
-    InsertRepr(GetOrInsertField(name), std::move(repr));
+  Status InsertRepr(int index, util::string_view repr) {
+    int dummy;
+    return repr_memos_[index]->GetOrInsert<StringType>(repr, &dummy);
   }
 
-  void InsertRepr(int index, std::string repr) { values_[index].insert(std::move(repr)); }
-
-  Result<std::shared_ptr<Schema>> Finish(ArrayVector* dictionaries) {
-    dictionaries->clear();
-
-    if (options_.max_partition_dictionary_size != 0) {
-      dictionaries->resize(name_to_index_.size());
-    }
+  Result<std::shared_ptr<Schema>> DoInpsect() {
+    dictionaries_.assign(name_to_index_.size(), nullptr);
 
     std::vector<std::shared_ptr<Field>> fields(name_to_index_.size());
 
     for (const auto& name_index : name_to_index_) {
       const auto& name = name_index.first;
       auto index = name_index.second;
-      ARROW_ASSIGN_OR_RAISE(auto type, InferType(name, values_[index],
-                                                 options_.max_partition_dictionary_size));
-      if (type->id() == Type::DICTIONARY) {
-        StringBuilder builder;
-        for (const auto& repr : values_[index]) {
-          RETURN_NOT_OK(builder.Append(repr));
-        }
-        RETURN_NOT_OK(builder.Finish(&dictionaries->at(index)));
+
+      std::shared_ptr<ArrayData> reprs;
+      RETURN_NOT_OK(repr_memos_[index]->GetArrayData(0, &reprs));
+
+      if (reprs->length == 0) {
+        return Status::Invalid("No segments were available for field '", name,
+                               "'; couldn't infer type");
+      }
+
+      // try casting to int32, otherwise bail and just use the string reprs
+      auto dict = compute::Cast(reprs, int32()).ValueOr(reprs).make_array();
+      auto type = dict->type();
+      if (options_.infer_dictionary) {
+        // wrap the inferred type in dictionary()
+        type = dictionary(int32(), std::move(type));
       }
+
       fields[index] = field(name, std::move(type));
+      dictionaries_[index] = std::move(dict);
     }
 
+    Reset();
     return ::arrow::schema(std::move(fields));
   }
 
@@ -387,38 +375,44 @@ class KeyValuePartitioningInspectImpl {
     return names;
   }
 
- private:
+  virtual void Reset() {
+    name_to_index_.clear();
+    repr_memos_.clear();
+  }
+
+  std::unique_ptr<internal::DictionaryMemoTable> MakeMemo() {
+    return internal::make_unique<internal::DictionaryMemoTable>(default_memory_pool(),
+                                                                utf8());
+  }
+
+  PartitioningFactoryOptions options_;
+  ArrayVector dictionaries_;
   std::unordered_map<std::string, int> name_to_index_;
-  std::vector<std::set<std::string>> values_;
-  const PartitioningFactoryOptions& options_;
+  std::vector<std::unique_ptr<internal::DictionaryMemoTable>> repr_memos_;
 };
 
-class DirectoryPartitioningFactory : public PartitioningFactory {
+class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
  public:
   DirectoryPartitioningFactory(std::vector<std::string> field_names,
                                PartitioningFactoryOptions options)
-      : field_names_(std::move(field_names)), options_(options) {}
+      : KeyValuePartitioningFactory(options), field_names_(std::move(field_names)) {
+    Reset();
+  }
 
   std::string type_name() const override { return "schema"; }
 
   Result<std::shared_ptr<Schema>> Inspect(
       const std::vector<std::string>& paths) override {
-    KeyValuePartitioningInspectImpl impl(options_);
-
-    for (const auto& name : field_names_) {
-      impl.GetOrInsertField(name);
-    }
-
     for (auto path : paths) {
       size_t field_index = 0;
       for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
         if (field_index == field_names_.size()) break;
 
-        impl.InsertRepr(static_cast<int>(field_index++), std::move(segment));
+        RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), segment));
       }
     }
 
-    return impl.Finish(&dictionaries_);
+    return DoInpsect();
   }
 
   Result<std::shared_ptr<Partitioning>> Finish(
@@ -435,9 +429,15 @@ class DirectoryPartitioningFactory : public PartitioningFactory {
   }
 
  private:
+  void Reset() override {
+    KeyValuePartitioningFactory::Reset();
+
+    for (const auto& name : field_names_) {
+      GetOrInsertField(name);
+    }
+  }
+
   std::vector<std::string> field_names_;
-  ArrayVector dictionaries_;
-  PartitioningFactoryOptions options_;
 };
 
 std::shared_ptr<PartitioningFactory> DirectoryPartitioning::MakeFactory(
@@ -490,27 +490,25 @@ Result<std::string> HivePartitioning::FormatValues(
   return fs::internal::JoinAbstractPath(std::move(segments));
 }
 
-class HivePartitioningFactory : public PartitioningFactory {
+class HivePartitioningFactory : public KeyValuePartitioningFactory {
  public:
   explicit HivePartitioningFactory(PartitioningFactoryOptions options)
-      : options_(options) {}
+      : KeyValuePartitioningFactory(options) {}
 
   std::string type_name() const override { return "hive"; }
 
   Result<std::shared_ptr<Schema>> Inspect(
       const std::vector<std::string>& paths) override {
-    KeyValuePartitioningInspectImpl impl(options_);
-
     for (auto path : paths) {
       for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
         if (auto key = HivePartitioning::ParseKey(segment)) {
-          impl.InsertRepr(key->name, key->value);
+          RETURN_NOT_OK(InsertRepr(key->name, key->value));
         }
       }
     }
 
-    field_names_ = impl.FieldNames();
-    return impl.Finish(&dictionaries_);
+    field_names_ = FieldNames();
+    return DoInpsect();
   }
 
   Result<std::shared_ptr<Partitioning>> Finish(
@@ -520,7 +518,7 @@ class HivePartitioningFactory : public PartitioningFactory {
     } else {
       for (FieldRef ref : field_names_) {
         // ensure all of field_names_ are present in schema
-        RETURN_NOT_OK(ref.FindOne(*schema).status());
+        RETURN_NOT_OK(ref.FindOne(*schema));
       }
 
       // drop fields which aren't in field_names_
@@ -532,8 +530,6 @@ class HivePartitioningFactory : public PartitioningFactory {
 
  private:
   std::vector<std::string> field_names_;
-  ArrayVector dictionaries_;
-  PartitioningFactoryOptions options_;
 };
 
 std::shared_ptr<PartitioningFactory> HivePartitioning::MakeFactory(
@@ -543,7 +539,7 @@ std::shared_ptr<PartitioningFactory> HivePartitioning::MakeFactory(
 
 std::string StripPrefixAndFilename(const std::string& path, const std::string& prefix) {
   auto maybe_base_less = fs::internal::RemoveAncestor(prefix, path);
-  auto base_less = maybe_base_less ? maybe_base_less->to_string() : path;
+  auto base_less = maybe_base_less ? std::string(*maybe_base_less) : path;
   auto basename_filename = fs::internal::GetAbstractPathParent(base_less);
   return basename_filename.first;
 }
diff --git a/cpp/src/arrow/dataset/partition.h b/cpp/src/arrow/dataset/partition.h
index 021f82245e2..165fcfb5248 100644
--- a/cpp/src/arrow/dataset/partition.h
+++ b/cpp/src/arrow/dataset/partition.h
@@ -60,12 +60,12 @@ class ARROW_DS_EXPORT Partitioning {
   virtual std::string type_name() const = 0;
 
   /// \brief If the input batch shares any fields with this partitioning,
-  /// produce slices of the batch which satisfy mutually exclusive Expressions.
-  struct PartitionedBatch {
-    std::shared_ptr<RecordBatch> batch;
-    std::shared_ptr<Expression> partition_expression;
+  /// produce sub-batches which satisfy mutually exclusive Expressions.
+  struct PartitionedBatches {
+    RecordBatchVector batches;
+    ExpressionVector expressions;
   };
-  virtual Result<std::vector<PartitionedBatch>> Partition(
+  virtual Result<PartitionedBatches> Partition(
       const std::shared_ptr<RecordBatch>& batch) const = 0;
 
   /// \brief Parse a path into a partition expression
@@ -85,14 +85,11 @@ class ARROW_DS_EXPORT Partitioning {
 };
 
 struct PartitioningFactoryOptions {
-  /// When inferring a schema for partition fields, string fields may be inferred as
-  /// a dictionary type instead. This can be more efficient when materializing virtual
-  /// columns. If the number of discovered unique values of a string field exceeds
-  /// max_partition_dictionary_size, it will instead be inferred as a string.
-  ///
-  /// max_partition_dictionary_size = 0: No fields will be inferred as dictionary.
-  /// max_partition_dictionary_size = -1: All fields will be inferred as dictionary.
-  int max_partition_dictionary_size = 0;
+  /// When inferring a schema for partition fields, yield dictionary encoded types
+  /// instead of plain. This can be more efficient when materializing virtual
+  /// columns, and Expressions parsed by the finished Partitioning will include
+  /// dictionaries of all unique inspected values for each field.
+  bool infer_dictionary = false;
 };
 
 /// \brief PartitioningFactory provides creation of a partitioning  when the
@@ -136,7 +133,7 @@ class ARROW_DS_EXPORT KeyValuePartitioning : public Partitioning {
   static Status SetDefaultValuesFromKeys(const Expression& expr,
                                          RecordBatchProjector* projector);
 
-  Result<std::vector<PartitionedBatch>> Partition(
+  Result<PartitionedBatches> Partition(
       const std::shared_ptr<RecordBatch>& batch) const override;
 
   Result<std::shared_ptr<Expression>> Parse(const std::string& path) const override;
@@ -243,7 +240,7 @@ class ARROW_DS_EXPORT FunctionPartitioning : public Partitioning {
     return Status::NotImplemented("formatting paths from ", type_name(), " Partitioning");
   }
 
-  Result<std::vector<PartitionedBatch>> Partition(
+  Result<PartitionedBatches> Partition(
       const std::shared_ptr<RecordBatch>& batch) const override {
     return Status::NotImplemented("partitioning batches from ", type_name(),
                                   " Partitioning");
diff --git a/cpp/src/arrow/dataset/partition_test.cc b/cpp/src/arrow/dataset/partition_test.cc
index 27ab00ab0c2..f49103a585a 100644
--- a/cpp/src/arrow/dataset/partition_test.cc
+++ b/cpp/src/arrow/dataset/partition_test.cc
@@ -89,10 +89,14 @@ class TestPartitioning : public ::testing::Test {
     return field(std::move(name), utf8());
   }
 
-  static std::shared_ptr<Field> Dict(std::string name) {
+  static std::shared_ptr<Field> DictStr(std::string name) {
     return field(std::move(name), dictionary(int32(), utf8()));
   }
 
+  static std::shared_ptr<Field> DictInt(std::string name) {
+    return field(std::move(name), dictionary(int32(), int32()));
+  }
+
   std::shared_ptr<Partitioning> partitioning_;
   std::shared_ptr<PartitioningFactory> factory_;
 };
@@ -152,34 +156,37 @@ TEST_F(TestPartitioning, DiscoverSchema) {
   // fall back to string if any segment for field alpha is not parseable as int
   AssertInspect({"/0/1", "/hello/1"}, {Str("alpha"), Int("beta")});
 
+  // If there are too many digits fall back to string
+  AssertInspect({"/3760212050/1"}, {Str("alpha"), Int("beta")});
+
   // missing segment for beta doesn't cause an error or fallback
   AssertInspect({"/0/1", "/hello"}, {Str("alpha"), Int("beta")});
 }
 
 TEST_F(TestPartitioning, DictionaryInference) {
   PartitioningFactoryOptions options;
-  options.max_partition_dictionary_size = 2;
+  options.infer_dictionary = true;
   factory_ = DirectoryPartitioning::MakeFactory({"alpha", "beta"}, options);
 
   // type is still int32 if possible
-  AssertInspect({"/0/1"}, {Int("alpha"), Int("beta")});
+  AssertInspect({"/0/1"}, {DictInt("alpha"), DictInt("beta")});
 
-  // successful dictionary inference
-  AssertInspect({"/a/0"}, {Dict("alpha"), Int("beta")});
-  AssertInspect({"/a/0", "/a/1"}, {Dict("alpha"), Int("beta")});
-  AssertInspect({"/a/0", "/b/0", "/a/1", "/b/1"}, {Dict("alpha"), Int("beta")});
-  AssertInspect({"/a/-", "/b/-", "/a/_", "/b/_"}, {Dict("alpha"), Dict("beta")});
+  // If there are too many digits fall back to string
+  AssertInspect({"/3760212050/1"}, {DictStr("alpha"), DictInt("beta")});
 
-  // fall back to string if max dictionary size is exceeded
-  AssertInspect({"/a/0", "/b/0", "/c/1", "/d/1"}, {Str("alpha"), Int("beta")});
+  // successful dictionary inference
+  AssertInspect({"/a/0"}, {DictStr("alpha"), DictInt("beta")});
+  AssertInspect({"/a/0", "/a/1"}, {DictStr("alpha"), DictInt("beta")});
+  AssertInspect({"/a/0", "/b/0", "/a/1", "/b/1"}, {DictStr("alpha"), DictInt("beta")});
+  AssertInspect({"/a/-", "/b/-", "/a/_", "/b/_"}, {DictStr("alpha"), DictStr("beta")});
 }
 
 TEST_F(TestPartitioning, DictionaryHasUniqueValues) {
   PartitioningFactoryOptions options;
-  options.max_partition_dictionary_size = -1;
+  options.infer_dictionary = true;
   factory_ = DirectoryPartitioning::MakeFactory({"alpha"}, options);
 
-  auto alpha = Dict("alpha");
+  auto alpha = DictStr("alpha");
   AssertInspect({"/a", "/b", "/a", "/b", "/c", "/a"}, {alpha});
   ASSERT_OK_AND_ASSIGN(auto partitioning, factory_->Finish(schema({alpha})));
 
@@ -255,6 +262,9 @@ TEST_F(TestPartitioning, DiscoverHiveSchema) {
   // (...so ensure your partitions are ordered the same for all paths)
   AssertInspect({"/alpha=0/beta=1", "/beta=2/alpha=3"}, {Int("alpha"), Int("beta")});
 
+  // If there are too many digits fall back to string
+  AssertInspect({"/alpha=3760212050"}, {Str("alpha")});
+
   // missing path segments will not cause an error
   AssertInspect({"/alpha=0/beta=1", "/beta=2/alpha=3", "/gamma=what"},
                 {Int("alpha"), Int("beta"), Str("gamma")});
@@ -262,34 +272,32 @@ TEST_F(TestPartitioning, DiscoverHiveSchema) {
 
 TEST_F(TestPartitioning, HiveDictionaryInference) {
   PartitioningFactoryOptions options;
-  options.max_partition_dictionary_size = 2;
+  options.infer_dictionary = true;
   factory_ = HivePartitioning::MakeFactory(options);
 
   // type is still int32 if possible
-  AssertInspect({"/alpha=0/beta=1"}, {Int("alpha"), Int("beta")});
+  AssertInspect({"/alpha=0/beta=1"}, {DictInt("alpha"), DictInt("beta")});
+
+  // If there are too many digits fall back to string
+  AssertInspect({"/alpha=3760212050"}, {DictStr("alpha")});
 
   // successful dictionary inference
-  AssertInspect({"/alpha=a/beta=0"}, {Dict("alpha"), Int("beta")});
-  AssertInspect({"/alpha=a/beta=0", "/alpha=a/1"}, {Dict("alpha"), Int("beta")});
+  AssertInspect({"/alpha=a/beta=0"}, {DictStr("alpha"), DictInt("beta")});
+  AssertInspect({"/alpha=a/beta=0", "/alpha=a/1"}, {DictStr("alpha"), DictInt("beta")});
   AssertInspect(
       {"/alpha=a/beta=0", "/alpha=b/beta=0", "/alpha=a/beta=1", "/alpha=b/beta=1"},
-      {Dict("alpha"), Int("beta")});
+      {DictStr("alpha"), DictInt("beta")});
   AssertInspect(
       {"/alpha=a/beta=-", "/alpha=b/beta=-", "/alpha=a/beta=_", "/alpha=b/beta=_"},
-      {Dict("alpha"), Dict("beta")});
-
-  // fall back to string if max dictionary size is exceeded
-  AssertInspect(
-      {"/alpha=a/beta=0", "/alpha=b/beta=0", "/alpha=c/beta=1", "/alpha=d/beta=1"},
-      {Str("alpha"), Int("beta")});
+      {DictStr("alpha"), DictStr("beta")});
 }
 
 TEST_F(TestPartitioning, HiveDictionaryHasUniqueValues) {
   PartitioningFactoryOptions options;
-  options.max_partition_dictionary_size = -1;
+  options.infer_dictionary = true;
   factory_ = HivePartitioning::MakeFactory(options);
 
-  auto alpha = Dict("alpha");
+  auto alpha = DictStr("alpha");
   AssertInspect({"/alpha=a", "/alpha=b", "/alpha=a", "/alpha=b", "/alpha=c", "/alpha=a"},
                 {alpha});
   ASSERT_OK_AND_ASSIGN(auto partitioning, factory_->Finish(schema({alpha})));
@@ -439,7 +447,7 @@ class RangePartitioning : public Partitioning {
   }
 
   Result<std::string> Format(const Expression&) const override { return ""; }
-  Result<std::vector<PartitionedBatch>> Partition(
+  Result<PartitionedBatches> Partition(
       const std::shared_ptr<RecordBatch>&) const override {
     return Status::OK();
   }
diff --git a/cpp/src/arrow/dataset/projector.cc b/cpp/src/arrow/dataset/projector.cc
index 9ce90ad0ed3..2ba679ce6e7 100644
--- a/cpp/src/arrow/dataset/projector.cc
+++ b/cpp/src/arrow/dataset/projector.cc
@@ -46,6 +46,15 @@ Status CheckProjectable(const Schema& from, const Schema& to) {
                                from);
     }
 
+    if (from_field->type()->id() == Type::NA) {
+      // promotion from null to any type is supported
+      if (to_field->nullable()) continue;
+
+      return Status::TypeError("field ", to_field->ToString(),
+                               " is not nullable but has type ", NullType(),
+                               " in origin schema ", from);
+    }
+
     if (!from_field->type()->Equals(to_field->type())) {
       return Status::TypeError("fields had matching names but differing types. From: ",
                                from_field->ToString(), " To: ", to_field->ToString());
@@ -98,7 +107,7 @@ Result<std::shared_ptr<RecordBatch>> RecordBatchProjector::Project(
     RETURN_NOT_OK(ResizeMissingColumns(batch.num_rows(), pool));
   }
 
-  std::vector<std::shared_ptr<Array>> columns(to_->num_fields());
+  ArrayVector columns(to_->num_fields());
 
   for (int i = 0; i < to_->num_fields(); ++i) {
     if (column_indices_[i] != kNoMatch) {
@@ -120,7 +129,8 @@ Status RecordBatchProjector::SetInputSchema(std::shared_ptr<Schema> from,
     ARROW_ASSIGN_OR_RAISE(auto match,
                           FieldRef(to_->field(i)->name()).FindOneOrNone(*from_));
 
-    if (match.indices().empty()) {
+    if (match.indices().empty() ||
+        from_->field(match.indices()[0])->type()->id() == Type::NA) {
       // Mark column i as missing by setting missing_columns_[i]
       // to a non-null placeholder.
       ARROW_ASSIGN_OR_RAISE(missing_columns_[i],
diff --git a/cpp/src/arrow/dataset/scanner.cc b/cpp/src/arrow/dataset/scanner.cc
index 22075241eaa..019416041aa 100644
--- a/cpp/src/arrow/dataset/scanner.cc
+++ b/cpp/src/arrow/dataset/scanner.cc
@@ -114,7 +114,7 @@ Status ScannerBuilder::Project(std::vector<std::string> columns) {
 
 Status ScannerBuilder::Filter(std::shared_ptr<Expression> filter) {
   RETURN_NOT_OK(schema()->CanReferenceFieldsByNames(FieldsInExpression(*filter)));
-  RETURN_NOT_OK(filter->Validate(*schema()).status());
+  RETURN_NOT_OK(filter->Validate(*schema()));
   scan_options_->filter = std::move(filter);
   return Status::OK();
 }
@@ -202,7 +202,7 @@ Result<std::shared_ptr<Table>> Scanner::ToTable() {
 
   size_t scan_task_id = 0;
   for (auto maybe_scan_task : scan_task_it) {
-    ARROW_ASSIGN_OR_RAISE(auto scan_task, std::move(maybe_scan_task));
+    ARROW_ASSIGN_OR_RAISE(auto scan_task, maybe_scan_task);
 
     auto id = scan_task_id++;
     task_group->Append([state, id, scan_task] {
diff --git a/cpp/src/arrow/dataset/scanner.h b/cpp/src/arrow/dataset/scanner.h
index 243df2fde39..950d416f615 100644
--- a/cpp/src/arrow/dataset/scanner.h
+++ b/cpp/src/arrow/dataset/scanner.h
@@ -21,7 +21,6 @@
 
 #include <memory>
 #include <string>
-#include <unordered_set>
 #include <utility>
 #include <vector>
 
@@ -36,6 +35,8 @@
 namespace arrow {
 namespace dataset {
 
+constexpr int64_t kDefaultBatchSize = 1 << 20;
+
 /// \brief Shared state for a Scan operation
 struct ARROW_DS_EXPORT ScanContext {
   /// A pool from which materialized and scanned arrays will be allocated.
@@ -73,7 +74,7 @@ class ARROW_DS_EXPORT ScanOptions {
   RecordBatchProjector projector;
 
   // Maximum row count for scanned batches.
-  int64_t batch_size = 1 << 15;
+  int64_t batch_size = kDefaultBatchSize;
 
   // Return a vector of fields that requires materialization.
   //
diff --git a/cpp/src/arrow/dataset/test_util.h b/cpp/src/arrow/dataset/test_util.h
index 0a686a19268..f504305a996 100644
--- a/cpp/src/arrow/dataset/test_util.h
+++ b/cpp/src/arrow/dataset/test_util.h
@@ -22,6 +22,7 @@
 #include <functional>
 #include <memory>
 #include <string>
+#include <unordered_set>
 #include <utility>
 #include <vector>
 
@@ -29,6 +30,7 @@
 #include <gtest/gtest.h>
 
 #include "arrow/dataset/dataset_internal.h"
+#include "arrow/dataset/discovery.h"
 #include "arrow/dataset/file_base.h"
 #include "arrow/dataset/filter.h"
 #include "arrow/filesystem/localfs.h"
@@ -36,6 +38,7 @@
 #include "arrow/filesystem/path_util.h"
 #include "arrow/filesystem/test_util.h"
 #include "arrow/record_batch.h"
+#include "arrow/table.h"
 #include "arrow/testing/generator.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/util/io_util.h"
@@ -48,6 +51,7 @@ namespace dataset {
 
 using fs::internal::GetAbstractPathExtension;
 using internal::checked_cast;
+using internal::checked_pointer_cast;
 using internal::TemporaryDir;
 
 class FileSourceFixtureMixin : public ::testing::Test {
@@ -196,6 +200,11 @@ class DummyFileFormat : public FileFormat {
 
   std::string type_name() const override { return "dummy"; }
 
+  bool Equals(const FileFormat& other) const override {
+    return type_name() == other.type_name() &&
+           schema_->Equals(checked_cast<const DummyFileFormat&>(other).schema_);
+  }
+
   Result<bool> IsSupported(const FileSource& source) const override { return true; }
 
   Result<std::shared_ptr<Schema>> Inspect(const FileSource& source) const override {
@@ -209,6 +218,14 @@ class DummyFileFormat : public FileFormat {
     return MakeEmptyIterator<std::shared_ptr<ScanTask>>();
   }
 
+  Result<std::shared_ptr<FileWriter>> MakeWriter(
+      std::shared_ptr<io::OutputStream> destination, std::shared_ptr<Schema> schema,
+      std::shared_ptr<FileWriteOptions> options) const override {
+    return Status::NotImplemented("writing fragment of DummyFileFormat");
+  }
+
+  std::shared_ptr<FileWriteOptions> DefaultWriteOptions() override { return nullptr; }
+
  protected:
   std::shared_ptr<Schema> schema_;
 };
@@ -223,6 +240,8 @@ class JSONRecordBatchFileFormat : public FileFormat {
   explicit JSONRecordBatchFileFormat(SchemaResolver resolver)
       : resolver_(std::move(resolver)) {}
 
+  bool Equals(const FileFormat& other) const override { return this == &other; }
+
   std::string type_name() const override { return "json_record_batch"; }
 
   /// \brief Return true if the given file extension
@@ -248,6 +267,14 @@ class JSONRecordBatchFileFormat : public FileFormat {
                                            std::move(context));
   }
 
+  Result<std::shared_ptr<FileWriter>> MakeWriter(
+      std::shared_ptr<io::OutputStream> destination, std::shared_ptr<Schema> schema,
+      std::shared_ptr<FileWriteOptions> options) const override {
+    return Status::NotImplemented("writing fragment of JSONRecordBatchFileFormat");
+  }
+
+  std::shared_ptr<FileWriteOptions> DefaultWriteOptions() override { return nullptr; }
+
  protected:
   SchemaResolver resolver_;
 };
@@ -317,7 +344,7 @@ struct MakeFileSystemDatasetMixin {
       fragments.push_back(std::move(fragment));
     }
 
-    ASSERT_OK_AND_ASSIGN(dataset_, FileSystemDataset::Make(s, root_partition, format,
+    ASSERT_OK_AND_ASSIGN(dataset_, FileSystemDataset::Make(s, root_partition, format, fs_,
                                                            std::move(fragments)));
   }
 
@@ -485,5 +512,279 @@ struct ArithmeticDatasetFixture {
   }
 };
 
+class WriteFileSystemDatasetMixin : public MakeFileSystemDatasetMixin {
+ public:
+  using PathAndContent = std::unordered_map<std::string, std::string>;
+
+  void MakeSourceDataset() {
+    PathAndContent source_files;
+
+    source_files["/dataset/year=2018/month=01/dat0.json"] = R"([
+        {"region": "NY", "model": "3", "sales": 742.0, "country": "US"},
+        {"region": "NY", "model": "S", "sales": 304.125, "country": "US"},
+        {"region": "NY", "model": "Y", "sales": 27.5, "country": "US"}
+      ])";
+    source_files["/dataset/year=2018/month=01/dat1.json"] = R"([
+        {"region": "QC", "model": "3", "sales": 512, "country": "CA"},
+        {"region": "QC", "model": "S", "sales": 978, "country": "CA"},
+        {"region": "NY", "model": "X", "sales": 136.25, "country": "US"},
+        {"region": "QC", "model": "X", "sales": 1.0, "country": "CA"},
+        {"region": "QC", "model": "Y", "sales": 69, "country": "CA"}
+      ])";
+    source_files["/dataset/year=2019/month=01/dat0.json"] = R"([
+        {"region": "CA", "model": "3", "sales": 273.5, "country": "US"},
+        {"region": "CA", "model": "S", "sales": 13, "country": "US"},
+        {"region": "CA", "model": "X", "sales": 54, "country": "US"},
+        {"region": "QC", "model": "S", "sales": 10, "country": "CA"},
+        {"region": "CA", "model": "Y", "sales": 21, "country": "US"}
+      ])";
+    source_files["/dataset/year=2019/month=01/dat1.json"] = R"([
+        {"region": "QC", "model": "3", "sales": 152.25, "country": "CA"},
+        {"region": "QC", "model": "X", "sales": 42, "country": "CA"},
+        {"region": "QC", "model": "Y", "sales": 37, "country": "CA"}
+      ])";
+    source_files["/dataset/.pesky"] = "garbage content";
+
+    auto mock_fs = std::make_shared<fs::internal::MockFileSystem>(fs::kNoTime);
+    for (const auto& f : source_files) {
+      ARROW_EXPECT_OK(mock_fs->CreateFile(f.first, f.second, /* recursive */ true));
+    }
+    fs_ = mock_fs;
+
+    /// schema for the whole dataset (both source and destination)
+    source_schema_ = schema({
+        field("region", utf8()),
+        field("model", utf8()),
+        field("sales", float64()),
+        field("year", int32()),
+        field("month", int32()),
+        field("country", utf8()),
+    });
+
+    /// Dummy file format for source dataset. Note that it isn't partitioned on country
+    auto source_format = std::make_shared<JSONRecordBatchFileFormat>(
+        SchemaFromColumnNames(source_schema_, {"region", "model", "sales", "country"}));
+
+    fs::FileSelector s;
+    s.base_dir = "/dataset";
+    s.recursive = true;
+
+    FileSystemFactoryOptions options;
+    options.selector_ignore_prefixes = {"."};
+    options.partitioning = std::make_shared<HivePartitioning>(
+        SchemaFromColumnNames(source_schema_, {"year", "month"}));
+    ASSERT_OK_AND_ASSIGN(auto factory,
+                         FileSystemDatasetFactory::Make(fs_, s, source_format, options));
+    ASSERT_OK_AND_ASSIGN(dataset_, factory->Finish());
+
+    scan_options_ = ScanOptions::Make(source_schema_);
+  }
+
+  void SetWriteOptions(std::shared_ptr<FileWriteOptions> file_write_options) {
+    write_options_.file_write_options = file_write_options;
+    write_options_.filesystem = fs_;
+    write_options_.base_dir = "new_root/";
+    write_options_.basename_template = "dat_{i}";
+  }
+
+  void DoWrite(std::shared_ptr<Partitioning> desired_partitioning) {
+    write_options_.partitioning = desired_partitioning;
+    auto scanner = std::make_shared<Scanner>(dataset_, scan_options_, scan_context_);
+    ASSERT_OK(FileSystemDataset::Write(write_options_, scanner));
+
+    // re-discover the written dataset
+    fs::FileSelector s;
+    s.recursive = true;
+    s.base_dir = "/new_root";
+
+    FileSystemFactoryOptions factory_options;
+    factory_options.partitioning = desired_partitioning;
+    ASSERT_OK_AND_ASSIGN(
+        auto factory, FileSystemDatasetFactory::Make(fs_, s, format_, factory_options));
+    ASSERT_OK_AND_ASSIGN(written_, factory->Finish());
+  }
+
+  void TestWriteWithIdenticalPartitioningSchema() {
+    DoWrite(std::make_shared<DirectoryPartitioning>(
+        SchemaFromColumnNames(source_schema_, {"year", "month"})));
+
+    expected_files_["/new_root/2018/1/dat_0"] = R"([
+        {"region": "NY", "model": "3", "sales": 742.0, "country": "US"},
+        {"region": "NY", "model": "S", "sales": 304.125, "country": "US"},
+        {"region": "NY", "model": "Y", "sales": 27.5, "country": "US"},
+        {"region": "QC", "model": "3", "sales": 512, "country": "CA"},
+        {"region": "QC", "model": "S", "sales": 978, "country": "CA"},
+        {"region": "NY", "model": "X", "sales": 136.25, "country": "US"},
+        {"region": "QC", "model": "X", "sales": 1.0, "country": "CA"},
+        {"region": "QC", "model": "Y", "sales": 69, "country": "CA"}
+      ])";
+    expected_files_["/new_root/2019/1/dat_1"] = R"([
+        {"region": "CA", "model": "3", "sales": 273.5, "country": "US"},
+        {"region": "CA", "model": "S", "sales": 13, "country": "US"},
+        {"region": "CA", "model": "X", "sales": 54, "country": "US"},
+        {"region": "QC", "model": "S", "sales": 10, "country": "CA"},
+        {"region": "CA", "model": "Y", "sales": 21, "country": "US"},
+        {"region": "QC", "model": "3", "sales": 152.25, "country": "CA"},
+        {"region": "QC", "model": "X", "sales": 42, "country": "CA"},
+        {"region": "QC", "model": "Y", "sales": 37, "country": "CA"}
+      ])";
+    expected_physical_schema_ =
+        SchemaFromColumnNames(source_schema_, {"region", "model", "sales", "country"});
+
+    AssertWrittenAsExpected();
+  }
+
+  void TestWriteWithUnrelatedPartitioningSchema() {
+    DoWrite(std::make_shared<DirectoryPartitioning>(
+        SchemaFromColumnNames(source_schema_, {"country", "region"})));
+
+    // XXX first thing a user will be annoyed by: we don't support left
+    // padding the month field with 0.
+    expected_files_["/new_root/US/NY/dat_0"] = R"([
+        {"year": 2018, "month": 1, "model": "3", "sales": 742.0},
+        {"year": 2018, "month": 1, "model": "S", "sales": 304.125},
+        {"year": 2018, "month": 1, "model": "Y", "sales": 27.5},
+        {"year": 2018, "month": 1, "model": "X", "sales": 136.25}
+  ])";
+    expected_files_["/new_root/CA/QC/dat_1"] = R"([
+        {"year": 2018, "month": 1, "model": "3", "sales": 512},
+        {"year": 2018, "month": 1, "model": "S", "sales": 978},
+        {"year": 2018, "month": 1, "model": "X", "sales": 1.0},
+        {"year": 2018, "month": 1, "model": "Y", "sales": 69},
+        {"year": 2019, "month": 1, "model": "S", "sales": 10},
+        {"year": 2019, "month": 1, "model": "3", "sales": 152.25},
+        {"year": 2019, "month": 1, "model": "X", "sales": 42},
+        {"year": 2019, "month": 1, "model": "Y", "sales": 37}
+  ])";
+    expected_files_["/new_root/US/CA/dat_2"] = R"([
+        {"year": 2019, "month": 1, "model": "3", "sales": 273.5},
+        {"year": 2019, "month": 1, "model": "S", "sales": 13},
+        {"year": 2019, "month": 1, "model": "X", "sales": 54},
+        {"year": 2019, "month": 1, "model": "Y", "sales": 21}
+  ])";
+    expected_physical_schema_ =
+        SchemaFromColumnNames(source_schema_, {"model", "sales", "year", "month"});
+
+    AssertWrittenAsExpected();
+  }
+
+  void TestWriteWithSupersetPartitioningSchema() {
+    DoWrite(std::make_shared<DirectoryPartitioning>(
+        SchemaFromColumnNames(source_schema_, {"year", "month", "country", "region"})));
+
+    // XXX first thing a user will be annoyed by: we don't support left
+    // padding the month field with 0.
+    expected_files_["/new_root/2018/1/US/NY/dat_0"] = R"([
+        {"model": "3", "sales": 742.0},
+        {"model": "S", "sales": 304.125},
+        {"model": "Y", "sales": 27.5},
+        {"model": "X", "sales": 136.25}
+  ])";
+    expected_files_["/new_root/2018/1/CA/QC/dat_1"] = R"([
+        {"model": "3", "sales": 512},
+        {"model": "S", "sales": 978},
+        {"model": "X", "sales": 1.0},
+        {"model": "Y", "sales": 69}
+  ])";
+    expected_files_["/new_root/2019/1/US/CA/dat_2"] = R"([
+        {"model": "3", "sales": 273.5},
+        {"model": "S", "sales": 13},
+        {"model": "X", "sales": 54},
+        {"model": "Y", "sales": 21}
+  ])";
+    expected_files_["/new_root/2019/1/CA/QC/dat_3"] = R"([
+        {"model": "S", "sales": 10},
+        {"model": "3", "sales": 152.25},
+        {"model": "X", "sales": 42},
+        {"model": "Y", "sales": 37}
+  ])";
+    expected_physical_schema_ = SchemaFromColumnNames(source_schema_, {"model", "sales"});
+
+    AssertWrittenAsExpected();
+  }
+
+  void TestWriteWithEmptyPartitioningSchema() {
+    DoWrite(std::make_shared<DirectoryPartitioning>(
+        SchemaFromColumnNames(source_schema_, {})));
+
+    expected_files_["/new_root/dat_0"] = R"([
+        {"country": "US", "region": "NY", "year": 2018, "month": 1, "model": "3", "sales": 742.0},
+        {"country": "US", "region": "NY", "year": 2018, "month": 1, "model": "S", "sales": 304.125},
+        {"country": "US", "region": "NY", "year": 2018, "month": 1, "model": "Y", "sales": 27.5},
+        {"country": "CA", "region": "QC", "year": 2018, "month": 1, "model": "3", "sales": 512},
+        {"country": "CA", "region": "QC", "year": 2018, "month": 1, "model": "S", "sales": 978},
+        {"country": "US", "region": "NY", "year": 2018, "month": 1, "model": "X", "sales": 136.25},
+        {"country": "CA", "region": "QC", "year": 2018, "month": 1, "model": "X", "sales": 1.0},
+        {"country": "CA", "region": "QC", "year": 2018, "month": 1, "model": "Y", "sales": 69},
+        {"country": "US", "region": "CA", "year": 2019, "month": 1, "model": "3", "sales": 273.5},
+        {"country": "US", "region": "CA", "year": 2019, "month": 1, "model": "S", "sales": 13},
+        {"country": "US", "region": "CA", "year": 2019, "month": 1, "model": "X", "sales": 54},
+        {"country": "CA", "region": "QC", "year": 2019, "month": 1, "model": "S", "sales": 10},
+        {"country": "US", "region": "CA", "year": 2019, "month": 1, "model": "Y", "sales": 21},
+        {"country": "CA", "region": "QC", "year": 2019, "month": 1, "model": "3", "sales": 152.25},
+        {"country": "CA", "region": "QC", "year": 2019, "month": 1, "model": "X", "sales": 42},
+        {"country": "CA", "region": "QC", "year": 2019, "month": 1, "model": "Y", "sales": 37}
+  ])";
+    expected_physical_schema_ = source_schema_;
+
+    AssertWrittenAsExpected();
+  }
+
+  void AssertWrittenAsExpected() {
+    std::unordered_set<std::string> expected_paths, actual_paths;
+    for (const auto& file_contents : expected_files_) {
+      expected_paths.insert(file_contents.first);
+    }
+    for (auto path : checked_pointer_cast<FileSystemDataset>(written_)->files()) {
+      actual_paths.insert(std::move(path));
+    }
+    EXPECT_THAT(actual_paths, testing::UnorderedElementsAreArray(expected_paths));
+
+    for (auto maybe_fragment : written_->GetFragments()) {
+      ASSERT_OK_AND_ASSIGN(auto fragment, maybe_fragment);
+
+      ASSERT_OK_AND_ASSIGN(auto actual_physical_schema, fragment->ReadPhysicalSchema());
+      AssertSchemaEqual(*expected_physical_schema_, *actual_physical_schema,
+                        check_metadata_);
+
+      const auto& path = checked_pointer_cast<FileFragment>(fragment)->source().path();
+
+      auto file_contents = expected_files_.find(path);
+      if (file_contents == expected_files_.end()) {
+        // file wasn't expected to be written at all; nothing to compare with
+        continue;
+      }
+
+      ASSERT_OK_AND_ASSIGN(auto scanner, ScannerBuilder(actual_physical_schema, fragment,
+                                                        std::make_shared<ScanContext>())
+                                             .Finish());
+      ASSERT_OK_AND_ASSIGN(auto actual_table, scanner->ToTable());
+      ASSERT_OK_AND_ASSIGN(actual_table, actual_table->CombineChunks());
+      std::shared_ptr<Array> actual_struct;
+
+      for (auto maybe_batch :
+           IteratorFromReader(std::make_shared<TableBatchReader>(*actual_table))) {
+        ASSERT_OK_AND_ASSIGN(auto batch, maybe_batch);
+        ASSERT_OK_AND_ASSIGN(actual_struct, batch->ToStructArray());
+      }
+
+      auto expected_struct = ArrayFromJSON(struct_(expected_physical_schema_->fields()),
+                                           {file_contents->second});
+
+      AssertArraysEqual(*expected_struct, *actual_struct, /*verbose=*/true);
+    }
+  }
+
+  bool check_metadata_ = true;
+  std::shared_ptr<Schema> source_schema_;
+  std::shared_ptr<FileFormat> format_;
+  PathAndContent expected_files_;
+  std::shared_ptr<Schema> expected_physical_schema_;
+  std::shared_ptr<Dataset> written_;
+  FileSystemDatasetWriteOptions write_options_;
+  std::shared_ptr<ScanOptions> scan_options_;
+  std::shared_ptr<ScanContext> scan_context_ = std::make_shared<ScanContext>();
+};
+
 }  // namespace dataset
 }  // namespace arrow
diff --git a/cpp/src/arrow/dataset/type_fwd.h b/cpp/src/arrow/dataset/type_fwd.h
index 089e1dffdde..73cfdff2b42 100644
--- a/cpp/src/arrow/dataset/type_fwd.h
+++ b/cpp/src/arrow/dataset/type_fwd.h
@@ -48,14 +48,23 @@ using FragmentVector = std::vector<std::shared_ptr<Fragment>>;
 class FileSource;
 class FileFormat;
 class FileFragment;
+class FileWriter;
+class FileWriteOptions;
 class FileSystemDataset;
+struct FileSystemDatasetWriteOptions;
+
+class InMemoryDataset;
 
 class CsvFileFormat;
 
 class IpcFileFormat;
+class IpcFileWriter;
+class IpcFileWriteOptions;
 
 class ParquetFileFormat;
 class ParquetFileFragment;
+class ParquetFileWriter;
+class ParquetFileWriteOptions;
 
 class Expression;
 using ExpressionVector = std::vector<std::shared_ptr<Expression>>;
@@ -63,7 +72,7 @@ class ExpressionEvaluator;
 
 /// forward declared to facilitate scalar(true) as a default for Expression parameters
 ARROW_DS_EXPORT
-std::shared_ptr<Expression> scalar(bool);
+const std::shared_ptr<Expression>& scalar(bool);
 
 class Partitioning;
 class PartitioningFactory;
diff --git a/cpp/src/arrow/dbi/hiveserver2/thrift/Types.thrift b/cpp/src/arrow/dbi/hiveserver2/thrift/Types.thrift
index 4238f9c26ba..39ae6d0ba6d 100644
--- a/cpp/src/arrow/dbi/hiveserver2/thrift/Types.thrift
+++ b/cpp/src/arrow/dbi/hiveserver2/thrift/Types.thrift
@@ -115,7 +115,7 @@ enum TRuntimeFilterMode {
   // fragment.
   LOCAL,
 
-  // All fiters are computed in the BE, and are published globally.
+  // All filters are computed in the BE, and are published globally.
   GLOBAL
 }
 
diff --git a/cpp/src/arrow/extension_type.cc b/cpp/src/arrow/extension_type.cc
index 23bb30955b2..e579b691023 100644
--- a/cpp/src/arrow/extension_type.cc
+++ b/cpp/src/arrow/extension_type.cc
@@ -25,6 +25,7 @@
 #include <utility>
 
 #include "arrow/array/util.h"
+#include "arrow/chunked_array.h"
 #include "arrow/status.h"
 #include "arrow/type.h"
 #include "arrow/util/checked_cast.h"
@@ -42,6 +43,31 @@ std::string ExtensionType::ToString() const {
   return ss.str();
 }
 
+std::shared_ptr<Array> ExtensionType::WrapArray(const std::shared_ptr<DataType>& type,
+                                                const std::shared_ptr<Array>& storage) {
+  DCHECK_EQ(type->id(), Type::EXTENSION);
+  const auto& ext_type = checked_cast<const ExtensionType&>(*type);
+  DCHECK_EQ(storage->type_id(), ext_type.storage_type()->id());
+  auto data = storage->data()->Copy();
+  data->type = type;
+  return ext_type.MakeArray(std::move(data));
+}
+
+std::shared_ptr<ChunkedArray> ExtensionType::WrapArray(
+    const std::shared_ptr<DataType>& type, const std::shared_ptr<ChunkedArray>& storage) {
+  DCHECK_EQ(type->id(), Type::EXTENSION);
+  const auto& ext_type = checked_cast<const ExtensionType&>(*type);
+  DCHECK_EQ(storage->type()->id(), ext_type.storage_type()->id());
+
+  ArrayVector out_chunks(storage->num_chunks());
+  for (int i = 0; i < storage->num_chunks(); i++) {
+    auto data = storage->chunk(i)->data()->Copy();
+    data->type = type;
+    out_chunks[i] = ext_type.MakeArray(std::move(data));
+  }
+  return std::make_shared<ChunkedArray>(std::move(out_chunks));
+}
+
 ExtensionArray::ExtensionArray(const std::shared_ptr<ArrayData>& data) { SetData(data); }
 
 ExtensionArray::ExtensionArray(const std::shared_ptr<DataType>& type,
diff --git a/cpp/src/arrow/extension_type.h b/cpp/src/arrow/extension_type.h
index ef7205eb058..a22d015195d 100644
--- a/cpp/src/arrow/extension_type.h
+++ b/cpp/src/arrow/extension_type.h
@@ -28,6 +28,7 @@
 #include "arrow/result.h"
 #include "arrow/status.h"
 #include "arrow/type.h"
+#include "arrow/type_fwd.h"
 #include "arrow/util/checked_cast.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
@@ -80,6 +81,15 @@ class ARROW_EXPORT ExtensionType : public DataType {
   /// \return the serialized representation
   virtual std::string Serialize() const = 0;
 
+  /// \brief Wrap the given storage array as an extension array
+  static std::shared_ptr<Array> WrapArray(const std::shared_ptr<DataType>& ext_type,
+                                          const std::shared_ptr<Array>& storage);
+
+  /// \brief Wrap the given chunked storage array as a chunked extension array
+  static std::shared_ptr<ChunkedArray> WrapArray(
+      const std::shared_ptr<DataType>& ext_type,
+      const std::shared_ptr<ChunkedArray>& storage);
+
  protected:
   explicit ExtensionType(std::shared_ptr<DataType> storage_type)
       : DataType(Type::EXTENSION), storage_type_(storage_type) {}
diff --git a/cpp/src/arrow/filesystem/CMakeLists.txt b/cpp/src/arrow/filesystem/CMakeLists.txt
index b2eb480859e..5b06646157b 100644
--- a/cpp/src/arrow/filesystem/CMakeLists.txt
+++ b/cpp/src/arrow/filesystem/CMakeLists.txt
@@ -31,6 +31,26 @@ add_arrow_test(filesystem-test
 
 if(ARROW_S3)
   add_arrow_test(s3fs_test EXTRA_LABELS filesystem)
+  if(TARGET arrow-s3fs-test)
+    set(ARROW_S3FS_TEST_COMPILE_DEFINITIONS ${ARROW_BOOST_PROCESS_COMPILE_DEFINITIONS})
+    get_target_property(AWS_CPP_SDK_S3_TYPE aws-cpp-sdk-s3 TYPE)
+    # We need to initialize AWS C++ SDK for direct use (not via
+    # arrow::fs::S3FileSystem) in arrow-s3fs-test if we use static AWS
+    # C++ SDK. Because AWS C++ SDK has internal static variables that
+    # aren't shared in libarrow and arrow-s3fs-test. It means that
+    # arrow::fs::InitializeS3() doesn't initialize AWS C++ SDK that is
+    # directly used in arrow-s3fs-test.
+    #
+    # But it seems that internal static variables in AWS C++ SDK are
+    # shared on macOS even if we link static AWS C++ SDK to both
+    # libarrow and arrow-s3fs-test. So we don't need to initialize AWS
+    # C++ SDK in arrow-s3fs-test on macOS.
+    if(AWS_CPP_SDK_S3_TYPE STREQUAL "STATIC_LIBRARY" AND NOT APPLE)
+      list(APPEND ARROW_S3FS_TEST_COMPILE_DEFINITIONS "AWS_CPP_SDK_S3_NOT_SHARED")
+    endif()
+    target_compile_definitions(arrow-s3fs-test PRIVATE
+                               ${ARROW_S3FS_TEST_COMPILE_DEFINITIONS})
+  endif()
 
   if(ARROW_BUILD_TESTS)
     add_executable(arrow-s3fs-narrative-test s3fs_narrative_test.cc)
@@ -41,6 +61,8 @@ if(ARROW_S3)
 
   if(ARROW_BUILD_BENCHMARKS AND ARROW_PARQUET)
     add_arrow_benchmark(s3fs_benchmark PREFIX "arrow-filesystem")
+    target_compile_definitions(arrow-filesystem-s3fs-benchmark PRIVATE
+                               ${ARROW_BOOST_PROCESS_COMPILE_DEFINITIONS})
     if(ARROW_TEST_LINKAGE STREQUAL "static")
       target_link_libraries(arrow-filesystem-s3fs-benchmark PRIVATE parquet_static)
     else()
diff --git a/cpp/src/arrow/filesystem/api.h b/cpp/src/arrow/filesystem/api.h
index bb6ad990c8a..5b0c97d150a 100644
--- a/cpp/src/arrow/filesystem/api.h
+++ b/cpp/src/arrow/filesystem/api.h
@@ -17,8 +17,12 @@
 
 #pragma once
 
+#include "arrow/util/config.h"  // IWYU pragma: export
+
 #include "arrow/filesystem/filesystem.h"  // IWYU pragma: export
 #include "arrow/filesystem/hdfs.h"        // IWYU pragma: export
 #include "arrow/filesystem/localfs.h"     // IWYU pragma: export
 #include "arrow/filesystem/mockfs.h"      // IWYU pragma: export
-#include "arrow/filesystem/s3fs.h"        // IWYU pragma: export
+#ifdef ARROW_S3
+#include "arrow/filesystem/s3fs.h"  // IWYU pragma: export
+#endif
diff --git a/cpp/src/arrow/filesystem/filesystem.cc b/cpp/src/arrow/filesystem/filesystem.cc
index 9558ba7a409..6945aa06465 100644
--- a/cpp/src/arrow/filesystem/filesystem.cc
+++ b/cpp/src/arrow/filesystem/filesystem.cc
@@ -18,6 +18,8 @@
 #include <sstream>
 #include <utility>
 
+#include "arrow/util/config.h"
+
 #include "arrow/filesystem/filesystem.h"
 #ifdef ARROW_HDFS
 #include "arrow/filesystem/hdfs.h"
@@ -35,6 +37,7 @@
 #include "arrow/util/checked_cast.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
+#include "arrow/util/parallel.h"
 #include "arrow/util/uri.h"
 #include "arrow/util/windows_fixup.h"
 
@@ -439,6 +442,69 @@ Result<std::shared_ptr<io::OutputStream>> SlowFileSystem::OpenAppendStream(
   return base_fs_->OpenAppendStream(path);
 }
 
+Status CopyFiles(const std::vector<FileLocator>& sources,
+                 const std::vector<FileLocator>& destinations, int64_t chunk_size,
+                 bool use_threads) {
+  if (sources.size() != destinations.size()) {
+    return Status::Invalid("Trying to copy ", sources.size(), " files into ",
+                           destinations.size(), " paths.");
+  }
+
+  return ::arrow::internal::OptionalParallelFor(
+      use_threads, static_cast<int>(sources.size()), [&](int i) {
+        if (sources[i].filesystem->Equals(destinations[i].filesystem)) {
+          return sources[i].filesystem->CopyFile(sources[i].path, destinations[i].path);
+        }
+
+        ARROW_ASSIGN_OR_RAISE(auto source,
+                              sources[i].filesystem->OpenInputStream(sources[i].path));
+
+        ARROW_ASSIGN_OR_RAISE(
+            auto destination,
+            destinations[i].filesystem->OpenOutputStream(destinations[i].path));
+        return internal::CopyStream(source, destination, chunk_size);
+      });
+}
+
+Status CopyFiles(const std::shared_ptr<FileSystem>& source_fs,
+                 const FileSelector& source_sel,
+                 const std::shared_ptr<FileSystem>& destination_fs,
+                 const std::string& destination_base_dir, int64_t chunk_size,
+                 bool use_threads) {
+  ARROW_ASSIGN_OR_RAISE(auto source_infos, source_fs->GetFileInfo(source_sel));
+  if (source_infos.empty()) {
+    return Status::OK();
+  }
+
+  std::vector<FileLocator> sources, destinations;
+  std::vector<std::string> dirs;
+
+  for (const FileInfo& source_info : source_infos) {
+    auto relative = internal::RemoveAncestor(source_sel.base_dir, source_info.path());
+    if (!relative.has_value()) {
+      return Status::Invalid("GetFileInfo() yielded path '", source_info.path(),
+                             "', which is outside base dir '", source_sel.base_dir, "'");
+    }
+
+    auto destination_path =
+        internal::ConcatAbstractPath(destination_base_dir, relative->to_string());
+
+    if (source_info.IsDirectory()) {
+      dirs.push_back(destination_path);
+    } else if (source_info.IsFile()) {
+      sources.push_back({source_fs, source_info.path()});
+      destinations.push_back({destination_fs, destination_path});
+    }
+  }
+
+  dirs = internal::MinimalCreateDirSet(std::move(dirs));
+  RETURN_NOT_OK(::arrow::internal::OptionalParallelFor(
+      use_threads, static_cast<int>(dirs.size()),
+      [&](int i) { return destination_fs->CreateDir(dirs[i]); }));
+
+  return CopyFiles(sources, destinations, chunk_size, use_threads);
+}
+
 namespace {
 
 Result<Uri> ParseFileSystemUri(const std::string& uri_string) {
diff --git a/cpp/src/arrow/filesystem/filesystem.h b/cpp/src/arrow/filesystem/filesystem.h
index d8466b99198..9eeb3d86841 100644
--- a/cpp/src/arrow/filesystem/filesystem.h
+++ b/cpp/src/arrow/filesystem/filesystem.h
@@ -125,13 +125,19 @@ struct ARROW_EXPORT FileSelector {
   std::string base_dir;
   /// The behavior if `base_dir` isn't found in the filesystem.  If false,
   /// an error is returned.  If true, an empty selection is returned.
-  bool allow_not_found = false;
+  bool allow_not_found;
   /// Whether to recurse into subdirectories.
-  bool recursive = false;
+  bool recursive;
   /// The maximum number of subdirectories to recurse into.
-  int32_t max_recursion = INT32_MAX;
+  int32_t max_recursion;
 
-  FileSelector() {}
+  FileSelector() : allow_not_found(false), recursive(false), max_recursion(INT32_MAX) {}
+};
+
+/// \brief FileSystem, path pair
+struct ARROW_EXPORT FileLocator {
+  std::shared_ptr<FileSystem> filesystem;
+  std::string path;
 };
 
 /// \brief Abstract file system API
@@ -387,6 +393,26 @@ Result<std::shared_ptr<FileSystem>> FileSystemFromUriOrPath(
 
 /// @}
 
+/// \brief Copy files, including from one FileSystem to another
+///
+/// If a source and destination are resident in the same FileSystem FileSystem::CopyFile
+/// will be used, otherwise the file will be opened as a stream in both FileSystems and
+/// chunks copied from the source to the destination. No directories will be created.
+ARROW_EXPORT
+Status CopyFiles(const std::vector<FileLocator>& sources,
+                 const std::vector<FileLocator>& destinations,
+                 int64_t chunk_size = 1024 * 1024, bool use_threads = true);
+
+/// \brief Copy selected files, including from one FileSystem to another
+///
+/// Directories will be created under the destination base directory as needed.
+ARROW_EXPORT
+Status CopyFiles(const std::shared_ptr<FileSystem>& source_fs,
+                 const FileSelector& source_sel,
+                 const std::shared_ptr<FileSystem>& destination_fs,
+                 const std::string& destination_base_dir,
+                 int64_t chunk_size = 1024 * 1024, bool use_threads = true);
+
 struct FileSystemGlobalOptions {
   /// Path to a single PEM file holding all TLS CA certificates
   ///
diff --git a/cpp/src/arrow/filesystem/filesystem_test.cc b/cpp/src/arrow/filesystem/filesystem_test.cc
index b679f8bd97e..f3b561fc581 100644
--- a/cpp/src/arrow/filesystem/filesystem_test.cc
+++ b/cpp/src/arrow/filesystem/filesystem_test.cc
@@ -188,6 +188,16 @@ TEST(PathUtil, RemoveLeadingSlash) {
   ASSERT_EQ("abc/def/", std::string(RemoveLeadingSlash("//abc/def/")));
 }
 
+TEST(PathUtil, IsAncestorOf) {
+  ASSERT_TRUE(IsAncestorOf("", ""));
+  ASSERT_TRUE(IsAncestorOf("", "/hello"));
+  ASSERT_TRUE(IsAncestorOf("/hello", "/hello"));
+  ASSERT_FALSE(IsAncestorOf("/hello", "/world"));
+  ASSERT_TRUE(IsAncestorOf("/hello", "/hello/world"));
+  ASSERT_TRUE(IsAncestorOf("/hello", "/hello/world/how/are/you"));
+  ASSERT_FALSE(IsAncestorOf("/hello/w", "/hello/world"));
+}
+
 TEST(PathUtil, MakeAbstractPathRelative) {
   ASSERT_OK_AND_EQ("", MakeAbstractPathRelative("/", "/"));
   ASSERT_OK_AND_EQ("foo/bar", MakeAbstractPathRelative("/", "/foo/bar"));
@@ -223,6 +233,21 @@ TEST(PathUtil, AncestorsFromBasePath) {
             V({"foo/bar", "foo/bar/baz"}));
 }
 
+TEST(PathUtil, MinimalCreateDirSet) {
+  using V = std::vector<std::string>;
+
+  ASSERT_EQ(MinimalCreateDirSet({}), V{});
+  ASSERT_EQ(MinimalCreateDirSet({"foo"}), V{"foo"});
+  ASSERT_EQ(MinimalCreateDirSet({"foo", "foo/bar"}), V{"foo/bar"});
+  ASSERT_EQ(MinimalCreateDirSet({"foo", "foo/bar/baz"}), V{"foo/bar/baz"});
+  ASSERT_EQ(MinimalCreateDirSet({"foo", "foo/bar", "foo/bar"}), V{"foo/bar"});
+  ASSERT_EQ(MinimalCreateDirSet({"foo", "foo/bar", "foo", "foo/baz", "foo/baz/quux"}),
+            V({"foo/bar", "foo/baz/quux"}));
+
+  ASSERT_EQ(MinimalCreateDirSet({""}), V{});
+  ASSERT_EQ(MinimalCreateDirSet({"", "/foo"}), V{"/foo"});
+}
+
 TEST(PathUtil, ToBackslashes) {
   ASSERT_EQ(ToBackslashes("foo/bar"), "foo\\bar");
   ASSERT_EQ(ToBackslashes("//foo/bar/"), "\\\\foo\\bar\\");
@@ -595,6 +620,32 @@ TEST_F(TestSubTreeFileSystem, CopyFile) {
               {"sub/tree/cd", time_, "data"}});
 }
 
+TEST_F(TestSubTreeFileSystem, CopyFiles) {
+  ASSERT_OK(subfs_->CreateDir("AB"));
+  ASSERT_OK(subfs_->CreateDir("CD/CD"));
+  ASSERT_OK(subfs_->CreateDir("EF/EF/EF"));
+
+  CreateFile("AB/ab", "ab");
+  CreateFile("CD/CD/cd", "cd");
+  CreateFile("EF/EF/EF/ef", "ef");
+
+  ASSERT_OK(fs_->CreateDir("sub/copy"));
+  auto dest_fs = std::make_shared<SubTreeFileSystem>("sub/copy", fs_);
+
+  FileSelector sel;
+  sel.recursive = true;
+  ASSERT_OK(CopyFiles(subfs_, sel, dest_fs, ""));
+
+  CheckFiles({
+      {"sub/copy/AB/ab", time_, "ab"},
+      {"sub/copy/CD/CD/cd", time_, "cd"},
+      {"sub/copy/EF/EF/EF/ef", time_, "ef"},
+      {"sub/tree/AB/ab", time_, "ab"},
+      {"sub/tree/CD/CD/cd", time_, "cd"},
+      {"sub/tree/EF/EF/EF/ef", time_, "ef"},
+  });
+}
+
 TEST_F(TestSubTreeFileSystem, OpenInputStream) {
   std::shared_ptr<io::InputStream> stream;
   CreateFile("ab", "data");
diff --git a/cpp/src/arrow/filesystem/mockfs.cc b/cpp/src/arrow/filesystem/mockfs.cc
index ad7d0092315..4253b2c168b 100644
--- a/cpp/src/arrow/filesystem/mockfs.cc
+++ b/cpp/src/arrow/filesystem/mockfs.cc
@@ -18,6 +18,7 @@
 #include <algorithm>
 #include <iterator>
 #include <map>
+#include <mutex>
 #include <sstream>
 #include <string>
 #include <utility>
@@ -224,10 +225,15 @@ class MockFileSystem::Impl {
   TimePoint current_time;
   // The root directory
   Entry root;
+  std::mutex mutex;
 
   explicit Impl(TimePoint current_time)
       : current_time(current_time), root(Directory("", current_time)) {}
 
+  std::unique_lock<std::mutex> lock_guard() {
+    return std::unique_lock<std::mutex>(mutex);
+  }
+
   Directory& RootDir() { return root.as_dir(); }
 
   template <typename It>
@@ -376,12 +382,14 @@ Status MockFileSystem::CreateDir(const std::string& path, bool recursive) {
   auto parts = SplitAbstractPath(path);
   RETURN_NOT_OK(ValidateAbstractPathParts(parts));
 
+  auto guard = impl_->lock_guard();
+
   size_t consumed;
   Entry* entry = impl_->FindEntry(parts, &consumed);
   if (!entry->is_dir()) {
     auto file_path = JoinAbstractPath(parts.begin(), parts.begin() + consumed);
     return Status::IOError("Cannot create directory '", path, "': ", "ancestor '",
-                           file_path, "' is a regular file");
+                           file_path, "' is not a directory");
   }
   if (!recursive && (parts.size() - consumed) > 1) {
     return Status::IOError("Cannot create directory '", path,
@@ -392,6 +400,7 @@ Status MockFileSystem::CreateDir(const std::string& path, bool recursive) {
     std::unique_ptr<Entry> child(new Entry(Directory(name, impl_->current_time)));
     Entry* child_ptr = child.get();
     bool inserted = entry->as_dir().CreateEntry(name, std::move(child));
+    // No race condition on insertion is possible, as all operations are locked
     DCHECK(inserted);
     entry = child_ptr;
   }
@@ -402,6 +411,8 @@ Status MockFileSystem::DeleteDir(const std::string& path) {
   auto parts = SplitAbstractPath(path);
   RETURN_NOT_OK(ValidateAbstractPathParts(parts));
 
+  auto guard = impl_->lock_guard();
+
   Entry* parent = impl_->FindParent(parts);
   if (parent == nullptr || !parent->is_dir()) {
     return PathNotFound(path);
@@ -424,6 +435,8 @@ Status MockFileSystem::DeleteDirContents(const std::string& path) {
   auto parts = SplitAbstractPath(path);
   RETURN_NOT_OK(ValidateAbstractPathParts(parts));
 
+  auto guard = impl_->lock_guard();
+
   if (parts.empty()) {
     // Wipe filesystem
     return internal::InvalidDeleteDirContents(path);
@@ -441,6 +454,8 @@ Status MockFileSystem::DeleteDirContents(const std::string& path) {
 }
 
 Status MockFileSystem::DeleteRootDirContents() {
+  auto guard = impl_->lock_guard();
+
   impl_->RootDir().entries.clear();
   return Status::OK();
 }
@@ -449,6 +464,8 @@ Status MockFileSystem::DeleteFile(const std::string& path) {
   auto parts = SplitAbstractPath(path);
   RETURN_NOT_OK(ValidateAbstractPathParts(parts));
 
+  auto guard = impl_->lock_guard();
+
   Entry* parent = impl_->FindParent(parts);
   if (parent == nullptr || !parent->is_dir()) {
     return PathNotFound(path);
@@ -470,6 +487,8 @@ Result<FileInfo> MockFileSystem::GetFileInfo(const std::string& path) {
   auto parts = SplitAbstractPath(path);
   RETURN_NOT_OK(ValidateAbstractPathParts(parts));
 
+  auto guard = impl_->lock_guard();
+
   FileInfo info;
   Entry* entry = impl_->FindEntry(parts);
   if (entry == nullptr) {
@@ -485,6 +504,8 @@ Result<std::vector<FileInfo>> MockFileSystem::GetFileInfo(const FileSelector& se
   auto parts = SplitAbstractPath(selector.base_dir);
   RETURN_NOT_OK(ValidateAbstractPathParts(parts));
 
+  auto guard = impl_->lock_guard();
+
   std::vector<FileInfo> results;
 
   Entry* base_dir = impl_->FindEntry(parts);
@@ -504,6 +525,8 @@ Result<std::vector<FileInfo>> MockFileSystem::GetFileInfo(const FileSelector& se
   return results;
 }
 
+namespace {
+
 // Helper for binary operations (move, copy)
 struct BinaryOp {
   std::vector<std::string> src_parts;
@@ -523,6 +546,8 @@ struct BinaryOp {
     RETURN_NOT_OK(ValidateAbstractPathParts(src_parts));
     RETURN_NOT_OK(ValidateAbstractPathParts(dest_parts));
 
+    auto guard = impl->lock_guard();
+
     // Both source and destination must have valid parents
     Entry* src_parent = impl->FindParent(src_parts);
     if (src_parent == nullptr || !src_parent->is_dir()) {
@@ -552,6 +577,8 @@ struct BinaryOp {
   }
 };
 
+}  // namespace
+
 Status MockFileSystem::Move(const std::string& src, const std::string& dest) {
   return BinaryOp::Run(impl_.get(), src, dest, [&](const BinaryOp& op) -> Status {
     if (op.src_entry == nullptr) {
@@ -609,31 +636,43 @@ Status MockFileSystem::CopyFile(const std::string& src, const std::string& dest)
 
 Result<std::shared_ptr<io::InputStream>> MockFileSystem::OpenInputStream(
     const std::string& path) {
+  auto guard = impl_->lock_guard();
+
   return impl_->OpenInputReader(path);
 }
 
 Result<std::shared_ptr<io::RandomAccessFile>> MockFileSystem::OpenInputFile(
     const std::string& path) {
+  auto guard = impl_->lock_guard();
+
   return impl_->OpenInputReader(path);
 }
 
 Result<std::shared_ptr<io::OutputStream>> MockFileSystem::OpenOutputStream(
     const std::string& path) {
+  auto guard = impl_->lock_guard();
+
   return impl_->OpenOutputStream(path, false /* append */);
 }
 
 Result<std::shared_ptr<io::OutputStream>> MockFileSystem::OpenAppendStream(
     const std::string& path) {
+  auto guard = impl_->lock_guard();
+
   return impl_->OpenOutputStream(path, true /* append */);
 }
 
 std::vector<MockDirInfo> MockFileSystem::AllDirs() {
+  auto guard = impl_->lock_guard();
+
   std::vector<MockDirInfo> result;
   impl_->DumpDirs("", impl_->RootDir(), &result);
   return result;
 }
 
 std::vector<MockFileInfo> MockFileSystem::AllFiles() {
+  auto guard = impl_->lock_guard();
+
   std::vector<MockFileInfo> result;
   impl_->DumpFiles("", impl_->RootDir(), &result);
   return result;
@@ -642,6 +681,7 @@ std::vector<MockFileInfo> MockFileSystem::AllFiles() {
 Status MockFileSystem::CreateFile(const std::string& path, const std::string& contents,
                                   bool recursive) {
   auto parent = fs::internal::GetAbstractPathParent(path).first;
+
   if (parent != "") {
     RETURN_NOT_OK(CreateDir(parent, recursive));
   }
diff --git a/cpp/src/arrow/filesystem/path_util.cc b/cpp/src/arrow/filesystem/path_util.cc
index 418ddb9e539..f1bd5c087bf 100644
--- a/cpp/src/arrow/filesystem/path_util.cc
+++ b/cpp/src/arrow/filesystem/path_util.cc
@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <algorithm>
+
 #include "arrow/filesystem/path_util.h"
 #include "arrow/result.h"
 #include "arrow/status.h"
@@ -80,7 +82,7 @@ std::string GetAbstractPathExtension(const std::string& s) {
     // Empty extension
     return "";
   }
-  return basename.substr(dot + 1).to_string();
+  return std::string(basename.substr(dot + 1));
 }
 
 Status ValidateAbstractPathParts(const std::vector<std::string>& parts) {
@@ -100,7 +102,7 @@ std::string ConcatAbstractPath(const std::string& base, const std::string& stem)
   if (base.empty()) {
     return stem;
   }
-  return EnsureTrailingSlash(base) + RemoveLeadingSlash(stem).to_string();
+  return EnsureTrailingSlash(base) + std::string(RemoveLeadingSlash(stem));
 }
 
 std::string EnsureTrailingSlash(util::string_view v) {
@@ -168,6 +170,11 @@ bool IsAncestorOf(util::string_view ancestor, util::string_view descendant) {
 
   descendant.remove_prefix(ancestor.size());
 
+  if (descendant.empty()) {
+    // "/hello" is an ancestor of "/hello"
+    return true;
+  }
+
   // "/hello/w" is not an ancestor of "/hello/world"
   return descendant.starts_with(std::string{kSep});
 }
@@ -186,7 +193,7 @@ std::vector<std::string> AncestorsFromBasePath(util::string_view base_path,
                                                util::string_view descendant) {
   std::vector<std::string> ancestry;
   if (auto relative = RemoveAncestor(base_path, descendant)) {
-    auto relative_segments = fs::internal::SplitAbstractPath(relative->to_string());
+    auto relative_segments = fs::internal::SplitAbstractPath(std::string(*relative));
 
     // the last segment indicates descendant
     relative_segments.pop_back();
@@ -198,13 +205,36 @@ std::vector<std::string> AncestorsFromBasePath(util::string_view base_path,
 
     for (auto&& relative_segment : relative_segments) {
       ancestry.push_back(JoinAbstractPath(
-          std::vector<std::string>{base_path.to_string(), std::move(relative_segment)}));
+          std::vector<std::string>{std::string(base_path), std::move(relative_segment)}));
       base_path = ancestry.back();
     }
   }
   return ancestry;
 }
 
+std::vector<std::string> MinimalCreateDirSet(std::vector<std::string> dirs) {
+  std::sort(dirs.begin(), dirs.end());
+
+  for (auto ancestor = dirs.begin(); ancestor != dirs.end(); ++ancestor) {
+    auto descendant = ancestor;
+    auto descendants_end = descendant + 1;
+
+    while (descendants_end != dirs.end() && IsAncestorOf(*descendant, *descendants_end)) {
+      ++descendant;
+      ++descendants_end;
+    }
+
+    ancestor = dirs.erase(ancestor, descendants_end - 1);
+  }
+
+  // the root directory need not be created
+  if (dirs.size() == 1 && IsAncestorOf(dirs[0], "")) {
+    return {};
+  }
+
+  return dirs;
+}
+
 std::string ToBackslashes(util::string_view v) {
   std::string s(v);
   for (auto& c : s) {
diff --git a/cpp/src/arrow/filesystem/path_util.h b/cpp/src/arrow/filesystem/path_util.h
index 46fa4151fb8..5701c11b5d8 100644
--- a/cpp/src/arrow/filesystem/path_util.h
+++ b/cpp/src/arrow/filesystem/path_util.h
@@ -87,6 +87,12 @@ ARROW_EXPORT
 std::vector<std::string> AncestorsFromBasePath(util::string_view base_path,
                                                util::string_view descendant);
 
+/// Given a vector of paths of directories which must be created, produce a the minimal
+/// subset for passing to CreateDir(recursive=true) by removing redundant parent
+/// directories
+ARROW_EXPORT
+std::vector<std::string> MinimalCreateDirSet(std::vector<std::string> dirs);
+
 // Join the components of an abstract path.
 template <class StringIt>
 std::string JoinAbstractPath(StringIt it, StringIt end) {
diff --git a/cpp/src/arrow/filesystem/s3_test_util.h b/cpp/src/arrow/filesystem/s3_test_util.h
index c6376ca0596..2bb614b9b80 100644
--- a/cpp/src/arrow/filesystem/s3_test_util.h
+++ b/cpp/src/arrow/filesystem/s3_test_util.h
@@ -22,12 +22,9 @@
 #include <string>
 #include <utility>
 
-// boost/process/detail/windows/handle_workaround.hpp doesn't work
-// without BOOST_USE_WINDOWS_H with MinGW because MinGW doesn't
-// provide __kernel_entry without winternl.h.
-//
-// See also:
-// https://github.com/boostorg/process/blob/develop/include/boost/process/detail/windows/handle_workaround.hpp
+// We need BOOST_USE_WINDOWS_H definition with MinGW when we use
+// boost/process.hpp. See ARROW_BOOST_PROCESS_COMPILE_DEFINITIONS in
+// cpp/cmake_modules/BuildUtils.cmake for details.
 #include <boost/process.hpp>
 
 #include <gtest/gtest.h>
diff --git a/cpp/src/arrow/filesystem/s3fs.cc b/cpp/src/arrow/filesystem/s3fs.cc
index 77cc6424faf..3cde8edd909 100644
--- a/cpp/src/arrow/filesystem/s3fs.cc
+++ b/cpp/src/arrow/filesystem/s3fs.cc
@@ -36,11 +36,14 @@
 #endif
 
 #include <aws/core/Aws.h>
+#include <aws/core/Region.h>
 #include <aws/core/auth/AWSCredentials.h>
 #include <aws/core/auth/AWSCredentialsProviderChain.h>
 #include <aws/core/client/RetryStrategy.h>
+#include <aws/core/http/HttpResponse.h>
 #include <aws/core/utils/logging/ConsoleLogSystem.h>
 #include <aws/core/utils/stream/PreallocatedStreamBuf.h>
+#include <aws/identity-management/auth/STSAssumeRoleCredentialsProvider.h>
 #include <aws/s3/S3Client.h>
 #include <aws/s3/model/AbortMultipartUploadRequest.h>
 #include <aws/s3/model/CompleteMultipartUploadRequest.h>
@@ -70,6 +73,7 @@
 #include "arrow/io/util_internal.h"
 #include "arrow/result.h"
 #include "arrow/status.h"
+#include "arrow/util/atomic_shared_ptr.h"
 #include "arrow/util/checked_cast.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/windows_fixup.h"
@@ -95,8 +99,6 @@ using internal::OutcomeToStatus;
 using internal::ToAwsString;
 using internal::ToURLEncodedAwsString;
 
-const char* kS3DefaultRegion = "us-east-1";
-
 static const char kSep = '/';
 
 namespace {
@@ -160,6 +162,9 @@ Status EnsureS3Initialized() {
   return Status::OK();
 }
 
+// -----------------------------------------------------------------------
+// S3Options implementation
+
 void S3Options::ConfigureDefaultCredentials() {
   credentials_provider =
       std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>();
@@ -170,9 +175,19 @@ void S3Options::ConfigureAnonymousCredentials() {
 }
 
 void S3Options::ConfigureAccessKey(const std::string& access_key,
-                                   const std::string& secret_key) {
+                                   const std::string& secret_key,
+                                   const std::string& session_token) {
   credentials_provider = std::make_shared<Aws::Auth::SimpleAWSCredentialsProvider>(
-      ToAwsString(access_key), ToAwsString(secret_key));
+      ToAwsString(access_key), ToAwsString(secret_key), ToAwsString(session_token));
+}
+
+void S3Options::ConfigureAssumeRoleCredentials(
+    const std::string& role_arn, const std::string& session_name,
+    const std::string& external_id, int load_frequency,
+    const std::shared_ptr<Aws::STS::STSClient>& stsClient) {
+  credentials_provider = std::make_shared<Aws::Auth::STSAssumeRoleCredentialsProvider>(
+      ToAwsString(role_arn), ToAwsString(session_name), ToAwsString(external_id),
+      load_frequency, stsClient);
 }
 
 std::string S3Options::GetAccessKey() const {
@@ -185,6 +200,11 @@ std::string S3Options::GetSecretKey() const {
   return std::string(FromAwsString(credentials.GetAWSSecretKey()));
 }
 
+std::string S3Options::GetSessionToken() const {
+  auto credentials = credentials_provider->GetAWSCredentials();
+  return std::string(FromAwsString(credentials.GetSessionToken()));
+}
+
 S3Options S3Options::Defaults() {
   S3Options options;
   options.ConfigureDefaultCredentials();
@@ -198,9 +218,24 @@ S3Options S3Options::Anonymous() {
 }
 
 S3Options S3Options::FromAccessKey(const std::string& access_key,
-                                   const std::string& secret_key) {
+                                   const std::string& secret_key,
+                                   const std::string& session_token) {
   S3Options options;
-  options.ConfigureAccessKey(access_key, secret_key);
+  options.ConfigureAccessKey(access_key, secret_key, session_token);
+  return options;
+}
+
+S3Options S3Options::FromAssumeRole(
+    const std::string& role_arn, const std::string& session_name,
+    const std::string& external_id, int load_frequency,
+    const std::shared_ptr<Aws::STS::STSClient>& stsClient) {
+  S3Options options;
+  options.role_arn = role_arn;
+  options.session_name = session_name;
+  options.external_id = external_id;
+  options.load_frequency = load_frequency;
+  options.ConfigureAssumeRoleCredentials(role_arn, session_name, external_id,
+                                         load_frequency, stsClient);
   return options;
 }
 
@@ -240,17 +275,23 @@ Result<S3Options> S3Options::FromUri(const Uri& uri, std::string* out_path) {
     options.ConfigureDefaultCredentials();
   }
 
-  auto it = options_map.find("region");
-  if (it != options_map.end()) {
-    options.region = it->second;
-  }
-  it = options_map.find("scheme");
-  if (it != options_map.end()) {
-    options.scheme = it->second;
+  bool region_set = false;
+  for (const auto& kv : options_map) {
+    if (kv.first == "region") {
+      options.region = kv.second;
+      region_set = true;
+    } else if (kv.first == "scheme") {
+      options.scheme = kv.second;
+    } else if (kv.first == "endpoint_override") {
+      options.endpoint_override = kv.second;
+    } else {
+      return Status::Invalid("Unexpected query parameter in S3 URI: '", kv.first, "'");
+    }
   }
-  it = options_map.find("endpoint_override");
-  if (it != options_map.end()) {
-    options.endpoint_override = it->second;
+
+  if (!region_set && !bucket.empty() && options.endpoint_override.empty()) {
+    // XXX Should we use a dedicated resolver with the given credentials?
+    ARROW_ASSIGN_OR_RAISE(options.region, ResolveBucketRegion(bucket));
   }
 
   return options;
@@ -267,7 +308,8 @@ bool S3Options::Equals(const S3Options& other) const {
   return (region == other.region && endpoint_override == other.endpoint_override &&
           scheme == other.scheme && background_writes == other.background_writes &&
           GetAccessKey() == other.GetAccessKey() &&
-          GetSecretKey() == other.GetSecretKey());
+          GetSecretKey() == other.GetSecretKey() &&
+          GetSessionToken() == other.GetSessionToken());
 }
 
 namespace {
@@ -372,6 +414,183 @@ std::string FormatRange(int64_t start, int64_t length) {
   return ss.str();
 }
 
+class S3Client : public Aws::S3::S3Client {
+ public:
+  using Aws::S3::S3Client::S3Client;
+
+  // To get a bucket's region, we must extract the "x-amz-bucket-region" header
+  // from the response to a HEAD bucket request.
+  // Unfortunately, the S3Client APIs don't let us access the headers of successful
+  // responses.  So we have to cook a AWS request and issue it ourselves.
+
+  Result<std::string> GetBucketRegion(const S3Model::HeadBucketRequest& request) {
+    auto uri = GeneratePresignedUrl(request.GetBucket(),
+                                    /*key=*/"", Aws::Http::HttpMethod::HTTP_HEAD);
+    // NOTE: The signer region argument isn't passed here, as there's no easy
+    // way of computing it (the relevant method is private).
+    auto outcome = MakeRequest(uri, request, Aws::Http::HttpMethod::HTTP_HEAD,
+                               Aws::Auth::SIGV4_SIGNER);
+    const auto code = outcome.IsSuccess() ? outcome.GetResult().GetResponseCode()
+                                          : outcome.GetError().GetResponseCode();
+    const auto& headers = outcome.IsSuccess()
+                              ? outcome.GetResult().GetHeaderValueCollection()
+                              : outcome.GetError().GetResponseHeaders();
+
+    const auto it = headers.find(ToAwsString("x-amz-bucket-region"));
+    if (it == headers.end()) {
+      if (code == Aws::Http::HttpResponseCode::NOT_FOUND) {
+        return Status::IOError("Bucket '", request.GetBucket(), "' not found");
+      } else if (!outcome.IsSuccess()) {
+        return ErrorToStatus(std::forward_as_tuple("When resolving region for bucket '",
+                                                   request.GetBucket(), "': "),
+                             outcome.GetError());
+      } else {
+        return Status::IOError("When resolving region for bucket '", request.GetBucket(),
+                               "': missing 'x-amz-bucket-region' header in response");
+      }
+    }
+    return std::string(FromAwsString(it->second));
+  }
+
+  Result<std::string> GetBucketRegion(const std::string& bucket) {
+    S3Model::HeadBucketRequest req;
+    req.SetBucket(ToAwsString(bucket));
+    return GetBucketRegion(req);
+  }
+};
+
+// In AWS SDK < 1.8, Aws::Client::ClientConfiguration::followRedirects is a bool.
+template <bool Never = false>
+void DisableRedirectsImpl(bool* followRedirects) {
+  *followRedirects = false;
+}
+
+// In AWS SDK >= 1.8, it's a Aws::Client::FollowRedirectsPolicy scoped enum.
+template <typename PolicyEnum, PolicyEnum Never = PolicyEnum::NEVER>
+void DisableRedirectsImpl(PolicyEnum* followRedirects) {
+  *followRedirects = Never;
+}
+
+void DisableRedirects(Aws::Client::ClientConfiguration* c) {
+  DisableRedirectsImpl(&c->followRedirects);
+}
+
+class ClientBuilder {
+ public:
+  explicit ClientBuilder(S3Options options) : options_(std::move(options)) {}
+
+  const Aws::Client::ClientConfiguration& config() const { return client_config_; }
+
+  Aws::Client::ClientConfiguration* mutable_config() { return &client_config_; }
+
+  Result<std::unique_ptr<S3Client>> BuildClient() {
+    credentials_provider_ = options_.credentials_provider;
+    if (!options_.region.empty()) {
+      client_config_.region = ToAwsString(options_.region);
+    }
+    client_config_.endpointOverride = ToAwsString(options_.endpoint_override);
+    if (options_.scheme == "http") {
+      client_config_.scheme = Aws::Http::Scheme::HTTP;
+    } else if (options_.scheme == "https") {
+      client_config_.scheme = Aws::Http::Scheme::HTTPS;
+    } else {
+      return Status::Invalid("Invalid S3 connection scheme '", options_.scheme, "'");
+    }
+    client_config_.retryStrategy = std::make_shared<ConnectRetryStrategy>();
+    if (!internal::global_options.tls_ca_file_path.empty()) {
+      client_config_.caFile = ToAwsString(internal::global_options.tls_ca_file_path);
+    }
+    if (!internal::global_options.tls_ca_dir_path.empty()) {
+      client_config_.caPath = ToAwsString(internal::global_options.tls_ca_dir_path);
+    }
+
+    const bool use_virtual_addressing = options_.endpoint_override.empty();
+    return std::unique_ptr<S3Client>(
+        new S3Client(credentials_provider_, client_config_,
+                     Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
+                     use_virtual_addressing));
+  }
+
+  const S3Options& options() const { return options_; }
+
+ protected:
+  S3Options options_;
+  Aws::Client::ClientConfiguration client_config_;
+  std::shared_ptr<Aws::Auth::AWSCredentialsProvider> credentials_provider_;
+};
+
+// -----------------------------------------------------------------------
+// S3 region resolver
+
+class RegionResolver {
+ public:
+  static Result<std::shared_ptr<RegionResolver>> Make(S3Options options) {
+    std::shared_ptr<RegionResolver> resolver(new RegionResolver(std::move(options)));
+    RETURN_NOT_OK(resolver->Init());
+    return resolver;
+  }
+
+  static Result<std::shared_ptr<RegionResolver>> DefaultInstance() {
+    static std::shared_ptr<RegionResolver> instance;
+    auto resolver = arrow::internal::atomic_load(&instance);
+    if (resolver) {
+      return resolver;
+    }
+    auto maybe_resolver = Make(S3Options::Anonymous());
+    if (!maybe_resolver.ok()) {
+      return maybe_resolver;
+    }
+    // Make sure to always return the same instance even if several threads
+    // call DefaultInstance at once.
+    std::shared_ptr<RegionResolver> existing;
+    if (arrow::internal::atomic_compare_exchange_strong(&instance, &existing,
+                                                        *maybe_resolver)) {
+      return *maybe_resolver;
+    } else {
+      return existing;
+    }
+  }
+
+  Result<std::string> ResolveRegion(const std::string& bucket) {
+    std::unique_lock<std::mutex> lock(cache_mutex_);
+    auto it = cache_.find(bucket);
+    if (it != cache_.end()) {
+      return it->second;
+    }
+    lock.unlock();
+    ARROW_ASSIGN_OR_RAISE(auto region, ResolveRegionUncached(bucket));
+    lock.lock();
+    // Note we don't cache a non-existent bucket, as the bucket could be created later
+    cache_[bucket] = region;
+    return region;
+  }
+
+  Result<std::string> ResolveRegionUncached(const std::string& bucket) {
+    return client_->GetBucketRegion(bucket);
+  }
+
+ protected:
+  explicit RegionResolver(S3Options options) : builder_(std::move(options)) {}
+
+  Status Init() {
+    DCHECK(builder_.options().endpoint_override.empty());
+    // On Windows with AWS SDK >= 1.8, it is necessary to disable redirects (ARROW-10085).
+    DisableRedirects(builder_.mutable_config());
+    return builder_.BuildClient().Value(&client_);
+  }
+
+  ClientBuilder builder_;
+  std::unique_ptr<S3Client> client_;
+
+  std::mutex cache_mutex_;
+  // XXX Should cache size be bounded?  It must be quite unusual to query millions
+  // of different buckets in a single program invocation...
+  std::unordered_map<std::string, std::string> cache_;
+};
+
+// -----------------------------------------------------------------------
+// S3 file stream implementations
+
 // A non-copying iostream.
 // See https://stackoverflow.com/questions/35322033/aws-c-sdk-uploadpart-times-out
 // https://stackoverflow.com/questions/13059091/creating-an-input-stream-from-constant-memory
@@ -404,13 +623,11 @@ Result<S3Model::GetObjectResult> GetObjectRange(Aws::S3::S3Client* client,
 }
 
 // A RandomAccessFile that reads from a S3 object
-class ObjectInputFile : public io::RandomAccessFile {
+class ObjectInputFile final : public io::RandomAccessFile {
  public:
-  ObjectInputFile(Aws::S3::S3Client* client, const S3Path& path)
-      : client_(client), path_(path) {}
-
-  ObjectInputFile(Aws::S3::S3Client* client, const S3Path& path, int64_t size)
-      : client_(client), path_(path), content_length_(size) {}
+  ObjectInputFile(std::shared_ptr<FileSystem> fs, Aws::S3::S3Client* client,
+                  const S3Path& path, int64_t size = kNoSize)
+      : fs_(std::move(fs)), client_(client), path_(path), content_length_(size) {}
 
   Status Init() {
     // Issue a HEAD Object to get the content-length and ensure any
@@ -460,6 +677,8 @@ class ObjectInputFile : public io::RandomAccessFile {
   // RandomAccessFile APIs
 
   Status Close() override {
+    fs_.reset();
+    client_ = nullptr;
     closed_ = true;
     return Status::OK();
   }
@@ -534,6 +753,7 @@ class ObjectInputFile : public io::RandomAccessFile {
   }
 
  protected:
+  std::shared_ptr<FileSystem> fs_;  // Owner of S3Client
   Aws::S3::S3Client* client_;
   S3Path path_;
   bool closed_ = false;
@@ -548,14 +768,14 @@ class ObjectInputFile : public io::RandomAccessFile {
 static constexpr int64_t kMinimumPartUpload = 5 * 1024 * 1024;
 
 // An OutputStream that writes to a S3 object
-class ObjectOutputStream : public io::OutputStream {
+class ObjectOutputStream final : public io::OutputStream {
  protected:
   struct UploadState;
 
  public:
-  ObjectOutputStream(Aws::S3::S3Client* client, const S3Path& path,
-                     const S3Options& options)
-      : client_(client), path_(path), options_(options) {}
+  ObjectOutputStream(std::shared_ptr<FileSystem> fs, Aws::S3::S3Client* client,
+                     const S3Path& path, const S3Options& options)
+      : fs_(std::move(fs)), client_(client), path_(path), options_(options) {}
 
   ~ObjectOutputStream() override {
     // For compliance with the rest of the IO stack, Close rather than Abort,
@@ -600,6 +820,8 @@ class ObjectOutputStream : public io::OutputStream {
           outcome.GetError());
     }
     current_part_.reset();
+    fs_.reset();
+    client_ = nullptr;
     closed_ = true;
     return Status::OK();
   }
@@ -645,6 +867,8 @@ class ObjectOutputStream : public io::OutputStream {
           outcome.GetError());
     }
 
+    fs_.reset();
+    client_ = nullptr;
     closed_ = true;
     return Status::OK();
   }
@@ -817,6 +1041,7 @@ class ObjectOutputStream : public io::OutputStream {
   }
 
  protected:
+  std::shared_ptr<FileSystem> fs_;  // Owner of S3Client
   Aws::S3::S3Client* client_;
   S3Path path_;
   const S3Options& options_;
@@ -857,11 +1082,12 @@ void FileObjectToInfo(const S3Model::Object& obj, FileInfo* info) {
 
 }  // namespace
 
+// -----------------------------------------------------------------------
+// S3 filesystem implementation
+
 class S3FileSystem::Impl {
  public:
-  S3Options options_;
-  Aws::Client::ClientConfiguration client_config_;
-  Aws::Auth::AWSCredentials credentials_;
+  ClientBuilder builder_;
   std::unique_ptr<Aws::S3::S3Client> client_;
 
   const int32_t kListObjectsMaxKeys = 1000;
@@ -870,36 +1096,15 @@ class S3FileSystem::Impl {
   // Limit recursing depth, since a recursion bomb can be created
   const int32_t kMaxNestingDepth = 100;
 
-  explicit Impl(S3Options options) : options_(std::move(options)) {}
+  explicit Impl(S3Options options) : builder_(std::move(options)) {}
 
-  Status Init() {
-    credentials_ = options_.credentials_provider->GetAWSCredentials();
-    client_config_.region = ToAwsString(options_.region);
-    client_config_.endpointOverride = ToAwsString(options_.endpoint_override);
-    if (options_.scheme == "http") {
-      client_config_.scheme = Aws::Http::Scheme::HTTP;
-    } else if (options_.scheme == "https") {
-      client_config_.scheme = Aws::Http::Scheme::HTTPS;
-    } else {
-      return Status::Invalid("Invalid S3 connection scheme '", options_.scheme, "'");
-    }
-    client_config_.retryStrategy = std::make_shared<ConnectRetryStrategy>();
-    if (!internal::global_options.tls_ca_file_path.empty()) {
-      client_config_.caFile = ToAwsString(internal::global_options.tls_ca_file_path);
-    }
-    if (!internal::global_options.tls_ca_dir_path.empty()) {
-      client_config_.caPath = ToAwsString(internal::global_options.tls_ca_dir_path);
-    }
+  Status Init() { return builder_.BuildClient().Value(&client_); }
 
-    bool use_virtual_addressing = options_.endpoint_override.empty();
-    client_.reset(
-        new Aws::S3::S3Client(credentials_, client_config_,
-                              Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
-                              use_virtual_addressing));
-    return Status::OK();
-  }
+  const S3Options& options() const { return builder_.options(); }
 
-  S3Options options() const { return options_; }
+  std::string region() const {
+    return std::string(FromAwsString(builder_.config().region));
+  }
 
   // Create a bucket.  Successful if bucket already exists.
   Status CreateBucket(const std::string& bucket) {
@@ -907,7 +1112,7 @@ class S3FileSystem::Impl {
     S3Model::CreateBucketRequest req;
     config.SetLocationConstraint(
         S3Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName(
-            ToAwsString(options_.region)));
+            ToAwsString(options().region)));
     req.SetBucket(ToAwsString(bucket));
     req.SetCreateBucketConfiguration(config);
 
@@ -1241,6 +1446,35 @@ class S3FileSystem::Impl {
     }
     return Status::OK();
   }
+
+  Result<std::shared_ptr<ObjectInputFile>> OpenInputFile(const std::string& s,
+                                                         S3FileSystem* fs) {
+    ARROW_ASSIGN_OR_RAISE(auto path, S3Path::FromString(s));
+    RETURN_NOT_OK(ValidateFilePath(path));
+
+    auto ptr =
+        std::make_shared<ObjectInputFile>(fs->shared_from_this(), client_.get(), path);
+    RETURN_NOT_OK(ptr->Init());
+    return ptr;
+  }
+
+  Result<std::shared_ptr<ObjectInputFile>> OpenInputFile(const FileInfo& info,
+                                                         S3FileSystem* fs) {
+    if (info.type() == FileType::NotFound) {
+      return ::arrow::fs::internal::PathNotFound(info.path());
+    }
+    if (info.type() != FileType::File && info.type() != FileType::Unknown) {
+      return ::arrow::fs::internal::NotAFile(info.path());
+    }
+
+    ARROW_ASSIGN_OR_RAISE(auto path, S3Path::FromString(info.path()));
+    RETURN_NOT_OK(ValidateFilePath(path));
+
+    auto ptr = std::make_shared<ObjectInputFile>(fs->shared_from_this(), client_.get(),
+                                                 path, info.size());
+    RETURN_NOT_OK(ptr->Init());
+    return ptr;
+  }
 };
 
 S3FileSystem::S3FileSystem(const S3Options& options) : impl_(new Impl{options}) {}
@@ -1268,6 +1502,8 @@ bool S3FileSystem::Equals(const FileSystem& other) const {
 
 S3Options S3FileSystem::options() const { return impl_->options(); }
 
+std::string S3FileSystem::region() const { return impl_->region(); }
+
 Result<FileInfo> S3FileSystem::GetFileInfo(const std::string& s) {
   ARROW_ASSIGN_OR_RAISE(auto path, S3Path::FromString(s));
   FileInfo info;
@@ -1496,56 +1732,22 @@ Status S3FileSystem::CopyFile(const std::string& src, const std::string& dest) {
 
 Result<std::shared_ptr<io::InputStream>> S3FileSystem::OpenInputStream(
     const std::string& s) {
-  ARROW_ASSIGN_OR_RAISE(auto path, S3Path::FromString(s));
-  RETURN_NOT_OK(ValidateFilePath(path));
-
-  auto ptr = std::make_shared<ObjectInputFile>(impl_->client_.get(), path);
-  RETURN_NOT_OK(ptr->Init());
-  return ptr;
+  return impl_->OpenInputFile(s, this);
 }
 
 Result<std::shared_ptr<io::InputStream>> S3FileSystem::OpenInputStream(
     const FileInfo& info) {
-  if (info.type() == FileType::NotFound) {
-    return ::arrow::fs::internal::PathNotFound(info.path());
-  }
-  if (info.type() != FileType::File && info.type() != FileType::Unknown) {
-    return ::arrow::fs::internal::NotAFile(info.path());
-  }
-
-  ARROW_ASSIGN_OR_RAISE(auto path, S3Path::FromString(info.path()));
-  RETURN_NOT_OK(ValidateFilePath(path));
-
-  auto ptr = std::make_shared<ObjectInputFile>(impl_->client_.get(), path, info.size());
-  RETURN_NOT_OK(ptr->Init());
-  return ptr;
+  return impl_->OpenInputFile(info, this);
 }
 
 Result<std::shared_ptr<io::RandomAccessFile>> S3FileSystem::OpenInputFile(
     const std::string& s) {
-  ARROW_ASSIGN_OR_RAISE(auto path, S3Path::FromString(s));
-  RETURN_NOT_OK(ValidateFilePath(path));
-
-  auto ptr = std::make_shared<ObjectInputFile>(impl_->client_.get(), path);
-  RETURN_NOT_OK(ptr->Init());
-  return ptr;
+  return impl_->OpenInputFile(s, this);
 }
 
 Result<std::shared_ptr<io::RandomAccessFile>> S3FileSystem::OpenInputFile(
     const FileInfo& info) {
-  if (info.type() == FileType::NotFound) {
-    return ::arrow::fs::internal::PathNotFound(info.path());
-  }
-  if (info.type() != FileType::File && info.type() != FileType::Unknown) {
-    return ::arrow::fs::internal::NotAFile(info.path());
-  }
-
-  ARROW_ASSIGN_OR_RAISE(auto path, S3Path::FromString(info.path()));
-  RETURN_NOT_OK(ValidateFilePath(path));
-
-  auto ptr = std::make_shared<ObjectInputFile>(impl_->client_.get(), path, info.size());
-  RETURN_NOT_OK(ptr->Init());
-  return ptr;
+  return impl_->OpenInputFile(info, this);
 }
 
 Result<std::shared_ptr<io::OutputStream>> S3FileSystem::OpenOutputStream(
@@ -1553,8 +1755,8 @@ Result<std::shared_ptr<io::OutputStream>> S3FileSystem::OpenOutputStream(
   ARROW_ASSIGN_OR_RAISE(auto path, S3Path::FromString(s));
   RETURN_NOT_OK(ValidateFilePath(path));
 
-  auto ptr =
-      std::make_shared<ObjectOutputStream>(impl_->client_.get(), path, impl_->options_);
+  auto ptr = std::make_shared<ObjectOutputStream>(
+      shared_from_this(), impl_->client_.get(), path, impl_->options());
   RETURN_NOT_OK(ptr->Init());
   return ptr;
 }
@@ -1567,5 +1769,14 @@ Result<std::shared_ptr<io::OutputStream>> S3FileSystem::OpenAppendStream(
   return Status::NotImplemented("It is not possible to append efficiently to S3 objects");
 }
 
+//
+// Top-level utility functions
+//
+
+Result<std::string> ResolveBucketRegion(const std::string& bucket) {
+  ARROW_ASSIGN_OR_RAISE(auto resolver, RegionResolver::DefaultInstance());
+  return resolver->ResolveRegion(bucket);
+}
+
 }  // namespace fs
 }  // namespace arrow
diff --git a/cpp/src/arrow/filesystem/s3fs.h b/cpp/src/arrow/filesystem/s3fs.h
index 6bf29938738..bd8f1fadbb9 100644
--- a/cpp/src/arrow/filesystem/s3fs.h
+++ b/cpp/src/arrow/filesystem/s3fs.h
@@ -29,19 +29,27 @@ namespace Aws {
 namespace Auth {
 
 class AWSCredentialsProvider;
+class STSAssumeRoleCredentialsProvider;
 
 }  // namespace Auth
+namespace STS {
+class STSClient;
+}
 }  // namespace Aws
 
 namespace arrow {
 namespace fs {
 
-extern ARROW_EXPORT const char* kS3DefaultRegion;
-
 /// Options for the S3FileSystem implementation.
 struct ARROW_EXPORT S3Options {
-  /// AWS region to connect to (default "us-east-1")
-  std::string region = kS3DefaultRegion;
+  /// AWS region to connect to.
+  ///
+  /// If unset, the AWS SDK will choose a default value.  The exact algorithm
+  /// depends on the SDK version.  Before 1.8, the default is hardcoded
+  /// to "us-east-1".  Since 1.8, several heuristics are used to determine
+  /// the region (environment variables, configuration profile, EC2 metadata
+  /// server).
+  std::string region;
 
   /// If non-empty, override region with a connect string such as "localhost:9000"
   // XXX perhaps instead take a URL like "http://localhost:9000"?
@@ -49,6 +57,15 @@ struct ARROW_EXPORT S3Options {
   /// S3 connection transport, default "https"
   std::string scheme = "https";
 
+  /// ARN of role to assume
+  std::string role_arn;
+  /// Optional identifier for an assumed role session.
+  std::string session_name;
+  /// Optional external idenitifer to pass to STS when assuming a role
+  std::string external_id;
+  /// Frequency (in seconds) to refresh temporary credentials from assumed role
+  int load_frequency;
+
   /// AWS credentials provider
   std::shared_ptr<Aws::Auth::AWSCredentialsProvider> credentials_provider;
 
@@ -62,10 +79,18 @@ struct ARROW_EXPORT S3Options {
   void ConfigureAnonymousCredentials();
 
   /// Configure with explicit access and secret key.
-  void ConfigureAccessKey(const std::string& access_key, const std::string& secret_key);
+  void ConfigureAccessKey(const std::string& access_key, const std::string& secret_key,
+                          const std::string& session_token = "");
+
+  /// Configure with credentials from an assumed role.
+  void ConfigureAssumeRoleCredentials(
+      const std::string& role_arn, const std::string& session_name = "",
+      const std::string& external_id = "", int load_frequency = 900,
+      const std::shared_ptr<Aws::STS::STSClient>& stsClient = NULLPTR);
 
   std::string GetAccessKey() const;
   std::string GetSecretKey() const;
+  std::string GetSessionToken() const;
 
   bool Equals(const S3Options& other) const;
 
@@ -74,13 +99,25 @@ struct ARROW_EXPORT S3Options {
   /// This is recommended if you use the standard AWS environment variables
   /// and/or configuration file.
   static S3Options Defaults();
+
   /// \brief Initialize with anonymous credentials.
   ///
   /// This will only let you access public buckets.
   static S3Options Anonymous();
-  /// \brief Initialize with explicit access and secret key
+
+  /// \brief Initialize with explicit access and secret key.
+  ///
+  /// Optionally, a session token may also be provided for temporary credentials
+  /// (from STS).
   static S3Options FromAccessKey(const std::string& access_key,
-                                 const std::string& secret_key);
+                                 const std::string& secret_key,
+                                 const std::string& session_token = "");
+
+  /// \brief Initialize from an assumed role.
+  static S3Options FromAssumeRole(
+      const std::string& role_arn, const std::string& session_name = "",
+      const std::string& external_id = "", int load_frequency = 900,
+      const std::shared_ptr<Aws::STS::STSClient>& stsClient = NULLPTR);
 
   static Result<S3Options> FromUri(const ::arrow::internal::Uri& uri,
                                    std::string* out_path = NULLPTR);
@@ -98,7 +135,11 @@ class ARROW_EXPORT S3FileSystem : public FileSystem {
   ~S3FileSystem() override;
 
   std::string type_name() const override { return "s3"; }
+
+  /// Return the original S3 options when constructing the filesystem
   S3Options options() const;
+  /// Return the actual region this filesystem connects to
+  std::string region() const;
 
   bool Equals(const FileSystem& other) const override;
 
@@ -187,5 +228,8 @@ Status EnsureS3Initialized();
 ARROW_EXPORT
 Status FinalizeS3();
 
+ARROW_EXPORT
+Result<std::string> ResolveBucketRegion(const std::string& bucket);
+
 }  // namespace fs
 }  // namespace arrow
diff --git a/cpp/src/arrow/filesystem/s3fs_narrative_test.cc b/cpp/src/arrow/filesystem/s3fs_narrative_test.cc
index b47df2f4966..0beb51d70e0 100644
--- a/cpp/src/arrow/filesystem/s3fs_narrative_test.cc
+++ b/cpp/src/arrow/filesystem/s3fs_narrative_test.cc
@@ -43,7 +43,7 @@ DEFINE_string(access_key, "", "S3 access key");
 DEFINE_string(secret_key, "", "S3 secret key");
 
 DEFINE_string(bucket, "", "bucket name");
-DEFINE_string(region, arrow::fs::kS3DefaultRegion, "AWS region");
+DEFINE_string(region, "", "AWS region");
 DEFINE_string(endpoint, "", "Endpoint override (e.g. '127.0.0.1:9000')");
 DEFINE_string(scheme, "https", "Connection scheme");
 
@@ -87,7 +87,7 @@ void PrintError(const std::string& context_msg, const Result<T>& result) {
 void ClearBucket(int argc, char** argv) {
   auto fs = MakeFileSystem();
 
-  ASSERT_OK(fs->DeleteDirContents(""));
+  ASSERT_OK(fs->DeleteRootDirContents());
 }
 
 void TestBucket(int argc, char** argv) {
@@ -201,6 +201,10 @@ void TestMain(int argc, char** argv) {
                           : (FLAGS_verbose ? S3LogLevel::Warn : S3LogLevel::Fatal);
   ASSERT_OK(InitializeS3(options));
 
+  if (FLAGS_region.empty()) {
+    ASSERT_OK_AND_ASSIGN(FLAGS_region, ResolveBucketRegion(FLAGS_bucket));
+  }
+
   if (FLAGS_clear) {
     ClearBucket(argc, argv);
   } else if (FLAGS_test) {
diff --git a/cpp/src/arrow/filesystem/s3fs_test.cc b/cpp/src/arrow/filesystem/s3fs_test.cc
index d71e1bb6b65..99e6b3f7dfd 100644
--- a/cpp/src/arrow/filesystem/s3fs_test.cc
+++ b/cpp/src/arrow/filesystem/s3fs_test.cc
@@ -22,14 +22,23 @@
 #include <utility>
 #include <vector>
 
-// boost/process/detail/windows/handle_workaround.hpp doesn't work
-// without BOOST_USE_WINDOWS_H with MinGW because MinGW doesn't
-// provide __kernel_entry without winternl.h.
+// This boost/asio/io_context.hpp include is needless for no MinGW
+// build.
 //
-// See also:
-// https://github.com/boostorg/process/blob/develop/include/boost/process/detail/windows/handle_workaround.hpp
+// This is for including boost/asio/detail/socket_types.hpp before any
+// "#include <windows.h>". boost/asio/detail/socket_types.hpp doesn't
+// work if windows.h is already included. boost/process.h ->
+// boost/process/args.hpp -> boost/process/detail/basic_cmd.hpp
+// includes windows.h. boost/process/args.hpp is included before
+// boost/process/async.h that includes
+// boost/asio/detail/socket_types.hpp implicitly is included.
+#include <boost/asio/io_context.hpp>
+// We need BOOST_USE_WINDOWS_H definition with MinGW when we use
+// boost/process.hpp. See ARROW_BOOST_PROCESS_COMPILE_DEFINITIONS in
+// cpp/cmake_modules/BuildUtils.cmake for details.
 #include <boost/process.hpp>
 
+#include <gmock/gmock-matchers.h>
 #include <gtest/gtest.h>
 
 #ifdef _WIN32
@@ -43,6 +52,7 @@
 #endif
 
 #include <aws/core/Aws.h>
+#include <aws/core/Version.h>
 #include <aws/core/auth/AWSCredentials.h>
 #include <aws/core/auth/AWSCredentialsProvider.h>
 #include <aws/core/client/RetryStrategy.h>
@@ -51,6 +61,7 @@
 #include <aws/s3/model/CreateBucketRequest.h>
 #include <aws/s3/model/GetObjectRequest.h>
 #include <aws/s3/model/PutObjectRequest.h>
+#include <aws/sts/STSClient.h>
 
 #include "arrow/filesystem/filesystem.h"
 #include "arrow/filesystem/s3_internal.h"
@@ -61,12 +72,15 @@
 #include "arrow/status.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/testing/util.h"
+#include "arrow/util/checked_cast.h"
+#include "arrow/util/io_util.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
 
 namespace arrow {
 namespace fs {
 
+using ::arrow::internal::checked_pointer_cast;
 using ::arrow::internal::PlatformFilename;
 using ::arrow::internal::UriEscape;
 
@@ -101,27 +115,65 @@ namespace bp = boost::process;
   ARROW_AWS_ASSIGN_OR_FAIL_IMPL(             \
       ARROW_AWS_ASSIGN_OR_FAIL_NAME(_aws_error_or_value, __COUNTER__), lhs, rexpr);
 
-class S3TestMixin : public ::testing::Test {
+class AwsTestMixin : public ::testing::Test {
+ public:
+  // We set this environment variable to speed up tests by ensuring
+  // DefaultAWSCredentialsProviderChain does not query (inaccessible)
+  // EC2 metadata endpoint
+  AwsTestMixin() : ec2_metadata_disabled_guard_("AWS_EC2_METADATA_DISABLED", "true") {}
+
+  void SetUp() override {
+#ifdef AWS_CPP_SDK_S3_NOT_SHARED
+    auto aws_log_level = Aws::Utils::Logging::LogLevel::Fatal;
+    aws_options_.loggingOptions.logLevel = aws_log_level;
+    aws_options_.loggingOptions.logger_create_fn = [&aws_log_level] {
+      return std::make_shared<Aws::Utils::Logging::ConsoleLogSystem>(aws_log_level);
+    };
+    Aws::InitAPI(aws_options_);
+#endif
+  }
+
+  void TearDown() override {
+#ifdef AWS_CPP_SDK_S3_NOT_SHARED
+    Aws::ShutdownAPI(aws_options_);
+#endif
+  }
+
+ private:
+  EnvVarGuard ec2_metadata_disabled_guard_;
+#ifdef AWS_CPP_SDK_S3_NOT_SHARED
+  Aws::SDKOptions aws_options_;
+#endif
+};
+
+class S3TestMixin : public AwsTestMixin {
  public:
   void SetUp() override {
+    AwsTestMixin::SetUp();
+
     ASSERT_OK(minio_.Start());
 
-    client_config_.endpointOverride = ToAwsString(minio_.connect_string());
-    client_config_.scheme = Aws::Http::Scheme::HTTP;
-    client_config_.retryStrategy = std::make_shared<ConnectRetryStrategy>();
+    client_config_.reset(new Aws::Client::ClientConfiguration());
+    client_config_->endpointOverride = ToAwsString(minio_.connect_string());
+    client_config_->scheme = Aws::Http::Scheme::HTTP;
+    client_config_->retryStrategy = std::make_shared<ConnectRetryStrategy>();
     credentials_ = {ToAwsString(minio_.access_key()), ToAwsString(minio_.secret_key())};
     bool use_virtual_addressing = false;
     client_.reset(
-        new Aws::S3::S3Client(credentials_, client_config_,
+        new Aws::S3::S3Client(credentials_, *client_config_,
                               Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
                               use_virtual_addressing));
   }
 
-  void TearDown() override { ASSERT_OK(minio_.Stop()); }
+  void TearDown() override {
+    ASSERT_OK(minio_.Stop());
+
+    AwsTestMixin::TearDown();
+  }
 
  protected:
   MinioTestServer minio_;
-  Aws::Client::ClientConfiguration client_config_;
+  std::unique_ptr<Aws::Client::ClientConfiguration> client_config_;
   Aws::Auth::AWSCredentials credentials_;
   std::unique_ptr<Aws::S3::S3Client> client_;
 };
@@ -152,12 +204,14 @@ void AssertObjectContents(Aws::S3::S3Client* client, const std::string& bucket,
 ////////////////////////////////////////////////////////////////////////////
 // S3Options tests
 
-TEST(S3Options, FromUri) {
+class S3OptionsTest : public AwsTestMixin {};
+
+TEST_F(S3OptionsTest, FromUri) {
   std::string path;
   S3Options options;
 
   ASSERT_OK_AND_ASSIGN(options, S3Options::FromUri("s3://", &path));
-  ASSERT_EQ(options.region, kS3DefaultRegion);
+  ASSERT_EQ(options.region, "");
   ASSERT_EQ(options.scheme, "https");
   ASSERT_EQ(options.endpoint_override, "");
   ASSERT_EQ(path, "");
@@ -172,17 +226,23 @@ TEST(S3Options, FromUri) {
   ASSERT_EQ(creds.GetAWSSecretKey(), "secret");
 
   ASSERT_OK_AND_ASSIGN(options, S3Options::FromUri("s3://mybucket/", &path));
-  ASSERT_EQ(options.region, kS3DefaultRegion);
+  ASSERT_NE(options.region, "");  // Some region was chosen
   ASSERT_EQ(options.scheme, "https");
   ASSERT_EQ(options.endpoint_override, "");
   ASSERT_EQ(path, "mybucket");
 
   ASSERT_OK_AND_ASSIGN(options, S3Options::FromUri("s3://mybucket/foo/bar/", &path));
-  ASSERT_EQ(options.region, kS3DefaultRegion);
+  ASSERT_NE(options.region, "");
   ASSERT_EQ(options.scheme, "https");
   ASSERT_EQ(options.endpoint_override, "");
   ASSERT_EQ(path, "mybucket/foo/bar");
 
+  // Region resolution with a well-known bucket
+  ASSERT_OK_AND_ASSIGN(
+      options, S3Options::FromUri("s3://aws-earth-mo-atmospheric-ukv-prd/", &path));
+  ASSERT_EQ(options.region, "eu-west-2");
+
+  // Explicit region override
   ASSERT_OK_AND_ASSIGN(
       options,
       S3Options::FromUri(
@@ -195,8 +255,100 @@ TEST(S3Options, FromUri) {
 
   // Missing bucket name
   ASSERT_RAISES(Invalid, S3Options::FromUri("s3:///foo/bar/", &path));
+
+  // Invalid option
+  ASSERT_RAISES(Invalid, S3Options::FromUri("s3://mybucket/?xxx=zzz", &path));
+}
+
+TEST_F(S3OptionsTest, FromAccessKey) {
+  S3Options options;
+
+  // session token is optional and should default to empty string
+  options = S3Options::FromAccessKey("access", "secret");
+  ASSERT_EQ(options.GetAccessKey(), "access");
+  ASSERT_EQ(options.GetSecretKey(), "secret");
+  ASSERT_EQ(options.GetSessionToken(), "");
+
+  options = S3Options::FromAccessKey("access", "secret", "token");
+  ASSERT_EQ(options.GetAccessKey(), "access");
+  ASSERT_EQ(options.GetSecretKey(), "secret");
+  ASSERT_EQ(options.GetSessionToken(), "token");
+}
+
+TEST_F(S3OptionsTest, FromAssumeRole) {
+  S3Options options;
+
+  // arn should be only required argument
+  options = S3Options::FromAssumeRole("my_role_arn");
+  options = S3Options::FromAssumeRole("my_role_arn", "session");
+  options = S3Options::FromAssumeRole("my_role_arn", "session", "id");
+  options = S3Options::FromAssumeRole("my_role_arn", "session", "id", 42);
+
+  // test w/ custom STSClient (will not use DefaultAWSCredentialsProviderChain)
+  Aws::Auth::AWSCredentials test_creds = Aws::Auth::AWSCredentials("access", "secret");
+  std::shared_ptr<Aws::STS::STSClient> sts_client =
+      std::make_shared<Aws::STS::STSClient>(Aws::STS::STSClient(test_creds));
+  options = S3Options::FromAssumeRole("my_role_arn", "session", "id", 42, sts_client);
 }
 
+////////////////////////////////////////////////////////////////////////////
+// Region resolution test
+
+class S3RegionResolutionTest : public AwsTestMixin {};
+
+TEST_F(S3RegionResolutionTest, PublicBucket) {
+  ASSERT_OK_AND_EQ("us-east-2", ResolveBucketRegion("ursa-labs-taxi-data"));
+
+  // Taken from a registry of open S3-hosted datasets
+  // at https://github.com/awslabs/open-data-registry
+  ASSERT_OK_AND_EQ("eu-west-2", ResolveBucketRegion("aws-earth-mo-atmospheric-ukv-prd"));
+  // Same again, cached
+  ASSERT_OK_AND_EQ("eu-west-2", ResolveBucketRegion("aws-earth-mo-atmospheric-ukv-prd"));
+}
+
+TEST_F(S3RegionResolutionTest, RestrictedBucket) {
+  ASSERT_OK_AND_EQ("us-west-2", ResolveBucketRegion("ursa-labs-r-test"));
+  // Same again, cached
+  ASSERT_OK_AND_EQ("us-west-2", ResolveBucketRegion("ursa-labs-r-test"));
+}
+
+TEST_F(S3RegionResolutionTest, NonExistentBucket) {
+  auto maybe_region = ResolveBucketRegion("ursa-labs-non-existent-bucket");
+  ASSERT_RAISES(IOError, maybe_region);
+  ASSERT_THAT(maybe_region.status().message(),
+              ::testing::HasSubstr("Bucket 'ursa-labs-non-existent-bucket' not found"));
+}
+
+////////////////////////////////////////////////////////////////////////////
+// S3FileSystem region test
+
+class S3FileSystemRegionTest : public AwsTestMixin {};
+
+TEST_F(S3FileSystemRegionTest, Default) {
+  ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFromUri("s3://"));
+  auto s3fs = checked_pointer_cast<S3FileSystem>(fs);
+  ASSERT_EQ(s3fs->region(), "us-east-1");
+}
+
+// Skipped on Windows, as the AWS SDK ignores runtime environment changes:
+// https://github.com/aws/aws-sdk-cpp/issues/1476
+
+#ifndef _WIN32
+TEST_F(S3FileSystemRegionTest, EnvironmentVariable) {
+  // Region override with environment variable (AWS SDK >= 1.8)
+  EnvVarGuard region_guard("AWS_DEFAULT_REGION", "eu-north-1");
+
+  ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFromUri("s3://"));
+  auto s3fs = checked_pointer_cast<S3FileSystem>(fs);
+
+  if (Aws::Version::GetVersionMajor() > 1 || Aws::Version::GetVersionMinor() >= 8) {
+    ASSERT_EQ(s3fs->region(), "eu-north-1");
+  } else {
+    ASSERT_EQ(s3fs->region(), "us-east-1");
+  }
+}
+#endif
+
 ////////////////////////////////////////////////////////////////////////////
 // Basic test for the Minio test server.
 
@@ -306,6 +458,16 @@ class TestS3FS : public S3TestMixin {
     ASSERT_OK_AND_ASSIGN(stream, fs_->OpenOutputStream("bucket/newfile1"));
     ASSERT_OK(stream->Close());
     AssertObjectContents(client_.get(), "bucket", "newfile1", "");
+
+    // Open file and then lose filesystem reference
+    ASSERT_EQ(fs_.use_count(), 1);  // needed for test to work
+    std::weak_ptr<S3FileSystem> weak_fs(fs_);
+    ASSERT_OK_AND_ASSIGN(stream, fs_->OpenOutputStream("bucket/newfile5"));
+    fs_.reset();
+    ASSERT_FALSE(weak_fs.expired());
+    ASSERT_OK(stream->Write("some data"));
+    ASSERT_OK(stream->Close());
+    ASSERT_TRUE(weak_fs.expired());
   }
 
   void TestOpenOutputStreamAbort() {
@@ -628,11 +790,23 @@ TEST_F(TestS3FS, OpenInputStream) {
   AssertBufferEqual(*buf, "sub data");
   ASSERT_OK_AND_ASSIGN(buf, stream->Read(100));
   AssertBufferEqual(*buf, "");
+  ASSERT_OK(stream->Close());
 
   // "Directories"
   ASSERT_RAISES(IOError, fs_->OpenInputStream("bucket/emptydir"));
   ASSERT_RAISES(IOError, fs_->OpenInputStream("bucket/somedir"));
   ASSERT_RAISES(IOError, fs_->OpenInputStream("bucket"));
+
+  // Open file and then lose filesystem reference
+  ASSERT_EQ(fs_.use_count(), 1);  // needed for test to work
+  std::weak_ptr<S3FileSystem> weak_fs(fs_);
+  ASSERT_OK_AND_ASSIGN(stream, fs_->OpenInputStream("bucket/somefile"));
+  fs_.reset();
+  ASSERT_FALSE(weak_fs.expired());
+  ASSERT_OK_AND_ASSIGN(buf, stream->Read(10));
+  AssertBufferEqual(*buf, "some data");
+  ASSERT_OK(stream->Close());
+  ASSERT_TRUE(weak_fs.expired());
 }
 
 TEST_F(TestS3FS, OpenInputFile) {
diff --git a/cpp/src/arrow/filesystem/test_util.cc b/cpp/src/arrow/filesystem/test_util.cc
index 9acf1c0b1ad..619caf4b5a3 100644
--- a/cpp/src/arrow/filesystem/test_util.cc
+++ b/cpp/src/arrow/filesystem/test_util.cc
@@ -106,10 +106,7 @@ void CreateFile(FileSystem* fs, const std::string& path, const std::string& data
 }
 
 void SortInfos(std::vector<FileInfo>* infos) {
-  std::sort(infos->begin(), infos->end(),
-            [](const FileInfo& left, const FileInfo& right) -> bool {
-              return left.path() < right.path();
-            });
+  std::sort(infos->begin(), infos->end(), FileInfo::ByPath{});
 }
 
 void AssertFileInfo(const FileInfo& info, const std::string& path, FileType type) {
diff --git a/cpp/src/arrow/flight/CMakeLists.txt b/cpp/src/arrow/flight/CMakeLists.txt
index 9ce4654f90b..edd4fdf1c33 100644
--- a/cpp/src/arrow/flight/CMakeLists.txt
+++ b/cpp/src/arrow/flight/CMakeLists.txt
@@ -61,6 +61,57 @@ set_source_files_properties(${FLIGHT_GENERATED_PROTO_FILES} PROPERTIES GENERATED
 
 add_custom_target(flight_grpc_gen ALL DEPENDS ${FLIGHT_GENERATED_PROTO_FILES})
 
+# <KLUDGE> -Werror / /WX cause try_compile to fail because there seems to be no
+# way to pass -isystem $GRPC_INCLUDE_DIR instead of -I$GRPC_INCLUDE_DIR
+set(CMAKE_CXX_FLAGS_BACKUP "${CMAKE_CXX_FLAGS}")
+string(REPLACE "/WX" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+string(REPLACE "-Werror " " " CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+
+# Probe the version of gRPC being used to see if it supports disabling server
+# verification when using TLS.
+if(NOT DEFINED HAS_GRPC_132)
+  message(STATUS "Checking support for TlsCredentialsOptions...")
+  get_property(CURRENT_INCLUDE_DIRECTORIES
+               DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+               PROPERTY INCLUDE_DIRECTORIES)
+  try_compile(HAS_GRPC_132 ${CMAKE_CURRENT_BINARY_DIR}/try_compile SOURCES
+              "${CMAKE_CURRENT_SOURCE_DIR}/try_compile/check_tls_opts_132.cc"
+              CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${CURRENT_INCLUDE_DIRECTORIES}"
+              LINK_LIBRARIES gRPC::grpc
+              OUTPUT_VARIABLE TSL_CREDENTIALS_OPTIONS_CHECK_OUTPUT CXX_STANDARD 11)
+
+  if(HAS_GRPC_132)
+    message(STATUS "TlsCredentialsOptions found in grpc::experimental.")
+    add_definitions(-DGRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS=grpc::experimental)
+  else()
+    message(STATUS "TlsCredentialsOptions not found in grpc::experimental.")
+    message(DEBUG "Build output:")
+    list(APPEND CMAKE_MESSAGE_INDENT "check_tls_opts_132.cc: ")
+    message(DEBUG ${TSL_CREDENTIALS_OPTIONS_CHECK_OUTPUT})
+    list(REMOVE_AT CMAKE_MESSAGE_INDENT -1)
+
+    try_compile(HAS_GRPC_127 ${CMAKE_CURRENT_BINARY_DIR}/try_compile SOURCES
+                "${CMAKE_CURRENT_SOURCE_DIR}/try_compile/check_tls_opts_127.cc"
+                CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${CURRENT_INCLUDE_DIRECTORIES}"
+                OUTPUT_VARIABLE TSL_CREDENTIALS_OPTIONS_CHECK_OUTPUT CXX_STANDARD 11)
+
+    if(HAS_GRPC_127)
+      message(STATUS "TlsCredentialsOptions found in grpc_impl::experimental.")
+      add_definitions(
+        -DGRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS=grpc_impl::experimental)
+    else()
+      message(STATUS "TlsCredentialsOptions not found in grpc_impl::experimental.")
+      message(DEBUG "Build output:")
+      list(APPEND CMAKE_MESSAGE_INDENT "check_tls_opts_127.cc: ")
+      message(DEBUG ${TSL_CREDENTIALS_OPTIONS_CHECK_OUTPUT})
+      list(REMOVE_AT CMAKE_MESSAGE_INDENT -1)
+    endif()
+  endif()
+endif()
+
+# </KLUDGE> Restore the CXXFLAGS that were modified above
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_BACKUP}")
+
 # Note, we do not compile the generated Protobuf sources directly, instead
 # compiling then via protocol_internal.cc which contains some gRPC template
 # overrides to enable Flight-specific optimizations. See comments in
@@ -101,7 +152,7 @@ foreach(LIB_TARGET ${ARROW_FLIGHT_LIBRARIES})
 endforeach()
 
 # Define arrow_flight_testing library
-if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS OR ARROW_BUILD_INTEGRATION)
+if(ARROW_TESTING)
   add_arrow_lib(arrow_flight_testing
                 CMAKE_PACKAGE_NAME
                 ArrowFlightTesting
@@ -130,7 +181,8 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS OR ARROW_BUILD_INTEGRATION)
 endif()
 
 foreach(LIB_TARGET ${ARROW_FLIGHT_TESTING_LIBRARIES})
-  target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_FLIGHT_EXPORTING)
+  target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_FLIGHT_EXPORTING
+                             ${ARROW_BOOST_PROCESS_COMPILE_DEFINITIONS})
 endforeach()
 
 add_arrow_test(flight_test
diff --git a/cpp/src/arrow/flight/client.cc b/cpp/src/arrow/flight/client.cc
index 400d07a2145..821da214a59 100644
--- a/cpp/src/arrow/flight/client.cc
+++ b/cpp/src/arrow/flight/client.cc
@@ -25,14 +25,20 @@
 #include <mutex>
 #include <sstream>
 #include <string>
+#include <unordered_map>
 #include <utility>
 
 #ifdef GRPCPP_PP_INCLUDE
 #include <grpcpp/grpcpp.h>
+#if defined(GRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS)
+#include <grpcpp/security/tls_credentials_options.h>
+#endif
 #else
 #include <grpc++/grpc++.h>
 #endif
 
+#include <grpc/grpc_security_constants.h>
+
 #include "arrow/buffer.h"
 #include "arrow/ipc/reader.h"
 #include "arrow/ipc/writer.h"
@@ -83,6 +89,9 @@ std::shared_ptr<FlightWriteSizeStatusDetail> FlightWriteSizeStatusDetail::Unwrap
   return std::dynamic_pointer_cast<FlightWriteSizeStatusDetail>(status.detail());
 }
 
+FlightClientOptions::FlightClientOptions()
+    : write_size_limit_bytes(0), disable_server_verification(false) {}
+
 FlightClientOptions FlightClientOptions::Defaults() { return FlightClientOptions(); }
 
 struct ClientRpc {
@@ -212,10 +221,10 @@ class FinishableWritableStream : public FinishableStream<Stream, ReadT> {
     // outstanding read.
     std::unique_lock<std::mutex> read_guard(*read_mutex_, std::try_to_lock);
     if (!read_guard.owns_lock()) {
-      return Finish(MakeFlightError(
+      return MakeFlightError(
           FlightStatusCode::Internal,
           "Cannot close stream with pending read operation. Client context: " +
-              st.ToString()));
+              st.ToString());
     }
 
     // Try to flush pending writes. Don't use our WritesDone() to
@@ -834,6 +843,31 @@ class GrpcMetadataReader : public FlightMetadataReader {
   std::shared_ptr<std::mutex> read_mutex_;
 };
 
+namespace {
+// Dummy self-signed certificate to be used because TlsCredentials
+// requires root CA certs, even if you are skipping server
+// verification.
+#if defined(GRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS)
+constexpr char BLANK_ROOT_PEM[] =
+    "-----BEGIN CERTIFICATE-----\n"
+    "MIICwzCCAaugAwIBAgIJAM12DOkcaqrhMA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV\n"
+    "BAMTCWxvY2FsaG9zdDAeFw0yMDEwMDcwODIyNDFaFw0zMDEwMDUwODIyNDFaMBQx\n"
+    "EjAQBgNVBAMTCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC\n"
+    "ggEBALjJ8KPEpF0P4GjMPrJhjIBHUL0AX9E4oWdgJRCSFkPKKEWzQabTQBikMOhI\n"
+    "W4VvBMaHEBuECE5OEyrzDRiAO354I4F4JbBfxMOY8NIW0uWD6THWm2KkCzoZRIPW\n"
+    "yZL6dN+mK6cEH+YvbNuy5ZQGNjGG43tyiXdOCAc4AI9POeTtjdMpbbpR2VY4Ad/E\n"
+    "oTEiS3gNnN7WIAdgMhCJxjzvPwKszV3f7pwuTHzFMsuHLKr6JeaVUYfbi4DxxC8Z\n"
+    "k6PF6dLlLf3ngTSLBJyaXP1BhKMvz0TaMK3F0y2OGwHM9J8np2zWjTlNVEzffQZx\n"
+    "SWMOQManlJGs60xYx9KCPJMZZsMCAwEAAaMYMBYwFAYDVR0RBA0wC4IJbG9jYWxo\n"
+    "b3N0MA0GCSqGSIb3DQEBBQUAA4IBAQC0LrmbcNKgO+D50d/wOc+vhi9K04EZh8bg\n"
+    "WYAK1kLOT4eShbzqWGV/1EggY4muQ6ypSELCLuSsg88kVtFQIeRilA6bHFqQSj6t\n"
+    "sqgh2cWsMwyllCtmX6Maf3CLb2ZdoJlqUwdiBdrbIbuyeAZj3QweCtLKGSQzGDyI\n"
+    "KH7G8nC5d0IoRPiCMB6RnMMKsrhviuCdWbAFHop7Ff36JaOJ8iRa2sSf2OXE8j/5\n"
+    "obCXCUvYHf4Zw27JcM2AnnQI9VJLnYxis83TysC5s2Z7t0OYNS9kFmtXQbUNlmpS\n"
+    "doQ/Eu47vWX7S0TXeGziGtbAOKxbHE0BGGPDOAB/jGW/JVbeTiXY\n"
+    "-----END CERTIFICATE-----\n";
+#endif
+}  // namespace
 class FlightClient::FlightClientImpl {
  public:
   Status Connect(const Location& location, const FlightClientOptions& options) {
@@ -844,18 +878,49 @@ class FlightClient::FlightClientImpl {
     if (scheme == kSchemeGrpc || scheme == kSchemeGrpcTcp || scheme == kSchemeGrpcTls) {
       grpc_uri << location.uri_->host() << ":" << location.uri_->port_text();
 
-      if (scheme == "grpc+tls") {
-        grpc::SslCredentialsOptions ssl_options;
-        if (!options.tls_root_certs.empty()) {
-          ssl_options.pem_root_certs = options.tls_root_certs;
-        }
-        if (!options.cert_chain.empty()) {
-          ssl_options.pem_cert_chain = options.cert_chain;
-        }
-        if (!options.private_key.empty()) {
-          ssl_options.pem_private_key = options.private_key;
+      if (scheme == kSchemeGrpcTls) {
+        if (options.disable_server_verification) {
+#if !defined(GRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS)
+          return Status::NotImplemented(
+              "Using encryption with server verification disabled is unsupported. "
+              "Please use a release of Arrow Flight built with gRPC 1.27 or higher.");
+#else
+          namespace ge = GRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS;
+
+          // A callback to supply to TlsCredentialsOptions that accepts any server
+          // arguments.
+          struct NoOpTlsAuthorizationCheck
+              : public ge::TlsServerAuthorizationCheckInterface {
+            int Schedule(ge::TlsServerAuthorizationCheckArg* arg) override {
+              arg->set_success(1);
+              arg->set_status(GRPC_STATUS_OK);
+              return 0;
+            }
+          };
+
+          noop_auth_check_ = std::make_shared<ge::TlsServerAuthorizationCheckConfig>(
+              std::make_shared<NoOpTlsAuthorizationCheck>());
+          auto materials_config = std::make_shared<ge::TlsKeyMaterialsConfig>();
+          materials_config->set_pem_root_certs(BLANK_ROOT_PEM);
+          ge::TlsCredentialsOptions tls_options(
+              GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE,
+              GRPC_TLS_SKIP_ALL_SERVER_VERIFICATION, materials_config,
+              std::shared_ptr<ge::TlsCredentialReloadConfig>(), noop_auth_check_);
+          creds = ge::TlsCredentials(tls_options);
+#endif
+        } else {
+          grpc::SslCredentialsOptions ssl_options;
+          if (!options.tls_root_certs.empty()) {
+            ssl_options.pem_root_certs = options.tls_root_certs;
+          }
+          if (!options.cert_chain.empty()) {
+            ssl_options.pem_cert_chain = options.cert_chain;
+          }
+          if (!options.private_key.empty()) {
+            ssl_options.pem_private_key = options.private_key;
+          }
+          creds = grpc::SslCredentials(ssl_options);
         }
-        creds = grpc::SslCredentials(ssl_options);
       } else {
         creds = grpc::InsecureChannelCredentials();
       }
@@ -867,13 +932,17 @@ class FlightClient::FlightClientImpl {
     }
 
     grpc::ChannelArguments args;
+    // We can't set the same config value twice, so for values where
+    // we want to set defaults, keep them in a map and update them;
+    // then update them all at once
+    std::unordered_map<std::string, int> default_args;
     // Try to reconnect quickly at first, in case the server is still starting up
-    args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS, 100);
+    default_args[GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS] = 100;
     // Receive messages of any size
-    args.SetMaxReceiveMessageSize(-1);
+    default_args[GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH] = -1;
     // Setting this arg enables each client to open it's own TCP connection to server,
     // not sharing one single connection, which becomes bottleneck under high load.
-    args.SetInt(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL, 1);
+    default_args[GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL] = 1;
 
     if (options.override_hostname != "") {
       args.SetSslTargetNameOverride(options.override_hostname);
@@ -882,12 +951,15 @@ class FlightClient::FlightClientImpl {
     // Allow setting generic gRPC options.
     for (const auto& arg : options.generic_options) {
       if (util::holds_alternative<int>(arg.second)) {
-        args.SetInt(arg.first, util::get<int>(arg.second));
+        default_args[arg.first] = util::get<int>(arg.second);
       } else if (util::holds_alternative<std::string>(arg.second)) {
         args.SetString(arg.first, util::get<std::string>(arg.second));
       }
       // Otherwise unimplemented
     }
+    for (const auto& pair : default_args) {
+      args.SetInt(pair.first, pair.second);
+    }
 
     std::vector<std::unique_ptr<grpc::experimental::ClientInterceptorFactoryInterface>>
         interceptors;
@@ -1093,6 +1165,15 @@ class FlightClient::FlightClientImpl {
  private:
   std::unique_ptr<pb::FlightService::Stub> stub_;
   std::shared_ptr<ClientAuthHandler> auth_handler_;
+#if defined(GRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS)
+  // Scope the TlsServerAuthorizationCheckConfig to be at the class instance level, since
+  // it gets created during Connect() and needs to persist to DoAction() calls. gRPC does
+  // not correctly increase the reference count of this object:
+  // https://github.com/grpc/grpc/issues/22287
+  std::shared_ptr<
+      GRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS::TlsServerAuthorizationCheckConfig>
+      noop_auth_check_;
+#endif
   int64_t write_size_limit_bytes_;
 };
 
diff --git a/cpp/src/arrow/flight/client.h b/cpp/src/arrow/flight/client.h
index a6ad17b9279..935e8fb92ba 100644
--- a/cpp/src/arrow/flight/client.h
+++ b/cpp/src/arrow/flight/client.h
@@ -90,6 +90,8 @@ class ARROW_FLIGHT_EXPORT FlightWriteSizeStatusDetail : public arrow::StatusDeta
 
 class ARROW_FLIGHT_EXPORT FlightClientOptions {
  public:
+  FlightClientOptions();
+
   /// \brief Root certificates to use for validating server
   /// certificates.
   std::string tls_root_certs;
@@ -108,10 +110,14 @@ class ARROW_FLIGHT_EXPORT FlightClientOptions {
   /// positive. When enabled, FlightStreamWriter.Write* may yield a
   /// IOError with error detail FlightWriteSizeStatusDetail.
   int64_t write_size_limit_bytes;
+
   /// \brief Generic connection options, passed to the underlying
   ///     transport; interpretation is implementation-dependent.
   std::vector<std::pair<std::string, util::variant<int, std::string>>> generic_options;
 
+  /// \brief Use TLS without validating the server certificate. Use with caution.
+  bool disable_server_verification;
+
   /// \brief Get default options.
   static FlightClientOptions Defaults();
 };
diff --git a/cpp/src/arrow/flight/flight_benchmark.cc b/cpp/src/arrow/flight/flight_benchmark.cc
index 6180e05cbfd..d2b686d8652 100644
--- a/cpp/src/arrow/flight/flight_benchmark.cc
+++ b/cpp/src/arrow/flight/flight_benchmark.cc
@@ -29,7 +29,6 @@
 
 #include <gflags/gflags.h>
 
-#include "arrow/api.h"
 #include "arrow/io/memory.h"
 #include "arrow/ipc/api.h"
 #include "arrow/record_batch.h"
diff --git a/cpp/src/arrow/flight/flight_test.cc b/cpp/src/arrow/flight/flight_test.cc
index 3808699ce11..a5fdfe8184e 100644
--- a/cpp/src/arrow/flight/flight_test.cc
+++ b/cpp/src/arrow/flight/flight_test.cc
@@ -354,8 +354,6 @@ class TestFlightClient : public ::testing::Test {
   template <typename EndpointCheckFunc>
   void CheckDoGet(const FlightDescriptor& descr, const BatchVector& expected_batches,
                   EndpointCheckFunc&& check_endpoints) {
-    auto num_batches = static_cast<int>(expected_batches.size());
-    ASSERT_GE(num_batches, 2);
     auto expected_schema = expected_batches[0]->schema();
 
     std::unique_ptr<FlightInfo> info;
@@ -369,6 +367,13 @@ class TestFlightClient : public ::testing::Test {
 
     // By convention, fetch the first endpoint
     Ticket ticket = info->endpoints()[0].ticket;
+    CheckDoGet(ticket, expected_batches);
+  }
+
+  void CheckDoGet(const Ticket& ticket, const BatchVector& expected_batches) {
+    auto num_batches = static_cast<int>(expected_batches.size());
+    ASSERT_GE(num_batches, 2);
+
     std::unique_ptr<FlightStreamReader> stream;
     ASSERT_OK(client_->DoGet(ticket, &stream));
 
@@ -1105,6 +1110,15 @@ TEST_F(TestFlightClient, DoGetDicts) {
   CheckDoGet(descr, expected_batches, check_endpoints);
 }
 
+// Ensure the gRPC client is configured to allow large messages
+// Tests a 32 MiB batch
+TEST_F(TestFlightClient, DoGetLargeBatch) {
+  BatchVector expected_batches;
+  ASSERT_OK(ExampleLargeBatches(&expected_batches));
+  Ticket ticket{"ticket-large-batch-1"};
+  CheckDoGet(ticket, expected_batches);
+}
+
 TEST_F(TestFlightClient, DoExchange) {
   auto descr = FlightDescriptor::Command("counter");
   BatchVector batches;
@@ -1403,7 +1417,7 @@ TEST_F(TestFlightClient, GenericOptions) {
   std::unique_ptr<FlightClient> client;
   auto options = FlightClientOptions::Defaults();
   // Set a very low limit at the gRPC layer to fail all calls
-  options.generic_options.emplace_back(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, 32);
+  options.generic_options.emplace_back(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, 4);
   Location location;
   ASSERT_OK(Location::ForGrpcTcp("localhost", server_->port(), &location));
   ASSERT_OK(FlightClient::Connect(location, options, &client));
@@ -1515,6 +1529,16 @@ TEST_F(TestDoPut, DoPutDicts) {
   CheckDoPut(descr, schema, batches);
 }
 
+// Ensure the gRPC server is configured to allow large messages
+// Tests a 32 MiB batch
+TEST_F(TestDoPut, DoPutLargeBatch) {
+  auto descr = FlightDescriptor::Path({"large-batches"});
+  auto schema = ExampleLargeSchema();
+  BatchVector batches;
+  ASSERT_OK(ExampleLargeBatches(&batches));
+  CheckDoPut(descr, schema, batches);
+}
+
 TEST_F(TestDoPut, DoPutSizeLimit) {
   const int64_t size_limit = 4096;
   Location location;
@@ -1776,6 +1800,32 @@ TEST_F(TestTls, DoAction) {
   ASSERT_EQ(result->body->ToString(), "Hello, world!");
 }
 
+#if defined(GRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS)
+TEST_F(TestTls, DisableServerVerification) {
+  std::unique_ptr<FlightClient> client;
+  auto client_options = FlightClientOptions();
+  // For security reasons, if encryption is being used,
+  // the client should be configured to verify the server by default.
+  ASSERT_EQ(client_options.disable_server_verification, false);
+  client_options.disable_server_verification = true;
+  ASSERT_OK(FlightClient::Connect(location_, client_options, &client));
+
+  FlightCallOptions options;
+  options.timeout = TimeoutDuration{5.0};
+  Action action;
+  action.type = "test";
+  action.body = Buffer::FromString("");
+  std::unique_ptr<ResultStream> results;
+  ASSERT_OK(client->DoAction(options, action, &results));
+  ASSERT_NE(results, nullptr);
+
+  std::unique_ptr<Result> result;
+  ASSERT_OK(results->Next(&result));
+  ASSERT_NE(result, nullptr);
+  ASSERT_EQ(result->body->ToString(), "Hello, world!");
+}
+#endif
+
 TEST_F(TestTls, OverrideHostname) {
   std::unique_ptr<FlightClient> client;
   auto client_options = FlightClientOptions();
diff --git a/cpp/src/arrow/flight/internal.cc b/cpp/src/arrow/flight/internal.cc
index 26252043c4b..f27de208ac3 100644
--- a/cpp/src/arrow/flight/internal.cc
+++ b/cpp/src/arrow/flight/internal.cc
@@ -465,11 +465,9 @@ Status FromProto(const pb::SchemaResult& pb_result, std::string* result) {
 }
 
 Status SchemaToString(const Schema& schema, std::string* out) {
-  // TODO(wesm): Do we care about better memory efficiency here?
   ipc::DictionaryMemo unused_dict_memo;
-  ARROW_ASSIGN_OR_RAISE(
-      std::shared_ptr<Buffer> serialized_schema,
-      ipc::SerializeSchema(schema, &unused_dict_memo, default_memory_pool()));
+  ARROW_ASSIGN_OR_RAISE(std::shared_ptr<Buffer> serialized_schema,
+                        ipc::SerializeSchema(schema));
   *out = std::string(reinterpret_cast<const char*>(serialized_schema->data()),
                      static_cast<size_t>(serialized_schema->size()));
   return Status::OK();
diff --git a/cpp/src/arrow/flight/perf_server.cc b/cpp/src/arrow/flight/perf_server.cc
index 59675b4ebe5..142dd88aa57 100644
--- a/cpp/src/arrow/flight/perf_server.cc
+++ b/cpp/src/arrow/flight/perf_server.cc
@@ -66,6 +66,7 @@ class PerfDataStream : public FlightDataStream {
         total_records_(total_records),
         records_sent_(0),
         schema_(schema),
+        mapper_(*schema),
         arrays_(arrays) {
     batch_ = RecordBatch::Make(schema, batch_length_, arrays_);
   }
@@ -73,8 +74,7 @@ class PerfDataStream : public FlightDataStream {
   std::shared_ptr<Schema> schema() override { return schema_; }
 
   Status GetSchemaPayload(FlightPayload* payload) override {
-    return ipc::GetSchemaPayload(*schema_, ipc_options_, &dictionary_memo_,
-                                 &payload->ipc_message);
+    return ipc::GetSchemaPayload(*schema_, ipc_options_, mapper_, &payload->ipc_message);
   }
 
   Status Next(FlightPayload* payload) override {
@@ -112,7 +112,7 @@ class PerfDataStream : public FlightDataStream {
   const int64_t total_records_;
   int64_t records_sent_;
   std::shared_ptr<Schema> schema_;
-  ipc::DictionaryMemo dictionary_memo_;
+  ipc::DictionaryFieldMapper mapper_;
   ipc::IpcWriteOptions ipc_options_;
   std::shared_ptr<RecordBatch> batch_;
   ArrayVector arrays_;
diff --git a/cpp/src/arrow/flight/server.cc b/cpp/src/arrow/flight/server.cc
index 8a3d9bf5d2f..0cdd3b43f51 100644
--- a/cpp/src/arrow/flight/server.cc
+++ b/cpp/src/arrow/flight/server.cc
@@ -288,8 +288,9 @@ class DoExchangeMessageWriter : public FlightMessageWriter {
     started_ = true;
     ipc_options_ = options;
 
+    RETURN_NOT_OK(mapper_.AddSchemaFields(*schema));
     FlightPayload schema_payload;
-    RETURN_NOT_OK(ipc::GetSchemaPayload(*schema, ipc_options_, &dictionary_memo_,
+    RETURN_NOT_OK(ipc::GetSchemaPayload(*schema, ipc_options_, mapper_,
                                         &schema_payload.ipc_message));
     return WritePayload(schema_payload);
   }
@@ -342,8 +343,9 @@ class DoExchangeMessageWriter : public FlightMessageWriter {
       return Status::OK();
     }
     dictionaries_written_ = true;
-    RETURN_NOT_OK(ipc::CollectDictionaries(batch, &dictionary_memo_));
-    for (auto& pair : dictionary_memo_.dictionaries()) {
+    ARROW_ASSIGN_OR_RAISE(const auto dictionaries,
+                          ipc::CollectDictionaries(batch, mapper_));
+    for (const auto& pair : dictionaries) {
       FlightPayload payload{};
       RETURN_NOT_OK(ipc::GetDictionaryPayload(pair.first, pair.second, ipc_options_,
                                               &payload.ipc_message));
@@ -354,7 +356,7 @@ class DoExchangeMessageWriter : public FlightMessageWriter {
 
   grpc::ServerReaderWriter<pb::FlightData, pb::FlightData>* stream_;
   ::arrow::ipc::IpcWriteOptions ipc_options_;
-  ipc::DictionaryMemo dictionary_memo_;
+  ipc::DictionaryFieldMapper mapper_;
   bool started_ = false;
   bool dictionaries_written_ = false;
 };
@@ -470,7 +472,16 @@ class FlightServiceImpl : public FlightService::Service {
   grpc::Status CheckAuth(const FlightMethod& method, ServerContext* context,
                          GrpcServerCallContext& flight_context) {
     if (!auth_handler_) {
-      flight_context.peer_identity_ = "";
+      const auto auth_context = context->auth_context();
+      if (auth_context && auth_context->IsPeerAuthenticated()) {
+        auto peer_identity = auth_context->GetPeerIdentity();
+        flight_context.peer_identity_ =
+            peer_identity.empty()
+                ? ""
+                : std::string(peer_identity.front().begin(), peer_identity.front().end());
+      } else {
+        flight_context.peer_identity_ = "";
+      }
     } else {
       const auto client_metadata = context->client_metadata();
       const auto auth_header = client_metadata.find(internal::kGrpcAuthHeader);
@@ -969,12 +980,12 @@ class RecordBatchStream::RecordBatchStreamImpl {
 
   RecordBatchStreamImpl(const std::shared_ptr<RecordBatchReader>& reader,
                         const ipc::IpcWriteOptions& options)
-      : reader_(reader), ipc_options_(options) {}
+      : reader_(reader), mapper_(*reader_->schema()), ipc_options_(options) {}
 
   std::shared_ptr<Schema> schema() { return reader_->schema(); }
 
   Status GetSchemaPayload(FlightPayload* payload) {
-    return ipc::GetSchemaPayload(*reader_->schema(), ipc_options_, &dictionary_memo_,
+    return ipc::GetSchemaPayload(*reader_->schema(), ipc_options_, mapper_,
                                  &payload->ipc_message);
   }
 
@@ -986,7 +997,8 @@ class RecordBatchStream::RecordBatchStreamImpl {
         payload->ipc_message.metadata = nullptr;
         return Status::OK();
       }
-      RETURN_NOT_OK(CollectDictionaries(*current_batch_));
+      ARROW_ASSIGN_OR_RAISE(dictionaries_,
+                            ipc::CollectDictionaries(*current_batch_, mapper_));
       stage_ = Stage::DICTIONARY;
     }
 
@@ -1020,15 +1032,9 @@ class RecordBatchStream::RecordBatchStreamImpl {
                                      &payload->ipc_message);
   }
 
-  Status CollectDictionaries(const RecordBatch& batch) {
-    RETURN_NOT_OK(ipc::CollectDictionaries(batch, &dictionary_memo_));
-    dictionaries_ = dictionary_memo_.dictionaries();
-    return Status::OK();
-  }
-
   Stage stage_ = Stage::NEW;
   std::shared_ptr<RecordBatchReader> reader_;
-  ipc::DictionaryMemo dictionary_memo_;
+  ipc::DictionaryFieldMapper mapper_;
   ipc::IpcWriteOptions ipc_options_;
   std::shared_ptr<RecordBatch> current_batch_;
   std::vector<std::pair<int64_t, std::shared_ptr<Array>>> dictionaries_;
diff --git a/cpp/src/arrow/flight/test_util.cc b/cpp/src/arrow/flight/test_util.cc
index 302fda1ab81..f5efa395909 100644
--- a/cpp/src/arrow/flight/test_util.cc
+++ b/cpp/src/arrow/flight/test_util.cc
@@ -26,18 +26,15 @@
 #include <sstream>
 
 #include <boost/filesystem.hpp>
-// boost/process/detail/windows/handle_workaround.hpp doesn't work
-// without BOOST_USE_WINDOWS_H with MinGW because MinGW doesn't
-// provide __kernel_entry without winternl.h.
-//
-// See also:
-// https://github.com/boostorg/process/blob/develop/include/boost/process/detail/windows/handle_workaround.hpp
-#define BOOST_USE_WINDOWS_H 1
+// We need BOOST_USE_WINDOWS_H definition with MinGW when we use
+// boost/process.hpp. See ARROW_BOOST_PROCESS_COMPILE_DEFINITIONS in
+// cpp/cmake_modules/BuildUtils.cmake for details.
 #include <boost/process.hpp>
 
 #include <gtest/gtest.h>
 
 #include "arrow/ipc/test_common.h"
+#include "arrow/testing/generator.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/testing/util.h"
 #include "arrow/util/logging.h"
@@ -156,6 +153,11 @@ Status GetBatchForFlight(const Ticket& ticket, std::shared_ptr<RecordBatchReader
     RETURN_NOT_OK(ExampleDictBatches(&batches));
     *out = std::make_shared<BatchIterator>(batches[0]->schema(), batches);
     return Status::OK();
+  } else if (ticket.ticket == "ticket-large-batch-1") {
+    BatchVector batches;
+    RETURN_NOT_OK(ExampleLargeBatches(&batches));
+    *out = std::make_shared<BatchIterator>(batches[0]->schema(), batches);
+    return Status::OK();
   } else {
     return Status::NotImplemented("no stream implemented for ticket: " + ticket.ticket);
   }
@@ -504,6 +506,15 @@ std::shared_ptr<Schema> ExampleDictSchema() {
   return batch->schema();
 }
 
+std::shared_ptr<Schema> ExampleLargeSchema() {
+  std::vector<std::shared_ptr<arrow::Field>> fields;
+  for (int i = 0; i < 128; i++) {
+    const auto field_name = "f" + std::to_string(i);
+    fields.push_back(arrow::field(field_name, arrow::float64()));
+  }
+  return arrow::schema(fields);
+}
+
 std::vector<FlightInfo> ExampleFlightInfo() {
   Location location1;
   Location location2;
@@ -582,6 +593,20 @@ Status ExampleNestedBatches(BatchVector* out) {
   return Status::OK();
 }
 
+Status ExampleLargeBatches(BatchVector* out) {
+  const auto array_length = 32768;
+  std::shared_ptr<RecordBatch> batch;
+  std::vector<std::shared_ptr<arrow::Array>> arrays;
+  const auto arr = arrow::ConstantArrayGenerator::Float64(array_length, 1.0);
+  for (int i = 0; i < 128; i++) {
+    arrays.push_back(arr);
+  }
+  auto schema = ExampleLargeSchema();
+  out->push_back(RecordBatch::Make(schema, array_length, arrays));
+  out->push_back(RecordBatch::Make(schema, array_length, arrays));
+  return Status::OK();
+}
+
 std::vector<ActionType> ExampleActionTypes() {
   return {{"drop", "drop a dataset"}, {"cache", "cache a dataset"}};
 }
diff --git a/cpp/src/arrow/flight/test_util.h b/cpp/src/arrow/flight/test_util.h
index 5cb5267c33c..38086f6bb86 100644
--- a/cpp/src/arrow/flight/test_util.h
+++ b/cpp/src/arrow/flight/test_util.h
@@ -140,6 +140,9 @@ std::shared_ptr<Schema> ExampleStringSchema();
 ARROW_FLIGHT_EXPORT
 std::shared_ptr<Schema> ExampleDictSchema();
 
+ARROW_FLIGHT_EXPORT
+std::shared_ptr<Schema> ExampleLargeSchema();
+
 ARROW_FLIGHT_EXPORT
 Status ExampleIntBatches(BatchVector* out);
 
@@ -152,6 +155,9 @@ Status ExampleDictBatches(BatchVector* out);
 ARROW_FLIGHT_EXPORT
 Status ExampleNestedBatches(BatchVector* out);
 
+ARROW_FLIGHT_EXPORT
+Status ExampleLargeBatches(BatchVector* out);
+
 ARROW_FLIGHT_EXPORT
 std::vector<FlightInfo> ExampleFlightInfo();
 
diff --git a/cpp/src/arrow/flight/try_compile/check_tls_opts_127.cc b/cpp/src/arrow/flight/try_compile/check_tls_opts_127.cc
new file mode 100644
index 00000000000..3815d13c5ca
--- /dev/null
+++ b/cpp/src/arrow/flight/try_compile/check_tls_opts_127.cc
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Dummy file for checking if TlsCredentialsOptions exists in
+// the grpc_impl::experimental namespace. gRPC versions 1.27-1.31
+// put it here. This is for supporting disabling server
+// validation when using TLS.
+
+#include <grpc/grpc_security_constants.h>
+#include <grpcpp/grpcpp.h>
+#include <grpcpp/security/tls_credentials_options.h>
+
+static grpc_tls_server_verification_option check(
+    const grpc_impl::experimental::TlsCredentialsOptions* options) {
+  grpc_tls_server_verification_option server_opt = options->server_verification_option();
+  return server_opt;
+}
+
+int main(int argc, const char** argv) {
+  grpc_tls_server_verification_option opt = check(nullptr);
+  return 0;
+}
diff --git a/cpp/src/arrow/flight/try_compile/check_tls_opts_132.cc b/cpp/src/arrow/flight/try_compile/check_tls_opts_132.cc
new file mode 100644
index 00000000000..d580aba6e44
--- /dev/null
+++ b/cpp/src/arrow/flight/try_compile/check_tls_opts_132.cc
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Dummy file for checking if TlsCredentialsOptions exists in
+// the grpc::experimental namespace. gRPC versions 1.32 and higher
+// put it here. This is for supporting disabling server
+// validation when using TLS.
+
+#include <grpc/grpc_security_constants.h>
+#include <grpcpp/grpcpp.h>
+#include <grpcpp/security/tls_credentials_options.h>
+
+static grpc_tls_server_verification_option check(
+    const grpc::experimental::TlsCredentialsOptions* options) {
+  grpc_tls_server_verification_option server_opt = options->server_verification_option();
+  return server_opt;
+}
+
+int main(int argc, const char** argv) {
+  grpc_tls_server_verification_option opt = check(nullptr);
+  return 0;
+}
diff --git a/cpp/src/arrow/gpu/cuda_test.cc b/cpp/src/arrow/gpu/cuda_test.cc
index 3a3b8be5d2c..6bb06443dc4 100644
--- a/cpp/src/arrow/gpu/cuda_test.cc
+++ b/cpp/src/arrow/gpu/cuda_test.cc
@@ -596,7 +596,7 @@ TEST_F(TestCudaArrowIpc, DictionaryWriteRead) {
   ASSERT_OK(ipc::test::MakeDictionary(&batch));
 
   ipc::DictionaryMemo dictionary_memo;
-  ASSERT_OK(ipc::CollectDictionaries(*batch, &dictionary_memo));
+  ASSERT_OK(ipc::internal::CollectDictionaries(*batch, &dictionary_memo));
 
   std::shared_ptr<CudaBuffer> device_serialized;
   ASSERT_OK_AND_ASSIGN(device_serialized, SerializeRecordBatch(*batch, context_.get()));
diff --git a/cpp/src/arrow/io/buffered_test.cc b/cpp/src/arrow/io/buffered_test.cc
index 38b9f51be4e..1fefc261b1d 100644
--- a/cpp/src/arrow/io/buffered_test.cc
+++ b/cpp/src/arrow/io/buffered_test.cc
@@ -42,11 +42,14 @@
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/testing/gtest_util.h"
+#include "arrow/util/io_util.h"
 #include "arrow/util/string_view.h"
 
 namespace arrow {
 namespace io {
 
+using ::arrow::internal::TemporaryDir;
+
 static std::string GenerateRandomData(size_t nbytes) {
   // MSVC doesn't accept uint8_t for std::independent_bits_engine<>
   typedef unsigned long UInt;  // NOLINT
@@ -61,7 +64,11 @@ template <typename FileType>
 class FileTestFixture : public ::testing::Test {
  public:
   void SetUp() {
-    path_ = "arrow-test-io-buffered-stream.txt";
+    ASSERT_OK_AND_ASSIGN(temp_dir_, TemporaryDir::Make("buffered-test-"));
+    path_ = temp_dir_->path()
+                .Join("arrow-test-io-buffered-stream.txt")
+                .ValueOrDie()
+                .ToString();
     EnsureFileDeleted();
   }
 
@@ -79,6 +86,7 @@ class FileTestFixture : public ::testing::Test {
   int fd_;
   std::shared_ptr<FileType> buffered_;
   std::string path_;
+  std::unique_ptr<TemporaryDir> temp_dir_;
 };
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/io/file_benchmark.cc b/cpp/src/arrow/io/file_benchmark.cc
index f0bf7b08974..5e7e55725da 100644
--- a/cpp/src/arrow/io/file_benchmark.cc
+++ b/cpp/src/arrow/io/file_benchmark.cc
@@ -15,7 +15,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/api.h"
 #include "arrow/io/buffered.h"
 #include "arrow/io/file.h"
 #include "arrow/testing/gtest_util.h"
diff --git a/cpp/src/arrow/io/file_test.cc b/cpp/src/arrow/io/file_test.cc
index 7fca2de3c18..7d3d1c621ce 100644
--- a/cpp/src/arrow/io/file_test.cc
+++ b/cpp/src/arrow/io/file_test.cc
@@ -54,16 +54,22 @@ using internal::FileOpenWritable;
 using internal::FileRead;
 using internal::FileSeek;
 using internal::PlatformFilename;
+using internal::TemporaryDir;
 
 namespace io {
 
 class FileTestFixture : public ::testing::Test {
  public:
   void SetUp() {
-    path_ = "arrow-test-io-file.txt";
+    ASSERT_OK_AND_ASSIGN(temp_dir_, TemporaryDir::Make("file-test-"));
+    path_ = TempFile("arrow-test-io-file.txt");
     EnsureFileDeleted();
   }
 
+  std::string TempFile(arrow::util::string_view path) {
+    return temp_dir_->path().Join(std::string(path)).ValueOrDie().ToString();
+  }
+
   void TearDown() { EnsureFileDeleted(); }
 
   void EnsureFileDeleted() {
@@ -73,6 +79,7 @@ class FileTestFixture : public ::testing::Test {
   }
 
  protected:
+  std::unique_ptr<TemporaryDir> temp_dir_;
   std::string path_;
 };
 
@@ -556,13 +563,24 @@ TEST_F(TestPipeIO, ReadableFileFails) {
 
 class TestMemoryMappedFile : public ::testing::Test, public MemoryMapFixture {
  public:
-  void TearDown() { MemoryMapFixture::TearDown(); }
+  void SetUp() override {
+    ASSERT_OK_AND_ASSIGN(temp_dir_, TemporaryDir::Make("memory-map-test-"));
+  }
+
+  void TearDown() override { MemoryMapFixture::TearDown(); }
+
+  std::string TempFile(arrow::util::string_view path) {
+    return temp_dir_->path().Join(std::string(path)).ValueOrDie().ToString();
+  }
+
+ protected:
+  std::unique_ptr<TemporaryDir> temp_dir_;
 };
 
 TEST_F(TestMemoryMappedFile, InvalidUsages) {}
 
 TEST_F(TestMemoryMappedFile, ZeroSizeFile) {
-  std::string path = "io-memory-map-zero-size";
+  std::string path = TempFile("io-memory-map-zero-size");
   ASSERT_OK_AND_ASSIGN(auto result, InitMemoryMap(0, path));
 
   ASSERT_OK_AND_EQ(0, result->Tell());
@@ -578,7 +596,7 @@ TEST_F(TestMemoryMappedFile, MapPartFile) {
 
   const int reps = 128;
 
-  std::string path = "io-memory-map-offset";
+  std::string path = TempFile("io-memory-map-offset");
 
   // file size = 128k
   CreateFile(path, reps * buffer_size);
@@ -622,7 +640,7 @@ TEST_F(TestMemoryMappedFile, WriteRead) {
 
   const int reps = 5;
 
-  std::string path = "io-memory-map-write-read-test";
+  std::string path = TempFile("io-memory-map-write-read-test");
   ASSERT_OK_AND_ASSIGN(auto result, InitMemoryMap(reps * buffer_size, path));
 
   int64_t position = 0;
@@ -641,7 +659,7 @@ TEST_F(TestMemoryMappedFile, ReadAsync) {
   std::vector<uint8_t> buffer(buffer_size);
   random_bytes(1024, 0, buffer.data());
 
-  std::string path = "io-memory-map-read-async-test";
+  std::string path = TempFile("io-memory-map-read-async-test");
   ASSERT_OK_AND_ASSIGN(auto mmap, InitMemoryMap(buffer_size, path));
   ASSERT_OK(mmap->Write(buffer.data(), buffer_size));
 
@@ -661,7 +679,7 @@ TEST_F(TestMemoryMappedFile, WillNeed) {
   std::vector<uint8_t> buffer(buffer_size);
   random_bytes(1024, 0, buffer.data());
 
-  std::string path = "io-memory-map-will-need-test";
+  std::string path = TempFile("io-memory-map-will-need-test");
   ASSERT_OK_AND_ASSIGN(auto mmap, InitMemoryMap(buffer_size, path));
   ASSERT_OK(mmap->Write(buffer.data(), buffer_size));
 
@@ -672,7 +690,7 @@ TEST_F(TestMemoryMappedFile, WillNeed) {
 }
 
 TEST_F(TestMemoryMappedFile, InvalidReads) {
-  std::string path = "io-memory-map-invalid-reads-test";
+  std::string path = TempFile("io-memory-map-invalid-reads-test");
   ASSERT_OK_AND_ASSIGN(auto result, InitMemoryMap(4096, path));
 
   uint8_t buffer[10];
@@ -692,7 +710,7 @@ TEST_F(TestMemoryMappedFile, WriteResizeRead) {
     random_bytes(buffer_size, 0, b.data());
   }
 
-  std::string path = "io-memory-map-write-read-test";
+  std::string path = TempFile("io-memory-map-write-read-test");
   ASSERT_OK_AND_ASSIGN(auto result, InitMemoryMap(buffer_size, path));
 
   int64_t position = 0;
@@ -716,7 +734,7 @@ TEST_F(TestMemoryMappedFile, ResizeRaisesOnExported) {
   std::vector<uint8_t> buffer(buffer_size);
   random_bytes(buffer_size, 0, buffer.data());
 
-  std::string path = "io-memory-map-write-read-test";
+  std::string path = TempFile("io-memory-map-write-read-test");
   ASSERT_OK_AND_ASSIGN(auto result, InitMemoryMap(buffer_size, path));
 
   ASSERT_OK(result->Write(buffer.data(), buffer_size));
@@ -745,7 +763,7 @@ TEST_F(TestMemoryMappedFile, WriteReadZeroInitSize) {
   std::vector<uint8_t> buffer(buffer_size);
   random_bytes(buffer_size, 0, buffer.data());
 
-  std::string path = "io-memory-map-write-read-test";
+  std::string path = TempFile("io-memory-map-write-read-test");
   ASSERT_OK_AND_ASSIGN(auto result, InitMemoryMap(0, path));
 
   ASSERT_OK(result->Resize(buffer_size));
@@ -761,7 +779,7 @@ TEST_F(TestMemoryMappedFile, WriteThenShrink) {
   std::vector<uint8_t> buffer(buffer_size);
   random_bytes(buffer_size, 0, buffer.data());
 
-  std::string path = "io-memory-map-write-read-test";
+  std::string path = TempFile("io-memory-map-write-read-test");
   ASSERT_OK_AND_ASSIGN(auto result, InitMemoryMap(buffer_size * 2, path));
 
   ASSERT_OK(result->Resize(buffer_size));
@@ -780,7 +798,7 @@ TEST_F(TestMemoryMappedFile, WriteThenShrinkToHalfThenWrite) {
   std::vector<uint8_t> buffer(buffer_size);
   random_bytes(buffer_size, 0, buffer.data());
 
-  std::string path = "io-memory-map-write-read-test";
+  std::string path = TempFile("io-memory-map-write-read-test");
   ASSERT_OK_AND_ASSIGN(auto result, InitMemoryMap(buffer_size, path));
 
   ASSERT_OK(result->Write(buffer.data(), buffer_size));
@@ -808,7 +826,7 @@ TEST_F(TestMemoryMappedFile, ResizeToZeroThanWrite) {
   std::vector<uint8_t> buffer(buffer_size);
   random_bytes(buffer_size, 0, buffer.data());
 
-  std::string path = "io-memory-map-write-read-test";
+  std::string path = TempFile("io-memory-map-write-read-test");
   ASSERT_OK_AND_ASSIGN(auto result, InitMemoryMap(buffer_size, path));
 
   // just a sanity check that writing works ook
@@ -841,7 +859,7 @@ TEST_F(TestMemoryMappedFile, WriteAt) {
   std::vector<uint8_t> buffer(buffer_size);
   random_bytes(buffer_size, 0, buffer.data());
 
-  std::string path = "io-memory-map-write-read-test";
+  std::string path = TempFile("io-memory-map-write-read-test");
   ASSERT_OK_AND_ASSIGN(auto result, InitMemoryMap(buffer_size, path));
 
   ASSERT_OK(result->WriteAt(0, buffer.data(), buffer_size / 2));
@@ -859,7 +877,7 @@ TEST_F(TestMemoryMappedFile, WriteBeyondEnd) {
   std::vector<uint8_t> buffer(buffer_size);
   random_bytes(buffer_size, 0, buffer.data());
 
-  std::string path = "io-memory-map-write-read-test";
+  std::string path = TempFile("io-memory-map-write-read-test");
   ASSERT_OK_AND_ASSIGN(auto result, InitMemoryMap(buffer_size, path));
 
   ASSERT_OK(result->Seek(1));
@@ -875,7 +893,7 @@ TEST_F(TestMemoryMappedFile, WriteAtBeyondEnd) {
   std::vector<uint8_t> buffer(buffer_size);
   random_bytes(buffer_size, 0, buffer.data());
 
-  std::string path = "io-memory-map-write-read-test";
+  std::string path = TempFile("io-memory-map-write-read-test");
   ASSERT_OK_AND_ASSIGN(auto result, InitMemoryMap(buffer_size, path));
 
   // Attempt to write beyond end of memory map
@@ -886,7 +904,7 @@ TEST_F(TestMemoryMappedFile, WriteAtBeyondEnd) {
 }
 
 TEST_F(TestMemoryMappedFile, GetSize) {
-  std::string path = "io-memory-map-get-size";
+  std::string path = TempFile("io-memory-map-get-size");
   ASSERT_OK_AND_ASSIGN(auto result, InitMemoryMap(16384, path));
 
   ASSERT_OK_AND_EQ(16384, result->GetSize());
@@ -902,7 +920,7 @@ TEST_F(TestMemoryMappedFile, ReadOnly) {
 
   const int reps = 5;
 
-  std::string path = "ipc-read-only-test";
+  std::string path = TempFile("ipc-read-only-test");
   ASSERT_OK_AND_ASSIGN(auto rwmmap, InitMemoryMap(reps * buffer_size, path));
 
   int64_t position = 0;
@@ -933,7 +951,7 @@ TEST_F(TestMemoryMappedFile, LARGE_MEMORY_TEST(ReadWriteOver4GbFile)) {
 
   const int64_t reps = 5000;
 
-  std::string path = "ipc-read-over-4gb-file-test";
+  std::string path = TempFile("ipc-read-over-4gb-file-test");
   ASSERT_OK_AND_ASSIGN(auto rwmmap, InitMemoryMap(reps * buffer_size, path));
   AppendFile(path);
 
@@ -964,7 +982,7 @@ TEST_F(TestMemoryMappedFile, RetainMemoryMapReference) {
 
   random_bytes(1024, 0, buffer.data());
 
-  std::string path = "ipc-read-only-test";
+  std::string path = TempFile("ipc-read-only-test");
   CreateFile(path, buffer_size);
 
   {
@@ -995,7 +1013,7 @@ TEST_F(TestMemoryMappedFile, InvalidMode) {
 
   random_bytes(1024, 0, buffer.data());
 
-  std::string path = "ipc-invalid-mode-test";
+  std::string path = TempFile("ipc-invalid-mode-test");
   CreateFile(path, buffer_size);
 
   ASSERT_OK_AND_ASSIGN(auto rommap, MemoryMappedFile::Open(path, FileMode::READ));
@@ -1015,7 +1033,7 @@ TEST_F(TestMemoryMappedFile, CastableToFileInterface) {
 
 TEST_F(TestMemoryMappedFile, ThreadSafety) {
   std::string data = "foobar";
-  std::string path = "ipc-multithreading-test";
+  std::string path = TempFile("ipc-multithreading-test");
   CreateFile(path, static_cast<int>(data.size()));
 
   ASSERT_OK_AND_ASSIGN(auto file, MemoryMappedFile::Open(path, FileMode::READWRITE));
diff --git a/cpp/src/arrow/io/memory_benchmark.cc b/cpp/src/arrow/io/memory_benchmark.cc
index eefcdb3df0e..fbb34f38654 100644
--- a/cpp/src/arrow/io/memory_benchmark.cc
+++ b/cpp/src/arrow/io/memory_benchmark.cc
@@ -17,7 +17,6 @@
 
 #include <iostream>
 
-#include "arrow/api.h"
 #include "arrow/io/memory.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/testing/util.h"
diff --git a/cpp/src/arrow/io/memory_test.cc b/cpp/src/arrow/io/memory_test.cc
index 8aef23d956c..059e2ef360b 100644
--- a/cpp/src/arrow/io/memory_test.cc
+++ b/cpp/src/arrow/io/memory_test.cc
@@ -203,11 +203,11 @@ TEST(TestBufferReader, Peek) {
   ASSERT_OK_AND_ASSIGN(view, reader.Peek(4));
 
   ASSERT_EQ(4, view.size());
-  ASSERT_EQ(data.substr(0, 4), view.to_string());
+  ASSERT_EQ(data.substr(0, 4), std::string(view));
 
   ASSERT_OK_AND_ASSIGN(view, reader.Peek(20));
   ASSERT_EQ(data.size(), view.size());
-  ASSERT_EQ(data, view.to_string());
+  ASSERT_EQ(data, std::string(view));
 }
 
 TEST(TestBufferReader, ReadAsync) {
diff --git a/cpp/src/arrow/io/slow.cc b/cpp/src/arrow/io/slow.cc
index 62a39bbcf41..1042691fa59 100644
--- a/cpp/src/arrow/io/slow.cc
+++ b/cpp/src/arrow/io/slow.cc
@@ -28,6 +28,7 @@
 #include "arrow/io/util_internal.h"
 #include "arrow/result.h"
 #include "arrow/status.h"
+#include "arrow/util/io_util.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
@@ -64,8 +65,8 @@ void LatencyGenerator::Sleep() {
 }
 
 std::shared_ptr<LatencyGenerator> LatencyGenerator::Make(double average_latency) {
-  auto seed = static_cast<int32_t>(std::random_device()());
-  return std::make_shared<LatencyGeneratorImpl>(average_latency, seed);
+  return std::make_shared<LatencyGeneratorImpl>(
+      average_latency, static_cast<int32_t>(::arrow::internal::GetRandomSeed()));
 }
 
 std::shared_ptr<LatencyGenerator> LatencyGenerator::Make(double average_latency,
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 63fc605f2b1..335a858dc35 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -45,8 +45,9 @@ function(ADD_ARROW_IPC_TEST REL_TEST_NAME)
 endfunction()
 
 add_arrow_test(feather_test)
-add_arrow_ipc_test(read_write_test)
 add_arrow_ipc_test(json_simple_test)
+add_arrow_ipc_test(read_write_test)
+add_arrow_ipc_test(tensor_test)
 
 # Headers: top level
 arrow_install_all_headers("arrow/ipc")
diff --git a/cpp/src/arrow/ipc/dictionary.cc b/cpp/src/arrow/ipc/dictionary.cc
index 8a4f4b642e1..31f5199e79b 100644
--- a/cpp/src/arrow/ipc/dictionary.cc
+++ b/cpp/src/arrow/ipc/dictionary.cc
@@ -20,6 +20,7 @@
 #include <algorithm>
 #include <cstdint>
 #include <memory>
+#include <unordered_map>
 #include <utility>
 #include <vector>
 
@@ -30,6 +31,14 @@
 #include "arrow/status.h"
 #include "arrow/type.h"
 #include "arrow/util/checked_cast.h"
+#include "arrow/util/logging.h"
+
+namespace std {
+template <>
+struct hash<arrow::FieldPath> {
+  size_t operator()(const arrow::FieldPath& path) const { return path.hash(); }
+};
+}  // namespace std
 
 namespace arrow {
 
@@ -37,164 +46,233 @@ using internal::checked_cast;
 
 namespace ipc {
 
+using internal::FieldPosition;
+
 // ----------------------------------------------------------------------
+// DictionaryFieldMapper implementation
 
-DictionaryMemo::DictionaryMemo() {}
+struct DictionaryFieldMapper::Impl {
+  using FieldPathMap = std::unordered_map<FieldPath, int64_t>;
 
-Status DictionaryMemo::GetDictionaryType(int64_t id,
-                                         std::shared_ptr<DataType>* type) const {
-  auto it = id_to_type_.find(id);
-  if (it == id_to_type_.end()) {
-    return Status::KeyError("No record of dictionary type with id ", id);
-  }
-  *type = it->second;
-  return Status::OK();
-}
+  FieldPathMap field_path_to_id;
 
-// Returns KeyError if dictionary not found
-Status DictionaryMemo::GetDictionary(int64_t id,
-                                     std::shared_ptr<Array>* dictionary) const {
-  auto it = id_to_dictionary_.find(id);
-  if (it == id_to_dictionary_.end()) {
-    return Status::KeyError("Dictionary with id ", id, " not found");
+  void ImportSchema(const Schema& schema) {
+    ImportFields(FieldPosition(), schema.fields());
   }
-  *dictionary = it->second;
-  return Status::OK();
-}
 
-Status DictionaryMemo::AddFieldInternal(int64_t id, const std::shared_ptr<Field>& field) {
-  field_to_id_[field.get()] = id;
+  Status AddSchemaFields(const Schema& schema) {
+    if (!field_path_to_id.empty()) {
+      return Status::Invalid("Non-empty DictionaryFieldMapper");
+    }
+    ImportSchema(schema);
+    return Status::OK();
+  }
 
-  auto field_type = field->type();
-  if (field_type->id() == Type::EXTENSION) {
-    field_type = checked_cast<const ExtensionType&>(*field_type).storage_type();
+  Status AddField(int64_t id, std::vector<int> field_path) {
+    const auto pair = field_path_to_id.emplace(FieldPath(std::move(field_path)), id);
+    if (!pair.second) {
+      return Status::KeyError("Field already mapped to id");
+    }
+    return Status::OK();
   }
-  if (field_type->id() != Type::DICTIONARY) {
-    return Status::Invalid("Field type was not DictionaryType: ", field_type->ToString());
+
+  Result<int64_t> GetFieldId(std::vector<int> field_path) const {
+    const auto it = field_path_to_id.find(FieldPath(std::move(field_path)));
+    if (it == field_path_to_id.end()) {
+      return Status::KeyError("Dictionary field not found");
+    }
+    return it->second;
   }
 
-  std::shared_ptr<DataType> value_type =
-      checked_cast<const DictionaryType&>(*field_type).value_type();
+  int num_fields() const { return static_cast<int>(field_path_to_id.size()); }
 
-  // Add the value type for the dictionary
-  auto it = id_to_type_.find(id);
-  if (it != id_to_type_.end()) {
-    if (!it->second->Equals(*value_type)) {
-      return Status::Invalid("Field with dictionary id 0 seen but had type ",
-                             it->second->ToString(), "and not ", value_type->ToString());
+ private:
+  void ImportFields(const FieldPosition& pos,
+                    const std::vector<std::shared_ptr<Field>>& fields) {
+    for (int i = 0; i < static_cast<int>(fields.size()); ++i) {
+      ImportField(pos.child(i), *fields[i]);
     }
-  } else {
-    // Newly-observed dictionary id
-    id_to_type_[id] = value_type;
   }
-  return Status::OK();
-}
 
-Status DictionaryMemo::GetOrAssignId(const std::shared_ptr<Field>& field, int64_t* out) {
-  auto it = field_to_id_.find(field.get());
-  if (it != field_to_id_.end()) {
-    // Field already observed, return the id
-    *out = it->second;
-  } else {
-    int64_t new_id = *out = static_cast<int64_t>(field_to_id_.size());
-    RETURN_NOT_OK(AddFieldInternal(new_id, field));
+  void ImportField(const FieldPosition& pos, const Field& field) {
+    const DataType* type = field.type().get();
+    if (type->id() == Type::EXTENSION) {
+      type = checked_cast<const ExtensionType&>(*type).storage_type().get();
+    }
+    if (type->id() == Type::DICTIONARY) {
+      InsertPath(pos);
+      // Import nested dictionaries
+      ImportFields(pos,
+                   checked_cast<const DictionaryType&>(*type).value_type()->fields());
+    } else {
+      ImportFields(pos, type->fields());
+    }
   }
-  return Status::OK();
-}
 
-Status DictionaryMemo::AddField(int64_t id, const std::shared_ptr<Field>& field) {
-  auto it = field_to_id_.find(field.get());
-  if (it != field_to_id_.end()) {
-    return Status::KeyError("Field is already in memo: ", field->ToString());
-  } else {
-    RETURN_NOT_OK(AddFieldInternal(id, field));
-    return Status::OK();
+  void InsertPath(const FieldPosition& pos) {
+    const int64_t id = field_path_to_id.size();
+    const auto pair = field_path_to_id.emplace(FieldPath(pos.path()), id);
+    DCHECK(pair.second);  // was inserted
+    ARROW_UNUSED(pair);
   }
+};
+
+DictionaryFieldMapper::DictionaryFieldMapper() : impl_(new Impl) {}
+
+DictionaryFieldMapper::DictionaryFieldMapper(const Schema& schema) : impl_(new Impl) {
+  impl_->ImportSchema(schema);
 }
 
-Status DictionaryMemo::GetId(const Field* field, int64_t* id) const {
-  auto it = field_to_id_.find(field);
-  if (it != field_to_id_.end()) {
-    // Field recorded, return the id
-    *id = it->second;
-    return Status::OK();
-  } else {
-    return Status::KeyError("Field with memory address ",
-                            reinterpret_cast<int64_t>(field), " not found");
+DictionaryFieldMapper::~DictionaryFieldMapper() {}
+
+Status DictionaryFieldMapper::AddSchemaFields(const Schema& schema) {
+  return impl_->AddSchemaFields(schema);
+}
+
+Status DictionaryFieldMapper::AddField(int64_t id, std::vector<int> field_path) {
+  return impl_->AddField(id, std::move(field_path));
+}
+
+Result<int64_t> DictionaryFieldMapper::GetFieldId(std::vector<int> field_path) const {
+  return impl_->GetFieldId(std::move(field_path));
+}
+
+int DictionaryFieldMapper::num_fields() const { return impl_->num_fields(); }
+
+// ----------------------------------------------------------------------
+// DictionaryMemo implementation
+
+struct DictionaryMemo::Impl {
+  // Map of dictionary id to dictionary array(s) (several in case of deltas)
+  std::unordered_map<int64_t, ArrayDataVector> id_to_dictionary_;
+  std::unordered_map<int64_t, std::shared_ptr<DataType>> id_to_type_;
+  DictionaryFieldMapper mapper_;
+
+  Result<decltype(id_to_dictionary_)::iterator> FindDictionary(int64_t id) {
+    auto it = id_to_dictionary_.find(id);
+    if (it == id_to_dictionary_.end()) {
+      return Status::KeyError("Dictionary with id ", id, " not found");
+    }
+    return it;
   }
+
+  Result<std::shared_ptr<ArrayData>> ReifyDictionary(int64_t id, MemoryPool* pool) {
+    ARROW_ASSIGN_OR_RAISE(auto it, FindDictionary(id));
+    ArrayDataVector* data_vector = &it->second;
+
+    DCHECK(!data_vector->empty());
+    if (data_vector->size() > 1) {
+      // There are deltas, we need to concatenate them to the first dictionary.
+      ArrayVector to_combine;
+      to_combine.reserve(data_vector->size());
+      // IMPORTANT: At this point, the dictionary data may be untrusted.
+      // We need to validate it, as concatenation can crash on invalid or
+      // corrupted data.  Full validation is necessary for certain types
+      // (for example nested dictionaries).
+      // XXX: this won't work if there are unresolved nested dictionaries.
+      for (const auto& data : *data_vector) {
+        to_combine.push_back(MakeArray(data));
+        RETURN_NOT_OK(to_combine.back()->ValidateFull());
+      }
+      ARROW_ASSIGN_OR_RAISE(auto combined_dict, Concatenate(to_combine, pool));
+      *data_vector = {combined_dict->data()};
+    }
+
+    return data_vector->back();
+  }
+};
+
+DictionaryMemo::DictionaryMemo() : impl_(new Impl()) {}
+
+DictionaryMemo::~DictionaryMemo() {}
+
+DictionaryFieldMapper& DictionaryMemo::fields() { return impl_->mapper_; }
+
+const DictionaryFieldMapper& DictionaryMemo::fields() const { return impl_->mapper_; }
+
+Result<std::shared_ptr<DataType>> DictionaryMemo::GetDictionaryType(int64_t id) const {
+  const auto it = impl_->id_to_type_.find(id);
+  if (it == impl_->id_to_type_.end()) {
+    return Status::KeyError("No record of dictionary type with id ", id);
+  }
+  return it->second;
 }
 
-bool DictionaryMemo::HasDictionary(const Field& field) const {
-  auto it = field_to_id_.find(&field);
-  return it != field_to_id_.end();
+// Returns KeyError if dictionary not found
+Result<std::shared_ptr<ArrayData>> DictionaryMemo::GetDictionary(int64_t id,
+                                                                 MemoryPool* pool) const {
+  return impl_->ReifyDictionary(id, pool);
+}
+
+Status DictionaryMemo::AddDictionaryType(int64_t id,
+                                         const std::shared_ptr<DataType>& type) {
+  // AddDictionaryType expects the dict value type
+  DCHECK_NE(type->id(), Type::DICTIONARY);
+  const auto pair = impl_->id_to_type_.emplace(id, type);
+  if (!pair.second && !pair.first->second->Equals(*type)) {
+    return Status::KeyError("Conflicting dictionary types for id ", id);
+  }
+  return Status::OK();
 }
 
 bool DictionaryMemo::HasDictionary(int64_t id) const {
-  auto it = id_to_dictionary_.find(id);
-  return it != id_to_dictionary_.end();
+  const auto it = impl_->id_to_dictionary_.find(id);
+  return it != impl_->id_to_dictionary_.end();
 }
 
 Status DictionaryMemo::AddDictionary(int64_t id,
-                                     const std::shared_ptr<Array>& dictionary) {
-  if (HasDictionary(id)) {
+                                     const std::shared_ptr<ArrayData>& dictionary) {
+  const auto pair = impl_->id_to_dictionary_.emplace(id, ArrayDataVector{dictionary});
+  if (!pair.second) {
     return Status::KeyError("Dictionary with id ", id, " already exists");
   }
-  id_to_dictionary_[id] = dictionary;
   return Status::OK();
 }
 
 Status DictionaryMemo::AddDictionaryDelta(int64_t id,
-                                          const std::shared_ptr<Array>& dictionary,
-                                          MemoryPool* pool) {
-  std::shared_ptr<Array> originalDict, combinedDict;
-  RETURN_NOT_OK(GetDictionary(id, &originalDict));
-  ArrayVector dictsToCombine{originalDict, dictionary};
-  ARROW_ASSIGN_OR_RAISE(combinedDict, Concatenate(dictsToCombine, pool));
-  id_to_dictionary_[id] = combinedDict;
+                                          const std::shared_ptr<ArrayData>& dictionary) {
+  ARROW_ASSIGN_OR_RAISE(auto it, impl_->FindDictionary(id));
+  it->second.push_back(dictionary);
   return Status::OK();
 }
 
-Status DictionaryMemo::AddOrReplaceDictionary(int64_t id,
-                                              const std::shared_ptr<Array>& dictionary) {
-  id_to_dictionary_[id] = dictionary;
-  return Status::OK();
-}
+Result<bool> DictionaryMemo::AddOrReplaceDictionary(
+    int64_t id, const std::shared_ptr<ArrayData>& dictionary) {
+  ArrayDataVector value{dictionary};
 
-DictionaryMemo::DictionaryVector DictionaryMemo::dictionaries() const {
-  // Sort dictionaries by ascending id.   This ensures that, in the case
-  // of nested dictionaries, inner dictionaries are emitted before outer
-  // dictionaries.
-  // XXX This shouldn't be required.  On the IPC write path, the
-  // DictionaryMemo only needs to store a vector of dictionaries
-  // (by-id lookups are only needed on the IPC read path).
-  using DictEntry = typename DictionaryVector::value_type;
-
-  std::vector<DictEntry> dict_entries(id_to_dictionary_.size());
-  std::copy(id_to_dictionary_.begin(), id_to_dictionary_.end(), dict_entries.begin());
-
-  const auto compare_entries = [](const DictEntry& l, const DictEntry& r) {
-    return l.first < r.first;
-  };
-  std::sort(dict_entries.begin(), dict_entries.end(), compare_entries);
-  return dict_entries;
+  auto pair = impl_->id_to_dictionary_.emplace(id, value);
+  if (pair.second) {
+    // Inserted
+    return true;
+  } else {
+    // Update existing value
+    pair.first->second = std::move(value);
+    return false;
+  }
 }
 
 // ----------------------------------------------------------------------
 // CollectDictionaries implementation
 
+namespace {
+
 struct DictionaryCollector {
-  DictionaryMemo* dictionary_memo_;
+  const DictionaryFieldMapper& mapper_;
+  DictionaryVector dictionaries_;
 
-  Status WalkChildren(const DataType& type, const Array& array) {
+  Status WalkChildren(const FieldPosition& position, const DataType& type,
+                      const Array& array) {
     for (int i = 0; i < type.num_fields(); ++i) {
       auto boxed_child = MakeArray(array.data()->child_data[i]);
-      RETURN_NOT_OK(Visit(type.field(i), boxed_child.get()));
+      RETURN_NOT_OK(Visit(position.child(i), type.field(i), boxed_child.get()));
     }
     return Status::OK();
   }
 
-  Status Visit(const std::shared_ptr<Field>& field, const Array* array) {
+  Status Visit(const FieldPosition& position, const std::shared_ptr<Field>& field,
+               const Array* array) {
     const DataType* type = array->type().get();
+
     if (type->id() == Type::EXTENSION) {
       type = checked_cast<const ExtensionType&>(*type).storage_type().get();
       array = checked_cast<const ExtensionArray&>(*array).storage().get();
@@ -202,31 +280,93 @@ struct DictionaryCollector {
     if (type->id() == Type::DICTIONARY) {
       const auto& dict_array = checked_cast<const DictionaryArray&>(*array);
       auto dictionary = dict_array.dictionary();
-      int64_t id = -1;
-      RETURN_NOT_OK(dictionary_memo_->GetOrAssignId(field, &id));
-      RETURN_NOT_OK(dictionary_memo_->AddDictionary(id, dictionary));
 
-      // Traverse the dictionary to gather any nested dictionaries
+      // Traverse the dictionary to first gather any nested dictionaries
+      // (so that they appear in the output before their parent)
       const auto& dict_type = checked_cast<const DictionaryType&>(*type);
-      RETURN_NOT_OK(WalkChildren(*dict_type.value_type(), *dictionary));
+      RETURN_NOT_OK(WalkChildren(position, *dict_type.value_type(), *dictionary));
+
+      // Then record the dictionary itself
+      ARROW_ASSIGN_OR_RAISE(int64_t id, mapper_.GetFieldId(position.path()));
+      dictionaries_.emplace_back(id, dictionary);
     } else {
-      RETURN_NOT_OK(WalkChildren(*type, *array));
+      RETURN_NOT_OK(WalkChildren(position, *type, *array));
     }
     return Status::OK();
   }
 
   Status Collect(const RecordBatch& batch) {
+    FieldPosition position;
     const Schema& schema = *batch.schema();
+    dictionaries_.reserve(mapper_.num_fields());
+
     for (int i = 0; i < schema.num_fields(); ++i) {
-      RETURN_NOT_OK(Visit(schema.field(i), batch.column(i).get()));
+      RETURN_NOT_OK(Visit(position.child(i), schema.field(i), batch.column(i).get()));
     }
     return Status::OK();
   }
 };
 
+struct DictionaryResolver {
+  const DictionaryMemo& memo_;
+  MemoryPool* pool_;
+
+  Status VisitChildren(const ArrayDataVector& data_vector, FieldPosition parent_pos) {
+    int i = 0;
+    for (const auto& data : data_vector) {
+      // Some data entries may be missing if reading only a subset of the schema
+      if (data != nullptr) {
+        RETURN_NOT_OK(VisitField(parent_pos.child(i), data.get()));
+      }
+      ++i;
+    }
+    return Status::OK();
+  }
+
+  Status VisitField(FieldPosition field_pos, ArrayData* data) {
+    const DataType* type = data->type.get();
+    if (type->id() == Type::EXTENSION) {
+      type = checked_cast<const ExtensionType&>(*type).storage_type().get();
+    }
+    if (type->id() == Type::DICTIONARY) {
+      ARROW_ASSIGN_OR_RAISE(const int64_t id,
+                            memo_.fields().GetFieldId(field_pos.path()));
+      ARROW_ASSIGN_OR_RAISE(data->dictionary, memo_.GetDictionary(id, pool_));
+      // Resolve nested dictionary data
+      RETURN_NOT_OK(VisitField(field_pos, data->dictionary.get()));
+    }
+    // Resolve child data
+    return VisitChildren(data->child_data, field_pos);
+  }
+};
+
+}  // namespace
+
+Result<DictionaryVector> CollectDictionaries(const RecordBatch& batch,
+                                             const DictionaryFieldMapper& mapper) {
+  DictionaryCollector collector{mapper, {}};
+  RETURN_NOT_OK(collector.Collect(batch));
+  return std::move(collector.dictionaries_);
+}
+
+namespace internal {
+
 Status CollectDictionaries(const RecordBatch& batch, DictionaryMemo* memo) {
-  DictionaryCollector collector{memo};
-  return collector.Collect(batch);
+  RETURN_NOT_OK(memo->fields().AddSchemaFields(*batch.schema()));
+  ARROW_ASSIGN_OR_RAISE(const auto dictionaries,
+                        CollectDictionaries(batch, memo->fields()));
+  for (const auto& pair : dictionaries) {
+    RETURN_NOT_OK(memo->AddDictionary(pair.first, pair.second->data()));
+  }
+  return Status::OK();
+}
+
+}  // namespace internal
+
+Status ResolveDictionaries(const ArrayDataVector& columns, const DictionaryMemo& memo,
+                           MemoryPool* pool) {
+  DictionaryResolver resolver{memo, pool};
+  return resolver.VisitChildren(columns, FieldPosition());
 }
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/dictionary.h b/cpp/src/arrow/ipc/dictionary.h
index c8b347cf182..263443a43cf 100644
--- a/cpp/src/arrow/ipc/dictionary.h
+++ b/cpp/src/arrow/ipc/dictionary.h
@@ -21,96 +21,153 @@
 
 #include <cstdint>
 #include <memory>
-#include <unordered_map>
 #include <utility>
 #include <vector>
 
-#include "arrow/memory_pool.h"
+#include "arrow/result.h"
 #include "arrow/status.h"
+#include "arrow/type_fwd.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
+namespace ipc {
 
-class Array;
-class DataType;
-class Field;
-class RecordBatch;
+namespace internal {
 
-namespace ipc {
+class FieldPosition {
+ public:
+  FieldPosition() : parent_(NULLPTR), index_(-1), depth_(0) {}
+
+  FieldPosition child(int index) const { return {this, index}; }
+
+  std::vector<int> path() const {
+    std::vector<int> path(depth_);
+    const FieldPosition* cur = this;
+    for (int i = depth_ - 1; i >= 0; --i) {
+      path[i] = cur->index_;
+      cur = cur->parent_;
+    }
+    return path;
+  }
+
+ protected:
+  FieldPosition(const FieldPosition* parent, int index)
+      : parent_(parent), index_(index), depth_(parent->depth_ + 1) {}
+
+  const FieldPosition* parent_;
+  int index_;
+  int depth_;
+};
 
-/// \brief Memoization data structure for assigning id numbers to
-/// dictionaries and tracking their current state through possible
-/// deltas in an IPC stream
-class ARROW_EXPORT DictionaryMemo {
+}  // namespace internal
+
+/// \brief Map fields in a schema to dictionary ids
+///
+/// The mapping is structural, i.e. the field path (as a vector of indices)
+/// is associated to the dictionary id.  A dictionary id may be associated
+/// to multiple fields.
+class ARROW_EXPORT DictionaryFieldMapper {
  public:
-  using DictionaryVector = std::vector<std::pair<int64_t, std::shared_ptr<Array>>>;
+  DictionaryFieldMapper();
+  explicit DictionaryFieldMapper(const Schema& schema);
+  ~DictionaryFieldMapper();
+
+  Status AddSchemaFields(const Schema& schema);
+  Status AddField(int64_t id, std::vector<int> field_path);
 
+  Result<int64_t> GetFieldId(std::vector<int> field_path) const;
+
+  int num_fields() const;
+
+ private:
+  struct Impl;
+  std::unique_ptr<Impl> impl_;
+};
+
+using DictionaryVector = std::vector<std::pair<int64_t, std::shared_ptr<Array>>>;
+
+/// \brief Memoization data structure for reading dictionaries from IPC streams
+///
+/// This structure tracks the following associations:
+/// - field position (structural) -> dictionary id
+/// - dictionary id -> value type
+/// - dictionary id -> dictionary (value) data
+///
+/// Together, they allow resolving dictionary data when reading an IPC stream,
+/// using metadata recorded in the schema message and data recorded in the
+/// dictionary batch messages (see ResolveDictionaries).
+///
+/// This structure isn't useful for writing an IPC stream, where only
+/// DictionaryFieldMapper is necessary.
+class ARROW_EXPORT DictionaryMemo {
+ public:
   DictionaryMemo();
-  DictionaryMemo(DictionaryMemo&&) = default;
-  DictionaryMemo& operator=(DictionaryMemo&&) = default;
+  ~DictionaryMemo();
+
+  DictionaryFieldMapper& fields();
+  const DictionaryFieldMapper& fields() const;
 
   /// \brief Return current dictionary corresponding to a particular
   /// id. Returns KeyError if id not found
-  Status GetDictionary(int64_t id, std::shared_ptr<Array>* dictionary) const;
+  Result<std::shared_ptr<ArrayData>> GetDictionary(int64_t id, MemoryPool* pool) const;
 
   /// \brief Return dictionary value type corresponding to a
-  /// particular dictionary id. This permits multiple fields to
-  /// reference the same dictionary in IPC and JSON
-  Status GetDictionaryType(int64_t id, std::shared_ptr<DataType>* type) const;
-
-  /// \brief Return id for dictionary, computing new id if necessary
-  Status GetOrAssignId(const std::shared_ptr<Field>& field, int64_t* out);
-
-  /// \brief Return id for dictionary if it exists, otherwise return
-  /// KeyError
-  Status GetId(const Field* type, int64_t* id) const;
-
-  /// \brief Return true if dictionary for type is in this memo
-  bool HasDictionary(const Field& type) const;
+  /// particular dictionary id.
+  Result<std::shared_ptr<DataType>> GetDictionaryType(int64_t id) const;
 
   /// \brief Return true if we have a dictionary for the input id
   bool HasDictionary(int64_t id) const;
 
-  /// \brief Add field to the memo, return KeyError if already present
-  Status AddField(int64_t id, const std::shared_ptr<Field>& field);
+  /// \brief Add a dictionary value type to the memo with a particular id.
+  /// Returns KeyError if a different type is already registered with the same id.
+  Status AddDictionaryType(int64_t id, const std::shared_ptr<DataType>& type);
 
   /// \brief Add a dictionary to the memo with a particular id. Returns
   /// KeyError if that dictionary already exists
-  Status AddDictionary(int64_t id, const std::shared_ptr<Array>& dictionary);
+  Status AddDictionary(int64_t id, const std::shared_ptr<ArrayData>& dictionary);
 
   /// \brief Append a dictionary delta to the memo with a particular id. Returns
   /// KeyError if that dictionary does not exists
-  Status AddDictionaryDelta(int64_t id, const std::shared_ptr<Array>& dictionary,
-                            MemoryPool* pool);
+  Status AddDictionaryDelta(int64_t id, const std::shared_ptr<ArrayData>& dictionary);
 
   /// \brief Add a dictionary to the memo if it does not have one with the id,
   /// otherwise, replace the dictionary with the new one.
-  Status AddOrReplaceDictionary(int64_t id, const std::shared_ptr<Array>& dictionary);
-
-  /// \brief The stored dictionaries, in ascending id order.
-  DictionaryVector dictionaries() const;
-
-  /// \brief The number of fields tracked in the memo
-  int num_fields() const { return static_cast<int>(field_to_id_.size()); }
-  int num_dictionaries() const { return static_cast<int>(id_to_dictionary_.size()); }
+  ///
+  /// Return true if the dictionary was added, false if replaced.
+  Result<bool> AddOrReplaceDictionary(int64_t id,
+                                      const std::shared_ptr<ArrayData>& dictionary);
 
  private:
-  Status AddFieldInternal(int64_t id, const std::shared_ptr<Field>& field);
+  struct Impl;
+  std::unique_ptr<Impl> impl_;
+};
 
-  // Dictionary memory addresses, to track whether a particular
-  // dictionary-encoded field has been seen before
-  std::unordered_map<const Field*, int64_t> field_to_id_;
+// For writing: collect dictionary entries to write to the IPC stream, in order
+// (i.e. inner dictionaries before dependent outer dictionaries).
+ARROW_EXPORT
+Result<DictionaryVector> CollectDictionaries(const RecordBatch& batch,
+                                             const DictionaryFieldMapper& mapper);
 
-  // Map of dictionary id to dictionary array
-  std::unordered_map<int64_t, std::shared_ptr<Array>> id_to_dictionary_;
-  std::unordered_map<int64_t, std::shared_ptr<DataType>> id_to_type_;
+// For reading: resolve all dictionaries in columns, according to the field
+// mapping and dictionary arrays stored in memo.
+// Columns may be sparse, i.e. some entries may be left null
+// (e.g. if an inclusion mask was used).
+ARROW_EXPORT
+Status ResolveDictionaries(const ArrayDataVector& columns, const DictionaryMemo& memo,
+                           MemoryPool* pool);
 
-  ARROW_DISALLOW_COPY_AND_ASSIGN(DictionaryMemo);
-};
+namespace internal {
 
+// Like CollectDictionaries above, but uses the memo's DictionaryFieldMapper
+// and all collected dictionaries are added to the memo using AddDictionary.
+//
+// This is used as a shortcut in some roundtripping tests (to avoid emitting
+// any actual dictionary batches).
 ARROW_EXPORT
 Status CollectDictionaries(const RecordBatch& batch, DictionaryMemo* memo);
 
+}  // namespace internal
+
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index 5f757b4e92c..5ce8885341c 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -795,11 +795,12 @@ Status WriteTable(const Table& table, io::OutputStream* dst,
     return WriteFeatherV1(table, dst);
   } else {
     IpcWriteOptions ipc_options = IpcWriteOptions::Defaults();
-    ipc_options.compression = properties.compression;
-    ipc_options.compression_level = properties.compression_level;
+    ARROW_ASSIGN_OR_RAISE(
+        ipc_options.codec,
+        util::Codec::Create(properties.compression, properties.compression_level));
 
     std::shared_ptr<RecordBatchWriter> writer;
-    ARROW_ASSIGN_OR_RAISE(writer, NewFileWriter(dst, table.schema(), ipc_options));
+    ARROW_ASSIGN_OR_RAISE(writer, MakeFileWriter(dst, table.schema(), ipc_options));
     RETURN_NOT_OK(writer->WriteTable(table, properties.chunksize));
     return writer->Close();
   }
diff --git a/cpp/src/arrow/ipc/feather.h b/cpp/src/arrow/ipc/feather.h
index 187e8277072..b40893c408f 100644
--- a/cpp/src/arrow/ipc/feather.h
+++ b/cpp/src/arrow/ipc/feather.h
@@ -25,7 +25,7 @@
 #include <string>
 #include <vector>
 
-#include "arrow/result.h"
+#include "arrow/type_fwd.h"
 #include "arrow/util/compression.h"
 #include "arrow/util/visibility.h"
 
diff --git a/cpp/src/arrow/ipc/feather_test.cc b/cpp/src/arrow/ipc/feather_test.cc
index a2961116c17..a7ee31f18c1 100644
--- a/cpp/src/arrow/ipc/feather_test.cc
+++ b/cpp/src/arrow/ipc/feather_test.cc
@@ -75,6 +75,7 @@ class TestFeatherBase {
     DoWrite(*table);
     std::shared_ptr<Table> result;
     ASSERT_OK(reader_->Read(&result));
+    ASSERT_OK(result->ValidateFull());
     if (table->num_rows() > 0) {
       AssertTablesEqual(*table, *result);
     } else {
@@ -101,6 +102,7 @@ class TestFeatherBase {
 
     std::shared_ptr<Table> read_table;
     ASSERT_OK(reader_->Read(&read_table));
+    ASSERT_OK(read_table->ValidateFull());
     AssertTablesEqual(*table, *read_table);
   }
 
@@ -329,23 +331,25 @@ namespace {
 
 const std::vector<test::MakeRecordBatch*> kBatchCases = {
     &ipc::test::MakeIntRecordBatch,
-    &ipc::test::MakeBooleanBatch,
-    &ipc::test::MakeFloatBatch,
     &ipc::test::MakeListRecordBatch,
+    &ipc::test::MakeFixedSizeListRecordBatch,
     &ipc::test::MakeNonNullRecordBatch,
     &ipc::test::MakeDeeplyNestedList,
     &ipc::test::MakeStringTypesRecordBatchWithNulls,
     &ipc::test::MakeStruct,
     &ipc::test::MakeUnion,
     &ipc::test::MakeDictionary,
-    &ipc::test::MakeDictionaryFlat,
     &ipc::test::MakeNestedDictionary,
+    &ipc::test::MakeMap,
+    &ipc::test::MakeMapOfDictionary,
     &ipc::test::MakeDates,
     &ipc::test::MakeTimestamps,
     &ipc::test::MakeTimes,
     &ipc::test::MakeFWBinary,
     &ipc::test::MakeNull,
     &ipc::test::MakeDecimal,
+    &ipc::test::MakeBooleanBatch,
+    &ipc::test::MakeFloatBatch,
     &ipc::test::MakeIntervals};
 
 }  // namespace
diff --git a/cpp/src/arrow/ipc/file_to_stream.cc b/cpp/src/arrow/ipc/file_to_stream.cc
index 8570b6f3051..c15eb6de21f 100644
--- a/cpp/src/arrow/ipc/file_to_stream.cc
+++ b/cpp/src/arrow/ipc/file_to_stream.cc
@@ -39,8 +39,8 @@ Status ConvertToStream(const char* path) {
 
   ARROW_ASSIGN_OR_RAISE(auto in_file, io::ReadableFile::Open(path));
   ARROW_ASSIGN_OR_RAISE(auto reader, ipc::RecordBatchFileReader::Open(in_file.get()));
-  ARROW_ASSIGN_OR_RAISE(auto writer, ipc::NewStreamWriter(&sink, reader->schema(),
-                                                          IpcWriteOptions::Defaults()));
+  ARROW_ASSIGN_OR_RAISE(auto writer, ipc::MakeStreamWriter(&sink, reader->schema(),
+                                                           IpcWriteOptions::Defaults()));
   for (int i = 0; i < reader->num_record_batches(); ++i) {
     ARROW_ASSIGN_OR_RAISE(std::shared_ptr<RecordBatch> chunk, reader->ReadRecordBatch(i));
     RETURN_NOT_OK(writer->WriteRecordBatch(*chunk));
diff --git a/cpp/src/arrow/ipc/generate_fuzz_corpus.cc b/cpp/src/arrow/ipc/generate_fuzz_corpus.cc
index e34abedd116..9e640030540 100644
--- a/cpp/src/arrow/ipc/generate_fuzz_corpus.cc
+++ b/cpp/src/arrow/ipc/generate_fuzz_corpus.cc
@@ -99,9 +99,9 @@ Result<std::shared_ptr<Buffer>> SerializeRecordBatch(
   ARROW_ASSIGN_OR_RAISE(auto sink, io::BufferOutputStream::Create(1024));
   std::shared_ptr<RecordBatchWriter> writer;
   if (is_stream_format) {
-    ARROW_ASSIGN_OR_RAISE(writer, NewStreamWriter(sink.get(), batch->schema()));
+    ARROW_ASSIGN_OR_RAISE(writer, MakeStreamWriter(sink, batch->schema()));
   } else {
-    ARROW_ASSIGN_OR_RAISE(writer, NewFileWriter(sink.get(), batch->schema()));
+    ARROW_ASSIGN_OR_RAISE(writer, MakeFileWriter(sink, batch->schema()));
   }
   RETURN_NOT_OK(writer->WriteRecordBatch(*batch));
   RETURN_NOT_OK(writer->Close());
diff --git a/cpp/src/arrow/ipc/json_simple.cc b/cpp/src/arrow/ipc/json_simple.cc
index d307373c129..fba8194aeb1 100644
--- a/cpp/src/arrow/ipc/json_simple.cc
+++ b/cpp/src/arrow/ipc/json_simple.cc
@@ -22,11 +22,18 @@
 #include <vector>
 
 #include "arrow/array/array_dict.h"
-#include "arrow/builder.h"
+#include "arrow/array/builder_binary.h"
+#include "arrow/array/builder_decimal.h"
+#include "arrow/array/builder_dict.h"
+#include "arrow/array/builder_nested.h"
+#include "arrow/array/builder_primitive.h"
+#include "arrow/array/builder_time.h"
+#include "arrow/array/builder_union.h"
 #include "arrow/ipc/json_simple.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/checked_cast.h"
 #include "arrow/util/decimal.h"
+#include "arrow/util/logging.h"
 #include "arrow/util/string_view.h"
 #include "arrow/util/value_parsing.h"
 
@@ -50,9 +57,11 @@ namespace json {
 using ::arrow::internal::checked_cast;
 using ::arrow::internal::checked_pointer_cast;
 
-static constexpr auto kParseFlags = rj::kParseFullPrecisionFlag | rj::kParseNanAndInfFlag;
+namespace {
 
-static Status JSONTypeError(const char* expected_type, rj::Type json_type) {
+constexpr auto kParseFlags = rj::kParseFullPrecisionFlag | rj::kParseNanAndInfFlag;
+
+Status JSONTypeError(const char* expected_type, rj::Type json_type) {
   return Status::Invalid("Expected ", expected_type, " or null, got JSON type ",
                          json_type);
 }
@@ -101,6 +110,22 @@ class ConcreteConverter : public Converter {
     }
     return Status::OK();
   }
+
+  const std::shared_ptr<DataType>& value_type() {
+    if (type_->id() != Type::DICTIONARY) {
+      return type_;
+    }
+    return checked_cast<const DictionaryType&>(*type_).value_type();
+  }
+
+  template <typename BuilderType>
+  Status MakeConcreteBuilder(std::shared_ptr<BuilderType>* out) {
+    std::unique_ptr<ArrayBuilder> builder;
+    RETURN_NOT_OK(MakeBuilder(default_memory_pool(), this->type_, &builder));
+    *out = checked_pointer_cast<BuilderType>(std::move(builder));
+    DCHECK(*out);
+    return Status::OK();
+  }
 };
 
 // ------------------------------------------------------------------------
@@ -213,20 +238,17 @@ enable_if_physical_floating_point<T, Status> ConvertNumber(const rj::Value& json
 // ------------------------------------------------------------------------
 // Converter for int arrays
 
-template <typename Type>
-class IntegerConverter final : public ConcreteConverter<IntegerConverter<Type>> {
+template <typename Type, typename BuilderType = typename TypeTraits<Type>::BuilderType>
+class IntegerConverter final
+    : public ConcreteConverter<IntegerConverter<Type, BuilderType>> {
   using c_type = typename Type::c_type;
+
   static constexpr auto is_signed = std::is_signed<c_type>::value;
 
  public:
   explicit IntegerConverter(const std::shared_ptr<DataType>& type) { this->type_ = type; }
 
-  Status Init() override {
-    std::unique_ptr<ArrayBuilder> builder;
-    RETURN_NOT_OK(MakeBuilder(default_memory_pool(), this->type_, &builder));
-    builder_ = checked_pointer_cast<NumericBuilder<Type>>(std::move(builder));
-    return Status::OK();
-  }
+  Status Init() override { return this->MakeConcreteBuilder(&builder_); }
 
   Status AppendValue(const rj::Value& json_obj) override {
     if (json_obj.IsNull()) {
@@ -240,21 +262,20 @@ class IntegerConverter final : public ConcreteConverter<IntegerConverter<Type>>
   std::shared_ptr<ArrayBuilder> builder() override { return builder_; }
 
  private:
-  std::shared_ptr<NumericBuilder<Type>> builder_;
+  std::shared_ptr<BuilderType> builder_;
 };
 
 // ------------------------------------------------------------------------
 // Converter for float arrays
 
-template <typename Type>
-class FloatConverter final : public ConcreteConverter<FloatConverter<Type>> {
+template <typename Type, typename BuilderType = typename TypeTraits<Type>::BuilderType>
+class FloatConverter final : public ConcreteConverter<FloatConverter<Type, BuilderType>> {
   using c_type = typename Type::c_type;
 
  public:
-  explicit FloatConverter(const std::shared_ptr<DataType>& type) {
-    this->type_ = type;
-    builder_ = std::make_shared<NumericBuilder<Type>>();
-  }
+  explicit FloatConverter(const std::shared_ptr<DataType>& type) { this->type_ = type; }
+
+  Status Init() override { return this->MakeConcreteBuilder(&builder_); }
 
   Status AppendValue(const rj::Value& json_obj) override {
     if (json_obj.IsNull()) {
@@ -268,29 +289,33 @@ class FloatConverter final : public ConcreteConverter<FloatConverter<Type>> {
   std::shared_ptr<ArrayBuilder> builder() override { return builder_; }
 
  private:
-  std::shared_ptr<NumericBuilder<Type>> builder_;
+  std::shared_ptr<BuilderType> builder_;
 };
 
 // ------------------------------------------------------------------------
 // Converter for decimal arrays
 
-class DecimalConverter final : public ConcreteConverter<DecimalConverter> {
+template <typename DecimalSubtype, typename DecimalValue, typename BuilderType>
+class DecimalConverter final
+    : public ConcreteConverter<
+          DecimalConverter<DecimalSubtype, DecimalValue, BuilderType>> {
  public:
   explicit DecimalConverter(const std::shared_ptr<DataType>& type) {
     this->type_ = type;
-    decimal_type_ = checked_cast<Decimal128Type*>(type.get());
-    builder_ = std::make_shared<DecimalBuilder>(type);
+    decimal_type_ = &checked_cast<const DecimalSubtype&>(*this->value_type());
   }
 
+  Status Init() override { return this->MakeConcreteBuilder(&builder_); }
+
   Status AppendValue(const rj::Value& json_obj) override {
     if (json_obj.IsNull()) {
       return this->AppendNull();
     }
     if (json_obj.IsString()) {
       int32_t precision, scale;
-      Decimal128 d;
+      DecimalValue d;
       auto view = util::string_view(json_obj.GetString(), json_obj.GetStringLength());
-      RETURN_NOT_OK(Decimal128::FromString(view, &d, &precision, &scale));
+      RETURN_NOT_OK(DecimalValue::FromString(view, &d, &precision, &scale));
       if (scale != decimal_type_->scale()) {
         return Status::Invalid("Invalid scale for decimal: expected ",
                                decimal_type_->scale(), ", got ", scale);
@@ -303,10 +328,15 @@ class DecimalConverter final : public ConcreteConverter<DecimalConverter> {
   std::shared_ptr<ArrayBuilder> builder() override { return builder_; }
 
  private:
-  std::shared_ptr<DecimalBuilder> builder_;
-  Decimal128Type* decimal_type_;
+  std::shared_ptr<BuilderType> builder_;
+  const DecimalSubtype* decimal_type_;
 };
 
+template <typename BuilderType = typename TypeTraits<Decimal128Type>::BuilderType>
+using Decimal128Converter = DecimalConverter<Decimal128Type, Decimal128, BuilderType>;
+template <typename BuilderType = typename TypeTraits<Decimal256Type>::BuilderType>
+using Decimal256Converter = DecimalConverter<Decimal256Type, Decimal256, BuilderType>;
+
 // ------------------------------------------------------------------------
 // Converter for timestamp arrays
 
@@ -381,15 +411,13 @@ class DayTimeIntervalConverter final
 // ------------------------------------------------------------------------
 // Converter for binary and string arrays
 
-template <typename TYPE>
-class StringConverter final : public ConcreteConverter<StringConverter<TYPE>> {
+template <typename Type, typename BuilderType = typename TypeTraits<Type>::BuilderType>
+class StringConverter final
+    : public ConcreteConverter<StringConverter<Type, BuilderType>> {
  public:
-  using BuilderType = typename TypeTraits<TYPE>::BuilderType;
+  explicit StringConverter(const std::shared_ptr<DataType>& type) { this->type_ = type; }
 
-  explicit StringConverter(const std::shared_ptr<DataType>& type) {
-    this->type_ = type;
-    builder_ = std::make_shared<BuilderType>(type, default_memory_pool());
-  }
+  Status Init() override { return this->MakeConcreteBuilder(&builder_); }
 
   Status AppendValue(const rj::Value& json_obj) override {
     if (json_obj.IsNull()) {
@@ -412,14 +440,16 @@ class StringConverter final : public ConcreteConverter<StringConverter<TYPE>> {
 // ------------------------------------------------------------------------
 // Converter for fixed-size binary arrays
 
+template <typename BuilderType = typename TypeTraits<FixedSizeBinaryType>::BuilderType>
 class FixedSizeBinaryConverter final
-    : public ConcreteConverter<FixedSizeBinaryConverter> {
+    : public ConcreteConverter<FixedSizeBinaryConverter<BuilderType>> {
  public:
   explicit FixedSizeBinaryConverter(const std::shared_ptr<DataType>& type) {
     this->type_ = type;
-    builder_ = std::make_shared<FixedSizeBinaryBuilder>(type, default_memory_pool());
   }
 
+  Status Init() override { return this->MakeConcreteBuilder(&builder_); }
+
   Status AppendValue(const rj::Value& json_obj) override {
     if (json_obj.IsNull()) {
       return this->AppendNull();
@@ -441,7 +471,7 @@ class FixedSizeBinaryConverter final
   std::shared_ptr<ArrayBuilder> builder() override { return builder_; }
 
  private:
-  std::shared_ptr<FixedSizeBinaryBuilder> builder_;
+  std::shared_ptr<BuilderType> builder_;
 };
 
 // ------------------------------------------------------------------------
@@ -720,14 +750,63 @@ class UnionConverter final : public ConcreteConverter<UnionConverter> {
 // ------------------------------------------------------------------------
 // General conversion functions
 
+Status ConversionNotImplemented(const std::shared_ptr<DataType>& type) {
+  return Status::NotImplemented("JSON conversion to ", type->ToString(),
+                                " not implemented");
+}
+
+Status GetDictConverter(const std::shared_ptr<DataType>& type,
+                        std::shared_ptr<Converter>* out) {
+  std::shared_ptr<Converter> res;
+
+  const auto value_type = checked_cast<const DictionaryType&>(*type).value_type();
+
+#define SIMPLE_CONVERTER_CASE(ID, CLASS, TYPE)                    \
+  case ID:                                                        \
+    res = std::make_shared<CLASS<DictionaryBuilder<TYPE>>>(type); \
+    break;
+
+#define PARAM_CONVERTER_CASE(ID, CLASS, TYPE)                           \
+  case ID:                                                              \
+    res = std::make_shared<CLASS<TYPE, DictionaryBuilder<TYPE>>>(type); \
+    break;
+
+  switch (value_type->id()) {
+    PARAM_CONVERTER_CASE(Type::INT8, IntegerConverter, Int8Type)
+    PARAM_CONVERTER_CASE(Type::INT16, IntegerConverter, Int16Type)
+    PARAM_CONVERTER_CASE(Type::INT32, IntegerConverter, Int32Type)
+    PARAM_CONVERTER_CASE(Type::INT64, IntegerConverter, Int64Type)
+    PARAM_CONVERTER_CASE(Type::UINT8, IntegerConverter, UInt8Type)
+    PARAM_CONVERTER_CASE(Type::UINT16, IntegerConverter, UInt16Type)
+    PARAM_CONVERTER_CASE(Type::UINT32, IntegerConverter, UInt32Type)
+    PARAM_CONVERTER_CASE(Type::UINT64, IntegerConverter, UInt64Type)
+    PARAM_CONVERTER_CASE(Type::STRING, StringConverter, StringType)
+    PARAM_CONVERTER_CASE(Type::BINARY, StringConverter, BinaryType)
+    PARAM_CONVERTER_CASE(Type::LARGE_STRING, StringConverter, LargeStringType)
+    PARAM_CONVERTER_CASE(Type::LARGE_BINARY, StringConverter, LargeBinaryType)
+    SIMPLE_CONVERTER_CASE(Type::FIXED_SIZE_BINARY, FixedSizeBinaryConverter,
+                          FixedSizeBinaryType)
+    SIMPLE_CONVERTER_CASE(Type::DECIMAL128, Decimal128Converter, Decimal128Type)
+    SIMPLE_CONVERTER_CASE(Type::DECIMAL256, Decimal256Converter, Decimal256Type)
+    default:
+      return ConversionNotImplemented(type);
+  }
+
+#undef SIMPLE_CONVERTER_CASE
+#undef PARAM_CONVERTER_CASE
+
+  RETURN_NOT_OK(res->Init());
+  *out = res;
+  return Status::OK();
+}
+
 Status GetConverter(const std::shared_ptr<DataType>& type,
                     std::shared_ptr<Converter>* out) {
-  std::shared_ptr<Converter> res;
+  if (type->id() == Type::DICTIONARY) {
+    return GetDictConverter(type, out);
+  }
 
-  auto not_implemented = [&]() -> Status {
-    return Status::NotImplemented("JSON conversion to ", type->ToString(),
-                                  " not implemented");
-  };
+  std::shared_ptr<Converter> res;
 
 #define SIMPLE_CONVERTER_CASE(ID, CLASS) \
   case ID:                               \
@@ -763,14 +842,15 @@ Status GetConverter(const std::shared_ptr<DataType>& type,
     SIMPLE_CONVERTER_CASE(Type::BINARY, StringConverter<BinaryType>)
     SIMPLE_CONVERTER_CASE(Type::LARGE_STRING, StringConverter<LargeStringType>)
     SIMPLE_CONVERTER_CASE(Type::LARGE_BINARY, StringConverter<LargeBinaryType>)
-    SIMPLE_CONVERTER_CASE(Type::FIXED_SIZE_BINARY, FixedSizeBinaryConverter)
-    SIMPLE_CONVERTER_CASE(Type::DECIMAL, DecimalConverter)
+    SIMPLE_CONVERTER_CASE(Type::FIXED_SIZE_BINARY, FixedSizeBinaryConverter<>)
+    SIMPLE_CONVERTER_CASE(Type::DECIMAL128, Decimal128Converter<>)
+    SIMPLE_CONVERTER_CASE(Type::DECIMAL256, Decimal256Converter<>)
     SIMPLE_CONVERTER_CASE(Type::SPARSE_UNION, UnionConverter)
     SIMPLE_CONVERTER_CASE(Type::DENSE_UNION, UnionConverter)
     SIMPLE_CONVERTER_CASE(Type::INTERVAL_MONTHS, IntegerConverter<MonthIntervalType>)
     SIMPLE_CONVERTER_CASE(Type::INTERVAL_DAY_TIME, DayTimeIntervalConverter)
     default:
-      return not_implemented();
+      return ConversionNotImplemented(type);
   }
 
 #undef SIMPLE_CONVERTER_CASE
@@ -780,6 +860,8 @@ Status GetConverter(const std::shared_ptr<DataType>& type,
   return Status::OK();
 }
 
+}  // namespace
+
 Status ArrayFromJSON(const std::shared_ptr<DataType>& type, util::string_view json_string,
                      std::shared_ptr<Array>* out) {
   std::shared_ptr<Converter> converter;
diff --git a/cpp/src/arrow/ipc/json_simple_test.cc b/cpp/src/arrow/ipc/json_simple_test.cc
index fe1b0273e57..c5358ac89f1 100644
--- a/cpp/src/arrow/ipc/json_simple_test.cc
+++ b/cpp/src/arrow/ipc/json_simple_test.cc
@@ -30,6 +30,10 @@
 #include <gtest/gtest.h>
 
 #include "arrow/array.h"
+#include "arrow/array/builder_decimal.h"
+#include "arrow/array/builder_nested.h"
+#include "arrow/array/builder_primitive.h"
+#include "arrow/array/builder_time.h"
 #include "arrow/ipc/json_simple.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/type.h"
@@ -113,6 +117,25 @@ void AssertJSONArray(const std::shared_ptr<DataType>& type, const std::string& j
   AssertArraysEqual(*expected, *actual);
 }
 
+void AssertJSONDictArray(const std::shared_ptr<DataType>& index_type,
+                         const std::shared_ptr<DataType>& value_type,
+                         const std::string& json,
+                         const std::string& expected_indices_json,
+                         const std::string& expected_values_json) {
+  auto type = dictionary(index_type, value_type);
+  std::shared_ptr<Array> actual, expected_indices, expected_values;
+
+  ASSERT_OK(ArrayFromJSON(index_type, expected_indices_json, &expected_indices));
+  ASSERT_OK(ArrayFromJSON(value_type, expected_values_json, &expected_values));
+
+  ASSERT_OK(ArrayFromJSON(type, json, &actual));
+  ASSERT_OK(actual->ValidateFull());
+
+  const auto& dict_array = checked_cast<const DictionaryArray&>(*actual);
+  AssertArraysEqual(*expected_indices, *dict_array.indices());
+  AssertArraysEqual(*expected_values, *dict_array.dictionary());
+}
+
 TEST(TestHelper, JSONArray) {
   // Test the JSONArray helper func
   std::string s =
@@ -133,7 +156,10 @@ TEST(TestHelper, SafeSignedAdd) {
 }
 
 template <typename T>
-class TestIntegers : public ::testing::Test {};
+class TestIntegers : public ::testing::Test {
+ public:
+  std::shared_ptr<DataType> type() { return TypeTraits<T>::type_singleton(); }
+};
 
 TYPED_TEST_SUITE_P(TestIntegers);
 
@@ -142,7 +168,7 @@ TYPED_TEST_P(TestIntegers, Basics) {
   using c_type = typename T::c_type;
 
   std::shared_ptr<Array> expected, actual;
-  std::shared_ptr<DataType> type = TypeTraits<T>::type_singleton();
+  auto type = this->type();
 
   AssertJSONArray<T>(type, "[]", {});
   AssertJSONArray<T>(type, "[4, 0, 5]", {4, 0, 5});
@@ -158,10 +184,8 @@ TYPED_TEST_P(TestIntegers, Basics) {
 }
 
 TYPED_TEST_P(TestIntegers, Errors) {
-  using T = TypeParam;
-
   std::shared_ptr<Array> array;
-  std::shared_ptr<DataType> type = TypeTraits<T>::type_singleton();
+  auto type = this->type();
 
   ASSERT_RAISES(Invalid, ArrayFromJSON(type, "", &array));
   ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[", &array));
@@ -177,7 +201,7 @@ TYPED_TEST_P(TestIntegers, OutOfBounds) {
   using c_type = typename T::c_type;
 
   std::shared_ptr<Array> array;
-  std::shared_ptr<DataType> type = TypeTraits<T>::type_singleton();
+  auto type = this->type();
 
   if (type->id() == Type::UINT64) {
     ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[18446744073709551616]", &array));
@@ -200,7 +224,21 @@ TYPED_TEST_P(TestIntegers, OutOfBounds) {
   }
 }
 
-REGISTER_TYPED_TEST_SUITE_P(TestIntegers, Basics, Errors, OutOfBounds);
+TYPED_TEST_P(TestIntegers, Dictionary) {
+  std::shared_ptr<Array> array;
+  std::shared_ptr<DataType> value_type = this->type();
+
+  if (value_type->id() == Type::HALF_FLOAT) {
+    // Unsupported, skip
+    return;
+  }
+
+  AssertJSONDictArray(int8(), value_type, "[1, 2, 3, null, 3, 1]",
+                      /*indices=*/"[0, 1, 2, null, 2, 0]",
+                      /*values=*/"[1, 2, 3]");
+}
+
+REGISTER_TYPED_TEST_SUITE_P(TestIntegers, Basics, Errors, OutOfBounds, Dictionary);
 
 INSTANTIATE_TYPED_TEST_SUITE_P(TestInt8, TestIntegers, Int8Type);
 INSTANTIATE_TYPED_TEST_SUITE_P(TestInt16, TestIntegers, Int16Type);
@@ -212,6 +250,66 @@ INSTANTIATE_TYPED_TEST_SUITE_P(TestUInt32, TestIntegers, UInt32Type);
 INSTANTIATE_TYPED_TEST_SUITE_P(TestUInt64, TestIntegers, UInt64Type);
 INSTANTIATE_TYPED_TEST_SUITE_P(TestHalfFloat, TestIntegers, HalfFloatType);
 
+template <typename T>
+class TestStrings : public ::testing::Test {
+ public:
+  std::shared_ptr<DataType> type() { return TypeTraits<T>::type_singleton(); }
+};
+
+TYPED_TEST_SUITE_P(TestStrings);
+
+TYPED_TEST_P(TestStrings, Basics) {
+  using T = TypeParam;
+  auto type = this->type();
+
+  std::shared_ptr<Array> expected, actual;
+
+  AssertJSONArray<T, std::string>(type, "[]", {});
+  AssertJSONArray<T, std::string>(type, "[\"\", \"foo\"]", {"", "foo"});
+  AssertJSONArray<T, std::string>(type, "[\"\", null]", {true, false}, {"", ""});
+  // NUL character in string
+  std::string s = "some";
+  s += '\x00';
+  s += "char";
+  AssertJSONArray<T, std::string>(type, "[\"\", \"some\\u0000char\"]", {"", s});
+  // UTF8 sequence in string
+  AssertJSONArray<T, std::string>(type, "[\"\xc3\xa9\"]", {"\xc3\xa9"});
+
+  if (!T::is_utf8) {
+    // Arbitrary binary (non-UTF8) sequence in string
+    s = "\xff\x9f";
+    AssertJSONArray<T, std::string>(type, "[\"" + s + "\"]", {s});
+  }
+
+  // Bytes < 0x20 can be represented as JSON unicode escapes
+  s = '\x00';
+  s += "\x1f";
+  AssertJSONArray<T, std::string>(type, "[\"\\u0000\\u001f\"]", {s});
+}
+
+TYPED_TEST_P(TestStrings, Errors) {
+  auto type = this->type();
+  std::shared_ptr<Array> array;
+
+  ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[0]", &array));
+  ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[[]]", &array));
+}
+
+TYPED_TEST_P(TestStrings, Dictionary) {
+  auto value_type = this->type();
+
+  AssertJSONDictArray(int16(), value_type, R"(["foo", "bar", null, "bar", "foo"])",
+                      /*indices=*/"[0, 1, null, 1, 0]",
+                      /*values=*/R"(["foo", "bar"])");
+}
+
+REGISTER_TYPED_TEST_SUITE_P(TestStrings, Basics, Errors, Dictionary);
+
+INSTANTIATE_TYPED_TEST_SUITE_P(TestString, TestStrings, StringType);
+INSTANTIATE_TYPED_TEST_SUITE_P(TestBinary, TestStrings, BinaryType);
+INSTANTIATE_TYPED_TEST_SUITE_P(TestLargeString, TestStrings, LargeStringType);
+INSTANTIATE_TYPED_TEST_SUITE_P(TestLargeBinary, TestStrings, LargeBinaryType);
+
 TEST(TestNull, Basics) {
   std::shared_ptr<DataType> type = null();
   std::shared_ptr<Array> expected, actual;
@@ -297,50 +395,6 @@ TEST(TestDouble, Errors) {
   ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[true]", &array));
 }
 
-TEST(TestString, Basics) {
-  // String type
-  std::shared_ptr<DataType> type = utf8();
-  std::shared_ptr<Array> expected, actual;
-
-  AssertJSONArray<StringType, std::string>(type, "[]", {});
-  AssertJSONArray<StringType, std::string>(type, "[\"\", \"foo\"]", {"", "foo"});
-  AssertJSONArray<StringType, std::string>(type, "[\"\", null]", {true, false}, {"", ""});
-  // NUL character in string
-  std::string s = "some";
-  s += '\x00';
-  s += "char";
-  AssertJSONArray<StringType, std::string>(type, "[\"\", \"some\\u0000char\"]", {"", s});
-  // UTF8 sequence in string
-  AssertJSONArray<StringType, std::string>(type, "[\"\xc3\xa9\"]", {"\xc3\xa9"});
-
-  // Binary type
-  type = binary();
-  AssertJSONArray<BinaryType, std::string>(type, "[\"\", \"foo\", null]",
-                                           {true, true, false}, {"", "foo", ""});
-  // Arbitrary binary (non-UTF8) sequence in string
-  s = "\xff\x9f";
-  AssertJSONArray<BinaryType, std::string>(type, "[\"" + s + "\"]", {s});
-  // Bytes < 0x20 can be represented as JSON unicode escapes
-  s = '\x00';
-  s += "\x1f";
-  AssertJSONArray<BinaryType, std::string>(type, "[\"\\u0000\\u001f\"]", {s});
-}
-
-TEST(TestLargeString, Basics) {
-  // Similar as TestString above, only testing the basics
-  std::shared_ptr<DataType> type = large_utf8();
-  std::shared_ptr<Array> expected, actual;
-
-  AssertJSONArray<LargeStringType, std::string>(type, "[\"\", \"foo\"]", {"", "foo"});
-  AssertJSONArray<LargeStringType, std::string>(type, "[\"\", null]", {true, false},
-                                                {"", ""});
-
-  // Large binary type
-  type = large_binary();
-  AssertJSONArray<LargeBinaryType, std::string>(type, "[\"\", \"foo\", null]",
-                                                {true, true, false}, {"", "foo", ""});
-}
-
 TEST(TestTimestamp, Basics) {
   // Timestamp type
   auto type = timestamp(TimeUnit::SECOND);
@@ -407,14 +461,6 @@ TEST(TestDayTimeInterval, Basics) {
                                        {{1, -600}, {}});
 }
 
-TEST(TestString, Errors) {
-  std::shared_ptr<DataType> type = utf8();
-  std::shared_ptr<Array> array;
-
-  ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[0]", &array));
-  ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[[]]", &array));
-}
-
 TEST(TestFixedSizeBinary, Basics) {
   std::shared_ptr<DataType> type = fixed_size_binary(3);
   std::shared_ptr<Array> expected, actual;
@@ -440,14 +486,26 @@ TEST(TestFixedSizeBinary, Errors) {
   ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[\"abcd\"]", &array));
 }
 
-TEST(TestDecimal, Basics) {
-  std::shared_ptr<DataType> type = decimal(10, 4);
+TEST(TestFixedSizeBinary, Dictionary) {
+  std::shared_ptr<DataType> type = fixed_size_binary(3);
+
+  AssertJSONDictArray(int8(), type, R"(["foo", "bar", "foo", null])",
+                      /*indices=*/"[0, 1, 0, null]",
+                      /*values=*/R"(["foo", "bar"])");
+
+  // Invalid length
+  std::shared_ptr<Array> array;
+  ASSERT_RAISES(Invalid, ArrayFromJSON(dictionary(int8(), type), R"(["x"])", &array));
+}
+
+template <typename DecimalValue, typename DecimalBuilder>
+void TestDecimalBasic(std::shared_ptr<DataType> type) {
   std::shared_ptr<Array> expected, actual;
 
   ASSERT_OK(ArrayFromJSON(type, "[]", &actual));
   ASSERT_OK(actual->ValidateFull());
   {
-    Decimal128Builder builder(type);
+    DecimalBuilder builder(type);
     ASSERT_OK(builder.Finish(&expected));
   }
   AssertArraysEqual(*expected, *actual);
@@ -455,9 +513,9 @@ TEST(TestDecimal, Basics) {
   ASSERT_OK(ArrayFromJSON(type, "[\"123.4567\", \"-78.9000\"]", &actual));
   ASSERT_OK(actual->ValidateFull());
   {
-    Decimal128Builder builder(type);
-    ASSERT_OK(builder.Append(Decimal128(1234567)));
-    ASSERT_OK(builder.Append(Decimal128(-789000)));
+    DecimalBuilder builder(type);
+    ASSERT_OK(builder.Append(DecimalValue(1234567)));
+    ASSERT_OK(builder.Append(DecimalValue(-789000)));
     ASSERT_OK(builder.Finish(&expected));
   }
   AssertArraysEqual(*expected, *actual);
@@ -465,23 +523,41 @@ TEST(TestDecimal, Basics) {
   ASSERT_OK(ArrayFromJSON(type, "[\"123.4567\", null]", &actual));
   ASSERT_OK(actual->ValidateFull());
   {
-    Decimal128Builder builder(type);
-    ASSERT_OK(builder.Append(Decimal128(1234567)));
+    DecimalBuilder builder(type);
+    ASSERT_OK(builder.Append(DecimalValue(1234567)));
     ASSERT_OK(builder.AppendNull());
     ASSERT_OK(builder.Finish(&expected));
   }
   AssertArraysEqual(*expected, *actual);
 }
 
+TEST(TestDecimal128, Basics) {
+  TestDecimalBasic<Decimal128, Decimal128Builder>(decimal128(10, 4));
+}
+
+TEST(TestDecimal256, Basics) {
+  TestDecimalBasic<Decimal256, Decimal256Builder>(decimal256(10, 4));
+}
+
 TEST(TestDecimal, Errors) {
-  std::shared_ptr<DataType> type = decimal(10, 4);
-  std::shared_ptr<Array> array;
+  for (std::shared_ptr<DataType> type : {decimal128(10, 4), decimal256(10, 4)}) {
+    std::shared_ptr<Array> array;
+
+    ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[0]", &array));
+    ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[12.3456]", &array));
+    // Bad scale
+    ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[\"12.345\"]", &array));
+    ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[\"12.34560\"]", &array));
+  }
+}
 
-  ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[0]", &array));
-  ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[12.3456]", &array));
-  // Bad scale
-  ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[\"12.345\"]", &array));
-  ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[\"12.34560\"]", &array));
+TEST(TestDecimal, Dictionary) {
+  for (std::shared_ptr<DataType> type : {decimal128(10, 2), decimal256(10, 2)}) {
+    AssertJSONDictArray(int32(), type,
+                        R"(["123.45", "-78.90", "-78.90", null, "123.45"])",
+                        /*indices=*/"[0, 1, 1, null, 0]",
+                        /*values=*/R"(["123.45", "-78.90"])");
+  }
 }
 
 TEST(TestList, IntegerList) {
@@ -1208,7 +1284,28 @@ TEST(TestSparseUnion, Errors) {
   ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[[8, true, 1]]", &array));
 }
 
-TEST(TestDictionary, Basics) {
+TEST(TestNestedDictionary, ListOfDict) {
+  auto index_type = int8();
+  auto value_type = utf8();
+  auto dict_type = dictionary(index_type, value_type);
+  auto type = list(dict_type);
+
+  std::shared_ptr<Array> array, expected, indices, values, dicts, offsets;
+
+  ASSERT_OK(ArrayFromJSON(type, R"([["ab", "cd", null], null, ["cd", "cd"]])", &array));
+  ASSERT_OK(array->ValidateFull());
+
+  // Build expected array
+  ASSERT_OK(ArrayFromJSON(index_type, "[0, 1, null, 1, 1]", &indices));
+  ASSERT_OK(ArrayFromJSON(value_type, R"(["ab", "cd"])", &values));
+  ASSERT_OK_AND_ASSIGN(dicts, DictionaryArray::FromArrays(dict_type, indices, values));
+  ASSERT_OK(ArrayFromJSON(int32(), "[0, null, 3, 5]", &offsets));
+  ASSERT_OK_AND_ASSIGN(expected, ListArray::FromArrays(*offsets, *dicts));
+
+  AssertArraysEqual(*expected, *array, /*verbose=*/true);
+}
+
+TEST(TestDictArrayFromJSON, Basics) {
   auto type = dictionary(int32(), utf8());
   auto array =
       DictArrayFromJSON(type, "[null, 2, 1, 0]", R"(["whiskey", "tango", "foxtrot"])");
@@ -1220,7 +1317,7 @@ TEST(TestDictionary, Basics) {
                       *array);
 }
 
-TEST(TestDictionary, Errors) {
+TEST(TestDictArrayFromJSON, Errors) {
   auto type = dictionary(int32(), utf8());
   std::shared_ptr<Array> array;
 
diff --git a/cpp/src/arrow/ipc/message.h b/cpp/src/arrow/ipc/message.h
index c6037b9a26c..6a7619d31b3 100644
--- a/cpp/src/arrow/ipc/message.h
+++ b/cpp/src/arrow/ipc/message.h
@@ -365,7 +365,7 @@ class ARROW_EXPORT MessageDecoder {
   ///   memcpy(buffer->mutable_data() + current_buffer_size,
   ///          small_chunk,
   ///          small_chunk_size);
-  ///   if (buffer->size() < decoder.next_requied_size()) {
+  ///   if (buffer->size() < decoder.next_required_size()) {
   ///     continue;
   ///   }
   ///   std::shared_ptr<arrow::Buffer> chunk(buffer.release());
diff --git a/cpp/src/arrow/ipc/metadata_internal.cc b/cpp/src/arrow/ipc/metadata_internal.cc
index b2420049394..250ac950208 100644
--- a/cpp/src/arrow/ipc/metadata_internal.cc
+++ b/cpp/src/arrow/ipc/metadata_internal.cc
@@ -153,10 +153,6 @@ Status FloatFromFlatbuffer(const flatbuf::FloatingPoint* float_data,
   return Status::OK();
 }
 
-// Forward declaration
-Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
-                         DictionaryMemo* dictionary_memo, FieldOffset* offset);
-
 Offset IntToFlatbuffer(FBB& fbb, int bitWidth, bool is_signed) {
   return flatbuf::CreateInt(fbb, bitWidth, is_signed).Union();
 }
@@ -165,17 +161,6 @@ Offset FloatToFlatbuffer(FBB& fbb, flatbuf::Precision precision) {
   return flatbuf::CreateFloatingPoint(fbb, precision).Union();
 }
 
-Status AppendChildFields(FBB& fbb, const DataType& type,
-                         std::vector<FieldOffset>* out_children,
-                         DictionaryMemo* dictionary_memo) {
-  FieldOffset field;
-  for (int i = 0; i < type.num_fields(); ++i) {
-    RETURN_NOT_OK(FieldToFlatbuffer(fbb, type.field(i), dictionary_memo, &field));
-    out_children->push_back(field);
-  }
-  return Status::OK();
-}
-
 // ----------------------------------------------------------------------
 // Union implementation
 
@@ -251,8 +236,6 @@ static inline TimeUnit::type FromFlatbufferUnit(flatbuf::TimeUnit unit) {
   return TimeUnit::SECOND;
 }
 
-constexpr int32_t kDecimalBitWidth = 128;
-
 Status ConcreteTypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
                                   const std::vector<std::shared_ptr<Field>>& children,
                                   std::shared_ptr<DataType>* out) {
@@ -275,8 +258,7 @@ Status ConcreteTypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
       return Status::OK();
     case flatbuf::Type::FixedSizeBinary: {
       auto fw_binary = static_cast<const flatbuf::FixedSizeBinary*>(type_data);
-      *out = fixed_size_binary(fw_binary->byteWidth());
-      return Status::OK();
+      return FixedSizeBinaryType::Make(fw_binary->byteWidth()).Value(out);
     }
     case flatbuf::Type::Utf8:
       *out = utf8();
@@ -289,10 +271,13 @@ Status ConcreteTypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
       return Status::OK();
     case flatbuf::Type::Decimal: {
       auto dec_type = static_cast<const flatbuf::Decimal*>(type_data);
-      if (dec_type->bitWidth() != kDecimalBitWidth) {
-        return Status::Invalid("Library only supports 128-bit decimal values");
+      if (dec_type->bitWidth() == 128) {
+        return Decimal128Type::Make(dec_type->precision(), dec_type->scale()).Value(out);
+      } else if (dec_type->bitWidth() == 256) {
+        return Decimal256Type::Make(dec_type->precision(), dec_type->scale()).Value(out);
+      } else {
+        return Status::Invalid("Library only supports 128-bit or 256-bit decimal values");
       }
-      return Decimal128Type::Make(dec_type->precision(), dec_type->scale()).Value(out);
     }
     case flatbuf::Type::Date: {
       auto date_type = static_cast<const flatbuf::Date*>(type_data);
@@ -440,20 +425,15 @@ Status TensorTypeToFlatbuffer(FBB& fbb, const DataType& type, flatbuf::Type* out
 }
 
 static Status GetDictionaryEncoding(FBB& fbb, const std::shared_ptr<Field>& field,
-                                    const DictionaryType& type, DictionaryMemo* memo,
+                                    const DictionaryType& type, int64_t dictionary_id,
                                     DictionaryOffset* out) {
-  int64_t dictionary_id = -1;
-  RETURN_NOT_OK(memo->GetOrAssignId(field, &dictionary_id));
-
   // We assume that the dictionary index type (as an integer) has already been
-  // validated elsewhere, and can safely assume we are dealing with signed
-  // integers
+  // validated elsewhere, and can safely assume we are dealing with integers
   const auto& index_type = checked_cast<const IntegerType&>(*type.index_type());
 
   auto index_type_offset =
       flatbuf::CreateInt(fbb, index_type.bit_width(), index_type.is_signed());
 
-  // TODO(wesm): ordered dictionaries
   *out = flatbuf::CreateDictionaryEncoding(fbb, dictionary_id, index_type_offset,
                                            type.ordered());
   return Status::OK();
@@ -474,8 +454,9 @@ static void AppendKeyValueMetadata(FBB& fbb, const KeyValueMetadata& metadata,
 
 class FieldToFlatbufferVisitor {
  public:
-  FieldToFlatbufferVisitor(FBB& fbb, DictionaryMemo* dictionary_memo)
-      : fbb_(fbb), dictionary_memo_(dictionary_memo) {}
+  FieldToFlatbufferVisitor(FBB& fbb, const DictionaryFieldMapper& mapper,
+                           const FieldPosition& field_pos)
+      : fbb_(fbb), mapper_(mapper), field_pos_(field_pos) {}
 
   Status VisitType(const DataType& type) { return VisitTypeInline(type, this); }
 
@@ -613,52 +594,62 @@ class FieldToFlatbufferVisitor {
     return Status::OK();
   }
 
-  Status Visit(const DecimalType& type) {
+  Status Visit(const Decimal128Type& type) {
     const auto& dec_type = checked_cast<const Decimal128Type&>(type);
     fb_type_ = flatbuf::Type::Decimal;
-    type_offset_ =
-        flatbuf::CreateDecimal(fbb_, dec_type.precision(), dec_type.scale()).Union();
+    type_offset_ = flatbuf::CreateDecimal(fbb_, dec_type.precision(), dec_type.scale(),
+                                          /*bitWidth=*/128)
+                       .Union();
+    return Status::OK();
+  }
+
+  Status Visit(const Decimal256Type& type) {
+    const auto& dec_type = checked_cast<const Decimal256Type&>(type);
+    fb_type_ = flatbuf::Type::Decimal;
+    type_offset_ = flatbuf::CreateDecimal(fbb_, dec_type.precision(), dec_type.scale(),
+                                          /*bitWith=*/256)
+                       .Union();
     return Status::OK();
   }
 
   Status Visit(const ListType& type) {
     fb_type_ = flatbuf::Type::List;
-    RETURN_NOT_OK(AppendChildFields(fbb_, type, &children_, dictionary_memo_));
+    RETURN_NOT_OK(VisitChildFields(type));
     type_offset_ = flatbuf::CreateList(fbb_).Union();
     return Status::OK();
   }
 
   Status Visit(const LargeListType& type) {
     fb_type_ = flatbuf::Type::LargeList;
-    RETURN_NOT_OK(AppendChildFields(fbb_, type, &children_, dictionary_memo_));
+    RETURN_NOT_OK(VisitChildFields(type));
     type_offset_ = flatbuf::CreateLargeList(fbb_).Union();
     return Status::OK();
   }
 
   Status Visit(const MapType& type) {
     fb_type_ = flatbuf::Type::Map;
-    RETURN_NOT_OK(AppendChildFields(fbb_, type, &children_, dictionary_memo_));
+    RETURN_NOT_OK(VisitChildFields(type));
     type_offset_ = flatbuf::CreateMap(fbb_, type.keys_sorted()).Union();
     return Status::OK();
   }
 
   Status Visit(const FixedSizeListType& type) {
     fb_type_ = flatbuf::Type::FixedSizeList;
-    RETURN_NOT_OK(AppendChildFields(fbb_, type, &children_, dictionary_memo_));
+    RETURN_NOT_OK(VisitChildFields(type));
     type_offset_ = flatbuf::CreateFixedSizeList(fbb_, type.list_size()).Union();
     return Status::OK();
   }
 
   Status Visit(const StructType& type) {
     fb_type_ = flatbuf::Type::Struct_;
-    RETURN_NOT_OK(AppendChildFields(fbb_, type, &children_, dictionary_memo_));
+    RETURN_NOT_OK(VisitChildFields(type));
     type_offset_ = flatbuf::CreateStruct_(fbb_).Union();
     return Status::OK();
   }
 
   Status Visit(const UnionType& type) {
     fb_type_ = flatbuf::Type::Union;
-    RETURN_NOT_OK(AppendChildFields(fbb_, type, &children_, dictionary_memo_));
+    RETURN_NOT_OK(VisitChildFields(type));
 
     const auto& union_type = checked_cast<const UnionType&>(type);
 
@@ -692,10 +683,18 @@ class FieldToFlatbufferVisitor {
     return Status::OK();
   }
 
+  Status VisitChildFields(const DataType& type) {
+    for (int i = 0; i < type.num_fields(); ++i) {
+      FieldOffset child_offset;
+      FieldToFlatbufferVisitor child_visitor(fbb_, mapper_, field_pos_.child(i));
+      RETURN_NOT_OK(child_visitor.GetResult(type.field(i), &child_offset));
+      children_.push_back(child_offset);
+    }
+    return Status::OK();
+  }
+
   Status GetResult(const std::shared_ptr<Field>& field, FieldOffset* offset) {
-    auto fb_name = fbb_.CreateString(field->name());
     RETURN_NOT_OK(VisitType(*field->type()));
-    auto fb_children = fbb_.CreateVector(children_.data(), children_.size());
 
     DictionaryOffset dictionary = 0;
     const DataType* storage_type = field->type().get();
@@ -704,13 +703,11 @@ class FieldToFlatbufferVisitor {
           checked_cast<const ExtensionType&>(*storage_type).storage_type().get();
     }
     if (storage_type->id() == Type::DICTIONARY) {
-      // Note we're emitting the dictionary encoding after potentially
-      // emitting any nested dictionaries in VisitType() above,
-      // so the outer dictionaries will naturally get an id larger than any
-      // inner dictionaries.
+      ARROW_ASSIGN_OR_RAISE(const auto dictionary_id,
+                            mapper_.GetFieldId(field_pos_.path()));
       RETURN_NOT_OK(GetDictionaryEncoding(
-          fbb_, field, checked_cast<const DictionaryType&>(*storage_type),
-          dictionary_memo_, &dictionary));
+          fbb_, field, checked_cast<const DictionaryType&>(*storage_type), dictionary_id,
+          &dictionary));
     }
 
     auto metadata = field->metadata();
@@ -728,6 +725,9 @@ class FieldToFlatbufferVisitor {
     if (key_values.size() > 0) {
       fb_custom_metadata = fbb_.CreateVector(key_values);
     }
+
+    auto fb_name = fbb_.CreateString(field->name());
+    auto fb_children = fbb_.CreateVector(children_.data(), children_.size());
     *offset =
         flatbuf::CreateField(fbb_, fb_name, field->nullable(), fb_type_, type_offset_,
                              dictionary, fb_children, fb_custom_metadata);
@@ -736,21 +736,16 @@ class FieldToFlatbufferVisitor {
 
  private:
   FBB& fbb_;
-  DictionaryMemo* dictionary_memo_;
+  const DictionaryFieldMapper& mapper_;
+  FieldPosition field_pos_;
   flatbuf::Type fb_type_;
   Offset type_offset_;
   std::vector<FieldOffset> children_;
   std::unordered_map<std::string, std::string> extra_type_metadata_;
 };
 
-Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
-                         DictionaryMemo* dictionary_memo, FieldOffset* offset) {
-  FieldToFlatbufferVisitor field_visitor(fbb, dictionary_memo);
-  return field_visitor.GetResult(field, offset);
-}
-
-Status FieldFromFlatbuffer(const flatbuf::Field* field, DictionaryMemo* dictionary_memo,
-                           std::shared_ptr<Field>* out) {
+Status FieldFromFlatbuffer(const flatbuf::Field* field, FieldPosition field_pos,
+                           DictionaryMemo* dictionary_memo, std::shared_ptr<Field>* out) {
   std::shared_ptr<DataType> type;
 
   std::shared_ptr<KeyValueMetadata> metadata;
@@ -762,8 +757,8 @@ Status FieldFromFlatbuffer(const flatbuf::Field* field, DictionaryMemo* dictiona
   CHECK_FLATBUFFERS_NOT_NULL(children, "Field.children");
   std::vector<std::shared_ptr<Field>> child_fields(children->size());
   for (int i = 0; i < static_cast<int>(children->size()); ++i) {
-    RETURN_NOT_OK(
-        FieldFromFlatbuffer(children->Get(i), dictionary_memo, &child_fields[i]));
+    RETURN_NOT_OK(FieldFromFlatbuffer(children->Get(i), field_pos.child(i),
+                                      dictionary_memo, &child_fields[i]));
   }
 
   // 2. Top-level concrete data type
@@ -774,6 +769,7 @@ Status FieldFromFlatbuffer(const flatbuf::Field* field, DictionaryMemo* dictiona
 
   // 3. Is it a dictionary type?
   int64_t dictionary_id = -1;
+  std::shared_ptr<DataType> dict_value_type;
   const flatbuf::DictionaryEncoding* encoding = field->dictionary();
   if (encoding != nullptr) {
     // The field is dictionary-encoded. Construct the DictionaryType
@@ -783,6 +779,7 @@ Status FieldFromFlatbuffer(const flatbuf::Field* field, DictionaryMemo* dictiona
     auto int_data = encoding->indexType();
     CHECK_FLATBUFFERS_NOT_NULL(int_data, "DictionaryEncoding.indexType");
     RETURN_NOT_OK(IntFromFlatbuffer(int_data, &index_type));
+    dict_value_type = type;
     ARROW_ASSIGN_OR_RAISE(type,
                           DictionaryType::Make(index_type, type, encoding->isOrdered()));
     dictionary_id = encoding->id();
@@ -817,7 +814,12 @@ Status FieldFromFlatbuffer(const flatbuf::Field* field, DictionaryMemo* dictiona
   *out =
       ::arrow::field(std::move(field_name), type, field->nullable(), std::move(metadata));
   if (dictionary_id != -1) {
-    RETURN_NOT_OK(dictionary_memo->AddField(dictionary_id, *out));
+    // We need both the id -> type mapping (to find the value type when
+    // reading a dictionary batch)
+    // and the field path -> id mapping (to find the dictionary when
+    // reading a record batch)
+    RETURN_NOT_OK(dictionary_memo->fields().AddField(dictionary_id, field_pos.path()));
+    RETURN_NOT_OK(dictionary_memo->AddDictionaryType(dictionary_id, dict_value_type));
   }
   return Status::OK();
 }
@@ -845,13 +847,15 @@ flatbuffers::Offset<KVVector> SerializeCustomMetadata(
   }
 }
 
-Status SchemaToFlatbuffer(FBB& fbb, const Schema& schema, DictionaryMemo* dictionary_memo,
+Status SchemaToFlatbuffer(FBB& fbb, const Schema& schema,
+                          const DictionaryFieldMapper& mapper,
                           flatbuffers::Offset<flatbuf::Schema>* out) {
-  /// Fields
   std::vector<FieldOffset> field_offsets;
+  FieldPosition pos;
   for (int i = 0; i < schema.num_fields(); ++i) {
     FieldOffset offset;
-    RETURN_NOT_OK(FieldToFlatbuffer(fbb, schema.field(i), dictionary_memo, &offset));
+    FieldToFlatbufferVisitor field_visitor(fbb, mapper, pos.child(i));
+    RETURN_NOT_OK(field_visitor.GetResult(schema.field(i), &offset));
     field_offsets.push_back(offset);
   }
 
@@ -864,12 +868,12 @@ Status SchemaToFlatbuffer(FBB& fbb, const Schema& schema, DictionaryMemo* dictio
 Result<std::shared_ptr<Buffer>> WriteFBMessage(
     FBB& fbb, flatbuf::MessageHeader header_type, flatbuffers::Offset<void> header,
     int64_t body_length, MetadataVersion version,
-    const std::shared_ptr<const KeyValueMetadata>& custom_metadata = nullptr) {
+    const std::shared_ptr<const KeyValueMetadata>& custom_metadata, MemoryPool* pool) {
   auto message = flatbuf::CreateMessage(fbb, MetadataVersionToFlatbuffer(version),
                                         header_type, header, body_length,
                                         SerializeCustomMetadata(fbb, custom_metadata));
   fbb.Finish(message);
-  return WriteFlatbufferBuilder(fbb);
+  return WriteFlatbufferBuilder(fbb, pool);
 }
 
 using FieldNodeVector =
@@ -909,15 +913,15 @@ static Status WriteBuffers(FBB& fbb, const std::vector<BufferMetadata>& buffers,
 
 static Status GetBodyCompression(FBB& fbb, const IpcWriteOptions& options,
                                  BodyCompressionOffset* out) {
-  if (options.compression != Compression::UNCOMPRESSED) {
+  if (options.codec != nullptr) {
     flatbuf::CompressionType codec;
-    if (options.compression == Compression::LZ4_FRAME) {
+    if (options.codec->compression_type() == Compression::LZ4_FRAME) {
       codec = flatbuf::CompressionType::LZ4_FRAME;
-    } else if (options.compression == Compression::ZSTD) {
+    } else if (options.codec->compression_type() == Compression::ZSTD) {
       codec = flatbuf::CompressionType::ZSTD;
     } else {
       return Status::Invalid("Unsupported IPC compression codec: ",
-                             util::Codec::GetCodecAsString(options.compression));
+                             options.codec->name());
     }
     *out = flatbuf::CreateBodyCompression(fbb, codec,
                                           flatbuf::BodyCompressionMethod::BUFFER);
@@ -1160,7 +1164,7 @@ Status GetKeyValueMetadata(const KVVector* fb_metadata,
   auto metadata = std::make_shared<KeyValueMetadata>();
 
   metadata->reserve(fb_metadata->size());
-  for (const auto& pair : *fb_metadata) {
+  for (const auto pair : *fb_metadata) {
     CHECK_FLATBUFFERS_NOT_NULL(pair->key(), "custom_metadata.key");
     CHECK_FLATBUFFERS_NOT_NULL(pair->value(), "custom_metadata.value");
     metadata->Append(pair->key()->str(), pair->value()->str());
@@ -1170,13 +1174,14 @@ Status GetKeyValueMetadata(const KVVector* fb_metadata,
   return Status::OK();
 }
 
-Status WriteSchemaMessage(const Schema& schema, DictionaryMemo* dictionary_memo,
+Status WriteSchemaMessage(const Schema& schema, const DictionaryFieldMapper& mapper,
                           const IpcWriteOptions& options, std::shared_ptr<Buffer>* out) {
   FBB fbb;
   flatbuffers::Offset<flatbuf::Schema> fb_schema;
-  RETURN_NOT_OK(SchemaToFlatbuffer(fbb, schema, dictionary_memo, &fb_schema));
+  RETURN_NOT_OK(SchemaToFlatbuffer(fbb, schema, mapper, &fb_schema));
   return WriteFBMessage(fbb, flatbuf::MessageHeader::Schema, fb_schema.Union(),
-                        /*body_length=*/0, options.metadata_version)
+                        /*body_length=*/0, options.metadata_version,
+                        /*custom_metadata=*/nullptr, options.memory_pool)
       .Value(out);
 }
 
@@ -1190,7 +1195,8 @@ Status WriteRecordBatchMessage(
   RETURN_NOT_OK(
       MakeRecordBatch(fbb, length, body_length, nodes, buffers, options, &record_batch));
   return WriteFBMessage(fbb, flatbuf::MessageHeader::RecordBatch, record_batch.Union(),
-                        body_length, options.metadata_version, custom_metadata)
+                        body_length, options.metadata_version, custom_metadata,
+                        options.memory_pool)
       .Value(out);
 }
 
@@ -1224,7 +1230,8 @@ Result<std::shared_ptr<Buffer>> WriteTensorMessage(const Tensor& tensor,
       flatbuf::CreateTensor(fbb, fb_type_type, fb_type, fb_shape, fb_strides, &buffer);
 
   return WriteFBMessage(fbb, flatbuf::MessageHeader::Tensor, fb_tensor.Union(),
-                        body_length, options.metadata_version);
+                        body_length, options.metadata_version,
+                        /*custom_metadata=*/nullptr, options.memory_pool);
 }
 
 Result<std::shared_ptr<Buffer>> WriteSparseTensorMessage(
@@ -1235,7 +1242,8 @@ Result<std::shared_ptr<Buffer>> WriteSparseTensorMessage(
   RETURN_NOT_OK(
       MakeSparseTensor(fbb, sparse_tensor, body_length, buffers, &fb_sparse_tensor));
   return WriteFBMessage(fbb, flatbuf::MessageHeader::SparseTensor,
-                        fb_sparse_tensor.Union(), body_length, options.metadata_version);
+                        fb_sparse_tensor.Union(), body_length, options.metadata_version,
+                        /*custom_metadata=*/nullptr, options.memory_pool);
 }
 
 Status WriteDictionaryMessage(
@@ -1250,7 +1258,8 @@ Status WriteDictionaryMessage(
   auto dictionary_batch =
       flatbuf::CreateDictionaryBatch(fbb, id, record_batch, is_delta).Union();
   return WriteFBMessage(fbb, flatbuf::MessageHeader::DictionaryBatch, dictionary_batch,
-                        body_length, options.metadata_version, custom_metadata)
+                        body_length, options.metadata_version, custom_metadata,
+                        options.memory_pool)
       .Value(out);
 }
 
@@ -1272,8 +1281,8 @@ Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dicti
   FBB fbb;
 
   flatbuffers::Offset<flatbuf::Schema> fb_schema;
-  DictionaryMemo dictionary_memo;  // unused
-  RETURN_NOT_OK(SchemaToFlatbuffer(fbb, schema, &dictionary_memo, &fb_schema));
+  DictionaryFieldMapper mapper(schema);
+  RETURN_NOT_OK(SchemaToFlatbuffer(fbb, schema, mapper, &fb_schema));
 
 #ifndef NDEBUG
   for (size_t i = 0; i < dictionaries.size(); ++i) {
@@ -1313,12 +1322,15 @@ Status GetSchema(const void* opaque_schema, DictionaryMemo* dictionary_memo,
   CHECK_FLATBUFFERS_NOT_NULL(schema->fields(), "Schema.fields");
   int num_fields = static_cast<int>(schema->fields()->size());
 
+  FieldPosition field_pos;
+
   std::vector<std::shared_ptr<Field>> fields(num_fields);
   for (int i = 0; i < num_fields; ++i) {
     const flatbuf::Field* field = schema->fields()->Get(i);
     // XXX I don't think this check is necessary (AP)
     CHECK_FLATBUFFERS_NOT_NULL(field, "DictionaryEncoding.indexType");
-    RETURN_NOT_OK(FieldFromFlatbuffer(field, dictionary_memo, &fields[i]));
+    RETURN_NOT_OK(
+        FieldFromFlatbuffer(field, field_pos.child(i), dictionary_memo, &fields[i]));
   }
 
   std::shared_ptr<KeyValueMetadata> metadata;
diff --git a/cpp/src/arrow/ipc/metadata_internal.h b/cpp/src/arrow/ipc/metadata_internal.h
index 8f432c43632..d5c697fe57b 100644
--- a/cpp/src/arrow/ipc/metadata_internal.h
+++ b/cpp/src/arrow/ipc/metadata_internal.h
@@ -48,6 +48,7 @@ namespace flatbuf = org::apache::arrow::flatbuf;
 
 namespace ipc {
 
+class DictionaryFieldMapper;
 class DictionaryMemo;
 
 namespace internal {
@@ -166,13 +167,7 @@ static inline Status VerifyMessage(const uint8_t* data, int64_t size,
 }
 
 // Serialize arrow::Schema as a Flatbuffer
-//
-// \param[in] schema a Schema instance
-// \param[in,out] dictionary_memo class for tracking dictionaries and assigning
-// dictionary ids
-// \param[out] out the serialized arrow::Buffer
-// \return Status outcome
-Status WriteSchemaMessage(const Schema& schema, DictionaryMemo* dictionary_memo,
+Status WriteSchemaMessage(const Schema& schema, const DictionaryFieldMapper& mapper,
                           const IpcWriteOptions& options, std::shared_ptr<Buffer>* out);
 
 // This function is used in a unit test
@@ -204,10 +199,11 @@ Status WriteDictionaryMessage(
     const IpcWriteOptions& options, std::shared_ptr<Buffer>* out);
 
 static inline Result<std::shared_ptr<Buffer>> WriteFlatbufferBuilder(
-    flatbuffers::FlatBufferBuilder& fbb) {
+    flatbuffers::FlatBufferBuilder& fbb,  // NOLINT non-const reference
+    MemoryPool* pool = default_memory_pool()) {
   int32_t size = fbb.GetSize();
 
-  ARROW_ASSIGN_OR_RAISE(auto result, AllocateBuffer(size));
+  ARROW_ASSIGN_OR_RAISE(auto result, AllocateBuffer(size, pool));
 
   uint8_t* dst = result->mutable_data();
   memcpy(dst, fbb.GetBufferPointer(), size);
diff --git a/cpp/src/arrow/ipc/options.h b/cpp/src/arrow/ipc/options.h
index 6bbd7b87de2..bf535cdacf3 100644
--- a/cpp/src/arrow/ipc/options.h
+++ b/cpp/src/arrow/ipc/options.h
@@ -59,8 +59,7 @@ struct ARROW_EXPORT IpcWriteOptions {
   /// \brief Compression codec to use for record batch body buffers
   ///
   /// May only be UNCOMPRESSED, LZ4_FRAME and ZSTD.
-  Compression::type compression = Compression::UNCOMPRESSED;
-  int compression_level = Compression::kUseDefaultCompressionLevel;
+  std::shared_ptr<util::Codec> codec;
 
   /// \brief Use global CPU thread pool to parallelize any computational tasks
   /// like compression
diff --git a/cpp/src/arrow/ipc/read_write_benchmark.cc b/cpp/src/arrow/ipc/read_write_benchmark.cc
index 7286003e86a..a56dd3579e2 100644
--- a/cpp/src/arrow/ipc/read_write_benchmark.cc
+++ b/cpp/src/arrow/ipc/read_write_benchmark.cc
@@ -21,11 +21,12 @@
 #include <sstream>
 #include <string>
 
-#include "arrow/api.h"
 #include "arrow/io/memory.h"
 #include "arrow/ipc/api.h"
+#include "arrow/record_batch.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/testing/random.h"
+#include "arrow/type.h"
 
 namespace arrow {
 
@@ -100,7 +101,7 @@ static void ReadFile(benchmark::State& state) {  // NOLINT non-const reference
     auto record_batch = MakeRecordBatch(kTotalSize, state.range(0));
 
     io::BufferOutputStream stream(buffer);
-    auto writer = *ipc::NewFileWriter(&stream, record_batch->schema(), options);
+    auto writer = *ipc::MakeFileWriter(&stream, record_batch->schema(), options);
     ABORT_NOT_OK(writer->WriteRecordBatch(*record_batch));
     ABORT_NOT_OK(writer->Close());
     ABORT_NOT_OK(stream.Close());
@@ -131,7 +132,7 @@ static void ReadStream(benchmark::State& state) {  // NOLINT non-const reference
 
     io::BufferOutputStream stream(buffer);
 
-    auto writer_result = ipc::NewStreamWriter(&stream, record_batch->schema(), options);
+    auto writer_result = ipc::MakeStreamWriter(&stream, record_batch->schema(), options);
     ABORT_NOT_OK(writer_result);
     auto writer = *writer_result;
     ABORT_NOT_OK(writer->WriteRecordBatch(*record_batch));
@@ -167,7 +168,7 @@ static void DecodeStream(benchmark::State& state) {  // NOLINT non-const referen
 
   io::BufferOutputStream stream(buffer);
 
-  auto writer_result = ipc::NewStreamWriter(&stream, record_batch->schema(), options);
+  auto writer_result = ipc::MakeStreamWriter(&stream, record_batch->schema(), options);
   ABORT_NOT_OK(writer_result);
   auto writer = *writer_result;
   ABORT_NOT_OK(writer->WriteRecordBatch(*record_batch));
diff --git a/cpp/src/arrow/ipc/read_write_test.cc b/cpp/src/arrow/ipc/read_write_test.cc
index 7aec9afd092..2e1e6ad615f 100644
--- a/cpp/src/arrow/ipc/read_write_test.cc
+++ b/cpp/src/arrow/ipc/read_write_test.cc
@@ -27,8 +27,7 @@
 #include <gtest/gtest.h>
 
 #include "arrow/array.h"
-#include "arrow/buffer.h"
-#include "arrow/builder.h"
+#include "arrow/array/builder_primitive.h"
 #include "arrow/io/file.h"
 #include "arrow/io/memory.h"
 #include "arrow/io/test_common.h"
@@ -37,18 +36,16 @@
 #include "arrow/ipc/reader.h"
 #include "arrow/ipc/test_common.h"
 #include "arrow/ipc/writer.h"
-#include "arrow/memory_pool.h"
 #include "arrow/record_batch.h"
-#include "arrow/sparse_tensor.h"
 #include "arrow/status.h"
-#include "arrow/tensor.h"
 #include "arrow/testing/extension_type.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/testing/random.h"
 #include "arrow/testing/util.h"
-#include "arrow/type.h"
+#include "arrow/type_fwd.h"
 #include "arrow/util/bit_util.h"
 #include "arrow/util/checked_cast.h"
+#include "arrow/util/io_util.h"
 #include "arrow/util/key_value_metadata.h"
 
 #include "generated/Message_generated.h"  // IWYU pragma: keep
@@ -57,8 +54,12 @@ namespace arrow {
 
 using internal::checked_cast;
 using internal::GetByteWidth;
+using internal::TemporaryDir;
 
 namespace ipc {
+
+using internal::FieldPosition;
+
 namespace test {
 
 using BatchVector = std::vector<std::shared_ptr<RecordBatch>>;
@@ -223,11 +224,10 @@ class TestSchemaMetadata : public ::testing::Test {
   void SetUp() {}
 
   void CheckSchemaRoundtrip(const Schema& schema) {
-    DictionaryMemo in_memo, out_memo;
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Buffer> buffer,
-                         SerializeSchema(schema, &out_memo, default_memory_pool()));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Buffer> buffer, SerializeSchema(schema));
 
     io::BufferReader reader(buffer);
+    DictionaryMemo in_memo;
     ASSERT_OK_AND_ASSIGN(auto actual_schema, ReadSchema(&reader, &in_memo));
     AssertSchemaEqual(schema, *actual_schema);
   }
@@ -334,6 +334,7 @@ TEST_F(TestSchemaMetadata, MetadataVersionForwardCompatibility) {
 const std::vector<test::MakeRecordBatch*> kBatchCases = {
     &MakeIntRecordBatch,
     &MakeListRecordBatch,
+    &MakeFixedSizeListRecordBatch,
     &MakeNonNullRecordBatch,
     &MakeZeroLengthRecordBatch,
     &MakeDeeplyNestedList,
@@ -342,6 +343,8 @@ const std::vector<test::MakeRecordBatch*> kBatchCases = {
     &MakeUnion,
     &MakeDictionary,
     &MakeNestedDictionary,
+    &MakeMap,
+    &MakeMapOfDictionary,
     &MakeDates,
     &MakeTimestamps,
     &MakeTimes,
@@ -370,17 +373,22 @@ class ExtensionTypesMixin {
 
 class IpcTestFixture : public io::MemoryMapFixture, public ExtensionTypesMixin {
  public:
-  void SetUp() { options_ = IpcWriteOptions::Defaults(); }
+  void SetUp() {
+    options_ = IpcWriteOptions::Defaults();
+    ASSERT_OK_AND_ASSIGN(temp_dir_, TemporaryDir::Make("ipc-test-"));
+  }
 
-  void DoSchemaRoundTrip(const Schema& schema, DictionaryMemo* out_memo,
-                         std::shared_ptr<Schema>* result) {
+  std::string TempFile(util::string_view file) {
+    return temp_dir_->path().Join(std::string(file)).ValueOrDie().ToString();
+  }
+
+  void DoSchemaRoundTrip(const Schema& schema, std::shared_ptr<Schema>* result) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<Buffer> serialized_schema,
-                         SerializeSchema(schema, out_memo, options_.memory_pool));
+                         SerializeSchema(schema, options_.memory_pool));
 
     DictionaryMemo in_memo;
     io::BufferReader buf_reader(serialized_schema);
     ASSERT_OK_AND_ASSIGN(*result, ReadSchema(&buf_reader, &in_memo));
-    ASSERT_EQ(out_memo->num_fields(), in_memo.num_fields());
   }
 
   Result<std::shared_ptr<RecordBatch>> DoStandardRoundTrip(
@@ -405,7 +413,7 @@ class IpcTestFixture : public io::MemoryMapFixture, public ExtensionTypesMixin {
     options.allow_64bit = true;
 
     ARROW_ASSIGN_OR_RAISE(auto file_writer,
-                          NewFileWriter(mmap_.get(), batch.schema(), options));
+                          MakeFileWriter(mmap_, batch.schema(), options));
     RETURN_NOT_OK(file_writer->WriteRecordBatch(batch));
     RETURN_NOT_OK(file_writer->Close());
 
@@ -434,16 +442,15 @@ class IpcTestFixture : public io::MemoryMapFixture, public ExtensionTypesMixin {
                       int64_t buffer_size = 1 << 20) {
     std::stringstream ss;
     ss << "test-write-row-batch-" << g_file_number++;
-    ASSERT_OK_AND_ASSIGN(mmap_,
-                         io::MemoryMapFixture::InitMemoryMap(buffer_size, ss.str()));
-
-    DictionaryMemo dictionary_memo;
+    ASSERT_OK_AND_ASSIGN(
+        mmap_, io::MemoryMapFixture::InitMemoryMap(buffer_size, TempFile(ss.str())));
 
     std::shared_ptr<Schema> schema_result;
-    DoSchemaRoundTrip(*batch.schema(), &dictionary_memo, &schema_result);
+    DoSchemaRoundTrip(*batch.schema(), &schema_result);
     ASSERT_TRUE(batch.schema()->Equals(*schema_result));
 
-    ASSERT_OK(CollectDictionaries(batch, &dictionary_memo));
+    DictionaryMemo dictionary_memo;
+    ASSERT_OK(::arrow::ipc::internal::CollectDictionaries(batch, &dictionary_memo));
 
     ASSERT_OK_AND_ASSIGN(
         auto result, DoStandardRoundTrip(batch, options, &dictionary_memo, read_options));
@@ -467,6 +474,7 @@ class IpcTestFixture : public io::MemoryMapFixture, public ExtensionTypesMixin {
  protected:
   std::shared_ptr<io::MemoryMappedFile> mmap_;
   IpcWriteOptions options_;
+  std::unique_ptr<TemporaryDir> temp_dir_;
 };
 
 TEST(MetadataVersion, ForwardsCompatCheck) {
@@ -609,7 +617,7 @@ TEST_F(TestWriteRecordBatch, WriteWithCompression) {
       continue;
     }
     IpcWriteOptions write_options = IpcWriteOptions::Defaults();
-    write_options.compression = codec;
+    ASSERT_OK_AND_ASSIGN(write_options.codec, util::Codec::Create(codec));
     CheckRoundtrip(*batch, write_options);
 
     // Check non-parallel read and write
@@ -626,9 +634,9 @@ TEST_F(TestWriteRecordBatch, WriteWithCompression) {
     if (!util::Codec::IsAvailable(codec)) {
       continue;
     }
-    IpcWriteOptions options = IpcWriteOptions::Defaults();
-    options.compression = codec;
-    ASSERT_RAISES(Invalid, SerializeRecordBatch(*batch, options));
+    IpcWriteOptions write_options = IpcWriteOptions::Defaults();
+    ASSERT_OK_AND_ASSIGN(write_options.codec, util::Codec::Create(codec));
+    ASSERT_RAISES(Invalid, SerializeRecordBatch(*batch, write_options));
   }
 }
 
@@ -642,10 +650,9 @@ TEST_F(TestWriteRecordBatch, SliceTruncatesBinaryOffsets) {
   auto batch = RecordBatch::Make(schema, array->length(), {array});
   auto sliced_batch = batch->Slice(0, 5);
 
-  std::stringstream ss;
-  ss << "test-truncate-offsets";
   ASSERT_OK_AND_ASSIGN(
-      mmap_, io::MemoryMapFixture::InitMemoryMap(/*buffer_size=*/1 << 20, ss.str()));
+      mmap_, io::MemoryMapFixture::InitMemoryMap(/*buffer_size=*/1 << 20,
+                                                 TempFile("test-truncate-offsets")));
   DictionaryMemo dictionary_memo;
   ASSERT_OK_AND_ASSIGN(
       auto result,
@@ -730,10 +737,9 @@ TEST_F(TestWriteRecordBatch, RoundtripPreservesBufferSizes) {
   auto arr = rg.String(length, 0, 10, 0.1);
   auto batch = RecordBatch::Make(::arrow::schema({field("f0", utf8())}), length, {arr});
 
-  std::stringstream ss;
-  ss << "test-roundtrip-buffer-sizes";
   ASSERT_OK_AND_ASSIGN(
-      mmap_, io::MemoryMapFixture::InitMemoryMap(/*buffer_size=*/1 << 20, ss.str()));
+      mmap_, io::MemoryMapFixture::InitMemoryMap(
+                 /*buffer_size=*/1 << 20, TempFile("test-roundtrip-buffer-sizes")));
   DictionaryMemo dictionary_memo;
   ASSERT_OK_AND_ASSIGN(
       auto result,
@@ -785,7 +791,15 @@ TEST_F(TestWriteRecordBatch, IntegerGetRecordBatchSize) {
 
 class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
  public:
-  void SetUp() { pool_ = default_memory_pool(); }
+  void SetUp() {
+    pool_ = default_memory_pool();
+    ASSERT_OK_AND_ASSIGN(temp_dir_, TemporaryDir::Make("ipc-recursion-limits-test-"));
+  }
+
+  std::string TempFile(util::string_view file) {
+    return temp_dir_->path().Join(std::string(file)).ValueOrDie().ToString();
+  }
+
   void TearDown() { io::MemoryMapFixture::TearDown(); }
 
   Status WriteToMmap(int recursion_level, bool override_level, int32_t* metadata_length,
@@ -811,8 +825,8 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
     std::stringstream ss;
     ss << "test-write-past-max-recursion-" << g_file_number++;
     const int memory_map_size = 1 << 20;
-    ARROW_ASSIGN_OR_RAISE(mmap_,
-                          io::MemoryMapFixture::InitMemoryMap(memory_map_size, ss.str()));
+    ARROW_ASSIGN_OR_RAISE(
+        mmap_, io::MemoryMapFixture::InitMemoryMap(memory_map_size, TempFile(ss.str())));
 
     auto options = IpcWriteOptions::Defaults();
     if (override_level) {
@@ -824,6 +838,7 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
 
  protected:
   std::shared_ptr<io::MemoryMappedFile> mmap_;
+  std::unique_ptr<TemporaryDir> temp_dir_;
   MemoryPool* pool_;
 };
 
@@ -895,13 +910,16 @@ TEST_F(RecursionLimits, StressLimit) {
 #endif  // !defined(_WIN32) || defined(NDEBUG)
 
 struct FileWriterHelper {
+  static constexpr bool kIsFileFormat = true;
+
   Status Init(const std::shared_ptr<Schema>& schema, const IpcWriteOptions& options,
               const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr) {
     num_batches_written_ = 0;
 
     ARROW_ASSIGN_OR_RAISE(buffer_, AllocateResizableBuffer(0));
     sink_.reset(new io::BufferOutputStream(buffer_));
-    ARROW_ASSIGN_OR_RAISE(writer_, NewFileWriter(sink_.get(), schema, options, metadata));
+    ARROW_ASSIGN_OR_RAISE(writer_,
+                          MakeFileWriter(sink_.get(), schema, options, metadata));
     return Status::OK();
   }
 
@@ -918,11 +936,11 @@ struct FileWriterHelper {
     return sink_->Tell().Value(&footer_offset_);
   }
 
-  Status ReadBatches(const IpcReadOptions& options, BatchVector* out_batches) {
+  virtual Status ReadBatches(const IpcReadOptions& options, BatchVector* out_batches,
+                             ReadStats* out_stats = nullptr) {
     auto buf_reader = std::make_shared<io::BufferReader>(buffer_);
-    std::shared_ptr<RecordBatchFileReader> reader;
-    ARROW_ASSIGN_OR_RAISE(
-        reader, RecordBatchFileReader::Open(buf_reader.get(), footer_offset_, options));
+    ARROW_ASSIGN_OR_RAISE(auto reader, RecordBatchFileReader::Open(
+                                           buf_reader.get(), footer_offset_, options));
 
     EXPECT_EQ(num_batches_written_, reader->num_record_batches());
     for (int i = 0; i < num_batches_written_; ++i) {
@@ -930,7 +948,9 @@ struct FileWriterHelper {
                             reader->ReadRecordBatch(i));
       out_batches->push_back(chunk);
     }
-
+    if (out_stats) {
+      *out_stats = reader->stats();
+    }
     return Status::OK();
   }
 
@@ -963,10 +983,12 @@ struct FileWriterHelper {
 };
 
 struct StreamWriterHelper {
+  static constexpr bool kIsFileFormat = false;
+
   Status Init(const std::shared_ptr<Schema>& schema, const IpcWriteOptions& options) {
     ARROW_ASSIGN_OR_RAISE(buffer_, AllocateResizableBuffer(0));
     sink_.reset(new io::BufferOutputStream(buffer_));
-    ARROW_ASSIGN_OR_RAISE(writer_, NewStreamWriter(sink_.get(), schema, options));
+    ARROW_ASSIGN_OR_RAISE(writer_, MakeStreamWriter(sink_.get(), schema, options));
     return Status::OK();
   }
 
@@ -980,11 +1002,15 @@ struct StreamWriterHelper {
     return sink_->Close();
   }
 
-  virtual Status ReadBatches(const IpcReadOptions& options, BatchVector* out_batches) {
+  virtual Status ReadBatches(const IpcReadOptions& options, BatchVector* out_batches,
+                             ReadStats* out_stats = nullptr) {
     auto buf_reader = std::make_shared<io::BufferReader>(buffer_);
-    std::shared_ptr<RecordBatchReader> reader;
-    ARROW_ASSIGN_OR_RAISE(reader, RecordBatchStreamReader::Open(buf_reader, options))
-    return reader->ReadAll(out_batches);
+    ARROW_ASSIGN_OR_RAISE(auto reader, RecordBatchStreamReader::Open(buf_reader, options))
+    RETURN_NOT_OK(reader->ReadAll(out_batches));
+    if (out_stats) {
+      *out_stats = reader->stats();
+    }
+    return Status::OK();
   }
 
   Status ReadSchema(std::shared_ptr<Schema>* out) {
@@ -1006,7 +1032,11 @@ struct StreamWriterHelper {
 };
 
 struct StreamDecoderWriterHelper : public StreamWriterHelper {
-  Status ReadBatches(const IpcReadOptions& options, BatchVector* out_batches) override {
+  Status ReadBatches(const IpcReadOptions& options, BatchVector* out_batches,
+                     ReadStats* out_stats = nullptr) override {
+    if (out_stats) {
+      return Status::NotImplemented("StreamDecoder does not support stats()");
+    }
     auto listener = std::make_shared<CollectListener>();
     StreamDecoder decoder(listener, options);
     RETURN_NOT_OK(DoConsume(&decoder));
@@ -1333,8 +1363,9 @@ class DictionaryBatchHelper {
 
     // write schema
     IpcPayload payload;
-    RETURN_NOT_OK(GetSchemaPayload(schema_, IpcWriteOptions::Defaults(),
-                                   &dictionary_memo_, &payload));
+    DictionaryFieldMapper mapper(schema_);
+    RETURN_NOT_OK(
+        GetSchemaPayload(schema_, IpcWriteOptions::Defaults(), mapper, &payload));
     return payload_writer_->WritePayload(payload);
   }
 
@@ -1369,7 +1400,6 @@ class DictionaryBatchHelper {
 
   std::unique_ptr<internal::IpcPayloadWriter> payload_writer_;
   const Schema& schema_;
-  DictionaryMemo dictionary_memo_;
   std::shared_ptr<ResizableBuffer> buffer_;
   std::unique_ptr<io::BufferOutputStream> sink_;
 };
@@ -1587,7 +1617,7 @@ TEST(TestRecordBatchStreamReader, EmptyStreamWithDictionaries) {
 
   ASSERT_OK_AND_ASSIGN(auto stream, io::BufferOutputStream::Create(0));
 
-  ASSERT_OK_AND_ASSIGN(auto writer, NewStreamWriter(stream.get(), schema));
+  ASSERT_OK_AND_ASSIGN(auto writer, MakeStreamWriter(stream, schema));
   ASSERT_OK(writer->Close());
 
   ASSERT_OK_AND_ASSIGN(auto buffer, stream->Finish());
@@ -1644,7 +1674,7 @@ TEST(TestRecordBatchStreamReader, NotEnoughDictionaries) {
   ASSERT_OK(MakeDictionaryFlat(&batch));
 
   ASSERT_OK_AND_ASSIGN(auto out, io::BufferOutputStream::Create(0));
-  ASSERT_OK_AND_ASSIGN(auto writer, NewStreamWriter(out.get(), batch->schema()));
+  ASSERT_OK_AND_ASSIGN(auto writer, MakeStreamWriter(out, batch->schema()));
   ASSERT_OK(writer->WriteRecordBatch(*batch));
   ASSERT_OK(writer->Close());
 
@@ -1677,509 +1707,350 @@ TEST(TestRecordBatchStreamReader, NotEnoughDictionaries) {
   AssertFailsWith(truncated_stream, ex_message);
 }
 
-class TestTensorRoundTrip : public ::testing::Test, public IpcTestFixture {
- public:
-  void SetUp() { IpcTestFixture::SetUp(); }
-  void TearDown() { IpcTestFixture::TearDown(); }
-
-  void CheckTensorRoundTrip(const Tensor& tensor) {
-    int32_t metadata_length;
-    int64_t body_length;
-    const int elem_size = GetByteWidth(*tensor.type());
-
-    ASSERT_OK(mmap_->Seek(0));
-
-    ASSERT_OK(WriteTensor(tensor, mmap_.get(), &metadata_length, &body_length));
-
-    const int64_t expected_body_length = elem_size * tensor.size();
-    ASSERT_EQ(expected_body_length, body_length);
-
-    ASSERT_OK(mmap_->Seek(0));
-
-    std::shared_ptr<Tensor> result;
-    ASSERT_OK_AND_ASSIGN(result, ReadTensor(mmap_.get()));
-
-    ASSERT_EQ(result->data()->size(), expected_body_length);
-    ASSERT_TRUE(tensor.Equals(*result));
-  }
-};
-
-TEST_F(TestTensorRoundTrip, BasicRoundtrip) {
-  std::string path = "test-write-tensor";
-  constexpr int64_t kBufferSize = 1 << 20;
-  ASSERT_OK_AND_ASSIGN(mmap_, io::MemoryMapFixture::InitMemoryMap(kBufferSize, path));
-
-  std::vector<int64_t> shape = {4, 6};
-  std::vector<int64_t> strides = {48, 8};
-  std::vector<std::string> dim_names = {"foo", "bar"};
-  int64_t size = 24;
-
-  std::vector<int64_t> values;
-  randint(size, 0, 100, &values);
-
-  auto data = Buffer::Wrap(values);
-
-  Tensor t0(int64(), data, shape, strides, dim_names);
-  Tensor t_no_dims(int64(), data, {}, {}, {});
-  Tensor t_zero_length_dim(int64(), data, {0}, {8}, {"foo"});
+TEST(TestRecordBatchStreamReader, MalformedInput) {
+  const std::string empty_str = "";
+  const std::string garbage_str = "12345678";
 
-  CheckTensorRoundTrip(t0);
-  CheckTensorRoundTrip(t_no_dims);
-  CheckTensorRoundTrip(t_zero_length_dim);
+  auto empty = std::make_shared<Buffer>(empty_str);
+  auto garbage = std::make_shared<Buffer>(garbage_str);
 
-  int64_t serialized_size;
-  ASSERT_OK(GetTensorSize(t0, &serialized_size));
-  ASSERT_TRUE(serialized_size > static_cast<int64_t>(size * sizeof(int64_t)));
+  io::BufferReader empty_reader(empty);
+  ASSERT_RAISES(Invalid, RecordBatchStreamReader::Open(&empty_reader));
 
-  // ARROW-2840: Check that padding/alignment minded
-  std::vector<int64_t> shape_2 = {1, 1};
-  std::vector<int64_t> strides_2 = {8, 8};
-  Tensor t0_not_multiple_64(int64(), data, shape_2, strides_2, dim_names);
-  CheckTensorRoundTrip(t0_not_multiple_64);
+  io::BufferReader garbage_reader(garbage);
+  ASSERT_RAISES(Invalid, RecordBatchStreamReader::Open(&garbage_reader));
 }
 
-TEST_F(TestTensorRoundTrip, NonContiguous) {
-  std::string path = "test-write-tensor-strided";
-  constexpr int64_t kBufferSize = 1 << 20;
-  ASSERT_OK_AND_ASSIGN(mmap_, io::MemoryMapFixture::InitMemoryMap(kBufferSize, path));
-
-  std::vector<int64_t> values;
-  randint(24, 0, 100, &values);
-
-  auto data = Buffer::Wrap(values);
-  Tensor tensor(int64(), data, {4, 3}, {48, 16});
-
-  CheckTensorRoundTrip(tensor);
+TEST(TestStreamDecoder, NextRequiredSize) {
+  auto listener = std::make_shared<CollectListener>();
+  StreamDecoder decoder(listener);
+  auto next_required_size = decoder.next_required_size();
+  const uint8_t data[1] = {0};
+  ASSERT_OK(decoder.Consume(data, 1));
+  ASSERT_EQ(next_required_size - 1, decoder.next_required_size());
 }
 
-template <typename IndexValueType>
-class TestSparseTensorRoundTrip : public ::testing::Test, public IpcTestFixture {
+template <typename WriterHelperType>
+class TestDictionaryReplacement : public ::testing::Test {
  public:
-  void SetUp() { IpcTestFixture::SetUp(); }
-  void TearDown() { IpcTestFixture::TearDown(); }
-
-  void CheckSparseCOOTensorRoundTrip(const SparseCOOTensor& sparse_tensor) {
-    const int elem_size = GetByteWidth(*sparse_tensor.type());
-    const int index_elem_size = sizeof(typename IndexValueType::c_type);
-
-    int32_t metadata_length;
-    int64_t body_length;
-
-    ASSERT_OK(mmap_->Seek(0));
-
-    ASSERT_OK(
-        WriteSparseTensor(sparse_tensor, mmap_.get(), &metadata_length, &body_length));
-
-    const auto& sparse_index =
-        checked_cast<const SparseCOOIndex&>(*sparse_tensor.sparse_index());
-    const int64_t indices_length =
-        BitUtil::RoundUpToMultipleOf8(index_elem_size * sparse_index.indices()->size());
-    const int64_t data_length =
-        BitUtil::RoundUpToMultipleOf8(elem_size * sparse_tensor.non_zero_length());
-    const int64_t expected_body_length = indices_length + data_length;
-    ASSERT_EQ(expected_body_length, body_length);
-
-    ASSERT_OK(mmap_->Seek(0));
-
-    std::shared_ptr<SparseTensor> result;
-    ASSERT_OK_AND_ASSIGN(result, ReadSparseTensor(mmap_.get()));
-    ASSERT_EQ(SparseTensorFormat::COO, result->format_id());
+  using WriterHelper = WriterHelperType;
 
-    const auto& resulted_sparse_index =
-        checked_cast<const SparseCOOIndex&>(*result->sparse_index());
-    ASSERT_EQ(resulted_sparse_index.indices()->data()->size(), indices_length);
-    ASSERT_EQ(resulted_sparse_index.is_canonical(), sparse_index.is_canonical());
-    ASSERT_EQ(result->data()->size(), data_length);
-    ASSERT_TRUE(result->Equals(sparse_tensor));
+  void TestSameDictPointer() {
+    auto type = dictionary(int8(), utf8());
+    auto values = ArrayFromJSON(utf8(), R"(["foo", "bar", "quux"])");
+    auto batch1 = MakeBatch(type, ArrayFromJSON(int8(), "[0, 2, null, 1]"), values);
+    auto batch2 = MakeBatch(type, ArrayFromJSON(int8(), "[1, 0, 0]"), values);
+    CheckRoundtrip({batch1, batch2});
+
+    EXPECT_EQ(read_stats_.num_messages, 4);  // including schema message
+    EXPECT_EQ(read_stats_.num_record_batches, 2);
+    EXPECT_EQ(read_stats_.num_dictionary_batches, 1);
+    EXPECT_EQ(read_stats_.num_replaced_dictionaries, 0);
+    EXPECT_EQ(read_stats_.num_dictionary_deltas, 0);
   }
 
-  template <typename SparseIndexType>
-  void CheckSparseCSXMatrixRoundTrip(
-      const SparseTensorImpl<SparseIndexType>& sparse_tensor) {
-    static_assert(std::is_same<SparseIndexType, SparseCSRIndex>::value ||
-                      std::is_same<SparseIndexType, SparseCSCIndex>::value,
-                  "SparseIndexType must be either SparseCSRIndex or SparseCSCIndex");
-
-    const int elem_size = GetByteWidth(*sparse_tensor.type());
-    const int index_elem_size = sizeof(typename IndexValueType::c_type);
-
-    int32_t metadata_length;
-    int64_t body_length;
-
-    ASSERT_OK(mmap_->Seek(0));
-
-    ASSERT_OK(
-        WriteSparseTensor(sparse_tensor, mmap_.get(), &metadata_length, &body_length));
-
-    const auto& sparse_index =
-        checked_cast<const SparseIndexType&>(*sparse_tensor.sparse_index());
-    const int64_t indptr_length =
-        BitUtil::RoundUpToMultipleOf8(index_elem_size * sparse_index.indptr()->size());
-    const int64_t indices_length =
-        BitUtil::RoundUpToMultipleOf8(index_elem_size * sparse_index.indices()->size());
-    const int64_t data_length =
-        BitUtil::RoundUpToMultipleOf8(elem_size * sparse_tensor.non_zero_length());
-    const int64_t expected_body_length = indptr_length + indices_length + data_length;
-    ASSERT_EQ(expected_body_length, body_length);
-
-    ASSERT_OK(mmap_->Seek(0));
-
-    std::shared_ptr<SparseTensor> result;
-    ASSERT_OK_AND_ASSIGN(result, ReadSparseTensor(mmap_.get()));
+  void TestSameDictValues() {
+    auto type = dictionary(int8(), utf8());
+    // Create two separate dictionaries, but with the same contents
+    auto batch1 = MakeBatch(ArrayFromJSON(type, R"(["foo", "foo", "bar", null])"));
+    auto batch2 = MakeBatch(ArrayFromJSON(type, R"(["foo", "bar", "foo"])"));
+    CheckRoundtrip({batch1, batch2});
+
+    EXPECT_EQ(read_stats_.num_messages, 4);  // including schema message
+    EXPECT_EQ(read_stats_.num_record_batches, 2);
+    EXPECT_EQ(read_stats_.num_dictionary_batches, 1);
+    EXPECT_EQ(read_stats_.num_replaced_dictionaries, 0);
+    EXPECT_EQ(read_stats_.num_dictionary_deltas, 0);
+  }
 
-    constexpr auto expected_format_id =
-        std::is_same<SparseIndexType, SparseCSRIndex>::value ? SparseTensorFormat::CSR
-                                                             : SparseTensorFormat::CSC;
-    ASSERT_EQ(expected_format_id, result->format_id());
+  void TestSameDictValuesNested() {
+    CheckRoundtrip(SameValuesNestedDictBatches());
 
-    const auto& resulted_sparse_index =
-        checked_cast<const SparseIndexType&>(*result->sparse_index());
-    ASSERT_EQ(resulted_sparse_index.indptr()->data()->size(), indptr_length);
-    ASSERT_EQ(resulted_sparse_index.indices()->data()->size(), indices_length);
-    ASSERT_EQ(result->data()->size(), data_length);
-    ASSERT_TRUE(result->Equals(sparse_tensor));
+    EXPECT_EQ(read_stats_.num_messages, 5);  // including schema message
+    EXPECT_EQ(read_stats_.num_record_batches, 2);
+    EXPECT_EQ(read_stats_.num_dictionary_batches, 2);
+    EXPECT_EQ(read_stats_.num_replaced_dictionaries, 0);
+    EXPECT_EQ(read_stats_.num_dictionary_deltas, 0);
   }
 
-  void CheckSparseCSFTensorRoundTrip(const SparseCSFTensor& sparse_tensor) {
-    const int elem_size = GetByteWidth(*sparse_tensor.type());
-    const int index_elem_size = sizeof(typename IndexValueType::c_type);
-
-    int32_t metadata_length;
-    int64_t body_length;
+  void TestDifferentDictValues() {
+    if (WriterHelper::kIsFileFormat) {
+      CheckWritingFails(DifferentOrderDictBatches(), 1);
+      CheckWritingFails(DifferentValuesDictBatches(), 1);
+      return;
+    }
+    CheckRoundtrip(DifferentOrderDictBatches());
 
-    ASSERT_OK(mmap_->Seek(0));
+    EXPECT_EQ(read_stats_.num_messages, 5);  // including schema message
+    EXPECT_EQ(read_stats_.num_record_batches, 2);
+    EXPECT_EQ(read_stats_.num_dictionary_batches, 2);
+    EXPECT_EQ(read_stats_.num_replaced_dictionaries, 1);
+    EXPECT_EQ(read_stats_.num_dictionary_deltas, 0);
 
-    ASSERT_OK(
-        WriteSparseTensor(sparse_tensor, mmap_.get(), &metadata_length, &body_length));
+    CheckRoundtrip(DifferentValuesDictBatches());
 
-    const auto& sparse_index =
-        checked_cast<const SparseCSFIndex&>(*sparse_tensor.sparse_index());
+    EXPECT_EQ(read_stats_.num_messages, 5);  // including schema message
+    EXPECT_EQ(read_stats_.num_record_batches, 2);
+    EXPECT_EQ(read_stats_.num_dictionary_batches, 2);
+    EXPECT_EQ(read_stats_.num_replaced_dictionaries, 1);
+    EXPECT_EQ(read_stats_.num_dictionary_deltas, 0);
+  }
 
-    const int64_t ndim = sparse_index.axis_order().size();
-    int64_t indptr_length = 0;
-    int64_t indices_length = 0;
+  void TestDifferentDictValuesNested() {
+    if (WriterHelper::kIsFileFormat) {
+      CheckWritingFails(DifferentValuesNestedDictBatches1(), 1);
+      CheckWritingFails(DifferentValuesNestedDictBatches2(), 1);
+      return;
+    }
+    CheckRoundtrip(DifferentValuesNestedDictBatches1());
+
+    EXPECT_EQ(read_stats_.num_messages, 7);  // including schema message
+    EXPECT_EQ(read_stats_.num_record_batches, 2);
+    // Both inner and outer dict were replaced
+    EXPECT_EQ(read_stats_.num_dictionary_batches, 4);
+    EXPECT_EQ(read_stats_.num_replaced_dictionaries, 2);
+    EXPECT_EQ(read_stats_.num_dictionary_deltas, 0);
+
+    CheckRoundtrip(DifferentValuesNestedDictBatches2());
+
+    EXPECT_EQ(read_stats_.num_messages, 6);  // including schema message
+    EXPECT_EQ(read_stats_.num_record_batches, 2);
+    // Only inner dict was replaced
+    EXPECT_EQ(read_stats_.num_dictionary_batches, 3);
+    EXPECT_EQ(read_stats_.num_replaced_dictionaries, 1);
+    EXPECT_EQ(read_stats_.num_dictionary_deltas, 0);
+  }
 
-    for (int64_t i = 0; i < ndim - 1; ++i) {
-      indptr_length += BitUtil::RoundUpToMultipleOf8(index_elem_size *
-                                                     sparse_index.indptr()[i]->size());
+  Status RoundTrip(const BatchVector& in_batches, BatchVector* out_batches) {
+    WriterHelper writer_helper;
+    RETURN_NOT_OK(writer_helper.Init(in_batches[0]->schema(), write_options_));
+    for (const auto& batch : in_batches) {
+      RETURN_NOT_OK(writer_helper.WriteBatch(batch));
     }
-    for (int64_t i = 0; i < ndim; ++i) {
-      indices_length += BitUtil::RoundUpToMultipleOf8(index_elem_size *
-                                                      sparse_index.indices()[i]->size());
+    RETURN_NOT_OK(writer_helper.Finish());
+    RETURN_NOT_OK(writer_helper.ReadBatches(read_options_, out_batches, &read_stats_));
+    for (const auto& batch : *out_batches) {
+      RETURN_NOT_OK(batch->ValidateFull());
     }
-    const int64_t data_length =
-        BitUtil::RoundUpToMultipleOf8(elem_size * sparse_tensor.non_zero_length());
-    const int64_t expected_body_length = indptr_length + indices_length + data_length;
-    ASSERT_EQ(expected_body_length, body_length);
-
-    ASSERT_OK(mmap_->Seek(0));
-
-    std::shared_ptr<SparseTensor> result;
-    ASSERT_OK_AND_ASSIGN(result, ReadSparseTensor(mmap_.get()));
-    ASSERT_EQ(SparseTensorFormat::CSF, result->format_id());
-
-    const auto& resulted_sparse_index =
-        checked_cast<const SparseCSFIndex&>(*result->sparse_index());
+    return Status::OK();
+  }
 
-    int64_t out_indptr_length = 0;
-    int64_t out_indices_length = 0;
-    for (int i = 0; i < ndim - 1; ++i) {
-      out_indptr_length += BitUtil::RoundUpToMultipleOf8(
-          index_elem_size * resulted_sparse_index.indptr()[i]->size());
-    }
-    for (int i = 0; i < ndim; ++i) {
-      out_indices_length += BitUtil::RoundUpToMultipleOf8(
-          index_elem_size * resulted_sparse_index.indices()[i]->size());
+  void CheckRoundtrip(const BatchVector& in_batches) {
+    BatchVector out_batches;
+    ASSERT_OK(RoundTrip(in_batches, &out_batches));
+    ASSERT_EQ(in_batches.size(), out_batches.size());
+    for (size_t i = 0; i < in_batches.size(); ++i) {
+      AssertBatchesEqual(*in_batches[i], *out_batches[i]);
     }
+  }
 
-    ASSERT_EQ(out_indptr_length, indptr_length);
-    ASSERT_EQ(out_indices_length, indices_length);
-    ASSERT_EQ(result->data()->size(), data_length);
-    ASSERT_TRUE(resulted_sparse_index.Equals(sparse_index));
-    ASSERT_TRUE(result->Equals(sparse_tensor));
+  void CheckWritingFails(const BatchVector& in_batches, size_t fails_at_batch_num) {
+    WriterHelper writer_helper;
+    ASSERT_OK(writer_helper.Init(in_batches[0]->schema(), write_options_));
+    for (size_t i = 0; i < fails_at_batch_num; ++i) {
+      ASSERT_OK(writer_helper.WriteBatch(in_batches[i]));
+    }
+    ASSERT_RAISES(Invalid, writer_helper.WriteBatch(in_batches[fails_at_batch_num]));
   }
 
- protected:
-  std::shared_ptr<SparseCOOIndex> MakeSparseCOOIndex(
-      const std::vector<int64_t>& coords_shape,
-      const std::vector<int64_t>& coords_strides,
-      std::vector<typename IndexValueType::c_type>& coords_values) const {
-    auto coords_data = Buffer::Wrap(coords_values);
-    auto coords = std::make_shared<NumericTensor<IndexValueType>>(
-        coords_data, coords_shape, coords_strides);
-    return std::make_shared<SparseCOOIndex>(coords);
-  }
-
-  template <typename ValueType>
-  Result<std::shared_ptr<SparseCOOTensor>> MakeSparseCOOTensor(
-      const std::shared_ptr<SparseCOOIndex>& si, std::vector<ValueType>& sparse_values,
-      const std::vector<int64_t>& shape,
-      const std::vector<std::string>& dim_names = {}) const {
-    auto data = Buffer::Wrap(sparse_values);
-    return SparseCOOTensor::Make(si, CTypeTraits<ValueType>::type_singleton(), data,
-                                 shape, dim_names);
+  BatchVector DifferentOrderDictBatches() {
+    // Create two separate dictionaries with different order
+    auto type = dictionary(int8(), utf8());
+    auto batch1 = MakeBatch(ArrayFromJSON(type, R"(["foo", "foo", "bar", null])"));
+    auto batch2 = MakeBatch(ArrayFromJSON(type, R"(["bar", "bar", "foo"])"));
+    return {batch1, batch2};
   }
-};
 
-TYPED_TEST_SUITE_P(TestSparseTensorRoundTrip);
-
-TYPED_TEST_P(TestSparseTensorRoundTrip, WithSparseCOOIndexRowMajor) {
-  using IndexValueType = TypeParam;
-  using c_index_value_type = typename IndexValueType::c_type;
-
-  std::string path = "test-write-sparse-coo-tensor";
-  constexpr int64_t kBufferSize = 1 << 20;
-  ASSERT_OK_AND_ASSIGN(this->mmap_,
-                       io::MemoryMapFixture::InitMemoryMap(kBufferSize, path));
-
-  // Dense representation:
-  // [
-  //   [
-  //     1 0 2 0
-  //     0 3 0 4
-  //     5 0 6 0
-  //   ],
-  //   [
-  //      0 11  0 12
-  //     13  0 14  0
-  //      0 15  0 16
-  //   ]
-  // ]
-  //
-  // Sparse representation:
-  // idx[0] = [0 0 0 0 0 0  1  1  1  1  1  1]
-  // idx[1] = [0 0 1 1 2 2  0  0  1  1  2  2]
-  // idx[2] = [0 2 1 3 0 2  1  3  0  2  1  3]
-  // data   = [1 2 3 4 5 6 11 12 13 14 15 16]
-
-  // canonical
-  std::vector<c_index_value_type> coords_values = {0, 0, 0, 0, 0, 2, 0, 1, 1, 0, 1, 3,
-                                                   0, 2, 0, 0, 2, 2, 1, 0, 1, 1, 0, 3,
-                                                   1, 1, 0, 1, 1, 2, 1, 2, 1, 1, 2, 3};
-  const int sizeof_index_value = sizeof(c_index_value_type);
-  std::shared_ptr<SparseCOOIndex> si;
-  ASSERT_OK_AND_ASSIGN(
-      si, SparseCOOIndex::Make(TypeTraits<IndexValueType>::type_singleton(), {12, 3},
-                               {sizeof_index_value * 3, sizeof_index_value},
-                               Buffer::Wrap(coords_values)));
-  ASSERT_TRUE(si->is_canonical());
+  BatchVector DifferentValuesDictBatches() {
+    // Create two separate dictionaries with different values
+    auto type = dictionary(int8(), utf8());
+    auto batch1 = MakeBatch(ArrayFromJSON(type, R"(["foo", "foo", "bar", null])"));
+    auto batch2 = MakeBatch(ArrayFromJSON(type, R"(["bar", "quux", "quux"])"));
+    return {batch1, batch2};
+  }
 
-  std::vector<int64_t> shape = {2, 3, 4};
-  std::vector<std::string> dim_names = {"foo", "bar", "baz"};
-  std::vector<int64_t> values = {1, 2, 3, 4, 5, 6, 11, 12, 13, 14, 15, 16};
-  std::shared_ptr<SparseCOOTensor> st;
-  ASSERT_OK_AND_ASSIGN(st, this->MakeSparseCOOTensor(si, values, shape, dim_names));
+  BatchVector SameValuesNestedDictBatches() {
+    auto value_type = list(dictionary(int8(), utf8()));
+    auto type = dictionary(int8(), value_type);
+    auto batch1_values = ArrayFromJSON(value_type, R"([[], ["a"], ["b"], ["a", "a"]])");
+    auto batch2_values = ArrayFromJSON(value_type, R"([[], ["a"], ["b"], ["a", "a"]])");
+    auto batch1 = MakeBatch(type, ArrayFromJSON(int8(), "[1, 3, 0, 3]"), batch1_values);
+    auto batch2 = MakeBatch(type, ArrayFromJSON(int8(), "[2, null, 2]"), batch2_values);
+    return {batch1, batch2};
+  }
 
-  this->CheckSparseCOOTensorRoundTrip(*st);
+  BatchVector DifferentValuesNestedDictBatches1() {
+    // Inner dictionary values differ
+    auto value_type = list(dictionary(int8(), utf8()));
+    auto type = dictionary(int8(), value_type);
+    auto batch1_values = ArrayFromJSON(value_type, R"([[], ["a"], ["b"], ["a", "a"]])");
+    auto batch2_values = ArrayFromJSON(value_type, R"([[], ["a"], ["c"], ["a", "a"]])");
+    auto batch1 = MakeBatch(type, ArrayFromJSON(int8(), "[1, 3, 0, 3]"), batch1_values);
+    auto batch2 = MakeBatch(type, ArrayFromJSON(int8(), "[2, null, 2]"), batch2_values);
+    return {batch1, batch2};
+  }
 
-  // non-canonical
-  ASSERT_OK_AND_ASSIGN(
-      si, SparseCOOIndex::Make(TypeTraits<IndexValueType>::type_singleton(), {12, 3},
-                               {sizeof_index_value * 3, sizeof_index_value},
-                               Buffer::Wrap(coords_values), false));
-  ASSERT_FALSE(si->is_canonical());
-  ASSERT_OK_AND_ASSIGN(st, this->MakeSparseCOOTensor(si, values, shape, dim_names));
+  BatchVector DifferentValuesNestedDictBatches2() {
+    // Outer dictionary values differ
+    auto value_type = list(dictionary(int8(), utf8()));
+    auto type = dictionary(int8(), value_type);
+    auto batch1_values = ArrayFromJSON(value_type, R"([[], ["a"], ["b"], ["a", "a"]])");
+    auto batch2_values = ArrayFromJSON(value_type, R"([["a"], ["b"], ["a", "a"]])");
+    auto batch1 = MakeBatch(type, ArrayFromJSON(int8(), "[1, 3, 0, 3]"), batch1_values);
+    auto batch2 = MakeBatch(type, ArrayFromJSON(int8(), "[2, null, 2]"), batch2_values);
+    return {batch1, batch2};
+  }
 
-  this->CheckSparseCOOTensorRoundTrip(*st);
-}
+  // Make one-column batch
+  std::shared_ptr<RecordBatch> MakeBatch(std::shared_ptr<Array> column) {
+    return RecordBatch::Make(schema({field("f", column->type())}), column->length(),
+                             {column});
+  }
 
-TYPED_TEST_P(TestSparseTensorRoundTrip, WithSparseCOOIndexColumnMajor) {
-  using IndexValueType = TypeParam;
-  using c_index_value_type = typename IndexValueType::c_type;
-
-  std::string path = "test-write-sparse-coo-tensor";
-  constexpr int64_t kBufferSize = 1 << 20;
-  ASSERT_OK_AND_ASSIGN(this->mmap_,
-                       io::MemoryMapFixture::InitMemoryMap(kBufferSize, path));
-
-  // Dense representation:
-  // [
-  //   [
-  //     1 0 2 0
-  //     0 3 0 4
-  //     5 0 6 0
-  //   ],
-  //   [
-  //      0 11  0 12
-  //     13  0 14  0
-  //      0 15  0 16
-  //   ]
-  // ]
-  //
-  // Sparse representation:
-  // idx[0] = [0 0 0 0 0 0  1  1  1  1  1  1]
-  // idx[1] = [0 0 1 1 2 2  0  0  1  1  2  2]
-  // idx[2] = [0 2 1 3 0 2  1  3  0  2  1  3]
-  // data   = [1 2 3 4 5 6 11 12 13 14 15 16]
-
-  // canonical
-  std::vector<c_index_value_type> coords_values = {0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1,
-                                                   0, 0, 1, 1, 2, 2, 0, 0, 1, 1, 2, 2,
-                                                   0, 2, 1, 3, 0, 2, 1, 3, 0, 2, 1, 3};
-  const int sizeof_index_value = sizeof(c_index_value_type);
-  std::shared_ptr<SparseCOOIndex> si;
-  ASSERT_OK_AND_ASSIGN(
-      si, SparseCOOIndex::Make(TypeTraits<IndexValueType>::type_singleton(), {12, 3},
-                               {sizeof_index_value, sizeof_index_value * 12},
-                               Buffer::Wrap(coords_values)));
-  ASSERT_TRUE(si->is_canonical());
+  // Make one-column batch with a dictionary array
+  std::shared_ptr<RecordBatch> MakeBatch(std::shared_ptr<DataType> type,
+                                         std::shared_ptr<Array> indices,
+                                         std::shared_ptr<Array> dictionary) {
+    auto array = *DictionaryArray::FromArrays(std::move(type), std::move(indices),
+                                              std::move(dictionary));
+    return MakeBatch(std::move(array));
+  }
 
-  std::vector<int64_t> shape = {2, 3, 4};
-  std::vector<std::string> dim_names = {"foo", "bar", "baz"};
-  std::vector<int64_t> values = {1, 2, 3, 4, 5, 6, 11, 12, 13, 14, 15, 16};
+ protected:
+  IpcWriteOptions write_options_ = IpcWriteOptions::Defaults();
+  IpcReadOptions read_options_ = IpcReadOptions::Defaults();
+  ReadStats read_stats_;
+};
 
-  std::shared_ptr<SparseCOOTensor> st;
-  ASSERT_OK_AND_ASSIGN(st, this->MakeSparseCOOTensor(si, values, shape, dim_names));
+TYPED_TEST_SUITE_P(TestDictionaryReplacement);
 
-  this->CheckSparseCOOTensorRoundTrip(*st);
+TYPED_TEST_P(TestDictionaryReplacement, SameDictPointer) { this->TestSameDictPointer(); }
 
-  // non-canonical
-  ASSERT_OK_AND_ASSIGN(
-      si, SparseCOOIndex::Make(TypeTraits<IndexValueType>::type_singleton(), {12, 3},
-                               {sizeof_index_value, sizeof_index_value * 12},
-                               Buffer::Wrap(coords_values), false));
-  ASSERT_FALSE(si->is_canonical());
-  ASSERT_OK_AND_ASSIGN(st, this->MakeSparseCOOTensor(si, values, shape, dim_names));
+TYPED_TEST_P(TestDictionaryReplacement, SameDictValues) { this->TestSameDictValues(); }
 
-  this->CheckSparseCOOTensorRoundTrip(*st);
+TYPED_TEST_P(TestDictionaryReplacement, SameDictValuesNested) {
+  this->TestSameDictValuesNested();
 }
 
-TYPED_TEST_P(TestSparseTensorRoundTrip, WithSparseCSRIndex) {
-  using IndexValueType = TypeParam;
-
-  std::string path = "test-write-sparse-csr-matrix";
-  constexpr int64_t kBufferSize = 1 << 20;
-  ASSERT_OK_AND_ASSIGN(this->mmap_,
-                       io::MemoryMapFixture::InitMemoryMap(kBufferSize, path));
-
-  std::vector<int64_t> shape = {4, 6};
-  std::vector<std::string> dim_names = {"foo", "bar", "baz"};
-  std::vector<int64_t> values = {1, 0,  2, 0,  0,  3, 0,  4, 5, 0,  6, 0,
-                                 0, 11, 0, 12, 13, 0, 14, 0, 0, 15, 0, 16};
-
-  auto data = Buffer::Wrap(values);
-  NumericTensor<Int64Type> t(data, shape, {}, dim_names);
-  std::shared_ptr<SparseCSRMatrix> st;
-  ASSERT_OK_AND_ASSIGN(
-      st, SparseCSRMatrix::Make(t, TypeTraits<IndexValueType>::type_singleton()));
-
-  this->CheckSparseCSXMatrixRoundTrip(*st);
+TYPED_TEST_P(TestDictionaryReplacement, DifferentDictValues) {
+  this->TestDifferentDictValues();
 }
 
-TYPED_TEST_P(TestSparseTensorRoundTrip, WithSparseCSCIndex) {
-  using IndexValueType = TypeParam;
-
-  std::string path = "test-write-sparse-csc-matrix";
-  constexpr int64_t kBufferSize = 1 << 20;
-  ASSERT_OK_AND_ASSIGN(this->mmap_,
-                       io::MemoryMapFixture::InitMemoryMap(kBufferSize, path));
-
-  std::vector<int64_t> shape = {4, 6};
-  std::vector<std::string> dim_names = {"foo", "bar", "baz"};
-  std::vector<int64_t> values = {1, 0,  2, 0,  0,  3, 0,  4, 5, 0,  6, 0,
-                                 0, 11, 0, 12, 13, 0, 14, 0, 0, 15, 0, 16};
-
-  auto data = Buffer::Wrap(values);
-  NumericTensor<Int64Type> t(data, shape, {}, dim_names);
-  std::shared_ptr<SparseCSCMatrix> st;
-  ASSERT_OK_AND_ASSIGN(
-      st, SparseCSCMatrix::Make(t, TypeTraits<IndexValueType>::type_singleton()));
-
-  this->CheckSparseCSXMatrixRoundTrip(*st);
+TYPED_TEST_P(TestDictionaryReplacement, DifferentDictValuesNested) {
+  this->TestDifferentDictValuesNested();
 }
 
-TYPED_TEST_P(TestSparseTensorRoundTrip, WithSparseCSFIndex) {
-  using IndexValueType = TypeParam;
+REGISTER_TYPED_TEST_SUITE_P(TestDictionaryReplacement, SameDictPointer, SameDictValues,
+                            SameDictValuesNested, DifferentDictValues,
+                            DifferentDictValuesNested);
 
-  std::string path = "test-write-sparse-csf-tensor";
-  constexpr int64_t kBufferSize = 1 << 20;
-  ASSERT_OK_AND_ASSIGN(this->mmap_,
-                       io::MemoryMapFixture::InitMemoryMap(kBufferSize, path));
+using DictionaryReplacementTestTypes =
+    ::testing::Types<StreamWriterHelper, FileWriterHelper>;
 
-  std::vector<int64_t> shape = {4, 6};
-  std::vector<std::string> dim_names = {"foo", "bar", "baz"};
-  std::vector<int64_t> values = {1, 0,  2, 0,  0,  3, 0,  4, 5, 0,  6, 0,
-                                 0, 11, 0, 12, 13, 0, 14, 0, 0, 15, 0, 16};
+INSTANTIATE_TYPED_TEST_SUITE_P(TestDictionaryReplacement, TestDictionaryReplacement,
+                               DictionaryReplacementTestTypes);
 
-  auto data = Buffer::Wrap(values);
-  NumericTensor<Int64Type> t(data, shape, {}, dim_names);
-  std::shared_ptr<SparseCSFTensor> st;
-  ASSERT_OK_AND_ASSIGN(
-      st, SparseCSFTensor::Make(t, TypeTraits<IndexValueType>::type_singleton()));
+// ----------------------------------------------------------------------
+// Miscellanea
 
-  this->CheckSparseCSFTensorRoundTrip(*st);
+TEST(FieldPosition, Basics) {
+  FieldPosition pos;
+  ASSERT_EQ(pos.path(), std::vector<int>{});
+  {
+    auto child = pos.child(6);
+    ASSERT_EQ(child.path(), std::vector<int>{6});
+    auto grand_child = child.child(42);
+    ASSERT_EQ(grand_child.path(), (std::vector<int>{6, 42}));
+  }
+  {
+    auto child = pos.child(12);
+    ASSERT_EQ(child.path(), std::vector<int>{12});
+  }
 }
-REGISTER_TYPED_TEST_SUITE_P(TestSparseTensorRoundTrip, WithSparseCOOIndexRowMajor,
-                            WithSparseCOOIndexColumnMajor, WithSparseCSRIndex,
-                            WithSparseCSCIndex, WithSparseCSFIndex);
-
-INSTANTIATE_TYPED_TEST_SUITE_P(TestInt8, TestSparseTensorRoundTrip, Int8Type);
-INSTANTIATE_TYPED_TEST_SUITE_P(TestUInt8, TestSparseTensorRoundTrip, UInt8Type);
-INSTANTIATE_TYPED_TEST_SUITE_P(TestInt16, TestSparseTensorRoundTrip, Int16Type);
-INSTANTIATE_TYPED_TEST_SUITE_P(TestUInt16, TestSparseTensorRoundTrip, UInt16Type);
-INSTANTIATE_TYPED_TEST_SUITE_P(TestInt32, TestSparseTensorRoundTrip, Int32Type);
-INSTANTIATE_TYPED_TEST_SUITE_P(TestUInt32, TestSparseTensorRoundTrip, UInt32Type);
-INSTANTIATE_TYPED_TEST_SUITE_P(TestInt64, TestSparseTensorRoundTrip, Int64Type);
 
-TEST(TestRecordBatchStreamReader, MalformedInput) {
-  const std::string empty_str = "";
-  const std::string garbage_str = "12345678";
+TEST(DictionaryFieldMapper, Basics) {
+  DictionaryFieldMapper mapper;
+
+  ASSERT_EQ(mapper.num_fields(), 0);
+
+  ASSERT_OK(mapper.AddField(42, {0, 1}));
+  ASSERT_OK(mapper.AddField(43, {0, 2}));
+  ASSERT_OK(mapper.AddField(44, {0, 1, 3}));
+  ASSERT_EQ(mapper.num_fields(), 3);
+
+  ASSERT_OK_AND_EQ(42, mapper.GetFieldId({0, 1}));
+  ASSERT_OK_AND_EQ(43, mapper.GetFieldId({0, 2}));
+  ASSERT_OK_AND_EQ(44, mapper.GetFieldId({0, 1, 3}));
+  ASSERT_RAISES(KeyError, mapper.GetFieldId({}));
+  ASSERT_RAISES(KeyError, mapper.GetFieldId({0}));
+  ASSERT_RAISES(KeyError, mapper.GetFieldId({0, 1, 2}));
+  ASSERT_RAISES(KeyError, mapper.GetFieldId({1}));
+
+  ASSERT_OK(mapper.AddField(41, {}));
+  ASSERT_EQ(mapper.num_fields(), 4);
+  ASSERT_OK_AND_EQ(41, mapper.GetFieldId({}));
+  ASSERT_OK_AND_EQ(42, mapper.GetFieldId({0, 1}));
+
+  // Duplicated dictionary ids are allowed
+  ASSERT_OK(mapper.AddField(42, {4, 5, 6}));
+  ASSERT_EQ(mapper.num_fields(), 5);
+  ASSERT_OK_AND_EQ(42, mapper.GetFieldId({0, 1}));
+  ASSERT_OK_AND_EQ(42, mapper.GetFieldId({4, 5, 6}));
+
+  // Duplicated fields paths are not
+  ASSERT_RAISES(KeyError, mapper.AddField(46, {0, 1}));
+}
 
-  auto empty = std::make_shared<Buffer>(empty_str);
-  auto garbage = std::make_shared<Buffer>(garbage_str);
+TEST(DictionaryFieldMapper, FromSchema) {
+  auto f0 = field("f0", int8());
+  auto f1 =
+      field("f1", struct_({field("a", null()), field("b", dictionary(int8(), utf8()))}));
+  auto f2 = field("f2", dictionary(int32(), list(dictionary(int8(), utf8()))));
 
-  io::BufferReader empty_reader(empty);
-  ASSERT_RAISES(Invalid, RecordBatchStreamReader::Open(&empty_reader));
+  Schema schema({f0, f1, f2});
+  DictionaryFieldMapper mapper(schema);
 
-  io::BufferReader garbage_reader(garbage);
-  ASSERT_RAISES(Invalid, RecordBatchStreamReader::Open(&garbage_reader));
+  ASSERT_EQ(mapper.num_fields(), 3);
+  std::unordered_set<int64_t> ids;
+  for (const auto& path : std::vector<std::vector<int>>{{1, 1}, {2}, {2, 0}}) {
+    ASSERT_OK_AND_ASSIGN(const int64_t id, mapper.GetFieldId(path));
+    ids.insert(id);
+  }
+  ASSERT_EQ(ids.size(), 3);  // All ids are distinct
 }
 
-TEST(TestStreamDecoder, NextRequiredSize) {
-  auto listener = std::make_shared<CollectListener>();
-  StreamDecoder decoder(listener);
-  auto next_required_size = decoder.next_required_size();
-  const uint8_t data[1] = {0};
-  ASSERT_OK(decoder.Consume(data, 1));
-  ASSERT_EQ(next_required_size - 1, decoder.next_required_size());
+static void AssertMemoDictionaryType(const DictionaryMemo& memo, int64_t id,
+                                     const std::shared_ptr<DataType>& expected) {
+  ASSERT_OK_AND_ASSIGN(const auto actual, memo.GetDictionaryType(id));
+  AssertTypeEqual(*expected, *actual);
 }
 
-// ----------------------------------------------------------------------
-// DictionaryMemo miscellanea
-
-TEST(TestDictionaryMemo, ReusedDictionaries) {
+TEST(DictionaryMemo, AddDictionaryType) {
   DictionaryMemo memo;
-
-  std::shared_ptr<Field> field1 = field("a", dictionary(int8(), utf8()));
-  std::shared_ptr<Field> field2 = field("b", dictionary(int16(), utf8()));
-
-  // Two fields referencing the same dictionary_id
-  int64_t dictionary_id = 0;
-  auto dict = ArrayFromJSON(utf8(), "[\"foo\", \"bar\", \"baz\"]");
-
-  ASSERT_OK(memo.AddField(dictionary_id, field1));
-  ASSERT_OK(memo.AddField(dictionary_id, field2));
-
-  std::shared_ptr<DataType> value_type;
-  ASSERT_OK(memo.GetDictionaryType(dictionary_id, &value_type));
-  ASSERT_TRUE(value_type->Equals(*utf8()));
-
-  ASSERT_FALSE(memo.HasDictionary(dictionary_id));
-  ASSERT_OK(memo.AddDictionary(dictionary_id, dict));
-  ASSERT_TRUE(memo.HasDictionary(dictionary_id));
-
-  ASSERT_EQ(2, memo.num_fields());
-  ASSERT_EQ(1, memo.num_dictionaries());
-
-  ASSERT_TRUE(memo.HasDictionary(*field1));
-  ASSERT_TRUE(memo.HasDictionary(*field2));
-
-  int64_t returned_id = -1;
-  ASSERT_OK(memo.GetId(field1.get(), &returned_id));
-  ASSERT_EQ(0, returned_id);
-  returned_id = -1;
-  ASSERT_OK(memo.GetId(field2.get(), &returned_id));
-  ASSERT_EQ(0, returned_id);
+  std::shared_ptr<DataType> type;
+
+  ASSERT_RAISES(KeyError, memo.GetDictionaryType(42));
+
+  ASSERT_OK(memo.AddDictionaryType(42, utf8()));
+  ASSERT_OK(memo.AddDictionaryType(43, large_binary()));
+  AssertMemoDictionaryType(memo, 42, utf8());
+  AssertMemoDictionaryType(memo, 43, large_binary());
+
+  // Re-adding same type with different id
+  ASSERT_OK(memo.AddDictionaryType(44, utf8()));
+  AssertMemoDictionaryType(memo, 42, utf8());
+  AssertMemoDictionaryType(memo, 44, utf8());
+
+  // Re-adding same type with same id
+  ASSERT_OK(memo.AddDictionaryType(42, utf8()));
+  AssertMemoDictionaryType(memo, 42, utf8());
+  AssertMemoDictionaryType(memo, 44, utf8());
+
+  // Trying to add different type with same id
+  ASSERT_RAISES(KeyError, memo.AddDictionaryType(42, large_utf8()));
+  AssertMemoDictionaryType(memo, 42, utf8());
+  AssertMemoDictionaryType(memo, 43, large_binary());
+  AssertMemoDictionaryType(memo, 44, utf8());
 }
 
 }  // namespace test
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index a7fb9f75ea4..92f2b70f294 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -72,6 +72,8 @@ using internal::kArrowMagicBytes;
 
 namespace {
 
+enum class DictionaryKind { New, Delta, Replacement };
+
 Status InvalidMessageType(MessageType expected, MessageType actual) {
   return Status::IOError("Expected IPC message of type ", FormatMessageType(expected),
                          " but got ", FormatMessageType(actual));
@@ -110,13 +112,11 @@ Status InvalidMessageType(MessageType expected, MessageType actual) {
 class ArrayLoader {
  public:
   explicit ArrayLoader(const flatbuf::RecordBatch* metadata,
-                       MetadataVersion metadata_version,
-                       const DictionaryMemo* dictionary_memo,
-                       const IpcReadOptions& options, io::RandomAccessFile* file)
+                       MetadataVersion metadata_version, const IpcReadOptions& options,
+                       io::RandomAccessFile* file)
       : metadata_(metadata),
         metadata_version_(metadata_version),
         file_(file),
-        dictionary_memo_(dictionary_memo),
         max_recursion_depth_(options.max_recursion_depth) {}
 
   Status ReadBuffer(int64_t offset, int64_t length, std::shared_ptr<Buffer>* out) {
@@ -244,15 +244,15 @@ class ArrayLoader {
     return LoadChildren(type.fields());
   }
 
-  Status LoadChildren(std::vector<std::shared_ptr<Field>> child_fields) {
+  Status LoadChildren(const std::vector<std::shared_ptr<Field>>& child_fields) {
     ArrayData* parent = out_;
-    parent->child_data.reserve(static_cast<int>(child_fields.size()));
-    for (const auto& child_field : child_fields) {
-      auto field_array = std::make_shared<ArrayData>();
+
+    parent->child_data.resize(child_fields.size());
+    for (int i = 0; i < static_cast<int>(child_fields.size()); ++i) {
+      parent->child_data[i] = std::make_shared<ArrayData>();
       --max_recursion_depth_;
-      RETURN_NOT_OK(Load(child_field.get(), field_array.get()));
+      RETURN_NOT_OK(Load(child_fields[i].get(), parent->child_data[i].get()));
       ++max_recursion_depth_;
-      parent->child_data.emplace_back(field_array);
     }
     out_ = parent;
     return Status::OK();
@@ -348,16 +348,8 @@ class ArrayLoader {
   }
 
   Status Visit(const DictionaryType& type) {
-    RETURN_NOT_OK(LoadType(*type.index_type()));
-
-    // Look up dictionary
-    int64_t id = -1;
-    RETURN_NOT_OK(dictionary_memo_->GetId(field_, &id));
-
-    std::shared_ptr<Array> boxed_dict;
-    RETURN_NOT_OK(dictionary_memo_->GetDictionary(id, &boxed_dict));
-    out_->dictionary = boxed_dict->data();
-    return Status::OK();
+    // out_->dictionary will be filled later in ResolveDictionaries()
+    return LoadType(*type.index_type());
   }
 
   Status Visit(const ExtensionType& type) { return LoadType(*type.storage_type()); }
@@ -366,7 +358,6 @@ class ArrayLoader {
   const flatbuf::RecordBatch* metadata_;
   const MetadataVersion metadata_version_;
   io::RandomAccessFile* file_;
-  const DictionaryMemo* dictionary_memo_;
   int max_recursion_depth_;
   int buffer_index_ = 0;
   int field_index_ = 0;
@@ -410,11 +401,11 @@ Result<std::shared_ptr<Buffer>> DecompressBuffer(const std::shared_ptr<Buffer>&
 }
 
 Status DecompressBuffers(Compression::type compression, const IpcReadOptions& options,
-                         std::vector<std::shared_ptr<ArrayData>>* fields) {
+                         ArrayDataVector* fields) {
   struct BufferAccumulator {
     using BufferPtrVector = std::vector<std::shared_ptr<Buffer>*>;
 
-    void AppendFrom(const std::vector<std::shared_ptr<ArrayData>>& fields) {
+    void AppendFrom(const ArrayDataVector& fields) {
       for (const auto& field : fields) {
         for (auto& buffer : field->buffers) {
           buffers_.push_back(&buffer);
@@ -423,7 +414,7 @@ Status DecompressBuffers(Compression::type compression, const IpcReadOptions& op
       }
     }
 
-    BufferPtrVector Get(const std::vector<std::shared_ptr<ArrayData>>& fields) && {
+    BufferPtrVector Get(const ArrayDataVector& fields) && {
       AppendFrom(fields);
       return std::move(buffers_);
     }
@@ -431,7 +422,7 @@ Status DecompressBuffers(Compression::type compression, const IpcReadOptions& op
     BufferPtrVector buffers_;
   };
 
-  // flatten all buffers
+  // Flatten all buffers
   auto buffers = BufferAccumulator{}.Get(*fields);
 
   std::unique_ptr<util::Codec> codec;
@@ -447,37 +438,54 @@ Status DecompressBuffers(Compression::type compression, const IpcReadOptions& op
 
 Result<std::shared_ptr<RecordBatch>> LoadRecordBatchSubset(
     const flatbuf::RecordBatch* metadata, const std::shared_ptr<Schema>& schema,
-    const std::vector<bool>& inclusion_mask, const DictionaryMemo* dictionary_memo,
+    const std::vector<bool>* inclusion_mask, const DictionaryMemo* dictionary_memo,
     const IpcReadOptions& options, MetadataVersion metadata_version,
     Compression::type compression, io::RandomAccessFile* file) {
-  ArrayLoader loader(metadata, metadata_version, dictionary_memo, options, file);
+  ArrayLoader loader(metadata, metadata_version, options, file);
 
-  std::vector<std::shared_ptr<ArrayData>> field_data;
-  std::vector<std::shared_ptr<Field>> schema_fields;
+  ArrayDataVector columns(schema->num_fields());
+  ArrayDataVector filtered_columns;
+  FieldVector filtered_fields;
+  std::shared_ptr<Schema> filtered_schema;
 
   for (int i = 0; i < schema->num_fields(); ++i) {
-    if (inclusion_mask[i]) {
+    const Field& field = *schema->field(i);
+    if (!inclusion_mask || (*inclusion_mask)[i]) {
       // Read field
-      auto arr = std::make_shared<ArrayData>();
-      RETURN_NOT_OK(loader.Load(schema->field(i).get(), arr.get()));
-      if (metadata->length() != arr->length) {
+      auto column = std::make_shared<ArrayData>();
+      RETURN_NOT_OK(loader.Load(&field, column.get()));
+      if (metadata->length() != column->length) {
         return Status::IOError("Array length did not match record batch length");
       }
-      field_data.emplace_back(std::move(arr));
-      schema_fields.emplace_back(schema->field(i));
+      columns[i] = std::move(column);
+      if (inclusion_mask) {
+        filtered_columns.push_back(columns[i]);
+        filtered_fields.push_back(schema->field(i));
+      }
     } else {
       // Skip field. This logic must be executed to advance the state of the
       // loader to the next field
-      RETURN_NOT_OK(loader.SkipField(schema->field(i).get()));
+      RETURN_NOT_OK(loader.SkipField(&field));
     }
   }
 
+  // Dictionary resolution needs to happen on the unfiltered columns,
+  // because fields are mapped structurally (by path in the original schema).
+  RETURN_NOT_OK(ResolveDictionaries(columns, *dictionary_memo, options.memory_pool));
+
+  if (inclusion_mask) {
+    filtered_schema = ::arrow::schema(std::move(filtered_fields), schema->metadata());
+    columns.clear();
+  } else {
+    filtered_schema = schema;
+    filtered_columns = std::move(columns);
+  }
   if (compression != Compression::UNCOMPRESSED) {
-    RETURN_NOT_OK(DecompressBuffers(compression, options, &field_data));
+    RETURN_NOT_OK(DecompressBuffers(compression, options, &filtered_columns));
   }
 
-  return RecordBatch::Make(::arrow::schema(std::move(schema_fields), schema->metadata()),
-                           metadata->length(), std::move(field_data));
+  return RecordBatch::Make(filtered_schema, metadata->length(),
+                           std::move(filtered_columns));
 }
 
 Result<std::shared_ptr<RecordBatch>> LoadRecordBatch(
@@ -486,24 +494,12 @@ Result<std::shared_ptr<RecordBatch>> LoadRecordBatch(
     const IpcReadOptions& options, MetadataVersion metadata_version,
     Compression::type compression, io::RandomAccessFile* file) {
   if (inclusion_mask.size() > 0) {
-    return LoadRecordBatchSubset(metadata, schema, inclusion_mask, dictionary_memo,
+    return LoadRecordBatchSubset(metadata, schema, &inclusion_mask, dictionary_memo,
                                  options, metadata_version, compression, file);
+  } else {
+    return LoadRecordBatchSubset(metadata, schema, nullptr, dictionary_memo, options,
+                                 metadata_version, compression, file);
   }
-
-  ArrayLoader loader(metadata, metadata_version, dictionary_memo, options, file);
-  std::vector<std::shared_ptr<ArrayData>> arrays(schema->num_fields());
-  for (int i = 0; i < schema->num_fields(); ++i) {
-    auto arr = std::make_shared<ArrayData>();
-    RETURN_NOT_OK(loader.Load(schema->field(i).get(), arr.get()));
-    if (metadata->length() != arr->length) {
-      return Status::IOError("Array length did not match record batch length");
-    }
-    arrays[i] = std::move(arr);
-  }
-  if (compression != Compression::UNCOMPRESSED) {
-    RETURN_NOT_OK(DecompressBuffers(compression, options, &arrays));
-  }
-  return RecordBatch::Make(schema, metadata->length(), std::move(arrays));
 }
 
 // ----------------------------------------------------------------------
@@ -675,17 +671,18 @@ Result<std::shared_ptr<RecordBatch>> ReadRecordBatch(
 }
 
 Status ReadDictionary(const Buffer& metadata, DictionaryMemo* dictionary_memo,
-                      const IpcReadOptions& options, io::RandomAccessFile* file) {
+                      const IpcReadOptions& options, DictionaryKind* kind,
+                      io::RandomAccessFile* file) {
   const flatbuf::Message* message = nullptr;
   RETURN_NOT_OK(internal::VerifyMessage(metadata.data(), metadata.size(), &message));
-  auto dictionary_batch = message->header_as_DictionaryBatch();
+  const auto dictionary_batch = message->header_as_DictionaryBatch();
   if (dictionary_batch == nullptr) {
     return Status::IOError(
         "Header-type of flatbuffer-encoded Message is not DictionaryBatch.");
   }
 
   // The dictionary is embedded in a record batch with a single column
-  auto batch_meta = dictionary_batch->data();
+  const auto batch_meta = dictionary_batch->data();
 
   CHECK_FLATBUFFERS_NOT_NULL(batch_meta, "DictionaryBatch.data");
 
@@ -698,45 +695,46 @@ Status ReadDictionary(const Buffer& metadata, DictionaryMemo* dictionary_memo,
     RETURN_NOT_OK(GetCompressionExperimental(message, &compression));
   }
 
-  int64_t id = dictionary_batch->id();
+  const int64_t id = dictionary_batch->id();
 
-  // Look up the field, which must have been added to the
+  // Look up the dictionary value type, which must have been added to the
   // DictionaryMemo already prior to invoking this function
-  std::shared_ptr<DataType> value_type;
-  RETURN_NOT_OK(dictionary_memo->GetDictionaryType(id, &value_type));
+  ARROW_ASSIGN_OR_RAISE(auto value_type, dictionary_memo->GetDictionaryType(id));
 
-  auto value_field = ::arrow::field("dummy", value_type);
+  // Load the dictionary data from the dictionary batch
+  ArrayLoader loader(batch_meta, internal::GetMetadataVersion(message->version()),
+                     options, file);
+  const auto dict_data = std::make_shared<ArrayData>();
+  const Field dummy_field("", value_type);
+  RETURN_NOT_OK(loader.Load(&dummy_field, dict_data.get()));
+
+  if (compression != Compression::UNCOMPRESSED) {
+    ArrayDataVector dict_fields{dict_data};
+    RETURN_NOT_OK(DecompressBuffers(compression, options, &dict_fields));
+  }
 
-  std::shared_ptr<RecordBatch> batch;
-  ARROW_ASSIGN_OR_RAISE(
-      batch, LoadRecordBatch(batch_meta, ::arrow::schema({value_field}),
-                             /*field_inclusion_mask=*/{}, dictionary_memo, options,
-                             internal::GetMetadataVersion(message->version()),
-                             compression, file));
-  if (batch->num_columns() != 1) {
-    return Status::Invalid("Dictionary record batch must only contain one field");
-  }
-  auto dictionary = batch->column(0);
-  // Validate the dictionary for safe delta concatenation
-  RETURN_NOT_OK(dictionary->Validate());
   if (dictionary_batch->isDelta()) {
-    return dictionary_memo->AddDictionaryDelta(id, dictionary, options.memory_pool);
+    if (kind != nullptr) {
+      *kind = DictionaryKind::Delta;
+    }
+    return dictionary_memo->AddDictionaryDelta(id, dict_data);
+  }
+  ARROW_ASSIGN_OR_RAISE(bool inserted,
+                        dictionary_memo->AddOrReplaceDictionary(id, dict_data));
+  if (kind != nullptr) {
+    *kind = inserted ? DictionaryKind::New : DictionaryKind::Replacement;
   }
-  return dictionary_memo->AddOrReplaceDictionary(id, dictionary);
+  return Status::OK();
 }
 
-Status ParseDictionary(const Message& message, DictionaryMemo* dictionary_memo,
-                       const IpcReadOptions& options) {
+Status ReadDictionary(const Message& message, DictionaryMemo* dictionary_memo,
+                      const IpcReadOptions& options, DictionaryKind* kind) {
   // Only invoke this method if we already know we have a dictionary message
   DCHECK_EQ(message.type(), MessageType::DICTIONARY_BATCH);
   CHECK_HAS_BODY(message);
   ARROW_ASSIGN_OR_RAISE(auto reader, Buffer::GetReader(message.body()));
-  return ReadDictionary(*message.metadata(), dictionary_memo, options, reader.get());
-}
-
-Status UpdateDictionaries(const Message& message, DictionaryMemo* dictionary_memo,
-                          const IpcReadOptions& options) {
-  return ParseDictionary(message, dictionary_memo, options);
+  return ReadDictionary(*message.metadata(), dictionary_memo, options, kind,
+                        reader.get());
 }
 
 // ----------------------------------------------------------------------
@@ -750,8 +748,7 @@ class RecordBatchStreamReaderImpl : public RecordBatchStreamReader {
     options_ = options;
 
     // Read schema
-    ARROW_ASSIGN_OR_RAISE(std::unique_ptr<Message> message,
-                          message_reader_->ReadNextMessage());
+    ARROW_ASSIGN_OR_RAISE(std::unique_ptr<Message> message, ReadNextMessage());
     if (!message) {
       return Status::Invalid("Tried reading schema message, was null or length 0");
     }
@@ -774,11 +771,11 @@ class RecordBatchStreamReaderImpl : public RecordBatchStreamReader {
 
     // Continue to read other dictionaries, if any
     std::unique_ptr<Message> message;
-    ARROW_ASSIGN_OR_RAISE(message, message_reader_->ReadNextMessage());
+    ARROW_ASSIGN_OR_RAISE(message, ReadNextMessage());
 
     while (message != nullptr && message->type() == MessageType::DICTIONARY_BATCH) {
-      RETURN_NOT_OK(UpdateDictionaries(*message, &dictionary_memo_, options_));
-      ARROW_ASSIGN_OR_RAISE(message, message_reader_->ReadNextMessage());
+      RETURN_NOT_OK(ReadDictionary(*message));
+      ARROW_ASSIGN_OR_RAISE(message, ReadNextMessage());
     }
 
     if (message == nullptr) {
@@ -796,7 +793,45 @@ class RecordBatchStreamReaderImpl : public RecordBatchStreamReader {
 
   std::shared_ptr<Schema> schema() const override { return out_schema_; }
 
+  ReadStats stats() const override { return stats_; }
+
  private:
+  Result<std::unique_ptr<Message>> ReadNextMessage() {
+    ARROW_ASSIGN_OR_RAISE(auto message, message_reader_->ReadNextMessage());
+    if (message) {
+      ++stats_.num_messages;
+      switch (message->type()) {
+        case MessageType::RECORD_BATCH:
+          ++stats_.num_record_batches;
+          break;
+        case MessageType::DICTIONARY_BATCH:
+          ++stats_.num_dictionary_batches;
+          break;
+        default:
+          break;
+      }
+    }
+    return std::move(message);
+  }
+
+  // Read dictionary from dictionary batch
+  Status ReadDictionary(const Message& message) {
+    DictionaryKind kind;
+    RETURN_NOT_OK(
+        ::arrow::ipc::ReadDictionary(message, &dictionary_memo_, options_, &kind));
+    switch (kind) {
+      case DictionaryKind::New:
+        break;
+      case DictionaryKind::Delta:
+        ++stats_.num_dictionary_deltas;
+        break;
+      case DictionaryKind::Replacement:
+        ++stats_.num_replaced_dictionaries;
+        break;
+    }
+    return Status::OK();
+  }
+
   Status ReadInitialDictionaries() {
     // We must receive all dictionaries before reconstructing the
     // first record batch. Subsequent dictionary deltas modify the memo
@@ -804,8 +839,9 @@ class RecordBatchStreamReaderImpl : public RecordBatchStreamReader {
 
     // TODO(wesm): In future, we may want to reconcile the ids in the stream with
     // those found in the schema
-    for (int i = 0; i < dictionary_memo_.num_fields(); ++i) {
-      ARROW_ASSIGN_OR_RAISE(message, message_reader_->ReadNextMessage());
+    const auto num_dicts = dictionary_memo_.fields().num_fields();
+    for (int i = 0; i < num_dicts; ++i) {
+      ARROW_ASSIGN_OR_RAISE(message, ReadNextMessage());
       if (!message) {
         if (i == 0) {
           /// ARROW-6006: If we fail to find any dictionaries in the stream, then
@@ -819,16 +855,15 @@ class RecordBatchStreamReaderImpl : public RecordBatchStreamReader {
           // ARROW-6126, the stream terminated before receiving the expected
           // number of dictionaries
           return Status::Invalid("IPC stream ended without reading the expected number (",
-                                 dictionary_memo_.num_fields(), ") of dictionaries");
+                                 num_dicts, ") of dictionaries");
         }
       }
 
       if (message->type() != MessageType::DICTIONARY_BATCH) {
-        return Status::Invalid("IPC stream did not have the expected number (",
-                               dictionary_memo_.num_fields(),
+        return Status::Invalid("IPC stream did not have the expected number (", num_dicts,
                                ") of dictionaries at the start of the stream");
       }
-      RETURN_NOT_OK(ParseDictionary(*message, &dictionary_memo_, options_));
+      RETURN_NOT_OK(ReadDictionary(*message));
     }
 
     have_read_initial_dictionaries_ = true;
@@ -845,6 +880,8 @@ class RecordBatchStreamReaderImpl : public RecordBatchStreamReader {
   // and so the reader should not attempt to parse any messages
   bool empty_stream_ = false;
 
+  ReadStats stats_;
+
   DictionaryMemo dictionary_memo_;
   std::shared_ptr<Schema> schema_, out_schema_;
 };
@@ -852,7 +889,7 @@ class RecordBatchStreamReaderImpl : public RecordBatchStreamReader {
 // ----------------------------------------------------------------------
 // Stream reader constructors
 
-Result<std::shared_ptr<RecordBatchReader>> RecordBatchStreamReader::Open(
+Result<std::shared_ptr<RecordBatchStreamReader>> RecordBatchStreamReader::Open(
     std::unique_ptr<MessageReader> message_reader, const IpcReadOptions& options) {
   // Private ctor
   auto result = std::make_shared<RecordBatchStreamReaderImpl>();
@@ -860,12 +897,12 @@ Result<std::shared_ptr<RecordBatchReader>> RecordBatchStreamReader::Open(
   return result;
 }
 
-Result<std::shared_ptr<RecordBatchReader>> RecordBatchStreamReader::Open(
+Result<std::shared_ptr<RecordBatchStreamReader>> RecordBatchStreamReader::Open(
     io::InputStream* stream, const IpcReadOptions& options) {
   return Open(MessageReader::Open(stream), options);
 }
 
-Result<std::shared_ptr<RecordBatchReader>> RecordBatchStreamReader::Open(
+Result<std::shared_ptr<RecordBatchStreamReader>> RecordBatchStreamReader::Open(
     const std::shared_ptr<io::InputStream>& stream, const IpcReadOptions& options) {
   return Open(MessageReader::Open(stream), options);
 }
@@ -898,13 +935,16 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader {
       read_dictionaries_ = true;
     }
 
-    std::unique_ptr<Message> message;
-    RETURN_NOT_OK(ReadMessageFromBlock(GetRecordBatchBlock(i), &message));
+    ARROW_ASSIGN_OR_RAISE(auto message, ReadMessageFromBlock(GetRecordBatchBlock(i)));
 
     CHECK_HAS_BODY(*message);
     ARROW_ASSIGN_OR_RAISE(auto reader, Buffer::GetReader(message->body()));
-    return ReadRecordBatchInternal(*message->metadata(), schema_, field_inclusion_mask_,
-                                   &dictionary_memo_, options_, reader.get());
+    ARROW_ASSIGN_OR_RAISE(
+        auto batch,
+        ReadRecordBatchInternal(*message->metadata(), schema_, field_inclusion_mask_,
+                                &dictionary_memo_, options_, reader.get()));
+    ++stats_.num_record_batches;
+    return batch;
   }
 
   Status Open(const std::shared_ptr<io::RandomAccessFile>& file, int64_t footer_offset,
@@ -921,14 +961,18 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader {
     RETURN_NOT_OK(ReadFooter());
 
     // Get the schema and record any observed dictionaries
-    return UnpackSchemaMessage(footer_->schema(), options, &dictionary_memo_, &schema_,
-                               &out_schema_, &field_inclusion_mask_);
+    RETURN_NOT_OK(UnpackSchemaMessage(footer_->schema(), options, &dictionary_memo_,
+                                      &schema_, &out_schema_, &field_inclusion_mask_));
+    ++stats_.num_messages;
+    return Status::OK();
   }
 
   std::shared_ptr<Schema> schema() const override { return out_schema_; }
 
   std::shared_ptr<const KeyValueMetadata> metadata() const override { return metadata_; }
 
+  ReadStats stats() const override { return stats_; }
+
  private:
   FileBlock GetRecordBatchBlock(int i) const {
     return FileBlockFromFlatbuffer(footer_->recordBatches()->Get(i));
@@ -938,7 +982,7 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader {
     return FileBlockFromFlatbuffer(footer_->dictionaries()->Get(i));
   }
 
-  Status ReadMessageFromBlock(const FileBlock& block, std::unique_ptr<Message>* out) {
+  Result<std::unique_ptr<Message>> ReadMessageFromBlock(const FileBlock& block) {
     if (!BitUtil::IsMultipleOf8(block.offset) ||
         !BitUtil::IsMultipleOf8(block.metadata_length) ||
         !BitUtil::IsMultipleOf8(block.body_length)) {
@@ -948,19 +992,28 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader {
     // TODO(wesm): this breaks integration tests, see ARROW-3256
     // DCHECK_EQ((*out)->body_length(), block.body_length);
 
-    return ReadMessage(block.offset, block.metadata_length, file_).Value(out);
+    ARROW_ASSIGN_OR_RAISE(auto message,
+                          ReadMessage(block.offset, block.metadata_length, file_));
+    ++stats_.num_messages;
+    return std::move(message);
   }
 
   Status ReadDictionaries() {
     // Read all the dictionaries
     for (int i = 0; i < num_dictionaries(); ++i) {
-      std::unique_ptr<Message> message;
-      RETURN_NOT_OK(ReadMessageFromBlock(GetDictionaryBlock(i), &message));
+      ARROW_ASSIGN_OR_RAISE(auto message, ReadMessageFromBlock(GetDictionaryBlock(i)));
 
       CHECK_HAS_BODY(*message);
       ARROW_ASSIGN_OR_RAISE(auto reader, Buffer::GetReader(message->body()));
+      DictionaryKind kind;
       RETURN_NOT_OK(ReadDictionary(*message->metadata(), &dictionary_memo_, options_,
-                                   reader.get()));
+                                   &kind, reader.get()));
+      ++stats_.num_dictionary_batches;
+      if (kind != DictionaryKind::New) {
+        return Status::Invalid(
+            "Unsupported dictionary replacement or "
+            "dictionary delta in IPC file");
+      }
     }
     return Status::OK();
   }
@@ -1040,6 +1093,8 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader {
   std::shared_ptr<Schema> schema_;
   // Schema with deselected fields dropped
   std::shared_ptr<Schema> out_schema_;
+
+  ReadStats stats_;
 };
 
 Result<std::shared_ptr<RecordBatchFileReader>> RecordBatchFileReader::Open(
@@ -1139,7 +1194,7 @@ class StreamDecoder::StreamDecoderImpl : public MessageDecoderListener {
     RETURN_NOT_OK(UnpackSchemaMessage(*message, options_, &dictionary_memo_, &schema_,
                                       &out_schema_, &field_inclusion_mask_));
 
-    n_required_dictionaries_ = dictionary_memo_.num_fields();
+    n_required_dictionaries_ = dictionary_memo_.fields().num_fields();
     if (n_required_dictionaries_ == 0) {
       state_ = State::RECORD_BATCHES;
       RETURN_NOT_OK(listener_->OnSchemaDecoded(schema_));
@@ -1152,10 +1207,10 @@ class StreamDecoder::StreamDecoderImpl : public MessageDecoderListener {
   Status OnInitialDictionaryMessageDecoded(std::unique_ptr<Message> message) {
     if (message->type() != MessageType::DICTIONARY_BATCH) {
       return Status::Invalid("IPC stream did not have the expected number (",
-                             dictionary_memo_.num_fields(),
+                             dictionary_memo_.fields().num_fields(),
                              ") of dictionaries at the start of the stream");
     }
-    RETURN_NOT_OK(ParseDictionary(*message, &dictionary_memo_, options_));
+    RETURN_NOT_OK(ReadDictionary(*message));
     n_required_dictionaries_--;
     if (n_required_dictionaries_ == 0) {
       state_ = State::RECORD_BATCHES;
@@ -1166,7 +1221,7 @@ class StreamDecoder::StreamDecoderImpl : public MessageDecoderListener {
 
   Status OnRecordBatchMessageDecoded(std::unique_ptr<Message> message) {
     if (message->type() == MessageType::DICTIONARY_BATCH) {
-      return UpdateDictionaries(*message, &dictionary_memo_, options_);
+      return ReadDictionary(*message);
     } else {
       CHECK_HAS_BODY(*message);
       ARROW_ASSIGN_OR_RAISE(auto reader, Buffer::GetReader(message->body()));
@@ -1178,6 +1233,14 @@ class StreamDecoder::StreamDecoderImpl : public MessageDecoderListener {
     }
   }
 
+  // Read dictionary from dictionary batch
+  Status ReadDictionary(const Message& message) {
+    // TODO accumulate and expose ReadStats
+    DictionaryKind unused_kind;
+    return ::arrow::ipc::ReadDictionary(message, &dictionary_memo_, options_,
+                                        &unused_kind);
+  }
+
   std::shared_ptr<Listener> listener_;
   IpcReadOptions options_;
   State state_;
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index d7ff661fb60..8a663879a40 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -25,28 +25,16 @@
 #include <utility>
 #include <vector>
 
+#include "arrow/io/type_fwd.h"
 #include "arrow/ipc/message.h"
 #include "arrow/ipc/options.h"
 #include "arrow/record_batch.h"
 #include "arrow/result.h"
+#include "arrow/type_fwd.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
-
-class Buffer;
-class Schema;
-class Status;
-class Tensor;
-class SparseTensor;
-
-namespace io {
-
-class InputStream;
-class RandomAccessFile;
-
-}  // namespace io
-
 namespace ipc {
 
 class DictionaryMemo;
@@ -54,6 +42,23 @@ struct IpcPayload;
 
 using RecordBatchReader = ::arrow::RecordBatchReader;
 
+struct ReadStats {
+  /// Number of IPC messages read.
+  int64_t num_messages = 0;
+  /// Number of record batches read.
+  int64_t num_record_batches = 0;
+  /// Number of dictionary batches read.
+  ///
+  /// Note: num_dictionary_batches >= num_dictionary_deltas + num_replaced_dictionaries
+  int64_t num_dictionary_batches = 0;
+
+  /// Number of dictionary deltas read.
+  int64_t num_dictionary_deltas = 0;
+  /// Number of replaced dictionaries (i.e. where a dictionary batch replaces
+  /// an existing dictionary with an unrelated new dictionary).
+  int64_t num_replaced_dictionaries = 0;
+};
+
 /// \class RecordBatchStreamReader
 /// \brief Synchronous batch stream reader that reads from io::InputStream
 ///
@@ -68,7 +73,7 @@ class ARROW_EXPORT RecordBatchStreamReader : public RecordBatchReader {
   /// \param[in] message_reader a MessageReader implementation
   /// \param[in] options any IPC reading options (optional)
   /// \return the created batch reader
-  static Result<std::shared_ptr<RecordBatchReader>> Open(
+  static Result<std::shared_ptr<RecordBatchStreamReader>> Open(
       std::unique_ptr<MessageReader> message_reader,
       const IpcReadOptions& options = IpcReadOptions::Defaults());
 
@@ -78,7 +83,7 @@ class ARROW_EXPORT RecordBatchStreamReader : public RecordBatchReader {
   /// lifetime of stream reader
   /// \param[in] options any IPC reading options (optional)
   /// \return the created batch reader
-  static Result<std::shared_ptr<RecordBatchReader>> Open(
+  static Result<std::shared_ptr<RecordBatchStreamReader>> Open(
       io::InputStream* stream,
       const IpcReadOptions& options = IpcReadOptions::Defaults());
 
@@ -86,9 +91,12 @@ class ARROW_EXPORT RecordBatchStreamReader : public RecordBatchReader {
   /// \param[in] stream the input stream
   /// \param[in] options any IPC reading options (optional)
   /// \return the created batch reader
-  static Result<std::shared_ptr<RecordBatchReader>> Open(
+  static Result<std::shared_ptr<RecordBatchStreamReader>> Open(
       const std::shared_ptr<io::InputStream>& stream,
       const IpcReadOptions& options = IpcReadOptions::Defaults());
+
+  /// \brief Return current read statistics
+  virtual ReadStats stats() const = 0;
 };
 
 /// \brief Reads the record batch file format
@@ -159,6 +167,9 @@ class ARROW_EXPORT RecordBatchFileReader {
   /// \param[in] i the index of the record batch to return
   /// \return the read batch
   virtual Result<std::shared_ptr<RecordBatch>> ReadRecordBatch(int i) = 0;
+
+  /// \brief Return current read statistics
+  virtual ReadStats stats() const = 0;
 };
 
 /// \class Listener
@@ -332,7 +343,7 @@ class ARROW_EXPORT StreamDecoder {
   ///   memcpy(buffer->mutable_data() + current_buffer_size,
   ///          small_chunk,
   ///          small_chunk_size);
-  ///   if (buffer->size() < decoder.next_requied_size()) {
+  ///   if (buffer->size() < decoder.next_required_size()) {
   ///     continue;
   ///   }
   ///   std::shared_ptr<arrow::Buffer> chunk(buffer.release());
diff --git a/cpp/src/arrow/ipc/stream_to_file.cc b/cpp/src/arrow/ipc/stream_to_file.cc
index 22518d3e46c..3a2a7fb49fe 100644
--- a/cpp/src/arrow/ipc/stream_to_file.cc
+++ b/cpp/src/arrow/ipc/stream_to_file.cc
@@ -38,7 +38,7 @@ Status ConvertToFile() {
 
   ARROW_ASSIGN_OR_RAISE(auto reader, RecordBatchStreamReader::Open(&input));
   ARROW_ASSIGN_OR_RAISE(
-      auto writer, NewFileWriter(&sink, reader->schema(), IpcWriteOptions::Defaults()));
+      auto writer, MakeFileWriter(&sink, reader->schema(), IpcWriteOptions::Defaults()));
   std::shared_ptr<RecordBatch> batch;
   while (true) {
     ARROW_ASSIGN_OR_RAISE(batch, reader->Next());
diff --git a/cpp/src/arrow/ipc/tensor_test.cc b/cpp/src/arrow/ipc/tensor_test.cc
new file mode 100644
index 00000000000..7af1492f624
--- /dev/null
+++ b/cpp/src/arrow/ipc/tensor_test.cc
@@ -0,0 +1,506 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <algorithm>
+#include <cstdint>
+#include <limits>
+#include <memory>
+#include <string>
+#include <unordered_set>
+
+#include <gtest/gtest.h>
+
+#include "arrow/io/file.h"
+#include "arrow/io/memory.h"
+#include "arrow/io/test_common.h"
+#include "arrow/ipc/reader.h"
+#include "arrow/ipc/test_common.h"
+#include "arrow/ipc/writer.h"
+#include "arrow/sparse_tensor.h"
+#include "arrow/status.h"
+#include "arrow/tensor.h"
+#include "arrow/testing/gtest_util.h"
+#include "arrow/testing/random.h"
+#include "arrow/type.h"
+#include "arrow/util/checked_cast.h"
+#include "arrow/util/io_util.h"
+
+namespace arrow {
+
+using internal::checked_cast;
+using internal::GetByteWidth;
+using internal::TemporaryDir;
+
+namespace ipc {
+namespace test {
+
+class BaseTensorTest : public ::testing::Test, public io::MemoryMapFixture {
+ public:
+  void SetUp() { ASSERT_OK_AND_ASSIGN(temp_dir_, TemporaryDir::Make("ipc-test-")); }
+
+  void TearDown() { io::MemoryMapFixture::TearDown(); }
+
+ protected:
+  std::shared_ptr<io::MemoryMappedFile> mmap_;
+  std::unique_ptr<TemporaryDir> temp_dir_;
+};
+
+class TestTensorRoundTrip : public BaseTensorTest {
+ public:
+  void CheckTensorRoundTrip(const Tensor& tensor) {
+    int32_t metadata_length;
+    int64_t body_length;
+    const int elem_size = GetByteWidth(*tensor.type());
+
+    ASSERT_OK(mmap_->Seek(0));
+
+    ASSERT_OK(WriteTensor(tensor, mmap_.get(), &metadata_length, &body_length));
+
+    const int64_t expected_body_length = elem_size * tensor.size();
+    ASSERT_EQ(expected_body_length, body_length);
+
+    ASSERT_OK(mmap_->Seek(0));
+
+    std::shared_ptr<Tensor> result;
+    ASSERT_OK_AND_ASSIGN(result, ReadTensor(mmap_.get()));
+
+    ASSERT_EQ(result->data()->size(), expected_body_length);
+    ASSERT_TRUE(tensor.Equals(*result));
+  }
+
+ protected:
+  std::shared_ptr<io::MemoryMappedFile> mmap_;
+  std::unique_ptr<TemporaryDir> temp_dir_;
+};
+
+TEST_F(TestTensorRoundTrip, BasicRoundtrip) {
+  std::string path = "test-write-tensor";
+  constexpr int64_t kBufferSize = 1 << 20;
+  ASSERT_OK_AND_ASSIGN(mmap_, io::MemoryMapFixture::InitMemoryMap(kBufferSize, path));
+
+  std::vector<int64_t> shape = {4, 6};
+  std::vector<int64_t> strides = {48, 8};
+  std::vector<std::string> dim_names = {"foo", "bar"};
+  int64_t size = 24;
+
+  std::vector<int64_t> values;
+  randint(size, 0, 100, &values);
+
+  auto data = Buffer::Wrap(values);
+
+  Tensor t0(int64(), data, shape, strides, dim_names);
+  Tensor t_no_dims(int64(), data, {}, {}, {});
+  Tensor t_zero_length_dim(int64(), data, {0}, {8}, {"foo"});
+
+  CheckTensorRoundTrip(t0);
+  CheckTensorRoundTrip(t_no_dims);
+  CheckTensorRoundTrip(t_zero_length_dim);
+
+  int64_t serialized_size;
+  ASSERT_OK(GetTensorSize(t0, &serialized_size));
+  ASSERT_TRUE(serialized_size > static_cast<int64_t>(size * sizeof(int64_t)));
+
+  // ARROW-2840: Check that padding/alignment minded
+  std::vector<int64_t> shape_2 = {1, 1};
+  std::vector<int64_t> strides_2 = {8, 8};
+  Tensor t0_not_multiple_64(int64(), data, shape_2, strides_2, dim_names);
+  CheckTensorRoundTrip(t0_not_multiple_64);
+}
+
+TEST_F(TestTensorRoundTrip, NonContiguous) {
+  std::string path = "test-write-tensor-strided";
+  constexpr int64_t kBufferSize = 1 << 20;
+  ASSERT_OK_AND_ASSIGN(mmap_, io::MemoryMapFixture::InitMemoryMap(kBufferSize, path));
+
+  std::vector<int64_t> values;
+  randint(24, 0, 100, &values);
+
+  auto data = Buffer::Wrap(values);
+  Tensor tensor(int64(), data, {4, 3}, {48, 16});
+
+  CheckTensorRoundTrip(tensor);
+}
+
+template <typename IndexValueType>
+class TestSparseTensorRoundTrip : public BaseTensorTest {
+ public:
+  void CheckSparseCOOTensorRoundTrip(const SparseCOOTensor& sparse_tensor) {
+    const int elem_size = GetByteWidth(*sparse_tensor.type());
+    const int index_elem_size = sizeof(typename IndexValueType::c_type);
+
+    int32_t metadata_length;
+    int64_t body_length;
+
+    ASSERT_OK(mmap_->Seek(0));
+
+    ASSERT_OK(
+        WriteSparseTensor(sparse_tensor, mmap_.get(), &metadata_length, &body_length));
+
+    const auto& sparse_index =
+        checked_cast<const SparseCOOIndex&>(*sparse_tensor.sparse_index());
+    const int64_t indices_length =
+        BitUtil::RoundUpToMultipleOf8(index_elem_size * sparse_index.indices()->size());
+    const int64_t data_length =
+        BitUtil::RoundUpToMultipleOf8(elem_size * sparse_tensor.non_zero_length());
+    const int64_t expected_body_length = indices_length + data_length;
+    ASSERT_EQ(expected_body_length, body_length);
+
+    ASSERT_OK(mmap_->Seek(0));
+
+    std::shared_ptr<SparseTensor> result;
+    ASSERT_OK_AND_ASSIGN(result, ReadSparseTensor(mmap_.get()));
+    ASSERT_EQ(SparseTensorFormat::COO, result->format_id());
+
+    const auto& resulted_sparse_index =
+        checked_cast<const SparseCOOIndex&>(*result->sparse_index());
+    ASSERT_EQ(resulted_sparse_index.indices()->data()->size(), indices_length);
+    ASSERT_EQ(resulted_sparse_index.is_canonical(), sparse_index.is_canonical());
+    ASSERT_EQ(result->data()->size(), data_length);
+    ASSERT_TRUE(result->Equals(sparse_tensor));
+  }
+
+  template <typename SparseIndexType>
+  void CheckSparseCSXMatrixRoundTrip(
+      const SparseTensorImpl<SparseIndexType>& sparse_tensor) {
+    static_assert(std::is_same<SparseIndexType, SparseCSRIndex>::value ||
+                      std::is_same<SparseIndexType, SparseCSCIndex>::value,
+                  "SparseIndexType must be either SparseCSRIndex or SparseCSCIndex");
+
+    const int elem_size = GetByteWidth(*sparse_tensor.type());
+    const int index_elem_size = sizeof(typename IndexValueType::c_type);
+
+    int32_t metadata_length;
+    int64_t body_length;
+
+    ASSERT_OK(mmap_->Seek(0));
+
+    ASSERT_OK(
+        WriteSparseTensor(sparse_tensor, mmap_.get(), &metadata_length, &body_length));
+
+    const auto& sparse_index =
+        checked_cast<const SparseIndexType&>(*sparse_tensor.sparse_index());
+    const int64_t indptr_length =
+        BitUtil::RoundUpToMultipleOf8(index_elem_size * sparse_index.indptr()->size());
+    const int64_t indices_length =
+        BitUtil::RoundUpToMultipleOf8(index_elem_size * sparse_index.indices()->size());
+    const int64_t data_length =
+        BitUtil::RoundUpToMultipleOf8(elem_size * sparse_tensor.non_zero_length());
+    const int64_t expected_body_length = indptr_length + indices_length + data_length;
+    ASSERT_EQ(expected_body_length, body_length);
+
+    ASSERT_OK(mmap_->Seek(0));
+
+    std::shared_ptr<SparseTensor> result;
+    ASSERT_OK_AND_ASSIGN(result, ReadSparseTensor(mmap_.get()));
+
+    constexpr auto expected_format_id =
+        std::is_same<SparseIndexType, SparseCSRIndex>::value ? SparseTensorFormat::CSR
+                                                             : SparseTensorFormat::CSC;
+    ASSERT_EQ(expected_format_id, result->format_id());
+
+    const auto& resulted_sparse_index =
+        checked_cast<const SparseIndexType&>(*result->sparse_index());
+    ASSERT_EQ(resulted_sparse_index.indptr()->data()->size(), indptr_length);
+    ASSERT_EQ(resulted_sparse_index.indices()->data()->size(), indices_length);
+    ASSERT_EQ(result->data()->size(), data_length);
+    ASSERT_TRUE(result->Equals(sparse_tensor));
+  }
+
+  void CheckSparseCSFTensorRoundTrip(const SparseCSFTensor& sparse_tensor) {
+    const int elem_size = GetByteWidth(*sparse_tensor.type());
+    const int index_elem_size = sizeof(typename IndexValueType::c_type);
+
+    int32_t metadata_length;
+    int64_t body_length;
+
+    ASSERT_OK(mmap_->Seek(0));
+
+    ASSERT_OK(
+        WriteSparseTensor(sparse_tensor, mmap_.get(), &metadata_length, &body_length));
+
+    const auto& sparse_index =
+        checked_cast<const SparseCSFIndex&>(*sparse_tensor.sparse_index());
+
+    const int64_t ndim = sparse_index.axis_order().size();
+    int64_t indptr_length = 0;
+    int64_t indices_length = 0;
+
+    for (int64_t i = 0; i < ndim - 1; ++i) {
+      indptr_length += BitUtil::RoundUpToMultipleOf8(index_elem_size *
+                                                     sparse_index.indptr()[i]->size());
+    }
+    for (int64_t i = 0; i < ndim; ++i) {
+      indices_length += BitUtil::RoundUpToMultipleOf8(index_elem_size *
+                                                      sparse_index.indices()[i]->size());
+    }
+    const int64_t data_length =
+        BitUtil::RoundUpToMultipleOf8(elem_size * sparse_tensor.non_zero_length());
+    const int64_t expected_body_length = indptr_length + indices_length + data_length;
+    ASSERT_EQ(expected_body_length, body_length);
+
+    ASSERT_OK(mmap_->Seek(0));
+
+    std::shared_ptr<SparseTensor> result;
+    ASSERT_OK_AND_ASSIGN(result, ReadSparseTensor(mmap_.get()));
+    ASSERT_EQ(SparseTensorFormat::CSF, result->format_id());
+
+    const auto& resulted_sparse_index =
+        checked_cast<const SparseCSFIndex&>(*result->sparse_index());
+
+    int64_t out_indptr_length = 0;
+    int64_t out_indices_length = 0;
+    for (int i = 0; i < ndim - 1; ++i) {
+      out_indptr_length += BitUtil::RoundUpToMultipleOf8(
+          index_elem_size * resulted_sparse_index.indptr()[i]->size());
+    }
+    for (int i = 0; i < ndim; ++i) {
+      out_indices_length += BitUtil::RoundUpToMultipleOf8(
+          index_elem_size * resulted_sparse_index.indices()[i]->size());
+    }
+
+    ASSERT_EQ(out_indptr_length, indptr_length);
+    ASSERT_EQ(out_indices_length, indices_length);
+    ASSERT_EQ(result->data()->size(), data_length);
+    ASSERT_TRUE(resulted_sparse_index.Equals(sparse_index));
+    ASSERT_TRUE(result->Equals(sparse_tensor));
+  }
+
+ protected:
+  std::shared_ptr<SparseCOOIndex> MakeSparseCOOIndex(
+      const std::vector<int64_t>& coords_shape,
+      const std::vector<int64_t>& coords_strides,
+      std::vector<typename IndexValueType::c_type>& coords_values) const {
+    auto coords_data = Buffer::Wrap(coords_values);
+    auto coords = std::make_shared<NumericTensor<IndexValueType>>(
+        coords_data, coords_shape, coords_strides);
+    return std::make_shared<SparseCOOIndex>(coords);
+  }
+
+  template <typename ValueType>
+  Result<std::shared_ptr<SparseCOOTensor>> MakeSparseCOOTensor(
+      const std::shared_ptr<SparseCOOIndex>& si, std::vector<ValueType>& sparse_values,
+      const std::vector<int64_t>& shape,
+      const std::vector<std::string>& dim_names = {}) const {
+    auto data = Buffer::Wrap(sparse_values);
+    return SparseCOOTensor::Make(si, CTypeTraits<ValueType>::type_singleton(), data,
+                                 shape, dim_names);
+  }
+};
+
+TYPED_TEST_SUITE_P(TestSparseTensorRoundTrip);
+
+TYPED_TEST_P(TestSparseTensorRoundTrip, WithSparseCOOIndexRowMajor) {
+  using IndexValueType = TypeParam;
+  using c_index_value_type = typename IndexValueType::c_type;
+
+  std::string path = "test-write-sparse-coo-tensor";
+  constexpr int64_t kBufferSize = 1 << 20;
+  ASSERT_OK_AND_ASSIGN(this->mmap_,
+                       io::MemoryMapFixture::InitMemoryMap(kBufferSize, path));
+
+  // Dense representation:
+  // [
+  //   [
+  //     1 0 2 0
+  //     0 3 0 4
+  //     5 0 6 0
+  //   ],
+  //   [
+  //      0 11  0 12
+  //     13  0 14  0
+  //      0 15  0 16
+  //   ]
+  // ]
+  //
+  // Sparse representation:
+  // idx[0] = [0 0 0 0 0 0  1  1  1  1  1  1]
+  // idx[1] = [0 0 1 1 2 2  0  0  1  1  2  2]
+  // idx[2] = [0 2 1 3 0 2  1  3  0  2  1  3]
+  // data   = [1 2 3 4 5 6 11 12 13 14 15 16]
+
+  // canonical
+  std::vector<c_index_value_type> coords_values = {0, 0, 0, 0, 0, 2, 0, 1, 1, 0, 1, 3,
+                                                   0, 2, 0, 0, 2, 2, 1, 0, 1, 1, 0, 3,
+                                                   1, 1, 0, 1, 1, 2, 1, 2, 1, 1, 2, 3};
+  const int sizeof_index_value = sizeof(c_index_value_type);
+  std::shared_ptr<SparseCOOIndex> si;
+  ASSERT_OK_AND_ASSIGN(
+      si, SparseCOOIndex::Make(TypeTraits<IndexValueType>::type_singleton(), {12, 3},
+                               {sizeof_index_value * 3, sizeof_index_value},
+                               Buffer::Wrap(coords_values)));
+  ASSERT_TRUE(si->is_canonical());
+
+  std::vector<int64_t> shape = {2, 3, 4};
+  std::vector<std::string> dim_names = {"foo", "bar", "baz"};
+  std::vector<int64_t> values = {1, 2, 3, 4, 5, 6, 11, 12, 13, 14, 15, 16};
+  std::shared_ptr<SparseCOOTensor> st;
+  ASSERT_OK_AND_ASSIGN(st, this->MakeSparseCOOTensor(si, values, shape, dim_names));
+
+  this->CheckSparseCOOTensorRoundTrip(*st);
+
+  // non-canonical
+  ASSERT_OK_AND_ASSIGN(
+      si, SparseCOOIndex::Make(TypeTraits<IndexValueType>::type_singleton(), {12, 3},
+                               {sizeof_index_value * 3, sizeof_index_value},
+                               Buffer::Wrap(coords_values), false));
+  ASSERT_FALSE(si->is_canonical());
+  ASSERT_OK_AND_ASSIGN(st, this->MakeSparseCOOTensor(si, values, shape, dim_names));
+
+  this->CheckSparseCOOTensorRoundTrip(*st);
+}
+
+TYPED_TEST_P(TestSparseTensorRoundTrip, WithSparseCOOIndexColumnMajor) {
+  using IndexValueType = TypeParam;
+  using c_index_value_type = typename IndexValueType::c_type;
+
+  std::string path = "test-write-sparse-coo-tensor";
+  constexpr int64_t kBufferSize = 1 << 20;
+  ASSERT_OK_AND_ASSIGN(this->mmap_,
+                       io::MemoryMapFixture::InitMemoryMap(kBufferSize, path));
+
+  // Dense representation:
+  // [
+  //   [
+  //     1 0 2 0
+  //     0 3 0 4
+  //     5 0 6 0
+  //   ],
+  //   [
+  //      0 11  0 12
+  //     13  0 14  0
+  //      0 15  0 16
+  //   ]
+  // ]
+  //
+  // Sparse representation:
+  // idx[0] = [0 0 0 0 0 0  1  1  1  1  1  1]
+  // idx[1] = [0 0 1 1 2 2  0  0  1  1  2  2]
+  // idx[2] = [0 2 1 3 0 2  1  3  0  2  1  3]
+  // data   = [1 2 3 4 5 6 11 12 13 14 15 16]
+
+  // canonical
+  std::vector<c_index_value_type> coords_values = {0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1,
+                                                   0, 0, 1, 1, 2, 2, 0, 0, 1, 1, 2, 2,
+                                                   0, 2, 1, 3, 0, 2, 1, 3, 0, 2, 1, 3};
+  const int sizeof_index_value = sizeof(c_index_value_type);
+  std::shared_ptr<SparseCOOIndex> si;
+  ASSERT_OK_AND_ASSIGN(
+      si, SparseCOOIndex::Make(TypeTraits<IndexValueType>::type_singleton(), {12, 3},
+                               {sizeof_index_value, sizeof_index_value * 12},
+                               Buffer::Wrap(coords_values)));
+  ASSERT_TRUE(si->is_canonical());
+
+  std::vector<int64_t> shape = {2, 3, 4};
+  std::vector<std::string> dim_names = {"foo", "bar", "baz"};
+  std::vector<int64_t> values = {1, 2, 3, 4, 5, 6, 11, 12, 13, 14, 15, 16};
+
+  std::shared_ptr<SparseCOOTensor> st;
+  ASSERT_OK_AND_ASSIGN(st, this->MakeSparseCOOTensor(si, values, shape, dim_names));
+
+  this->CheckSparseCOOTensorRoundTrip(*st);
+
+  // non-canonical
+  ASSERT_OK_AND_ASSIGN(
+      si, SparseCOOIndex::Make(TypeTraits<IndexValueType>::type_singleton(), {12, 3},
+                               {sizeof_index_value, sizeof_index_value * 12},
+                               Buffer::Wrap(coords_values), false));
+  ASSERT_FALSE(si->is_canonical());
+  ASSERT_OK_AND_ASSIGN(st, this->MakeSparseCOOTensor(si, values, shape, dim_names));
+
+  this->CheckSparseCOOTensorRoundTrip(*st);
+}
+
+TYPED_TEST_P(TestSparseTensorRoundTrip, WithSparseCSRIndex) {
+  using IndexValueType = TypeParam;
+
+  std::string path = "test-write-sparse-csr-matrix";
+  constexpr int64_t kBufferSize = 1 << 20;
+  ASSERT_OK_AND_ASSIGN(this->mmap_,
+                       io::MemoryMapFixture::InitMemoryMap(kBufferSize, path));
+
+  std::vector<int64_t> shape = {4, 6};
+  std::vector<std::string> dim_names = {"foo", "bar", "baz"};
+  std::vector<int64_t> values = {1, 0,  2, 0,  0,  3, 0,  4, 5, 0,  6, 0,
+                                 0, 11, 0, 12, 13, 0, 14, 0, 0, 15, 0, 16};
+
+  auto data = Buffer::Wrap(values);
+  NumericTensor<Int64Type> t(data, shape, {}, dim_names);
+  std::shared_ptr<SparseCSRMatrix> st;
+  ASSERT_OK_AND_ASSIGN(
+      st, SparseCSRMatrix::Make(t, TypeTraits<IndexValueType>::type_singleton()));
+
+  this->CheckSparseCSXMatrixRoundTrip(*st);
+}
+
+TYPED_TEST_P(TestSparseTensorRoundTrip, WithSparseCSCIndex) {
+  using IndexValueType = TypeParam;
+
+  std::string path = "test-write-sparse-csc-matrix";
+  constexpr int64_t kBufferSize = 1 << 20;
+  ASSERT_OK_AND_ASSIGN(this->mmap_,
+                       io::MemoryMapFixture::InitMemoryMap(kBufferSize, path));
+
+  std::vector<int64_t> shape = {4, 6};
+  std::vector<std::string> dim_names = {"foo", "bar", "baz"};
+  std::vector<int64_t> values = {1, 0,  2, 0,  0,  3, 0,  4, 5, 0,  6, 0,
+                                 0, 11, 0, 12, 13, 0, 14, 0, 0, 15, 0, 16};
+
+  auto data = Buffer::Wrap(values);
+  NumericTensor<Int64Type> t(data, shape, {}, dim_names);
+  std::shared_ptr<SparseCSCMatrix> st;
+  ASSERT_OK_AND_ASSIGN(
+      st, SparseCSCMatrix::Make(t, TypeTraits<IndexValueType>::type_singleton()));
+
+  this->CheckSparseCSXMatrixRoundTrip(*st);
+}
+
+TYPED_TEST_P(TestSparseTensorRoundTrip, WithSparseCSFIndex) {
+  using IndexValueType = TypeParam;
+
+  std::string path = "test-write-sparse-csf-tensor";
+  constexpr int64_t kBufferSize = 1 << 20;
+  ASSERT_OK_AND_ASSIGN(this->mmap_,
+                       io::MemoryMapFixture::InitMemoryMap(kBufferSize, path));
+
+  std::vector<int64_t> shape = {4, 6};
+  std::vector<std::string> dim_names = {"foo", "bar", "baz"};
+  std::vector<int64_t> values = {1, 0,  2, 0,  0,  3, 0,  4, 5, 0,  6, 0,
+                                 0, 11, 0, 12, 13, 0, 14, 0, 0, 15, 0, 16};
+
+  auto data = Buffer::Wrap(values);
+  NumericTensor<Int64Type> t(data, shape, {}, dim_names);
+  std::shared_ptr<SparseCSFTensor> st;
+  ASSERT_OK_AND_ASSIGN(
+      st, SparseCSFTensor::Make(t, TypeTraits<IndexValueType>::type_singleton()));
+
+  this->CheckSparseCSFTensorRoundTrip(*st);
+}
+REGISTER_TYPED_TEST_SUITE_P(TestSparseTensorRoundTrip, WithSparseCOOIndexRowMajor,
+                            WithSparseCOOIndexColumnMajor, WithSparseCSRIndex,
+                            WithSparseCSCIndex, WithSparseCSFIndex);
+
+INSTANTIATE_TYPED_TEST_SUITE_P(TestInt8, TestSparseTensorRoundTrip, Int8Type);
+INSTANTIATE_TYPED_TEST_SUITE_P(TestUInt8, TestSparseTensorRoundTrip, UInt8Type);
+INSTANTIATE_TYPED_TEST_SUITE_P(TestInt16, TestSparseTensorRoundTrip, Int16Type);
+INSTANTIATE_TYPED_TEST_SUITE_P(TestUInt16, TestSparseTensorRoundTrip, UInt16Type);
+INSTANTIATE_TYPED_TEST_SUITE_P(TestInt32, TestSparseTensorRoundTrip, Int32Type);
+INSTANTIATE_TYPED_TEST_SUITE_P(TestUInt32, TestSparseTensorRoundTrip, UInt32Type);
+INSTANTIATE_TYPED_TEST_SUITE_P(TestInt64, TestSparseTensorRoundTrip, Int64Type);
+
+}  // namespace test
+}  // namespace ipc
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/test_common.cc b/cpp/src/arrow/ipc/test_common.cc
index 9cb7793b194..31b1f655cd4 100644
--- a/cpp/src/arrow/ipc/test_common.cc
+++ b/cpp/src/arrow/ipc/test_common.cc
@@ -23,10 +23,10 @@
 #include <vector>
 
 #include "arrow/array.h"
-#include "arrow/buffer.h"
-#include "arrow/builder.h"
+#include "arrow/array/builder_binary.h"
+#include "arrow/array/builder_primitive.h"
+#include "arrow/array/builder_time.h"
 #include "arrow/ipc/test_common.h"
-#include "arrow/memory_pool.h"
 #include "arrow/pretty_print.h"
 #include "arrow/record_batch.h"
 #include "arrow/status.h"
@@ -35,6 +35,7 @@
 #include "arrow/testing/random.h"
 #include "arrow/testing/util.h"
 #include "arrow/type.h"
+#include "arrow/type_fwd.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit_util.h"
 #include "arrow/util/bitmap_builders.h"
@@ -184,14 +185,14 @@ Status MakeRandomMapArray(const std::shared_ptr<Array>& key_array,
   auto pair_type = struct_(
       {field("key", key_array->type(), false), field("value", item_array->type())});
 
-  auto pair_array = std::make_shared<StructArray>(pair_type, num_maps,
+  auto pair_array = std::make_shared<StructArray>(pair_type, key_array->length(),
                                                   ArrayVector{key_array, item_array});
 
   RETURN_NOT_OK(MakeRandomListArray(pair_array, num_maps, include_nulls, pool, out));
   auto map_data = (*out)->data();
   map_data->type = map(key_array->type(), item_array->type());
   out->reset(new MapArray(map_data));
-  return Status::OK();
+  return (**out).Validate();
 }
 
 Status MakeRandomBooleanArray(const int length, bool include_nulls,
@@ -682,26 +683,107 @@ Status MakeDictionaryFlat(std::shared_ptr<RecordBatch>* out) {
 Status MakeNestedDictionary(std::shared_ptr<RecordBatch>* out) {
   const int64_t length = 7;
 
-  auto inner_dict_values = ArrayFromJSON(utf8(), "[\"foo\", \"bar\", \"baz\"]");
-  ARROW_ASSIGN_OR_RAISE(auto inner_dict,
+  auto values0 = ArrayFromJSON(utf8(), "[\"foo\", \"bar\", \"baz\"]");
+  auto values1 = ArrayFromJSON(int64(), "[1234567890, 987654321]");
+
+  // NOTE: it is important to test several levels of nesting, with non-trivial
+  // numbers of child fields, to exercise structural mapping of fields to dict ids.
+
+  // Field 0: dict(int32, list(dict(int8, utf8)))
+  ARROW_ASSIGN_OR_RAISE(auto inner0,
                         DictionaryArray::FromArrays(
-                            dictionary(int8(), inner_dict_values->type()),
+                            dictionary(int8(), values0->type()),
                             /*indices=*/ArrayFromJSON(int8(), "[0, 1, 2, null, 2, 1, 0]"),
-                            /*dictionary=*/inner_dict_values));
+                            /*dictionary=*/values0));
 
-  ARROW_ASSIGN_OR_RAISE(auto outer_dict_values,
+  ARROW_ASSIGN_OR_RAISE(auto nested_values0,
                         ListArray::FromArrays(
                             /*offsets=*/*ArrayFromJSON(int32(), "[0, 3, 3, 6, 7]"),
-                            /*values=*/*inner_dict));
+                            /*values=*/*inner0));
+  ARROW_ASSIGN_OR_RAISE(
+      auto outer0, DictionaryArray::FromArrays(
+                       dictionary(int32(), nested_values0->type()),
+                       /*indices=*/ArrayFromJSON(int32(), "[0, 1, 3, 3, null, 3, 2]"),
+                       /*dictionary=*/nested_values0));
+  DCHECK_EQ(outer0->length(), length);
+
+  // Field 1: struct(a: dict(int8, int64), b: dict(int16, utf8))
+  ARROW_ASSIGN_OR_RAISE(
+      auto inner1, DictionaryArray::FromArrays(
+                       dictionary(int8(), values1->type()),
+                       /*indices=*/ArrayFromJSON(int8(), "[0, 1, 1, null, null, 1, 0]"),
+                       /*dictionary=*/values1));
+  ARROW_ASSIGN_OR_RAISE(
+      auto inner2, DictionaryArray::FromArrays(
+                       dictionary(int16(), values0->type()),
+                       /*indices=*/ArrayFromJSON(int16(), "[2, 1, null, null, 2, 1, 0]"),
+                       /*dictionary=*/values0));
+  ARROW_ASSIGN_OR_RAISE(
+      auto outer1, StructArray::Make({inner1, inner2}, {field("a", inner1->type()),
+                                                        field("b", inner2->type())}));
+  DCHECK_EQ(outer1->length(), length);
+
+  // Field 2: dict(int8, struct(c: dict(int8, int64), d: dict(int16, list(dict(int8,
+  // utf8)))))
+  ARROW_ASSIGN_OR_RAISE(auto nested_values2,
+                        ListArray::FromArrays(
+                            /*offsets=*/*ArrayFromJSON(int32(), "[0, 1, 5, 5, 7]"),
+                            /*values=*/*inner0));
+  ARROW_ASSIGN_OR_RAISE(
+      auto inner3, DictionaryArray::FromArrays(
+                       dictionary(int16(), nested_values2->type()),
+                       /*indices=*/ArrayFromJSON(int16(), "[0, 1, 3, null, 3, 2, 1]"),
+                       /*dictionary=*/nested_values2));
   ARROW_ASSIGN_OR_RAISE(
-      auto outer_dict, DictionaryArray::FromArrays(
-                           dictionary(int32(), outer_dict_values->type()),
-                           /*indices=*/ArrayFromJSON(int32(), "[0, 1, 3, 3, null, 3, 2]"),
-                           /*dictionary=*/outer_dict_values));
-  DCHECK_EQ(outer_dict->length(), length);
-
-  auto schema = ::arrow::schema({field("f0", outer_dict->type())});
-  *out = RecordBatch::Make(schema, length, {outer_dict});
+      auto inner4, StructArray::Make({inner1, inner3}, {field("c", inner1->type()),
+                                                        field("d", inner3->type())}));
+  ARROW_ASSIGN_OR_RAISE(auto outer2,
+                        DictionaryArray::FromArrays(
+                            dictionary(int8(), inner4->type()),
+                            /*indices=*/ArrayFromJSON(int8(), "[0, 2, 4, 6, 1, 3, 5]"),
+                            /*dictionary=*/inner4));
+  DCHECK_EQ(outer2->length(), length);
+
+  auto schema = ::arrow::schema({
+      field("f0", outer0->type()),
+      field("f1", outer1->type()),
+      field("f2", outer2->type()),
+  });
+  *out = RecordBatch::Make(schema, length, {outer0, outer1, outer2});
+  return Status::OK();
+}
+
+Status MakeMap(std::shared_ptr<RecordBatch>* out) {
+  constexpr int64_t kNumRows = 3;
+  std::shared_ptr<Array> a0, a1;
+
+  auto key_array = ArrayFromJSON(utf8(), R"(["k1", "k2", "k1", "k3", "k1", "k4"])");
+  auto item_array = ArrayFromJSON(int16(), "[0, -1, 2, -3, 4, null]");
+  RETURN_NOT_OK(MakeRandomMapArray(key_array, item_array, kNumRows,
+                                   /*include_nulls=*/false, default_memory_pool(), &a0));
+  RETURN_NOT_OK(MakeRandomMapArray(key_array, item_array, kNumRows,
+                                   /*include_nulls=*/true, default_memory_pool(), &a1));
+  auto f0 = field("f0", a0->type());
+  auto f1 = field("f1", a1->type());
+  *out = RecordBatch::Make(::arrow::schema({f0, f1}), kNumRows, {a0, a1});
+  return Status::OK();
+}
+
+Status MakeMapOfDictionary(std::shared_ptr<RecordBatch>* out) {
+  // Exercises ARROW-9660
+  constexpr int64_t kNumRows = 3;
+  std::shared_ptr<Array> a0, a1;
+
+  auto key_array = DictArrayFromJSON(dictionary(int32(), utf8()), "[0, 1, 0, 2, 0, 3]",
+                                     R"(["k1", "k2", "k3", "k4"])");
+  auto item_array = ArrayFromJSON(int16(), "[0, -1, 2, -3, 4, null]");
+  RETURN_NOT_OK(MakeRandomMapArray(key_array, item_array, kNumRows,
+                                   /*include_nulls=*/false, default_memory_pool(), &a0));
+  RETURN_NOT_OK(MakeRandomMapArray(key_array, item_array, kNumRows,
+                                   /*include_nulls=*/true, default_memory_pool(), &a1));
+  auto f0 = field("f0", a0->type());
+  auto f1 = field("f1", a1->type());
+  *out = RecordBatch::Make(::arrow::schema({f0, f1}), kNumRows, {a0, a1});
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/ipc/test_common.h b/cpp/src/arrow/ipc/test_common.h
index d2e001d865b..021d70258b8 100644
--- a/cpp/src/arrow/ipc/test_common.h
+++ b/cpp/src/arrow/ipc/test_common.h
@@ -53,11 +53,6 @@ Status MakeRandomLargeListArray(const std::shared_ptr<Array>& child_array, int n
                                 bool include_nulls, MemoryPool* pool,
                                 std::shared_ptr<Array>* out);
 
-ARROW_TESTING_EXPORT
-Status MakeRandomMapArray(const std::shared_ptr<Array>& child_array, int num_lists,
-                          bool include_nulls, MemoryPool* pool,
-                          std::shared_ptr<Array>* out);
-
 ARROW_TESTING_EXPORT
 Status MakeRandomBooleanArray(const int length, bool include_nulls,
                               std::shared_ptr<Array>* out);
@@ -133,6 +128,12 @@ Status MakeDictionaryFlat(std::shared_ptr<RecordBatch>* out);
 ARROW_TESTING_EXPORT
 Status MakeNestedDictionary(std::shared_ptr<RecordBatch>* out);
 
+ARROW_TESTING_EXPORT
+Status MakeMap(std::shared_ptr<RecordBatch>* out);
+
+ARROW_TESTING_EXPORT
+Status MakeMapOfDictionary(std::shared_ptr<RecordBatch>* out);
+
 ARROW_TESTING_EXPORT
 Status MakeDates(std::shared_ptr<RecordBatch>* out);
 
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index d3af24d7f9a..624edec0e2f 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -24,6 +24,7 @@
 #include <sstream>
 #include <string>
 #include <type_traits>
+#include <unordered_map>
 #include <utility>
 #include <vector>
 
@@ -66,7 +67,7 @@ namespace ipc {
 using internal::FileBlock;
 using internal::kArrowMagicBytes;
 
-namespace internal {
+namespace {
 
 Status GetTruncatedBitmap(int64_t offset, int64_t length,
                           const std::shared_ptr<Buffer> input, MemoryPool* pool,
@@ -184,17 +185,13 @@ class RecordBatchSerializer {
   }
 
   Status CompressBodyBuffers() {
-    std::unique_ptr<util::Codec> codec;
-
-    RETURN_NOT_OK(internal::CheckCompressionSupported(options_.compression));
-
-    ARROW_ASSIGN_OR_RAISE(
-        codec, util::Codec::Create(options_.compression, options_.compression_level));
+    RETURN_NOT_OK(
+        internal::CheckCompressionSupported(options_.codec->compression_type()));
 
     auto CompressOne = [&](size_t i) {
       if (out_->body_buffers[i]->size() > 0) {
-        RETURN_NOT_OK(
-            CompressBuffer(*out_->body_buffers[i], codec.get(), &out_->body_buffers[i]));
+        RETURN_NOT_OK(CompressBuffer(*out_->body_buffers[i], options_.codec.get(),
+                                     &out_->body_buffers[i]));
       }
       return Status::OK();
     };
@@ -215,7 +212,7 @@ class RecordBatchSerializer {
       RETURN_NOT_OK(VisitArray(*batch.column(i)));
     }
 
-    if (options_.compression != Compression::UNCOMPRESSED) {
+    if (options_.codec != nullptr) {
       RETURN_NOT_OK(CompressBodyBuffers());
     }
 
@@ -226,8 +223,7 @@ class RecordBatchSerializer {
     buffer_meta_.reserve(out_->body_buffers.size());
 
     // Construct the buffer metadata for the record batch header
-    for (size_t i = 0; i < out_->body_buffers.size(); ++i) {
-      const Buffer* buffer = out_->body_buffers[i].get();
+    for (const auto& buffer : out_->body_buffers) {
       int64_t size = 0;
       int64_t padding = 0;
 
@@ -523,8 +519,8 @@ class RecordBatchSerializer {
 
   std::shared_ptr<KeyValueMetadata> custom_metadata_;
 
-  std::vector<FieldMetadata> field_nodes_;
-  std::vector<BufferMetadata> buffer_meta_;
+  std::vector<internal::FieldMetadata> field_nodes_;
+  std::vector<internal::BufferMetadata> buffer_meta_;
 
   const IpcWriteOptions& options_;
   int64_t max_recursion_depth_;
@@ -557,7 +553,7 @@ class DictionarySerializer : public RecordBatchSerializer {
   bool is_delta_;
 };
 
-}  // namespace internal
+}  // namespace
 
 Status WriteIpcPayload(const IpcPayload& payload, const IpcWriteOptions& options,
                        io::OutputStream* dst, int32_t* metadata_length) {
@@ -596,9 +592,9 @@ Status WriteIpcPayload(const IpcPayload& payload, const IpcWriteOptions& options
 }
 
 Status GetSchemaPayload(const Schema& schema, const IpcWriteOptions& options,
-                        DictionaryMemo* dictionary_memo, IpcPayload* out) {
+                        const DictionaryFieldMapper& mapper, IpcPayload* out) {
   out->type = MessageType::SCHEMA;
-  return internal::WriteSchemaMessage(schema, dictionary_memo, options, &out->metadata);
+  return internal::WriteSchemaMessage(schema, mapper, options, &out->metadata);
 }
 
 Status GetDictionaryPayload(int64_t id, const std::shared_ptr<Array>& dictionary,
@@ -611,15 +607,14 @@ Status GetDictionaryPayload(int64_t id, bool is_delta,
                             const IpcWriteOptions& options, IpcPayload* out) {
   out->type = MessageType::DICTIONARY_BATCH;
   // Frame of reference is 0, see ARROW-384
-  internal::DictionarySerializer assembler(id, is_delta, /*buffer_start_offset=*/0,
-                                           options, out);
+  DictionarySerializer assembler(id, is_delta, /*buffer_start_offset=*/0, options, out);
   return assembler.Assemble(dictionary);
 }
 
 Status GetRecordBatchPayload(const RecordBatch& batch, const IpcWriteOptions& options,
                              IpcPayload* out) {
   out->type = MessageType::RECORD_BATCH;
-  internal::RecordBatchSerializer assembler(/*buffer_start_offset=*/0, options, out);
+  RecordBatchSerializer assembler(/*buffer_start_offset=*/0, options, out);
   return assembler.Assemble(batch);
 }
 
@@ -627,7 +622,7 @@ Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
                         io::OutputStream* dst, int32_t* metadata_length,
                         int64_t* body_length, const IpcWriteOptions& options) {
   IpcPayload payload;
-  internal::RecordBatchSerializer assembler(buffer_start_offset, options, &payload);
+  RecordBatchSerializer assembler(buffer_start_offset, options, &payload);
   RETURN_NOT_OK(assembler.Assemble(batch));
 
   // TODO: it's a rough edge that the metadata and body length here are
@@ -642,7 +637,7 @@ Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
 Status WriteRecordBatchStream(const std::vector<std::shared_ptr<RecordBatch>>& batches,
                               const IpcWriteOptions& options, io::OutputStream* dst) {
   ASSIGN_OR_RAISE(std::shared_ptr<RecordBatchWriter> writer,
-                  NewStreamWriter(dst, batches[0]->schema(), options));
+                  MakeStreamWriter(dst, batches[0]->schema(), options));
   for (const auto& batch : batches) {
     DCHECK(batch->schema()->Equals(*batches[0]->schema())) << "Schemas unequal";
     RETURN_NOT_OK(writer->WriteRecordBatch(*batch));
@@ -960,24 +955,21 @@ Status IpcPayloadWriter::Start() { return Status::OK(); }
 
 class ARROW_EXPORT IpcFormatWriter : public RecordBatchWriter {
  public:
-  /// A RecordBatchWriter implementation that writes to a IpcPayloadWriter.
+  // A RecordBatchWriter implementation that writes to a IpcPayloadWriter.
   IpcFormatWriter(std::unique_ptr<internal::IpcPayloadWriter> payload_writer,
                   const Schema& schema, const IpcWriteOptions& options,
-                  DictionaryMemo* out_memo = nullptr)
+                  bool is_file_format)
       : payload_writer_(std::move(payload_writer)),
         schema_(schema),
-        dictionary_memo_(out_memo),
-        options_(options) {
-    if (out_memo == nullptr) {
-      dictionary_memo_ = &internal_dict_memo_;
-    }
-  }
+        mapper_(schema),
+        is_file_format_(is_file_format),
+        options_(options) {}
 
   // A Schema-owning constructor variant
   IpcFormatWriter(std::unique_ptr<internal::IpcPayloadWriter> payload_writer,
                   const std::shared_ptr<Schema>& schema, const IpcWriteOptions& options,
-                  DictionaryMemo* out_memo = nullptr)
-      : IpcFormatWriter(std::move(payload_writer), *schema, options, out_memo) {
+                  bool is_file_format)
+      : IpcFormatWriter(std::move(payload_writer), *schema, options, is_file_format) {
     shared_schema_ = schema;
   }
 
@@ -988,13 +980,7 @@ class ARROW_EXPORT IpcFormatWriter : public RecordBatchWriter {
 
     RETURN_NOT_OK(CheckStarted());
 
-    if (!wrote_dictionaries_) {
-      RETURN_NOT_OK(WriteDictionaries(batch));
-      wrote_dictionaries_ = true;
-    }
-
-    // TODO: Check for delta dictionaries. Can we scan for deltas while computing
-    // the RecordBatch payload to save time?
+    RETURN_NOT_OK(WriteDictionaries(batch));
 
     IpcPayload payload;
     RETURN_NOT_OK(GetRecordBatchPayload(batch, options_, &payload));
@@ -1011,7 +997,7 @@ class ARROW_EXPORT IpcFormatWriter : public RecordBatchWriter {
     RETURN_NOT_OK(payload_writer_->Start());
 
     IpcPayload payload;
-    RETURN_NOT_OK(GetSchemaPayload(schema_, options_, dictionary_memo_, &payload));
+    RETURN_NOT_OK(GetSchemaPayload(schema_, options_, mapper_, &payload));
     return payload_writer_->WritePayload(payload);
   }
 
@@ -1024,15 +1010,43 @@ class ARROW_EXPORT IpcFormatWriter : public RecordBatchWriter {
   }
 
   Status WriteDictionaries(const RecordBatch& batch) {
-    RETURN_NOT_OK(CollectDictionaries(batch, dictionary_memo_));
+    ARROW_ASSIGN_OR_RAISE(const auto dictionaries, CollectDictionaries(batch, mapper_));
 
-    for (const auto& pair : dictionary_memo_->dictionaries()) {
+    for (const auto& pair : dictionaries) {
       IpcPayload payload;
       int64_t dictionary_id = pair.first;
       const auto& dictionary = pair.second;
 
+      // If a dictionary with this id was already emitted, check if it was the same.
+      auto* last_dictionary = &last_dictionaries_[dictionary_id];
+      const bool dictionary_exists = (*last_dictionary != nullptr);
+      if (dictionary_exists) {
+        if ((*last_dictionary)->data() == dictionary->data()) {
+          // Fast shortcut for a common case.
+          // Same dictionary data by pointer => no need to emit it again
+          continue;
+        }
+        if ((*last_dictionary)->Equals(dictionary, EqualOptions().nans_equal(true))) {
+          // Same dictionary by value => no need to emit it again
+          // (while this can have a CPU cost, this code path is required
+          //  for the IPC file format)
+          continue;
+        }
+        // TODO check for possible delta?
+      }
+
+      if (is_file_format_ && dictionary_exists) {
+        return Status::Invalid(
+            "Dictionary replacement detected when writing IPC file format. "
+            "Arrow IPC files only support a single dictionary for a given field "
+            "across all batches.");
+      }
+
       RETURN_NOT_OK(GetDictionaryPayload(dictionary_id, dictionary, options_, &payload));
       RETURN_NOT_OK(payload_writer_->WritePayload(payload));
+
+      // Remember dictionary for next batches
+      *last_dictionary = dictionary;
     }
     return Status::OK();
   }
@@ -1040,17 +1054,29 @@ class ARROW_EXPORT IpcFormatWriter : public RecordBatchWriter {
   std::unique_ptr<IpcPayloadWriter> payload_writer_;
   std::shared_ptr<Schema> shared_schema_;
   const Schema& schema_;
-  DictionaryMemo* dictionary_memo_;
-  DictionaryMemo internal_dict_memo_;
+  const DictionaryFieldMapper mapper_;
+  const bool is_file_format_;
+
+  // A map of last-written dictionaries by id.
+  // This is required to avoid the same dictionary again and again,
+  // and also for correctness when writing the IPC file format
+  // (where replacements and deltas are unsupported).
+  // The latter is also why we can't use weak_ptr.
+  std::unordered_map<int64_t, std::shared_ptr<Array>> last_dictionaries_;
+
   bool started_ = false;
-  bool wrote_dictionaries_ = false;
   IpcWriteOptions options_;
 };
 
 class StreamBookKeeper {
  public:
-  explicit StreamBookKeeper(const IpcWriteOptions& options, io::OutputStream* sink)
+  StreamBookKeeper(const IpcWriteOptions& options, io::OutputStream* sink)
       : options_(options), sink_(sink), position_(-1) {}
+  StreamBookKeeper(const IpcWriteOptions& options, std::shared_ptr<io::OutputStream> sink)
+      : options_(options),
+        sink_(sink.get()),
+        owned_sink_(std::move(sink)),
+        position_(-1) {}
 
   Status UpdatePosition() { return sink_->Tell().Value(&position_); }
 
@@ -1089,6 +1115,7 @@ class StreamBookKeeper {
  protected:
   IpcWriteOptions options_;
   io::OutputStream* sink_;
+  std::shared_ptr<io::OutputStream> owned_sink_;
   int64_t position_;
 };
 
@@ -1099,6 +1126,9 @@ class PayloadStreamWriter : public IpcPayloadWriter, protected StreamBookKeeper
   PayloadStreamWriter(io::OutputStream* sink,
                       const IpcWriteOptions& options = IpcWriteOptions::Defaults())
       : StreamBookKeeper(options, sink) {}
+  PayloadStreamWriter(std::shared_ptr<io::OutputStream> sink,
+                      const IpcWriteOptions& options = IpcWriteOptions::Defaults())
+      : StreamBookKeeper(options, std::move(sink)) {}
 
   ~PayloadStreamWriter() override = default;
 
@@ -1125,6 +1155,12 @@ class PayloadFileWriter : public internal::IpcPayloadWriter, protected StreamBoo
                     const std::shared_ptr<const KeyValueMetadata>& metadata,
                     io::OutputStream* sink)
       : StreamBookKeeper(options, sink), schema_(schema), metadata_(metadata) {}
+  PayloadFileWriter(const IpcWriteOptions& options, const std::shared_ptr<Schema>& schema,
+                    const std::shared_ptr<const KeyValueMetadata>& metadata,
+                    std::shared_ptr<io::OutputStream> sink)
+      : StreamBookKeeper(options, std::move(sink)),
+        schema_(schema),
+        metadata_(metadata) {}
 
   ~PayloadFileWriter() override = default;
 
@@ -1201,22 +1237,54 @@ class PayloadFileWriter : public internal::IpcPayloadWriter, protected StreamBoo
 
 }  // namespace internal
 
-Result<std::shared_ptr<RecordBatchWriter>> NewStreamWriter(
+Result<std::shared_ptr<RecordBatchWriter>> MakeStreamWriter(
     io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
     const IpcWriteOptions& options) {
   return std::make_shared<internal::IpcFormatWriter>(
       ::arrow::internal::make_unique<internal::PayloadStreamWriter>(sink, options),
-      schema, options);
+      schema, options, /*is_file_format=*/false);
 }
 
-Result<std::shared_ptr<RecordBatchWriter>> NewFileWriter(
+Result<std::shared_ptr<RecordBatchWriter>> MakeStreamWriter(
+    std::shared_ptr<io::OutputStream> sink, const std::shared_ptr<Schema>& schema,
+    const IpcWriteOptions& options) {
+  return std::make_shared<internal::IpcFormatWriter>(
+      ::arrow::internal::make_unique<internal::PayloadStreamWriter>(std::move(sink),
+                                                                    options),
+      schema, options, /*is_file_format=*/false);
+}
+
+Result<std::shared_ptr<RecordBatchWriter>> NewStreamWriter(
+    io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
+    const IpcWriteOptions& options) {
+  return MakeStreamWriter(sink, schema, options);
+}
+
+Result<std::shared_ptr<RecordBatchWriter>> MakeFileWriter(
     io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
     const IpcWriteOptions& options,
     const std::shared_ptr<const KeyValueMetadata>& metadata) {
   return std::make_shared<internal::IpcFormatWriter>(
       ::arrow::internal::make_unique<internal::PayloadFileWriter>(options, schema,
                                                                   metadata, sink),
-      schema, options);
+      schema, options, /*is_file_format=*/true);
+}
+
+Result<std::shared_ptr<RecordBatchWriter>> MakeFileWriter(
+    std::shared_ptr<io::OutputStream> sink, const std::shared_ptr<Schema>& schema,
+    const IpcWriteOptions& options,
+    const std::shared_ptr<const KeyValueMetadata>& metadata) {
+  return std::make_shared<internal::IpcFormatWriter>(
+      ::arrow::internal::make_unique<internal::PayloadFileWriter>(
+          options, schema, metadata, std::move(sink)),
+      schema, options, /*is_file_format=*/true);
+}
+
+Result<std::shared_ptr<RecordBatchWriter>> NewFileWriter(
+    io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
+    const IpcWriteOptions& options,
+    const std::shared_ptr<const KeyValueMetadata>& metadata) {
+  return MakeFileWriter(sink, schema, options, metadata);
 }
 
 namespace internal {
@@ -1225,8 +1293,8 @@ Result<std::unique_ptr<RecordBatchWriter>> OpenRecordBatchWriter(
     std::unique_ptr<IpcPayloadWriter> sink, const std::shared_ptr<Schema>& schema,
     const IpcWriteOptions& options) {
   // XXX should we call Start()?
-  return ::arrow::internal::make_unique<internal::IpcFormatWriter>(std::move(sink),
-                                                                   schema, options);
+  return ::arrow::internal::make_unique<internal::IpcFormatWriter>(
+      std::move(sink), schema, options, /*is_file_format=*/false);
 }
 
 Result<std::unique_ptr<IpcPayloadWriter>> MakePayloadStreamWriter(
@@ -1284,16 +1352,14 @@ Status SerializeRecordBatch(const RecordBatch& batch, const IpcWriteOptions& opt
   return WriteRecordBatch(batch, 0, out, &metadata_length, &body_length, options);
 }
 
-Result<std::shared_ptr<Buffer>> SerializeSchema(const Schema& schema,
-                                                DictionaryMemo* dictionary_memo,
-                                                MemoryPool* pool) {
+Result<std::shared_ptr<Buffer>> SerializeSchema(const Schema& schema, MemoryPool* pool) {
   ARROW_ASSIGN_OR_RAISE(auto stream, io::BufferOutputStream::Create(1024, pool));
 
   auto options = IpcWriteOptions::Defaults();
+  const bool is_file_format = false;  // indifferent as we don't write dictionaries
   internal::IpcFormatWriter writer(
       ::arrow::internal::make_unique<internal::PayloadStreamWriter>(stream.get()), schema,
-      options, dictionary_memo);
-  // Write schema and populate fields (but not dictionaries) in dictionary_memo
+      options, is_file_format);
   RETURN_NOT_OK(writer.Start());
   return stream->Finish();
 }
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index 6a89202e68c..48f3a9a3b58 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -97,6 +97,24 @@ class ARROW_EXPORT RecordBatchWriter {
 /// \param[in] options options for serialization
 /// \return Result<std::shared_ptr<RecordBatchWriter>>
 ARROW_EXPORT
+Result<std::shared_ptr<RecordBatchWriter>> MakeStreamWriter(
+    io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
+    const IpcWriteOptions& options = IpcWriteOptions::Defaults());
+
+/// Create a new IPC stream writer from stream sink and schema. User is
+/// responsible for closing the actual OutputStream.
+///
+/// \param[in] sink output stream to write to
+/// \param[in] schema the schema of the record batches to be written
+/// \param[in] options options for serialization
+/// \return Result<std::shared_ptr<RecordBatchWriter>>
+ARROW_EXPORT
+Result<std::shared_ptr<RecordBatchWriter>> MakeStreamWriter(
+    std::shared_ptr<io::OutputStream> sink, const std::shared_ptr<Schema>& schema,
+    const IpcWriteOptions& options = IpcWriteOptions::Defaults());
+
+ARROW_DEPRECATED("Use MakeStreamWriter")
+ARROW_EXPORT
 Result<std::shared_ptr<RecordBatchWriter>> NewStreamWriter(
     io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
     const IpcWriteOptions& options = IpcWriteOptions::Defaults());
@@ -109,6 +127,26 @@ Result<std::shared_ptr<RecordBatchWriter>> NewStreamWriter(
 /// \param[in] metadata custom metadata for File Footer, optional
 /// \return Result<std::shared_ptr<RecordBatchWriter>>
 ARROW_EXPORT
+Result<std::shared_ptr<RecordBatchWriter>> MakeFileWriter(
+    io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
+    const IpcWriteOptions& options = IpcWriteOptions::Defaults(),
+    const std::shared_ptr<const KeyValueMetadata>& metadata = NULLPTR);
+
+/// Create a new IPC file writer from stream sink and schema
+///
+/// \param[in] sink output stream to write to
+/// \param[in] schema the schema of the record batches to be written
+/// \param[in] options options for serialization, optional
+/// \param[in] metadata custom metadata for File Footer, optional
+/// \return Result<std::shared_ptr<RecordBatchWriter>>
+ARROW_EXPORT
+Result<std::shared_ptr<RecordBatchWriter>> MakeFileWriter(
+    std::shared_ptr<io::OutputStream> sink, const std::shared_ptr<Schema>& schema,
+    const IpcWriteOptions& options = IpcWriteOptions::Defaults(),
+    const std::shared_ptr<const KeyValueMetadata>& metadata = NULLPTR);
+
+ARROW_DEPRECATED("Use MakeFileWriter")
+ARROW_EXPORT
 Result<std::shared_ptr<RecordBatchWriter>> NewFileWriter(
     io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
     const IpcWriteOptions& options = IpcWriteOptions::Defaults(),
@@ -166,12 +204,10 @@ Status SerializeRecordBatch(const RecordBatch& batch, const IpcWriteOptions& opt
 /// \brief Serialize schema as encapsulated IPC message
 ///
 /// \param[in] schema the schema to write
-/// \param[in] dictionary_memo a DictionaryMemo for recording dictionary ids
 /// \param[in] pool a MemoryPool to allocate memory from
 /// \return the serialized schema
 ARROW_EXPORT
 Result<std::shared_ptr<Buffer>> SerializeSchema(const Schema& schema,
-                                                DictionaryMemo* dictionary_memo,
                                                 MemoryPool* pool = default_memory_pool());
 
 /// \brief Write multiple record batches to OutputStream, including schema
@@ -280,12 +316,12 @@ Status WriteSparseTensor(const SparseTensor& sparse_tensor, io::OutputStream* ds
 /// \brief Compute IpcPayload for the given schema
 /// \param[in] schema the Schema that is being serialized
 /// \param[in] options options for serialization
-/// \param[in,out] dictionary_memo class to populate with assigned dictionary ids
+/// \param[in] mapper object mapping dictionary fields to dictionary ids
 /// \param[out] out the returned vector of IpcPayloads
 /// \return Status
 ARROW_EXPORT
 Status GetSchemaPayload(const Schema& schema, const IpcWriteOptions& options,
-                        DictionaryMemo* dictionary_memo, IpcPayload* out);
+                        const DictionaryFieldMapper& mapper, IpcPayload* out);
 
 /// \brief Compute IpcPayload for a dictionary
 /// \param[in] id the dictionary id
@@ -378,6 +414,9 @@ Result<std::unique_ptr<IpcPayloadWriter>> MakePayloadFileWriter(
 
 /// Create a new RecordBatchWriter from IpcPayloadWriter and schema.
 ///
+/// The format is implicitly the IPC stream format (allowing dictionary
+/// replacement and deltas).
+///
 /// \param[in] sink the IpcPayloadWriter to write to
 /// \param[in] schema the schema of the record batches to be written
 /// \param[in] options options for serialization
diff --git a/cpp/src/arrow/json/chunked_builder.cc b/cpp/src/arrow/json/chunked_builder.cc
index 7789a5d038a..01385d2b8e1 100644
--- a/cpp/src/arrow/json/chunked_builder.cc
+++ b/cpp/src/arrow/json/chunked_builder.cc
@@ -246,9 +246,7 @@ class ChunkedListArrayBuilder : public ChunkedArrayBuilder {
     value_builder_->Insert(block_index, value_field_, std::make_shared<NullArray>(0));
 
     ARROW_ASSIGN_OR_RAISE(null_bitmap_chunks_[block_index],
-                          AllocateBitmap(length, pool_));
-    std::memset(null_bitmap_chunks_[block_index]->mutable_data(), 0,
-                null_bitmap_chunks_[block_index]->size());
+                          AllocateEmptyBitmap(length, pool_));
 
     int64_t offsets_length = (length + 1) * sizeof(int32_t);
     ARROW_ASSIGN_OR_RAISE(offset_chunks_[block_index],
diff --git a/cpp/src/arrow/json/converter.cc b/cpp/src/arrow/json/converter.cc
index dab66952ab7..f71169a4422 100644
--- a/cpp/src/arrow/json/converter.cc
+++ b/cpp/src/arrow/json/converter.cc
@@ -21,7 +21,9 @@
 #include <utility>
 
 #include "arrow/array.h"
-#include "arrow/builder.h"
+#include "arrow/array/builder_binary.h"
+#include "arrow/array/builder_primitive.h"
+#include "arrow/array/builder_time.h"
 #include "arrow/json/parser.h"
 #include "arrow/type.h"
 #include "arrow/util/checked_cast.h"
diff --git a/cpp/src/arrow/json/parser.cc b/cpp/src/arrow/json/parser.cc
index 0ff04f0e946..9d921c57994 100644
--- a/cpp/src/arrow/json/parser.cc
+++ b/cpp/src/arrow/json/parser.cc
@@ -29,9 +29,8 @@
 #include "rapidjson/reader.h"
 
 #include "arrow/array.h"
+#include "arrow/array/builder_binary.h"
 #include "arrow/buffer_builder.h"
-#include "arrow/builder.h"
-#include "arrow/memory_pool.h"
 #include "arrow/type.h"
 #include "arrow/util/bitset_stack.h"
 #include "arrow/util/logging.h"
@@ -372,7 +371,7 @@ class RawArrayBuilder<Kind::kObject> {
       std::shared_ptr<Array> field_values;
       RETURN_NOT_OK(finish_child(field_builders_[i], &field_values));
       child_data[i] = field_values->data();
-      fields[i] = field(field_names[i].to_string(), field_values->type(),
+      fields[i] = field(std::string(field_names[i]), field_values->type(),
                         field_builders_[i].nullable, Kind::Tag(field_builders_[i].kind));
     }
 
diff --git a/cpp/src/arrow/json/parser_test.cc b/cpp/src/arrow/json/parser_test.cc
index 49332990b2e..e22648e84b3 100644
--- a/cpp/src/arrow/json/parser_test.cc
+++ b/cpp/src/arrow/json/parser_test.cc
@@ -15,15 +15,16 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <string>
-#include <utility>
-#include <vector>
+#include "arrow/json/parser.h"
 
 #include <gmock/gmock-matchers.h>
 #include <gtest/gtest.h>
 
+#include <string>
+#include <utility>
+#include <vector>
+
 #include "arrow/json/options.h"
-#include "arrow/json/parser.h"
 #include "arrow/json/test_common.h"
 #include "arrow/status.h"
 #include "arrow/testing/gtest_util.h"
@@ -188,7 +189,7 @@ TEST(BlockParserWithSchema, Nested) {
                       field("nuf", struct_({field("ps", utf8())}))},
                      {"[\"thing\", null, \"\xe5\xbf\x8d\", null]",
                       R"([["1", "2", "3"], ["2"], [], null])",
-                      R"([{"ps":null}, null, {"ps":"78"}, {"ps":"90"}])"});
+                      R"([{"ps":null}, {}, {"ps":"78"}, {"ps":"90"}])"});
 }
 
 TEST(BlockParserWithSchema, FailOnIncompleteJson) {
@@ -217,7 +218,7 @@ TEST(BlockParser, Nested) {
                       field("nuf", struct_({field("ps", utf8())}))},
                      {"[\"thing\", null, \"\xe5\xbf\x8d\", null]",
                       R"([["1", "2", "3"], ["2"], [], null])",
-                      R"([{"ps":null}, null, {"ps":"78"}, {"ps":"90"}])"});
+                      R"([{"ps":null}, {}, {"ps":"78"}, {"ps":"90"}])"});
 }
 
 TEST(BlockParser, AdHoc) {
diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index 787c01b153e..c0ff19c4747 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -264,10 +264,6 @@ class MimallocAllocator {
 
 }  // namespace
 
-MemoryPool::MemoryPool() {}
-
-MemoryPool::~MemoryPool() {}
-
 int64_t MemoryPool::max_memory() const { return -1; }
 
 ///////////////////////////////////////////////////////////////////////
diff --git a/cpp/src/arrow/memory_pool.h b/cpp/src/arrow/memory_pool.h
index b875e54b440..71a39b9cb25 100644
--- a/cpp/src/arrow/memory_pool.h
+++ b/cpp/src/arrow/memory_pool.h
@@ -63,7 +63,7 @@ class MemoryPoolStats {
 /// take care of the required 64-byte alignment.
 class ARROW_EXPORT MemoryPool {
  public:
-  virtual ~MemoryPool();
+  virtual ~MemoryPool() = default;
 
   /// \brief EXPERIMENTAL. Create a new instance of the default MemoryPool
   static std::unique_ptr<MemoryPool> CreateDefault();
@@ -101,7 +101,7 @@ class ARROW_EXPORT MemoryPool {
   virtual std::string backend_name() const = 0;
 
  protected:
-  MemoryPool();
+  MemoryPool() = default;
 };
 
 class ARROW_EXPORT LoggingMemoryPool : public MemoryPool {
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index 4b3fd8682c2..8c2ac376d1e 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include "arrow/pretty_print.h"
+
 #include <algorithm>
 #include <chrono>
 #include <cstddef>
@@ -28,7 +30,6 @@
 
 #include "arrow/array.h"
 #include "arrow/chunked_array.h"
-#include "arrow/pretty_print.h"
 #include "arrow/record_batch.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
@@ -226,6 +227,11 @@ class ArrayPrinter : public PrettyPrinter {
     return Status::OK();
   }
 
+  Status WriteDataValues(const Decimal256Array& array) {
+    WriteValues(array, [&](int64_t i) { (*sink_) << array.FormatValue(i); });
+    return Status::OK();
+  }
+
   template <typename T>
   enable_if_list_like<typename T::TypeClass, Status> WriteDataValues(const T& array) {
     bool skip_comma = true;
@@ -327,7 +333,6 @@ class ArrayPrinter : public PrettyPrinter {
       if (offset != 0) {
         field = field->Slice(offset, length);
       }
-
       RETURN_NOT_OK(PrettyPrint(*field, indent_ + options_.indent_size, sink_));
     }
     return Status::OK();
@@ -552,7 +557,7 @@ Status PrettyPrint(const Table& table, const PrettyPrintOptions& options,
 }
 
 Status DebugPrint(const Array& arr, int indent) {
-  return PrettyPrint(arr, indent, &std::cout);
+  return PrettyPrint(arr, indent, &std::cerr);
 }
 
 class SchemaPrinter : public PrettyPrinter {
diff --git a/cpp/src/arrow/pretty_print_test.cc b/cpp/src/arrow/pretty_print_test.cc
index 6124b8f2ddc..538e7365185 100644
--- a/cpp/src/arrow/pretty_print_test.cc
+++ b/cpp/src/arrow/pretty_print_test.cc
@@ -15,6 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include "arrow/pretty_print.h"
+
+#include <gtest/gtest.h>
+
 #include <cstdint>
 #include <cstring>
 #include <memory>
@@ -22,11 +26,7 @@
 #include <string>
 #include <vector>
 
-#include <gtest/gtest.h>
-
 #include "arrow/array.h"
-#include "arrow/builder.h"
-#include "arrow/pretty_print.h"
 #include "arrow/table.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/type.h"
@@ -324,13 +324,13 @@ TEST_F(TestPrettyPrint, StructTypeAdvanced) {
 -- child 0 type: int32
   [
     11,
-    null,
+    0,
     null
   ]
 -- child 1 type: int32
   [
     22,
-    null,
+    0,
     33
   ])expected";
   CheckStream(*array, {0, 10}, ex);
@@ -498,15 +498,16 @@ TEST_F(TestPrettyPrint, FixedSizeBinaryType) {
   CheckArray(*array, {2, 1}, ex_2);
 }
 
-TEST_F(TestPrettyPrint, Decimal128Type) {
+TEST_F(TestPrettyPrint, DecimalTypes) {
   int32_t p = 19;
   int32_t s = 4;
 
-  auto type = decimal(p, s);
-  auto array = ArrayFromJSON(type, "[\"123.4567\", \"456.7891\", null]");
+  for (auto type : {decimal128(p, s), decimal256(p, s)}) {
+    auto array = ArrayFromJSON(type, "[\"123.4567\", \"456.7891\", null]");
 
-  static const char* ex = "[\n  123.4567,\n  456.7891,\n  null\n]";
-  CheckArray(*array, {0}, ex);
+    static const char* ex = "[\n  123.4567,\n  456.7891,\n  null\n]";
+    CheckArray(*array, {0}, ex);
+  }
 }
 
 TEST_F(TestPrettyPrint, DictionaryType) {
diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
index b972b0d8606..960155703e1 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -38,6 +38,7 @@ set(ARROW_PYTHON_SRCS
     inference.cc
     init.cc
     io.cc
+    ipc.cc
     numpy_convert.cc
     numpy_to_arrow.cc
     python_to_arrow.cc
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index bc4e25b08df..09245285030 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -17,9 +17,8 @@
 
 // Functions for pandas conversion via NumPy
 
-#include "arrow/python/numpy_interop.h"  // IWYU pragma: expand
-
 #include "arrow/python/arrow_to_pandas.h"
+#include "arrow/python/numpy_interop.h"  // IWYU pragma: expand
 
 #include <cmath>
 #include <cstdint>
@@ -68,6 +67,30 @@ using internal::CheckIndexBounds;
 using internal::GetByteWidth;
 using internal::OptionalParallelFor;
 
+namespace py {
+namespace {
+
+// Fix options for conversion of an inner (child) array.
+PandasOptions MakeInnerOptions(PandasOptions options) {
+  // Make sure conversion of inner dictionary arrays always returns an array,
+  // not a dict {'indices': array, 'dictionary': array, 'ordered': bool}
+  options.decode_dictionaries = true;
+  options.categorical_columns.clear();
+  options.strings_to_categorical = false;
+
+  // In ARROW-7723, we found as a result of ARROW-3789 that second
+  // through microsecond resolution tz-aware timestamps were being promoted to
+  // use the DATETIME_NANO_TZ conversion path, yielding a datetime64[ns] NumPy
+  // array in this function. PyArray_GETITEM returns datetime.datetime for
+  // units second through microsecond but PyLong for nanosecond (because
+  // datetime.datetime does not support nanoseconds).
+  // We force the object conversion to preserve the value of the timezone.
+  // Nanoseconds are returned as integers.
+  options.coerce_temporal_nanoseconds = false;
+
+  return options;
+}
+
 // ----------------------------------------------------------------------
 // PyCapsule code for setting ndarray base to reference C++ object
 
@@ -79,8 +102,6 @@ struct BufferCapsule {
   std::shared_ptr<Buffer> buffer;
 };
 
-namespace {
-
 void ArrayCapsule_Destructor(PyObject* capsule) {
   delete reinterpret_cast<ArrayCapsule*>(PyCapsule_GetPointer(capsule, "arrow::Array"));
 }
@@ -89,13 +110,9 @@ void BufferCapsule_Destructor(PyObject* capsule) {
   delete reinterpret_cast<BufferCapsule*>(PyCapsule_GetPointer(capsule, "arrow::Buffer"));
 }
 
-}  // namespace
-
 // ----------------------------------------------------------------------
 // pandas 0.x DataFrame conversion internals
 
-namespace py {
-
 using internal::arrow_traits;
 using internal::npy_traits;
 
@@ -150,7 +167,8 @@ static inline bool ListTypeSupported(const DataType& type) {
     case Type::UINT64:
     case Type::FLOAT:
     case Type::DOUBLE:
-    case Type::DECIMAL:
+    case Type::DECIMAL128:
+    case Type::DECIMAL256:
     case Type::BINARY:
     case Type::LARGE_BINARY:
     case Type::STRING:
@@ -629,8 +647,8 @@ inline Status ConvertAsPyObjects(const PandasOptions& options, const ChunkedArra
   return Status::OK();
 }
 
-inline Status ConvertStruct(const PandasOptions& options, const ChunkedArray& data,
-                            PyObject** out_values) {
+Status ConvertStruct(PandasOptions options, const ChunkedArray& data,
+                     PyObject** out_values) {
   if (data.num_chunks() == 0) {
     return Status::OK();
   }
@@ -642,25 +660,18 @@ inline Status ConvertStruct(const PandasOptions& options, const ChunkedArray& da
   std::vector<OwnedRef> fields_data(num_fields);
   OwnedRef dict_item;
 
-  // XXX(wesm): In ARROW-7723, we found as a result of ARROW-3789 that second
-  // through microsecond resolution tz-aware timestamps were being promoted to
-  // use the DATETIME_NANO_TZ conversion path, yielding a datetime64[ns] NumPy
-  // array in this function. PyArray_GETITEM returns datetime.datetime for
-  // units second through microsecond but PyLong for nanosecond (because
-  // datetime.datetime does not support nanoseconds). We inserted this hack to
-  // preserve the <= 0.15.1 behavior until a better solution can be devised
-  PandasOptions modified_options = options;
-  modified_options.ignore_timezone = true;
-  modified_options.coerce_temporal_nanoseconds = false;
+  // See notes in MakeInnerOptions.
+  options = MakeInnerOptions(std::move(options));
+  // Don't blindly convert because timestamps in lists are handled differently.
+  options.timestamp_as_object = true;
 
   for (int c = 0; c < data.num_chunks(); c++) {
     auto arr = checked_cast<const StructArray*>(data.chunk(c).get());
     // Convert the struct arrays first
     for (int32_t i = 0; i < num_fields; i++) {
-      PyObject* numpy_array;
-      RETURN_NOT_OK(ConvertArrayToPandas(
-          modified_options, arr->field(static_cast<int>(i)), nullptr, &numpy_array));
-      fields_data[i].reset(numpy_array);
+      const auto field = arr->field(static_cast<int>(i));
+      RETURN_NOT_OK(ConvertArrayToPandas(options, field, nullptr, fields_data[i].ref()));
+      DCHECK(PyArray_Check(fields_data[i].obj()));
     }
 
     // Construct a dictionary for each row
@@ -704,7 +715,7 @@ inline Status ConvertStruct(const PandasOptions& options, const ChunkedArray& da
 }
 
 Status DecodeDictionaries(MemoryPool* pool, const std::shared_ptr<DataType>& dense_type,
-                          std::vector<std::shared_ptr<Array>>* arrays) {
+                          ArrayVector* arrays) {
   compute::ExecContext ctx(pool);
   compute::CastOptions options;
   for (size_t i = 0; i < arrays->size(); ++i) {
@@ -714,11 +725,19 @@ Status DecodeDictionaries(MemoryPool* pool, const std::shared_ptr<DataType>& den
   return Status::OK();
 }
 
+Status DecodeDictionaries(MemoryPool* pool, const std::shared_ptr<DataType>& dense_type,
+                          std::shared_ptr<ChunkedArray>* array) {
+  auto chunks = (*array)->chunks();
+  RETURN_NOT_OK(DecodeDictionaries(pool, dense_type, &chunks));
+  *array = std::make_shared<ChunkedArray>(std::move(chunks), dense_type);
+  return Status::OK();
+}
+
 template <typename ListArrayT>
-Status ConvertListsLike(const PandasOptions& options, const ChunkedArray& data,
+Status ConvertListsLike(PandasOptions options, const ChunkedArray& data,
                         PyObject** out_values) {
   // Get column of underlying value arrays
-  std::vector<std::shared_ptr<Array>> value_arrays;
+  ArrayVector value_arrays;
   for (int c = 0; c < data.num_chunks(); c++) {
     const auto& arr = checked_cast<const ListArrayT&>(*data.chunk(c));
     value_arrays.emplace_back(arr.values());
@@ -727,44 +746,29 @@ Status ConvertListsLike(const PandasOptions& options, const ChunkedArray& data,
   const auto& list_type = checked_cast<const ListArrayType&>(*data.type());
   auto value_type = list_type.value_type();
 
-  if (value_type->id() == Type::DICTIONARY) {
-    // ARROW-6899: Convert dictionary-encoded children to dense instead of
-    // failing below. A more efficient conversion than this could be done later
-    auto dense_type = checked_cast<const DictionaryType&>(*value_type).value_type();
-    RETURN_NOT_OK(DecodeDictionaries(options.pool, dense_type, &value_arrays));
-    value_type = dense_type;
-  }
-
   auto flat_column = std::make_shared<ChunkedArray>(value_arrays, value_type);
-  // TODO(ARROW-489): Currently we don't have a Python reference for single columns.
-  //    Storing a reference to the whole Array would be too expensive.
-
-  // ARROW-3789(wesm): During refactoring I found that unit tests assumed that
-  // timestamp units would be preserved on list<timestamp UNIT> conversions in
-  // Table.to_pandas. So we set the option here to not coerce things to
-  // nanoseconds. Bit of a hack but this seemed the simplest thing to satisfy
-  // the existing unit tests
-  PandasOptions modified_options = options;
-  modified_options.coerce_temporal_nanoseconds = false;
+
+  options = MakeInnerOptions(std::move(options));
 
   OwnedRefNoGIL owned_numpy_array;
-  RETURN_NOT_OK(ConvertChunkedArrayToPandas(modified_options, flat_column, nullptr,
+  RETURN_NOT_OK(ConvertChunkedArrayToPandas(options, flat_column, nullptr,
                                             owned_numpy_array.ref()));
 
   PyObject* numpy_array = owned_numpy_array.obj();
+  DCHECK(PyArray_Check(numpy_array));
 
   int64_t chunk_offset = 0;
   for (int c = 0; c < data.num_chunks(); c++) {
-    auto arr = std::static_pointer_cast<ListArrayT>(data.chunk(c));
+    const auto& arr = checked_cast<const ListArrayT&>(*data.chunk(c));
 
     const bool has_nulls = data.null_count() > 0;
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      if (has_nulls && arr->IsNull(i)) {
+    for (int64_t i = 0; i < arr.length(); ++i) {
+      if (has_nulls && arr.IsNull(i)) {
         Py_INCREF(Py_None);
         *out_values = Py_None;
       } else {
-        OwnedRef start(PyLong_FromLongLong(arr->value_offset(i) + chunk_offset));
-        OwnedRef end(PyLong_FromLongLong(arr->value_offset(i + 1) + chunk_offset));
+        OwnedRef start(PyLong_FromLongLong(arr.value_offset(i) + chunk_offset));
+        OwnedRef end(PyLong_FromLongLong(arr.value_offset(i + 1) + chunk_offset));
         OwnedRef slice(PySlice_New(start.obj(), end.obj(), nullptr));
 
         if (ARROW_PREDICT_FALSE(slice.obj() == nullptr)) {
@@ -782,7 +786,111 @@ Status ConvertListsLike(const PandasOptions& options, const ChunkedArray& data,
     }
     RETURN_IF_PYERROR();
 
-    chunk_offset += arr->values()->length();
+    chunk_offset += arr.values()->length();
+  }
+
+  return Status::OK();
+}
+
+Status ConvertMap(PandasOptions options, const ChunkedArray& data,
+                  PyObject** out_values) {
+  // Get columns of underlying key/item arrays
+  std::vector<std::shared_ptr<Array>> key_arrays;
+  std::vector<std::shared_ptr<Array>> item_arrays;
+  for (int c = 0; c < data.num_chunks(); ++c) {
+    const auto& map_arr = checked_cast<const MapArray&>(*data.chunk(c));
+    key_arrays.emplace_back(map_arr.keys());
+    item_arrays.emplace_back(map_arr.items());
+  }
+
+  const auto& map_type = checked_cast<const MapType&>(*data.type());
+  auto key_type = map_type.key_type();
+  auto item_type = map_type.item_type();
+
+  // ARROW-6899: Convert dictionary-encoded children to dense instead of
+  // failing below. A more efficient conversion than this could be done later
+  if (key_type->id() == Type::DICTIONARY) {
+    auto dense_type = checked_cast<const DictionaryType&>(*key_type).value_type();
+    RETURN_NOT_OK(DecodeDictionaries(options.pool, dense_type, &key_arrays));
+    key_type = dense_type;
+  }
+  if (item_type->id() == Type::DICTIONARY) {
+    auto dense_type = checked_cast<const DictionaryType&>(*item_type).value_type();
+    RETURN_NOT_OK(DecodeDictionaries(options.pool, dense_type, &item_arrays));
+    item_type = dense_type;
+  }
+
+  // See notes in MakeInnerOptions.
+  options = MakeInnerOptions(std::move(options));
+  // Don't blindly convert because timestamps in lists are handled differently.
+  options.timestamp_as_object = true;
+
+  auto flat_keys = std::make_shared<ChunkedArray>(key_arrays, key_type);
+  auto flat_items = std::make_shared<ChunkedArray>(item_arrays, item_type);
+  OwnedRef list_item;
+  OwnedRef key_value;
+  OwnedRef item_value;
+  OwnedRefNoGIL owned_numpy_keys;
+  RETURN_NOT_OK(
+      ConvertChunkedArrayToPandas(options, flat_keys, nullptr, owned_numpy_keys.ref()));
+  OwnedRefNoGIL owned_numpy_items;
+  RETURN_NOT_OK(
+      ConvertChunkedArrayToPandas(options, flat_items, nullptr, owned_numpy_items.ref()));
+  PyArrayObject* py_keys = reinterpret_cast<PyArrayObject*>(owned_numpy_keys.obj());
+  PyArrayObject* py_items = reinterpret_cast<PyArrayObject*>(owned_numpy_items.obj());
+
+  int64_t chunk_offset = 0;
+  for (int c = 0; c < data.num_chunks(); ++c) {
+    const auto& arr = checked_cast<const MapArray&>(*data.chunk(c));
+    const bool has_nulls = data.null_count() > 0;
+
+    // Make a list of key/item pairs for each row in array
+    for (int64_t i = 0; i < arr.length(); ++i) {
+      if (has_nulls && arr.IsNull(i)) {
+        Py_INCREF(Py_None);
+        *out_values = Py_None;
+      } else {
+        int64_t entry_offset = arr.value_offset(i);
+        int64_t num_maps = arr.value_offset(i + 1) - entry_offset;
+
+        // Build the new list object for the row of maps
+        list_item.reset(PyList_New(num_maps));
+        RETURN_IF_PYERROR();
+
+        // Add each key/item pair in the row
+        for (int64_t j = 0; j < num_maps; ++j) {
+          // Get key value, key is non-nullable for a valid row
+          auto ptr_key = reinterpret_cast<const char*>(
+              PyArray_GETPTR1(py_keys, chunk_offset + entry_offset + j));
+          key_value.reset(PyArray_GETITEM(py_keys, ptr_key));
+          RETURN_IF_PYERROR();
+
+          if (item_arrays[c]->IsNull(entry_offset + j)) {
+            // Translate the Null to a None
+            Py_INCREF(Py_None);
+            item_value.reset(Py_None);
+          } else {
+            // Get valid value from item array
+            auto ptr_item = reinterpret_cast<const char*>(
+                PyArray_GETPTR1(py_items, chunk_offset + entry_offset + j));
+            item_value.reset(PyArray_GETITEM(py_items, ptr_item));
+            RETURN_IF_PYERROR();
+          }
+
+          // Add the key/item pair to the list for the row
+          PyList_SET_ITEM(list_item.obj(), j,
+                          PyTuple_Pack(2, key_value.obj(), item_value.obj()));
+          RETURN_IF_PYERROR();
+        }
+
+        // Pass ownership to the resulting array
+        *out_values = list_item.detach();
+      }
+      ++out_values;
+    }
+    RETURN_IF_PYERROR();
+
+    chunk_offset += arr.values()->length();
   }
 
   return Status::OK();
@@ -951,12 +1059,39 @@ struct ObjectWriterVisitor {
   template <typename Type>
   enable_if_timestamp<Type, Status> Visit(const Type& type) {
     const TimeUnit::type unit = type.unit();
-    auto WrapValue = [unit](typename Type::c_type value, PyObject** out) {
+    OwnedRef tzinfo;
+
+    auto ConvertTimezoneNaive = [&](typename Type::c_type value, PyObject** out) {
       RETURN_NOT_OK(internal::PyDateTime_from_int(value, unit, out));
       RETURN_IF_PYERROR();
       return Status::OK();
     };
-    return ConvertAsPyObjects<Type>(options, data, WrapValue, out_values);
+    auto ConvertTimezoneAware = [&](typename Type::c_type value, PyObject** out) {
+      PyObject* naive_datetime;
+      RETURN_NOT_OK(ConvertTimezoneNaive(value, &naive_datetime));
+      // convert the timezone naive datetime object to timezone aware
+      *out = PyObject_CallMethod(tzinfo.obj(), "fromutc", "O", naive_datetime);
+      // the timezone naive object is no longer required
+      Py_DECREF(naive_datetime);
+      RETURN_IF_PYERROR();
+      return Status::OK();
+    };
+
+    if (!type.timezone().empty() && !options.ignore_timezone) {
+      // convert timezone aware
+      PyObject* tzobj;
+      ARROW_ASSIGN_OR_RAISE(tzobj, internal::StringToTzinfo(type.timezone()));
+      tzinfo.reset(tzobj);
+      RETURN_IF_PYERROR();
+      RETURN_NOT_OK(
+          ConvertAsPyObjects<Type>(options, data, ConvertTimezoneAware, out_values));
+    } else {
+      // convert timezone naive
+      RETURN_NOT_OK(
+          ConvertAsPyObjects<Type>(options, data, ConvertTimezoneNaive, out_values));
+    }
+
+    return Status::OK();
   }
 
   Status Visit(const Decimal128Type& type) {
@@ -984,6 +1119,31 @@ struct ObjectWriterVisitor {
     return Status::OK();
   }
 
+  Status Visit(const Decimal256Type& type) {
+    OwnedRef decimal;
+    OwnedRef Decimal;
+    RETURN_NOT_OK(internal::ImportModule("decimal", &decimal));
+    RETURN_NOT_OK(internal::ImportFromModule(decimal.obj(), "Decimal", &Decimal));
+    PyObject* decimal_constructor = Decimal.obj();
+
+    for (int c = 0; c < data.num_chunks(); c++) {
+      const auto& arr = checked_cast<const arrow::Decimal256Array&>(*data.chunk(c));
+
+      for (int64_t i = 0; i < arr.length(); ++i) {
+        if (arr.IsNull(i)) {
+          Py_INCREF(Py_None);
+          *out_values++ = Py_None;
+        } else {
+          *out_values++ =
+              internal::DecimalFromString(decimal_constructor, arr.FormatValue(i));
+          RETURN_IF_PYERROR();
+        }
+      }
+    }
+
+    return Status::OK();
+  }
+
   template <typename T>
   enable_if_t<is_fixed_size_list_type<T>::value || is_var_length_list_type<T>::value,
               Status>
@@ -997,6 +1157,8 @@ struct ObjectWriterVisitor {
     return ConvertListsLike<ArrayType>(options, data, out_values);
   }
 
+  Status Visit(const MapType& type) { return ConvertMap(options, data, out_values); }
+
   Status Visit(const StructType& type) {
     return ConvertStruct(options, data, out_values);
   }
@@ -1709,7 +1871,8 @@ static Status GetPandasWriterType(const ChunkedArray& data, const PandasOptions&
     case Type::STRUCT:             // fall through
     case Type::TIME32:             // fall through
     case Type::TIME64:             // fall through
-    case Type::DECIMAL:            // fall through
+    case Type::DECIMAL128:         // fall through
+    case Type::DECIMAL256:         // fall through
       *output_type = PandasWriter::OBJECT;
       break;
     case Type::DATE32:  // fall through
@@ -1727,8 +1890,7 @@ static Status GetPandasWriterType(const ChunkedArray& data, const PandasOptions&
         // Nanoseconds are never out of bounds for pandas, so in that case
         // we don't convert to object
         *output_type = PandasWriter::OBJECT;
-      } else if (ts_type.timezone() != "" && !options.ignore_timezone) {
-        // XXX: ignore_timezone: hack here for ARROW-7723
+      } else if (!ts_type.timezone().empty()) {
         *output_type = PandasWriter::DATETIME_NANO_TZ;
       } else if (options.coerce_temporal_nanoseconds) {
         *output_type = PandasWriter::DATETIME_NANO;
@@ -1772,7 +1934,8 @@ static Status GetPandasWriterType(const ChunkedArray& data, const PandasOptions&
     } break;
     case Type::FIXED_SIZE_LIST:
     case Type::LIST:
-    case Type::LARGE_LIST: {
+    case Type::LARGE_LIST:
+    case Type::MAP: {
       auto list_type = std::static_pointer_cast<BaseListType>(data.type());
       if (!ListTypeSupported(*list_type->value_type())) {
         return Status::NotImplemented("Not implemented type for Arrow list to pandas: ",
@@ -1805,9 +1968,8 @@ class PandasBlockCreator {
  public:
   using WriterMap = std::unordered_map<int, std::shared_ptr<PandasWriter>>;
 
-  explicit PandasBlockCreator(const PandasOptions& options,
-                              std::vector<std::shared_ptr<Field>> fields,
-                              std::vector<std::shared_ptr<ChunkedArray>> arrays)
+  explicit PandasBlockCreator(const PandasOptions& options, FieldVector fields,
+                              ChunkedArrayVector arrays)
       : options_(options), fields_(std::move(fields)), arrays_(std::move(arrays)) {
     num_columns_ = static_cast<int>(arrays_.size());
     if (num_columns_ > 0) {
@@ -1835,8 +1997,8 @@ class PandasBlockCreator {
  protected:
   PandasOptions options_;
 
-  std::vector<std::shared_ptr<Field>> fields_;
-  std::vector<std::shared_ptr<ChunkedArray>> arrays_;
+  FieldVector fields_;
+  ChunkedArrayVector arrays_;
   int num_columns_;
   int64_t num_rows_;
 
@@ -2009,9 +2171,8 @@ class SplitBlockCreator : public PandasBlockCreator {
   std::vector<std::shared_ptr<PandasWriter>> writers_;
 };
 
-Status ConvertCategoricals(const PandasOptions& options,
-                           std::vector<std::shared_ptr<ChunkedArray>>* arrays,
-                           std::vector<std::shared_ptr<Field>>* fields) {
+Status ConvertCategoricals(const PandasOptions& options, ChunkedArrayVector* arrays,
+                           FieldVector* fields) {
   std::vector<int> columns_to_encode;
 
   // For Categorical conversions
@@ -2047,6 +2208,8 @@ Status ConvertCategoricals(const PandasOptions& options,
                              static_cast<int>(columns_to_encode.size()), EncodeColumn);
 }
 
+}  // namespace
+
 Status ConvertArrayToPandas(const PandasOptions& options, std::shared_ptr<Array> arr,
                             PyObject* py_ref, PyObject** out) {
   return ConvertChunkedArrayToPandas(
@@ -2056,6 +2219,13 @@ Status ConvertArrayToPandas(const PandasOptions& options, std::shared_ptr<Array>
 Status ConvertChunkedArrayToPandas(const PandasOptions& options,
                                    std::shared_ptr<ChunkedArray> arr, PyObject* py_ref,
                                    PyObject** out) {
+  if (options.decode_dictionaries && arr->type()->id() == Type::DICTIONARY) {
+    const auto& dense_type =
+        checked_cast<const DictionaryType&>(*arr->type()).value_type();
+    RETURN_NOT_OK(DecodeDictionaries(options.pool, dense_type, &arr));
+    DCHECK_NE(arr->type()->id(), Type::DICTIONARY);
+  }
+
   if (options.strings_to_categorical && is_base_binary_like(arr->type()->id())) {
     if (options.zero_copy_only) {
       return Status::Invalid("Need to dictionary encode a column, but ",
@@ -2076,6 +2246,9 @@ Status ConvertChunkedArrayToPandas(const PandasOptions& options,
 
   PandasWriter::type output_type;
   RETURN_NOT_OK(GetPandasWriterType(*arr, modified_options, &output_type));
+  if (options.decode_dictionaries) {
+    DCHECK_NE(output_type, PandasWriter::CATEGORICAL);
+  }
 
   std::shared_ptr<PandasWriter> writer;
   RETURN_NOT_OK(MakeWriter(modified_options, output_type, *arr->type(), arr->length(),
@@ -2086,8 +2259,8 @@ Status ConvertChunkedArrayToPandas(const PandasOptions& options,
 
 Status ConvertTableToPandas(const PandasOptions& options, std::shared_ptr<Table> table,
                             PyObject** out) {
-  std::vector<std::shared_ptr<ChunkedArray>> arrays = table->columns();
-  std::vector<std::shared_ptr<Field>> fields = table->fields();
+  ChunkedArrayVector arrays = table->columns();
+  FieldVector fields = table->fields();
 
   // ARROW-3789: allow "self-destructing" by releasing references to columns as
   // we convert them to pandas
diff --git a/cpp/src/arrow/python/arrow_to_pandas.h b/cpp/src/arrow/python/arrow_to_pandas.h
index 79a72bcb1ef..6570364b8d2 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.h
+++ b/cpp/src/arrow/python/arrow_to_pandas.h
@@ -56,8 +56,9 @@ struct PandasOptions {
   /// Coerce all date and timestamp to datetime64[ns]
   bool coerce_temporal_nanoseconds = false;
 
-  /// XXX(wesm): Hack for ARROW-7723 to opt out of DATETIME_NANO_TZ conversion
-  /// path
+  /// Used to maintain backwards compatibility for
+  /// timezone bugs (see ARROW-9528).  Should be removed
+  /// after Arrow 2.0 release.
   bool ignore_timezone = false;
 
   /// \brief If true, do not create duplicate PyObject versions of equal
@@ -89,6 +90,9 @@ struct PandasOptions {
   /// conversions
   bool self_destruct = false;
 
+  // Used internally for nested arrays.
+  bool decode_dictionaries = false;
+
   // Columns that should be casted to categorical
   std::unordered_set<std::string> categorical_columns;
 
diff --git a/cpp/src/arrow/python/common.h b/cpp/src/arrow/python/common.h
index 52a3f334d4e..8560fa2d6f4 100644
--- a/cpp/src/arrow/python/common.h
+++ b/cpp/src/arrow/python/common.h
@@ -23,6 +23,7 @@
 #include "arrow/buffer.h"
 #include "arrow/python/pyarrow.h"
 #include "arrow/python/visibility.h"
+#include "arrow/result.h"
 #include "arrow/util/macros.h"
 
 namespace arrow {
@@ -188,84 +189,80 @@ class ARROW_PYTHON_EXPORT OwnedRefNoGIL : public OwnedRef {
 struct PyBytesView {
   const char* bytes;
   Py_ssize_t size;
+  bool is_utf8;
 
-  PyBytesView() : bytes(NULLPTR), size(0), ref(NULLPTR) {}
-
-  // View the given Python object as binary-like, i.e. bytes
-  Status FromBinary(PyObject* obj) { return FromBinary(obj, "a bytes object"); }
-
-  Status FromString(PyObject* obj) {
-    bool ignored = false;
-    return FromString(obj, false, &ignored);
+  static Result<PyBytesView> FromString(PyObject* obj, bool check_utf8 = false) {
+    PyBytesView self;
+    ARROW_RETURN_NOT_OK(self.ParseString(obj, check_utf8));
+    return std::move(self);
   }
 
-  Status FromString(PyObject* obj, bool* is_utf8) {
-    return FromString(obj, true, is_utf8);
+  static Result<PyBytesView> FromUnicode(PyObject* obj) {
+    PyBytesView self;
+    ARROW_RETURN_NOT_OK(self.ParseUnicode(obj));
+    return std::move(self);
   }
 
-  Status FromUnicode(PyObject* obj) {
-    Py_ssize_t size;
-    // The utf-8 representation is cached on the unicode object
-    const char* data = PyUnicode_AsUTF8AndSize(obj, &size);
-    RETURN_IF_PYERROR();
-    this->bytes = data;
-    this->size = size;
-    this->ref.reset();
-    return Status::OK();
+  static Result<PyBytesView> FromBinary(PyObject* obj) {
+    PyBytesView self;
+    ARROW_RETURN_NOT_OK(self.ParseBinary(obj));
+    return std::move(self);
   }
 
- protected:
-  PyBytesView(const char* b, Py_ssize_t s, PyObject* obj = NULLPTR)
-      : bytes(b), size(s), ref(obj) {}
-
   // View the given Python object as string-like, i.e. str or (utf8) bytes
-  Status FromString(PyObject* obj, bool check_utf8, bool* is_utf8) {
+  Status ParseString(PyObject* obj, bool check_utf8 = false) {
     if (PyUnicode_Check(obj)) {
-      *is_utf8 = true;
-      return FromUnicode(obj);
+      return ParseUnicode(obj);
     } else {
-      ARROW_RETURN_NOT_OK(FromBinary(obj, "a string or bytes object"));
+      ARROW_RETURN_NOT_OK(ParseBinary(obj));
       if (check_utf8) {
         // Check the bytes are utf8 utf-8
         OwnedRef decoded(PyUnicode_FromStringAndSize(bytes, size));
         if (ARROW_PREDICT_TRUE(!PyErr_Occurred())) {
-          *is_utf8 = true;
+          is_utf8 = true;
         } else {
-          *is_utf8 = false;
           PyErr_Clear();
+          is_utf8 = false;
         }
-      } else {
-        *is_utf8 = false;
       }
       return Status::OK();
     }
   }
 
-  Status FromBinary(PyObject* obj, const char* expected_msg) {
+  // View the given Python object as unicode string
+  Status ParseUnicode(PyObject* obj) {
+    // The utf-8 representation is cached on the unicode object
+    bytes = PyUnicode_AsUTF8AndSize(obj, &size);
+    RETURN_IF_PYERROR();
+    is_utf8 = true;
+    return Status::OK();
+  }
+
+  // View the given Python object as binary-like, i.e. bytes
+  Status ParseBinary(PyObject* obj) {
     if (PyBytes_Check(obj)) {
-      this->bytes = PyBytes_AS_STRING(obj);
-      this->size = PyBytes_GET_SIZE(obj);
-      this->ref.reset();
-      return Status::OK();
+      bytes = PyBytes_AS_STRING(obj);
+      size = PyBytes_GET_SIZE(obj);
+      is_utf8 = false;
     } else if (PyByteArray_Check(obj)) {
-      this->bytes = PyByteArray_AS_STRING(obj);
-      this->size = PyByteArray_GET_SIZE(obj);
-      this->ref.reset();
-      return Status::OK();
+      bytes = PyByteArray_AS_STRING(obj);
+      size = PyByteArray_GET_SIZE(obj);
+      is_utf8 = false;
     } else if (PyMemoryView_Check(obj)) {
-      PyObject* contig_view = PyMemoryView_GetContiguous(obj, PyBUF_READ, 'C');
+      PyObject* ref = PyMemoryView_GetContiguous(obj, PyBUF_READ, 'C');
       RETURN_IF_PYERROR();
-      this->ref.reset(contig_view);
-      Py_buffer* buf = PyMemoryView_GET_BUFFER(contig_view);
-      this->bytes = reinterpret_cast<const char*>(buf->buf);
-      this->size = buf->len;
-      return Status::OK();
+      Py_buffer* buffer = PyMemoryView_GET_BUFFER(ref);
+      bytes = reinterpret_cast<const char*>(buffer->buf);
+      size = buffer->len;
+      is_utf8 = false;
     } else {
-      return Status::TypeError("Expected ", expected_msg, ", got a '",
-                               Py_TYPE(obj)->tp_name, "' object");
+      return Status::TypeError("Expected bytes, got a '", Py_TYPE(obj)->tp_name,
+                               "' object");
     }
+    return Status::OK();
   }
 
+ protected:
   OwnedRef ref;
 };
 
diff --git a/cpp/src/arrow/python/datetime.cc b/cpp/src/arrow/python/datetime.cc
index 8cec87bdd36..4b18918cbcf 100644
--- a/cpp/src/arrow/python/datetime.cc
+++ b/cpp/src/arrow/python/datetime.cc
@@ -14,22 +14,65 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+#include "arrow/python/datetime.h"
 
 #include <algorithm>
 #include <chrono>
-#include <iostream>
+#include <iomanip>
 
 #include "arrow/python/common.h"
-#include "arrow/python/datetime.h"
+#include "arrow/python/helpers.h"
 #include "arrow/python/platform.h"
 #include "arrow/status.h"
 #include "arrow/type.h"
 #include "arrow/util/logging.h"
+#include "arrow/util/value_parsing.h"
 
 namespace arrow {
 namespace py {
 namespace internal {
 
+namespace {
+
+// Same as Regex '([+-])(0[0-9]|1[0-9]|2[0-3]):([0-5][0-9])$'.
+// GCC 4.9 doesn't support regex, so handcode until support for it
+// is dropped.
+bool MatchFixedOffset(const std::string& tz, util::string_view* sign,
+                      util::string_view* hour, util::string_view* minute) {
+  if (tz.size() < 5) {
+    return false;
+  }
+  const char* iter = tz.data();
+  if (*iter == '+' || *iter == '-') {
+    *sign = util::string_view(iter, 1);
+    iter++;
+    if (tz.size() < 6) {
+      return false;
+    }
+  }
+  if ((((*iter == '0' || *iter == '1') && *(iter + 1) >= '0' && *(iter + 1) <= '9') ||
+       (*iter == '2' && *(iter + 1) >= '0' && *(iter + 1) <= '3'))) {
+    *hour = util::string_view(iter, 2);
+    iter += 2;
+  } else {
+    return false;
+  }
+  if (*iter != ':') {
+    return false;
+  }
+  iter++;
+
+  if (*iter >= '0' && *iter <= '5' && *(iter + 1) >= '0' && *(iter + 1) <= '9') {
+    *minute = util::string_view(iter, 2);
+    iter += 2;
+  } else {
+    return false;
+  }
+  return iter == (tz.data() + tz.size());
+}
+
+}  // namespace
+
 PyDateTime_CAPI* datetime_api = nullptr;
 
 void InitDatetime() {
@@ -262,6 +305,151 @@ int64_t PyDate_to_days(PyDateTime_Date* pydate) {
                             PyDateTime_GET_DAY(pydate));
 }
 
+Result<int64_t> PyDateTime_utcoffset_s(PyObject* obj) {
+  // calculate offset from UTC timezone in seconds
+  // supports only PyDateTime_DateTime and PyDateTime_Time objects
+  OwnedRef pyoffset(PyObject_CallMethod(obj, "utcoffset", NULL));
+  RETURN_IF_PYERROR();
+  if (pyoffset.obj() != nullptr && pyoffset.obj() != Py_None) {
+    auto delta = reinterpret_cast<PyDateTime_Delta*>(pyoffset.obj());
+    return internal::PyDelta_to_s(delta);
+  } else {
+    return 0;
+  }
+}
+
+Result<std::string> PyTZInfo_utcoffset_hhmm(PyObject* pytzinfo) {
+  // attempt to convert timezone offset objects to "+/-{hh}:{mm}" format
+  OwnedRef pydelta_object(PyObject_CallMethod(pytzinfo, "utcoffset", "O", Py_None));
+  RETURN_IF_PYERROR();
+
+  if (!PyDelta_Check(pydelta_object.obj())) {
+    return Status::Invalid(
+        "Object returned by tzinfo.utcoffset(None) is not an instance of "
+        "datetime.timedelta");
+  }
+  auto pydelta = reinterpret_cast<PyDateTime_Delta*>(pydelta_object.obj());
+
+  // retrieve the offset as seconds
+  auto total_seconds = internal::PyDelta_to_s(pydelta);
+
+  // determine whether the offset is positive or negative
+  auto sign = (total_seconds < 0) ? "-" : "+";
+  total_seconds = abs(total_seconds);
+
+  // calculate offset components
+  int64_t hours, minutes, seconds;
+  seconds = split_time(total_seconds, 60, &minutes);
+  minutes = split_time(minutes, 60, &hours);
+  if (seconds > 0) {
+    // check there are no remaining seconds
+    return Status::Invalid("Offset must represent whole number of minutes");
+  }
+
+  // construct the timezone string
+  std::stringstream stream;
+  stream << sign << std::setfill('0') << std::setw(2) << hours << ":" << std::setfill('0')
+         << std::setw(2) << minutes;
+  return stream.str();
+}
+
+// Converted from python.  See https://github.com/apache/arrow/pull/7604
+// for details.
+Result<PyObject*> StringToTzinfo(const std::string& tz) {
+  util::string_view sign_str, hour_str, minute_str;
+  OwnedRef pytz;
+  RETURN_NOT_OK(internal::ImportModule("pytz", &pytz));
+
+  if (MatchFixedOffset(tz, &sign_str, &hour_str, &minute_str)) {
+    int sign = -1;
+    if (sign_str == "+") {
+      sign = 1;
+    }
+    OwnedRef fixed_offset;
+    RETURN_NOT_OK(internal::ImportFromModule(pytz.obj(), "FixedOffset", &fixed_offset));
+    uint32_t minutes, hours;
+    if (!::arrow::internal::ParseUnsigned(hour_str.data(), hour_str.size(), &hours) ||
+        !::arrow::internal::ParseUnsigned(minute_str.data(), minute_str.size(),
+                                          &minutes)) {
+      return Status::Invalid("Invalid timezone: ", tz);
+    }
+    OwnedRef total_minutes(PyLong_FromLong(
+        sign * ((static_cast<int>(hours) * 60) + static_cast<int>(minutes))));
+    RETURN_IF_PYERROR();
+    auto tzinfo =
+        PyObject_CallFunctionObjArgs(fixed_offset.obj(), total_minutes.obj(), NULL);
+    RETURN_IF_PYERROR();
+    return tzinfo;
+  }
+
+  OwnedRef timezone;
+  RETURN_NOT_OK(internal::ImportFromModule(pytz.obj(), "timezone", &timezone));
+  OwnedRef py_tz_string(
+      PyUnicode_FromStringAndSize(tz.c_str(), static_cast<Py_ssize_t>(tz.size())));
+  auto tzinfo = PyObject_CallFunctionObjArgs(timezone.obj(), py_tz_string.obj(), NULL);
+  RETURN_IF_PYERROR();
+  return tzinfo;
+}
+
+Result<std::string> TzinfoToString(PyObject* tzinfo) {
+  OwnedRef module_pytz;        // import pytz
+  OwnedRef module_datetime;    // import datetime
+  OwnedRef class_timezone;     // from datetime import timezone
+  OwnedRef class_fixedoffset;  // from pytz import _FixedOffset
+
+  // import necessary modules
+  RETURN_NOT_OK(internal::ImportModule("pytz", &module_pytz));
+  RETURN_NOT_OK(internal::ImportModule("datetime", &module_datetime));
+  // import necessary classes
+  RETURN_NOT_OK(
+      internal::ImportFromModule(module_pytz.obj(), "_FixedOffset", &class_fixedoffset));
+  RETURN_NOT_OK(
+      internal::ImportFromModule(module_datetime.obj(), "timezone", &class_timezone));
+
+  // check that it's a valid tzinfo object
+  if (!PyTZInfo_Check(tzinfo)) {
+    return Status::TypeError("Not an instance of datetime.tzinfo");
+  }
+
+  // if tzinfo is an instance of pytz._FixedOffset or datetime.timezone return the
+  // HH:MM offset string representation
+  if (PyObject_IsInstance(tzinfo, class_timezone.obj()) ||
+      PyObject_IsInstance(tzinfo, class_fixedoffset.obj())) {
+    // still recognize datetime.timezone.utc as UTC (instead of +00:00)
+    OwnedRef tzname_object(PyObject_CallMethod(tzinfo, "tzname", "O", Py_None));
+    RETURN_IF_PYERROR();
+    if (PyUnicode_Check(tzname_object.obj())) {
+      std::string result;
+      RETURN_NOT_OK(internal::PyUnicode_AsStdString(tzname_object.obj(), &result));
+      if (result == "UTC") {
+        return result;
+      }
+    }
+    return PyTZInfo_utcoffset_hhmm(tzinfo);
+  }
+
+  // try to look up zone attribute
+  if (PyObject_HasAttrString(tzinfo, "zone")) {
+    OwnedRef zone(PyObject_GetAttrString(tzinfo, "zone"));
+    RETURN_IF_PYERROR();
+    std::string result;
+    RETURN_NOT_OK(internal::PyUnicode_AsStdString(zone.obj(), &result));
+    return result;
+  }
+
+  // attempt to call tzinfo.tzname(None)
+  OwnedRef tzname_object(PyObject_CallMethod(tzinfo, "tzname", "O", Py_None));
+  RETURN_IF_PYERROR();
+  if (PyUnicode_Check(tzname_object.obj())) {
+    std::string result;
+    RETURN_NOT_OK(internal::PyUnicode_AsStdString(tzname_object.obj(), &result));
+    return result;
+  }
+
+  // fall back to HH:MM offset string representation based on tzinfo.utcoffset(None)
+  return PyTZInfo_utcoffset_hhmm(tzinfo);
+}
+
 }  // namespace internal
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/datetime.h b/cpp/src/arrow/python/datetime.h
index a8b22da4741..0072cdda4cb 100644
--- a/cpp/src/arrow/python/datetime.h
+++ b/cpp/src/arrow/python/datetime.h
@@ -44,9 +44,9 @@ void InitDatetime();
 
 ARROW_PYTHON_EXPORT
 inline int64_t PyTime_to_us(PyObject* pytime) {
-  return (static_cast<int64_t>(PyDateTime_TIME_GET_HOUR(pytime)) * 3600000000LL +
-          static_cast<int64_t>(PyDateTime_TIME_GET_MINUTE(pytime)) * 60000000LL +
-          static_cast<int64_t>(PyDateTime_TIME_GET_SECOND(pytime)) * 1000000LL +
+  return (PyDateTime_TIME_GET_HOUR(pytime) * 3600000000LL +
+          PyDateTime_TIME_GET_MINUTE(pytime) * 60000000LL +
+          PyDateTime_TIME_GET_SECOND(pytime) * 1000000LL +
           PyDateTime_TIME_GET_MICROSECOND(pytime));
 }
 
@@ -76,39 +76,39 @@ using TimePoint =
 ARROW_PYTHON_EXPORT
 int64_t PyDate_to_days(PyDateTime_Date* pydate);
 
+ARROW_PYTHON_EXPORT
+inline int64_t PyDate_to_s(PyDateTime_Date* pydate) {
+  return PyDate_to_days(pydate) * 86400LL;
+}
+
 ARROW_PYTHON_EXPORT
 inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
-  return PyDate_to_days(pydate) * 24 * 3600 * 1000;
+  return PyDate_to_days(pydate) * 86400000LL;
 }
 
 ARROW_PYTHON_EXPORT
 inline int64_t PyDateTime_to_s(PyDateTime_DateTime* pydatetime) {
-  int64_t total_seconds = 0;
-  total_seconds += PyDateTime_DATE_GET_SECOND(pydatetime);
-  total_seconds += PyDateTime_DATE_GET_MINUTE(pydatetime) * 60;
-  total_seconds += PyDateTime_DATE_GET_HOUR(pydatetime) * 3600;
-
-  return total_seconds +
-         (PyDate_to_ms(reinterpret_cast<PyDateTime_Date*>(pydatetime)) / 1000LL);
+  return (PyDate_to_s(reinterpret_cast<PyDateTime_Date*>(pydatetime)) +
+          PyDateTime_DATE_GET_HOUR(pydatetime) * 3600LL +
+          PyDateTime_DATE_GET_MINUTE(pydatetime) * 60LL +
+          PyDateTime_DATE_GET_SECOND(pydatetime));
 }
 
 ARROW_PYTHON_EXPORT
 inline int64_t PyDateTime_to_ms(PyDateTime_DateTime* pydatetime) {
-  int64_t date_ms = PyDateTime_to_s(pydatetime) * 1000;
-  int ms = PyDateTime_DATE_GET_MICROSECOND(pydatetime) / 1000;
-  return date_ms + ms;
+  return (PyDateTime_to_s(pydatetime) * 1000LL +
+          PyDateTime_DATE_GET_MICROSECOND(pydatetime) / 1000);
 }
 
 ARROW_PYTHON_EXPORT
 inline int64_t PyDateTime_to_us(PyDateTime_DateTime* pydatetime) {
-  int64_t ms = PyDateTime_to_s(pydatetime) * 1000;
-  int us = PyDateTime_DATE_GET_MICROSECOND(pydatetime);
-  return ms * 1000 + us;
+  return (PyDateTime_to_s(pydatetime) * 1000000LL +
+          PyDateTime_DATE_GET_MICROSECOND(pydatetime));
 }
 
 ARROW_PYTHON_EXPORT
 inline int64_t PyDateTime_to_ns(PyDateTime_DateTime* pydatetime) {
-  return PyDateTime_to_us(pydatetime) * 1000;
+  return PyDateTime_to_us(pydatetime) * 1000LL;
 }
 
 ARROW_PYTHON_EXPORT
@@ -131,32 +131,53 @@ inline TimePoint TimePoint_from_ns(int64_t val) {
 
 ARROW_PYTHON_EXPORT
 inline int64_t PyDelta_to_s(PyDateTime_Delta* pytimedelta) {
-  int64_t total_seconds = 0;
-  total_seconds += PyDateTime_DELTA_GET_SECONDS(pytimedelta);
-  total_seconds += PyDateTime_DELTA_GET_DAYS(pytimedelta) * 24 * 3600;
-  return total_seconds;
+  return (PyDateTime_DELTA_GET_DAYS(pytimedelta) * 86400LL +
+          PyDateTime_DELTA_GET_SECONDS(pytimedelta));
 }
 
 ARROW_PYTHON_EXPORT
 inline int64_t PyDelta_to_ms(PyDateTime_Delta* pytimedelta) {
-  int64_t total_ms = PyDelta_to_s(pytimedelta) * 1000;
-  total_ms += PyDateTime_DELTA_GET_MICROSECONDS(pytimedelta) / 1000;
-  return total_ms;
+  return (PyDelta_to_s(pytimedelta) * 1000LL +
+          PyDateTime_DELTA_GET_MICROSECONDS(pytimedelta) / 1000);
 }
 
 ARROW_PYTHON_EXPORT
 inline int64_t PyDelta_to_us(PyDateTime_Delta* pytimedelta) {
-  int64_t total_us = 0;
-  total_us += PyDelta_to_s(pytimedelta) * 1000 * 1000;
-  total_us += PyDateTime_DELTA_GET_MICROSECONDS(pytimedelta);
-  return total_us;
+  return (PyDelta_to_s(pytimedelta) * 1000000LL +
+          PyDateTime_DELTA_GET_MICROSECONDS(pytimedelta));
 }
 
 ARROW_PYTHON_EXPORT
 inline int64_t PyDelta_to_ns(PyDateTime_Delta* pytimedelta) {
-  return PyDelta_to_us(pytimedelta) * 1000;
+  return PyDelta_to_us(pytimedelta) * 1000LL;
 }
 
+ARROW_PYTHON_EXPORT
+Result<int64_t> PyDateTime_utcoffset_s(PyObject* pydatetime);
+
+/// \brief Convert a time zone name into a time zone object.
+///
+/// Supported input strings are:
+/// * As used in the Olson time zone database (the "tz database" or
+///   "tzdata"), such as "America/New_York"
+/// * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30
+/// GIL must be held when calling this method.
+ARROW_PYTHON_EXPORT
+Result<PyObject*> StringToTzinfo(const std::string& tz);
+
+/// \brief Convert a time zone object to a string representation.
+///
+/// The output strings are:
+/// * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30
+///   if the input object is either an instance of pytz._FixedOffset or
+///   datetime.timedelta
+/// * The timezone's name if the input object's tzname() method returns with a
+///   non-empty timezone name such as "UTC" or "America/New_York"
+///
+/// GIL must be held when calling this method.
+ARROW_PYTHON_EXPORT
+Result<std::string> TzinfoToString(PyObject* pytzinfo);
+
 }  // namespace internal
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/decimal.cc b/cpp/src/arrow/python/decimal.cc
index 18712015df8..ab28b0f2b9f 100644
--- a/cpp/src/arrow/python/decimal.cc
+++ b/cpp/src/arrow/python/decimal.cc
@@ -21,11 +21,10 @@
 #include "arrow/python/common.h"
 #include "arrow/python/decimal.h"
 #include "arrow/python/helpers.h"
+#include "arrow/type_fwd.h"
 #include "arrow/util/decimal.h"
 #include "arrow/util/logging.h"
 
-#include <arrow/api.h>
-
 namespace arrow {
 namespace py {
 namespace internal {
@@ -109,13 +108,14 @@ PyObject* DecimalFromString(PyObject* decimal_constructor,
 
 namespace {
 
+template <typename ArrowDecimal>
 Status DecimalFromStdString(const std::string& decimal_string,
-                            const DecimalType& arrow_type, Decimal128* out) {
+                            const DecimalType& arrow_type, ArrowDecimal* out) {
   int32_t inferred_precision;
   int32_t inferred_scale;
 
-  RETURN_NOT_OK(
-      Decimal128::FromString(decimal_string, out, &inferred_precision, &inferred_scale));
+  RETURN_NOT_OK(ArrowDecimal::FromString(decimal_string, out, &inferred_precision,
+                                         &inferred_scale));
 
   const int32_t precision = arrow_type.precision();
   const int32_t scale = arrow_type.scale();
@@ -133,10 +133,10 @@ Status DecimalFromStdString(const std::string& decimal_string,
   return Status::OK();
 }
 
-}  // namespace
-
-Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type,
-                                Decimal128* out) {
+template <typename ArrowDecimal>
+Status InternalDecimalFromPythonDecimal(PyObject* python_decimal,
+                                        const DecimalType& arrow_type,
+                                        ArrowDecimal* out) {
   DCHECK_NE(python_decimal, NULLPTR);
   DCHECK_NE(out, NULLPTR);
 
@@ -145,8 +145,9 @@ Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arr
   return DecimalFromStdString(string, arrow_type, out);
 }
 
-Status DecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type,
-                           Decimal128* out) {
+template <typename ArrowDecimal>
+Status InternalDecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type,
+                                   ArrowDecimal* out) {
   DCHECK_NE(obj, NULLPTR);
   DCHECK_NE(out, NULLPTR);
 
@@ -156,13 +157,35 @@ Status DecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type,
     RETURN_NOT_OK(PyObject_StdStringStr(obj, &string));
     return DecimalFromStdString(string, arrow_type, out);
   } else if (PyDecimal_Check(obj)) {
-    return DecimalFromPythonDecimal(obj, arrow_type, out);
+    return InternalDecimalFromPythonDecimal<ArrowDecimal>(obj, arrow_type, out);
   } else {
     return Status::TypeError("int or Decimal object expected, got ",
                              Py_TYPE(obj)->tp_name);
   }
 }
 
+}  // namespace
+
+Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type,
+                                Decimal128* out) {
+  return InternalDecimalFromPythonDecimal(python_decimal, arrow_type, out);
+}
+
+Status DecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type,
+                           Decimal128* out) {
+  return InternalDecimalFromPyObject(obj, arrow_type, out);
+}
+
+Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type,
+                                Decimal256* out) {
+  return InternalDecimalFromPythonDecimal(python_decimal, arrow_type, out);
+}
+
+Status DecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type,
+                           Decimal256* out) {
+  return InternalDecimalFromPyObject(obj, arrow_type, out);
+}
+
 bool PyDecimal_Check(PyObject* obj) {
   static OwnedRef decimal_type;
   if (!decimal_type.obj()) {
diff --git a/cpp/src/arrow/python/decimal.h b/cpp/src/arrow/python/decimal.h
index 3d20b014010..1187037aed2 100644
--- a/cpp/src/arrow/python/decimal.h
+++ b/cpp/src/arrow/python/decimal.h
@@ -25,6 +25,7 @@
 namespace arrow {
 
 class Decimal128;
+class Decimal256;
 
 namespace py {
 
@@ -72,6 +73,23 @@ Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arr
 ARROW_PYTHON_EXPORT
 Status DecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type, Decimal128* out);
 
+// \brief Convert a Python decimal to an Arrow Decimal256 object
+// \param[in] python_decimal A Python decimal.Decimal instance
+// \param[in] arrow_type An instance of arrow::DecimalType
+// \param[out] out A pointer to a Decimal256
+// \return The status of the operation
+ARROW_PYTHON_EXPORT
+Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type,
+                                Decimal256* out);
+
+// \brief Convert a Python object to an Arrow Decimal256 object
+// \param[in] python_decimal A Python int or decimal.Decimal instance
+// \param[in] arrow_type An instance of arrow::DecimalType
+// \param[out] out A pointer to a Decimal256
+// \return The status of the operation
+ARROW_PYTHON_EXPORT
+Status DecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type, Decimal256* out);
+
 // \brief Check whether obj is an instance of Decimal
 ARROW_PYTHON_EXPORT
 bool PyDecimal_Check(PyObject* obj);
diff --git a/cpp/src/arrow/python/extension_type.cc b/cpp/src/arrow/python/extension_type.cc
index 8c69003d298..3ccc171c871 100644
--- a/cpp/src/arrow/python/extension_type.cc
+++ b/cpp/src/arrow/python/extension_type.cc
@@ -16,6 +16,7 @@
 // under the License.
 
 #include <memory>
+#include <sstream>
 #include <utility>
 
 #include "arrow/python/extension_type.h"
@@ -71,6 +72,16 @@ PyObject* DeserializeExtInstance(PyObject* type_class,
 
 static const char* kExtensionName = "arrow.py_extension_type";
 
+std::string PyExtensionType::ToString() const {
+  PyAcquireGIL lock;
+
+  std::stringstream ss;
+  OwnedRef instance(GetInstance());
+  ss << "extension<" << this->extension_name() << "<" << Py_TYPE(instance.obj())->tp_name
+     << ">>";
+  return ss.str();
+}
+
 PyExtensionType::PyExtensionType(std::shared_ptr<DataType> storage_type, PyObject* typ,
                                  PyObject* inst)
     : ExtensionType(storage_type),
diff --git a/cpp/src/arrow/python/extension_type.h b/cpp/src/arrow/python/extension_type.h
index 0041c8af6a4..e433d9aca70 100644
--- a/cpp/src/arrow/python/extension_type.h
+++ b/cpp/src/arrow/python/extension_type.h
@@ -33,6 +33,8 @@ class ARROW_PYTHON_EXPORT PyExtensionType : public ExtensionType {
   // Implement extensionType API
   std::string extension_name() const override { return extension_name_; }
 
+  std::string ToString() const override;
+
   bool ExtensionEquals(const ExtensionType& other) const override;
 
   std::shared_ptr<Array> MakeArray(std::shared_ptr<ArrayData> data) const override;
@@ -44,6 +46,7 @@ class ARROW_PYTHON_EXPORT PyExtensionType : public ExtensionType {
   std::string Serialize() const override;
 
   // For use from Cython
+  // Assumes that `typ` is borrowed
   static Status FromClass(const std::shared_ptr<DataType> storage_type,
                           const std::string extension_name, PyObject* typ,
                           std::shared_ptr<ExtensionType>* out);
diff --git a/cpp/src/arrow/python/flight.cc b/cpp/src/arrow/python/flight.cc
index 1ae54b64a93..ee1491e0d14 100644
--- a/cpp/src/arrow/python/flight.cc
+++ b/cpp/src/arrow/python/flight.cc
@@ -232,7 +232,7 @@ Status PyFlightDataStream::Next(FlightPayload* payload) { return stream_->Next(p
 PyGeneratorFlightDataStream::PyGeneratorFlightDataStream(
     PyObject* generator, std::shared_ptr<arrow::Schema> schema,
     PyGeneratorFlightDataStreamCallback callback, const ipc::IpcWriteOptions& options)
-    : schema_(schema), options_(options), callback_(callback) {
+    : schema_(schema), mapper_(*schema_), options_(options), callback_(callback) {
   Py_INCREF(generator);
   generator_.reset(generator);
 }
@@ -240,8 +240,7 @@ PyGeneratorFlightDataStream::PyGeneratorFlightDataStream(
 std::shared_ptr<Schema> PyGeneratorFlightDataStream::schema() { return schema_; }
 
 Status PyGeneratorFlightDataStream::GetSchemaPayload(FlightPayload* payload) {
-  return ipc::GetSchemaPayload(*schema_, options_, &dictionary_memo_,
-                               &payload->ipc_message);
+  return ipc::GetSchemaPayload(*schema_, options_, mapper_, &payload->ipc_message);
 }
 
 Status PyGeneratorFlightDataStream::Next(FlightPayload* payload) {
diff --git a/cpp/src/arrow/python/flight.h b/cpp/src/arrow/python/flight.h
index e04d3c3e179..45a090ef445 100644
--- a/cpp/src/arrow/python/flight.h
+++ b/cpp/src/arrow/python/flight.h
@@ -328,7 +328,7 @@ class ARROW_PYFLIGHT_EXPORT PyGeneratorFlightDataStream
  private:
   OwnedRefNoGIL generator_;
   std::shared_ptr<arrow::Schema> schema_;
-  ipc::DictionaryMemo dictionary_memo_;
+  ipc::DictionaryFieldMapper mapper_;
   ipc::IpcWriteOptions options_;
   PyGeneratorFlightDataStreamCallback callback_;
 };
diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
index 852bf763afc..6c8f0fe1e3a 100644
--- a/cpp/src/arrow/python/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -20,19 +20,18 @@
 
 #include "arrow/python/helpers.h"
 
+#include <cmath>
 #include <limits>
 #include <mutex>
 #include <sstream>
 #include <type_traits>
-#include <typeinfo>
 
 #include "arrow/python/common.h"
 #include "arrow/python/decimal.h"
+#include "arrow/type_fwd.h"
 #include "arrow/util/checked_cast.h"
 #include "arrow/util/logging.h"
 
-#include <arrow/api.h>
-
 namespace arrow {
 
 using internal::checked_cast;
@@ -128,6 +127,14 @@ Status PyObject_StdStringStr(PyObject* obj, std::string* out) {
   return PyUnicode_AsStdString(string_ref.obj(), out);
 }
 
+Result<bool> IsModuleImported(const std::string& module_name) {
+  // PyImport_GetModuleDict returns with a borrowed reference
+  OwnedRef key(PyUnicode_FromString(module_name.c_str()));
+  auto is_imported = PyDict_Contains(PyImport_GetModuleDict(), key.obj());
+  RETURN_IF_PYERROR();
+  return is_imported;
+}
+
 Status ImportModule(const std::string& module_name, OwnedRef* ref) {
   PyObject* module = PyImport_ImportModule(module_name.c_str());
   RETURN_IF_PYERROR();
@@ -258,30 +265,44 @@ bool PyFloat_IsNaN(PyObject* obj) {
 namespace {
 
 static std::once_flag pandas_static_initialized;
-static PyTypeObject* pandas_NaTType = nullptr;
+
 static PyObject* pandas_NA = nullptr;
+static PyObject* pandas_NaT = nullptr;
+static PyObject* pandas_Timedelta = nullptr;
+static PyObject* pandas_Timestamp = nullptr;
+static PyTypeObject* pandas_NaTType = nullptr;
 
 void GetPandasStaticSymbols() {
   OwnedRef pandas;
+
+  // import pandas
   Status s = ImportModule("pandas", &pandas);
   if (!s.ok()) {
     return;
   }
 
   OwnedRef ref;
-  s = ImportFromModule(pandas.obj(), "NaT", &ref);
-  if (!s.ok()) {
-    return;
+
+  // set NaT sentinel and its type
+  if (ImportFromModule(pandas.obj(), "NaT", &ref).ok()) {
+    pandas_NaT = ref.obj();
+    // PyObject_Type returns a new reference but we trust that pandas.NaT will
+    // outlive our use of this PyObject*
+    pandas_NaTType = Py_TYPE(ref.obj());
   }
-  PyObject* nat_type = PyObject_Type(ref.obj());
-  pandas_NaTType = reinterpret_cast<PyTypeObject*>(nat_type);
 
-  // PyObject_Type returns a new reference but we trust that pandas.NaT will
-  // outlive our use of this PyObject*
-  Py_DECREF(nat_type);
+  // retain a reference to Timedelta
+  if (ImportFromModule(pandas.obj(), "Timedelta", &ref).ok()) {
+    pandas_Timedelta = ref.obj();
+  }
+
+  // retain a reference to Timestamp
+  if (ImportFromModule(pandas.obj(), "Timestamp", &ref).ok()) {
+    pandas_Timestamp = ref.obj();
+  }
 
+  // if pandas.NA exists, retain a reference to it
   if (ImportFromModule(pandas.obj(), "NA", &ref).ok()) {
-    // If pandas.NA exists, retain a reference to it
     pandas_NA = ref.obj();
   }
 }
@@ -307,6 +328,14 @@ bool PandasObjectIsNull(PyObject* obj) {
   return false;
 }
 
+bool IsPandasTimedelta(PyObject* obj) {
+  return pandas_Timedelta && PyObject_IsInstance(obj, pandas_Timedelta);
+}
+
+bool IsPandasTimestamp(PyObject* obj) {
+  return pandas_Timestamp && PyObject_IsInstance(obj, pandas_Timestamp);
+}
+
 Status InvalidValue(PyObject* obj, const std::string& why) {
   std::string obj_as_str;
   RETURN_NOT_OK(internal::PyObject_StdStringStr(obj, &obj_as_str));
@@ -328,6 +357,8 @@ Status UnboxIntegerAsInt64(PyObject* obj, int64_t* out) {
     if (overflow) {
       return Status::Invalid("PyLong is too large to fit int64");
     }
+  } else if (PyArray_IsScalar(obj, Byte)) {
+    *out = reinterpret_cast<PyByteScalarObject*>(obj)->obval;
   } else if (PyArray_IsScalar(obj, UByte)) {
     *out = reinterpret_cast<PyUByteScalarObject*>(obj)->obval;
   } else if (PyArray_IsScalar(obj, Short)) {
diff --git a/cpp/src/arrow/python/helpers.h b/cpp/src/arrow/python/helpers.h
index 4d27c2c11a1..19288756c0e 100644
--- a/cpp/src/arrow/python/helpers.h
+++ b/cpp/src/arrow/python/helpers.h
@@ -51,6 +51,10 @@ ARROW_PYTHON_EXPORT Status PyFloat_AsHalf(PyObject* obj, npy_half* out);
 
 namespace internal {
 
+// \brief Check that a Python module has been already imported
+// \param[in] module_name The name of the module
+Result<bool> IsModuleImported(const std::string& module_name);
+
 // \brief Import a Python module
 // \param[in] module_name The name of the module
 // \param[out] ref The OwnedRef containing the module PyObject*
@@ -76,6 +80,13 @@ void InitPandasStaticData();
 ARROW_PYTHON_EXPORT
 bool PandasObjectIsNull(PyObject* obj);
 
+// \brief Check that obj is a pandas.Timedelta instance
+ARROW_PYTHON_EXPORT
+bool IsPandasTimedelta(PyObject* obj);
+
+// \brief Check that obj is a pandas.Timestamp instance
+bool IsPandasTimestamp(PyObject* obj);
+
 // \brief Check whether obj is a floating-point NaN
 ARROW_PYTHON_EXPORT
 bool PyFloat_IsNaN(PyObject* obj);
diff --git a/cpp/src/arrow/python/inference.cc b/cpp/src/arrow/python/inference.cc
index c2fc06e554c..9d6707aa11d 100644
--- a/cpp/src/arrow/python/inference.cc
+++ b/cpp/src/arrow/python/inference.cc
@@ -295,10 +295,7 @@ class TypeInferrer {
         int_count_(0),
         date_count_(0),
         time_count_(0),
-        timestamp_second_count_(0),
-        timestamp_milli_count_(0),
         timestamp_micro_count_(0),
-        timestamp_nano_count_(0),
         duration_count_(0),
         float_count_(0),
         binary_count_(0),
@@ -331,6 +328,13 @@ class TypeInferrer {
     } else if (internal::IsPyInteger(obj)) {
       ++int_count_;
     } else if (PyDateTime_Check(obj)) {
+      // infer timezone from the first encountered datetime object
+      if (!timestamp_micro_count_) {
+        OwnedRef tzinfo(PyObject_GetAttrString(obj, "tzinfo"));
+        if (tzinfo.obj() != nullptr && tzinfo.obj() != Py_None) {
+          ARROW_ASSIGN_OR_RAISE(timezone_, internal::TzinfoToString(tzinfo.obj()));
+        }
+      }
       ++timestamp_micro_count_;
       *keep_going = make_unions_;
     } else if (PyDelta_Check(obj)) {
@@ -446,9 +450,16 @@ class TypeInferrer {
     } else if (struct_count_) {
       RETURN_NOT_OK(GetStructType(out));
     } else if (decimal_count_) {
-      // the default constructor does not validate the precision and scale
-      ARROW_ASSIGN_OR_RAISE(*out, Decimal128Type::Make(max_decimal_metadata_.precision(),
-                                                       max_decimal_metadata_.scale()));
+      if (max_decimal_metadata_.precision() > Decimal128Type::kMaxPrecision) {
+        // the default constructor does not validate the precision and scale
+        ARROW_ASSIGN_OR_RAISE(*out,
+                              Decimal256Type::Make(max_decimal_metadata_.precision(),
+                                                   max_decimal_metadata_.scale()));
+      } else {
+        ARROW_ASSIGN_OR_RAISE(*out,
+                              Decimal128Type::Make(max_decimal_metadata_.precision(),
+                                                   max_decimal_metadata_.scale()));
+      }
     } else if (float_count_) {
       // Prioritize floats before integers
       *out = float64();
@@ -458,14 +469,8 @@ class TypeInferrer {
       *out = date32();
     } else if (time_count_) {
       *out = time64(TimeUnit::MICRO);
-    } else if (timestamp_nano_count_) {
-      *out = timestamp(TimeUnit::NANO);
     } else if (timestamp_micro_count_) {
-      *out = timestamp(TimeUnit::MICRO);
-    } else if (timestamp_milli_count_) {
-      *out = timestamp(TimeUnit::MILLI);
-    } else if (timestamp_second_count_) {
-      *out = timestamp(TimeUnit::SECOND);
+      *out = timestamp(TimeUnit::MICRO, timezone_);
     } else if (duration_count_) {
       *out = duration(TimeUnit::MICRO);
     } else if (bool_count_) {
@@ -597,10 +602,8 @@ class TypeInferrer {
   int64_t int_count_;
   int64_t date_count_;
   int64_t time_count_;
-  int64_t timestamp_second_count_;
-  int64_t timestamp_milli_count_;
   int64_t timestamp_micro_count_;
-  int64_t timestamp_nano_count_;
+  std::string timezone_;
   int64_t duration_count_;
   int64_t float_count_;
   int64_t binary_count_;
@@ -624,39 +627,23 @@ class TypeInferrer {
 };
 
 // Non-exhaustive type inference
-Status InferArrowType(PyObject* obj, PyObject* mask, bool pandas_null_sentinels,
-                      std::shared_ptr<DataType>* out_type) {
+Result<std::shared_ptr<DataType>> InferArrowType(PyObject* obj, PyObject* mask,
+                                                 bool pandas_null_sentinels) {
   if (pandas_null_sentinels) {
     // ARROW-842: If pandas is not installed then null checks will be less
     // comprehensive, but that is okay.
     internal::InitPandasStaticData();
   }
 
+  std::shared_ptr<DataType> out_type;
   TypeInferrer inferrer(pandas_null_sentinels);
   RETURN_NOT_OK(inferrer.VisitSequence(obj, mask));
-  RETURN_NOT_OK(inferrer.GetType(out_type));
-  if (*out_type == nullptr) {
+  RETURN_NOT_OK(inferrer.GetType(&out_type));
+  if (out_type == nullptr) {
     return Status::TypeError("Unable to determine data type");
+  } else {
+    return std::move(out_type);
   }
-
-  return Status::OK();
-}
-
-Status InferArrowTypeAndSize(PyObject* obj, PyObject* mask, bool pandas_null_sentinels,
-                             int64_t* size, std::shared_ptr<DataType>* out_type) {
-  if (!PySequence_Check(obj)) {
-    return Status::TypeError("Object is not a sequence");
-  }
-  *size = static_cast<int64_t>(PySequence_Size(obj));
-
-  // For 0-length sequences, refuse to guess
-  if (*size == 0) {
-    *out_type = null();
-    return Status::OK();
-  }
-  RETURN_NOT_OK(InferArrowType(obj, mask, pandas_null_sentinels, out_type));
-
-  return Status::OK();
 }
 
 ARROW_PYTHON_EXPORT
diff --git a/cpp/src/arrow/python/inference.h b/cpp/src/arrow/python/inference.h
index 74d1b78161c..eff18362934 100644
--- a/cpp/src/arrow/python/inference.h
+++ b/cpp/src/arrow/python/inference.h
@@ -44,15 +44,9 @@ namespace py {
 /// \param[in] mask an optional mask where True values are null. May
 /// be nullptr
 /// \param[in] pandas_null_sentinels use pandas's null value markers
-/// \param[out] out_type the inferred type
 ARROW_PYTHON_EXPORT
-arrow::Status InferArrowType(PyObject* obj, PyObject* mask, bool pandas_null_sentinels,
-                             std::shared_ptr<arrow::DataType>* out_type);
-
-ARROW_PYTHON_EXPORT
-arrow::Status InferArrowTypeAndSize(PyObject* obj, PyObject* mask,
-                                    bool pandas_null_sentinels, int64_t* size,
-                                    std::shared_ptr<arrow::DataType>* out_type);
+Result<std::shared_ptr<arrow::DataType>> InferArrowType(PyObject* obj, PyObject* mask,
+                                                        bool pandas_null_sentinels);
 
 /// Checks whether the passed Python object is a boolean scalar
 ARROW_PYTHON_EXPORT
diff --git a/cpp/src/arrow/python/ipc.cc b/cpp/src/arrow/python/ipc.cc
new file mode 100644
index 00000000000..2e6c9d91275
--- /dev/null
+++ b/cpp/src/arrow/python/ipc.cc
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/python/ipc.h"
+
+#include <memory>
+
+#include "arrow/python/pyarrow.h"
+
+namespace arrow {
+namespace py {
+
+PyRecordBatchReader::PyRecordBatchReader() {}
+
+Status PyRecordBatchReader::Init(std::shared_ptr<Schema> schema, PyObject* iterable) {
+  schema_ = std::move(schema);
+
+  iterator_.reset(PyObject_GetIter(iterable));
+  return CheckPyError();
+}
+
+std::shared_ptr<Schema> PyRecordBatchReader::schema() const { return schema_; }
+
+Status PyRecordBatchReader::ReadNext(std::shared_ptr<RecordBatch>* batch) {
+  PyAcquireGIL lock;
+
+  if (!iterator_) {
+    // End of stream
+    batch->reset();
+    return Status::OK();
+  }
+
+  OwnedRef py_batch(PyIter_Next(iterator_.obj()));
+  if (!py_batch) {
+    RETURN_IF_PYERROR();
+    // End of stream
+    batch->reset();
+    iterator_.reset();
+    return Status::OK();
+  }
+
+  return unwrap_batch(py_batch.obj()).Value(batch);
+}
+
+Result<std::shared_ptr<RecordBatchReader>> PyRecordBatchReader::Make(
+    std::shared_ptr<Schema> schema, PyObject* iterable) {
+  auto reader = std::shared_ptr<PyRecordBatchReader>(new PyRecordBatchReader());
+  RETURN_NOT_OK(reader->Init(std::move(schema), iterable));
+  return reader;
+}
+
+}  // namespace py
+}  // namespace arrow
diff --git a/cpp/src/arrow/python/ipc.h b/cpp/src/arrow/python/ipc.h
new file mode 100644
index 00000000000..92232ed8300
--- /dev/null
+++ b/cpp/src/arrow/python/ipc.h
@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+
+#include "arrow/python/common.h"
+#include "arrow/python/visibility.h"
+#include "arrow/record_batch.h"
+#include "arrow/result.h"
+#include "arrow/util/macros.h"
+
+namespace arrow {
+namespace py {
+
+class ARROW_PYTHON_EXPORT PyRecordBatchReader : public RecordBatchReader {
+ public:
+  std::shared_ptr<Schema> schema() const override;
+
+  Status ReadNext(std::shared_ptr<RecordBatch>* batch) override;
+
+  // For use from Cython
+  // Assumes that `iterable` is borrowed
+  static Result<std::shared_ptr<RecordBatchReader>> Make(std::shared_ptr<Schema>,
+                                                         PyObject* iterable);
+
+ protected:
+  PyRecordBatchReader();
+
+  Status Init(std::shared_ptr<Schema>, PyObject* iterable);
+
+  std::shared_ptr<Schema> schema_;
+  OwnedRefNoGIL iterator_;
+};
+
+}  // namespace py
+}  // namespace arrow
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index af608dfc360..07322d199e5 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -31,6 +31,7 @@
 #include <vector>
 
 #include "arrow/array.h"
+#include "arrow/array/builder_binary.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type_fwd.h"
@@ -314,11 +315,11 @@ Status NumPyConverter::Convert() {
     PyConversionOptions py_options;
     py_options.type = type_;
     py_options.from_pandas = from_pandas_;
-    std::shared_ptr<ChunkedArray> res;
-    RETURN_NOT_OK(ConvertPySequence(reinterpret_cast<PyObject*>(arr_),
-                                    reinterpret_cast<PyObject*>(mask_), py_options,
-                                    &res));
-    out_arrays_ = res->chunks();
+    ARROW_ASSIGN_OR_RAISE(
+        auto chunked_array,
+        ConvertPySequence(reinterpret_cast<PyObject*>(arr_),
+                          reinterpret_cast<PyObject*>(mask_), py_options, pool_));
+    out_arrays_ = chunked_array->chunks();
     return Status::OK();
   }
 
diff --git a/cpp/src/arrow/python/python_test.cc b/cpp/src/arrow/python/python_test.cc
index b21c16af50a..11d3593be23 100644
--- a/cpp/src/arrow/python/python_test.cc
+++ b/cpp/src/arrow/python/python_test.cc
@@ -24,10 +24,11 @@
 #include "arrow/python/platform.h"
 
 #include "arrow/array.h"
-#include "arrow/builder.h"
+#include "arrow/array/builder_binary.h"
 #include "arrow/table.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/util/decimal.h"
+#include "arrow/util/optional.h"
 
 #include "arrow/python/arrow_to_pandas.h"
 #include "arrow/python/decimal.h"
@@ -329,58 +330,65 @@ TEST(BuiltinConversionTest, TestMixedTypeFails) {
   ASSERT_EQ(PyList_SetItem(list, 1, integer), 0);
   ASSERT_EQ(PyList_SetItem(list, 2, doub), 0);
 
-  std::shared_ptr<ChunkedArray> arr;
-  ASSERT_RAISES(TypeError, ConvertPySequence(list, {}, &arr));
+  ASSERT_RAISES(TypeError, ConvertPySequence(list, nullptr, {}));
+}
+
+template <typename DecimalValue>
+void DecimalTestFromPythonDecimalRescale(std::shared_ptr<DataType> type,
+                                         OwnedRef python_decimal,
+                                         ::arrow::util::optional<int> expected) {
+  DecimalValue value;
+  const auto& decimal_type = checked_cast<const DecimalType&>(*type);
+
+  if (expected.has_value()) {
+    ASSERT_OK(
+        internal::DecimalFromPythonDecimal(python_decimal.obj(), decimal_type, &value));
+    ASSERT_EQ(expected.value(), value);
+
+    ASSERT_OK(internal::DecimalFromPyObject(python_decimal.obj(), decimal_type, &value));
+    ASSERT_EQ(expected.value(), value);
+  } else {
+    ASSERT_RAISES(Invalid, internal::DecimalFromPythonDecimal(python_decimal.obj(),
+                                                              decimal_type, &value));
+    ASSERT_RAISES(Invalid, internal::DecimalFromPyObject(python_decimal.obj(),
+                                                         decimal_type, &value));
+  }
 }
 
 TEST_F(DecimalTest, FromPythonDecimalRescaleNotTruncateable) {
   // We fail when truncating values that would lose data if cast to a decimal type with
   // lower scale
-  Decimal128 value;
-  OwnedRef python_decimal(this->CreatePythonDecimal("1.001"));
-  auto type = ::arrow::decimal(10, 2);
-  const auto& decimal_type = checked_cast<const DecimalType&>(*type);
-  ASSERT_RAISES(Invalid, internal::DecimalFromPythonDecimal(python_decimal.obj(),
-                                                            decimal_type, &value));
+  DecimalTestFromPythonDecimalRescale<Decimal128>(::arrow::decimal128(10, 2),
+                                                  this->CreatePythonDecimal("1.001"), {});
+  // TODO: Test Decimal256 after implementing scaling.
 }
 
 TEST_F(DecimalTest, FromPythonDecimalRescaleTruncateable) {
   // We allow truncation of values that do not lose precision when dividing by 10 * the
   // difference between the scales, e.g., 1.000 -> 1.00
-  Decimal128 value;
-  OwnedRef python_decimal(this->CreatePythonDecimal("1.000"));
-  auto type = ::arrow::decimal(10, 2);
-  const auto& decimal_type = checked_cast<const DecimalType&>(*type);
-  ASSERT_OK(
-      internal::DecimalFromPythonDecimal(python_decimal.obj(), decimal_type, &value));
-  ASSERT_EQ(100, value.low_bits());
-  ASSERT_EQ(0, value.high_bits());
-
-  ASSERT_OK(internal::DecimalFromPyObject(python_decimal.obj(), decimal_type, &value));
-  ASSERT_EQ(100, value.low_bits());
-  ASSERT_EQ(0, value.high_bits());
+  DecimalTestFromPythonDecimalRescale<Decimal128>(
+      ::arrow::decimal128(10, 2), this->CreatePythonDecimal("1.000"), 100);
+  // TODO: Test Decimal256 after implementing scaling.
 }
 
 TEST_F(DecimalTest, FromPythonNegativeDecimalRescale) {
-  Decimal128 value;
-  OwnedRef python_decimal(this->CreatePythonDecimal("-1.000"));
-  auto type = ::arrow::decimal(10, 9);
-  const auto& decimal_type = checked_cast<const DecimalType&>(*type);
-  ASSERT_OK(
-      internal::DecimalFromPythonDecimal(python_decimal.obj(), decimal_type, &value));
-  ASSERT_EQ(-1000000000, value);
+  DecimalTestFromPythonDecimalRescale<Decimal128>(
+      ::arrow::decimal128(10, 9), this->CreatePythonDecimal("-1.000"), -1000000000);
+  // TODO: Test Decimal256 after implementing scaling.
 }
 
-TEST_F(DecimalTest, FromPythonInteger) {
+TEST_F(DecimalTest, Decimal128FromPythonInteger) {
   Decimal128 value;
   OwnedRef python_long(PyLong_FromLong(42));
-  auto type = ::arrow::decimal(10, 2);
+  auto type = ::arrow::decimal128(10, 2);
   const auto& decimal_type = checked_cast<const DecimalType&>(*type);
   ASSERT_OK(internal::DecimalFromPyObject(python_long.obj(), decimal_type, &value));
   ASSERT_EQ(4200, value);
 }
 
-TEST_F(DecimalTest, TestOverflowFails) {
+// TODO: Test Decimal256 from python after implementing scaling.
+
+TEST_F(DecimalTest, TestDecimal128OverflowFails) {
   Decimal128 value;
   OwnedRef python_decimal(
       this->CreatePythonDecimal("9999999999999999999999999999999999999.9"));
@@ -395,6 +403,8 @@ TEST_F(DecimalTest, TestOverflowFails) {
                                                             decimal_type, &value));
 }
 
+// TODO: Test Decimal256 overflow after implementing scaling.
+
 TEST_F(DecimalTest, TestNoneAndNaN) {
   OwnedRef list_ref(PyList_New(4));
   PyObject* list = list_ref.obj();
@@ -422,17 +432,18 @@ TEST_F(DecimalTest, TestNoneAndNaN) {
   ASSERT_EQ(0, PyList_SetItem(list, 2, missing_value2));
   ASSERT_EQ(0, PyList_SetItem(list, 3, missing_value3));
 
-  std::shared_ptr<ChunkedArray> arr, arr_from_pandas;
   PyConversionOptions options;
-  ASSERT_RAISES(TypeError, ConvertPySequence(list, options, &arr));
+  ASSERT_RAISES(TypeError, ConvertPySequence(list, nullptr, options));
 
   options.from_pandas = true;
-  ASSERT_OK(ConvertPySequence(list, options, &arr_from_pandas));
-  auto c0 = arr_from_pandas->chunk(0);
-  ASSERT_TRUE(c0->IsValid(0));
-  ASSERT_TRUE(c0->IsNull(1));
-  ASSERT_TRUE(c0->IsNull(2));
-  ASSERT_TRUE(c0->IsNull(3));
+  ASSERT_OK_AND_ASSIGN(auto chunked, ConvertPySequence(list, nullptr, options));
+  ASSERT_EQ(chunked->num_chunks(), 1);
+
+  auto arr = chunked->chunk(0);
+  ASSERT_TRUE(arr->IsValid(0));
+  ASSERT_TRUE(arr->IsNull(1));
+  ASSERT_TRUE(arr->IsNull(2));
+  ASSERT_TRUE(arr->IsNull(3));
 }
 
 TEST_F(DecimalTest, TestMixedPrecisionAndScale) {
@@ -451,8 +462,7 @@ TEST_F(DecimalTest, TestMixedPrecisionAndScale) {
     ASSERT_EQ(0, result);
   }
 
-  std::shared_ptr<ChunkedArray> arr;
-  ASSERT_OK(ConvertPySequence(list, {}, &arr));
+  ASSERT_OK_AND_ASSIGN(auto arr, ConvertPySequence(list, nullptr, {}))
   const auto& type = checked_cast<const DecimalType&>(*arr->type());
 
   int32_t expected_precision = 9;
@@ -476,9 +486,7 @@ TEST_F(DecimalTest, TestMixedPrecisionAndScaleSequenceConvert) {
   ASSERT_EQ(PyList_SetItem(list, 0, value1), 0);
   ASSERT_EQ(PyList_SetItem(list, 1, value2), 0);
 
-  std::shared_ptr<ChunkedArray> arr;
-  ASSERT_OK(ConvertPySequence(list, {}, &arr));
-
+  ASSERT_OK_AND_ASSIGN(auto arr, ConvertPySequence(list, nullptr, {}));
   const auto& type = checked_cast<const Decimal128Type&>(*arr->type());
   ASSERT_EQ(3, type.precision());
   ASSERT_EQ(3, type.scale());
diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
index d75c0aceff6..b136bec9709 100644
--- a/cpp/src/arrow/python/python_to_arrow.cc
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -22,20 +22,25 @@
 
 #include <algorithm>
 #include <limits>
-#include <map>
 #include <sstream>
 #include <string>
 #include <utility>
 #include <vector>
 
 #include "arrow/array.h"
-#include "arrow/builder.h"
+#include "arrow/array/builder_binary.h"
+#include "arrow/array/builder_decimal.h"
+#include "arrow/array/builder_dict.h"
+#include "arrow/array/builder_nested.h"
+#include "arrow/array/builder_primitive.h"
 #include "arrow/chunked_array.h"
 #include "arrow/status.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/checked_cast.h"
+#include "arrow/util/converter.h"
 #include "arrow/util/decimal.h"
+#include "arrow/util/int_util_internal.h"
 #include "arrow/util/logging.h"
 
 #include "arrow/python/datetime.h"
@@ -45,48 +50,60 @@
 #include "arrow/python/iterators.h"
 #include "arrow/python/numpy_convert.h"
 #include "arrow/python/type_traits.h"
+#include "arrow/visitor_inline.h"
 
 namespace arrow {
 
 using internal::checked_cast;
 using internal::checked_pointer_cast;
 
-namespace py {
-
-// ----------------------------------------------------------------------
-// NullCoding
+using internal::Converter;
+using internal::DictionaryConverter;
+using internal::ListConverter;
+using internal::PrimitiveConverter;
+using internal::StructConverter;
 
-enum class NullCoding : char { NONE_ONLY, PANDAS_SENTINELS };
+using internal::MakeChunker;
+using internal::MakeConverter;
 
-template <NullCoding kind>
-struct NullChecker {};
+namespace py {
 
-template <>
-struct NullChecker<NullCoding::NONE_ONLY> {
-  static inline bool Check(PyObject* obj) { return obj == Py_None; }
-};
+// Utility for converting single python objects to their intermediate C representations
+// which can be fed to the typed builders
+class PyValue {
+ public:
+  // Type aliases for shorter signature definitions
+  using I = PyObject*;
+  using O = PyConversionOptions;
+
+  // Used for null checking before actually converting the values
+  static bool IsNull(const O& options, I obj) {
+    if (options.from_pandas) {
+      return internal::PandasObjectIsNull(obj);
+    } else {
+      return obj == Py_None;
+    }
+  }
 
-template <>
-struct NullChecker<NullCoding::PANDAS_SENTINELS> {
-  static inline bool Check(PyObject* obj) { return internal::PandasObjectIsNull(obj); }
-};
+  // Used for post-conversion numpy NaT sentinel checking
+  static bool IsNaT(const TimestampType*, int64_t value) {
+    return internal::npy_traits<NPY_DATETIME>::isnull(value);
+  }
 
-// ----------------------------------------------------------------------
-// ValueConverters
-//
-// Typed conversion logic for single python objects are encapsulated in
-// ValueConverter structs using SFINAE for specialization.
-//
-// The FromPython medthod is responsible to convert the python object to the
-// C++ value counterpart which can be directly appended to the ArrayBuilder or
-// Scalar can be constructed from.
+  // Used for post-conversion numpy NaT sentinel checking
+  static bool IsNaT(const DurationType*, int64_t value) {
+    return internal::npy_traits<NPY_TIMEDELTA>::isnull(value);
+  }
 
-template <typename Type, typename Enable = void>
-struct ValueConverter {};
+  static Result<std::nullptr_t> Convert(const NullType*, const O&, I obj) {
+    if (obj == Py_None) {
+      return nullptr;
+    } else {
+      return Status::Invalid("Invalid null value");
+    }
+  }
 
-template <>
-struct ValueConverter<BooleanType> {
-  static inline Result<bool> FromPython(PyObject* obj) {
+  static Result<bool> Convert(const BooleanType*, const O&, I obj) {
     if (obj == Py_True) {
       return true;
     } else if (obj == Py_False) {
@@ -97,33 +114,28 @@ struct ValueConverter<BooleanType> {
       return internal::InvalidValue(obj, "tried to convert to boolean");
     }
   }
-};
 
-template <typename Type>
-struct ValueConverter<Type, enable_if_integer<Type>> {
-  using ValueType = typename Type::c_type;
-
-  static inline Result<ValueType> FromPython(PyObject* obj) {
-    ValueType value;
-    RETURN_NOT_OK(internal::CIntFromPython(obj, &value));
-    return value;
+  template <typename T>
+  static enable_if_integer<T, Result<typename T::c_type>> Convert(const T*, const O&,
+                                                                  I obj) {
+    typename T::c_type value;
+    auto status = internal::CIntFromPython(obj, &value);
+    if (ARROW_PREDICT_TRUE(status.ok())) {
+      return value;
+    } else if (!internal::PyIntScalar_Check(obj)) {
+      return internal::InvalidValue(obj, "tried to convert to int");
+    } else {
+      return status;
+    }
   }
-};
 
-template <>
-struct ValueConverter<HalfFloatType> {
-  using ValueType = typename HalfFloatType::c_type;
-
-  static inline Result<ValueType> FromPython(PyObject* obj) {
-    ValueType value;
+  static Result<uint16_t> Convert(const HalfFloatType*, const O&, I obj) {
+    uint16_t value;
     RETURN_NOT_OK(PyFloat_AsHalf(obj, &value));
     return value;
   }
-};
 
-template <>
-struct ValueConverter<FloatType> {
-  static inline Result<float> FromPython(PyObject* obj) {
+  static Result<float> Convert(const FloatType*, const O&, I obj) {
     float value;
     if (internal::PyFloatScalar_Check(obj)) {
       value = static_cast<float>(PyFloat_AsDouble(obj));
@@ -135,11 +147,8 @@ struct ValueConverter<FloatType> {
     }
     return value;
   }
-};
 
-template <>
-struct ValueConverter<DoubleType> {
-  static inline Result<double> FromPython(PyObject* obj) {
+  static Result<double> Convert(const DoubleType*, const O&, I obj) {
     double value;
     if (PyFloat_Check(obj)) {
       value = PyFloat_AS_DOUBLE(obj);
@@ -154,11 +163,20 @@ struct ValueConverter<DoubleType> {
     }
     return value;
   }
-};
 
-template <>
-struct ValueConverter<Date32Type> {
-  static inline Result<int32_t> FromPython(PyObject* obj) {
+  static Result<Decimal128> Convert(const Decimal128Type* type, const O&, I obj) {
+    Decimal128 value;
+    RETURN_NOT_OK(internal::DecimalFromPyObject(obj, *type, &value));
+    return value;
+  }
+
+  static Result<Decimal256> Convert(const Decimal256Type* type, const O&, I obj) {
+    Decimal256 value;
+    RETURN_NOT_OK(internal::DecimalFromPyObject(obj, *type, &value));
+    return value;
+  }
+
+  static Result<int32_t> Convert(const Date32Type*, const O&, I obj) {
     int32_t value;
     if (PyDate_Check(obj)) {
       auto pydate = reinterpret_cast<PyDateTime_Date*>(obj);
@@ -169,16 +187,14 @@ struct ValueConverter<Date32Type> {
     }
     return value;
   }
-};
 
-template <>
-struct ValueConverter<Date64Type> {
-  static inline Result<int64_t> FromPython(PyObject* obj) {
+  static Result<int64_t> Convert(const Date64Type*, const O&, I obj) {
     int64_t value;
     if (PyDateTime_Check(obj)) {
       auto pydate = reinterpret_cast<PyDateTime_DateTime*>(obj);
       value = internal::PyDateTime_to_ms(pydate);
       // Truncate any intraday milliseconds
+      // TODO: introduce an option for this
       value -= value % 86400000LL;
     } else if (PyDate_Check(obj)) {
       auto pydate = reinterpret_cast<PyDateTime_Date*>(obj);
@@ -189,15 +205,11 @@ struct ValueConverter<Date64Type> {
     }
     return value;
   }
-};
 
-template <>
-struct ValueConverter<Time32Type> {
-  static inline Result<int32_t> FromPython(PyObject* obj, TimeUnit::type unit) {
+  static Result<int32_t> Convert(const Time32Type* type, const O&, I obj) {
     int32_t value;
     if (PyTime_Check(obj)) {
-      // datetime.time stores microsecond resolution
-      switch (unit) {
+      switch (type->unit()) {
         case TimeUnit::SECOND:
           value = static_cast<int32_t>(internal::PyTime_to_s(obj));
           break;
@@ -212,15 +224,11 @@ struct ValueConverter<Time32Type> {
     }
     return value;
   }
-};
 
-template <>
-struct ValueConverter<Time64Type> {
-  static inline Result<int64_t> FromPython(PyObject* obj, TimeUnit::type unit) {
+  static Result<int64_t> Convert(const Time64Type* type, const O&, I obj) {
     int64_t value;
     if (PyTime_Check(obj)) {
-      // datetime.time stores microsecond resolution
-      switch (unit) {
+      switch (type->unit()) {
         case TimeUnit::MICRO:
           value = internal::PyTime_to_us(obj);
           break;
@@ -235,61 +243,72 @@ struct ValueConverter<Time64Type> {
     }
     return value;
   }
-};
 
-template <>
-struct ValueConverter<TimestampType> {
-  static inline Result<int64_t> FromPython(PyObject* obj, TimeUnit::type unit) {
-    int64_t value;
+  static Result<int64_t> Convert(const TimestampType* type, const O& options, I obj) {
+    int64_t value, offset;
     if (PyDateTime_Check(obj)) {
+      if (ARROW_PREDICT_FALSE(options.ignore_timezone)) {
+        offset = 0;
+      } else {
+        ARROW_ASSIGN_OR_RAISE(offset, internal::PyDateTime_utcoffset_s(obj));
+      }
       auto dt = reinterpret_cast<PyDateTime_DateTime*>(obj);
-      switch (unit) {
+      switch (type->unit()) {
         case TimeUnit::SECOND:
-          value = internal::PyDateTime_to_s(dt);
+          value = internal::PyDateTime_to_s(dt) - offset;
           break;
         case TimeUnit::MILLI:
-          value = internal::PyDateTime_to_ms(dt);
+          value = internal::PyDateTime_to_ms(dt) - offset * 1000LL;
           break;
         case TimeUnit::MICRO:
-          value = internal::PyDateTime_to_us(dt);
+          value = internal::PyDateTime_to_us(dt) - offset * 1000000LL;
           break;
         case TimeUnit::NANO:
-          value = internal::PyDateTime_to_ns(dt);
+          if (internal::IsPandasTimestamp(obj)) {
+            // pd.Timestamp value attribute contains the offset from unix epoch
+            // so no adjustment for timezone is need.
+            OwnedRef nanos(PyObject_GetAttrString(obj, "value"));
+            RETURN_IF_PYERROR();
+            RETURN_NOT_OK(internal::CIntFromPython(nanos.obj(), &value));
+          } else {
+            // Conversion to nanoseconds can overflow -> check multiply of microseconds
+            value = internal::PyDateTime_to_us(dt);
+            if (arrow::internal::MultiplyWithOverflow(value, 1000LL, &value)) {
+              return internal::InvalidValue(obj,
+                                            "out of bounds for nanosecond resolution");
+            }
+
+            // Adjust with offset and check for overflow
+            if (arrow::internal::SubtractWithOverflow(value, offset * 1000000000LL,
+                                                      &value)) {
+              return internal::InvalidValue(obj,
+                                            "out of bounds for nanosecond resolution");
+            }
+          }
           break;
         default:
           return Status::UnknownError("Invalid time unit");
       }
+    } else if (PyArray_CheckAnyScalarExact(obj)) {
+      // validate that the numpy scalar has np.datetime64 dtype
+      std::shared_ptr<DataType> numpy_type;
+      RETURN_NOT_OK(NumPyDtypeToArrow(PyArray_DescrFromScalar(obj), &numpy_type));
+      if (!numpy_type->Equals(*type)) {
+        return Status::NotImplemented("Expected np.datetime64 but got: ",
+                                      numpy_type->ToString());
+      }
+      return reinterpret_cast<PyDatetimeScalarObject*>(obj)->obval;
     } else {
       RETURN_NOT_OK(internal::CIntFromPython(obj, &value));
     }
     return value;
   }
 
-  static inline Result<int64_t> FromNumpy(PyObject* obj, TimeUnit::type unit) {
-    // validate that the numpy scalar has np.datetime64 dtype
-    std::shared_ptr<DataType> type;
-    RETURN_NOT_OK(NumPyDtypeToArrow(PyArray_DescrFromScalar(obj), &type));
-    if (type->id() != TimestampType::type_id) {
-      // TODO(kszucs): the message should highlight the received numpy dtype
-      return Status::Invalid("Expected np.datetime64 but got: ", type->ToString());
-    }
-    // validate that the time units are matching
-    if (unit != checked_cast<const TimestampType&>(*type).unit()) {
-      return Status::NotImplemented(
-          "Cannot convert NumPy np.datetime64 objects with differing unit");
-    }
-    // convert the numpy value
-    return reinterpret_cast<PyDatetimeScalarObject*>(obj)->obval;
-  }
-};
-
-template <>
-struct ValueConverter<DurationType> {
-  static inline Result<int64_t> FromPython(PyObject* obj, TimeUnit::type unit) {
+  static Result<int64_t> Convert(const DurationType* type, const O&, I obj) {
     int64_t value;
     if (PyDelta_Check(obj)) {
       auto dt = reinterpret_cast<PyDateTime_Delta*>(obj);
-      switch (unit) {
+      switch (type->unit()) {
         case TimeUnit::SECOND:
           value = internal::PyDelta_to_s(dt);
           break;
@@ -300,971 +319,646 @@ struct ValueConverter<DurationType> {
           value = internal::PyDelta_to_us(dt);
           break;
         case TimeUnit::NANO:
-          value = internal::PyDelta_to_ns(dt);
+          if (internal::IsPandasTimedelta(obj)) {
+            OwnedRef nanos(PyObject_GetAttrString(obj, "value"));
+            RETURN_IF_PYERROR();
+            RETURN_NOT_OK(internal::CIntFromPython(nanos.obj(), &value));
+          } else {
+            value = internal::PyDelta_to_ns(dt);
+          }
           break;
         default:
           return Status::UnknownError("Invalid time unit");
       }
+    } else if (PyArray_CheckAnyScalarExact(obj)) {
+      // validate that the numpy scalar has np.datetime64 dtype
+      std::shared_ptr<DataType> numpy_type;
+      RETURN_NOT_OK(NumPyDtypeToArrow(PyArray_DescrFromScalar(obj), &numpy_type));
+      if (!numpy_type->Equals(*type)) {
+        return Status::NotImplemented("Expected np.timedelta64 but got: ",
+                                      numpy_type->ToString());
+      }
+      return reinterpret_cast<PyTimedeltaScalarObject*>(obj)->obval;
     } else {
       RETURN_NOT_OK(internal::CIntFromPython(obj, &value));
     }
     return value;
   }
 
-  static inline Result<int64_t> FromNumpy(PyObject* obj, TimeUnit::type unit) {
-    // validate that the numpy scalar has np.timedelta64 dtype
-    std::shared_ptr<DataType> type;
-    RETURN_NOT_OK(NumPyDtypeToArrow(PyArray_DescrFromScalar(obj), &type));
-    if (type->id() != DurationType::type_id) {
-      // TODO(kszucs): the message should highlight the received numpy dtype
-      return Status::Invalid("Expected np.timedelta64 but got: ", type->ToString());
-    }
-    // validate that the time units are matching
-    if (unit != checked_cast<const DurationType&>(*type).unit()) {
-      return Status::NotImplemented(
-          "Cannot convert NumPy np.timedelta64 objects with differing unit");
-    }
-    // convert the numpy value
-    return reinterpret_cast<PyTimedeltaScalarObject*>(obj)->obval;
-  }
-};
+  // The binary-like intermediate representation is PyBytesView because it keeps temporary
+  // python objects alive (non-contiguous memoryview) and stores whether the original
+  // object was unicode encoded or not, which is used for unicode -> bytes coersion if
+  // there is a non-unicode object observed.
 
-template <typename Type>
-struct ValueConverter<Type, enable_if_any_binary<Type>> {
-  static inline Result<PyBytesView> FromPython(PyObject* obj) {
-    PyBytesView view;
-    RETURN_NOT_OK(view.FromString(obj));
-    return std::move(view);
+  static Status Convert(const BaseBinaryType*, const O&, I obj, PyBytesView& view) {
+    return view.ParseString(obj);
   }
-};
 
-template <typename Type>
-struct ValueConverter<Type, enable_if_string_like<Type>> {
-  static inline Result<PyBytesView> FromPython(PyObject* obj) {
-    // strict conversion, force output to be unicode / utf8 and validate that
-    // any binary values are utf8
-    bool is_utf8 = false;
-    PyBytesView view;
-
-    RETURN_NOT_OK(view.FromString(obj, &is_utf8));
-    if (!is_utf8) {
-      return internal::InvalidValue(obj, "was not a utf8 string");
+  static Status Convert(const FixedSizeBinaryType* type, const O&, I obj,
+                        PyBytesView& view) {
+    ARROW_RETURN_NOT_OK(view.ParseString(obj));
+    if (view.size != type->byte_width()) {
+      std::stringstream ss;
+      ss << "expected to be length " << type->byte_width() << " was " << view.size;
+      return internal::InvalidValue(obj, ss.str());
+    } else {
+      return Status::OK();
     }
-    return std::move(view);
   }
 
-  static inline Result<PyBytesView> FromPython(PyObject* obj, bool* is_utf8) {
-    PyBytesView view;
-
-    // Non-strict conversion; keep track of whether values are unicode or bytes
-    if (PyUnicode_Check(obj)) {
-      *is_utf8 = true;
-      RETURN_NOT_OK(view.FromUnicode(obj));
+  template <typename T>
+  static enable_if_string<T, Status> Convert(const T*, const O& options, I obj,
+                                             PyBytesView& view) {
+    if (options.strict) {
+      // Strict conversion, force output to be unicode / utf8 and validate that
+      // any binary values are utf8
+      ARROW_RETURN_NOT_OK(view.ParseString(obj, true));
+      if (!view.is_utf8) {
+        return internal::InvalidValue(obj, "was not a utf8 string");
+      }
+      return Status::OK();
     } else {
-      // If not unicode or bytes, FromBinary will error
-      *is_utf8 = false;
-      RETURN_NOT_OK(view.FromBinary(obj));
+      // Non-strict conversion; keep track of whether values are unicode or bytes
+      return view.ParseString(obj);
     }
-    return std::move(view);
   }
-};
 
-template <typename Type>
-struct ValueConverter<Type, enable_if_fixed_size_binary<Type>> {
-  static inline Result<PyBytesView> FromPython(PyObject* obj, int32_t byte_width) {
-    PyBytesView view;
-    RETURN_NOT_OK(view.FromString(obj));
-    if (ARROW_PREDICT_FALSE(view.size != byte_width)) {
-      std::stringstream ss;
-      ss << "expected to be length " << byte_width << " was " << view.size;
-      return internal::InvalidValue(obj, ss.str());
-    } else {
-      return std::move(view);
-    }
+  static Result<bool> Convert(const DataType* type, const O&, I obj) {
+    return Status::NotImplemented("PyValue::Convert is not implemented for type ", type);
   }
 };
 
-// ----------------------------------------------------------------------
-// Sequence converter base and CRTP "middle" subclasses
-
-class SeqConverter;
-
-// Forward-declare converter factory
-Status GetConverter(const std::shared_ptr<DataType>& type, bool from_pandas,
-                    bool strict_conversions, std::unique_ptr<SeqConverter>* out);
-
-// Marshal Python sequence (list, tuple, etc.) to Arrow array
-class SeqConverter {
- public:
-  virtual ~SeqConverter() = default;
+template <typename T>
+Status Extend(T* converter, PyObject* values, int64_t size) {
+  /// Ensure we've allocated enough space
+  RETURN_NOT_OK(converter->Reserve(size));
+  // Iterate over the items adding each one
+  return internal::VisitSequence(values, [converter](PyObject* item, bool* /* unused */) {
+    return converter->Append(item);
+  });
+}
 
-  // Initialize the sequence converter with an ArrayBuilder created
-  // externally. The reason for this interface is that we have
-  // arrow::MakeBuilder which also creates child builders for nested types, so
-  // we have to pass in the child builders to child SeqConverter in the case of
-  // converting Python objects to Arrow nested types
-  virtual Status Init(ArrayBuilder* builder) = 0;
+// Convert and append a sequence of values masked with a numpy array
+template <typename T>
+Status ExtendMasked(T* converter, PyObject* values, PyObject* mask, int64_t size) {
+  /// Ensure we've allocated enough space
+  RETURN_NOT_OK(converter->Reserve(size));
+  // Iterate over the items adding each one
+  return internal::VisitSequenceMasked(
+      values, mask, [converter](PyObject* item, bool is_masked, bool* /* unused */) {
+        if (is_masked) {
+          return converter->AppendNull();
+        } else {
+          // This will also apply the null-checking convention in the event
+          // that the value is not masked
+          return converter->Append(item);  // perhaps use AppendValue instead?
+        }
+      });
+}
 
-  // Append a single null value to the builder
-  virtual Status AppendNull() = 0;
+// The base Converter class is a mixin with predefined behavior and constructors.
+using PyConverter = Converter<PyObject*, PyConversionOptions>;
 
-  // Append a valid value
-  virtual Status AppendValue(PyObject* seq) = 0;
+template <typename T, typename Enable = void>
+class PyPrimitiveConverter;
 
-  // Append a single python object handling Null values
-  virtual Status Append(PyObject* seq) = 0;
+template <typename T>
+class PyListConverter;
 
-  // Append the contents of a Python sequence to the underlying builder,
-  // virtual version
-  virtual Status Extend(PyObject* seq, int64_t size) = 0;
+template <typename U, typename Enable = void>
+class PyDictionaryConverter;
 
-  // Append the contents of a Python sequence to the underlying builder,
-  // virtual version
-  virtual Status ExtendMasked(PyObject* seq, PyObject* mask, int64_t size) = 0;
+class PyStructConverter;
 
-  virtual Status Close() {
-    if (chunks_.size() == 0 || builder_->length() > 0) {
-      std::shared_ptr<Array> last_chunk;
-      RETURN_NOT_OK(builder_->Finish(&last_chunk));
-      chunks_.emplace_back(std::move(last_chunk));
-    }
-    return Status::OK();
-  }
+template <typename T, typename Enable = void>
+struct PyConverterTrait;
 
-  virtual Status GetResult(std::shared_ptr<ChunkedArray>* out) {
-    // Still some accumulated data in the builder. If there are no chunks, we
-    // always call Finish to deal with the edge case where a size-0 sequence
-    // was converted with a specific output type, like array([], type=t)
-    RETURN_NOT_OK(Close());
-    *out = std::make_shared<ChunkedArray>(this->chunks_, builder_->type());
-    return Status::OK();
-  }
+template <typename T>
+struct PyConverterTrait<
+    T, enable_if_t<!is_nested_type<T>::value && !is_interval_type<T>::value &&
+                   !is_extension_type<T>::value>> {
+  using type = PyPrimitiveConverter<T>;
+};
 
-  ArrayBuilder* builder() const { return builder_; }
+template <typename T>
+struct PyConverterTrait<T, enable_if_list_like<T>> {
+  using type = PyListConverter<T>;
+};
 
-  int num_chunks() const { return static_cast<int>(chunks_.size()); }
+template <>
+struct PyConverterTrait<StructType> {
+  using type = PyStructConverter;
+};
 
- protected:
-  ArrayBuilder* builder_;
-  bool unfinished_builder_;
-  std::vector<std::shared_ptr<Array>> chunks_;
+template <>
+struct PyConverterTrait<DictionaryType> {
+  template <typename T>
+  using dictionary_type = PyDictionaryConverter<T>;
 };
 
-template <typename Type, NullCoding null_coding = NullCoding::NONE_ONLY>
-class TypedConverter : public SeqConverter {
+template <typename T>
+class PyPrimitiveConverter<T, enable_if_null<T>>
+    : public PrimitiveConverter<T, PyConverter> {
  public:
-  using BuilderType = typename TypeTraits<Type>::BuilderType;
-
-  Status Init(ArrayBuilder* builder) override {
-    builder_ = builder;
-    DCHECK_NE(builder_, nullptr);
-    typed_builder_ = checked_cast<BuilderType*>(builder);
-    return Status::OK();
-  }
-
-  // Append a missing item (default implementation)
-  Status AppendNull() override { return this->typed_builder_->AppendNull(); }
-
-  // Append null if the obj is None or pandas null otherwise the valid value
-  Status Append(PyObject* obj) override {
-    return NullChecker<null_coding>::Check(obj) ? AppendNull() : AppendValue(obj);
-  }
-
-  Status Extend(PyObject* obj, int64_t size) override {
-    /// Ensure we've allocated enough space
-    RETURN_NOT_OK(typed_builder_->Reserve(size));
-    // Iterate over the items adding each one
-    return internal::VisitSequence(
-        obj, [this](PyObject* item, bool* /* unused */) { return this->Append(item); });
-  }
-
-  Status ExtendMasked(PyObject* obj, PyObject* mask, int64_t size) override {
-    /// Ensure we've allocated enough space
-    RETURN_NOT_OK(typed_builder_->Reserve(size));
-    // Iterate over the items adding each one
-    return internal::VisitSequenceMasked(
-        obj, mask, [this](PyObject* item, bool is_masked, bool* /* unused */) {
-          if (is_masked) {
-            return this->AppendNull();
-          } else {
-            // This will also apply the null-checking convention in the event
-            // that the value is not masked
-            return this->Append(item);  // perhaps use AppendValue instead?
-          }
-        });
+  Status Append(PyObject* value) override {
+    if (PyValue::IsNull(this->options_, value)) {
+      return this->primitive_builder_->AppendNull();
+    } else {
+      ARROW_ASSIGN_OR_RAISE(
+          auto converted, PyValue::Convert(this->primitive_type_, this->options_, value));
+      return this->primitive_builder_->Append(converted);
+    }
   }
-
- protected:
-  BuilderType* typed_builder_;
 };
 
-// ----------------------------------------------------------------------
-// Sequence converter for null type
-
-template <NullCoding null_coding>
-class NullConverter : public TypedConverter<NullType, null_coding> {
+template <typename T>
+class PyPrimitiveConverter<
+    T, enable_if_t<is_boolean_type<T>::value || is_number_type<T>::value ||
+                   is_decimal_type<T>::value || is_date_type<T>::value ||
+                   is_time_type<T>::value>> : public PrimitiveConverter<T, PyConverter> {
  public:
-  Status AppendValue(PyObject* obj) override {
-    return internal::InvalidValue(obj, "converting to null type");
+  Status Append(PyObject* value) override {
+    // Since the required space has been already allocated in the Extend functions we can
+    // rely on the Unsafe builder API which improves the performance.
+    if (PyValue::IsNull(this->options_, value)) {
+      this->primitive_builder_->UnsafeAppendNull();
+    } else {
+      ARROW_ASSIGN_OR_RAISE(
+          auto converted, PyValue::Convert(this->primitive_type_, this->options_, value));
+      this->primitive_builder_->UnsafeAppend(converted);
+    }
+    return Status::OK();
   }
 };
 
-// ----------------------------------------------------------------------
-// Sequence converter template for primitive (integer and floating point bool) types
-
-template <typename Type, NullCoding null_coding>
-class PrimitiveConverter : public TypedConverter<Type, null_coding> {
-  Status AppendValue(PyObject* obj) override {
-    ARROW_ASSIGN_OR_RAISE(auto value, ValueConverter<Type>::FromPython(obj));
-    return this->typed_builder_->Append(value);
+template <typename T>
+class PyPrimitiveConverter<
+    T, enable_if_t<is_timestamp_type<T>::value || is_duration_type<T>::value>>
+    : public PrimitiveConverter<T, PyConverter> {
+ public:
+  Status Append(PyObject* value) override {
+    if (PyValue::IsNull(this->options_, value)) {
+      this->primitive_builder_->UnsafeAppendNull();
+    } else {
+      ARROW_ASSIGN_OR_RAISE(
+          auto converted, PyValue::Convert(this->primitive_type_, this->options_, value));
+      // Numpy NaT sentinels can be checked after the conversion
+      if (PyArray_CheckAnyScalarExact(value) &&
+          PyValue::IsNaT(this->primitive_type_, converted)) {
+        this->primitive_builder_->UnsafeAppendNull();
+      } else {
+        this->primitive_builder_->UnsafeAppend(converted);
+      }
+    }
+    return Status::OK();
   }
 };
 
-// ----------------------------------------------------------------------
-// Sequence converters for temporal types
-
-template <typename Type, NullCoding null_coding>
-class TimeConverter : public TypedConverter<Type, null_coding> {
+template <typename T>
+class PyPrimitiveConverter<T, enable_if_binary<T>>
+    : public PrimitiveConverter<T, PyConverter> {
  public:
-  explicit TimeConverter(TimeUnit::type unit) : unit_(unit) {}
+  using OffsetType = typename T::offset_type;
 
-  // TODO(kszucs): support numpy values for date and time converters
-  Status AppendValue(PyObject* obj) override {
-    ARROW_ASSIGN_OR_RAISE(auto value, ValueConverter<Type>::FromPython(obj, unit_));
-    return this->typed_builder_->Append(value);
+  Status Append(PyObject* value) override {
+    if (PyValue::IsNull(this->options_, value)) {
+      this->primitive_builder_->UnsafeAppendNull();
+    } else {
+      ARROW_RETURN_NOT_OK(
+          PyValue::Convert(this->primitive_type_, this->options_, value, view_));
+      // Since we don't know the varying length input size in advance, we need to
+      // reserve space in the value builder one by one. ReserveData raises CapacityError
+      // if the value would not fit into the array.
+      ARROW_RETURN_NOT_OK(this->primitive_builder_->ReserveData(view_.size));
+      this->primitive_builder_->UnsafeAppend(view_.bytes,
+                                             static_cast<OffsetType>(view_.size));
+    }
+    return Status::OK();
   }
 
  protected:
-  TimeUnit::type unit_;
+  // Create a single instance of PyBytesView here to prevent unnecessary object
+  // creation/destruction. This significantly improves the conversion performance.
+  PyBytesView view_;
 };
 
-// TODO(kszucs): move it to the type_traits
 template <typename T>
-struct NumpyType {};
-
-template <>
-struct NumpyType<TimestampType> {
-  static inline bool isnull(int64_t v) {
-    return internal::npy_traits<NPY_DATETIME>::isnull(v);
-  }
-};
-
-template <>
-struct NumpyType<DurationType> {
-  static inline bool isnull(int64_t v) {
-    return internal::npy_traits<NPY_TIMEDELTA>::isnull(v);
-  }
-};
-
-template <typename Type, NullCoding null_coding>
-class TemporalConverter : public TimeConverter<Type, null_coding> {
+class PyPrimitiveConverter<T, enable_if_t<std::is_same<T, FixedSizeBinaryType>::value>>
+    : public PrimitiveConverter<T, PyConverter> {
  public:
-  using TimeConverter<Type, null_coding>::TimeConverter;
-
-  Status AppendValue(PyObject* obj) override {
-    int64_t value;
-    if (PyArray_CheckAnyScalarExact(obj)) {
-      // convert np.datetime64 / np.timedelta64 depending on Type
-      ARROW_ASSIGN_OR_RAISE(value, ValueConverter<Type>::FromNumpy(obj, this->unit_));
-      if (NumpyType<Type>::isnull(value)) {
-        // checks numpy NaT sentinel after conversion
-        return this->typed_builder_->AppendNull();
-      }
+  Status Append(PyObject* value) override {
+    if (PyValue::IsNull(this->options_, value)) {
+      this->primitive_builder_->UnsafeAppendNull();
     } else {
-      // convert builtin python objects
-      ARROW_ASSIGN_OR_RAISE(value, ValueConverter<Type>::FromPython(obj, this->unit_));
+      ARROW_RETURN_NOT_OK(
+          PyValue::Convert(this->primitive_type_, this->options_, value, view_));
+      ARROW_RETURN_NOT_OK(this->primitive_builder_->ReserveData(view_.size));
+      this->primitive_builder_->UnsafeAppend(view_.bytes);
     }
-    return this->typed_builder_->Append(value);
+    return Status::OK();
   }
-};
 
-// ----------------------------------------------------------------------
-// Sequence converters for Binary, FixedSizeBinary, String
+ protected:
+  PyBytesView view_;
+};
 
-template <typename Type, NullCoding null_coding>
-class BinaryLikeConverter : public TypedConverter<Type, null_coding> {
+template <typename T>
+class PyPrimitiveConverter<T, enable_if_string_like<T>>
+    : public PrimitiveConverter<T, PyConverter> {
  public:
-  using BuilderType = typename TypeTraits<Type>::BuilderType;
-
-  inline Status AutoChunk(Py_ssize_t size) {
-    // did we reach the builder size limit?
-    if (ARROW_PREDICT_FALSE(this->typed_builder_->value_data_length() + size >
-                            BuilderType::memory_limit())) {
-      // builder would be full, so need to add a new chunk
-      std::shared_ptr<Array> chunk;
-      RETURN_NOT_OK(this->typed_builder_->Finish(&chunk));
-      this->chunks_.emplace_back(std::move(chunk));
+  using OffsetType = typename T::offset_type;
+
+  Status Append(PyObject* value) override {
+    if (PyValue::IsNull(this->options_, value)) {
+      this->primitive_builder_->UnsafeAppendNull();
+    } else {
+      ARROW_RETURN_NOT_OK(
+          PyValue::Convert(this->primitive_type_, this->options_, value, view_));
+      if (!view_.is_utf8) {
+        // observed binary value
+        observed_binary_ = true;
+      }
+      ARROW_RETURN_NOT_OK(this->primitive_builder_->ReserveData(view_.size));
+      this->primitive_builder_->UnsafeAppend(view_.bytes,
+                                             static_cast<OffsetType>(view_.size));
     }
     return Status::OK();
   }
 
-  Status AppendString(const PyBytesView& view) {
-    // check that the value fits in the datatype
-    if (view.size > BuilderType::memory_limit()) {
-      return Status::Invalid("string too large for datatype");
+  Result<std::shared_ptr<Array>> ToArray() override {
+    ARROW_ASSIGN_OR_RAISE(auto array, (PrimitiveConverter<T, PyConverter>::ToArray()));
+    if (observed_binary_) {
+      // if we saw any non-unicode, cast results to BinaryArray
+      auto binary_type = TypeTraits<typename T::PhysicalType>::type_singleton();
+      return array->View(binary_type);
+    } else {
+      return array;
     }
-    DCHECK_GE(view.size, 0);
-
-    // create a new chunk if the value would overflow the builder
-    RETURN_NOT_OK(AutoChunk(view.size));
-
-    // now we can safely append the value to the builder
-    RETURN_NOT_OK(
-        this->typed_builder_->Append(::arrow::util::string_view(view.bytes, view.size)));
-
-    return Status::OK();
   }
 
  protected:
-  // Create a single instance of PyBytesView here to prevent unnecessary object
-  // creation/destruction
-  PyBytesView string_view_;
+  PyBytesView view_;
+  bool observed_binary_ = false;
 };
 
-template <typename Type, NullCoding null_coding>
-class BinaryConverter : public BinaryLikeConverter<Type, null_coding> {
+template <typename U>
+class PyDictionaryConverter<U, enable_if_has_c_type<U>>
+    : public DictionaryConverter<U, PyConverter> {
  public:
-  Status AppendValue(PyObject* obj) override {
-    ARROW_ASSIGN_OR_RAISE(auto view, ValueConverter<Type>::FromPython(obj));
-    return this->AppendString(view);
+  Status Append(PyObject* value) override {
+    if (PyValue::IsNull(this->options_, value)) {
+      return this->value_builder_->AppendNull();
+    } else {
+      ARROW_ASSIGN_OR_RAISE(auto converted,
+                            PyValue::Convert(this->value_type_, this->options_, value));
+      return this->value_builder_->Append(converted);
+    }
   }
 };
 
-template <NullCoding null_coding>
-class FixedSizeBinaryConverter
-    : public BinaryLikeConverter<FixedSizeBinaryType, null_coding> {
+template <typename U>
+class PyDictionaryConverter<U, enable_if_has_string_view<U>>
+    : public DictionaryConverter<U, PyConverter> {
  public:
-  explicit FixedSizeBinaryConverter(int32_t byte_width) : byte_width_(byte_width) {}
-
-  Status AppendValue(PyObject* obj) override {
-    ARROW_ASSIGN_OR_RAISE(
-        this->string_view_,
-        ValueConverter<FixedSizeBinaryType>::FromPython(obj, byte_width_));
-    return this->AppendString(this->string_view_);
+  Status Append(PyObject* value) override {
+    if (PyValue::IsNull(this->options_, value)) {
+      return this->value_builder_->AppendNull();
+    } else {
+      ARROW_RETURN_NOT_OK(
+          PyValue::Convert(this->value_type_, this->options_, value, view_));
+      return this->value_builder_->Append(view_.bytes, static_cast<int32_t>(view_.size));
+    }
   }
 
  protected:
-  int32_t byte_width_;
+  PyBytesView view_;
 };
 
-// For String/UTF8, if strict_conversions enabled, we reject any non-UTF8,
-// otherwise we allow but return results as BinaryArray
-template <typename Type, bool STRICT, NullCoding null_coding>
-class StringConverter : public BinaryLikeConverter<Type, null_coding> {
+template <typename T>
+class PyListConverter : public ListConverter<T, PyConverter, PyConverterTrait> {
  public:
-  StringConverter() : binary_count_(0) {}
+  Status Append(PyObject* value) override {
+    if (PyValue::IsNull(this->options_, value)) {
+      return this->list_builder_->AppendNull();
+    }
 
-  Status AppendValue(PyObject* obj) override {
-    if (STRICT) {
-      // raise if the object is not unicode or not an utf-8 encoded bytes
-      ARROW_ASSIGN_OR_RAISE(this->string_view_, ValueConverter<Type>::FromPython(obj));
+    RETURN_NOT_OK(this->list_builder_->Append());
+    if (PyArray_Check(value)) {
+      RETURN_NOT_OK(AppendNdarray(value));
+    } else if (PySequence_Check(value)) {
+      RETURN_NOT_OK(AppendSequence(value));
     } else {
-      // keep track of whether values are unicode or bytes; if any bytes are
-      // observe, the result will be bytes
-      bool is_utf8;
-      ARROW_ASSIGN_OR_RAISE(this->string_view_,
-                            ValueConverter<Type>::FromPython(obj, &is_utf8));
-      if (!is_utf8) {
-        ++binary_count_;
-      }
+      return internal::InvalidType(
+          value, "was not a sequence or recognized null for conversion to list type");
     }
-    return this->AppendString(this->string_view_);
-  }
-
-  Status GetResult(std::shared_ptr<ChunkedArray>* out) override {
-    RETURN_NOT_OK(SeqConverter::GetResult(out));
 
-    // If we saw any non-unicode, cast results to BinaryArray
-    if (binary_count_) {
-      // We should have bailed out earlier
-      DCHECK(!STRICT);
-      auto binary_type = TypeTraits<typename Type::PhysicalType>::type_singleton();
-      return (*out)->View(binary_type).Value(out);
-    }
-    return Status::OK();
+    return ValidateBuilder(this->list_type_);
   }
 
  protected:
-  int64_t binary_count_;
-};
-
-// ----------------------------------------------------------------------
-// Convert lists (NumPy arrays containing lists or ndarrays as values)
-
-// If the value type does not match the expected NumPy dtype, then fall through
-// to a slower PySequence-based path
-#define LIST_FAST_CASE(TYPE, NUMPY_TYPE, ArrowType)            \
-  case Type::TYPE: {                                           \
-    if (PyArray_DESCR(arr)->type_num != NUMPY_TYPE) {          \
-      return value_converter_->Extend(obj, value_length);      \
-    }                                                          \
-    return AppendNdarrayTypedItem<NUMPY_TYPE, ArrowType>(arr); \
-  }
-
-// Use internal::VisitSequence, fast for NPY_OBJECT but slower otherwise
-#define LIST_SLOW_CASE(TYPE)                            \
-  case Type::TYPE: {                                    \
-    return value_converter_->Extend(obj, value_length); \
+  Status ValidateBuilder(const MapType*) {
+    if (this->list_builder_->key_builder()->null_count() > 0) {
+      return Status::Invalid("Invalid Map: key field can not contain null values");
+    } else {
+      return Status::OK();
+    }
   }
 
-// Base class for ListConverter and FixedSizeListConverter (to have both work with CRTP)
-template <typename TypeClass, NullCoding null_coding>
-class BaseListConverter : public TypedConverter<TypeClass, null_coding> {
- public:
-  using BuilderType = typename TypeTraits<TypeClass>::BuilderType;
+  Status ValidateBuilder(const BaseListType*) { return Status::OK(); }
 
-  explicit BaseListConverter(bool from_pandas, bool strict_conversions)
-      : from_pandas_(from_pandas), strict_conversions_(strict_conversions) {}
+  Status AppendSequence(PyObject* value) {
+    int64_t size = static_cast<int64_t>(PySequence_Size(value));
+    RETURN_NOT_OK(this->list_builder_->ValidateOverflow(size));
+    return Extend(this->value_converter_.get(), value, size);
+  }
 
-  Status Init(ArrayBuilder* builder) override {
-    this->builder_ = builder;
-    this->typed_builder_ = checked_cast<BuilderType*>(builder);
+  Status AppendNdarray(PyObject* value) {
+    PyArrayObject* ndarray = reinterpret_cast<PyArrayObject*>(value);
+    if (PyArray_NDIM(ndarray) != 1) {
+      return Status::Invalid("Can only convert 1-dimensional array values");
+    }
+    const int64_t size = PyArray_SIZE(ndarray);
+    RETURN_NOT_OK(this->list_builder_->ValidateOverflow(size));
 
-    this->value_type_ = checked_cast<const TypeClass&>(*builder->type()).value_type();
-    RETURN_NOT_OK(
-        GetConverter(value_type_, from_pandas_, strict_conversions_, &value_converter_));
-    return this->value_converter_->Init(this->typed_builder_->value_builder());
+    const auto value_type = this->value_converter_->builder()->type();
+    switch (value_type->id()) {
+// If the value type does not match the expected NumPy dtype, then fall through
+// to a slower PySequence-based path
+#define LIST_FAST_CASE(TYPE_ID, TYPE, NUMPY_TYPE)               \
+  case Type::TYPE_ID: {                                         \
+    if (PyArray_DESCR(ndarray)->type_num != NUMPY_TYPE) {       \
+      return Extend(this->value_converter_.get(), value, size); \
+    }                                                           \
+    return AppendNdarrayTyped<TYPE, NUMPY_TYPE>(ndarray);       \
+  }
+      LIST_FAST_CASE(BOOL, BooleanType, NPY_BOOL)
+      LIST_FAST_CASE(UINT8, UInt8Type, NPY_UINT8)
+      LIST_FAST_CASE(INT8, Int8Type, NPY_INT8)
+      LIST_FAST_CASE(UINT16, UInt16Type, NPY_UINT16)
+      LIST_FAST_CASE(INT16, Int16Type, NPY_INT16)
+      LIST_FAST_CASE(UINT32, UInt32Type, NPY_UINT32)
+      LIST_FAST_CASE(INT32, Int32Type, NPY_INT32)
+      LIST_FAST_CASE(UINT64, UInt64Type, NPY_UINT64)
+      LIST_FAST_CASE(INT64, Int64Type, NPY_INT64)
+      LIST_FAST_CASE(HALF_FLOAT, HalfFloatType, NPY_FLOAT16)
+      LIST_FAST_CASE(FLOAT, FloatType, NPY_FLOAT)
+      LIST_FAST_CASE(DOUBLE, DoubleType, NPY_DOUBLE)
+      LIST_FAST_CASE(TIMESTAMP, TimestampType, NPY_DATETIME)
+      LIST_FAST_CASE(DURATION, DurationType, NPY_TIMEDELTA)
+#undef LIST_FAST_CASE
+      default: {
+        return Extend(this->value_converter_.get(), value, size);
+      }
+    }
   }
 
-  template <int NUMPY_TYPE, typename Type>
-  Status AppendNdarrayTypedItem(PyArrayObject* arr) {
-    using traits = internal::npy_traits<NUMPY_TYPE>;
-    using T = typename traits::value_type;
-    using ValueBuilderType = typename TypeTraits<Type>::BuilderType;
+  template <typename ArrowType, int NUMPY_TYPE>
+  Status AppendNdarrayTyped(PyArrayObject* ndarray) {
+    // no need to go through the conversion
+    using NumpyTrait = internal::npy_traits<NUMPY_TYPE>;
+    using NumpyType = typename NumpyTrait::value_type;
+    using ValueBuilderType = typename TypeTraits<ArrowType>::BuilderType;
 
     const bool null_sentinels_possible =
         // Always treat Numpy's NaT as null
         NUMPY_TYPE == NPY_DATETIME || NUMPY_TYPE == NPY_TIMEDELTA ||
         // Observing pandas's null sentinels
-        (from_pandas_ && traits::supports_nulls);
+        (this->options_.from_pandas && NumpyTrait::supports_nulls);
 
-    auto child_builder = checked_cast<ValueBuilderType*>(value_converter_->builder());
+    auto value_builder =
+        checked_cast<ValueBuilderType*>(this->value_converter_->builder().get());
 
     // TODO(wesm): Vector append when not strided
-    Ndarray1DIndexer<T> values(arr);
+    Ndarray1DIndexer<NumpyType> values(ndarray);
     if (null_sentinels_possible) {
       for (int64_t i = 0; i < values.size(); ++i) {
-        if (traits::isnull(values[i])) {
-          RETURN_NOT_OK(child_builder->AppendNull());
+        if (NumpyTrait::isnull(values[i])) {
+          RETURN_NOT_OK(value_builder->AppendNull());
         } else {
-          RETURN_NOT_OK(child_builder->Append(values[i]));
+          RETURN_NOT_OK(value_builder->Append(values[i]));
         }
       }
     } else {
       for (int64_t i = 0; i < values.size(); ++i) {
-        RETURN_NOT_OK(child_builder->Append(values[i]));
+        RETURN_NOT_OK(value_builder->Append(values[i]));
       }
     }
     return Status::OK();
   }
-
-  Status AppendNdarrayItem(PyObject* obj) {
-    PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(obj);
-
-    if (PyArray_NDIM(arr) != 1) {
-      return Status::Invalid("Can only convert 1-dimensional array values");
-    }
-
-    const int64_t value_length = PyArray_SIZE(arr);
-
-    switch (value_type_->id()) {
-      LIST_SLOW_CASE(NA)
-      LIST_FAST_CASE(UINT8, NPY_UINT8, UInt8Type)
-      LIST_FAST_CASE(INT8, NPY_INT8, Int8Type)
-      LIST_FAST_CASE(UINT16, NPY_UINT16, UInt16Type)
-      LIST_FAST_CASE(INT16, NPY_INT16, Int16Type)
-      LIST_FAST_CASE(UINT32, NPY_UINT32, UInt32Type)
-      LIST_FAST_CASE(INT32, NPY_INT32, Int32Type)
-      LIST_FAST_CASE(UINT64, NPY_UINT64, UInt64Type)
-      LIST_FAST_CASE(INT64, NPY_INT64, Int64Type)
-      LIST_SLOW_CASE(DATE32)
-      LIST_SLOW_CASE(DATE64)
-      LIST_SLOW_CASE(TIME32)
-      LIST_SLOW_CASE(TIME64)
-      LIST_FAST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
-      LIST_FAST_CASE(DURATION, NPY_TIMEDELTA, DurationType)
-      LIST_FAST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
-      LIST_FAST_CASE(FLOAT, NPY_FLOAT, FloatType)
-      LIST_FAST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
-      LIST_SLOW_CASE(BINARY)
-      LIST_SLOW_CASE(FIXED_SIZE_BINARY)
-      LIST_SLOW_CASE(STRING)
-      case Type::LIST: {
-        if (PyArray_DESCR(arr)->type_num != NPY_OBJECT) {
-          return Status::Invalid(
-              "Can only convert list types from NumPy object "
-              "array input");
-        }
-        return internal::VisitSequence(obj, [this](PyObject* item, bool*) {
-          return value_converter_->Append(item);
-        });
-      }
-      default: {
-        return Status::TypeError("Unknown list item type: ", value_type_->ToString());
-      }
-    }
-  }
-
-  Status AppendValue(PyObject* obj) override {
-    RETURN_NOT_OK(this->typed_builder_->Append());
-    if (PyArray_Check(obj)) {
-      return AppendNdarrayItem(obj);
-    }
-    if (!PySequence_Check(obj)) {
-      return internal::InvalidType(obj,
-                                   "was not a sequence or recognized null"
-                                   " for conversion to list type");
-    }
-    int64_t list_size = static_cast<int64_t>(PySequence_Size(obj));
-    return value_converter_->Extend(obj, list_size);
-  }
-
-  Status GetResult(std::shared_ptr<ChunkedArray>* out) override {
-    // TODO: Improved handling of chunked children
-    if (value_converter_->num_chunks() > 0) {
-      return Status::Invalid("List child type ",
-                             value_converter_->builder()->type()->ToString(),
-                             " overflowed the capacity of a single chunk");
-    }
-    return SeqConverter::GetResult(out);
-  }
-
- protected:
-  std::shared_ptr<DataType> value_type_;
-  std::unique_ptr<SeqConverter> value_converter_;
-  bool from_pandas_;
-  bool strict_conversions_;
-};
-
-template <typename TypeClass, NullCoding null_coding>
-class ListConverter : public BaseListConverter<TypeClass, null_coding> {
- public:
-  using BASE = BaseListConverter<TypeClass, null_coding>;
-  using BASE::BASE;
 };
 
-template <NullCoding null_coding>
-class FixedSizeListConverter : public BaseListConverter<FixedSizeListType, null_coding> {
+class PyStructConverter : public StructConverter<PyConverter, PyConverterTrait> {
  public:
-  using BASE = BaseListConverter<FixedSizeListType, null_coding>;
-  using BASE::BASE;
-
-  Status Init(ArrayBuilder* builder) override {
-    RETURN_NOT_OK(BASE::Init(builder));
-    list_size_ = checked_pointer_cast<FixedSizeListType>(builder->type())->list_size();
-    return Status::OK();
-  }
-
-  Status AppendValue(PyObject* obj) override {
-    // the same as BaseListConverter but with additional length checks
-    RETURN_NOT_OK(this->typed_builder_->Append());
-    if (PyArray_Check(obj)) {
-      int64_t list_size = static_cast<int64_t>(PyArray_Size(obj));
-      if (list_size != list_size_) {
-        return Status::Invalid("Length of item not correct: expected ", list_size_,
-                               " but got array of size ", list_size);
-      }
-      return this->AppendNdarrayItem(obj);
-    }
-    if (!PySequence_Check(obj)) {
-      return internal::InvalidType(obj,
-                                   "was not a sequence or recognized null"
-                                   " for conversion to list type");
+  Status Append(PyObject* value) override {
+    if (PyValue::IsNull(this->options_, value)) {
+      return this->struct_builder_->AppendNull();
     }
-    int64_t list_size = static_cast<int64_t>(PySequence_Size(obj));
-    if (list_size != list_size_) {
-      return Status::Invalid("Length of item not correct: expected ", list_size_,
-                             " but got list of size ", list_size);
+    switch (input_kind_) {
+      case InputKind::DICT:
+        RETURN_NOT_OK(this->struct_builder_->Append());
+        return AppendDict(value);
+      case InputKind::TUPLE:
+        RETURN_NOT_OK(this->struct_builder_->Append());
+        return AppendTuple(value);
+      case InputKind::ITEMS:
+        RETURN_NOT_OK(this->struct_builder_->Append());
+        return AppendItems(value);
+      default:
+        RETURN_NOT_OK(InferInputKind(value));
+        return Append(value);
     }
-    return this->value_converter_->Extend(obj, list_size);
   }
 
  protected:
-  int64_t list_size_;
-};
-
-// ----------------------------------------------------------------------
-// Convert maps
-
-// Define a MapConverter as a ListConverter that uses MapBuilder.value_builder
-// to append struct of key/value pairs
-template <NullCoding null_coding>
-class MapConverter : public BaseListConverter<MapType, null_coding> {
- public:
-  using BASE = BaseListConverter<MapType, null_coding>;
-
-  explicit MapConverter(bool from_pandas, bool strict_conversions)
-      : BASE(from_pandas, strict_conversions), key_builder_(nullptr) {}
+  Status Init(MemoryPool* pool) override {
+    RETURN_NOT_OK((StructConverter<PyConverter, PyConverterTrait>::Init(pool)));
 
-  Status Append(PyObject* obj) override {
-    RETURN_NOT_OK(BASE::Append(obj));
-    return VerifyLastStructAppended();
-  }
-
-  Status Extend(PyObject* seq, int64_t size) override {
-    RETURN_NOT_OK(BASE::Extend(seq, size));
-    return VerifyLastStructAppended();
-  }
+    // Store the field names as a PyObjects for dict matching
+    num_fields_ = this->struct_type_->num_fields();
+    bytes_field_names_.reset(PyList_New(num_fields_));
+    unicode_field_names_.reset(PyList_New(num_fields_));
+    RETURN_IF_PYERROR();
 
-  Status ExtendMasked(PyObject* seq, PyObject* mask, int64_t size) override {
-    RETURN_NOT_OK(BASE::ExtendMasked(seq, mask, size));
-    return VerifyLastStructAppended();
+    for (int i = 0; i < num_fields_; i++) {
+      const auto& field_name = this->struct_type_->field(i)->name();
+      PyObject* bytes = PyBytes_FromStringAndSize(field_name.c_str(), field_name.size());
+      PyObject* unicode =
+          PyUnicode_FromStringAndSize(field_name.c_str(), field_name.size());
+      RETURN_IF_PYERROR();
+      PyList_SET_ITEM(bytes_field_names_.obj(), i, bytes);
+      PyList_SET_ITEM(unicode_field_names_.obj(), i, unicode);
+    }
+    return Status::OK();
   }
 
- protected:
-  Status VerifyLastStructAppended() {
-    // The struct_builder may not have field_builders initialized in constructor, so
-    // assign key_builder lazily
-    if (key_builder_ == nullptr) {
-      auto struct_builder =
-          checked_cast<StructBuilder*>(BASE::value_converter_->builder());
-      key_builder_ = struct_builder->field_builder(0);
-    }
-    if (key_builder_->null_count() > 0) {
-      return Status::Invalid("Invalid Map: key field can not contain null values");
+  Status InferInputKind(PyObject* value) {
+    // Infer input object's type, note that heterogeneous sequences are not allowed
+    if (PyDict_Check(value)) {
+      input_kind_ = InputKind::DICT;
+    } else if (PyTuple_Check(value)) {
+      input_kind_ = InputKind::TUPLE;
+    } else if (PySequence_Check(value)) {
+      input_kind_ = InputKind::ITEMS;
+    } else {
+      return internal::InvalidType(value,
+                                   "was not a dict, tuple, or recognized null value "
+                                   "for conversion to struct type");
     }
     return Status::OK();
   }
 
- private:
-  ArrayBuilder* key_builder_;
-};
-
-// ----------------------------------------------------------------------
-// Convert structs
-
-template <NullCoding null_coding>
-class StructConverter : public TypedConverter<StructType, null_coding> {
- public:
-  explicit StructConverter(bool from_pandas, bool strict_conversions)
-      : from_pandas_(from_pandas), strict_conversions_(strict_conversions) {}
+  Status InferKeyKind(PyObject* items) {
+    for (int i = 0; i < PySequence_Length(items); i++) {
+      // retrieve the key from the passed key-value pairs
+      ARROW_ASSIGN_OR_RAISE(auto pair, GetKeyValuePair(items, i));
 
-  Status Init(ArrayBuilder* builder) override {
-    this->builder_ = builder;
-    this->typed_builder_ = checked_cast<StructBuilder*>(builder);
-    auto struct_type = checked_pointer_cast<StructType>(builder->type());
-
-    num_fields_ = this->typed_builder_->num_fields();
-    DCHECK_EQ(num_fields_, struct_type->num_fields());
-
-    field_name_bytes_list_.reset(PyList_New(num_fields_));
-    field_name_unicode_list_.reset(PyList_New(num_fields_));
-    RETURN_IF_PYERROR();
-
-    // Initialize the child converters and field names
-    for (int i = 0; i < num_fields_; i++) {
-      const std::string& field_name(struct_type->field(i)->name());
-      std::shared_ptr<DataType> field_type(struct_type->field(i)->type());
+      // check key exists between the unicode field names
+      bool do_contain = PySequence_Contains(unicode_field_names_.obj(), pair.first);
+      RETURN_IF_PYERROR();
+      if (do_contain) {
+        key_kind_ = KeyKind::UNICODE;
+        return Status::OK();
+      }
 
-      std::unique_ptr<SeqConverter> value_converter;
-      RETURN_NOT_OK(
-          GetConverter(field_type, from_pandas_, strict_conversions_, &value_converter));
-      RETURN_NOT_OK(value_converter->Init(this->typed_builder_->field_builder(i)));
-      value_converters_.push_back(std::move(value_converter));
-
-      // Store the field name as a PyObject, for dict matching
-      PyObject* bytesobj =
-          PyBytes_FromStringAndSize(field_name.c_str(), field_name.size());
-      PyObject* unicodeobj =
-          PyUnicode_FromStringAndSize(field_name.c_str(), field_name.size());
+      // check key exists between the bytes field names
+      do_contain = PySequence_Contains(bytes_field_names_.obj(), pair.first);
       RETURN_IF_PYERROR();
-      PyList_SET_ITEM(field_name_bytes_list_.obj(), i, bytesobj);
-      PyList_SET_ITEM(field_name_unicode_list_.obj(), i, unicodeobj);
+      if (do_contain) {
+        key_kind_ = KeyKind::BYTES;
+        return Status::OK();
+      }
     }
-
     return Status::OK();
   }
 
-  Status AppendValue(PyObject* obj) override {
-    RETURN_NOT_OK(this->typed_builder_->Append());
-    // Note heterogeneous sequences are not allowed
-    if (ARROW_PREDICT_FALSE(source_kind_ == SourceKind::UNKNOWN)) {
-      if (PyDict_Check(obj)) {
-        source_kind_ = SourceKind::DICTS;
-      } else if (PyTuple_Check(obj)) {
-        source_kind_ = SourceKind::TUPLES;
-      }
-    }
-    if (PyDict_Check(obj) && source_kind_ == SourceKind::DICTS) {
-      return AppendDictItem(obj);
-    } else if (PyTuple_Check(obj) && source_kind_ == SourceKind::TUPLES) {
-      return AppendTupleItem(obj);
-    } else {
-      return internal::InvalidType(obj,
-                                   "was not a dict, tuple, or recognized null value"
-                                   " for conversion to struct type");
+  Status AppendEmpty() {
+    for (int i = 0; i < num_fields_; i++) {
+      RETURN_NOT_OK(this->children_[i]->Append(Py_None));
     }
+    return Status::OK();
   }
 
-  // Append a missing item
-  Status AppendNull() override { return this->typed_builder_->AppendNull(); }
-
- protected:
-  Status AppendDictItem(PyObject* obj) {
-    if (dict_key_kind_ == DictKeyKind::UNICODE) {
-      return AppendDictItemWithUnicodeKeys(obj);
-    }
-    if (dict_key_kind_ == DictKeyKind::BYTES) {
-      return AppendDictItemWithBytesKeys(obj);
+  Status AppendTuple(PyObject* tuple) {
+    if (!PyTuple_Check(tuple)) {
+      return internal::InvalidType(tuple, "was expecting a tuple");
     }
-    for (int i = 0; i < num_fields_; i++) {
-      PyObject* nameobj = PyList_GET_ITEM(field_name_unicode_list_.obj(), i);
-      PyObject* valueobj = PyDict_GetItem(obj, nameobj);
-      if (valueobj != NULL) {
-        dict_key_kind_ = DictKeyKind::UNICODE;
-        return AppendDictItemWithUnicodeKeys(obj);
-      }
-      RETURN_IF_PYERROR();
-      // Unicode key not present, perhaps bytes key is?
-      nameobj = PyList_GET_ITEM(field_name_bytes_list_.obj(), i);
-      valueobj = PyDict_GetItem(obj, nameobj);
-      if (valueobj != NULL) {
-        dict_key_kind_ = DictKeyKind::BYTES;
-        return AppendDictItemWithBytesKeys(obj);
-      }
-      RETURN_IF_PYERROR();
+    if (PyTuple_GET_SIZE(tuple) != num_fields_) {
+      return Status::Invalid("Tuple size must be equal to number of struct fields");
     }
-    // If we come here, it means all keys are absent
     for (int i = 0; i < num_fields_; i++) {
-      RETURN_NOT_OK(value_converters_[i]->Append(Py_None));
+      PyObject* value = PyTuple_GET_ITEM(tuple, i);
+      RETURN_NOT_OK(this->children_[i]->Append(value));
     }
     return Status::OK();
   }
 
-  Status AppendDictItemWithBytesKeys(PyObject* obj) {
-    return AppendDictItem(obj, field_name_bytes_list_.obj());
+  Status AppendDict(PyObject* dict) {
+    if (!PyDict_Check(dict)) {
+      return internal::InvalidType(dict, "was expecting a dict");
+    }
+    switch (key_kind_) {
+      case KeyKind::UNICODE:
+        return AppendDict(dict, unicode_field_names_.obj());
+      case KeyKind::BYTES:
+        return AppendDict(dict, bytes_field_names_.obj());
+      default:
+        RETURN_NOT_OK(InferKeyKind(PyDict_Items(dict)));
+        if (key_kind_ == KeyKind::UNKNOWN) {
+          // was unable to infer the type which means that all keys are absent
+          return AppendEmpty();
+        } else {
+          return AppendDict(dict);
+        }
+    }
   }
 
-  Status AppendDictItemWithUnicodeKeys(PyObject* obj) {
-    return AppendDictItem(obj, field_name_unicode_list_.obj());
+  Status AppendItems(PyObject* items) {
+    if (!PySequence_Check(items)) {
+      return internal::InvalidType(items, "was expecting a sequence of key-value items");
+    }
+    switch (key_kind_) {
+      case KeyKind::UNICODE:
+        return AppendItems(items, unicode_field_names_.obj());
+      case KeyKind::BYTES:
+        return AppendItems(items, bytes_field_names_.obj());
+      default:
+        RETURN_NOT_OK(InferKeyKind(items));
+        if (key_kind_ == KeyKind::UNKNOWN) {
+          // was unable to infer the type which means that all keys are absent
+          return AppendEmpty();
+        } else {
+          return AppendItems(items);
+        }
+    }
   }
 
-  Status AppendDictItem(PyObject* obj, PyObject* field_name_list) {
+  Status AppendDict(PyObject* dict, PyObject* field_names) {
     // NOTE we're ignoring any extraneous dict items
     for (int i = 0; i < num_fields_; i++) {
-      PyObject* nameobj = PyList_GET_ITEM(field_name_list, i);  // borrowed
-      PyObject* valueobj = PyDict_GetItem(obj, nameobj);        // borrowed
-      if (valueobj == NULL) {
+      PyObject* name = PyList_GET_ITEM(field_names, i);  // borrowed
+      PyObject* value = PyDict_GetItem(dict, name);      // borrowed
+      if (value == NULL) {
         RETURN_IF_PYERROR();
       }
-      RETURN_NOT_OK(value_converters_[i]->Append(valueobj ? valueobj : Py_None));
+      RETURN_NOT_OK(this->children_[i]->Append(value ? value : Py_None));
     }
     return Status::OK();
   }
 
-  Status AppendTupleItem(PyObject* obj) {
-    if (PyTuple_GET_SIZE(obj) != num_fields_) {
-      return Status::Invalid("Tuple size must be equal to number of struct fields");
-    }
-    for (int i = 0; i < num_fields_; i++) {
-      PyObject* valueobj = PyTuple_GET_ITEM(obj, i);
-      RETURN_NOT_OK(value_converters_[i]->Append(valueobj));
+  Result<std::pair<PyObject*, PyObject*>> GetKeyValuePair(PyObject* seq, int index) {
+    PyObject* pair = PySequence_GetItem(seq, index);
+    RETURN_IF_PYERROR();
+    if (!PyTuple_Check(pair) || PyTuple_Size(pair) != 2) {
+      return internal::InvalidType(pair, "was expecting tuple of (key, value) pair");
     }
-    return Status::OK();
+    PyObject* key = PyTuple_GetItem(pair, 0);
+    RETURN_IF_PYERROR();
+    PyObject* value = PyTuple_GetItem(pair, 1);
+    RETURN_IF_PYERROR();
+    return std::make_pair(key, value);
   }
 
-  std::vector<std::unique_ptr<SeqConverter>> value_converters_;
-  OwnedRef field_name_unicode_list_;
-  OwnedRef field_name_bytes_list_;
-  int num_fields_;
-  // Whether we're converting from a sequence of dicts or tuples
-  enum class SourceKind { UNKNOWN, DICTS, TUPLES } source_kind_ = SourceKind::UNKNOWN;
-  enum class DictKeyKind {
-    UNKNOWN,
-    BYTES,
-    UNICODE
-  } dict_key_kind_ = DictKeyKind::UNKNOWN;
-  bool from_pandas_;
-  bool strict_conversions_;
-};
-
-template <NullCoding null_coding>
-class DecimalConverter : public TypedConverter<arrow::Decimal128Type, null_coding> {
- public:
-  using BASE = TypedConverter<arrow::Decimal128Type, null_coding>;
-
-  Status Init(ArrayBuilder* builder) override {
-    RETURN_NOT_OK(BASE::Init(builder));
-    decimal_type_ = checked_pointer_cast<DecimalType>(this->typed_builder_->type());
-    return Status::OK();
-  }
+  Status AppendItems(PyObject* items, PyObject* field_names) {
+    auto length = static_cast<int>(PySequence_Size(items));
+    RETURN_IF_PYERROR();
 
-  Status AppendValue(PyObject* obj) override {
-    Decimal128 value;
-    RETURN_NOT_OK(internal::DecimalFromPyObject(obj, *decimal_type_, &value));
-    return this->typed_builder_->Append(value);
-  }
+    // append the values for the defined fields
+    for (int i = 0; i < std::min(num_fields_, length); i++) {
+      // retrieve the key-value pair
+      ARROW_ASSIGN_OR_RAISE(auto pair, GetKeyValuePair(items, i));
 
- private:
-  std::shared_ptr<DecimalType> decimal_type_;
-};
+      // validate that the key and the field name are equal
+      PyObject* name = PyList_GET_ITEM(field_names, i);
+      bool are_equal = PyObject_RichCompareBool(pair.first, name, Py_EQ);
+      RETURN_IF_PYERROR();
 
-#define PRIMITIVE(TYPE_ENUM, TYPE)                                                   \
-  case Type::TYPE_ENUM:                                                              \
-    *out = std::unique_ptr<SeqConverter>(new PrimitiveConverter<TYPE, null_coding>); \
-    break;
-
-#define SIMPLE_CONVERTER_CASE(TYPE_ENUM, TYPE_CLASS)                   \
-  case Type::TYPE_ENUM:                                                \
-    *out = std::unique_ptr<SeqConverter>(new TYPE_CLASS<null_coding>); \
-    break;
-
-// Dynamic constructor for sequence converters
-template <NullCoding null_coding>
-Status GetConverterFlat(const std::shared_ptr<DataType>& type, bool strict_conversions,
-                        std::unique_ptr<SeqConverter>* out) {
-  switch (type->id()) {
-    SIMPLE_CONVERTER_CASE(NA, NullConverter);
-    PRIMITIVE(BOOL, BooleanType);
-    PRIMITIVE(INT8, Int8Type);
-    PRIMITIVE(INT16, Int16Type);
-    PRIMITIVE(INT32, Int32Type);
-    PRIMITIVE(INT64, Int64Type);
-    PRIMITIVE(UINT8, UInt8Type);
-    PRIMITIVE(UINT16, UInt16Type);
-    PRIMITIVE(UINT32, UInt32Type);
-    PRIMITIVE(UINT64, UInt64Type);
-    PRIMITIVE(HALF_FLOAT, HalfFloatType);
-    PRIMITIVE(FLOAT, FloatType);
-    PRIMITIVE(DOUBLE, DoubleType);
-    PRIMITIVE(DATE32, Date32Type);
-    PRIMITIVE(DATE64, Date64Type);
-    SIMPLE_CONVERTER_CASE(DECIMAL, DecimalConverter);
-    case Type::BINARY:
-      *out =
-          std::unique_ptr<SeqConverter>(new BinaryConverter<BinaryType, null_coding>());
-      break;
-    case Type::LARGE_BINARY:
-      *out = std::unique_ptr<SeqConverter>(
-          new BinaryConverter<LargeBinaryType, null_coding>());
-      break;
-    case Type::FIXED_SIZE_BINARY:
-      *out = std::unique_ptr<SeqConverter>(new FixedSizeBinaryConverter<null_coding>(
-          checked_cast<const FixedSizeBinaryType&>(*type).byte_width()));
-      break;
-    case Type::STRING:
-      if (strict_conversions) {
-        *out = std::unique_ptr<SeqConverter>(
-            new StringConverter<StringType, true, null_coding>());
-      } else {
-        *out = std::unique_ptr<SeqConverter>(
-            new StringConverter<StringType, false, null_coding>());
-      }
-      break;
-    case Type::LARGE_STRING:
-      if (strict_conversions) {
-        *out = std::unique_ptr<SeqConverter>(
-            new StringConverter<LargeStringType, true, null_coding>());
+      // finally append to the respective child builder
+      if (are_equal) {
+        RETURN_NOT_OK(this->children_[i]->Append(pair.second));
       } else {
-        *out = std::unique_ptr<SeqConverter>(
-            new StringConverter<LargeStringType, false, null_coding>());
+        ARROW_ASSIGN_OR_RAISE(auto key_view, PyBytesView::FromString(pair.first));
+        ARROW_ASSIGN_OR_RAISE(auto name_view, PyBytesView::FromString(name));
+        return Status::Invalid("The expected field name is `", name_view.bytes, "` but `",
+                               key_view.bytes, "` was given");
       }
-      break;
-    case Type::TIME32: {
-      *out = std::unique_ptr<SeqConverter>(new TimeConverter<Time32Type, null_coding>(
-          checked_cast<const Time32Type&>(*type).unit()));
-      break;
-    }
-    case Type::TIME64: {
-      *out = std::unique_ptr<SeqConverter>(new TimeConverter<Time64Type, null_coding>(
-          checked_cast<const Time64Type&>(*type).unit()));
-      break;
-    }
-    case Type::TIMESTAMP: {
-      *out =
-          std::unique_ptr<SeqConverter>(new TemporalConverter<TimestampType, null_coding>(
-              checked_cast<const TimestampType&>(*type).unit()));
-      break;
     }
-    case Type::DURATION: {
-      *out =
-          std::unique_ptr<SeqConverter>(new TemporalConverter<DurationType, null_coding>(
-              checked_cast<const DurationType&>(*type).unit()));
-      break;
+    // insert null values for missing fields
+    for (int i = length; i < num_fields_; i++) {
+      RETURN_NOT_OK(this->children_[i]->AppendNull());
     }
-    default:
-      return Status::NotImplemented("Sequence converter for type ", type->ToString(),
-                                    " not implemented");
-  }
-  return Status::OK();
-}
-
-Status GetConverter(const std::shared_ptr<DataType>& type, bool from_pandas,
-                    bool strict_conversions, std::unique_ptr<SeqConverter>* out) {
-  if (from_pandas) {
-    // ARROW-842: If pandas is not installed then null checks will be less
-    // comprehensive, but that is okay.
-    internal::InitPandasStaticData();
-  }
-
-  switch (type->id()) {
-    case Type::LIST:
-      if (from_pandas) {
-        *out = std::unique_ptr<SeqConverter>(
-            new ListConverter<ListType, NullCoding::PANDAS_SENTINELS>(
-                from_pandas, strict_conversions));
-      } else {
-        *out = std::unique_ptr<SeqConverter>(
-            new ListConverter<ListType, NullCoding::NONE_ONLY>(from_pandas,
-                                                               strict_conversions));
-      }
-      return Status::OK();
-    case Type::LARGE_LIST:
-      if (from_pandas) {
-        *out = std::unique_ptr<SeqConverter>(
-            new ListConverter<LargeListType, NullCoding::PANDAS_SENTINELS>(
-                from_pandas, strict_conversions));
-      } else {
-        *out = std::unique_ptr<SeqConverter>(
-            new ListConverter<LargeListType, NullCoding::NONE_ONLY>(from_pandas,
-                                                                    strict_conversions));
-      }
-      return Status::OK();
-    case Type::MAP:
-      if (from_pandas) {
-        *out =
-            std::unique_ptr<SeqConverter>(new MapConverter<NullCoding::PANDAS_SENTINELS>(
-                from_pandas, strict_conversions));
-      } else {
-        *out = std::unique_ptr<SeqConverter>(
-            new MapConverter<NullCoding::NONE_ONLY>(from_pandas, strict_conversions));
-      }
-      return Status::OK();
-    case Type::FIXED_SIZE_LIST:
-      if (from_pandas) {
-        *out = std::unique_ptr<SeqConverter>(
-            new FixedSizeListConverter<NullCoding::PANDAS_SENTINELS>(from_pandas,
-                                                                     strict_conversions));
-      } else {
-        *out = std::unique_ptr<SeqConverter>(
-            new FixedSizeListConverter<NullCoding::NONE_ONLY>(from_pandas,
-                                                              strict_conversions));
-      }
-      return Status::OK();
-    case Type::STRUCT:
-      if (from_pandas) {
-        *out = std::unique_ptr<SeqConverter>(
-            new StructConverter<NullCoding::PANDAS_SENTINELS>(from_pandas,
-                                                              strict_conversions));
-      } else {
-        *out = std::unique_ptr<SeqConverter>(
-            new StructConverter<NullCoding::NONE_ONLY>(from_pandas, strict_conversions));
-      }
-      return Status::OK();
-    default:
-      break;
-  }
-
-  if (from_pandas) {
-    RETURN_NOT_OK(
-        GetConverterFlat<NullCoding::PANDAS_SENTINELS>(type, strict_conversions, out));
-  } else {
-    RETURN_NOT_OK(GetConverterFlat<NullCoding::NONE_ONLY>(type, strict_conversions, out));
+    return Status::OK();
   }
-  return Status::OK();
-}
 
-// ----------------------------------------------------------------------
+  // Whether we're converting from a sequence of dicts or tuples or list of pairs
+  enum class InputKind { UNKNOWN, DICT, TUPLE, ITEMS } input_kind_ = InputKind::UNKNOWN;
+  // Whether the input dictionary keys' type is python bytes or unicode
+  enum class KeyKind { UNKNOWN, BYTES, UNICODE } key_kind_ = KeyKind::UNKNOWN;
+  // Store the field names as a PyObjects for dict matching
+  OwnedRef bytes_field_names_;
+  OwnedRef unicode_field_names_;
+  // Store the number of fields for later reuse
+  int num_fields_;
+};
 
 // Convert *obj* to a sequence if necessary
 // Fill *size* to its length.  If >= 0 on entry, *size* is an upper size
@@ -1309,60 +1003,59 @@ Status ConvertToSequenceAndInferSize(PyObject* obj, PyObject** seq, int64_t* siz
   return Status::OK();
 }
 
-Status ConvertPySequence(PyObject* sequence_source, PyObject* mask,
-                         const PyConversionOptions& options,
-                         std::shared_ptr<ChunkedArray>* out) {
+Result<std::shared_ptr<ChunkedArray>> ConvertPySequence(PyObject* obj, PyObject* mask,
+                                                        PyConversionOptions options,
+                                                        MemoryPool* pool) {
   PyAcquireGIL lock;
 
   PyObject* seq;
   OwnedRef tmp_seq_nanny;
 
-  std::shared_ptr<DataType> real_type;
+  ARROW_ASSIGN_OR_RAISE(auto is_pandas_imported, internal::IsModuleImported("pandas"));
+  if (is_pandas_imported) {
+    // If pandas has been already imported initialize the static pandas objects to
+    // support converting from pd.Timedelta and pd.Timestamp objects
+    internal::InitPandasStaticData();
+  }
 
   int64_t size = options.size;
-  RETURN_NOT_OK(ConvertToSequenceAndInferSize(sequence_source, &seq, &size));
+  RETURN_NOT_OK(ConvertToSequenceAndInferSize(obj, &seq, &size));
   tmp_seq_nanny.reset(seq);
 
   // In some cases, type inference may be "loose", like strings. If the user
   // passed pa.string(), then we will error if we encounter any non-UTF8
   // value. If not, then we will allow the result to be a BinaryArray
-  bool strict_conversions = false;
-
   if (options.type == nullptr) {
-    RETURN_NOT_OK(InferArrowType(seq, mask, options.from_pandas, &real_type));
+    ARROW_ASSIGN_OR_RAISE(options.type, InferArrowType(seq, mask, options.from_pandas));
+    options.strict = false;
   } else {
-    real_type = options.type;
-    strict_conversions = true;
+    options.strict = true;
   }
   DCHECK_GE(size, 0);
 
-  // Create the sequence converter, initialize with the builder
-  std::unique_ptr<SeqConverter> converter;
-  RETURN_NOT_OK(
-      GetConverter(real_type, options.from_pandas, strict_conversions, &converter));
-
-  // Create ArrayBuilder for type, then pass into the SeqConverter
-  // instance. The reason this is created here rather than in GetConverter is
-  // because of nested types (child SeqConverter objects need the child
-  // builders created by MakeBuilder)
-  std::unique_ptr<ArrayBuilder> type_builder;
-  RETURN_NOT_OK(MakeBuilder(options.pool, real_type, &type_builder));
-  RETURN_NOT_OK(converter->Init(type_builder.get()));
-
-  // Convert values
-  if (mask != nullptr && mask != Py_None) {
-    RETURN_NOT_OK(converter->ExtendMasked(seq, mask, size));
+  ARROW_ASSIGN_OR_RAISE(auto converter, (MakeConverter<PyConverter, PyConverterTrait>(
+                                            options.type, options, pool)));
+  if (converter->may_overflow()) {
+    // The converter hierarchy contains binary- or list-like builders which can overflow
+    // depending on the input values. Wrap the converter with a chunker which detects
+    // the overflow and automatically creates new chunks.
+    ARROW_ASSIGN_OR_RAISE(auto chunked_converter, MakeChunker(std::move(converter)));
+    if (mask != nullptr && mask != Py_None) {
+      RETURN_NOT_OK(ExtendMasked(chunked_converter.get(), seq, mask, size));
+    } else {
+      RETURN_NOT_OK(Extend(chunked_converter.get(), seq, size));
+    }
+    return chunked_converter->ToChunkedArray();
   } else {
-    RETURN_NOT_OK(converter->Extend(seq, size));
+    // If the converter can't overflow spare the capacity error checking on the hot-path,
+    // this improves the performance roughly by ~10% for primitive types.
+    if (mask != nullptr && mask != Py_None) {
+      RETURN_NOT_OK(ExtendMasked(converter.get(), seq, mask, size));
+    } else {
+      RETURN_NOT_OK(Extend(converter.get(), seq, size));
+    }
+    return converter->ToChunkedArray();
   }
-
-  // Retrieve result. Conversion may yield one or more array values
-  return converter->GetResult(out);
-}
-
-Status ConvertPySequence(PyObject* obj, const PyConversionOptions& options,
-                         std::shared_ptr<ChunkedArray>* out) {
-  return ConvertPySequence(obj, nullptr, options, out);
 }
 
 }  // namespace py
diff --git a/cpp/src/arrow/python/python_to_arrow.h b/cpp/src/arrow/python/python_to_arrow.h
index 5c8052ac6ce..d167996ba8d 100644
--- a/cpp/src/arrow/python/python_to_arrow.h
+++ b/cpp/src/arrow/python/python_to_arrow.h
@@ -39,23 +39,26 @@ class Status;
 namespace py {
 
 struct PyConversionOptions {
-  PyConversionOptions() : type(NULLPTR), size(-1), pool(NULLPTR), from_pandas(false) {}
+  PyConversionOptions() = default;
 
   PyConversionOptions(const std::shared_ptr<DataType>& type, int64_t size,
                       MemoryPool* pool, bool from_pandas)
-      : type(type), size(size), pool(default_memory_pool()), from_pandas(from_pandas) {}
+      : type(type), size(size), from_pandas(from_pandas) {}
 
   // Set to null if to be inferred
   std::shared_ptr<DataType> type;
 
-  // Default is -1: infer from data
-  int64_t size;
+  // Default is -1, which indicates the size should the same as the input sequence
+  int64_t size = -1;
 
-  // Memory pool to use for allocations
-  MemoryPool* pool;
+  bool from_pandas = false;
 
-  // Default false
-  bool from_pandas;
+  /// Used to maintain backwards compatibility for
+  /// timezone bugs (see ARROW-9528).  Should be removed
+  /// after Arrow 2.0 release.
+  bool ignore_timezone = false;
+
+  bool strict = false;
 };
 
 /// \brief Convert sequence (list, generator, NumPy array with dtype object) of
@@ -65,16 +68,13 @@ struct PyConversionOptions {
 /// values in the sequence are null (true) or not null (false). This parameter
 /// may be null
 /// \param[in] options various conversion options
-/// \param[out] out a ChunkedArray containing one or more chunks
-/// \return Status
-ARROW_PYTHON_EXPORT
-Status ConvertPySequence(PyObject* obj, PyObject* mask,
-                         const PyConversionOptions& options,
-                         std::shared_ptr<ChunkedArray>* out);
-
+/// \param[in] pool MemoryPool to use for allocations
+/// \return Result ChunkedArray
 ARROW_PYTHON_EXPORT
-Status ConvertPySequence(PyObject* obj, const PyConversionOptions& options,
-                         std::shared_ptr<ChunkedArray>* out);
+Result<std::shared_ptr<ChunkedArray>> ConvertPySequence(
+    PyObject* obj, PyObject* mask, PyConversionOptions options,
+    MemoryPool* pool = default_memory_pool());
 
 }  // namespace py
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/serialize.cc b/cpp/src/arrow/python/serialize.cc
index cefa97abeea..ad079cbd9c7 100644
--- a/cpp/src/arrow/python/serialize.cc
+++ b/cpp/src/arrow/python/serialize.cc
@@ -29,14 +29,16 @@
 #include <numpy/arrayscalars.h>
 
 #include "arrow/array.h"
+#include "arrow/array/builder_binary.h"
+#include "arrow/array/builder_nested.h"
+#include "arrow/array/builder_primitive.h"
 #include "arrow/array/builder_union.h"
-#include "arrow/builder.h"
 #include "arrow/io/interfaces.h"
 #include "arrow/io/memory.h"
 #include "arrow/ipc/util.h"
 #include "arrow/ipc/writer.h"
-#include "arrow/memory_pool.h"
 #include "arrow/record_batch.h"
+#include "arrow/result.h"
 #include "arrow/tensor.h"
 #include "arrow/util/logging.h"
 
@@ -482,8 +484,7 @@ Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder,
     RETURN_NOT_OK(internal::CastSize(PyBytes_GET_SIZE(elem), &size));
     RETURN_NOT_OK(builder->AppendBytes(data, size));
   } else if (PyUnicode_Check(elem)) {
-    PyBytesView view;
-    RETURN_NOT_OK(view.FromString(elem));
+    ARROW_ASSIGN_OR_RAISE(auto view, PyBytesView::FromUnicode(elem));
     int32_t size = -1;
     RETURN_NOT_OK(internal::CastSize(view.size, &size));
     RETURN_NOT_OK(builder->AppendString(view.bytes, size));
diff --git a/cpp/src/arrow/python/type_traits.h b/cpp/src/arrow/python/type_traits.h
index 2f704701ce6..a941577f765 100644
--- a/cpp/src/arrow/python/type_traits.h
+++ b/cpp/src/arrow/python/type_traits.h
@@ -28,8 +28,7 @@
 
 #include <numpy/halffloat.h>
 
-#include "arrow/builder.h"
-#include "arrow/type.h"
+#include "arrow/type_fwd.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
@@ -105,7 +104,9 @@ struct npy_traits<NPY_FLOAT32> {
   using TypeClass = FloatType;
   using BuilderClass = FloatBuilder;
 
-  static constexpr float na_sentinel = NAN;
+  // We need to use quiet_NaN here instead of the NAN macro as on Windows
+  // the NAN macro leads to "division-by-zero" compile-time error with clang.
+  static constexpr float na_sentinel = std::numeric_limits<float>::quiet_NaN();
 
   static constexpr bool supports_nulls = true;
 
@@ -118,7 +119,7 @@ struct npy_traits<NPY_FLOAT64> {
   using TypeClass = DoubleType;
   using BuilderClass = DoubleBuilder;
 
-  static constexpr double na_sentinel = NAN;
+  static constexpr double na_sentinel = std::numeric_limits<double>::quiet_NaN();
 
   static constexpr bool supports_nulls = true;
 
@@ -178,13 +179,13 @@ struct arrow_traits<Type::BOOL> {
   typedef typename npy_traits<NPY_BOOL>::value_type T;
 };
 
-#define INT_DECL(TYPE)                                     \
-  template <>                                              \
-  struct arrow_traits<Type::TYPE> {                        \
-    static constexpr int npy_type = NPY_##TYPE;            \
-    static constexpr bool supports_nulls = false;          \
-    static constexpr double na_value = NAN;                \
-    typedef typename npy_traits<NPY_##TYPE>::value_type T; \
+#define INT_DECL(TYPE)                                                           \
+  template <>                                                                    \
+  struct arrow_traits<Type::TYPE> {                                              \
+    static constexpr int npy_type = NPY_##TYPE;                                  \
+    static constexpr bool supports_nulls = false;                                \
+    static constexpr double na_value = std::numeric_limits<double>::quiet_NaN(); \
+    typedef typename npy_traits<NPY_##TYPE>::value_type T;                       \
   };
 
 INT_DECL(INT8);
@@ -208,7 +209,7 @@ template <>
 struct arrow_traits<Type::FLOAT> {
   static constexpr int npy_type = NPY_FLOAT32;
   static constexpr bool supports_nulls = true;
-  static constexpr float na_value = NAN;
+  static constexpr float na_value = std::numeric_limits<float>::quiet_NaN();
   typedef typename npy_traits<NPY_FLOAT32>::value_type T;
 };
 
@@ -216,7 +217,7 @@ template <>
 struct arrow_traits<Type::DOUBLE> {
   static constexpr int npy_type = NPY_FLOAT64;
   static constexpr bool supports_nulls = true;
-  static constexpr double na_value = NAN;
+  static constexpr double na_value = std::numeric_limits<double>::quiet_NaN();
   typedef typename npy_traits<NPY_FLOAT64>::value_type T;
 };
 
diff --git a/cpp/src/arrow/result.h b/cpp/src/arrow/result.h
index 96863ecad6d..9c43b324f08 100644
--- a/cpp/src/arrow/result.h
+++ b/cpp/src/arrow/result.h
@@ -28,6 +28,9 @@
 
 namespace arrow {
 
+template <typename>
+struct EnsureResult;
+
 namespace internal {
 
 #if __cplusplus >= 201703L
@@ -379,7 +382,7 @@ class ARROW_MUST_USE_TYPE Result : public util::EqualityComparable<Result<T>> {
   /// Apply a function to the internally stored value to produce a new result or propagate
   /// the stored error.
   template <typename M>
-  typename std::result_of<M && (T)>::type Map(M&& m) && {
+  typename EnsureResult<typename std::result_of<M && (T)>::type>::type Map(M&& m) && {
     if (!ok()) {
       return status();
     }
@@ -389,7 +392,8 @@ class ARROW_MUST_USE_TYPE Result : public util::EqualityComparable<Result<T>> {
   /// Apply a function to the internally stored value to produce a new result or propagate
   /// the stored error.
   template <typename M>
-  typename std::result_of<M && (const T&)>::type Map(M&& m) const& {
+  typename EnsureResult<typename std::result_of<M && (const T&)>::type>::type Map(
+      M&& m) const& {
     if (!ok()) {
       return status();
     }
@@ -425,9 +429,9 @@ class ARROW_MUST_USE_TYPE Result : public util::EqualityComparable<Result<T>> {
 };
 
 #define ARROW_ASSIGN_OR_RAISE_IMPL(result_name, lhs, rexpr) \
-  auto result_name = (rexpr);                               \
+  auto&& result_name = (rexpr);                             \
   ARROW_RETURN_NOT_OK((result_name).status());              \
-  lhs = std::move(result_name).MoveValueUnsafe();
+  lhs = std::move(result_name).ValueUnsafe();
 
 #define ARROW_ASSIGN_OR_RAISE_NAME(x, y) ARROW_CONCAT(x, y)
 
@@ -467,4 +471,14 @@ Result<T> ToResult(T t) {
   return Result<T>(std::move(t));
 }
 
+template <typename T>
+struct EnsureResult {
+  using type = Result<T>;
+};
+
+template <typename T>
+struct EnsureResult<Result<T>> {
+  using type = Result<T>;
+};
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/scalar.cc b/cpp/src/arrow/scalar.cc
index b953177a459..9e038024e06 100644
--- a/cpp/src/arrow/scalar.cc
+++ b/cpp/src/arrow/scalar.cc
@@ -40,7 +40,9 @@ namespace arrow {
 using internal::checked_cast;
 using internal::checked_pointer_cast;
 
-bool Scalar::Equals(const Scalar& other) const { return ScalarEquals(*this, other); }
+bool Scalar::Equals(const Scalar& other, const EqualOptions& options) const {
+  return ScalarEquals(*this, other, options);
+}
 
 struct ScalarHashImpl {
   static std::hash<std::string> string_hash;
@@ -67,6 +69,14 @@ struct ScalarHashImpl {
     return StdHash(s.value.low_bits()) & StdHash(s.value.high_bits());
   }
 
+  Status Visit(const Decimal256Scalar& s) {
+    Status status = Status::OK();
+    for (uint64_t elem : s.value.little_endian_array()) {
+      status &= StdHash(elem);
+    }
+    return status;
+  }
+
   Status Visit(const BaseListScalar& s) { return ArrayHash(*s.value); }
 
   Status Visit(const StructScalar& s) {
@@ -325,7 +335,7 @@ struct ScalarParseImpl {
     return MakeScalar(std::move(type_), std::forward<Arg>(arg)).Value(&out_);
   }
 
-  Status FinishWithBuffer() { return Finish(Buffer::FromString(s_.to_string())); }
+  Status FinishWithBuffer() { return Finish(Buffer::FromString(std::string(s_))); }
 
   Result<std::shared_ptr<Scalar>> Finish() && {
     RETURN_NOT_OK(VisitTypeInline(*type_, this));
@@ -364,8 +374,9 @@ std::shared_ptr<Buffer> FormatToBuffer(Formatter&& formatter, const ScalarType&
   if (!from.is_valid) {
     return Buffer::FromString("null");
   }
-  return formatter(
-      from.value, [&](util::string_view v) { return Buffer::FromString(v.to_string()); });
+  return formatter(from.value, [&](util::string_view v) {
+    return Buffer::FromString(std::string(v));
+  });
 }
 
 // error fallback
diff --git a/cpp/src/arrow/scalar.h b/cpp/src/arrow/scalar.h
index d15c44ce9db..80157a750cb 100644
--- a/cpp/src/arrow/scalar.h
+++ b/cpp/src/arrow/scalar.h
@@ -28,6 +28,7 @@
 #include <utility>
 #include <vector>
 
+#include "arrow/compare.h"
 #include "arrow/result.h"
 #include "arrow/status.h"
 #include "arrow/type.h"
@@ -61,7 +62,8 @@ struct ARROW_EXPORT Scalar : public util::EqualityComparable<Scalar> {
 
   using util::EqualityComparable<Scalar>::operator==;
   using util::EqualityComparable<Scalar>::Equals;
-  bool Equals(const Scalar& other) const;
+  bool Equals(const Scalar& other,
+              const EqualOptions& options = EqualOptions::Defaults()) const;
 
   struct ARROW_EXPORT Hash {
     size_t operator()(const Scalar& scalar) const { return hash(scalar); }
@@ -212,7 +214,7 @@ struct ARROW_EXPORT BaseBinaryScalar : public Scalar {
 
 struct ARROW_EXPORT BinaryScalar : public BaseBinaryScalar {
   using BaseBinaryScalar::BaseBinaryScalar;
-  using TypeClass = BinaryScalar;
+  using TypeClass = BinaryType;
 
   BinaryScalar(std::shared_ptr<Buffer> value, std::shared_ptr<DataType> type)
       : BaseBinaryScalar(std::move(value), std::move(type)) {}
@@ -237,7 +239,7 @@ struct ARROW_EXPORT StringScalar : public BinaryScalar {
 
 struct ARROW_EXPORT LargeBinaryScalar : public BaseBinaryScalar {
   using BaseBinaryScalar::BaseBinaryScalar;
-  using TypeClass = LargeBinaryScalar;
+  using TypeClass = LargeBinaryType;
 
   LargeBinaryScalar(std::shared_ptr<Buffer> value, std::shared_ptr<DataType> type)
       : BaseBinaryScalar(std::move(value), std::move(type)) {}
@@ -345,6 +347,17 @@ struct ARROW_EXPORT Decimal128Scalar : public Scalar {
   Decimal128 value;
 };
 
+struct ARROW_EXPORT Decimal256Scalar : public Scalar {
+  using Scalar::Scalar;
+  using TypeClass = Decimal256Type;
+  using ValueType = Decimal256;
+
+  Decimal256Scalar(Decimal256 value, std::shared_ptr<DataType> type)
+      : Scalar(std::move(type), true), value(value) {}
+
+  Decimal256 value;
+};
+
 struct ARROW_EXPORT BaseListScalar : public Scalar {
   using Scalar::Scalar;
   using ValueType = std::shared_ptr<Array>;
@@ -426,8 +439,8 @@ struct ARROW_EXPORT DictionaryScalar : public Scalar {
 
   explicit DictionaryScalar(std::shared_ptr<DataType> type);
 
-  DictionaryScalar(ValueType value, std::shared_ptr<DataType> type)
-      : Scalar(std::move(type), true), value(std::move(value)) {}
+  DictionaryScalar(ValueType value, std::shared_ptr<DataType> type, bool is_valid = true)
+      : Scalar(std::move(type), is_valid), value(std::move(value)) {}
 
   Result<std::shared_ptr<Scalar>> GetEncodedValue() const;
 };
diff --git a/cpp/src/arrow/scalar_test.cc b/cpp/src/arrow/scalar_test.cc
index f1ab6d6cfab..71f1ae04ce2 100644
--- a/cpp/src/arrow/scalar_test.cc
+++ b/cpp/src/arrow/scalar_test.cc
@@ -127,8 +127,8 @@ TYPED_TEST(TestNumericScalar, MakeScalar) {
   ASSERT_EQ(ScalarType(3), *three);
 }
 
-TEST(TestDecimalScalar, Basics) {
-  auto ty = decimal(3, 2);
+TEST(TestDecimal128Scalar, Basics) {
+  auto ty = decimal128(3, 2);
   auto pi = Decimal128Scalar(Decimal128("3.14"), ty);
   auto null = MakeNullScalar(ty);
 
@@ -144,6 +144,23 @@ TEST(TestDecimalScalar, Basics) {
   ASSERT_FALSE(second->Equals(null));
 }
 
+TEST(TestDecimal256Scalar, Basics) {
+  auto ty = decimal256(3, 2);
+  auto pi = Decimal256Scalar(Decimal256("3.14"), ty);
+  auto null = MakeNullScalar(ty);
+
+  ASSERT_EQ(pi.value, Decimal256("3.14"));
+
+  // test Array.GetScalar
+  auto arr = ArrayFromJSON(ty, "[null, \"3.14\"]");
+  ASSERT_OK_AND_ASSIGN(auto first, arr->GetScalar(0));
+  ASSERT_OK_AND_ASSIGN(auto second, arr->GetScalar(1));
+  ASSERT_TRUE(first->Equals(null));
+  ASSERT_FALSE(first->Equals(pi));
+  ASSERT_TRUE(second->Equals(pi));
+  ASSERT_FALSE(second->Equals(null));
+}
+
 TEST(TestBinaryScalar, Basics) {
   std::string data = "test data";
   auto buf = std::make_shared<Buffer>(data);
@@ -567,7 +584,7 @@ TYPED_TEST(TestNumericScalar, Cast) {
     }
 
     ASSERT_OK_AND_ASSIGN(auto cast_from_string,
-                         StringScalar(repr.to_string()).CastTo(type));
+                         StringScalar(std::string(repr)).CastTo(type));
     ASSERT_EQ(*cast_from_string, *scalar);
 
     if (is_integer_type<TypeParam>::value) {
@@ -627,6 +644,8 @@ TEST(TestDictionaryScalar, Basics) {
     gamma.dictionary = dict;
 
     auto scalar_null = MakeNullScalar(ty);
+    checked_cast<DictionaryScalar&>(*scalar_null).value.dictionary = dict;
+
     auto scalar_alpha = DictionaryScalar(alpha, ty);
     auto scalar_gamma = DictionaryScalar(gamma, ty);
 
@@ -654,6 +673,12 @@ TEST(TestDictionaryScalar, Basics) {
     ASSERT_TRUE(first->Equals(scalar_gamma));
     ASSERT_TRUE(second->Equals(scalar_alpha));
     ASSERT_TRUE(last->Equals(scalar_null));
+
+    auto first_dict_scalar = checked_cast<const DictionaryScalar&>(*first);
+    ASSERT_TRUE(first_dict_scalar.value.dictionary->Equals(arr.dictionary()));
+
+    auto second_dict_scalar = checked_cast<const DictionaryScalar&>(*second);
+    ASSERT_TRUE(second_dict_scalar.value.dictionary->Equals(arr.dictionary()));
   }
 }
 
diff --git a/cpp/src/arrow/stl.h b/cpp/src/arrow/stl.h
index 5ba7a15072d..146c5706766 100644
--- a/cpp/src/arrow/stl.h
+++ b/cpp/src/arrow/stl.h
@@ -28,13 +28,15 @@
 #include <vector>
 
 #include "arrow/array.h"
-#include "arrow/builder.h"
+#include "arrow/array/builder_base.h"
+#include "arrow/array/builder_binary.h"
+#include "arrow/array/builder_nested.h"
+#include "arrow/array/builder_primitive.h"
 #include "arrow/chunked_array.h"
 #include "arrow/compute/api.h"
-#include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
-#include "arrow/type.h"
+#include "arrow/type_fwd.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/checked_cast.h"
 #include "arrow/util/macros.h"
diff --git a/cpp/src/arrow/table_builder.cc b/cpp/src/arrow/table_builder.cc
index 2c2a5b59e0a..78034c92868 100644
--- a/cpp/src/arrow/table_builder.cc
+++ b/cpp/src/arrow/table_builder.cc
@@ -21,7 +21,7 @@
 #include <utility>
 
 #include "arrow/array/array_base.h"
-#include "arrow/builder.h"
+#include "arrow/array/builder_base.h"
 #include "arrow/record_batch.h"
 #include "arrow/status.h"
 #include "arrow/type.h"
@@ -62,7 +62,21 @@ Status RecordBatchBuilder::Flush(bool reset_builders,
     }
     length = fields[i]->length();
   }
-  *batch = RecordBatch::Make(schema_, length, std::move(fields));
+
+  // For certain types like dictionaries, types may not be fully
+  // determined before we have flushed. Make sure that the RecordBatch
+  // gets the correct types in schema.
+  // See: #ARROW-9969
+  std::vector<std::shared_ptr<Field>> schema_fields(schema_->fields());
+  for (int i = 0; i < this->num_fields(); ++i) {
+    if (!schema_fields[i]->type()->Equals(fields[i]->type())) {
+      schema_fields[i] = schema_fields[i]->WithType(fields[i]->type());
+    }
+  }
+  std::shared_ptr<Schema> schema =
+      std::make_shared<Schema>(schema_fields, schema_->metadata());
+
+  *batch = RecordBatch::Make(schema, length, std::move(fields));
   if (reset_builders) {
     return InitBuilders();
   } else {
diff --git a/cpp/src/arrow/table_builder_test.cc b/cpp/src/arrow/table_builder_test.cc
index 16c20fbbb12..c730913122a 100644
--- a/cpp/src/arrow/table_builder_test.cc
+++ b/cpp/src/arrow/table_builder_test.cc
@@ -25,6 +25,7 @@
 
 #include "arrow/array/array_base.h"
 #include "arrow/array/builder_binary.h"
+#include "arrow/array/builder_dict.h"
 #include "arrow/array/builder_nested.h"
 #include "arrow/array/builder_primitive.h"
 #include "arrow/record_batch.h"
@@ -151,4 +152,31 @@ TEST_F(TestRecordBatchBuilder, InvalidFieldLength) {
   ASSERT_RAISES(Invalid, builder->Flush(&dummy));
 }
 
+// In #ARROW-9969 dictionary types were not updated
+// in schema when the index width grew.
+TEST_F(TestRecordBatchBuilder, DictionaryTypes) {
+  const int num_rows = static_cast<int>(UINT8_MAX) + 2;
+  std::vector<std::string> f0_values;
+  std::vector<bool> is_valid(num_rows, true);
+  for (int i = 0; i < num_rows; i++) {
+    f0_values.push_back(std::to_string(i));
+  }
+
+  auto f0 = field("f0", dictionary(int8(), utf8()));
+
+  auto schema = ::arrow::schema({f0});
+
+  std::unique_ptr<RecordBatchBuilder> builder;
+  ASSERT_OK(RecordBatchBuilder::Make(schema, pool_, &builder));
+
+  auto b0 = builder->GetFieldAs<StringDictionaryBuilder>(0);
+
+  AppendValues<StringDictionaryBuilder, std::string>(b0, f0_values, is_valid);
+
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK(builder->Flush(&batch));
+
+  AssertTypeEqual(batch->column(0)->type(), batch->schema()->field(0)->type());
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/testing/generator.h b/cpp/src/arrow/testing/generator.h
index d0fe61d2232..9188dca5709 100644
--- a/cpp/src/arrow/testing/generator.h
+++ b/cpp/src/arrow/testing/generator.h
@@ -166,6 +166,8 @@ class ARROW_TESTING_EXPORT ConstantArrayGenerator {
   static std::shared_ptr<arrow::Array> Zeroes(int64_t size,
                                               const std::shared_ptr<DataType>& type) {
     switch (type->id()) {
+      case Type::NA:
+        return std::make_shared<NullArray>(size);
       case Type::BOOL:
         return Boolean(size);
       case Type::UINT8:
@@ -184,6 +186,19 @@ class ARROW_TESTING_EXPORT ConstantArrayGenerator {
         return UInt64(size);
       case Type::INT64:
         return Int64(size);
+      case Type::TIME64:
+      case Type::DATE64:
+      case Type::TIMESTAMP: {
+        EXPECT_OK_AND_ASSIGN(auto viewed, Int64(size)->View(type));
+        return viewed;
+      }
+      case Type::INTERVAL_DAY_TIME:
+      case Type::INTERVAL_MONTHS:
+      case Type::TIME32:
+      case Type::DATE32: {
+        EXPECT_OK_AND_ASSIGN(auto viewed, Int32(size)->View(type));
+        return viewed;
+      }
       case Type::FLOAT:
         return Float32(size);
       case Type::DOUBLE:
diff --git a/cpp/src/arrow/testing/gtest_common.h b/cpp/src/arrow/testing/gtest_common.h
index cf6af369420..8b48238ed8a 100644
--- a/cpp/src/arrow/testing/gtest_common.h
+++ b/cpp/src/arrow/testing/gtest_common.h
@@ -27,11 +27,11 @@
 #include <gtest/gtest.h>
 
 #include "arrow/array.h"
-#include "arrow/buffer.h"
-#include "arrow/builder.h"
-#include "arrow/memory_pool.h"
+#include "arrow/array/builder_binary.h"
+#include "arrow/array/builder_primitive.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/testing/util.h"
+#include "arrow/type_fwd.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/testing/gtest_util.cc b/cpp/src/arrow/testing/gtest_util.cc
index b2f55668977..41ab7a4e8b5 100644
--- a/cpp/src/arrow/testing/gtest_util.cc
+++ b/cpp/src/arrow/testing/gtest_util.cc
@@ -47,6 +47,7 @@
 #include "arrow/table.h"
 #include "arrow/type.h"
 #include "arrow/util/checked_cast.h"
+#include "arrow/util/io_util.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
@@ -68,7 +69,8 @@ std::vector<Type::type> AllTypeIds() {
           Type::HALF_FLOAT,
           Type::FLOAT,
           Type::DOUBLE,
-          Type::DECIMAL,
+          Type::DECIMAL128,
+          Type::DECIMAL256,
           Type::DATE32,
           Type::DATE64,
           Type::TIME32,
@@ -135,22 +137,24 @@ void AssertArraysEqual(const Array& expected, const Array& actual, bool verbose)
       });
 }
 
-void AssertArraysApproxEqual(const Array& expected, const Array& actual, bool verbose) {
+void AssertArraysApproxEqual(const Array& expected, const Array& actual, bool verbose,
+                             const EqualOptions& option) {
   return AssertArraysEqualWith(
       expected, actual, verbose,
-      [](const Array& expected, const Array& actual, std::stringstream* diff) {
-        return expected.ApproxEquals(actual, EqualOptions().diff_sink(diff));
+      [&option](const Array& expected, const Array& actual, std::stringstream* diff) {
+        return expected.ApproxEquals(actual, option.diff_sink(diff));
       });
 }
 
-void AssertScalarsEqual(const Scalar& expected, const Scalar& actual, bool verbose) {
+void AssertScalarsEqual(const Scalar& expected, const Scalar& actual, bool verbose,
+                        const EqualOptions& options) {
   std::stringstream diff;
   // ARROW-8956, ScalarEquals returns false when both are null
   if (!expected.is_valid && !actual.is_valid) {
     // We consider both being null to be equal in this function
     return;
   }
-  if (!expected.Equals(actual)) {
+  if (!expected.Equals(actual, options)) {
     if (verbose) {
       diff << "Expected:\n" << expected.ToString();
       diff << "\nActual:\n" << actual.ToString();
@@ -229,12 +233,21 @@ void AssertBufferEqual(const Buffer& buffer, const Buffer& expected) {
   ASSERT_TRUE(buffer.Equals(expected));
 }
 
+template <typename T>
+std::string ToStringWithMetadata(const T& t, bool show_metadata) {
+  return t.ToString(show_metadata);
+}
+
+std::string ToStringWithMetadata(const DataType& t, bool show_metadata) {
+  return t.ToString();
+}
+
 template <typename T>
 void AssertFingerprintablesEqual(const T& left, const T& right, bool check_metadata,
                                  const char* types_plural) {
   ASSERT_TRUE(left.Equals(right, check_metadata))
-      << types_plural << " '" << left.ToString() << "' and '" << right.ToString()
-      << "' should have compared equal";
+      << types_plural << " '" << ToStringWithMetadata(left, check_metadata) << "' and '"
+      << ToStringWithMetadata(right, check_metadata) << "' should have compared equal";
   auto lfp = left.fingerprint();
   auto rfp = right.fingerprint();
   // Note: all types tested in this file should implement fingerprinting,
@@ -243,8 +256,10 @@ void AssertFingerprintablesEqual(const T& left, const T& right, bool check_metad
     lfp += left.metadata_fingerprint();
     rfp += right.metadata_fingerprint();
   }
-  ASSERT_EQ(lfp, rfp) << "Fingerprints for " << types_plural << " '" << left.ToString()
-                      << "' and '" << right.ToString() << "' should have compared equal";
+  ASSERT_EQ(lfp, rfp) << "Fingerprints for " << types_plural << " '"
+                      << ToStringWithMetadata(left, check_metadata) << "' and '"
+                      << ToStringWithMetadata(right, check_metadata)
+                      << "' should have compared equal";
 }
 
 template <typename T>
@@ -260,8 +275,8 @@ template <typename T>
 void AssertFingerprintablesNotEqual(const T& left, const T& right, bool check_metadata,
                                     const char* types_plural) {
   ASSERT_FALSE(left.Equals(right, check_metadata))
-      << types_plural << " '" << left.ToString() << "' and '" << right.ToString()
-      << "' should have compared unequal";
+      << types_plural << " '" << ToStringWithMetadata(left, check_metadata) << "' and '"
+      << ToStringWithMetadata(right, check_metadata) << "' should have compared unequal";
   auto lfp = left.fingerprint();
   auto rfp = right.fingerprint();
   // Note: all types tested in this file should implement fingerprinting,
@@ -271,8 +286,9 @@ void AssertFingerprintablesNotEqual(const T& left, const T& right, bool check_me
       lfp += left.metadata_fingerprint();
       rfp += right.metadata_fingerprint();
     }
-    ASSERT_NE(lfp, rfp) << "Fingerprints for " << types_plural << " '" << left.ToString()
-                        << "' and '" << right.ToString()
+    ASSERT_NE(lfp, rfp) << "Fingerprints for " << types_plural << " '"
+                        << ToStringWithMetadata(left, check_metadata) << "' and '"
+                        << ToStringWithMetadata(right, check_metadata)
                         << "' should have compared unequal";
   }
 }
@@ -480,6 +496,26 @@ LocaleGuard::LocaleGuard(const char* new_locale) : impl_(new Impl(new_locale)) {
 
 LocaleGuard::~LocaleGuard() {}
 
+EnvVarGuard::EnvVarGuard(const std::string& name, const std::string& value)
+    : name_(name) {
+  auto maybe_value = arrow::internal::GetEnvVar(name);
+  if (maybe_value.ok()) {
+    was_set_ = true;
+    old_value_ = *std::move(maybe_value);
+  } else {
+    was_set_ = false;
+  }
+  ARROW_CHECK_OK(arrow::internal::SetEnvVar(name, value));
+}
+
+EnvVarGuard::~EnvVarGuard() {
+  if (was_set_) {
+    ARROW_CHECK_OK(arrow::internal::SetEnvVar(name_, old_value_));
+  } else {
+    ARROW_CHECK_OK(arrow::internal::DelEnvVar(name_));
+  }
+}
+
 namespace {
 
 // Used to prevent compiler optimizing away side-effect-less statements
diff --git a/cpp/src/arrow/testing/gtest_util.h b/cpp/src/arrow/testing/gtest_util.h
index 1411e705bcf..7a0c9b7c257 100644
--- a/cpp/src/arrow/testing/gtest_util.h
+++ b/cpp/src/arrow/testing/gtest_util.h
@@ -29,8 +29,8 @@
 
 #include <gtest/gtest.h>
 
-#include "arrow/buffer.h"
-#include "arrow/builder.h"
+#include "arrow/array/builder_binary.h"
+#include "arrow/array/builder_primitive.h"
 #include "arrow/result.h"
 #include "arrow/status.h"
 #include "arrow/testing/gtest_compat.h"
@@ -108,7 +108,7 @@
   } while (false);
 
 #define ASSIGN_OR_HANDLE_ERROR_IMPL(handle_error, status_name, lhs, rexpr) \
-  auto status_name = (rexpr);                                              \
+  auto&& status_name = (rexpr);                                            \
   handle_error(status_name.status());                                      \
   lhs = std::move(status_name).ValueOrDie();
 
@@ -164,12 +164,13 @@ std::vector<Type::type> AllTypeIds();
 // If verbose is true, then the arrays will be pretty printed
 ARROW_TESTING_EXPORT void AssertArraysEqual(const Array& expected, const Array& actual,
                                             bool verbose = false);
-ARROW_TESTING_EXPORT void AssertArraysApproxEqual(const Array& expected,
-                                                  const Array& actual,
-                                                  bool verbose = false);
+ARROW_TESTING_EXPORT void AssertArraysApproxEqual(
+    const Array& expected, const Array& actual, bool verbose = false,
+    const EqualOptions& option = EqualOptions::Defaults());
 // Returns true when values are both null
-ARROW_TESTING_EXPORT void AssertScalarsEqual(const Scalar& expected, const Scalar& actual,
-                                             bool verbose = false);
+ARROW_TESTING_EXPORT void AssertScalarsEqual(
+    const Scalar& expected, const Scalar& actual, bool verbose = false,
+    const EqualOptions& options = EqualOptions::Defaults());
 ARROW_TESTING_EXPORT void AssertBatchesEqual(const RecordBatch& expected,
                                              const RecordBatch& actual,
                                              bool check_metadata = false);
@@ -253,7 +254,7 @@ ARROW_TESTING_EXPORT void TestInitialized(const Array& array);
 
 template <typename BuilderType>
 void FinishAndCheckPadding(BuilderType* builder, std::shared_ptr<Array>* out) {
-  ASSERT_OK(builder->Finish(out));
+  ASSERT_OK_AND_ASSIGN(*out, builder->Finish());
   AssertZeroPadded(**out);
   TestInitialized(**out);
 }
@@ -445,6 +446,17 @@ class ARROW_TESTING_EXPORT LocaleGuard {
   std::unique_ptr<Impl> impl_;
 };
 
+class ARROW_TESTING_EXPORT EnvVarGuard {
+ public:
+  EnvVarGuard(const std::string& name, const std::string& value);
+  ~EnvVarGuard();
+
+ protected:
+  const std::string name_;
+  std::string old_value_;
+  bool was_set_;
+};
+
 #ifndef ARROW_LARGE_MEMORY_TESTS
 #define LARGE_MEMORY_TEST(name) DISABLED_##name
 #else
@@ -452,3 +464,15 @@ class ARROW_TESTING_EXPORT LocaleGuard {
 #endif
 
 }  // namespace arrow
+
+namespace nonstd {
+namespace sv_lite {
+
+// Without this hint, GTest will print string_views as a container of char
+template <class Char, class Traits = std::char_traits<Char>>
+void PrintTo(const basic_string_view<Char, Traits>& view, std::ostream* os) {
+  *os << view;
+}
+
+}  // namespace sv_lite
+}  // namespace nonstd
diff --git a/cpp/src/arrow/testing/json_integration.cc b/cpp/src/arrow/testing/json_integration.cc
index d724bd10ebf..2af094a781b 100644
--- a/cpp/src/arrow/testing/json_integration.cc
+++ b/cpp/src/arrow/testing/json_integration.cc
@@ -41,6 +41,7 @@ using std::size_t;
 
 namespace arrow {
 
+using ipc::DictionaryFieldMapper;
 using ipc::DictionaryMemo;
 
 namespace testing {
@@ -51,24 +52,24 @@ namespace testing {
 class IntegrationJsonWriter::Impl {
  public:
   explicit Impl(const std::shared_ptr<Schema>& schema)
-      : schema_(schema), first_batch_written_(false) {
+      : schema_(schema), mapper_(*schema), first_batch_written_(false) {
     writer_.reset(new RjWriter(string_buffer_));
   }
 
   Status Start() {
     writer_->StartObject();
-    RETURN_NOT_OK(json::WriteSchema(*schema_, &dictionary_memo_, writer_.get()));
+    RETURN_NOT_OK(json::WriteSchema(*schema_, mapper_, writer_.get()));
     return Status::OK();
   }
 
   Status FirstRecordBatch(const RecordBatch& batch) {
-    RETURN_NOT_OK(CollectDictionaries(batch, &dictionary_memo_));
+    ARROW_ASSIGN_OR_RAISE(const auto dictionaries, CollectDictionaries(batch, mapper_));
 
     // Write dictionaries, if any
-    if (dictionary_memo_.num_dictionaries() > 0) {
+    if (!dictionaries.empty()) {
       writer_->Key("dictionaries");
       writer_->StartArray();
-      for (const auto& entry : dictionary_memo_.dictionaries()) {
+      for (const auto& entry : dictionaries) {
         RETURN_NOT_OK(json::WriteDictionary(entry.first, entry.second, writer_.get()));
       }
       writer_->EndArray();
@@ -100,7 +101,7 @@ class IntegrationJsonWriter::Impl {
 
  private:
   std::shared_ptr<Schema> schema_;
-  DictionaryMemo dictionary_memo_;
+  DictionaryFieldMapper mapper_;
 
   bool first_batch_written_;
 
diff --git a/cpp/src/arrow/testing/json_integration_test.cc b/cpp/src/arrow/testing/json_integration_test.cc
index b432da0d9f6..627c1e9c9c6 100644
--- a/cpp/src/arrow/testing/json_integration_test.cc
+++ b/cpp/src/arrow/testing/json_integration_test.cc
@@ -29,15 +29,14 @@
 #include <gtest/gtest.h>
 
 #include "arrow/array.h"
-#include "arrow/buffer.h"
-#include "arrow/builder.h"
+#include "arrow/array/builder_binary.h"
+#include "arrow/array/builder_primitive.h"
 #include "arrow/io/file.h"
 #include "arrow/ipc/dictionary.h"
 #include "arrow/ipc/reader.h"
 #include "arrow/ipc/test_common.h"
 #include "arrow/ipc/writer.h"
 #include "arrow/pretty_print.h"
-#include "arrow/record_batch.h"
 #include "arrow/status.h"
 #include "arrow/testing/extension_type.h"
 #include "arrow/testing/gtest_util.h"
@@ -45,6 +44,7 @@
 #include "arrow/testing/json_internal.h"
 #include "arrow/testing/random.h"
 #include "arrow/type.h"
+#include "arrow/type_fwd.h"
 #include "arrow/util/io_util.h"
 
 DEFINE_string(arrow, "", "Arrow file name");
@@ -58,6 +58,7 @@ DEFINE_bool(verbose, true, "Verbose output");
 namespace arrow {
 
 using internal::TemporaryDir;
+using ipc::DictionaryFieldMapper;
 using ipc::DictionaryMemo;
 using ipc::IpcWriteOptions;
 using ipc::MetadataVersion;
@@ -83,8 +84,8 @@ static Status ConvertJsonToArrow(const std::string& json_path,
               << reader->schema()->ToString(/* show_metadata = */ true) << std::endl;
   }
 
-  ARROW_ASSIGN_OR_RAISE(auto writer, ipc::NewFileWriter(out_file.get(), reader->schema(),
-                                                        IpcWriteOptions::Defaults()));
+  ARROW_ASSIGN_OR_RAISE(auto writer, ipc::MakeFileWriter(out_file, reader->schema(),
+                                                         IpcWriteOptions::Defaults()));
   for (int i = 0; i < reader->num_record_batches(); ++i) {
     std::shared_ptr<RecordBatch> batch;
     RETURN_NOT_OK(reader->ReadRecordBatch(i, &batch));
@@ -509,7 +510,7 @@ static const char* json_example3 = R"example(
     {
       "id": 0,
       "data": {
-        "count": 10,
+        "count": 3,
         "columns": [
           {
             "name": "DICT0",
@@ -638,10 +639,10 @@ void TestSchemaRoundTrip(const Schema& schema) {
   rj::StringBuffer sb;
   rj::Writer<rj::StringBuffer> writer(sb);
 
-  DictionaryMemo out_memo;
+  DictionaryFieldMapper mapper(schema);
 
   writer.StartObject();
-  ASSERT_OK(json::WriteSchema(schema, &out_memo, &writer));
+  ASSERT_OK(json::WriteSchema(schema, mapper, &writer));
   writer.EndObject();
 
   std::string json_schema = sb.GetString();
@@ -681,11 +682,9 @@ void TestArrayRoundTrip(const Array& array) {
     FAIL() << "JSON parsing failed";
   }
 
-  DictionaryMemo out_memo;
-
   std::shared_ptr<Array> out;
   ASSERT_OK(json::ReadArray(default_memory_pool(), d, ::arrow::field(name, array.type()),
-                            &out_memo, &out));
+                            &out));
 
   // std::cout << array_as_json << std::endl;
   CompareArraysDetailed(0, *out, array);
@@ -1014,15 +1013,6 @@ TEST(TestJsonFileReadWrite, JsonExample4) {
   AssertArraysEqual(*batch->column(0), *expected_array);
 }
 
-#define BATCH_CASES()                                                             \
-  ::testing::Values(                                                              \
-      &MakeIntRecordBatch, &MakeListRecordBatch, &MakeFixedSizeListRecordBatch,   \
-      &MakeNonNullRecordBatch, &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, \
-      &MakeStringTypesRecordBatchWithNulls, &MakeStruct, &MakeUnion, &MakeDates,  \
-      &MakeTimestamps, &MakeTimes, &MakeFWBinary, &MakeDecimal, &MakeFloatBatch,  \
-      &MakeDictionary, &MakeNestedDictionary, &MakeIntervals, &MakeUuid,          \
-      &MakeDictExtension)
-
 class TestJsonRoundTrip : public ::testing::TestWithParam<MakeRecordBatch*> {
  public:
   void SetUp() {}
@@ -1061,7 +1051,34 @@ TEST_P(TestJsonRoundTrip, RoundTrip) {
   CheckRoundtrip(*batch);
 }
 
-INSTANTIATE_TEST_SUITE_P(TestJsonRoundTrip, TestJsonRoundTrip, BATCH_CASES());
+const std::vector<ipc::test::MakeRecordBatch*> kBatchCases = {
+    &MakeIntRecordBatch,
+    &MakeListRecordBatch,
+    &MakeFixedSizeListRecordBatch,
+    &MakeNonNullRecordBatch,
+    &MakeZeroLengthRecordBatch,
+    &MakeDeeplyNestedList,
+    &MakeStringTypesRecordBatchWithNulls,
+    &MakeStruct,
+    &MakeUnion,
+    &MakeDictionary,
+    &MakeNestedDictionary,
+    &MakeMap,
+    &MakeMapOfDictionary,
+    &MakeDates,
+    &MakeTimestamps,
+    &MakeTimes,
+    &MakeFWBinary,
+    &MakeNull,
+    &MakeDecimal,
+    &MakeBooleanBatch,
+    &MakeFloatBatch,
+    &MakeIntervals,
+    &MakeUuid,
+    &MakeDictExtension};
+
+INSTANTIATE_TEST_SUITE_P(TestJsonRoundTrip, TestJsonRoundTrip,
+                         ::testing::ValuesIn(kBatchCases));
 
 }  // namespace testing
 }  // namespace arrow
diff --git a/cpp/src/arrow/testing/json_internal.cc b/cpp/src/arrow/testing/json_internal.cc
index 31a64f52e91..edc8675434f 100644
--- a/cpp/src/arrow/testing/json_internal.cc
+++ b/cpp/src/arrow/testing/json_internal.cc
@@ -27,14 +27,17 @@
 #include <vector>
 
 #include "arrow/array.h"
-#include "arrow/buffer.h"
-#include "arrow/builder.h"  // IWYU pragma: keep
+#include "arrow/array/builder_binary.h"
+#include "arrow/array/builder_decimal.h"
+#include "arrow/array/builder_primitive.h"
+#include "arrow/array/builder_time.h"
 #include "arrow/extension_type.h"
 #include "arrow/ipc/dictionary.h"
 #include "arrow/record_batch.h"
 #include "arrow/result.h"
 #include "arrow/status.h"
 #include "arrow/type.h"
+#include "arrow/type_fwd.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit_util.h"
 #include "arrow/util/checked_cast.h"
@@ -47,26 +50,27 @@
 #include "arrow/visitor_inline.h"
 
 namespace arrow {
-namespace {
-constexpr char kData[] = "DATA";
-constexpr char kDays[] = "days";
-constexpr char kDayTime[] = "DAY_TIME";
-constexpr char kDuration[] = "duration";
-constexpr char kMilliseconds[] = "milliseconds";
-constexpr char kYearMonth[] = "YEAR_MONTH";
-}  // namespace
-
-class MemoryPool;
 
 using internal::checked_cast;
 using internal::ParseValue;
 
+using ipc::DictionaryFieldMapper;
+using ipc::DictionaryMemo;
+using ipc::internal::FieldPosition;
+
 namespace testing {
 namespace json {
 
-using ::arrow::ipc::DictionaryMemo;
+namespace {
 
-static std::string GetFloatingPrecisionName(FloatingPointType::Precision precision) {
+constexpr char kData[] = "DATA";
+constexpr char kDays[] = "days";
+constexpr char kDayTime[] = "DAY_TIME";
+constexpr char kDuration[] = "duration";
+constexpr char kMilliseconds[] = "milliseconds";
+constexpr char kYearMonth[] = "YEAR_MONTH";
+
+std::string GetFloatingPrecisionName(FloatingPointType::Precision precision) {
   switch (precision) {
     case FloatingPointType::HALF:
       return "HALF";
@@ -80,7 +84,7 @@ static std::string GetFloatingPrecisionName(FloatingPointType::Precision precisi
   return "UNKNOWN";
 }
 
-static std::string GetTimeUnitName(TimeUnit::type unit) {
+std::string GetTimeUnitName(TimeUnit::type unit) {
   switch (unit) {
     case TimeUnit::SECOND:
       return "SECOND";
@@ -98,17 +102,21 @@ static std::string GetTimeUnitName(TimeUnit::type unit) {
 
 class SchemaWriter {
  public:
-  explicit SchemaWriter(const Schema& schema, DictionaryMemo* dictionary_memo,
+  explicit SchemaWriter(const Schema& schema, const DictionaryFieldMapper& mapper,
                         RjWriter* writer)
-      : schema_(schema), dictionary_memo_(dictionary_memo), writer_(writer) {}
+      : schema_(schema), mapper_(mapper), writer_(writer) {}
 
   Status Write() {
     writer_->Key("schema");
     writer_->StartObject();
     writer_->Key("fields");
     writer_->StartArray();
+
+    FieldPosition field_pos;
+    int i = 0;
     for (const std::shared_ptr<Field>& field : schema_.fields()) {
-      RETURN_NOT_OK(VisitField(field));
+      RETURN_NOT_OK(VisitField(field, field_pos.child(i)));
+      ++i;
     }
     writer_->EndArray();
     WriteKeyValueMetadata(schema_.metadata());
@@ -168,7 +176,7 @@ class SchemaWriter {
     return Status::OK();
   }
 
-  Status VisitField(const std::shared_ptr<Field>& field) {
+  Status VisitField(const std::shared_ptr<Field>& field, FieldPosition field_pos) {
     writer_->StartObject();
 
     writer_->Key("name");
@@ -196,12 +204,12 @@ class SchemaWriter {
       const auto& dict_type = checked_cast<const DictionaryType&>(*type);
       // Ensure we visit child fields first so that, in the case of nested
       // dictionaries, inner dictionaries get a smaller id than outer dictionaries.
-      RETURN_NOT_OK(WriteChildren(dict_type.value_type()->fields()));
-      int64_t dictionary_id = -1;
-      RETURN_NOT_OK(dictionary_memo_->GetOrAssignId(field, &dictionary_id));
+      RETURN_NOT_OK(WriteChildren(dict_type.value_type()->fields(), field_pos));
+      ARROW_ASSIGN_OR_RAISE(const int64_t dictionary_id,
+                            mapper_.GetFieldId(field_pos.path()));
       RETURN_NOT_OK(WriteDictionaryMetadata(dictionary_id, dict_type));
     } else {
-      RETURN_NOT_OK(WriteChildren(type->fields()));
+      RETURN_NOT_OK(WriteChildren(type->fields(), field_pos));
     }
 
     WriteKeyValueMetadata(field->metadata(), additional_metadata);
@@ -220,7 +228,7 @@ class SchemaWriter {
 
   void WriteTypeMetadata(const MapType& type) {
     writer_->Key("keysSorted");
-    writer_->Int(type.keys_sorted());
+    writer_->Bool(type.keys_sorted());
   }
 
   void WriteTypeMetadata(const IntegerType& type) {
@@ -297,6 +305,13 @@ class SchemaWriter {
     writer_->Int(type.scale());
   }
 
+  void WriteTypeMetadata(const Decimal256Type& type) {
+    writer_->Key("precision");
+    writer_->Int(type.precision());
+    writer_->Key("scale");
+    writer_->Int(type.scale());
+  }
+
   void WriteTypeMetadata(const UnionType& type) {
     writer_->Key("mode");
     switch (type.mode()) {
@@ -338,11 +353,14 @@ class SchemaWriter {
     return Status::OK();
   }
 
-  Status WriteChildren(const std::vector<std::shared_ptr<Field>>& children) {
+  Status WriteChildren(const std::vector<std::shared_ptr<Field>>& children,
+                       FieldPosition field_pos) {
     writer_->Key("children");
     writer_->StartArray();
+    int i = 0;
     for (const std::shared_ptr<Field>& field : children) {
-      RETURN_NOT_OK(VisitField(field));
+      RETURN_NOT_OK(VisitField(field, field_pos.child(i)));
+      ++i;
     }
     writer_->EndArray();
     return Status::OK();
@@ -367,6 +385,7 @@ class SchemaWriter {
   }
 
   Status Visit(const Decimal128Type& type) { return WritePrimitive("decimal", type); }
+  Status Visit(const Decimal256Type& type) { return WritePrimitive("decimal256", type); }
   Status Visit(const TimestampType& type) { return WritePrimitive("timestamp", type); }
   Status Visit(const DurationType& type) { return WritePrimitive(kDuration, type); }
   Status Visit(const MonthIntervalType& type) { return WritePrimitive("interval", type); }
@@ -411,7 +430,7 @@ class SchemaWriter {
 
  private:
   const Schema& schema_;
-  DictionaryMemo* dictionary_memo_;
+  const DictionaryFieldMapper& mapper_;
   RjWriter* writer_;
 };
 
@@ -537,6 +556,18 @@ class ArrayWriter {
     }
   }
 
+  void WriteDataValues(const Decimal256Array& arr) {
+    static const char null_string[] = "0";
+    for (int64_t i = 0; i < arr.length(); ++i) {
+      if (arr.IsValid(i)) {
+        const Decimal256 value(arr.GetValue(i));
+        writer_->String(value.ToIntegerString());
+      } else {
+        writer_->String(null_string, sizeof(null_string));
+      }
+    }
+  }
+
   void WriteDataValues(const BooleanArray& arr) {
     for (int64_t i = 0; i < arr.length(); ++i) {
       if (arr.IsValid(i)) {
@@ -686,38 +717,60 @@ class ArrayWriter {
   RjWriter* writer_;
 };
 
-static Status GetObjectInt(const RjObject& obj, const std::string& key, int* out) {
+Result<TimeUnit::type> GetUnitFromString(const std::string& unit_str) {
+  if (unit_str == "SECOND") {
+    return TimeUnit::SECOND;
+  } else if (unit_str == "MILLISECOND") {
+    return TimeUnit::MILLI;
+  } else if (unit_str == "MICROSECOND") {
+    return TimeUnit::MICRO;
+  } else if (unit_str == "NANOSECOND") {
+    return TimeUnit::NANO;
+  } else {
+    return Status::Invalid("Invalid time unit: ", unit_str);
+  }
+}
+
+template <typename IntType = int>
+Result<IntType> GetMemberInt(const RjObject& obj, const std::string& key) {
   const auto& it = obj.FindMember(key);
   RETURN_NOT_INT(key, it, obj);
-  *out = it->value.GetInt();
-  return Status::OK();
+  return static_cast<IntType>(it->value.GetInt64());
 }
 
-static Status GetObjectBool(const RjObject& obj, const std::string& key, bool* out) {
+Result<bool> GetMemberBool(const RjObject& obj, const std::string& key) {
   const auto& it = obj.FindMember(key);
   RETURN_NOT_BOOL(key, it, obj);
-  *out = it->value.GetBool();
-  return Status::OK();
+  return it->value.GetBool();
 }
 
-static Status GetObjectString(const RjObject& obj, const std::string& key,
-                              std::string* out) {
+Result<std::string> GetMemberString(const RjObject& obj, const std::string& key) {
   const auto& it = obj.FindMember(key);
   RETURN_NOT_STRING(key, it, obj);
-  *out = it->value.GetString();
-  return Status::OK();
+  return it->value.GetString();
+}
+
+Result<const RjObject> GetMemberObject(const RjObject& obj, const std::string& key) {
+  const auto& it = obj.FindMember(key);
+  RETURN_NOT_OBJECT(key, it, obj);
+  return it->value.GetObject();
 }
 
-static Status GetInteger(const rj::Value::ConstObject& json_type,
-                         std::shared_ptr<DataType>* type) {
-  const auto& it_bit_width = json_type.FindMember("bitWidth");
-  RETURN_NOT_INT("bitWidth", it_bit_width, json_type);
+Result<const RjArray> GetMemberArray(const RjObject& obj, const std::string& key) {
+  const auto& it = obj.FindMember(key);
+  RETURN_NOT_ARRAY(key, it, obj);
+  return it->value.GetArray();
+}
 
-  const auto& it_is_signed = json_type.FindMember("isSigned");
-  RETURN_NOT_BOOL("isSigned", it_is_signed, json_type);
+Result<TimeUnit::type> GetMemberTimeUnit(const RjObject& obj, const std::string& key) {
+  ARROW_ASSIGN_OR_RAISE(const auto unit_str, GetMemberString(obj, key));
+  return GetUnitFromString(unit_str);
+}
 
-  bool is_signed = it_is_signed->value.GetBool();
-  int bit_width = it_bit_width->value.GetInt();
+Status GetInteger(const rj::Value::ConstObject& json_type,
+                  std::shared_ptr<DataType>* type) {
+  ARROW_ASSIGN_OR_RAISE(const bool is_signed, GetMemberBool(json_type, "isSigned"));
+  ARROW_ASSIGN_OR_RAISE(const int bit_width, GetMemberInt<int>(json_type, "bitWidth"));
 
   switch (bit_width) {
     case 8:
@@ -738,12 +791,8 @@ static Status GetInteger(const rj::Value::ConstObject& json_type,
   return Status::OK();
 }
 
-static Status GetFloatingPoint(const RjObject& json_type,
-                               std::shared_ptr<DataType>* type) {
-  const auto& it_precision = json_type.FindMember("precision");
-  RETURN_NOT_STRING("precision", it_precision, json_type);
-
-  std::string precision = it_precision->value.GetString();
+Status GetFloatingPoint(const RjObject& json_type, std::shared_ptr<DataType>* type) {
+  ARROW_ASSIGN_OR_RAISE(const auto precision, GetMemberString(json_type, "precision"));
 
   if (precision == "DOUBLE") {
     *type = float64();
@@ -757,61 +806,60 @@ static Status GetFloatingPoint(const RjObject& json_type,
   return Status::OK();
 }
 
-static Status GetMap(const RjObject& json_type,
-                     const std::vector<std::shared_ptr<Field>>& children,
-                     std::shared_ptr<DataType>* type) {
+Status GetMap(const RjObject& json_type,
+              const std::vector<std::shared_ptr<Field>>& children,
+              std::shared_ptr<DataType>* type) {
   if (children.size() != 1) {
     return Status::Invalid("Map must have exactly one child");
   }
 
-  const auto& it_keys_sorted = json_type.FindMember("keysSorted");
-  RETURN_NOT_BOOL("keysSorted", it_keys_sorted, json_type);
-  bool keys_sorted = it_keys_sorted->value.GetBool();
-
+  ARROW_ASSIGN_OR_RAISE(const bool keys_sorted, GetMemberBool(json_type, "keysSorted"));
   return MapType::Make(children[0], keys_sorted).Value(type);
 }
 
-static Status GetFixedSizeBinary(const RjObject& json_type,
-                                 std::shared_ptr<DataType>* type) {
-  const auto& it_byte_width = json_type.FindMember("byteWidth");
-  RETURN_NOT_INT("byteWidth", it_byte_width, json_type);
-
-  int32_t byte_width = it_byte_width->value.GetInt();
+Status GetFixedSizeBinary(const RjObject& json_type, std::shared_ptr<DataType>* type) {
+  ARROW_ASSIGN_OR_RAISE(const int32_t byte_width,
+                        GetMemberInt<int32_t>(json_type, "byteWidth"));
   *type = fixed_size_binary(byte_width);
   return Status::OK();
 }
 
-static Status GetFixedSizeList(const RjObject& json_type,
-                               const std::vector<std::shared_ptr<Field>>& children,
-                               std::shared_ptr<DataType>* type) {
+Status GetFixedSizeList(const RjObject& json_type,
+                        const std::vector<std::shared_ptr<Field>>& children,
+                        std::shared_ptr<DataType>* type) {
   if (children.size() != 1) {
     return Status::Invalid("FixedSizeList must have exactly one child");
   }
 
-  const auto& it_list_size = json_type.FindMember("listSize");
-  RETURN_NOT_INT("listSize", it_list_size, json_type);
-
-  int32_t list_size = it_list_size->value.GetInt();
+  ARROW_ASSIGN_OR_RAISE(const int32_t list_size,
+                        GetMemberInt<int32_t>(json_type, "listSize"));
   *type = fixed_size_list(children[0], list_size);
   return Status::OK();
 }
 
-static Status GetDecimal(const RjObject& json_type, std::shared_ptr<DataType>* type) {
-  const auto& it_precision = json_type.FindMember("precision");
-  const auto& it_scale = json_type.FindMember("scale");
-
-  RETURN_NOT_INT("precision", it_precision, json_type);
-  RETURN_NOT_INT("scale", it_scale, json_type);
+Status GetDecimal(const RjObject& json_type, std::shared_ptr<DataType>* type) {
+  ARROW_ASSIGN_OR_RAISE(const int32_t precision,
+                        GetMemberInt<int32_t>(json_type, "precision"));
+  ARROW_ASSIGN_OR_RAISE(const int32_t scale, GetMemberInt<int32_t>(json_type, "scale"));
+  int32_t bit_width = 128;
+  Result<int32_t> maybe_bit_width = GetMemberInt<int32_t>(json_type, "bitWidth");
+  if (maybe_bit_width.ok()) {
+    bit_width = maybe_bit_width.ValueOrDie();
+  }
 
-  *type = decimal(it_precision->value.GetInt(), it_scale->value.GetInt());
+  if (bit_width == 128) {
+    *type = decimal128(precision, scale);
+  } else if (bit_width == 256) {
+    *type = decimal256(precision, scale);
+  } else {
+    return Status::Invalid("Only 128 bit and 256 Decimals are supported. Received",
+                           bit_width);
+  }
   return Status::OK();
 }
 
-static Status GetDate(const RjObject& json_type, std::shared_ptr<DataType>* type) {
-  const auto& it_unit = json_type.FindMember("unit");
-  RETURN_NOT_STRING("unit", it_unit, json_type);
-
-  std::string unit_str = it_unit->value.GetString();
+Status GetDate(const RjObject& json_type, std::shared_ptr<DataType>* type) {
+  ARROW_ASSIGN_OR_RAISE(const auto unit_str, GetMemberString(json_type, "unit"));
 
   if (unit_str == "DAY") {
     *type = date32();
@@ -823,14 +871,9 @@ static Status GetDate(const RjObject& json_type, std::shared_ptr<DataType>* type
   return Status::OK();
 }
 
-static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type) {
-  const auto& it_unit = json_type.FindMember("unit");
-  RETURN_NOT_STRING("unit", it_unit, json_type);
-
-  const auto& it_bit_width = json_type.FindMember("bitWidth");
-  RETURN_NOT_INT("bitWidth", it_bit_width, json_type);
-
-  std::string unit_str = it_unit->value.GetString();
+Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type) {
+  ARROW_ASSIGN_OR_RAISE(const auto unit_str, GetMemberString(json_type, "unit"));
+  ARROW_ASSIGN_OR_RAISE(const int bit_width, GetMemberInt<int>(json_type, "bitWidth"));
 
   if (unit_str == "SECOND") {
     *type = time32(TimeUnit::SECOND);
@@ -846,7 +889,6 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
 
   const auto& fw_type = checked_cast<const FixedWidthType&>(**type);
 
-  int bit_width = it_bit_width->value.GetInt();
   if (bit_width != fw_type.bit_width()) {
     return Status::Invalid("Indicated bit width does not match unit");
   }
@@ -854,79 +896,45 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
   return Status::OK();
 }
 
-static Status GetUnitFromString(const std::string& unit_str, TimeUnit::type* unit) {
-  if (unit_str == "SECOND") {
-    *unit = TimeUnit::SECOND;
-  } else if (unit_str == "MILLISECOND") {
-    *unit = TimeUnit::MILLI;
-  } else if (unit_str == "MICROSECOND") {
-    *unit = TimeUnit::MICRO;
-  } else if (unit_str == "NANOSECOND") {
-    *unit = TimeUnit::NANO;
-  } else {
-    return Status::Invalid("Invalid time unit: ", unit_str);
-  }
-  return Status::OK();
-}
-
-static Status GetDuration(const RjObject& json_type, std::shared_ptr<DataType>* type) {
-  const auto& it_unit = json_type.FindMember("unit");
-  RETURN_NOT_STRING("unit", it_unit, json_type);
-
-  std::string unit_str = it_unit->value.GetString();
-
-  TimeUnit::type unit;
-  RETURN_NOT_OK(GetUnitFromString(unit_str, &unit));
-
+Status GetDuration(const RjObject& json_type, std::shared_ptr<DataType>* type) {
+  ARROW_ASSIGN_OR_RAISE(const TimeUnit::type unit, GetMemberTimeUnit(json_type, "unit"));
   *type = duration(unit);
-
   return Status::OK();
 }
 
-static Status GetTimestamp(const RjObject& json_type, std::shared_ptr<DataType>* type) {
-  const auto& it_unit = json_type.FindMember("unit");
-  RETURN_NOT_STRING("unit", it_unit, json_type);
-
-  std::string unit_str = it_unit->value.GetString();
-
-  TimeUnit::type unit;
-  RETURN_NOT_OK(GetUnitFromString(unit_str, &unit));
+Status GetTimestamp(const RjObject& json_type, std::shared_ptr<DataType>* type) {
+  ARROW_ASSIGN_OR_RAISE(const TimeUnit::type unit, GetMemberTimeUnit(json_type, "unit"));
 
   const auto& it_tz = json_type.FindMember("timezone");
   if (it_tz == json_type.MemberEnd()) {
     *type = timestamp(unit);
   } else {
+    RETURN_NOT_STRING("timezone", it_tz, json_type);
     *type = timestamp(unit, it_tz->value.GetString());
   }
 
   return Status::OK();
 }
 
-static Status GetInterval(const RjObject& json_type, std::shared_ptr<DataType>* type) {
-  const auto& it_unit = json_type.FindMember("unit");
-  RETURN_NOT_STRING("unit", it_unit, json_type);
-
-  std::string unit_name = it_unit->value.GetString();
+Status GetInterval(const RjObject& json_type, std::shared_ptr<DataType>* type) {
+  ARROW_ASSIGN_OR_RAISE(const auto unit_str, GetMemberString(json_type, "unit"));
 
-  if (unit_name == kDayTime) {
+  if (unit_str == kDayTime) {
     *type = day_time_interval();
-  } else if (unit_name == kYearMonth) {
+  } else if (unit_str == kYearMonth) {
     *type = month_interval();
   } else {
-    return Status::Invalid("Invalid interval unit: " + unit_name);
+    return Status::Invalid("Invalid interval unit: " + unit_str);
   }
   return Status::OK();
 }
 
-static Status GetUnion(const RjObject& json_type,
-                       const std::vector<std::shared_ptr<Field>>& children,
-                       std::shared_ptr<DataType>* type) {
-  const auto& it_mode = json_type.FindMember("mode");
-  RETURN_NOT_STRING("mode", it_mode, json_type);
+Status GetUnion(const RjObject& json_type,
+                const std::vector<std::shared_ptr<Field>>& children,
+                std::shared_ptr<DataType>* type) {
+  ARROW_ASSIGN_OR_RAISE(const auto mode_str, GetMemberString(json_type, "mode"));
 
-  std::string mode_str = it_mode->value.GetString();
   UnionMode::type mode;
-
   if (mode_str == "SPARSE") {
     mode = UnionMode::SPARSE;
   } else if (mode_str == "DENSE") {
@@ -935,14 +943,14 @@ static Status GetUnion(const RjObject& json_type,
     return Status::Invalid("Invalid union mode: ", mode_str);
   }
 
-  const auto& it_type_codes = json_type.FindMember("typeIds");
-  RETURN_NOT_ARRAY("typeIds", it_type_codes, json_type);
+  ARROW_ASSIGN_OR_RAISE(const auto json_type_codes, GetMemberArray(json_type, "typeIds"));
 
   std::vector<int8_t> type_codes;
-  const auto& id_array = it_type_codes->value.GetArray();
-  type_codes.reserve(id_array.Size());
-  for (const rj::Value& val : id_array) {
-    DCHECK(val.IsInt());
+  type_codes.reserve(json_type_codes.Size());
+  for (const rj::Value& val : json_type_codes) {
+    if (!val.IsInt()) {
+      return Status::Invalid("Union type codes must be integers");
+    }
     type_codes.push_back(static_cast<int8_t>(val.GetInt()));
   }
 
@@ -955,13 +963,10 @@ static Status GetUnion(const RjObject& json_type,
   return Status::OK();
 }
 
-static Status GetType(const RjObject& json_type,
-                      const std::vector<std::shared_ptr<Field>>& children,
-                      std::shared_ptr<DataType>* type) {
-  const auto& it_type_name = json_type.FindMember("name");
-  RETURN_NOT_STRING("name", it_type_name, json_type);
-
-  std::string type_name = it_type_name->value.GetString();
+Status GetType(const RjObject& json_type,
+               const std::vector<std::shared_ptr<Field>>& children,
+               std::shared_ptr<DataType>* type) {
+  ARROW_ASSIGN_OR_RAISE(const auto type_name, GetMemberString(json_type, "name"));
 
   if (type_name == "int") {
     return GetInteger(json_type, type);
@@ -1017,35 +1022,28 @@ static Status GetType(const RjObject& json_type,
   return Status::OK();
 }
 
-static Status GetField(const rj::Value& obj, DictionaryMemo* dictionary_memo,
-                       std::shared_ptr<Field>* field);
+Status GetField(const rj::Value& obj, FieldPosition field_pos,
+                DictionaryMemo* dictionary_memo, std::shared_ptr<Field>* field);
 
-static Status GetFieldsFromArray(const rj::Value& obj, DictionaryMemo* dictionary_memo,
-                                 std::vector<std::shared_ptr<Field>>* fields) {
-  const auto& values = obj.GetArray();
-
-  fields->resize(values.Size());
-  for (rj::SizeType i = 0; i < fields->size(); ++i) {
-    RETURN_NOT_OK(GetField(values[i], dictionary_memo, &(*fields)[i]));
+Status GetFieldsFromArray(const RjArray& json_fields, FieldPosition parent_pos,
+                          DictionaryMemo* dictionary_memo,
+                          std::vector<std::shared_ptr<Field>>* fields) {
+  fields->resize(json_fields.Size());
+  for (rj::SizeType i = 0; i < json_fields.Size(); ++i) {
+    RETURN_NOT_OK(GetField(json_fields[i], parent_pos.child(static_cast<int>(i)),
+                           dictionary_memo, &(*fields)[i]));
   }
   return Status::OK();
 }
 
-static Status ParseDictionary(const RjObject& obj, int64_t* id, bool* is_ordered,
-                              std::shared_ptr<DataType>* index_type) {
-  int32_t int32_id;
-  RETURN_NOT_OK(GetObjectInt(obj, "id", &int32_id));
-  *id = int32_id;
-
-  RETURN_NOT_OK(GetObjectBool(obj, "isOrdered", is_ordered));
-
-  const auto& it_index_type = obj.FindMember("indexType");
-  RETURN_NOT_OBJECT("indexType", it_index_type, obj);
+Status ParseDictionary(const RjObject& obj, int64_t* id, bool* is_ordered,
+                       std::shared_ptr<DataType>* index_type) {
+  ARROW_ASSIGN_OR_RAISE(*id, GetMemberInt<int64_t>(obj, "id"));
+  ARROW_ASSIGN_OR_RAISE(*is_ordered, GetMemberBool(obj, "isOrdered"));
 
-  const auto& json_index_type = it_index_type->value.GetObject();
+  ARROW_ASSIGN_OR_RAISE(const auto json_index_type, GetMemberObject(obj, "indexType"));
 
-  std::string type_name;
-  RETURN_NOT_OK(GetObjectString(json_index_type, "name", &type_name));
+  ARROW_ASSIGN_OR_RAISE(const auto type_name, GetMemberString(json_index_type, "name"));
   if (type_name != "int") {
     return Status::Invalid("Dictionary indices can only be integers");
   }
@@ -1053,66 +1051,56 @@ static Status ParseDictionary(const RjObject& obj, int64_t* id, bool* is_ordered
 }
 
 template <typename FieldOrStruct>
-static Status GetKeyValueMetadata(const FieldOrStruct& field_or_struct,
-                                  std::shared_ptr<KeyValueMetadata>* out) {
+Status GetKeyValueMetadata(const FieldOrStruct& field_or_struct,
+                           std::shared_ptr<KeyValueMetadata>* out) {
   out->reset(new KeyValueMetadata);
   auto it = field_or_struct.FindMember("metadata");
-  if (it == field_or_struct.MemberEnd()) {
+  if (it == field_or_struct.MemberEnd() || it->value.IsNull()) {
     return Status::OK();
   }
-
-  if (it->value.IsNull()) {
-    return Status::OK();
-  }
-
   if (!it->value.IsArray()) {
     return Status::Invalid("Metadata was not a JSON array");
   }
-  const auto& key_value_pairs = it->value.GetArray();
 
-  for (auto it = key_value_pairs.Begin(); it != key_value_pairs.End(); ++it) {
-    if (!it->IsObject()) {
+  for (const auto& val : it->value.GetArray()) {
+    if (!val.IsObject()) {
       return Status::Invalid("Metadata KeyValue was not a JSON object");
     }
-    const auto& key_value_pair = it->GetObject();
+    const auto& key_value_pair = val.GetObject();
 
-    std::string key, value;
-    RETURN_NOT_OK(GetObjectString(key_value_pair, "key", &key));
-    RETURN_NOT_OK(GetObjectString(key_value_pair, "value", &value));
+    ARROW_ASSIGN_OR_RAISE(const auto key, GetMemberString(key_value_pair, "key"));
+    ARROW_ASSIGN_OR_RAISE(const auto value, GetMemberString(key_value_pair, "value"));
 
     (*out)->Append(std::move(key), std::move(value));
   }
   return Status::OK();
 }
 
-static Status GetField(const rj::Value& obj, DictionaryMemo* dictionary_memo,
-                       std::shared_ptr<Field>* field) {
+Status GetField(const rj::Value& obj, FieldPosition field_pos,
+                DictionaryMemo* dictionary_memo, std::shared_ptr<Field>* field) {
   if (!obj.IsObject()) {
     return Status::Invalid("Field was not a JSON object");
   }
   const auto& json_field = obj.GetObject();
 
-  std::string name;
-  bool nullable;
-  RETURN_NOT_OK(GetObjectString(json_field, "name", &name));
-  RETURN_NOT_OK(GetObjectBool(json_field, "nullable", &nullable));
-
   std::shared_ptr<DataType> type;
-  const auto& it_type = json_field.FindMember("type");
-  RETURN_NOT_OBJECT("type", it_type, json_field);
 
-  const auto& it_children = json_field.FindMember("children");
-  RETURN_NOT_ARRAY("children", it_children, json_field);
+  ARROW_ASSIGN_OR_RAISE(const auto name, GetMemberString(json_field, "name"));
+  ARROW_ASSIGN_OR_RAISE(const bool nullable, GetMemberBool(json_field, "nullable"));
+
+  ARROW_ASSIGN_OR_RAISE(const auto json_type, GetMemberObject(json_field, "type"));
+  ARROW_ASSIGN_OR_RAISE(const auto json_children, GetMemberArray(json_field, "children"));
 
   std::vector<std::shared_ptr<Field>> children;
-  RETURN_NOT_OK(GetFieldsFromArray(it_children->value, dictionary_memo, &children));
-  RETURN_NOT_OK(GetType(it_type->value.GetObject(), children, &type));
+  RETURN_NOT_OK(GetFieldsFromArray(json_children, field_pos, dictionary_memo, &children));
+  RETURN_NOT_OK(GetType(json_type, children, &type));
 
   std::shared_ptr<KeyValueMetadata> metadata;
   RETURN_NOT_OK(GetKeyValueMetadata(json_field, &metadata));
 
   // Is it a dictionary type?
   int64_t dictionary_id = -1;
+  std::shared_ptr<DataType> dict_value_type;
   const auto& it_dictionary = json_field.FindMember("dictionary");
   if (dictionary_memo != nullptr && it_dictionary != json_field.MemberEnd()) {
     // Parse dictionary id in JSON and add dictionary field to the
@@ -1123,6 +1111,7 @@ static Status GetField(const rj::Value& obj, DictionaryMemo* dictionary_memo,
     RETURN_NOT_OK(ParseDictionary(it_dictionary->value.GetObject(), &dictionary_id,
                                   &is_ordered, &index_type));
 
+    dict_value_type = type;
     type = ::arrow::dictionary(index_type, type, is_ordered);
   }
 
@@ -1151,7 +1140,8 @@ static Status GetField(const rj::Value& obj, DictionaryMemo* dictionary_memo,
   // Create field
   *field = ::arrow::field(name, type, nullable, metadata);
   if (dictionary_id != -1) {
-    RETURN_NOT_OK(dictionary_memo->AddField(dictionary_id, *field));
+    RETURN_NOT_OK(dictionary_memo->fields().AddField(dictionary_id, field_pos.path()));
+    RETURN_NOT_OK(dictionary_memo->AddDictionaryType(dictionary_id, dict_value_type));
   }
 
   return Status::OK();
@@ -1192,36 +1182,40 @@ enable_if_physical_floating_point<T, typename T::c_type> UnboxValue(
 
 class ArrayReader {
  public:
-  ArrayReader(const RjObject& obj, MemoryPool* pool, const std::shared_ptr<Field>& field,
-              DictionaryMemo* dictionary_memo)
-      : obj_(obj),
-        pool_(pool),
-        field_(field),
-        type_(field->type()),
-        dictionary_memo_(dictionary_memo),
-        dictionary_id_(-1) {}
+  ArrayReader(const RjObject& obj, MemoryPool* pool, const std::shared_ptr<Field>& field)
+      : obj_(obj), pool_(pool), field_(field), type_(field->type()) {}
+
+  template <typename BuilderType>
+  Status FinishBuilder(BuilderType* builder) {
+    std::shared_ptr<Array> array;
+    RETURN_NOT_OK(builder->Finish(&array));
+    data_ = array->data();
+    return Status::OK();
+  }
+
+  Result<const RjArray> GetDataArray(const RjObject& obj) {
+    ARROW_ASSIGN_OR_RAISE(const auto json_data_arr, GetMemberArray(obj, kData));
+    if (static_cast<int32_t>(json_data_arr.Size()) != length_) {
+      return Status::Invalid("JSON DATA array size differs from advertised array length");
+    }
+    return json_data_arr;
+  }
 
   template <typename T>
   enable_if_has_c_type<T, Status> Visit(const T& type) {
     typename TypeTraits<T>::BuilderType builder(type_, pool_);
 
-    const auto& json_data = obj_.FindMember(kData);
-    RETURN_NOT_ARRAY(kData, json_data, obj_);
+    ARROW_ASSIGN_OR_RAISE(const auto json_data_arr, GetDataArray(obj_));
 
-    const auto& json_data_arr = json_data->value.GetArray();
-
-    DCHECK_EQ(static_cast<int32_t>(json_data_arr.Size()), length_);
     for (int i = 0; i < length_; ++i) {
       if (!is_valid_[i]) {
         RETURN_NOT_OK(builder.AppendNull());
         continue;
       }
-
       const rj::Value& val = json_data_arr[i];
       RETURN_NOT_OK(builder.Append(UnboxValue<T>(val)));
     }
-
-    return builder.Finish(&result_);
+    return FinishBuilder(&builder);
   }
 
   template <typename T>
@@ -1229,19 +1223,13 @@ class ArrayReader {
     typename TypeTraits<T>::BuilderType builder(pool_);
     using offset_type = typename T::offset_type;
 
-    const auto& json_data = obj_.FindMember(kData);
-    RETURN_NOT_ARRAY(kData, json_data, obj_);
-
-    const auto& json_data_arr = json_data->value.GetArray();
-
-    DCHECK_EQ(static_cast<int32_t>(json_data_arr.Size()), length_);
+    ARROW_ASSIGN_OR_RAISE(const auto json_data_arr, GetDataArray(obj_));
 
     for (int i = 0; i < length_; ++i) {
       if (!is_valid_[i]) {
         RETURN_NOT_OK(builder.AppendNull());
         continue;
       }
-
       const rj::Value& val = json_data_arr[i];
       DCHECK(val.IsString());
 
@@ -1266,20 +1254,13 @@ class ArrayReader {
             builder.Append(byte_buffer_data, static_cast<offset_type>(value_len)));
       }
     }
-
-    return builder.Finish(&result_);
+    return FinishBuilder(&builder);
   }
 
   Status Visit(const DayTimeIntervalType& type) {
     DayTimeIntervalBuilder builder(pool_);
 
-    const auto& json_data = obj_.FindMember(kData);
-    RETURN_NOT_ARRAY(kData, json_data, obj_);
-
-    const auto& json_data_arr = json_data->value.GetArray();
-
-    DCHECK_EQ(static_cast<int32_t>(json_data_arr.Size()), length_)
-        << "data length: " << json_data_arr.Size() << " != length_: " << length_;
+    ARROW_ASSIGN_OR_RAISE(const auto json_data_arr, GetDataArray(obj_));
 
     for (int i = 0; i < length_; ++i) {
       if (!is_valid_[i]) {
@@ -1294,7 +1275,7 @@ class ArrayReader {
       dm.milliseconds = val[kMilliseconds].GetInt();
       RETURN_NOT_OK(builder.Append(dm));
     }
-    return builder.Finish(&result_);
+    return FinishBuilder(&builder);
   }
 
   template <typename T>
@@ -1302,12 +1283,8 @@ class ArrayReader {
   Visit(const T& type) {
     typename TypeTraits<T>::BuilderType builder(type_, pool_);
 
-    const auto& json_data = obj_.FindMember(kData);
-    RETURN_NOT_ARRAY(kData, json_data, obj_);
+    ARROW_ASSIGN_OR_RAISE(const auto json_data_arr, GetDataArray(obj_));
 
-    const auto& json_data_arr = json_data->value.GetArray();
-
-    DCHECK_EQ(static_cast<int32_t>(json_data_arr.Size()), length_);
     int32_t byte_width = type.byte_width();
 
     // Allocate space for parsed values
@@ -1334,19 +1311,14 @@ class ArrayReader {
         RETURN_NOT_OK(builder.Append(byte_buffer_data));
       }
     }
-    return builder.Finish(&result_);
+    return FinishBuilder(&builder);
   }
 
   template <typename T>
   enable_if_decimal<T, Status> Visit(const T& type) {
     typename TypeTraits<T>::BuilderType builder(type_, pool_);
 
-    const auto& json_data = obj_.FindMember(kData);
-    RETURN_NOT_ARRAY(kData, json_data, obj_);
-
-    const auto& json_data_arr = json_data->value.GetArray();
-
-    DCHECK_EQ(static_cast<int32_t>(json_data_arr.Size()), length_);
+    ARROW_ASSIGN_OR_RAISE(const auto json_data_arr, GetDataArray(obj_));
 
     for (int i = 0; i < length_; ++i) {
       if (!is_valid_[i]) {
@@ -1358,12 +1330,14 @@ class ArrayReader {
         DCHECK_GT(val.GetStringLength(), 0)
             << "Empty string found when parsing Decimal128 value";
 
-        Decimal128 value;
-        ARROW_ASSIGN_OR_RAISE(value, Decimal128::FromString(val.GetString()));
+        using Value = typename TypeTraits<T>::ScalarType::ValueType;
+        Value value;
+        ARROW_ASSIGN_OR_RAISE(value, Value::FromString(val.GetString()));
         RETURN_NOT_OK(builder.Append(value));
       }
     }
-    return builder.Finish(&result_);
+
+    return FinishBuilder(&builder);
   }
 
   template <typename T>
@@ -1386,7 +1360,6 @@ class ArrayReader {
     } else {
       // Read 64-bit integers as strings, as JSON numbers cannot represent
       // them exactly.
-
       for (int i = 0; i < length; ++i) {
         const rj::Value& val = json_array[i];
         DCHECK(val.IsString());
@@ -1403,174 +1376,137 @@ class ArrayReader {
   }
 
   template <typename T>
-  Status CreateList(const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out) {
+  Status CreateList(const std::shared_ptr<DataType>& type) {
     using offset_type = typename T::offset_type;
-    using ArrayType = typename TypeTraits<T>::ArrayType;
-
-    int32_t null_count = 0;
-    std::shared_ptr<Buffer> validity_buffer;
-    RETURN_NOT_OK(GetValidityBuffer(is_valid_, &null_count, &validity_buffer));
 
-    const auto& json_offsets = obj_.FindMember("OFFSET");
-    RETURN_NOT_ARRAY("OFFSET", json_offsets, obj_);
-    std::shared_ptr<Buffer> offsets_buffer;
-    RETURN_NOT_OK(GetIntArray<offset_type>(json_offsets->value.GetArray(), length_ + 1,
-                                           &offsets_buffer));
-
-    std::vector<std::shared_ptr<Array>> children;
-    RETURN_NOT_OK(GetChildren(obj_, *type, &children));
-    DCHECK_EQ(children.size(), 1);
+    RETURN_NOT_OK(InitializeData(2));
 
-    out->reset(new ArrayType(type, length_, offsets_buffer, children[0], validity_buffer,
-                             null_count));
+    RETURN_NOT_OK(GetNullBitmap());
+    ARROW_ASSIGN_OR_RAISE(const auto json_offsets, GetMemberArray(obj_, "OFFSET"));
+    RETURN_NOT_OK(
+        GetIntArray<offset_type>(json_offsets, length_ + 1, &data_->buffers[1]));
+    RETURN_NOT_OK(GetChildren(obj_, *type));
     return Status::OK();
   }
 
   template <typename T>
   enable_if_var_size_list<T, Status> Visit(const T& type) {
-    return CreateList<T>(type_, &result_);
+    return CreateList<T>(type_);
   }
 
   Status Visit(const MapType& type) {
     auto list_type = std::make_shared<ListType>(type.value_field());
-    std::shared_ptr<Array> list_array;
-    RETURN_NOT_OK(CreateList<ListType>(list_type, &list_array));
-    auto map_data = list_array->data();
-    map_data->type = type_;
-    result_ = std::make_shared<MapArray>(map_data);
+    RETURN_NOT_OK(CreateList<ListType>(list_type));
+    data_->type = type_;
     return Status::OK();
   }
 
   Status Visit(const FixedSizeListType& type) {
-    int32_t null_count = 0;
-    std::shared_ptr<Buffer> validity_buffer;
-    RETURN_NOT_OK(GetValidityBuffer(is_valid_, &null_count, &validity_buffer));
-
-    std::vector<std::shared_ptr<Array>> children;
-    RETURN_NOT_OK(GetChildren(obj_, type, &children));
-    DCHECK_EQ(children.size(), 1);
-    DCHECK_EQ(children[0]->length(), type.list_size() * length_);
-
-    result_ = std::make_shared<FixedSizeListArray>(type_, length_, children[0],
-                                                   validity_buffer, null_count);
+    RETURN_NOT_OK(InitializeData(1));
+    RETURN_NOT_OK(GetNullBitmap());
 
+    RETURN_NOT_OK(GetChildren(obj_, type));
+    DCHECK_EQ(data_->child_data[0]->length, type.list_size() * length_);
     return Status::OK();
   }
 
   Status Visit(const StructType& type) {
-    int32_t null_count = 0;
-    std::shared_ptr<Buffer> validity_buffer;
-    RETURN_NOT_OK(GetValidityBuffer(is_valid_, &null_count, &validity_buffer));
-
-    std::vector<std::shared_ptr<Array>> fields;
-    RETURN_NOT_OK(GetChildren(obj_, type, &fields));
-
-    result_ = std::make_shared<StructArray>(type_, length_, fields, validity_buffer,
-                                            null_count);
+    RETURN_NOT_OK(InitializeData(1));
 
+    RETURN_NOT_OK(GetNullBitmap());
+    RETURN_NOT_OK(GetChildren(obj_, type));
     return Status::OK();
   }
 
-  Status Visit(const UnionType& type) {
-    std::shared_ptr<Buffer> type_id_buffer;
-
-    const auto& json_type_ids = obj_.FindMember("TYPE_ID");
-    RETURN_NOT_ARRAY("TYPE_ID", json_type_ids, obj_);
-    RETURN_NOT_OK(
-        GetIntArray<uint8_t>(json_type_ids->value.GetArray(), length_, &type_id_buffer));
+  Status GetUnionTypeIds() {
+    ARROW_ASSIGN_OR_RAISE(const auto json_type_ids, GetMemberArray(obj_, "TYPE_ID"));
+    return GetIntArray<uint8_t>(json_type_ids, length_, &data_->buffers[1]);
+  }
 
-    std::vector<std::shared_ptr<Array>> children;
-    RETURN_NOT_OK(GetChildren(obj_, type, &children));
+  Status Visit(const SparseUnionType& type) {
+    RETURN_NOT_OK(InitializeData(2));
 
-    if (type.mode() == UnionMode::SPARSE) {
-      result_ =
-          std::make_shared<SparseUnionArray>(type_, length_, children, type_id_buffer);
-    } else {
-      const auto& json_offsets = obj_.FindMember("OFFSET");
-      RETURN_NOT_ARRAY("OFFSET", json_offsets, obj_);
+    RETURN_NOT_OK(GetNullBitmap());
+    RETURN_NOT_OK(GetUnionTypeIds());
+    RETURN_NOT_OK(GetChildren(obj_, type));
+    return Status::OK();
+  }
 
-      std::shared_ptr<Buffer> offsets_buffer;
-      RETURN_NOT_OK(
-          GetIntArray<int32_t>(json_offsets->value.GetArray(), length_, &offsets_buffer));
+  Status Visit(const DenseUnionType& type) {
+    RETURN_NOT_OK(InitializeData(3));
 
-      result_ = std::make_shared<DenseUnionArray>(type_, length_, children,
-                                                  type_id_buffer, offsets_buffer);
-    }
+    RETURN_NOT_OK(GetNullBitmap());
+    RETURN_NOT_OK(GetUnionTypeIds());
+    RETURN_NOT_OK(GetChildren(obj_, type));
 
-    return Status::OK();
+    ARROW_ASSIGN_OR_RAISE(const auto json_offsets, GetMemberArray(obj_, "OFFSET"));
+    return GetIntArray<int32_t>(json_offsets, length_, &data_->buffers[2]);
   }
 
   Status Visit(const NullType& type) {
-    result_ = std::make_shared<NullArray>(length_);
+    data_ = std::make_shared<NullArray>(length_)->data();
     return Status::OK();
   }
 
   Status Visit(const DictionaryType& type) {
-    std::shared_ptr<Array> indices;
-
-    ArrayReader parser(obj_, pool_, ::arrow::field("indices", type.index_type()),
-                       dictionary_memo_);
-    RETURN_NOT_OK(parser.Parse(&indices));
-
-    RETURN_NOT_OK(LookupDictionaryId(field_));
-    std::shared_ptr<Array> dictionary;
-    RETURN_NOT_OK(dictionary_memo_->GetDictionary(dictionary_id_, &dictionary));
+    ArrayReader parser(obj_, pool_, ::arrow::field("indices", type.index_type()));
+    ARROW_ASSIGN_OR_RAISE(data_, parser.Parse());
 
-    result_ = std::make_shared<DictionaryArray>(field_->type(), indices, dictionary);
+    data_->type = field_->type();
+    // data_->dictionary will be filled later by ResolveDictionaries()
     return Status::OK();
   }
 
   Status Visit(const ExtensionType& type) {
-    std::shared_ptr<Array> storage_array;
-
-    ArrayReader parser(obj_, pool_, field_->WithType(type.storage_type()),
-                       dictionary_memo_);
+    ArrayReader parser(obj_, pool_, field_->WithType(type.storage_type()));
+    ARROW_ASSIGN_OR_RAISE(data_, parser.Parse());
+    data_->type = type_;
     // If the storage array is a dictionary array, lookup its dictionary id
     // using the extension field.
     // (the field is looked up by pointer, so the Field instance constructed
     //  above wouldn't work)
-    if (parser.type_->id() == Type::DICTIONARY) {
-      RETURN_NOT_OK(parser.LookupDictionaryId(field_));
-    }
-    RETURN_NOT_OK(parser.Parse(&storage_array));
-    result_ = std::make_shared<ExtensionArray>(type_, storage_array);
     return Status::OK();
   }
 
-  Status GetValidityBuffer(const std::vector<bool>& is_valid, int32_t* null_count,
-                           std::shared_ptr<Buffer>* validity_buffer) {
-    int length = static_cast<int>(is_valid.size());
+  Status InitializeData(int num_buffers) {
+    data_ = std::make_shared<ArrayData>(type_, length_);
+    data_->buffers.resize(num_buffers);
+    return Status::OK();
+  }
 
-    ARROW_ASSIGN_OR_RAISE(auto out_buffer, AllocateEmptyBitmap(length, pool_));
-    uint8_t* bitmap = out_buffer->mutable_data();
+  Status GetNullBitmap() {
+    const int64_t length = static_cast<int64_t>(is_valid_.size());
 
-    *null_count = 0;
-    for (int i = 0; i < length; ++i) {
-      if (!is_valid[i]) {
-        ++(*null_count);
-        continue;
+    ARROW_ASSIGN_OR_RAISE(data_->buffers[0], AllocateEmptyBitmap(length, pool_));
+    uint8_t* bitmap = data_->buffers[0]->mutable_data();
+
+    data_->null_count = 0;
+    for (int64_t i = 0; i < length; ++i) {
+      if (is_valid_[i]) {
+        BitUtil::SetBit(bitmap, i);
+      } else {
+        ++data_->null_count;
       }
-      BitUtil::SetBit(bitmap, i);
+    }
+    if (data_->null_count == 0) {
+      data_->buffers[0].reset();
     }
 
-    *validity_buffer = out_buffer;
     return Status::OK();
   }
+  Status GetChildren(const RjObject& obj, const DataType& type) {
+    ARROW_ASSIGN_OR_RAISE(const auto json_children, GetMemberArray(obj, "children"));
 
-  Status GetChildren(const RjObject& obj, const DataType& type,
-                     std::vector<std::shared_ptr<Array>>* array) {
-    const auto& json_children = obj.FindMember("children");
-    RETURN_NOT_ARRAY("children", json_children, obj);
-    const auto& json_children_arr = json_children->value.GetArray();
-
-    if (type.num_fields() != static_cast<int>(json_children_arr.Size())) {
+    if (type.num_fields() != static_cast<int>(json_children.Size())) {
       return Status::Invalid("Expected ", type.num_fields(), " children, but got ",
-                             json_children_arr.Size());
+                             json_children.Size());
     }
 
-    for (int i = 0; i < static_cast<int>(json_children_arr.Size()); ++i) {
-      const rj::Value& json_child = json_children_arr[i];
+    data_->child_data.resize(type.num_fields());
+    for (int i = 0; i < type.num_fields(); ++i) {
+      const rj::Value& json_child = json_children[i];
       DCHECK(json_child.IsObject());
+      const auto& child_obj = json_child.GetObject();
 
       std::shared_ptr<Field> child_field = type.field(i);
 
@@ -1578,20 +1514,18 @@ class ArrayReader {
       RETURN_NOT_STRING("name", it, json_child);
 
       DCHECK_EQ(it->value.GetString(), child_field->name());
-      std::shared_ptr<Array> child;
-      RETURN_NOT_OK(
-          ReadArray(pool_, json_children_arr[i], child_field, dictionary_memo_, &child));
-      array->emplace_back(child);
+      ArrayReader child_reader(child_obj, pool_, child_field);
+      ARROW_ASSIGN_OR_RAISE(data_->child_data[i], child_reader.Parse());
     }
 
     return Status::OK();
   }
 
   Status ParseValidityBitmap() {
-    const auto& json_valid_iter = obj_.FindMember("VALIDITY");
-    RETURN_NOT_ARRAY("VALIDITY", json_valid_iter, obj_);
-    const auto& json_validity = json_valid_iter->value.GetArray();
-    DCHECK_EQ(static_cast<int>(json_validity.Size()), length_);
+    ARROW_ASSIGN_OR_RAISE(const auto json_validity, GetMemberArray(obj_, "VALIDITY"));
+    if (static_cast<int>(json_validity.Size()) != length_) {
+      return Status::Invalid("JSON VALIDITY size differs from advertised array length");
+    }
     is_valid_.reserve(json_validity.Size());
     for (const rj::Value& val : json_validity) {
       DCHECK(val.IsInt());
@@ -1600,8 +1534,8 @@ class ArrayReader {
     return Status::OK();
   }
 
-  Status Parse(std::shared_ptr<Array>* out) {
-    RETURN_NOT_OK(GetObjectInt(obj_, "count", &length_));
+  Result<std::shared_ptr<ArrayData>> Parse() {
+    ARROW_ASSIGN_OR_RAISE(length_, GetMemberInt<int32_t>(obj_, "count"));
 
     if (::arrow::internal::HasValidityBitmap(type_->id())) {
       // Null and union types don't have a validity bitmap
@@ -1609,16 +1543,7 @@ class ArrayReader {
     }
 
     RETURN_NOT_OK(VisitTypeInline(*type_, this));
-
-    *out = result_;
-    return Status::OK();
-  }
-
-  Status LookupDictionaryId(const std::shared_ptr<Field>& field) {
-    if (dictionary_id_ == -1) {
-      RETURN_NOT_OK(dictionary_memo_->GetId(field.get(), &dictionary_id_));
-    }
-    return Status::OK();
+    return data_;
   }
 
  private:
@@ -1626,47 +1551,51 @@ class ArrayReader {
   MemoryPool* pool_;
   std::shared_ptr<Field> field_;
   std::shared_ptr<DataType> type_;
-  DictionaryMemo* dictionary_memo_;
-  int64_t dictionary_id_;
 
   // Parsed common attributes
   std::vector<bool> is_valid_;
   int32_t length_;
-  std::shared_ptr<Array> result_;
+  std::shared_ptr<ArrayData> data_;
 };
 
-Status WriteSchema(const Schema& schema, DictionaryMemo* dictionary_memo,
-                   RjWriter* json_writer) {
-  SchemaWriter converter(schema, dictionary_memo, json_writer);
-  return converter.Write();
+Result<std::shared_ptr<ArrayData>> ReadArrayData(MemoryPool* pool,
+                                                 const rj::Value& json_array,
+                                                 const std::shared_ptr<Field>& field) {
+  if (!json_array.IsObject()) {
+    return Status::Invalid("Array element was not a JSON object");
+  }
+  auto obj = json_array.GetObject();
+  ArrayReader parser(obj, pool, field);
+  return parser.Parse();
 }
 
-static Status ReadDictionary(const RjObject& obj, MemoryPool* pool,
-                             DictionaryMemo* dictionary_memo) {
-  int id;
-  RETURN_NOT_OK(GetObjectInt(obj, "id", &id));
+Status ReadDictionary(const RjObject& obj, MemoryPool* pool,
+                      DictionaryMemo* dictionary_memo) {
+  ARROW_ASSIGN_OR_RAISE(int64_t dictionary_id, GetMemberInt<int64_t>(obj, "id"));
 
-  const auto& it_data = obj.FindMember("data");
-  RETURN_NOT_OBJECT("data", it_data, obj);
+  ARROW_ASSIGN_OR_RAISE(const auto batch_obj, GetMemberObject(obj, "data"));
 
-  std::shared_ptr<DataType> value_type;
-  RETURN_NOT_OK(dictionary_memo->GetDictionaryType(id, &value_type));
-  auto value_field = ::arrow::field("dummy", value_type);
+  ARROW_ASSIGN_OR_RAISE(auto value_type,
+                        dictionary_memo->GetDictionaryType(dictionary_id));
 
-  // We need a placeholder schema to read the record, because the dictionary
-  // is embedded in a record batch with a single column.
-  std::shared_ptr<RecordBatch> batch;
-  RETURN_NOT_OK(ReadRecordBatch(it_data->value, ::arrow::schema({value_field}),
-                                dictionary_memo, pool, &batch));
+  ARROW_ASSIGN_OR_RAISE(const int64_t num_rows,
+                        GetMemberInt<int64_t>(batch_obj, "count"));
+  ARROW_ASSIGN_OR_RAISE(const auto json_columns, GetMemberArray(batch_obj, "columns"));
+  if (json_columns.Size() != 1) {
+    return Status::Invalid("Dictionary batch must contain only one column");
+  }
 
-  if (batch->num_columns() != 1) {
-    return Status::Invalid("Dictionary record batch must only contain one field");
+  ARROW_ASSIGN_OR_RAISE(auto dict_data,
+                        ReadArrayData(pool, json_columns[0], field("dummy", value_type)));
+  if (num_rows != dict_data->length) {
+    return Status::Invalid("Dictionary batch length mismatch: advertised (", num_rows,
+                           ") != actual (", dict_data->length, ")");
   }
-  return dictionary_memo->AddDictionary(id, batch->column(0));
+  return dictionary_memo->AddDictionary(dictionary_id, dict_data);
 }
 
-static Status ReadDictionaries(const rj::Value& doc, MemoryPool* pool,
-                               DictionaryMemo* dictionary_memo) {
+Status ReadDictionaries(const rj::Value& doc, MemoryPool* pool,
+                        DictionaryMemo* dictionary_memo) {
   auto it = doc.FindMember("dictionaries");
   if (it == doc.MemberEnd()) {
     // No dictionaries
@@ -1683,20 +1612,22 @@ static Status ReadDictionaries(const rj::Value& doc, MemoryPool* pool,
   return Status::OK();
 }
 
+}  // namespace
+
 Status ReadSchema(const rj::Value& json_schema, MemoryPool* pool,
                   DictionaryMemo* dictionary_memo, std::shared_ptr<Schema>* schema) {
-  auto it = json_schema.FindMember("schema");
-  RETURN_NOT_OBJECT("schema", it, json_schema);
-  const auto& obj_schema = it->value.GetObject();
+  DCHECK(json_schema.IsObject());
+  ARROW_ASSIGN_OR_RAISE(const auto obj_schema,
+                        GetMemberObject(json_schema.GetObject(), "schema"));
 
-  const auto& it_fields = obj_schema.FindMember("fields");
-  RETURN_NOT_ARRAY("fields", it_fields, obj_schema);
+  ARROW_ASSIGN_OR_RAISE(const auto json_fields, GetMemberArray(obj_schema, "fields"));
 
   std::shared_ptr<KeyValueMetadata> metadata;
   RETURN_NOT_OK(GetKeyValueMetadata(obj_schema, &metadata));
 
   std::vector<std::shared_ptr<Field>> fields;
-  RETURN_NOT_OK(GetFieldsFromArray(it_fields->value, dictionary_memo, &fields));
+  RETURN_NOT_OK(
+      GetFieldsFromArray(json_fields, FieldPosition(), dictionary_memo, &fields));
 
   // Read the dictionaries (if any) and cache in the memo
   RETURN_NOT_OK(ReadDictionaries(json_schema, pool, dictionary_memo));
@@ -1705,30 +1636,42 @@ Status ReadSchema(const rj::Value& json_schema, MemoryPool* pool,
   return Status::OK();
 }
 
+Status ReadArray(MemoryPool* pool, const rj::Value& json_array,
+                 const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
+  ARROW_ASSIGN_OR_RAISE(auto data, ReadArrayData(pool, json_array, field));
+  *out = MakeArray(data);
+  return Status::OK();
+}
+
 Status ReadRecordBatch(const rj::Value& json_obj, const std::shared_ptr<Schema>& schema,
                        DictionaryMemo* dictionary_memo, MemoryPool* pool,
                        std::shared_ptr<RecordBatch>* batch) {
   DCHECK(json_obj.IsObject());
   const auto& batch_obj = json_obj.GetObject();
 
-  auto it = batch_obj.FindMember("count");
-  RETURN_NOT_INT("count", it, batch_obj);
-  int32_t num_rows = static_cast<int32_t>(it->value.GetInt());
+  ARROW_ASSIGN_OR_RAISE(const int64_t num_rows,
+                        GetMemberInt<int64_t>(batch_obj, "count"));
 
-  it = batch_obj.FindMember("columns");
-  RETURN_NOT_ARRAY("columns", it, batch_obj);
-  const auto& json_columns = it->value.GetArray();
+  ARROW_ASSIGN_OR_RAISE(const auto json_columns, GetMemberArray(batch_obj, "columns"));
 
-  std::vector<std::shared_ptr<Array>> columns(json_columns.Size());
+  ArrayDataVector columns(json_columns.Size());
   for (int i = 0; i < static_cast<int>(columns.size()); ++i) {
-    RETURN_NOT_OK(
-        ReadArray(pool, json_columns[i], schema->field(i), dictionary_memo, &columns[i]));
+    ARROW_ASSIGN_OR_RAISE(columns[i],
+                          ReadArrayData(pool, json_columns[i], schema->field(i)));
   }
 
+  RETURN_NOT_OK(ResolveDictionaries(columns, *dictionary_memo, pool));
+
   *batch = RecordBatch::Make(schema, num_rows, columns);
   return Status::OK();
 }
 
+Status WriteSchema(const Schema& schema, const DictionaryFieldMapper& mapper,
+                   RjWriter* json_writer) {
+  SchemaWriter converter(schema, mapper, json_writer);
+  return converter.Write();
+}
+
 Status WriteDictionary(int64_t id, const std::shared_ptr<Array>& dictionary,
                        RjWriter* writer) {
   writer->StartObject();
@@ -1772,37 +1715,6 @@ Status WriteArray(const std::string& name, const Array& array, RjWriter* json_wr
   return converter.Write();
 }
 
-Status ReadArray(MemoryPool* pool, const rj::Value& json_array,
-                 const std::shared_ptr<Field>& field, DictionaryMemo* dictionary_memo,
-                 std::shared_ptr<Array>* out) {
-  if (!json_array.IsObject()) {
-    return Status::Invalid("Array element was not a JSON object");
-  }
-  auto obj = json_array.GetObject();
-  ArrayReader parser(obj, pool, field, dictionary_memo);
-  return parser.Parse(out);
-}
-
-Status ReadArray(MemoryPool* pool, const rj::Value& json_array, const Schema& schema,
-                 DictionaryMemo* dictionary_memo, std::shared_ptr<Array>* array) {
-  if (!json_array.IsObject()) {
-    return Status::Invalid("Element was not a JSON object");
-  }
-
-  const auto& json_obj = json_array.GetObject();
-
-  const auto& it_name = json_obj.FindMember("name");
-  RETURN_NOT_STRING("name", it_name, json_obj);
-
-  std::string name = it_name->value.GetString();
-  std::shared_ptr<Field> result = schema.GetFieldByName(name);
-  if (result == nullptr) {
-    return Status::KeyError("Field named ", name, " not found in schema");
-  }
-
-  return ReadArray(pool, json_array, result, dictionary_memo, array);
-}
-
 }  // namespace json
 }  // namespace testing
 }  // namespace arrow
diff --git a/cpp/src/arrow/testing/json_internal.h b/cpp/src/arrow/testing/json_internal.h
index 053ba7e208e..0870dd1e797 100644
--- a/cpp/src/arrow/testing/json_internal.h
+++ b/cpp/src/arrow/testing/json_internal.h
@@ -84,6 +84,7 @@ class Schema;
 
 namespace ipc {
 
+class DictionaryFieldMapper;
 class DictionaryMemo;
 
 }  // namespace ipc
@@ -93,7 +94,7 @@ namespace json {
 
 /// \brief Append integration test Schema format to rapidjson writer
 ARROW_TESTING_EXPORT
-Status WriteSchema(const Schema& schema, ipc::DictionaryMemo* dict_memo,
+Status WriteSchema(const Schema& schema, const ipc::DictionaryFieldMapper& mapper,
                    RjWriter* writer);
 
 ARROW_TESTING_EXPORT
@@ -115,14 +116,10 @@ Status ReadRecordBatch(const rj::Value& json_obj, const std::shared_ptr<Schema>&
                        ipc::DictionaryMemo* dict_memo, MemoryPool* pool,
                        std::shared_ptr<RecordBatch>* batch);
 
+// NOTE: Doesn't work with dictionary arrays, use ReadRecordBatch instead.
 ARROW_TESTING_EXPORT
 Status ReadArray(MemoryPool* pool, const rj::Value& json_obj,
-                 const std::shared_ptr<Field>& type, ipc::DictionaryMemo* dict_memo,
-                 std::shared_ptr<Array>* array);
-
-ARROW_TESTING_EXPORT
-Status ReadArray(MemoryPool* pool, const rj::Value& json_obj, const Schema& schema,
-                 ipc::DictionaryMemo* dict_memo, std::shared_ptr<Array>* array);
+                 const std::shared_ptr<Field>& type, std::shared_ptr<Array>* array);
 
 }  // namespace json
 }  // namespace testing
diff --git a/cpp/src/arrow/testing/random.cc b/cpp/src/arrow/testing/random.cc
index 2bd03dca197..32007f810e9 100644
--- a/cpp/src/arrow/testing/random.cc
+++ b/cpp/src/arrow/testing/random.cc
@@ -31,11 +31,14 @@
 #include "arrow/type_fwd.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit_util.h"
+#include "arrow/util/bitmap_reader.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
 namespace random {
 
+namespace {
+
 template <typename ValueType, typename DistributionType>
 struct GenerateOptions {
   GenerateOptions(SeedType seed, ValueType min, ValueType max, double probability)
@@ -76,6 +79,21 @@ struct GenerateOptions {
   double probability_;
 };
 
+}  // namespace
+
+std::shared_ptr<Buffer> RandomArrayGenerator::NullBitmap(int64_t size,
+                                                         double null_probability) {
+  // The bitmap generator does not care about the value distribution since it
+  // only calls the GenerateBitmap method.
+  using GenOpt = GenerateOptions<int, std::uniform_int_distribution<int>>;
+
+  GenOpt null_gen(seed(), 0, 1, null_probability);
+  std::shared_ptr<Buffer> bitmap = *AllocateEmptyBitmap(size);
+  null_gen.GenerateBitmap(bitmap->mutable_data(), size, nullptr);
+
+  return bitmap;
+}
+
 std::shared_ptr<Array> RandomArrayGenerator::Boolean(int64_t size,
                                                      double true_probability,
                                                      double null_probability) {
@@ -250,12 +268,23 @@ std::shared_ptr<Array> RandomArrayGenerator::StringWithRepeats(int64_t size,
 }
 
 std::shared_ptr<Array> RandomArrayGenerator::Offsets(int64_t size, int32_t first_offset,
-                                                     int32_t last_offset) {
+                                                     int32_t last_offset,
+                                                     double null_probability,
+                                                     bool force_empty_nulls) {
   using GenOpt = GenerateOptions<int32_t, std::uniform_int_distribution<int32_t>>;
-  GenOpt options(seed(), first_offset, last_offset, /*null_probability=*/0);
+  GenOpt options(seed(), first_offset, last_offset, null_probability);
 
   BufferVector buffers{2};
 
+  int64_t null_count = 0;
+
+  buffers[0] = *AllocateEmptyBitmap(size);
+  uint8_t* null_bitmap = buffers[0]->mutable_data();
+  options.GenerateBitmap(null_bitmap, size, &null_count);
+  // Make sure the first and last entry are non-null
+  arrow::BitUtil::SetBit(null_bitmap, 0);
+  arrow::BitUtil::SetBit(null_bitmap, size - 1);
+
   buffers[1] = *AllocateBuffer(sizeof(int32_t) * size);
   auto data = reinterpret_cast<int32_t*>(buffers[1]->mutable_data());
   options.GenerateTypedData(data, size);
@@ -267,10 +296,33 @@ std::shared_ptr<Array> RandomArrayGenerator::Offsets(int64_t size, int32_t first
   data[0] = first_offset;
   data[size - 1] = last_offset;
 
-  auto array_data = ArrayData::Make(int32(), size, buffers, /*null_count=*/0);
+  if (force_empty_nulls) {
+    arrow::internal::BitmapReader reader(null_bitmap, 0, size);
+    for (int64_t i = 0; i < size; ++i) {
+      if (reader.IsNotSet()) {
+        // Ensure a null entry corresponds to a 0-sized list extent
+        // (note this can be neither the first nor the last list entry, see above)
+        data[i + 1] = data[i];
+      }
+      reader.Next();
+    }
+  }
+
+  auto array_data = ArrayData::Make(int32(), size, buffers, null_count);
   return std::make_shared<Int32Array>(array_data);
 }
 
+std::shared_ptr<Array> RandomArrayGenerator::List(const Array& values, int64_t size,
+                                                  double null_probability,
+                                                  bool force_empty_nulls) {
+  auto offsets = Offsets(size, static_cast<int32_t>(values.offset()),
+                         static_cast<int32_t>(values.offset() + values.length()),
+                         null_probability, force_empty_nulls);
+  return *::arrow::ListArray::FromArrays(*offsets, values);
+}
+
+namespace {
+
 struct RandomArrayGeneratorOfImpl {
   Status Visit(const NullType&) {
     out_ = std::make_shared<NullArray>(size_);
@@ -353,6 +405,8 @@ struct RandomArrayGeneratorOfImpl {
   std::shared_ptr<Array> out_;
 };
 
+}  // namespace
+
 std::shared_ptr<Array> RandomArrayGenerator::ArrayOf(std::shared_ptr<DataType> type,
                                                      int64_t size,
                                                      double null_probability) {
diff --git a/cpp/src/arrow/testing/random.h b/cpp/src/arrow/testing/random.h
index 2aabdd6df49..e2ec405543f 100644
--- a/cpp/src/arrow/testing/random.h
+++ b/cpp/src/arrow/testing/random.h
@@ -34,7 +34,7 @@ class Array;
 
 namespace random {
 
-using SeedType = std::random_device::result_type;
+using SeedType = int32_t;
 constexpr SeedType kSeedMax = std::numeric_limits<SeedType>::max();
 
 class ARROW_TESTING_EXPORT RandomArrayGenerator {
@@ -42,7 +42,15 @@ class ARROW_TESTING_EXPORT RandomArrayGenerator {
   explicit RandomArrayGenerator(SeedType seed)
       : seed_distribution_(static_cast<SeedType>(1), kSeedMax), seed_rng_(seed) {}
 
-  /// \brief Generates a random BooleanArray
+  /// \brief Generate a null bitmap
+  ///
+  /// \param[in] size the size of the bitmap to generate
+  /// \param[in] null_probability the probability of a bit being zero
+  ///
+  /// \return a generated Buffer
+  std::shared_ptr<Buffer> NullBitmap(int64_t size, double null_probability = 0);
+
+  /// \brief Generate a random BooleanArray
   ///
   /// \param[in] size the size of the array to generate
   /// \param[in] true_probability the probability of a value being 1 / bit-set
@@ -52,7 +60,7 @@ class ARROW_TESTING_EXPORT RandomArrayGenerator {
   std::shared_ptr<Array> Boolean(int64_t size, double true_probability,
                                  double null_probability = 0);
 
-  /// \brief Generates a random UInt8Array
+  /// \brief Generate a random UInt8Array
   ///
   /// \param[in] size the size of the array to generate
   /// \param[in] min the lower bound of the uniform distribution
@@ -63,7 +71,7 @@ class ARROW_TESTING_EXPORT RandomArrayGenerator {
   std::shared_ptr<Array> UInt8(int64_t size, uint8_t min, uint8_t max,
                                double null_probability = 0);
 
-  /// \brief Generates a random Int8Array
+  /// \brief Generate a random Int8Array
   ///
   /// \param[in] size the size of the array to generate
   /// \param[in] min the lower bound of the uniform distribution
@@ -74,7 +82,7 @@ class ARROW_TESTING_EXPORT RandomArrayGenerator {
   std::shared_ptr<Array> Int8(int64_t size, int8_t min, int8_t max,
                               double null_probability = 0);
 
-  /// \brief Generates a random UInt16Array
+  /// \brief Generate a random UInt16Array
   ///
   /// \param[in] size the size of the array to generate
   /// \param[in] min the lower bound of the uniform distribution
@@ -85,7 +93,7 @@ class ARROW_TESTING_EXPORT RandomArrayGenerator {
   std::shared_ptr<Array> UInt16(int64_t size, uint16_t min, uint16_t max,
                                 double null_probability = 0);
 
-  /// \brief Generates a random Int16Array
+  /// \brief Generate a random Int16Array
   ///
   /// \param[in] size the size of the array to generate
   /// \param[in] min the lower bound of the uniform distribution
@@ -96,7 +104,7 @@ class ARROW_TESTING_EXPORT RandomArrayGenerator {
   std::shared_ptr<Array> Int16(int64_t size, int16_t min, int16_t max,
                                double null_probability = 0);
 
-  /// \brief Generates a random UInt32Array
+  /// \brief Generate a random UInt32Array
   ///
   /// \param[in] size the size of the array to generate
   /// \param[in] min the lower bound of the uniform distribution
@@ -107,7 +115,7 @@ class ARROW_TESTING_EXPORT RandomArrayGenerator {
   std::shared_ptr<Array> UInt32(int64_t size, uint32_t min, uint32_t max,
                                 double null_probability = 0);
 
-  /// \brief Generates a random Int32Array
+  /// \brief Generate a random Int32Array
   ///
   /// \param[in] size the size of the array to generate
   /// \param[in] min the lower bound of the uniform distribution
@@ -118,7 +126,7 @@ class ARROW_TESTING_EXPORT RandomArrayGenerator {
   std::shared_ptr<Array> Int32(int64_t size, int32_t min, int32_t max,
                                double null_probability = 0);
 
-  /// \brief Generates a random UInt64Array
+  /// \brief Generate a random UInt64Array
   ///
   /// \param[in] size the size of the array to generate
   /// \param[in] min the lower bound of the uniform distribution
@@ -129,7 +137,7 @@ class ARROW_TESTING_EXPORT RandomArrayGenerator {
   std::shared_ptr<Array> UInt64(int64_t size, uint64_t min, uint64_t max,
                                 double null_probability = 0);
 
-  /// \brief Generates a random Int64Array
+  /// \brief Generate a random Int64Array
   ///
   /// \param[in] size the size of the array to generate
   /// \param[in] min the lower bound of the uniform distribution
@@ -140,7 +148,7 @@ class ARROW_TESTING_EXPORT RandomArrayGenerator {
   std::shared_ptr<Array> Int64(int64_t size, int64_t min, int64_t max,
                                double null_probability = 0);
 
-  /// \brief Generates a random HalfFloatArray
+  /// \brief Generate a random HalfFloatArray
   ///
   /// \param[in] size the size of the array to generate
   /// \param[in] min the lower bound of the distribution
@@ -151,7 +159,7 @@ class ARROW_TESTING_EXPORT RandomArrayGenerator {
   std::shared_ptr<Array> Float16(int64_t size, int16_t min, int16_t max,
                                  double null_probability = 0);
 
-  /// \brief Generates a random FloatArray
+  /// \brief Generate a random FloatArray
   ///
   /// \param[in] size the size of the array to generate
   /// \param[in] min the lower bound of the uniform distribution
@@ -162,7 +170,7 @@ class ARROW_TESTING_EXPORT RandomArrayGenerator {
   std::shared_ptr<Array> Float32(int64_t size, float min, float max,
                                  double null_probability = 0);
 
-  /// \brief Generates a random DoubleArray
+  /// \brief Generate a random DoubleArray
   ///
   /// \param[in] size the size of the array to generate
   /// \param[in] min the lower bound of the uniform distribution
@@ -215,9 +223,20 @@ class ARROW_TESTING_EXPORT RandomArrayGenerator {
     }
   }
 
-  std::shared_ptr<Array> Offsets(int64_t size, int32_t first_offset, int32_t last_offset);
+  /// \brief Generate an array of offsets (for use in e.g. ListArray::FromArrays)
+  ///
+  /// \param[in] size the size of the array to generate
+  /// \param[in] first_offset the first offset value (usually 0)
+  /// \param[in] last_offset the last offset value (usually the size of the child array)
+  /// \param[in] null_probability the probability of an offset being null
+  /// \param[in] force_empty_nulls if true, null offsets must have 0 "length"
+  ///
+  /// \return a generated Array
+  std::shared_ptr<Array> Offsets(int64_t size, int32_t first_offset, int32_t last_offset,
+                                 double null_probability = 0,
+                                 bool force_empty_nulls = false);
 
-  /// \brief Generates a random StringArray
+  /// \brief Generate a random StringArray
   ///
   /// \param[in] size the size of the array to generate
   /// \param[in] min_length the lower bound of the string length
@@ -230,7 +249,7 @@ class ARROW_TESTING_EXPORT RandomArrayGenerator {
   std::shared_ptr<Array> String(int64_t size, int32_t min_length, int32_t max_length,
                                 double null_probability = 0);
 
-  /// \brief Generates a random LargeStringArray
+  /// \brief Generate a random LargeStringArray
   ///
   /// \param[in] size the size of the array to generate
   /// \param[in] min_length the lower bound of the string length
@@ -243,7 +262,7 @@ class ARROW_TESTING_EXPORT RandomArrayGenerator {
   std::shared_ptr<Array> LargeString(int64_t size, int32_t min_length, int32_t max_length,
                                      double null_probability = 0);
 
-  /// \brief Generates a random StringArray with repeated values
+  /// \brief Generate a random StringArray with repeated values
   ///
   /// \param[in] size the size of the array to generate
   /// \param[in] unique the number of unique string values used
@@ -264,7 +283,18 @@ class ARROW_TESTING_EXPORT RandomArrayGenerator {
                                            int32_t min_length, int32_t max_length,
                                            double null_probability = 0);
 
-  /// \brief Randomly generate an Array of the specified type, size, and null_probability.
+  /// \brief Generate a random ListArray
+  ///
+  /// \param[in] values The underlying values array
+  /// \param[in] size The size of the generated list array
+  /// \param[in] null_probability the probability of a list value being null
+  /// \param[in] force_empty_nulls if true, null list entries must have 0 length
+  ///
+  /// \return a generated Array
+  std::shared_ptr<Array> List(const Array& values, int64_t size, double null_probability,
+                              bool force_empty_nulls = false);
+
+  /// \brief Generate a random Array of the specified type, size, and null_probability.
   ///
   /// Generation parameters other than size and null_probability are determined based on
   /// the type of Array to be generated.
diff --git a/cpp/src/arrow/testing/util.h b/cpp/src/arrow/testing/util.h
index 0b873dc0b93..99b438db9c7 100644
--- a/cpp/src/arrow/testing/util.h
+++ b/cpp/src/arrow/testing/util.h
@@ -28,8 +28,7 @@
 #include <utility>
 #include <vector>
 
-#include "arrow/buffer.h"
-#include "arrow/builder.h"
+#include "arrow/array/builder_primitive.h"
 #include "arrow/record_batch.h"
 #include "arrow/status.h"
 #include "arrow/testing/visibility.h"
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 186daace56d..cbf18a08734 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -20,6 +20,7 @@
 #include <algorithm>
 #include <climits>
 #include <cstddef>
+#include <limits>
 #include <ostream>
 #include <sstream>  // IWYU pragma: keep
 #include <string>
@@ -69,6 +70,8 @@ constexpr Type::type StructType::type_id;
 
 constexpr Type::type Decimal128Type::type_id;
 
+constexpr Type::type Decimal256Type::type_id;
+
 constexpr Type::type SparseUnionType::type_id;
 
 constexpr Type::type DenseUnionType::type_id;
@@ -129,7 +132,8 @@ std::string ToString(Type::type id) {
     TO_STRING_CASE(HALF_FLOAT)
     TO_STRING_CASE(FLOAT)
     TO_STRING_CASE(DOUBLE)
-    TO_STRING_CASE(DECIMAL)
+    TO_STRING_CASE(DECIMAL128)
+    TO_STRING_CASE(DECIMAL256)
     TO_STRING_CASE(DATE32)
     TO_STRING_CASE(DATE64)
     TO_STRING_CASE(TIME32)
@@ -465,6 +469,17 @@ std::string LargeStringType::ToString() const { return "large_string"; }
 
 int FixedSizeBinaryType::bit_width() const { return CHAR_BIT * byte_width(); }
 
+Result<std::shared_ptr<DataType>> FixedSizeBinaryType::Make(int32_t byte_width) {
+  if (byte_width < 0) {
+    return Status::Invalid("Negative FixedSizeBinaryType byte width");
+  }
+  if (byte_width > std::numeric_limits<int>::max() / CHAR_BIT) {
+    // bit_width() would overflow
+    return Status::Invalid("byte width of FixedSizeBinaryType too large");
+  }
+  return std::make_shared<FixedSizeBinaryType>(byte_width);
+}
+
 std::string FixedSizeBinaryType::ToString() const {
   std::stringstream ss;
   ss << "fixed_size_binary[" << byte_width_ << "]";
@@ -736,7 +751,7 @@ std::vector<std::shared_ptr<Field>> StructType::GetAllFieldsByName(
 // Decimal128 type
 
 Decimal128Type::Decimal128Type(int32_t precision, int32_t scale)
-    : DecimalType(16, precision, scale) {
+    : DecimalType(type_id, 16, precision, scale) {
   ARROW_CHECK_GE(precision, kMinPrecision);
   ARROW_CHECK_LE(precision, kMaxPrecision);
 }
@@ -748,6 +763,22 @@ Result<std::shared_ptr<DataType>> Decimal128Type::Make(int32_t precision, int32_
   return std::make_shared<Decimal128Type>(precision, scale);
 }
 
+// ----------------------------------------------------------------------
+// Decimal256 type
+
+Decimal256Type::Decimal256Type(int32_t precision, int32_t scale)
+    : DecimalType(type_id, 32, precision, scale) {
+  ARROW_CHECK_GE(precision, kMinPrecision);
+  ARROW_CHECK_LE(precision, kMaxPrecision);
+}
+
+Result<std::shared_ptr<DataType>> Decimal256Type::Make(int32_t precision, int32_t scale) {
+  if (precision < kMinPrecision || precision > kMaxPrecision) {
+    return Status::Invalid("Decimal precision out of range: ", precision);
+  }
+  return std::make_shared<Decimal256Type>(precision, scale);
+}
+
 // ----------------------------------------------------------------------
 // Dictionary-encoded type
 
@@ -2126,13 +2157,28 @@ std::shared_ptr<Field> field(std::string name, std::shared_ptr<DataType> type,
 }
 
 std::shared_ptr<DataType> decimal(int32_t precision, int32_t scale) {
+  return precision <= Decimal128Type::kMaxPrecision ? decimal128(precision, scale)
+                                                    : decimal256(precision, scale);
+}
+
+std::shared_ptr<DataType> decimal128(int32_t precision, int32_t scale) {
   return std::make_shared<Decimal128Type>(precision, scale);
 }
 
+std::shared_ptr<DataType> decimal256(int32_t precision, int32_t scale) {
+  return std::make_shared<Decimal256Type>(precision, scale);
+}
+
 std::string Decimal128Type::ToString() const {
   std::stringstream s;
   s << "decimal(" << precision_ << ", " << scale_ << ")";
   return s.str();
 }
 
+std::string Decimal256Type::ToString() const {
+  std::stringstream s;
+  s << "decimal256(" << precision_ << ", " << scale_ << ")";
+  return s.str();
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index e2532d2489f..c45cf857a2a 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -126,7 +126,7 @@ class ARROW_EXPORT DataType : public detail::Fingerprintable {
   ARROW_DEPRECATED("Use field(i)")
   const std::shared_ptr<Field>& child(int i) const { return field(i); }
 
-  /// Returns the the child-field at index i.
+  /// Returns the child-field at index i.
   const std::shared_ptr<Field>& field(int i) const { return children_[i]; }
 
   ARROW_DEPRECATED("Use fields()")
@@ -811,6 +811,9 @@ class ARROW_EXPORT FixedSizeBinaryType : public FixedWidthType, public Parametri
   int32_t byte_width() const { return byte_width_; }
   int bit_width() const override;
 
+  // Validating constructor
+  static Result<std::shared_ptr<DataType>> Make(int32_t byte_width);
+
  protected:
   std::string ComputeFingerprint() const override;
 
@@ -858,10 +861,9 @@ class ARROW_EXPORT StructType : public NestedType {
 /// \brief Base type class for (fixed-size) decimal data
 class ARROW_EXPORT DecimalType : public FixedSizeBinaryType {
  public:
-  explicit DecimalType(int32_t byte_width, int32_t precision, int32_t scale)
-      : FixedSizeBinaryType(byte_width, Type::DECIMAL),
-        precision_(precision),
-        scale_(scale) {}
+  explicit DecimalType(Type::type type_id, int32_t byte_width, int32_t precision,
+                       int32_t scale)
+      : FixedSizeBinaryType(byte_width, type_id), precision_(precision), scale_(scale) {}
 
   int32_t precision() const { return precision_; }
   int32_t scale() const { return scale_; }
@@ -876,7 +878,7 @@ class ARROW_EXPORT DecimalType : public FixedSizeBinaryType {
 /// \brief Concrete type class for 128-bit decimal data
 class ARROW_EXPORT Decimal128Type : public DecimalType {
  public:
-  static constexpr Type::type type_id = Type::DECIMAL;
+  static constexpr Type::type type_id = Type::DECIMAL128;
 
   static constexpr const char* type_name() { return "decimal"; }
 
@@ -893,6 +895,26 @@ class ARROW_EXPORT Decimal128Type : public DecimalType {
   static constexpr int32_t kMaxPrecision = 38;
 };
 
+/// \brief Concrete type class for 256-bit decimal data
+class ARROW_EXPORT Decimal256Type : public DecimalType {
+ public:
+  static constexpr Type::type type_id = Type::DECIMAL256;
+
+  static constexpr const char* type_name() { return "decimal256"; }
+
+  /// Decimal256Type constructor that aborts on invalid input.
+  explicit Decimal256Type(int32_t precision, int32_t scale);
+
+  /// Decimal256Type constructor that returns an error on invalid input.
+  static Result<std::shared_ptr<DataType>> Make(int32_t precision, int32_t scale);
+
+  std::string ToString() const override;
+  std::string name() const override { return "decimal256"; }
+
+  static constexpr int32_t kMinPrecision = 1;
+  static constexpr int32_t kMaxPrecision = 76;
+};
+
 /// \brief Concrete type class for union data
 class ARROW_EXPORT UnionType : public NestedType {
  public:
@@ -1696,18 +1718,18 @@ class ARROW_EXPORT SchemaBuilder {
   };
 
   /// \brief Construct an empty SchemaBuilder
-  /// `field_merge_options` is only effecitive when `conflict_policy` == `CONFLICT_MERGE`.
+  /// `field_merge_options` is only effective when `conflict_policy` == `CONFLICT_MERGE`.
   SchemaBuilder(
       ConflictPolicy conflict_policy = CONFLICT_APPEND,
       Field::MergeOptions field_merge_options = Field::MergeOptions::Defaults());
   /// \brief Construct a SchemaBuilder from a list of fields
-  /// `field_merge_options` is only effecitive when `conflict_policy` == `CONFLICT_MERGE`.
+  /// `field_merge_options` is only effective when `conflict_policy` == `CONFLICT_MERGE`.
   SchemaBuilder(
       std::vector<std::shared_ptr<Field>> fields,
       ConflictPolicy conflict_policy = CONFLICT_APPEND,
       Field::MergeOptions field_merge_options = Field::MergeOptions::Defaults());
   /// \brief Construct a SchemaBuilder from a schema, preserving the metadata
-  /// `field_merge_options` is only effecitive when `conflict_policy` == `CONFLICT_MERGE`.
+  /// `field_merge_options` is only effective when `conflict_policy` == `CONFLICT_MERGE`.
   SchemaBuilder(
       const std::shared_ptr<Schema>& schema,
       ConflictPolicy conflict_policy = CONFLICT_APPEND,
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
index fc25b27238c..5ad0a199be3 100644
--- a/cpp/src/arrow/type_fwd.h
+++ b/cpp/src/arrow/type_fwd.h
@@ -60,7 +60,6 @@ using FieldVector = std::vector<std::shared_ptr<Field>>;
 class Array;
 struct ArrayData;
 class ArrayBuilder;
-class Tensor;
 struct Scalar;
 
 using ArrayDataVector = std::vector<std::shared_ptr<ArrayData>>;
@@ -143,11 +142,16 @@ class StructBuilder;
 struct StructScalar;
 
 class Decimal128;
+class Decimal256;
 class DecimalType;
 class Decimal128Type;
+class Decimal256Type;
 class Decimal128Array;
+class Decimal256Array;
 class Decimal128Builder;
+class Decimal256Builder;
 struct Decimal128Scalar;
+struct Decimal256Scalar;
 
 struct UnionMode {
   enum type { SPARSE, DENSE };
@@ -246,6 +250,9 @@ class ExtensionType;
 class ExtensionArray;
 struct ExtensionScalar;
 
+class Tensor;
+class SparseTensor;
+
 // ----------------------------------------------------------------------
 
 struct Type {
@@ -326,9 +333,14 @@ struct Type {
     /// DAY_TIME interval in SQL style
     INTERVAL_DAY_TIME,
 
-    /// Precision- and scale-based decimal type. Storage type depends on the
-    /// parameters.
-    DECIMAL,
+    /// Precision- and scale-based decimal type with 128 bits.
+    DECIMAL128,
+
+    /// Defined for backward-compatibility.
+    DECIMAL = DECIMAL128,
+
+    /// Precision- and scale-based decimal type with 256 bits.
+    DECIMAL256,
 
     /// A list of some logical data type
     LIST,
@@ -423,10 +435,18 @@ std::shared_ptr<DataType> ARROW_EXPORT date64();
 ARROW_EXPORT
 std::shared_ptr<DataType> fixed_size_binary(int32_t byte_width);
 
-/// \brief Create a Decimal128Type instance
+/// \brief Create a Decimal128Type or Decimal256Type instance depending on the precision
 ARROW_EXPORT
 std::shared_ptr<DataType> decimal(int32_t precision, int32_t scale);
 
+/// \brief Create a Decimal128Type instance
+ARROW_EXPORT
+std::shared_ptr<DataType> decimal128(int32_t precision, int32_t scale);
+
+/// \brief Create a Decimal256Type instance
+ARROW_EXPORT
+std::shared_ptr<DataType> decimal256(int32_t precision, int32_t scale);
+
 /// \brief Create a ListType instance from its child Field type
 ARROW_EXPORT
 std::shared_ptr<DataType> list(const std::shared_ptr<Field>& value_type);
diff --git a/cpp/src/arrow/type_test.cc b/cpp/src/arrow/type_test.cc
index e53d259c0fb..d5ece2eea8e 100644
--- a/cpp/src/arrow/type_test.cc
+++ b/cpp/src/arrow/type_test.cc
@@ -1770,43 +1770,85 @@ TEST(TestDictionaryType, UnifyLarge) {
 TEST(TypesTest, TestDecimal128Small) {
   Decimal128Type t1(8, 4);
 
-  ASSERT_EQ(t1.id(), Type::DECIMAL);
-  ASSERT_EQ(t1.precision(), 8);
-  ASSERT_EQ(t1.scale(), 4);
+  EXPECT_EQ(t1.id(), Type::DECIMAL128);
+  EXPECT_EQ(t1.precision(), 8);
+  EXPECT_EQ(t1.scale(), 4);
 
-  ASSERT_EQ(t1.ToString(), std::string("decimal(8, 4)"));
+  EXPECT_EQ(t1.ToString(), std::string("decimal(8, 4)"));
 
   // Test properties
-  ASSERT_EQ(t1.byte_width(), 16);
-  ASSERT_EQ(t1.bit_width(), 128);
+  EXPECT_EQ(t1.byte_width(), 16);
+  EXPECT_EQ(t1.bit_width(), 128);
 }
 
 TEST(TypesTest, TestDecimal128Medium) {
   Decimal128Type t1(12, 5);
 
-  ASSERT_EQ(t1.id(), Type::DECIMAL);
-  ASSERT_EQ(t1.precision(), 12);
-  ASSERT_EQ(t1.scale(), 5);
+  EXPECT_EQ(t1.id(), Type::DECIMAL128);
+  EXPECT_EQ(t1.precision(), 12);
+  EXPECT_EQ(t1.scale(), 5);
 
-  ASSERT_EQ(t1.ToString(), std::string("decimal(12, 5)"));
+  EXPECT_EQ(t1.ToString(), std::string("decimal(12, 5)"));
 
   // Test properties
-  ASSERT_EQ(t1.byte_width(), 16);
-  ASSERT_EQ(t1.bit_width(), 128);
+  EXPECT_EQ(t1.byte_width(), 16);
+  EXPECT_EQ(t1.bit_width(), 128);
 }
 
 TEST(TypesTest, TestDecimal128Large) {
   Decimal128Type t1(27, 7);
 
-  ASSERT_EQ(t1.id(), Type::DECIMAL);
-  ASSERT_EQ(t1.precision(), 27);
-  ASSERT_EQ(t1.scale(), 7);
+  EXPECT_EQ(t1.id(), Type::DECIMAL128);
+  EXPECT_EQ(t1.precision(), 27);
+  EXPECT_EQ(t1.scale(), 7);
 
-  ASSERT_EQ(t1.ToString(), std::string("decimal(27, 7)"));
+  EXPECT_EQ(t1.ToString(), std::string("decimal(27, 7)"));
 
   // Test properties
-  ASSERT_EQ(t1.byte_width(), 16);
-  ASSERT_EQ(t1.bit_width(), 128);
+  EXPECT_EQ(t1.byte_width(), 16);
+  EXPECT_EQ(t1.bit_width(), 128);
+}
+
+TEST(TypesTest, TestDecimal256Small) {
+  Decimal256Type t1(8, 4);
+
+  EXPECT_EQ(t1.id(), Type::DECIMAL256);
+  EXPECT_EQ(t1.precision(), 8);
+  EXPECT_EQ(t1.scale(), 4);
+
+  EXPECT_EQ(t1.ToString(), std::string("decimal256(8, 4)"));
+
+  // Test properties
+  EXPECT_EQ(t1.byte_width(), 32);
+  EXPECT_EQ(t1.bit_width(), 256);
+}
+
+TEST(TypesTest, TestDecimal256Medium) {
+  Decimal256Type t1(12, 5);
+
+  EXPECT_EQ(t1.id(), Type::DECIMAL256);
+  EXPECT_EQ(t1.precision(), 12);
+  EXPECT_EQ(t1.scale(), 5);
+
+  EXPECT_EQ(t1.ToString(), std::string("decimal256(12, 5)"));
+
+  // Test properties
+  EXPECT_EQ(t1.byte_width(), 32);
+  EXPECT_EQ(t1.bit_width(), 256);
+}
+
+TEST(TypesTest, TestDecimal256Large) {
+  Decimal256Type t1(76, 38);
+
+  EXPECT_EQ(t1.id(), Type::DECIMAL256);
+  EXPECT_EQ(t1.precision(), 76);
+  EXPECT_EQ(t1.scale(), 38);
+
+  EXPECT_EQ(t1.ToString(), std::string("decimal256(76, 38)"));
+
+  // Test properties
+  EXPECT_EQ(t1.byte_width(), 32);
+  EXPECT_EQ(t1.bit_width(), 256);
 }
 
 TEST(TypesTest, TestDecimalEquals) {
@@ -1815,12 +1857,24 @@ TEST(TypesTest, TestDecimalEquals) {
   Decimal128Type t3(8, 5);
   Decimal128Type t4(27, 5);
 
+  Decimal256Type t5(8, 4);
+  Decimal256Type t6(8, 4);
+  Decimal256Type t7(8, 5);
+  Decimal256Type t8(27, 5);
+
   FixedSizeBinaryType t9(16);
+  FixedSizeBinaryType t10(32);
 
   AssertTypeEqual(t1, t2);
   AssertTypeNotEqual(t1, t3);
   AssertTypeNotEqual(t1, t4);
   AssertTypeNotEqual(t1, t9);
+
+  AssertTypeEqual(t5, t6);
+  AssertTypeNotEqual(t5, t1);
+  AssertTypeNotEqual(t5, t7);
+  AssertTypeNotEqual(t5, t8);
+  AssertTypeNotEqual(t5, t10);
 }
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index 7cf9503ce79..2dcfc77c437 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -22,7 +22,7 @@
 #include <type_traits>
 #include <vector>
 
-#include "arrow/type_fwd.h"
+#include "arrow/type.h"
 #include "arrow/util/bit_util.h"
 
 namespace arrow {
@@ -66,7 +66,8 @@ TYPE_ID_TRAIT(TIMESTAMP, TimestampType)
 TYPE_ID_TRAIT(INTERVAL_DAY_TIME, DayTimeIntervalType)
 TYPE_ID_TRAIT(INTERVAL_MONTHS, MonthIntervalType)
 TYPE_ID_TRAIT(DURATION, DurationType)
-TYPE_ID_TRAIT(DECIMAL, Decimal128Type)  // XXX or DecimalType?
+TYPE_ID_TRAIT(DECIMAL128, Decimal128Type)
+TYPE_ID_TRAIT(DECIMAL256, Decimal256Type)
 TYPE_ID_TRAIT(STRUCT, StructType)
 TYPE_ID_TRAIT(LIST, ListType)
 TYPE_ID_TRAIT(LARGE_LIST, LargeListType)
@@ -288,6 +289,14 @@ struct TypeTraits<Decimal128Type> {
   constexpr static bool is_parameter_free = false;
 };
 
+template <>
+struct TypeTraits<Decimal256Type> {
+  using ArrayType = Decimal256Array;
+  using BuilderType = Decimal256Builder;
+  using ScalarType = Decimal256Scalar;
+  constexpr static bool is_parameter_free = false;
+};
+
 template <>
 struct TypeTraits<BinaryType> {
   using ArrayType = BinaryArray;
@@ -530,12 +539,20 @@ using enable_if_base_binary = enable_if_t<is_base_binary_type<T>::value, R>;
 
 // Any binary excludes string from Base binary
 template <typename T>
-using is_any_binary_type =
+using is_binary_type =
     std::integral_constant<bool, std::is_same<BinaryType, T>::value ||
                                      std::is_same<LargeBinaryType, T>::value>;
 
 template <typename T, typename R = void>
-using enable_if_any_binary = enable_if_t<is_any_binary_type<T>::value, R>;
+using enable_if_binary = enable_if_t<is_binary_type<T>::value, R>;
+
+template <typename T>
+using is_string_type =
+    std::integral_constant<bool, std::is_same<StringType, T>::value ||
+                                     std::is_same<LargeStringType, T>::value>;
+
+template <typename T, typename R = void>
+using enable_if_string = enable_if_t<is_string_type<T>::value, R>;
 
 template <typename T>
 using is_string_like_type =
@@ -544,6 +561,9 @@ using is_string_like_type =
 template <typename T, typename R = void>
 using enable_if_string_like = enable_if_t<is_string_like_type<T>::value, R>;
 
+template <typename T, typename U, typename R = void>
+using enable_if_same = enable_if_t<std::is_same<T, U>::value, R>;
+
 // Note that this also includes DecimalType
 template <typename T>
 using is_fixed_size_binary_type = std::is_base_of<FixedSizeBinaryType, T>;
@@ -566,6 +586,18 @@ using is_decimal_type = std::is_base_of<DecimalType, T>;
 template <typename T, typename R = void>
 using enable_if_decimal = enable_if_t<is_decimal_type<T>::value, R>;
 
+template <typename T>
+using is_decimal128_type = std::is_base_of<Decimal128Type, T>;
+
+template <typename T, typename R = void>
+using enable_if_decimal128 = enable_if_t<is_decimal128_type<T>::value, R>;
+
+template <typename T>
+using is_decimal256_type = std::is_base_of<Decimal256Type, T>;
+
+template <typename T, typename R = void>
+using enable_if_decimal256 = enable_if_t<is_decimal256_type<T>::value, R>;
+
 // Nested Types
 
 template <typename T>
@@ -574,6 +606,9 @@ using is_nested_type = std::is_base_of<NestedType, T>;
 template <typename T, typename R = void>
 using enable_if_nested = enable_if_t<is_nested_type<T>::value, R>;
 
+template <typename T, typename R = void>
+using enable_if_not_nested = enable_if_t<!is_nested_type<T>::value, R>;
+
 template <typename T>
 using is_var_length_list_type =
     std::integral_constant<bool, std::is_base_of<LargeListType, T>::value ||
@@ -596,6 +631,15 @@ using is_fixed_size_list_type = std::is_same<FixedSizeListType, T>;
 template <typename T, typename R = void>
 using enable_if_fixed_size_list = enable_if_t<is_fixed_size_list_type<T>::value, R>;
 
+template <typename T>
+using is_list_type =
+    std::integral_constant<bool, std::is_same<T, ListType>::value ||
+                                     std::is_same<T, LargeListType>::value ||
+                                     std::is_same<T, FixedSizeListType>::value>;
+
+template <typename T, typename R = void>
+using enable_if_list_type = enable_if_t<is_list_type<T>::value, R>;
+
 template <typename T>
 using is_list_like_type =
     std::integral_constant<bool, is_base_list_type<T>::value ||
@@ -654,6 +698,18 @@ using is_interval_type = std::is_base_of<IntervalType, T>;
 template <typename T, typename R = void>
 using enable_if_interval = enable_if_t<is_interval_type<T>::value, R>;
 
+template <typename T>
+using is_dictionary_type = std::is_base_of<DictionaryType, T>;
+
+template <typename T, typename R = void>
+using enable_if_dictionary = enable_if_t<is_dictionary_type<T>::value, R>;
+
+template <typename T>
+using is_extension_type = std::is_base_of<ExtensionType, T>;
+
+template <typename T, typename R = void>
+using enable_if_extension = enable_if_t<is_extension_type<T>::value, R>;
+
 // Attribute differentiation
 
 template <typename T>
@@ -670,8 +726,12 @@ template <typename T, typename R = void>
 using enable_if_has_c_type = enable_if_t<has_c_type<T>::value, R>;
 
 template <typename T>
-using has_string_view = std::integral_constant<bool, is_binary_like_type<T>::value ||
-                                                         is_string_like_type<T>::value>;
+using has_string_view =
+    std::integral_constant<bool, std::is_same<BinaryType, T>::value ||
+                                     std::is_same<LargeBinaryType, T>::value ||
+                                     std::is_same<StringType, T>::value ||
+                                     std::is_same<LargeStringType, T>::value ||
+                                     std::is_same<FixedSizeBinaryType, T>::value>;
 
 template <typename T, typename R = void>
 using enable_if_has_string_view = enable_if_t<has_string_view<T>::value, R>;
@@ -747,6 +807,32 @@ static inline bool is_integer(Type::type type_id) {
   return false;
 }
 
+static inline bool is_signed_integer(Type::type type_id) {
+  switch (type_id) {
+    case Type::INT8:
+    case Type::INT16:
+    case Type::INT32:
+    case Type::INT64:
+      return true;
+    default:
+      break;
+  }
+  return false;
+}
+
+static inline bool is_unsigned_integer(Type::type type_id) {
+  switch (type_id) {
+    case Type::UINT8:
+    case Type::UINT16:
+    case Type::UINT32:
+    case Type::UINT64:
+      return true;
+    default:
+      break;
+  }
+  return false;
+}
+
 static inline bool is_floating(Type::type type_id) {
   switch (type_id) {
     case Type::HALF_FLOAT:
@@ -829,7 +915,8 @@ static inline bool is_dictionary(Type::type type_id) {
 
 static inline bool is_fixed_size_binary(Type::type type_id) {
   switch (type_id) {
-    case Type::DECIMAL:
+    case Type::DECIMAL128:
+    case Type::DECIMAL256:
     case Type::FIXED_SIZE_BINARY:
       return true;
     default:
diff --git a/cpp/src/arrow/util/atomic_shared_ptr.h b/cpp/src/arrow/util/atomic_shared_ptr.h
index 08f12b351b0..d93ad921db6 100644
--- a/cpp/src/arrow/util/atomic_shared_ptr.h
+++ b/cpp/src/arrow/util/atomic_shared_ptr.h
@@ -47,13 +47,13 @@ using enable_if_atomic_load_shared_ptr_unavailable =
     enable_if_t<!is_atomic_load_shared_ptr_available<T>::value, T>;
 
 template <class T>
-inline enable_if_atomic_load_shared_ptr_available<std::shared_ptr<T>> atomic_load(
+enable_if_atomic_load_shared_ptr_available<std::shared_ptr<T>> atomic_load(
     const std::shared_ptr<T>* p) {
   return std::atomic_load(p);
 }
 
 template <class T>
-inline enable_if_atomic_load_shared_ptr_unavailable<std::shared_ptr<T>> atomic_load(
+enable_if_atomic_load_shared_ptr_unavailable<std::shared_ptr<T>> atomic_load(
     const std::shared_ptr<T>* p) {
   return *p;
 }
@@ -76,18 +76,36 @@ using enable_if_atomic_store_shared_ptr_unavailable =
     enable_if_t<!is_atomic_store_shared_ptr_available<T>::value, T>;
 
 template <class T>
-inline void atomic_store(
-    enable_if_atomic_store_shared_ptr_available<std::shared_ptr<T>*> p,
-    std::shared_ptr<T> r) {
+void atomic_store(enable_if_atomic_store_shared_ptr_available<std::shared_ptr<T>*> p,
+                  std::shared_ptr<T> r) {
   std::atomic_store(p, std::move(r));
 }
 
 template <class T>
-inline void atomic_store(
-    enable_if_atomic_store_shared_ptr_unavailable<std::shared_ptr<T>*> p,
-    std::shared_ptr<T> r) {
+void atomic_store(enable_if_atomic_store_shared_ptr_unavailable<std::shared_ptr<T>*> p,
+                  std::shared_ptr<T> r) {
   *p = r;
 }
 
+template <class T>
+bool atomic_compare_exchange_strong(
+    enable_if_atomic_store_shared_ptr_available<std::shared_ptr<T>*> p,
+    std::shared_ptr<T>* expected, std::shared_ptr<T> desired) {
+  return std::atomic_compare_exchange_strong(p, expected, std::move(desired));
+}
+
+template <class T>
+bool atomic_compare_exchange_strong(
+    enable_if_atomic_store_shared_ptr_unavailable<std::shared_ptr<T>*> p,
+    std::shared_ptr<T>* expected, std::shared_ptr<T> desired) {
+  if (*p == *expected) {
+    *p = std::move(desired);
+    return true;
+  } else {
+    *expected = *p;
+    return false;
+  }
+}
+
 }  // namespace internal
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/basic_decimal.cc b/cpp/src/arrow/util/basic_decimal.cc
index e46834d7d1f..e9799e2b59d 100644
--- a/cpp/src/arrow/util/basic_decimal.cc
+++ b/cpp/src/arrow/util/basic_decimal.cc
@@ -28,6 +28,7 @@
 #include <string>
 
 #include "arrow/util/bit_util.h"
+#include "arrow/util/int128_internal.h"
 #include "arrow/util/int_util_internal.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
@@ -119,8 +120,11 @@ static const BasicDecimal128 ScaleMultipliersHalf[] = {
     BasicDecimal128(271050543121376108LL, 9257742014424809472ULL),
     BasicDecimal128(2710505431213761085LL, 343699775700336640ULL)};
 
-static constexpr uint64_t kIntMask = 0xFFFFFFFF;
-static constexpr auto kCarryBit = static_cast<uint64_t>(1) << static_cast<uint64_t>(32);
+#ifdef ARROW_USE_NATIVE_INT128
+static constexpr uint64_t kInt64Mask = 0xFFFFFFFFFFFFFFFF;
+#else
+static constexpr uint64_t kInt32Mask = 0xFFFFFFFF;
+#endif
 
 // same as ScaleMultipliers[38] - 1
 static constexpr BasicDecimal128 kMaxValue =
@@ -248,40 +252,146 @@ BasicDecimal128& BasicDecimal128::operator>>=(uint32_t bits) {
   return *this;
 }
 
-BasicDecimal128& BasicDecimal128::operator*=(const BasicDecimal128& right) {
-  // Break the left and right numbers into 32 bit chunks
-  // so that we can multiply them without overflow.
-  const uint64_t L0 = static_cast<uint64_t>(high_bits_) >> 32;
-  const uint64_t L1 = static_cast<uint64_t>(high_bits_) & kIntMask;
-  const uint64_t L2 = low_bits_ >> 32;
-  const uint64_t L3 = low_bits_ & kIntMask;
+namespace {
+
+// Convenience wrapper type over 128 bit unsigned integers. We opt not to
+// replace the uint128_t type in int128_internal.h because it would require
+// significantly more implementation work to be done. This class merely
+// provides the minimum necessary set of functions to perform 128+ bit
+// multiplication operations when there may or may not be native support.
+#ifdef ARROW_USE_NATIVE_INT128
+struct uint128_t {
+  uint128_t() {}
+  uint128_t(uint64_t hi, uint64_t lo) : val_((static_cast<__uint128_t>(hi) << 64) | lo) {}
+  explicit uint128_t(const BasicDecimal128& decimal) {
+    val_ = (static_cast<__uint128_t>(decimal.high_bits()) << 64) | decimal.low_bits();
+  }
+
+  explicit uint128_t(uint64_t value) : val_(value) {}
 
-  const uint64_t R0 = static_cast<uint64_t>(right.high_bits_) >> 32;
-  const uint64_t R1 = static_cast<uint64_t>(right.high_bits_) & kIntMask;
-  const uint64_t R2 = right.low_bits_ >> 32;
-  const uint64_t R3 = right.low_bits_ & kIntMask;
+  uint64_t hi() { return val_ >> 64; }
+  uint64_t lo() { return val_ & kInt64Mask; }
 
-  uint64_t product = L3 * R3;
-  low_bits_ = product & kIntMask;
+  uint128_t& operator+=(const uint128_t& other) {
+    val_ += other.val_;
+    return *this;
+  }
 
-  uint64_t sum = product >> 32;
+  uint128_t& operator*=(const uint128_t& other) {
+    val_ *= other.val_;
+    return *this;
+  }
 
-  product = L2 * R3;
-  sum += product;
-  high_bits_ = static_cast<int64_t>(sum < product ? kCarryBit : 0);
+  __uint128_t val_;
+};
 
-  product = L3 * R2;
-  sum += product;
+#else
+// Multiply two 64 bit word components into a 128 bit result, with high bits
+// stored in hi and low bits in lo.
+inline void ExtendAndMultiply(uint64_t x, uint64_t y, uint64_t* hi, uint64_t* lo) {
+  // Perform multiplication on two 64 bit words x and y into a 128 bit result
+  // by splitting up x and y into 32 bit high/low bit components,
+  // allowing us to represent the multiplication as
+  // x * y = x_lo * y_lo + x_hi * y_lo * 2^32 + y_hi * x_lo * 2^32
+  // + x_hi * y_hi * 2^64
+  //
+  // Now, consider the final output as lo_lo || lo_hi || hi_lo || hi_hi
+  // Therefore,
+  // lo_lo is (x_lo * y_lo)_lo,
+  // lo_hi is ((x_lo * y_lo)_hi + (x_hi * y_lo)_lo + (x_lo * y_hi)_lo)_lo,
+  // hi_lo is ((x_hi * y_hi)_lo + (x_hi * y_lo)_hi + (x_lo * y_hi)_hi)_hi,
+  // hi_hi is (x_hi * y_hi)_hi
+  const uint64_t x_lo = x & kInt32Mask;
+  const uint64_t y_lo = y & kInt32Mask;
+  const uint64_t x_hi = x >> 32;
+  const uint64_t y_hi = y >> 32;
+
+  const uint64_t t = x_lo * y_lo;
+  const uint64_t t_lo = t & kInt32Mask;
+  const uint64_t t_hi = t >> 32;
+
+  const uint64_t u = x_hi * y_lo + t_hi;
+  const uint64_t u_lo = u & kInt32Mask;
+  const uint64_t u_hi = u >> 32;
+
+  const uint64_t v = x_lo * y_hi + u_lo;
+  const uint64_t v_hi = v >> 32;
+
+  *hi = x_hi * y_hi + u_hi + v_hi;
+  *lo = (v << 32) + t_lo;
+}
+
+struct uint128_t {
+  uint128_t() {}
+  uint128_t(uint64_t hi, uint64_t lo) : hi_(hi), lo_(lo) {}
+  explicit uint128_t(const BasicDecimal128& decimal) {
+    hi_ = decimal.high_bits();
+    lo_ = decimal.low_bits();
+  }
 
-  low_bits_ += sum << 32;
+  uint64_t hi() const { return hi_; }
+  uint64_t lo() const { return lo_; }
+
+  uint128_t& operator+=(const uint128_t& other) {
+    // To deduce the carry bit, we perform "65 bit" addition on the low bits and
+    // seeing if the resulting high bit is 1. This is accomplished by shifting the
+    // low bits to the right by 1 (chopping off the lowest bit), then adding 1 if the
+    // result of adding the two chopped bits would have produced a carry.
+    uint64_t carry = (((lo_ & other.lo_) & 1) + (lo_ >> 1) + (other.lo_ >> 1)) >> 63;
+    hi_ += other.hi_ + carry;
+    lo_ += other.lo_;
+    return *this;
+  }
 
-  if (sum < product) {
-    high_bits_ += kCarryBit;
+  uint128_t& operator*=(const uint128_t& other) {
+    uint128_t r;
+    ExtendAndMultiply(lo_, other.lo_, &r.hi_, &r.lo_);
+    r.hi_ += (hi_ * other.lo_) + (lo_ * other.hi_);
+    *this = r;
+    return *this;
   }
 
-  high_bits_ += static_cast<int64_t>(sum >> 32);
-  high_bits_ += L1 * R3 + L2 * R2 + L3 * R1;
-  high_bits_ += (L0 * R3 + L1 * R2 + L2 * R1 + L3 * R0) << 32;
+  uint64_t hi_;
+  uint64_t lo_;
+};
+#endif
+
+// Multiplies two N * 64 bit unsigned integer types, represented by a uint64_t
+// array into a same sized output. Elements in the array should be in
+// little endian order, and output will be the same. Overflow in multiplication
+// will result in the lower N * 64 bits of the result being set.
+template <int N>
+inline void MultiplyUnsignedArray(const std::array<uint64_t, N>& lh,
+                                  const std::array<uint64_t, N>& rh,
+                                  std::array<uint64_t, N>* result) {
+  for (int j = 0; j < N; ++j) {
+    uint64_t carry = 0;
+    for (int i = 0; i < N - j; ++i) {
+      uint128_t tmp(lh[i]);
+      tmp *= uint128_t(rh[j]);
+      tmp += uint128_t((*result)[i + j]);
+      tmp += uint128_t(carry);
+      (*result)[i + j] = tmp.lo();
+      carry = tmp.hi();
+    }
+  }
+}
+
+}  // namespace
+
+BasicDecimal128& BasicDecimal128::operator*=(const BasicDecimal128& right) {
+  // Since the max value of BasicDecimal128 is supposed to be 1e38 - 1 and the
+  // min the negation taking the absolute values here should always be safe.
+  const bool negate = Sign() != right.Sign();
+  BasicDecimal128 x = BasicDecimal128::Abs(*this);
+  BasicDecimal128 y = BasicDecimal128::Abs(right);
+  uint128_t r(x);
+  r *= uint128_t{y};
+  high_bits_ = r.hi();
+  low_bits_ = r.lo();
+  if (negate) {
+    Negate();
+  }
   return *this;
 }
 
@@ -725,4 +835,99 @@ int32_t BasicDecimal128::CountLeadingBinaryZeros() const {
   }
 }
 
+#if ARROW_LITTLE_ENDIAN
+BasicDecimal256::BasicDecimal256(const uint8_t* bytes)
+    : little_endian_array_(
+          std::array<uint64_t, 4>({reinterpret_cast<const uint64_t*>(bytes)[0],
+                                   reinterpret_cast<const uint64_t*>(bytes)[1],
+                                   reinterpret_cast<const uint64_t*>(bytes)[2],
+                                   reinterpret_cast<const uint64_t*>(bytes)[3]})) {}
+#else
+BasicDecimal256::BasicDecimal256(const uint8_t* bytes)
+    : little_endian_array_(
+          std::array<uint64_t, 4>({reinterpret_cast<const uint64_t*>(bytes)[3],
+                                   reinterpret_cast<const uint64_t*>(bytes)[2],
+                                   reinterpret_cast<const uint64_t*>(bytes)[1],
+                                   reinterpret_cast<const uint64_t*>(bytes)[0]})) {}
+#endif
+
+BasicDecimal256& BasicDecimal256::Negate() {
+  uint64_t carry = 1;
+  for (uint64_t& elem : little_endian_array_) {
+    elem = ~elem + carry;
+    carry &= (elem == 0);
+  }
+  return *this;
+}
+
+BasicDecimal256& BasicDecimal256::Abs() { return *this < 0 ? Negate() : *this; }
+
+BasicDecimal256 BasicDecimal256::Abs(const BasicDecimal256& in) {
+  BasicDecimal256 result(in);
+  return result.Abs();
+}
+
+std::array<uint8_t, 32> BasicDecimal256::ToBytes() const {
+  std::array<uint8_t, 32> out{{0}};
+  ToBytes(out.data());
+  return out;
+}
+
+void BasicDecimal256::ToBytes(uint8_t* out) const {
+  DCHECK_NE(out, nullptr);
+#if ARROW_LITTLE_ENDIAN
+  reinterpret_cast<int64_t*>(out)[0] = little_endian_array_[0];
+  reinterpret_cast<int64_t*>(out)[1] = little_endian_array_[1];
+  reinterpret_cast<int64_t*>(out)[2] = little_endian_array_[2];
+  reinterpret_cast<int64_t*>(out)[3] = little_endian_array_[3];
+#else
+  reinterpret_cast<int64_t*>(out)[0] = little_endian_array_[3];
+  reinterpret_cast<int64_t*>(out)[1] = little_endian_array_[2];
+  reinterpret_cast<int64_t*>(out)[2] = little_endian_array_[1];
+  reinterpret_cast<int64_t*>(out)[3] = little_endian_array_[0];
+#endif
+}
+
+BasicDecimal256& BasicDecimal256::operator*=(const BasicDecimal256& right) {
+  // Since the max value of BasicDecimal256 is supposed to be 1e76 - 1 and the
+  // min the negation taking the absolute values here should always be safe.
+  const bool negate = Sign() != right.Sign();
+  BasicDecimal256 x = BasicDecimal256::Abs(*this);
+  BasicDecimal256 y = BasicDecimal256::Abs(right);
+
+  uint128_t r_hi;
+  uint128_t r_lo;
+  std::array<uint64_t, 4> res{0, 0, 0, 0};
+  MultiplyUnsignedArray<4>(x.little_endian_array_, y.little_endian_array_, &res);
+  little_endian_array_ = res;
+  if (negate) {
+    Negate();
+  }
+  return *this;
+}
+
+DecimalStatus BasicDecimal256::Rescale(int32_t original_scale, int32_t new_scale,
+                                       BasicDecimal256* out) const {
+  if (original_scale == new_scale) {
+    return DecimalStatus::kSuccess;
+  }
+  // TODO: implement.
+  return DecimalStatus::kRescaleDataLoss;
+}
+
+BasicDecimal256 operator*(const BasicDecimal256& left, const BasicDecimal256& right) {
+  BasicDecimal256 result = left;
+  result *= right;
+  return result;
+}
+
+bool operator<(const BasicDecimal256& left, const BasicDecimal256& right) {
+  const std::array<uint64_t, 4>& lhs = left.little_endian_array();
+  const std::array<uint64_t, 4>& rhs = right.little_endian_array();
+  return lhs[3] != rhs[3]
+             ? static_cast<int64_t>(lhs[3]) < static_cast<int64_t>(rhs[3])
+             : lhs[2] != rhs[2] ? lhs[2] < rhs[2]
+                                : lhs[1] != rhs[1] ? lhs[1] < rhs[1] : lhs[0] < rhs[0];
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/basic_decimal.h b/cpp/src/arrow/util/basic_decimal.h
index 23c38bbb9d3..55a23183830 100644
--- a/cpp/src/arrow/util/basic_decimal.h
+++ b/cpp/src/arrow/util/basic_decimal.h
@@ -109,10 +109,10 @@ class ARROW_EXPORT BasicDecimal128 {
   BasicDecimal128& operator>>=(uint32_t bits);
 
   /// \brief Get the high bits of the two's complement representation of the number.
-  inline int64_t high_bits() const { return high_bits_; }
+  inline constexpr int64_t high_bits() const { return high_bits_; }
 
   /// \brief Get the low bits of the two's complement representation of the number.
-  inline uint64_t low_bits() const { return low_bits_; }
+  inline constexpr uint64_t low_bits() const { return low_bits_; }
 
   /// \brief Return the raw bytes of the value in native-endian byte order.
   std::array<uint8_t, 16> ToBytes() const;
@@ -178,4 +178,104 @@ ARROW_EXPORT BasicDecimal128 operator/(const BasicDecimal128& left,
 ARROW_EXPORT BasicDecimal128 operator%(const BasicDecimal128& left,
                                        const BasicDecimal128& right);
 
+class ARROW_EXPORT BasicDecimal256 {
+ private:
+  // Due to a bug in clang, we have to declare the extend method prior to its
+  // usage.
+  template <typename T>
+  inline static constexpr uint64_t extend(T low_bits) noexcept {
+    return low_bits >= T() ? uint64_t{0} : ~uint64_t{0};
+  }
+
+ public:
+  /// \brief Create a BasicDecimal256 from the two's complement representation.
+  constexpr BasicDecimal256(const std::array<uint64_t, 4>& little_endian_array) noexcept
+      : little_endian_array_(little_endian_array) {}
+
+  /// \brief Empty constructor creates a BasicDecimal256 with a value of 0.
+  constexpr BasicDecimal256() noexcept : little_endian_array_({0, 0, 0, 0}) {}
+
+  /// \brief Convert any integer value into a BasicDecimal256.
+  template <typename T,
+            typename = typename std::enable_if<
+                std::is_integral<T>::value && (sizeof(T) <= sizeof(uint64_t)), T>::type>
+  constexpr BasicDecimal256(T value) noexcept
+      : little_endian_array_({static_cast<uint64_t>(value), extend(value), extend(value),
+                              extend(value)}) {}
+
+  constexpr BasicDecimal256(const BasicDecimal128& value) noexcept
+      : little_endian_array_({value.low_bits(), static_cast<uint64_t>(value.high_bits()),
+                              extend(value.high_bits()), extend(value.high_bits())}) {}
+
+  /// \brief Create a BasicDecimal256 from an array of bytes. Bytes are assumed to be in
+  /// native-endian byte order.
+  explicit BasicDecimal256(const uint8_t* bytes);
+
+  /// \brief Negate the current value (in-place)
+  BasicDecimal256& Negate();
+
+  /// \brief Absolute value (in-place)
+  BasicDecimal256& Abs();
+
+  /// \brief Absolute value
+  static BasicDecimal256 Abs(const BasicDecimal256& left);
+
+  /// \brief Get the bits of the two's complement representation of the number. The 4
+  /// elements are in little endian order. The bits within each uint64_t element are in
+  /// native endian order. For example,
+  /// BasicDecimal256(123).little_endian_array() = {123, 0, 0, 0};
+  /// BasicDecimal256(-2).little_endian_array() = {0xFF...FE, 0xFF...FF, 0xFF...FF,
+  /// 0xFF...FF}.
+  inline const std::array<uint64_t, 4>& little_endian_array() const {
+    return little_endian_array_;
+  }
+
+  /// \brief Return the raw bytes of the value in native-endian byte order.
+  std::array<uint8_t, 32> ToBytes() const;
+  void ToBytes(uint8_t* out) const;
+
+  /// \brief Convert BasicDecimal128 from one scale to another
+  DecimalStatus Rescale(int32_t original_scale, int32_t new_scale,
+                        BasicDecimal256* out) const;
+
+  inline int64_t Sign() const {
+    return 1 | (static_cast<int64_t>(little_endian_array_[3]) >> 63);
+  }
+
+  /// \brief Multiply this number by another number. The result is truncated to 256 bits.
+  BasicDecimal256& operator*=(const BasicDecimal256& right);
+
+ private:
+  std::array<uint64_t, 4> little_endian_array_;
+};
+
+ARROW_EXPORT inline bool operator==(const BasicDecimal256& left,
+                                    const BasicDecimal256& right) {
+  return left.little_endian_array() == right.little_endian_array();
+}
+
+ARROW_EXPORT inline bool operator!=(const BasicDecimal256& left,
+                                    const BasicDecimal256& right) {
+  return left.little_endian_array() != right.little_endian_array();
+}
+
+ARROW_EXPORT bool operator<(const BasicDecimal256& left, const BasicDecimal256& right);
+
+ARROW_EXPORT inline bool operator<=(const BasicDecimal256& left,
+                                    const BasicDecimal256& right) {
+  return !operator<(right, left);
+}
+
+ARROW_EXPORT inline bool operator>(const BasicDecimal256& left,
+                                   const BasicDecimal256& right) {
+  return operator<(right, left);
+}
+
+ARROW_EXPORT inline bool operator>=(const BasicDecimal256& left,
+                                    const BasicDecimal256& right) {
+  return !operator<(left, right);
+}
+
+ARROW_EXPORT BasicDecimal256 operator*(const BasicDecimal256& left,
+                                       const BasicDecimal256& right);
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/bit_block_counter.cc b/cpp/src/arrow/util/bit_block_counter.cc
index 516695ef3dd..ffe6dbb7517 100644
--- a/cpp/src/arrow/util/bit_block_counter.cc
+++ b/cpp/src/arrow/util/bit_block_counter.cc
@@ -111,12 +111,19 @@ BitBlockCount BitBlockCounter::NextFourWords() {
   return {256, static_cast<int16_t>(total_popcount)};
 }
 
+// Prevent pointer arithmetic on nullptr, which is undefined behavior even if the pointer
+// is never dereferenced.
+inline const uint8_t* EnsureNotNull(const uint8_t* ptr) {
+  static const uint8_t byte{};
+  return ptr == nullptr ? &byte : ptr;
+}
+
 OptionalBitBlockCounter::OptionalBitBlockCounter(const uint8_t* validity_bitmap,
                                                  int64_t offset, int64_t length)
     : has_bitmap_(validity_bitmap != nullptr),
       position_(0),
       length_(length),
-      counter_(validity_bitmap, offset, length) {}
+      counter_(EnsureNotNull(validity_bitmap), offset, length) {}
 
 OptionalBitBlockCounter::OptionalBitBlockCounter(
     const std::shared_ptr<Buffer>& validity_bitmap, int64_t offset, int64_t length)
@@ -188,9 +195,10 @@ OptionalBinaryBitBlockCounter::OptionalBinaryBitBlockCounter(const uint8_t* left
     : has_bitmap_(HasBitmapFromBitmaps(left_bitmap != nullptr, right_bitmap != nullptr)),
       position_(0),
       length_(length),
-      unary_counter_(left_bitmap != nullptr ? left_bitmap : right_bitmap,
+      unary_counter_(EnsureNotNull(left_bitmap != nullptr ? left_bitmap : right_bitmap),
                      left_bitmap != nullptr ? left_offset : right_offset, length),
-      binary_counter_(left_bitmap, left_offset, right_bitmap, right_offset, length) {}
+      binary_counter_(EnsureNotNull(left_bitmap), left_offset,
+                      EnsureNotNull(right_bitmap), right_offset, length) {}
 
 OptionalBinaryBitBlockCounter::OptionalBinaryBitBlockCounter(
     const std::shared_ptr<Buffer>& left_bitmap, int64_t left_offset,
diff --git a/cpp/src/arrow/util/bit_run_reader.h b/cpp/src/arrow/util/bit_run_reader.h
index eaa85bf3d7c..6dba30e0bc2 100644
--- a/cpp/src/arrow/util/bit_run_reader.h
+++ b/cpp/src/arrow/util/bit_run_reader.h
@@ -64,7 +64,7 @@ class BitRunReaderLinear {
 };
 
 #if ARROW_LITTLE_ENDIAN
-/// A convenience class for counting the number of continguous set/unset bits
+/// A convenience class for counting the number of contiguous set/unset bits
 /// in a bitmap.
 class ARROW_EXPORT BitRunReader {
  public:
@@ -145,7 +145,7 @@ class ARROW_EXPORT BitRunReader {
     }
 
     // Two cases:
-    //   1. For unset, CountTrailingZeros works natually so we don't
+    //   1. For unset, CountTrailingZeros works naturally so we don't
     //   invert the word.
     //   2. Otherwise invert so we can use CountTrailingZeros.
     if (current_run_bit_set_) {
diff --git a/cpp/src/arrow/util/bit_stream_utils.h b/cpp/src/arrow/util/bit_stream_utils.h
index 85a9ef10e9b..b9e695dfcb0 100644
--- a/cpp/src/arrow/util/bit_stream_utils.h
+++ b/cpp/src/arrow/util/bit_stream_utils.h
@@ -27,6 +27,7 @@
 #include "arrow/util/bpacking.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
+#include "arrow/util/ubsan.h"
 
 namespace arrow {
 namespace BitUtil {
@@ -292,7 +293,8 @@ inline int BitReader::GetBatch(int num_bits, T* v, int batch_size) {
   const uint8_t* buffer = buffer_;
 
   uint64_t needed_bits = num_bits * batch_size;
-  uint64_t remaining_bits = (max_bytes - byte_offset) * 8 - bit_offset;
+  constexpr uint64_t kBitsPerByte = 8;
+  uint64_t remaining_bits = (max_bytes - byte_offset) * kBitsPerByte - bit_offset;
   if (remaining_bits < needed_bits) {
     batch_size = static_cast<int>(remaining_bits) / num_bits;
   }
diff --git a/cpp/src/arrow/util/bit_util_benchmark.cc b/cpp/src/arrow/util/bit_util_benchmark.cc
index 63783da77ef..a9c472b27f2 100644
--- a/cpp/src/arrow/util/bit_util_benchmark.cc
+++ b/cpp/src/arrow/util/bit_util_benchmark.cc
@@ -28,12 +28,11 @@
 #include "arrow/array/array_base.h"
 #include "arrow/array/array_primitive.h"
 #include "arrow/buffer.h"
-#include "arrow/builder.h"
-#include "arrow/memory_pool.h"
 #include "arrow/result.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/testing/random.h"
 #include "arrow/testing/util.h"
+#include "arrow/type_fwd.h"
 #include "arrow/util/bit_run_reader.h"
 #include "arrow/util/bit_util.h"
 #include "arrow/util/bitmap.h"
@@ -421,7 +420,7 @@ static void CopyBitmapWithOffset(benchmark::State& state) {  // NOLINT non-const
   CopyBitmap<4>(state);
 }
 
-// Trigger the slow path where both source and dest buffer are not byte aligend.
+// Trigger the slow path where both source and dest buffer are not byte aligned.
 static void CopyBitmapWithOffsetBoth(benchmark::State& state) { CopyBitmap<3, 7>(state); }
 
 // Benchmark the worst case of comparing two identical bitmap
diff --git a/cpp/src/arrow/util/bit_util_test.cc b/cpp/src/arrow/util/bit_util_test.cc
index adb1e0f7ef4..ef6f1254348 100644
--- a/cpp/src/arrow/util/bit_util_test.cc
+++ b/cpp/src/arrow/util/bit_util_test.cc
@@ -689,7 +689,7 @@ TEST(FirstTimeBitmapWriter, AppendWordShiftsBitsCorrectly) {
   check_append("11111110", "10110010", "10110010", 15, true);
 }
 
-TEST(TestAppendBitmap, AppendWordOnlyApproriateBytesWritten) {
+TEST(TestAppendBitmap, AppendWordOnlyAppropriateBytesWritten) {
   std::vector<uint8_t> valid_bits = {0x00, 0x00};
 
   uint64_t bitmap = 0x1FF;
diff --git a/cpp/src/arrow/util/bitmap_ops.cc b/cpp/src/arrow/util/bitmap_ops.cc
index 32f21d1e08c..eb2aec73e81 100644
--- a/cpp/src/arrow/util/bitmap_ops.cc
+++ b/cpp/src/arrow/util/bitmap_ops.cc
@@ -52,8 +52,24 @@ int64_t CountSetBits(const uint8_t* data, int64_t bit_offset, int64_t length) {
     DCHECK_EQ(reinterpret_cast<size_t>(u64_data) & 7, 0);
     const uint64_t* end = u64_data + p.aligned_words;
 
-    for (auto iter = u64_data; iter < end; ++iter) {
-      count += BitUtil::PopCount(*iter);
+    constexpr int64_t kCountUnrollFactor = 4;
+    const int64_t words_rounded = BitUtil::RoundDown(p.aligned_words, kCountUnrollFactor);
+    int64_t count_unroll[kCountUnrollFactor] = {0};
+
+    // Unroll the loop for better performance
+    for (int64_t i = 0; i < words_rounded; i += kCountUnrollFactor) {
+      for (int64_t k = 0; k < kCountUnrollFactor; k++) {
+        count_unroll[k] += BitUtil::PopCount(u64_data[k]);
+      }
+      u64_data += kCountUnrollFactor;
+    }
+    for (int64_t k = 0; k < kCountUnrollFactor; k++) {
+      count += count_unroll[k];
+    }
+
+    // The trailing part
+    for (; u64_data < end; ++u64_data) {
+      count += BitUtil::PopCount(*u64_data);
     }
   }
 
diff --git a/cpp/src/arrow/util/bpacking.cc b/cpp/src/arrow/util/bpacking.cc
new file mode 100644
index 00000000000..02634755bd0
--- /dev/null
+++ b/cpp/src/arrow/util/bpacking.cc
@@ -0,0 +1,171 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/util/bpacking.h"
+#include "arrow/util/bpacking_default.h"
+#include "arrow/util/cpu_info.h"
+#include "arrow/util/dispatch.h"
+#include "arrow/util/logging.h"
+
+#if defined(ARROW_HAVE_RUNTIME_AVX2)
+#include "arrow/util/bpacking_avx2.h"
+#endif
+#if defined(ARROW_HAVE_RUNTIME_AVX512)
+#include "arrow/util/bpacking_avx512.h"
+#endif
+
+namespace arrow {
+namespace internal {
+
+namespace {
+
+int unpack32_default(const uint32_t* in, uint32_t* out, int batch_size, int num_bits) {
+  batch_size = batch_size / 32 * 32;
+  int num_loops = batch_size / 32;
+
+  switch (num_bits) {
+    case 0:
+      for (int i = 0; i < num_loops; ++i) in = nullunpacker32(in, out + i * 32);
+      break;
+    case 1:
+      for (int i = 0; i < num_loops; ++i) in = unpack1_32(in, out + i * 32);
+      break;
+    case 2:
+      for (int i = 0; i < num_loops; ++i) in = unpack2_32(in, out + i * 32);
+      break;
+    case 3:
+      for (int i = 0; i < num_loops; ++i) in = unpack3_32(in, out + i * 32);
+      break;
+    case 4:
+      for (int i = 0; i < num_loops; ++i) in = unpack4_32(in, out + i * 32);
+      break;
+    case 5:
+      for (int i = 0; i < num_loops; ++i) in = unpack5_32(in, out + i * 32);
+      break;
+    case 6:
+      for (int i = 0; i < num_loops; ++i) in = unpack6_32(in, out + i * 32);
+      break;
+    case 7:
+      for (int i = 0; i < num_loops; ++i) in = unpack7_32(in, out + i * 32);
+      break;
+    case 8:
+      for (int i = 0; i < num_loops; ++i) in = unpack8_32(in, out + i * 32);
+      break;
+    case 9:
+      for (int i = 0; i < num_loops; ++i) in = unpack9_32(in, out + i * 32);
+      break;
+    case 10:
+      for (int i = 0; i < num_loops; ++i) in = unpack10_32(in, out + i * 32);
+      break;
+    case 11:
+      for (int i = 0; i < num_loops; ++i) in = unpack11_32(in, out + i * 32);
+      break;
+    case 12:
+      for (int i = 0; i < num_loops; ++i) in = unpack12_32(in, out + i * 32);
+      break;
+    case 13:
+      for (int i = 0; i < num_loops; ++i) in = unpack13_32(in, out + i * 32);
+      break;
+    case 14:
+      for (int i = 0; i < num_loops; ++i) in = unpack14_32(in, out + i * 32);
+      break;
+    case 15:
+      for (int i = 0; i < num_loops; ++i) in = unpack15_32(in, out + i * 32);
+      break;
+    case 16:
+      for (int i = 0; i < num_loops; ++i) in = unpack16_32(in, out + i * 32);
+      break;
+    case 17:
+      for (int i = 0; i < num_loops; ++i) in = unpack17_32(in, out + i * 32);
+      break;
+    case 18:
+      for (int i = 0; i < num_loops; ++i) in = unpack18_32(in, out + i * 32);
+      break;
+    case 19:
+      for (int i = 0; i < num_loops; ++i) in = unpack19_32(in, out + i * 32);
+      break;
+    case 20:
+      for (int i = 0; i < num_loops; ++i) in = unpack20_32(in, out + i * 32);
+      break;
+    case 21:
+      for (int i = 0; i < num_loops; ++i) in = unpack21_32(in, out + i * 32);
+      break;
+    case 22:
+      for (int i = 0; i < num_loops; ++i) in = unpack22_32(in, out + i * 32);
+      break;
+    case 23:
+      for (int i = 0; i < num_loops; ++i) in = unpack23_32(in, out + i * 32);
+      break;
+    case 24:
+      for (int i = 0; i < num_loops; ++i) in = unpack24_32(in, out + i * 32);
+      break;
+    case 25:
+      for (int i = 0; i < num_loops; ++i) in = unpack25_32(in, out + i * 32);
+      break;
+    case 26:
+      for (int i = 0; i < num_loops; ++i) in = unpack26_32(in, out + i * 32);
+      break;
+    case 27:
+      for (int i = 0; i < num_loops; ++i) in = unpack27_32(in, out + i * 32);
+      break;
+    case 28:
+      for (int i = 0; i < num_loops; ++i) in = unpack28_32(in, out + i * 32);
+      break;
+    case 29:
+      for (int i = 0; i < num_loops; ++i) in = unpack29_32(in, out + i * 32);
+      break;
+    case 30:
+      for (int i = 0; i < num_loops; ++i) in = unpack30_32(in, out + i * 32);
+      break;
+    case 31:
+      for (int i = 0; i < num_loops; ++i) in = unpack31_32(in, out + i * 32);
+      break;
+    case 32:
+      for (int i = 0; i < num_loops; ++i) in = unpack32_32(in, out + i * 32);
+      break;
+    default:
+      DCHECK(false) << "Unsupported num_bits";
+  }
+
+  return batch_size;
+}
+
+struct Unpack32DynamicFunction {
+  using FunctionType = decltype(&unpack32_default);
+
+  static std::vector<std::pair<DispatchLevel, FunctionType>> implementations() {
+    return {
+      { DispatchLevel::NONE, unpack32_default }
+#if defined(ARROW_HAVE_RUNTIME_AVX2)
+      , { DispatchLevel::AVX2, unpack32_avx2 }
+#endif
+#if defined(ARROW_HAVE_RUNTIME_AVX512)
+      , { DispatchLevel::AVX512, unpack32_avx512 }
+#endif
+    };
+  }
+};
+
+}  // namespace
+
+int unpack32(const uint32_t* in, uint32_t* out, int batch_size, int num_bits) {
+  static DynamicDispatch<Unpack32DynamicFunction> dispatch;
+  return dispatch.func(in, out, batch_size, num_bits);
+}
+
+}  // namespace internal
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/bpacking.h b/cpp/src/arrow/util/bpacking.h
index a7730cba8f6..71714c4c7d8 100644
--- a/cpp/src/arrow/util/bpacking.h
+++ b/cpp/src/arrow/util/bpacking.h
@@ -17,127 +17,15 @@
 
 #pragma once
 
-#include "arrow/util/logging.h"
-#include "arrow/util/ubsan.h"
-#if defined(ARROW_HAVE_AVX512)
-#include "arrow/util/bpacking_avx512_generated.h"
-#else
-#include "arrow/util/bpacking_default.h"
-#endif
+#include "arrow/util/visibility.h"
+
+#include <stdint.h>
 
 namespace arrow {
 namespace internal {
 
-inline int unpack32(const uint32_t* in, uint32_t* out, int batch_size, int num_bits) {
-  batch_size = batch_size / 32 * 32;
-  int num_loops = batch_size / 32;
-
-  switch (num_bits) {
-    case 0:
-      for (int i = 0; i < num_loops; ++i) in = nullunpacker32(in, out + i * 32);
-      break;
-    case 1:
-      for (int i = 0; i < num_loops; ++i) in = unpack1_32(in, out + i * 32);
-      break;
-    case 2:
-      for (int i = 0; i < num_loops; ++i) in = unpack2_32(in, out + i * 32);
-      break;
-    case 3:
-      for (int i = 0; i < num_loops; ++i) in = unpack3_32(in, out + i * 32);
-      break;
-    case 4:
-      for (int i = 0; i < num_loops; ++i) in = unpack4_32(in, out + i * 32);
-      break;
-    case 5:
-      for (int i = 0; i < num_loops; ++i) in = unpack5_32(in, out + i * 32);
-      break;
-    case 6:
-      for (int i = 0; i < num_loops; ++i) in = unpack6_32(in, out + i * 32);
-      break;
-    case 7:
-      for (int i = 0; i < num_loops; ++i) in = unpack7_32(in, out + i * 32);
-      break;
-    case 8:
-      for (int i = 0; i < num_loops; ++i) in = unpack8_32(in, out + i * 32);
-      break;
-    case 9:
-      for (int i = 0; i < num_loops; ++i) in = unpack9_32(in, out + i * 32);
-      break;
-    case 10:
-      for (int i = 0; i < num_loops; ++i) in = unpack10_32(in, out + i * 32);
-      break;
-    case 11:
-      for (int i = 0; i < num_loops; ++i) in = unpack11_32(in, out + i * 32);
-      break;
-    case 12:
-      for (int i = 0; i < num_loops; ++i) in = unpack12_32(in, out + i * 32);
-      break;
-    case 13:
-      for (int i = 0; i < num_loops; ++i) in = unpack13_32(in, out + i * 32);
-      break;
-    case 14:
-      for (int i = 0; i < num_loops; ++i) in = unpack14_32(in, out + i * 32);
-      break;
-    case 15:
-      for (int i = 0; i < num_loops; ++i) in = unpack15_32(in, out + i * 32);
-      break;
-    case 16:
-      for (int i = 0; i < num_loops; ++i) in = unpack16_32(in, out + i * 32);
-      break;
-    case 17:
-      for (int i = 0; i < num_loops; ++i) in = unpack17_32(in, out + i * 32);
-      break;
-    case 18:
-      for (int i = 0; i < num_loops; ++i) in = unpack18_32(in, out + i * 32);
-      break;
-    case 19:
-      for (int i = 0; i < num_loops; ++i) in = unpack19_32(in, out + i * 32);
-      break;
-    case 20:
-      for (int i = 0; i < num_loops; ++i) in = unpack20_32(in, out + i * 32);
-      break;
-    case 21:
-      for (int i = 0; i < num_loops; ++i) in = unpack21_32(in, out + i * 32);
-      break;
-    case 22:
-      for (int i = 0; i < num_loops; ++i) in = unpack22_32(in, out + i * 32);
-      break;
-    case 23:
-      for (int i = 0; i < num_loops; ++i) in = unpack23_32(in, out + i * 32);
-      break;
-    case 24:
-      for (int i = 0; i < num_loops; ++i) in = unpack24_32(in, out + i * 32);
-      break;
-    case 25:
-      for (int i = 0; i < num_loops; ++i) in = unpack25_32(in, out + i * 32);
-      break;
-    case 26:
-      for (int i = 0; i < num_loops; ++i) in = unpack26_32(in, out + i * 32);
-      break;
-    case 27:
-      for (int i = 0; i < num_loops; ++i) in = unpack27_32(in, out + i * 32);
-      break;
-    case 28:
-      for (int i = 0; i < num_loops; ++i) in = unpack28_32(in, out + i * 32);
-      break;
-    case 29:
-      for (int i = 0; i < num_loops; ++i) in = unpack29_32(in, out + i * 32);
-      break;
-    case 30:
-      for (int i = 0; i < num_loops; ++i) in = unpack30_32(in, out + i * 32);
-      break;
-    case 31:
-      for (int i = 0; i < num_loops; ++i) in = unpack31_32(in, out + i * 32);
-      break;
-    case 32:
-      for (int i = 0; i < num_loops; ++i) in = unpack32_32(in, out + i * 32);
-      break;
-    default:
-      DCHECK(false) << "Unsupported num_bits";
-  }
-
-  return batch_size;
-}
+ARROW_EXPORT
+int unpack32(const uint32_t* in, uint32_t* out, int batch_size, int num_bits);
 
 }  // namespace internal
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/bpacking_avx2.cc b/cpp/src/arrow/util/bpacking_avx2.cc
new file mode 100644
index 00000000000..63b914b578a
--- /dev/null
+++ b/cpp/src/arrow/util/bpacking_avx2.cc
@@ -0,0 +1,137 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/util/bpacking_avx2.h"
+#include "arrow/util/bpacking_avx2_generated.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+namespace internal {
+
+int unpack32_avx2(const uint32_t* in, uint32_t* out, int batch_size, int num_bits) {
+  batch_size = batch_size / 32 * 32;
+  int num_loops = batch_size / 32;
+
+  switch (num_bits) {
+    case 0:
+      for (int i = 0; i < num_loops; ++i) in = unpack0_32_avx2(in, out + i * 32);
+      break;
+    case 1:
+      for (int i = 0; i < num_loops; ++i) in = unpack1_32_avx2(in, out + i * 32);
+      break;
+    case 2:
+      for (int i = 0; i < num_loops; ++i) in = unpack2_32_avx2(in, out + i * 32);
+      break;
+    case 3:
+      for (int i = 0; i < num_loops; ++i) in = unpack3_32_avx2(in, out + i * 32);
+      break;
+    case 4:
+      for (int i = 0; i < num_loops; ++i) in = unpack4_32_avx2(in, out + i * 32);
+      break;
+    case 5:
+      for (int i = 0; i < num_loops; ++i) in = unpack5_32_avx2(in, out + i * 32);
+      break;
+    case 6:
+      for (int i = 0; i < num_loops; ++i) in = unpack6_32_avx2(in, out + i * 32);
+      break;
+    case 7:
+      for (int i = 0; i < num_loops; ++i) in = unpack7_32_avx2(in, out + i * 32);
+      break;
+    case 8:
+      for (int i = 0; i < num_loops; ++i) in = unpack8_32_avx2(in, out + i * 32);
+      break;
+    case 9:
+      for (int i = 0; i < num_loops; ++i) in = unpack9_32_avx2(in, out + i * 32);
+      break;
+    case 10:
+      for (int i = 0; i < num_loops; ++i) in = unpack10_32_avx2(in, out + i * 32);
+      break;
+    case 11:
+      for (int i = 0; i < num_loops; ++i) in = unpack11_32_avx2(in, out + i * 32);
+      break;
+    case 12:
+      for (int i = 0; i < num_loops; ++i) in = unpack12_32_avx2(in, out + i * 32);
+      break;
+    case 13:
+      for (int i = 0; i < num_loops; ++i) in = unpack13_32_avx2(in, out + i * 32);
+      break;
+    case 14:
+      for (int i = 0; i < num_loops; ++i) in = unpack14_32_avx2(in, out + i * 32);
+      break;
+    case 15:
+      for (int i = 0; i < num_loops; ++i) in = unpack15_32_avx2(in, out + i * 32);
+      break;
+    case 16:
+      for (int i = 0; i < num_loops; ++i) in = unpack16_32_avx2(in, out + i * 32);
+      break;
+    case 17:
+      for (int i = 0; i < num_loops; ++i) in = unpack17_32_avx2(in, out + i * 32);
+      break;
+    case 18:
+      for (int i = 0; i < num_loops; ++i) in = unpack18_32_avx2(in, out + i * 32);
+      break;
+    case 19:
+      for (int i = 0; i < num_loops; ++i) in = unpack19_32_avx2(in, out + i * 32);
+      break;
+    case 20:
+      for (int i = 0; i < num_loops; ++i) in = unpack20_32_avx2(in, out + i * 32);
+      break;
+    case 21:
+      for (int i = 0; i < num_loops; ++i) in = unpack21_32_avx2(in, out + i * 32);
+      break;
+    case 22:
+      for (int i = 0; i < num_loops; ++i) in = unpack22_32_avx2(in, out + i * 32);
+      break;
+    case 23:
+      for (int i = 0; i < num_loops; ++i) in = unpack23_32_avx2(in, out + i * 32);
+      break;
+    case 24:
+      for (int i = 0; i < num_loops; ++i) in = unpack24_32_avx2(in, out + i * 32);
+      break;
+    case 25:
+      for (int i = 0; i < num_loops; ++i) in = unpack25_32_avx2(in, out + i * 32);
+      break;
+    case 26:
+      for (int i = 0; i < num_loops; ++i) in = unpack26_32_avx2(in, out + i * 32);
+      break;
+    case 27:
+      for (int i = 0; i < num_loops; ++i) in = unpack27_32_avx2(in, out + i * 32);
+      break;
+    case 28:
+      for (int i = 0; i < num_loops; ++i) in = unpack28_32_avx2(in, out + i * 32);
+      break;
+    case 29:
+      for (int i = 0; i < num_loops; ++i) in = unpack29_32_avx2(in, out + i * 32);
+      break;
+    case 30:
+      for (int i = 0; i < num_loops; ++i) in = unpack30_32_avx2(in, out + i * 32);
+      break;
+    case 31:
+      for (int i = 0; i < num_loops; ++i) in = unpack31_32_avx2(in, out + i * 32);
+      break;
+    case 32:
+      for (int i = 0; i < num_loops; ++i) in = unpack32_32_avx2(in, out + i * 32);
+      break;
+    default:
+      DCHECK(false) << "Unsupported num_bits";
+  }
+
+  return batch_size;
+}
+
+}  // namespace internal
+}  // namespace arrow
diff --git a/cpp/src/arrow/compute/kernels/scalar_string_internal.h b/cpp/src/arrow/util/bpacking_avx2.h
similarity index 51%
rename from cpp/src/arrow/compute/kernels/scalar_string_internal.h
rename to cpp/src/arrow/util/bpacking_avx2.h
index 897c75ee8c6..7a7d8bf8c44 100644
--- a/cpp/src/arrow/compute/kernels/scalar_string_internal.h
+++ b/cpp/src/arrow/util/bpacking_avx2.h
@@ -15,30 +15,14 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <memory>
-#include <string>
-#include <utility>
+#pragma once
 
-#include "arrow/array/builder_binary.h"
-#include "arrow/compute/kernels/common.h"
+#include <stdint.h>
 
 namespace arrow {
-namespace compute {
 namespace internal {
 
-// Apply a scalar function to each string and yield same output type
-template <typename Op>
-void MakeUnaryStringToString(std::string name, FunctionRegistry* registry) {
-  auto func = std::make_shared<ScalarFunction>(name, Arity::Unary());
-  ArrayKernelExec exec_offset_32 =
-      applicator::ScalarUnaryNotNull<StringType, StringType, Op>::Exec;
-  ArrayKernelExec exec_offset_64 =
-      applicator::ScalarUnaryNotNull<LargeStringType, LargeStringType, Op>::Exec;
-  DCHECK_OK(func->AddKernel({utf8()}, utf8(), exec_offset_32));
-  DCHECK_OK(func->AddKernel({large_utf8()}, large_utf8(), exec_offset_64));
-  DCHECK_OK(registry->AddFunction(std::move(func)));
-}
+int unpack32_avx2(const uint32_t* in, uint32_t* out, int batch_size, int num_bits);
 
 }  // namespace internal
-}  // namespace compute
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/bpacking_avx2_codegen.py b/cpp/src/arrow/util/bpacking_avx2_codegen.py
new file mode 100644
index 00000000000..ec4084c1a44
--- /dev/null
+++ b/cpp/src/arrow/util/bpacking_avx2_codegen.py
@@ -0,0 +1,200 @@
+#!/bin/python
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Usage: python bpacking_avx2_codegen.py > bpacking_avx2_generated.h
+
+
+def print_unpack_bit_func(bit):
+    shift = 0
+    shifts = []
+    in_index = 0
+    inls = []
+    mask = (1 << bit) - 1
+    bracket = "{"
+
+    print(
+        f"inline static const uint32_t* unpack{bit}_32_avx2(const uint32_t* in, uint32_t* out) {bracket}")
+    print("  uint32_t mask = 0x%x;" % mask)
+    print("  __m256i reg_shifts, reg_inls, reg_masks;")
+    print("  __m256i results;")
+
+    print("")
+    for i in range(32):
+        if shift + bit == 32:
+            shifts.append(shift)
+            inls.append(f"in[{in_index}]")
+            in_index += 1
+            shift = 0
+        elif shift + bit > 32:  # cross the boundary
+            inls.append(
+                f"in[{in_index}] >> {shift} | in[{in_index + 1}] << {32 - shift}")
+            in_index += 1
+            shift = bit - (32 - shift)
+            shifts.append(0)  # zero shift
+        else:
+            shifts.append(shift)
+            inls.append(f"in[{in_index}]")
+            shift += bit
+
+    print("  reg_masks = _mm256_set1_epi32(mask);")
+    print("")
+
+    print("  // shift the first 8 outs")
+    print(
+        f"  reg_shifts = _mm256_set_epi32({shifts[7]}, {shifts[6]}, {shifts[5]}, {shifts[4]},")
+    print(
+        f"                               {shifts[3]}, {shifts[2]}, {shifts[1]}, {shifts[0]});")
+    print(f"  reg_inls = _mm256_set_epi32({inls[7]}, {inls[6]},")
+    print(f"                             {inls[5]}, {inls[4]},")
+    print(f"                             {inls[3]}, {inls[2]},")
+    print(f"                             {inls[1]}, {inls[0]});")
+    print(
+        "  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);")
+    print("  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);")
+    print("  out += 8;")
+    print("")
+
+    print("  // shift the second 8 outs")
+    print(
+        f"  reg_shifts = _mm256_set_epi32({shifts[15]}, {shifts[14]}, {shifts[13]}, {shifts[12]},")
+    print(
+        f"                                {shifts[11]}, {shifts[10]}, {shifts[9]}, {shifts[8]});")
+    print(f"  reg_inls = _mm256_set_epi32({inls[15]}, {inls[14]},")
+    print(f"                              {inls[13]}, {inls[12]},")
+    print(f"                              {inls[11]}, {inls[10]},")
+    print(f"                              {inls[9]}, {inls[8]});")
+    print(
+        "  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);")
+    print("  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);")
+    print("  out += 8;")
+    print("")
+
+    print("  // shift the third 8 outs")
+    print(
+        f"  reg_shifts = _mm256_set_epi32({shifts[23]}, {shifts[22]}, {shifts[21]}, {shifts[20]},")
+    print(
+        f"                                {shifts[19]}, {shifts[18]}, {shifts[17]}, {shifts[16]});")
+    print(f"  reg_inls = _mm256_set_epi32({inls[23]}, {inls[22]},")
+    print(f"                              {inls[21]}, {inls[20]},")
+    print(f"                              {inls[19]}, {inls[18]},")
+    print(f"                              {inls[17]}, {inls[16]});")
+    print(
+        "  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);")
+    print("  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);")
+    print("  out += 8;")
+    print("")
+
+    print("  // shift the last 8 outs")
+    print(
+        f"  reg_shifts = _mm256_set_epi32({shifts[31]}, {shifts[30]}, {shifts[29]}, {shifts[28]},")
+    print(
+        f"                                {shifts[27]}, {shifts[26]}, {shifts[25]}, {shifts[24]});")
+    print(f"  reg_inls = _mm256_set_epi32({inls[31]}, {inls[30]},")
+    print(f"                              {inls[29]}, {inls[28]},")
+    print(f"                              {inls[27]}, {inls[26]},")
+    print(f"                              {inls[25]}, {inls[24]});")
+    print(
+        "  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);")
+    print("  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);")
+    print("  out += 8;")
+
+    print("")
+    print(f"  in += {bit};")
+    print("")
+    print("  return in;")
+    print("}")
+
+
+def print_unpack_bit0_func():
+    print(
+        "inline static const uint32_t* unpack0_32_avx2(const uint32_t* in, uint32_t* out) {")
+    print("  memset(out, 0x0, 32 * sizeof(*out));")
+    print("  out += 32;")
+    print("")
+    print("  return in;")
+    print("}")
+
+
+def print_unpack_bit32_func():
+    print(
+        "inline static const uint32_t* unpack32_32_avx2(const uint32_t* in, uint32_t* out) {")
+    print("  memcpy(out, in, 32 * sizeof(*out));")
+    print("  in += 32;")
+    print("  out += 32;")
+    print("")
+    print("  return in;")
+    print("}")
+
+
+def print_copyright():
+    print(
+        """// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.""")
+
+
+def print_note():
+    print("//")
+    print("// Automatically generated file; DO NOT EDIT.")
+
+
+def main():
+    print_copyright()
+    print_note()
+    print("")
+    print("#pragma once")
+    print("")
+    print("#include <stdint.h>")
+    print("#include <string.h>")
+    print("")
+    print("#ifdef _MSC_VER")
+    print("#include <intrin.h>")
+    print("#else")
+    print("#include <immintrin.h>")
+    print("#endif")
+    print("")
+    print("namespace arrow {")
+    print("namespace internal {")
+    print("")
+    print_unpack_bit0_func()
+    print("")
+    for i in range(1, 32):
+        print_unpack_bit_func(i)
+        print("")
+    print_unpack_bit32_func()
+    print("")
+    print("}  // namespace internal")
+    print("}  // namespace arrow")
+
+
+if __name__ == '__main__':
+    main()
diff --git a/cpp/src/arrow/util/bpacking_avx2_generated.h b/cpp/src/arrow/util/bpacking_avx2_generated.h
new file mode 100644
index 00000000000..b2733e346e0
--- /dev/null
+++ b/cpp/src/arrow/util/bpacking_avx2_generated.h
@@ -0,0 +1,1786 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Automatically generated file; DO NOT EDIT.
+
+#pragma once
+
+#include <stdint.h>
+#include <string.h>
+
+#ifdef _MSC_VER
+#include <intrin.h>
+#else
+#include <immintrin.h>
+#endif
+
+namespace arrow {
+namespace internal {
+
+inline const uint32_t* unpack0_32_avx2(const uint32_t* in, uint32_t* out) {
+  memset(out, 0x0, 32 * sizeof(*out));
+  out += 32;
+
+  return in;
+}
+
+inline static const uint32_t* unpack1_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x1;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(7, 6, 5, 4,
+                               3, 2, 1, 0);
+  reg_inls = _mm256_set_epi32(in[0], in[0],
+                             in[0], in[0],
+                             in[0], in[0],
+                             in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(15, 14, 13, 12,
+                                11, 10, 9, 8);
+  reg_inls = _mm256_set_epi32(in[0], in[0],
+                              in[0], in[0],
+                              in[0], in[0],
+                              in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(23, 22, 21, 20,
+                                19, 18, 17, 16);
+  reg_inls = _mm256_set_epi32(in[0], in[0],
+                              in[0], in[0],
+                              in[0], in[0],
+                              in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(31, 30, 29, 28,
+                                27, 26, 25, 24);
+  reg_inls = _mm256_set_epi32(in[0], in[0],
+                              in[0], in[0],
+                              in[0], in[0],
+                              in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 1;
+
+  return in;
+}
+
+inline static const uint32_t* unpack2_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x3;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(14, 12, 10, 8,
+                               6, 4, 2, 0);
+  reg_inls = _mm256_set_epi32(in[0], in[0],
+                             in[0], in[0],
+                             in[0], in[0],
+                             in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(30, 28, 26, 24,
+                                22, 20, 18, 16);
+  reg_inls = _mm256_set_epi32(in[0], in[0],
+                              in[0], in[0],
+                              in[0], in[0],
+                              in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(14, 12, 10, 8,
+                                6, 4, 2, 0);
+  reg_inls = _mm256_set_epi32(in[1], in[1],
+                              in[1], in[1],
+                              in[1], in[1],
+                              in[1], in[1]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(30, 28, 26, 24,
+                                22, 20, 18, 16);
+  reg_inls = _mm256_set_epi32(in[1], in[1],
+                              in[1], in[1],
+                              in[1], in[1],
+                              in[1], in[1]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 2;
+
+  return in;
+}
+
+inline static const uint32_t* unpack3_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x7;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(21, 18, 15, 12,
+                               9, 6, 3, 0);
+  reg_inls = _mm256_set_epi32(in[0], in[0],
+                             in[0], in[0],
+                             in[0], in[0],
+                             in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(13, 10, 7, 4,
+                                1, 0, 27, 24);
+  reg_inls = _mm256_set_epi32(in[1], in[1],
+                              in[1], in[1],
+                              in[1], in[0] >> 30 | in[1] << 2,
+                              in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(5, 2, 0, 28,
+                                25, 22, 19, 16);
+  reg_inls = _mm256_set_epi32(in[2], in[2],
+                              in[1] >> 31 | in[2] << 1, in[1],
+                              in[1], in[1],
+                              in[1], in[1]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(29, 26, 23, 20,
+                                17, 14, 11, 8);
+  reg_inls = _mm256_set_epi32(in[2], in[2],
+                              in[2], in[2],
+                              in[2], in[2],
+                              in[2], in[2]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 3;
+
+  return in;
+}
+
+inline static const uint32_t* unpack4_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0xf;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(28, 24, 20, 16,
+                               12, 8, 4, 0);
+  reg_inls = _mm256_set_epi32(in[0], in[0],
+                             in[0], in[0],
+                             in[0], in[0],
+                             in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(28, 24, 20, 16,
+                                12, 8, 4, 0);
+  reg_inls = _mm256_set_epi32(in[1], in[1],
+                              in[1], in[1],
+                              in[1], in[1],
+                              in[1], in[1]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(28, 24, 20, 16,
+                                12, 8, 4, 0);
+  reg_inls = _mm256_set_epi32(in[2], in[2],
+                              in[2], in[2],
+                              in[2], in[2],
+                              in[2], in[2]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(28, 24, 20, 16,
+                                12, 8, 4, 0);
+  reg_inls = _mm256_set_epi32(in[3], in[3],
+                              in[3], in[3],
+                              in[3], in[3],
+                              in[3], in[3]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 4;
+
+  return in;
+}
+
+inline static const uint32_t* unpack5_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x1f;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(3, 0, 25, 20,
+                               15, 10, 5, 0);
+  reg_inls = _mm256_set_epi32(in[1], in[0] >> 30 | in[1] << 2,
+                             in[0], in[0],
+                             in[0], in[0],
+                             in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(11, 6, 1, 0,
+                                23, 18, 13, 8);
+  reg_inls = _mm256_set_epi32(in[2], in[2],
+                              in[2], in[1] >> 28 | in[2] << 4,
+                              in[1], in[1],
+                              in[1], in[1]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(19, 14, 9, 4,
+                                0, 26, 21, 16);
+  reg_inls = _mm256_set_epi32(in[3], in[3],
+                              in[3], in[3],
+                              in[2] >> 31 | in[3] << 1, in[2],
+                              in[2], in[2]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(27, 22, 17, 12,
+                                7, 2, 0, 24);
+  reg_inls = _mm256_set_epi32(in[4], in[4],
+                              in[4], in[4],
+                              in[4], in[4],
+                              in[3] >> 29 | in[4] << 3, in[3]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 5;
+
+  return in;
+}
+
+inline static const uint32_t* unpack6_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x3f;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(10, 4, 0, 24,
+                               18, 12, 6, 0);
+  reg_inls = _mm256_set_epi32(in[1], in[1],
+                             in[0] >> 30 | in[1] << 2, in[0],
+                             in[0], in[0],
+                             in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(26, 20, 14, 8,
+                                2, 0, 22, 16);
+  reg_inls = _mm256_set_epi32(in[2], in[2],
+                              in[2], in[2],
+                              in[2], in[1] >> 28 | in[2] << 4,
+                              in[1], in[1]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(10, 4, 0, 24,
+                                18, 12, 6, 0);
+  reg_inls = _mm256_set_epi32(in[4], in[4],
+                              in[3] >> 30 | in[4] << 2, in[3],
+                              in[3], in[3],
+                              in[3], in[3]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(26, 20, 14, 8,
+                                2, 0, 22, 16);
+  reg_inls = _mm256_set_epi32(in[5], in[5],
+                              in[5], in[5],
+                              in[5], in[4] >> 28 | in[5] << 4,
+                              in[4], in[4]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 6;
+
+  return in;
+}
+
+inline static const uint32_t* unpack7_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x7f;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(17, 10, 3, 0,
+                               21, 14, 7, 0);
+  reg_inls = _mm256_set_epi32(in[1], in[1],
+                             in[1], in[0] >> 28 | in[1] << 4,
+                             in[0], in[0],
+                             in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(9, 2, 0, 20,
+                                13, 6, 0, 24);
+  reg_inls = _mm256_set_epi32(in[3], in[3],
+                              in[2] >> 27 | in[3] << 5, in[2],
+                              in[2], in[2],
+                              in[1] >> 31 | in[2] << 1, in[1]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(1, 0, 19, 12,
+                                5, 0, 23, 16);
+  reg_inls = _mm256_set_epi32(in[5], in[4] >> 26 | in[5] << 6,
+                              in[4], in[4],
+                              in[4], in[3] >> 30 | in[4] << 2,
+                              in[3], in[3]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(25, 18, 11, 4,
+                                0, 22, 15, 8);
+  reg_inls = _mm256_set_epi32(in[6], in[6],
+                              in[6], in[6],
+                              in[5] >> 29 | in[6] << 3, in[5],
+                              in[5], in[5]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 7;
+
+  return in;
+}
+
+inline static const uint32_t* unpack8_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0xff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(24, 16, 8, 0,
+                               24, 16, 8, 0);
+  reg_inls = _mm256_set_epi32(in[1], in[1],
+                             in[1], in[1],
+                             in[0], in[0],
+                             in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(24, 16, 8, 0,
+                                24, 16, 8, 0);
+  reg_inls = _mm256_set_epi32(in[3], in[3],
+                              in[3], in[3],
+                              in[2], in[2],
+                              in[2], in[2]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(24, 16, 8, 0,
+                                24, 16, 8, 0);
+  reg_inls = _mm256_set_epi32(in[5], in[5],
+                              in[5], in[5],
+                              in[4], in[4],
+                              in[4], in[4]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(24, 16, 8, 0,
+                                24, 16, 8, 0);
+  reg_inls = _mm256_set_epi32(in[7], in[7],
+                              in[7], in[7],
+                              in[6], in[6],
+                              in[6], in[6]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 8;
+
+  return in;
+}
+
+inline static const uint32_t* unpack9_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x1ff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(0, 22, 13, 4,
+                               0, 18, 9, 0);
+  reg_inls = _mm256_set_epi32(in[1] >> 31 | in[2] << 1, in[1],
+                             in[1], in[1],
+                             in[0] >> 27 | in[1] << 5, in[0],
+                             in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(7, 0, 21, 12,
+                                3, 0, 17, 8);
+  reg_inls = _mm256_set_epi32(in[4], in[3] >> 30 | in[4] << 2,
+                              in[3], in[3],
+                              in[3], in[2] >> 26 | in[3] << 6,
+                              in[2], in[2]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(15, 6, 0, 20,
+                                11, 2, 0, 16);
+  reg_inls = _mm256_set_epi32(in[6], in[6],
+                              in[5] >> 29 | in[6] << 3, in[5],
+                              in[5], in[5],
+                              in[4] >> 25 | in[5] << 7, in[4]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(23, 14, 5, 0,
+                                19, 10, 1, 0);
+  reg_inls = _mm256_set_epi32(in[8], in[8],
+                              in[8], in[7] >> 28 | in[8] << 4,
+                              in[7], in[7],
+                              in[7], in[6] >> 24 | in[7] << 8);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 9;
+
+  return in;
+}
+
+inline static const uint32_t* unpack10_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x3ff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(6, 0, 18, 8,
+                               0, 20, 10, 0);
+  reg_inls = _mm256_set_epi32(in[2], in[1] >> 28 | in[2] << 4,
+                             in[1], in[1],
+                             in[0] >> 30 | in[1] << 2, in[0],
+                             in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(22, 12, 2, 0,
+                                14, 4, 0, 16);
+  reg_inls = _mm256_set_epi32(in[4], in[4],
+                              in[4], in[3] >> 24 | in[4] << 8,
+                              in[3], in[3],
+                              in[2] >> 26 | in[3] << 6, in[2]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(6, 0, 18, 8,
+                                0, 20, 10, 0);
+  reg_inls = _mm256_set_epi32(in[7], in[6] >> 28 | in[7] << 4,
+                              in[6], in[6],
+                              in[5] >> 30 | in[6] << 2, in[5],
+                              in[5], in[5]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(22, 12, 2, 0,
+                                14, 4, 0, 16);
+  reg_inls = _mm256_set_epi32(in[9], in[9],
+                              in[9], in[8] >> 24 | in[9] << 8,
+                              in[8], in[8],
+                              in[7] >> 26 | in[8] << 6, in[7]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 10;
+
+  return in;
+}
+
+inline static const uint32_t* unpack11_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x7ff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(13, 2, 0, 12,
+                               1, 0, 11, 0);
+  reg_inls = _mm256_set_epi32(in[2], in[2],
+                             in[1] >> 23 | in[2] << 9, in[1],
+                             in[1], in[0] >> 22 | in[1] << 10,
+                             in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(5, 0, 15, 4,
+                                0, 14, 3, 0);
+  reg_inls = _mm256_set_epi32(in[5], in[4] >> 26 | in[5] << 6,
+                              in[4], in[4],
+                              in[3] >> 25 | in[4] << 7, in[3],
+                              in[3], in[2] >> 24 | in[3] << 8);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(0, 18, 7, 0,
+                                17, 6, 0, 16);
+  reg_inls = _mm256_set_epi32(in[7] >> 29 | in[8] << 3, in[7],
+                              in[7], in[6] >> 28 | in[7] << 4,
+                              in[6], in[6],
+                              in[5] >> 27 | in[6] << 5, in[5]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(21, 10, 0, 20,
+                                9, 0, 19, 8);
+  reg_inls = _mm256_set_epi32(in[10], in[10],
+                              in[9] >> 31 | in[10] << 1, in[9],
+                              in[9], in[8] >> 30 | in[9] << 2,
+                              in[8], in[8]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 11;
+
+  return in;
+}
+
+inline static const uint32_t* unpack12_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0xfff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(20, 8, 0, 16,
+                               4, 0, 12, 0);
+  reg_inls = _mm256_set_epi32(in[2], in[2],
+                             in[1] >> 28 | in[2] << 4, in[1],
+                             in[1], in[0] >> 24 | in[1] << 8,
+                             in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(20, 8, 0, 16,
+                                4, 0, 12, 0);
+  reg_inls = _mm256_set_epi32(in[5], in[5],
+                              in[4] >> 28 | in[5] << 4, in[4],
+                              in[4], in[3] >> 24 | in[4] << 8,
+                              in[3], in[3]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(20, 8, 0, 16,
+                                4, 0, 12, 0);
+  reg_inls = _mm256_set_epi32(in[8], in[8],
+                              in[7] >> 28 | in[8] << 4, in[7],
+                              in[7], in[6] >> 24 | in[7] << 8,
+                              in[6], in[6]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(20, 8, 0, 16,
+                                4, 0, 12, 0);
+  reg_inls = _mm256_set_epi32(in[11], in[11],
+                              in[10] >> 28 | in[11] << 4, in[10],
+                              in[10], in[9] >> 24 | in[10] << 8,
+                              in[9], in[9]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 12;
+
+  return in;
+}
+
+inline static const uint32_t* unpack13_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x1fff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(0, 14, 1, 0,
+                               7, 0, 13, 0);
+  reg_inls = _mm256_set_epi32(in[2] >> 27 | in[3] << 5, in[2],
+                             in[2], in[1] >> 20 | in[2] << 12,
+                             in[1], in[0] >> 26 | in[1] << 6,
+                             in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(3, 0, 9, 0,
+                                15, 2, 0, 8);
+  reg_inls = _mm256_set_epi32(in[6], in[5] >> 22 | in[6] << 10,
+                              in[5], in[4] >> 28 | in[5] << 4,
+                              in[4], in[4],
+                              in[3] >> 21 | in[4] << 11, in[3]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(11, 0, 17, 4,
+                                0, 10, 0, 16);
+  reg_inls = _mm256_set_epi32(in[9], in[8] >> 30 | in[9] << 2,
+                              in[8], in[8],
+                              in[7] >> 23 | in[8] << 9, in[7],
+                              in[6] >> 29 | in[7] << 3, in[6]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(19, 6, 0, 12,
+                                0, 18, 5, 0);
+  reg_inls = _mm256_set_epi32(in[12], in[12],
+                              in[11] >> 25 | in[12] << 7, in[11],
+                              in[10] >> 31 | in[11] << 1, in[10],
+                              in[10], in[9] >> 24 | in[10] << 8);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 13;
+
+  return in;
+}
+
+inline static const uint32_t* unpack14_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x3fff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(2, 0, 6, 0,
+                               10, 0, 14, 0);
+  reg_inls = _mm256_set_epi32(in[3], in[2] >> 20 | in[3] << 12,
+                             in[2], in[1] >> 24 | in[2] << 8,
+                             in[1], in[0] >> 28 | in[1] << 4,
+                             in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(18, 4, 0, 8,
+                                0, 12, 0, 16);
+  reg_inls = _mm256_set_epi32(in[6], in[6],
+                              in[5] >> 22 | in[6] << 10, in[5],
+                              in[4] >> 26 | in[5] << 6, in[4],
+                              in[3] >> 30 | in[4] << 2, in[3]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(2, 0, 6, 0,
+                                10, 0, 14, 0);
+  reg_inls = _mm256_set_epi32(in[10], in[9] >> 20 | in[10] << 12,
+                              in[9], in[8] >> 24 | in[9] << 8,
+                              in[8], in[7] >> 28 | in[8] << 4,
+                              in[7], in[7]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(18, 4, 0, 8,
+                                0, 12, 0, 16);
+  reg_inls = _mm256_set_epi32(in[13], in[13],
+                              in[12] >> 22 | in[13] << 10, in[12],
+                              in[11] >> 26 | in[12] << 6, in[11],
+                              in[10] >> 30 | in[11] << 2, in[10]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 14;
+
+  return in;
+}
+
+inline static const uint32_t* unpack15_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x7fff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(9, 0, 11, 0,
+                               13, 0, 15, 0);
+  reg_inls = _mm256_set_epi32(in[3], in[2] >> 26 | in[3] << 6,
+                             in[2], in[1] >> 28 | in[2] << 4,
+                             in[1], in[0] >> 30 | in[1] << 2,
+                             in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(1, 0, 3, 0,
+                                5, 0, 7, 0);
+  reg_inls = _mm256_set_epi32(in[7], in[6] >> 18 | in[7] << 14,
+                              in[6], in[5] >> 20 | in[6] << 12,
+                              in[5], in[4] >> 22 | in[5] << 10,
+                              in[4], in[3] >> 24 | in[4] << 8);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(0, 10, 0, 12,
+                                0, 14, 0, 16);
+  reg_inls = _mm256_set_epi32(in[10] >> 25 | in[11] << 7, in[10],
+                              in[9] >> 27 | in[10] << 5, in[9],
+                              in[8] >> 29 | in[9] << 3, in[8],
+                              in[7] >> 31 | in[8] << 1, in[7]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(17, 2, 0, 4,
+                                0, 6, 0, 8);
+  reg_inls = _mm256_set_epi32(in[14], in[14],
+                              in[13] >> 19 | in[14] << 13, in[13],
+                              in[12] >> 21 | in[13] << 11, in[12],
+                              in[11] >> 23 | in[12] << 9, in[11]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 15;
+
+  return in;
+}
+
+inline static const uint32_t* unpack16_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0xffff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(16, 0, 16, 0,
+                               16, 0, 16, 0);
+  reg_inls = _mm256_set_epi32(in[3], in[3],
+                             in[2], in[2],
+                             in[1], in[1],
+                             in[0], in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(16, 0, 16, 0,
+                                16, 0, 16, 0);
+  reg_inls = _mm256_set_epi32(in[7], in[7],
+                              in[6], in[6],
+                              in[5], in[5],
+                              in[4], in[4]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(16, 0, 16, 0,
+                                16, 0, 16, 0);
+  reg_inls = _mm256_set_epi32(in[11], in[11],
+                              in[10], in[10],
+                              in[9], in[9],
+                              in[8], in[8]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(16, 0, 16, 0,
+                                16, 0, 16, 0);
+  reg_inls = _mm256_set_epi32(in[15], in[15],
+                              in[14], in[14],
+                              in[13], in[13],
+                              in[12], in[12]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 16;
+
+  return in;
+}
+
+inline static const uint32_t* unpack17_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x1ffff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(0, 6, 0, 4,
+                               0, 2, 0, 0);
+  reg_inls = _mm256_set_epi32(in[3] >> 23 | in[4] << 9, in[3],
+                             in[2] >> 21 | in[3] << 11, in[2],
+                             in[1] >> 19 | in[2] << 13, in[1],
+                             in[0] >> 17 | in[1] << 15, in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(0, 14, 0, 12,
+                                0, 10, 0, 8);
+  reg_inls = _mm256_set_epi32(in[7] >> 31 | in[8] << 1, in[7],
+                              in[6] >> 29 | in[7] << 3, in[6],
+                              in[5] >> 27 | in[6] << 5, in[5],
+                              in[4] >> 25 | in[5] << 7, in[4]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(7, 0, 5, 0,
+                                3, 0, 1, 0);
+  reg_inls = _mm256_set_epi32(in[12], in[11] >> 22 | in[12] << 10,
+                              in[11], in[10] >> 20 | in[11] << 12,
+                              in[10], in[9] >> 18 | in[10] << 14,
+                              in[9], in[8] >> 16 | in[9] << 16);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(15, 0, 13, 0,
+                                11, 0, 9, 0);
+  reg_inls = _mm256_set_epi32(in[16], in[15] >> 30 | in[16] << 2,
+                              in[15], in[14] >> 28 | in[15] << 4,
+                              in[14], in[13] >> 26 | in[14] << 6,
+                              in[13], in[12] >> 24 | in[13] << 8);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 17;
+
+  return in;
+}
+
+inline static const uint32_t* unpack18_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x3ffff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(0, 12, 0, 8,
+                               0, 4, 0, 0);
+  reg_inls = _mm256_set_epi32(in[3] >> 30 | in[4] << 2, in[3],
+                             in[2] >> 26 | in[3] << 6, in[2],
+                             in[1] >> 22 | in[2] << 10, in[1],
+                             in[0] >> 18 | in[1] << 14, in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(14, 0, 10, 0,
+                                6, 0, 2, 0);
+  reg_inls = _mm256_set_epi32(in[8], in[7] >> 28 | in[8] << 4,
+                              in[7], in[6] >> 24 | in[7] << 8,
+                              in[6], in[5] >> 20 | in[6] << 12,
+                              in[5], in[4] >> 16 | in[5] << 16);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(0, 12, 0, 8,
+                                0, 4, 0, 0);
+  reg_inls = _mm256_set_epi32(in[12] >> 30 | in[13] << 2, in[12],
+                              in[11] >> 26 | in[12] << 6, in[11],
+                              in[10] >> 22 | in[11] << 10, in[10],
+                              in[9] >> 18 | in[10] << 14, in[9]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(14, 0, 10, 0,
+                                6, 0, 2, 0);
+  reg_inls = _mm256_set_epi32(in[17], in[16] >> 28 | in[17] << 4,
+                              in[16], in[15] >> 24 | in[16] << 8,
+                              in[15], in[14] >> 20 | in[15] << 12,
+                              in[14], in[13] >> 16 | in[14] << 16);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 18;
+
+  return in;
+}
+
+inline static const uint32_t* unpack19_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x7ffff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(5, 0, 0, 12,
+                               0, 6, 0, 0);
+  reg_inls = _mm256_set_epi32(in[4], in[3] >> 18 | in[4] << 14,
+                             in[2] >> 31 | in[3] << 1, in[2],
+                             in[1] >> 25 | in[2] << 7, in[1],
+                             in[0] >> 19 | in[1] << 13, in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(0, 10, 0, 4,
+                                0, 0, 11, 0);
+  reg_inls = _mm256_set_epi32(in[8] >> 29 | in[9] << 3, in[8],
+                              in[7] >> 23 | in[8] << 9, in[7],
+                              in[6] >> 17 | in[7] << 15, in[5] >> 30 | in[6] << 2,
+                              in[5], in[4] >> 24 | in[5] << 8);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(0, 2, 0, 0,
+                                9, 0, 3, 0);
+  reg_inls = _mm256_set_epi32(in[13] >> 21 | in[14] << 11, in[13],
+                              in[12] >> 15 | in[13] << 17, in[11] >> 28 | in[12] << 4,
+                              in[11], in[10] >> 22 | in[11] << 10,
+                              in[10], in[9] >> 16 | in[10] << 16);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(13, 0, 7, 0,
+                                1, 0, 0, 8);
+  reg_inls = _mm256_set_epi32(in[18], in[17] >> 26 | in[18] << 6,
+                              in[17], in[16] >> 20 | in[17] << 12,
+                              in[16], in[15] >> 14 | in[16] << 18,
+                              in[14] >> 27 | in[15] << 5, in[14]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 19;
+
+  return in;
+}
+
+inline static const uint32_t* unpack20_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0xfffff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(12, 0, 4, 0,
+                               0, 8, 0, 0);
+  reg_inls = _mm256_set_epi32(in[4], in[3] >> 24 | in[4] << 8,
+                             in[3], in[2] >> 16 | in[3] << 16,
+                             in[1] >> 28 | in[2] << 4, in[1],
+                             in[0] >> 20 | in[1] << 12, in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(12, 0, 4, 0,
+                                0, 8, 0, 0);
+  reg_inls = _mm256_set_epi32(in[9], in[8] >> 24 | in[9] << 8,
+                              in[8], in[7] >> 16 | in[8] << 16,
+                              in[6] >> 28 | in[7] << 4, in[6],
+                              in[5] >> 20 | in[6] << 12, in[5]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(12, 0, 4, 0,
+                                0, 8, 0, 0);
+  reg_inls = _mm256_set_epi32(in[14], in[13] >> 24 | in[14] << 8,
+                              in[13], in[12] >> 16 | in[13] << 16,
+                              in[11] >> 28 | in[12] << 4, in[11],
+                              in[10] >> 20 | in[11] << 12, in[10]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(12, 0, 4, 0,
+                                0, 8, 0, 0);
+  reg_inls = _mm256_set_epi32(in[19], in[18] >> 24 | in[19] << 8,
+                              in[18], in[17] >> 16 | in[18] << 16,
+                              in[16] >> 28 | in[17] << 4, in[16],
+                              in[15] >> 20 | in[16] << 12, in[15]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 20;
+
+  return in;
+}
+
+inline static const uint32_t* unpack21_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x1fffff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(0, 0, 9, 0,
+                               0, 10, 0, 0);
+  reg_inls = _mm256_set_epi32(in[4] >> 19 | in[5] << 13, in[3] >> 30 | in[4] << 2,
+                             in[3], in[2] >> 20 | in[3] << 12,
+                             in[1] >> 31 | in[2] << 1, in[1],
+                             in[0] >> 21 | in[1] << 11, in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(0, 6, 0, 0,
+                                7, 0, 0, 8);
+  reg_inls = _mm256_set_epi32(in[9] >> 27 | in[10] << 5, in[9],
+                              in[8] >> 17 | in[9] << 15, in[7] >> 28 | in[8] << 4,
+                              in[7], in[6] >> 18 | in[7] << 14,
+                              in[5] >> 29 | in[6] << 3, in[5]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(3, 0, 0, 4,
+                                0, 0, 5, 0);
+  reg_inls = _mm256_set_epi32(in[15], in[14] >> 14 | in[15] << 18,
+                              in[13] >> 25 | in[14] << 7, in[13],
+                              in[12] >> 15 | in[13] << 17, in[11] >> 26 | in[12] << 6,
+                              in[11], in[10] >> 16 | in[11] << 16);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(11, 0, 1, 0,
+                                0, 2, 0, 0);
+  reg_inls = _mm256_set_epi32(in[20], in[19] >> 22 | in[20] << 10,
+                              in[19], in[18] >> 12 | in[19] << 20,
+                              in[17] >> 23 | in[18] << 9, in[17],
+                              in[16] >> 13 | in[17] << 19, in[15] >> 24 | in[16] << 8);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 21;
+
+  return in;
+}
+
+inline static const uint32_t* unpack22_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x3fffff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(0, 4, 0, 0,
+                               2, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[4] >> 26 | in[5] << 6, in[4],
+                             in[3] >> 14 | in[4] << 18, in[2] >> 24 | in[3] << 8,
+                             in[2], in[1] >> 12 | in[2] << 20,
+                             in[0] >> 22 | in[1] << 10, in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(10, 0, 0, 8,
+                                0, 0, 6, 0);
+  reg_inls = _mm256_set_epi32(in[10], in[9] >> 20 | in[10] << 12,
+                              in[8] >> 30 | in[9] << 2, in[8],
+                              in[7] >> 18 | in[8] << 14, in[6] >> 28 | in[7] << 4,
+                              in[6], in[5] >> 16 | in[6] << 16);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(0, 4, 0, 0,
+                                2, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[15] >> 26 | in[16] << 6, in[15],
+                              in[14] >> 14 | in[15] << 18, in[13] >> 24 | in[14] << 8,
+                              in[13], in[12] >> 12 | in[13] << 20,
+                              in[11] >> 22 | in[12] << 10, in[11]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(10, 0, 0, 8,
+                                0, 0, 6, 0);
+  reg_inls = _mm256_set_epi32(in[21], in[20] >> 20 | in[21] << 12,
+                              in[19] >> 30 | in[20] << 2, in[19],
+                              in[18] >> 18 | in[19] << 14, in[17] >> 28 | in[18] << 4,
+                              in[17], in[16] >> 16 | in[17] << 16);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 22;
+
+  return in;
+}
+
+inline static const uint32_t* unpack23_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x7fffff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(1, 0, 0, 0,
+                               5, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[5], in[4] >> 10 | in[5] << 22,
+                             in[3] >> 19 | in[4] << 13, in[2] >> 28 | in[3] << 4,
+                             in[2], in[1] >> 14 | in[2] << 18,
+                             in[0] >> 23 | in[1] << 9, in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(0, 2, 0, 0,
+                                0, 6, 0, 0);
+  reg_inls = _mm256_set_epi32(in[10] >> 25 | in[11] << 7, in[10],
+                              in[9] >> 11 | in[10] << 21, in[8] >> 20 | in[9] << 12,
+                              in[7] >> 29 | in[8] << 3, in[7],
+                              in[6] >> 15 | in[7] << 17, in[5] >> 24 | in[6] << 8);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(0, 0, 3, 0,
+                                0, 0, 7, 0);
+  reg_inls = _mm256_set_epi32(in[16] >> 17 | in[17] << 15, in[15] >> 26 | in[16] << 6,
+                              in[15], in[14] >> 12 | in[15] << 20,
+                              in[13] >> 21 | in[14] << 11, in[12] >> 30 | in[13] << 2,
+                              in[12], in[11] >> 16 | in[12] << 16);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(9, 0, 0, 4,
+                                0, 0, 0, 8);
+  reg_inls = _mm256_set_epi32(in[22], in[21] >> 18 | in[22] << 14,
+                              in[20] >> 27 | in[21] << 5, in[20],
+                              in[19] >> 13 | in[20] << 19, in[18] >> 22 | in[19] << 10,
+                              in[17] >> 31 | in[18] << 1, in[17]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 23;
+
+  return in;
+}
+
+inline static const uint32_t* unpack24_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0xffffff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(8, 0, 0, 0,
+                               8, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[5], in[4] >> 16 | in[5] << 16,
+                             in[3] >> 24 | in[4] << 8, in[3],
+                             in[2], in[1] >> 16 | in[2] << 16,
+                             in[0] >> 24 | in[1] << 8, in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(8, 0, 0, 0,
+                                8, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[11], in[10] >> 16 | in[11] << 16,
+                              in[9] >> 24 | in[10] << 8, in[9],
+                              in[8], in[7] >> 16 | in[8] << 16,
+                              in[6] >> 24 | in[7] << 8, in[6]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(8, 0, 0, 0,
+                                8, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[17], in[16] >> 16 | in[17] << 16,
+                              in[15] >> 24 | in[16] << 8, in[15],
+                              in[14], in[13] >> 16 | in[14] << 16,
+                              in[12] >> 24 | in[13] << 8, in[12]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(8, 0, 0, 0,
+                                8, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[23], in[22] >> 16 | in[23] << 16,
+                              in[21] >> 24 | in[22] << 8, in[21],
+                              in[20], in[19] >> 16 | in[20] << 16,
+                              in[18] >> 24 | in[19] << 8, in[18]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 24;
+
+  return in;
+}
+
+inline static const uint32_t* unpack25_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x1ffffff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(0, 0, 0, 4,
+                               0, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[5] >> 15 | in[6] << 17, in[4] >> 22 | in[5] << 10,
+                             in[3] >> 29 | in[4] << 3, in[3],
+                             in[2] >> 11 | in[3] << 21, in[1] >> 18 | in[2] << 14,
+                             in[0] >> 25 | in[1] << 7, in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(0, 0, 5, 0,
+                                0, 0, 1, 0);
+  reg_inls = _mm256_set_epi32(in[11] >> 23 | in[12] << 9, in[10] >> 30 | in[11] << 2,
+                              in[10], in[9] >> 12 | in[10] << 20,
+                              in[8] >> 19 | in[9] << 13, in[7] >> 26 | in[8] << 6,
+                              in[7], in[6] >> 8 | in[7] << 24);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(0, 6, 0, 0,
+                                0, 2, 0, 0);
+  reg_inls = _mm256_set_epi32(in[17] >> 31 | in[18] << 1, in[17],
+                              in[16] >> 13 | in[17] << 19, in[15] >> 20 | in[16] << 12,
+                              in[14] >> 27 | in[15] << 5, in[14],
+                              in[13] >> 9 | in[14] << 23, in[12] >> 16 | in[13] << 16);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(7, 0, 0, 0,
+                                3, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[24], in[23] >> 14 | in[24] << 18,
+                              in[22] >> 21 | in[23] << 11, in[21] >> 28 | in[22] << 4,
+                              in[21], in[20] >> 10 | in[21] << 22,
+                              in[19] >> 17 | in[20] << 15, in[18] >> 24 | in[19] << 8);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 25;
+
+  return in;
+}
+
+inline static const uint32_t* unpack26_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x3ffffff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(0, 0, 2, 0,
+                               0, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[5] >> 22 | in[6] << 10, in[4] >> 28 | in[5] << 4,
+                             in[4], in[3] >> 8 | in[4] << 24,
+                             in[2] >> 14 | in[3] << 18, in[1] >> 20 | in[2] << 12,
+                             in[0] >> 26 | in[1] << 6, in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(6, 0, 0, 0,
+                                0, 4, 0, 0);
+  reg_inls = _mm256_set_epi32(in[12], in[11] >> 12 | in[12] << 20,
+                              in[10] >> 18 | in[11] << 14, in[9] >> 24 | in[10] << 8,
+                              in[8] >> 30 | in[9] << 2, in[8],
+                              in[7] >> 10 | in[8] << 22, in[6] >> 16 | in[7] << 16);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(0, 0, 2, 0,
+                                0, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[18] >> 22 | in[19] << 10, in[17] >> 28 | in[18] << 4,
+                              in[17], in[16] >> 8 | in[17] << 24,
+                              in[15] >> 14 | in[16] << 18, in[14] >> 20 | in[15] << 12,
+                              in[13] >> 26 | in[14] << 6, in[13]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(6, 0, 0, 0,
+                                0, 4, 0, 0);
+  reg_inls = _mm256_set_epi32(in[25], in[24] >> 12 | in[25] << 20,
+                              in[23] >> 18 | in[24] << 14, in[22] >> 24 | in[23] << 8,
+                              in[21] >> 30 | in[22] << 2, in[21],
+                              in[20] >> 10 | in[21] << 22, in[19] >> 16 | in[20] << 16);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 26;
+
+  return in;
+}
+
+inline static const uint32_t* unpack27_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x7ffffff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(0, 2, 0, 0,
+                               0, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[5] >> 29 | in[6] << 3, in[5],
+                             in[4] >> 7 | in[5] << 25, in[3] >> 12 | in[4] << 20,
+                             in[2] >> 17 | in[3] << 15, in[1] >> 22 | in[2] << 10,
+                             in[0] >> 27 | in[1] << 5, in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(0, 0, 0, 4,
+                                0, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[12] >> 21 | in[13] << 11, in[11] >> 26 | in[12] << 6,
+                              in[10] >> 31 | in[11] << 1, in[10],
+                              in[9] >> 9 | in[10] << 23, in[8] >> 14 | in[9] << 18,
+                              in[7] >> 19 | in[8] << 13, in[6] >> 24 | in[7] << 8);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(0, 0, 0, 0,
+                                1, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[19] >> 13 | in[20] << 19, in[18] >> 18 | in[19] << 14,
+                              in[17] >> 23 | in[18] << 9, in[16] >> 28 | in[17] << 4,
+                              in[16], in[15] >> 6 | in[16] << 26,
+                              in[14] >> 11 | in[15] << 21, in[13] >> 16 | in[14] << 16);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(5, 0, 0, 0,
+                                0, 0, 3, 0);
+  reg_inls = _mm256_set_epi32(in[26], in[25] >> 10 | in[26] << 22,
+                              in[24] >> 15 | in[25] << 17, in[23] >> 20 | in[24] << 12,
+                              in[22] >> 25 | in[23] << 7, in[21] >> 30 | in[22] << 2,
+                              in[21], in[20] >> 8 | in[21] << 24);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 27;
+
+  return in;
+}
+
+inline static const uint32_t* unpack28_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0xfffffff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(4, 0, 0, 0,
+                               0, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[6], in[5] >> 8 | in[6] << 24,
+                             in[4] >> 12 | in[5] << 20, in[3] >> 16 | in[4] << 16,
+                             in[2] >> 20 | in[3] << 12, in[1] >> 24 | in[2] << 8,
+                             in[0] >> 28 | in[1] << 4, in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(4, 0, 0, 0,
+                                0, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[13], in[12] >> 8 | in[13] << 24,
+                              in[11] >> 12 | in[12] << 20, in[10] >> 16 | in[11] << 16,
+                              in[9] >> 20 | in[10] << 12, in[8] >> 24 | in[9] << 8,
+                              in[7] >> 28 | in[8] << 4, in[7]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(4, 0, 0, 0,
+                                0, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[20], in[19] >> 8 | in[20] << 24,
+                              in[18] >> 12 | in[19] << 20, in[17] >> 16 | in[18] << 16,
+                              in[16] >> 20 | in[17] << 12, in[15] >> 24 | in[16] << 8,
+                              in[14] >> 28 | in[15] << 4, in[14]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(4, 0, 0, 0,
+                                0, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[27], in[26] >> 8 | in[27] << 24,
+                              in[25] >> 12 | in[26] << 20, in[24] >> 16 | in[25] << 16,
+                              in[23] >> 20 | in[24] << 12, in[22] >> 24 | in[23] << 8,
+                              in[21] >> 28 | in[22] << 4, in[21]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 28;
+
+  return in;
+}
+
+inline static const uint32_t* unpack29_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x1fffffff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(0, 0, 0, 0,
+                               0, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[6] >> 11 | in[7] << 21, in[5] >> 14 | in[6] << 18,
+                             in[4] >> 17 | in[5] << 15, in[3] >> 20 | in[4] << 12,
+                             in[2] >> 23 | in[3] << 9, in[1] >> 26 | in[2] << 6,
+                             in[0] >> 29 | in[1] << 3, in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(0, 0, 0, 0,
+                                0, 2, 0, 0);
+  reg_inls = _mm256_set_epi32(in[13] >> 19 | in[14] << 13, in[12] >> 22 | in[13] << 10,
+                              in[11] >> 25 | in[12] << 7, in[10] >> 28 | in[11] << 4,
+                              in[9] >> 31 | in[10] << 1, in[9],
+                              in[8] >> 5 | in[9] << 27, in[7] >> 8 | in[8] << 24);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(0, 0, 1, 0,
+                                0, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[20] >> 27 | in[21] << 5, in[19] >> 30 | in[20] << 2,
+                              in[19], in[18] >> 4 | in[19] << 28,
+                              in[17] >> 7 | in[18] << 25, in[16] >> 10 | in[17] << 22,
+                              in[15] >> 13 | in[16] << 19, in[14] >> 16 | in[15] << 16);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(3, 0, 0, 0,
+                                0, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[28], in[27] >> 6 | in[28] << 26,
+                              in[26] >> 9 | in[27] << 23, in[25] >> 12 | in[26] << 20,
+                              in[24] >> 15 | in[25] << 17, in[23] >> 18 | in[24] << 14,
+                              in[22] >> 21 | in[23] << 11, in[21] >> 24 | in[22] << 8);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 29;
+
+  return in;
+}
+
+inline static const uint32_t* unpack30_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x3fffffff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(0, 0, 0, 0,
+                               0, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[6] >> 18 | in[7] << 14, in[5] >> 20 | in[6] << 12,
+                             in[4] >> 22 | in[5] << 10, in[3] >> 24 | in[4] << 8,
+                             in[2] >> 26 | in[3] << 6, in[1] >> 28 | in[2] << 4,
+                             in[0] >> 30 | in[1] << 2, in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(2, 0, 0, 0,
+                                0, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[14], in[13] >> 4 | in[14] << 28,
+                              in[12] >> 6 | in[13] << 26, in[11] >> 8 | in[12] << 24,
+                              in[10] >> 10 | in[11] << 22, in[9] >> 12 | in[10] << 20,
+                              in[8] >> 14 | in[9] << 18, in[7] >> 16 | in[8] << 16);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(0, 0, 0, 0,
+                                0, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[21] >> 18 | in[22] << 14, in[20] >> 20 | in[21] << 12,
+                              in[19] >> 22 | in[20] << 10, in[18] >> 24 | in[19] << 8,
+                              in[17] >> 26 | in[18] << 6, in[16] >> 28 | in[17] << 4,
+                              in[15] >> 30 | in[16] << 2, in[15]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(2, 0, 0, 0,
+                                0, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[29], in[28] >> 4 | in[29] << 28,
+                              in[27] >> 6 | in[28] << 26, in[26] >> 8 | in[27] << 24,
+                              in[25] >> 10 | in[26] << 22, in[24] >> 12 | in[25] << 20,
+                              in[23] >> 14 | in[24] << 18, in[22] >> 16 | in[23] << 16);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 30;
+
+  return in;
+}
+
+inline static const uint32_t* unpack31_32_avx2(const uint32_t* in, uint32_t* out) {
+  uint32_t mask = 0x7fffffff;
+  __m256i reg_shifts, reg_inls, reg_masks;
+  __m256i results;
+
+  reg_masks = _mm256_set1_epi32(mask);
+
+  // shift the first 8 outs
+  reg_shifts = _mm256_set_epi32(0, 0, 0, 0,
+                               0, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[6] >> 25 | in[7] << 7, in[5] >> 26 | in[6] << 6,
+                             in[4] >> 27 | in[5] << 5, in[3] >> 28 | in[4] << 4,
+                             in[2] >> 29 | in[3] << 3, in[1] >> 30 | in[2] << 2,
+                             in[0] >> 31 | in[1] << 1, in[0]);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the second 8 outs
+  reg_shifts = _mm256_set_epi32(0, 0, 0, 0,
+                                0, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[14] >> 17 | in[15] << 15, in[13] >> 18 | in[14] << 14,
+                              in[12] >> 19 | in[13] << 13, in[11] >> 20 | in[12] << 12,
+                              in[10] >> 21 | in[11] << 11, in[9] >> 22 | in[10] << 10,
+                              in[8] >> 23 | in[9] << 9, in[7] >> 24 | in[8] << 8);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the third 8 outs
+  reg_shifts = _mm256_set_epi32(0, 0, 0, 0,
+                                0, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[22] >> 9 | in[23] << 23, in[21] >> 10 | in[22] << 22,
+                              in[20] >> 11 | in[21] << 21, in[19] >> 12 | in[20] << 20,
+                              in[18] >> 13 | in[19] << 19, in[17] >> 14 | in[18] << 18,
+                              in[16] >> 15 | in[17] << 17, in[15] >> 16 | in[16] << 16);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  // shift the last 8 outs
+  reg_shifts = _mm256_set_epi32(1, 0, 0, 0,
+                                0, 0, 0, 0);
+  reg_inls = _mm256_set_epi32(in[30], in[29] >> 2 | in[30] << 30,
+                              in[28] >> 3 | in[29] << 29, in[27] >> 4 | in[28] << 28,
+                              in[26] >> 5 | in[27] << 27, in[25] >> 6 | in[26] << 26,
+                              in[24] >> 7 | in[25] << 25, in[23] >> 8 | in[24] << 24);
+  results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);
+  out += 8;
+
+  in += 31;
+
+  return in;
+}
+
+inline const uint32_t* unpack32_32_avx2(const uint32_t* in, uint32_t* out) {
+  memcpy(out, in, 32 * sizeof(*out));
+  in += 32;
+  out += 32;
+
+  return in;
+}
+
+}  // namespace internal
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/bpacking_avx512.cc b/cpp/src/arrow/util/bpacking_avx512.cc
new file mode 100644
index 00000000000..98eb4d325af
--- /dev/null
+++ b/cpp/src/arrow/util/bpacking_avx512.cc
@@ -0,0 +1,137 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/util/bpacking_avx512.h"
+#include "arrow/util/bpacking_avx512_generated.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+namespace internal {
+
+int unpack32_avx512(const uint32_t* in, uint32_t* out, int batch_size, int num_bits) {
+  batch_size = batch_size / 32 * 32;
+  int num_loops = batch_size / 32;
+
+  switch (num_bits) {
+    case 0:
+      for (int i = 0; i < num_loops; ++i) in = unpack0_32_avx512(in, out + i * 32);
+      break;
+    case 1:
+      for (int i = 0; i < num_loops; ++i) in = unpack1_32_avx512(in, out + i * 32);
+      break;
+    case 2:
+      for (int i = 0; i < num_loops; ++i) in = unpack2_32_avx512(in, out + i * 32);
+      break;
+    case 3:
+      for (int i = 0; i < num_loops; ++i) in = unpack3_32_avx512(in, out + i * 32);
+      break;
+    case 4:
+      for (int i = 0; i < num_loops; ++i) in = unpack4_32_avx512(in, out + i * 32);
+      break;
+    case 5:
+      for (int i = 0; i < num_loops; ++i) in = unpack5_32_avx512(in, out + i * 32);
+      break;
+    case 6:
+      for (int i = 0; i < num_loops; ++i) in = unpack6_32_avx512(in, out + i * 32);
+      break;
+    case 7:
+      for (int i = 0; i < num_loops; ++i) in = unpack7_32_avx512(in, out + i * 32);
+      break;
+    case 8:
+      for (int i = 0; i < num_loops; ++i) in = unpack8_32_avx512(in, out + i * 32);
+      break;
+    case 9:
+      for (int i = 0; i < num_loops; ++i) in = unpack9_32_avx512(in, out + i * 32);
+      break;
+    case 10:
+      for (int i = 0; i < num_loops; ++i) in = unpack10_32_avx512(in, out + i * 32);
+      break;
+    case 11:
+      for (int i = 0; i < num_loops; ++i) in = unpack11_32_avx512(in, out + i * 32);
+      break;
+    case 12:
+      for (int i = 0; i < num_loops; ++i) in = unpack12_32_avx512(in, out + i * 32);
+      break;
+    case 13:
+      for (int i = 0; i < num_loops; ++i) in = unpack13_32_avx512(in, out + i * 32);
+      break;
+    case 14:
+      for (int i = 0; i < num_loops; ++i) in = unpack14_32_avx512(in, out + i * 32);
+      break;
+    case 15:
+      for (int i = 0; i < num_loops; ++i) in = unpack15_32_avx512(in, out + i * 32);
+      break;
+    case 16:
+      for (int i = 0; i < num_loops; ++i) in = unpack16_32_avx512(in, out + i * 32);
+      break;
+    case 17:
+      for (int i = 0; i < num_loops; ++i) in = unpack17_32_avx512(in, out + i * 32);
+      break;
+    case 18:
+      for (int i = 0; i < num_loops; ++i) in = unpack18_32_avx512(in, out + i * 32);
+      break;
+    case 19:
+      for (int i = 0; i < num_loops; ++i) in = unpack19_32_avx512(in, out + i * 32);
+      break;
+    case 20:
+      for (int i = 0; i < num_loops; ++i) in = unpack20_32_avx512(in, out + i * 32);
+      break;
+    case 21:
+      for (int i = 0; i < num_loops; ++i) in = unpack21_32_avx512(in, out + i * 32);
+      break;
+    case 22:
+      for (int i = 0; i < num_loops; ++i) in = unpack22_32_avx512(in, out + i * 32);
+      break;
+    case 23:
+      for (int i = 0; i < num_loops; ++i) in = unpack23_32_avx512(in, out + i * 32);
+      break;
+    case 24:
+      for (int i = 0; i < num_loops; ++i) in = unpack24_32_avx512(in, out + i * 32);
+      break;
+    case 25:
+      for (int i = 0; i < num_loops; ++i) in = unpack25_32_avx512(in, out + i * 32);
+      break;
+    case 26:
+      for (int i = 0; i < num_loops; ++i) in = unpack26_32_avx512(in, out + i * 32);
+      break;
+    case 27:
+      for (int i = 0; i < num_loops; ++i) in = unpack27_32_avx512(in, out + i * 32);
+      break;
+    case 28:
+      for (int i = 0; i < num_loops; ++i) in = unpack28_32_avx512(in, out + i * 32);
+      break;
+    case 29:
+      for (int i = 0; i < num_loops; ++i) in = unpack29_32_avx512(in, out + i * 32);
+      break;
+    case 30:
+      for (int i = 0; i < num_loops; ++i) in = unpack30_32_avx512(in, out + i * 32);
+      break;
+    case 31:
+      for (int i = 0; i < num_loops; ++i) in = unpack31_32_avx512(in, out + i * 32);
+      break;
+    case 32:
+      for (int i = 0; i < num_loops; ++i) in = unpack32_32_avx512(in, out + i * 32);
+      break;
+    default:
+      DCHECK(false) << "Unsupported num_bits";
+  }
+
+  return batch_size;
+}
+
+}  // namespace internal
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/bpacking_avx512.h b/cpp/src/arrow/util/bpacking_avx512.h
new file mode 100644
index 00000000000..96723f803e0
--- /dev/null
+++ b/cpp/src/arrow/util/bpacking_avx512.h
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <stdint.h>
+
+namespace arrow {
+namespace internal {
+
+int unpack32_avx512(const uint32_t* in, uint32_t* out, int batch_size, int num_bits);
+
+}  // namespace internal
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/bpacking_avx512_codegen.py b/cpp/src/arrow/util/bpacking_avx512_codegen.py
index 867ae6884c6..1a56189017d 100644
--- a/cpp/src/arrow/util/bpacking_avx512_codegen.py
+++ b/cpp/src/arrow/util/bpacking_avx512_codegen.py
@@ -29,10 +29,10 @@ def print_unpack_bit_func(bit):
     bracket = "{"
 
     print(
-        f"inline const uint32_t* unpack{bit}_32(const uint32_t* in, uint32_t* out) {bracket}")
+        f"inline static const uint32_t* unpack{bit}_32_avx512(const uint32_t* in, uint32_t* out) {bracket}")
     print("  uint32_t mask = 0x%x;" % mask)
     print("  __m512i reg_shifts, reg_inls, reg_masks;")
-    print("  __m512i results[2];")
+    print("  __m512i results;")
 
     print("")
     for i in range(32):
@@ -41,7 +41,7 @@ def print_unpack_bit_func(bit):
             inls.append(f"in[{in_index}]")
             in_index += 1
             shift = 0
-        elif shift + bit > 32:  # croos the boundary
+        elif shift + bit > 32:  # cross the boundary
             inls.append(
                 f"in[{in_index}] >> {shift} | in[{in_index + 1}] << {32 - shift}")
             in_index += 1
@@ -72,7 +72,9 @@ def print_unpack_bit_func(bit):
     print(f"                              {inls[3]}, {inls[2]},")
     print(f"                              {inls[1]}, {inls[0]});")
     print(
-        "  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);")
+        "  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);")
+    print("  _mm512_storeu_si512(out, results);")
+    print("  out += 16;")
     print("")
     print("  // shift the second 16 outs")
     print(
@@ -92,10 +94,10 @@ def print_unpack_bit_func(bit):
     print(f"                              {inls[19]}, {inls[18]},")
     print(f"                              {inls[17]}, {inls[16]});")
     print(
-        "  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);")
+        "  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);")
+    print("  _mm512_storeu_si512(out, results);")
+    print("  out += 16;")
     print("")
-    print("  memcpy(out, &results, 32 * sizeof(*out));")
-    print("  out += 32;")
     print(f"  in += {bit};")
     print("")
     print("  return in;")
@@ -104,7 +106,7 @@ def print_unpack_bit_func(bit):
 
 def print_unpack_bit0_func():
     print(
-        "inline const uint32_t* nullunpacker32(const uint32_t* in, uint32_t* out) {")
+        "inline static const uint32_t* unpack0_32_avx512(const uint32_t* in, uint32_t* out) {")
     print("  memset(out, 0x0, 32 * sizeof(*out));")
     print("  out += 32;")
     print("")
@@ -114,7 +116,7 @@ def print_unpack_bit0_func():
 
 def print_unpack_bit32_func():
     print(
-        "inline const uint32_t* unpack32_32(const uint32_t* in, uint32_t* out) {")
+        "inline static const uint32_t* unpack32_32_avx512(const uint32_t* in, uint32_t* out) {")
     print("  memcpy(out, in, 32 * sizeof(*out));")
     print("  in += 32;")
     print("  out += 32;")
@@ -154,7 +156,14 @@ def main():
     print("")
     print("#pragma once")
     print("")
+    print("#include <stdint.h>")
+    print("#include <string.h>")
+    print("")
+    print("#ifdef _MSC_VER")
+    print("#include <intrin.h>")
+    print("#else")
     print("#include <immintrin.h>")
+    print("#endif")
     print("")
     print("namespace arrow {")
     print("namespace internal {")
diff --git a/cpp/src/arrow/util/bpacking_avx512_generated.h b/cpp/src/arrow/util/bpacking_avx512_generated.h
index b63ff572d9a..b65bde5561e 100644
--- a/cpp/src/arrow/util/bpacking_avx512_generated.h
+++ b/cpp/src/arrow/util/bpacking_avx512_generated.h
@@ -19,22 +19,29 @@
 
 #pragma once
 
+#include <stdint.h>
+#include <string.h>
+
+#ifdef _MSC_VER
+#include <intrin.h>
+#else
 #include <immintrin.h>
+#endif
 
 namespace arrow {
 namespace internal {
 
-inline const uint32_t* nullunpacker32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack0_32_avx512(const uint32_t* in, uint32_t* out) {
   memset(out, 0x0, 32 * sizeof(*out));
   out += 32;
 
   return in;
 }
 
-inline const uint32_t* unpack1_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack1_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x1;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -51,7 +58,9 @@ inline const uint32_t* unpack1_32(const uint32_t* in, uint32_t* out) {
                               in[0], in[0],
                               in[0], in[0],
                               in[0], in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(31, 30, 29, 28,
@@ -66,19 +75,19 @@ inline const uint32_t* unpack1_32(const uint32_t* in, uint32_t* out) {
                               in[0], in[0],
                               in[0], in[0],
                               in[0], in[0]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 1;
 
   return in;
 }
 
-inline const uint32_t* unpack2_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack2_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x3;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -95,7 +104,9 @@ inline const uint32_t* unpack2_32(const uint32_t* in, uint32_t* out) {
                               in[0], in[0],
                               in[0], in[0],
                               in[0], in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(30, 28, 26, 24,
@@ -110,19 +121,19 @@ inline const uint32_t* unpack2_32(const uint32_t* in, uint32_t* out) {
                               in[1], in[1],
                               in[1], in[1],
                               in[1], in[1]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 2;
 
   return in;
 }
 
-inline const uint32_t* unpack3_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack3_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x7;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -139,7 +150,9 @@ inline const uint32_t* unpack3_32(const uint32_t* in, uint32_t* out) {
                               in[0], in[0],
                               in[0], in[0],
                               in[0], in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(29, 26, 23, 20,
@@ -154,19 +167,19 @@ inline const uint32_t* unpack3_32(const uint32_t* in, uint32_t* out) {
                               in[1] >> 31 | in[2] << 1, in[1],
                               in[1], in[1],
                               in[1], in[1]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 3;
 
   return in;
 }
 
-inline const uint32_t* unpack4_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack4_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0xf;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -183,7 +196,9 @@ inline const uint32_t* unpack4_32(const uint32_t* in, uint32_t* out) {
                               in[0], in[0],
                               in[0], in[0],
                               in[0], in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(28, 24, 20, 16,
@@ -198,19 +213,19 @@ inline const uint32_t* unpack4_32(const uint32_t* in, uint32_t* out) {
                               in[2], in[2],
                               in[2], in[2],
                               in[2], in[2]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 4;
 
   return in;
 }
 
-inline const uint32_t* unpack5_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack5_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x1f;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -227,7 +242,9 @@ inline const uint32_t* unpack5_32(const uint32_t* in, uint32_t* out) {
                               in[0], in[0],
                               in[0], in[0],
                               in[0], in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(27, 22, 17, 12,
@@ -242,19 +259,19 @@ inline const uint32_t* unpack5_32(const uint32_t* in, uint32_t* out) {
                               in[3], in[3],
                               in[2] >> 31 | in[3] << 1, in[2],
                               in[2], in[2]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 5;
 
   return in;
 }
 
-inline const uint32_t* unpack6_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack6_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x3f;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -271,7 +288,9 @@ inline const uint32_t* unpack6_32(const uint32_t* in, uint32_t* out) {
                               in[0] >> 30 | in[1] << 2, in[0],
                               in[0], in[0],
                               in[0], in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(26, 20, 14, 8,
@@ -286,19 +305,19 @@ inline const uint32_t* unpack6_32(const uint32_t* in, uint32_t* out) {
                               in[3] >> 30 | in[4] << 2, in[3],
                               in[3], in[3],
                               in[3], in[3]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 6;
 
   return in;
 }
 
-inline const uint32_t* unpack7_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack7_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x7f;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -315,7 +334,9 @@ inline const uint32_t* unpack7_32(const uint32_t* in, uint32_t* out) {
                               in[1], in[0] >> 28 | in[1] << 4,
                               in[0], in[0],
                               in[0], in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(25, 18, 11, 4,
@@ -330,19 +351,19 @@ inline const uint32_t* unpack7_32(const uint32_t* in, uint32_t* out) {
                               in[4], in[4],
                               in[4], in[3] >> 30 | in[4] << 2,
                               in[3], in[3]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 7;
 
   return in;
 }
 
-inline const uint32_t* unpack8_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack8_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0xff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -359,7 +380,9 @@ inline const uint32_t* unpack8_32(const uint32_t* in, uint32_t* out) {
                               in[1], in[1],
                               in[0], in[0],
                               in[0], in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(24, 16, 8, 0,
@@ -374,19 +397,19 @@ inline const uint32_t* unpack8_32(const uint32_t* in, uint32_t* out) {
                               in[5], in[5],
                               in[4], in[4],
                               in[4], in[4]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 8;
 
   return in;
 }
 
-inline const uint32_t* unpack9_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack9_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x1ff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -403,7 +426,9 @@ inline const uint32_t* unpack9_32(const uint32_t* in, uint32_t* out) {
                               in[1], in[1],
                               in[0] >> 27 | in[1] << 5, in[0],
                               in[0], in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(23, 14, 5, 0,
@@ -418,19 +443,19 @@ inline const uint32_t* unpack9_32(const uint32_t* in, uint32_t* out) {
                               in[5] >> 29 | in[6] << 3, in[5],
                               in[5], in[5],
                               in[4] >> 25 | in[5] << 7, in[4]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 9;
 
   return in;
 }
 
-inline const uint32_t* unpack10_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack10_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x3ff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -447,7 +472,9 @@ inline const uint32_t* unpack10_32(const uint32_t* in, uint32_t* out) {
                               in[1], in[1],
                               in[0] >> 30 | in[1] << 2, in[0],
                               in[0], in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(22, 12, 2, 0,
@@ -462,19 +489,19 @@ inline const uint32_t* unpack10_32(const uint32_t* in, uint32_t* out) {
                               in[6], in[6],
                               in[5] >> 30 | in[6] << 2, in[5],
                               in[5], in[5]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 10;
 
   return in;
 }
 
-inline const uint32_t* unpack11_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack11_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x7ff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -491,7 +518,9 @@ inline const uint32_t* unpack11_32(const uint32_t* in, uint32_t* out) {
                               in[1] >> 23 | in[2] << 9, in[1],
                               in[1], in[0] >> 22 | in[1] << 10,
                               in[0], in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(21, 10, 0, 20,
@@ -506,19 +535,19 @@ inline const uint32_t* unpack11_32(const uint32_t* in, uint32_t* out) {
                               in[7], in[6] >> 28 | in[7] << 4,
                               in[6], in[6],
                               in[5] >> 27 | in[6] << 5, in[5]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 11;
 
   return in;
 }
 
-inline const uint32_t* unpack12_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack12_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0xfff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -535,7 +564,9 @@ inline const uint32_t* unpack12_32(const uint32_t* in, uint32_t* out) {
                               in[1] >> 28 | in[2] << 4, in[1],
                               in[1], in[0] >> 24 | in[1] << 8,
                               in[0], in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(20, 8, 0, 16,
@@ -550,19 +581,19 @@ inline const uint32_t* unpack12_32(const uint32_t* in, uint32_t* out) {
                               in[7] >> 28 | in[8] << 4, in[7],
                               in[7], in[6] >> 24 | in[7] << 8,
                               in[6], in[6]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 12;
 
   return in;
 }
 
-inline const uint32_t* unpack13_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack13_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x1fff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -579,7 +610,9 @@ inline const uint32_t* unpack13_32(const uint32_t* in, uint32_t* out) {
                               in[2], in[1] >> 20 | in[2] << 12,
                               in[1], in[0] >> 26 | in[1] << 6,
                               in[0], in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(19, 6, 0, 12,
@@ -594,19 +627,19 @@ inline const uint32_t* unpack13_32(const uint32_t* in, uint32_t* out) {
                               in[8], in[8],
                               in[7] >> 23 | in[8] << 9, in[7],
                               in[6] >> 29 | in[7] << 3, in[6]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 13;
 
   return in;
 }
 
-inline const uint32_t* unpack14_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack14_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x3fff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -623,7 +656,9 @@ inline const uint32_t* unpack14_32(const uint32_t* in, uint32_t* out) {
                               in[2], in[1] >> 24 | in[2] << 8,
                               in[1], in[0] >> 28 | in[1] << 4,
                               in[0], in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(18, 4, 0, 8,
@@ -638,19 +673,19 @@ inline const uint32_t* unpack14_32(const uint32_t* in, uint32_t* out) {
                               in[9], in[8] >> 24 | in[9] << 8,
                               in[8], in[7] >> 28 | in[8] << 4,
                               in[7], in[7]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 14;
 
   return in;
 }
 
-inline const uint32_t* unpack15_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack15_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x7fff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -667,7 +702,9 @@ inline const uint32_t* unpack15_32(const uint32_t* in, uint32_t* out) {
                               in[2], in[1] >> 28 | in[2] << 4,
                               in[1], in[0] >> 30 | in[1] << 2,
                               in[0], in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(17, 2, 0, 4,
@@ -682,19 +719,19 @@ inline const uint32_t* unpack15_32(const uint32_t* in, uint32_t* out) {
                               in[9] >> 27 | in[10] << 5, in[9],
                               in[8] >> 29 | in[9] << 3, in[8],
                               in[7] >> 31 | in[8] << 1, in[7]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 15;
 
   return in;
 }
 
-inline const uint32_t* unpack16_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack16_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0xffff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -711,7 +748,9 @@ inline const uint32_t* unpack16_32(const uint32_t* in, uint32_t* out) {
                               in[2], in[2],
                               in[1], in[1],
                               in[0], in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(16, 0, 16, 0,
@@ -726,19 +765,19 @@ inline const uint32_t* unpack16_32(const uint32_t* in, uint32_t* out) {
                               in[10], in[10],
                               in[9], in[9],
                               in[8], in[8]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 16;
 
   return in;
 }
 
-inline const uint32_t* unpack17_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack17_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x1ffff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -755,7 +794,9 @@ inline const uint32_t* unpack17_32(const uint32_t* in, uint32_t* out) {
                               in[2] >> 21 | in[3] << 11, in[2],
                               in[1] >> 19 | in[2] << 13, in[1],
                               in[0] >> 17 | in[1] << 15, in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(15, 0, 13, 0,
@@ -770,19 +811,19 @@ inline const uint32_t* unpack17_32(const uint32_t* in, uint32_t* out) {
                               in[11], in[10] >> 20 | in[11] << 12,
                               in[10], in[9] >> 18 | in[10] << 14,
                               in[9], in[8] >> 16 | in[9] << 16);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 17;
 
   return in;
 }
 
-inline const uint32_t* unpack18_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack18_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x3ffff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -799,7 +840,9 @@ inline const uint32_t* unpack18_32(const uint32_t* in, uint32_t* out) {
                               in[2] >> 26 | in[3] << 6, in[2],
                               in[1] >> 22 | in[2] << 10, in[1],
                               in[0] >> 18 | in[1] << 14, in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(14, 0, 10, 0,
@@ -814,19 +857,19 @@ inline const uint32_t* unpack18_32(const uint32_t* in, uint32_t* out) {
                               in[11] >> 26 | in[12] << 6, in[11],
                               in[10] >> 22 | in[11] << 10, in[10],
                               in[9] >> 18 | in[10] << 14, in[9]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 18;
 
   return in;
 }
 
-inline const uint32_t* unpack19_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack19_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x7ffff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -843,7 +886,9 @@ inline const uint32_t* unpack19_32(const uint32_t* in, uint32_t* out) {
                               in[2] >> 31 | in[3] << 1, in[2],
                               in[1] >> 25 | in[2] << 7, in[1],
                               in[0] >> 19 | in[1] << 13, in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(13, 0, 7, 0,
@@ -858,19 +903,19 @@ inline const uint32_t* unpack19_32(const uint32_t* in, uint32_t* out) {
                               in[12] >> 15 | in[13] << 17, in[11] >> 28 | in[12] << 4,
                               in[11], in[10] >> 22 | in[11] << 10,
                               in[10], in[9] >> 16 | in[10] << 16);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 19;
 
   return in;
 }
 
-inline const uint32_t* unpack20_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack20_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0xfffff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -887,7 +932,9 @@ inline const uint32_t* unpack20_32(const uint32_t* in, uint32_t* out) {
                               in[3], in[2] >> 16 | in[3] << 16,
                               in[1] >> 28 | in[2] << 4, in[1],
                               in[0] >> 20 | in[1] << 12, in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(12, 0, 4, 0,
@@ -902,19 +949,19 @@ inline const uint32_t* unpack20_32(const uint32_t* in, uint32_t* out) {
                               in[13], in[12] >> 16 | in[13] << 16,
                               in[11] >> 28 | in[12] << 4, in[11],
                               in[10] >> 20 | in[11] << 12, in[10]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 20;
 
   return in;
 }
 
-inline const uint32_t* unpack21_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack21_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x1fffff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -931,7 +978,9 @@ inline const uint32_t* unpack21_32(const uint32_t* in, uint32_t* out) {
                               in[3], in[2] >> 20 | in[3] << 12,
                               in[1] >> 31 | in[2] << 1, in[1],
                               in[0] >> 21 | in[1] << 11, in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(11, 0, 1, 0,
@@ -946,19 +995,19 @@ inline const uint32_t* unpack21_32(const uint32_t* in, uint32_t* out) {
                               in[13] >> 25 | in[14] << 7, in[13],
                               in[12] >> 15 | in[13] << 17, in[11] >> 26 | in[12] << 6,
                               in[11], in[10] >> 16 | in[11] << 16);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 21;
 
   return in;
 }
 
-inline const uint32_t* unpack22_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack22_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x3fffff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -975,7 +1024,9 @@ inline const uint32_t* unpack22_32(const uint32_t* in, uint32_t* out) {
                               in[3] >> 14 | in[4] << 18, in[2] >> 24 | in[3] << 8,
                               in[2], in[1] >> 12 | in[2] << 20,
                               in[0] >> 22 | in[1] << 10, in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(10, 0, 0, 8,
@@ -990,19 +1041,19 @@ inline const uint32_t* unpack22_32(const uint32_t* in, uint32_t* out) {
                               in[14] >> 14 | in[15] << 18, in[13] >> 24 | in[14] << 8,
                               in[13], in[12] >> 12 | in[13] << 20,
                               in[11] >> 22 | in[12] << 10, in[11]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 22;
 
   return in;
 }
 
-inline const uint32_t* unpack23_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack23_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x7fffff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -1019,7 +1070,9 @@ inline const uint32_t* unpack23_32(const uint32_t* in, uint32_t* out) {
                               in[3] >> 19 | in[4] << 13, in[2] >> 28 | in[3] << 4,
                               in[2], in[1] >> 14 | in[2] << 18,
                               in[0] >> 23 | in[1] << 9, in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(9, 0, 0, 4,
@@ -1034,19 +1087,19 @@ inline const uint32_t* unpack23_32(const uint32_t* in, uint32_t* out) {
                               in[15], in[14] >> 12 | in[15] << 20,
                               in[13] >> 21 | in[14] << 11, in[12] >> 30 | in[13] << 2,
                               in[12], in[11] >> 16 | in[12] << 16);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 23;
 
   return in;
 }
 
-inline const uint32_t* unpack24_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack24_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0xffffff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -1063,7 +1116,9 @@ inline const uint32_t* unpack24_32(const uint32_t* in, uint32_t* out) {
                               in[3] >> 24 | in[4] << 8, in[3],
                               in[2], in[1] >> 16 | in[2] << 16,
                               in[0] >> 24 | in[1] << 8, in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(8, 0, 0, 0,
@@ -1078,19 +1133,19 @@ inline const uint32_t* unpack24_32(const uint32_t* in, uint32_t* out) {
                               in[15] >> 24 | in[16] << 8, in[15],
                               in[14], in[13] >> 16 | in[14] << 16,
                               in[12] >> 24 | in[13] << 8, in[12]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 24;
 
   return in;
 }
 
-inline const uint32_t* unpack25_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack25_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x1ffffff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -1107,7 +1162,9 @@ inline const uint32_t* unpack25_32(const uint32_t* in, uint32_t* out) {
                               in[3] >> 29 | in[4] << 3, in[3],
                               in[2] >> 11 | in[3] << 21, in[1] >> 18 | in[2] << 14,
                               in[0] >> 25 | in[1] << 7, in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(7, 0, 0, 0,
@@ -1122,19 +1179,19 @@ inline const uint32_t* unpack25_32(const uint32_t* in, uint32_t* out) {
                               in[16] >> 13 | in[17] << 19, in[15] >> 20 | in[16] << 12,
                               in[14] >> 27 | in[15] << 5, in[14],
                               in[13] >> 9 | in[14] << 23, in[12] >> 16 | in[13] << 16);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 25;
 
   return in;
 }
 
-inline const uint32_t* unpack26_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack26_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x3ffffff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -1151,7 +1208,9 @@ inline const uint32_t* unpack26_32(const uint32_t* in, uint32_t* out) {
                               in[4], in[3] >> 8 | in[4] << 24,
                               in[2] >> 14 | in[3] << 18, in[1] >> 20 | in[2] << 12,
                               in[0] >> 26 | in[1] << 6, in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(6, 0, 0, 0,
@@ -1166,19 +1225,19 @@ inline const uint32_t* unpack26_32(const uint32_t* in, uint32_t* out) {
                               in[17], in[16] >> 8 | in[17] << 24,
                               in[15] >> 14 | in[16] << 18, in[14] >> 20 | in[15] << 12,
                               in[13] >> 26 | in[14] << 6, in[13]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 26;
 
   return in;
 }
 
-inline const uint32_t* unpack27_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack27_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x7ffffff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -1195,7 +1254,9 @@ inline const uint32_t* unpack27_32(const uint32_t* in, uint32_t* out) {
                               in[4] >> 7 | in[5] << 25, in[3] >> 12 | in[4] << 20,
                               in[2] >> 17 | in[3] << 15, in[1] >> 22 | in[2] << 10,
                               in[0] >> 27 | in[1] << 5, in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(5, 0, 0, 0,
@@ -1210,19 +1271,19 @@ inline const uint32_t* unpack27_32(const uint32_t* in, uint32_t* out) {
                               in[17] >> 23 | in[18] << 9, in[16] >> 28 | in[17] << 4,
                               in[16], in[15] >> 6 | in[16] << 26,
                               in[14] >> 11 | in[15] << 21, in[13] >> 16 | in[14] << 16);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 27;
 
   return in;
 }
 
-inline const uint32_t* unpack28_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack28_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0xfffffff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -1239,7 +1300,9 @@ inline const uint32_t* unpack28_32(const uint32_t* in, uint32_t* out) {
                               in[4] >> 12 | in[5] << 20, in[3] >> 16 | in[4] << 16,
                               in[2] >> 20 | in[3] << 12, in[1] >> 24 | in[2] << 8,
                               in[0] >> 28 | in[1] << 4, in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(4, 0, 0, 0,
@@ -1254,19 +1317,19 @@ inline const uint32_t* unpack28_32(const uint32_t* in, uint32_t* out) {
                               in[18] >> 12 | in[19] << 20, in[17] >> 16 | in[18] << 16,
                               in[16] >> 20 | in[17] << 12, in[15] >> 24 | in[16] << 8,
                               in[14] >> 28 | in[15] << 4, in[14]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 28;
 
   return in;
 }
 
-inline const uint32_t* unpack29_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack29_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x1fffffff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -1283,7 +1346,9 @@ inline const uint32_t* unpack29_32(const uint32_t* in, uint32_t* out) {
                               in[4] >> 17 | in[5] << 15, in[3] >> 20 | in[4] << 12,
                               in[2] >> 23 | in[3] << 9, in[1] >> 26 | in[2] << 6,
                               in[0] >> 29 | in[1] << 3, in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(3, 0, 0, 0,
@@ -1298,19 +1363,19 @@ inline const uint32_t* unpack29_32(const uint32_t* in, uint32_t* out) {
                               in[19], in[18] >> 4 | in[19] << 28,
                               in[17] >> 7 | in[18] << 25, in[16] >> 10 | in[17] << 22,
                               in[15] >> 13 | in[16] << 19, in[14] >> 16 | in[15] << 16);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 29;
 
   return in;
 }
 
-inline const uint32_t* unpack30_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack30_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x3fffffff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -1327,7 +1392,9 @@ inline const uint32_t* unpack30_32(const uint32_t* in, uint32_t* out) {
                               in[4] >> 22 | in[5] << 10, in[3] >> 24 | in[4] << 8,
                               in[2] >> 26 | in[3] << 6, in[1] >> 28 | in[2] << 4,
                               in[0] >> 30 | in[1] << 2, in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(2, 0, 0, 0,
@@ -1342,19 +1409,19 @@ inline const uint32_t* unpack30_32(const uint32_t* in, uint32_t* out) {
                               in[19] >> 22 | in[20] << 10, in[18] >> 24 | in[19] << 8,
                               in[17] >> 26 | in[18] << 6, in[16] >> 28 | in[17] << 4,
                               in[15] >> 30 | in[16] << 2, in[15]);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 30;
 
   return in;
 }
 
-inline const uint32_t* unpack31_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack31_32_avx512(const uint32_t* in, uint32_t* out) {
   uint32_t mask = 0x7fffffff;
   __m512i reg_shifts, reg_inls, reg_masks;
-  __m512i results[2];
+  __m512i results;
 
   reg_masks = _mm512_set1_epi32(mask);
 
@@ -1371,7 +1438,9 @@ inline const uint32_t* unpack31_32(const uint32_t* in, uint32_t* out) {
                               in[4] >> 27 | in[5] << 5, in[3] >> 28 | in[4] << 4,
                               in[2] >> 29 | in[3] << 3, in[1] >> 30 | in[2] << 2,
                               in[0] >> 31 | in[1] << 1, in[0]);
-  results[0] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
   // shift the second 16 outs
   reg_shifts = _mm512_set_epi32(1, 0, 0, 0,
@@ -1386,16 +1455,16 @@ inline const uint32_t* unpack31_32(const uint32_t* in, uint32_t* out) {
                               in[20] >> 11 | in[21] << 21, in[19] >> 12 | in[20] << 20,
                               in[18] >> 13 | in[19] << 19, in[17] >> 14 | in[18] << 18,
                               in[16] >> 15 | in[17] << 17, in[15] >> 16 | in[16] << 16);
-  results[1] = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);
+  _mm512_storeu_si512(out, results);
+  out += 16;
 
-  memcpy(out, &results, 32 * sizeof(*out));
-  out += 32;
   in += 31;
 
   return in;
 }
 
-inline const uint32_t* unpack32_32(const uint32_t* in, uint32_t* out) {
+inline static const uint32_t* unpack32_32_avx512(const uint32_t* in, uint32_t* out) {
   memcpy(out, in, 32 * sizeof(*out));
   in += 32;
   out += 32;
diff --git a/cpp/src/arrow/util/bpacking_default.h b/cpp/src/arrow/util/bpacking_default.h
index 62c87b98e39..d2516effa4d 100644
--- a/cpp/src/arrow/util/bpacking_default.h
+++ b/cpp/src/arrow/util/bpacking_default.h
@@ -26,6 +26,9 @@
 
 #pragma once
 
+#include "arrow/util/bit_util.h"
+#include "arrow/util/ubsan.h"
+
 namespace arrow {
 namespace internal {
 
diff --git a/cpp/src/arrow/util/byte_stream_split.h b/cpp/src/arrow/util/byte_stream_split.h
index bab50746064..28dcce52bb8 100644
--- a/cpp/src/arrow/util/byte_stream_split.h
+++ b/cpp/src/arrow/util/byte_stream_split.h
@@ -58,7 +58,7 @@ void ByteStreamSplitDecodeSse2(const uint8_t* data, int64_t num_values, int64_t
     out[i] = arrow::util::SafeLoadAs<T>(&gathered_byte_data[0]);
   }
 
-  // The blocks get processed hierahically using the unpack intrinsics.
+  // The blocks get processed hierarchically using the unpack intrinsics.
   // Example with four streams:
   // Stage 1: AAAA BBBB CCCC DDDD
   // Stage 2: ACAC ACAC BDBD BDBD
@@ -200,7 +200,7 @@ void ByteStreamSplitDecodeAvx2(const uint8_t* data, int64_t num_values, int64_t
     out[i] = arrow::util::SafeLoadAs<T>(&gathered_byte_data[0]);
   }
 
-  // Processed hierahically using unpack intrinsics, then permute intrinsics.
+  // Processed hierarchically using unpack intrinsics, then permute intrinsics.
   __m256i stage[kNumStreamsLog2 + 1U][kNumStreams];
   __m256i final_result[kNumStreams];
   constexpr size_t kNumStreamsHalf = kNumStreams / 2U;
@@ -292,7 +292,7 @@ void ByteStreamSplitEncodeAvx2(const uint8_t* raw_values, const size_t num_value
   }
 
   // Path for float.
-  // 1. Processed hierahically to 32i blcok using the unpack intrinsics.
+  // 1. Processed hierarchically to 32i blcok using the unpack intrinsics.
   // 2. Pack 128i block using _mm256_permutevar8x32_epi32.
   // 3. Pack final 256i block with _mm256_permute2x128_si256.
   constexpr size_t kNumUnpack = 3U;
@@ -358,7 +358,7 @@ void ByteStreamSplitDecodeAvx512(const uint8_t* data, int64_t num_values, int64_
     out[i] = arrow::util::SafeLoadAs<T>(&gathered_byte_data[0]);
   }
 
-  // Processed hierahically using the unpack, then two shuffles.
+  // Processed hierarchically using the unpack, then two shuffles.
   __m512i stage[kNumStreamsLog2 + 1U][kNumStreams];
   __m512i shuffle[kNumStreams];
   __m512i final_result[kNumStreams];
@@ -526,7 +526,7 @@ void ByteStreamSplitEncodeAvx512(const uint8_t* raw_values, const size_t num_val
       final_result[7] = _mm512_shuffle_i32x4(shuffle[6], shuffle[7], 0b11011101);
     } else {
       // Path for float.
-      // 1. Processed hierahically to 32i blcok using the unpack intrinsics.
+      // 1. Processed hierarchically to 32i blcok using the unpack intrinsics.
       // 2. Pack 128i block using _mm256_permutevar8x32_epi32.
       // 3. Pack final 256i block with _mm256_permute2x128_si256.
       for (size_t i = 0; i < kNumStreams; ++i)
diff --git a/cpp/src/arrow/util/compression.cc b/cpp/src/arrow/util/compression.cc
index c5a84a1a3ef..f9c084f6c26 100644
--- a/cpp/src/arrow/util/compression.cc
+++ b/cpp/src/arrow/util/compression.cc
@@ -24,146 +24,156 @@
 #include "arrow/result.h"
 #include "arrow/status.h"
 #include "arrow/util/compression_internal.h"
+#include "arrow/util/logging.h"
 
 namespace arrow {
 namespace util {
 
-Compressor::~Compressor() {}
-
-Decompressor::~Decompressor() {}
-
-Codec::~Codec() {}
-
 int Codec::UseDefaultCompressionLevel() { return kUseDefaultCompressionLevel; }
 
 Status Codec::Init() { return Status::OK(); }
 
-std::string Codec::GetCodecAsString(Compression::type t) {
+const std::string& Codec::GetCodecAsString(Compression::type t) {
+  static const std::string uncompressed = "uncompressed", snappy = "snappy",
+                           gzip = "gzip", lzo = "lzo", brotli = "brotli",
+                           lz4_raw = "lz4_raw", lz4 = "lz4", lz4_hadoop = "lz4_hadoop",
+                           zstd = "zstd", bz2 = "bz2", unknown = "unknown";
+
   switch (t) {
     case Compression::UNCOMPRESSED:
-      return "UNCOMPRESSED";
+      return uncompressed;
     case Compression::SNAPPY:
-      return "SNAPPY";
+      return snappy;
     case Compression::GZIP:
-      return "GZIP";
+      return gzip;
     case Compression::LZO:
-      return "LZO";
+      return lzo;
     case Compression::BROTLI:
-      return "BROTLI";
+      return brotli;
     case Compression::LZ4:
-      return "LZ4_RAW";
+      return lz4_raw;
     case Compression::LZ4_FRAME:
-      return "LZ4";
+      return lz4;
+    case Compression::LZ4_HADOOP:
+      return lz4_hadoop;
     case Compression::ZSTD:
-      return "ZSTD";
+      return zstd;
     case Compression::BZ2:
-      return "BZ2";
+      return bz2;
     default:
-      return "UNKNOWN";
+      return unknown;
   }
 }
 
 Result<Compression::type> Codec::GetCompressionType(const std::string& name) {
-  if (name == "UNCOMPRESSED") {
+  if (name == "uncompressed") {
     return Compression::UNCOMPRESSED;
-  } else if (name == "GZIP") {
+  } else if (name == "gzip") {
     return Compression::GZIP;
-  } else if (name == "SNAPPY") {
+  } else if (name == "snappy") {
     return Compression::SNAPPY;
-  } else if (name == "LZO") {
+  } else if (name == "lzo") {
     return Compression::LZO;
-  } else if (name == "BROTLI") {
+  } else if (name == "brotli") {
     return Compression::BROTLI;
-  } else if (name == "LZ4_RAW") {
+  } else if (name == "lz4_raw") {
     return Compression::LZ4;
-  } else if (name == "LZ4") {
+  } else if (name == "lz4") {
     return Compression::LZ4_FRAME;
-  } else if (name == "ZSTD") {
+  } else if (name == "lz4_hadoop") {
+    return Compression::LZ4_HADOOP;
+  } else if (name == "zstd") {
     return Compression::ZSTD;
-  } else if (name == "BZ2") {
+  } else if (name == "bz2") {
     return Compression::BZ2;
   } else {
     return Status::Invalid("Unrecognized compression type: ", name);
   }
 }
 
+bool Codec::SupportsCompressionLevel(Compression::type codec) {
+  switch (codec) {
+    case Compression::GZIP:
+    case Compression::BROTLI:
+    case Compression::ZSTD:
+    case Compression::BZ2:
+      return true;
+    default:
+      return false;
+  }
+}
+
 Result<std::unique_ptr<Codec>> Codec::Create(Compression::type codec_type,
                                              int compression_level) {
+  if (!IsAvailable(codec_type)) {
+    if (codec_type == Compression::LZO) {
+      return Status::NotImplemented("LZO codec not implemented");
+    }
+
+    auto name = GetCodecAsString(codec_type);
+    if (name == "unknown") {
+      return Status::Invalid("Unrecognized codec");
+    }
+
+    return Status::NotImplemented("Support for codec '", GetCodecAsString(codec_type),
+                                  "' not built");
+  }
+
+  if (compression_level != kUseDefaultCompressionLevel &&
+      !SupportsCompressionLevel(codec_type)) {
+    return Status::Invalid("Codec '", GetCodecAsString(codec_type),
+                           "' doesn't support setting a compression level.");
+  }
+
   std::unique_ptr<Codec> codec;
-  const bool compression_level_set{compression_level != kUseDefaultCompressionLevel};
   switch (codec_type) {
     case Compression::UNCOMPRESSED:
-      if (compression_level_set) {
-        return Status::Invalid("Compression level cannot be specified for UNCOMPRESSED.");
-      }
       return nullptr;
     case Compression::SNAPPY:
 #ifdef ARROW_WITH_SNAPPY
-      if (compression_level_set) {
-        return Status::Invalid("Snappy doesn't support setting a compression level.");
-      }
       codec = internal::MakeSnappyCodec();
-      break;
-#else
-      return Status::NotImplemented("Snappy codec support not built");
 #endif
+      break;
     case Compression::GZIP:
 #ifdef ARROW_WITH_ZLIB
       codec = internal::MakeGZipCodec(compression_level);
-      break;
-#else
-      return Status::NotImplemented("Gzip codec support not built");
 #endif
-    case Compression::LZO:
-      if (compression_level_set) {
-        return Status::Invalid("LZ0 doesn't support setting a compression level.");
-      }
-      return Status::NotImplemented("LZO codec not implemented");
+      break;
     case Compression::BROTLI:
 #ifdef ARROW_WITH_BROTLI
       codec = internal::MakeBrotliCodec(compression_level);
-      break;
-#else
-      return Status::NotImplemented("Brotli codec support not built");
 #endif
+      break;
     case Compression::LZ4:
 #ifdef ARROW_WITH_LZ4
-      if (compression_level_set) {
-        return Status::Invalid("LZ4 doesn't support setting a compression level.");
-      }
       codec = internal::MakeLz4RawCodec();
-      break;
-#else
-      return Status::NotImplemented("LZ4 codec support not built");
 #endif
+      break;
     case Compression::LZ4_FRAME:
 #ifdef ARROW_WITH_LZ4
-      if (compression_level_set) {
-        return Status::Invalid("LZ4 doesn't support setting a compression level.");
-      }
       codec = internal::MakeLz4FrameCodec();
+#endif
       break;
-#else
-      return Status::NotImplemented("LZ4 codec support not built");
+    case Compression::LZ4_HADOOP:
+#ifdef ARROW_WITH_LZ4
+      codec = internal::MakeLz4HadoopRawCodec();
 #endif
+      break;
     case Compression::ZSTD:
 #ifdef ARROW_WITH_ZSTD
       codec = internal::MakeZSTDCodec(compression_level);
-      break;
-#else
-      return Status::NotImplemented("ZSTD codec support not built");
 #endif
+      break;
     case Compression::BZ2:
 #ifdef ARROW_WITH_BZ2
       codec = internal::MakeBZ2Codec(compression_level);
-      break;
-#else
-      return Status::NotImplemented("BZ2 codec support not built");
 #endif
+      break;
     default:
-      return Status::Invalid("Unrecognized codec");
+      break;
   }
 
+  DCHECK_NE(codec, nullptr);
   RETURN_NOT_OK(codec->Init());
   return std::move(codec);
 }
@@ -194,6 +204,7 @@ bool Codec::IsAvailable(Compression::type codec_type) {
 #endif
     case Compression::LZ4:
     case Compression::LZ4_FRAME:
+    case Compression::LZ4_HADOOP:
 #ifdef ARROW_WITH_LZ4
       return true;
 #else
diff --git a/cpp/src/arrow/util/compression.h b/cpp/src/arrow/util/compression.h
index bbcb9e83d32..d3e8e1e62f1 100644
--- a/cpp/src/arrow/util/compression.h
+++ b/cpp/src/arrow/util/compression.h
@@ -30,7 +30,18 @@ namespace arrow {
 
 struct Compression {
   /// \brief Compression algorithm
-  enum type { UNCOMPRESSED, SNAPPY, GZIP, BROTLI, ZSTD, LZ4, LZ4_FRAME, LZO, BZ2 };
+  enum type {
+    UNCOMPRESSED,
+    SNAPPY,
+    GZIP,
+    BROTLI,
+    ZSTD,
+    LZ4,
+    LZ4_FRAME,
+    LZO,
+    BZ2,
+    LZ4_HADOOP
+  };
 
   static constexpr int kUseDefaultCompressionLevel = std::numeric_limits<int>::min();
 };
@@ -43,7 +54,7 @@ constexpr int kUseDefaultCompressionLevel = Compression::kUseDefaultCompressionL
 ///
 class ARROW_EXPORT Compressor {
  public:
-  virtual ~Compressor();
+  virtual ~Compressor() = default;
 
   struct CompressResult {
     int64_t bytes_read;
@@ -85,7 +96,7 @@ class ARROW_EXPORT Compressor {
 ///
 class ARROW_EXPORT Decompressor {
  public:
-  virtual ~Decompressor();
+  virtual ~Decompressor() = default;
 
   struct DecompressResult {
     // XXX is need_more_output necessary? (Brotli?)
@@ -117,14 +128,14 @@ class ARROW_EXPORT Decompressor {
 /// \brief Compression codec
 class ARROW_EXPORT Codec {
  public:
-  virtual ~Codec();
+  virtual ~Codec() = default;
 
   /// \brief Return special value to indicate that a codec implementation
   /// should use its default compression level
   static int UseDefaultCompressionLevel();
 
   /// \brief Return a string name for compression type
-  static std::string GetCodecAsString(Compression::type t);
+  static const std::string& GetCodecAsString(Compression::type t);
 
   /// \brief Return compression type for name (all upper case)
   static Result<Compression::type> GetCompressionType(const std::string& name);
@@ -136,6 +147,9 @@ class ARROW_EXPORT Codec {
   /// \brief Return true if support for indicated codec has been enabled
   static bool IsAvailable(Compression::type codec);
 
+  /// \brief Return true if indicated codec supports setting a compression level
+  static bool SupportsCompressionLevel(Compression::type codec);
+
   /// \brief One-shot decompression function
   ///
   /// output_buffer_len must be correct and therefore be obtained in advance.
@@ -167,7 +181,14 @@ class ARROW_EXPORT Codec {
   /// \brief Create a streaming compressor instance
   virtual Result<std::shared_ptr<Decompressor>> MakeDecompressor() = 0;
 
-  virtual const char* name() const = 0;
+  /// \brief This Codec's compression type
+  virtual Compression::type compression_type() const = 0;
+
+  /// \brief The name of this Codec's compression type
+  const std::string& name() const { return GetCodecAsString(compression_type()); }
+
+  /// \brief This Codec's compression level, if applicable
+  virtual int compression_level() const { return UseDefaultCompressionLevel(); }
 
  private:
   /// \brief Initializes the codec's resources.
diff --git a/cpp/src/arrow/util/compression_brotli.cc b/cpp/src/arrow/util/compression_brotli.cc
index ca4f523a657..4feabe23345 100644
--- a/cpp/src/arrow/util/compression_brotli.cc
+++ b/cpp/src/arrow/util/compression_brotli.cc
@@ -38,8 +38,6 @@ namespace {
 
 class BrotliDecompressor : public Decompressor {
  public:
-  BrotliDecompressor() {}
-
   ~BrotliDecompressor() override {
     if (state_ != nullptr) {
       BrotliDecoderDestroyInstance(state_);
@@ -167,7 +165,7 @@ class BrotliCompressor : public Compressor {
   BrotliEncoderState* state_ = nullptr;
 
  private:
-  int compression_level_;
+  const int compression_level_;
 };
 
 // ----------------------------------------------------------------------
@@ -175,11 +173,10 @@ class BrotliCompressor : public Compressor {
 
 class BrotliCodec : public Codec {
  public:
-  explicit BrotliCodec(int compression_level) {
-    compression_level_ = compression_level == kUseDefaultCompressionLevel
-                             ? kBrotliDefaultCompressionLevel
-                             : compression_level;
-  }
+  explicit BrotliCodec(int compression_level)
+      : compression_level_(compression_level == kUseDefaultCompressionLevel
+                               ? kBrotliDefaultCompressionLevel
+                               : compression_level) {}
 
   Result<int64_t> Decompress(int64_t input_len, const uint8_t* input,
                              int64_t output_buffer_len, uint8_t* output_buffer) override {
@@ -224,10 +221,12 @@ class BrotliCodec : public Codec {
     return ptr;
   }
 
-  const char* name() const override { return "brotli"; }
+  Compression::type compression_type() const override { return Compression::BROTLI; }
+
+  int compression_level() const override { return compression_level_; }
 
  private:
-  int compression_level_;
+  const int compression_level_;
 };
 
 }  // namespace
diff --git a/cpp/src/arrow/util/compression_bz2.cc b/cpp/src/arrow/util/compression_bz2.cc
index dfb266ab870..8a8c1cb7a45 100644
--- a/cpp/src/arrow/util/compression_bz2.cc
+++ b/cpp/src/arrow/util/compression_bz2.cc
@@ -262,7 +262,9 @@ class BZ2Codec : public Codec {
     return ptr;
   }
 
-  const char* name() const override { return "bz2"; }
+  Compression::type compression_type() const override { return Compression::BZ2; }
+
+  int compression_level() const override { return compression_level_; }
 
  private:
   int compression_level_;
diff --git a/cpp/src/arrow/util/compression_internal.h b/cpp/src/arrow/util/compression_internal.h
index a2f7f7b6c23..268672e14e2 100644
--- a/cpp/src/arrow/util/compression_internal.h
+++ b/cpp/src/arrow/util/compression_internal.h
@@ -61,6 +61,9 @@ std::unique_ptr<Codec> MakeSnappyCodec();
 // Lz4 "raw" format codec.
 std::unique_ptr<Codec> MakeLz4RawCodec();
 
+// Lz4 "Hadoop" format codec (== Lz4 raw codec prefixed with lengths header)
+std::unique_ptr<Codec> MakeLz4HadoopRawCodec();
+
 // Lz4 frame format codec.
 std::unique_ptr<Codec> MakeLz4FrameCodec();
 
diff --git a/cpp/src/arrow/util/compression_lz4.cc b/cpp/src/arrow/util/compression_lz4.cc
index 27446a983eb..365cd0f523e 100644
--- a/cpp/src/arrow/util/compression_lz4.cc
+++ b/cpp/src/arrow/util/compression_lz4.cc
@@ -26,8 +26,10 @@
 
 #include "arrow/result.h"
 #include "arrow/status.h"
+#include "arrow/util/bit_util.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
+#include "arrow/util/ubsan.h"
 
 #ifndef LZ4F_HEADER_SIZE_MAX
 #define LZ4F_HEADER_SIZE_MAX 19
@@ -296,10 +298,10 @@ class Lz4FrameCodec : public Codec {
     return ptr;
   }
 
-  const char* name() const override { return "lz4"; }
+  Compression::type compression_type() const override { return Compression::LZ4_FRAME; }
 
  protected:
-  LZ4F_preferences_t prefs_;
+  const LZ4F_preferences_t prefs_;
 };
 
 // ----------------------------------------------------------------------
@@ -346,7 +348,108 @@ class Lz4Codec : public Codec {
         "Try using LZ4 frame format instead.");
   }
 
-  const char* name() const override { return "lz4_raw"; }
+  Compression::type compression_type() const override { return Compression::LZ4; }
+};
+
+// ----------------------------------------------------------------------
+// Lz4 Hadoop "raw" codec implementation
+
+class Lz4HadoopCodec : public Lz4Codec {
+ public:
+  Result<int64_t> Decompress(int64_t input_len, const uint8_t* input,
+                             int64_t output_buffer_len, uint8_t* output_buffer) override {
+    const int64_t decompressed_size =
+        TryDecompressHadoop(input_len, input, output_buffer_len, output_buffer);
+    if (decompressed_size != kNotHadoop) {
+      return decompressed_size;
+    }
+    // Fall back on raw LZ4 codec (for files produces by earlier versions of Parquet C++)
+    return Lz4Codec::Decompress(input_len, input, output_buffer_len, output_buffer);
+  }
+
+  int64_t MaxCompressedLen(int64_t input_len,
+                           const uint8_t* ARROW_ARG_UNUSED(input)) override {
+    return kPrefixLength + Lz4Codec::MaxCompressedLen(input_len, nullptr);
+  }
+
+  Result<int64_t> Compress(int64_t input_len, const uint8_t* input,
+                           int64_t output_buffer_len, uint8_t* output_buffer) override {
+    if (output_buffer_len < kPrefixLength) {
+      return Status::Invalid("Output buffer too small for Lz4HadoopCodec compression");
+    }
+
+    ARROW_ASSIGN_OR_RAISE(
+        int64_t output_len,
+        Lz4Codec::Compress(input_len, input, output_buffer_len - kPrefixLength,
+                           output_buffer + kPrefixLength));
+
+    // Prepend decompressed size in bytes and compressed size in bytes
+    // to be compatible with Hadoop Lz4Codec
+    const uint32_t decompressed_size =
+        BitUtil::ToBigEndian(static_cast<uint32_t>(input_len));
+    const uint32_t compressed_size =
+        BitUtil::ToBigEndian(static_cast<uint32_t>(output_len));
+    SafeStore(output_buffer, decompressed_size);
+    SafeStore(output_buffer + sizeof(uint32_t), compressed_size);
+
+    return kPrefixLength + output_len;
+  }
+
+  Result<std::shared_ptr<Compressor>> MakeCompressor() override {
+    return Status::NotImplemented(
+        "Streaming compression unsupported with LZ4 Hadoop raw format. "
+        "Try using LZ4 frame format instead.");
+  }
+
+  Result<std::shared_ptr<Decompressor>> MakeDecompressor() override {
+    return Status::NotImplemented(
+        "Streaming decompression unsupported with LZ4 Hadoop raw format. "
+        "Try using LZ4 frame format instead.");
+  }
+
+  Compression::type compression_type() const override { return Compression::LZ4_HADOOP; }
+
+ protected:
+  // Offset starting at which page data can be read/written
+  static const int64_t kPrefixLength = sizeof(uint32_t) * 2;
+
+  static const int64_t kNotHadoop = -1;
+
+  int64_t TryDecompressHadoop(int64_t input_len, const uint8_t* input,
+                              int64_t output_buffer_len, uint8_t* output_buffer) {
+    // Parquet files written with the Hadoop Lz4Codec contain at the beginning
+    // of the input buffer two uint32_t's representing (in this order) expected
+    // decompressed size in bytes and expected compressed size in bytes.
+    //
+    // The Hadoop Lz4Codec source code can be found here:
+    // https://github.com/apache/hadoop/blob/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc
+    if (input_len < kPrefixLength) {
+      return kNotHadoop;
+    }
+
+    const uint32_t expected_decompressed_size =
+        BitUtil::FromBigEndian(SafeLoadAs<uint32_t>(input));
+    const uint32_t expected_compressed_size =
+        BitUtil::FromBigEndian(SafeLoadAs<uint32_t>(input + sizeof(uint32_t)));
+    const int64_t lz4_compressed_buffer_size = input_len - kPrefixLength;
+
+    // We use a heuristic to determine if the parquet file being read
+    // was compressed using the Hadoop Lz4Codec.
+    if (lz4_compressed_buffer_size == expected_compressed_size) {
+      // Parquet file was likely compressed with Hadoop Lz4Codec.
+      auto maybe_decompressed_size =
+          Lz4Codec::Decompress(lz4_compressed_buffer_size, input + kPrefixLength,
+                               output_buffer_len, output_buffer);
+
+      if (maybe_decompressed_size.ok() &&
+          *maybe_decompressed_size == expected_decompressed_size) {
+        return *maybe_decompressed_size;
+      }
+    }
+
+    // Parquet file was compressed without Hadoop Lz4Codec (or data is corrupt)
+    return kNotHadoop;
+  }
 };
 
 }  // namespace
@@ -357,6 +460,10 @@ std::unique_ptr<Codec> MakeLz4FrameCodec() {
   return std::unique_ptr<Codec>(new Lz4FrameCodec());
 }
 
+std::unique_ptr<Codec> MakeLz4HadoopRawCodec() {
+  return std::unique_ptr<Codec>(new Lz4HadoopCodec());
+}
+
 std::unique_ptr<Codec> MakeLz4RawCodec() {
   return std::unique_ptr<Codec>(new Lz4Codec());
 }
diff --git a/cpp/src/arrow/util/compression_snappy.cc b/cpp/src/arrow/util/compression_snappy.cc
index 9cd06dc4f02..9b016874b56 100644
--- a/cpp/src/arrow/util/compression_snappy.cc
+++ b/cpp/src/arrow/util/compression_snappy.cc
@@ -41,8 +41,6 @@ namespace {
 
 class SnappyCodec : public Codec {
  public:
-  SnappyCodec() {}
-
   Result<int64_t> Decompress(int64_t input_len, const uint8_t* input,
                              int64_t output_buffer_len, uint8_t* output_buffer) override {
     size_t decompressed_size;
@@ -87,7 +85,7 @@ class SnappyCodec : public Codec {
     return Status::NotImplemented("Streaming decompression unsupported with Snappy");
   }
 
-  const char* name() const override { return "snappy"; }
+  Compression::type compression_type() const override { return Compression::SNAPPY; }
 };
 
 }  // namespace
diff --git a/cpp/src/arrow/util/compression_test.cc b/cpp/src/arrow/util/compression_test.cc
index 74c040a7964..8b184f2dc41 100644
--- a/cpp/src/arrow/util/compression_test.cc
+++ b/cpp/src/arrow/util/compression_test.cc
@@ -55,7 +55,7 @@ std::vector<uint8_t> MakeCompressibleData(int data_size) {
 
 // Check roundtrip of one-shot compression and decompression functions.
 void CheckCodecRoundtrip(std::unique_ptr<Codec>& c1, std::unique_ptr<Codec>& c2,
-                         const std::vector<uint8_t>& data) {
+                         const std::vector<uint8_t>& data, bool check_reverse = true) {
   int max_compressed_len =
       static_cast<int>(c1->MaxCompressedLen(data.size(), data.data()));
   std::vector<uint8_t> compressed(max_compressed_len);
@@ -76,26 +76,29 @@ void CheckCodecRoundtrip(std::unique_ptr<Codec>& c1, std::unique_ptr<Codec>& c2,
   ASSERT_EQ(data, decompressed);
   ASSERT_EQ(data.size(), actual_decompressed_size);
 
-  // compress with c2
-  ASSERT_EQ(max_compressed_len,
-            static_cast<int>(c2->MaxCompressedLen(data.size(), data.data())));
-  // Resize to prevent ASAN from detecting container overflow.
-  compressed.resize(max_compressed_len);
-
-  int64_t actual_size2;
-  ASSERT_OK_AND_ASSIGN(actual_size2, c2->Compress(data.size(), data.data(),
-                                                  max_compressed_len, compressed.data()));
-  ASSERT_EQ(actual_size2, actual_size);
-  compressed.resize(actual_size2);
-
-  // decompress with c1
-  int64_t actual_decompressed_size2;
-  ASSERT_OK_AND_ASSIGN(actual_decompressed_size2,
-                       c1->Decompress(compressed.size(), compressed.data(),
-                                      decompressed.size(), decompressed.data()));
-
-  ASSERT_EQ(data, decompressed);
-  ASSERT_EQ(data.size(), actual_decompressed_size2);
+  if (check_reverse) {
+    // compress with c2
+    ASSERT_EQ(max_compressed_len,
+              static_cast<int>(c2->MaxCompressedLen(data.size(), data.data())));
+    // Resize to prevent ASAN from detecting container overflow.
+    compressed.resize(max_compressed_len);
+
+    int64_t actual_size2;
+    ASSERT_OK_AND_ASSIGN(
+        actual_size2,
+        c2->Compress(data.size(), data.data(), max_compressed_len, compressed.data()));
+    ASSERT_EQ(actual_size2, actual_size);
+    compressed.resize(actual_size2);
+
+    // decompress with c1
+    int64_t actual_decompressed_size2;
+    ASSERT_OK_AND_ASSIGN(actual_decompressed_size2,
+                         c1->Decompress(compressed.size(), compressed.data(),
+                                        decompressed.size(), decompressed.data()));
+
+    ASSERT_EQ(data, decompressed);
+    ASSERT_EQ(data.size(), actual_decompressed_size2);
+  }
 }
 
 // Check the streaming compressor against one-shot decompression
@@ -317,30 +320,30 @@ class CodecTest : public ::testing::TestWithParam<Compression::type> {
 };
 
 TEST(TestCodecMisc, GetCodecAsString) {
-  ASSERT_EQ("UNCOMPRESSED", Codec::GetCodecAsString(Compression::UNCOMPRESSED));
-  ASSERT_EQ("SNAPPY", Codec::GetCodecAsString(Compression::SNAPPY));
-  ASSERT_EQ("GZIP", Codec::GetCodecAsString(Compression::GZIP));
-  ASSERT_EQ("LZO", Codec::GetCodecAsString(Compression::LZO));
-  ASSERT_EQ("BROTLI", Codec::GetCodecAsString(Compression::BROTLI));
-  ASSERT_EQ("LZ4_RAW", Codec::GetCodecAsString(Compression::LZ4));
-  ASSERT_EQ("LZ4", Codec::GetCodecAsString(Compression::LZ4_FRAME));
-  ASSERT_EQ("ZSTD", Codec::GetCodecAsString(Compression::ZSTD));
-  ASSERT_EQ("BZ2", Codec::GetCodecAsString(Compression::BZ2));
+  EXPECT_EQ(Codec::GetCodecAsString(Compression::UNCOMPRESSED), "uncompressed");
+  EXPECT_EQ(Codec::GetCodecAsString(Compression::SNAPPY), "snappy");
+  EXPECT_EQ(Codec::GetCodecAsString(Compression::GZIP), "gzip");
+  EXPECT_EQ(Codec::GetCodecAsString(Compression::LZO), "lzo");
+  EXPECT_EQ(Codec::GetCodecAsString(Compression::BROTLI), "brotli");
+  EXPECT_EQ(Codec::GetCodecAsString(Compression::LZ4), "lz4_raw");
+  EXPECT_EQ(Codec::GetCodecAsString(Compression::LZ4_FRAME), "lz4");
+  EXPECT_EQ(Codec::GetCodecAsString(Compression::ZSTD), "zstd");
+  EXPECT_EQ(Codec::GetCodecAsString(Compression::BZ2), "bz2");
 }
 
 TEST(TestCodecMisc, GetCompressionType) {
-  ASSERT_OK_AND_EQ(Compression::UNCOMPRESSED, Codec::GetCompressionType("UNCOMPRESSED"));
-  ASSERT_OK_AND_EQ(Compression::SNAPPY, Codec::GetCompressionType("SNAPPY"));
-  ASSERT_OK_AND_EQ(Compression::GZIP, Codec::GetCompressionType("GZIP"));
-  ASSERT_OK_AND_EQ(Compression::LZO, Codec::GetCompressionType("LZO"));
-  ASSERT_OK_AND_EQ(Compression::BROTLI, Codec::GetCompressionType("BROTLI"));
-  ASSERT_OK_AND_EQ(Compression::LZ4, Codec::GetCompressionType("LZ4_RAW"));
-  ASSERT_OK_AND_EQ(Compression::LZ4_FRAME, Codec::GetCompressionType("LZ4"));
-  ASSERT_OK_AND_EQ(Compression::ZSTD, Codec::GetCompressionType("ZSTD"));
-  ASSERT_OK_AND_EQ(Compression::BZ2, Codec::GetCompressionType("BZ2"));
+  ASSERT_OK_AND_EQ(Compression::UNCOMPRESSED, Codec::GetCompressionType("uncompressed"));
+  ASSERT_OK_AND_EQ(Compression::SNAPPY, Codec::GetCompressionType("snappy"));
+  ASSERT_OK_AND_EQ(Compression::GZIP, Codec::GetCompressionType("gzip"));
+  ASSERT_OK_AND_EQ(Compression::LZO, Codec::GetCompressionType("lzo"));
+  ASSERT_OK_AND_EQ(Compression::BROTLI, Codec::GetCompressionType("brotli"));
+  ASSERT_OK_AND_EQ(Compression::LZ4, Codec::GetCompressionType("lz4_raw"));
+  ASSERT_OK_AND_EQ(Compression::LZ4_FRAME, Codec::GetCompressionType("lz4"));
+  ASSERT_OK_AND_EQ(Compression::ZSTD, Codec::GetCompressionType("zstd"));
+  ASSERT_OK_AND_EQ(Compression::BZ2, Codec::GetCompressionType("bz2"));
 
   ASSERT_RAISES(Invalid, Codec::GetCompressionType("unk"));
-  ASSERT_RAISES(Invalid, Codec::GetCompressionType("snappy"));
+  ASSERT_RAISES(Invalid, Codec::GetCompressionType("SNAPPY"));
 }
 
 TEST_P(CodecTest, CodecRoundtrip) {
@@ -433,7 +436,8 @@ TEST_P(CodecTest, StreamingCompressor) {
     // SKIP: BZ2 doesn't support one-shot decompression
     return;
   }
-  if (GetCompression() == Compression::LZ4) {
+  if (GetCompression() == Compression::LZ4 ||
+      GetCompression() == Compression::LZ4_HADOOP) {
     // SKIP: LZ4 raw format doesn't support streaming compression.
     return;
   }
@@ -459,7 +463,8 @@ TEST_P(CodecTest, StreamingDecompressor) {
     // SKIP: BZ2 doesn't support one-shot compression
     return;
   }
-  if (GetCompression() == Compression::LZ4) {
+  if (GetCompression() == Compression::LZ4 ||
+      GetCompression() == Compression::LZ4_HADOOP) {
     // SKIP: LZ4 raw format doesn't support streaming decompression.
     return;
   }
@@ -481,7 +486,8 @@ TEST_P(CodecTest, StreamingRoundtrip) {
     // SKIP: snappy doesn't support streaming decompression
     return;
   }
-  if (GetCompression() == Compression::LZ4) {
+  if (GetCompression() == Compression::LZ4 ||
+      GetCompression() == Compression::LZ4_HADOOP) {
     // SKIP: LZ4 raw format doesn't support streaming compression.
     return;
   }
@@ -503,7 +509,8 @@ TEST_P(CodecTest, StreamingDecompressorReuse) {
     // SKIP: snappy doesn't support streaming decompression
     return;
   }
-  if (GetCompression() == Compression::LZ4) {
+  if (GetCompression() == Compression::LZ4 ||
+      GetCompression() == Compression::LZ4_HADOOP) {
     // SKIP: LZ4 raw format doesn't support streaming decompression.
     return;
   }
@@ -533,6 +540,8 @@ INSTANTIATE_TEST_SUITE_P(TestSnappy, CodecTest, ::testing::Values(Compression::S
 
 #ifdef ARROW_WITH_LZ4
 INSTANTIATE_TEST_SUITE_P(TestLZ4, CodecTest, ::testing::Values(Compression::LZ4));
+INSTANTIATE_TEST_SUITE_P(TestLZ4Hadoop, CodecTest,
+                         ::testing::Values(Compression::LZ4_HADOOP));
 #endif
 
 #ifdef ARROW_WITH_LZ4
@@ -552,5 +561,16 @@ INSTANTIATE_TEST_SUITE_P(TestBZ2, CodecTest, ::testing::Values(Compression::BZ2)
 INSTANTIATE_TEST_SUITE_P(TestZSTD, CodecTest, ::testing::Values(Compression::ZSTD));
 #endif
 
+#ifdef ARROW_WITH_LZ4
+TEST(TestCodecLZ4Hadoop, Compatibility) {
+  // LZ4 Hadoop codec should be able to read back LZ4 raw blocks
+  ASSERT_OK_AND_ASSIGN(auto c1, Codec::Create(Compression::LZ4));
+  ASSERT_OK_AND_ASSIGN(auto c2, Codec::Create(Compression::LZ4_HADOOP));
+
+  std::vector<uint8_t> data = MakeRandomData(100);
+  CheckCodecRoundtrip(c1, c2, data, /*check_reverse=*/false);
+}
+#endif
+
 }  // namespace util
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/compression_zlib.cc b/cpp/src/arrow/util/compression_zlib.cc
index 717bbc61d46..84e517e2632 100644
--- a/cpp/src/arrow/util/compression_zlib.cc
+++ b/cpp/src/arrow/util/compression_zlib.cc
@@ -463,7 +463,9 @@ class GZipCodec : public Codec {
     return InitDecompressor();
   }
 
-  const char* name() const override { return "gzip"; }
+  Compression::type compression_type() const override { return Compression::GZIP; }
+
+  int compression_level() const override { return compression_level_; }
 
  private:
   // zlib is stateful and the z_stream state variable must be initialized
diff --git a/cpp/src/arrow/util/compression_zstd.cc b/cpp/src/arrow/util/compression_zstd.cc
index e7409604a4c..382e0573b29 100644
--- a/cpp/src/arrow/util/compression_zstd.cc
+++ b/cpp/src/arrow/util/compression_zstd.cc
@@ -173,20 +173,19 @@ class ZSTDCompressor : public Compressor {
 
 class ZSTDCodec : public Codec {
  public:
-  explicit ZSTDCodec(int compression_level) {
-    compression_level_ = compression_level == kUseDefaultCompressionLevel
-                             ? kZSTDDefaultCompressionLevel
-                             : compression_level;
-  }
+  explicit ZSTDCodec(int compression_level)
+      : compression_level_(compression_level == kUseDefaultCompressionLevel
+                               ? kZSTDDefaultCompressionLevel
+                               : compression_level) {}
 
   Result<int64_t> Decompress(int64_t input_len, const uint8_t* input,
                              int64_t output_buffer_len, uint8_t* output_buffer) override {
     if (output_buffer == nullptr) {
       // We may pass a NULL 0-byte output buffer but some zstd versions demand
       // a valid pointer: https://github.com/facebook/zstd/issues/1385
-      static uint8_t empty_buffer[1];
+      static uint8_t empty_buffer;
       DCHECK_EQ(output_buffer_len, 0);
-      output_buffer = empty_buffer;
+      output_buffer = &empty_buffer;
     }
 
     size_t ret = ZSTD_decompress(output_buffer, static_cast<size_t>(output_buffer_len),
@@ -228,10 +227,12 @@ class ZSTDCodec : public Codec {
     return ptr;
   }
 
-  const char* name() const override { return "zstd"; }
+  Compression::type compression_type() const override { return Compression::ZSTD; }
+
+  int compression_level() const override { return compression_level_; }
 
  private:
-  int compression_level_;
+  const int compression_level_;
 };
 
 }  // namespace
diff --git a/cpp/src/arrow/util/config.h.cmake b/cpp/src/arrow/util/config.h.cmake
index 8fb255b5fae..4b3025c2e17 100644
--- a/cpp/src/arrow/util/config.h.cmake
+++ b/cpp/src/arrow/util/config.h.cmake
@@ -34,4 +34,6 @@
 
 #define ARROW_PACKAGE_KIND "@ARROW_PACKAGE_KIND@"
 
+#cmakedefine ARROW_S3
+
 #cmakedefine GRPCPP_PP_INCLUDE
diff --git a/cpp/src/arrow/util/converter.h b/cpp/src/arrow/util/converter.h
new file mode 100644
index 00000000000..8a9cd2b415a
--- /dev/null
+++ b/cpp/src/arrow/util/converter.h
@@ -0,0 +1,323 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/chunked_array.h"
+#include "arrow/status.h"
+#include "arrow/type.h"
+#include "arrow/type_traits.h"
+#include "arrow/util/checked_cast.h"
+#include "arrow/util/make_unique.h"
+#include "arrow/visitor_inline.h"
+
+namespace arrow {
+namespace internal {
+
+template <typename BaseConverter, template <typename...> class ConverterTrait>
+static Result<std::unique_ptr<BaseConverter>> MakeConverter(
+    std::shared_ptr<DataType> type, typename BaseConverter::OptionsType options,
+    MemoryPool* pool);
+
+template <typename Input, typename Options>
+class Converter {
+ public:
+  using Self = Converter<Input, Options>;
+  using InputType = Input;
+  using OptionsType = Options;
+
+  virtual ~Converter() = default;
+
+  Status Construct(std::shared_ptr<DataType> type, OptionsType options,
+                   MemoryPool* pool) {
+    type_ = std::move(type);
+    options_ = std::move(options);
+    return Init(pool);
+  }
+
+  virtual Status Append(InputType value) = 0;
+
+  const std::shared_ptr<ArrayBuilder>& builder() const { return builder_; }
+
+  const std::shared_ptr<DataType>& type() const { return type_; }
+
+  OptionsType options() const { return options_; }
+
+  bool may_overflow() const { return may_overflow_; }
+
+  virtual Status Reserve(int64_t additional_capacity) {
+    return builder_->Reserve(additional_capacity);
+  }
+
+  Status AppendNull() { return builder_->AppendNull(); }
+
+  virtual Result<std::shared_ptr<Array>> ToArray() { return builder_->Finish(); }
+
+  virtual Result<std::shared_ptr<Array>> ToArray(int64_t length) {
+    ARROW_ASSIGN_OR_RAISE(auto arr, this->ToArray());
+    return arr->Slice(0, length);
+  }
+
+  virtual Result<std::shared_ptr<ChunkedArray>> ToChunkedArray() {
+    ARROW_ASSIGN_OR_RAISE(auto array, ToArray());
+    std::vector<std::shared_ptr<Array>> chunks = {std::move(array)};
+    return std::make_shared<ChunkedArray>(chunks);
+  }
+
+ protected:
+  virtual Status Init(MemoryPool* pool) { return Status::OK(); }
+
+  std::shared_ptr<DataType> type_;
+  std::shared_ptr<ArrayBuilder> builder_;
+  OptionsType options_;
+  bool may_overflow_ = false;
+};
+
+template <typename ArrowType, typename BaseConverter>
+class PrimitiveConverter : public BaseConverter {
+ public:
+  using BuilderType = typename TypeTraits<ArrowType>::BuilderType;
+
+ protected:
+  Status Init(MemoryPool* pool) override {
+    this->builder_ = std::make_shared<BuilderType>(this->type_, pool);
+    this->may_overflow_ =
+        is_base_binary_like(this->type_->id()) || is_fixed_size_binary(this->type_->id());
+    primitive_type_ = checked_cast<const ArrowType*>(this->type_.get());
+    primitive_builder_ = checked_cast<BuilderType*>(this->builder_.get());
+    return Status::OK();
+  }
+
+  const ArrowType* primitive_type_;
+  BuilderType* primitive_builder_;
+};
+
+template <typename ArrowType, typename BaseConverter,
+          template <typename...> class ConverterTrait>
+class ListConverter : public BaseConverter {
+ public:
+  using BuilderType = typename TypeTraits<ArrowType>::BuilderType;
+  using ConverterType = typename ConverterTrait<ArrowType>::type;
+
+ protected:
+  Status Init(MemoryPool* pool) override {
+    list_type_ = checked_cast<const ArrowType*>(this->type_.get());
+    ARROW_ASSIGN_OR_RAISE(value_converter_,
+                          (MakeConverter<BaseConverter, ConverterTrait>(
+                              list_type_->value_type(), this->options_, pool)));
+    this->builder_ =
+        std::make_shared<BuilderType>(pool, value_converter_->builder(), this->type_);
+    list_builder_ = checked_cast<BuilderType*>(this->builder_.get());
+    this->may_overflow_ = true;
+    return Status::OK();
+  }
+
+  const ArrowType* list_type_;
+  BuilderType* list_builder_;
+  std::unique_ptr<BaseConverter> value_converter_;
+};
+
+template <typename BaseConverter, template <typename...> class ConverterTrait>
+class StructConverter : public BaseConverter {
+ public:
+  using ConverterType = typename ConverterTrait<StructType>::type;
+
+  Status Reserve(int64_t additional_capacity) override {
+    ARROW_RETURN_NOT_OK(this->builder_->Reserve(additional_capacity));
+    for (const auto& child : children_) {
+      ARROW_RETURN_NOT_OK(child->Reserve(additional_capacity));
+    }
+    return Status::OK();
+  }
+
+ protected:
+  Status Init(MemoryPool* pool) override {
+    std::unique_ptr<BaseConverter> child_converter;
+    std::vector<std::shared_ptr<ArrayBuilder>> child_builders;
+
+    struct_type_ = checked_cast<const StructType*>(this->type_.get());
+    for (const auto& field : struct_type_->fields()) {
+      ARROW_ASSIGN_OR_RAISE(child_converter,
+                            (MakeConverter<BaseConverter, ConverterTrait>(
+                                field->type(), this->options_, pool)));
+      this->may_overflow_ |= child_converter->may_overflow();
+      child_builders.push_back(child_converter->builder());
+      children_.push_back(std::move(child_converter));
+    }
+
+    this->builder_ =
+        std::make_shared<StructBuilder>(this->type_, pool, std::move(child_builders));
+    struct_builder_ = checked_cast<StructBuilder*>(this->builder_.get());
+
+    return Status::OK();
+  }
+
+  const StructType* struct_type_;
+  StructBuilder* struct_builder_;
+  std::vector<std::unique_ptr<BaseConverter>> children_;
+};
+
+template <typename ValueType, typename BaseConverter>
+class DictionaryConverter : public BaseConverter {
+ public:
+  using BuilderType = DictionaryBuilder<ValueType>;
+
+ protected:
+  Status Init(MemoryPool* pool) override {
+    std::unique_ptr<ArrayBuilder> builder;
+    ARROW_RETURN_NOT_OK(MakeDictionaryBuilder(pool, this->type_, NULLPTR, &builder));
+    this->builder_ = std::move(builder);
+    this->may_overflow_ = false;
+    dict_type_ = checked_cast<const DictionaryType*>(this->type_.get());
+    value_type_ = checked_cast<const ValueType*>(dict_type_->value_type().get());
+    value_builder_ = checked_cast<BuilderType*>(this->builder_.get());
+    return Status::OK();
+  }
+
+  const DictionaryType* dict_type_;
+  const ValueType* value_type_;
+  BuilderType* value_builder_;
+};
+
+template <typename BaseConverter, template <typename...> class ConverterTrait>
+struct MakeConverterImpl {
+  template <typename T, typename ConverterType = typename ConverterTrait<T>::type>
+  Status Visit(const T&) {
+    out.reset(new ConverterType());
+    return out->Construct(std::move(type), std::move(options), pool);
+  }
+
+  Status Visit(const DictionaryType& t) {
+    switch (t.value_type()->id()) {
+#define DICTIONARY_CASE(TYPE)                                                       \
+  case TYPE::type_id:                                                               \
+    out = internal::make_unique<                                                    \
+        typename ConverterTrait<DictionaryType>::template dictionary_type<TYPE>>(); \
+    break;
+      DICTIONARY_CASE(BooleanType);
+      DICTIONARY_CASE(Int8Type);
+      DICTIONARY_CASE(Int16Type);
+      DICTIONARY_CASE(Int32Type);
+      DICTIONARY_CASE(Int64Type);
+      DICTIONARY_CASE(UInt8Type);
+      DICTIONARY_CASE(UInt16Type);
+      DICTIONARY_CASE(UInt32Type);
+      DICTIONARY_CASE(UInt64Type);
+      DICTIONARY_CASE(FloatType);
+      DICTIONARY_CASE(DoubleType);
+      DICTIONARY_CASE(BinaryType);
+      DICTIONARY_CASE(StringType);
+      DICTIONARY_CASE(FixedSizeBinaryType);
+#undef DICTIONARY_CASE
+      default:
+        return Status::NotImplemented("DictionaryArray converter for type ", t.ToString(),
+                                      " not implemented");
+    }
+    return out->Construct(std::move(type), std::move(options), pool);
+  }
+
+  Status Visit(const DataType& t) { return Status::NotImplemented(t.name()); }
+
+  std::shared_ptr<DataType> type;
+  typename BaseConverter::OptionsType options;
+  MemoryPool* pool;
+  std::unique_ptr<BaseConverter> out;
+};
+
+template <typename BaseConverter, template <typename...> class ConverterTrait>
+static Result<std::unique_ptr<BaseConverter>> MakeConverter(
+    std::shared_ptr<DataType> type, typename BaseConverter::OptionsType options,
+    MemoryPool* pool) {
+  MakeConverterImpl<BaseConverter, ConverterTrait> visitor{
+      std::move(type), std::move(options), pool, NULLPTR};
+  ARROW_RETURN_NOT_OK(VisitTypeInline(*visitor.type, &visitor));
+  return std::move(visitor.out);
+}
+
+template <typename Converter>
+class Chunker {
+ public:
+  using InputType = typename Converter::InputType;
+
+  explicit Chunker(std::unique_ptr<Converter> converter)
+      : converter_(std::move(converter)) {}
+
+  Status Reserve(int64_t additional_capacity) {
+    ARROW_RETURN_NOT_OK(converter_->Reserve(additional_capacity));
+    reserved_ += additional_capacity;
+    return Status::OK();
+  }
+
+  Status AppendNull() {
+    auto status = converter_->AppendNull();
+    if (ARROW_PREDICT_FALSE(status.IsCapacityError())) {
+      ARROW_RETURN_NOT_OK(FinishChunk());
+      return converter_->AppendNull();
+    }
+    ++length_;
+    return status;
+  }
+
+  Status Append(InputType value) {
+    auto status = converter_->Append(value);
+    if (ARROW_PREDICT_FALSE(status.IsCapacityError())) {
+      ARROW_RETURN_NOT_OK(FinishChunk());
+      return Append(value);
+    }
+    ++length_;
+    return status;
+  }
+
+  Status FinishChunk() {
+    ARROW_ASSIGN_OR_RAISE(auto chunk, converter_->ToArray(length_));
+    chunks_.push_back(chunk);
+    // reserve space for the remaining items, besides being an optimization it is also
+    // required if the converter's implementation relies on unsafe builder methods in
+    // conveter->Append()
+    auto remaining = reserved_ - length_;
+    Reset();
+    return Reserve(remaining);
+  }
+
+  Result<std::shared_ptr<ChunkedArray>> ToChunkedArray() {
+    ARROW_RETURN_NOT_OK(FinishChunk());
+    return std::make_shared<ChunkedArray>(chunks_);
+  }
+
+ protected:
+  void Reset() {
+    converter_->builder()->Reset();
+    length_ = 0;
+    reserved_ = 0;
+  }
+
+  int64_t length_ = 0;
+  int64_t reserved_ = 0;
+  std::unique_ptr<Converter> converter_;
+  std::vector<std::shared_ptr<Array>> chunks_;
+};
+
+template <typename T>
+static Result<std::unique_ptr<Chunker<T>>> MakeChunker(std::unique_ptr<T> converter) {
+  return internal::make_unique<Chunker<T>>(std::move(converter));
+}
+
+}  // namespace internal
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/cpu_info.cc b/cpp/src/arrow/util/cpu_info.cc
index 2fd1ad07930..ce3ad892af0 100644
--- a/cpp/src/arrow/util/cpu_info.cc
+++ b/cpp/src/arrow/util/cpu_info.cc
@@ -39,6 +39,7 @@
 #endif
 
 #include <algorithm>
+#include <cctype>
 #include <cerrno>
 #include <cstdint>
 #include <fstream>
@@ -46,6 +47,8 @@
 #include <mutex>
 #include <string>
 
+#include "arrow/result.h"
+#include "arrow/util/io_util.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/string.h"
 
@@ -123,10 +126,12 @@ static struct {
   int64_t flag;
 } flag_mappings[] = {
 #if (defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64))
-    {"ssse3", CpuInfo::SSSE3},    {"sse4_1", CpuInfo::SSE4_1},
-    {"sse4_2", CpuInfo::SSE4_2},  {"popcnt", CpuInfo::POPCNT},
-    {"avx", CpuInfo::AVX},        {"avx2", CpuInfo::AVX2},
-    {"avx512f", CpuInfo::AVX512}, {"bmi1", CpuInfo::BMI1},
+    {"ssse3", CpuInfo::SSSE3},       {"sse4_1", CpuInfo::SSE4_1},
+    {"sse4_2", CpuInfo::SSE4_2},     {"popcnt", CpuInfo::POPCNT},
+    {"avx", CpuInfo::AVX},           {"avx2", CpuInfo::AVX2},
+    {"avx512f", CpuInfo::AVX512F},   {"avx512cd", CpuInfo::AVX512CD},
+    {"avx512vl", CpuInfo::AVX512VL}, {"avx512dq", CpuInfo::AVX512DQ},
+    {"avx512bw", CpuInfo::AVX512BW}, {"bmi1", CpuInfo::BMI1},
     {"bmi2", CpuInfo::BMI2},
 #endif
 #if defined(__aarch64__)
@@ -202,8 +207,9 @@ bool RetrieveCacheSize(int64_t* cache_sizes) {
 }
 
 // Source: https://en.wikipedia.org/wiki/CPUID
-bool RetrieveCPUInfo(int64_t* hardware_flags, std::string* model_name) {
-  if (!hardware_flags || !model_name) {
+bool RetrieveCPUInfo(int64_t* hardware_flags, std::string* model_name,
+                     CpuInfo::Vendor* vendor) {
+  if (!hardware_flags || !model_name || !vendor) {
     return false;
   }
   int register_EAX_id = 1;
@@ -215,6 +221,15 @@ bool RetrieveCPUInfo(int64_t* hardware_flags, std::string* model_name) {
   // Get highest valid id
   __cpuid(cpu_info.data(), 0);
   highest_valid_id = cpu_info[0];
+  // HEX of "GenuineIntel": 47656E75 696E6549 6E74656C
+  // HEX of "AuthenticAMD": 41757468 656E7469 63414D44
+  if (cpu_info[1] == 0x756e6547 && cpu_info[2] == 0x49656e69 &&
+      cpu_info[3] == 0x6c65746e) {
+    *vendor = CpuInfo::Vendor::Intel;
+  } else if (cpu_info[1] == 0x68747541 && cpu_info[2] == 0x69746e65 &&
+             cpu_info[3] == 0x444d4163) {
+    *vendor = CpuInfo::Vendor::AMD;
+  }
 
   if (highest_valid_id <= register_EAX_id) return false;
 
@@ -251,14 +266,22 @@ bool RetrieveCPUInfo(int64_t* hardware_flags, std::string* model_name) {
     if (features_EBX[3]) *hardware_flags |= CpuInfo::BMI1;
     if (features_EBX[5]) *hardware_flags |= CpuInfo::AVX2;
     if (features_EBX[8]) *hardware_flags |= CpuInfo::BMI2;
-    if (features_EBX[16]) *hardware_flags |= CpuInfo::AVX512;
+    if (features_EBX[16]) *hardware_flags |= CpuInfo::AVX512F;
+    if (features_EBX[17]) *hardware_flags |= CpuInfo::AVX512DQ;
+    if (features_EBX[28]) *hardware_flags |= CpuInfo::AVX512CD;
+    if (features_EBX[30]) *hardware_flags |= CpuInfo::AVX512BW;
+    if (features_EBX[31]) *hardware_flags |= CpuInfo::AVX512VL;
   }
 
   return true;
 }
 #endif
 
-CpuInfo::CpuInfo() : hardware_flags_(0), num_cores_(1), model_name_("unknown") {}
+CpuInfo::CpuInfo()
+    : hardware_flags_(0),
+      num_cores_(1),
+      model_name_("unknown"),
+      vendor_(Vendor::Unknown) {}
 
 std::unique_ptr<CpuInfo> g_cpu_info;
 static std::once_flag cpuinfo_initialized;
@@ -312,6 +335,12 @@ void CpuInfo::Init() {
         ++num_cores;
       } else if (name.compare("model name") == 0) {
         model_name_ = value;
+      } else if (name.compare("vendor_id") == 0) {
+        if (value.compare("GenuineIntel") == 0) {
+          vendor_ = Vendor::Intel;
+        } else if (value.compare("AuthenticAMD") == 0) {
+          vendor_ = Vendor::AMD;
+        }
       }
     }
   }
@@ -332,7 +361,7 @@ void CpuInfo::Init() {
   if (!RetrieveCacheSize(cache_sizes_)) {
     SetDefaultCacheSize();
   }
-  RetrieveCPUInfo(&hardware_flags_, &model_name_);
+  RetrieveCPUInfo(&hardware_flags_, &model_name_, &vendor_);
 #else
   SetDefaultCacheSize();
 #endif
@@ -420,24 +449,29 @@ void CpuInfo::SetDefaultCacheSize() {
 }
 
 void CpuInfo::ParseUserSimdLevel() {
-  const char* user = std::getenv("ARROW_USER_SIMD_LEVEL");
-  if (!user) {
+  auto maybe_env_var = GetEnvVar("ARROW_USER_SIMD_LEVEL");
+  if (!maybe_env_var.ok()) {
     // No user settings
     return;
   }
+  std::string s = *std::move(maybe_env_var);
+  std::transform(s.begin(), s.end(), s.begin(),
+                 [](unsigned char c) { return std::toupper(c); });
 
   int level = USER_SIMD_MAX;
   // Parse the level
-  if (0 == strcmp("avx512", user)) {
+  if (s == "AVX512") {
     level = USER_SIMD_AVX512;
-  } else if (0 == strcmp("avx2", user)) {
+  } else if (s == "AVX2") {
     level = USER_SIMD_AVX2;
-  } else if (0 == strcmp("avx", user)) {
+  } else if (s == "AVX") {
     level = USER_SIMD_AVX;
-  } else if (0 == strcmp("sse4_2", user)) {
+  } else if (s == "SSE4_2") {
     level = USER_SIMD_SSE4_2;
-  } else if (0 == strcmp("none", user)) {
+  } else if (s == "NONE") {
     level = USER_SIMD_NONE;
+  } else if (!s.empty()) {
+    ARROW_LOG(WARNING) << "Invalid value for ARROW_USER_SIMD_LEVEL: " << s;
   }
 
   // Disable feature as the level
diff --git a/cpp/src/arrow/util/cpu_info.h b/cpp/src/arrow/util/cpu_info.h
index e18a632bbbd..a57ffd29467 100644
--- a/cpp/src/arrow/util/cpu_info.h
+++ b/cpp/src/arrow/util/cpu_info.h
@@ -41,9 +41,16 @@ class ARROW_EXPORT CpuInfo {
   static constexpr int64_t ASIMD = (1 << 5);
   static constexpr int64_t AVX = (1 << 6);
   static constexpr int64_t AVX2 = (1 << 7);
-  static constexpr int64_t AVX512 = (1 << 8);
-  static constexpr int64_t BMI1 = (1 << 9);
-  static constexpr int64_t BMI2 = (1 << 10);
+  static constexpr int64_t AVX512F = (1 << 8);
+  static constexpr int64_t AVX512CD = (1 << 9);
+  static constexpr int64_t AVX512VL = (1 << 10);
+  static constexpr int64_t AVX512DQ = (1 << 11);
+  static constexpr int64_t AVX512BW = (1 << 12);
+  static constexpr int64_t BMI1 = (1 << 13);
+  static constexpr int64_t BMI2 = (1 << 14);
+
+  /// Typical AVX512 subsets consists of AVX512F,AVX512BW,AVX512VL,AVX512CD,AVX512DQ
+  static constexpr int64_t AVX512 = AVX512F | AVX512CD | AVX512VL | AVX512DQ | AVX512BW;
 
   /// Cache enums for L1 (data), L2 and L3
   enum CacheLevel {
@@ -52,6 +59,8 @@ class ARROW_EXPORT CpuInfo {
     L3_CACHE = 2,
   };
 
+  enum class Vendor : int { Unknown = 0, Intel, AMD };
+
   /// The SIMD level set by user
   enum UserSimdLevel {
     USER_SIMD_NONE = 0,
@@ -71,8 +80,8 @@ class ARROW_EXPORT CpuInfo {
   /// Returns all the flags for this cpu
   int64_t hardware_flags();
 
-  /// Returns whether of not the cpu supports this flag
-  bool IsSupported(int64_t flag) const { return (hardware_flags_ & flag) != 0; }
+  /// Returns whether or not the cpu supports all the flags
+  bool IsSupported(int64_t flags) const { return (hardware_flags_ & flags) == flags; }
 
   /// \brief The processor supports SSE4.2 and the Arrow libraries are built
   /// with support for it
@@ -94,6 +103,14 @@ class ARROW_EXPORT CpuInfo {
   /// Returns the model name of the cpu (e.g. Intel i7-2600)
   std::string model_name();
 
+  /// Returns the vendor of the cpu.
+  Vendor vendor() const { return vendor_; }
+
+  bool HasEfficientBmi2() const {
+    // BMI2 (pext, pdep) is only efficient on Intel X86 processors.
+    return vendor() == Vendor::Intel && IsSupported(BMI2);
+  }
+
  private:
   CpuInfo();
 
@@ -111,6 +128,7 @@ class ARROW_EXPORT CpuInfo {
   int64_t cycles_per_ms_;
   int num_cores_;
   std::string model_name_;
+  Vendor vendor_;
 };
 
 }  // namespace internal
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index c9c63f8ad18..c38e66ca810 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -31,6 +31,8 @@
 #include "arrow/status.h"
 #include "arrow/util/bit_util.h"
 #include "arrow/util/decimal.h"
+#include "arrow/util/formatting.h"
+#include "arrow/util/int128_internal.h"
 #include "arrow/util/int_util_internal.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
@@ -40,39 +42,36 @@ namespace arrow {
 
 using internal::SafeLeftShift;
 using internal::SafeSignedAdd;
+using internal::uint128_t;
 
 Decimal128::Decimal128(const std::string& str) : Decimal128() {
   *this = Decimal128::FromString(str).ValueOrDie();
 }
 
-static const Decimal128 kTenTo36(static_cast<int64_t>(0xC097CE7BC90715),
-                                 0xB34B9F1000000000);
-static const Decimal128 kTenTo18(0xDE0B6B3A7640000);
-
 static constexpr auto kInt64DecimalDigits =
     static_cast<size_t>(std::numeric_limits<int64_t>::digits10);
 
-static constexpr int64_t kInt64PowersOfTen[kInt64DecimalDigits + 1] = {
+static constexpr uint64_t kUInt64PowersOfTen[kInt64DecimalDigits + 1] = {
     // clang-format off
-    1LL,
-    10LL,
-    100LL,
-    1000LL,
-    10000LL,
-    100000LL,
-    1000000LL,
-    10000000LL,
-    100000000LL,
-    1000000000LL,
-    10000000000LL,
-    100000000000LL,
-    1000000000000LL,
-    10000000000000LL,
-    100000000000000LL,
-    1000000000000000LL,
-    10000000000000000LL,
-    100000000000000000LL,
-    1000000000000000000LL
+    1ULL,
+    10ULL,
+    100ULL,
+    1000ULL,
+    10000ULL,
+    100000ULL,
+    1000000ULL,
+    10000000ULL,
+    100000000ULL,
+    1000000000ULL,
+    10000000000ULL,
+    100000000000ULL,
+    1000000000000ULL,
+    10000000000000ULL,
+    100000000000000ULL,
+    1000000000000000ULL,
+    10000000000000000ULL,
+    100000000000000000ULL,
+    1000000000000000000ULL
     // clang-format on
 };
 
@@ -195,42 +194,86 @@ double Decimal128::ToDouble(int32_t scale) const {
   return DecimalDoubleConversion::ToReal(*this, scale);
 }
 
-std::string Decimal128::ToIntegerString() const {
-  Decimal128 remainder;
-  std::stringstream buf;
-  bool need_fill = false;
-
-  // get anything above 10 ** 36 and print it
-  Decimal128 top;
-  std::tie(top, remainder) = Divide(kTenTo36).ValueOrDie();
-
-  if (top != 0) {
-    buf << static_cast<int64_t>(top);
-    remainder.Abs();
-    need_fill = true;
-  }
-
-  // now get anything above 10 ** 18 and print it
-  Decimal128 tail;
-  std::tie(top, tail) = remainder.Divide(kTenTo18).ValueOrDie();
-
-  if (need_fill || top != 0) {
-    if (need_fill) {
-      buf << std::setw(18) << std::setfill('0');
-    } else {
-      need_fill = true;
-      tail.Abs();
-    }
-
-    buf << static_cast<int64_t>(top);
-  }
+template <size_t n>
+static void AppendLittleEndianArrayToString(const std::array<uint64_t, n>& array,
+                                            std::string* result) {
+  const auto most_significant_non_zero =
+      find_if(array.rbegin(), array.rend(), [](uint64_t v) { return v != 0; });
+  if (most_significant_non_zero == array.rend()) {
+    result->push_back('0');
+    return;
+  }
+
+  size_t most_significant_elem_idx = &*most_significant_non_zero - array.data();
+  std::array<uint64_t, n> copy = array;
+  constexpr uint32_t k1e9 = 1000000000U;
+  constexpr size_t kNumBits = n * 64;
+  // Segments will contain the array split into groups that map to decimal digits,
+  // in little endian order. Each segment will hold at most 9 decimal digits.
+  // For example, if the input represents 9876543210123456789, then segments will be
+  // [123456789, 876543210, 9].
+  // The max number of segments needed = ceil(kNumBits * log(2) / log(1e9))
+  // = ceil(kNumBits / 29.897352854) <= ceil(kNumBits / 29).
+  std::array<uint32_t, (kNumBits + 28) / 29> segments;
+  size_t num_segments = 0;
+  uint64_t* most_significant_elem = &copy[most_significant_elem_idx];
+  do {
+    // Compute remainder = copy % 1e9 and copy = copy / 1e9.
+    uint32_t remainder = 0;
+    uint64_t* elem = most_significant_elem;
+    do {
+      // Compute dividend = (remainder << 32) | *elem  (a virtual 96-bit integer);
+      // *elem = dividend / 1e9;
+      // remainder = dividend % 1e9.
+      uint32_t hi = static_cast<uint32_t>(*elem >> 32);
+      uint32_t lo = static_cast<uint32_t>(*elem & BitUtil::LeastSignficantBitMask(32));
+      uint64_t dividend_hi = (static_cast<uint64_t>(remainder) << 32) | hi;
+      uint64_t quotient_hi = dividend_hi / k1e9;
+      remainder = static_cast<uint32_t>(dividend_hi % k1e9);
+      uint64_t dividend_lo = (static_cast<uint64_t>(remainder) << 32) | lo;
+      uint64_t quotient_lo = dividend_lo / k1e9;
+      remainder = static_cast<uint32_t>(dividend_lo % k1e9);
+      *elem = (quotient_hi << 32) | quotient_lo;
+    } while (elem-- != copy.data());
+
+    segments[num_segments++] = remainder;
+  } while (*most_significant_elem != 0 || most_significant_elem-- != copy.data());
+
+  size_t old_size = result->size();
+  size_t new_size = old_size + num_segments * 9;
+  result->resize(new_size, '0');
+  char* output = &result->at(old_size);
+  const uint32_t* segment = &segments[num_segments - 1];
+  internal::StringFormatter<UInt32Type> format;
+  // First segment is formatted as-is.
+  format(*segment, [&output](util::string_view formatted) {
+    memcpy(output, formatted.data(), formatted.size());
+    output += formatted.size();
+  });
+  while (segment != segments.data()) {
+    --segment;
+    // Right-pad formatted segment such that e.g. 123 is formatted as "000000123".
+    output += 9;
+    format(*segment, [output](util::string_view formatted) {
+      memcpy(output - formatted.size(), formatted.data(), formatted.size());
+    });
+  }
+  result->resize(output - result->data());
+}
 
-  // finally print the tail, which is less than 10**18
-  if (need_fill) {
-    buf << std::setw(18) << std::setfill('0');
+std::string Decimal128::ToIntegerString() const {
+  std::string result;
+  if (high_bits() < 0) {
+    result.push_back('-');
+    Decimal128 abs = *this;
+    abs.Negate();
+    AppendLittleEndianArrayToString<2>(
+        {abs.low_bits(), static_cast<uint64_t>(abs.high_bits())}, &result);
+  } else {
+    AppendLittleEndianArrayToString<2>({low_bits(), static_cast<uint64_t>(high_bits())},
+                                       &result);
   }
-  buf << static_cast<int64_t>(tail);
-  return buf.str();
+  return result;
 }
 
 Decimal128::operator int64_t() const {
@@ -241,97 +284,95 @@ Decimal128::operator int64_t() const {
   return static_cast<int64_t>(low_bits());
 }
 
-static std::string ToStringNegativeScale(const std::string& str,
-                                         int32_t adjusted_exponent, bool is_negative) {
-  std::stringstream buf;
-
-  size_t offset = 0;
-  buf << str[offset++];
-
-  if (is_negative) {
-    buf << str[offset++];
-  }
-
-  buf << '.' << str.substr(offset, std::string::npos) << 'E' << std::showpos
-      << adjusted_exponent;
-  return buf.str();
-}
-
-std::string Decimal128::ToString(int32_t scale) const {
-  const std::string str(ToIntegerString());
-
+static void AdjustIntegerStringWithScale(int32_t scale, std::string* str) {
   if (scale == 0) {
-    return str;
+    return;
   }
-
-  const bool is_negative = *this < 0;
-
-  const auto len = static_cast<int32_t>(str.size());
+  DCHECK(str != nullptr);
+  DCHECK(!str->empty());
+  const bool is_negative = str->front() == '-';
   const auto is_negative_offset = static_cast<int32_t>(is_negative);
-  const int32_t adjusted_exponent = -scale + (len - 1 - is_negative_offset);
+  const auto len = static_cast<int32_t>(str->size());
+  const int32_t num_digits = len - is_negative_offset;
+  const int32_t adjusted_exponent = num_digits - 1 - scale;
 
   /// Note that the -6 is taken from the Java BigDecimal documentation.
   if (scale < 0 || adjusted_exponent < -6) {
-    return ToStringNegativeScale(str, adjusted_exponent, is_negative);
-  }
-
-  if (is_negative) {
-    if (len - 1 > scale) {
-      const auto n = static_cast<size_t>(len - scale);
-      return str.substr(0, n) + "." + str.substr(n, static_cast<size_t>(scale));
+    // Example 1:
+    // Precondition: *str = "123", is_negative_offset = 0, num_digits = 3, scale = -2,
+    //               adjusted_exponent = 4
+    // After inserting decimal point: *str = "1.23"
+    // After appending exponent: *str = "1.23E+4"
+    // Example 2:
+    // Precondition: *str = "-123", is_negative_offset = 1, num_digits = 3, scale = 9,
+    //               adjusted_exponent = -7
+    // After inserting decimal point: *str = "-1.23"
+    // After appending exponent: *str = "-1.23E-7"
+    str->insert(str->begin() + 1 + is_negative_offset, '.');
+    str->push_back('E');
+    if (adjusted_exponent >= 0) {
+      str->push_back('+');
     }
-
-    if (len - 1 == scale) {
-      return "-0." + str.substr(1, std::string::npos);
-    }
-
-    std::string result("-0." + std::string(static_cast<size_t>(scale - len + 1), '0'));
-    return result + str.substr(1, std::string::npos);
+    internal::StringFormatter<Int32Type> format;
+    format(adjusted_exponent, [str](util::string_view formatted) {
+      str->append(formatted.data(), formatted.size());
+    });
+    return;
   }
 
-  if (len > scale) {
+  if (num_digits > scale) {
     const auto n = static_cast<size_t>(len - scale);
-    return str.substr(0, n) + "." + str.substr(n, static_cast<size_t>(scale));
-  }
-
-  if (len == scale) {
-    return "0." + str;
-  }
+    // Example 1:
+    // Precondition: *str = "123", len = num_digits = 3, scale = 1, n = 2
+    // After inserting decimal point: *str = "12.3"
+    // Example 2:
+    // Precondition: *str = "-123", len = 4, num_digits = 3, scale = 1, n = 3
+    // After inserting decimal point: *str = "-12.3"
+    str->insert(str->begin() + n, '.');
+    return;
+  }
+
+  // Example 1:
+  // Precondition: *str = "123", is_negative_offset = 0, num_digits = 3, scale = 4
+  // After insert: *str = "000123"
+  // After setting decimal point: *str = "0.0123"
+  // Example 2:
+  // Precondition: *str = "-123", is_negative_offset = 1, num_digits = 3, scale = 4
+  // After insert: *str = "-000123"
+  // After setting decimal point: *str = "-0.0123"
+  str->insert(is_negative_offset, scale - num_digits + 2, '0');
+  str->at(is_negative_offset + 1) = '.';
+}
 
-  return "0." + std::string(static_cast<size_t>(scale - len), '0') + str;
+std::string Decimal128::ToString(int32_t scale) const {
+  std::string str(ToIntegerString());
+  AdjustIntegerStringWithScale(scale, &str);
+  return str;
 }
 
-// Iterates over data and for each group of kInt64DecimalDigits multiple out by
+// Iterates over input and for each group of kInt64DecimalDigits multiple out by
 // the appropriate power of 10 necessary to add source parsed as uint64 and
 // then adds the parsed value of source.
-static inline void ShiftAndAdd(const char* data, size_t length, Decimal128* out) {
-  for (size_t posn = 0; posn < length;) {
-    const size_t group_size = std::min(kInt64DecimalDigits, length - posn);
-    const int64_t multiple = kInt64PowersOfTen[group_size];
-    int64_t chunk = 0;
-    ARROW_CHECK(internal::ParseValue<Int64Type>(data + posn, group_size, &chunk));
-
-    *out *= multiple;
-    *out += chunk;
+static inline void ShiftAndAdd(const util::string_view& input, uint64_t out[],
+                               size_t out_size) {
+  for (size_t posn = 0; posn < input.size();) {
+    const size_t group_size = std::min(kInt64DecimalDigits, input.size() - posn);
+    const uint64_t multiple = kUInt64PowersOfTen[group_size];
+    uint64_t chunk = 0;
+    ARROW_CHECK(
+        internal::ParseValue<UInt64Type>(input.data() + posn, group_size, &chunk));
+
+    for (size_t i = 0; i < out_size; ++i) {
+      uint128_t tmp = out[i];
+      tmp *= multiple;
+      tmp += chunk;
+      out[i] = static_cast<uint64_t>(tmp & 0xFFFFFFFFFFFFFFFFULL);
+      chunk = static_cast<uint64_t>(tmp >> 64);
+    }
     posn += group_size;
   }
 }
 
-static void StringToInteger(const util::string_view whole_digits,
-                            util::string_view fractional_digits, Decimal128* out) {
-  using std::size_t;
-
-  DCHECK_NE(out, nullptr) << "Decimal128 output variable cannot be nullptr";
-  DCHECK_EQ(*out, 0)
-      << "When converting a string to Decimal128 the initial output must be 0";
-
-  DCHECK_GT(whole_digits.size() + fractional_digits.size(), 0)
-      << "length of parsed decimal string should be greater than 0";
-
-  ShiftAndAdd(whole_digits.data(), whole_digits.length(), out);
-  ShiftAndAdd(fractional_digits.data(), fractional_digits.length(), out);
-}
-
 namespace {
 
 struct DecimalComponents {
@@ -404,6 +445,24 @@ bool ParseDecimalComponents(const char* s, size_t size, DecimalComponents* out)
   return pos == size;
 }
 
+inline Status ToArrowStatus(DecimalStatus dstatus, int num_bits) {
+  switch (dstatus) {
+    case DecimalStatus::kSuccess:
+      return Status::OK();
+
+    case DecimalStatus::kDivideByZero:
+      return Status::Invalid("Division by 0 in Decimal", num_bits);
+
+    case DecimalStatus::kOverflow:
+      return Status::Invalid("Overflow occurred during Decimal", num_bits, " operation.");
+
+    case DecimalStatus::kRescaleDataLoss:
+      return Status::Invalid("Rescaling Decimal", num_bits,
+                             " value would cause data loss");
+  }
+  return Status::OK();
+}
+
 }  // namespace
 
 Status Decimal128::FromString(const util::string_view& s, Decimal128* out,
@@ -423,38 +482,43 @@ Status Decimal128::FromString(const util::string_view& s, Decimal128* out,
   if (first_non_zero != std::string::npos) {
     significant_digits += dec.whole_digits.size() - first_non_zero;
   }
+  int32_t parsed_precision = static_cast<int32_t>(significant_digits);
 
-  if (precision != nullptr) {
-    *precision = static_cast<int32_t>(significant_digits);
-  }
-
-  if (scale != nullptr) {
-    if (dec.has_exponent) {
-      auto adjusted_exponent = dec.exponent;
-      auto len = static_cast<int32_t>(significant_digits);
-      *scale = -adjusted_exponent + len - 1;
-    } else {
-      *scale = static_cast<int32_t>(dec.fractional_digits.size());
-    }
+  int32_t parsed_scale = 0;
+  if (dec.has_exponent) {
+    auto adjusted_exponent = dec.exponent;
+    auto len = static_cast<int32_t>(significant_digits);
+    parsed_scale = -adjusted_exponent + len - 1;
+  } else {
+    parsed_scale = static_cast<int32_t>(dec.fractional_digits.size());
   }
 
   if (out != nullptr) {
-    *out = 0;
-    StringToInteger(dec.whole_digits, dec.fractional_digits, out);
+    std::array<uint64_t, 2> little_endian_array = {0, 0};
+    ShiftAndAdd(dec.whole_digits, little_endian_array.data(), little_endian_array.size());
+    ShiftAndAdd(dec.fractional_digits, little_endian_array.data(),
+                little_endian_array.size());
+    *out =
+        Decimal128(static_cast<int64_t>(little_endian_array[1]), little_endian_array[0]);
+    if (parsed_scale < 0) {
+      *out *= GetScaleMultiplier(-parsed_scale);
+    }
 
     if (dec.sign == '-') {
       out->Negate();
     }
+  }
 
-    if (scale != nullptr && *scale < 0) {
-      const int32_t abs_scale = std::abs(*scale);
-      *out *= GetScaleMultiplier(abs_scale);
+  if (parsed_scale < 0) {
+    parsed_precision -= parsed_scale;
+    parsed_scale = 0;
+  }
 
-      if (precision != nullptr) {
-        *precision += abs_scale;
-      }
-      *scale = 0;
-    }
+  if (precision != nullptr) {
+    *precision = parsed_precision;
+  }
+  if (scale != nullptr) {
+    *scale = parsed_scale;
   }
 
   return Status::OK();
@@ -552,31 +616,114 @@ Result<Decimal128> Decimal128::FromBigEndian(const uint8_t* bytes, int32_t lengt
 }
 
 Status Decimal128::ToArrowStatus(DecimalStatus dstatus) const {
-  Status status;
+  return arrow::ToArrowStatus(dstatus, 128);
+}
 
-  switch (dstatus) {
-    case DecimalStatus::kSuccess:
-      status = Status::OK();
-      break;
+std::ostream& operator<<(std::ostream& os, const Decimal128& decimal) {
+  os << decimal.ToIntegerString();
+  return os;
+}
 
-    case DecimalStatus::kDivideByZero:
-      status = Status::Invalid("Division by 0 in Decimal128");
-      break;
+Decimal256::Decimal256(const std::string& str) : Decimal256() {
+  *this = Decimal256::FromString(str).ValueOrDie();
+}
 
-    case DecimalStatus::kOverflow:
-      status = Status::Invalid("Overflow occurred during Decimal128 operation.");
-      break;
+std::string Decimal256::ToIntegerString() const {
+  std::string result;
+  if (static_cast<int64_t>(little_endian_array()[3]) < 0) {
+    result.push_back('-');
+    Decimal256 abs = *this;
+    abs.Negate();
+    AppendLittleEndianArrayToString(abs.little_endian_array(), &result);
+  } else {
+    AppendLittleEndianArrayToString(little_endian_array(), &result);
+  }
+  return result;
+}
 
-    case DecimalStatus::kRescaleDataLoss:
-      status = Status::Invalid("Rescaling decimal value would cause data loss");
-      break;
+std::string Decimal256::ToString(int32_t scale) const {
+  std::string str(ToIntegerString());
+  AdjustIntegerStringWithScale(scale, &str);
+  return str;
+}
+
+Status Decimal256::FromString(const util::string_view& s, Decimal256* out,
+                              int32_t* precision, int32_t* scale) {
+  if (s.empty()) {
+    return Status::Invalid("Empty string cannot be converted to decimal");
+  }
+
+  DecimalComponents dec;
+  if (!ParseDecimalComponents(s.data(), s.size(), &dec)) {
+    return Status::Invalid("The string '", s, "' is not a valid decimal number");
+  }
+
+  // Count number of significant digits (without leading zeros)
+  size_t first_non_zero = dec.whole_digits.find_first_not_of('0');
+  size_t significant_digits = dec.fractional_digits.size();
+  if (first_non_zero != std::string::npos) {
+    significant_digits += dec.whole_digits.size() - first_non_zero;
+  }
+
+  if (precision != nullptr) {
+    *precision = static_cast<int32_t>(significant_digits);
+  }
+
+  if (scale != nullptr) {
+    if (dec.has_exponent) {
+      auto adjusted_exponent = dec.exponent;
+      auto len = static_cast<int32_t>(significant_digits);
+      *scale = -adjusted_exponent + len - 1;
+    } else {
+      *scale = static_cast<int32_t>(dec.fractional_digits.size());
+    }
   }
-  return status;
+
+  if (out != nullptr) {
+    std::array<uint64_t, 4> little_endian_array = {0, 0, 0, 0};
+    ShiftAndAdd(dec.whole_digits, little_endian_array.data(), little_endian_array.size());
+    ShiftAndAdd(dec.fractional_digits, little_endian_array.data(),
+                little_endian_array.size());
+    *out = Decimal256(little_endian_array);
+
+    if (dec.sign == '-') {
+      out->Negate();
+    }
+  }
+
+  return Status::OK();
 }
 
-std::ostream& operator<<(std::ostream& os, const Decimal128& decimal) {
+Status Decimal256::FromString(const std::string& s, Decimal256* out, int32_t* precision,
+                              int32_t* scale) {
+  return FromString(util::string_view(s), out, precision, scale);
+}
+
+Status Decimal256::FromString(const char* s, Decimal256* out, int32_t* precision,
+                              int32_t* scale) {
+  return FromString(util::string_view(s), out, precision, scale);
+}
+
+Result<Decimal256> Decimal256::FromString(const util::string_view& s) {
+  Decimal256 out;
+  RETURN_NOT_OK(FromString(s, &out, nullptr, nullptr));
+  return std::move(out);
+}
+
+Result<Decimal256> Decimal256::FromString(const std::string& s) {
+  return FromString(util::string_view(s));
+}
+
+Result<Decimal256> Decimal256::FromString(const char* s) {
+  return FromString(util::string_view(s));
+}
+
+Status Decimal256::ToArrowStatus(DecimalStatus dstatus) const {
+  return arrow::ToArrowStatus(dstatus, 256);
+}
+
+std::ostream& operator<<(std::ostream& os, const Decimal256& decimal) {
   os << decimal.ToIntegerString();
   return os;
 }
-
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
index 1f727057c13..3b159bc8d88 100644
--- a/cpp/src/arrow/util/decimal.h
+++ b/cpp/src/arrow/util/decimal.h
@@ -172,4 +172,67 @@ struct Decimal128::ToRealConversion<double> {
   }
 };
 
+/// Represents a signed 256-bit integer in two's complement.
+/// The max decimal precision that can be safely represented is
+/// 76 significant digits.
+///
+/// The implementation is split into two parts :
+///
+/// 1. BasicDecimal256
+///    - can be safely compiled to IR without references to libstdc++.
+/// 2. Decimal256
+///    - (TODO) has additional functionality on top of BasicDecimal256 to deal with
+///      strings and streams.
+class ARROW_EXPORT Decimal256 : public BasicDecimal256 {
+ public:
+  /// \cond FALSE
+  // (need to avoid a duplicate definition in Sphinx)
+  using BasicDecimal256::BasicDecimal256;
+  /// \endcond
+
+  /// \brief constructor creates a Decimal256 from a BasicDecimal128.
+  constexpr Decimal256(const BasicDecimal256& value) noexcept : BasicDecimal256(value) {}
+
+  /// \brief Parse the number from a base 10 string representation.
+  explicit Decimal256(const std::string& value);
+
+  /// \brief Empty constructor creates a Decimal256 with a value of 0.
+  // This is required on some older compilers.
+  constexpr Decimal256() noexcept : BasicDecimal256() {}
+
+  /// \brief Convert the Decimal256 value to a base 10 decimal string with the given
+  /// scale.
+  std::string ToString(int32_t scale) const;
+
+  /// \brief Convert the value to an integer string
+  std::string ToIntegerString() const;
+
+  /// \brief Convert a decimal string to a Decimal256 value, optionally including
+  /// precision and scale if they're passed in and not null.
+  static Status FromString(const util::string_view& s, Decimal256* out,
+                           int32_t* precision, int32_t* scale = NULLPTR);
+  static Status FromString(const std::string& s, Decimal256* out, int32_t* precision,
+                           int32_t* scale = NULLPTR);
+  static Status FromString(const char* s, Decimal256* out, int32_t* precision,
+                           int32_t* scale = NULLPTR);
+  static Result<Decimal256> FromString(const util::string_view& s);
+  static Result<Decimal256> FromString(const std::string& s);
+  static Result<Decimal256> FromString(const char* s);
+
+  /// \brief Convert Decimal256 from one scale to another
+  Result<Decimal256> Rescale(int32_t original_scale, int32_t new_scale) const {
+    Decimal256 out;
+    auto dstatus = BasicDecimal256::Rescale(original_scale, new_scale, &out);
+    ARROW_RETURN_NOT_OK(ToArrowStatus(dstatus));
+    return std::move(out);
+  }
+
+  friend ARROW_EXPORT std::ostream& operator<<(std::ostream& os,
+                                               const Decimal256& decimal);
+
+ private:
+  /// Converts internal error code to Status
+  Status ToArrowStatus(DecimalStatus dstatus) const;
+};
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/decimal_benchmark.cc b/cpp/src/arrow/util/decimal_benchmark.cc
index 620593cf7d8..8e2a63dcf9d 100644
--- a/cpp/src/arrow/util/decimal_benchmark.cc
+++ b/cpp/src/arrow/util/decimal_benchmark.cc
@@ -21,19 +21,40 @@
 #include <vector>
 
 #include "arrow/util/decimal.h"
+#include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
 
 namespace arrow {
 namespace Decimal {
 
-static void FromString(benchmark::State& state) {  // NOLINT non-const reference
-  std::vector<std::string> values = {"0",
-                                     "1.23",
-                                     "12.345e6",
-                                     "-12.345e-6",
-                                     "123456789.123456789",
-                                     "1231234567890.451234567890"};
+static const std::vector<std::string>& GetValuesAsString() {
+  static const std::vector<std::string> kValues = {"0",
+                                                   "1.23",
+                                                   "12.345e6",
+                                                   "-12.345e-6",
+                                                   "123456789.123456789",
+                                                   "1231234567890.451234567890"};
+  return kValues;
+}
+
+struct DecimalValueAndScale {
+  Decimal128 decimal;
+  int32_t scale;
+};
+
+static std::vector<DecimalValueAndScale> GetDecimalValuesAndScales() {
+  const std::vector<std::string>& value_strs = GetValuesAsString();
+  std::vector<DecimalValueAndScale> result(value_strs.size());
+  for (size_t i = 0; i < value_strs.size(); ++i) {
+    int32_t precision;
+    ARROW_CHECK_OK(Decimal128::FromString(value_strs[i], &result[i].decimal,
+                                          &result[i].scale, &precision));
+  }
+  return result;
+}
 
+static void FromString(benchmark::State& state) {  // NOLINT non-const reference
+  const std::vector<std::string>& values = GetValuesAsString();
   for (auto _ : state) {
     for (const auto& value : values) {
       Decimal128 dec;
@@ -44,6 +65,16 @@ static void FromString(benchmark::State& state) {  // NOLINT non-const reference
   state.SetItemsProcessed(state.iterations() * values.size());
 }
 
+static void ToString(benchmark::State& state) {  // NOLINT non-const reference
+  static const std::vector<DecimalValueAndScale> values = GetDecimalValuesAndScales();
+  for (auto _ : state) {
+    for (const DecimalValueAndScale& item : values) {
+      benchmark::DoNotOptimize(item.decimal.ToString(item.scale));
+    }
+  }
+  state.SetItemsProcessed(state.iterations() * values.size());
+}
+
 constexpr int32_t kValueSize = 10;
 
 static void BinaryCompareOp(benchmark::State& state) {  // NOLINT non-const reference
@@ -98,7 +129,7 @@ static void BinaryMathOpAggregate(
   state.SetItemsProcessed(state.iterations() * kValueSize);
 }
 
-static void BinaryMathOp(benchmark::State& state) {  // NOLINT non-const reference
+static void BinaryMathOp128(benchmark::State& state) {  // NOLINT non-const reference
   std::vector<BasicDecimal128> v1, v2;
   for (int x = 0; x < kValueSize; x++) {
     v1.emplace_back(100 + x, 100 + x);
@@ -117,6 +148,21 @@ static void BinaryMathOp(benchmark::State& state) {  // NOLINT non-const referen
   state.SetItemsProcessed(state.iterations() * kValueSize);
 }
 
+static void BinaryMathOp256(benchmark::State& state) {  // NOLINT non-const reference
+  std::vector<BasicDecimal256> v1, v2;
+  for (uint64_t x = 0; x < kValueSize; x++) {
+    v1.push_back(BasicDecimal256({100 + x, 100 + x, 100 + x, 100 + x}));
+    v2.push_back(BasicDecimal256({200 + x, 200 + x, 200 + x, 200 + x}));
+  }
+
+  for (auto _ : state) {
+    for (int x = 0; x < kValueSize; x += 5) {
+      benchmark::DoNotOptimize(v1[x + 2] * v2[x + 2]);
+    }
+  }
+  state.SetItemsProcessed(state.iterations() * kValueSize);
+}
+
 static void UnaryOp(benchmark::State& state) {  // NOLINT non-const reference
   std::vector<BasicDecimal128> v;
   for (int x = 0; x < kValueSize; x++) {
@@ -158,7 +204,9 @@ static void BinaryBitOp(benchmark::State& state) {  // NOLINT non-const referenc
 }
 
 BENCHMARK(FromString);
-BENCHMARK(BinaryMathOp);
+BENCHMARK(ToString);
+BENCHMARK(BinaryMathOp128);
+BENCHMARK(BinaryMathOp256);
 BENCHMARK(BinaryMathOpAggregate);
 BENCHMARK(BinaryCompareOp);
 BENCHMARK(BinaryCompareOpConstant);
diff --git a/cpp/src/arrow/util/decimal_test.cc b/cpp/src/arrow/util/decimal_test.cc
index 856f10e626b..3c2f5335e74 100644
--- a/cpp/src/arrow/util/decimal_test.cc
+++ b/cpp/src/arrow/util/decimal_test.cc
@@ -32,12 +32,14 @@
 #include "arrow/testing/gtest_util.h"
 #include "arrow/testing/random.h"
 #include "arrow/util/decimal.h"
+#include "arrow/util/int128_internal.h"
 #include "arrow/util/macros.h"
 
-using boost::multiprecision::int128_t;
-
 namespace arrow {
 
+using internal::int128_t;
+using internal::uint128_t;
+
 class DecimalTestFixture : public ::testing::Test {
  public:
   DecimalTestFixture() : integer_value_(23423445), string_value_("234.23445") {}
@@ -45,13 +47,6 @@ class DecimalTestFixture : public ::testing::Test {
   std::string string_value_;
 };
 
-TEST_F(DecimalTestFixture, TestToString) {
-  Decimal128 decimal(this->integer_value_);
-  int32_t scale = 5;
-  std::string result = decimal.ToString(scale);
-  ASSERT_EQ(result, this->string_value_);
-}
-
 TEST_F(DecimalTestFixture, TestFromString) {
   Decimal128 expected(this->integer_value_);
   Decimal128 result;
@@ -106,6 +101,35 @@ TEST(DecimalTest, TestFromDecimalString128) {
   ASSERT_NE(result.high_bits(), 0);
 }
 
+TEST(DecimalTest, TestStringRoundTrip) {
+  static constexpr uint64_t kTestBits[] = {
+      0,
+      1,
+      999,
+      1000,
+      std::numeric_limits<int32_t>::max(),
+      (1ull << 31),
+      std::numeric_limits<uint32_t>::max(),
+      (1ull << 32),
+      std::numeric_limits<int64_t>::max(),
+      (1ull << 63),
+      std::numeric_limits<uint64_t>::max(),
+  };
+  static constexpr int32_t kScales[] = {0, 1, 10};
+  for (uint64_t high_bits : kTestBits) {
+    for (uint64_t low_bits : kTestBits) {
+      // When high_bits = 1ull << 63 or std::numeric_limits<uint64_t>::max(), decimal is
+      // negative.
+      Decimal128 decimal(high_bits, low_bits);
+      for (int32_t scale : kScales) {
+        std::string str = decimal.ToString(scale);
+        ASSERT_OK_AND_ASSIGN(Decimal128 result, Decimal128::FromString(str));
+        EXPECT_EQ(decimal, result);
+      }
+    }
+  }
+}
+
 TEST(DecimalTest, TestDecimal32SignedRoundTrip) {
   Decimal128 expected("-3402692");
 
@@ -291,28 +315,93 @@ TEST(Decimal128Test, PrintMinValue) {
   ASSERT_EQ(string_value, printed_value);
 }
 
-class Decimal128PrintingTest
-    : public ::testing::TestWithParam<std::tuple<int32_t, int32_t, std::string>> {};
-
-TEST_P(Decimal128PrintingTest, Print) {
-  int32_t test_value;
+struct ToStringTestParam {
+  int64_t test_value;
   int32_t scale;
   std::string expected_string;
-  std::tie(test_value, scale, expected_string) = GetParam();
-  const Decimal128 value(test_value);
-  const std::string printed_value = value.ToString(scale);
-  ASSERT_EQ(expected_string, printed_value);
-}
-
-INSTANTIATE_TEST_SUITE_P(Decimal128PrintingTest, Decimal128PrintingTest,
-                         ::testing::Values(std::make_tuple(123, 1, "12.3"),
-                                           std::make_tuple(123, 5, "0.00123"),
-                                           std::make_tuple(123, 10, "1.23E-8"),
-                                           std::make_tuple(123, -1, "1.23E+3"),
-                                           std::make_tuple(-123, -1, "-1.23E+3"),
-                                           std::make_tuple(123, -3, "1.23E+5"),
-                                           std::make_tuple(-123, -3, "-1.23E+5"),
-                                           std::make_tuple(12345, -3, "1.2345E+7")));
+
+  // Avoid Valgrind uninitialized memory reads with the default GTest print routine.
+  friend std::ostream& operator<<(std::ostream& os, const ToStringTestParam& param) {
+    return os << "<value: " << param.test_value << ">";
+  }
+};
+
+static const ToStringTestParam kToStringTestData[] = {
+    {0, -1, "0.E+1"},
+    {0, 0, "0"},
+    {0, 1, "0.0"},
+    {0, 6, "0.000000"},
+    {2, 7, "2.E-7"},
+    {2, -1, "2.E+1"},
+    {2, 0, "2"},
+    {2, 1, "0.2"},
+    {2, 6, "0.000002"},
+    {-2, 7, "-2.E-7"},
+    {-2, 7, "-2.E-7"},
+    {-2, -1, "-2.E+1"},
+    {-2, 0, "-2"},
+    {-2, 1, "-0.2"},
+    {-2, 6, "-0.000002"},
+    {-2, 7, "-2.E-7"},
+    {123, -3, "1.23E+5"},
+    {123, -1, "1.23E+3"},
+    {123, 1, "12.3"},
+    {123, 0, "123"},
+    {123, 5, "0.00123"},
+    {123, 8, "0.00000123"},
+    {123, 9, "1.23E-7"},
+    {123, 10, "1.23E-8"},
+    {-123, -3, "-1.23E+5"},
+    {-123, -1, "-1.23E+3"},
+    {-123, 1, "-12.3"},
+    {-123, 0, "-123"},
+    {-123, 5, "-0.00123"},
+    {-123, 8, "-0.00000123"},
+    {-123, 9, "-1.23E-7"},
+    {-123, 10, "-1.23E-8"},
+    {1000000000, -3, "1.000000000E+12"},
+    {1000000000, -1, "1.000000000E+10"},
+    {1000000000, 0, "1000000000"},
+    {1000000000, 1, "100000000.0"},
+    {1000000000, 5, "10000.00000"},
+    {1000000000, 15, "0.000001000000000"},
+    {1000000000, 16, "1.000000000E-7"},
+    {1000000000, 17, "1.000000000E-8"},
+    {-1000000000, -3, "-1.000000000E+12"},
+    {-1000000000, -1, "-1.000000000E+10"},
+    {-1000000000, 0, "-1000000000"},
+    {-1000000000, 1, "-100000000.0"},
+    {-1000000000, 5, "-10000.00000"},
+    {-1000000000, 15, "-0.000001000000000"},
+    {-1000000000, 16, "-1.000000000E-7"},
+    {-1000000000, 17, "-1.000000000E-8"},
+    {1234567890123456789LL, -3, "1.234567890123456789E+21"},
+    {1234567890123456789LL, -1, "1.234567890123456789E+19"},
+    {1234567890123456789LL, 0, "1234567890123456789"},
+    {1234567890123456789LL, 1, "123456789012345678.9"},
+    {1234567890123456789LL, 5, "12345678901234.56789"},
+    {1234567890123456789LL, 24, "0.000001234567890123456789"},
+    {1234567890123456789LL, 25, "1.234567890123456789E-7"},
+    {-1234567890123456789LL, -3, "-1.234567890123456789E+21"},
+    {-1234567890123456789LL, -1, "-1.234567890123456789E+19"},
+    {-1234567890123456789LL, 0, "-1234567890123456789"},
+    {-1234567890123456789LL, 1, "-123456789012345678.9"},
+    {-1234567890123456789LL, 5, "-12345678901234.56789"},
+    {-1234567890123456789LL, 24, "-0.000001234567890123456789"},
+    {-1234567890123456789LL, 25, "-1.234567890123456789E-7"},
+};
+
+class Decimal128ToStringTest : public ::testing::TestWithParam<ToStringTestParam> {};
+
+TEST_P(Decimal128ToStringTest, ToString) {
+  const ToStringTestParam& param = GetParam();
+  const Decimal128 value(param.test_value);
+  const std::string printed_value = value.ToString(param.scale);
+  ASSERT_EQ(param.expected_string, printed_value);
+}
+
+INSTANTIATE_TEST_SUITE_P(Decimal128ToStringTest, Decimal128ToStringTest,
+                         ::testing::ValuesIn(kToStringTestData));
 
 class Decimal128ParsingTest
     : public ::testing::TestWithParam<std::tuple<std::string, uint64_t, int32_t>> {};
@@ -382,8 +471,7 @@ struct FromRealTestParam {
   int32_t scale;
   std::string expected;
 
-  // Weird, but we need to define this to avoid Valgrind issues
-  // with the default GTest print routine.
+  // Avoid Valgrind uninitialized memory reads with the default GTest print routine.
   friend std::ostream& operator<<(std::ostream& os,
                                   const FromRealTestParam<Real>& param) {
     return os << "<real: " << param.real << ">";
@@ -706,15 +794,6 @@ TEST_F(TestDecimalToRealDouble, Precision) {
 
 #endif  // __MINGW32__
 
-TEST(Decimal128Test, TestSmallNumberFormat) {
-  Decimal128 value("0.2");
-  std::string expected("0.2");
-
-  const int32_t scale = 1;
-  std::string result = value.ToString(scale);
-  ASSERT_EQ(expected, result);
-}
-
 TEST(Decimal128Test, TestNoDecimalPointExponential) {
   Decimal128 value;
   int32_t precision;
@@ -825,6 +904,11 @@ std::vector<CType> GetRandomNumbers(int32_t size) {
   return ret;
 }
 
+Decimal128 Decimal128FromInt128(int128_t value) {
+  return Decimal128(static_cast<int64_t>(value >> 64),
+                    static_cast<uint64_t>(value & 0xFFFFFFFFFFFFFFFFULL));
+}
+
 TEST(Decimal128Test, Multiply) {
   ASSERT_EQ(Decimal128(60501), Decimal128(301) * Decimal128(201));
 
@@ -840,6 +924,14 @@ TEST(Decimal128Test, Multiply) {
       Decimal128 result = Decimal128(x) * Decimal128(y);
       ASSERT_EQ(Decimal128(static_cast<int64_t>(x) * y), result)
           << " x: " << x << " y: " << y;
+      // Test by multiplying with an additional 32 bit factor, then additional
+      // factor of 2^30 to test results in the range of -2^123 to 2^123 without overflow.
+      for (auto z : GetRandomNumbers<Int32Type>(32)) {
+        int128_t w = static_cast<int128_t>(x) * y * (1ull << 30);
+        Decimal128 expected = Decimal128FromInt128(static_cast<int128_t>(w) * z);
+        Decimal128 actual = Decimal128FromInt128(w) * Decimal128(z);
+        ASSERT_EQ(expected, actual) << " w: " << x << " * " << y << " * 2^30 z: " << z;
+      }
     }
   }
 
@@ -847,8 +939,11 @@ TEST(Decimal128Test, Multiply) {
   for (auto x : std::vector<int128_t>{-INT64_MAX, -INT32_MAX, 0, INT32_MAX, INT64_MAX}) {
     for (auto y :
          std::vector<int128_t>{-INT32_MAX, -32, -2, -1, 0, 1, 2, 32, INT32_MAX}) {
-      Decimal128 result = Decimal128(x.str()) * Decimal128(y.str());
-      ASSERT_EQ(Decimal128((x * y).str()), result) << " x: " << x << " y: " << y;
+      Decimal128 decimal_x = Decimal128FromInt128(x);
+      Decimal128 decimal_y = Decimal128FromInt128(y);
+      Decimal128 result = decimal_x * decimal_y;
+      EXPECT_EQ(Decimal128FromInt128(x * y), result)
+          << " x: " << decimal_x << " y: " << decimal_y;
     }
   }
 }
@@ -878,8 +973,11 @@ TEST(Decimal128Test, Divide) {
   // Test some edge cases
   for (auto x : std::vector<int128_t>{-INT64_MAX, -INT32_MAX, 0, INT32_MAX, INT64_MAX}) {
     for (auto y : std::vector<int128_t>{-INT32_MAX, -32, -2, -1, 1, 2, 32, INT32_MAX}) {
-      Decimal128 result = Decimal128(x.str()) * Decimal128(y.str());
-      ASSERT_EQ(Decimal128((x * y).str()), result) << " x: " << x << " y: " << y;
+      Decimal128 decimal_x = Decimal128FromInt128(x);
+      Decimal128 decimal_y = Decimal128FromInt128(y);
+      Decimal128 result = decimal_x / decimal_y;
+      EXPECT_EQ(Decimal128FromInt128(x / y), result)
+          << " x: " << decimal_x << " y: " << decimal_y;
     }
   }
 }
@@ -909,8 +1007,11 @@ TEST(Decimal128Test, Mod) {
   // Test some edge cases
   for (auto x : std::vector<int128_t>{-INT64_MAX, -INT32_MAX, 0, INT32_MAX, INT64_MAX}) {
     for (auto y : std::vector<int128_t>{-INT32_MAX, -32, -2, -1, 1, 2, 32, INT32_MAX}) {
-      Decimal128 result = Decimal128(x.str()) * Decimal128(y.str());
-      ASSERT_EQ(Decimal128((x * y).str()), result) << " x: " << x << " y: " << y;
+      Decimal128 decimal_x = Decimal128FromInt128(x);
+      Decimal128 decimal_y = Decimal128FromInt128(y);
+      Decimal128 result = decimal_x % decimal_y;
+      EXPECT_EQ(Decimal128FromInt128(x % y), result)
+          << " x: " << decimal_x << " y: " << decimal_y;
     }
   }
 }
@@ -1055,4 +1156,156 @@ TEST(Decimal128Test, FitsInPrecision) {
       Decimal128("-100000000000000000000000000000000000000").FitsInPrecision(38));
 }
 
+static constexpr std::array<uint64_t, 4> kSortedDecimal256Bits[] = {
+    {0, 0, 0, 0x8000000000000000ULL},  // min
+    {0xFFFFFFFFFFFFFFFEULL, 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL,
+     0xFFFFFFFFFFFFFFFFULL},  // -2
+    {0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL,
+     0xFFFFFFFFFFFFFFFFULL},  // -1
+    {0, 0, 0, 0},
+    {1, 0, 0, 0},
+    {2, 0, 0, 0},
+    {0xFFFFFFFFFFFFFFFFULL, 0, 0, 0},
+    {0, 1, 0, 0},
+    {0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL, 0, 0},
+    {0, 0, 1, 0},
+    {0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL, 0},
+    {0, 0, 0, 1},
+    {0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL,
+     0x7FFFFFFFFFFFFFFFULL},  // max
+};
+
+TEST(Decimal256Test, TestComparators) {
+  constexpr size_t num_values =
+      sizeof(kSortedDecimal256Bits) / sizeof(kSortedDecimal256Bits[0]);
+  for (size_t i = 0; i < num_values; ++i) {
+    Decimal256 left(kSortedDecimal256Bits[i]);
+    for (size_t j = 0; j < num_values; ++j) {
+      Decimal256 right(kSortedDecimal256Bits[j]);
+      EXPECT_EQ(i == j, left == right);
+      EXPECT_EQ(i != j, left != right);
+      EXPECT_EQ(i < j, left < right);
+      EXPECT_EQ(i > j, left > right);
+      EXPECT_EQ(i <= j, left <= right);
+      EXPECT_EQ(i >= j, left >= right);
+    }
+  }
+}
+
+TEST(Decimal256Test, TestToBytesRoundTrip) {
+  for (const std::array<uint64_t, 4>& bits : kSortedDecimal256Bits) {
+    Decimal256 decimal(bits);
+    EXPECT_EQ(decimal, Decimal256(decimal.ToBytes().data()));
+  }
+}
+
+template <typename T>
+class Decimal256Test : public ::testing::Test {
+ public:
+  Decimal256Test() {}
+};
+
+using Decimal256Types =
+    ::testing::Types<char, unsigned char, short, unsigned short,  // NOLINT
+                     int, unsigned int, long, unsigned long,      // NOLINT
+                     long long, unsigned long long                // NOLINT
+                     >;
+
+TYPED_TEST_SUITE(Decimal256Test, Decimal256Types);
+
+TYPED_TEST(Decimal256Test, ConstructibleFromAnyIntegerType) {
+  using UInt64Array = std::array<uint64_t, 4>;
+  Decimal256 value(TypeParam{42});
+  EXPECT_EQ(UInt64Array({42, 0, 0, 0}), value.little_endian_array());
+
+  TypeParam max = std::numeric_limits<TypeParam>::max();
+  Decimal256 max_value(max);
+  EXPECT_EQ(UInt64Array({static_cast<uint64_t>(max), 0, 0, 0}),
+            max_value.little_endian_array());
+
+  TypeParam min = std::numeric_limits<TypeParam>::min();
+  Decimal256 min_value(min);
+  uint64_t high_bits = std::is_signed<TypeParam>::value ? ~uint64_t{0} : uint64_t{0};
+  EXPECT_EQ(UInt64Array({static_cast<uint64_t>(min), high_bits, high_bits, high_bits}),
+            min_value.little_endian_array());
+}
+
+TEST(Decimal256Test, ConstructibleFromBool) {
+  EXPECT_EQ(Decimal256(0), Decimal256(false));
+  EXPECT_EQ(Decimal256(1), Decimal256(true));
+}
+
+Decimal256 Decimal256FromInt128(int128_t value) {
+  return Decimal256(Decimal128(static_cast<int64_t>(value >> 64),
+                               static_cast<uint64_t>(value & 0xFFFFFFFFFFFFFFFFULL)));
+}
+
+TEST(Decimal256Test, Multiply) {
+  using boost::multiprecision::int256_t;
+  using boost::multiprecision::uint256_t;
+
+  ASSERT_EQ(Decimal256(60501), Decimal256(301) * Decimal256(201));
+
+  ASSERT_EQ(Decimal256(-60501), Decimal256(-301) * Decimal256(201));
+
+  ASSERT_EQ(Decimal256(-60501), Decimal256(301) * Decimal256(-201));
+
+  ASSERT_EQ(Decimal256(60501), Decimal256(-301) * Decimal256(-201));
+
+  // Test some random numbers.
+  std::vector<int128_t> left;
+  std::vector<int128_t> right;
+  for (auto x : GetRandomNumbers<Int32Type>(16)) {
+    for (auto y : GetRandomNumbers<Int32Type>(16)) {
+      for (auto z : GetRandomNumbers<Int32Type>(16)) {
+        for (auto w : GetRandomNumbers<Int32Type>(16)) {
+          // Test two 128 bit numbers which have a large amount of bits set.
+          int128_t l = static_cast<uint128_t>(x) << 96 | static_cast<uint128_t>(y) << 64 |
+                       static_cast<uint128_t>(z) << 32 | static_cast<uint128_t>(w);
+          int128_t r = static_cast<uint128_t>(w) << 96 | static_cast<uint128_t>(z) << 64 |
+                       static_cast<uint128_t>(y) << 32 | static_cast<uint128_t>(x);
+          int256_t expected = int256_t(l) * r;
+          Decimal256 actual = Decimal256FromInt128(l) * Decimal256FromInt128(r);
+          ASSERT_EQ(expected.str(), actual.ToIntegerString())
+              << " " << int256_t(l).str() << " * " << int256_t(r).str();
+          // Test a 96 bit number against a 160 bit number.
+          int128_t s = l >> 32;
+          uint256_t b = uint256_t(r) << 32;
+          Decimal256 b_dec =
+              Decimal256FromInt128(r) * Decimal256(static_cast<uint64_t>(1) << 32);
+          ASSERT_EQ(b.str(), b_dec.ToIntegerString()) << int256_t(r).str();
+          expected = int256_t(s) * b;
+          actual = Decimal256FromInt128(s) * b_dec;
+          ASSERT_EQ(expected.str(), actual.ToIntegerString())
+              << " " << int256_t(s).str() << " * " << int256_t(b).str();
+        }
+      }
+    }
+  }
+
+  // Test some edge cases
+  for (auto x : std::vector<int128_t>{-INT64_MAX, -INT32_MAX, 0, INT32_MAX, INT64_MAX}) {
+    for (auto y :
+         std::vector<int128_t>{-INT32_MAX, -32, -2, -1, 0, 1, 2, 32, INT32_MAX}) {
+      Decimal256 decimal_x = Decimal256FromInt128(x);
+      Decimal256 decimal_y = Decimal256FromInt128(y);
+      Decimal256 result = decimal_x * decimal_y;
+      EXPECT_EQ(Decimal256FromInt128(x * y), result)
+          << " x: " << decimal_x << " y: " << decimal_y;
+    }
+  }
+}
+
+class Decimal256ToStringTest : public ::testing::TestWithParam<ToStringTestParam> {};
+
+TEST_P(Decimal256ToStringTest, ToString) {
+  const ToStringTestParam& data = GetParam();
+  const Decimal256 value(data.test_value);
+  const std::string printed_value = value.ToString(data.scale);
+  ASSERT_EQ(data.expected_string, printed_value);
+}
+
+INSTANTIATE_TEST_SUITE_P(Decimal256ToStringTest, Decimal256ToStringTest,
+                         ::testing::ValuesIn(kToStringTestData));
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/dispatch.h b/cpp/src/arrow/util/dispatch.h
new file mode 100644
index 00000000000..fae9293f9e7
--- /dev/null
+++ b/cpp/src/arrow/util/dispatch.h
@@ -0,0 +1,115 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <utility>
+#include <vector>
+
+#include "arrow/status.h"
+#include "arrow/util/cpu_info.h"
+
+namespace arrow {
+namespace internal {
+
+enum class DispatchLevel : int {
+  // These dispatch levels, corresponding to instruction set features,
+  // are sorted in increasing order of preference.
+  NONE = 0,
+  SSE4_2,
+  AVX2,
+  AVX512,
+  NEON,
+  MAX
+};
+
+/*
+  A facility for dynamic dispatch according to available DispatchLevel.
+
+  Typical use:
+
+    static void my_function_default(...);
+    static void my_function_avx2(...);
+
+    struct MyDynamicFunction {
+      using FunctionType = decltype(&my_function_default);
+
+      static std::vector<std::pair<DispatchLevel, FunctionType>> implementations() {
+        return {
+          { DispatchLevel::NONE, my_function_default }
+    #if defined(ARROW_HAVE_RUNTIME_AVX2)
+          , { DispatchLevel::AVX2, my_function_avx2 }
+    #endif
+        };
+      }
+    };
+
+    void my_function(...) {
+      static DynamicDispatch<MyDynamicFunction> dispatch;
+      return dispatch.func(...);
+    }
+*/
+template <typename DynamicFunction>
+class DynamicDispatch {
+ protected:
+  using FunctionType = typename DynamicFunction::FunctionType;
+  using Implementation = std::pair<DispatchLevel, FunctionType>;
+
+ public:
+  DynamicDispatch() { Resolve(DynamicFunction::implementations()); }
+
+  FunctionType func = {};
+
+ protected:
+  // Use the Implementation with the highest DispatchLevel
+  void Resolve(const std::vector<Implementation>& implementations) {
+    Implementation cur{DispatchLevel::NONE, {}};
+
+    for (const auto& impl : implementations) {
+      if (impl.first >= cur.first && IsSupported(impl.first)) {
+        // Higher (or same) level than current
+        cur = impl;
+      }
+    }
+
+    if (!cur.second) {
+      Status::Invalid("No appropriate implementation found").Abort();
+    }
+    func = cur.second;
+  }
+
+ private:
+  bool IsSupported(DispatchLevel level) const {
+    static const auto cpu_info = arrow::internal::CpuInfo::GetInstance();
+
+    switch (level) {
+      case DispatchLevel::NONE:
+        return true;
+      case DispatchLevel::SSE4_2:
+        return cpu_info->IsSupported(CpuInfo::SSE4_2);
+      case DispatchLevel::AVX2:
+        return cpu_info->IsSupported(CpuInfo::AVX2);
+      case DispatchLevel::AVX512:
+        return cpu_info->IsSupported(CpuInfo::AVX512);
+      default:
+        return false;
+    }
+  }
+};
+
+}  // namespace internal
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/future.h b/cpp/src/arrow/util/future.h
index 19fd632c065..575f5cb3c41 100644
--- a/cpp/src/arrow/util/future.h
+++ b/cpp/src/arrow/util/future.h
@@ -154,9 +154,8 @@ class FutureStorage : public FutureStorageBase {
 
   Status status() const { return result_.status(); }
 
-  template <typename U>
-  void MarkFinished(U&& value) {
-    result_ = std::forward<U>(value);
+  void MarkFinished(Result<T> result) {
+    result_ = std::move(result);
     if (ARROW_PREDICT_TRUE(result_.ok())) {
       impl_->MarkFinished();
     } else {
@@ -250,7 +249,7 @@ class Future {
   // The default constructor creates an invalid Future.  Use Future::Make()
   // for a valid Future.  This constructor is mostly for the convenience
   // of being able to presize a vector of Futures.
-  Future() : impl_(NULLPTR) {}
+  Future() = default;
 
   // Consumer API
 
@@ -311,6 +310,15 @@ class Future {
     return impl_->Wait(seconds);
   }
 
+  /// If a Result<Future> holds an error instead of a Future, construct a finished Future
+  /// holding that error.
+  static Future DeferNotOk(Result<Future> maybe_future) {
+    if (ARROW_PREDICT_FALSE(!maybe_future.ok())) {
+      return MakeFinished(std::move(maybe_future).status());
+    }
+    return std::move(maybe_future).MoveValueUnsafe();
+  }
+
   // Producer API
 
   /// \brief Producer API: execute function and mark Future finished
@@ -368,7 +376,7 @@ class Future {
   }
 
   std::shared_ptr<FutureStorage<T>> storage_;
-  FutureImpl* impl_;
+  FutureImpl* impl_ = NULLPTR;
 
   friend class FutureWaiter;
 };
@@ -419,15 +427,4 @@ inline std::vector<int> WaitForAny(const std::vector<Future<T>*>& futures,
   return waiter->MoveFinishedFutures();
 }
 
-#define ARROW_ASSIGN_OR_RETURN_FUTURE_IMPL(result_name, lhs, T, rexpr) \
-  auto result_name = (rexpr);                                          \
-  if (ARROW_PREDICT_FALSE(!(result_name).ok())) {                      \
-    return Future<T>::MakeFinished(std::move(result_name).status());   \
-  }                                                                    \
-  lhs = std::move(result_name).MoveValueUnsafe();
-
-#define ARROW_ASSIGN_OR_RETURN_FUTURE(lhs, T, rexpr) \
-  ARROW_ASSIGN_OR_RETURN_FUTURE_IMPL(                \
-      ARROW_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, T, rexpr);
-
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/hashing.h b/cpp/src/arrow/util/hashing.h
index ba644dceabb..00fb745f529 100644
--- a/cpp/src/arrow/util/hashing.h
+++ b/cpp/src/arrow/util/hashing.h
@@ -44,18 +44,9 @@
 #include "arrow/util/ubsan.h"
 
 #define XXH_INLINE_ALL
-#define XXH_PRIVATE_API
-#define XXH_NAMESPACE arrow_hashing_
 
 #include "arrow/vendored/xxhash.h"  // IWYU pragma: keep
 
-// ARROW-9415: See https://github.com/Cyan4973/xxHash/pull/426. altivec.h on
-// gcc leaks the "bool" define which causes a compilation failure on Power9
-// architecture.
-#if XXH_VECTOR == XXH_VSX  // altivec
-#undef bool
-#endif
-
 namespace arrow {
 namespace internal {
 
@@ -851,6 +842,11 @@ struct HashTraits<T, enable_if_t<has_string_view<T>::value &&
   using MemoTableType = BinaryMemoTable<BinaryBuilder>;
 };
 
+template <typename T>
+struct HashTraits<T, enable_if_decimal<T>> {
+  using MemoTableType = BinaryMemoTable<BinaryBuilder>;
+};
+
 template <typename T>
 struct HashTraits<T, enable_if_t<std::is_base_of<LargeBinaryType, T>::value>> {
   using MemoTableType = BinaryMemoTable<LargeBinaryBuilder>;
diff --git a/cpp/src/arrow/util/int128_internal.h b/cpp/src/arrow/util/int128_internal.h
new file mode 100644
index 00000000000..1d7c89806aa
--- /dev/null
+++ b/cpp/src/arrow/util/int128_internal.h
@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include "arrow/util/macros.h"
+
+// NOTE: Avoid native int128_t on clang with UBSan as it produces linker errors
+// (such as "undefined reference to '__muloti4'")
+#if defined(__SIZEOF_INT128__) && !defined(ARROW_UBSAN)
+#define ARROW_USE_NATIVE_INT128
+#else
+#include <boost/multiprecision/cpp_int.hpp>
+#endif
+
+namespace arrow {
+namespace internal {
+
+// NOTE: __int128_t and boost::multiprecision::int128_t are not interchangeable.
+// For example, __int128_t does not have any member function, and does not have
+// operator<<(std::ostream, __int128_t). On the other hand, the behavior of
+// boost::multiprecision::int128_t might be surprising with some configs (e.g.,
+// static_cast<uint64_t>(boost::multiprecision::uint128_t) might return
+// ~uint64_t{0} instead of the lower 64 bits of the input).
+// Try to minimize the usage of int128_t and uint128_t.
+#ifdef ARROW_USE_NATIVE_INT128
+using int128_t = __int128_t;
+using uint128_t = __uint128_t;
+#else
+using boost::multiprecision::int128_t;
+using boost::multiprecision::uint128_t;
+#endif
+
+}  // namespace internal
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/int_util_internal.h b/cpp/src/arrow/util/int_util_internal.h
index 5de92a99ec5..de39229cfdd 100644
--- a/cpp/src/arrow/util/int_util_internal.h
+++ b/cpp/src/arrow/util/int_util_internal.h
@@ -58,6 +58,7 @@ namespace internal {
 OPS_WITH_OVERFLOW(AddWithOverflow, add)
 OPS_WITH_OVERFLOW(SubtractWithOverflow, sub)
 OPS_WITH_OVERFLOW(MultiplyWithOverflow, mul)
+OPS_WITH_OVERFLOW(DivideWithOverflow, div)
 
 #undef OP_WITH_OVERFLOW
 #undef OPS_WITH_OVERFLOW
diff --git a/cpp/src/arrow/util/io_util.cc b/cpp/src/arrow/util/io_util.cc
index f2e4adefd35..5d36ba91d4a 100644
--- a/cpp/src/arrow/util/io_util.cc
+++ b/cpp/src/arrow/util/io_util.cc
@@ -41,15 +41,6 @@
 #include <sys/stat.h>
 #include <sys/types.h>  // IWYU pragma: keep
 
-// Defines that don't exist in MinGW
-#if defined(__MINGW32__)
-#define ARROW_WRITE_SHMODE S_IRUSR | S_IWUSR
-#elif defined(_MSC_VER)  // Visual Studio
-
-#else  // gcc / clang on POSIX platforms
-#define ARROW_WRITE_SHMODE S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH
-#endif
-
 // ----------------------------------------------------------------------
 // file compatibility stuff
 
@@ -927,7 +918,7 @@ Result<int> FileOpenWritable(const PlatformFilename& file_name, bool write_only,
     oflag |= O_RDWR;
   }
 
-  fd = open(file_name.ToNative().c_str(), oflag, ARROW_WRITE_SHMODE);
+  fd = open(file_name.ToNative().c_str(), oflag, 0666);
   errno_actual = errno;
 #endif
 
@@ -1476,14 +1467,8 @@ std::vector<NativePathString> GetPlatformTemporaryDirs() {
 
 std::string MakeRandomName(int num_chars) {
   static const std::string chars = "0123456789abcdefghijklmnopqrstuvwxyz";
-#ifdef ARROW_VALGRIND
-  // Valgrind can crash, hang or enter an infinite loop on std::random_device,
-  // use a PRNG instead.
-  static std::random_device::result_type seed = 42;
-  std::default_random_engine gen(seed++);
-#else
-  std::random_device gen;
-#endif
+  std::default_random_engine gen(
+      static_cast<std::default_random_engine::result_type>(GetRandomSeed()));
   std::uniform_int_distribution<int> dist(0, static_cast<int>(chars.length() - 1));
 
   std::string s;
@@ -1493,6 +1478,7 @@ std::string MakeRandomName(int num_chars) {
   }
   return s;
 }
+
 }  // namespace
 
 Result<std::unique_ptr<TemporaryDir>> TemporaryDir::Make(const std::string& prefix) {
@@ -1602,5 +1588,34 @@ Result<SignalHandler> SetSignalHandler(int signum, const SignalHandler& handler)
   return Status::OK();
 }
 
+namespace {
+
+std::mt19937_64 GetSeedGenerator() {
+  // Initialize Mersenne Twister PRNG with a true random seed.
+#ifdef ARROW_VALGRIND
+  // Valgrind can crash, hang or enter an infinite loop on std::random_device,
+  // use a crude initializer instead.
+  // Make sure to mix in process id to avoid clashes when parallel testing.
+  const uint8_t dummy = 0;
+  ARROW_UNUSED(dummy);
+  std::mt19937_64 seed_gen(reinterpret_cast<uintptr_t>(&dummy) ^
+                           static_cast<uintptr_t>(getpid()));
+#else
+  std::random_device true_random;
+  std::mt19937_64 seed_gen(static_cast<uint64_t>(true_random()) ^
+                           (static_cast<uint64_t>(true_random()) << 32));
+#endif
+  return seed_gen;
+}
+
+}  // namespace
+
+int64_t GetRandomSeed() {
+  // The process-global seed generator to aims to avoid calling std::random_device
+  // unless truly necessary (it can block on some systems, see ARROW-10287).
+  static auto seed_gen = GetSeedGenerator();
+  return static_cast<int64_t>(seed_gen());
+}
+
 }  // namespace internal
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/io_util.h b/cpp/src/arrow/util/io_util.h
index baa69a4bcfc..541041a6acd 100644
--- a/cpp/src/arrow/util/io_util.h
+++ b/cpp/src/arrow/util/io_util.h
@@ -31,6 +31,7 @@
 #endif
 
 #include "arrow/io/interfaces.h"
+#include "arrow/status.h"
 #include "arrow/type_fwd.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/windows_fixup.h"
@@ -346,11 +347,21 @@ class ARROW_EXPORT SignalHandler {
 /// \brief Return the current handler for the given signal number.
 ARROW_EXPORT
 Result<SignalHandler> GetSignalHandler(int signum);
+
 /// \brief Set a new handler for the given signal number.
 ///
 /// The old signal handler is returned.
 ARROW_EXPORT
 Result<SignalHandler> SetSignalHandler(int signum, const SignalHandler& handler);
 
+/// \brief Get an unpredictable random seed
+///
+/// This function may be slightly costly, so should only be used to initialize
+/// a PRNG, not to generate a large amount of random numbers.
+/// It is better to use this function rather than std::random_device, unless
+/// absolutely necessary (e.g. to generate a cryptographic secret).
+ARROW_EXPORT
+int64_t GetRandomSeed();
+
 }  // namespace internal
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/io_util_test.cc b/cpp/src/arrow/util/io_util_test.cc
index 74ee16dc690..d84a2b76e39 100644
--- a/cpp/src/arrow/util/io_util_test.cc
+++ b/cpp/src/arrow/util/io_util_test.cc
@@ -1,4 +1,5 @@
 // Licensed to the Apache Software Foundation (ASF) under one
+// std::unique_ptr<TemporaryDir> temp_dir;
 // or more contributor license agreements.  See the NOTICE file
 // distributed with this work for additional information
 // regarding copyright ownership.  The ASF licenses this file
@@ -332,7 +333,10 @@ TEST(PlatformFilename, Parent) {
 }
 
 TEST(CreateDirDeleteDir, Basics) {
-  const std::string BASE = "xxx-io-util-test-dir";
+  std::unique_ptr<TemporaryDir> temp_dir;
+  ASSERT_OK_AND_ASSIGN(temp_dir, TemporaryDir::Make("deletedirtest-"));
+  const std::string BASE =
+      temp_dir->path().Join("xxx-io-util-test-dir2").ValueOrDie().ToString();
   bool created, deleted;
   PlatformFilename parent, child;
 
@@ -378,7 +382,10 @@ TEST(CreateDirDeleteDir, Basics) {
 }
 
 TEST(DeleteDirContents, Basics) {
-  const std::string BASE = "xxx-io-util-test-dir2";
+  std::unique_ptr<TemporaryDir> temp_dir;
+  ASSERT_OK_AND_ASSIGN(temp_dir, TemporaryDir::Make("deletedirtest-"));
+  const std::string BASE =
+      temp_dir->path().Join("xxx-io-util-test-dir2").ValueOrDie().ToString();
   bool created, deleted;
   PlatformFilename parent, child1, child2;
 
diff --git a/cpp/src/arrow/util/iterator.h b/cpp/src/arrow/util/iterator.h
index 795b8d370af..58dda5df2a7 100644
--- a/cpp/src/arrow/util/iterator.h
+++ b/cpp/src/arrow/util/iterator.h
@@ -154,7 +154,7 @@ class Iterator : public util::EqualityComparable<Iterator<T>> {
   Result<std::vector<T>> ToVector() {
     std::vector<T> out;
     for (auto maybe_element : *this) {
-      ARROW_ASSIGN_OR_RAISE(auto element, std::move(maybe_element));
+      ARROW_ASSIGN_OR_RAISE(auto element, maybe_element);
       out.push_back(std::move(element));
     }
     // ARROW-8193: On gcc-4.8 without the explicit move it tries to use the
@@ -489,7 +489,7 @@ class ReadaheadIterator {
 
     ARROW_RETURN_NOT_OK(queue_->Append(MakePromise()));
 
-    ARROW_ASSIGN_OR_RAISE(auto out, std::move(it_promise->out_));
+    ARROW_ASSIGN_OR_RAISE(auto out, it_promise->out_);
     if (out == IterationTraits<T>::End()) {
       done_ = true;
     }
diff --git a/cpp/src/arrow/util/iterator_test.cc b/cpp/src/arrow/util/iterator_test.cc
index 4449104562c..7295627b7c8 100644
--- a/cpp/src/arrow/util/iterator_test.cc
+++ b/cpp/src/arrow/util/iterator_test.cc
@@ -208,7 +208,7 @@ TEST(TestVectorIterator, RangeForLoop) {
   // also works with move only types
   ints_it = ints.begin();
   for (auto maybe_i_ptr : MakeVectorIterator(std::move(intptrs))) {
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TestInt> i_ptr, std::move(maybe_i_ptr));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TestInt> i_ptr, maybe_i_ptr);
     ASSERT_EQ(*i_ptr, *ints_it++);
   }
   ASSERT_EQ(ints_it, ints.end());
diff --git a/cpp/src/arrow/util/key_value_metadata.h b/cpp/src/arrow/util/key_value_metadata.h
index 2f6452256e3..d4207a53dc4 100644
--- a/cpp/src/arrow/util/key_value_metadata.h
+++ b/cpp/src/arrow/util/key_value_metadata.h
@@ -51,10 +51,13 @@ class ARROW_EXPORT KeyValueMetadata {
   Status Set(const std::string& key, const std::string& value);
 
   void reserve(int64_t n);
-  int64_t size() const;
 
+  int64_t size() const;
   const std::string& key(int64_t i) const;
   const std::string& value(int64_t i) const;
+  const std::vector<std::string>& keys() const { return keys_; }
+  const std::vector<std::string>& values() const { return values_; }
+
   std::vector<std::pair<std::string, std::string>> sorted_pairs() const;
 
   /// \brief Perform linear search for key, returning -1 if not found
diff --git a/cpp/src/arrow/util/map.h b/cpp/src/arrow/util/map.h
new file mode 100644
index 00000000000..5523909061d
--- /dev/null
+++ b/cpp/src/arrow/util/map.h
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <utility>
+
+#include "arrow/result.h"
+
+namespace arrow {
+namespace internal {
+
+/// Helper providing single-lookup conditional insertion into std::map or
+/// std::unordered_map. If `key` exists in the container, an iterator to that pair
+/// will be returned. If `key` does not exist in the container, `gen(key)` will be
+/// invoked and its return value inserted.
+template <typename Map, typename Gen>
+auto GetOrInsertGenerated(Map* map, typename Map::key_type key, Gen&& gen)
+    -> decltype(map->begin()->second = gen(map->begin()->first), map->begin()) {
+  decltype(gen(map->begin()->first)) placeholder{};
+
+  auto it_success = map->emplace(std::move(key), std::move(placeholder));
+  if (it_success.second) {
+    // insertion of placeholder succeeded, overwrite it with gen()
+    const auto& inserted_key = it_success.first->first;
+    auto* value = &it_success.first->second;
+    *value = gen(inserted_key);
+  }
+  return it_success.first;
+}
+
+template <typename Map, typename Gen>
+auto GetOrInsertGenerated(Map* map, typename Map::key_type key, Gen&& gen)
+    -> Result<decltype(map->begin()->second = gen(map->begin()->first).ValueOrDie(),
+                       map->begin())> {
+  decltype(gen(map->begin()->first).ValueOrDie()) placeholder{};
+
+  auto it_success = map->emplace(std::move(key), std::move(placeholder));
+  if (it_success.second) {
+    // insertion of placeholder succeeded, overwrite it with gen()
+    const auto& inserted_key = it_success.first->first;
+    auto* value = &it_success.first->second;
+    ARROW_ASSIGN_OR_RAISE(*value, gen(inserted_key));
+  }
+  return it_success.first;
+}
+
+}  // namespace internal
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/mutex.cc b/cpp/src/arrow/util/mutex.cc
index fa900fd164b..7456d7889d8 100644
--- a/cpp/src/arrow/util/mutex.cc
+++ b/cpp/src/arrow/util/mutex.cc
@@ -19,6 +19,8 @@
 
 #include <mutex>
 
+#include "arrow/util/logging.h"
+
 namespace arrow {
 namespace util {
 
@@ -27,9 +29,13 @@ struct Mutex::Impl {
 };
 
 Mutex::Guard::Guard(Mutex* locked)
-    : locked_(locked, [](Mutex* locked) { locked->impl_->mutex_.unlock(); }) {}
+    : locked_(locked, [](Mutex* locked) {
+        DCHECK(!locked->impl_->mutex_.try_lock());
+        locked->impl_->mutex_.unlock();
+      }) {}
 
 Mutex::Guard Mutex::TryLock() {
+  DCHECK_NE(impl_, nullptr);
   if (impl_->mutex_.try_lock()) {
     return Guard{this};
   }
@@ -37,6 +43,7 @@ Mutex::Guard Mutex::TryLock() {
 }
 
 Mutex::Guard Mutex::Lock() {
+  DCHECK_NE(impl_, nullptr);
   impl_->mutex_.lock();
   return Guard{this};
 }
diff --git a/cpp/src/arrow/util/mutex.h b/cpp/src/arrow/util/mutex.h
index a0365b710fb..f4fc64181fb 100644
--- a/cpp/src/arrow/util/mutex.h
+++ b/cpp/src/arrow/util/mutex.h
@@ -31,9 +31,11 @@ namespace util {
 class ARROW_EXPORT Mutex {
  public:
   Mutex();
+  Mutex(Mutex&&) = default;
+  Mutex& operator=(Mutex&&) = default;
 
   /// A Guard is falsy if a lock could not be acquired.
-  class Guard {
+  class ARROW_EXPORT Guard {
    public:
     Guard() : locked_(NULLPTR, [](Mutex* mutex) {}) {}
     Guard(Guard&&) = default;
@@ -41,6 +43,8 @@ class ARROW_EXPORT Mutex {
 
     explicit operator bool() const { return bool(locked_); }
 
+    void Unlock() { locked_.reset(); }
+
    private:
     explicit Guard(Mutex* locked);
 
diff --git a/cpp/src/arrow/util/rle_encoding_test.cc b/cpp/src/arrow/util/rle_encoding_test.cc
index 449956520d1..362f8253ccd 100644
--- a/cpp/src/arrow/util/rle_encoding_test.cc
+++ b/cpp/src/arrow/util/rle_encoding_test.cc
@@ -30,6 +30,7 @@
 #include "arrow/type.h"
 #include "arrow/util/bit_stream_utils.h"
 #include "arrow/util/bit_util.h"
+#include "arrow/util/io_util.h"
 #include "arrow/util/rle_encoding.h"
 
 namespace arrow {
@@ -418,14 +419,13 @@ TEST(BitRle, Random) {
   std::vector<int> values(ngroups + max_group_size);
 
   // prng setup
-  std::random_device rd;
+  const auto seed = ::arrow::internal::GetRandomSeed();
+  std::default_random_engine gen(
+      static_cast<std::default_random_engine::result_type>(seed));
   std::uniform_int_distribution<int> dist(1, 20);
 
   for (int iter = 0; iter < niters; ++iter) {
     // generate a seed with device entropy
-    uint32_t seed = rd();
-    std::mt19937 gen(seed);
-
     bool parity = 0;
     values.resize(0);
 
diff --git a/cpp/src/arrow/util/simd.h b/cpp/src/arrow/util/simd.h
index 84c93a825cf..259641dd456 100644
--- a/cpp/src/arrow/util/simd.h
+++ b/cpp/src/arrow/util/simd.h
@@ -29,6 +29,10 @@
 #else
 // gcc/clang (possibly others)
 
+#if defined(ARROW_HAVE_BMI2)
+#include <x86intrin.h>
+#endif
+
 #if defined(ARROW_HAVE_AVX2) || defined(ARROW_HAVE_AVX512)
 #include <immintrin.h>
 #elif defined(ARROW_HAVE_SSE4_2)
diff --git a/cpp/src/arrow/util/spaced.h b/cpp/src/arrow/util/spaced.h
index 710775811d7..f2f369adb63 100644
--- a/cpp/src/arrow/util/spaced.h
+++ b/cpp/src/arrow/util/spaced.h
@@ -161,7 +161,7 @@ inline int SpacedExpand(T* buffer, int num_values, int null_count,
       idx_buffer -= kBatchSize;
       idx_decode -= current_block.popcount;
 
-      // Foward scan and pack the target data to temp
+      // Forward scan and pack the target data to temp
       int idx = idx_decode + 1;
       for (uint64_t i = 0; i < kBatchSize; i++) {
         if (valid_bits_reader.IsSet()) {
diff --git a/cpp/src/arrow/util/string.cc b/cpp/src/arrow/util/string.cc
index 4ee0d0fe4a5..691f10e2793 100644
--- a/cpp/src/arrow/util/string.cc
+++ b/cpp/src/arrow/util/string.cc
@@ -97,7 +97,7 @@ std::string JoinStrings(const std::vector<util::string_view>& strings,
   if (strings.size() == 0) {
     return "";
   }
-  std::string out = strings.front().to_string();
+  std::string out = std::string(strings.front());
   for (size_t i = 1; i < strings.size(); ++i) {
     out.append(delimiter.begin(), delimiter.end());
     out.append(strings[i].begin(), strings[i].end());
@@ -144,5 +144,23 @@ std::string AsciiToLower(util::string_view value) {
   return result;
 }
 
+std::string AsciiToUpper(util::string_view value) {
+  // TODO: ASCII validation
+  std::string result = std::string(value);
+  std::transform(result.begin(), result.end(), result.begin(),
+                 [](unsigned char c) { return std::toupper(c); });
+  return result;
+}
+
+util::optional<std::string> Replace(util::string_view s, util::string_view token,
+                                    util::string_view replacement) {
+  size_t token_start = s.find(token);
+  if (token_start == std::string::npos) {
+    return util::nullopt;
+  }
+  return s.substr(0, token_start).to_string() + replacement.to_string() +
+         s.substr(token_start + token.size()).to_string();
+}
+
 }  // namespace internal
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/string.h b/cpp/src/arrow/util/string.h
index ea445c71ef4..feb75d45b35 100644
--- a/cpp/src/arrow/util/string.h
+++ b/cpp/src/arrow/util/string.h
@@ -20,6 +20,7 @@
 #include <string>
 #include <vector>
 
+#include "arrow/util/optional.h"
 #include "arrow/util/string_view.h"
 #include "arrow/util/visibility.h"
 
@@ -56,5 +57,14 @@ bool AsciiEqualsCaseInsensitive(util::string_view left, util::string_view right)
 ARROW_EXPORT
 std::string AsciiToLower(util::string_view value);
 
+ARROW_EXPORT
+std::string AsciiToUpper(util::string_view value);
+
+/// \brief Search for the first instance of a token and replace it or return nullopt if
+/// the token is not found.
+ARROW_EXPORT
+util::optional<std::string> Replace(util::string_view s, util::string_view token,
+                                    util::string_view replacement);
+
 }  // namespace internal
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/string_test.cc b/cpp/src/arrow/util/string_test.cc
index c56525732b0..d9b3749fdbd 100644
--- a/cpp/src/arrow/util/string_test.cc
+++ b/cpp/src/arrow/util/string_test.cc
@@ -88,5 +88,19 @@ TEST(ParseHexValue, Invalid) {
   ASSERT_RAISES(Invalid, ParseHexValue(input.c_str(), &output));
 }
 
+TEST(Replace, Basics) {
+  auto s = Replace("dat_{i}.txt", "{i}", "23");
+  EXPECT_TRUE(s);
+  EXPECT_EQ(*s, "dat_23.txt");
+
+  // only replace the first occurrence of token
+  s = Replace("dat_{i}_{i}.txt", "{i}", "23");
+  EXPECT_TRUE(s);
+  EXPECT_EQ(*s, "dat_23_{i}.txt");
+
+  s = Replace("dat_.txt", "{nope}", "23");
+  EXPECT_FALSE(s);
+}
+
 }  // namespace internal
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/thread_pool.h b/cpp/src/arrow/util/thread_pool.h
index fa6583e7371..45dc1ca7145 100644
--- a/cpp/src/arrow/util/thread_pool.h
+++ b/cpp/src/arrow/util/thread_pool.h
@@ -143,10 +143,8 @@ class ARROW_EXPORT Executor {
       typename RT = typename detail::ExecutorResultTraits<FunctionRetType>,
       typename ValueType = typename RT::ValueType>
   Future<ValueType> SubmitAsFuture(Function&& func, Args&&... args) {
-    ARROW_ASSIGN_OR_RETURN_FUTURE(
-        auto future, ValueType,
+    return Future<ValueType>::DeferNotOk(
         Submit(std::forward<Function>(func), std::forward<Args>(args)...));
-    return future;
   }
 
   // Return the level of parallelism (the number of tasks that may be executed
diff --git a/cpp/src/arrow/util/thread_pool_test.cc b/cpp/src/arrow/util/thread_pool_test.cc
index ac0eae78165..a54880b21b1 100644
--- a/cpp/src/arrow/util/thread_pool_test.cc
+++ b/cpp/src/arrow/util/thread_pool_test.cc
@@ -126,7 +126,7 @@ class AddTester {
 
 class TestThreadPool : public ::testing::Test {
  public:
-  void TearDown() {
+  void TearDown() override {
     fflush(stdout);
     fflush(stderr);
   }
diff --git a/cpp/src/arrow/util/uri.cc b/cpp/src/arrow/util/uri.cc
index b116a136277..1261607b6c1 100644
--- a/cpp/src/arrow/util/uri.cc
+++ b/cpp/src/arrow/util/uri.cc
@@ -49,12 +49,21 @@ std::string TextRangeToString(const UriTextRangeStructA& range) {
 bool IsTextRangeSet(const UriTextRangeStructA& range) { return range.first != nullptr; }
 
 #ifdef _WIN32
-bool IsDriveSpec(util::string_view s) {
+bool IsDriveSpec(const util::string_view s) {
   return (s.length() >= 2 && s[1] == ':' &&
           ((s[0] >= 'A' && s[0] <= 'Z') || (s[0] >= 'a' && s[0] <= 'z')));
 }
 #endif
 
+std::string UriUnescape(const util::string_view s) {
+  std::string result(s);
+  if (!result.empty()) {
+    auto end = uriUnescapeInPlaceA(&result[0]);
+    result.resize(end - &result[0]);
+  }
+  return result;
+}
+
 }  // namespace
 
 std::string UriEscape(const std::string& s) {
@@ -125,9 +134,9 @@ std::string Uri::username() const {
   auto userpass = TextRangeToView(impl_->uri_.userInfo);
   auto sep_pos = userpass.find_first_of(':');
   if (sep_pos == util::string_view::npos) {
-    return std::string(userpass);
+    return UriUnescape(userpass);
   } else {
-    return std::string(userpass.substr(0, sep_pos));
+    return UriUnescape(userpass.substr(0, sep_pos));
   }
 }
 
@@ -137,7 +146,7 @@ std::string Uri::password() const {
   if (sep_pos == util::string_view::npos) {
     return std::string();
   } else {
-    return std::string(userpass.substr(sep_pos + 1));
+    return UriUnescape(userpass.substr(sep_pos + 1));
   }
 }
 
@@ -160,7 +169,7 @@ std::string Uri::path() const {
     ss << "/";
   }
   bool first = true;
-  for (const auto seg : segments) {
+  for (const auto& seg : segments) {
     if (!first) {
       ss << "/";
     }
diff --git a/cpp/src/arrow/util/uri_test.cc b/cpp/src/arrow/util/uri_test.cc
index f4d5d078ff4..44804704c43 100644
--- a/cpp/src/arrow/util/uri_test.cc
+++ b/cpp/src/arrow/util/uri_test.cc
@@ -212,6 +212,19 @@ TEST(Uri, ParseUserPass) {
   ASSERT_EQ(uri.host(), "localhost");
   ASSERT_EQ(uri.username(), "someuser");
   ASSERT_EQ(uri.password(), "somepass");
+
+  // With %-encoding
+  ASSERT_OK(uri.Parse("http://some%20user%2Fname:somepass@localhost"));
+  ASSERT_EQ(uri.scheme(), "http");
+  ASSERT_EQ(uri.host(), "localhost");
+  ASSERT_EQ(uri.username(), "some user/name");
+  ASSERT_EQ(uri.password(), "somepass");
+
+  ASSERT_OK(uri.Parse("http://some%20user%2Fname:some%20pass%2Fword@localhost"));
+  ASSERT_EQ(uri.scheme(), "http");
+  ASSERT_EQ(uri.host(), "localhost");
+  ASSERT_EQ(uri.username(), "some user/name");
+  ASSERT_EQ(uri.password(), "some pass/word");
 }
 
 TEST(Uri, FileScheme) {
diff --git a/cpp/src/arrow/util/utf8.cc b/cpp/src/arrow/util/utf8.cc
index d28f6a7da61..478d8ade95f 100644
--- a/cpp/src/arrow/util/utf8.cc
+++ b/cpp/src/arrow/util/utf8.cc
@@ -24,7 +24,12 @@
 #include "arrow/result.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/utf8.h"
-#include "arrow/vendored/utf8cpp/checked.h"
+#include "arrow/vendored/utfcpp/checked.h"
+
+// Can be defined by utfcpp
+#ifdef NOEXCEPT
+#undef NOEXCEPT
+#endif
 
 namespace arrow {
 namespace util {
diff --git a/cpp/src/arrow/util/utf8.h b/cpp/src/arrow/util/utf8.h
index d5875c4590b..2c7dfbbadf6 100644
--- a/cpp/src/arrow/util/utf8.h
+++ b/cpp/src/arrow/util/utf8.h
@@ -27,6 +27,7 @@
 #include "arrow/util/macros.h"
 #include "arrow/util/simd.h"
 #include "arrow/util/string_view.h"
+#include "arrow/util/ubsan.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
@@ -87,8 +88,9 @@ ARROW_EXPORT void InitializeUTF8();
 
 inline bool ValidateUTF8(const uint8_t* data, int64_t size) {
   static constexpr uint64_t high_bits_64 = 0x8080808080808080ULL;
-  // For some reason, defining this variable outside the loop helps clang
-  uint64_t mask;
+  static constexpr uint32_t high_bits_32 = 0x80808080UL;
+  static constexpr uint16_t high_bits_16 = 0x8080U;
+  static constexpr uint8_t high_bits_8 = 0x80U;
 
 #ifndef NDEBUG
   internal::CheckUTF8Initialized();
@@ -98,8 +100,8 @@ inline bool ValidateUTF8(const uint8_t* data, int64_t size) {
     // XXX This is doing an unaligned access.  Contemporary architectures
     // (x86-64, AArch64, PPC64) support it natively and often have good
     // performance nevertheless.
-    memcpy(&mask, data, 8);
-    if (ARROW_PREDICT_TRUE((mask & high_bits_64) == 0)) {
+    uint64_t mask64 = SafeLoadAs<uint64_t>(data);
+    if (ARROW_PREDICT_TRUE((mask64 & high_bits_64) == 0)) {
       // 8 bytes of pure ASCII, move forward
       size -= 8;
       data += 8;
@@ -154,13 +156,50 @@ inline bool ValidateUTF8(const uint8_t* data, int64_t size) {
     return false;
   }
 
-  // Validate string tail one byte at a time
+  // Check if string tail is full ASCII (common case, fast)
+  if (size >= 4) {
+    uint32_t tail_mask = SafeLoadAs<uint32_t>(data + size - 4);
+    uint32_t head_mask = SafeLoadAs<uint32_t>(data);
+    if (ARROW_PREDICT_TRUE(((head_mask | tail_mask) & high_bits_32) == 0)) {
+      return true;
+    }
+  } else if (size >= 2) {
+    uint16_t tail_mask = SafeLoadAs<uint16_t>(data + size - 2);
+    uint16_t head_mask = SafeLoadAs<uint16_t>(data);
+    if (ARROW_PREDICT_TRUE(((head_mask | tail_mask) & high_bits_16) == 0)) {
+      return true;
+    }
+  } else if (size == 1) {
+    if (ARROW_PREDICT_TRUE((*data & high_bits_8) == 0)) {
+      return true;
+    }
+  } else {
+    /* size == 0 */
+    return true;
+  }
+
+  // Fall back to UTF8 validation of tail string.
   // Note the state table is designed so that, once in the reject state,
   // we remain in that state until the end.  So we needn't check for
   // rejection at each char (we don't gain much by short-circuiting here).
   uint16_t state = internal::kUTF8ValidateAccept;
-  while (size-- > 0) {
-    state = internal::ValidateOneUTF8Byte(*data++, state);
+  switch (size) {
+    case 7:
+      state = internal::ValidateOneUTF8Byte(data[size - 7], state);
+    case 6:
+      state = internal::ValidateOneUTF8Byte(data[size - 6], state);
+    case 5:
+      state = internal::ValidateOneUTF8Byte(data[size - 5], state);
+    case 4:
+      state = internal::ValidateOneUTF8Byte(data[size - 4], state);
+    case 3:
+      state = internal::ValidateOneUTF8Byte(data[size - 3], state);
+    case 2:
+      state = internal::ValidateOneUTF8Byte(data[size - 2], state);
+    case 1:
+      state = internal::ValidateOneUTF8Byte(data[size - 1], state);
+    default:
+      break;
   }
   return ARROW_PREDICT_TRUE(state == internal::kUTF8ValidateAccept);
 }
@@ -282,6 +321,18 @@ static inline bool Utf8IsContinuation(const uint8_t codeunit) {
   return (codeunit & 0xC0) == 0x80;  // upper two bits should be 10
 }
 
+static inline bool Utf8Is2ByteStart(const uint8_t codeunit) {
+  return (codeunit & 0xE0) == 0xC0;  // upper three bits should be 110
+}
+
+static inline bool Utf8Is3ByteStart(const uint8_t codeunit) {
+  return (codeunit & 0xF0) == 0xE0;  // upper four bits should be 1110
+}
+
+static inline bool Utf8Is4ByteStart(const uint8_t codeunit) {
+  return (codeunit & 0xF8) == 0xF0;  // upper five bits should be 11110
+}
+
 static inline uint8_t* UTF8Encode(uint8_t* str, uint32_t codepoint) {
   if (codepoint < 0x80) {
     *str++ = codepoint;
@@ -305,7 +356,7 @@ static inline uint8_t* UTF8Encode(uint8_t* str, uint32_t codepoint) {
 
 static inline bool UTF8Decode(const uint8_t** data, uint32_t* codepoint) {
   const uint8_t* str = *data;
-  if (*str < 0x80) {  // ascci
+  if (*str < 0x80) {  // ascii
     *codepoint = *str++;
   } else if (ARROW_PREDICT_FALSE(*str < 0xC0)) {  // invalid non-ascii char
     return false;
@@ -350,6 +401,45 @@ static inline bool UTF8Decode(const uint8_t** data, uint32_t* codepoint) {
   return true;
 }
 
+static inline bool UTF8DecodeReverse(const uint8_t** data, uint32_t* codepoint) {
+  const uint8_t* str = *data;
+  if (*str < 0x80) {  // ascii
+    *codepoint = *str--;
+  } else {
+    if (ARROW_PREDICT_FALSE(!Utf8IsContinuation(*str))) {
+      return false;
+    }
+    uint8_t code_unit_N = (*str--) & 0x3F;  // take last 6 bits
+    if (Utf8Is2ByteStart(*str)) {
+      uint8_t code_unit_1 = (*str--) & 0x1F;  // take last 5 bits
+      *codepoint = (code_unit_1 << 6) + code_unit_N;
+    } else {
+      if (ARROW_PREDICT_FALSE(!Utf8IsContinuation(*str))) {
+        return false;
+      }
+      uint8_t code_unit_Nmin1 = (*str--) & 0x3F;  // take last 6 bits
+      if (Utf8Is3ByteStart(*str)) {
+        uint8_t code_unit_1 = (*str--) & 0x0F;  // take last 4 bits
+        *codepoint = (code_unit_1 << 12) + (code_unit_Nmin1 << 6) + code_unit_N;
+      } else {
+        if (ARROW_PREDICT_FALSE(!Utf8IsContinuation(*str))) {
+          return false;
+        }
+        uint8_t code_unit_Nmin2 = (*str--) & 0x3F;  // take last 6 bits
+        if (ARROW_PREDICT_TRUE(Utf8Is4ByteStart(*str))) {
+          uint8_t code_unit_1 = (*str--) & 0x07;  // take last 3 bits
+          *codepoint = (code_unit_1 << 18) + (code_unit_Nmin2 << 12) +
+                       (code_unit_Nmin1 << 6) + code_unit_N;
+        } else {
+          return false;
+        }
+      }
+    }
+  }
+  *data = str;
+  return true;
+}
+
 template <class UnaryOperation>
 static inline bool UTF8Transform(const uint8_t* first, const uint8_t* last,
                                  uint8_t** destination, UnaryOperation&& unary_op) {
diff --git a/cpp/src/arrow/util/utf8_util_test.cc b/cpp/src/arrow/util/utf8_util_test.cc
index 167b402933d..44caf365089 100644
--- a/cpp/src/arrow/util/utf8_util_test.cc
+++ b/cpp/src/arrow/util/utf8_util_test.cc
@@ -374,5 +374,43 @@ TEST(WideStringToUTF8, Basics) {
 #endif
 }
 
+TEST(UTF8DecodeReverse, Basics) {
+  auto CheckOk = [](const std::string& s) -> void {
+    const uint8_t* begin = reinterpret_cast<const uint8_t*>(s.c_str());
+    const uint8_t* end = begin + s.length();
+    const uint8_t* i = end - 1;
+    uint32_t codepoint;
+    EXPECT_TRUE(UTF8DecodeReverse(&i, &codepoint));
+    EXPECT_EQ(i, begin - 1);
+  };
+
+  // 0x80 == 0b10000000
+  // 0xC0 == 0b11000000
+  // 0xE0 == 0b11100000
+  // 0xF0 == 0b11110000
+  CheckOk("a");
+  CheckOk("\xC0\x80");
+  CheckOk("\xE0\x80\x80");
+  CheckOk("\xF0\x80\x80\x80");
+
+  auto CheckInvalid = [](const std::string& s) -> void {
+    const uint8_t* begin = reinterpret_cast<const uint8_t*>(s.c_str());
+    const uint8_t* end = begin + s.length();
+    const uint8_t* i = end - 1;
+    uint32_t codepoint;
+    EXPECT_FALSE(UTF8DecodeReverse(&i, &codepoint));
+  };
+
+  // too many continuation code units
+  CheckInvalid("a\x80");
+  CheckInvalid("\xC0\x80\x80");
+  CheckInvalid("\xE0\x80\x80\x80");
+  CheckInvalid("\xF0\x80\x80\x80\x80");
+  // not enough continuation code units
+  CheckInvalid("\xC0");
+  CheckInvalid("\xE0\x80");
+  CheckInvalid("\xF0\x80\x80");
+}
+
 }  // namespace util
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/value_parsing.cc b/cpp/src/arrow/util/value_parsing.cc
index cd12dbcdd7e..adc333ecfcc 100644
--- a/cpp/src/arrow/util/value_parsing.cc
+++ b/cpp/src/arrow/util/value_parsing.cc
@@ -20,70 +20,19 @@
 #include <string>
 #include <utility>
 
-#include "arrow/util/double_conversion.h"
+#include "arrow/vendored/fast_float/fast_float.h"
 
 namespace arrow {
 namespace internal {
 
-namespace {
-
-struct StringToFloatConverterImpl {
-  StringToFloatConverterImpl()
-      : main_converter_(flags_, main_junk_value_, main_junk_value_, "inf", "nan"),
-        fallback_converter_(flags_, fallback_junk_value_, fallback_junk_value_, "inf",
-                            "nan") {}
-
-  // NOTE: This is only supported in double-conversion 3.1+
-  static constexpr int flags_ =
-      util::double_conversion::StringToDoubleConverter::ALLOW_CASE_INSENSIBILITY;
-
-  // Two unlikely values to signal a parsing error
-  static constexpr double main_junk_value_ = 0.7066424364107089;
-  static constexpr double fallback_junk_value_ = 0.40088499148279166;
-
-  util::double_conversion::StringToDoubleConverter main_converter_;
-  util::double_conversion::StringToDoubleConverter fallback_converter_;
-};
-
-static const StringToFloatConverterImpl g_string_to_float;
-
-// Older clang versions need an explicit implementation definition.
-constexpr double StringToFloatConverterImpl::main_junk_value_;
-constexpr double StringToFloatConverterImpl::fallback_junk_value_;
-
-}  // namespace
-
 bool StringToFloat(const char* s, size_t length, float* out) {
-  int processed_length;
-  float v;
-  v = g_string_to_float.main_converter_.StringToFloat(s, static_cast<int>(length),
-                                                      &processed_length);
-  if (ARROW_PREDICT_FALSE(v == static_cast<float>(g_string_to_float.main_junk_value_))) {
-    v = g_string_to_float.fallback_converter_.StringToFloat(s, static_cast<int>(length),
-                                                            &processed_length);
-    if (ARROW_PREDICT_FALSE(v ==
-                            static_cast<float>(g_string_to_float.fallback_junk_value_))) {
-      return false;
-    }
-  }
-  *out = v;
-  return true;
+  const auto res = ::arrow_vendored::fast_float::from_chars(s, s + length, *out);
+  return res.ec == std::errc() && res.ptr == s + length;
 }
 
 bool StringToFloat(const char* s, size_t length, double* out) {
-  int processed_length;
-  double v;
-  v = g_string_to_float.main_converter_.StringToDouble(s, static_cast<int>(length),
-                                                       &processed_length);
-  if (ARROW_PREDICT_FALSE(v == g_string_to_float.main_junk_value_)) {
-    v = g_string_to_float.fallback_converter_.StringToDouble(s, static_cast<int>(length),
-                                                             &processed_length);
-    if (ARROW_PREDICT_FALSE(v == g_string_to_float.fallback_junk_value_)) {
-      return false;
-    }
-  }
-  *out = v;
-  return true;
+  const auto res = ::arrow_vendored::fast_float::from_chars(s, s + length, *out);
+  return res.ec == std::errc() && res.ptr == s + length;
 }
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/value_parsing.h b/cpp/src/arrow/util/value_parsing.h
index 6278fbd5bdf..0d18a13bd35 100644
--- a/cpp/src/arrow/util/value_parsing.h
+++ b/cpp/src/arrow/util/value_parsing.h
@@ -165,6 +165,8 @@ inline uint8_t ParseDecimalDigit(char c) { return static_cast<uint8_t>(c - '0');
       return false;                               \
     }                                             \
     result = static_cast<C_TYPE>(result + digit); \
+  } else {                                        \
+    break;                                        \
   }
 
 #define PARSE_UNSIGNED_ITERATION_LAST(C_TYPE)                                     \
@@ -194,71 +196,76 @@ inline uint8_t ParseDecimalDigit(char c) { return static_cast<uint8_t>(c - '0');
 inline bool ParseUnsigned(const char* s, size_t length, uint8_t* out) {
   uint8_t result = 0;
 
-  PARSE_UNSIGNED_ITERATION(uint8_t);
-  PARSE_UNSIGNED_ITERATION(uint8_t);
-  PARSE_UNSIGNED_ITERATION_LAST(uint8_t);
+  do {
+    PARSE_UNSIGNED_ITERATION(uint8_t);
+    PARSE_UNSIGNED_ITERATION(uint8_t);
+    PARSE_UNSIGNED_ITERATION_LAST(uint8_t);
+  } while (false);
   *out = result;
   return true;
 }
 
 inline bool ParseUnsigned(const char* s, size_t length, uint16_t* out) {
   uint16_t result = 0;
-
-  PARSE_UNSIGNED_ITERATION(uint16_t);
-  PARSE_UNSIGNED_ITERATION(uint16_t);
-  PARSE_UNSIGNED_ITERATION(uint16_t);
-  PARSE_UNSIGNED_ITERATION(uint16_t);
-  PARSE_UNSIGNED_ITERATION_LAST(uint16_t);
+  do {
+    PARSE_UNSIGNED_ITERATION(uint16_t);
+    PARSE_UNSIGNED_ITERATION(uint16_t);
+    PARSE_UNSIGNED_ITERATION(uint16_t);
+    PARSE_UNSIGNED_ITERATION(uint16_t);
+    PARSE_UNSIGNED_ITERATION_LAST(uint16_t);
+  } while (false);
   *out = result;
   return true;
 }
 
 inline bool ParseUnsigned(const char* s, size_t length, uint32_t* out) {
   uint32_t result = 0;
-
-  PARSE_UNSIGNED_ITERATION(uint32_t);
-  PARSE_UNSIGNED_ITERATION(uint32_t);
-  PARSE_UNSIGNED_ITERATION(uint32_t);
-  PARSE_UNSIGNED_ITERATION(uint32_t);
-  PARSE_UNSIGNED_ITERATION(uint32_t);
-
-  PARSE_UNSIGNED_ITERATION(uint32_t);
-  PARSE_UNSIGNED_ITERATION(uint32_t);
-  PARSE_UNSIGNED_ITERATION(uint32_t);
-  PARSE_UNSIGNED_ITERATION(uint32_t);
-
-  PARSE_UNSIGNED_ITERATION_LAST(uint32_t);
+  do {
+    PARSE_UNSIGNED_ITERATION(uint32_t);
+    PARSE_UNSIGNED_ITERATION(uint32_t);
+    PARSE_UNSIGNED_ITERATION(uint32_t);
+    PARSE_UNSIGNED_ITERATION(uint32_t);
+    PARSE_UNSIGNED_ITERATION(uint32_t);
+
+    PARSE_UNSIGNED_ITERATION(uint32_t);
+    PARSE_UNSIGNED_ITERATION(uint32_t);
+    PARSE_UNSIGNED_ITERATION(uint32_t);
+    PARSE_UNSIGNED_ITERATION(uint32_t);
+
+    PARSE_UNSIGNED_ITERATION_LAST(uint32_t);
+  } while (false);
   *out = result;
   return true;
 }
 
 inline bool ParseUnsigned(const char* s, size_t length, uint64_t* out) {
   uint64_t result = 0;
-
-  PARSE_UNSIGNED_ITERATION(uint64_t);
-  PARSE_UNSIGNED_ITERATION(uint64_t);
-  PARSE_UNSIGNED_ITERATION(uint64_t);
-  PARSE_UNSIGNED_ITERATION(uint64_t);
-  PARSE_UNSIGNED_ITERATION(uint64_t);
-
-  PARSE_UNSIGNED_ITERATION(uint64_t);
-  PARSE_UNSIGNED_ITERATION(uint64_t);
-  PARSE_UNSIGNED_ITERATION(uint64_t);
-  PARSE_UNSIGNED_ITERATION(uint64_t);
-  PARSE_UNSIGNED_ITERATION(uint64_t);
-
-  PARSE_UNSIGNED_ITERATION(uint64_t);
-  PARSE_UNSIGNED_ITERATION(uint64_t);
-  PARSE_UNSIGNED_ITERATION(uint64_t);
-  PARSE_UNSIGNED_ITERATION(uint64_t);
-  PARSE_UNSIGNED_ITERATION(uint64_t);
-
-  PARSE_UNSIGNED_ITERATION(uint64_t);
-  PARSE_UNSIGNED_ITERATION(uint64_t);
-  PARSE_UNSIGNED_ITERATION(uint64_t);
-  PARSE_UNSIGNED_ITERATION(uint64_t);
-
-  PARSE_UNSIGNED_ITERATION_LAST(uint64_t);
+  do {
+    PARSE_UNSIGNED_ITERATION(uint64_t);
+    PARSE_UNSIGNED_ITERATION(uint64_t);
+    PARSE_UNSIGNED_ITERATION(uint64_t);
+    PARSE_UNSIGNED_ITERATION(uint64_t);
+    PARSE_UNSIGNED_ITERATION(uint64_t);
+
+    PARSE_UNSIGNED_ITERATION(uint64_t);
+    PARSE_UNSIGNED_ITERATION(uint64_t);
+    PARSE_UNSIGNED_ITERATION(uint64_t);
+    PARSE_UNSIGNED_ITERATION(uint64_t);
+    PARSE_UNSIGNED_ITERATION(uint64_t);
+
+    PARSE_UNSIGNED_ITERATION(uint64_t);
+    PARSE_UNSIGNED_ITERATION(uint64_t);
+    PARSE_UNSIGNED_ITERATION(uint64_t);
+    PARSE_UNSIGNED_ITERATION(uint64_t);
+    PARSE_UNSIGNED_ITERATION(uint64_t);
+
+    PARSE_UNSIGNED_ITERATION(uint64_t);
+    PARSE_UNSIGNED_ITERATION(uint64_t);
+    PARSE_UNSIGNED_ITERATION(uint64_t);
+    PARSE_UNSIGNED_ITERATION(uint64_t);
+
+    PARSE_UNSIGNED_ITERATION_LAST(uint64_t);
+  } while (false);
   *out = result;
   return true;
 }
diff --git a/cpp/src/arrow/util/value_parsing_test.cc b/cpp/src/arrow/util/value_parsing_test.cc
index 1682e1d7f1d..547f6f51533 100644
--- a/cpp/src/arrow/util/value_parsing_test.cc
+++ b/cpp/src/arrow/util/value_parsing_test.cc
@@ -239,6 +239,31 @@ TEST(StringConversion, ToUInt64) {
   AssertConversionFails<UInt64Type>("e");
 }
 
+TEST(StringConversion, ToDate32) {
+  AssertConversion<Date32Type>("1970-01-01", 0);
+  AssertConversion<Date32Type>("1970-01-02", 1);
+  AssertConversion<Date32Type>("2020-03-15", 18336);
+  AssertConversion<Date32Type>("1945-05-08", -9004);
+  AssertConversion<Date32Type>("4707-11-28", 999999);
+  AssertConversion<Date32Type>("0001-01-01", -719162);
+
+  // Invalid format
+  AssertConversionFails<Date32Type>("");
+  AssertConversionFails<Date32Type>("1970");
+  AssertConversionFails<Date32Type>("1970-01");
+  AssertConversionFails<Date32Type>("1970-01-01 00:00:00");
+  AssertConversionFails<Date32Type>("1970/01/01");
+}
+
+TEST(StringConversion, ToDate64) {
+  AssertConversion<Date64Type>("1970-01-01", 0);
+  AssertConversion<Date64Type>("1970-01-02", 86400000);
+  AssertConversion<Date64Type>("2020-03-15", 1584230400000LL);
+  AssertConversion<Date64Type>("1945-05-08", -777945600000LL);
+  AssertConversion<Date64Type>("4707-11-28", 86399913600000LL);
+  AssertConversion<Date64Type>("0001-01-01", -62135596800000LL);
+}
+
 TEST(StringConversion, ToTimestampDate_ISO8601) {
   {
     TimestampType type{TimeUnit::SECOND};
diff --git a/cpp/src/arrow/vendored/datetime.h b/cpp/src/arrow/vendored/datetime.h
index f3b8b9d3fea..e437cdcbc2d 100644
--- a/cpp/src/arrow/vendored/datetime.h
+++ b/cpp/src/arrow/vendored/datetime.h
@@ -19,3 +19,8 @@
 
 #include "arrow/vendored/datetime/date.h"  // IWYU pragma: export
 #include "arrow/vendored/datetime/tz.h"    // IWYU pragma: export
+
+// Can be defined by date.h.
+#ifdef NOEXCEPT
+#undef NOEXCEPT
+#endif
diff --git a/cpp/src/arrow/vendored/datetime/README.md b/cpp/src/arrow/vendored/datetime/README.md
index ff156ea3100..811b6935ff2 100644
--- a/cpp/src/arrow/vendored/datetime/README.md
+++ b/cpp/src/arrow/vendored/datetime/README.md
@@ -17,5 +17,5 @@ copies or substantial portions of the Software.
 Sources for datetime are adapted from Howard Hinnant's date library
 (https://github.com/HowardHinnant/date).
 
-Sources are taken from v2.4.1 release of the above project.
+Sources are taken from v3.0.0 release of the above project.
 
diff --git a/cpp/src/arrow/vendored/datetime/date.h b/cpp/src/arrow/vendored/datetime/date.h
index 02a4909132a..6d0455a354b 100644
--- a/cpp/src/arrow/vendored/datetime/date.h
+++ b/cpp/src/arrow/vendored/datetime/date.h
@@ -8,6 +8,7 @@
 // Copyright (c) 2017 Florian Dang
 // Copyright (c) 2017 Paul Thompson
 // Copyright (c) 2018, 2019 Tomasz Kamiński
+// Copyright (c) 2019 Jiangang Zhuang
 //
 // Permission is hereby granted, free of charge, to any person obtaining a copy
 // of this software and associated documentation files (the "Software"), to deal
@@ -70,7 +71,9 @@
 
 #ifdef __GNUC__
 # pragma GCC diagnostic push
-# pragma GCC diagnostic ignored "-Wpedantic"
+# if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 7)
+#  pragma GCC diagnostic ignored "-Wpedantic"
+# endif
 # if __GNUC__ < 5
    // GCC 4.9 Bug 61489 Wrong warning with -Wmissing-field-initializers
 #  pragma GCC diagnostic ignored "-Wmissing-field-initializers"
@@ -137,7 +140,7 @@ namespace date
 #endif
 
 #ifndef HAS_UNCAUGHT_EXCEPTIONS
-#  if __cplusplus > 201703
+#  if __cplusplus > 201703 || (defined(_MSVC_LANG) && _MSVC_LANG > 201703L)
 #    define HAS_UNCAUGHT_EXCEPTIONS 1
 #  else
 #    define HAS_UNCAUGHT_EXCEPTIONS 0
@@ -145,7 +148,7 @@ namespace date
 #endif  // HAS_UNCAUGHT_EXCEPTIONS
 
 #ifndef HAS_VOID_T
-#  if __cplusplus >= 201703
+#  if __cplusplus >= 201703 || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L)
 #    define HAS_VOID_T 1
 #  else
 #    define HAS_VOID_T 0
@@ -157,6 +160,17 @@ namespace date
 #  undef sun
 #endif
 
+// Work around for a NVCC compiler bug which causes it to fail
+// to compile std::ratio_{multiply,divide} when used directly
+// in the std::chrono::duration template instantiations below
+namespace detail {
+template <typename R1, typename R2>
+using ratio_multiply = decltype(std::ratio_multiply<R1, R2>{});
+
+template <typename R1, typename R2>
+using ratio_divide = decltype(std::ratio_divide<R1, R2>{});
+}  // namespace detail
+
 //-----------+
 // Interface |
 //-----------+
@@ -164,16 +178,16 @@ namespace date
 // durations
 
 using days = std::chrono::duration
-    <int, std::ratio_multiply<std::ratio<24>, std::chrono::hours::period>>;
+    <int, detail::ratio_multiply<std::ratio<24>, std::chrono::hours::period>>;
 
 using weeks = std::chrono::duration
-    <int, std::ratio_multiply<std::ratio<7>, days::period>>;
+    <int, detail::ratio_multiply<std::ratio<7>, days::period>>;
 
 using years = std::chrono::duration
-    <int, std::ratio_multiply<std::ratio<146097, 400>, days::period>>;
+    <int, detail::ratio_multiply<std::ratio<146097, 400>, days::period>>;
 
 using months = std::chrono::duration
-    <int, std::ratio_divide<years::period, std::ratio<12>>>;
+    <int, detail::ratio_divide<years::period, std::ratio<12>>>;
 
 // time_point
 
@@ -406,8 +420,8 @@ class year
     CONSTCD11 explicit operator int() const NOEXCEPT;
     CONSTCD11 bool ok() const NOEXCEPT;
 
-    static CONSTCD11 year min() NOEXCEPT;
-    static CONSTCD11 year max() NOEXCEPT;
+    static CONSTCD11 year min() NOEXCEPT { return year{-32767}; }
+    static CONSTCD11 year max() NOEXCEPT { return year{32767}; }
 };
 
 CONSTCD11 bool operator==(const year& x, const year& y) NOEXCEPT;
@@ -1154,8 +1168,7 @@ struct no_overflow
     static const std::intmax_t d1 = R1::den / gcd_d1_d2;
     static const std::intmax_t n2 = R2::num / gcd_n1_n2;
     static const std::intmax_t d2 = R2::den / gcd_d1_d2;
-    static const std::intmax_t max = -((std::intmax_t(1) <<
-                                       (sizeof(std::intmax_t) * CHAR_BIT - 1)) + 1);
+    static const std::intmax_t max = std::numeric_limits<std::intmax_t>::max();
 
     template <std::intmax_t Xp, std::intmax_t Yp, bool overflow>
     struct mul    // overflow == false
@@ -1356,7 +1369,7 @@ trunc(const std::chrono::time_point<Clock, FromDuration>& tp)
 
 // day
 
-CONSTCD11 inline day::day(unsigned d) NOEXCEPT : d_(static_cast<unsigned char>(d)) {}
+CONSTCD11 inline day::day(unsigned d) NOEXCEPT : d_(static_cast<decltype(d_)>(d)) {}
 CONSTCD14 inline day& day::operator++() NOEXCEPT {++d_; return *this;}
 CONSTCD14 inline day day::operator++(int) NOEXCEPT {auto tmp(*this); ++(*this); return tmp;}
 CONSTCD14 inline day& day::operator--() NOEXCEPT {--d_; return *this;}
@@ -1553,7 +1566,7 @@ inline
 month
 operator+(const month& x, const months& y) NOEXCEPT
 {
-    auto const mu = static_cast<long long>(static_cast<unsigned>(x)) + (y.count() - 1);
+    auto const mu = static_cast<long long>(static_cast<unsigned>(x)) + y.count() - 1;
     auto const yr = (mu >= 0 ? mu : mu-11) / 12;
     return month{static_cast<unsigned>(mu - yr * 12 + 1)};
 }
@@ -1619,22 +1632,6 @@ year::ok() const NOEXCEPT
     return y_ != std::numeric_limits<short>::min();
 }
 
-CONSTCD11
-inline
-year
-year::min() NOEXCEPT
-{
-    return year{-32767};
-}
-
-CONSTCD11
-inline
-year
-year::max() NOEXCEPT
-{
-    return year{32767};
-}
-
 CONSTCD11
 inline
 bool
@@ -1724,6 +1721,7 @@ operator<<(std::basic_ostream<CharT, Traits>& os, const year& y)
     os.fill('0');
     os.flags(std::ios::dec | std::ios::internal);
     os.width(4 + (y < year{0}));
+    os.imbue(std::locale::classic());
     os << static_cast<int>(y);
     if (!y.ok())
         os << " is not a valid year";
@@ -2849,6 +2847,7 @@ operator<<(std::basic_ostream<CharT, Traits>& os, const year_month_day& ymd)
     detail::save_ostream<CharT, Traits> _(os);
     os.fill('0');
     os.flags(std::ios::dec | std::ios::right);
+    os.imbue(std::locale::classic());
     os << ymd.year() << '-';
     os.width(2);
     os << static_cast<unsigned>(ymd.month()) << '-';
@@ -3735,8 +3734,7 @@ class decimal_format_seconds
 
     CONSTCD11 explicit decimal_format_seconds(const Duration& d) NOEXCEPT
         : s_(std::chrono::duration_cast<std::chrono::seconds>(d))
-        , sub_s_(std::chrono::treat_as_floating_point<rep>::value ? d - s_ :
-                     std::chrono::duration_cast<precision>(d - s_))
+        , sub_s_(std::chrono::duration_cast<precision>(d - s_))
         {}
 
     CONSTCD14 std::chrono::seconds& seconds() NOEXCEPT {return s_;}
@@ -3789,6 +3787,8 @@ class decimal_format_seconds
 #else
             os << '.';
 #endif
+            date::detail::save_ostream<CharT, Traits> _s(os);
+            os.imbue(std::locale::classic());
             os.width(width);
             os << sub_s_.count();
         }
@@ -3860,7 +3860,7 @@ class hh_mm_ss
 
     CONSTCD11 explicit operator  precision()   const NOEXCEPT {return to_duration();}
     CONSTCD11          precision to_duration() const NOEXCEPT
-        {return (h_ + m_ + s_.to_duration()) * (1-2*neg_);}
+        {return (s_.to_duration() + m_ + h_) * (1-2*neg_);}
 
     CONSTCD11 bool in_conventional_range() const NOEXCEPT
     {
@@ -3933,7 +3933,7 @@ make12(std::chrono::hours h) NOEXCEPT
     else
     {
         if (h != hours{12})
-            h -= hours{12};
+            h = h - hours{12};
     }
     return h;
 }
@@ -3947,7 +3947,7 @@ make24(std::chrono::hours h, bool is_pm) NOEXCEPT
     if (is_pm)
     {
         if (h != hours{12})
-            h += hours{12};
+            h = h + hours{12};
     }
     else if (h == hours{12})
         h = hours{0};
@@ -4150,10 +4150,10 @@ operator+(std::basic_string<CharT, Traits, Alloc> x, const string_literal<CharT,
                            && (!defined(__SUNPRO_CC) || __SUNPRO_CC > 0x5150)
 
 template <class CharT,
-          class = std::enable_if_t<std::is_same<CharT, char>{} ||
-                                   std::is_same<CharT, wchar_t>{} ||
-                                   std::is_same<CharT, char16_t>{} ||
-                                   std::is_same<CharT, char32_t>{}>>
+          class = std::enable_if_t<std::is_same<CharT, char>::value ||
+                                   std::is_same<CharT, wchar_t>::value ||
+                                   std::is_same<CharT, char16_t>::value ||
+                                   std::is_same<CharT, char32_t>::value>>
 CONSTCD14
 inline
 string_literal<CharT, 2>
@@ -5078,6 +5078,7 @@ to_stream(std::basic_ostream<CharT, Traits>& os, const CharT* fmt,
                         os.setstate(std::ios::failbit);
                     auto const& ymd = fds.ymd;
                     save_ostream<CharT, Traits> _(os);
+                    os.imbue(std::locale::classic());
                     os.fill('0');
                     os.flags(std::ios::dec | std::ios::right);
                     os.width(4);
@@ -5152,7 +5153,7 @@ to_stream(std::basic_ostream<CharT, Traits>& os, const CharT* fmt,
                     if (modified == CharT{})
 #endif
                     {
-                        auto h = *fmt == CharT{'I'} ? make12(hms.hours()) : hms.hours();
+                        auto h = *fmt == CharT{'I'} ? date::make12(hms.hours()) : hms.hours();
                         if (h < hours{10})
                             os << CharT{'0'};
                         os << h.count();
@@ -5308,7 +5309,7 @@ to_stream(std::basic_ostream<CharT, Traits>& os, const CharT* fmt,
                     tm.tm_hour = static_cast<int>(fds.tod.hours().count());
                     facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f));
 #else
-                    if (is_am(fds.tod.hours()))
+                    if (date::is_am(fds.tod.hours()))
                         os << ampm_names().first[0];
                     else
                         os << ampm_names().first[1];
@@ -5366,12 +5367,12 @@ to_stream(std::basic_ostream<CharT, Traits>& os, const CharT* fmt,
                     save_ostream<CharT, Traits> _(os);
                     os.fill('0');
                     os.width(2);
-                    os << make12(tod.hours()).count() << CharT{':'};
+                    os << date::make12(tod.hours()).count() << CharT{':'};
                     os.width(2);
                     os << tod.minutes().count() << CharT{':'};
                     os.width(2);
                     os << tod.seconds().count() << CharT{' '};
-                    if (is_am(tod.hours()))
+                    if (date::is_am(tod.hours()))
                         os << ampm_names().first[0];
                     else
                         os << ampm_names().first[1];
@@ -5752,6 +5753,8 @@ to_stream(std::basic_ostream<CharT, Traits>& os, const CharT* fmt,
                     if (modified == CharT{})
 #endif
                     {
+                        save_ostream<CharT, Traits> _(os);
+                        os.imbue(std::locale::classic());
                         os << y;
                     }
 #if !ONLY_C_LOCALE
@@ -6253,7 +6256,7 @@ read(std::basic_istream<CharT, Traits>& is, int a0, Args&& ...args)
     if (a0 != -1)
     {
         auto u = static_cast<unsigned>(a0);
-        CharT buf[std::numeric_limits<unsigned>::digits10+2] = {};
+        CharT buf[std::numeric_limits<unsigned>::digits10+2u] = {};
         auto e = buf;
         do
         {
@@ -6363,7 +6366,7 @@ from_stream(std::basic_istream<CharT, Traits>& is, const CharT* fmt,
         using detail::ru;
         using detail::rld;
         using detail::checked_set;
-        for (; *fmt && is.rdstate() == std::ios::goodbit; ++fmt)
+        for (; *fmt != CharT{} && !is.fail(); ++fmt)
         {
             switch (*fmt)
             {
@@ -7364,14 +7367,18 @@ from_stream(std::basic_istream<CharT, Traits>& is, const CharT* fmt,
                 else  // !command
                 {
                     if (isspace(static_cast<unsigned char>(*fmt)))
-                        ws(is); // space matches 0 or more white space characters
+                    {
+                        // space matches 0 or more white space characters
+                        if (is.good())
+                           ws(is);
+                    }
                     else
                         read(is, *fmt);
                 }
                 break;
             }
         }
-        // is.rdstate() != ios::goodbit || *fmt == CharT{}
+        // is.fail() || *fmt == CharT{}
         if (is.rdstate() == ios::goodbit && command)
         {
             if (modified == CharT{})
@@ -7379,8 +7386,6 @@ from_stream(std::basic_istream<CharT, Traits>& is, const CharT* fmt,
             else
                 read(is, CharT{'%'}, width, modified);
         }
-        if (is.rdstate() != ios::goodbit && *fmt != CharT{} && !is.fail())
-            is.setstate(ios::failbit);
         if (!is.fail())
         {
             if (y != not_a_2digit_year)
diff --git a/cpp/src/arrow/vendored/datetime/ios.h b/cpp/src/arrow/vendored/datetime/ios.h
index acad28d13b5..46567d69b18 100644
--- a/cpp/src/arrow/vendored/datetime/ios.h
+++ b/cpp/src/arrow/vendored/datetime/ios.h
@@ -38,10 +38,10 @@
     {
     namespace iOSUtils
     {
-
+    
     std::string get_tzdata_path();
     std::string get_current_timezone();
-
+    
     }  // namespace iOSUtils
     }  // namespace date
     }  // namespace arrow_vendored
diff --git a/cpp/src/arrow/vendored/datetime/ios.mm b/cpp/src/arrow/vendored/datetime/ios.mm
index 24125583076..18c521201d3 100644
--- a/cpp/src/arrow/vendored/datetime/ios.mm
+++ b/cpp/src/arrow/vendored/datetime/ios.mm
@@ -51,289 +51,289 @@
 {
 namespace date
 {
-namespace iOSUtils
-{
-    
-struct TarInfo
-{
-    char objType;
-    std::string objName;
-    size_t realContentSize; // writable size without padding zeroes
-    size_t blocksContentSize; // adjusted size to 512 bytes blocks
-    bool success;
-};
-
-std::string convertCFStringRefPathToCStringPath(CFStringRef ref);
-bool extractTzdata(CFURLRef homeUrl, CFURLRef archiveUrl, std::string destPath);
-TarInfo getTarObjectInfo(std::ifstream &readStream);
-std::string getTarObject(std::ifstream &readStream, int64_t size);
-bool writeFile(const std::string &tzdataPath, const std::string &fileName,
-            const std::string &data, size_t realContentSize);
-
-std::string
-get_current_timezone()
-{
-    CFTimeZoneRef tzRef = CFTimeZoneCopySystem();
-    CFStringRef tzNameRef = CFTimeZoneGetName(tzRef);
-    CFIndex bufferSize = CFStringGetLength(tzNameRef) + 1;
-    char buffer[bufferSize];
-    
-    if (CFStringGetCString(tzNameRef, buffer, bufferSize, kCFStringEncodingUTF8))
-    {
-        CFRelease(tzRef);
-        return std::string(buffer);
-    }
-    
-    CFRelease(tzRef);
-    
-    return "";
-}
-
-std::string
-get_tzdata_path()
-{
-    CFURLRef homeUrlRef = CFCopyHomeDirectoryURL();
-    CFStringRef homePath = CFURLCopyPath(homeUrlRef);
-    std::string path(std::string(convertCFStringRefPathToCStringPath(homePath)) +
-                    INTERNAL_DIR + "/" + TZDATA_DIR);
-    std::string result_path(std::string(convertCFStringRefPathToCStringPath(homePath)) +
-                            INTERNAL_DIR);
-    
-    if (access(path.c_str(), F_OK) == 0)
-    {
-#if TAR_DEBUG
-        printf("tzdata dir exists\n");
-#endif
-        CFRelease(homeUrlRef);
-        CFRelease(homePath);
-        
-        return result_path;
-    }
-    
-    CFBundleRef mainBundle = CFBundleGetMainBundle();
-    CFArrayRef paths = CFBundleCopyResourceURLsOfType(mainBundle, CFSTR(TARGZ_EXTENSION),
-                                                    NULL);
-    
-    if (CFArrayGetCount(paths) != 0)
+    namespace iOSUtils
     {
-        // get archive path, assume there is no other tar.gz in bundle
-        CFURLRef archiveUrl = static_cast<CFURLRef>(CFArrayGetValueAtIndex(paths, 0));
-        CFStringRef archiveName = CFURLCopyPath(archiveUrl);
-        archiveUrl = CFBundleCopyResourceURL(mainBundle, archiveName, NULL, NULL);
         
-        extractTzdata(homeUrlRef, archiveUrl, path);
+        struct TarInfo
+        {
+            char objType;
+            std::string objName;
+            size_t realContentSize; // writable size without padding zeroes
+            size_t blocksContentSize; // adjusted size to 512 bytes blocks
+            bool success;
+        };
         
-        CFRelease(archiveUrl);
-        CFRelease(archiveName);
-    }
-    
-    CFRelease(homeUrlRef);
-    CFRelease(homePath);
-    CFRelease(paths);
-    
-    return result_path;
-}
-
-std::string
-convertCFStringRefPathToCStringPath(CFStringRef ref)
-{
-    CFIndex bufferSize = CFStringGetMaximumSizeOfFileSystemRepresentation(ref);
-    char *buffer = new char[bufferSize];
-    CFStringGetFileSystemRepresentation(ref, buffer, bufferSize);
-    auto result = std::string(buffer);
-    delete[] buffer;
-    return result;
-}
-
-bool
-extractTzdata(CFURLRef homeUrl, CFURLRef archiveUrl, std::string destPath)
-{
-    std::string TAR_TMP_PATH = "/tmp.tar";
-    
-    CFStringRef homeStringRef = CFURLCopyPath(homeUrl);
-    auto homePath = convertCFStringRefPathToCStringPath(homeStringRef);
-    CFRelease(homeStringRef);
-    
-    CFStringRef archiveStringRef = CFURLCopyPath(archiveUrl);
-    auto archivePath = convertCFStringRefPathToCStringPath(archiveStringRef);
-    CFRelease(archiveStringRef);
-    
-    // create Library path
-    auto libraryPath = homePath + INTERNAL_DIR;
-    
-    // create tzdata path
-    auto tzdataPath = libraryPath + "/" + TZDATA_DIR;
-    
-    // -- replace %20 with " "
-    const std::string search = "%20";
-    const std::string replacement = " ";
-    size_t pos = 0;
-    
-    while ((pos = archivePath.find(search, pos)) != std::string::npos) {
-        archivePath.replace(pos, search.length(), replacement);
-        pos += replacement.length();
-    }
-    
-    gzFile tarFile = gzopen(archivePath.c_str(), "rb");
-    
-    // create tar unpacking path
-    auto tarPath = libraryPath + TAR_TMP_PATH;
-    
-    // create tzdata directory
-    mkdir(destPath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
-    
-    // ======= extract tar ========
-    
-    std::ofstream os(tarPath.c_str(), std::ofstream::out | std::ofstream::app);
-    unsigned int bufferLength = 1024 * 256;  // 256Kb
-    unsigned char *buffer = (unsigned char *)malloc(bufferLength);
-    bool success = true;
-    
-    while (true)
-    {
-        int readBytes = gzread(tarFile, buffer, bufferLength);
+        std::string convertCFStringRefPathToCStringPath(CFStringRef ref);
+        bool extractTzdata(CFURLRef homeUrl, CFURLRef archiveUrl, std::string destPath);
+        TarInfo getTarObjectInfo(std::ifstream &readStream);
+        std::string getTarObject(std::ifstream &readStream, int64_t size);
+        bool writeFile(const std::string &tzdataPath, const std::string &fileName,
+                       const std::string &data, size_t realContentSize);
         
-        if (readBytes > 0)
+        std::string
+        get_current_timezone()
         {
-            os.write((char *) &buffer[0], readBytes);
+            CFTimeZoneRef tzRef = CFTimeZoneCopySystem();
+            CFStringRef tzNameRef = CFTimeZoneGetName(tzRef);
+            CFIndex bufferSize = CFStringGetLength(tzNameRef) + 1;
+            char buffer[bufferSize];
+            
+            if (CFStringGetCString(tzNameRef, buffer, bufferSize, kCFStringEncodingUTF8))
+            {
+                CFRelease(tzRef);
+                return std::string(buffer);
+            }
+            
+            CFRelease(tzRef);
+            
+            return "";
         }
-        else
-            if (readBytes == 0)
+        
+        std::string
+        get_tzdata_path()
+        {
+            CFURLRef homeUrlRef = CFCopyHomeDirectoryURL();
+            CFStringRef homePath = CFURLCopyPath(homeUrlRef);
+            std::string path(std::string(convertCFStringRefPathToCStringPath(homePath)) +
+                             INTERNAL_DIR + "/" + TZDATA_DIR);
+            std::string result_path(std::string(convertCFStringRefPathToCStringPath(homePath)) +
+                                    INTERNAL_DIR);
+            
+            if (access(path.c_str(), F_OK) == 0)
             {
-                break;
+#if TAR_DEBUG
+                printf("tzdata dir exists\n");
+#endif
+                CFRelease(homeUrlRef);
+                CFRelease(homePath);
+                
+                return result_path;
             }
-            else
-                if (readBytes == -1)
+            
+            CFBundleRef mainBundle = CFBundleGetMainBundle();
+            CFArrayRef paths = CFBundleCopyResourceURLsOfType(mainBundle, CFSTR(TARGZ_EXTENSION),
+                                                              NULL);
+            
+            if (CFArrayGetCount(paths) != 0)
+            {
+                // get archive path, assume there is no other tar.gz in bundle
+                CFURLRef archiveUrl = static_cast<CFURLRef>(CFArrayGetValueAtIndex(paths, 0));
+                CFStringRef archiveName = CFURLCopyPath(archiveUrl);
+                archiveUrl = CFBundleCopyResourceURL(mainBundle, archiveName, NULL, NULL);
+                
+                extractTzdata(homeUrlRef, archiveUrl, path);
+                
+                CFRelease(archiveUrl);
+                CFRelease(archiveName);
+            }
+            
+            CFRelease(homeUrlRef);
+            CFRelease(homePath);
+            CFRelease(paths);
+            
+            return result_path;
+        }
+        
+        std::string
+        convertCFStringRefPathToCStringPath(CFStringRef ref)
+        {
+            CFIndex bufferSize = CFStringGetMaximumSizeOfFileSystemRepresentation(ref);
+            char *buffer = new char[bufferSize];
+            CFStringGetFileSystemRepresentation(ref, buffer, bufferSize);
+            auto result = std::string(buffer);
+            delete[] buffer;
+            return result;
+        }
+        
+        bool
+        extractTzdata(CFURLRef homeUrl, CFURLRef archiveUrl, std::string destPath)
+        {
+            std::string TAR_TMP_PATH = "/tmp.tar";
+            
+            CFStringRef homeStringRef = CFURLCopyPath(homeUrl);
+            auto homePath = convertCFStringRefPathToCStringPath(homeStringRef);
+            CFRelease(homeStringRef);
+            
+            CFStringRef archiveStringRef = CFURLCopyPath(archiveUrl);
+            auto archivePath = convertCFStringRefPathToCStringPath(archiveStringRef);
+            CFRelease(archiveStringRef);
+            
+            // create Library path
+            auto libraryPath = homePath + INTERNAL_DIR;
+            
+            // create tzdata path
+            auto tzdataPath = libraryPath + "/" + TZDATA_DIR;
+            
+            // -- replace %20 with " "
+            const std::string search = "%20";
+            const std::string replacement = " ";
+            size_t pos = 0;
+            
+            while ((pos = archivePath.find(search, pos)) != std::string::npos) {
+                archivePath.replace(pos, search.length(), replacement);
+                pos += replacement.length();
+            }
+            
+            gzFile tarFile = gzopen(archivePath.c_str(), "rb");
+            
+            // create tar unpacking path
+            auto tarPath = libraryPath + TAR_TMP_PATH;
+            
+            // create tzdata directory
+            mkdir(destPath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
+            
+            // ======= extract tar ========
+            
+            std::ofstream os(tarPath.c_str(), std::ofstream::out | std::ofstream::app);
+            unsigned int bufferLength = 1024 * 256;  // 256Kb
+            unsigned char *buffer = (unsigned char *)malloc(bufferLength);
+            bool success = true;
+            
+            while (true)
+            {
+                int readBytes = gzread(tarFile, buffer, bufferLength);
+                
+                if (readBytes > 0)
                 {
-                    printf("decompression failed\n");
-                    success = false;
-                    break;
+                    os.write((char *) &buffer[0], readBytes);
                 }
                 else
+                    if (readBytes == 0)
+                    {
+                        break;
+                    }
+                    else
+                        if (readBytes == -1)
+                        {
+                            printf("decompression failed\n");
+                            success = false;
+                            break;
+                        }
+                        else
+                        {
+                            printf("unexpected zlib state\n");
+                            success = false;
+                            break;
+                        }
+            }
+            
+            os.close();
+            free(buffer);
+            gzclose(tarFile);
+            
+            if (!success)
+            {
+                remove(tarPath.c_str());
+                return false;
+            }
+            
+            // ======== extract files =========
+            
+            uint64_t location = 0; // Position in the file
+            
+            // get file size
+            struct stat stat_buf;
+            int res = stat(tarPath.c_str(), &stat_buf);
+            if (res != 0)
+            {
+                printf("error file size\n");
+                remove(tarPath.c_str());
+                return false;
+            }
+            int64_t tarSize = stat_buf.st_size;
+            
+            // create read stream
+            std::ifstream is(tarPath.c_str(), std::ifstream::in | std::ifstream::binary);
+            
+            // process files
+            while (location < tarSize)
+            {
+                TarInfo info = getTarObjectInfo(is);
+                
+                if (!info.success || info.realContentSize == 0)
                 {
-                    printf("unexpected zlib state\n");
-                    success = false;
-                    break;
+                    break; // something wrong or all files are read
                 }
-    }
-    
-    os.close();
-    free(buffer);
-    gzclose(tarFile);
-    
-    if (!success)
-    {
-        remove(tarPath.c_str());
-        return false;
-    }
-    
-    // ======== extract files =========
-    
-    uint64_t location = 0; // Position in the file
-    
-    // get file size
-    struct stat stat_buf;
-    int res = stat(tarPath.c_str(), &stat_buf);
-    if (res != 0)
-    {
-        printf("error file size\n");
-        remove(tarPath.c_str());
-        return false;
-    }
-    int64_t tarSize = stat_buf.st_size;
-    
-    // create read stream
-    std::ifstream is(tarPath.c_str(), std::ifstream::in | std::ifstream::binary);
-    
-    // process files
-    while (location < tarSize)
-    {
-        TarInfo info = getTarObjectInfo(is);
+                
+                switch (info.objType)
+                {
+                    case '0':   // file
+                    case '\0':  //
+                    {
+                        std::string obj = getTarObject(is, info.blocksContentSize);
+#if TAR_DEBUG
+                        size += info.realContentSize;
+                        printf("#%i %s file size %lld written total %ld from %lld\n", ++count,
+                               info.objName.c_str(), info.realContentSize, size, tarSize);
+#endif
+                        writeFile(tzdataPath, info.objName, obj, info.realContentSize);
+                        location += info.blocksContentSize;
+                        
+                        break;
+                    }
+                }
+            }
+            
+            remove(tarPath.c_str());
+            
+            return true;
+        }
         
-        if (!info.success || info.realContentSize == 0)
+        TarInfo
+        getTarObjectInfo(std::ifstream &readStream)
         {
-            break; // something wrong or all files are read
+            int64_t length = TAR_BLOCK_SIZE;
+            char buffer[length];
+            char type;
+            char name[TAR_NAME_SIZE + 1];
+            char sizeBuf[TAR_SIZE_SIZE + 1];
+            
+            readStream.read(buffer, length);
+            
+            memcpy(&type, &buffer[TAR_TYPE_POSITION], 1);
+            
+            memset(&name, '\0', TAR_NAME_SIZE + 1);
+            memcpy(&name, &buffer[TAR_NAME_POSITION], TAR_NAME_SIZE);
+            
+            memset(&sizeBuf, '\0', TAR_SIZE_SIZE + 1);
+            memcpy(&sizeBuf, &buffer[TAR_SIZE_POSITION], TAR_SIZE_SIZE);
+            size_t realSize = strtol(sizeBuf, NULL, 8);
+            size_t blocksSize = realSize + (TAR_BLOCK_SIZE - (realSize % TAR_BLOCK_SIZE));
+            
+            return {type, std::string(name), realSize, blocksSize, true};
         }
         
-        switch (info.objType)
+        std::string
+        getTarObject(std::ifstream &readStream, int64_t size)
         {
-            case '0':   // file
-            case '\0':  //
-            {
-                std::string obj = getTarObject(is, info.blocksContentSize);
-#if TAR_DEBUG
-                size += info.realContentSize;
-                printf("#%i %s file size %lld written total %ld from %lld\n", ++count,
-                    info.objName.c_str(), info.realContentSize, size, tarSize);
-#endif
-                writeFile(tzdataPath, info.objName, obj, info.realContentSize);
-                location += info.blocksContentSize;
-                
-                break;
+            char buffer[size];
+            readStream.read(buffer, size);
+            return std::string(buffer);
+        }
+        
+        bool
+        writeFile(const std::string &tzdataPath, const std::string &fileName, const std::string &data,
+                  size_t realContentSize)
+        {
+            std::ofstream os(tzdataPath + "/" + fileName, std::ofstream::out | std::ofstream::binary);
+            
+            if (!os) {
+                return false;
             }
+            
+            // trim empty space
+            char trimmedData[realContentSize + 1];
+            memset(&trimmedData, '\0', realContentSize);
+            memcpy(&trimmedData, data.c_str(), realContentSize);
+            
+            // write
+            os.write(trimmedData, realContentSize);
+            os.close();
+            
+            return true;
         }
-    }
-    
-    remove(tarPath.c_str());
-    
-    return true;
-}
-
-TarInfo
-getTarObjectInfo(std::ifstream &readStream)
-{
-    int64_t length = TAR_BLOCK_SIZE;
-    char buffer[length];
-    char type;
-    char name[TAR_NAME_SIZE + 1];
-    char sizeBuf[TAR_SIZE_SIZE + 1];
-    
-    readStream.read(buffer, length);
-    
-    memcpy(&type, &buffer[TAR_TYPE_POSITION], 1);
-    
-    memset(&name, '\0', TAR_NAME_SIZE + 1);
-    memcpy(&name, &buffer[TAR_NAME_POSITION], TAR_NAME_SIZE);
-    
-    memset(&sizeBuf, '\0', TAR_SIZE_SIZE + 1);
-    memcpy(&sizeBuf, &buffer[TAR_SIZE_POSITION], TAR_SIZE_SIZE);
-    size_t realSize = strtol(sizeBuf, NULL, 8);
-    size_t blocksSize = realSize + (TAR_BLOCK_SIZE - (realSize % TAR_BLOCK_SIZE));
-    
-    return {type, std::string(name), realSize, blocksSize, true};
-}
-
-std::string
-getTarObject(std::ifstream &readStream, int64_t size)
-{
-    char buffer[size];
-    readStream.read(buffer, size);
-    return std::string(buffer);
-}
-
-bool
-writeFile(const std::string &tzdataPath, const std::string &fileName, const std::string &data,
-        size_t realContentSize)
-{
-    std::ofstream os(tzdataPath + "/" + fileName, std::ofstream::out | std::ofstream::binary);
-    
-    if (!os) {
-        return false;
-    }
-    
-    // trim empty space
-    char trimmedData[realContentSize + 1];
-    memset(&trimmedData, '\0', realContentSize);
-    memcpy(&trimmedData, data.c_str(), realContentSize);
-    
-    // write
-    os.write(trimmedData, realContentSize);
-    os.close();
-    
-    return true;
-}
-    
-}  // namespace iOSUtils
+        
+    }  // namespace iOSUtils
 }  // namespace date
 }  // namespace arrow_vendored
 
diff --git a/cpp/src/arrow/vendored/datetime/tz.cpp b/cpp/src/arrow/vendored/datetime/tz.cpp
index a57ac49e2c0..9efc01d8d7c 100644
--- a/cpp/src/arrow/vendored/datetime/tz.cpp
+++ b/cpp/src/arrow/vendored/datetime/tz.cpp
@@ -30,7 +30,7 @@
 // been invented (that would involve another several millennia of evolution).
 // We did not mean to shout.
 
-// wesm: This is required so that symbols are properly exported from the DLL
+// NOTE(ARROW): This is required so that symbols are properly exported from the DLL
 #include "visibility.h"
 
 #ifdef _WIN32
@@ -92,6 +92,7 @@
 #  include "ios.h"
 #else
 #  define TARGET_OS_IPHONE 0
+#  define TARGET_OS_SIMULATOR 0
 #endif
 
 #if USE_OS_TZDB
@@ -239,7 +240,7 @@ get_download_folder()
 #    endif // WINRT
 #  else // !_WIN32
 
-#    if !defined(INSTALL) || HAS_REMOTE_API
+#    if !defined(INSTALL)
 
 static
 std::string
@@ -265,7 +266,7 @@ get_download_folder()
     return expand_path("~/Downloads");
 }
 
-#    endif // !defined(INSTALL) || HAS_REMOTE_API
+#    endif // !defined(INSTALL)
 
 #  endif  // !_WIN32
 
@@ -366,7 +367,11 @@ discover_tz_dir()
         throw runtime_error("discover_tz_dir failed to find zoneinfo\n");
 #  else  // __APPLE__
 #      if TARGET_OS_IPHONE
+#          if TARGET_OS_SIMULATOR
+    return "/usr/share/zoneinfo";
+#          else
     return "/var/db/timezone/zoneinfo";
+#          endif
 #      else
     CONSTDATA auto timezone = "/etc/localtime";
     if (!(lstat(timezone, &sb) == 0 && S_ISLNK(sb.st_mode) && sb.st_size > 0))
@@ -1929,13 +1934,13 @@ load_abbreviations(std::istream& inf, std::int32_t tzh_charcnt)
 
 template <class TimeType>
 static
-std::vector<leap>
+std::vector<leap_second>
 load_leaps(std::istream& inf, std::int32_t tzh_leapcnt)
 {
     // Read tzh_leapcnt pairs
     using namespace std::chrono;
-    std::vector<leap> leap_seconds;
-    leap_seconds.reserve(tzh_leapcnt);
+    std::vector<leap_second> leap_seconds;
+    leap_seconds.reserve(static_cast<std::size_t>(tzh_leapcnt));
     for (std::int32_t i = 0; i < tzh_leapcnt; ++i)
     {
         TimeType     t0;
@@ -1952,17 +1957,18 @@ load_leaps(std::istream& inf, std::int32_t tzh_leapcnt)
 
 template <class TimeType>
 static
-std::vector<leap>
+std::vector<leap_second>
 load_leap_data(std::istream& inf,
                std::int32_t tzh_leapcnt, std::int32_t tzh_timecnt,
                std::int32_t tzh_typecnt, std::int32_t tzh_charcnt)
 {
-    inf.ignore(tzh_timecnt*sizeof(TimeType) + tzh_timecnt + tzh_typecnt*6 + tzh_charcnt);
+    inf.ignore(tzh_timecnt*static_cast<std::int32_t>(sizeof(TimeType)) + tzh_timecnt +
+               tzh_typecnt*6 + tzh_charcnt);
     return load_leaps<TimeType>(inf, tzh_leapcnt);
 }
 
 static
-std::vector<leap>
+std::vector<leap_second>
 load_just_leaps(std::istream& inf)
 {
     // Read tzh_leapcnt pairs
@@ -2008,7 +2014,7 @@ time_zone::load_data(std::istream& inf,
     auto infos = load_ttinfo(inf, tzh_typecnt);
     auto abbrev = load_abbreviations(inf, tzh_charcnt);
 #if !MISSING_LEAP_SECONDS
-    auto& leap_seconds = get_tzdb_list().front().leaps;
+    auto& leap_seconds = get_tzdb_list().front().leap_seconds;
     if (leap_seconds.empty() && tzh_leapcnt > 0)
         leap_seconds = load_leaps<TimeType>(inf, tzh_leapcnt);
 #endif
@@ -2076,7 +2082,7 @@ time_zone::init_impl()
 #if !MISSING_LEAP_SECONDS
     if (tzh_leapcnt > 0)
     {
-        auto& leap_seconds = get_tzdb_list().front().leaps;
+        auto& leap_seconds = get_tzdb_list().front().leap_seconds;
         auto itr = leap_seconds.begin();
         auto l = itr->date();
         seconds leap_count{0};
@@ -2211,7 +2217,7 @@ operator<<(std::ostream& os, const time_zone& z)
 
 #if !MISSING_LEAP_SECONDS
 
-leap::leap(const sys_seconds& s, detail::undocumented)
+leap_second::leap_second(const sys_seconds& s, detail::undocumented)
     : date_(s)
 {
 }
@@ -2617,7 +2623,7 @@ operator<<(std::ostream& os, const time_zone& z)
 #if !MISSING_LEAP_SECONDS
 
 std::ostream&
-operator<<(std::ostream& os, const leap& x)
+operator<<(std::ostream& os, const leap_second& x)
 {
     using namespace date;
     return os << x.date_ << "  +";
@@ -2703,7 +2709,7 @@ init_tzdb()
     if (in)
     {
         in.exceptions(std::ios::failbit | std::ios::badbit);
-        db->leaps = load_just_leaps(in);
+        db->leap_seconds = load_just_leaps(in);
     }
     else
     {
@@ -2713,7 +2719,7 @@ init_tzdb()
         if (!in)
             throw std::runtime_error("Unable to extract leap second information");
         in.exceptions(std::ios::failbit | std::ios::badbit);
-        db->leaps = load_just_leaps(in);
+        db->leap_seconds = load_just_leaps(in);
     }
 #  endif  // !MISSING_LEAP_SECONDS
 #  ifdef __APPLE__
@@ -2724,9 +2730,9 @@ init_tzdb()
 
 #else  // !USE_OS_TZDB
 
-// link
+// time_zone_link
 
-link::link(const std::string& s)
+time_zone_link::time_zone_link(const std::string& s)
 {
     using namespace date;
     std::istringstream in(s);
@@ -2736,7 +2742,7 @@ link::link(const std::string& s)
 }
 
 std::ostream&
-operator<<(std::ostream& os, const link& x)
+operator<<(std::ostream& os, const time_zone_link& x)
 {
     using namespace date;
     detail::save_ostream<char> _(os);
@@ -2746,9 +2752,9 @@ operator<<(std::ostream& os, const link& x)
     return os << x.name_ << " --> " << x.target_;
 }
 
-// leap
+// leap_second
 
-leap::leap(const std::string& s, detail::undocumented)
+leap_second::leap_second(const std::string& s, detail::undocumented)
 {
     using namespace date;
     std::istringstream in(s);
@@ -2840,13 +2846,15 @@ namespace
 static
 bool
 download_to_file(const std::string& url, const std::string& local_filename,
-                 download_file_options opts)
+                 download_file_options opts, char* error_buffer)
 {
     auto curl = curl_init();
     if (!curl)
         return false;
     curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str());
     curl_easy_setopt(curl.get(), CURLOPT_SSL_VERIFYPEER, false);
+    if (error_buffer)
+       curl_easy_setopt(curl.get(), CURLOPT_ERRORBUFFER, error_buffer);
     curl_write_callback write_cb = [](char* contents, std::size_t size, std::size_t nmemb,
                                       void* userp) -> std::size_t
     {
@@ -3266,7 +3274,7 @@ extract_gz_file(const std::string&, const std::string& gz_file, const std::strin
 #  endif // !_WIN32
 
 bool
-remote_download(const std::string& version)
+remote_download(const std::string& version, char* error_buffer)
 {
     assert(!version.empty());
 
@@ -3285,7 +3293,7 @@ remote_download(const std::string& version)
     auto url = "https://data.iana.org/time-zones/releases/tzdata" + version +
                ".tar.gz";
     bool result = download_to_file(url, get_download_gz_file(version),
-                                   download_file_options::binary);
+                                   download_file_options::binary, error_buffer);
 #  ifdef _WIN32
     if (result)
     {
@@ -3293,7 +3301,7 @@ remote_download(const std::string& version)
         result = download_to_file(
 			"https://raw.githubusercontent.com/unicode-org/cldr/master/"
 			"common/supplemental/windowsZones.xml",
-            mapping_file, download_file_options::text);
+            mapping_file, download_file_options::text, error_buffer);
     }
 #  endif  // _WIN32
     return result;
@@ -3442,12 +3450,12 @@ init_tzdb()
                 }
                 else if (word == "Link")
                 {
-                    db->links.push_back(link(line));
+                    db->links.push_back(time_zone_link(line));
                     continue_zone = false;
                 }
                 else if (word == "Leap")
                 {
-                    db->leaps.push_back(leap(line, detail::undocumented{}));
+                    db->leap_seconds.push_back(leap_second(line, detail::undocumented{}));
                     continue_zone = false;
                 }
                 else if (word == "Zone")
@@ -3472,8 +3480,8 @@ init_tzdb()
     db->zones.shrink_to_fit();
     std::sort(db->links.begin(), db->links.end());
     db->links.shrink_to_fit();
-    std::sort(db->leaps.begin(), db->leaps.end());
-    db->leaps.shrink_to_fit();
+    std::sort(db->leap_seconds.begin(), db->leap_seconds.end());
+    db->leap_seconds.shrink_to_fit();
 
 #ifdef _WIN32
     std::string mapping_file = get_install() + folder_delimiter + "windowsZones.xml";
@@ -3525,9 +3533,9 @@ tzdb::locate_zone(const std::string& tz_name) const
 #if !USE_OS_TZDB
         auto li = std::lower_bound(links.begin(), links.end(), tz_name,
 #if HAS_STRING_VIEW
-        [](const link& z, const std::string_view& nm)
+        [](const time_zone_link& z, const std::string_view& nm)
 #else
-        [](const link& z, const std::string& nm)
+        [](const time_zone_link& z, const std::string& nm)
 #endif
         {
             return z.name() < nm;
@@ -3568,7 +3576,7 @@ operator<<(std::ostream& os, const tzdb& db)
         os << x << '\n';
 #if !MISSING_LEAP_SECONDS
     os << '\n';
-    for (const auto& x : db.leaps)
+    for (const auto& x : db.leap_seconds)
         os << x << '\n';
 #endif  // !MISSING_LEAP_SECONDS
     return os;
@@ -3628,7 +3636,7 @@ operator<<(std::ostream& os, const tzdb& db)
                         "---------------------------------------------------------"
                         "--------------------------------------------------------\n");
     os << title;
-    for (const auto& x : db.leaps)
+    for (const auto& x : db.leap_seconds)
         os << x << '\n';
     return os;
 }
@@ -3675,6 +3683,56 @@ tzdb::current_zone() const
 
 #else  // !_WIN32
 
+#if HAS_STRING_VIEW
+
+static
+std::string_view
+extract_tz_name(char const* rp)
+{
+    using namespace std;
+    string_view result = rp;
+    CONSTDATA string_view zoneinfo = "zoneinfo";
+    size_t pos = result.rfind(zoneinfo);
+    if (pos == result.npos)
+        throw runtime_error(
+            "current_zone() failed to find \"zoneinfo\" in " + string(result));
+    pos = result.find('/', pos);
+    result.remove_prefix(pos + 1);
+    return result;
+}
+
+#else  // !HAS_STRING_VIEW
+
+static
+std::string
+extract_tz_name(char const* rp)
+{
+    using namespace std;
+    string result = rp;
+    CONSTDATA char zoneinfo[] = "zoneinfo";
+    size_t pos = result.rfind(zoneinfo);
+    if (pos == result.npos)
+        throw runtime_error(
+            "current_zone() failed to find \"zoneinfo\" in " + result);
+    pos = result.find('/', pos);
+    result.erase(0, pos + 1);
+    return result;
+}
+
+#endif  // HAS_STRING_VIEW
+
+static
+bool
+sniff_realpath(const char* timezone)
+{
+    using namespace std;
+    char rp[PATH_MAX+1] = {};
+    if (realpath(timezone, rp) == nullptr)
+        throw system_error(errno, system_category(), "realpath() failed");
+    auto result = extract_tz_name(rp);
+    return result != "posixrules";
+}
+
 const time_zone*
 tzdb::current_zone() const
 {
@@ -3694,19 +3752,22 @@ tzdb::current_zone() const
     {
         struct stat sb;
         CONSTDATA auto timezone = "/etc/localtime";
-        if (lstat(timezone, &sb) == 0 && S_ISLNK(sb.st_mode) && sb.st_size > 0) {
+        if (lstat(timezone, &sb) == 0 && S_ISLNK(sb.st_mode) && sb.st_size > 0)
+        {
             using namespace std;
-            string result;
+            static const bool use_realpath = sniff_realpath(timezone);
             char rp[PATH_MAX+1] = {};
-            if (readlink(timezone, rp, sizeof(rp)-1) > 0)
-                result = string(rp);
+            if (use_realpath)
+            {
+                if (realpath(timezone, rp) == nullptr)
+                    throw system_error(errno, system_category(), "realpath() failed");
+            }
             else
-                throw system_error(errno, system_category(), "readlink() failed");
-
-            const size_t pos = result.find(get_tz_dir());
-            if (pos != result.npos)
-                result.erase(0, get_tz_dir().size() + 1 + pos);
-            return locate_zone(result);
+            {
+                if (readlink(timezone, rp, sizeof(rp)-1) <= 0)
+                    throw system_error(errno, system_category(), "readlink() failed");
+            }
+            return locate_zone(extract_tz_name(rp));
         }
     }
     // On embedded systems e.g. buildroot with uclibc the timezone is linked
diff --git a/cpp/src/arrow/vendored/datetime/tz.h b/cpp/src/arrow/vendored/datetime/tz.h
index 3f32cec5fcf..23c6742143c 100644
--- a/cpp/src/arrow/vendored/datetime/tz.h
+++ b/cpp/src/arrow/vendored/datetime/tz.h
@@ -30,13 +30,6 @@
 // been invented (that would involve another several millennia of evolution).
 // We did not mean to shout.
 
-// NOTE(wesm): If this is not set, then the library will attempt to
-// use libcurl to obtain a timezone database, and we do not yet have
-// curl in our build toolchain
-#ifndef _WIN32
-#define USE_OS_TZDB 1
-#endif
-
 // Get more recent database at http://www.iana.org/time-zones
 
 // The notion of "current timezone" is something the operating system is expected to "just
@@ -50,6 +43,13 @@
 // required. On Windows, the names are never "Standard" so mapping is always required.
 // Technically any OS may use the mapping process but currently only Windows does use it.
 
+// NOTE(ARROW): If this is not set, then the library will attempt to
+// use libcurl to obtain a timezone database, and we do not yet have
+// curl in our build toolchain
+#ifndef _WIN32
+#define USE_OS_TZDB 1
+#endif
+
 #ifndef USE_OS_TZDB
 #  define USE_OS_TZDB 0
 #endif
@@ -160,6 +160,15 @@ enum class choose {earliest, latest};
 namespace detail
 {
     struct undocumented;
+
+    template<typename T>
+    struct nodeduct
+    {
+       using type = T;
+    };
+
+    template<typename T>
+    using nodeduct_t = typename nodeduct<T>::type;
 }
 
 struct sys_info
@@ -352,13 +361,13 @@ class zoned_time
     sys_time<duration> tp_;
 
 public:
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
     template <class T = TimeZonePtr,
               class = decltype(zoned_traits<T>::default_zone())>
 #endif
         zoned_time();
 
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
     template <class T = TimeZonePtr,
               class = decltype(zoned_traits<T>::default_zone())>
 #endif
@@ -377,7 +386,7 @@ class zoned_time
               >::type>
         explicit zoned_time(std::string_view name);
 #else
-#  if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#  if !defined(_MSC_VER) || (_MSC_VER > 1916)
     template <class T = TimeZonePtr,
               class = typename std::enable_if
               <
@@ -401,7 +410,7 @@ class zoned_time
 
     zoned_time(TimeZonePtr z, const sys_time<Duration>& st);
 
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
     template <class T = TimeZonePtr,
               class = typename std::enable_if
               <
@@ -414,7 +423,7 @@ class zoned_time
 #endif
         zoned_time(TimeZonePtr z, const local_time<Duration>& tp);
 
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
     template <class T = TimeZonePtr,
               class = typename std::enable_if
               <
@@ -456,7 +465,7 @@ class zoned_time
                       sys_time<Duration>
                   >::value
               >::type>
-        zoned_time(std::string_view name, const sys_time<Duration>& st);
+        zoned_time(std::string_view name, detail::nodeduct_t<const sys_time<Duration>&> st);
 
     template <class T = TimeZonePtr,
               class = typename std::enable_if
@@ -468,7 +477,7 @@ class zoned_time
                       local_time<Duration>
                   >::value
               >::type>
-        zoned_time(std::string_view name, const local_time<Duration>& tp);
+        zoned_time(std::string_view name, detail::nodeduct_t<const local_time<Duration>&> tp);
 
     template <class T = TimeZonePtr,
               class = typename std::enable_if
@@ -481,7 +490,7 @@ class zoned_time
                       choose
                   >::value
               >::type>
-        zoned_time(std::string_view name,   const local_time<Duration>& tp, choose c);
+        zoned_time(std::string_view name, detail::nodeduct_t<const local_time<Duration>&> tp, choose c);
 
     template <class Duration2, class TimeZonePtr2, class T = TimeZonePtr,
               class = typename std::enable_if
@@ -514,7 +523,7 @@ class zoned_time
 
 #else  // !HAS_STRING_VIEW
 
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
     template <class T = TimeZonePtr,
               class = typename std::enable_if
               <
@@ -528,7 +537,7 @@ class zoned_time
 #endif
         zoned_time(const std::string& name, const sys_time<Duration>& st);
 
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
     template <class T = TimeZonePtr,
               class = typename std::enable_if
               <
@@ -542,7 +551,7 @@ class zoned_time
 #endif
         zoned_time(const char* name, const sys_time<Duration>& st);
 
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
     template <class T = TimeZonePtr,
               class = typename std::enable_if
               <
@@ -556,7 +565,7 @@ class zoned_time
 #endif
         zoned_time(const std::string& name, const local_time<Duration>& tp);
 
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
     template <class T = TimeZonePtr,
               class = typename std::enable_if
               <
@@ -570,7 +579,7 @@ class zoned_time
 #endif
         zoned_time(const char* name, const local_time<Duration>& tp);
 
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
     template <class T = TimeZonePtr,
               class = typename std::enable_if
               <
@@ -585,7 +594,7 @@ class zoned_time
 #endif
         zoned_time(const std::string& name, const local_time<Duration>& tp, choose c);
 
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
     template <class T = TimeZonePtr,
               class = typename std::enable_if
               <
@@ -600,7 +609,7 @@ class zoned_time
 #endif
         zoned_time(const char* name, const local_time<Duration>& tp, choose c);
 
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
     template <class Duration2, class TimeZonePtr2, class T = TimeZonePtr,
               class = typename std::enable_if
                       <
@@ -618,7 +627,7 @@ class zoned_time
 #endif
         zoned_time(const std::string& name, const zoned_time<Duration2, TimeZonePtr2>& zt);
 
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
     template <class Duration2, class TimeZonePtr2, class T = TimeZonePtr,
               class = typename std::enable_if
                       <
@@ -636,7 +645,7 @@ class zoned_time
 #endif
         zoned_time(const char* name, const zoned_time<Duration2, TimeZonePtr2>& zt);
 
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
     template <class Duration2, class TimeZonePtr2, class T = TimeZonePtr,
               class = typename std::enable_if
                       <
@@ -656,7 +665,7 @@ class zoned_time
         zoned_time(const std::string& name, const zoned_time<Duration2, TimeZonePtr2>& zt,
                    choose);
 
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
     template <class Duration2, class TimeZonePtr2, class T = TimeZonePtr,
               class = typename std::enable_if
                       <
@@ -709,6 +718,18 @@ using zoned_seconds = zoned_time<std::chrono::seconds>;
 
 #if HAS_DEDUCTION_GUIDES
 
+namespace detail
+{
+   template<typename TimeZonePtrOrName>
+   using time_zone_representation =
+       std::conditional_t
+       <
+           std::is_convertible<TimeZonePtrOrName, std::string_view>::value,
+           time_zone const*,
+           std::remove_cv_t<std::remove_reference_t<TimeZonePtrOrName>>
+       >;
+}
+
 zoned_time()
     -> zoned_time<std::chrono::seconds>;
 
@@ -717,44 +738,20 @@ zoned_time(sys_time<Duration>)
     -> zoned_time<std::common_type_t<Duration, std::chrono::seconds>>;
 
 template <class TimeZonePtrOrName>
-zoned_time(TimeZonePtrOrName)
-    -> zoned_time<std::chrono::seconds,
-                  std::conditional_t
-                  <
-                      std::is_convertible<TimeZonePtrOrName, std::string_view>::value,
-                      time_zone const*,
-                      TimeZonePtrOrName
-                  >>;
+zoned_time(TimeZonePtrOrName&&)
+    -> zoned_time<std::chrono::seconds, detail::time_zone_representation<TimeZonePtrOrName>>;
 
 template <class TimeZonePtrOrName, class Duration>
-zoned_time(TimeZonePtrOrName, sys_time<Duration>)
-    -> zoned_time<std::common_type_t<Duration, std::chrono::seconds>,
-                  std::conditional_t
-                  <
-                      std::is_convertible<TimeZonePtrOrName, std::string_view>::value,
-                      time_zone const*,
-                      TimeZonePtrOrName
-                  >>;
+zoned_time(TimeZonePtrOrName&&, sys_time<Duration>)
+    -> zoned_time<std::common_type_t<Duration, std::chrono::seconds>, detail::time_zone_representation<TimeZonePtrOrName>>;
 
 template <class TimeZonePtrOrName, class Duration>
-zoned_time(TimeZonePtrOrName, local_time<Duration>, choose = choose::earliest)
-    -> zoned_time<std::common_type_t<Duration, std::chrono::seconds>,
-                  std::conditional_t
-                  <
-                      std::is_convertible<TimeZonePtrOrName, std::string_view>::value,
-                      time_zone const*,
-                      TimeZonePtrOrName
-                  >>;
+zoned_time(TimeZonePtrOrName&&, local_time<Duration>, choose = choose::earliest)
+    -> zoned_time<std::common_type_t<Duration, std::chrono::seconds>, detail::time_zone_representation<TimeZonePtrOrName>>;
 
 template <class Duration, class TimeZonePtrOrName, class TimeZonePtr2>
-zoned_time(TimeZonePtrOrName, zoned_time<Duration, TimeZonePtr2>, choose = choose::earliest)
-    -> zoned_time<std::common_type_t<Duration, std::chrono::seconds>,
-                  std::conditional_t
-                  <
-                      std::is_convertible<TimeZonePtrOrName, std::string_view>::value,
-                      time_zone const*,
-                      TimeZonePtrOrName
-                  >>;
+zoned_time(TimeZonePtrOrName&&, zoned_time<Duration, TimeZonePtr2>, choose = choose::earliest)
+    -> zoned_time<std::common_type_t<Duration, std::chrono::seconds>, detail::time_zone_representation<TimeZonePtrOrName>>;
 
 #endif  // HAS_DEDUCTION_GUIDES
 
@@ -976,53 +973,55 @@ time_zone::to_sys_impl(local_time<Duration> tp, choose, std::true_type) const
 
 #if !USE_OS_TZDB
 
-class link
+class time_zone_link
 {
 private:
     std::string name_;
     std::string target_;
 public:
-    DATE_API explicit link(const std::string& s);
+    DATE_API explicit time_zone_link(const std::string& s);
 
     const std::string& name() const {return name_;}
     const std::string& target() const {return target_;}
 
-    friend bool operator==(const link& x, const link& y) {return x.name_ == y.name_;}
-    friend bool operator< (const link& x, const link& y) {return x.name_ < y.name_;}
+    friend bool operator==(const time_zone_link& x, const time_zone_link& y) {return x.name_ == y.name_;}
+    friend bool operator< (const time_zone_link& x, const time_zone_link& y) {return x.name_ < y.name_;}
 
-    friend DATE_API std::ostream& operator<<(std::ostream& os, const link& x);
+    friend DATE_API std::ostream& operator<<(std::ostream& os, const time_zone_link& x);
 };
 
-inline bool operator!=(const link& x, const link& y) {return !(x == y);}
-inline bool operator> (const link& x, const link& y) {return   y < x;}
-inline bool operator<=(const link& x, const link& y) {return !(y < x);}
-inline bool operator>=(const link& x, const link& y) {return !(x < y);}
+using link = time_zone_link;
+
+inline bool operator!=(const time_zone_link& x, const time_zone_link& y) {return !(x == y);}
+inline bool operator> (const time_zone_link& x, const time_zone_link& y) {return   y < x;}
+inline bool operator<=(const time_zone_link& x, const time_zone_link& y) {return !(y < x);}
+inline bool operator>=(const time_zone_link& x, const time_zone_link& y) {return !(x < y);}
 
 #endif  // !USE_OS_TZDB
 
 #if !MISSING_LEAP_SECONDS
 
-class leap
+class leap_second
 {
 private:
     sys_seconds date_;
 
 public:
 #if USE_OS_TZDB
-    DATE_API explicit leap(const sys_seconds& s, detail::undocumented);
+    DATE_API explicit leap_second(const sys_seconds& s, detail::undocumented);
 #else
-    DATE_API explicit leap(const std::string& s, detail::undocumented);
+    DATE_API explicit leap_second(const std::string& s, detail::undocumented);
 #endif
 
     sys_seconds date() const {return date_;}
 
-    friend bool operator==(const leap& x, const leap& y) {return x.date_ == y.date_;}
-    friend bool operator< (const leap& x, const leap& y) {return x.date_ < y.date_;}
+    friend bool operator==(const leap_second& x, const leap_second& y) {return x.date_ == y.date_;}
+    friend bool operator< (const leap_second& x, const leap_second& y) {return x.date_ < y.date_;}
 
     template <class Duration>
     friend
     bool
-    operator==(const leap& x, const sys_time<Duration>& y)
+    operator==(const leap_second& x, const sys_time<Duration>& y)
     {
         return x.date_ == y;
     }
@@ -1030,7 +1029,7 @@ class leap
     template <class Duration>
     friend
     bool
-    operator< (const leap& x, const sys_time<Duration>& y)
+    operator< (const leap_second& x, const sys_time<Duration>& y)
     {
         return x.date_ < y;
     }
@@ -1038,23 +1037,23 @@ class leap
     template <class Duration>
     friend
     bool
-    operator< (const sys_time<Duration>& x, const leap& y)
+    operator< (const sys_time<Duration>& x, const leap_second& y)
     {
         return x < y.date_;
     }
 
-    friend DATE_API std::ostream& operator<<(std::ostream& os, const leap& x);
+    friend DATE_API std::ostream& operator<<(std::ostream& os, const leap_second& x);
 };
 
-inline bool operator!=(const leap& x, const leap& y) {return !(x == y);}
-inline bool operator> (const leap& x, const leap& y) {return   y < x;}
-inline bool operator<=(const leap& x, const leap& y) {return !(y < x);}
-inline bool operator>=(const leap& x, const leap& y) {return !(x < y);}
+inline bool operator!=(const leap_second& x, const leap_second& y) {return !(x == y);}
+inline bool operator> (const leap_second& x, const leap_second& y) {return   y < x;}
+inline bool operator<=(const leap_second& x, const leap_second& y) {return !(y < x);}
+inline bool operator>=(const leap_second& x, const leap_second& y) {return !(x < y);}
 
 template <class Duration>
 inline
 bool
-operator==(const sys_time<Duration>& x, const leap& y)
+operator==(const sys_time<Duration>& x, const leap_second& y)
 {
     return y == x;
 }
@@ -1062,7 +1061,7 @@ operator==(const sys_time<Duration>& x, const leap& y)
 template <class Duration>
 inline
 bool
-operator!=(const leap& x, const sys_time<Duration>& y)
+operator!=(const leap_second& x, const sys_time<Duration>& y)
 {
     return !(x == y);
 }
@@ -1070,7 +1069,7 @@ operator!=(const leap& x, const sys_time<Duration>& y)
 template <class Duration>
 inline
 bool
-operator!=(const sys_time<Duration>& x, const leap& y)
+operator!=(const sys_time<Duration>& x, const leap_second& y)
 {
     return !(x == y);
 }
@@ -1078,7 +1077,7 @@ operator!=(const sys_time<Duration>& x, const leap& y)
 template <class Duration>
 inline
 bool
-operator> (const leap& x, const sys_time<Duration>& y)
+operator> (const leap_second& x, const sys_time<Duration>& y)
 {
     return y < x;
 }
@@ -1086,7 +1085,7 @@ operator> (const leap& x, const sys_time<Duration>& y)
 template <class Duration>
 inline
 bool
-operator> (const sys_time<Duration>& x, const leap& y)
+operator> (const sys_time<Duration>& x, const leap_second& y)
 {
     return y < x;
 }
@@ -1094,7 +1093,7 @@ operator> (const sys_time<Duration>& x, const leap& y)
 template <class Duration>
 inline
 bool
-operator<=(const leap& x, const sys_time<Duration>& y)
+operator<=(const leap_second& x, const sys_time<Duration>& y)
 {
     return !(y < x);
 }
@@ -1102,7 +1101,7 @@ operator<=(const leap& x, const sys_time<Duration>& y)
 template <class Duration>
 inline
 bool
-operator<=(const sys_time<Duration>& x, const leap& y)
+operator<=(const sys_time<Duration>& x, const leap_second& y)
 {
     return !(y < x);
 }
@@ -1110,7 +1109,7 @@ operator<=(const sys_time<Duration>& x, const leap& y)
 template <class Duration>
 inline
 bool
-operator>=(const leap& x, const sys_time<Duration>& y)
+operator>=(const leap_second& x, const sys_time<Duration>& y)
 {
     return !(x < y);
 }
@@ -1118,11 +1117,13 @@ operator>=(const leap& x, const sys_time<Duration>& y)
 template <class Duration>
 inline
 bool
-operator>=(const sys_time<Duration>& x, const leap& y)
+operator>=(const sys_time<Duration>& x, const leap_second& y)
 {
     return !(x < y);
 }
 
+using leap = leap_second;
+
 #endif  // !MISSING_LEAP_SECONDS
 
 #ifdef _WIN32
@@ -1165,16 +1166,16 @@ struct timezone_mapping
 
 struct tzdb
 {
-    std::string               version = "unknown";
-    std::vector<time_zone>    zones;
+    std::string                 version = "unknown";
+    std::vector<time_zone>      zones;
 #if !USE_OS_TZDB
-    std::vector<link>         links;
+    std::vector<time_zone_link> links;
 #endif
 #if !MISSING_LEAP_SECONDS
-    std::vector<leap>         leaps;
+    std::vector<leap_second>    leap_seconds;
 #endif
 #if !USE_OS_TZDB
-    std::vector<detail::Rule> rules;
+    std::vector<detail::Rule>   rules;
 #endif
 #ifdef _WIN32
     std::vector<detail::timezone_mapping> mappings;
@@ -1190,7 +1191,7 @@ struct tzdb
         : version(std::move(src.version))
         , zones(std::move(src.zones))
         , links(std::move(src.links))
-        , leaps(std::move(src.leaps))
+        , leap_seconds(std::move(src.leap_seconds))
         , rules(std::move(src.rules))
         , mappings(std::move(src.mappings))
     {}
@@ -1200,7 +1201,7 @@ struct tzdb
         version = std::move(src.version);
         zones = std::move(src.zones);
         links = std::move(src.links);
-        leaps = std::move(src.leaps);
+        leap_seconds = std::move(src.leap_seconds);
         rules = std::move(src.rules);
         mappings = std::move(src.mappings);
         return *this;
@@ -1322,7 +1323,8 @@ DATE_API void        set_install(const std::string& install);
 #if HAS_REMOTE_API
 
 DATE_API std::string remote_version();
-DATE_API bool        remote_download(const std::string& version);
+// if provided error_buffer size should be at least CURL_ERROR_SIZE
+DATE_API bool        remote_download(const std::string& version, char* error_buffer = nullptr);
 DATE_API bool        remote_install(const std::string& version);
 
 #endif
@@ -1352,7 +1354,7 @@ to_raw_pointer(Pointer p) noexcept
 }  // namespace detail
 
 template <class Duration, class TimeZonePtr>
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
 template <class T, class>
 #endif
 inline
@@ -1361,7 +1363,7 @@ zoned_time<Duration, TimeZonePtr>::zoned_time()
     {}
 
 template <class Duration, class TimeZonePtr>
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
 template <class T, class>
 #endif
 inline
@@ -1388,7 +1390,7 @@ zoned_time<Duration, TimeZonePtr>::zoned_time(std::string_view name)
 #else  // !HAS_STRING_VIEW
 
 template <class Duration, class TimeZonePtr>
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
 template <class T, class>
 #endif
 inline
@@ -1414,7 +1416,7 @@ zoned_time<Duration, TimeZonePtr>::zoned_time(TimeZonePtr z, const sys_time<Dura
     {}
 
 template <class Duration, class TimeZonePtr>
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
 template <class T, class>
 #endif
 inline
@@ -1424,7 +1426,7 @@ zoned_time<Duration, TimeZonePtr>::zoned_time(TimeZonePtr z, const local_time<Du
     {}
 
 template <class Duration, class TimeZonePtr>
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
 template <class T, class>
 #endif
 inline
@@ -1457,7 +1459,7 @@ template <class Duration, class TimeZonePtr>
 template <class T, class>
 inline
 zoned_time<Duration, TimeZonePtr>::zoned_time(std::string_view name,
-                                              const sys_time<Duration>& st)
+                                              detail::nodeduct_t<const sys_time<Duration>&> st)
     : zoned_time(zoned_traits<TimeZonePtr>::locate_zone(name), st)
     {}
 
@@ -1465,7 +1467,7 @@ template <class Duration, class TimeZonePtr>
 template <class T, class>
 inline
 zoned_time<Duration, TimeZonePtr>::zoned_time(std::string_view name,
-                                              const local_time<Duration>& t)
+                                              detail::nodeduct_t<const local_time<Duration>&> t)
     : zoned_time(zoned_traits<TimeZonePtr>::locate_zone(name), t)
     {}
 
@@ -1473,7 +1475,7 @@ template <class Duration, class TimeZonePtr>
 template <class T, class>
 inline
 zoned_time<Duration, TimeZonePtr>::zoned_time(std::string_view name,
-                                              const local_time<Duration>& t, choose c)
+                                              detail::nodeduct_t<const local_time<Duration>&> t, choose c)
     : zoned_time(zoned_traits<TimeZonePtr>::locate_zone(name), t, c)
     {}
 
@@ -1497,7 +1499,7 @@ zoned_time<Duration, TimeZonePtr>::zoned_time(std::string_view name,
 #else  // !HAS_STRING_VIEW
 
 template <class Duration, class TimeZonePtr>
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
 template <class T, class>
 #endif
 inline
@@ -1507,7 +1509,7 @@ zoned_time<Duration, TimeZonePtr>::zoned_time(const std::string& name,
     {}
 
 template <class Duration, class TimeZonePtr>
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
 template <class T, class>
 #endif
 inline
@@ -1517,7 +1519,7 @@ zoned_time<Duration, TimeZonePtr>::zoned_time(const char* name,
     {}
 
 template <class Duration, class TimeZonePtr>
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
 template <class T, class>
 #endif
 inline
@@ -1527,7 +1529,7 @@ zoned_time<Duration, TimeZonePtr>::zoned_time(const std::string& name,
     {}
 
 template <class Duration, class TimeZonePtr>
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
 template <class T, class>
 #endif
 inline
@@ -1537,7 +1539,7 @@ zoned_time<Duration, TimeZonePtr>::zoned_time(const char* name,
     {}
 
 template <class Duration, class TimeZonePtr>
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
 template <class T, class>
 #endif
 inline
@@ -1547,7 +1549,7 @@ zoned_time<Duration, TimeZonePtr>::zoned_time(const std::string& name,
     {}
 
 template <class Duration, class TimeZonePtr>
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
 template <class T, class>
 #endif
 inline
@@ -1557,7 +1559,7 @@ zoned_time<Duration, TimeZonePtr>::zoned_time(const char* name,
     {}
 
 template <class Duration, class TimeZonePtr>
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
 template <class Duration2, class TimeZonePtr2, class, class>
 #else
 template <class Duration2, class TimeZonePtr2>
@@ -1569,7 +1571,7 @@ zoned_time<Duration, TimeZonePtr>::zoned_time(const std::string& name,
     {}
 
 template <class Duration, class TimeZonePtr>
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
 template <class Duration2, class TimeZonePtr2, class, class>
 #else
 template <class Duration2, class TimeZonePtr2>
@@ -1581,7 +1583,7 @@ zoned_time<Duration, TimeZonePtr>::zoned_time(const char* name,
     {}
 
 template <class Duration, class TimeZonePtr>
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
 template <class Duration2, class TimeZonePtr2, class, class>
 #else
 template <class Duration2, class TimeZonePtr2>
@@ -1594,7 +1596,7 @@ zoned_time<Duration, TimeZonePtr>::zoned_time(const std::string& name,
     {}
 
 template <class Duration, class TimeZonePtr>
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
 template <class Duration2, class TimeZonePtr2, class, class>
 #else
 template <class Duration2, class TimeZonePtr2>
@@ -1690,7 +1692,8 @@ make_zoned(const sys_time<Duration>& tp)
 }
 
 template <class TimeZonePtr
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
+#if !defined(__INTEL_COMPILER) || (__INTEL_COMPILER > 1600)
           , class = typename std::enable_if
           <
             std::is_class
@@ -1701,6 +1704,7 @@ template <class TimeZonePtr
                 >::type
             >{}
           >::type
+#endif
 #endif
          >
 inline
@@ -1718,11 +1722,13 @@ make_zoned(const std::string& name)
 }
 
 template <class Duration, class TimeZonePtr
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
+#if !defined(__INTEL_COMPILER) || (__INTEL_COMPILER > 1600)
           , class = typename std::enable_if
           <
             std::is_class<typename std::decay<decltype(*std::declval<TimeZonePtr&>())>::type>{}
           >::type
+#endif
 #endif
          >
 inline
@@ -1734,11 +1740,13 @@ make_zoned(TimeZonePtr zone, const local_time<Duration>& tp)
 }
 
 template <class Duration, class TimeZonePtr
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
+#if !defined(__INTEL_COMPILER) || (__INTEL_COMPILER > 1600)
           , class = typename std::enable_if
           <
             std::is_class<typename std::decay<decltype(*std::declval<TimeZonePtr&>())>::type>{}
           >::type
+#endif
 #endif
          >
 inline
@@ -1800,11 +1808,13 @@ make_zoned(const std::string& name, const zoned_time<Duration, TimeZonePtr>& zt,
 }
 
 template <class Duration, class TimeZonePtr
-#if !defined(_MSC_VER) || (_MSC_VER > 1900)
+#if !defined(_MSC_VER) || (_MSC_VER > 1916)
+#if !defined(__INTEL_COMPILER) || (__INTEL_COMPILER > 1600)
           , class = typename std::enable_if
           <
             std::is_class<typename std::decay<decltype(*std::declval<TimeZonePtr&>())>::type>{}
           >::type
+#endif
 #endif
          >
 inline
@@ -1891,7 +1901,7 @@ utc_clock::from_sys(const sys_time<Duration>& st)
 {
     using std::chrono::seconds;
     using CD = typename std::common_type<Duration, seconds>::type;
-    auto const& leaps = get_tzdb().leaps;
+    auto const& leaps = get_tzdb().leap_seconds;
     auto const lt = std::upper_bound(leaps.begin(), leaps.end(), st);
     return utc_time<CD>{st.time_since_epoch() + seconds{lt-leaps.begin()}};
 }
@@ -1905,7 +1915,7 @@ is_leap_second(date::utc_time<Duration> const& ut)
 {
     using std::chrono::seconds;
     using duration = typename std::common_type<Duration, seconds>::type;
-    auto const& leaps = get_tzdb().leaps;
+    auto const& leaps = get_tzdb().leap_seconds;
     auto tp = sys_time<duration>{ut.time_since_epoch()};
     auto const lt = std::upper_bound(leaps.begin(), leaps.end(), tp);
     auto ds = seconds{lt-leaps.begin()};
@@ -2287,6 +2297,7 @@ template <>
 struct clock_time_conversion<std::chrono::system_clock, std::chrono::system_clock>
 {
     template <class Duration>
+    CONSTCD14
     sys_time<Duration>
     operator()(const sys_time<Duration>& st) const
     {
@@ -2298,6 +2309,7 @@ template <>
 struct clock_time_conversion<utc_clock, utc_clock>
 {
     template <class Duration>
+    CONSTCD14
     utc_time<Duration>
     operator()(const utc_time<Duration>& ut) const
     {
@@ -2309,6 +2321,7 @@ template<>
 struct clock_time_conversion<local_t, local_t>
 {
     template <class Duration>
+    CONSTCD14
     local_time<Duration>
     operator()(const local_time<Duration>& lt) const
     {
@@ -2342,6 +2355,7 @@ template<>
 struct clock_time_conversion<local_t, std::chrono::system_clock>
 {
     template <class Duration>
+    CONSTCD14
     local_time<Duration>
     operator()(const sys_time<Duration>& st) const
     {
@@ -2353,6 +2367,7 @@ template<>
 struct clock_time_conversion<std::chrono::system_clock, local_t>
 {
     template <class Duration>
+    CONSTCD14
     sys_time<Duration>
     operator()(const local_time<Duration>& lt) const
     {
@@ -2386,6 +2401,7 @@ template<typename Clock>
 struct clock_time_conversion<Clock, Clock>
 {
     template <class Duration>
+    CONSTCD14
     std::chrono::time_point<Clock, Duration>
     operator()(const std::chrono::time_point<Clock, Duration>& tp) const
     {
@@ -2532,6 +2548,7 @@ template <class SrcClock>
 struct clock_time_conversion<std::chrono::system_clock, SrcClock>
 {
     template <class Duration>
+    CONSTCD14
     typename ctc_detail::return_to_sys<SrcClock, Duration>::type
     operator()(const std::chrono::time_point<SrcClock, Duration>& tp) const
     {
@@ -2543,6 +2560,7 @@ template <class DstClock>
 struct clock_time_conversion<DstClock, std::chrono::system_clock>
 {
     template <class Duration>
+    CONSTCD14
     typename ctc_detail::return_from_sys<DstClock, Duration>::type
     operator()(const sys_time<Duration>& st) const
     {
@@ -2554,6 +2572,7 @@ template <class SrcClock>
 struct clock_time_conversion<utc_clock, SrcClock>
 {
     template <class Duration>
+    CONSTCD14
     typename ctc_detail::return_to_utc<SrcClock, Duration>::type
     operator()(const std::chrono::time_point<SrcClock, Duration>& tp) const
     {
@@ -2565,6 +2584,7 @@ template <class DstClock>
 struct clock_time_conversion<DstClock, utc_clock>
 {
     template <class Duration>
+    CONSTCD14
     typename ctc_detail::return_from_utc<DstClock, Duration>::type
     operator()(const utc_time<Duration>& ut) const
     {
@@ -2576,6 +2596,7 @@ template<typename SrcClock>
 struct clock_time_conversion<local_t, SrcClock>
 {
     template <class Duration>
+    CONSTCD14
     typename ctc_detail::return_to_local<SrcClock, Duration>::type
     operator()(const std::chrono::time_point<SrcClock, Duration>& tp) const
     {
@@ -2587,6 +2608,7 @@ template<typename DstClock>
 struct clock_time_conversion<DstClock, local_t>
 {
     template <class Duration>
+    CONSTCD14
     typename ctc_detail::return_from_local<DstClock, Duration>::type
     operator()(const local_time<Duration>& lt) const
     {
@@ -2602,6 +2624,7 @@ template <class Clock, class Duration>
 using std::chrono::system_clock;
 
 template <class DstClock, class SrcClock, class Duration>
+CONSTCD14
 auto
 conv_clock(const time_point<SrcClock, Duration>& t)
     -> decltype(std::declval<clock_time_conversion<DstClock, SrcClock>>()(t))
@@ -2611,6 +2634,7 @@ conv_clock(const time_point<SrcClock, Duration>& t)
 
 //direct trait conversion, 1st candidate
 template <class DstClock, class SrcClock, class Duration>
+CONSTCD14
 auto
 cc_impl(const time_point<SrcClock, Duration>& t, const time_point<SrcClock, Duration>*)
     -> decltype(conv_clock<DstClock>(t))
@@ -2620,6 +2644,7 @@ cc_impl(const time_point<SrcClock, Duration>& t, const time_point<SrcClock, Dura
 
 //conversion through sys, 2nd candidate
 template <class DstClock, class SrcClock, class Duration>
+CONSTCD14
 auto
 cc_impl(const time_point<SrcClock, Duration>& t, const void*)
     -> decltype(conv_clock<DstClock>(conv_clock<system_clock>(t)))
@@ -2629,6 +2654,7 @@ cc_impl(const time_point<SrcClock, Duration>& t, const void*)
 
 //conversion through utc, 2nd candidate
 template <class DstClock, class SrcClock, class Duration>
+CONSTCD14
 auto
 cc_impl(const time_point<SrcClock, Duration>& t, const void*)
     -> decltype(0,  // MSVC_WORKAROUND
@@ -2639,6 +2665,7 @@ cc_impl(const time_point<SrcClock, Duration>& t, const void*)
 
 //conversion through sys and utc, 3rd candidate
 template <class DstClock, class SrcClock, class Duration>
+CONSTCD14
 auto
 cc_impl(const time_point<SrcClock, Duration>& t, ...)
     -> decltype(conv_clock<DstClock>(conv_clock<utc_clock>(conv_clock<system_clock>(t))))
@@ -2648,6 +2675,7 @@ cc_impl(const time_point<SrcClock, Duration>& t, ...)
 
 //conversion through utc and sys, 3rd candidate
 template <class DstClock, class SrcClock, class Duration>
+CONSTCD14
 auto
 cc_impl(const time_point<SrcClock, Duration>& t, ...)
     -> decltype(0,  // MSVC_WORKAROUND
@@ -2659,6 +2687,7 @@ cc_impl(const time_point<SrcClock, Duration>& t, ...)
 }  // namespace clock_cast_detail
 
 template <class DstClock, class SrcClock, class Duration>
+CONSTCD14
 auto
 clock_cast(const std::chrono::time_point<SrcClock, Duration>& tp)
     -> decltype(clock_cast_detail::cc_impl<DstClock>(tp, &tp))
diff --git a/cpp/src/arrow/vendored/datetime/tz_private.h b/cpp/src/arrow/vendored/datetime/tz_private.h
index 039e56585a3..282842e7441 100644
--- a/cpp/src/arrow/vendored/datetime/tz_private.h
+++ b/cpp/src/arrow/vendored/datetime/tz_private.h
@@ -97,9 +97,9 @@ class MonthDayTime
         U& operator=(const pair& x);
     } u;
 
-    std::chrono::hours           h_{0};
-    std::chrono::minutes         m_{0};
-    std::chrono::seconds         s_{0};
+    std::chrono::hours           h_{};
+    std::chrono::minutes         m_{};
+    std::chrono::seconds         s_{};
     tz                           zone_{tz::local};
 
 public:
diff --git a/cpp/src/arrow/vendored/fast_float/README.md b/cpp/src/arrow/vendored/fast_float/README.md
new file mode 100644
index 00000000000..4e0728c69ec
--- /dev/null
+++ b/cpp/src/arrow/vendored/fast_float/README.md
@@ -0,0 +1,9 @@
+The files in this directory are vendored from fast_float
+git changeset `dc46ad4c606dc35cb63c947496a18ef8ab1e0f44`.
+
+See https://github.com/lemire/fast_float
+
+Changes:
+- fixed include paths
+- disabled unused `print()` function
+- enclosed in `arrow_vendored` namespace.
diff --git a/cpp/src/arrow/vendored/fast_float/ascii_number.h b/cpp/src/arrow/vendored/fast_float/ascii_number.h
new file mode 100644
index 00000000000..d1f8af4087c
--- /dev/null
+++ b/cpp/src/arrow/vendored/fast_float/ascii_number.h
@@ -0,0 +1,313 @@
+#ifndef FASTFLOAT_ASCII_NUMBER_H
+#define FASTFLOAT_ASCII_NUMBER_H
+
+#include <cstdio>
+#include <cctype>
+#include <cstdint>
+#include <cstring>
+
+#include "float_common.h"
+
+namespace arrow_vendored {
+namespace fast_float {
+
+fastfloat_really_inline bool is_integer(char c)  noexcept  { return (c >= '0' && c <= '9'); }
+
+
+// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/
+fastfloat_really_inline uint32_t parse_eight_digits_unrolled(const char *chars)  noexcept  {
+  uint64_t val;
+  memcpy(&val, chars, sizeof(uint64_t));
+  val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8;
+  val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16;
+  return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32);
+}
+
+fastfloat_really_inline bool is_made_of_eight_digits_fast(const char *chars)  noexcept  {
+  uint64_t val;
+  memcpy(&val, chars, 8);
+  return (((val & 0xF0F0F0F0F0F0F0F0) |
+           (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) ==
+          0x3333333333333333);
+}
+
+
+fastfloat_really_inline uint32_t parse_four_digits_unrolled(const char *chars)  noexcept  {
+  uint32_t val;
+  memcpy(&val, chars, sizeof(uint32_t));
+  val = (val & 0x0F0F0F0F) * 2561 >> 8;
+  return (val & 0x00FF00FF) * 6553601 >> 16;
+}
+
+fastfloat_really_inline bool is_made_of_four_digits_fast(const char *chars)  noexcept  {
+  uint32_t val;
+  memcpy(&val, chars, 4);
+  return (((val & 0xF0F0F0F0) |
+           (((val + 0x06060606) & 0xF0F0F0F0) >> 4)) ==
+          0x33333333);
+}
+
+struct parsed_number_string {
+  int64_t exponent;
+  uint64_t mantissa;
+  const char *lastmatch;
+  bool negative;
+  bool valid;
+  bool too_many_digits;
+};
+
+
+// Assuming that you use no more than 17 digits, this will
+// parse an ASCII string.
+fastfloat_really_inline
+parsed_number_string parse_number_string(const char *p, const char *pend, chars_format fmt) noexcept {
+  parsed_number_string answer;
+  answer.valid = false;
+  answer.negative = (*p == '-');
+  if ((*p == '-') || (*p == '+')) {
+    ++p;
+    if (p == pend) {
+      return answer;
+    }
+    if (!is_integer(*p) && (*p != '.')) { // a  sign must be followed by an integer or the dot
+      return answer;
+    }
+  }
+  const char *const start_digits = p;
+
+  uint64_t i = 0; // an unsigned int avoids signed overflows (which are bad)
+
+  while ((p != pend) && is_integer(*p)) {
+    // a multiplication by 10 is cheaper than an arbitrary integer
+    // multiplication
+    i = 10 * i +
+        (*p - '0'); // might overflow, we will handle the overflow later
+    ++p;
+  }
+  int64_t exponent = 0;
+  if ((p != pend) && (*p == '.')) {
+    ++p;
+    const char *first_after_period = p;
+    if ((p + 8 <= pend) && is_made_of_eight_digits_fast(p)) {
+      i = i * 100000000 + parse_eight_digits_unrolled(p); // in rare cases, this will overflow, but that's ok
+      p += 8;
+      if ((p + 8 <= pend) && is_made_of_eight_digits_fast(p)) {
+        i = i * 100000000 + parse_eight_digits_unrolled(p); // in rare cases, this will overflow, but that's ok
+        p += 8;
+      }
+    }
+    while ((p != pend) && is_integer(*p)) {
+      uint8_t digit = uint8_t(*p - '0');
+      ++p;
+      i = i * 10 + digit; // in rare cases, this will overflow, but that's ok
+    }
+    exponent = first_after_period - p;
+  }
+  // we must have encountered at least one integer!
+  if ((start_digits == p) || ((start_digits == p - 1) && (*start_digits == '.') )) {
+    return answer;
+  }
+
+  int32_t digit_count =
+      int32_t(p - start_digits - 1); // used later to guard against overflows
+  
+  if ((p != pend) && (('e' == *p) || ('E' == *p))) {
+    if((fmt & chars_format::fixed) && !(fmt & chars_format::scientific)) { return answer; } 
+    int64_t exp_number = 0;            // exponential part
+    ++p;
+    bool neg_exp = false;
+    if ((p != pend) && ('-' == *p)) {
+      neg_exp = true;
+      ++p;
+    } else if ((p != pend) && ('+' == *p)) {
+      ++p;
+    }
+    if ((p == pend) || !is_integer(*p)) {
+      return answer;
+    }
+    while ((p != pend) && is_integer(*p)) {
+      uint8_t digit = uint8_t(*p - '0');
+      if (exp_number < 0x10000) {
+        exp_number = 10 * exp_number + digit;
+      }
+      ++p;
+    }
+    exponent += (neg_exp ? -exp_number : exp_number);
+  } else {
+    if((fmt & chars_format::scientific) && !(fmt & chars_format::fixed)) { return answer; } 
+  }
+  answer.lastmatch = p;
+  answer.valid = true;
+
+  // If we frequently had to deal with long strings of digits,
+  // we could extend our code by using a 128-bit integer instead
+  // of a 64-bit integer. However, this is uncommon.
+  if (((digit_count >= 19))) { // this is uncommon
+    // It is possible that the integer had an overflow.
+    // We have to handle the case where we have 0.0000somenumber.
+    const char *start = start_digits;
+    while (*start == '0' || (*start == '.')) {
+      start++;
+    }
+    // we over-decrement by one when there is a decimal separator
+    digit_count -= int(start - start_digits);
+    if (digit_count >= 19) {
+      answer.mantissa = 0xFFFFFFFFFFFFFFFF; // important: we don't want the mantissa to be used in a fast path uninitialized.
+      answer.too_many_digits = true;
+      return answer;
+    }
+  }
+  answer.too_many_digits = false;
+  answer.exponent = exponent;
+  answer.mantissa = i;
+  return answer;
+}
+
+// This should always succeed since it follows a call to parse_number_string.
+// It assumes that there are more than 19 mantissa digits to parse.
+parsed_number_string parse_truncated_decimal(const char *&p, const char *pend)  noexcept  {
+  parsed_number_string answer;
+  answer.valid = true;
+  answer.negative = (*p == '-');
+  if ((*p == '-') || (*p == '+')) {
+    ++p;
+  }
+  size_t number_of_digits{0};
+
+
+  uint64_t i = 0; 
+
+  while ((p != pend) && is_integer(*p)) {
+    // a multiplication by 10 is cheaper than an arbitrary integer
+    // multiplication
+    if(number_of_digits < 19) {
+
+      uint8_t digit = uint8_t(*p - '0');
+      i = 10 * i + digit;
+      number_of_digits ++;
+    }
+    ++p;
+  }
+  int64_t exponent = 0;
+  if ((p != pend) && (*p == '.')) {
+    ++p;
+    const char *first_after_period = p;
+   
+    while ((p != pend) && is_integer(*p)) {
+      if(number_of_digits < 19) {
+        uint8_t digit = uint8_t(*p - '0');
+        i = i * 10 + digit;
+        number_of_digits ++;
+      } else if (exponent == 0) {
+        exponent = first_after_period - p;
+      }
+      ++p;
+    }
+  }
+
+  if ((p != pend) && (('e' == *p) || ('E' == *p))) {
+    int64_t exp_number = 0;            // exponential part
+    ++p;
+    bool neg_exp = false;
+    if ((p != pend) && ('-' == *p)) {
+      neg_exp = true;
+      ++p;
+    } else if ((p != pend) && ('+' == *p)) {
+      ++p;
+    }
+    if ((p == pend) || !is_integer(*p)) {
+      return answer;
+    }
+    while ((p != pend) && is_integer(*p)) {
+      uint8_t digit = uint8_t(*p - '0');
+      if (exp_number < 0x10000) {
+        exp_number = 10 * exp_number + digit;
+      }
+      ++p;
+    }
+    exponent += (neg_exp ? -exp_number : exp_number);
+  } 
+  answer.lastmatch = p;
+  answer.valid = true;
+  answer.too_many_digits = true; // assumed
+  answer.exponent = exponent;
+  answer.mantissa = i;
+  return answer;
+}
+
+
+// This should always succeed since it follows a call to parse_number_string.
+decimal parse_decimal(const char *&p, const char *pend)  noexcept  {
+  decimal answer;
+  answer.num_digits = 0;
+  answer.decimal_point = 0;
+  answer.negative = false;
+  answer.truncated = false;
+  // skip leading whitespace
+  while (fast_float::is_space(*p)) {
+    p++;
+  }
+  answer.negative = (*p == '-');
+  if ((*p == '-') || (*p == '+')) {
+    ++p;
+  }
+
+  while ((p != pend) && (*p == '0')) {
+    ++p;
+  }
+  while ((p != pend) && is_integer(*p)) {
+    if (answer.num_digits + 1 < max_digits) {
+      answer.digits[answer.num_digits++] = uint8_t(*p - '0');
+    } else {
+      answer.truncated = true;
+    }
+    ++p;
+  }
+  const char *first_after_period{};
+  if ((p != pend) && (*p == '.')) {
+    ++p;
+    first_after_period = p;
+    // if we have not yet encountered a zero, we have to skip it as well
+    if(answer.num_digits == 0) {
+      // skip zeros
+      while ((p != pend) && (*p == '0')) {
+       ++p;
+      }
+    }
+    while ((p != pend) && is_integer(*p)) {
+      if (answer.num_digits + 1 < max_digits) {
+        answer.digits[answer.num_digits++] = uint8_t(*p - '0');
+      } else {
+        answer.truncated = true;
+      }
+      ++p;
+    }
+    answer.decimal_point = int32_t(first_after_period - p);
+  }
+  
+  if ((p != pend) && (('e' == *p) || ('E' == *p))) {
+    ++p;
+    bool neg_exp = false;
+    if ((p != pend) && ('-' == *p)) {
+      neg_exp = true;
+      ++p;
+    } else if ((p != pend) && ('+' == *p)) {
+      ++p;
+    }
+    int32_t exp_number = 0; // exponential part
+    while ((p != pend) && is_integer(*p)) {
+      uint8_t digit = uint8_t(*p - '0');
+      if (exp_number < 0x10000) {
+        exp_number = 10 * exp_number + digit;
+      }      
+      ++p;
+    }
+    answer.decimal_point += (neg_exp ? -exp_number : exp_number);
+  }
+  answer.decimal_point += answer.num_digits;
+  return answer;
+}
+} // namespace fast_float
+}  // namespace arrow_vendored
+
+#endif
diff --git a/cpp/src/arrow/vendored/fast_float/decimal_to_binary.h b/cpp/src/arrow/vendored/fast_float/decimal_to_binary.h
new file mode 100644
index 00000000000..a64a7aaca17
--- /dev/null
+++ b/cpp/src/arrow/vendored/fast_float/decimal_to_binary.h
@@ -0,0 +1,167 @@
+#ifndef FASTFLOAT_DECIMAL_TO_BINARY_H
+#define FASTFLOAT_DECIMAL_TO_BINARY_H
+
+#include "float_common.h"
+#include "fast_table.h"
+#include <cfloat>
+#include <cinttypes>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+
+namespace arrow_vendored {
+namespace fast_float {
+
+
+
+
+// This will compute or rather approximate w * 5**q and return a pair of 64-bit words approximating
+// the result, with the "high" part corresponding to the most significant bits and the
+// low part corresponding to the least significant bits.
+// 
+template <int bit_precision>
+fastfloat_really_inline
+value128 compute_product_approximation(int64_t q, uint64_t w) {
+  const int index = 2 * int(q - smallest_power_of_five);
+  // For small values of q, e.g., q in [0,27], the answer is always exact because
+  // The line value128 firstproduct = full_multiplication(w, power_of_five_128[index]);
+  // gives the exact answer. 
+  value128 firstproduct = full_multiplication(w, power_of_five_128[index]);
+  static_assert((bit_precision >= 0) && (bit_precision <= 64), " precision should  be in (0,64]");
+  constexpr uint64_t precision_mask = (bit_precision < 64) ? 
+               (uint64_t(0xFFFFFFFFFFFFFFFF) >> bit_precision) 
+               : uint64_t(0xFFFFFFFFFFFFFFFF);
+  if((firstproduct.high & precision_mask) == precision_mask) { // could further guard with  (lower + w < lower)
+    // regarding the second product, we only need secondproduct.high, but our expectation is that the compiler will optimize this extra work away if needed.
+    value128 secondproduct = full_multiplication(w, power_of_five_128[index + 1]);
+    firstproduct.low += secondproduct.high;
+    if(secondproduct.high > firstproduct.low) {
+      firstproduct.high++;
+    }
+  }
+  return firstproduct;
+}
+
+namespace {
+/**
+ * For q in (-400,350), we have that
+ *  f = (((152170 + 65536) * q ) >> 16);
+ * is equal to
+ *   floor(p) + q
+ * where
+ *   p = log(5**q)/log(2) = q * log(5)/log(2)
+ *
+ */
+  fastfloat_really_inline unsigned int power(int q)  noexcept  {
+    return (((152170 + 65536) * q) >> 16) + 63;
+  }
+} // namespace
+
+// w * 10 ** q
+// The returned value should be a valid ieee64 number that simply need to be packed.
+// However, in some very rare cases, the computation will fail. In such cases, we 
+// return an adjusted_mantissa with a negative power of 2: the caller should recompute
+// in such cases.
+template <typename binary>
+fastfloat_really_inline
+adjusted_mantissa compute_float(int64_t q, uint64_t w)  noexcept  {
+  adjusted_mantissa answer;
+  if ((w == 0) || (q < smallest_power_of_five) ){
+    answer.power2 = 0;
+    answer.mantissa = 0;
+    // result should be zero
+    return answer;
+  }
+  if (q > largest_power_of_five) {
+    // we want to get infinity:
+    answer.power2 = binary::infinite_power();
+    answer.mantissa = 0;
+    return answer;
+  }
+  // At this point in time q is in [smallest_power_of_five, largest_power_of_five].
+
+  // We want the most significant bit of i to be 1. Shift if needed.
+  int lz = leading_zeroes(w);
+  w <<= lz;
+
+  // The required precision is binary::mantissa_explicit_bits() + 3 because
+  // 1. We need the implicit bit
+  // 2. We need an extra bit for rounding purposes
+  // 3. We might lose a bit due to the "upperbit" routine (result too small, requiring a shift)
+  value128 product = compute_product_approximation<binary::mantissa_explicit_bits() + 3>(q, w);
+  if(product.low == 0xFFFFFFFFFFFFFFFF) { //  could guard it further
+    // In some very rare cases, this could happen, in which case we might need a more accurate
+    // computation that what we can provide cheaply. This is very, very unlikely.
+    answer.power2 = -1;
+    return answer;
+  }
+  // The "compute_product_approximation" function can be slightly slower than a branchless approach:
+  // value128 product = compute_product(q, w);
+  // but in practice, we can win big with the compute_product_approximation if its additional branch
+  // is easily predicted. Which is best is data specific.
+  uint64_t upperbit = product.high >> 63;
+
+  answer.mantissa = product.high >> (upperbit + 64 - binary::mantissa_explicit_bits() - 3);
+  lz += int(1 ^ upperbit);
+  answer.power2 = power(int(q)) - lz - binary::minimum_exponent() + 1;
+
+  if (answer.power2 <= 0) { // we have a subnormal?
+    // Here have that answer.power2 <= 0 so -answer.power2 >= 0
+    if(-answer.power2 + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure.
+      answer.power2 = 0;
+      answer.mantissa = 0;
+      // result should be zero
+      return answer;
+    } 
+    // next line is safe because -answer.power2 + 1 < 0
+    answer.mantissa >>= -answer.power2 + 1;
+    // Thankfully, we can't have both "round-to-even" and subnormals because
+    // "round-to-even" only occurs for powers close to 0.
+    answer.mantissa += (answer.mantissa & 1); // round up
+    answer.mantissa >>= 1;
+    // There is a weird scenario where we don't have a subnormal but just.
+    // Suppose we start with 2.2250738585072013e-308, we end up
+    // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal
+    // whereas 0x40000000000000 x 2^-1023-53  is normal. Now, we need to round
+    // up 0x3fffffffffffff x 2^-1023-53  and once we do, we are no longer
+    // subnormal, but we can only know this after rounding.
+    // So we only declare a subnormal if we are smaller than the threshold.
+    answer.power2 = (answer.mantissa < (uint64_t(1) << binary::mantissa_explicit_bits())) ? 0 : 1;
+    return answer;
+  }
+
+  // usually, we round *up*, but if we fall right in between and and we have an
+  // even basis, we need to round down
+  // We are only concerned with the cases where 5**q fits in single 64-bit word.
+  if ((product.low <= 1) &&  (q >= binary::min_exponent_round_to_even()) && (q <= binary::max_exponent_round_to_even()) && 
+      ((answer.mantissa & 3) == 1) ) { // we may fall between two floats!
+    // To be in-between two floats we need that in doing
+    //   answer.mantissa = product.high >> (upperbit + 64 - binary::mantissa_explicit_bits() - 3);
+    // ... we dropped out only zeroes. But if this happened, then we can go back!!!
+    if((answer.mantissa  << (upperbit + 64 - binary::mantissa_explicit_bits() - 3)) ==  product.high) {
+      answer.mantissa &= ~1;          // flip it so that we do not round up
+    }
+  }
+
+  answer.mantissa += (answer.mantissa & 1); // round up
+  answer.mantissa >>= 1;
+  if (answer.mantissa >= (uint64_t(2) << binary::mantissa_explicit_bits())) {
+    answer.mantissa = (uint64_t(1) << binary::mantissa_explicit_bits());
+    answer.power2++; // undo previous addition
+  }
+
+  answer.mantissa &= ~(uint64_t(1) << binary::mantissa_explicit_bits());
+  if (answer.power2 >= binary::infinite_power()) { // infinity
+    answer.power2 = binary::infinite_power();
+    answer.mantissa = 0;
+  }
+  return answer;
+}
+
+} // namespace fast_float
+}  // namespace arrow_vendored
+
+#endif
diff --git a/cpp/src/arrow/vendored/fast_float/fast_float.h b/cpp/src/arrow/vendored/fast_float/fast_float.h
new file mode 100644
index 00000000000..0e7acf5f84d
--- /dev/null
+++ b/cpp/src/arrow/vendored/fast_float/fast_float.h
@@ -0,0 +1,47 @@
+#ifndef FASTFLOAT_FAST_FLOAT_H
+#define FASTFLOAT_FAST_FLOAT_H
+
+#include <system_error>
+
+namespace arrow_vendored {
+namespace fast_float {
+enum chars_format {
+    scientific = 1<<0,
+    fixed = 1<<2,
+    hex = 1<<3,
+    general = fixed | scientific
+};
+
+
+struct from_chars_result {
+  const char *ptr;
+  std::errc ec;
+};
+
+/**
+ * This function parses the character sequence [first,last) for a number. It parses floating-point numbers expecting
+ * a locale-indepent format equivalent to what is used by std::strtod in the default ("C") locale. 
+ * The resulting floating-point value is the closest floating-point values (using either float or double), 
+ * using the "round to even" convention for values that would otherwise fall right in-between two values.
+ * That is, we provide exact parsing according to the IEEE standard.
+ * 
+ * Given a successful parse, the pointer (`ptr`) in the returned value is set to point right after the
+ * parsed number, and the `value` referenced is set to the parsed value. In case of error, the returned
+ * `ec` contains a representative error, otherwise the default (`std::errc()`) value is stored.
+ * 
+ * The implementation does not throw and does not allocate memory (e.g., with `new` or `malloc`).
+ * 
+ * Like the C++17 standard, the `fast_float::from_chars` functions take an optional last argument of
+ * the type `fast_float::chars_format`. It is a bitset value: we check whether 
+ * `fmt & fast_float::chars_format::fixed` and `fmt & fast_float::chars_format::scientific` are set
+ * to determine whether we allowe the fixed point and scientific notation respectively.
+ * The default is  `fast_float::chars_format::general` which allows both `fixed` and `scientific`.
+ */
+template<typename T>
+from_chars_result from_chars(const char *first, const char *last,
+                             T &value, chars_format fmt = chars_format::general)  noexcept;
+
+}
+}  // namespace arrow_vendored
+#include "parse_number.h"
+#endif // FASTFLOAT_FAST_FLOAT_H
diff --git a/cpp/src/arrow/vendored/fast_float/fast_table.h b/cpp/src/arrow/vendored/fast_float/fast_table.h
new file mode 100644
index 00000000000..ac34fe7cefc
--- /dev/null
+++ b/cpp/src/arrow/vendored/fast_float/fast_table.h
@@ -0,0 +1,691 @@
+#ifndef FASTFLOAT_FAST_TABLE_H
+#define FASTFLOAT_FAST_TABLE_H
+#include <cstdint>
+
+namespace arrow_vendored {
+namespace fast_float {
+
+/**
+ * When mapping numbers from decimal to binary,
+ * we go from w * 10^q to m * 2^p but we have
+ * 10^q = 5^q * 2^q, so effectively
+ * we are trying to match
+ * w * 2^q * 5^q to m * 2^p. Thus the powers of two
+ * are not a concern since they can be represented
+ * exactly using the binary notation, only the powers of five
+ * affect the binary significand.
+ */
+
+/**
+ * The smallest non-zero float (binary64) is 2^−1074.
+ * We take as input numbers of the form w x 10^q where w < 2^64.
+ * We have that w * 10^-343  <  2^(64-344) 5^-343 < 2^-1076.
+ * However, we have that 
+ * (2^64-1) * 10^-342 =  (2^64-1) * 2^-342 * 5^-342 > 2^−1074.
+ * Thus it is possible for a number of the form w * 10^-342 where 
+ * w is a 64-bit value to be a non-zero floating-point number.
+ *********
+ * Any number of form w * 10^309 where w>= 1 is going to be 
+ * infinite in binary64 so we never need to worry about powers
+ * of 5 greater than 308.
+ */
+constexpr int smallest_power_of_five = -342;
+constexpr int largest_power_of_five = 308;
+// truncated powers of five from 5^-344 all the way to 5^308
+const uint64_t power_of_five_128[]= {
+        0xeef453d6923bd65a,0x113faa2906a13b3f,
+        0x9558b4661b6565f8,0x4ac7ca59a424c507,
+        0xbaaee17fa23ebf76,0x5d79bcf00d2df649,
+        0xe95a99df8ace6f53,0xf4d82c2c107973dc,
+        0x91d8a02bb6c10594,0x79071b9b8a4be869,
+        0xb64ec836a47146f9,0x9748e2826cdee284,
+        0xe3e27a444d8d98b7,0xfd1b1b2308169b25,
+        0x8e6d8c6ab0787f72,0xfe30f0f5e50e20f7,
+        0xb208ef855c969f4f,0xbdbd2d335e51a935,
+        0xde8b2b66b3bc4723,0xad2c788035e61382,
+        0x8b16fb203055ac76,0x4c3bcb5021afcc31,
+        0xaddcb9e83c6b1793,0xdf4abe242a1bbf3d,
+        0xd953e8624b85dd78,0xd71d6dad34a2af0d,
+        0x87d4713d6f33aa6b,0x8672648c40e5ad68,
+        0xa9c98d8ccb009506,0x680efdaf511f18c2,
+        0xd43bf0effdc0ba48,0x212bd1b2566def2,
+        0x84a57695fe98746d,0x14bb630f7604b57,
+        0xa5ced43b7e3e9188,0x419ea3bd35385e2d,
+        0xcf42894a5dce35ea,0x52064cac828675b9,
+        0x818995ce7aa0e1b2,0x7343efebd1940993,
+        0xa1ebfb4219491a1f,0x1014ebe6c5f90bf8,
+        0xca66fa129f9b60a6,0xd41a26e077774ef6,
+        0xfd00b897478238d0,0x8920b098955522b4,
+        0x9e20735e8cb16382,0x55b46e5f5d5535b0,
+        0xc5a890362fddbc62,0xeb2189f734aa831d,
+        0xf712b443bbd52b7b,0xa5e9ec7501d523e4,
+        0x9a6bb0aa55653b2d,0x47b233c92125366e,
+        0xc1069cd4eabe89f8,0x999ec0bb696e840a,
+        0xf148440a256e2c76,0xc00670ea43ca250d,
+        0x96cd2a865764dbca,0x380406926a5e5728,
+        0xbc807527ed3e12bc,0xc605083704f5ecf2,
+        0xeba09271e88d976b,0xf7864a44c633682e,
+        0x93445b8731587ea3,0x7ab3ee6afbe0211d,
+        0xb8157268fdae9e4c,0x5960ea05bad82964,
+        0xe61acf033d1a45df,0x6fb92487298e33bd,
+        0x8fd0c16206306bab,0xa5d3b6d479f8e056,
+        0xb3c4f1ba87bc8696,0x8f48a4899877186c,
+        0xe0b62e2929aba83c,0x331acdabfe94de87,
+        0x8c71dcd9ba0b4925,0x9ff0c08b7f1d0b14,
+        0xaf8e5410288e1b6f,0x7ecf0ae5ee44dd9,
+        0xdb71e91432b1a24a,0xc9e82cd9f69d6150,
+        0x892731ac9faf056e,0xbe311c083a225cd2,
+        0xab70fe17c79ac6ca,0x6dbd630a48aaf406,
+        0xd64d3d9db981787d,0x92cbbccdad5b108,
+        0x85f0468293f0eb4e,0x25bbf56008c58ea5,
+        0xa76c582338ed2621,0xaf2af2b80af6f24e,
+        0xd1476e2c07286faa,0x1af5af660db4aee1,
+        0x82cca4db847945ca,0x50d98d9fc890ed4d,
+        0xa37fce126597973c,0xe50ff107bab528a0,
+        0xcc5fc196fefd7d0c,0x1e53ed49a96272c8,
+        0xff77b1fcbebcdc4f,0x25e8e89c13bb0f7a,
+        0x9faacf3df73609b1,0x77b191618c54e9ac,
+        0xc795830d75038c1d,0xd59df5b9ef6a2417,
+        0xf97ae3d0d2446f25,0x4b0573286b44ad1d,
+        0x9becce62836ac577,0x4ee367f9430aec32,
+        0xc2e801fb244576d5,0x229c41f793cda73f,
+        0xf3a20279ed56d48a,0x6b43527578c1110f,
+        0x9845418c345644d6,0x830a13896b78aaa9,
+        0xbe5691ef416bd60c,0x23cc986bc656d553,
+        0xedec366b11c6cb8f,0x2cbfbe86b7ec8aa8,
+        0x94b3a202eb1c3f39,0x7bf7d71432f3d6a9,
+        0xb9e08a83a5e34f07,0xdaf5ccd93fb0cc53,
+        0xe858ad248f5c22c9,0xd1b3400f8f9cff68,
+        0x91376c36d99995be,0x23100809b9c21fa1,
+        0xb58547448ffffb2d,0xabd40a0c2832a78a,
+        0xe2e69915b3fff9f9,0x16c90c8f323f516c,
+        0x8dd01fad907ffc3b,0xae3da7d97f6792e3,
+        0xb1442798f49ffb4a,0x99cd11cfdf41779c,
+        0xdd95317f31c7fa1d,0x40405643d711d583,
+        0x8a7d3eef7f1cfc52,0x482835ea666b2572,
+        0xad1c8eab5ee43b66,0xda3243650005eecf,
+        0xd863b256369d4a40,0x90bed43e40076a82,
+        0x873e4f75e2224e68,0x5a7744a6e804a291,
+        0xa90de3535aaae202,0x711515d0a205cb36,
+        0xd3515c2831559a83,0xd5a5b44ca873e03,
+        0x8412d9991ed58091,0xe858790afe9486c2,
+        0xa5178fff668ae0b6,0x626e974dbe39a872,
+        0xce5d73ff402d98e3,0xfb0a3d212dc8128f,
+        0x80fa687f881c7f8e,0x7ce66634bc9d0b99,
+        0xa139029f6a239f72,0x1c1fffc1ebc44e80,
+        0xc987434744ac874e,0xa327ffb266b56220,
+        0xfbe9141915d7a922,0x4bf1ff9f0062baa8,
+        0x9d71ac8fada6c9b5,0x6f773fc3603db4a9,
+        0xc4ce17b399107c22,0xcb550fb4384d21d3,
+        0xf6019da07f549b2b,0x7e2a53a146606a48,
+        0x99c102844f94e0fb,0x2eda7444cbfc426d,
+        0xc0314325637a1939,0xfa911155fefb5308,
+        0xf03d93eebc589f88,0x793555ab7eba27ca,
+        0x96267c7535b763b5,0x4bc1558b2f3458de,
+        0xbbb01b9283253ca2,0x9eb1aaedfb016f16,
+        0xea9c227723ee8bcb,0x465e15a979c1cadc,
+        0x92a1958a7675175f,0xbfacd89ec191ec9,
+        0xb749faed14125d36,0xcef980ec671f667b,
+        0xe51c79a85916f484,0x82b7e12780e7401a,
+        0x8f31cc0937ae58d2,0xd1b2ecb8b0908810,
+        0xb2fe3f0b8599ef07,0x861fa7e6dcb4aa15,
+        0xdfbdcece67006ac9,0x67a791e093e1d49a,
+        0x8bd6a141006042bd,0xe0c8bb2c5c6d24e0,
+        0xaecc49914078536d,0x58fae9f773886e18,
+        0xda7f5bf590966848,0xaf39a475506a899e,
+        0x888f99797a5e012d,0x6d8406c952429603,
+        0xaab37fd7d8f58178,0xc8e5087ba6d33b83,
+        0xd5605fcdcf32e1d6,0xfb1e4a9a90880a64,
+        0x855c3be0a17fcd26,0x5cf2eea09a55067f,
+        0xa6b34ad8c9dfc06f,0xf42faa48c0ea481e,
+        0xd0601d8efc57b08b,0xf13b94daf124da26,
+        0x823c12795db6ce57,0x76c53d08d6b70858,
+        0xa2cb1717b52481ed,0x54768c4b0c64ca6e,
+        0xcb7ddcdda26da268,0xa9942f5dcf7dfd09,
+        0xfe5d54150b090b02,0xd3f93b35435d7c4c,
+        0x9efa548d26e5a6e1,0xc47bc5014a1a6daf,
+        0xc6b8e9b0709f109a,0x359ab6419ca1091b,
+        0xf867241c8cc6d4c0,0xc30163d203c94b62,
+        0x9b407691d7fc44f8,0x79e0de63425dcf1d,
+        0xc21094364dfb5636,0x985915fc12f542e4,
+        0xf294b943e17a2bc4,0x3e6f5b7b17b2939d,
+        0x979cf3ca6cec5b5a,0xa705992ceecf9c42,
+        0xbd8430bd08277231,0x50c6ff782a838353,
+        0xece53cec4a314ebd,0xa4f8bf5635246428,
+        0x940f4613ae5ed136,0x871b7795e136be99,
+        0xb913179899f68584,0x28e2557b59846e3f,
+        0xe757dd7ec07426e5,0x331aeada2fe589cf,
+        0x9096ea6f3848984f,0x3ff0d2c85def7621,
+        0xb4bca50b065abe63,0xfed077a756b53a9,
+        0xe1ebce4dc7f16dfb,0xd3e8495912c62894,
+        0x8d3360f09cf6e4bd,0x64712dd7abbbd95c,
+        0xb080392cc4349dec,0xbd8d794d96aacfb3,
+        0xdca04777f541c567,0xecf0d7a0fc5583a0,
+        0x89e42caaf9491b60,0xf41686c49db57244,
+        0xac5d37d5b79b6239,0x311c2875c522ced5,
+        0xd77485cb25823ac7,0x7d633293366b828b,
+        0x86a8d39ef77164bc,0xae5dff9c02033197,
+        0xa8530886b54dbdeb,0xd9f57f830283fdfc,
+        0xd267caa862a12d66,0xd072df63c324fd7b,
+        0x8380dea93da4bc60,0x4247cb9e59f71e6d,
+        0xa46116538d0deb78,0x52d9be85f074e608,
+        0xcd795be870516656,0x67902e276c921f8b,
+        0x806bd9714632dff6,0xba1cd8a3db53b6,
+        0xa086cfcd97bf97f3,0x80e8a40eccd228a4,
+        0xc8a883c0fdaf7df0,0x6122cd128006b2cd,
+        0xfad2a4b13d1b5d6c,0x796b805720085f81,
+        0x9cc3a6eec6311a63,0xcbe3303674053bb0,
+        0xc3f490aa77bd60fc,0xbedbfc4411068a9c,
+        0xf4f1b4d515acb93b,0xee92fb5515482d44,
+        0x991711052d8bf3c5,0x751bdd152d4d1c4a,
+        0xbf5cd54678eef0b6,0xd262d45a78a0635d,
+        0xef340a98172aace4,0x86fb897116c87c34,
+        0x9580869f0e7aac0e,0xd45d35e6ae3d4da0,
+        0xbae0a846d2195712,0x8974836059cca109,
+        0xe998d258869facd7,0x2bd1a438703fc94b,
+        0x91ff83775423cc06,0x7b6306a34627ddcf,
+        0xb67f6455292cbf08,0x1a3bc84c17b1d542,
+        0xe41f3d6a7377eeca,0x20caba5f1d9e4a93,
+        0x8e938662882af53e,0x547eb47b7282ee9c,
+        0xb23867fb2a35b28d,0xe99e619a4f23aa43,
+        0xdec681f9f4c31f31,0x6405fa00e2ec94d4,
+        0x8b3c113c38f9f37e,0xde83bc408dd3dd04,
+        0xae0b158b4738705e,0x9624ab50b148d445,
+        0xd98ddaee19068c76,0x3badd624dd9b0957,
+        0x87f8a8d4cfa417c9,0xe54ca5d70a80e5d6,
+        0xa9f6d30a038d1dbc,0x5e9fcf4ccd211f4c,
+        0xd47487cc8470652b,0x7647c3200069671f,
+        0x84c8d4dfd2c63f3b,0x29ecd9f40041e073,
+        0xa5fb0a17c777cf09,0xf468107100525890,
+        0xcf79cc9db955c2cc,0x7182148d4066eeb4,
+        0x81ac1fe293d599bf,0xc6f14cd848405530,
+        0xa21727db38cb002f,0xb8ada00e5a506a7c,
+        0xca9cf1d206fdc03b,0xa6d90811f0e4851c,
+        0xfd442e4688bd304a,0x908f4a166d1da663,
+        0x9e4a9cec15763e2e,0x9a598e4e043287fe,
+        0xc5dd44271ad3cdba,0x40eff1e1853f29fd,
+        0xf7549530e188c128,0xd12bee59e68ef47c,
+        0x9a94dd3e8cf578b9,0x82bb74f8301958ce,
+        0xc13a148e3032d6e7,0xe36a52363c1faf01,
+        0xf18899b1bc3f8ca1,0xdc44e6c3cb279ac1,
+        0x96f5600f15a7b7e5,0x29ab103a5ef8c0b9,
+        0xbcb2b812db11a5de,0x7415d448f6b6f0e7,
+        0xebdf661791d60f56,0x111b495b3464ad21,
+        0x936b9fcebb25c995,0xcab10dd900beec34,
+        0xb84687c269ef3bfb,0x3d5d514f40eea742,
+        0xe65829b3046b0afa,0xcb4a5a3112a5112,
+        0x8ff71a0fe2c2e6dc,0x47f0e785eaba72ab,
+        0xb3f4e093db73a093,0x59ed216765690f56,
+        0xe0f218b8d25088b8,0x306869c13ec3532c,
+        0x8c974f7383725573,0x1e414218c73a13fb,
+        0xafbd2350644eeacf,0xe5d1929ef90898fa,
+        0xdbac6c247d62a583,0xdf45f746b74abf39,
+        0x894bc396ce5da772,0x6b8bba8c328eb783,
+        0xab9eb47c81f5114f,0x66ea92f3f326564,
+        0xd686619ba27255a2,0xc80a537b0efefebd,
+        0x8613fd0145877585,0xbd06742ce95f5f36,
+        0xa798fc4196e952e7,0x2c48113823b73704,
+        0xd17f3b51fca3a7a0,0xf75a15862ca504c5,
+        0x82ef85133de648c4,0x9a984d73dbe722fb,
+        0xa3ab66580d5fdaf5,0xc13e60d0d2e0ebba,
+        0xcc963fee10b7d1b3,0x318df905079926a8,
+        0xffbbcfe994e5c61f,0xfdf17746497f7052,
+        0x9fd561f1fd0f9bd3,0xfeb6ea8bedefa633,
+        0xc7caba6e7c5382c8,0xfe64a52ee96b8fc0,
+        0xf9bd690a1b68637b,0x3dfdce7aa3c673b0,
+        0x9c1661a651213e2d,0x6bea10ca65c084e,
+        0xc31bfa0fe5698db8,0x486e494fcff30a62,
+        0xf3e2f893dec3f126,0x5a89dba3c3efccfa,
+        0x986ddb5c6b3a76b7,0xf89629465a75e01c,
+        0xbe89523386091465,0xf6bbb397f1135823,
+        0xee2ba6c0678b597f,0x746aa07ded582e2c,
+        0x94db483840b717ef,0xa8c2a44eb4571cdc,
+        0xba121a4650e4ddeb,0x92f34d62616ce413,
+        0xe896a0d7e51e1566,0x77b020baf9c81d17,
+        0x915e2486ef32cd60,0xace1474dc1d122e,
+        0xb5b5ada8aaff80b8,0xd819992132456ba,
+        0xe3231912d5bf60e6,0x10e1fff697ed6c69,
+        0x8df5efabc5979c8f,0xca8d3ffa1ef463c1,
+        0xb1736b96b6fd83b3,0xbd308ff8a6b17cb2,
+        0xddd0467c64bce4a0,0xac7cb3f6d05ddbde,
+        0x8aa22c0dbef60ee4,0x6bcdf07a423aa96b,
+        0xad4ab7112eb3929d,0x86c16c98d2c953c6,
+        0xd89d64d57a607744,0xe871c7bf077ba8b7,
+        0x87625f056c7c4a8b,0x11471cd764ad4972,
+        0xa93af6c6c79b5d2d,0xd598e40d3dd89bcf,
+        0xd389b47879823479,0x4aff1d108d4ec2c3,
+        0x843610cb4bf160cb,0xcedf722a585139ba,
+        0xa54394fe1eedb8fe,0xc2974eb4ee658828,
+        0xce947a3da6a9273e,0x733d226229feea32,
+        0x811ccc668829b887,0x806357d5a3f525f,
+        0xa163ff802a3426a8,0xca07c2dcb0cf26f7,
+        0xc9bcff6034c13052,0xfc89b393dd02f0b5,
+        0xfc2c3f3841f17c67,0xbbac2078d443ace2,
+        0x9d9ba7832936edc0,0xd54b944b84aa4c0d,
+        0xc5029163f384a931,0xa9e795e65d4df11,
+        0xf64335bcf065d37d,0x4d4617b5ff4a16d5,
+        0x99ea0196163fa42e,0x504bced1bf8e4e45,
+        0xc06481fb9bcf8d39,0xe45ec2862f71e1d6,
+        0xf07da27a82c37088,0x5d767327bb4e5a4c,
+        0x964e858c91ba2655,0x3a6a07f8d510f86f,
+        0xbbe226efb628afea,0x890489f70a55368b,
+        0xeadab0aba3b2dbe5,0x2b45ac74ccea842e,
+        0x92c8ae6b464fc96f,0x3b0b8bc90012929d,
+        0xb77ada0617e3bbcb,0x9ce6ebb40173744,
+        0xe55990879ddcaabd,0xcc420a6a101d0515,
+        0x8f57fa54c2a9eab6,0x9fa946824a12232d,
+        0xb32df8e9f3546564,0x47939822dc96abf9,
+        0xdff9772470297ebd,0x59787e2b93bc56f7,
+        0x8bfbea76c619ef36,0x57eb4edb3c55b65a,
+        0xaefae51477a06b03,0xede622920b6b23f1,
+        0xdab99e59958885c4,0xe95fab368e45eced,
+        0x88b402f7fd75539b,0x11dbcb0218ebb414,
+        0xaae103b5fcd2a881,0xd652bdc29f26a119,
+        0xd59944a37c0752a2,0x4be76d3346f0495f,
+        0x857fcae62d8493a5,0x6f70a4400c562ddb,
+        0xa6dfbd9fb8e5b88e,0xcb4ccd500f6bb952,
+        0xd097ad07a71f26b2,0x7e2000a41346a7a7,
+        0x825ecc24c873782f,0x8ed400668c0c28c8,
+        0xa2f67f2dfa90563b,0x728900802f0f32fa,
+        0xcbb41ef979346bca,0x4f2b40a03ad2ffb9,
+        0xfea126b7d78186bc,0xe2f610c84987bfa8,
+        0x9f24b832e6b0f436,0xdd9ca7d2df4d7c9,
+        0xc6ede63fa05d3143,0x91503d1c79720dbb,
+        0xf8a95fcf88747d94,0x75a44c6397ce912a,
+        0x9b69dbe1b548ce7c,0xc986afbe3ee11aba,
+        0xc24452da229b021b,0xfbe85badce996168,
+        0xf2d56790ab41c2a2,0xfae27299423fb9c3,
+        0x97c560ba6b0919a5,0xdccd879fc967d41a,
+        0xbdb6b8e905cb600f,0x5400e987bbc1c920,
+        0xed246723473e3813,0x290123e9aab23b68,
+        0x9436c0760c86e30b,0xf9a0b6720aaf6521,
+        0xb94470938fa89bce,0xf808e40e8d5b3e69,
+        0xe7958cb87392c2c2,0xb60b1d1230b20e04,
+        0x90bd77f3483bb9b9,0xb1c6f22b5e6f48c2,
+        0xb4ecd5f01a4aa828,0x1e38aeb6360b1af3,
+        0xe2280b6c20dd5232,0x25c6da63c38de1b0,
+        0x8d590723948a535f,0x579c487e5a38ad0e,
+        0xb0af48ec79ace837,0x2d835a9df0c6d851,
+        0xdcdb1b2798182244,0xf8e431456cf88e65,
+        0x8a08f0f8bf0f156b,0x1b8e9ecb641b58ff,
+        0xac8b2d36eed2dac5,0xe272467e3d222f3f,
+        0xd7adf884aa879177,0x5b0ed81dcc6abb0f,
+        0x86ccbb52ea94baea,0x98e947129fc2b4e9,
+        0xa87fea27a539e9a5,0x3f2398d747b36224,
+        0xd29fe4b18e88640e,0x8eec7f0d19a03aad,
+        0x83a3eeeef9153e89,0x1953cf68300424ac,
+        0xa48ceaaab75a8e2b,0x5fa8c3423c052dd7,
+        0xcdb02555653131b6,0x3792f412cb06794d,
+        0x808e17555f3ebf11,0xe2bbd88bbee40bd0,
+        0xa0b19d2ab70e6ed6,0x5b6aceaeae9d0ec4,
+        0xc8de047564d20a8b,0xf245825a5a445275,
+        0xfb158592be068d2e,0xeed6e2f0f0d56712,
+        0x9ced737bb6c4183d,0x55464dd69685606b,
+        0xc428d05aa4751e4c,0xaa97e14c3c26b886,
+        0xf53304714d9265df,0xd53dd99f4b3066a8,
+        0x993fe2c6d07b7fab,0xe546a8038efe4029,
+        0xbf8fdb78849a5f96,0xde98520472bdd033,
+        0xef73d256a5c0f77c,0x963e66858f6d4440,
+        0x95a8637627989aad,0xdde7001379a44aa8,
+        0xbb127c53b17ec159,0x5560c018580d5d52,
+        0xe9d71b689dde71af,0xaab8f01e6e10b4a6,
+        0x9226712162ab070d,0xcab3961304ca70e8,
+        0xb6b00d69bb55c8d1,0x3d607b97c5fd0d22,
+        0xe45c10c42a2b3b05,0x8cb89a7db77c506a,
+        0x8eb98a7a9a5b04e3,0x77f3608e92adb242,
+        0xb267ed1940f1c61c,0x55f038b237591ed3,
+        0xdf01e85f912e37a3,0x6b6c46dec52f6688,
+        0x8b61313bbabce2c6,0x2323ac4b3b3da015,
+        0xae397d8aa96c1b77,0xabec975e0a0d081a,
+        0xd9c7dced53c72255,0x96e7bd358c904a21,
+        0x881cea14545c7575,0x7e50d64177da2e54,
+        0xaa242499697392d2,0xdde50bd1d5d0b9e9,
+        0xd4ad2dbfc3d07787,0x955e4ec64b44e864,
+        0x84ec3c97da624ab4,0xbd5af13bef0b113e,
+        0xa6274bbdd0fadd61,0xecb1ad8aeacdd58e,
+        0xcfb11ead453994ba,0x67de18eda5814af2,
+        0x81ceb32c4b43fcf4,0x80eacf948770ced7,
+        0xa2425ff75e14fc31,0xa1258379a94d028d,
+        0xcad2f7f5359a3b3e,0x96ee45813a04330,
+        0xfd87b5f28300ca0d,0x8bca9d6e188853fc,
+        0x9e74d1b791e07e48,0x775ea264cf55347e,
+        0xc612062576589dda,0x95364afe032a81a0,
+        0xf79687aed3eec551,0x3a83ddbd83f52210,
+        0x9abe14cd44753b52,0xc4926a9672793580,
+        0xc16d9a0095928a27,0x75b7053c0f178400,
+        0xf1c90080baf72cb1,0x5324c68b12dd6800,
+        0x971da05074da7bee,0xd3f6fc16ebca8000,
+        0xbce5086492111aea,0x88f4bb1ca6bd0000,
+        0xec1e4a7db69561a5,0x2b31e9e3d0700000,
+        0x9392ee8e921d5d07,0x3aff322e62600000,
+        0xb877aa3236a4b449,0x9befeb9fad487c3,
+        0xe69594bec44de15b,0x4c2ebe687989a9b4,
+        0x901d7cf73ab0acd9,0xf9d37014bf60a11,
+        0xb424dc35095cd80f,0x538484c19ef38c95,
+        0xe12e13424bb40e13,0x2865a5f206b06fba,
+        0x8cbccc096f5088cb,0xf93f87b7442e45d4,
+        0xafebff0bcb24aafe,0xf78f69a51539d749,
+        0xdbe6fecebdedd5be,0xb573440e5a884d1c,
+        0x89705f4136b4a597,0x31680a88f8953031,
+        0xabcc77118461cefc,0xfdc20d2b36ba7c3e,
+        0xd6bf94d5e57a42bc,0x3d32907604691b4d,
+        0x8637bd05af6c69b5,0xa63f9a49c2c1b110,
+        0xa7c5ac471b478423,0xfcf80dc33721d54,
+        0xd1b71758e219652b,0xd3c36113404ea4a9,
+        0x83126e978d4fdf3b,0x645a1cac083126ea,
+        0xa3d70a3d70a3d70a,0x3d70a3d70a3d70a4,
+        0xcccccccccccccccc,0xcccccccccccccccd,
+        0x8000000000000000,0x0,
+        0xa000000000000000,0x0,
+        0xc800000000000000,0x0,
+        0xfa00000000000000,0x0,
+        0x9c40000000000000,0x0,
+        0xc350000000000000,0x0,
+        0xf424000000000000,0x0,
+        0x9896800000000000,0x0,
+        0xbebc200000000000,0x0,
+        0xee6b280000000000,0x0,
+        0x9502f90000000000,0x0,
+        0xba43b74000000000,0x0,
+        0xe8d4a51000000000,0x0,
+        0x9184e72a00000000,0x0,
+        0xb5e620f480000000,0x0,
+        0xe35fa931a0000000,0x0,
+        0x8e1bc9bf04000000,0x0,
+        0xb1a2bc2ec5000000,0x0,
+        0xde0b6b3a76400000,0x0,
+        0x8ac7230489e80000,0x0,
+        0xad78ebc5ac620000,0x0,
+        0xd8d726b7177a8000,0x0,
+        0x878678326eac9000,0x0,
+        0xa968163f0a57b400,0x0,
+        0xd3c21bcecceda100,0x0,
+        0x84595161401484a0,0x0,
+        0xa56fa5b99019a5c8,0x0,
+        0xcecb8f27f4200f3a,0x0,
+        0x813f3978f8940984,0x4000000000000000,
+        0xa18f07d736b90be5,0x5000000000000000,
+        0xc9f2c9cd04674ede,0xa400000000000000,
+        0xfc6f7c4045812296,0x4d00000000000000,
+        0x9dc5ada82b70b59d,0xf020000000000000,
+        0xc5371912364ce305,0x6c28000000000000,
+        0xf684df56c3e01bc6,0xc732000000000000,
+        0x9a130b963a6c115c,0x3c7f400000000000,
+        0xc097ce7bc90715b3,0x4b9f100000000000,
+        0xf0bdc21abb48db20,0x1e86d40000000000,
+        0x96769950b50d88f4,0x1314448000000000,
+        0xbc143fa4e250eb31,0x17d955a000000000,
+        0xeb194f8e1ae525fd,0x5dcfab0800000000,
+        0x92efd1b8d0cf37be,0x5aa1cae500000000,
+        0xb7abc627050305ad,0xf14a3d9e40000000,
+        0xe596b7b0c643c719,0x6d9ccd05d0000000,
+        0x8f7e32ce7bea5c6f,0xe4820023a2000000,
+        0xb35dbf821ae4f38b,0xdda2802c8a800000,
+        0xe0352f62a19e306e,0xd50b2037ad200000,
+        0x8c213d9da502de45,0x4526f422cc340000,
+        0xaf298d050e4395d6,0x9670b12b7f410000,
+        0xdaf3f04651d47b4c,0x3c0cdd765f114000,
+        0x88d8762bf324cd0f,0xa5880a69fb6ac800,
+        0xab0e93b6efee0053,0x8eea0d047a457a00,
+        0xd5d238a4abe98068,0x72a4904598d6d880,
+        0x85a36366eb71f041,0x47a6da2b7f864750,
+        0xa70c3c40a64e6c51,0x999090b65f67d924,
+        0xd0cf4b50cfe20765,0xfff4b4e3f741cf6d,
+        0x82818f1281ed449f,0xbff8f10e7a8921a4,
+        0xa321f2d7226895c7,0xaff72d52192b6a0d,
+        0xcbea6f8ceb02bb39,0x9bf4f8a69f764490,
+        0xfee50b7025c36a08,0x2f236d04753d5b4,
+        0x9f4f2726179a2245,0x1d762422c946590,
+        0xc722f0ef9d80aad6,0x424d3ad2b7b97ef5,
+        0xf8ebad2b84e0d58b,0xd2e0898765a7deb2,
+        0x9b934c3b330c8577,0x63cc55f49f88eb2f,
+        0xc2781f49ffcfa6d5,0x3cbf6b71c76b25fb,
+        0xf316271c7fc3908a,0x8bef464e3945ef7a,
+        0x97edd871cfda3a56,0x97758bf0e3cbb5ac,
+        0xbde94e8e43d0c8ec,0x3d52eeed1cbea317,
+        0xed63a231d4c4fb27,0x4ca7aaa863ee4bdd,
+        0x945e455f24fb1cf8,0x8fe8caa93e74ef6a,
+        0xb975d6b6ee39e436,0xb3e2fd538e122b44,
+        0xe7d34c64a9c85d44,0x60dbbca87196b616,
+        0x90e40fbeea1d3a4a,0xbc8955e946fe31cd,
+        0xb51d13aea4a488dd,0x6babab6398bdbe41,
+        0xe264589a4dcdab14,0xc696963c7eed2dd1,
+        0x8d7eb76070a08aec,0xfc1e1de5cf543ca2,
+        0xb0de65388cc8ada8,0x3b25a55f43294bcb,
+        0xdd15fe86affad912,0x49ef0eb713f39ebe,
+        0x8a2dbf142dfcc7ab,0x6e3569326c784337,
+        0xacb92ed9397bf996,0x49c2c37f07965404,
+        0xd7e77a8f87daf7fb,0xdc33745ec97be906,
+        0x86f0ac99b4e8dafd,0x69a028bb3ded71a3,
+        0xa8acd7c0222311bc,0xc40832ea0d68ce0c,
+        0xd2d80db02aabd62b,0xf50a3fa490c30190,
+        0x83c7088e1aab65db,0x792667c6da79e0fa,
+        0xa4b8cab1a1563f52,0x577001b891185938,
+        0xcde6fd5e09abcf26,0xed4c0226b55e6f86,
+        0x80b05e5ac60b6178,0x544f8158315b05b4,
+        0xa0dc75f1778e39d6,0x696361ae3db1c721,
+        0xc913936dd571c84c,0x3bc3a19cd1e38e9,
+        0xfb5878494ace3a5f,0x4ab48a04065c723,
+        0x9d174b2dcec0e47b,0x62eb0d64283f9c76,
+        0xc45d1df942711d9a,0x3ba5d0bd324f8394,
+        0xf5746577930d6500,0xca8f44ec7ee36479,
+        0x9968bf6abbe85f20,0x7e998b13cf4e1ecb,
+        0xbfc2ef456ae276e8,0x9e3fedd8c321a67e,
+        0xefb3ab16c59b14a2,0xc5cfe94ef3ea101e,
+        0x95d04aee3b80ece5,0xbba1f1d158724a12,
+        0xbb445da9ca61281f,0x2a8a6e45ae8edc97,
+        0xea1575143cf97226,0xf52d09d71a3293bd,
+        0x924d692ca61be758,0x593c2626705f9c56,
+        0xb6e0c377cfa2e12e,0x6f8b2fb00c77836c,
+        0xe498f455c38b997a,0xb6dfb9c0f956447,
+        0x8edf98b59a373fec,0x4724bd4189bd5eac,
+        0xb2977ee300c50fe7,0x58edec91ec2cb657,
+        0xdf3d5e9bc0f653e1,0x2f2967b66737e3ed,
+        0x8b865b215899f46c,0xbd79e0d20082ee74,
+        0xae67f1e9aec07187,0xecd8590680a3aa11,
+        0xda01ee641a708de9,0xe80e6f4820cc9495,
+        0x884134fe908658b2,0x3109058d147fdcdd,
+        0xaa51823e34a7eede,0xbd4b46f0599fd415,
+        0xd4e5e2cdc1d1ea96,0x6c9e18ac7007c91a,
+        0x850fadc09923329e,0x3e2cf6bc604ddb0,
+        0xa6539930bf6bff45,0x84db8346b786151c,
+        0xcfe87f7cef46ff16,0xe612641865679a63,
+        0x81f14fae158c5f6e,0x4fcb7e8f3f60c07e,
+        0xa26da3999aef7749,0xe3be5e330f38f09d,
+        0xcb090c8001ab551c,0x5cadf5bfd3072cc5,
+        0xfdcb4fa002162a63,0x73d9732fc7c8f7f6,
+        0x9e9f11c4014dda7e,0x2867e7fddcdd9afa,
+        0xc646d63501a1511d,0xb281e1fd541501b8,
+        0xf7d88bc24209a565,0x1f225a7ca91a4226,
+        0x9ae757596946075f,0x3375788de9b06958,
+        0xc1a12d2fc3978937,0x52d6b1641c83ae,
+        0xf209787bb47d6b84,0xc0678c5dbd23a49a,
+        0x9745eb4d50ce6332,0xf840b7ba963646e0,
+        0xbd176620a501fbff,0xb650e5a93bc3d898,
+        0xec5d3fa8ce427aff,0xa3e51f138ab4cebe,
+        0x93ba47c980e98cdf,0xc66f336c36b10137,
+        0xb8a8d9bbe123f017,0xb80b0047445d4184,
+        0xe6d3102ad96cec1d,0xa60dc059157491e5,
+        0x9043ea1ac7e41392,0x87c89837ad68db2f,
+        0xb454e4a179dd1877,0x29babe4598c311fb,
+        0xe16a1dc9d8545e94,0xf4296dd6fef3d67a,
+        0x8ce2529e2734bb1d,0x1899e4a65f58660c,
+        0xb01ae745b101e9e4,0x5ec05dcff72e7f8f,
+        0xdc21a1171d42645d,0x76707543f4fa1f73,
+        0x899504ae72497eba,0x6a06494a791c53a8,
+        0xabfa45da0edbde69,0x487db9d17636892,
+        0xd6f8d7509292d603,0x45a9d2845d3c42b6,
+        0x865b86925b9bc5c2,0xb8a2392ba45a9b2,
+        0xa7f26836f282b732,0x8e6cac7768d7141e,
+        0xd1ef0244af2364ff,0x3207d795430cd926,
+        0x8335616aed761f1f,0x7f44e6bd49e807b8,
+        0xa402b9c5a8d3a6e7,0x5f16206c9c6209a6,
+        0xcd036837130890a1,0x36dba887c37a8c0f,
+        0x802221226be55a64,0xc2494954da2c9789,
+        0xa02aa96b06deb0fd,0xf2db9baa10b7bd6c,
+        0xc83553c5c8965d3d,0x6f92829494e5acc7,
+        0xfa42a8b73abbf48c,0xcb772339ba1f17f9,
+        0x9c69a97284b578d7,0xff2a760414536efb,
+        0xc38413cf25e2d70d,0xfef5138519684aba,
+        0xf46518c2ef5b8cd1,0x7eb258665fc25d69,
+        0x98bf2f79d5993802,0xef2f773ffbd97a61,
+        0xbeeefb584aff8603,0xaafb550ffacfd8fa,
+        0xeeaaba2e5dbf6784,0x95ba2a53f983cf38,
+        0x952ab45cfa97a0b2,0xdd945a747bf26183,
+        0xba756174393d88df,0x94f971119aeef9e4,
+        0xe912b9d1478ceb17,0x7a37cd5601aab85d,
+        0x91abb422ccb812ee,0xac62e055c10ab33a,
+        0xb616a12b7fe617aa,0x577b986b314d6009,
+        0xe39c49765fdf9d94,0xed5a7e85fda0b80b,
+        0x8e41ade9fbebc27d,0x14588f13be847307,
+        0xb1d219647ae6b31c,0x596eb2d8ae258fc8,
+        0xde469fbd99a05fe3,0x6fca5f8ed9aef3bb,
+        0x8aec23d680043bee,0x25de7bb9480d5854,
+        0xada72ccc20054ae9,0xaf561aa79a10ae6a,
+        0xd910f7ff28069da4,0x1b2ba1518094da04,
+        0x87aa9aff79042286,0x90fb44d2f05d0842,
+        0xa99541bf57452b28,0x353a1607ac744a53,
+        0xd3fa922f2d1675f2,0x42889b8997915ce8,
+        0x847c9b5d7c2e09b7,0x69956135febada11,
+        0xa59bc234db398c25,0x43fab9837e699095,
+        0xcf02b2c21207ef2e,0x94f967e45e03f4bb,
+        0x8161afb94b44f57d,0x1d1be0eebac278f5,
+        0xa1ba1ba79e1632dc,0x6462d92a69731732,
+        0xca28a291859bbf93,0x7d7b8f7503cfdcfe,
+        0xfcb2cb35e702af78,0x5cda735244c3d43e,
+        0x9defbf01b061adab,0x3a0888136afa64a7,
+        0xc56baec21c7a1916,0x88aaa1845b8fdd0,
+        0xf6c69a72a3989f5b,0x8aad549e57273d45,
+        0x9a3c2087a63f6399,0x36ac54e2f678864b,
+        0xc0cb28a98fcf3c7f,0x84576a1bb416a7dd,
+        0xf0fdf2d3f3c30b9f,0x656d44a2a11c51d5,
+        0x969eb7c47859e743,0x9f644ae5a4b1b325,
+        0xbc4665b596706114,0x873d5d9f0dde1fee,
+        0xeb57ff22fc0c7959,0xa90cb506d155a7ea,
+        0x9316ff75dd87cbd8,0x9a7f12442d588f2,
+        0xb7dcbf5354e9bece,0xc11ed6d538aeb2f,
+        0xe5d3ef282a242e81,0x8f1668c8a86da5fa,
+        0x8fa475791a569d10,0xf96e017d694487bc,
+        0xb38d92d760ec4455,0x37c981dcc395a9ac,
+        0xe070f78d3927556a,0x85bbe253f47b1417,
+        0x8c469ab843b89562,0x93956d7478ccec8e,
+        0xaf58416654a6babb,0x387ac8d1970027b2,
+        0xdb2e51bfe9d0696a,0x6997b05fcc0319e,
+        0x88fcf317f22241e2,0x441fece3bdf81f03,
+        0xab3c2fddeeaad25a,0xd527e81cad7626c3,
+        0xd60b3bd56a5586f1,0x8a71e223d8d3b074,
+        0x85c7056562757456,0xf6872d5667844e49,
+        0xa738c6bebb12d16c,0xb428f8ac016561db,
+        0xd106f86e69d785c7,0xe13336d701beba52,
+        0x82a45b450226b39c,0xecc0024661173473,
+        0xa34d721642b06084,0x27f002d7f95d0190,
+        0xcc20ce9bd35c78a5,0x31ec038df7b441f4,
+        0xff290242c83396ce,0x7e67047175a15271,
+        0x9f79a169bd203e41,0xf0062c6e984d386,
+        0xc75809c42c684dd1,0x52c07b78a3e60868,
+        0xf92e0c3537826145,0xa7709a56ccdf8a82,
+        0x9bbcc7a142b17ccb,0x88a66076400bb691,
+        0xc2abf989935ddbfe,0x6acff893d00ea435,
+        0xf356f7ebf83552fe,0x583f6b8c4124d43,
+        0x98165af37b2153de,0xc3727a337a8b704a,
+        0xbe1bf1b059e9a8d6,0x744f18c0592e4c5c,
+        0xeda2ee1c7064130c,0x1162def06f79df73,
+        0x9485d4d1c63e8be7,0x8addcb5645ac2ba8,
+        0xb9a74a0637ce2ee1,0x6d953e2bd7173692,
+        0xe8111c87c5c1ba99,0xc8fa8db6ccdd0437,
+        0x910ab1d4db9914a0,0x1d9c9892400a22a2,
+        0xb54d5e4a127f59c8,0x2503beb6d00cab4b,
+        0xe2a0b5dc971f303a,0x2e44ae64840fd61d,
+        0x8da471a9de737e24,0x5ceaecfed289e5d2,
+        0xb10d8e1456105dad,0x7425a83e872c5f47,
+        0xdd50f1996b947518,0xd12f124e28f77719,
+        0x8a5296ffe33cc92f,0x82bd6b70d99aaa6f,
+        0xace73cbfdc0bfb7b,0x636cc64d1001550b,
+        0xd8210befd30efa5a,0x3c47f7e05401aa4e,
+        0x8714a775e3e95c78,0x65acfaec34810a71,
+        0xa8d9d1535ce3b396,0x7f1839a741a14d0d,
+        0xd31045a8341ca07c,0x1ede48111209a050,
+        0x83ea2b892091e44d,0x934aed0aab460432,
+        0xa4e4b66b68b65d60,0xf81da84d5617853f,
+        0xce1de40642e3f4b9,0x36251260ab9d668e,
+        0x80d2ae83e9ce78f3,0xc1d72b7c6b426019,
+        0xa1075a24e4421730,0xb24cf65b8612f81f,
+        0xc94930ae1d529cfc,0xdee033f26797b627,
+        0xfb9b7cd9a4a7443c,0x169840ef017da3b1,
+        0x9d412e0806e88aa5,0x8e1f289560ee864e,
+        0xc491798a08a2ad4e,0xf1a6f2bab92a27e2,
+        0xf5b5d7ec8acb58a2,0xae10af696774b1db,
+        0x9991a6f3d6bf1765,0xacca6da1e0a8ef29,
+        0xbff610b0cc6edd3f,0x17fd090a58d32af3,
+        0xeff394dcff8a948e,0xddfc4b4cef07f5b0,
+        0x95f83d0a1fb69cd9,0x4abdaf101564f98e,
+        0xbb764c4ca7a4440f,0x9d6d1ad41abe37f1,
+        0xea53df5fd18d5513,0x84c86189216dc5ed,
+        0x92746b9be2f8552c,0x32fd3cf5b4e49bb4,
+        0xb7118682dbb66a77,0x3fbc8c33221dc2a1,
+        0xe4d5e82392a40515,0xfabaf3feaa5334a,
+        0x8f05b1163ba6832d,0x29cb4d87f2a7400e,
+        0xb2c71d5bca9023f8,0x743e20e9ef511012,
+        0xdf78e4b2bd342cf6,0x914da9246b255416,
+        0x8bab8eefb6409c1a,0x1ad089b6c2f7548e,
+        0xae9672aba3d0c320,0xa184ac2473b529b1,
+        0xda3c0f568cc4f3e8,0xc9e5d72d90a2741e,
+        0x8865899617fb1871,0x7e2fa67c7a658892,
+        0xaa7eebfb9df9de8d,0xddbb901b98feeab7,
+        0xd51ea6fa85785631,0x552a74227f3ea565,
+        0x8533285c936b35de,0xd53a88958f87275f,
+        0xa67ff273b8460356,0x8a892abaf368f137,
+        0xd01fef10a657842c,0x2d2b7569b0432d85,
+        0x8213f56a67f6b29b,0x9c3b29620e29fc73,
+        0xa298f2c501f45f42,0x8349f3ba91b47b8f,
+        0xcb3f2f7642717713,0x241c70a936219a73,
+        0xfe0efb53d30dd4d7,0xed238cd383aa0110,
+        0x9ec95d1463e8a506,0xf4363804324a40aa,
+        0xc67bb4597ce2ce48,0xb143c6053edcd0d5,
+        0xf81aa16fdc1b81da,0xdd94b7868e94050a,
+        0x9b10a4e5e9913128,0xca7cf2b4191c8326,
+        0xc1d4ce1f63f57d72,0xfd1c2f611f63a3f0,
+        0xf24a01a73cf2dccf,0xbc633b39673c8cec,
+        0x976e41088617ca01,0xd5be0503e085d813,
+        0xbd49d14aa79dbc82,0x4b2d8644d8a74e18,
+        0xec9c459d51852ba2,0xddf8e7d60ed1219e,
+        0x93e1ab8252f33b45,0xcabb90e5c942b503,
+        0xb8da1662e7b00a17,0x3d6a751f3b936243,
+        0xe7109bfba19c0c9d,0xcc512670a783ad4,
+        0x906a617d450187e2,0x27fb2b80668b24c5,
+        0xb484f9dc9641e9da,0xb1f9f660802dedf6,
+        0xe1a63853bbd26451,0x5e7873f8a0396973,
+        0x8d07e33455637eb2,0xdb0b487b6423e1e8,
+        0xb049dc016abc5e5f,0x91ce1a9a3d2cda62,
+        0xdc5c5301c56b75f7,0x7641a140cc7810fb,
+        0x89b9b3e11b6329ba,0xa9e904c87fcb0a9d,
+        0xac2820d9623bf429,0x546345fa9fbdcd44,
+        0xd732290fbacaf133,0xa97c177947ad4095,
+        0x867f59a9d4bed6c0,0x49ed8eabcccc485d,
+        0xa81f301449ee8c70,0x5c68f256bfff5a74,
+        0xd226fc195c6a2f8c,0x73832eec6fff3111,
+        0x83585d8fd9c25db7,0xc831fd53c5ff7eab,
+        0xa42e74f3d032f525,0xba3e7ca8b77f5e55,
+        0xcd3a1230c43fb26f,0x28ce1bd2e55f35eb,
+        0x80444b5e7aa7cf85,0x7980d163cf5b81b3,
+        0xa0555e361951c366,0xd7e105bcc332621f,
+        0xc86ab5c39fa63440,0x8dd9472bf3fefaa7,
+        0xfa856334878fc150,0xb14f98f6f0feb951,
+        0x9c935e00d4b9d8d2,0x6ed1bf9a569f33d3,
+        0xc3b8358109e84f07,0xa862f80ec4700c8,
+        0xf4a642e14c6262c8,0xcd27bb612758c0fa,
+        0x98e7e9cccfbd7dbd,0x8038d51cb897789c,
+        0xbf21e44003acdd2c,0xe0470a63e6bd56c3,
+        0xeeea5d5004981478,0x1858ccfce06cac74,
+        0x95527a5202df0ccb,0xf37801e0c43ebc8,
+        0xbaa718e68396cffd,0xd30560258f54e6ba,
+        0xe950df20247c83fd,0x47c6b82ef32a2069,
+        0x91d28b7416cdd27e,0x4cdc331d57fa5441,
+        0xb6472e511c81471d,0xe0133fe4adf8e952,
+        0xe3d8f9e563a198e5,0x58180fddd97723a6,
+        0x8e679c2f5e44ff8f,0x570f09eaa7ea7648,};
+
+}
+}  // namespace arrow_vendored
+
+#endif
diff --git a/cpp/src/arrow/vendored/fast_float/float_common.h b/cpp/src/arrow/vendored/fast_float/float_common.h
new file mode 100644
index 00000000000..4d82e8769c2
--- /dev/null
+++ b/cpp/src/arrow/vendored/fast_float/float_common.h
@@ -0,0 +1,263 @@
+#ifndef FASTFLOAT_FLOAT_COMMON_H
+#define FASTFLOAT_FLOAT_COMMON_H
+
+#include <cfloat>
+#include <cstdint>
+#ifndef _WIN32
+// strcasecmp, strncasecmp 
+#include <strings.h>
+#endif
+
+#ifdef _MSC_VER
+#define fastfloat_really_inline __forceinline
+#else
+#define fastfloat_really_inline inline __attribute__((always_inline))
+#endif 
+
+#ifdef _WIN32
+#define fastfloat_strcasecmp _stricmp
+#define fastfloat_strncasecmp _strnicmp
+#else
+#define fastfloat_strcasecmp strcasecmp
+#define fastfloat_strncasecmp strncasecmp
+#endif
+namespace arrow_vendored {
+namespace fast_float {
+#ifndef FLT_EVAL_METHOD
+#error "FLT_EVAL_METHOD should be defined, please include cfloat."
+#endif
+
+
+
+
+
+
+bool is_space(uint8_t c) {
+    static const bool table[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+    return table[c];
+}
+
+namespace {
+constexpr uint32_t max_digits = 768;
+
+constexpr int32_t decimal_point_range = 2047;
+} // namespace
+
+
+struct value128 {
+  uint64_t low;
+  uint64_t high;
+  value128(uint64_t _low, uint64_t _high) : low(_low), high(_high) {}
+  value128() : low(0), high(0) {}
+};
+
+
+/* result might be undefined when input_num is zero */
+fastfloat_really_inline 
+int leading_zeroes(uint64_t input_num) {
+#ifdef _MSC_VER
+  unsigned long leading_zero = 0;
+  // Search the mask data from most significant bit (MSB)
+  // to least significant bit (LSB) for a set bit (1).
+  if (_BitScanReverse64(&leading_zero, input_num))
+    return (int)(63 - leading_zero);
+  else
+    return 64;
+#else
+  return __builtin_clzll(input_num);
+#endif
+}
+
+
+#if defined(_WIN32) && !defined(__clang__)
+// Note MinGW falls here too
+#include <intrin.h>
+
+#if !defined(_M_X64) && !defined(_M_ARM64)// _umul128 for x86, arm
+// this is a slow emulation routine for 32-bit Windows
+//
+fastfloat_really_inline uint64_t __emulu(uint32_t x, uint32_t y) {
+  return x * (uint64_t)y;
+}
+fastfloat_really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) {
+  uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd);
+  uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd);
+  uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32));
+  uint64_t adbc_carry = !!(adbc < ad);
+  uint64_t lo = bd + (adbc << 32);
+  *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) +
+        (adbc_carry << 32) + !!(lo < bd);
+  return lo;
+}
+#endif
+
+fastfloat_really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) {
+  value128 answer;
+#ifdef _M_ARM64
+  // ARM64 has native support for 64-bit multiplications, no need to emultate
+  answer.high = __umulh(value1, value2);
+  answer.low = value1 * value2;
+#else
+  answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64
+#endif // _M_ARM64
+  return answer;
+}
+
+#else
+
+// compute value1 * value2
+fastfloat_really_inline
+value128 full_multiplication(uint64_t value1, uint64_t value2) {
+  value128 answer;
+  __uint128_t r = ((__uint128_t)value1) * value2;
+  answer.low = uint64_t(r);
+  answer.high = uint64_t(r >> 64);
+  return answer;
+}
+
+#endif
+
+struct adjusted_mantissa {
+  uint64_t mantissa;
+  int power2;
+  adjusted_mantissa() : mantissa(0), power2(0) {}
+};
+
+struct decimal {
+  uint32_t num_digits;
+  int32_t decimal_point;
+  bool negative;
+  bool truncated;
+  uint8_t digits[max_digits];
+};
+
+constexpr static double powers_of_ten_double[] = {
+    1e0,  1e1,  1e2,  1e3,  1e4,  1e5,  1e6,  1e7,  1e8,  1e9,  1e10, 1e11,
+    1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22};
+constexpr static float powers_of_ten_float[] = {
+    1e0,  1e1,  1e2,  1e3,  1e4,  1e5,  1e6,  1e7,  1e8,  1e9,  1e10};
+
+template <typename T>
+struct binary_format {
+  static constexpr int mantissa_explicit_bits();
+  static constexpr int minimum_exponent();
+  static constexpr int infinite_power();
+  static constexpr int sign_index();
+  static constexpr int min_exponent_fast_path();
+  static constexpr int max_exponent_fast_path();
+  static constexpr int max_exponent_round_to_even();
+  static constexpr int min_exponent_round_to_even();
+  static constexpr uint64_t max_mantissa_fast_path();
+  static constexpr T exact_power_of_ten(int64_t power);
+};
+
+template <>
+constexpr int binary_format<double>::mantissa_explicit_bits() {
+  return 52;
+}
+template <>
+constexpr int binary_format<float>::mantissa_explicit_bits() { 
+  return 23;
+}
+
+template <>
+constexpr int binary_format<double>::max_exponent_round_to_even() {
+  return 23;
+}
+
+template <>
+constexpr int binary_format<float>::max_exponent_round_to_even() {
+  return 10;
+}
+
+
+template <>
+constexpr int binary_format<double>::min_exponent_round_to_even() {
+  return -4;
+}
+
+template <>
+constexpr int binary_format<float>::min_exponent_round_to_even() {
+  return -17;
+}
+
+template <>
+constexpr int binary_format<double>::minimum_exponent() { 
+  return -1023;
+}
+template <>
+constexpr int binary_format<float>::minimum_exponent() {
+  return -127;
+}
+
+template <>
+constexpr int binary_format<double>::infinite_power() {
+  return 0x7FF; 
+}
+template <>
+constexpr int binary_format<float>::infinite_power() { 
+  return 0xFF;
+}
+
+template <>
+constexpr int binary_format<double>::sign_index() { 
+  return 63;
+}
+template <>
+constexpr int binary_format<float>::sign_index() {
+  return 31;
+}
+
+template <>
+constexpr int binary_format<double>::min_exponent_fast_path() { 
+#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0)
+  return 0;
+#else
+  return -22;
+#endif
+}
+template <>
+constexpr int binary_format<float>::min_exponent_fast_path() {
+#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0)
+  return 0;
+#else
+  return -10;
+#endif
+}
+
+
+template <>
+constexpr int binary_format<double>::max_exponent_fast_path() { 
+  return 22;
+}
+template <>
+constexpr int binary_format<float>::max_exponent_fast_path() {
+  return 10;
+}
+
+
+template <>
+constexpr uint64_t binary_format<double>::max_mantissa_fast_path() { 
+  return uint64_t(2) << mantissa_explicit_bits();
+}
+template <>
+constexpr uint64_t binary_format<float>::max_mantissa_fast_path() {
+  return uint64_t(2) << mantissa_explicit_bits();
+}
+
+template <>
+constexpr double binary_format<double>::exact_power_of_ten(int64_t power) { 
+  return powers_of_ten_double[power];
+}
+template <>
+constexpr float binary_format<float>::exact_power_of_ten(int64_t power) {
+
+  return powers_of_ten_float[power];
+}
+
+ 
+
+} // namespace fast_float
+}  // namespace arrow_vendored
+
+#endif
diff --git a/cpp/src/arrow/vendored/fast_float/parse_number.h b/cpp/src/arrow/vendored/fast_float/parse_number.h
new file mode 100644
index 00000000000..f27a6d82f91
--- /dev/null
+++ b/cpp/src/arrow/vendored/fast_float/parse_number.h
@@ -0,0 +1,118 @@
+#ifndef FASTFLOAT_PARSE_NUMBER_H
+#define FASTFLOAT_PARSE_NUMBER_H
+#include "ascii_number.h"
+#include "decimal_to_binary.h"
+#include "thompson_tao.h"
+
+#include <cassert>
+#include <cmath>
+#include <cstring>
+#include <limits>
+#include <system_error>
+
+namespace arrow_vendored {
+namespace fast_float {
+
+
+namespace {
+/**
+ * Special case +inf, -inf, nan, infinity, -infinity.
+ * The case comparisons could be made much faster given that we know that the 
+ * strings a null-free and fixed.
+ **/
+template <typename T>
+from_chars_result parse_infnan(const char *first, const char *last, T &value)  noexcept  {
+  from_chars_result answer;
+  answer.ec = std::errc(); // be optimistic
+  if (last - first >= 3) {
+    if (fastfloat_strncasecmp(first, "nan", 3) == 0) {
+      answer.ptr = first + 3;
+      value = std::numeric_limits<T>::quiet_NaN();
+      return answer;
+    }
+    if (fastfloat_strncasecmp(first, "inf", 3) == 0) {
+
+      if ((last - first >= 8) && (fastfloat_strncasecmp(first, "infinity", 8) == 0)) {
+        answer.ptr = first + 8;
+      } else {
+        answer.ptr = first + 3;
+      }
+      value = std::numeric_limits<T>::infinity();
+      return answer;
+    }
+    if (last - first >= 4) {
+      if ((fastfloat_strncasecmp(first, "+nan", 4) == 0) || (fastfloat_strncasecmp(first, "-nan", 4) == 0)) {
+        answer.ptr = first + 4;
+        value = std::numeric_limits<T>::quiet_NaN();
+        if (first[0] == '-') {
+          value = -value;
+        }
+        return answer;
+      }
+
+      if ((fastfloat_strncasecmp(first, "+inf", 4) == 0) || (fastfloat_strncasecmp(first, "-inf", 4) == 0)) {
+        if ((last - first >= 8) && (fastfloat_strncasecmp(first + 1, "infinity", 8) == 0)) {
+          answer.ptr = first + 9;
+        } else {
+          answer.ptr = first + 4;
+        }
+        value = std::numeric_limits<T>::infinity();
+        if (first[0] == '-') {
+          value = -value;
+        }
+        return answer;
+      }
+    }
+  }
+  answer.ec = std::errc::invalid_argument;
+  return answer;
+}
+} // namespace
+
+
+
+template<typename T>
+from_chars_result from_chars(const char *first, const char *last,
+                             T &value, chars_format fmt /*= chars_format::general*/)  noexcept  {
+  static_assert (std::is_same<T, double>::value || std::is_same<T, float>::value, "only float and double are supported");
+
+
+  from_chars_result answer;
+  while ((first != last) && fast_float::is_space(*first)) {
+    first++;
+  }
+  if (first == last) {
+    answer.ec = std::errc::invalid_argument;
+    answer.ptr = first;
+    return answer;
+  }
+  parsed_number_string pns = parse_number_string(first, last, fmt);
+  if (!pns.valid) {
+    return parse_infnan(first, last, value);
+  }
+  answer.ec = std::errc(); // be optimistic
+  answer.ptr = pns.lastmatch;
+
+  if (binary_format<T>::min_exponent_fast_path() <= pns.exponent && pns.exponent <= binary_format<T>::max_exponent_fast_path() && pns.mantissa <=binary_format<T>::max_mantissa_fast_path()) {
+    value = T(pns.mantissa);
+    if (pns.exponent < 0) { value = value / binary_format<T>::exact_power_of_ten(-pns.exponent); } 
+    else { value = value * binary_format<T>::exact_power_of_ten(pns.exponent); }
+    if (pns.negative) { value = -value; }
+    return answer;
+  }
+  adjusted_mantissa am = pns.too_many_digits ? parse_long_mantissa<binary_format<T>>(first,last) : compute_float<binary_format<T>>(pns.exponent, pns.mantissa);
+  if(am.power2 < 0) {
+    am = parse_long_mantissa<binary_format<T>>(first,last);
+  }
+  uint64_t word = am.mantissa;
+  word |= uint64_t(am.power2) << binary_format<T>::mantissa_explicit_bits();
+  word = pns.negative 
+  ? word | (uint64_t(1) << binary_format<T>::sign_index()) : word;
+  memcpy(&value, &word, sizeof(T));
+  return answer;
+}
+
+} // namespace fast_float
+}  // namespace arrow_vendored
+
+#endif
diff --git a/cpp/src/arrow/vendored/fast_float/thompson_tao.h b/cpp/src/arrow/vendored/fast_float/thompson_tao.h
new file mode 100644
index 00000000000..c9ec1870c99
--- /dev/null
+++ b/cpp/src/arrow/vendored/fast_float/thompson_tao.h
@@ -0,0 +1,375 @@
+#ifndef FASTFLOAT_GENERIC_DECIMAL_TO_BINARY_H
+#define FASTFLOAT_GENERIC_DECIMAL_TO_BINARY_H
+
+/**
+ * This code is meant to handle the case where we have more than 19 digits.
+ * 
+ * Based on work by Nigel Tao (at https://github.com/google/wuffs/)
+ * who credits Ken Thompson for the design (via a reference to the Go source
+ * code). See
+ * https://github.com/google/wuffs/blob/aa46859ea40c72516deffa1b146121952d6dfd3b/internal/cgen/base/floatconv-submodule-data.c
+ * https://github.com/google/wuffs/blob/46cd8105f47ca07ae2ba8e6a7818ef9c0df6c152/internal/cgen/base/floatconv-submodule-code.c
+ * It is probably not very fast but it is a fallback that should almost never
+ * be used in reallife.
+ **/
+#include "ascii_number.h"
+#include "decimal_to_binary.h"
+#include <cstdint>
+
+namespace arrow_vendored {
+namespace fast_float {
+
+namespace {
+
+// remove all final zeroes
+inline void trim(decimal &h) {
+  while ((h.num_digits > 0) && (h.digits[h.num_digits - 1] == 0)) {
+    h.num_digits--;
+  }
+}
+
+#if 0
+/** If you ever want to see what is going on, the following function might prove handy:
+ * **/
+void print(const decimal d, int32_t exp2 = 0) {
+  printf("0.");
+  for(size_t i = 0; i < d.num_digits; i++) {
+    printf("%d", int(d.digits[i]));
+  }
+  printf(" * 10 **%d ", d.decimal_point);
+  printf(" * 2 **%d ", exp2);
+}
+#endif
+
+
+
+uint32_t number_of_digits_decimal_left_shift(decimal &h, uint32_t shift) {
+  shift &= 63;
+  const static uint16_t number_of_digits_decimal_left_shift_table[65] = {
+    0x0000, 0x0800, 0x0801, 0x0803, 0x1006, 0x1009, 0x100D, 0x1812, 0x1817,
+    0x181D, 0x2024, 0x202B, 0x2033, 0x203C, 0x2846, 0x2850, 0x285B, 0x3067,
+    0x3073, 0x3080, 0x388E, 0x389C, 0x38AB, 0x38BB, 0x40CC, 0x40DD, 0x40EF,
+    0x4902, 0x4915, 0x4929, 0x513E, 0x5153, 0x5169, 0x5180, 0x5998, 0x59B0,
+    0x59C9, 0x61E3, 0x61FD, 0x6218, 0x6A34, 0x6A50, 0x6A6D, 0x6A8B, 0x72AA,
+    0x72C9, 0x72E9, 0x7B0A, 0x7B2B, 0x7B4D, 0x8370, 0x8393, 0x83B7, 0x83DC,
+    0x8C02, 0x8C28, 0x8C4F, 0x9477, 0x949F, 0x94C8, 0x9CF2, 0x051C, 0x051C,
+    0x051C, 0x051C,
+  };
+  uint32_t x_a = number_of_digits_decimal_left_shift_table[shift];
+  uint32_t x_b = number_of_digits_decimal_left_shift_table[shift + 1];
+  uint32_t num_new_digits = x_a >> 11;
+  uint32_t pow5_a = 0x7FF & x_a;
+  uint32_t pow5_b = 0x7FF & x_b;
+  const static uint8_t
+    number_of_digits_decimal_left_shift_table_powers_of_5[0x051C] = {
+        5, 2, 5, 1, 2, 5, 6, 2, 5, 3, 1, 2, 5, 1, 5, 6, 2, 5, 7, 8, 1, 2, 5, 3,
+        9, 0, 6, 2, 5, 1, 9, 5, 3, 1, 2, 5, 9, 7, 6, 5, 6, 2, 5, 4, 8, 8, 2, 8,
+        1, 2, 5, 2, 4, 4, 1, 4, 0, 6, 2, 5, 1, 2, 2, 0, 7, 0, 3, 1, 2, 5, 6, 1,
+        0, 3, 5, 1, 5, 6, 2, 5, 3, 0, 5, 1, 7, 5, 7, 8, 1, 2, 5, 1, 5, 2, 5, 8,
+        7, 8, 9, 0, 6, 2, 5, 7, 6, 2, 9, 3, 9, 4, 5, 3, 1, 2, 5, 3, 8, 1, 4, 6,
+        9, 7, 2, 6, 5, 6, 2, 5, 1, 9, 0, 7, 3, 4, 8, 6, 3, 2, 8, 1, 2, 5, 9, 5,
+        3, 6, 7, 4, 3, 1, 6, 4, 0, 6, 2, 5, 4, 7, 6, 8, 3, 7, 1, 5, 8, 2, 0, 3,
+        1, 2, 5, 2, 3, 8, 4, 1, 8, 5, 7, 9, 1, 0, 1, 5, 6, 2, 5, 1, 1, 9, 2, 0,
+        9, 2, 8, 9, 5, 5, 0, 7, 8, 1, 2, 5, 5, 9, 6, 0, 4, 6, 4, 4, 7, 7, 5, 3,
+        9, 0, 6, 2, 5, 2, 9, 8, 0, 2, 3, 2, 2, 3, 8, 7, 6, 9, 5, 3, 1, 2, 5, 1,
+        4, 9, 0, 1, 1, 6, 1, 1, 9, 3, 8, 4, 7, 6, 5, 6, 2, 5, 7, 4, 5, 0, 5, 8,
+        0, 5, 9, 6, 9, 2, 3, 8, 2, 8, 1, 2, 5, 3, 7, 2, 5, 2, 9, 0, 2, 9, 8, 4,
+        6, 1, 9, 1, 4, 0, 6, 2, 5, 1, 8, 6, 2, 6, 4, 5, 1, 4, 9, 2, 3, 0, 9, 5,
+        7, 0, 3, 1, 2, 5, 9, 3, 1, 3, 2, 2, 5, 7, 4, 6, 1, 5, 4, 7, 8, 5, 1, 5,
+        6, 2, 5, 4, 6, 5, 6, 6, 1, 2, 8, 7, 3, 0, 7, 7, 3, 9, 2, 5, 7, 8, 1, 2,
+        5, 2, 3, 2, 8, 3, 0, 6, 4, 3, 6, 5, 3, 8, 6, 9, 6, 2, 8, 9, 0, 6, 2, 5,
+        1, 1, 6, 4, 1, 5, 3, 2, 1, 8, 2, 6, 9, 3, 4, 8, 1, 4, 4, 5, 3, 1, 2, 5,
+        5, 8, 2, 0, 7, 6, 6, 0, 9, 1, 3, 4, 6, 7, 4, 0, 7, 2, 2, 6, 5, 6, 2, 5,
+        2, 9, 1, 0, 3, 8, 3, 0, 4, 5, 6, 7, 3, 3, 7, 0, 3, 6, 1, 3, 2, 8, 1, 2,
+        5, 1, 4, 5, 5, 1, 9, 1, 5, 2, 2, 8, 3, 6, 6, 8, 5, 1, 8, 0, 6, 6, 4, 0,
+        6, 2, 5, 7, 2, 7, 5, 9, 5, 7, 6, 1, 4, 1, 8, 3, 4, 2, 5, 9, 0, 3, 3, 2,
+        0, 3, 1, 2, 5, 3, 6, 3, 7, 9, 7, 8, 8, 0, 7, 0, 9, 1, 7, 1, 2, 9, 5, 1,
+        6, 6, 0, 1, 5, 6, 2, 5, 1, 8, 1, 8, 9, 8, 9, 4, 0, 3, 5, 4, 5, 8, 5, 6,
+        4, 7, 5, 8, 3, 0, 0, 7, 8, 1, 2, 5, 9, 0, 9, 4, 9, 4, 7, 0, 1, 7, 7, 2,
+        9, 2, 8, 2, 3, 7, 9, 1, 5, 0, 3, 9, 0, 6, 2, 5, 4, 5, 4, 7, 4, 7, 3, 5,
+        0, 8, 8, 6, 4, 6, 4, 1, 1, 8, 9, 5, 7, 5, 1, 9, 5, 3, 1, 2, 5, 2, 2, 7,
+        3, 7, 3, 6, 7, 5, 4, 4, 3, 2, 3, 2, 0, 5, 9, 4, 7, 8, 7, 5, 9, 7, 6, 5,
+        6, 2, 5, 1, 1, 3, 6, 8, 6, 8, 3, 7, 7, 2, 1, 6, 1, 6, 0, 2, 9, 7, 3, 9,
+        3, 7, 9, 8, 8, 2, 8, 1, 2, 5, 5, 6, 8, 4, 3, 4, 1, 8, 8, 6, 0, 8, 0, 8,
+        0, 1, 4, 8, 6, 9, 6, 8, 9, 9, 4, 1, 4, 0, 6, 2, 5, 2, 8, 4, 2, 1, 7, 0,
+        9, 4, 3, 0, 4, 0, 4, 0, 0, 7, 4, 3, 4, 8, 4, 4, 9, 7, 0, 7, 0, 3, 1, 2,
+        5, 1, 4, 2, 1, 0, 8, 5, 4, 7, 1, 5, 2, 0, 2, 0, 0, 3, 7, 1, 7, 4, 2, 2,
+        4, 8, 5, 3, 5, 1, 5, 6, 2, 5, 7, 1, 0, 5, 4, 2, 7, 3, 5, 7, 6, 0, 1, 0,
+        0, 1, 8, 5, 8, 7, 1, 1, 2, 4, 2, 6, 7, 5, 7, 8, 1, 2, 5, 3, 5, 5, 2, 7,
+        1, 3, 6, 7, 8, 8, 0, 0, 5, 0, 0, 9, 2, 9, 3, 5, 5, 6, 2, 1, 3, 3, 7, 8,
+        9, 0, 6, 2, 5, 1, 7, 7, 6, 3, 5, 6, 8, 3, 9, 4, 0, 0, 2, 5, 0, 4, 6, 4,
+        6, 7, 7, 8, 1, 0, 6, 6, 8, 9, 4, 5, 3, 1, 2, 5, 8, 8, 8, 1, 7, 8, 4, 1,
+        9, 7, 0, 0, 1, 2, 5, 2, 3, 2, 3, 3, 8, 9, 0, 5, 3, 3, 4, 4, 7, 2, 6, 5,
+        6, 2, 5, 4, 4, 4, 0, 8, 9, 2, 0, 9, 8, 5, 0, 0, 6, 2, 6, 1, 6, 1, 6, 9,
+        4, 5, 2, 6, 6, 7, 2, 3, 6, 3, 2, 8, 1, 2, 5, 2, 2, 2, 0, 4, 4, 6, 0, 4,
+        9, 2, 5, 0, 3, 1, 3, 0, 8, 0, 8, 4, 7, 2, 6, 3, 3, 3, 6, 1, 8, 1, 6, 4,
+        0, 6, 2, 5, 1, 1, 1, 0, 2, 2, 3, 0, 2, 4, 6, 2, 5, 1, 5, 6, 5, 4, 0, 4,
+        2, 3, 6, 3, 1, 6, 6, 8, 0, 9, 0, 8, 2, 0, 3, 1, 2, 5, 5, 5, 5, 1, 1, 1,
+        5, 1, 2, 3, 1, 2, 5, 7, 8, 2, 7, 0, 2, 1, 1, 8, 1, 5, 8, 3, 4, 0, 4, 5,
+        4, 1, 0, 1, 5, 6, 2, 5, 2, 7, 7, 5, 5, 5, 7, 5, 6, 1, 5, 6, 2, 8, 9, 1,
+        3, 5, 1, 0, 5, 9, 0, 7, 9, 1, 7, 0, 2, 2, 7, 0, 5, 0, 7, 8, 1, 2, 5, 1,
+        3, 8, 7, 7, 7, 8, 7, 8, 0, 7, 8, 1, 4, 4, 5, 6, 7, 5, 5, 2, 9, 5, 3, 9,
+        5, 8, 5, 1, 1, 3, 5, 2, 5, 3, 9, 0, 6, 2, 5, 6, 9, 3, 8, 8, 9, 3, 9, 0,
+        3, 9, 0, 7, 2, 2, 8, 3, 7, 7, 6, 4, 7, 6, 9, 7, 9, 2, 5, 5, 6, 7, 6, 2,
+        6, 9, 5, 3, 1, 2, 5, 3, 4, 6, 9, 4, 4, 6, 9, 5, 1, 9, 5, 3, 6, 1, 4, 1,
+        8, 8, 8, 2, 3, 8, 4, 8, 9, 6, 2, 7, 8, 3, 8, 1, 3, 4, 7, 6, 5, 6, 2, 5,
+        1, 7, 3, 4, 7, 2, 3, 4, 7, 5, 9, 7, 6, 8, 0, 7, 0, 9, 4, 4, 1, 1, 9, 2,
+        4, 4, 8, 1, 3, 9, 1, 9, 0, 6, 7, 3, 8, 2, 8, 1, 2, 5, 8, 6, 7, 3, 6, 1,
+        7, 3, 7, 9, 8, 8, 4, 0, 3, 5, 4, 7, 2, 0, 5, 9, 6, 2, 2, 4, 0, 6, 9, 5,
+        9, 5, 3, 3, 6, 9, 1, 4, 0, 6, 2, 5,
+  };
+  const uint8_t *pow5 =
+      &number_of_digits_decimal_left_shift_table_powers_of_5[pow5_a];
+  uint32_t i = 0;
+  uint32_t n = pow5_b - pow5_a;
+  for (; i < n; i++) {
+    if (i >= h.num_digits) {
+      return num_new_digits - 1;
+    } else if (h.digits[i] == pow5[i]) {
+      continue;
+    } else if (h.digits[i] < pow5[i]) {
+      return num_new_digits - 1;
+    } else {
+      return num_new_digits;
+    }
+  }
+  return num_new_digits;
+}
+
+} // end of anonymous namespace
+
+uint64_t round(decimal &h) {
+  if ((h.num_digits == 0) || (h.decimal_point < 0)) {
+    return 0;
+  } else if (h.decimal_point > 18) {
+    return UINT64_MAX;
+  }
+  // at this point, we know that h.decimal_point >= 0
+  uint32_t dp = uint32_t(h.decimal_point);
+  uint64_t n = 0;
+  for (uint32_t i = 0; i < dp; i++) {
+    n = (10 * n) + ((i < h.num_digits) ? h.digits[i] : 0);
+  }
+  bool round_up = false;
+  if (dp < h.num_digits) {
+    round_up = h.digits[dp] >= 5; // normally, we round up    
+    // but we may need to round to even!
+    if ((h.digits[dp] == 5) && (dp + 1 == h.num_digits)) {
+      round_up = h.truncated || ((dp > 0) && (1 & h.digits[dp - 1]));
+    }
+  }
+  if (round_up) {
+    n++;
+  }
+  return n;
+}
+
+// computes h * 2^-shift
+void decimal_left_shift(decimal &h, uint32_t shift) {
+  if (h.num_digits == 0) {
+    return;
+  }
+  uint32_t num_new_digits = number_of_digits_decimal_left_shift(h, shift);
+  int32_t read_index = int32_t(h.num_digits - 1);
+  uint32_t write_index = h.num_digits - 1 + num_new_digits;
+  uint64_t n = 0;
+
+  while (read_index >= 0) {
+    n += uint64_t(h.digits[read_index]) << shift;
+    uint64_t quotient = n / 10;
+    uint64_t remainder = n - (10 * quotient);
+    if (write_index < max_digits) {
+      h.digits[write_index] = uint8_t(remainder);
+    } else if (remainder > 0) {
+      h.truncated = true;
+    }
+    n = quotient;
+    write_index--;
+    read_index--;
+  }
+  while (n > 0) {
+    uint64_t quotient = n / 10;
+    uint64_t remainder = n - (10 * quotient);
+    if (write_index < max_digits) {
+      h.digits[write_index] = uint8_t(remainder);
+    } else if (remainder > 0) {
+      h.truncated = true;
+    }
+    n = quotient;
+    write_index--;
+  }
+  h.num_digits += num_new_digits;
+  if (h.num_digits > max_digits) {
+    h.num_digits = max_digits;
+  }
+  h.decimal_point += int32_t(num_new_digits);
+  trim(h);
+}
+
+// computes h * 2^shift
+void decimal_right_shift(decimal &h, uint32_t shift) {
+  uint32_t read_index = 0;
+  uint32_t write_index = 0;
+
+  uint64_t n = 0;
+
+  while ((n >> shift) == 0) {
+    if (read_index < h.num_digits) {
+      n = (10 * n) + h.digits[read_index++];
+    } else if (n == 0) {
+      return;
+    } else {
+      while ((n >> shift) == 0) {
+        n = 10 * n;
+        read_index++;
+      }
+      break;
+    }
+  }
+  h.decimal_point -= int32_t(read_index - 1);
+  if (h.decimal_point < -decimal_point_range) { // it is zero
+    h.num_digits = 0;
+    h.decimal_point = 0;
+    h.negative = false;
+    h.truncated = false;
+    return;
+  }
+  uint64_t mask = (uint64_t(1) << shift) - 1;
+  while (read_index < h.num_digits) {
+    uint8_t new_digit = uint8_t(n >> shift);
+    n = (10 * (n & mask)) + h.digits[read_index++];
+    h.digits[write_index++] = new_digit;
+  }
+  while (n > 0) {
+    uint8_t new_digit = uint8_t(n >> shift);
+    n = 10 * (n & mask);
+    if (write_index < max_digits) {
+      h.digits[write_index++] = new_digit;
+    } else if (new_digit > 0) {
+      h.truncated = true;
+    }
+  }
+  h.num_digits = write_index;
+  trim(h);
+}
+
+
+template <typename binary>
+adjusted_mantissa compute_float(decimal &d) {
+  adjusted_mantissa answer;
+  if (d.num_digits == 0) {
+    // should be zero
+    answer.power2 = 0;
+    answer.mantissa = 0;
+    return answer;
+  }
+  // At this point, going further, we can assume that d.num_digits > 0.
+  //
+  // We want to guard against excessive decimal point values because
+  // they can result in long running times. Indeed, we do 
+  // shifts by at most 60 bits. We have that log(10**400)/log(2**60) ~= 22
+  // which is fine, but log(10**299995)/log(2**60) ~= 16609 which is not
+  // fine (runs for a long time).
+  //
+  if(d.decimal_point < -324) {
+    // We have something smaller than 1e-324 which is always zero
+    // in binary64 and binary32. 
+    // It should be zero.
+    answer.power2 = 0;
+    answer.mantissa = 0;
+    return answer;
+  } else if(d.decimal_point >= 310) {
+    // We have something at least as large as 0.1e310 which is
+    // always infinite.    
+    answer.power2 = binary::infinite_power();
+    answer.mantissa = 0;
+    return answer;
+  }
+  static const uint32_t max_shift = 60;
+  static const uint32_t num_powers = 19;
+  static const uint8_t powers[19] = {
+      0,  3,  6,  9,  13, 16, 19, 23, 26, 29, //
+      33, 36, 39, 43, 46, 49, 53, 56, 59,     //
+  };
+  int32_t exp2 = 0;
+  while (d.decimal_point > 0) {
+    uint32_t n = uint32_t(d.decimal_point);
+    uint32_t shift = (n < num_powers) ? powers[n] : max_shift;
+    decimal_right_shift(d, shift);
+    if (d.decimal_point < -decimal_point_range) {
+      // should be zero
+      answer.power2 = 0;
+      answer.mantissa = 0;
+      return answer;
+    }
+    exp2 += int32_t(shift);
+  }
+  // We shift left toward [1/2 ... 1].
+  while (d.decimal_point <= 0) {
+    uint32_t shift;
+    if (d.decimal_point == 0) {
+      if (d.digits[0] >= 5) {
+        break;
+      }
+      shift = (d.digits[0] < 2) ? 2 : 1;
+    } else {
+      uint32_t n = uint32_t(-d.decimal_point);
+      shift = (n < num_powers) ? powers[n] : max_shift;
+    }
+    decimal_left_shift(d, shift);
+    if (d.decimal_point > decimal_point_range) {
+      // we want to get infinity:
+      answer.power2 = 0xFF;
+      answer.mantissa = 0;
+      return answer;
+    }
+    exp2 -= int32_t(shift);
+  }
+  // We are now in the range [1/2 ... 1] but the binary format uses [1 ... 2].
+  exp2--;
+  constexpr int32_t minimum_exponent = binary::minimum_exponent();
+  while ((minimum_exponent + 1) > exp2) {
+    uint32_t n = uint32_t((minimum_exponent + 1) - exp2);
+    if (n > max_shift) {
+      n = max_shift;
+    }
+    decimal_right_shift(d, n);
+    exp2 += int32_t(n);
+  }
+  if ((exp2 - minimum_exponent) >= binary::infinite_power()) {
+    answer.power2 = binary::infinite_power();
+    answer.mantissa = 0;
+    return answer;
+  }
+
+  const int mantissa_size_in_bits = binary::mantissa_explicit_bits() + 1;
+  decimal_left_shift(d, mantissa_size_in_bits);
+
+  uint64_t mantissa = round(d);
+  // It is possible that we have an overflow, in which case we need
+  // to shift back.
+  if(mantissa >= (uint64_t(1) << mantissa_size_in_bits)) {
+    decimal_right_shift(d, 1);
+    exp2 += 1;
+    mantissa = round(d);
+    if ((exp2 - minimum_exponent) >= binary::infinite_power()) {
+      answer.power2 = binary::infinite_power();
+      answer.mantissa = 0;
+      return answer;
+    }
+  }
+  answer.power2 = exp2  - binary::minimum_exponent();
+  if(mantissa < (uint64_t(1) << binary::mantissa_explicit_bits())) { answer.power2--; }
+  answer.mantissa = mantissa & ((uint64_t(1) << binary::mantissa_explicit_bits()) - 1);
+  return answer;
+}
+
+template <typename binary>
+adjusted_mantissa parse_long_mantissa(const char *first, const char* last) {
+    decimal d = parse_decimal(first, last);
+    return compute_float<binary>(d);
+}
+
+} // namespace fast_float
+}  // namespace arrow_vendored
+#endif
diff --git a/cpp/src/arrow/vendored/string_view.hpp b/cpp/src/arrow/vendored/string_view.hpp
index 55a934874bc..a2d5567854f 100644
--- a/cpp/src/arrow/vendored/string_view.hpp
+++ b/cpp/src/arrow/vendored/string_view.hpp
@@ -1,6 +1,6 @@
-// Vendored from git changeset 88ad9b5b3771b2ceb01879ed3c8f296b3a0c9bdd
+// Vendored from git changeset v1.4.0
 
-// Copyright 2017-2019 by Martin Moene
+// Copyright 2017-2020 by Martin Moene
 //
 // string-view lite, a C++17-like string_view for C++98 and later.
 // For more information see https://github.com/martinmoene/string-view-lite
@@ -201,16 +201,17 @@ using std::operator<<;
 
 // Compiler versions:
 //
-// MSVC++ 6.0  _MSC_VER == 1200 (Visual Studio 6.0)
-// MSVC++ 7.0  _MSC_VER == 1300 (Visual Studio .NET 2002)
-// MSVC++ 7.1  _MSC_VER == 1310 (Visual Studio .NET 2003)
-// MSVC++ 8.0  _MSC_VER == 1400 (Visual Studio 2005)
-// MSVC++ 9.0  _MSC_VER == 1500 (Visual Studio 2008)
-// MSVC++ 10.0 _MSC_VER == 1600 (Visual Studio 2010)
-// MSVC++ 11.0 _MSC_VER == 1700 (Visual Studio 2012)
-// MSVC++ 12.0 _MSC_VER == 1800 (Visual Studio 2013)
-// MSVC++ 14.0 _MSC_VER == 1900 (Visual Studio 2015)
-// MSVC++ 14.1 _MSC_VER >= 1910 (Visual Studio 2017)
+// MSVC++  6.0  _MSC_VER == 1200  nssv_COMPILER_MSVC_VERSION ==  60  (Visual Studio 6.0)
+// MSVC++  7.0  _MSC_VER == 1300  nssv_COMPILER_MSVC_VERSION ==  70  (Visual Studio .NET 2002)
+// MSVC++  7.1  _MSC_VER == 1310  nssv_COMPILER_MSVC_VERSION ==  71  (Visual Studio .NET 2003)
+// MSVC++  8.0  _MSC_VER == 1400  nssv_COMPILER_MSVC_VERSION ==  80  (Visual Studio 2005)
+// MSVC++  9.0  _MSC_VER == 1500  nssv_COMPILER_MSVC_VERSION ==  90  (Visual Studio 2008)
+// MSVC++ 10.0  _MSC_VER == 1600  nssv_COMPILER_MSVC_VERSION == 100  (Visual Studio 2010)
+// MSVC++ 11.0  _MSC_VER == 1700  nssv_COMPILER_MSVC_VERSION == 110  (Visual Studio 2012)
+// MSVC++ 12.0  _MSC_VER == 1800  nssv_COMPILER_MSVC_VERSION == 120  (Visual Studio 2013)
+// MSVC++ 14.0  _MSC_VER == 1900  nssv_COMPILER_MSVC_VERSION == 140  (Visual Studio 2015)
+// MSVC++ 14.1  _MSC_VER >= 1910  nssv_COMPILER_MSVC_VERSION == 141  (Visual Studio 2017)
+// MSVC++ 14.2  _MSC_VER >= 1920  nssv_COMPILER_MSVC_VERSION == 142  (Visual Studio 2019)
 
 #if defined(_MSC_VER ) && !defined(__clang__)
 # define nssv_COMPILER_MSVC_VER      (_MSC_VER )
@@ -410,6 +411,22 @@ namespace nonstd { namespace sv_lite {
 
 namespace detail {
 
+#if nssv_CPP14_OR_GREATER
+
+template< typename CharT >
+inline constexpr std::size_t length( CharT * s, std::size_t result = 0 )
+{
+    CharT * v = s;
+    std::size_t r = result;
+    while ( *v != '\0' ) {
+       ++v;
+       ++r;
+    }
+    return r;
+}
+
+#else // nssv_CPP14_OR_GREATER
+
 // Expect tail call optimization to make length() non-recursive:
 
 template< typename CharT >
@@ -418,6 +435,8 @@ inline constexpr std::size_t length( CharT * s, std::size_t result = 0 )
     return *s == '\0' ? result : length( s + 1, result + 1 );
 }
 
+#endif // nssv_CPP14_OR_GREATER
+
 } // namespace detail
 
 #endif // nssv_CPP11_OR_GREATER
@@ -857,7 +876,7 @@ class basic_string_view
     {
         const basic_string_view v;
 
-        nssv_constexpr explicit not_in_view( basic_string_view v ) : v( v ) {}
+        nssv_constexpr explicit not_in_view( basic_string_view v_ ) : v( v_ ) {}
 
         nssv_constexpr bool operator()( CharT c ) const
         {
@@ -993,12 +1012,12 @@ nssv_constexpr bool operator>= (
 template< class CharT, class Traits>
 nssv_constexpr bool operator==(
     basic_string_view<CharT, Traits> lhs,
-    char const * rhs ) nssv_noexcept
+    CharT const * rhs ) nssv_noexcept
 { return lhs.compare( rhs ) == 0; }
 
 template< class CharT, class Traits>
 nssv_constexpr bool operator==(
-    char const * lhs,
+    CharT const * lhs,
     basic_string_view<CharT, Traits> rhs ) nssv_noexcept
 { return rhs.compare( lhs ) == 0; }
 
diff --git a/cpp/src/arrow/vendored/uriparser/UriDefsConfig.h b/cpp/src/arrow/vendored/uriparser/UriDefsConfig.h
index 9af4968d50e..d604494b002 100644
--- a/cpp/src/arrow/vendored/uriparser/UriDefsConfig.h
+++ b/cpp/src/arrow/vendored/uriparser/UriDefsConfig.h
@@ -82,7 +82,8 @@
 /* Intel C/C++ */
 /* http://predef.sourceforge.net/precomp.html#sec20 */
 /* http://www.intel.com/support/performancetools/c/windows/sb/CS-007751.htm#2 */
-# define URI_INLINE __force_inline
+/* EDIT 11/5/20. Intel changed __force_inline to __forceinline */
+# define URI_INLINE __forceinline
 #elif defined(_MSC_VER)
 /* Microsoft Visual C++ */
 /* http://predef.sourceforge.net/precomp.html#sec32 */
diff --git a/cpp/src/arrow/vendored/utfcpp/README.md b/cpp/src/arrow/vendored/utfcpp/README.md
new file mode 100644
index 00000000000..c0abfd7d11d
--- /dev/null
+++ b/cpp/src/arrow/vendored/utfcpp/README.md
@@ -0,0 +1,28 @@
+<!---
+    Boost Software License - Version 1.0 - August 17th, 2003
+
+    Permission is hereby granted, free of charge, to any person or organization
+    obtaining a copy of the software and accompanying documentation covered by
+    this license (the "Software") to use, reproduce, display, distribute,
+    execute, and transmit the Software, and to prepare derivative works of the
+    Software, and to permit third-parties to whom the Software is furnished to
+    do so, all subject to the following:
+
+    The copyright notices in the Software and this entire statement, including
+    the above license grant, this restriction and the following disclaimer,
+    must be included in all copies of the Software, in whole or in part, and
+    all derivative works of the Software, unless such copies or derivative
+    works are solely in the form of machine-executable object code generated by
+    a source language processor.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+    SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+    FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+    ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+    DEALINGS IN THE SOFTWARE.
+-->
+
+The files in this directory are vendored from utfcpp git tag v3.1.1
+(https://github.com/nemtrif/utfcpp).
diff --git a/cpp/src/arrow/vendored/utf8cpp/checked.h b/cpp/src/arrow/vendored/utfcpp/checked.h
similarity index 87%
rename from cpp/src/arrow/vendored/utf8cpp/checked.h
rename to cpp/src/arrow/vendored/utfcpp/checked.h
index 38f1748cd74..648636e4686 100644
--- a/cpp/src/arrow/vendored/utf8cpp/checked.h
+++ b/cpp/src/arrow/vendored/utfcpp/checked.h
@@ -1,6 +1,4 @@
-// Vendored from v2.3.4 (http://utfcpp.sourceforge.net/)
-
-// Copyright 2006 Nemanja Trifunovic
+// Copyright 2006-2016 Nemanja Trifunovic
 
 /*
 Permission is hereby granted, free of charge, to any person or organization
@@ -43,8 +41,8 @@ namespace utf8
     class invalid_code_point : public exception {
         uint32_t cp;
     public:
-        invalid_code_point(uint32_t cp) : cp(cp) {}
-        virtual const char* what() const throw() { return "Invalid code point"; }
+        invalid_code_point(uint32_t codepoint) : cp(codepoint) {}
+        virtual const char* what() const NOEXCEPT OVERRIDE { return "Invalid code point"; }
         uint32_t code_point() const {return cp;}
     };
 
@@ -52,7 +50,7 @@ namespace utf8
         uint8_t u8;
     public:
         invalid_utf8 (uint8_t u) : u8(u) {}
-        virtual const char* what() const throw() { return "Invalid UTF-8"; }
+        virtual const char* what() const NOEXCEPT OVERRIDE { return "Invalid UTF-8"; }
         uint8_t utf8_octet() const {return u8;}
     };
 
@@ -60,13 +58,13 @@ namespace utf8
         uint16_t u16;
     public:
         invalid_utf16 (uint16_t u) : u16(u) {}
-        virtual const char* what() const throw() { return "Invalid UTF-16"; }
+        virtual const char* what() const NOEXCEPT OVERRIDE { return "Invalid UTF-16"; }
         uint16_t utf16_word() const {return u16;}
     };
 
     class not_enough_room : public exception {
     public:
-        virtual const char* what() const throw() { return "Not enough space"; }
+        virtual const char* what() const NOEXCEPT OVERRIDE { return "Not enough space"; }
     };
 
     /// The library API - functions intended to be called by the users
@@ -109,7 +107,9 @@ namespace utf8
                         *out++ = *it;
                     break;
                 case internal::NOT_ENOUGH_ROOM:
-                    throw not_enough_room();
+                    out = utf8::append (replacement, out);
+                    start = end;
+                    break;
                 case internal::INVALID_LEAD:
                     out = utf8::append (replacement, out);
                     ++start;
@@ -176,23 +176,19 @@ namespace utf8
         return utf8::peek_next(it, end);
     }
 
-    /// Deprecated in versions that include "prior"
-    template <typename octet_iterator>
-    uint32_t previous(octet_iterator& it, octet_iterator pass_start)
-    {
-        octet_iterator end = it;
-        while (utf8::internal::is_trail(*(--it)))
-            if (it == pass_start)
-                throw invalid_utf8(*it); // error - no lead byte in the sequence
-        octet_iterator temp = it;
-        return utf8::next(temp, end);
-    }
-
     template <typename octet_iterator, typename distance_type>
     void advance (octet_iterator& it, distance_type n, octet_iterator end)
     {
-        for (distance_type i = 0; i < n; ++i)
-            utf8::next(it, end);
+        const distance_type zero(0);
+        if (n < zero) {
+            // backward
+            for (distance_type i = n; i < zero; ++i)
+                utf8::prior(it, end);
+        } else {
+            // forward
+            for (distance_type i = zero; i < n; ++i)
+                utf8::next(it, end);
+        }
     }
 
     template <typename octet_iterator>
@@ -235,7 +231,7 @@ namespace utf8
     template <typename u16bit_iterator, typename octet_iterator>
     u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
     {
-        while (start != end) {
+        while (start < end) {
             uint32_t cp = utf8::next(start, end);
             if (cp > 0xffff) { //make a surrogate pair
                 *result++ = static_cast<uint16_t>((cp >> 10)   + internal::LEAD_OFFSET);
@@ -259,7 +255,7 @@ namespace utf8
     template <typename octet_iterator, typename u32bit_iterator>
     u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result)
     {
-        while (start != end)
+        while (start < end)
             (*result++) = utf8::next(start, end);
 
         return result;
@@ -267,16 +263,21 @@ namespace utf8
 
     // The iterator class
     template <typename octet_iterator>
-    class iterator : public std::iterator <std::bidirectional_iterator_tag, uint32_t> {
+    class iterator {
       octet_iterator it;
       octet_iterator range_start;
       octet_iterator range_end;
       public:
+      typedef uint32_t value_type;
+      typedef uint32_t* pointer;
+      typedef uint32_t& reference;
+      typedef std::ptrdiff_t difference_type;
+      typedef std::bidirectional_iterator_tag iterator_category;
       iterator () {}
       explicit iterator (const octet_iterator& octet_it,
-                         const octet_iterator& range_start,
-                         const octet_iterator& range_end) :
-               it(octet_it), range_start(range_start), range_end(range_end)
+                         const octet_iterator& rangestart,
+                         const octet_iterator& rangeend) :
+               it(octet_it), range_start(rangestart), range_end(rangeend)
       {
           if (it < range_start || it > range_end)
               throw std::out_of_range("Invalid utf-8 iterator position");
@@ -324,6 +325,9 @@ namespace utf8
 
 } // namespace utf8
 
-#endif //header guard
+#if UTF_CPP_CPLUSPLUS >= 201103L // C++ 11 or later
+#include "cpp11.h"
+#endif // C++ 11 or later
 
+#endif //header guard
 
diff --git a/cpp/src/arrow/vendored/utf8cpp/core.h b/cpp/src/arrow/vendored/utfcpp/core.h
similarity index 93%
rename from cpp/src/arrow/vendored/utf8cpp/core.h
rename to cpp/src/arrow/vendored/utfcpp/core.h
index 98e6c6e7dea..244e8923112 100644
--- a/cpp/src/arrow/vendored/utf8cpp/core.h
+++ b/cpp/src/arrow/vendored/utfcpp/core.h
@@ -1,5 +1,3 @@
-// Vendored from v2.3.4 (http://utfcpp.sourceforge.net/)
-
 // Copyright 2006 Nemanja Trifunovic
 
 /*
@@ -32,6 +30,23 @@ DEALINGS IN THE SOFTWARE.
 
 #include <iterator>
 
+// Determine the C++ standard version.
+// If the user defines UTF_CPP_CPLUSPLUS, use that.
+// Otherwise, trust the unreliable predefined macro __cplusplus
+
+#if !defined UTF_CPP_CPLUSPLUS
+    #define UTF_CPP_CPLUSPLUS __cplusplus
+#endif
+
+#if UTF_CPP_CPLUSPLUS >= 201103L // C++ 11 or later
+    #define OVERRIDE override
+    #define NOEXCEPT noexcept
+#else // C++ 98/03
+    #define OVERRIDE
+    #define NOEXCEPT throw()
+#endif // C++ 11 or later
+
+
 namespace utf8
 {
     // The typedefs for 8-bit, 16-bit and 32-bit unsigned integers
@@ -51,8 +66,8 @@ namespace internal
     const uint16_t LEAD_SURROGATE_MAX  = 0xdbffu;
     const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u;
     const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu;
-    const uint16_t LEAD_OFFSET         = LEAD_SURROGATE_MIN - (0x10000 >> 10);
-    const uint32_t SURROGATE_OFFSET    = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN;
+    const uint16_t LEAD_OFFSET         = 0xd7c0u;       // LEAD_SURROGATE_MIN - (0x10000 >> 10)
+    const uint32_t SURROGATE_OFFSET    = 0xfca02400u;   // 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN
 
     // Maximum valid value for a Unicode code point
     const uint32_t CODE_POINT_MAX      = 0x0010ffffu;
@@ -144,7 +159,7 @@ namespace internal
 
         if (!utf8::internal::is_trail(*it))
             return INCOMPLETE_SEQUENCE;
-        
+
         return UTF8_OK;
     }
 
@@ -167,7 +182,7 @@ namespace internal
     {
         if (it == end) 
             return NOT_ENOUGH_ROOM;
-        
+
         code_point = utf8::internal::mask8(*it);
 
         UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
@@ -224,6 +239,9 @@ namespace internal
     template <typename octet_iterator>
     utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point)
     {
+        if (it == end)
+            return NOT_ENOUGH_ROOM;
+
         // Save the original value of it so we can go back in case of failure
         // Of course, it does not make much sense with i.e. stream iterators
         octet_iterator original_it = it;
@@ -236,7 +254,7 @@ namespace internal
         // Get trail octets and calculate the code point
         utf_error err = UTF8_OK;
         switch (length) {
-            case 0: 
+            case 0:
                 return INVALID_LEAD;
             case 1:
                 err = utf8::internal::get_sequence_1(it, end, cp);
@@ -312,18 +330,7 @@ namespace internal
             ((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) &&
             ((it != end) && (utf8::internal::mask8(*it))   == bom[2])
            );
-    }
-	
-    //Deprecated in release 2.3 
-    template <typename octet_iterator>
-    inline bool is_bom (octet_iterator it)
-    {
-        return (
-            (utf8::internal::mask8(*it++)) == bom[0] &&
-            (utf8::internal::mask8(*it++)) == bom[1] &&
-            (utf8::internal::mask8(*it))   == bom[2]
-           );
-    }
+    }	
 } // namespace utf8
 
 #endif // header guard
diff --git a/cpp/src/arrow/vendored/utfcpp/cpp11.h b/cpp/src/arrow/vendored/utfcpp/cpp11.h
new file mode 100644
index 00000000000..d93961b04f8
--- /dev/null
+++ b/cpp/src/arrow/vendored/utfcpp/cpp11.h
@@ -0,0 +1,103 @@
+// Copyright 2018 Nemanja Trifunovic
+
+/*
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+*/
+
+
+#ifndef UTF8_FOR_CPP_a184c22c_d012_11e8_a8d5_f2801f1b9fd1
+#define UTF8_FOR_CPP_a184c22c_d012_11e8_a8d5_f2801f1b9fd1
+
+#include "checked.h"
+#include <string>
+
+namespace utf8
+{
+
+    inline void append(char32_t cp, std::string& s)
+    {
+        append(uint32_t(cp), std::back_inserter(s));
+    }
+
+    inline std::string utf16to8(const std::u16string& s)
+    {
+        std::string result;
+        utf16to8(s.begin(), s.end(), std::back_inserter(result));
+        return result;
+    }
+
+    inline std::u16string utf8to16(const std::string& s)
+    {
+        std::u16string result;
+        utf8to16(s.begin(), s.end(), std::back_inserter(result));
+        return result;
+    }
+
+    inline std::string utf32to8(const std::u32string& s)
+    {
+        std::string result;
+        utf32to8(s.begin(), s.end(), std::back_inserter(result));
+        return result;
+    }
+
+    inline std::u32string utf8to32(const std::string& s)
+    {
+        std::u32string result;
+        utf8to32(s.begin(), s.end(), std::back_inserter(result));
+        return result;
+    }
+
+    inline std::size_t find_invalid(const std::string& s)
+    {
+        std::string::const_iterator invalid = find_invalid(s.begin(), s.end());
+        return (invalid == s.end()) ? std::string::npos : (invalid - s.begin());
+    }
+
+    inline bool is_valid(const std::string& s)
+    {
+        return is_valid(s.begin(), s.end());
+    }
+
+    inline std::string replace_invalid(const std::string& s, char32_t replacement)
+    {
+        std::string result;
+        replace_invalid(s.begin(), s.end(), std::back_inserter(result), replacement);
+        return result;
+    }
+
+    inline std::string replace_invalid(const std::string& s)
+    {
+        std::string result;
+        replace_invalid(s.begin(), s.end(), std::back_inserter(result));
+        return result;
+    }
+
+    inline bool starts_with_bom(const std::string& s)
+    {
+        return starts_with_bom(s.begin(), s.end());
+    }
+ 
+} // namespace utf8
+
+#endif // header guard
+
diff --git a/cpp/src/arrow/vendored/xxhash.h b/cpp/src/arrow/vendored/xxhash.h
index 014c766888a..a33cdf8610d 100644
--- a/cpp/src/arrow/vendored/xxhash.h
+++ b/cpp/src/arrow/vendored/xxhash.h
@@ -15,14 +15,4 @@
 // specific language governing permissions and limitations
 // under the License.
 
-// Workaround https://github.com/Cyan4973/xxHash/issues/249
-#if defined(_MSC_VER)
-#pragma warning(push)
-#pragma warning(disable : 4146)
-#endif
-
 #include "arrow/vendored/xxhash/xxhash.h"
-
-#if defined(_MSC_VER)
-#pragma warning(pop)
-#endif
diff --git a/cpp/src/arrow/vendored/xxhash/README.md b/cpp/src/arrow/vendored/xxhash/README.md
index 06a7f731a83..a24fa68d868 100644
--- a/cpp/src/arrow/vendored/xxhash/README.md
+++ b/cpp/src/arrow/vendored/xxhash/README.md
@@ -17,4 +17,5 @@
   under the License.
 -->
 
-The files in this directory are vendored from xxHash git tag v0.7.1.
+The files in this directory are vendored from xxHash git tag v0.8.0
+(https://github.com/Cyan4973/xxHash).
diff --git a/cpp/src/arrow/vendored/xxhash/xxh3.h b/cpp/src/arrow/vendored/xxhash/xxh3.h
deleted file mode 100644
index 77dc955dd9e..00000000000
--- a/cpp/src/arrow/vendored/xxhash/xxh3.h
+++ /dev/null
@@ -1,1583 +0,0 @@
-/*
-   xxHash - Extremely Fast Hash algorithm
-   Development source file for `xxh3`
-   Copyright (C) 2019-present, Yann Collet.
-
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions are
-   met:
-
-       * Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-       * Redistributions in binary form must reproduce the above
-   copyright notice, this list of conditions and the following disclaimer
-   in the documentation and/or other materials provided with the
-   distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-   You can contact the author at :
-   - xxHash source repository : https://github.com/Cyan4973/xxHash
-*/
-
-/* Note :
-   This file is separated for development purposes.
-   It will be integrated into `xxhash.c` when development phase is complete.
-*/
-
-#ifndef XXH3_H
-#define XXH3_H
-
-
-/* ===   Dependencies   === */
-
-#undef XXH_INLINE_ALL   /* in case it's already defined */
-#define XXH_INLINE_ALL
-#include "xxhash.h"
-
-
-/* ===   Compiler specifics   === */
-
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* >= C99 */
-#  define XXH_RESTRICT   restrict
-#else
-/* note : it might be useful to define __restrict or __restrict__ for some C++ compilers */
-#  define XXH_RESTRICT   /* disable */
-#endif
-
-#if defined(__GNUC__)
-#  if defined(__AVX2__)
-#    include <immintrin.h>
-#  elif defined(__SSE2__)
-#    include <emmintrin.h>
-#  elif defined(__ARM_NEON__) || defined(__ARM_NEON)
-#    define inline __inline__  /* clang bug */
-#    include <arm_neon.h>
-#    undef inline
-#  endif
-#elif defined(_MSC_VER)
-#  include <intrin.h>
-#endif
-
-
-
-/* ==========================================
- * Vectorization detection
- * ========================================== */
-#define XXH_SCALAR 0
-#define XXH_SSE2   1
-#define XXH_AVX2   2
-#define XXH_NEON   3
-#define XXH_VSX    4
-
-#ifndef XXH_VECTOR    /* can be defined on command line */
-#  if defined(__AVX2__)
-#    define XXH_VECTOR XXH_AVX2
-#  elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2))
-#    define XXH_VECTOR XXH_SSE2
-#  elif defined(__GNUC__) /* msvc support maybe later */ \
-  && (defined(__ARM_NEON__) || defined(__ARM_NEON)) \
-  && defined(__LITTLE_ENDIAN__) /* ARM big endian is a thing */
-#    define XXH_VECTOR XXH_NEON
-#  elif defined(__PPC64__) && defined(__VSX__) && defined(__GNUC__)
-#    define XXH_VECTOR XXH_VSX
-#  else
-#    define XXH_VECTOR XXH_SCALAR
-#  endif
-#endif
-
-/* control alignment of accumulator,
- * for compatibility with fast vector loads */
-#ifndef XXH_ACC_ALIGN
-#  if XXH_VECTOR == 0   /* scalar */
-#     define XXH_ACC_ALIGN 8
-#  elif XXH_VECTOR == 1  /* sse2 */
-#     define XXH_ACC_ALIGN 16
-#  elif XXH_VECTOR == 2  /* avx2 */
-#     define XXH_ACC_ALIGN 32
-#  elif XXH_VECTOR == 3  /* neon */
-#     define XXH_ACC_ALIGN 16
-#  elif XXH_VECTOR == 4  /* vsx */
-#     define XXH_ACC_ALIGN 16
-#  endif
-#endif
-
-/* U64 XXH_mult32to64(U32 a, U64 b) { return (U64)a * (U64)b; } */
-#if defined(_MSC_VER) && defined(_M_IX86)
-#    include <intrin.h>
-#    define XXH_mult32to64(x, y) __emulu(x, y)
-#else
-#    define XXH_mult32to64(x, y) ((U64)((x) & 0xFFFFFFFF) * (U64)((y) & 0xFFFFFFFF))
-#endif
-
-/* VSX stuff */
-#if XXH_VECTOR == XXH_VSX
-#  include <altivec.h>
-#  undef vector
-typedef __vector unsigned long long U64x2;
-typedef __vector unsigned U32x4;
-/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */
-XXH_FORCE_INLINE U64x2 XXH_vsxMultOdd(U32x4 a, U32x4 b) {
-    U64x2 result;
-    __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
-    return result;
-}
-XXH_FORCE_INLINE U64x2 XXH_vsxMultEven(U32x4 a, U32x4 b) {
-    U64x2 result;
-    __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
-    return result;
-}
-#endif
-
-
-/* ==========================================
- * XXH3 default settings
- * ========================================== */
-
-#define XXH_SECRET_DEFAULT_SIZE 192   /* minimum XXH3_SECRET_SIZE_MIN */
-
-#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN)
-#  error "default keyset is not large enough"
-#endif
-
-XXH_ALIGN(64) static const BYTE kSecret[XXH_SECRET_DEFAULT_SIZE] = {
-    0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
-    0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
-    0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
-    0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
-    0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
-    0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
-    0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
-    0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
-
-    0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
-    0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
-    0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
-    0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e,
-};
-
-
-static XXH128_hash_t
-XXH3_mul128(U64 ll1, U64 ll2)
-{
-/* __uint128_t seems a bad choice with emscripten current, see https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 */
-#if !defined(__wasm__) && defined(__SIZEOF_INT128__) || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
-
-    __uint128_t lll = (__uint128_t)ll1 * ll2;
-    XXH128_hash_t const r128 = { (U64)(lll), (U64)(lll >> 64) };
-    return r128;
-
-#elif defined(_M_X64) || defined(_M_IA64)
-
-#ifndef _MSC_VER
-#   pragma intrinsic(_umul128)
-#endif
-    U64 llhigh;
-    U64 const lllow = _umul128(ll1, ll2, &llhigh);
-    XXH128_hash_t const r128 = { lllow, llhigh };
-    return r128;
-
-#else /* Portable scalar version */
-
-    /* emulate 64x64->128b multiplication, using four 32x32->64 */
-    U32 const h1 = (U32)(ll1 >> 32);
-    U32 const h2 = (U32)(ll2 >> 32);
-    U32 const l1 = (U32)ll1;
-    U32 const l2 = (U32)ll2;
-
-    U64 const llh  = XXH_mult32to64(h1, h2);
-    U64 const llm1 = XXH_mult32to64(l1, h2);
-    U64 const llm2 = XXH_mult32to64(h1, l2);
-    U64 const lll  = XXH_mult32to64(l1, l2);
-
-    U64 const t = lll + (llm1 << 32);
-    U64 const carry1 = t < lll;
-
-    U64 const lllow = t + (llm2 << 32);
-    U64 const carry2 = lllow < t;
-    U64 const llhigh = llh + (llm1 >> 32) + (llm2 >> 32) + carry1 + carry2;
-
-    XXH128_hash_t const r128 = { lllow, llhigh };
-    return r128;
-
-#endif
-}
-
-
-#if defined(__GNUC__) && defined(__i386__)
-/* GCC is stupid and tries to vectorize this.
- * This tells GCC that it is wrong. */
-__attribute__((__target__("no-sse")))
-#endif
-static U64
-XXH3_mul128_fold64(U64 ll1, U64 ll2)
-{
-/* __uint128_t seems a bad choice with emscripten current, see https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 */
-#if !defined(__wasm__) && defined(__SIZEOF_INT128__) || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
-
-    __uint128_t lll = (__uint128_t)ll1 * ll2;
-    return (U64)lll ^ (U64)(lll >> 64);
-
-#elif defined(_M_X64) || defined(_M_IA64)
-
-#ifndef _MSC_VER
-#   pragma intrinsic(_umul128)
-#endif
-    U64 llhigh;
-    U64 const lllow = _umul128(ll1, ll2, &llhigh);
-    return lllow ^ llhigh;
-
-    /* We have to do it out manually on 32-bit.
-     * This is a modified, unrolled, widened, and optimized version of the
-     * mulqdu routine from Hacker's Delight.
-     *
-     *   https://www.hackersdelight.org/hdcodetxt/mulqdu.c.txt
-     *
-     * This was modified to use U32->U64 multiplication instead
-     * of U16->U32, to add the high and low values in the end,
-     * be endian-independent, and I added a partial assembly
-     * implementation for ARM. */
-
-    /* An easy 128-bit folding multiply on ARMv6T2 and ARMv7-A/R can be done with
-     * the mighty umaal (Unsigned Multiply Accumulate Accumulate Long) which takes 4 cycles
-     * or less, doing a long multiply and adding two 32-bit integers:
-     *
-     *     void umaal(U32 *RdLo, U32 *RdHi, U32 Rn, U32 Rm)
-     *     {
-     *         U64 prodAcc = (U64)Rn * (U64)Rm;
-     *         prodAcc += *RdLo;
-     *         prodAcc += *RdHi;
-     *         *RdLo = prodAcc & 0xFFFFFFFF;
-     *         *RdHi = prodAcc >> 32;
-     *     }
-     *
-     * This is compared to umlal which adds to a single 64-bit integer:
-     *
-     *     void umlal(U32 *RdLo, U32 *RdHi, U32 Rn, U32 Rm)
-     *     {
-     *         U64 prodAcc = (U64)Rn * (U64)Rm;
-     *         prodAcc += (*RdLo | ((U64)*RdHi << 32);
-     *         *RdLo = prodAcc & 0xFFFFFFFF;
-     *         *RdHi = prodAcc >> 32;
-     *     }
-     *
-     * Getting the compiler to emit them is like pulling teeth, and checking
-     * for it is annoying because ARMv7-M lacks this instruction. However, it
-     * is worth it, because this is an otherwise expensive operation. */
-
-     /* GCC-compatible, ARMv6t2 or ARMv7+, non-M variant, and 32-bit */
-#elif defined(__GNUC__) /* GCC-compatible */ \
-    && defined(__ARM_ARCH) && !defined(__aarch64__) && !defined(__arm64__) /* 32-bit ARM */\
-    && !defined(__ARM_ARCH_7M__) /* <- Not ARMv7-M  vv*/ \
-        && !(defined(__TARGET_ARCH_ARM) && __TARGET_ARCH_ARM == 0 && __TARGET_ARCH_THUMB == 4) \
-    && (defined(__ARM_ARCH_6T2__) || __ARM_ARCH > 6) /* ARMv6T2 or later */
-
-    U32 w[4] = { 0 };
-    U32 u[2] = { (U32)(ll1 >> 32), (U32)ll1 };
-    U32 v[2] = { (U32)(ll2 >> 32), (U32)ll2 };
-    U32 k;
-
-    /* U64 t = (U64)u[1] * (U64)v[1];
-     * w[3] = t & 0xFFFFFFFF;
-     * k = t >> 32; */
-    __asm__("umull %0, %1, %2, %3"
-            : "=r" (w[3]), "=r" (k)
-            : "r" (u[1]), "r" (v[1]));
-
-    /* t = (U64)u[0] * (U64)v[1] + w[2] + k;
-     * w[2] = t & 0xFFFFFFFF;
-     * k = t >> 32; */
-    __asm__("umaal %0, %1, %2, %3"
-            : "+r" (w[2]), "+r" (k)
-            : "r" (u[0]), "r" (v[1]));
-    w[1] = k;
-    k = 0;
-
-    /* t = (U64)u[1] * (U64)v[0] + w[2] + k;
-     * w[2] = t & 0xFFFFFFFF;
-     * k = t >> 32; */
-    __asm__("umaal %0, %1, %2, %3"
-            : "+r" (w[2]), "+r" (k)
-            : "r" (u[1]), "r" (v[0]));
-
-    /* t = (U64)u[0] * (U64)v[0] + w[1] + k;
-     * w[1] = t & 0xFFFFFFFF;
-     * k = t >> 32; */
-    __asm__("umaal %0, %1, %2, %3"
-            : "+r" (w[1]), "+r" (k)
-            : "r" (u[0]), "r" (v[0]));
-    w[0] = k;
-
-    return (w[1] | ((U64)w[0] << 32)) ^ (w[3] | ((U64)w[2] << 32));
-
-#else /* Portable scalar version */
-
-    /* emulate 64x64->128b multiplication, using four 32x32->64 */
-    U32 const h1 = (U32)(ll1 >> 32);
-    U32 const h2 = (U32)(ll2 >> 32);
-    U32 const l1 = (U32)ll1;
-    U32 const l2 = (U32)ll2;
-
-    U64 const llh  = XXH_mult32to64(h1, h2);
-    U64 const llm1 = XXH_mult32to64(l1, h2);
-    U64 const llm2 = XXH_mult32to64(h1, l2);
-    U64 const lll  = XXH_mult32to64(l1, l2);
-
-    U64 const t = lll + (llm1 << 32);
-    U64 const carry1 = t < lll;
-
-    U64 const lllow = t + (llm2 << 32);
-    U64 const carry2 = lllow < t;
-    U64 const llhigh = llh + (llm1 >> 32) + (llm2 >> 32) + carry1 + carry2;
-
-    return llhigh ^ lllow;
-
-#endif
-}
-
-
-static XXH64_hash_t XXH3_avalanche(U64 h64)
-{
-    h64 ^= h64 >> 37;
-    h64 *= PRIME64_3;
-    h64 ^= h64 >> 32;
-    return h64;
-}
-
-
-/* ==========================================
- * Short keys
- * ========================================== */
-
-XXH_FORCE_INLINE XXH64_hash_t
-XXH3_len_1to3_64b(const void* data, size_t len, const void* keyPtr, XXH64_hash_t seed)
-{
-    XXH_ASSERT(data != NULL);
-    XXH_ASSERT(1 <= len && len <= 3);
-    XXH_ASSERT(keyPtr != NULL);
-    {   BYTE const c1 = ((const BYTE*)data)[0];
-        BYTE const c2 = ((const BYTE*)data)[len >> 1];
-        BYTE const c3 = ((const BYTE*)data)[len - 1];
-        U32  const combined = ((U32)c1) + (((U32)c2) << 8) + (((U32)c3) << 16) + (((U32)len) << 24);
-        U64  const keyed = (U64)combined ^ (XXH_readLE32(keyPtr) + seed);
-        U64  const mixed = keyed * PRIME64_1;
-        return XXH3_avalanche(mixed);
-    }
-}
-
-XXH_FORCE_INLINE XXH64_hash_t
-XXH3_len_4to8_64b(const void* data, size_t len, const void* keyPtr, XXH64_hash_t seed)
-{
-    XXH_ASSERT(data != NULL);
-    XXH_ASSERT(keyPtr != NULL);
-    XXH_ASSERT(4 <= len && len <= 8);
-    {   U32 const in1 = XXH_readLE32(data);
-        U32 const in2 = XXH_readLE32((const BYTE*)data + len - 4);
-        U64 const in64 = in1 + ((U64)in2 << 32);
-        U64 const keyed = in64 ^ (XXH_readLE64(keyPtr) + seed);
-        U64 const mix64 = len + ((keyed ^ (keyed >> 51)) * PRIME32_1);
-        return XXH3_avalanche((mix64 ^ (mix64 >> 47)) * PRIME64_2);
-    }
-}
-
-XXH_FORCE_INLINE XXH64_hash_t
-XXH3_len_9to16_64b(const void* data, size_t len, const void* keyPtr, XXH64_hash_t seed)
-{
-    XXH_ASSERT(data != NULL);
-    XXH_ASSERT(keyPtr != NULL);
-    XXH_ASSERT(9 <= len && len <= 16);
-    {   const U64* const key64 = (const U64*) keyPtr;
-        U64 const ll1 = XXH_readLE64(data) ^ (XXH_readLE64(key64) + seed);
-        U64 const ll2 = XXH_readLE64((const BYTE*)data + len - 8) ^ (XXH_readLE64(key64+1) - seed);
-        U64 const acc = len + (ll1 + ll2) + XXH3_mul128_fold64(ll1, ll2);
-        return XXH3_avalanche(acc);
-    }
-}
-
-XXH_FORCE_INLINE XXH64_hash_t
-XXH3_len_0to16_64b(const void* data, size_t len, const void* keyPtr, XXH64_hash_t seed)
-{
-    XXH_ASSERT(len <= 16);
-    {   if (len > 8) return XXH3_len_9to16_64b(data, len, keyPtr, seed);
-        if (len >= 4) return XXH3_len_4to8_64b(data, len, keyPtr, seed);
-        if (len) return XXH3_len_1to3_64b(data, len, keyPtr, seed);
-        return 0;
-    }
-}
-
-
-/* ===    Long Keys    === */
-
-#define STRIPE_LEN 64
-#define XXH_SECRET_CONSUME_RATE 8   /* nb of secret bytes consumed at each accumulation */
-#define ACC_NB (STRIPE_LEN / sizeof(U64))
-
-typedef enum { XXH3_acc_64bits, XXH3_acc_128bits } XXH3_accWidth_e;
-
-XXH_FORCE_INLINE void
-XXH3_accumulate_512(      void* XXH_RESTRICT acc,
-                    const void* XXH_RESTRICT data,
-                    const void* XXH_RESTRICT key,
-                    XXH3_accWidth_e accWidth)
-{
-#if (XXH_VECTOR == XXH_AVX2)
-
-    XXH_ASSERT((((size_t)acc) & 31) == 0);
-    {   XXH_ALIGN(32) __m256i* const xacc  =       (__m256i *) acc;
-        const         __m256i* const xdata = (const __m256i *) data;  /* not really aligned, just for ptr arithmetic, and because _mm256_loadu_si256() requires this type */
-        const         __m256i* const xkey  = (const __m256i *) key;   /* not really aligned, just for ptr arithmetic, and because _mm256_loadu_si256() requires this type */
-
-        size_t i;
-        for (i=0; i < STRIPE_LEN/sizeof(__m256i); i++) {
-            __m256i const d   = _mm256_loadu_si256 (xdata+i);
-            __m256i const k   = _mm256_loadu_si256 (xkey+i);
-            __m256i const dk  = _mm256_xor_si256 (d,k);                                  /* uint32 dk[8]  = {d0+k0, d1+k1, d2+k2, d3+k3, ...} */
-            __m256i const mul = _mm256_mul_epu32 (dk, _mm256_shuffle_epi32 (dk, 0x31));  /* uint64 mul[4] = {dk0*dk1, dk2*dk3, ...} */
-            if (accWidth == XXH3_acc_128bits) {
-                __m256i const dswap = _mm256_shuffle_epi32(d, _MM_SHUFFLE(1,0,3,2));
-                __m256i const add = _mm256_add_epi64(xacc[i], dswap);
-                xacc[i]  = _mm256_add_epi64(mul, add);
-            } else {  /* XXH3_acc_64bits */
-                __m256i const add = _mm256_add_epi64(xacc[i], d);
-                xacc[i]  = _mm256_add_epi64(mul, add);
-            }
-    }   }
-
-#elif (XXH_VECTOR == XXH_SSE2)
-
-    XXH_ASSERT((((size_t)acc) & 15) == 0);
-    {   XXH_ALIGN(16) __m128i* const xacc  =       (__m128i *) acc;   /* presumed */
-        const         __m128i* const xdata = (const __m128i *) data;  /* not really aligned, just for ptr arithmetic, and because _mm_loadu_si128() requires this type */
-        const         __m128i* const xkey  = (const __m128i *) key;   /* not really aligned, just for ptr arithmetic, and because _mm_loadu_si128() requires this type */
-
-        size_t i;
-        for (i=0; i < STRIPE_LEN/sizeof(__m128i); i++) {
-            __m128i const d   = _mm_loadu_si128 (xdata+i);
-            __m128i const k   = _mm_loadu_si128 (xkey+i);
-            __m128i const dk  = _mm_xor_si128 (d,k);                                 /* uint32 dk[4]  = {d0+k0, d1+k1, d2+k2, d3+k3} */
-            __m128i const mul = _mm_mul_epu32 (dk, _mm_shuffle_epi32 (dk, 0x31));    /* uint64 mul[2] = {dk0*dk1,dk2*dk3} */
-            if (accWidth == XXH3_acc_128bits) {
-                __m128i const dswap = _mm_shuffle_epi32(d, _MM_SHUFFLE(1,0,3,2));
-                __m128i const add = _mm_add_epi64(xacc[i], dswap);
-                xacc[i]  = _mm_add_epi64(mul, add);
-            } else {  /* XXH3_acc_64bits */
-                __m128i const add = _mm_add_epi64(xacc[i], d);
-                xacc[i]  = _mm_add_epi64(mul, add);
-            }
-    }   }
-
-#elif (XXH_VECTOR == XXH_NEON)
-
-    XXH_ASSERT((((size_t)acc) & 15) == 0);
-    {
-        XXH_ALIGN(16) uint64x2_t* const xacc = (uint64x2_t *) acc;
-        /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */
-        uint32_t const* const xdata = (const uint32_t *) data;
-        uint32_t const* const xkey  = (const uint32_t *) key;
-
-        size_t i;
-        for (i=0; i < STRIPE_LEN / sizeof(uint64x2_t); i++) {
-#if !defined(__aarch64__) && !defined(__arm64__) && defined(__GNUC__) /* ARM32-specific hack */
-            /* vzip on ARMv7 Clang generates a lot of vmovs (technically vorrs) without this.
-             * vzip on 32-bit ARM NEON will overwrite the original register, and I think that Clang
-             * assumes I don't want to destroy it and tries to make a copy. This slows down the code
-             * a lot.
-             * aarch64 not only uses an entirely different syntax, but it requires three
-             * instructions...
-             *    ext    v1.16B, v0.16B, #8    // select high bits because aarch64 can't address them directly
-             *    zip1   v3.2s, v0.2s, v1.2s   // first zip
-             *    zip2   v2.2s, v0.2s, v1.2s   // second zip
-             * ...to do what ARM does in one:
-             *    vzip.32 d0, d1               // Interleave high and low bits and overwrite. */
-
-            /* data_vec = xdata[i]; */
-            uint32x4_t const data_vec    = vld1q_u32(xdata + (i * 4));
-            /* key_vec  = xkey[i];  */
-            uint32x4_t const key_vec     = vld1q_u32(xkey  + (i * 4));
-            /* data_key = data_vec ^ key_vec; */
-            uint32x4_t       data_key;
-
-            if (accWidth == XXH3_acc_64bits) {
-                /* Add first to prevent register swaps */
-                /* xacc[i] += data_vec; */
-                xacc[i] = vaddq_u64 (xacc[i], vreinterpretq_u64_u32(data_vec));
-            } else {  /* XXH3_acc_128bits */
-                /* xacc[i] += swap(data_vec); */
-                /* can probably be optimized better */
-                uint64x2_t const data64 = vreinterpretq_u64_u32(data_vec);
-                uint64x2_t const swapped= vextq_u64(data64, data64, 1);
-                xacc[i] = vaddq_u64 (xacc[i], swapped);
-            }
-
-            data_key = veorq_u32(data_vec, key_vec);
-
-            /* Here's the magic. We use the quirkiness of vzip to shuffle data_key in place.
-             * shuffle: data_key[0, 1, 2, 3] = data_key[0, 2, 1, 3] */
-            __asm__("vzip.32 %e0, %f0" : "+w" (data_key));
-            /* xacc[i] += (uint64x2_t) data_key[0, 1] * (uint64x2_t) data_key[2, 3]; */
-            xacc[i] = vmlal_u32(xacc[i], vget_low_u32(data_key), vget_high_u32(data_key));
-
-#else
-            /* On aarch64, vshrn/vmovn seems to be equivalent to, if not faster than, the vzip method. */
-
-            /* data_vec = xdata[i]; */
-            uint32x4_t const data_vec    = vld1q_u32(xdata + (i * 4));
-            /* key_vec  = xkey[i];  */
-            uint32x4_t const key_vec     = vld1q_u32(xkey  + (i * 4));
-            /* data_key = data_vec ^ key_vec; */
-            uint32x4_t const data_key    = veorq_u32(data_vec, key_vec);
-            /* data_key_lo = (uint32x2_t) (data_key & 0xFFFFFFFF); */
-            uint32x2_t const data_key_lo = vmovn_u64  (vreinterpretq_u64_u32(data_key));
-            /* data_key_hi = (uint32x2_t) (data_key >> 32); */
-            uint32x2_t const data_key_hi = vshrn_n_u64 (vreinterpretq_u64_u32(data_key), 32);
-            if (accWidth == XXH3_acc_64bits) {
-                /* xacc[i] += data_vec; */
-                xacc[i] = vaddq_u64 (xacc[i], vreinterpretq_u64_u32(data_vec));
-            } else {  /* XXH3_acc_128bits */
-                /* xacc[i] += swap(data_vec); */
-                uint64x2_t const data64 = vreinterpretq_u64_u32(data_vec);
-                uint64x2_t const swapped= vextq_u64(data64, data64, 1);
-                xacc[i] = vaddq_u64 (xacc[i], swapped);
-            }
-            /* xacc[i] += (uint64x2_t) data_key_lo * (uint64x2_t) data_key_hi; */
-            xacc[i] = vmlal_u32 (xacc[i], data_key_lo, data_key_hi);
-
-#endif
-        }
-    }
-
-#elif (XXH_VECTOR == XXH_VSX) && 0   /* <=========================== DISABLED : MUST BE VALIDATED */
-    /* note : vsx code path currently not tested in CI (limitation of cross-compiler and/or emulator)
-     *        for vsx code path to be shipped and supported, it is critical to create a CI test for it */
-          U64x2* const xacc =        (U64x2*) acc;    /* presumed aligned */
-    U64x2 const* const xdata = (U64x2 const*) data;   /* no alignment restriction */
-    U64x2 const* const xkey  = (U64x2 const*) key;    /* no alignment restriction */
-    U64x2 const v32 = { 32,  32 };
-
-    size_t i;
-    for (i = 0; i < STRIPE_LEN / sizeof(U64x2); i++) {
-        /* data_vec = xdata[i]; */
-        /* key_vec = xkey[i]; */
-#ifdef __BIG_ENDIAN__
-        /* byteswap */
-        U64x2 const data_vec = vec_revb(vec_vsx_ld(0, xdata + i));  /* note : vec_revb is power9+ */
-        U64x2 const key_vec = vec_revb(vec_vsx_ld(0, xkey + i));    /* note : vec_revb is power9+ */
-#else
-        U64x2 const data_vec = vec_vsx_ld(0, xdata + i);
-        U64x2 const key_vec = vec_vsx_ld(0, xkey + i);
-#endif
-        U64x2 const data_key = data_vec ^ key_vec;
-        /* shuffled = (data_key << 32) | (data_key >> 32); */
-        U32x4 const shuffled = (U32x4)vec_rl(data_key, v32);
-        /* product = ((U64x2)data_key & 0xFFFFFFFF) * ((U64x2)shuffled & 0xFFFFFFFF); */
-        U64x2 const product = XXH_vsxMultOdd((U32x4)data_key, shuffled);
-
-        xacc[i] += product;
-
-        if (accWidth == XXH3_acc_64bits) {
-            xacc[i] += data_vec;
-        } else {  /* XXH3_acc_128bits */
-            U64x2 const data_swapped = vec_permi(data_vec, data_vec, 2);   /* <===== untested !!! */
-            xacc[i] += data_swapped;
-        }
-    }
-
-#else   /* scalar variant of Accumulator - universal */
-
-    XXH_ALIGN(XXH_ACC_ALIGN) U64* const xacc = (U64*) acc;    /* presumed aligned on 32-bytes boundaries, little hint for the auto-vectorizer */
-    const char* const xdata = (const char*) data;  /* no alignment restriction */
-    const char* const xkey  = (const char*) key;   /* no alignment restriction */
-    size_t i;
-    XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0);
-    for (i=0; i < ACC_NB; i+=2) {
-        U64 const in1 = XXH_readLE64(xdata + 8*i);
-        U64 const in2 = XXH_readLE64(xdata + 8*(i+1));
-        U64 const key1  = XXH_readLE64(xkey + 8*i);
-        U64 const key2  = XXH_readLE64(xkey + 8*(i+1));
-        U64 const data_key1 = key1 ^ in1;
-        U64 const data_key2 = key2 ^ in2;
-        xacc[i]   += XXH_mult32to64(data_key1 & 0xFFFFFFFF, data_key1 >> 32);
-        xacc[i+1] += XXH_mult32to64(data_key2 & 0xFFFFFFFF, data_key2 >> 32);
-        if (accWidth == XXH3_acc_128bits) {
-            xacc[i]   += in2;
-            xacc[i+1] += in1;
-        } else {  /* XXH3_acc_64bits */
-            xacc[i]   += in1;
-            xacc[i+1] += in2;
-        }
-    }
-#endif
-}
-
-XXH_FORCE_INLINE void
-XXH3_scrambleAcc(void* XXH_RESTRICT acc, const void* XXH_RESTRICT key)
-{
-#if (XXH_VECTOR == XXH_AVX2)
-
-    XXH_ASSERT((((size_t)acc) & 31) == 0);
-    {   XXH_ALIGN(32) __m256i* const xacc = (__m256i*) acc;
-        const         __m256i* const xkey = (const __m256i *) key;   /* not really aligned, just for ptr arithmetic, and because _mm256_loadu_si256() requires this argument type */
-        const __m256i prime32 = _mm256_set1_epi32((int)PRIME32_1);
-
-        size_t i;
-        for (i=0; i < STRIPE_LEN/sizeof(__m256i); i++) {
-            __m256i data = xacc[i];
-            __m256i const shifted = _mm256_srli_epi64(data, 47);
-            data = _mm256_xor_si256(data, shifted);
-
-            {   __m256i const k   = _mm256_loadu_si256 (xkey+i);
-                __m256i const dk  = _mm256_xor_si256   (data, k);
-
-                __m256i const dk1 = _mm256_mul_epu32 (dk, prime32);
-
-                __m256i const d2  = _mm256_shuffle_epi32 (dk, 0x31);
-                __m256i const dk2 = _mm256_mul_epu32 (d2, prime32);
-                __m256i const dk2h= _mm256_slli_epi64 (dk2, 32);
-
-                xacc[i] = _mm256_add_epi64(dk1, dk2h);
-        }   }
-    }
-
-#elif (XXH_VECTOR == XXH_SSE2)
-
-    {   XXH_ALIGN(16) __m128i* const xacc = (__m128i*) acc;
-        const         __m128i* const xkey = (const __m128i *) key;   /* not really aligned, just for ptr arithmetic */
-        const __m128i prime32 = _mm_set1_epi32((int)PRIME32_1);
-
-        size_t i;
-        for (i=0; i < STRIPE_LEN/sizeof(__m128i); i++) {
-            __m128i data = xacc[i];
-            __m128i const shifted = _mm_srli_epi64(data, 47);
-            data = _mm_xor_si128(data, shifted);
-
-            {   __m128i const k   = _mm_loadu_si128 (xkey+i);
-                __m128i const dk  = _mm_xor_si128   (data,k);
-
-                __m128i const dk1 = _mm_mul_epu32 (dk, prime32);
-
-                __m128i const d2  = _mm_shuffle_epi32 (dk, 0x31);
-                __m128i const dk2 = _mm_mul_epu32 (d2, prime32);
-                __m128i const dk2h= _mm_slli_epi64(dk2, 32);
-
-                xacc[i] = _mm_add_epi64(dk1, dk2h);
-        }   }
-    }
-
-#elif (XXH_VECTOR == XXH_NEON)
-
-    XXH_ASSERT((((size_t)acc) & 15) == 0);
-
-    {   uint64x2_t* const xacc =     (uint64x2_t*) acc;
-        uint32_t const* const xkey = (uint32_t const*) key;
-        uint32x2_t const prime     = vdup_n_u32 (PRIME32_1);
-
-        size_t i;
-        for (i=0; i < STRIPE_LEN/sizeof(uint64x2_t); i++) {
-            /* data_vec = xacc[i] ^ (xacc[i] >> 47); */
-            uint64x2_t const   acc_vec  = xacc[i];
-            uint64x2_t const   shifted  = vshrq_n_u64 (acc_vec, 47);
-            uint64x2_t const   data_vec = veorq_u64   (acc_vec, shifted);
-
-            /* key_vec  = xkey[i]; */
-            uint32x4_t const   key_vec  = vld1q_u32   (xkey + (i * 4));
-            /* data_key = data_vec ^ key_vec; */
-            uint32x4_t const   data_key = veorq_u32   (vreinterpretq_u32_u64(data_vec), key_vec);
-            /* shuffled = { data_key[0, 2], data_key[1, 3] }; */
-            uint32x2x2_t const shuffled = vzip_u32    (vget_low_u32(data_key), vget_high_u32(data_key));
-
-            /* data_key *= PRIME32_1 */
-
-            /* prod_hi = (data_key >> 32) * PRIME32_1; */
-            uint64x2_t const   prod_hi = vmull_u32    (shuffled.val[1], prime);
-            /* xacc[i] = prod_hi << 32; */
-            xacc[i] = vshlq_n_u64(prod_hi, 32);
-            /* xacc[i] += (prod_hi & 0xFFFFFFFF) * PRIME32_1; */
-            xacc[i] = vmlal_u32(xacc[i], shuffled.val[0], prime);
-    }   }
-
-#elif (XXH_VECTOR == XXH_VSX)
-
-          U64x2* const xacc =       (U64x2*) acc;
-    const U64x2* const xkey = (const U64x2*) key;
-    /* constants */
-    U64x2 const v32  = { 32, 32 };
-    U64x2 const v47 = { 47, 47 };
-    U32x4 const prime = { PRIME32_1, PRIME32_1, PRIME32_1, PRIME32_1 };
-    size_t i;
-
-    for (i = 0; i < STRIPE_LEN / sizeof(U64x2); i++) {
-        U64x2 const acc_vec  = xacc[i];
-        U64x2 const data_vec = acc_vec ^ (acc_vec >> v47);
-        /* key_vec = xkey[i]; */
-#ifdef __BIG_ENDIAN__
-        /* swap 32-bit words */
-        U64x2 const key_vec  = vec_rl(vec_vsx_ld(0, xkey + i), v32);
-#else
-        U64x2 const key_vec  = vec_vsx_ld(0, xkey + i);
-#endif
-        U64x2 const data_key = data_vec ^ key_vec;
-
-        /* data_key *= PRIME32_1 */
-
-        /* prod_lo = ((U64x2)data_key & 0xFFFFFFFF) * ((U64x2)prime & 0xFFFFFFFF);  */
-        U64x2 const prod_lo  = XXH_vsxMultOdd((U32x4)data_key, prime);
-        /* prod_hi = ((U64x2)data_key >> 32) * ((U64x2)prime >> 32);  */
-        U64x2 const prod_hi  = XXH_vsxMultEven((U32x4)data_key, prime);
-        xacc[i] = prod_lo + (prod_hi << v32);
-    }
-
-#else   /* scalar variant of Scrambler - universal */
-
-    XXH_ALIGN(XXH_ACC_ALIGN) U64* const xacc = (U64*) acc;   /* presumed aligned on 32-bytes boundaries, little hint for the auto-vectorizer */
-    const char* const xkey = (const char*) key;   /* no alignment restriction */
-    int i;
-    XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0);
-
-    for (i=0; i < (int)ACC_NB; i++) {
-        U64 const key64 = XXH_readLE64(xkey + 8*i);
-        U64 acc64 = xacc[i];
-        acc64 ^= acc64 >> 47;
-        acc64 ^= key64;
-        acc64 *= PRIME32_1;
-        xacc[i] = acc64;
-    }
-
-#endif
-}
-
-/* assumption : nbStripes will not overflow secret size */
-XXH_FORCE_INLINE void
-XXH3_accumulate(       U64* XXH_RESTRICT acc,
-                const void* XXH_RESTRICT data,
-                const void* XXH_RESTRICT secret,
-                      size_t nbStripes,
-                      XXH3_accWidth_e accWidth)
-{
-    size_t n;
-    /* Clang doesn't unroll this loop without the pragma. Unrolling can be up to 1.4x faster.
-     * The unroll statement seems detrimental for WASM (@aras-p) and ARM though.
-     */
-#if defined(__clang__) && !defined(__OPTIMIZE_SIZE__) && !defined(__ARM_ARCH) && !defined(__EMSCRIPTEN__)
-#  pragma clang loop unroll(enable)
-#endif
-
-    for (n = 0; n < nbStripes; n++ ) {
-        XXH3_accumulate_512(acc,
-               (const char*)data   + n*STRIPE_LEN,
-               (const char*)secret + n*XXH_SECRET_CONSUME_RATE,
-                            accWidth);
-    }
-}
-
-/* note : clang auto-vectorizes well in SS2 mode _if_ this function is `static`,
- *        and doesn't auto-vectorize it at all if it is `FORCE_INLINE`.
- *        However, it auto-vectorizes better AVX2 if it is `FORCE_INLINE`
- *        Pretty much every other modes and compilers prefer `FORCE_INLINE`.
- */
-#if defined(__clang__) && (XXH_VECTOR==0) && !defined(__AVX2__)
-static void
-#else
-XXH_FORCE_INLINE void
-#endif
-XXH3_hashLong_internal_loop( U64* XXH_RESTRICT acc,
-                      const void* XXH_RESTRICT data, size_t len,
-                      const void* XXH_RESTRICT secret, size_t secretSize,
-                            XXH3_accWidth_e accWidth)
-{
-    size_t const nb_rounds = (secretSize - STRIPE_LEN) / XXH_SECRET_CONSUME_RATE;
-    size_t const block_len = STRIPE_LEN * nb_rounds;
-    size_t const nb_blocks = len / block_len;
-
-    size_t n;
-
-    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
-
-    for (n = 0; n < nb_blocks; n++) {
-        XXH3_accumulate(acc, (const char*)data + n*block_len, secret, nb_rounds, accWidth);
-        XXH3_scrambleAcc(acc, (const char*)secret + secretSize - STRIPE_LEN);
-    }
-
-    /* last partial block */
-    XXH_ASSERT(len > STRIPE_LEN);
-    {   size_t const nbStripes = (len - (block_len * nb_blocks)) / STRIPE_LEN;
-        XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE));
-        XXH3_accumulate(acc, (const char*)data + nb_blocks*block_len, secret, nbStripes, accWidth);
-
-        /* last stripe */
-        if (len & (STRIPE_LEN - 1)) {
-            const void* const p = (const char*)data + len - STRIPE_LEN;
-#define XXH_SECRET_LASTACC_START 7  /* do not align on 8, so that secret is different from scrambler */
-            XXH3_accumulate_512(acc, p, (const char*)secret + secretSize - STRIPE_LEN - XXH_SECRET_LASTACC_START, accWidth);
-    }   }
-}
-
-XXH_FORCE_INLINE U64
-XXH3_mix2Accs(const U64* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
-{
-    const U64* const key64 = (const U64*)secret;
-    return XXH3_mul128_fold64(
-               acc[0] ^ XXH_readLE64(key64),
-               acc[1] ^ XXH_readLE64(key64+1) );
-}
-
-static XXH64_hash_t
-XXH3_mergeAccs(const U64* XXH_RESTRICT acc, const void* XXH_RESTRICT secret, U64 start)
-{
-    U64 result64 = start;
-
-    result64 += XXH3_mix2Accs(acc+0, (const char*)secret +  0);
-    result64 += XXH3_mix2Accs(acc+2, (const char*)secret + 16);
-    result64 += XXH3_mix2Accs(acc+4, (const char*)secret + 32);
-    result64 += XXH3_mix2Accs(acc+6, (const char*)secret + 48);
-
-    return XXH3_avalanche(result64);
-}
-
-#define XXH3_INIT_ACC { PRIME32_3, PRIME64_1, PRIME64_2, PRIME64_3, \
-                        PRIME64_4, PRIME32_2, PRIME64_5, PRIME32_1 };
-
-XXH_FORCE_INLINE XXH64_hash_t
-XXH3_hashLong_internal(const void* XXH_RESTRICT data, size_t len,
-                       const void* XXH_RESTRICT secret, size_t secretSize)
-{
-    XXH_ALIGN(XXH_ACC_ALIGN) U64 acc[ACC_NB] = XXH3_INIT_ACC;
-
-    XXH3_hashLong_internal_loop(acc, data, len, secret, secretSize, XXH3_acc_64bits);
-
-    /* converge into final hash */
-    XXH_STATIC_ASSERT(sizeof(acc) == 64);
-#define XXH_SECRET_MERGEACCS_START 11  /* do not align on 8, so that secret is different from accumulator */
-    XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
-    return XXH3_mergeAccs(acc, (const char*)secret + XXH_SECRET_MERGEACCS_START, (U64)len * PRIME64_1);
-}
-
-
-XXH_NO_INLINE XXH64_hash_t    /* It's important for performance that XXH3_hashLong is not inlined. Not sure why (uop cache maybe ?), but difference is large and easily measurable */
-XXH3_hashLong_64b_defaultSecret(const void* XXH_RESTRICT data, size_t len)
-{
-    return XXH3_hashLong_internal(data, len, kSecret, sizeof(kSecret));
-}
-
-XXH_NO_INLINE XXH64_hash_t    /* It's important for performance that XXH3_hashLong is not inlined. Not sure why (uop cache maybe ?), but difference is large and easily measurable */
-XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT data, size_t len,
-                             const void* XXH_RESTRICT secret, size_t secretSize)
-{
-    return XXH3_hashLong_internal(data, len, secret, secretSize);
-}
-
-
-XXH_FORCE_INLINE void XXH_writeLE64(void* dst, U64 v64)
-{
-    if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64);
-    memcpy(dst, &v64, sizeof(v64));
-}
-
-/* XXH3_initKeySeed() :
- * destination `customSecret` is presumed allocated and same size as `kSecret`.
- */
-XXH_FORCE_INLINE void XXH3_initKeySeed(void* customSecret, U64 seed64)
-{
-          char* const dst = (char*)customSecret;
-    const char* const src = (const char*)kSecret;
-    int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16;
-    int i;
-
-    XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
-
-    for (i=0; i < nbRounds; i++) {
-        XXH_writeLE64(dst + 16*i,     XXH_readLE64(src + 16*i)     + seed64);
-        XXH_writeLE64(dst + 16*i + 8, XXH_readLE64(src + 16*i + 8) - seed64);
-    }
-}
-
-
-/* XXH3_hashLong_64b_withSeed() :
- * Generate a custom key,
- * based on alteration of default kSecret with the seed,
- * and then use this key for long mode hashing.
- * This operation is decently fast but nonetheless costs a little bit of time.
- * Try to avoid it whenever possible (typically when seed==0).
- */
-XXH_NO_INLINE XXH64_hash_t    /* It's important for performance that XXH3_hashLong is not inlined. Not sure why (uop cache maybe ?), but difference is large and easily measurable */
-XXH3_hashLong_64b_withSeed(const void* data, size_t len, XXH64_hash_t seed)
-{
-    XXH_ALIGN(8) char secret[XXH_SECRET_DEFAULT_SIZE];
-    if (seed==0) return XXH3_hashLong_64b_defaultSecret(data, len);
-    XXH3_initKeySeed(secret, seed);
-    return XXH3_hashLong_internal(data, len, secret, sizeof(secret));
-}
-
-
-XXH_FORCE_INLINE U64 XXH3_mix16B(const void* XXH_RESTRICT data,
-                                 const void* XXH_RESTRICT key, U64 seed64)
-{
-    const U64* const key64 = (const U64*)key;
-    U64 const ll1 = XXH_readLE64(data);
-    U64 const ll2 = XXH_readLE64((const BYTE*)data+8);
-    return XXH3_mul128_fold64(
-               ll1 ^ (XXH_readLE64(key64)   + seed64),
-               ll2 ^ (XXH_readLE64(key64+1) - seed64) );
-}
-
-
-XXH_FORCE_INLINE XXH64_hash_t
-XXH3_len_17to128_64b(const void* XXH_RESTRICT data, size_t len,
-                     const void* XXH_RESTRICT secret, size_t secretSize,
-                     XXH64_hash_t seed)
-{
-    const BYTE* const p = (const BYTE*)data;
-    const char* const key = (const char*)secret;
-
-    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
-    XXH_ASSERT(16 < len && len <= 128);
-
-    {   U64 acc = len * PRIME64_1;
-        if (len > 32) {
-            if (len > 64) {
-                if (len > 96) {
-                    acc += XXH3_mix16B(p+48, key+96, seed);
-                    acc += XXH3_mix16B(p+len-64, key+112, seed);
-                }
-                acc += XXH3_mix16B(p+32, key+64, seed);
-                acc += XXH3_mix16B(p+len-48, key+80, seed);
-            }
-            acc += XXH3_mix16B(p+16, key+32, seed);
-            acc += XXH3_mix16B(p+len-32, key+48, seed);
-        }
-        acc += XXH3_mix16B(p+0, key+0, seed);
-        acc += XXH3_mix16B(p+len-16, key+16, seed);
-
-        return XXH3_avalanche(acc);
-    }
-}
-
-#define XXH3_MIDSIZE_MAX 240
-
-XXH_NO_INLINE XXH64_hash_t
-XXH3_len_129to240_64b(const void* XXH_RESTRICT data, size_t len,
-                      const void* XXH_RESTRICT secret, size_t secretSize,
-                      XXH64_hash_t seed)
-{
-    const BYTE* const p = (const BYTE*)data;
-    const char* const key = (const char*)secret;
-
-    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
-    XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
-
-    #define XXH3_MIDSIZE_STARTOFFSET 3
-    #define XXH3_MIDSIZE_LASTOFFSET  17
-
-    {   U64 acc = len * PRIME64_1;
-        int const nbRounds = (int)len / 16;
-        int i;
-        for (i=0; i<8; i++) {
-            acc += XXH3_mix16B(p+(16*i), key+(16*i), seed);
-        }
-        acc = XXH3_avalanche(acc);
-        XXH_ASSERT(nbRounds >= 8);
-        for (i=8 ; i < nbRounds; i++) {
-            acc += XXH3_mix16B(p+(16*i), key+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed);
-        }
-        /* last bytes */
-        acc += XXH3_mix16B(p + len - 16, key + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed);
-        return XXH3_avalanche(acc);
-    }
-}
-
-/* ===   Public entry point   === */
-
-XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* data, size_t len)
-{
-    if (len <= 16) return XXH3_len_0to16_64b(data, len, kSecret, 0);
-    if (len <= 128) return XXH3_len_17to128_64b(data, len, kSecret, sizeof(kSecret), 0);
-    if (len <= XXH3_MIDSIZE_MAX) return XXH3_len_129to240_64b(data, len, kSecret, sizeof(kSecret), 0);
-    return XXH3_hashLong_64b_defaultSecret(data, len);
-}
-
-XXH_PUBLIC_API XXH64_hash_t
-XXH3_64bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize)
-{
-    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
-    /* if an action must be taken should `secret` conditions not be respected,
-     * it should be done here.
-     * For now, it's a contract pre-condition.
-     * Adding a check and a branch here would cost performance at every hash */
-     if (len <= 16) return XXH3_len_0to16_64b(data, len, secret, 0);
-     if (len <= 128) return XXH3_len_17to128_64b(data, len, secret, secretSize, 0);
-     if (len <= XXH3_MIDSIZE_MAX) return XXH3_len_129to240_64b(data, len, secret, secretSize, 0);
-     return XXH3_hashLong_64b_withSecret(data, len, secret, secretSize);
-}
-
-XXH_PUBLIC_API XXH64_hash_t
-XXH3_64bits_withSeed(const void* data, size_t len, XXH64_hash_t seed)
-{
-    if (len <= 16) return XXH3_len_0to16_64b(data, len, kSecret, seed);
-    if (len <= 128) return XXH3_len_17to128_64b(data, len, kSecret, sizeof(kSecret), seed);
-    if (len <= XXH3_MIDSIZE_MAX) return XXH3_len_129to240_64b(data, len, kSecret, sizeof(kSecret), seed);
-    return XXH3_hashLong_64b_withSeed(data, len, seed);
-}
-
-/* ===   XXH3 streaming   === */
-
-XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void)
-{
-    return (XXH3_state_t*)XXH_malloc(sizeof(XXH3_state_t));
-}
-
-XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr)
-{
-    XXH_free(statePtr);
-    return XXH_OK;
-}
-
-XXH_PUBLIC_API void
-XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state)
-{
-    memcpy(dst_state, src_state, sizeof(*dst_state));
-}
-
-static void
-XXH3_64bits_reset_internal(XXH3_state_t* statePtr,
-                           XXH64_hash_t seed,
-                           const void* secret, size_t secretSize)
-{
-    XXH_ASSERT(statePtr != NULL);
-    memset(statePtr, 0, sizeof(*statePtr));
-    statePtr->acc[0] = PRIME32_3;
-    statePtr->acc[1] = PRIME64_1;
-    statePtr->acc[2] = PRIME64_2;
-    statePtr->acc[3] = PRIME64_3;
-    statePtr->acc[4] = PRIME64_4;
-    statePtr->acc[5] = PRIME32_2;
-    statePtr->acc[6] = PRIME64_5;
-    statePtr->acc[7] = PRIME32_1;
-    statePtr->seed = seed;
-    XXH_ASSERT(secret != NULL);
-    statePtr->secret = secret;
-    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
-    statePtr->secretLimit = (XXH32_hash_t)(secretSize - STRIPE_LEN);
-    statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE;
-}
-
-XXH_PUBLIC_API XXH_errorcode
-XXH3_64bits_reset(XXH3_state_t* statePtr)
-{
-    if (statePtr == NULL) return XXH_ERROR;
-    XXH3_64bits_reset_internal(statePtr, 0, kSecret, XXH_SECRET_DEFAULT_SIZE);
-    return XXH_OK;
-}
-
-XXH_PUBLIC_API XXH_errorcode
-XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize)
-{
-    if (statePtr == NULL) return XXH_ERROR;
-    XXH3_64bits_reset_internal(statePtr, 0, secret, secretSize);
-    if (secret == NULL) return XXH_ERROR;
-    if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
-    return XXH_OK;
-}
-
-XXH_PUBLIC_API XXH_errorcode
-XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed)
-{
-    if (statePtr == NULL) return XXH_ERROR;
-    XXH3_64bits_reset_internal(statePtr, seed, kSecret, XXH_SECRET_DEFAULT_SIZE);
-    XXH3_initKeySeed(statePtr->customSecret, seed);
-    statePtr->secret = statePtr->customSecret;
-    return XXH_OK;
-}
-
-XXH_FORCE_INLINE void
-XXH3_consumeStripes( U64* acc,
-                            XXH32_hash_t* nbStripesSoFarPtr, XXH32_hash_t nbStripesPerBlock,
-                            const void* data, size_t totalStripes,
-                            const void* secret, size_t secretLimit,
-                            XXH3_accWidth_e accWidth)
-{
-    XXH_ASSERT(*nbStripesSoFarPtr < nbStripesPerBlock);
-    if (nbStripesPerBlock - *nbStripesSoFarPtr <= totalStripes) {
-        /* need a scrambling operation */
-        size_t const nbStripes = nbStripesPerBlock - *nbStripesSoFarPtr;
-        XXH3_accumulate(acc, data, (const char*)secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripes, accWidth);
-        XXH3_scrambleAcc(acc, (const char*)secret + secretLimit);
-        XXH3_accumulate(acc, (const char*)data + nbStripes * STRIPE_LEN, secret, totalStripes - nbStripes, accWidth);
-        *nbStripesSoFarPtr = (XXH32_hash_t)(totalStripes - nbStripes);
-    } else {
-        XXH3_accumulate(acc, data, (const char*)secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, totalStripes, accWidth);
-        *nbStripesSoFarPtr += (XXH32_hash_t)totalStripes;
-    }
-}
-
-XXH_FORCE_INLINE XXH_errorcode
-XXH3_update(XXH3_state_t* state, const void* input, size_t len, XXH3_accWidth_e accWidth)
-{
-    if (input==NULL)
-#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
-        return XXH_OK;
-#else
-        return XXH_ERROR;
-#endif
-
-    {   const BYTE* p = (const BYTE*)input;
-        const BYTE* const bEnd = p + len;
-
-        state->totalLen += len;
-
-        if (state->bufferedSize + len <= XXH3_INTERNALBUFFER_SIZE) {  /* fill in tmp buffer */
-            XXH_memcpy(state->buffer + state->bufferedSize, input, len);
-            state->bufferedSize += (XXH32_hash_t)len;
-            return XXH_OK;
-        }
-        /* input now > XXH3_INTERNALBUFFER_SIZE */
-
-        #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / STRIPE_LEN)
-        XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % STRIPE_LEN == 0);   /* clean multiple */
-
-        if (state->bufferedSize) {   /* some data within internal buffer: fill then consume it */
-            size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize;
-            XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize);
-            p += loadSize;
-            XXH3_consumeStripes(state->acc,
-                               &state->nbStripesSoFar, state->nbStripesPerBlock,
-                                state->buffer, XXH3_INTERNALBUFFER_STRIPES,
-                                state->secret, state->secretLimit,
-                                accWidth);
-            state->bufferedSize = 0;
-        }
-
-        /* consume input by full buffer quantities */
-        if (p+XXH3_INTERNALBUFFER_SIZE <= bEnd) {
-            const BYTE* const limit = bEnd - XXH3_INTERNALBUFFER_SIZE;
-            do {
-                XXH3_consumeStripes(state->acc,
-                                   &state->nbStripesSoFar, state->nbStripesPerBlock,
-                                    p, XXH3_INTERNALBUFFER_STRIPES,
-                                    state->secret, state->secretLimit,
-                                    accWidth);
-                p += XXH3_INTERNALBUFFER_SIZE;
-            } while (p<=limit);
-        }
-
-        if (p < bEnd) { /* some remaining input data : buffer it */
-            XXH_memcpy(state->buffer, p, (size_t)(bEnd-p));
-            state->bufferedSize = (XXH32_hash_t)(bEnd-p);
-        }
-    }
-
-    return XXH_OK;
-}
-
-XXH_PUBLIC_API XXH_errorcode
-XXH3_64bits_update(XXH3_state_t* state, const void* input, size_t len)
-{
-    return XXH3_update(state, input, len, XXH3_acc_64bits);
-}
-
-
-XXH_FORCE_INLINE void
-XXH3_digest_long (XXH64_hash_t* acc, const XXH3_state_t* state, XXH3_accWidth_e accWidth)
-{
-    memcpy(acc, state->acc, sizeof(state->acc));  /* digest locally, state remains unaltered, and can continue ingesting more data afterwards */
-    if (state->bufferedSize >= STRIPE_LEN) {
-        size_t const totalNbStripes = state->bufferedSize / STRIPE_LEN;
-        XXH32_hash_t nbStripesSoFar = state->nbStripesSoFar;
-        XXH3_consumeStripes(acc,
-                           &nbStripesSoFar, state->nbStripesPerBlock,
-                            state->buffer, totalNbStripes,
-                            state->secret, state->secretLimit,
-                            accWidth);
-        if (state->bufferedSize % STRIPE_LEN) {  /* one last partial stripe */
-            XXH3_accumulate_512(acc,
-                                state->buffer + state->bufferedSize - STRIPE_LEN,
-                   (const char*)state->secret + state->secretLimit - XXH_SECRET_LASTACC_START,
-                                accWidth);
-        }
-    } else {  /* bufferedSize < STRIPE_LEN */
-        if (state->bufferedSize) { /* one last stripe */
-            char lastStripe[STRIPE_LEN];
-            size_t const catchupSize = STRIPE_LEN - state->bufferedSize;
-            memcpy(lastStripe, (const char*)state->buffer + sizeof(state->buffer) - catchupSize, catchupSize);
-            memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize);
-            XXH3_accumulate_512(acc,
-                                lastStripe,
-                   (const char*)state->secret + state->secretLimit - XXH_SECRET_LASTACC_START,
-                                accWidth);
-    }   }
-}
-
-XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* state)
-{
-    if (state->totalLen > XXH3_MIDSIZE_MAX) {
-        XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[ACC_NB];
-        XXH3_digest_long(acc, state, XXH3_acc_64bits);
-        return XXH3_mergeAccs(acc, (const char*)state->secret + XXH_SECRET_MERGEACCS_START, (U64)state->totalLen * PRIME64_1);
-    }
-    /* len <= XXH3_MIDSIZE_MAX : short code */
-    if (state->seed)
-        return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
-    return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), state->secret, state->secretLimit + STRIPE_LEN);
-}
-
-/* ==========================================
- * XXH3 128 bits (=> XXH128)
- * ========================================== */
-
-XXH_FORCE_INLINE XXH128_hash_t
-XXH3_len_1to3_128b(const void* data, size_t len, const void* keyPtr, XXH64_hash_t seed)
-{
-    XXH_ASSERT(data != NULL);
-    XXH_ASSERT(1 <= len && len <= 3);
-    XXH_ASSERT(keyPtr != NULL);
-    {   const U32* const key32 = (const U32*) keyPtr;
-        BYTE const c1 = ((const BYTE*)data)[0];
-        BYTE const c2 = ((const BYTE*)data)[len >> 1];
-        BYTE const c3 = ((const BYTE*)data)[len - 1];
-        U32  const combinedl = ((U32)c1) + (((U32)c2) << 8) + (((U32)c3) << 16) + (((U32)len) << 24);
-        U32  const combinedh = XXH_swap32(combinedl);
-        U64  const keyedl = (U64)combinedl ^ (XXH_readLE32(key32)   + seed);
-        U64  const keyedh = (U64)combinedh ^ (XXH_readLE32(key32+1) - seed);
-        U64  const mixedl = keyedl * PRIME64_1;
-        U64  const mixedh = keyedh * PRIME64_2;
-        XXH128_hash_t const h128 = { XXH3_avalanche(mixedl) /*low64*/, XXH3_avalanche(mixedh) /*high64*/ };
-        return h128;
-    }
-}
-
-
-XXH_FORCE_INLINE XXH128_hash_t
-XXH3_len_4to8_128b(const void* data, size_t len, const void* keyPtr, XXH64_hash_t seed)
-{
-    XXH_ASSERT(data != NULL);
-    XXH_ASSERT(keyPtr != NULL);
-    XXH_ASSERT(4 <= len && len <= 8);
-    {   U32 const in1 = XXH_readLE32(data);
-        U32 const in2 = XXH_readLE32((const BYTE*)data + len - 4);
-        U64 const in64l = in1 + ((U64)in2 << 32);
-        U64 const in64h = XXH_swap64(in64l);
-        U64 const keyedl = in64l ^ (XXH_readLE64(keyPtr) + seed);
-        U64 const keyedh = in64h ^ (XXH_readLE64((const char*)keyPtr + 8) - seed);
-        U64 const mix64l1 = len + ((keyedl ^ (keyedl >> 51)) * PRIME32_1);
-        U64 const mix64l2 = (mix64l1 ^ (mix64l1 >> 47)) * PRIME64_2;
-        U64 const mix64h1 = ((keyedh ^ (keyedh >> 47)) * PRIME64_1) - len;
-        U64 const mix64h2 = (mix64h1 ^ (mix64h1 >> 43)) * PRIME64_4;
-        {   XXH128_hash_t const h128 = { XXH3_avalanche(mix64l2) /*low64*/, XXH3_avalanche(mix64h2) /*high64*/ };
-            return h128;
-    }   }
-}
-
-XXH_FORCE_INLINE XXH128_hash_t
-XXH3_len_9to16_128b(const void* data, size_t len, const void* keyPtr, XXH64_hash_t seed)
-{
-    XXH_ASSERT(data != NULL);
-    XXH_ASSERT(keyPtr != NULL);
-    XXH_ASSERT(9 <= len && len <= 16);
-    {   const U64* const key64 = (const U64*) keyPtr;
-        U64 const ll1 = XXH_readLE64(data) ^ (XXH_readLE64(key64) + seed);
-        U64 const ll2 = XXH_readLE64((const BYTE*)data + len - 8) ^ (XXH_readLE64(key64+1) - seed);
-        U64 const inlow = ll1 ^ ll2;
-        XXH128_hash_t m128 = XXH3_mul128(inlow, PRIME64_1);
-        m128.high64 += ll2 * PRIME64_1;
-        m128.low64  ^= (m128.high64 >> 32);
-        {   XXH128_hash_t h128 = XXH3_mul128(m128.low64, PRIME64_2);
-            h128.high64 += m128.high64 * PRIME64_2;
-            h128.low64   = XXH3_avalanche(h128.low64);
-            h128.high64  = XXH3_avalanche(h128.high64);
-            return h128;
-    }   }
-}
-
-/* Assumption : `secret` size is >= 16
- * Note : it should be >= XXH3_SECRET_SIZE_MIN anyway */
-XXH_FORCE_INLINE XXH128_hash_t
-XXH3_len_0to16_128b(const void* data, size_t len, const void* secret, XXH64_hash_t seed)
-{
-    XXH_ASSERT(len <= 16);
-    {   if (len > 8) return XXH3_len_9to16_128b(data, len, secret, seed);
-        if (len >= 4) return XXH3_len_4to8_128b(data, len, secret, seed);
-        if (len) return XXH3_len_1to3_128b(data, len, secret, seed);
-        {   XXH128_hash_t const h128 = { 0, 0 };
-            return h128;
-    }   }
-}
-
-XXH_FORCE_INLINE XXH128_hash_t
-XXH3_hashLong_128b_internal(const void* XXH_RESTRICT data, size_t len,
-                            const void* XXH_RESTRICT secret, size_t secretSize)
-{
-    XXH_ALIGN(XXH_ACC_ALIGN) U64 acc[ACC_NB] = XXH3_INIT_ACC;
-
-    XXH3_hashLong_internal_loop(acc, data, len, secret, secretSize, XXH3_acc_128bits);
-
-    /* converge into final hash */
-    XXH_STATIC_ASSERT(sizeof(acc) == 64);
-    XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
-    {   U64 const low64 = XXH3_mergeAccs(acc, (const char*)secret + XXH_SECRET_MERGEACCS_START, (U64)len * PRIME64_1);
-        U64 const high64 = XXH3_mergeAccs(acc, (const char*)secret + secretSize - sizeof(acc) - XXH_SECRET_MERGEACCS_START, ~((U64)len * PRIME64_2));
-        XXH128_hash_t const h128 = { low64, high64 };
-        return h128;
-    }
-}
-
-XXH_NO_INLINE XXH128_hash_t    /* It's important for performance that XXH3_hashLong is not inlined. Not sure why (uop cache maybe ?), but difference is large and easily measurable */
-XXH3_hashLong_128b_defaultSecret(const void* data, size_t len)
-{
-    return XXH3_hashLong_128b_internal(data, len, kSecret, sizeof(kSecret));
-}
-
-XXH_NO_INLINE XXH128_hash_t    /* It's important for performance that XXH3_hashLong is not inlined. Not sure why (uop cache maybe ?), but difference is large and easily measurable */
-XXH3_hashLong_128b_withSecret(const void* data, size_t len,
-                              const void* secret, size_t secretSize)
-{
-    return XXH3_hashLong_128b_internal(data, len, secret, secretSize);
-}
-
-XXH_NO_INLINE XXH128_hash_t    /* It's important for performance that XXH3_hashLong is not inlined. Not sure why (uop cache maybe ?), but difference is large and easily measurable */
-XXH3_hashLong_128b_withSeed(const void* data, size_t len, XXH64_hash_t seed)
-{
-    XXH_ALIGN(8) char secret[XXH_SECRET_DEFAULT_SIZE];
-    if (seed == 0) return XXH3_hashLong_128b_defaultSecret(data, len);
-    XXH3_initKeySeed(secret, seed);
-    return XXH3_hashLong_128b_internal(data, len, secret, sizeof(secret));
-}
-
-XXH_NO_INLINE XXH128_hash_t
-XXH3_len_129to240_128b(const void* XXH_RESTRICT data, size_t len,
-                      const void* XXH_RESTRICT secret, size_t secretSize,
-                      XXH64_hash_t seed)
-{
-    const BYTE* const p = (const BYTE*)data;
-    const char* const key = (const char*)secret;
-
-    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
-    XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
-
-    {   U64 acc1 = len * PRIME64_1;
-        U64 acc2 = 0;
-        int const nbRounds = (int)len / 32;
-        int i;
-        for (i=0; i<4; i++) {
-            acc1 += XXH3_mix16B(p+(32*i),    key+(32*i),     seed);
-            acc2 += XXH3_mix16B(p+(32*i)+16, key+(32*i)+16, -seed);
-        }
-        acc1 = XXH3_avalanche(acc1);
-        acc2 = XXH3_avalanche(acc2);
-        XXH_ASSERT(nbRounds >= 4);
-        for (i=4 ; i < nbRounds; i++) {
-            acc1 += XXH3_mix16B(p+(32*i)   , key+(32*(i-4))    + XXH3_MIDSIZE_STARTOFFSET,  seed);
-            acc2 += XXH3_mix16B(p+(32*i)+16, key+(32*(i-4))+16 + XXH3_MIDSIZE_STARTOFFSET, -seed);
-        }
-        /* last bytes */
-        acc1 += XXH3_mix16B(p + len - 16, key + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET     ,  seed);
-        acc2 += XXH3_mix16B(p + len - 32, key + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, -seed);
-
-        {   U64 const low64 = acc1 + acc2;
-            U64 const high64 = (acc1 * PRIME64_1) + (acc2 * PRIME64_4) + ((len - seed) * PRIME64_2);
-            XXH128_hash_t const h128 = { XXH3_avalanche(low64), (XXH64_hash_t)0 - XXH3_avalanche(high64) };
-            return h128;
-        }
-    }
-}
-
-XXH_FORCE_INLINE XXH128_hash_t
-XXH3_len_17to128_128b(const void* XXH_RESTRICT data, size_t len,
-                     const void* XXH_RESTRICT secret, size_t secretSize,
-                     XXH64_hash_t seed)
-{
-    const BYTE* const p = (const BYTE*)data;
-    const char* const key = (const char*)secret;
-
-    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
-    XXH_ASSERT(16 < len && len <= 128);
-
-    {   U64 acc1 = len * PRIME64_1;
-        U64 acc2 = 0;
-        if (len > 32) {
-            if (len > 64) {
-                if (len > 96) {
-                    acc1 += XXH3_mix16B(p+48, key+96, seed);
-                    acc2 += XXH3_mix16B(p+len-64, key+112, seed);
-                }
-                acc1 += XXH3_mix16B(p+32, key+64, seed);
-                acc2 += XXH3_mix16B(p+len-48, key+80, seed);
-            }
-            acc1 += XXH3_mix16B(p+16, key+32, seed);
-            acc2 += XXH3_mix16B(p+len-32, key+48, seed);
-        }
-        acc1 += XXH3_mix16B(p+0, key+0, seed);
-        acc2 += XXH3_mix16B(p+len-16, key+16, seed);
-
-        {   U64 const low64 = acc1 + acc2;
-            U64 const high64 = (acc1 * PRIME64_1) + (acc2 * PRIME64_4) + ((len - seed) * PRIME64_2);
-            XXH128_hash_t const h128 = { XXH3_avalanche(low64), (XXH64_hash_t)0 - XXH3_avalanche(high64) };
-            return h128;
-        }
-    }
-}
-
-XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* data, size_t len)
-{
-    if (len <= 16) return XXH3_len_0to16_128b(data, len, kSecret, 0);
-    if (len <= 128) return XXH3_len_17to128_128b(data, len, kSecret, sizeof(kSecret), 0);
-    if (len <= XXH3_MIDSIZE_MAX) return XXH3_len_129to240_128b(data, len, kSecret, sizeof(kSecret), 0);
-    return XXH3_hashLong_128b_defaultSecret(data, len);
-}
-
-XXH_PUBLIC_API XXH128_hash_t
-XXH3_128bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize)
-{
-    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
-    /* if an action must be taken should `secret` conditions not be respected,
-     * it should be done here.
-     * For now, it's a contract pre-condition.
-     * Adding a check and a branch here would cost performance at every hash */
-     if (len <= 16) return XXH3_len_0to16_128b(data, len, secret, 0);
-     if (len <= 128) return XXH3_len_17to128_128b(data, len, secret, secretSize, 0);
-     if (len <= XXH3_MIDSIZE_MAX) return XXH3_len_129to240_128b(data, len, secret, secretSize, 0);
-     return XXH3_hashLong_128b_withSecret(data, len, secret, secretSize);
-}
-
-XXH_PUBLIC_API XXH128_hash_t
-XXH3_128bits_withSeed(const void* data, size_t len, XXH64_hash_t seed)
-{
-    if (len <= 16) return XXH3_len_0to16_128b(data, len, kSecret, seed);
-    if (len <= 128) return XXH3_len_17to128_128b(data, len, kSecret, sizeof(kSecret), seed);
-    if (len <= XXH3_MIDSIZE_MAX) return XXH3_len_129to240_128b(data, len, kSecret, sizeof(kSecret), seed);
-    return XXH3_hashLong_128b_withSeed(data, len, seed);
-}
-
-XXH_PUBLIC_API XXH128_hash_t
-XXH128(const void* data, size_t len, XXH64_hash_t seed)
-{
-    return XXH3_128bits_withSeed(data, len, seed);
-}
-
-
-/* ===   XXH3 128-bit streaming   === */
-
-/* all the functions are actually the same as for 64-bit streaming variant,
-   just the reset one is different (different initial acc values for 0,5,6,7),
-   and near the end of the digest function */
-
-static void
-XXH3_128bits_reset_internal(XXH3_state_t* statePtr,
-                           XXH64_hash_t seed,
-                           const void* secret, size_t secretSize)
-{
-    XXH3_64bits_reset_internal(statePtr, seed, secret, secretSize);
-}
-
-XXH_PUBLIC_API XXH_errorcode
-XXH3_128bits_reset(XXH3_state_t* statePtr)
-{
-    if (statePtr == NULL) return XXH_ERROR;
-    XXH3_128bits_reset_internal(statePtr, 0, kSecret, XXH_SECRET_DEFAULT_SIZE);
-    return XXH_OK;
-}
-
-XXH_PUBLIC_API XXH_errorcode
-XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize)
-{
-    if (statePtr == NULL) return XXH_ERROR;
-    XXH3_128bits_reset_internal(statePtr, 0, secret, secretSize);
-    if (secret == NULL) return XXH_ERROR;
-    if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
-    return XXH_OK;
-}
-
-XXH_PUBLIC_API XXH_errorcode
-XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed)
-{
-    if (statePtr == NULL) return XXH_ERROR;
-    XXH3_128bits_reset_internal(statePtr, seed, kSecret, XXH_SECRET_DEFAULT_SIZE);
-    XXH3_initKeySeed(statePtr->customSecret, seed);
-    statePtr->secret = statePtr->customSecret;
-    return XXH_OK;
-}
-
-XXH_PUBLIC_API XXH_errorcode
-XXH3_128bits_update(XXH3_state_t* state, const void* input, size_t len)
-{
-    return XXH3_update(state, input, len, XXH3_acc_128bits);
-}
-
-XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* state)
-{
-    if (state->totalLen > XXH3_MIDSIZE_MAX) {
-        XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[ACC_NB];
-        XXH3_digest_long(acc, state, XXH3_acc_128bits);
-        XXH_ASSERT(state->secretLimit + STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
-        {   U64 const low64 = XXH3_mergeAccs(acc, (const char*)state->secret + XXH_SECRET_MERGEACCS_START, (U64)state->totalLen * PRIME64_1);
-            U64 const high64 = XXH3_mergeAccs(acc, (const char*)state->secret + state->secretLimit + STRIPE_LEN - sizeof(acc) - XXH_SECRET_MERGEACCS_START, ~((U64)state->totalLen * PRIME64_2));
-            XXH128_hash_t const h128 = { low64, high64 };
-            return h128;
-        }
-    }
-    /* len <= XXH3_MIDSIZE_MAX : short code */
-    if (state->seed)
-        return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
-    return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), state->secret, state->secretLimit + STRIPE_LEN);
-}
-
-/* 128-bit utility functions */
-
-#include <string.h>   /* memcmp */
-
-/* return : 1 is equal, 0 if different */
-XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2)
-{
-    /* note : XXH128_hash_t is compact, it has no padding byte */
-    return !(memcmp(&h1, &h2, sizeof(h1)));
-}
-
-/* This prototype is compatible with stdlib's qsort().
- * return : >0 if *h128_1  > *h128_2
- *          <0 if *h128_1  < *h128_2
- *          =0 if *h128_1 == *h128_2  */
-XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2)
-{
-    XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1;
-    XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2;
-    int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64);
-    /* note : bets that, in most cases, hash values are different */
-    if (hcmp) return hcmp;
-    return (h1.low64 > h2.low64) - (h2.low64 > h1.low64);
-}
-
-
-/*======   Canonical representation   ======*/
-XXH_PUBLIC_API void
-XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash)
-{
-    XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t));
-    if (XXH_CPU_LITTLE_ENDIAN) {
-        hash.high64 = XXH_swap64(hash.high64);
-        hash.low64  = XXH_swap64(hash.low64);
-    }
-    memcpy(dst, &hash.high64, sizeof(hash.high64));
-    memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64));
-}
-
-XXH_PUBLIC_API XXH128_hash_t
-XXH128_hashFromCanonical(const XXH128_canonical_t* src)
-{
-    XXH128_hash_t h;
-    h.high64 = XXH_readBE64(src);
-    h.low64  = XXH_readBE64(src->digest + 8);
-    return h;
-}
-
-
-
-#endif  /* XXH3_H */
diff --git a/cpp/src/arrow/vendored/xxhash/xxhash.c b/cpp/src/arrow/vendored/xxhash/xxhash.c
index c960f49fe00..0fae88c5d67 100644
--- a/cpp/src/arrow/vendored/xxhash/xxhash.c
+++ b/cpp/src/arrow/vendored/xxhash/xxhash.c
@@ -1,1118 +1,43 @@
 /*
-*  xxHash - Fast Hash algorithm
-*  Copyright (C) 2012-2016, Yann Collet
-*
-*  BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-*
-*  Redistribution and use in source and binary forms, with or without
-*  modification, are permitted provided that the following conditions are
-*  met:
-*
-*  * Redistributions of source code must retain the above copyright
-*  notice, this list of conditions and the following disclaimer.
-*  * Redistributions in binary form must reproduce the above
-*  copyright notice, this list of conditions and the following disclaimer
-*  in the documentation and/or other materials provided with the
-*  distribution.
-*
-*  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-*  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-*  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-*  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-*  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-*  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-*  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-*  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-*  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-*  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-*  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*
-*  You can contact the author at :
-*  - xxHash homepage: http://www.xxhash.com
-*  - xxHash source repository : https://github.com/Cyan4973/xxHash
-*/
-
-
-/* *************************************
-*  Tuning parameters
-***************************************/
-/*!XXH_FORCE_MEMORY_ACCESS :
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
- * The below switch allow to select different access method for improved performance.
- * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
- *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
- * Method 2 : direct access. This method doesn't depend on compiler but violate C standard.
- *            It can generate buggy code on targets which do not support unaligned memory accesses.
- *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://stackoverflow.com/a/32095106/646947 for details.
- * Prefer these methods in priority order (0 > 1 > 2)
- */
-#ifndef XXH_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) \
-                        || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) \
-                        || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
-#    define XXH_FORCE_MEMORY_ACCESS 2
-#  elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \
-  (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) \
-                    || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) \
-                    || defined(__ARM_ARCH_7S__) ))
-#    define XXH_FORCE_MEMORY_ACCESS 1
-#  endif
-#endif
-
-/*!XXH_ACCEPT_NULL_INPUT_POINTER :
- * If input pointer is NULL, xxHash default behavior is to dereference it, triggering a segfault.
- * When this macro is enabled, xxHash actively checks input for null pointer.
- * It it is, result for null input pointers is the same as a null-length input.
- */
-#ifndef XXH_ACCEPT_NULL_INPUT_POINTER   /* can be defined externally */
-#  define XXH_ACCEPT_NULL_INPUT_POINTER 0
-#endif
-
-/*!XXH_FORCE_ALIGN_CHECK :
- * This is a minor performance trick, only useful with lots of very small keys.
- * It means : check for aligned/unaligned input.
- * The check costs one initial branch per hash;
- * set it to 0 when the input is guaranteed to be aligned,
- * or when alignment doesn't matter for performance.
- */
-#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
-#  if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
-#    define XXH_FORCE_ALIGN_CHECK 0
-#  else
-#    define XXH_FORCE_ALIGN_CHECK 1
-#  endif
-#endif
-
-/*!XXH_REROLL:
- * Whether to reroll XXH32_finalize, and XXH64_finalize,
- * instead of using an unrolled jump table/if statement loop.
+ * xxHash - Extremely Fast Hash algorithm
+ * Copyright (C) 2012-2020 Yann Collet
  *
- * This is automatically defined on -Os/-Oz on GCC and Clang. */
-#ifndef XXH_REROLL
-#  if defined(__OPTIMIZE_SIZE__)
-#    define XXH_REROLL 1
-#  else
-#    define XXH_REROLL 0
-#  endif
-#endif
-
-/* *************************************
-*  Includes & Memory related functions
-***************************************/
-/*! Modify the local functions below should you wish to use some other memory routines
-*   for malloc(), free() */
-#include <stdlib.h>
-static void* XXH_malloc(size_t s) { return malloc(s); }
-static void  XXH_free  (void* p)  { free(p); }
-/*! and for memcpy() */
-#include <string.h>
-static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
-
-#include <limits.h>   /* ULLONG_MAX */
-
-#define XXH_STATIC_LINKING_ONLY
-#include "xxhash.h"
-
-
-/* *************************************
-*  Compiler Specific Options
-***************************************/
-#ifdef _MSC_VER    /* Visual Studio */
-#  pragma warning(disable : 4127)      /* disable: C4127: conditional expression is constant */
-#  define XXH_FORCE_INLINE static __forceinline
-#  define XXH_NO_INLINE static __declspec(noinline)
-#else
-#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
-#    ifdef __GNUC__
-#      define XXH_FORCE_INLINE static inline __attribute__((always_inline))
-#      define XXH_NO_INLINE static __attribute__((noinline))
-#    else
-#      define XXH_FORCE_INLINE static inline
-#      define XXH_NO_INLINE static
-#    endif
-#  else
-#    define XXH_FORCE_INLINE static
-#    define XXH_NO_INLINE static
-#  endif /* __STDC_VERSION__ */
-#endif
-
-
-
-/* *************************************
-*  Debug
-***************************************/
-/* DEBUGLEVEL is expected to be defined externally,
- * typically through compiler command line.
- * Value must be a number. */
-#ifndef DEBUGLEVEL
-#  define DEBUGLEVEL 0
-#endif
-
-#if (DEBUGLEVEL>=1)
-#  include <assert.h>   /* note : can still be disabled with NDEBUG */
-#  define XXH_ASSERT(c)   assert(c)
-#else
-#  define XXH_ASSERT(c)   ((void)0)
-#endif
-
-/* note : use after variable declarations */
-#define XXH_STATIC_ASSERT(c)  { enum { XXH_sa = 1/(int)(!!(c)) }; }
-
-
-/* *************************************
-*  Basic Types
-***************************************/
-#ifndef MEM_MODULE
-# if !defined (__VMS) \
-  && (defined (__cplusplus) \
-  || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
-#   include <stdint.h>
-    typedef uint8_t  BYTE;
-    typedef uint16_t U16;
-    typedef uint32_t U32;
-# else
-    typedef unsigned char      BYTE;
-    typedef unsigned short     U16;
-    typedef unsigned int       U32;
-# endif
-#endif
-
-
-/* ===   Memory access   === */
-
-#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
-
-/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
-static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; }
-
-#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
-
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-typedef union { U32 u32; } __attribute__((packed)) unalign;
-static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-
-#else
-
-/* portable and safe solution. Generally efficient.
- * see : http://stackoverflow.com/a/32095106/646947
- */
-static U32 XXH_read32(const void* memPtr)
-{
-    U32 val;
-    memcpy(&val, memPtr, sizeof(val));
-    return val;
-}
-
-#endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
-
-
-/* ===   Endianess   === */
-typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
-
-/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */
-#ifndef XXH_CPU_LITTLE_ENDIAN
-static int XXH_isLittleEndian(void)
-{
-    const union { U32 u; BYTE c[4]; } one = { 1 };   /* don't use static : performance detrimental  */
-    return one.c[0];
-}
-#   define XXH_CPU_LITTLE_ENDIAN   XXH_isLittleEndian()
-#endif
-
-
-
-
-/* ****************************************
-*  Compiler-specific Functions and Macros
-******************************************/
-#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-
-#ifndef __has_builtin
-#  define __has_builtin(x) 0
-#endif
-
-#if !defined(NO_CLANG_BUILTIN) && __has_builtin(__builtin_rotateleft32) && __has_builtin(__builtin_rotateleft64)
-#  define XXH_rotl32 __builtin_rotateleft32
-#  define XXH_rotl64 __builtin_rotateleft64
-/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */
-#elif defined(_MSC_VER)
-#  define XXH_rotl32(x,r) _rotl(x,r)
-#  define XXH_rotl64(x,r) _rotl64(x,r)
-#else
-#  define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r))))
-#  define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r))))
-#endif
-
-#if defined(_MSC_VER)     /* Visual Studio */
-#  define XXH_swap32 _byteswap_ulong
-#elif XXH_GCC_VERSION >= 403
-#  define XXH_swap32 __builtin_bswap32
-#else
-static U32 XXH_swap32 (U32 x)
-{
-    return  ((x << 24) & 0xff000000 ) |
-            ((x <<  8) & 0x00ff0000 ) |
-            ((x >>  8) & 0x0000ff00 ) |
-            ((x >> 24) & 0x000000ff );
-}
-#endif
-
-
-/* ***************************
-*  Memory reads
-*****************************/
-typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
-
-XXH_FORCE_INLINE U32 XXH_readLE32(const void* ptr)
-{
-    return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
-}
-
-static U32 XXH_readBE32(const void* ptr)
-{
-    return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
-}
-
-XXH_FORCE_INLINE U32
-XXH_readLE32_align(const void* ptr, XXH_alignment align)
-{
-    if (align==XXH_unaligned) {
-        return XXH_readLE32(ptr);
-    } else {
-        return XXH_CPU_LITTLE_ENDIAN ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
-    }
-}
-
-
-/* *************************************
-*  Misc
-***************************************/
-XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
-
-
-/* *******************************************************************
-*  32-bit hash functions
-*********************************************************************/
-static const U32 PRIME32_1 = 0x9E3779B1U;   /* 0b10011110001101110111100110110001 */
-static const U32 PRIME32_2 = 0x85EBCA77U;   /* 0b10000101111010111100101001110111 */
-static const U32 PRIME32_3 = 0xC2B2AE3DU;   /* 0b11000010101100101010111000111101 */
-static const U32 PRIME32_4 = 0x27D4EB2FU;   /* 0b00100111110101001110101100101111 */
-static const U32 PRIME32_5 = 0x165667B1U;   /* 0b00010110010101100110011110110001 */
-
-static U32 XXH32_round(U32 acc, U32 input)
-{
-    acc += input * PRIME32_2;
-    acc  = XXH_rotl32(acc, 13);
-    acc *= PRIME32_1;
-#if defined(__GNUC__) && defined(__SSE4_1__) && !defined(XXH_ENABLE_AUTOVECTORIZE)
-    /* UGLY HACK:
-     * This inline assembly hack forces acc into a normal register. This is the
-     * only thing that prevents GCC and Clang from autovectorizing the XXH32 loop
-     * (pragmas and attributes don't work for some resason) without globally
-     * disabling SSE4.1.
-     *
-     * The reason we want to avoid vectorization is because despite working on
-     * 4 integers at a time, there are multiple factors slowing XXH32 down on
-     * SSE4:
-     * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on newer chips!)
-     *   making it slightly slower to multiply four integers at once compared to four
-     *   integers independently. Even when pmulld was fastest, Sandy/Ivy Bridge, it is
-     *   still not worth it to go into SSE just to multiply unless doing a long operation.
-     *
-     * - Four instructions are required to rotate,
-     *      movqda tmp,  v // not required with VEX encoding
-     *      pslld  tmp, 13 // tmp <<= 13
-     *      psrld  v,   19 // x >>= 19
-     *      por    v,  tmp // x |= tmp
-     *   compared to one for scalar:
-     *      roll   v, 13    // reliably fast across the board
-     *      shldl  v, v, 13 // Sandy Bridge and later prefer this for some reason
-     *
-     * - Instruction level parallelism is actually more beneficial here because the
-     *   SIMD actually serializes this operation: While v1 is rotating, v2 can load data,
-     *   while v3 can multiply. SSE forces them to operate together.
-     *
-     * How this hack works:
-     * __asm__(""       // Declare an assembly block but don't declare any instructions
-     *          :       // However, as an Input/Output Operand,
-     *          "+r"    // constrain a read/write operand (+) as a general purpose register (r).
-     *          (acc)   // and set acc as the operand
-     * );
-     *
-     * Because of the 'r', the compiler has promised that seed will be in a
-     * general purpose register and the '+' says that it will be 'read/write',
-     * so it has to assume it has changed. It is like volatile without all the
-     * loads and stores.
-     *
-     * Since the argument has to be in a normal register (not an SSE register),
-     * each time XXH32_round is called, it is impossible to vectorize. */
-    __asm__("" : "+r" (acc));
-#endif
-    return acc;
-}
-
-/* mix all bits */
-static U32 XXH32_avalanche(U32 h32)
-{
-    h32 ^= h32 >> 15;
-    h32 *= PRIME32_2;
-    h32 ^= h32 >> 13;
-    h32 *= PRIME32_3;
-    h32 ^= h32 >> 16;
-    return(h32);
-}
-
-#define XXH_get32bits(p) XXH_readLE32_align(p, align)
-
-static U32
-XXH32_finalize(U32 h32, const void* ptr, size_t len, XXH_alignment align)
-{
-    const BYTE* p = (const BYTE*)ptr;
-
-#define PROCESS1               \
-    h32 += (*p++) * PRIME32_5; \
-    h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
-
-#define PROCESS4                         \
-    h32 += XXH_get32bits(p) * PRIME32_3; \
-    p+=4;                                \
-    h32  = XXH_rotl32(h32, 17) * PRIME32_4 ;
-
-    /* Compact rerolled version */
-    if (XXH_REROLL) {
-        len &= 15;
-        while (len >= 4) {
-            PROCESS4;
-            len -= 4;
-        }
-        while (len > 0) {
-            PROCESS1;
-            --len;
-        }
-        return XXH32_avalanche(h32);
-    } else {
-         switch(len&15) /* or switch(bEnd - p) */ {
-           case 12:      PROCESS4;
-                         /* fallthrough */
-           case 8:       PROCESS4;
-                         /* fallthrough */
-           case 4:       PROCESS4;
-                         return XXH32_avalanche(h32);
-
-           case 13:      PROCESS4;
-                         /* fallthrough */
-           case 9:       PROCESS4;
-                         /* fallthrough */
-           case 5:       PROCESS4;
-                         PROCESS1;
-                         return XXH32_avalanche(h32);
-
-           case 14:      PROCESS4;
-                         /* fallthrough */
-           case 10:      PROCESS4;
-                         /* fallthrough */
-           case 6:       PROCESS4;
-                         PROCESS1;
-                         PROCESS1;
-                         return XXH32_avalanche(h32);
-
-           case 15:      PROCESS4;
-                         /* fallthrough */
-           case 11:      PROCESS4;
-                         /* fallthrough */
-           case 7:       PROCESS4;
-                         /* fallthrough */
-           case 3:       PROCESS1;
-                         /* fallthrough */
-           case 2:       PROCESS1;
-                         /* fallthrough */
-           case 1:       PROCESS1;
-                         /* fallthrough */
-           case 0:       return XXH32_avalanche(h32);
-        }
-        XXH_ASSERT(0);
-        return h32;   /* reaching this point is deemed impossible */
-    }
-}
-
-XXH_FORCE_INLINE U32
-XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_alignment align)
-{
-    const BYTE* p = (const BYTE*)input;
-    const BYTE* bEnd = p + len;
-    U32 h32;
-
-#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
-    if (p==NULL) {
-        len=0;
-        bEnd=p=(const BYTE*)(size_t)16;
-    }
-#endif
-
-    if (len>=16) {
-        const BYTE* const limit = bEnd - 15;
-        U32 v1 = seed + PRIME32_1 + PRIME32_2;
-        U32 v2 = seed + PRIME32_2;
-        U32 v3 = seed + 0;
-        U32 v4 = seed - PRIME32_1;
-
-        do {
-            v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4;
-            v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4;
-            v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4;
-            v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4;
-        } while (p < limit);
-
-        h32 = XXH_rotl32(v1, 1)  + XXH_rotl32(v2, 7)
-            + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
-    } else {
-        h32  = seed + PRIME32_5;
-    }
-
-    h32 += (U32)len;
-
-    return XXH32_finalize(h32, p, len&15, align);
-}
-
-
-XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed)
-{
-#if 0
-    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
-    XXH32_state_t state;
-    XXH32_reset(&state, seed);
-    XXH32_update(&state, input, len);
-    return XXH32_digest(&state);
-
-#else
-
-    if (XXH_FORCE_ALIGN_CHECK) {
-        if ((((size_t)input) & 3) == 0) {   /* Input is 4-bytes aligned, leverage the speed benefit */
-            return XXH32_endian_align(input, len, seed, XXH_aligned);
-    }   }
-
-    return XXH32_endian_align(input, len, seed, XXH_unaligned);
-#endif
-}
-
-
-
-/*======   Hash streaming   ======*/
-
-XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
-{
-    return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
-}
-XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
-{
-    XXH_free(statePtr);
-    return XXH_OK;
-}
-
-XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState)
-{
-    memcpy(dstState, srcState, sizeof(*dstState));
-}
-
-XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed)
-{
-    XXH32_state_t state;   /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
-    memset(&state, 0, sizeof(state));
-    state.v1 = seed + PRIME32_1 + PRIME32_2;
-    state.v2 = seed + PRIME32_2;
-    state.v3 = seed + 0;
-    state.v4 = seed - PRIME32_1;
-    /* do not write into reserved, planned to be removed in a future version */
-    memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved));
-    return XXH_OK;
-}
-
-
-XXH_PUBLIC_API XXH_errorcode
-XXH32_update(XXH32_state_t* state, const void* input, size_t len)
-{
-    if (input==NULL)
-#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
-        return XXH_OK;
-#else
-        return XXH_ERROR;
-#endif
-
-    {   const BYTE* p = (const BYTE*)input;
-        const BYTE* const bEnd = p + len;
-
-        state->total_len_32 += (XXH32_hash_t)len;
-        state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16));
-
-        if (state->memsize + len < 16)  {   /* fill in tmp buffer */
-            XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
-            state->memsize += (XXH32_hash_t)len;
-            return XXH_OK;
-        }
-
-        if (state->memsize) {   /* some data left from previous update */
-            XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
-            {   const U32* p32 = state->mem32;
-                state->v1 = XXH32_round(state->v1, XXH_readLE32(p32)); p32++;
-                state->v2 = XXH32_round(state->v2, XXH_readLE32(p32)); p32++;
-                state->v3 = XXH32_round(state->v3, XXH_readLE32(p32)); p32++;
-                state->v4 = XXH32_round(state->v4, XXH_readLE32(p32));
-            }
-            p += 16-state->memsize;
-            state->memsize = 0;
-        }
-
-        if (p <= bEnd-16) {
-            const BYTE* const limit = bEnd - 16;
-            U32 v1 = state->v1;
-            U32 v2 = state->v2;
-            U32 v3 = state->v3;
-            U32 v4 = state->v4;
-
-            do {
-                v1 = XXH32_round(v1, XXH_readLE32(p)); p+=4;
-                v2 = XXH32_round(v2, XXH_readLE32(p)); p+=4;
-                v3 = XXH32_round(v3, XXH_readLE32(p)); p+=4;
-                v4 = XXH32_round(v4, XXH_readLE32(p)); p+=4;
-            } while (p<=limit);
-
-            state->v1 = v1;
-            state->v2 = v2;
-            state->v3 = v3;
-            state->v4 = v4;
-        }
-
-        if (p < bEnd) {
-            XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
-            state->memsize = (unsigned)(bEnd-p);
-        }
-    }
-
-    return XXH_OK;
-}
-
-
-XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state)
-{
-    U32 h32;
-
-    if (state->large_len) {
-        h32 = XXH_rotl32(state->v1, 1)
-            + XXH_rotl32(state->v2, 7)
-            + XXH_rotl32(state->v3, 12)
-            + XXH_rotl32(state->v4, 18);
-    } else {
-        h32 = state->v3 /* == seed */ + PRIME32_5;
-    }
-
-    h32 += state->total_len_32;
-
-    return XXH32_finalize(h32, state->mem32, state->memsize, XXH_aligned);
-}
-
-
-/*======   Canonical representation   ======*/
-
-/*! Default XXH result types are basic unsigned 32 and 64 bits.
-*   The canonical representation follows human-readable write convention, aka big-endian (large digits first).
-*   These functions allow transformation of hash result into and from its canonical format.
-*   This way, hash values can be written into a file or buffer, remaining comparable across different systems.
-*/
-
-XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
-{
-    XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
-    if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
-    memcpy(dst, &hash, sizeof(*dst));
-}
-
-XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
-{
-    return XXH_readBE32(src);
-}
-
-
-#ifndef XXH_NO_LONG_LONG
-
-/* *******************************************************************
-*  64-bit hash functions
-*********************************************************************/
-
-/*======   Memory access   ======*/
-
-#ifndef MEM_MODULE
-# define MEM_MODULE
-# if !defined (__VMS) \
-  && (defined (__cplusplus) \
-  || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
-#   include <stdint.h>
-    typedef uint64_t U64;
-# else
-    /* if compiler doesn't support unsigned long long, replace by another 64-bit type */
-    typedef unsigned long long U64;
-# endif
-#endif
-
-/*! XXH_REROLL_XXH64:
- * Whether to reroll the XXH64_finalize() loop.
+ * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
  *
- * Just like XXH32, we can unroll the XXH64_finalize() loop. This can be a performance gain
- * on 64-bit hosts, as only one jump is required.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
  *
- * However, on 32-bit hosts, because arithmetic needs to be done with two 32-bit registers,
- * and 64-bit arithmetic needs to be simulated, it isn't beneficial to unroll. The code becomes
- * ridiculously large (the largest function in the binary on i386!), and rerolling it saves
- * anywhere from 3kB to 20kB. It is also slightly faster because it fits into cache better
- * and is more likely to be inlined by the compiler.
+ *    * Redistributions of source code must retain the above copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *    * Redistributions in binary form must reproduce the above
+ *      copyright notice, this list of conditions and the following disclaimer
+ *      in the documentation and/or other materials provided with the
+ *      distribution.
  *
- * If XXH_REROLL is defined, this is ignored and the loop is always rerolled. */
-#ifndef XXH_REROLL_XXH64
-#  if (defined(__ILP32__) || defined(_ILP32)) /* ILP32 is often defined on 32-bit GCC family */ \
-   || !(defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) /* x86-64 */ \
-     || defined(_M_ARM64) || defined(__aarch64__) || defined(__arm64__) /* aarch64 */ \
-     || defined(__PPC64__) || defined(__PPC64LE__) || defined(__ppc64__) || defined(__powerpc64__) /* ppc64 */ \
-     || defined(__mips64__) || defined(__mips64)) /* mips64 */ \
-   || (!defined(SIZE_MAX) || SIZE_MAX < ULLONG_MAX) /* check limits */
-#    define XXH_REROLL_XXH64 1
-#  else
-#    define XXH_REROLL_XXH64 0
-#  endif
-#endif /* !defined(XXH_REROLL_XXH64) */
-
-#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
-
-/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
-static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; }
-
-#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
-
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign64;
-static U64 XXH_read64(const void* ptr) { return ((const unalign64*)ptr)->u64; }
-
-#else
-
-/* portable and safe solution. Generally efficient.
- * see : http://stackoverflow.com/a/32095106/646947
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * You can contact the author at:
+ *   - xxHash homepage: https://www.xxhash.com
+ *   - xxHash source repository: https://github.com/Cyan4973/xxHash
  */
 
-static U64 XXH_read64(const void* memPtr)
-{
-    U64 val;
-    memcpy(&val, memPtr, sizeof(val));
-    return val;
-}
-
-#endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
-
-#if defined(_MSC_VER)     /* Visual Studio */
-#  define XXH_swap64 _byteswap_uint64
-#elif XXH_GCC_VERSION >= 403
-#  define XXH_swap64 __builtin_bswap64
-#else
-static U64 XXH_swap64 (U64 x)
-{
-    return  ((x << 56) & 0xff00000000000000ULL) |
-            ((x << 40) & 0x00ff000000000000ULL) |
-            ((x << 24) & 0x0000ff0000000000ULL) |
-            ((x << 8)  & 0x000000ff00000000ULL) |
-            ((x >> 8)  & 0x00000000ff000000ULL) |
-            ((x >> 24) & 0x0000000000ff0000ULL) |
-            ((x >> 40) & 0x000000000000ff00ULL) |
-            ((x >> 56) & 0x00000000000000ffULL);
-}
-#endif
-
-XXH_FORCE_INLINE U64 XXH_readLE64(const void* ptr)
-{
-    return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
-}
-
-static U64 XXH_readBE64(const void* ptr)
-{
-    return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
-}
-
-XXH_FORCE_INLINE U64
-XXH_readLE64_align(const void* ptr, XXH_alignment align)
-{
-    if (align==XXH_unaligned)
-        return XXH_readLE64(ptr);
-    else
-        return XXH_CPU_LITTLE_ENDIAN ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
-}
-
-
-/*======   xxh64   ======*/
-
-static const U64 PRIME64_1 = 0x9E3779B185EBCA87ULL;   /* 0b1001111000110111011110011011000110000101111010111100101010000111 */
-static const U64 PRIME64_2 = 0xC2B2AE3D27D4EB4FULL;   /* 0b1100001010110010101011100011110100100111110101001110101101001111 */
-static const U64 PRIME64_3 = 0x165667B19E3779F9ULL;   /* 0b0001011001010110011001111011000110011110001101110111100111111001 */
-static const U64 PRIME64_4 = 0x85EBCA77C2B2AE63ULL;   /* 0b1000010111101011110010100111011111000010101100101010111001100011 */
-static const U64 PRIME64_5 = 0x27D4EB2F165667C5ULL;   /* 0b0010011111010100111010110010111100010110010101100110011111000101 */
-
-static U64 XXH64_round(U64 acc, U64 input)
-{
-    acc += input * PRIME64_2;
-    acc  = XXH_rotl64(acc, 31);
-    acc *= PRIME64_1;
-    return acc;
-}
-
-static U64 XXH64_mergeRound(U64 acc, U64 val)
-{
-    val  = XXH64_round(0, val);
-    acc ^= val;
-    acc  = acc * PRIME64_1 + PRIME64_4;
-    return acc;
-}
-
-static U64 XXH64_avalanche(U64 h64)
-{
-    h64 ^= h64 >> 33;
-    h64 *= PRIME64_2;
-    h64 ^= h64 >> 29;
-    h64 *= PRIME64_3;
-    h64 ^= h64 >> 32;
-    return h64;
-}
-
-
-#define XXH_get64bits(p) XXH_readLE64_align(p, align)
-
-static U64
-XXH64_finalize(U64 h64, const void* ptr, size_t len, XXH_alignment align)
-{
-    const BYTE* p = (const BYTE*)ptr;
-
-#define PROCESS1_64            \
-    h64 ^= (*p++) * PRIME64_5; \
-    h64 = XXH_rotl64(h64, 11) * PRIME64_1;
-
-#define PROCESS4_64          \
-    h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; \
-    p+=4;                    \
-    h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
-
-#define PROCESS8_64 {        \
-    U64 const k1 = XXH64_round(0, XXH_get64bits(p)); \
-    p+=8;                    \
-    h64 ^= k1;               \
-    h64  = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; \
-}
-
-    /* Rerolled version for 32-bit targets is faster and much smaller. */
-    if (XXH_REROLL || XXH_REROLL_XXH64) {
-        len &= 31;
-        while (len >= 8) {
-            PROCESS8_64;
-            len -= 8;
-        }
-        if (len >= 4) {
-            PROCESS4_64;
-            len -= 4;
-        }
-        while (len > 0) {
-            PROCESS1_64;
-            --len;
-        }
-         return  XXH64_avalanche(h64);
-    } else {
-        switch(len & 31) {
-           case 24: PROCESS8_64;
-                         /* fallthrough */
-           case 16: PROCESS8_64;
-                         /* fallthrough */
-           case  8: PROCESS8_64;
-                    return XXH64_avalanche(h64);
-
-           case 28: PROCESS8_64;
-                         /* fallthrough */
-           case 20: PROCESS8_64;
-                         /* fallthrough */
-           case 12: PROCESS8_64;
-                         /* fallthrough */
-           case  4: PROCESS4_64;
-                    return XXH64_avalanche(h64);
-
-           case 25: PROCESS8_64;
-                         /* fallthrough */
-           case 17: PROCESS8_64;
-                         /* fallthrough */
-           case  9: PROCESS8_64;
-                    PROCESS1_64;
-                    return XXH64_avalanche(h64);
-
-           case 29: PROCESS8_64;
-                         /* fallthrough */
-           case 21: PROCESS8_64;
-                         /* fallthrough */
-           case 13: PROCESS8_64;
-                         /* fallthrough */
-           case  5: PROCESS4_64;
-                    PROCESS1_64;
-                    return XXH64_avalanche(h64);
-
-           case 26: PROCESS8_64;
-                         /* fallthrough */
-           case 18: PROCESS8_64;
-                         /* fallthrough */
-           case 10: PROCESS8_64;
-                    PROCESS1_64;
-                    PROCESS1_64;
-                    return XXH64_avalanche(h64);
-
-           case 30: PROCESS8_64;
-                         /* fallthrough */
-           case 22: PROCESS8_64;
-                         /* fallthrough */
-           case 14: PROCESS8_64;
-                         /* fallthrough */
-           case  6: PROCESS4_64;
-                    PROCESS1_64;
-                    PROCESS1_64;
-                    return XXH64_avalanche(h64);
-
-           case 27: PROCESS8_64;
-                         /* fallthrough */
-           case 19: PROCESS8_64;
-                         /* fallthrough */
-           case 11: PROCESS8_64;
-                    PROCESS1_64;
-                    PROCESS1_64;
-                    PROCESS1_64;
-                    return XXH64_avalanche(h64);
 
-           case 31: PROCESS8_64;
-                         /* fallthrough */
-           case 23: PROCESS8_64;
-                         /* fallthrough */
-           case 15: PROCESS8_64;
-                         /* fallthrough */
-           case  7: PROCESS4_64;
-                         /* fallthrough */
-           case  3: PROCESS1_64;
-                         /* fallthrough */
-           case  2: PROCESS1_64;
-                         /* fallthrough */
-           case  1: PROCESS1_64;
-                         /* fallthrough */
-           case  0: return XXH64_avalanche(h64);
-        }
-    }
-    /* impossible to reach */
-    XXH_ASSERT(0);
-    return 0;  /* unreachable, but some compilers complain without it */
-}
-
-XXH_FORCE_INLINE U64
-XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_alignment align)
-{
-    const BYTE* p = (const BYTE*)input;
-    const BYTE* bEnd = p + len;
-    U64 h64;
-
-#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
-    if (p==NULL) {
-        len=0;
-        bEnd=p=(const BYTE*)(size_t)32;
-    }
-#endif
-
-    if (len>=32) {
-        const BYTE* const limit = bEnd - 32;
-        U64 v1 = seed + PRIME64_1 + PRIME64_2;
-        U64 v2 = seed + PRIME64_2;
-        U64 v3 = seed + 0;
-        U64 v4 = seed - PRIME64_1;
-
-        do {
-            v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8;
-            v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8;
-            v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8;
-            v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8;
-        } while (p<=limit);
-
-        h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
-        h64 = XXH64_mergeRound(h64, v1);
-        h64 = XXH64_mergeRound(h64, v2);
-        h64 = XXH64_mergeRound(h64, v3);
-        h64 = XXH64_mergeRound(h64, v4);
-
-    } else {
-        h64  = seed + PRIME64_5;
-    }
-
-    h64 += (U64) len;
-
-    return XXH64_finalize(h64, p, len, align);
-}
-
-
-XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t len, unsigned long long seed)
-{
-#if 0
-    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
-    XXH64_state_t state;
-    XXH64_reset(&state, seed);
-    XXH64_update(&state, input, len);
-    return XXH64_digest(&state);
-
-#else
-
-    if (XXH_FORCE_ALIGN_CHECK) {
-        if ((((size_t)input) & 7)==0) {  /* Input is aligned, let's leverage the speed advantage */
-            return XXH64_endian_align(input, len, seed, XXH_aligned);
-    }   }
-
-    return XXH64_endian_align(input, len, seed, XXH_unaligned);
-
-#endif
-}
-
-/*======   Hash Streaming   ======*/
-
-XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
-{
-    return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
-}
-XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
-{
-    XXH_free(statePtr);
-    return XXH_OK;
-}
-
-XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState)
-{
-    memcpy(dstState, srcState, sizeof(*dstState));
-}
-
-XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed)
-{
-    XXH64_state_t state;   /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
-    memset(&state, 0, sizeof(state));
-    state.v1 = seed + PRIME64_1 + PRIME64_2;
-    state.v2 = seed + PRIME64_2;
-    state.v3 = seed + 0;
-    state.v4 = seed - PRIME64_1;
-     /* do not write into reserved, might be removed in a future version */
-    memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved));
-    return XXH_OK;
-}
-
-XXH_PUBLIC_API XXH_errorcode
-XXH64_update (XXH64_state_t* state, const void* input, size_t len)
-{
-    if (input==NULL)
-#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
-        return XXH_OK;
-#else
-        return XXH_ERROR;
-#endif
-
-    {   const BYTE* p = (const BYTE*)input;
-        const BYTE* const bEnd = p + len;
-
-        state->total_len += len;
-
-        if (state->memsize + len < 32) {  /* fill in tmp buffer */
-            XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
-            state->memsize += (U32)len;
-            return XXH_OK;
-        }
-
-        if (state->memsize) {   /* tmp buffer is full */
-            XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
-            state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0));
-            state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1));
-            state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2));
-            state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3));
-            p += 32-state->memsize;
-            state->memsize = 0;
-        }
-
-        if (p+32 <= bEnd) {
-            const BYTE* const limit = bEnd - 32;
-            U64 v1 = state->v1;
-            U64 v2 = state->v2;
-            U64 v3 = state->v3;
-            U64 v4 = state->v4;
-
-            do {
-                v1 = XXH64_round(v1, XXH_readLE64(p)); p+=8;
-                v2 = XXH64_round(v2, XXH_readLE64(p)); p+=8;
-                v3 = XXH64_round(v3, XXH_readLE64(p)); p+=8;
-                v4 = XXH64_round(v4, XXH_readLE64(p)); p+=8;
-            } while (p<=limit);
-
-            state->v1 = v1;
-            state->v2 = v2;
-            state->v3 = v3;
-            state->v4 = v4;
-        }
-
-        if (p < bEnd) {
-            XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
-            state->memsize = (unsigned)(bEnd-p);
-        }
-    }
-
-    return XXH_OK;
-}
-
-
-XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* state)
-{
-    U64 h64;
-
-    if (state->total_len >= 32) {
-        U64 const v1 = state->v1;
-        U64 const v2 = state->v2;
-        U64 const v3 = state->v3;
-        U64 const v4 = state->v4;
-
-        h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
-        h64 = XXH64_mergeRound(h64, v1);
-        h64 = XXH64_mergeRound(h64, v2);
-        h64 = XXH64_mergeRound(h64, v3);
-        h64 = XXH64_mergeRound(h64, v4);
-    } else {
-        h64  = state->v3 /*seed*/ + PRIME64_5;
-    }
-
-    h64 += (U64) state->total_len;
-
-    return XXH64_finalize(h64, state->mem64, (size_t)state->total_len, XXH_aligned);
-}
-
-
-/*====== Canonical representation   ======*/
-
-XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
-{
-    XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
-    if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
-    memcpy(dst, &hash, sizeof(*dst));
-}
-
-XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
-{
-    return XXH_readBE64(src);
-}
-
-
-
-/* *********************************************************************
-*  XXH3
-*  New generation hash designed for speed on small keys and vectorization
-************************************************************************ */
-
-#include "arrow/vendored/xxhash/xxh3.h"
+/*
+ * xxhash.c instantiates functions defined in xxhash.h
+ */
 
+#define XXH_STATIC_LINKING_ONLY   /* access advanced declarations */
+#define XXH_IMPLEMENTATION   /* access definitions */
 
-#endif  /* XXH_NO_LONG_LONG */
+#include "xxhash.h"
diff --git a/cpp/src/arrow/vendored/xxhash/xxhash.h b/cpp/src/arrow/vendored/xxhash/xxhash.h
index 7533eb29a54..2d56d23c5d0 100644
--- a/cpp/src/arrow/vendored/xxhash/xxhash.h
+++ b/cpp/src/arrow/vendored/xxhash/xxhash.h
@@ -1,40 +1,42 @@
 /*
-   xxHash - Extremely Fast Hash algorithm
-   Header File
-   Copyright (C) 2012-2016, Yann Collet.
-
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions are
-   met:
-
-       * Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-       * Redistributions in binary form must reproduce the above
-   copyright notice, this list of conditions and the following disclaimer
-   in the documentation and/or other materials provided with the
-   distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-   You can contact the author at :
-   - xxHash source repository : https://github.com/Cyan4973/xxHash
-*/
+ * xxHash - Extremely Fast Hash algorithm
+ * Header File
+ * Copyright (C) 2012-2020 Yann Collet
+ *
+ * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *    * Redistributions of source code must retain the above copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *    * Redistributions in binary form must reproduce the above
+ *      copyright notice, this list of conditions and the following disclaimer
+ *      in the documentation and/or other materials provided with the
+ *      distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * You can contact the author at:
+ *   - xxHash homepage: https://www.xxhash.com
+ *   - xxHash source repository: https://github.com/Cyan4973/xxHash
+ */
 
-/* Notice extracted from xxHash homepage :
+/* TODO: update */
+/* Notice extracted from xxHash homepage:
 
-xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
+xxHash is an extremely fast hash algorithm, running at RAM speed limits.
 It also successfully passes all tests from the SMHasher suite.
 
 Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
@@ -57,6 +59,11 @@ Q.Score is a measure of quality of the hash function.
 It depends on successfully passing SMHasher test set.
 10 is a perfect score.
 
+Note: SMHasher's CRC32 implementation is not the fastest one.
+Other speed-oriented implementations can be faster,
+especially in combination with PCLMUL instruction:
+https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html?showComment=1552696407071#c3490092340461170735
+
 A 64-bit version, named XXH64, is available since r35.
 It offers much better speed, but for 64-bit applications only.
 Name     Speed on 64 bits    Speed on 32 bits
@@ -64,40 +71,38 @@ XXH64       13.8 GB/s            1.9 GB/s
 XXH32        6.8 GB/s            6.0 GB/s
 */
 
-#ifndef XXHASH_H_5627135585666179
-#define XXHASH_H_5627135585666179 1
-
 #if defined (__cplusplus)
 extern "C" {
 #endif
 
-
-/* ****************************
-*  Definitions
-******************************/
-#include <stddef.h>   /* size_t */
-typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
-
-
 /* ****************************
- *  API modifier
+ *  INLINE mode
  ******************************/
-/** XXH_INLINE_ALL (and XXH_PRIVATE_API)
- *  This build macro includes xxhash functions in `static` mode
- *  in order to inline them, and remove their symbol from the public list.
- *  Inlining offers great performance improvement on small keys,
- *  and dramatic ones when length is expressed as a compile-time constant.
- *  See https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html .
- *  Methodology :
+/*!
+ * XXH_INLINE_ALL (and XXH_PRIVATE_API)
+ * Use these build macros to inline xxhash into the target unit.
+ * Inlining improves performance on small inputs, especially when the length is
+ * expressed as a compile-time constant:
+ *
+ *      https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html
+ *
+ * It also keeps xxHash symbols private to the unit, so they are not exported.
+ *
+ * Usage:
  *     #define XXH_INLINE_ALL
  *     #include "xxhash.h"
- * `xxhash.c` is automatically included.
- *  It's not useful to compile and link it as a separate object.
+ *
+ * Do not compile and link xxhash.o as a separate object, as it is not useful.
  */
-#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
-#  ifndef XXH_STATIC_LINKING_ONLY
-#    define XXH_STATIC_LINKING_ONLY
-#  endif
+#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \
+    && !defined(XXH_INLINE_ALL_31684351384)
+   /* this section should be traversed only once */
+#  define XXH_INLINE_ALL_31684351384
+   /* give access to the advanced API, required to compile implementations */
+#  undef XXH_STATIC_LINKING_ONLY   /* avoid macro redef */
+#  define XXH_STATIC_LINKING_ONLY
+   /* make all functions private */
+#  undef XXH_PUBLIC_API
 #  if defined(__GNUC__)
 #    define XXH_PUBLIC_API static __inline __attribute__((unused))
 #  elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
@@ -105,10 +110,63 @@ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
 #  elif defined(_MSC_VER)
 #    define XXH_PUBLIC_API static __inline
 #  else
-     /* this version may generate warnings for unused static functions */
+     /* note: this version may generate warnings for unused static functions */
 #    define XXH_PUBLIC_API static
 #  endif
-#else
+
+   /*
+    * This part deals with the special case where a unit wants to inline xxHash,
+    * but "xxhash.h" has previously been included without XXH_INLINE_ALL, such
+    * as part of some previously included *.h header file.
+    * Without further action, the new include would just be ignored,
+    * and functions would effectively _not_ be inlined (silent failure).
+    * The following macros solve this situation by prefixing all inlined names,
+    * avoiding naming collision with previous inclusions.
+    */
+#  ifdef XXH_NAMESPACE
+#    error "XXH_INLINE_ALL with XXH_NAMESPACE is not supported"
+     /*
+      * Note: Alternative: #undef all symbols (it's a pretty large list).
+      * Without #error: it compiles, but functions are actually not inlined.
+      */
+#  endif
+#  define XXH_NAMESPACE XXH_INLINE_
+   /*
+    * Some identifiers (enums, type names) are not symbols, but they must
+    * still be renamed to avoid redeclaration.
+    * Alternative solution: do not redeclare them.
+    * However, this requires some #ifdefs, and is a more dispersed action.
+    * Meanwhile, renaming can be achieved in a single block
+    */
+#  define XXH_IPREF(Id)   XXH_INLINE_ ## Id
+#  define XXH_OK XXH_IPREF(XXH_OK)
+#  define XXH_ERROR XXH_IPREF(XXH_ERROR)
+#  define XXH_errorcode XXH_IPREF(XXH_errorcode)
+#  define XXH32_canonical_t  XXH_IPREF(XXH32_canonical_t)
+#  define XXH64_canonical_t  XXH_IPREF(XXH64_canonical_t)
+#  define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t)
+#  define XXH32_state_s XXH_IPREF(XXH32_state_s)
+#  define XXH32_state_t XXH_IPREF(XXH32_state_t)
+#  define XXH64_state_s XXH_IPREF(XXH64_state_s)
+#  define XXH64_state_t XXH_IPREF(XXH64_state_t)
+#  define XXH3_state_s  XXH_IPREF(XXH3_state_s)
+#  define XXH3_state_t  XXH_IPREF(XXH3_state_t)
+#  define XXH128_hash_t XXH_IPREF(XXH128_hash_t)
+   /* Ensure the header is parsed again, even if it was previously included */
+#  undef XXHASH_H_5627135585666179
+#  undef XXHASH_H_STATIC_13879238742
+#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */
+
+
+
+/* ****************************************************************
+ *  Stable API
+ *****************************************************************/
+#ifndef XXHASH_H_5627135585666179
+#define XXHASH_H_5627135585666179 1
+
+/* specific declaration modes for Windows */
+#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API)
 #  if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT))
 #    ifdef XXH_EXPORT
 #      define XXH_PUBLIC_API __declspec(dllexport)
@@ -118,23 +176,26 @@ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
 #  else
 #    define XXH_PUBLIC_API   /* do nothing */
 #  endif
-#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */
+#endif
 
-/*! XXH_NAMESPACE, aka Namespace Emulation :
- *
- * If you want to include _and expose_ xxHash functions from within your own library,
- * but also want to avoid symbol collisions with other libraries which may also include xxHash,
+/*!
+ * XXH_NAMESPACE, aka Namespace Emulation:
  *
- * you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library
- * with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values).
+ * If you want to include _and expose_ xxHash functions from within your own
+ * library, but also want to avoid symbol collisions with other libraries which
+ * may also include xxHash, you can use XXH_NAMESPACE to automatically prefix
+ * any public symbol from xxhash library with the value of XXH_NAMESPACE
+ * (therefore, avoid empty or numeric values).
  *
- * Note that no change is required within the calling program as long as it includes `xxhash.h` :
- * regular symbol name will be automatically translated by this header.
+ * Note that no change is required within the calling program as long as it
+ * includes `xxhash.h`: Regular symbol names will be automatically translated
+ * by this header.
  */
 #ifdef XXH_NAMESPACE
 #  define XXH_CAT(A,B) A##B
 #  define XXH_NAME2(A,B) XXH_CAT(A,B)
 #  define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
+/* XXH32 */
 #  define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
 #  define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
 #  define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
@@ -144,6 +205,7 @@ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
 #  define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
 #  define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
 #  define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
+/* XXH64 */
 #  define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
 #  define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
 #  define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
@@ -153,6 +215,33 @@ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
 #  define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
 #  define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
 #  define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
+/* XXH3_64bits */
+#  define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits)
+#  define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret)
+#  define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed)
+#  define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState)
+#  define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState)
+#  define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState)
+#  define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset)
+#  define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed)
+#  define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret)
+#  define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update)
+#  define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest)
+#  define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret)
+/* XXH3_128bits */
+#  define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128)
+#  define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits)
+#  define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed)
+#  define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret)
+#  define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset)
+#  define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed)
+#  define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret)
+#  define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update)
+#  define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest)
+#  define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual)
+#  define XXH128_cmp     XXH_NAME2(XXH_NAMESPACE, XXH128_cmp)
+#  define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash)
+#  define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical)
 #endif
 
 
@@ -160,12 +249,19 @@ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
 *  Version
 ***************************************/
 #define XXH_VERSION_MAJOR    0
-#define XXH_VERSION_MINOR    7
-#define XXH_VERSION_RELEASE  1
+#define XXH_VERSION_MINOR    8
+#define XXH_VERSION_RELEASE  0
 #define XXH_VERSION_NUMBER  (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
 XXH_PUBLIC_API unsigned XXH_versionNumber (void);
 
 
+/* ****************************
+*  Definitions
+******************************/
+#include <stddef.h>   /* size_t */
+typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
+
+
 /*-**********************************************************************
 *  32-bit hash
 ************************************************************************/
@@ -175,59 +271,90 @@ XXH_PUBLIC_API unsigned XXH_versionNumber (void);
 #   include <stdint.h>
     typedef uint32_t XXH32_hash_t;
 #else
-    typedef unsigned int XXH32_hash_t;
+#   include <limits.h>
+#   if UINT_MAX == 0xFFFFFFFFUL
+      typedef unsigned int XXH32_hash_t;
+#   else
+#     if ULONG_MAX == 0xFFFFFFFFUL
+        typedef unsigned long XXH32_hash_t;
+#     else
+#       error "unsupported platform: need a 32-bit type"
+#     endif
+#   endif
 #endif
 
-/*! XXH32() :
-    Calculate the 32-bit hash of sequence "length" bytes stored at memory address "input".
-    The memory between input & input+length must be valid (allocated and read-accessible).
-    "seed" can be used to alter the result predictably.
-    Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */
-XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed);
+/*!
+ * XXH32():
+ *  Calculate the 32-bit hash of sequence "length" bytes stored at memory address "input".
+ *  The memory between input & input+length must be valid (allocated and read-accessible).
+ *  "seed" can be used to alter the result predictably.
+ *  Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s
+ *
+ * Note: XXH3 provides competitive speed for both 32-bit and 64-bit systems,
+ * and offers true 64/128 bit hash results. It provides a superior level of
+ * dispersion, and greatly reduces the risks of collisions.
+ */
+XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed);
+
+/*******   Streaming   *******/
+
+/*
+ * Streaming functions generate the xxHash value from an incrememtal input.
+ * This method is slower than single-call functions, due to state management.
+ * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized.
+ *
+ * An XXH state must first be allocated using `XXH*_createState()`.
+ *
+ * Start a new hash by initializing the state with a seed using `XXH*_reset()`.
+ *
+ * Then, feed the hash state by calling `XXH*_update()` as many times as necessary.
+ *
+ * The function returns an error code, with 0 meaning OK, and any other value
+ * meaning there is an error.
+ *
+ * Finally, a hash value can be produced anytime, by using `XXH*_digest()`.
+ * This function returns the nn-bits hash as an int or long long.
+ *
+ * It's still possible to continue inserting input into the hash state after a
+ * digest, and generate new hash values later on by invoking `XXH*_digest()`.
+ *
+ * When done, release the state using `XXH*_freeState()`.
+ */
 
-/*======   Streaming   ======*/
 typedef struct XXH32_state_s XXH32_state_t;   /* incomplete type */
 XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
 XXH_PUBLIC_API XXH_errorcode  XXH32_freeState(XXH32_state_t* statePtr);
 XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state);
 
-XXH_PUBLIC_API XXH_errorcode XXH32_reset  (XXH32_state_t* statePtr, unsigned int seed);
+XXH_PUBLIC_API XXH_errorcode XXH32_reset  (XXH32_state_t* statePtr, XXH32_hash_t seed);
 XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
 XXH_PUBLIC_API XXH32_hash_t  XXH32_digest (const XXH32_state_t* statePtr);
 
+/*******   Canonical representation   *******/
+
 /*
- * Streaming functions generate the xxHash of an input provided in multiple segments.
- * Note that, for small input, they are slower than single-call functions, due to state management.
- * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized.
- *
- * XXH state must first be allocated, using XXH*_createState() .
- *
- * Start a new hash by initializing state with a seed, using XXH*_reset().
+ * The default return values from XXH functions are unsigned 32 and 64 bit
+ * integers.
+ * This the simplest and fastest format for further post-processing.
  *
- * Then, feed the hash state by calling XXH*_update() as many times as necessary.
- * The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
+ * However, this leaves open the question of what is the order on the byte level,
+ * since little and big endian conventions will store the same number differently.
  *
- * Finally, a hash value can be produced anytime, by using XXH*_digest().
- * This function returns the nn-bits hash as an int or long long.
+ * The canonical representation settles this issue by mandating big-endian
+ * convention, the same convention as human-readable numbers (large digits first).
  *
- * It's still possible to continue inserting input into the hash state after a digest,
- * and generate some new hashes later on, by calling again XXH*_digest().
+ * When writing hash values to storage, sending them over a network, or printing
+ * them, it's highly recommended to use the canonical representation to ensure
+ * portability across a wider range of systems, present and future.
  *
- * When done, free XXH state space if it was allocated dynamically.
+ * The following functions allow transformation of hash values to and from
+ * canonical format.
  */
 
-/*======   Canonical representation   ======*/
-
 typedef struct { unsigned char digest[4]; } XXH32_canonical_t;
 XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
 XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
 
-/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
- * The canonical representation uses human-readable write convention, aka big-endian (large digits first).
- * These functions allow transformation of hash result into and from its canonical format.
- * This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
- */
-
 
 #ifndef XXH_NO_LONG_LONG
 /*-**********************************************************************
@@ -239,262 +366,195 @@ XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src
 #   include <stdint.h>
     typedef uint64_t XXH64_hash_t;
 #else
+    /* the following type must have a width of 64-bit */
     typedef unsigned long long XXH64_hash_t;
 #endif
 
-/*! XXH64() :
-    Calculate the 64-bit hash of sequence of length "len" stored at memory address "input".
-    "seed" can be used to alter the result predictably.
-    This function runs faster on 64-bit systems, but slower on 32-bit systems (see benchmark).
-*/
-XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed);
+/*!
+ * XXH64():
+ * Returns the 64-bit hash of sequence of length @length stored at memory
+ * address @input.
+ * @seed can be used to alter the result predictably.
+ *
+ * This function usually runs faster on 64-bit systems, but slower on 32-bit
+ * systems (see benchmark).
+ *
+ * Note: XXH3 provides competitive speed for both 32-bit and 64-bit systems,
+ * and offers true 64/128 bit hash results. It provides a superior level of
+ * dispersion, and greatly reduces the risks of collisions.
+ */
+XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, XXH64_hash_t seed);
 
-/*======   Streaming   ======*/
+/*******   Streaming   *******/
 typedef struct XXH64_state_s XXH64_state_t;   /* incomplete type */
 XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void);
 XXH_PUBLIC_API XXH_errorcode  XXH64_freeState(XXH64_state_t* statePtr);
 XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state);
 
-XXH_PUBLIC_API XXH_errorcode XXH64_reset  (XXH64_state_t* statePtr, unsigned long long seed);
+XXH_PUBLIC_API XXH_errorcode XXH64_reset  (XXH64_state_t* statePtr, XXH64_hash_t seed);
 XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
 XXH_PUBLIC_API XXH64_hash_t  XXH64_digest (const XXH64_state_t* statePtr);
 
-/*======   Canonical representation   ======*/
-typedef struct { unsigned char digest[8]; } XXH64_canonical_t;
+/*******   Canonical representation   *******/
+typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t;
 XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash);
 XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
 
 
-#endif  /* XXH_NO_LONG_LONG */
-
-
-
-#ifdef XXH_STATIC_LINKING_ONLY
-
-/* ================================================================================================
-   This section contains declarations which are not guaranteed to remain stable.
-   They may change in future versions, becoming incompatible with a different version of the library.
-   These declarations should only be used with static linking.
-   Never use them in association with dynamic linking !
-=================================================================================================== */
-
-/* These definitions are only present to allow
- * static allocation of XXH state, on stack or in a struct for example.
- * Never **ever** use members directly. */
-
-struct XXH32_state_s {
-   XXH32_hash_t total_len_32;
-   XXH32_hash_t large_len;
-   XXH32_hash_t v1;
-   XXH32_hash_t v2;
-   XXH32_hash_t v3;
-   XXH32_hash_t v4;
-   XXH32_hash_t mem32[4];
-   XXH32_hash_t memsize;
-   XXH32_hash_t reserved;   /* never read nor write, might be removed in a future version */
-};   /* typedef'd to XXH32_state_t */
-
-#ifndef XXH_NO_LONG_LONG  /* remove 64-bit support */
-struct XXH64_state_s {
-   XXH64_hash_t total_len;
-   XXH64_hash_t v1;
-   XXH64_hash_t v2;
-   XXH64_hash_t v3;
-   XXH64_hash_t v4;
-   XXH64_hash_t mem64[4];
-   XXH32_hash_t memsize;
-   XXH32_hash_t reserved[2];     /* never read nor write, might be removed in a future version */
-};   /* typedef'd to XXH64_state_t */
-#endif   /* XXH_NO_LONG_LONG */
-
-
 /*-**********************************************************************
-*  XXH3
-*  New experimental hash
+*  XXH3 64-bit variant
 ************************************************************************/
-#ifndef XXH_NO_LONG_LONG
 
-
-/* ============================================
- * XXH3 is a new hash algorithm,
- * featuring improved speed performance for both small and large inputs.
- * See full speed analysis at : http://fastcompression.blogspot.com/2019/03/presenting-xxh3.html
- * In general, expect XXH3 to run about ~2x faster on large inputs,
- * and >3x faster on small ones, though exact differences depend on platform.
+/* ************************************************************************
+ * XXH3 is a new hash algorithm featuring:
+ *  - Improved speed for both small and large inputs
+ *  - True 64-bit and 128-bit outputs
+ *  - SIMD acceleration
+ *  - Improved 32-bit viability
+ *
+ * Speed analysis methodology is explained here:
+ *
+ *    https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html
+ *
+ * In general, expect XXH3 to run about ~2x faster on large inputs and >3x
+ * faster on small ones compared to XXH64, though exact differences depend on
+ * the platform.
+ *
+ * The algorithm is portable: Like XXH32 and XXH64, it generates the same hash
+ * on all platforms.
+ *
+ * It benefits greatly from SIMD and 64-bit arithmetic, but does not require it.
  *
- * The algorithm is portable, will generate the same hash on all platforms.
- * It benefits greatly from vectorization units, but does not require it.
+ * Almost all 32-bit and 64-bit targets that can run XXH32 smoothly can run
+ * XXH3 at competitive speeds, even if XXH64 runs slowly. Further details are
+ * explained in the implementation.
+ *
+ * Optimized implementations are provided for AVX512, AVX2, SSE2, NEON, POWER8,
+ * ZVector and scalar targets. This can be controlled with the XXH_VECTOR macro.
  *
  * XXH3 offers 2 variants, _64bits and _128bits.
- * When only 64 bits are needed, prefer calling the _64bits variant :
- * it reduces the amount of mixing, resulting in faster speed on small inputs.
+ * When only 64 bits are needed, prefer calling the _64bits variant, as it
+ * reduces the amount of mixing, resulting in faster speed on small inputs.
+ *
  * It's also generally simpler to manipulate a scalar return type than a struct.
  *
- * The XXH3 algorithm is still considered experimental.
- * Produced results can still change between versions.
- * For example, results produced by v0.7.1 are not comparable with results from v0.7.0 .
- * It's nonetheless possible to use XXH3 for ephemeral data (local sessions),
- * but avoid storing values in long-term storage for later re-use.
+ * The 128-bit version adds additional strength, but it is slightly slower.
  *
- * The API supports one-shot hashing, streaming mode, and custom secrets.
+ * The XXH3 algorithm is still in development.
+ * The results it produces may still change in future versions.
  *
- * There are still a number of opened questions that community can influence during the experimental period.
- * I'm trying to list a few of them below, though don't consider this list as complete.
+ * Results produced by v0.7.x are not comparable with results from v0.7.y.
+ * However, the API is completely stable, and it can safely be used for
+ * ephemeral data (local sessions).
  *
- * - 128-bits output type : currently defined as a structure of two 64-bits fields.
- *                          That's because 128-bit values do not exist in C standard.
- *                          Note that it means that, at byte level, result is not identical depending on endianess.
- *                          However, at field level, they are identical on all platforms.
- *                          The canonical representation solves the issue of identical byte-level representation across platforms,
- *                          which is necessary for serialization.
- *                          Would there be a better representation for a 128-bit hash result ?
- *                          Are the names of the inner 64-bit fields important ? Should they be changed ?
+ * Avoid storing values in long-term storage until the algorithm is finalized.
+ * XXH3's return values will be officially finalized upon reaching v0.8.0.
  *
- * - Seed type for 128-bits variant : currently, it's a single 64-bit value, like the 64-bit variant.
- *                          It could be argued that it's more logical to offer a 128-bit seed input parameter for a 128-bit hash.
- *                          But 128-bit seed is more difficult to use, since it requires to pass a structure instead of a scalar value.
- *                          Such a variant could either replace current one, or become an additional one.
- *                          Farmhash, for example, offers both variants (the 128-bits seed variant is called `doubleSeed`).
- *                          If both 64-bit and 128-bit seeds are possible, which variant should be called XXH128 ?
+ * After which, return values of XXH3 and XXH128 will no longer change in
+ * future versions.
  *
- * - Result for len==0 : Currently, the result of hashing a zero-length input is `0`.
- *                          It seems okay as a return value when using all "default" secret and seed (it used to be a request for XXH32/XXH64).
- *                          But is it still fine to return `0` when secret or seed are non-default ?
- *                          Are there use cases which could depend on generating a different hash result for zero-length input when the secret is different ?
+ * The API supports one-shot hashing, streaming mode, and custom secrets.
  */
 
-#ifdef XXH_NAMESPACE
-#  define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits)
-#  define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret)
-#  define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed)
-
-#  define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState)
-#  define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState)
-#  define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState)
-
-#  define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset)
-#  define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed)
-#  define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret)
-#  define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update)
-#  define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest)
-#endif
-
-/* XXH3_64bits() :
+/* XXH3_64bits():
  * default 64-bit variant, using default secret and default seed of 0.
  * It's the fastest variant. */
 XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* data, size_t len);
 
-/* XXH3_64bits_withSecret() :
+/*
+ * XXH3_64bits_withSeed():
+ * This variant generates a custom secret on the fly
+ * based on default secret altered using the `seed` value.
+ * While this operation is decently fast, note that it's not completely free.
+ * Note: seed==0 produces the same results as XXH3_64bits().
+ */
+XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void* data, size_t len, XXH64_hash_t seed);
+
+/*
+ * XXH3_64bits_withSecret():
  * It's possible to provide any blob of bytes as a "secret" to generate the hash.
  * This makes it more difficult for an external actor to prepare an intentional collision.
- * The secret *must* be large enough (>= XXH3_SECRET_SIZE_MIN).
- * It should consist of random bytes.
- * Avoid repeating same character, or sequences of bytes,
- * and especially avoid swathes of \0.
- * Failure to respect these conditions will result in a poor quality hash.
+ * The main condition is that secretSize *must* be large enough (>= XXH3_SECRET_SIZE_MIN).
+ * However, the quality of produced hash values depends on secret's entropy.
+ * Technically, the secret must look like a bunch of random bytes.
+ * Avoid "trivial" or structured data such as repeated sequences or a text document.
+ * Whenever unsure about the "randomness" of the blob of bytes,
+ * consider relabelling it as a "custom seed" instead,
+ * and employ "XXH3_generateSecret()" (see below)
+ * to generate a high entropy secret derived from the custom seed.
  */
 #define XXH3_SECRET_SIZE_MIN 136
 XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize);
 
-/* XXH3_64bits_withSeed() :
- * This variant generates on the fly a custom secret,
- * based on the default secret, altered using the `seed` value.
- * While this operation is decently fast, note that it's not completely free.
- * note : seed==0 produces same results as XXH3_64bits() */
-XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void* data, size_t len, XXH64_hash_t seed);
-
-
-/* streaming 64-bit */
-
-#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)   /* C11+ */
-#  include <stdalign.h>
-#  define XXH_ALIGN(n)      alignas(n)
-#elif defined(__GNUC__)
-#  define XXH_ALIGN(n)      __attribute__ ((aligned(n)))
-#elif defined(_MSC_VER)
-#  define XXH_ALIGN(n)      __declspec(align(n))
-#else
-#  define XXH_ALIGN(n)   /* disabled */
-#endif
-
-typedef struct XXH3_state_s XXH3_state_t;
-
-#define XXH3_SECRET_DEFAULT_SIZE 192   /* minimum XXH3_SECRET_SIZE_MIN */
-#define XXH3_INTERNALBUFFER_SIZE 256
-struct XXH3_state_s {
-   XXH_ALIGN(64) XXH64_hash_t acc[8];
-   XXH_ALIGN(64) char customSecret[XXH3_SECRET_DEFAULT_SIZE];  /* used to store a custom secret generated from the seed. Makes state larger. Design might change */
-   XXH_ALIGN(64) char buffer[XXH3_INTERNALBUFFER_SIZE];
-   const void* secret;
-   XXH32_hash_t bufferedSize;
-   XXH32_hash_t nbStripesPerBlock;
-   XXH32_hash_t nbStripesSoFar;
-   XXH32_hash_t reserved32;
-   XXH32_hash_t reserved32_2;
-   XXH32_hash_t secretLimit;
-   XXH64_hash_t totalLen;
-   XXH64_hash_t seed;
-   XXH64_hash_t reserved64;
-};   /* typedef'd to XXH3_state_t */
 
-/* Streaming requires state maintenance.
- * This operation costs memory and cpu.
+/*******   Streaming   *******/
+/*
+ * Streaming requires state maintenance.
+ * This operation costs memory and CPU.
  * As a consequence, streaming is slower than one-shot hashing.
- * For better performance, prefer using one-shot functions whenever possible. */
-
+ * For better performance, prefer one-shot functions whenever applicable.
+ */
+typedef struct XXH3_state_s XXH3_state_t;
 XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void);
 XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr);
 XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state);
 
-
-/* XXH3_64bits_reset() :
- * initialize with default parameters.
- * result will be equivalent to `XXH3_64bits()`. */
+/*
+ * XXH3_64bits_reset():
+ * Initialize with default parameters.
+ * digest will be equivalent to `XXH3_64bits()`.
+ */
 XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t* statePtr);
-/* XXH3_64bits_reset_withSeed() :
- * generate a custom secret from `seed`, and store it into state.
- * digest will be equivalent to `XXH3_64bits_withSeed()`. */
+/*
+ * XXH3_64bits_reset_withSeed():
+ * Generate a custom secret from `seed`, and store it into `statePtr`.
+ * digest will be equivalent to `XXH3_64bits_withSeed()`.
+ */
 XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed);
-/* XXH3_64bits_reset_withSecret() :
- * `secret` is referenced, and must outlive the hash streaming session.
- * secretSize must be >= XXH3_SECRET_SIZE_MIN.
+/*
+ * XXH3_64bits_reset_withSecret():
+ * `secret` is referenced, it _must outlive_ the hash streaming session.
+ * Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`,
+ * and the quality of produced hash values depends on secret's entropy
+ * (secret's content should look like a bunch of random bytes).
+ * When in doubt about the randomness of a candidate `secret`,
+ * consider employing `XXH3_generateSecret()` instead (see below).
  */
 XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize);
 
 XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH3_state_t* statePtr, const void* input, size_t length);
 XXH_PUBLIC_API XXH64_hash_t  XXH3_64bits_digest (const XXH3_state_t* statePtr);
 
+/* note : canonical representation of XXH3 is the same as XXH64
+ * since they both produce XXH64_hash_t values */
 
-/* 128-bit */
-
-#ifdef XXH_NAMESPACE
-#  define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128)
-#  define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits)
-#  define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed)
-#  define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret)
-
-#  define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset)
-#  define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed)
-#  define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret)
-#  define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update)
-#  define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest)
 
-#  define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual)
-#  define XXH128_cmp     XXH_NAME2(XXH_NAMESPACE, XXH128_cmp)
-#  define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash)
-#  define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical)
-#endif
+/*-**********************************************************************
+*  XXH3 128-bit variant
+************************************************************************/
 
 typedef struct {
-    XXH64_hash_t low64;
-    XXH64_hash_t high64;
+ XXH64_hash_t low64;
+ XXH64_hash_t high64;
 } XXH128_hash_t;
 
-XXH_PUBLIC_API XXH128_hash_t XXH128(const void* data, size_t len, XXH64_hash_t seed);
 XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* data, size_t len);
-XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void* data, size_t len, XXH64_hash_t seed);  /* == XXH128() */
+XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void* data, size_t len, XXH64_hash_t seed);
 XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize);
 
+/*******   Streaming   *******/
+/*
+ * Streaming requires state maintenance.
+ * This operation costs memory and CPU.
+ * As a consequence, streaming is slower than one-shot hashing.
+ * For better performance, prefer one-shot functions whenever applicable.
+ *
+ * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits().
+ * Use already declared XXH3_createState() and XXH3_freeState().
+ *
+ * All reset and streaming functions have same meaning as their 64-bit counterpart.
+ */
+
 XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t* statePtr);
 XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed);
 XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize);
@@ -502,43 +562,4205 @@ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH3_state_t* statePt
 XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH3_state_t* statePtr, const void* input, size_t length);
 XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* statePtr);
 
+/* Following helper functions make it possible to compare XXH128_hast_t values.
+ * Since XXH128_hash_t is a structure, this capability is not offered by the language.
+ * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */
 
-/* Note : for better performance, following functions should be inlined,
- * using XXH_INLINE_ALL */
-
-/* return : 1 is equal, 0 if different */
+/*!
+ * XXH128_isEqual():
+ * Return: 1 if `h1` and `h2` are equal, 0 if they are not.
+ */
 XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2);
 
-/* This comparator is compatible with stdlib's qsort().
- * return : >0 if *h128_1  > *h128_2
- *          <0 if *h128_1  < *h128_2
- *          =0 if *h128_1 == *h128_2  */
+/*!
+ * XXH128_cmp():
+ *
+ * This comparator is compatible with stdlib's `qsort()`/`bsearch()`.
+ *
+ * return: >0 if *h128_1  > *h128_2
+ *         =0 if *h128_1 == *h128_2
+ *         <0 if *h128_1  < *h128_2
+ */
 XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2);
 
 
-/*======   Canonical representation   ======*/
-typedef struct { unsigned char digest[16]; } XXH128_canonical_t;
+/*******   Canonical representation   *******/
+typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t;
 XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash);
 XXH_PUBLIC_API XXH128_hash_t XXH128_hashFromCanonical(const XXH128_canonical_t* src);
 
 
 #endif  /* XXH_NO_LONG_LONG */
 
+#endif /* XXHASH_H_5627135585666179 */
 
-/*-**********************************************************************
-*  XXH_INLINE_ALL
-************************************************************************/
-#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
-#  include "xxhash.c"   /* include xxhash function bodies as `static`, for inlining */
-#endif
 
 
+#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742)
+#define XXHASH_H_STATIC_13879238742
+/* ****************************************************************************
+ * This section contains declarations which are not guaranteed to remain stable.
+ * They may change in future versions, becoming incompatible with a different
+ * version of the library.
+ * These declarations should only be used with static linking.
+ * Never use them in association with dynamic linking!
+ ***************************************************************************** */
+
+/*
+ * These definitions are only present to allow static allocation
+ * of XXH states, on stack or in a struct, for example.
+ * Never **ever** access their members directly.
+ */
 
-#endif /* XXH_STATIC_LINKING_ONLY */
+struct XXH32_state_s {
+   XXH32_hash_t total_len_32;
+   XXH32_hash_t large_len;
+   XXH32_hash_t v1;
+   XXH32_hash_t v2;
+   XXH32_hash_t v3;
+   XXH32_hash_t v4;
+   XXH32_hash_t mem32[4];
+   XXH32_hash_t memsize;
+   XXH32_hash_t reserved;   /* never read nor write, might be removed in a future version */
+};   /* typedef'd to XXH32_state_t */
 
 
-#if defined (__cplusplus)
-}
-#endif
+#ifndef XXH_NO_LONG_LONG  /* defined when there is no 64-bit support */
 
-#endif /* XXHASH_H_5627135585666179 */
+struct XXH64_state_s {
+   XXH64_hash_t total_len;
+   XXH64_hash_t v1;
+   XXH64_hash_t v2;
+   XXH64_hash_t v3;
+   XXH64_hash_t v4;
+   XXH64_hash_t mem64[4];
+   XXH32_hash_t memsize;
+   XXH32_hash_t reserved32;  /* required for padding anyway */
+   XXH64_hash_t reserved64;  /* never read nor write, might be removed in a future version */
+};   /* typedef'd to XXH64_state_t */
+
+#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)   /* C11+ */
+#  include <stdalign.h>
+#  define XXH_ALIGN(n)      alignas(n)
+#elif defined(__GNUC__)
+#  define XXH_ALIGN(n)      __attribute__ ((aligned(n)))
+#elif defined(_MSC_VER)
+#  define XXH_ALIGN(n)      __declspec(align(n))
+#else
+#  define XXH_ALIGN(n)   /* disabled */
+#endif
+
+/* Old GCC versions only accept the attribute after the type in structures. */
+#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L))   /* C11+ */ \
+    && defined(__GNUC__)
+#   define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align)
+#else
+#   define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type
+#endif
+
+#define XXH3_INTERNALBUFFER_SIZE 256
+#define XXH3_SECRET_DEFAULT_SIZE 192
+struct XXH3_state_s {
+   XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]);
+   /* used to store a custom secret generated from a seed */
+   XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]);
+   XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]);
+   XXH32_hash_t bufferedSize;
+   XXH32_hash_t reserved32;
+   size_t nbStripesSoFar;
+   XXH64_hash_t totalLen;
+   size_t nbStripesPerBlock;
+   size_t secretLimit;
+   XXH64_hash_t seed;
+   XXH64_hash_t reserved64;
+   const unsigned char* extSecret;  /* reference to external secret;
+                                     * if == NULL, use .customSecret instead */
+   /* note: there may be some padding at the end due to alignment on 64 bytes */
+}; /* typedef'd to XXH3_state_t */
+
+#undef XXH_ALIGN_MEMBER
+
+/* When the XXH3_state_t structure is merely emplaced on stack,
+ * it should be initialized with XXH3_INITSTATE() or a memset()
+ * in case its first reset uses XXH3_NNbits_reset_withSeed().
+ * This init can be omitted if the first reset uses default or _withSecret mode.
+ * This operation isn't necessary when the state is created with XXH3_createState().
+ * Note that this doesn't prepare the state for a streaming operation,
+ * it's still necessary to use XXH3_NNbits_reset*() afterwards.
+ */
+#define XXH3_INITSTATE(XXH3_state_ptr)   { (XXH3_state_ptr)->seed = 0; }
+
+
+/* ===   Experimental API   === */
+/* Symbols defined below must be considered tied to a specific library version. */
+
+/*
+ * XXH3_generateSecret():
+ *
+ * Derive a high-entropy secret from any user-defined content, named customSeed.
+ * The generated secret can be used in combination with `*_withSecret()` functions.
+ * The `_withSecret()` variants are useful to provide a higher level of protection than 64-bit seed,
+ * as it becomes much more difficult for an external actor to guess how to impact the calculation logic.
+ *
+ * The function accepts as input a custom seed of any length and any content,
+ * and derives from it a high-entropy secret of length XXH3_SECRET_DEFAULT_SIZE
+ * into an already allocated buffer secretBuffer.
+ * The generated secret is _always_ XXH_SECRET_DEFAULT_SIZE bytes long.
+ *
+ * The generated secret can then be used with any `*_withSecret()` variant.
+ * Functions `XXH3_128bits_withSecret()`, `XXH3_64bits_withSecret()`,
+ * `XXH3_128bits_reset_withSecret()` and `XXH3_64bits_reset_withSecret()`
+ * are part of this list. They all accept a `secret` parameter
+ * which must be very long for implementation reasons (>= XXH3_SECRET_SIZE_MIN)
+ * _and_ feature very high entropy (consist of random-looking bytes).
+ * These conditions can be a high bar to meet, so
+ * this function can be used to generate a secret of proper quality.
+ *
+ * customSeed can be anything. It can have any size, even small ones,
+ * and its content can be anything, even stupidly "low entropy" source such as a bunch of zeroes.
+ * The resulting `secret` will nonetheless provide all expected qualities.
+ *
+ * Supplying NULL as the customSeed copies the default secret into `secretBuffer`.
+ * When customSeedSize > 0, supplying NULL as customSeed is undefined behavior.
+ */
+XXH_PUBLIC_API void XXH3_generateSecret(void* secretBuffer, const void* customSeed, size_t customSeedSize);
+
+
+/* simple short-cut to pre-selected XXH3_128bits variant */
+XXH_PUBLIC_API XXH128_hash_t XXH128(const void* data, size_t len, XXH64_hash_t seed);
+
+
+#endif  /* XXH_NO_LONG_LONG */
+
+
+#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
+#  define XXH_IMPLEMENTATION
+#endif
+
+#endif  /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */
+
+
+/* ======================================================================== */
+/* ======================================================================== */
+/* ======================================================================== */
+
+
+/*-**********************************************************************
+ * xxHash implementation
+ *-**********************************************************************
+ * xxHash's implementation used to be hosted inside xxhash.c.
+ *
+ * However, inlining requires implementation to be visible to the compiler,
+ * hence be included alongside the header.
+ * Previously, implementation was hosted inside xxhash.c,
+ * which was then #included when inlining was activated.
+ * This construction created issues with a few build and install systems,
+ * as it required xxhash.c to be stored in /include directory.
+ *
+ * xxHash implementation is now directly integrated within xxhash.h.
+ * As a consequence, xxhash.c is no longer needed in /include.
+ *
+ * xxhash.c is still available and is still useful.
+ * In a "normal" setup, when xxhash is not inlined,
+ * xxhash.h only exposes the prototypes and public symbols,
+ * while xxhash.c can be built into an object file xxhash.o
+ * which can then be linked into the final binary.
+ ************************************************************************/
+
+#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \
+   || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387)
+#  define XXH_IMPLEM_13a8737387
+
+/* *************************************
+*  Tuning parameters
+***************************************/
+/*!
+ * XXH_FORCE_MEMORY_ACCESS:
+ * By default, access to unaligned memory is controlled by `memcpy()`, which is
+ * safe and portable.
+ *
+ * Unfortunately, on some target/compiler combinations, the generated assembly
+ * is sub-optimal.
+ *
+ * The below switch allow selection of a different access method
+ * in the search for improved performance.
+ * Method 0 (default):
+ *     Use `memcpy()`. Safe and portable. Default.
+ * Method 1:
+ *     `__attribute__((packed))` statement. It depends on compiler extensions
+ *     and is therefore not portable.
+ *     This method is safe if your compiler supports it, and *generally* as
+ *     fast or faster than `memcpy`.
+ * Method 2:
+ *     Direct access via cast. This method doesn't depend on the compiler but
+ *     violates the C standard.
+ *     It can generate buggy code on targets which do not support unaligned
+ *     memory accesses.
+ *     But in some circumstances, it's the only known way to get the most
+ *     performance (example: GCC + ARMv6)
+ * Method 3:
+ *     Byteshift. This can generate the best code on old compilers which don't
+ *     inline small `memcpy()` calls, and it might also be faster on big-endian
+ *     systems which lack a native byteswap instruction.
+ * See https://stackoverflow.com/a/32095106/646947 for details.
+ * Prefer these methods in priority order (0 > 1 > 2 > 3)
+ */
+#ifndef XXH_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
+#  if !defined(__clang__) && defined(__GNUC__) && defined(__ARM_FEATURE_UNALIGNED) && defined(__ARM_ARCH) && (__ARM_ARCH == 6)
+#    define XXH_FORCE_MEMORY_ACCESS 2
+#  elif !defined(__clang__) && ((defined(__INTEL_COMPILER) && !defined(_WIN32)) || \
+  (defined(__GNUC__) && (defined(__ARM_ARCH) && __ARM_ARCH >= 7)))
+#    define XXH_FORCE_MEMORY_ACCESS 1
+#  endif
+#endif
+
+/*!
+ * XXH_ACCEPT_NULL_INPUT_POINTER:
+ * If the input pointer is NULL, xxHash's default behavior is to dereference it,
+ * triggering a segfault.
+ * When this macro is enabled, xxHash actively checks the input for a null pointer.
+ * If it is, the result for null input pointers is the same as a zero-length input.
+ */
+#ifndef XXH_ACCEPT_NULL_INPUT_POINTER   /* can be defined externally */
+#  define XXH_ACCEPT_NULL_INPUT_POINTER 0
+#endif
+
+/*!
+ * XXH_FORCE_ALIGN_CHECK:
+ * This is an important performance trick
+ * for architectures without decent unaligned memory access performance.
+ * It checks for input alignment, and when conditions are met,
+ * uses a "fast path" employing direct 32-bit/64-bit read,
+ * resulting in _dramatically faster_ read speed.
+ *
+ * The check costs one initial branch per hash, which is generally negligible, but not zero.
+ * Moreover, it's not useful to generate binary for an additional code path
+ * if memory access uses same instruction for both aligned and unaligned adresses.
+ *
+ * In these cases, the alignment check can be removed by setting this macro to 0.
+ * Then the code will always use unaligned memory access.
+ * Align check is automatically disabled on x86, x64 & arm64,
+ * which are platforms known to offer good unaligned memory accesses performance.
+ *
+ * This option does not affect XXH3 (only XXH32 and XXH64).
+ */
+#ifndef XXH_FORCE_ALIGN_CHECK  /* can be defined externally */
+#  if defined(__i386)  || defined(__x86_64__) || defined(__aarch64__) \
+   || defined(_M_IX86) || defined(_M_X64)     || defined(_M_ARM64) /* visual */
+#    define XXH_FORCE_ALIGN_CHECK 0
+#  else
+#    define XXH_FORCE_ALIGN_CHECK 1
+#  endif
+#endif
+
+/*!
+ * XXH_NO_INLINE_HINTS:
+ *
+ * By default, xxHash tries to force the compiler to inline almost all internal
+ * functions.
+ *
+ * This can usually improve performance due to reduced jumping and improved
+ * constant folding, but significantly increases the size of the binary which
+ * might not be favorable.
+ *
+ * Additionally, sometimes the forced inlining can be detrimental to performance,
+ * depending on the architecture.
+ *
+ * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the
+ * compiler full control on whether to inline or not.
+ *
+ * When not optimizing (-O0), optimizing for size (-Os, -Oz), or using
+ * -fno-inline with GCC or Clang, this will automatically be defined.
+ */
+#ifndef XXH_NO_INLINE_HINTS
+#  if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ \
+   || defined(__NO_INLINE__)     /* -O0, -fno-inline */
+#    define XXH_NO_INLINE_HINTS 1
+#  else
+#    define XXH_NO_INLINE_HINTS 0
+#  endif
+#endif
+
+/*!
+ * XXH_REROLL:
+ * Whether to reroll XXH32_finalize, and XXH64_finalize,
+ * instead of using an unrolled jump table/if statement loop.
+ *
+ * This is automatically defined on -Os/-Oz on GCC and Clang.
+ */
+#ifndef XXH_REROLL
+#  if defined(__OPTIMIZE_SIZE__)
+#    define XXH_REROLL 1
+#  else
+#    define XXH_REROLL 0
+#  endif
+#endif
+
+
+/* *************************************
+*  Includes & Memory related functions
+***************************************/
+/*!
+ * Modify the local functions below should you wish to use
+ * different memory routines for malloc() and free()
+ */
+#include <stdlib.h>
+
+static void* XXH_malloc(size_t s) { return malloc(s); }
+static void XXH_free(void* p) { free(p); }
+
+/*! and for memcpy() */
+#include <string.h>
+static void* XXH_memcpy(void* dest, const void* src, size_t size)
+{
+    return memcpy(dest,src,size);
+}
+
+#include <limits.h>   /* ULLONG_MAX */
+
+
+/* *************************************
+*  Compiler Specific Options
+***************************************/
+#ifdef _MSC_VER /* Visual Studio warning fix */
+#  pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+#endif
+
+#if XXH_NO_INLINE_HINTS  /* disable inlining hints */
+#  if defined(__GNUC__)
+#    define XXH_FORCE_INLINE static __attribute__((unused))
+#  else
+#    define XXH_FORCE_INLINE static
+#  endif
+#  define XXH_NO_INLINE static
+/* enable inlining hints */
+#elif defined(_MSC_VER)  /* Visual Studio */
+#  define XXH_FORCE_INLINE static __forceinline
+#  define XXH_NO_INLINE static __declspec(noinline)
+#elif defined(__GNUC__)
+#  define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused))
+#  define XXH_NO_INLINE static __attribute__((noinline))
+#elif defined (__cplusplus) \
+  || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L))   /* C99 */
+#  define XXH_FORCE_INLINE static inline
+#  define XXH_NO_INLINE static
+#else
+#  define XXH_FORCE_INLINE static
+#  define XXH_NO_INLINE static
+#endif
+
+
+
+/* *************************************
+*  Debug
+***************************************/
+/*
+ * XXH_DEBUGLEVEL is expected to be defined externally, typically via the
+ * compiler's command line options. The value must be a number.
+ */
+#ifndef XXH_DEBUGLEVEL
+#  ifdef DEBUGLEVEL /* backwards compat */
+#    define XXH_DEBUGLEVEL DEBUGLEVEL
+#  else
+#    define XXH_DEBUGLEVEL 0
+#  endif
+#endif
+
+#if (XXH_DEBUGLEVEL>=1)
+#  include <assert.h>   /* note: can still be disabled with NDEBUG */
+#  define XXH_ASSERT(c)   assert(c)
+#else
+#  define XXH_ASSERT(c)   ((void)0)
+#endif
+
+/* note: use after variable declarations */
+#define XXH_STATIC_ASSERT(c)  do { enum { XXH_sa = 1/(int)(!!(c)) }; } while (0)
+
+
+/* *************************************
+*  Basic Types
+***************************************/
+#if !defined (__VMS) \
+ && (defined (__cplusplus) \
+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+# include <stdint.h>
+  typedef uint8_t xxh_u8;
+#else
+  typedef unsigned char xxh_u8;
+#endif
+typedef XXH32_hash_t xxh_u32;
+
+#ifdef XXH_OLD_NAMES
+#  define BYTE xxh_u8
+#  define U8   xxh_u8
+#  define U32  xxh_u32
+#endif
+
+/* ***   Memory access   *** */
+
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
+/*
+ * Manual byteshift. Best for old compilers which don't inline memcpy.
+ * We actually directly use XXH_readLE32 and XXH_readBE32.
+ */
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
+
+/*
+ * Force direct memory access. Only works on CPU which support unaligned memory
+ * access in hardware.
+ */
+static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; }
+
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
+
+/*
+ * __pack instructions are safer but compiler specific, hence potentially
+ * problematic for some compilers.
+ *
+ * Currently only defined for GCC and ICC.
+ */
+#ifdef XXH_OLD_NAMES
+typedef union { xxh_u32 u32; } __attribute__((packed)) unalign;
+#endif
+static xxh_u32 XXH_read32(const void* ptr)
+{
+    typedef union { xxh_u32 u32; } __attribute__((packed)) xxh_unalign;
+    return ((const xxh_unalign*)ptr)->u32;
+}
+
+#else
+
+/*
+ * Portable and safe solution. Generally efficient.
+ * see: https://stackoverflow.com/a/32095106/646947
+ */
+static xxh_u32 XXH_read32(const void* memPtr)
+{
+    xxh_u32 val;
+    memcpy(&val, memPtr, sizeof(val));
+    return val;
+}
+
+#endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
+
+
+/* ***   Endianess   *** */
+typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
+
+/*!
+ * XXH_CPU_LITTLE_ENDIAN:
+ * Defined to 1 if the target is little endian, or 0 if it is big endian.
+ * It can be defined externally, for example on the compiler command line.
+ *
+ * If it is not defined, a runtime check (which is usually constant folded)
+ * is used instead.
+ */
+#ifndef XXH_CPU_LITTLE_ENDIAN
+/*
+ * Try to detect endianness automatically, to avoid the nonstandard behavior
+ * in `XXH_isLittleEndian()`
+ */
+#  if defined(_WIN32) /* Windows is always little endian */ \
+     || defined(__LITTLE_ENDIAN__) \
+     || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+#    define XXH_CPU_LITTLE_ENDIAN 1
+#  elif defined(__BIG_ENDIAN__) \
+     || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+#    define XXH_CPU_LITTLE_ENDIAN 0
+#  else
+/*
+ * runtime test, presumed to simplify to a constant by compiler
+ */
+static int XXH_isLittleEndian(void)
+{
+    /*
+     * Portable and well-defined behavior.
+     * Don't use static: it is detrimental to performance.
+     */
+    const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 };
+    return one.c[0];
+}
+#   define XXH_CPU_LITTLE_ENDIAN   XXH_isLittleEndian()
+#  endif
+#endif
+
+
+
+
+/* ****************************************
+*  Compiler-specific Functions and Macros
+******************************************/
+#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+
+#ifdef __has_builtin
+#  define XXH_HAS_BUILTIN(x) __has_builtin(x)
+#else
+#  define XXH_HAS_BUILTIN(x) 0
+#endif
+
+#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \
+                               && XXH_HAS_BUILTIN(__builtin_rotateleft64)
+#  define XXH_rotl32 __builtin_rotateleft32
+#  define XXH_rotl64 __builtin_rotateleft64
+/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */
+#elif defined(_MSC_VER)
+#  define XXH_rotl32(x,r) _rotl(x,r)
+#  define XXH_rotl64(x,r) _rotl64(x,r)
+#else
+#  define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r))))
+#  define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r))))
+#endif
+
+#if defined(_MSC_VER)     /* Visual Studio */
+#  define XXH_swap32 _byteswap_ulong
+#elif XXH_GCC_VERSION >= 403
+#  define XXH_swap32 __builtin_bswap32
+#else
+static xxh_u32 XXH_swap32 (xxh_u32 x)
+{
+    return  ((x << 24) & 0xff000000 ) |
+            ((x <<  8) & 0x00ff0000 ) |
+            ((x >>  8) & 0x0000ff00 ) |
+            ((x >> 24) & 0x000000ff );
+}
+#endif
+
+
+/* ***************************
+*  Memory reads
+*****************************/
+typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
+
+/*
+ * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load.
+ *
+ * This is ideal for older compilers which don't inline memcpy.
+ */
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
+
+XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr)
+{
+    const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
+    return bytePtr[0]
+         | ((xxh_u32)bytePtr[1] << 8)
+         | ((xxh_u32)bytePtr[2] << 16)
+         | ((xxh_u32)bytePtr[3] << 24);
+}
+
+XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr)
+{
+    const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
+    return bytePtr[3]
+         | ((xxh_u32)bytePtr[2] << 8)
+         | ((xxh_u32)bytePtr[1] << 16)
+         | ((xxh_u32)bytePtr[0] << 24);
+}
+
+#else
+XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr)
+{
+    return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
+}
+
+static xxh_u32 XXH_readBE32(const void* ptr)
+{
+    return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
+}
+#endif
+
+XXH_FORCE_INLINE xxh_u32
+XXH_readLE32_align(const void* ptr, XXH_alignment align)
+{
+    if (align==XXH_unaligned) {
+        return XXH_readLE32(ptr);
+    } else {
+        return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr);
+    }
+}
+
+
+/* *************************************
+*  Misc
+***************************************/
+XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
+
+
+/* *******************************************************************
+*  32-bit hash functions
+*********************************************************************/
+static const xxh_u32 XXH_PRIME32_1 = 0x9E3779B1U;   /* 0b10011110001101110111100110110001 */
+static const xxh_u32 XXH_PRIME32_2 = 0x85EBCA77U;   /* 0b10000101111010111100101001110111 */
+static const xxh_u32 XXH_PRIME32_3 = 0xC2B2AE3DU;   /* 0b11000010101100101010111000111101 */
+static const xxh_u32 XXH_PRIME32_4 = 0x27D4EB2FU;   /* 0b00100111110101001110101100101111 */
+static const xxh_u32 XXH_PRIME32_5 = 0x165667B1U;   /* 0b00010110010101100110011110110001 */
+
+#ifdef XXH_OLD_NAMES
+#  define PRIME32_1 XXH_PRIME32_1
+#  define PRIME32_2 XXH_PRIME32_2
+#  define PRIME32_3 XXH_PRIME32_3
+#  define PRIME32_4 XXH_PRIME32_4
+#  define PRIME32_5 XXH_PRIME32_5
+#endif
+
+static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input)
+{
+    acc += input * XXH_PRIME32_2;
+    acc  = XXH_rotl32(acc, 13);
+    acc *= XXH_PRIME32_1;
+#if defined(__GNUC__) && defined(__SSE4_1__) && !defined(XXH_ENABLE_AUTOVECTORIZE)
+    /*
+     * UGLY HACK:
+     * This inline assembly hack forces acc into a normal register. This is the
+     * only thing that prevents GCC and Clang from autovectorizing the XXH32
+     * loop (pragmas and attributes don't work for some resason) without globally
+     * disabling SSE4.1.
+     *
+     * The reason we want to avoid vectorization is because despite working on
+     * 4 integers at a time, there are multiple factors slowing XXH32 down on
+     * SSE4:
+     * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on
+     *   newer chips!) making it slightly slower to multiply four integers at
+     *   once compared to four integers independently. Even when pmulld was
+     *   fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE
+     *   just to multiply unless doing a long operation.
+     *
+     * - Four instructions are required to rotate,
+     *      movqda tmp,  v // not required with VEX encoding
+     *      pslld  tmp, 13 // tmp <<= 13
+     *      psrld  v,   19 // x >>= 19
+     *      por    v,  tmp // x |= tmp
+     *   compared to one for scalar:
+     *      roll   v, 13    // reliably fast across the board
+     *      shldl  v, v, 13 // Sandy Bridge and later prefer this for some reason
+     *
+     * - Instruction level parallelism is actually more beneficial here because
+     *   the SIMD actually serializes this operation: While v1 is rotating, v2
+     *   can load data, while v3 can multiply. SSE forces them to operate
+     *   together.
+     *
+     * How this hack works:
+     * __asm__(""       // Declare an assembly block but don't declare any instructions
+     *          :       // However, as an Input/Output Operand,
+     *          "+r"    // constrain a read/write operand (+) as a general purpose register (r).
+     *          (acc)   // and set acc as the operand
+     * );
+     *
+     * Because of the 'r', the compiler has promised that seed will be in a
+     * general purpose register and the '+' says that it will be 'read/write',
+     * so it has to assume it has changed. It is like volatile without all the
+     * loads and stores.
+     *
+     * Since the argument has to be in a normal register (not an SSE register),
+     * each time XXH32_round is called, it is impossible to vectorize.
+     */
+    __asm__("" : "+r" (acc));
+#endif
+    return acc;
+}
+
+/* mix all bits */
+static xxh_u32 XXH32_avalanche(xxh_u32 h32)
+{
+    h32 ^= h32 >> 15;
+    h32 *= XXH_PRIME32_2;
+    h32 ^= h32 >> 13;
+    h32 *= XXH_PRIME32_3;
+    h32 ^= h32 >> 16;
+    return(h32);
+}
+
+#define XXH_get32bits(p) XXH_readLE32_align(p, align)
+
+static xxh_u32
+XXH32_finalize(xxh_u32 h32, const xxh_u8* ptr, size_t len, XXH_alignment align)
+{
+#define XXH_PROCESS1 do {                           \
+    h32 += (*ptr++) * XXH_PRIME32_5;                \
+    h32 = XXH_rotl32(h32, 11) * XXH_PRIME32_1;      \
+} while (0)
+
+#define XXH_PROCESS4 do {                           \
+    h32 += XXH_get32bits(ptr) * XXH_PRIME32_3;      \
+    ptr += 4;                                   \
+    h32  = XXH_rotl32(h32, 17) * XXH_PRIME32_4;     \
+} while (0)
+
+    /* Compact rerolled version */
+    if (XXH_REROLL) {
+        len &= 15;
+        while (len >= 4) {
+            XXH_PROCESS4;
+            len -= 4;
+        }
+        while (len > 0) {
+            XXH_PROCESS1;
+            --len;
+        }
+        return XXH32_avalanche(h32);
+    } else {
+         switch(len&15) /* or switch(bEnd - p) */ {
+           case 12:      XXH_PROCESS4;
+                         /* fallthrough */
+           case 8:       XXH_PROCESS4;
+                         /* fallthrough */
+           case 4:       XXH_PROCESS4;
+                         return XXH32_avalanche(h32);
+
+           case 13:      XXH_PROCESS4;
+                         /* fallthrough */
+           case 9:       XXH_PROCESS4;
+                         /* fallthrough */
+           case 5:       XXH_PROCESS4;
+                         XXH_PROCESS1;
+                         return XXH32_avalanche(h32);
+
+           case 14:      XXH_PROCESS4;
+                         /* fallthrough */
+           case 10:      XXH_PROCESS4;
+                         /* fallthrough */
+           case 6:       XXH_PROCESS4;
+                         XXH_PROCESS1;
+                         XXH_PROCESS1;
+                         return XXH32_avalanche(h32);
+
+           case 15:      XXH_PROCESS4;
+                         /* fallthrough */
+           case 11:      XXH_PROCESS4;
+                         /* fallthrough */
+           case 7:       XXH_PROCESS4;
+                         /* fallthrough */
+           case 3:       XXH_PROCESS1;
+                         /* fallthrough */
+           case 2:       XXH_PROCESS1;
+                         /* fallthrough */
+           case 1:       XXH_PROCESS1;
+                         /* fallthrough */
+           case 0:       return XXH32_avalanche(h32);
+        }
+        XXH_ASSERT(0);
+        return h32;   /* reaching this point is deemed impossible */
+    }
+}
+
+#ifdef XXH_OLD_NAMES
+#  define PROCESS1 XXH_PROCESS1
+#  define PROCESS4 XXH_PROCESS4
+#else
+#  undef XXH_PROCESS1
+#  undef XXH_PROCESS4
+#endif
+
+XXH_FORCE_INLINE xxh_u32
+XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align)
+{
+    const xxh_u8* bEnd = input + len;
+    xxh_u32 h32;
+
+#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
+    if (input==NULL) {
+        len=0;
+        bEnd=input=(const xxh_u8*)(size_t)16;
+    }
+#endif
+
+    if (len>=16) {
+        const xxh_u8* const limit = bEnd - 15;
+        xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
+        xxh_u32 v2 = seed + XXH_PRIME32_2;
+        xxh_u32 v3 = seed + 0;
+        xxh_u32 v4 = seed - XXH_PRIME32_1;
+
+        do {
+            v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4;
+            v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4;
+            v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4;
+            v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4;
+        } while (input < limit);
+
+        h32 = XXH_rotl32(v1, 1)  + XXH_rotl32(v2, 7)
+            + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
+    } else {
+        h32  = seed + XXH_PRIME32_5;
+    }
+
+    h32 += (xxh_u32)len;
+
+    return XXH32_finalize(h32, input, len&15, align);
+}
+
+
+XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed)
+{
+#if 0
+    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+    XXH32_state_t state;
+    XXH32_reset(&state, seed);
+    XXH32_update(&state, (const xxh_u8*)input, len);
+    return XXH32_digest(&state);
+
+#else
+
+    if (XXH_FORCE_ALIGN_CHECK) {
+        if ((((size_t)input) & 3) == 0) {   /* Input is 4-bytes aligned, leverage the speed benefit */
+            return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
+    }   }
+
+    return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
+#endif
+}
+
+
+
+/*******   Hash streaming   *******/
+
+XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
+{
+    return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
+}
+XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
+{
+    XXH_free(statePtr);
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState)
+{
+    memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed)
+{
+    XXH32_state_t state;   /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
+    memset(&state, 0, sizeof(state));
+    state.v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
+    state.v2 = seed + XXH_PRIME32_2;
+    state.v3 = seed + 0;
+    state.v4 = seed - XXH_PRIME32_1;
+    /* do not write into reserved, planned to be removed in a future version */
+    memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved));
+    return XXH_OK;
+}
+
+
+XXH_PUBLIC_API XXH_errorcode
+XXH32_update(XXH32_state_t* state, const void* input, size_t len)
+{
+    if (input==NULL)
+#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
+        return XXH_OK;
+#else
+        return XXH_ERROR;
+#endif
+
+    {   const xxh_u8* p = (const xxh_u8*)input;
+        const xxh_u8* const bEnd = p + len;
+
+        state->total_len_32 += (XXH32_hash_t)len;
+        state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16));
+
+        if (state->memsize + len < 16)  {   /* fill in tmp buffer */
+            XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len);
+            state->memsize += (XXH32_hash_t)len;
+            return XXH_OK;
+        }
+
+        if (state->memsize) {   /* some data left from previous update */
+            XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize);
+            {   const xxh_u32* p32 = state->mem32;
+                state->v1 = XXH32_round(state->v1, XXH_readLE32(p32)); p32++;
+                state->v2 = XXH32_round(state->v2, XXH_readLE32(p32)); p32++;
+                state->v3 = XXH32_round(state->v3, XXH_readLE32(p32)); p32++;
+                state->v4 = XXH32_round(state->v4, XXH_readLE32(p32));
+            }
+            p += 16-state->memsize;
+            state->memsize = 0;
+        }
+
+        if (p <= bEnd-16) {
+            const xxh_u8* const limit = bEnd - 16;
+            xxh_u32 v1 = state->v1;
+            xxh_u32 v2 = state->v2;
+            xxh_u32 v3 = state->v3;
+            xxh_u32 v4 = state->v4;
+
+            do {
+                v1 = XXH32_round(v1, XXH_readLE32(p)); p+=4;
+                v2 = XXH32_round(v2, XXH_readLE32(p)); p+=4;
+                v3 = XXH32_round(v3, XXH_readLE32(p)); p+=4;
+                v4 = XXH32_round(v4, XXH_readLE32(p)); p+=4;
+            } while (p<=limit);
+
+            state->v1 = v1;
+            state->v2 = v2;
+            state->v3 = v3;
+            state->v4 = v4;
+        }
+
+        if (p < bEnd) {
+            XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
+            state->memsize = (unsigned)(bEnd-p);
+        }
+    }
+
+    return XXH_OK;
+}
+
+
+XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* state)
+{
+    xxh_u32 h32;
+
+    if (state->large_len) {
+        h32 = XXH_rotl32(state->v1, 1)
+            + XXH_rotl32(state->v2, 7)
+            + XXH_rotl32(state->v3, 12)
+            + XXH_rotl32(state->v4, 18);
+    } else {
+        h32 = state->v3 /* == seed */ + XXH_PRIME32_5;
+    }
+
+    h32 += state->total_len_32;
+
+    return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned);
+}
+
+
+/*******   Canonical representation   *******/
+
+/*
+ * The default return values from XXH functions are unsigned 32 and 64 bit
+ * integers.
+ *
+ * The canonical representation uses big endian convention, the same convention
+ * as human-readable numbers (large digits first).
+ *
+ * This way, hash values can be written into a file or buffer, remaining
+ * comparable across different systems.
+ *
+ * The following functions allow transformation of hash values to and from their
+ * canonical format.
+ */
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
+{
+    XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
+    if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
+    memcpy(dst, &hash, sizeof(*dst));
+}
+
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
+{
+    return XXH_readBE32(src);
+}
+
+
+#ifndef XXH_NO_LONG_LONG
+
+/* *******************************************************************
+*  64-bit hash functions
+*********************************************************************/
+
+/*******   Memory access   *******/
+
+typedef XXH64_hash_t xxh_u64;
+
+#ifdef XXH_OLD_NAMES
+#  define U64 xxh_u64
+#endif
+
+/*!
+ * XXH_REROLL_XXH64:
+ * Whether to reroll the XXH64_finalize() loop.
+ *
+ * Just like XXH32, we can unroll the XXH64_finalize() loop. This can be a
+ * performance gain on 64-bit hosts, as only one jump is required.
+ *
+ * However, on 32-bit hosts, because arithmetic needs to be done with two 32-bit
+ * registers, and 64-bit arithmetic needs to be simulated, it isn't beneficial
+ * to unroll. The code becomes ridiculously large (the largest function in the
+ * binary on i386!), and rerolling it saves anywhere from 3kB to 20kB. It is
+ * also slightly faster because it fits into cache better and is more likely
+ * to be inlined by the compiler.
+ *
+ * If XXH_REROLL is defined, this is ignored and the loop is always rerolled.
+ */
+#ifndef XXH_REROLL_XXH64
+#  if (defined(__ILP32__) || defined(_ILP32)) /* ILP32 is often defined on 32-bit GCC family */ \
+   || !(defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) /* x86-64 */ \
+     || defined(_M_ARM64) || defined(__aarch64__) || defined(__arm64__) /* aarch64 */ \
+     || defined(__PPC64__) || defined(__PPC64LE__) || defined(__ppc64__) || defined(__powerpc64__) /* ppc64 */ \
+     || defined(__mips64__) || defined(__mips64)) /* mips64 */ \
+   || (!defined(SIZE_MAX) || SIZE_MAX < ULLONG_MAX) /* check limits */
+#    define XXH_REROLL_XXH64 1
+#  else
+#    define XXH_REROLL_XXH64 0
+#  endif
+#endif /* !defined(XXH_REROLL_XXH64) */
+
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
+/*
+ * Manual byteshift. Best for old compilers which don't inline memcpy.
+ * We actually directly use XXH_readLE64 and XXH_readBE64.
+ */
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
+
+/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
+static xxh_u64 XXH_read64(const void* memPtr) { return *(const xxh_u64*) memPtr; }
+
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
+
+/*
+ * __pack instructions are safer, but compiler specific, hence potentially
+ * problematic for some compilers.
+ *
+ * Currently only defined for GCC and ICC.
+ */
+#ifdef XXH_OLD_NAMES
+typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64;
+#endif
+static xxh_u64 XXH_read64(const void* ptr)
+{
+    typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) xxh_unalign64;
+    return ((const xxh_unalign64*)ptr)->u64;
+}
+
+#else
+
+/*
+ * Portable and safe solution. Generally efficient.
+ * see: https://stackoverflow.com/a/32095106/646947
+ */
+static xxh_u64 XXH_read64(const void* memPtr)
+{
+    xxh_u64 val;
+    memcpy(&val, memPtr, sizeof(val));
+    return val;
+}
+
+#endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
+
+#if defined(_MSC_VER)     /* Visual Studio */
+#  define XXH_swap64 _byteswap_uint64
+#elif XXH_GCC_VERSION >= 403
+#  define XXH_swap64 __builtin_bswap64
+#else
+static xxh_u64 XXH_swap64 (xxh_u64 x)
+{
+    return  ((x << 56) & 0xff00000000000000ULL) |
+            ((x << 40) & 0x00ff000000000000ULL) |
+            ((x << 24) & 0x0000ff0000000000ULL) |
+            ((x << 8)  & 0x000000ff00000000ULL) |
+            ((x >> 8)  & 0x00000000ff000000ULL) |
+            ((x >> 24) & 0x0000000000ff0000ULL) |
+            ((x >> 40) & 0x000000000000ff00ULL) |
+            ((x >> 56) & 0x00000000000000ffULL);
+}
+#endif
+
+
+/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
+
+XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr)
+{
+    const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
+    return bytePtr[0]
+         | ((xxh_u64)bytePtr[1] << 8)
+         | ((xxh_u64)bytePtr[2] << 16)
+         | ((xxh_u64)bytePtr[3] << 24)
+         | ((xxh_u64)bytePtr[4] << 32)
+         | ((xxh_u64)bytePtr[5] << 40)
+         | ((xxh_u64)bytePtr[6] << 48)
+         | ((xxh_u64)bytePtr[7] << 56);
+}
+
+XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr)
+{
+    const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
+    return bytePtr[7]
+         | ((xxh_u64)bytePtr[6] << 8)
+         | ((xxh_u64)bytePtr[5] << 16)
+         | ((xxh_u64)bytePtr[4] << 24)
+         | ((xxh_u64)bytePtr[3] << 32)
+         | ((xxh_u64)bytePtr[2] << 40)
+         | ((xxh_u64)bytePtr[1] << 48)
+         | ((xxh_u64)bytePtr[0] << 56);
+}
+
+#else
+XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr)
+{
+    return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
+}
+
+static xxh_u64 XXH_readBE64(const void* ptr)
+{
+    return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
+}
+#endif
+
+XXH_FORCE_INLINE xxh_u64
+XXH_readLE64_align(const void* ptr, XXH_alignment align)
+{
+    if (align==XXH_unaligned)
+        return XXH_readLE64(ptr);
+    else
+        return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr);
+}
+
+
+/*******   xxh64   *******/
+
+static const xxh_u64 XXH_PRIME64_1 = 0x9E3779B185EBCA87ULL;   /* 0b1001111000110111011110011011000110000101111010111100101010000111 */
+static const xxh_u64 XXH_PRIME64_2 = 0xC2B2AE3D27D4EB4FULL;   /* 0b1100001010110010101011100011110100100111110101001110101101001111 */
+static const xxh_u64 XXH_PRIME64_3 = 0x165667B19E3779F9ULL;   /* 0b0001011001010110011001111011000110011110001101110111100111111001 */
+static const xxh_u64 XXH_PRIME64_4 = 0x85EBCA77C2B2AE63ULL;   /* 0b1000010111101011110010100111011111000010101100101010111001100011 */
+static const xxh_u64 XXH_PRIME64_5 = 0x27D4EB2F165667C5ULL;   /* 0b0010011111010100111010110010111100010110010101100110011111000101 */
+
+#ifdef XXH_OLD_NAMES
+#  define PRIME64_1 XXH_PRIME64_1
+#  define PRIME64_2 XXH_PRIME64_2
+#  define PRIME64_3 XXH_PRIME64_3
+#  define PRIME64_4 XXH_PRIME64_4
+#  define PRIME64_5 XXH_PRIME64_5
+#endif
+
+static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input)
+{
+    acc += input * XXH_PRIME64_2;
+    acc  = XXH_rotl64(acc, 31);
+    acc *= XXH_PRIME64_1;
+    return acc;
+}
+
+static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val)
+{
+    val  = XXH64_round(0, val);
+    acc ^= val;
+    acc  = acc * XXH_PRIME64_1 + XXH_PRIME64_4;
+    return acc;
+}
+
+static xxh_u64 XXH64_avalanche(xxh_u64 h64)
+{
+    h64 ^= h64 >> 33;
+    h64 *= XXH_PRIME64_2;
+    h64 ^= h64 >> 29;
+    h64 *= XXH_PRIME64_3;
+    h64 ^= h64 >> 32;
+    return h64;
+}
+
+
+#define XXH_get64bits(p) XXH_readLE64_align(p, align)
+
+static xxh_u64
+XXH64_finalize(xxh_u64 h64, const xxh_u8* ptr, size_t len, XXH_alignment align)
+{
+#define XXH_PROCESS1_64 do {                                   \
+    h64 ^= (*ptr++) * XXH_PRIME64_5;                           \
+    h64 = XXH_rotl64(h64, 11) * XXH_PRIME64_1;                 \
+} while (0)
+
+#define XXH_PROCESS4_64 do {                                   \
+    h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1;      \
+    ptr += 4;                                              \
+    h64 = XXH_rotl64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3;     \
+} while (0)
+
+#define XXH_PROCESS8_64 do {                                   \
+    xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); \
+    ptr += 8;                                              \
+    h64 ^= k1;                                             \
+    h64  = XXH_rotl64(h64,27) * XXH_PRIME64_1 + XXH_PRIME64_4;     \
+} while (0)
+
+    /* Rerolled version for 32-bit targets is faster and much smaller. */
+    if (XXH_REROLL || XXH_REROLL_XXH64) {
+        len &= 31;
+        while (len >= 8) {
+            XXH_PROCESS8_64;
+            len -= 8;
+        }
+        if (len >= 4) {
+            XXH_PROCESS4_64;
+            len -= 4;
+        }
+        while (len > 0) {
+            XXH_PROCESS1_64;
+            --len;
+        }
+         return  XXH64_avalanche(h64);
+    } else {
+        switch(len & 31) {
+           case 24: XXH_PROCESS8_64;
+                         /* fallthrough */
+           case 16: XXH_PROCESS8_64;
+                         /* fallthrough */
+           case  8: XXH_PROCESS8_64;
+                    return XXH64_avalanche(h64);
+
+           case 28: XXH_PROCESS8_64;
+                         /* fallthrough */
+           case 20: XXH_PROCESS8_64;
+                         /* fallthrough */
+           case 12: XXH_PROCESS8_64;
+                         /* fallthrough */
+           case  4: XXH_PROCESS4_64;
+                    return XXH64_avalanche(h64);
+
+           case 25: XXH_PROCESS8_64;
+                         /* fallthrough */
+           case 17: XXH_PROCESS8_64;
+                         /* fallthrough */
+           case  9: XXH_PROCESS8_64;
+                    XXH_PROCESS1_64;
+                    return XXH64_avalanche(h64);
+
+           case 29: XXH_PROCESS8_64;
+                         /* fallthrough */
+           case 21: XXH_PROCESS8_64;
+                         /* fallthrough */
+           case 13: XXH_PROCESS8_64;
+                         /* fallthrough */
+           case  5: XXH_PROCESS4_64;
+                    XXH_PROCESS1_64;
+                    return XXH64_avalanche(h64);
+
+           case 26: XXH_PROCESS8_64;
+                         /* fallthrough */
+           case 18: XXH_PROCESS8_64;
+                         /* fallthrough */
+           case 10: XXH_PROCESS8_64;
+                    XXH_PROCESS1_64;
+                    XXH_PROCESS1_64;
+                    return XXH64_avalanche(h64);
+
+           case 30: XXH_PROCESS8_64;
+                         /* fallthrough */
+           case 22: XXH_PROCESS8_64;
+                         /* fallthrough */
+           case 14: XXH_PROCESS8_64;
+                         /* fallthrough */
+           case  6: XXH_PROCESS4_64;
+                    XXH_PROCESS1_64;
+                    XXH_PROCESS1_64;
+                    return XXH64_avalanche(h64);
+
+           case 27: XXH_PROCESS8_64;
+                         /* fallthrough */
+           case 19: XXH_PROCESS8_64;
+                         /* fallthrough */
+           case 11: XXH_PROCESS8_64;
+                    XXH_PROCESS1_64;
+                    XXH_PROCESS1_64;
+                    XXH_PROCESS1_64;
+                    return XXH64_avalanche(h64);
+
+           case 31: XXH_PROCESS8_64;
+                         /* fallthrough */
+           case 23: XXH_PROCESS8_64;
+                         /* fallthrough */
+           case 15: XXH_PROCESS8_64;
+                         /* fallthrough */
+           case  7: XXH_PROCESS4_64;
+                         /* fallthrough */
+           case  3: XXH_PROCESS1_64;
+                         /* fallthrough */
+           case  2: XXH_PROCESS1_64;
+                         /* fallthrough */
+           case  1: XXH_PROCESS1_64;
+                         /* fallthrough */
+           case  0: return XXH64_avalanche(h64);
+        }
+    }
+    /* impossible to reach */
+    XXH_ASSERT(0);
+    return 0;  /* unreachable, but some compilers complain without it */
+}
+
+#ifdef XXH_OLD_NAMES
+#  define PROCESS1_64 XXH_PROCESS1_64
+#  define PROCESS4_64 XXH_PROCESS4_64
+#  define PROCESS8_64 XXH_PROCESS8_64
+#else
+#  undef XXH_PROCESS1_64
+#  undef XXH_PROCESS4_64
+#  undef XXH_PROCESS8_64
+#endif
+
+XXH_FORCE_INLINE xxh_u64
+XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align)
+{
+    const xxh_u8* bEnd = input + len;
+    xxh_u64 h64;
+
+#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
+    if (input==NULL) {
+        len=0;
+        bEnd=input=(const xxh_u8*)(size_t)32;
+    }
+#endif
+
+    if (len>=32) {
+        const xxh_u8* const limit = bEnd - 32;
+        xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
+        xxh_u64 v2 = seed + XXH_PRIME64_2;
+        xxh_u64 v3 = seed + 0;
+        xxh_u64 v4 = seed - XXH_PRIME64_1;
+
+        do {
+            v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8;
+            v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8;
+            v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8;
+            v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8;
+        } while (input<=limit);
+
+        h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+        h64 = XXH64_mergeRound(h64, v1);
+        h64 = XXH64_mergeRound(h64, v2);
+        h64 = XXH64_mergeRound(h64, v3);
+        h64 = XXH64_mergeRound(h64, v4);
+
+    } else {
+        h64  = seed + XXH_PRIME64_5;
+    }
+
+    h64 += (xxh_u64) len;
+
+    return XXH64_finalize(h64, input, len, align);
+}
+
+
+XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t len, XXH64_hash_t seed)
+{
+#if 0
+    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+    XXH64_state_t state;
+    XXH64_reset(&state, seed);
+    XXH64_update(&state, (const xxh_u8*)input, len);
+    return XXH64_digest(&state);
+
+#else
+
+    if (XXH_FORCE_ALIGN_CHECK) {
+        if ((((size_t)input) & 7)==0) {  /* Input is aligned, let's leverage the speed advantage */
+            return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
+    }   }
+
+    return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
+
+#endif
+}
+
+/*******   Hash Streaming   *******/
+
+XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
+{
+    return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
+}
+XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
+{
+    XXH_free(statePtr);
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState)
+{
+    memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, XXH64_hash_t seed)
+{
+    XXH64_state_t state;   /* use a local state to memcpy() in order to avoid strict-aliasing warnings */
+    memset(&state, 0, sizeof(state));
+    state.v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
+    state.v2 = seed + XXH_PRIME64_2;
+    state.v3 = seed + 0;
+    state.v4 = seed - XXH_PRIME64_1;
+     /* do not write into reserved64, might be removed in a future version */
+    memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved64));
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API XXH_errorcode
+XXH64_update (XXH64_state_t* state, const void* input, size_t len)
+{
+    if (input==NULL)
+#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
+        return XXH_OK;
+#else
+        return XXH_ERROR;
+#endif
+
+    {   const xxh_u8* p = (const xxh_u8*)input;
+        const xxh_u8* const bEnd = p + len;
+
+        state->total_len += len;
+
+        if (state->memsize + len < 32) {  /* fill in tmp buffer */
+            XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len);
+            state->memsize += (xxh_u32)len;
+            return XXH_OK;
+        }
+
+        if (state->memsize) {   /* tmp buffer is full */
+            XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize);
+            state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0));
+            state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1));
+            state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2));
+            state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3));
+            p += 32-state->memsize;
+            state->memsize = 0;
+        }
+
+        if (p+32 <= bEnd) {
+            const xxh_u8* const limit = bEnd - 32;
+            xxh_u64 v1 = state->v1;
+            xxh_u64 v2 = state->v2;
+            xxh_u64 v3 = state->v3;
+            xxh_u64 v4 = state->v4;
+
+            do {
+                v1 = XXH64_round(v1, XXH_readLE64(p)); p+=8;
+                v2 = XXH64_round(v2, XXH_readLE64(p)); p+=8;
+                v3 = XXH64_round(v3, XXH_readLE64(p)); p+=8;
+                v4 = XXH64_round(v4, XXH_readLE64(p)); p+=8;
+            } while (p<=limit);
+
+            state->v1 = v1;
+            state->v2 = v2;
+            state->v3 = v3;
+            state->v4 = v4;
+        }
+
+        if (p < bEnd) {
+            XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
+            state->memsize = (unsigned)(bEnd-p);
+        }
+    }
+
+    return XXH_OK;
+}
+
+
+XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* state)
+{
+    xxh_u64 h64;
+
+    if (state->total_len >= 32) {
+        xxh_u64 const v1 = state->v1;
+        xxh_u64 const v2 = state->v2;
+        xxh_u64 const v3 = state->v3;
+        xxh_u64 const v4 = state->v4;
+
+        h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+        h64 = XXH64_mergeRound(h64, v1);
+        h64 = XXH64_mergeRound(h64, v2);
+        h64 = XXH64_mergeRound(h64, v3);
+        h64 = XXH64_mergeRound(h64, v4);
+    } else {
+        h64  = state->v3 /*seed*/ + XXH_PRIME64_5;
+    }
+
+    h64 += (xxh_u64) state->total_len;
+
+    return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned);
+}
+
+
+/******* Canonical representation   *******/
+
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
+{
+    XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
+    if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
+    memcpy(dst, &hash, sizeof(*dst));
+}
+
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
+{
+    return XXH_readBE64(src);
+}
+
+
+
+/* *********************************************************************
+*  XXH3
+*  New generation hash designed for speed on small keys and vectorization
+************************************************************************ */
+
+/* ===   Compiler specifics   === */
+
+#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* >= C99 */
+#  define XXH_RESTRICT   restrict
+#else
+/* Note: it might be useful to define __restrict or __restrict__ for some C++ compilers */
+#  define XXH_RESTRICT   /* disable */
+#endif
+
+#if (defined(__GNUC__) && (__GNUC__ >= 3))  \
+  || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \
+  || defined(__clang__)
+#    define XXH_likely(x) __builtin_expect(x, 1)
+#    define XXH_unlikely(x) __builtin_expect(x, 0)
+#else
+#    define XXH_likely(x) (x)
+#    define XXH_unlikely(x) (x)
+#endif
+
+#if defined(__GNUC__)
+#  if defined(__AVX2__)
+#    include <immintrin.h>
+#  elif defined(__SSE2__)
+#    include <emmintrin.h>
+#  elif defined(__ARM_NEON__) || defined(__ARM_NEON)
+#    define inline __inline__  /* circumvent a clang bug */
+#    include <arm_neon.h>
+#    undef inline
+#  endif
+#elif defined(_MSC_VER)
+#  include <intrin.h>
+#endif
+
+/*
+ * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while
+ * remaining a true 64-bit/128-bit hash function.
+ *
+ * This is done by prioritizing a subset of 64-bit operations that can be
+ * emulated without too many steps on the average 32-bit machine.
+ *
+ * For example, these two lines seem similar, and run equally fast on 64-bit:
+ *
+ *   xxh_u64 x;
+ *   x ^= (x >> 47); // good
+ *   x ^= (x >> 13); // bad
+ *
+ * However, to a 32-bit machine, there is a major difference.
+ *
+ * x ^= (x >> 47) looks like this:
+ *
+ *   x.lo ^= (x.hi >> (47 - 32));
+ *
+ * while x ^= (x >> 13) looks like this:
+ *
+ *   // note: funnel shifts are not usually cheap.
+ *   x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13));
+ *   x.hi ^= (x.hi >> 13);
+ *
+ * The first one is significantly faster than the second, simply because the
+ * shift is larger than 32. This means:
+ *  - All the bits we need are in the upper 32 bits, so we can ignore the lower
+ *    32 bits in the shift.
+ *  - The shift result will always fit in the lower 32 bits, and therefore,
+ *    we can ignore the upper 32 bits in the xor.
+ *
+ * Thanks to this optimization, XXH3 only requires these features to be efficient:
+ *
+ *  - Usable unaligned access
+ *  - A 32-bit or 64-bit ALU
+ *      - If 32-bit, a decent ADC instruction
+ *  - A 32 or 64-bit multiply with a 64-bit result
+ *  - For the 128-bit variant, a decent byteswap helps short inputs.
+ *
+ * The first two are already required by XXH32, and almost all 32-bit and 64-bit
+ * platforms which can run XXH32 can run XXH3 efficiently.
+ *
+ * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one
+ * notable exception.
+ *
+ * First of all, Thumb-1 lacks support for the UMULL instruction which
+ * performs the important long multiply. This means numerous __aeabi_lmul
+ * calls.
+ *
+ * Second of all, the 8 functional registers are just not enough.
+ * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need
+ * Lo registers, and this shuffling results in thousands more MOVs than A32.
+ *
+ * A32 and T32 don't have this limitation. They can access all 14 registers,
+ * do a 32->64 multiply with UMULL, and the flexible operand allowing free
+ * shifts is helpful, too.
+ *
+ * Therefore, we do a quick sanity check.
+ *
+ * If compiling Thumb-1 for a target which supports ARM instructions, we will
+ * emit a warning, as it is not a "sane" platform to compile for.
+ *
+ * Usually, if this happens, it is because of an accident and you probably need
+ * to specify -march, as you likely meant to compile for a newer architecture.
+ *
+ * Credit: large sections of the vectorial and asm source code paths
+ *         have been contributed by @easyaspi314
+ */
+#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM)
+#   warning "XXH3 is highly inefficient without ARM or Thumb-2."
+#endif
+
+/* ==========================================
+ * Vectorization detection
+ * ========================================== */
+#define XXH_SCALAR 0  /* Portable scalar version */
+#define XXH_SSE2   1  /* SSE2 for Pentium 4 and all x86_64 */
+#define XXH_AVX2   2  /* AVX2 for Haswell and Bulldozer */
+#define XXH_AVX512 3  /* AVX512 for Skylake and Icelake */
+#define XXH_NEON   4  /* NEON for most ARMv7-A and all AArch64 */
+#define XXH_VSX    5  /* VSX and ZVector for POWER8/z13 */
+
+#ifndef XXH_VECTOR    /* can be defined on command line */
+#  if defined(__AVX512F__)
+#    define XXH_VECTOR XXH_AVX512
+#  elif defined(__AVX2__)
+#    define XXH_VECTOR XXH_AVX2
+#  elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2))
+#    define XXH_VECTOR XXH_SSE2
+#  elif defined(__GNUC__) /* msvc support maybe later */ \
+  && (defined(__ARM_NEON__) || defined(__ARM_NEON)) \
+  && (defined(__LITTLE_ENDIAN__) /* We only support little endian NEON */ \
+    || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
+#    define XXH_VECTOR XXH_NEON
+#  elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \
+     || (defined(__s390x__) && defined(__VEC__)) \
+     && defined(__GNUC__) /* TODO: IBM XL */
+#    define XXH_VECTOR XXH_VSX
+#  else
+#    define XXH_VECTOR XXH_SCALAR
+#  endif
+#endif
+
+/*
+ * Controls the alignment of the accumulator,
+ * for compatibility with aligned vector loads, which are usually faster.
+ */
+#ifndef XXH_ACC_ALIGN
+#  if defined(XXH_X86DISPATCH)
+#     define XXH_ACC_ALIGN 64  /* for compatibility with avx512 */
+#  elif XXH_VECTOR == XXH_SCALAR  /* scalar */
+#     define XXH_ACC_ALIGN 8
+#  elif XXH_VECTOR == XXH_SSE2  /* sse2 */
+#     define XXH_ACC_ALIGN 16
+#  elif XXH_VECTOR == XXH_AVX2  /* avx2 */
+#     define XXH_ACC_ALIGN 32
+#  elif XXH_VECTOR == XXH_NEON  /* neon */
+#     define XXH_ACC_ALIGN 16
+#  elif XXH_VECTOR == XXH_VSX   /* vsx */
+#     define XXH_ACC_ALIGN 16
+#  elif XXH_VECTOR == XXH_AVX512  /* avx512 */
+#     define XXH_ACC_ALIGN 64
+#  endif
+#endif
+
+#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \
+    || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512
+#  define XXH_SEC_ALIGN XXH_ACC_ALIGN
+#else
+#  define XXH_SEC_ALIGN 8
+#endif
+
+/*
+ * UGLY HACK:
+ * GCC usually generates the best code with -O3 for xxHash.
+ *
+ * However, when targeting AVX2, it is overzealous in its unrolling resulting
+ * in code roughly 3/4 the speed of Clang.
+ *
+ * There are other issues, such as GCC splitting _mm256_loadu_si256 into
+ * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which
+ * only applies to Sandy and Ivy Bridge... which don't even support AVX2.
+ *
+ * That is why when compiling the AVX2 version, it is recommended to use either
+ *   -O2 -mavx2 -march=haswell
+ * or
+ *   -O2 -mavx2 -mno-avx256-split-unaligned-load
+ * for decent performance, or to use Clang instead.
+ *
+ * Fortunately, we can control the first one with a pragma that forces GCC into
+ * -O2, but the other one we can't control without "failed to inline always
+ * inline function due to target mismatch" warnings.
+ */
+#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
+  && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
+  && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */
+#  pragma GCC push_options
+#  pragma GCC optimize("-O2")
+#endif
+
+
+#if XXH_VECTOR == XXH_NEON
+/*
+ * NEON's setup for vmlal_u32 is a little more complicated than it is on
+ * SSE2, AVX2, and VSX.
+ *
+ * While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an upcast.
+ *
+ * To do the same operation, the 128-bit 'Q' register needs to be split into
+ * two 64-bit 'D' registers, performing this operation::
+ *
+ *   [                a                 |                 b                ]
+ *            |              '---------. .--------'                |
+ *            |                         x                          |
+ *            |              .---------' '--------.                |
+ *   [ a & 0xFFFFFFFF | b & 0xFFFFFFFF ],[    a >> 32     |     b >> 32    ]
+ *
+ * Due to significant changes in aarch64, the fastest method for aarch64 is
+ * completely different than the fastest method for ARMv7-A.
+ *
+ * ARMv7-A treats D registers as unions overlaying Q registers, so modifying
+ * D11 will modify the high half of Q5. This is similar to how modifying AH
+ * will only affect bits 8-15 of AX on x86.
+ *
+ * VZIP takes two registers, and puts even lanes in one register and odd lanes
+ * in the other.
+ *
+ * On ARMv7-A, this strangely modifies both parameters in place instead of
+ * taking the usual 3-operand form.
+ *
+ * Therefore, if we want to do this, we can simply use a D-form VZIP.32 on the
+ * lower and upper halves of the Q register to end up with the high and low
+ * halves where we want - all in one instruction.
+ *
+ *   vzip.32   d10, d11       @ d10 = { d10[0], d11[0] }; d11 = { d10[1], d11[1] }
+ *
+ * Unfortunately we need inline assembly for this: Instructions modifying two
+ * registers at once is not possible in GCC or Clang's IR, and they have to
+ * create a copy.
+ *
+ * aarch64 requires a different approach.
+ *
+ * In order to make it easier to write a decent compiler for aarch64, many
+ * quirks were removed, such as conditional execution.
+ *
+ * NEON was also affected by this.
+ *
+ * aarch64 cannot access the high bits of a Q-form register, and writes to a
+ * D-form register zero the high bits, similar to how writes to W-form scalar
+ * registers (or DWORD registers on x86_64) work.
+ *
+ * The formerly free vget_high intrinsics now require a vext (with a few
+ * exceptions)
+ *
+ * Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the equivalent
+ * of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to only modify one
+ * operand.
+ *
+ * The equivalent of the VZIP.32 on the lower and upper halves would be this
+ * mess:
+ *
+ *   ext     v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], v0[1] }
+ *   zip1    v1.2s, v0.2s, v2.2s     // v1 = { v0[0], v2[0] }
+ *   zip2    v0.2s, v0.2s, v1.2s     // v0 = { v0[1], v2[1] }
+ *
+ * Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 (SHRN):
+ *
+ *   shrn    v1.2s, v0.2d, #32  // v1 = (uint32x2_t)(v0 >> 32);
+ *   xtn     v0.2s, v0.2d       // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF);
+ *
+ * This is available on ARMv7-A, but is less efficient than a single VZIP.32.
+ */
+
+/*
+ * Function-like macro:
+ * void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t &outHi)
+ * {
+ *     outLo = (uint32x2_t)(in & 0xFFFFFFFF);
+ *     outHi = (uint32x2_t)(in >> 32);
+ *     in = UNDEFINED;
+ * }
+ */
+# if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \
+   && defined(__GNUC__) \
+   && !defined(__aarch64__) && !defined(__arm64__)
+#  define XXH_SPLIT_IN_PLACE(in, outLo, outHi)                                              \
+    do {                                                                                    \
+      /* Undocumented GCC/Clang operand modifier: %e0 = lower D half, %f0 = upper D half */ \
+      /* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 */     \
+      /* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 */ \
+      __asm__("vzip.32  %e0, %f0" : "+w" (in));                                             \
+      (outLo) = vget_low_u32 (vreinterpretq_u32_u64(in));                                   \
+      (outHi) = vget_high_u32(vreinterpretq_u32_u64(in));                                   \
+   } while (0)
+# else
+#  define XXH_SPLIT_IN_PLACE(in, outLo, outHi)                                            \
+    do {                                                                                  \
+      (outLo) = vmovn_u64    (in);                                                        \
+      (outHi) = vshrn_n_u64  ((in), 32);                                                  \
+    } while (0)
+# endif
+#endif  /* XXH_VECTOR == XXH_NEON */
+
+/*
+ * VSX and Z Vector helpers.
+ *
+ * This is very messy, and any pull requests to clean this up are welcome.
+ *
+ * There are a lot of problems with supporting VSX and s390x, due to
+ * inconsistent intrinsics, spotty coverage, and multiple endiannesses.
+ */
+#if XXH_VECTOR == XXH_VSX
+#  if defined(__s390x__)
+#    include <s390intrin.h>
+#  else
+/* gcc's altivec.h can have the unwanted consequence to unconditionally
+ * #define bool, vector, and pixel keywords,
+ * with bad consequences for programs already using these keywords for other purposes.
+ * The paragraph defining these macros is skipped when __APPLE_ALTIVEC__ is defined.
+ * __APPLE_ALTIVEC__ is _generally_ defined automatically by the compiler,
+ * but it seems that, in some cases, it isn't.
+ * Force the build macro to be defined, so that keywords are not altered.
+ */
+#    if defined(__GNUC__) && !defined(__APPLE_ALTIVEC__)
+#      define __APPLE_ALTIVEC__
+#    endif
+#    include <altivec.h>
+#  endif
+
+typedef __vector unsigned long long xxh_u64x2;
+typedef __vector unsigned char xxh_u8x16;
+typedef __vector unsigned xxh_u32x4;
+
+# ifndef XXH_VSX_BE
+#  if defined(__BIG_ENDIAN__) \
+  || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+#    define XXH_VSX_BE 1
+#  elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__
+#    warning "-maltivec=be is not recommended. Please use native endianness."
+#    define XXH_VSX_BE 1
+#  else
+#    define XXH_VSX_BE 0
+#  endif
+# endif /* !defined(XXH_VSX_BE) */
+
+# if XXH_VSX_BE
+/* A wrapper for POWER9's vec_revb. */
+#  if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__))
+#    define XXH_vec_revb vec_revb
+#  else
+XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val)
+{
+    xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
+                                  0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 };
+    return vec_perm(val, val, vByteSwap);
+}
+#  endif
+# endif /* XXH_VSX_BE */
+
+/*
+ * Performs an unaligned load and byte swaps it on big endian.
+ */
+XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr)
+{
+    xxh_u64x2 ret;
+    memcpy(&ret, ptr, sizeof(xxh_u64x2));
+# if XXH_VSX_BE
+    ret = XXH_vec_revb(ret);
+# endif
+    return ret;
+}
+
+/*
+ * vec_mulo and vec_mule are very problematic intrinsics on PowerPC
+ *
+ * These intrinsics weren't added until GCC 8, despite existing for a while,
+ * and they are endian dependent. Also, their meaning swap depending on version.
+ * */
+# if defined(__s390x__)
+ /* s390x is always big endian, no issue on this platform */
+#  define XXH_vec_mulo vec_mulo
+#  define XXH_vec_mule vec_mule
+# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw)
+/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */
+#  define XXH_vec_mulo __builtin_altivec_vmulouw
+#  define XXH_vec_mule __builtin_altivec_vmuleuw
+# else
+/* gcc needs inline assembly */
+/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */
+XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b)
+{
+    xxh_u64x2 result;
+    __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
+    return result;
+}
+XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b)
+{
+    xxh_u64x2 result;
+    __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
+    return result;
+}
+# endif /* XXH_vec_mulo, XXH_vec_mule */
+#endif /* XXH_VECTOR == XXH_VSX */
+
+
+/* prefetch
+ * can be disabled, by declaring XXH_NO_PREFETCH build macro */
+#if defined(XXH_NO_PREFETCH)
+#  define XXH_PREFETCH(ptr)  (void)(ptr)  /* disabled */
+#else
+#  if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86))  /* _mm_prefetch() is not defined outside of x86/x64 */
+#    include <mmintrin.h>   /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
+#    define XXH_PREFETCH(ptr)  _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
+#  elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
+#    define XXH_PREFETCH(ptr)  __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
+#  else
+#    define XXH_PREFETCH(ptr) (void)(ptr)  /* disabled */
+#  endif
+#endif  /* XXH_NO_PREFETCH */
+
+
+/* ==========================================
+ * XXH3 default settings
+ * ========================================== */
+
+#define XXH_SECRET_DEFAULT_SIZE 192   /* minimum XXH3_SECRET_SIZE_MIN */
+
+#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN)
+#  error "default keyset is not large enough"
+#endif
+
+/* Pseudorandom secret taken directly from FARSH */
+XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = {
+    0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
+    0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
+    0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
+    0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
+    0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
+    0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
+    0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
+    0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
+    0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
+    0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
+    0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
+    0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e,
+};
+
+
+#ifdef XXH_OLD_NAMES
+#  define kSecret XXH3_kSecret
+#endif
+
+/*
+ * Calculates a 32-bit to 64-bit long multiply.
+ *
+ * Wraps __emulu on MSVC x86 because it tends to call __allmul when it doesn't
+ * need to (but it shouldn't need to anyways, it is about 7 instructions to do
+ * a 64x64 multiply...). Since we know that this will _always_ emit MULL, we
+ * use that instead of the normal method.
+ *
+ * If you are compiling for platforms like Thumb-1 and don't have a better option,
+ * you may also want to write your own long multiply routine here.
+ *
+ * XXH_FORCE_INLINE xxh_u64 XXH_mult32to64(xxh_u64 x, xxh_u64 y)
+ * {
+ *    return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF);
+ * }
+ */
+#if defined(_MSC_VER) && defined(_M_IX86)
+#    include <intrin.h>
+#    define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y))
+#else
+/*
+ * Downcast + upcast is usually better than masking on older compilers like
+ * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers.
+ *
+ * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands
+ * and perform a full 64x64 multiply -- entirely redundant on 32-bit.
+ */
+#    define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y))
+#endif
+
+/*
+ * Calculates a 64->128-bit long multiply.
+ *
+ * Uses __uint128_t and _umul128 if available, otherwise uses a scalar version.
+ */
+static XXH128_hash_t
+XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs)
+{
+    /*
+     * GCC/Clang __uint128_t method.
+     *
+     * On most 64-bit targets, GCC and Clang define a __uint128_t type.
+     * This is usually the best way as it usually uses a native long 64-bit
+     * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64.
+     *
+     * Usually.
+     *
+     * Despite being a 32-bit platform, Clang (and emscripten) define this type
+     * despite not having the arithmetic for it. This results in a laggy
+     * compiler builtin call which calculates a full 128-bit multiply.
+     * In that case it is best to use the portable one.
+     * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677
+     */
+#if defined(__GNUC__) && !defined(__wasm__) \
+    && defined(__SIZEOF_INT128__) \
+    || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+
+    __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs;
+    XXH128_hash_t r128;
+    r128.low64  = (xxh_u64)(product);
+    r128.high64 = (xxh_u64)(product >> 64);
+    return r128;
+
+    /*
+     * MSVC for x64's _umul128 method.
+     *
+     * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct);
+     *
+     * This compiles to single operand MUL on x64.
+     */
+#elif defined(_M_X64) || defined(_M_IA64)
+
+#ifndef _MSC_VER
+#   pragma intrinsic(_umul128)
+#endif
+    xxh_u64 product_high;
+    xxh_u64 const product_low = _umul128(lhs, rhs, &product_high);
+    XXH128_hash_t r128;
+    r128.low64  = product_low;
+    r128.high64 = product_high;
+    return r128;
+
+#else
+    /*
+     * Portable scalar method. Optimized for 32-bit and 64-bit ALUs.
+     *
+     * This is a fast and simple grade school multiply, which is shown below
+     * with base 10 arithmetic instead of base 0x100000000.
+     *
+     *           9 3 // D2 lhs = 93
+     *         x 7 5 // D2 rhs = 75
+     *     ----------
+     *           1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15
+     *         4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45
+     *         2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21
+     *     + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63
+     *     ---------
+     *         2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27
+     *     + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67
+     *     ---------
+     *       6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975
+     *
+     * The reasons for adding the products like this are:
+     *  1. It avoids manual carry tracking. Just like how
+     *     (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX.
+     *     This avoids a lot of complexity.
+     *
+     *  2. It hints for, and on Clang, compiles to, the powerful UMAAL
+     *     instruction available in ARM's Digital Signal Processing extension
+     *     in 32-bit ARMv6 and later, which is shown below:
+     *
+     *         void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm)
+     *         {
+     *             xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm;
+     *             *RdLo = (xxh_u32)(product & 0xFFFFFFFF);
+     *             *RdHi = (xxh_u32)(product >> 32);
+     *         }
+     *
+     *     This instruction was designed for efficient long multiplication, and
+     *     allows this to be calculated in only 4 instructions at speeds
+     *     comparable to some 64-bit ALUs.
+     *
+     *  3. It isn't terrible on other platforms. Usually this will be a couple
+     *     of 32-bit ADD/ADCs.
+     */
+
+    /* First calculate all of the cross products. */
+    xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF);
+    xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32,        rhs & 0xFFFFFFFF);
+    xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32);
+    xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32,        rhs >> 32);
+
+    /* Now add the products together. These will never overflow. */
+    xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi;
+    xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32)        + hi_hi;
+    xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF);
+
+    XXH128_hash_t r128;
+    r128.low64  = lower;
+    r128.high64 = upper;
+    return r128;
+#endif
+}
+
+/*
+ * Does a 64-bit to 128-bit multiply, then XOR folds it.
+ *
+ * The reason for the separate function is to prevent passing too many structs
+ * around by value. This will hopefully inline the multiply, but we don't force it.
+ */
+static xxh_u64
+XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs)
+{
+    XXH128_hash_t product = XXH_mult64to128(lhs, rhs);
+    return product.low64 ^ product.high64;
+}
+
+/* Seems to produce slightly better code on GCC for some reason. */
+XXH_FORCE_INLINE xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift)
+{
+    XXH_ASSERT(0 <= shift && shift < 64);
+    return v64 ^ (v64 >> shift);
+}
+
+/*
+ * This is a fast avalanche stage,
+ * suitable when input bits are already partially mixed
+ */
+static XXH64_hash_t XXH3_avalanche(xxh_u64 h64)
+{
+    h64 = XXH_xorshift64(h64, 37);
+    h64 *= 0x165667919E3779F9ULL;
+    h64 = XXH_xorshift64(h64, 32);
+    return h64;
+}
+
+/*
+ * This is a stronger avalanche,
+ * inspired by Pelle Evensen's rrmxmx
+ * preferable when input has not been previously mixed
+ */
+static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len)
+{
+    /* this mix is inspired by Pelle Evensen's rrmxmx */
+    h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24);
+    h64 *= 0x9FB21C651E98DF25ULL;
+    h64 ^= (h64 >> 35) + len ;
+    h64 *= 0x9FB21C651E98DF25ULL;
+    return XXH_xorshift64(h64, 28);
+}
+
+
+/* ==========================================
+ * Short keys
+ * ==========================================
+ * One of the shortcomings of XXH32 and XXH64 was that their performance was
+ * sub-optimal on short lengths. It used an iterative algorithm which strongly
+ * favored lengths that were a multiple of 4 or 8.
+ *
+ * Instead of iterating over individual inputs, we use a set of single shot
+ * functions which piece together a range of lengths and operate in constant time.
+ *
+ * Additionally, the number of multiplies has been significantly reduced. This
+ * reduces latency, especially when emulating 64-bit multiplies on 32-bit.
+ *
+ * Depending on the platform, this may or may not be faster than XXH32, but it
+ * is almost guaranteed to be faster than XXH64.
+ */
+
+/*
+ * At very short lengths, there isn't enough input to fully hide secrets, or use
+ * the entire secret.
+ *
+ * There is also only a limited amount of mixing we can do before significantly
+ * impacting performance.
+ *
+ * Therefore, we use different sections of the secret and always mix two secret
+ * samples with an XOR. This should have no effect on performance on the
+ * seedless or withSeed variants because everything _should_ be constant folded
+ * by modern compilers.
+ *
+ * The XOR mixing hides individual parts of the secret and increases entropy.
+ *
+ * This adds an extra layer of strength for custom secrets.
+ */
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+    XXH_ASSERT(input != NULL);
+    XXH_ASSERT(1 <= len && len <= 3);
+    XXH_ASSERT(secret != NULL);
+    /*
+     * len = 1: combined = { input[0], 0x01, input[0], input[0] }
+     * len = 2: combined = { input[1], 0x02, input[0], input[1] }
+     * len = 3: combined = { input[2], 0x03, input[0], input[1] }
+     */
+    {   xxh_u8  const c1 = input[0];
+        xxh_u8  const c2 = input[len >> 1];
+        xxh_u8  const c3 = input[len - 1];
+        xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2  << 24)
+                               | ((xxh_u32)c3 <<  0) | ((xxh_u32)len << 8);
+        xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
+        xxh_u64 const keyed = (xxh_u64)combined ^ bitflip;
+        return XXH64_avalanche(keyed);
+    }
+}
+
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+    XXH_ASSERT(input != NULL);
+    XXH_ASSERT(secret != NULL);
+    XXH_ASSERT(4 <= len && len < 8);
+    seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
+    {   xxh_u32 const input1 = XXH_readLE32(input);
+        xxh_u32 const input2 = XXH_readLE32(input + len - 4);
+        xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed;
+        xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32);
+        xxh_u64 const keyed = input64 ^ bitflip;
+        return XXH3_rrmxmx(keyed, len);
+    }
+}
+
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+    XXH_ASSERT(input != NULL);
+    XXH_ASSERT(secret != NULL);
+    XXH_ASSERT(8 <= len && len <= 16);
+    {   xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed;
+        xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed;
+        xxh_u64 const input_lo = XXH_readLE64(input)           ^ bitflip1;
+        xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2;
+        xxh_u64 const acc = len
+                          + XXH_swap64(input_lo) + input_hi
+                          + XXH3_mul128_fold64(input_lo, input_hi);
+        return XXH3_avalanche(acc);
+    }
+}
+
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+    XXH_ASSERT(len <= 16);
+    {   if (XXH_likely(len >  8)) return XXH3_len_9to16_64b(input, len, secret, seed);
+        if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed);
+        if (len) return XXH3_len_1to3_64b(input, len, secret, seed);
+        return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64)));
+    }
+}
+
+/*
+ * DISCLAIMER: There are known *seed-dependent* multicollisions here due to
+ * multiplication by zero, affecting hashes of lengths 17 to 240.
+ *
+ * However, they are very unlikely.
+ *
+ * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all
+ * unseeded non-cryptographic hashes, it does not attempt to defend itself
+ * against specially crafted inputs, only random inputs.
+ *
+ * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes
+ * cancelling out the secret is taken an arbitrary number of times (addressed
+ * in XXH3_accumulate_512), this collision is very unlikely with random inputs
+ * and/or proper seeding:
+ *
+ * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a
+ * function that is only called up to 16 times per hash with up to 240 bytes of
+ * input.
+ *
+ * This is not too bad for a non-cryptographic hash function, especially with
+ * only 64 bit outputs.
+ *
+ * The 128-bit variant (which trades some speed for strength) is NOT affected
+ * by this, although it is always a good idea to use a proper seed if you care
+ * about strength.
+ */
+XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input,
+                                     const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64)
+{
+#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
+  && defined(__i386__) && defined(__SSE2__)  /* x86 + SSE2 */ \
+  && !defined(XXH_ENABLE_AUTOVECTORIZE)      /* Define to disable like XXH32 hack */
+    /*
+     * UGLY HACK:
+     * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in
+     * slower code.
+     *
+     * By forcing seed64 into a register, we disrupt the cost model and
+     * cause it to scalarize. See `XXH32_round()`
+     *
+     * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600,
+     * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on
+     * GCC 9.2, despite both emitting scalar code.
+     *
+     * GCC generates much better scalar code than Clang for the rest of XXH3,
+     * which is why finding a more optimal codepath is an interest.
+     */
+    __asm__ ("" : "+r" (seed64));
+#endif
+    {   xxh_u64 const input_lo = XXH_readLE64(input);
+        xxh_u64 const input_hi = XXH_readLE64(input+8);
+        return XXH3_mul128_fold64(
+            input_lo ^ (XXH_readLE64(secret)   + seed64),
+            input_hi ^ (XXH_readLE64(secret+8) - seed64)
+        );
+    }
+}
+
+/* For mid range keys, XXH3 uses a Mum-hash variant. */
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
+                     const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+                     XXH64_hash_t seed)
+{
+    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
+    XXH_ASSERT(16 < len && len <= 128);
+
+    {   xxh_u64 acc = len * XXH_PRIME64_1;
+        if (len > 32) {
+            if (len > 64) {
+                if (len > 96) {
+                    acc += XXH3_mix16B(input+48, secret+96, seed);
+                    acc += XXH3_mix16B(input+len-64, secret+112, seed);
+                }
+                acc += XXH3_mix16B(input+32, secret+64, seed);
+                acc += XXH3_mix16B(input+len-48, secret+80, seed);
+            }
+            acc += XXH3_mix16B(input+16, secret+32, seed);
+            acc += XXH3_mix16B(input+len-32, secret+48, seed);
+        }
+        acc += XXH3_mix16B(input+0, secret+0, seed);
+        acc += XXH3_mix16B(input+len-16, secret+16, seed);
+
+        return XXH3_avalanche(acc);
+    }
+}
+
+#define XXH3_MIDSIZE_MAX 240
+
+XXH_NO_INLINE XXH64_hash_t
+XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
+                      const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+                      XXH64_hash_t seed)
+{
+    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
+    XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
+
+    #define XXH3_MIDSIZE_STARTOFFSET 3
+    #define XXH3_MIDSIZE_LASTOFFSET  17
+
+    {   xxh_u64 acc = len * XXH_PRIME64_1;
+        int const nbRounds = (int)len / 16;
+        int i;
+        for (i=0; i<8; i++) {
+            acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed);
+        }
+        acc = XXH3_avalanche(acc);
+        XXH_ASSERT(nbRounds >= 8);
+#if defined(__clang__)                                /* Clang */ \
+    && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \
+    && !defined(XXH_ENABLE_AUTOVECTORIZE)             /* Define to disable */
+        /*
+         * UGLY HACK:
+         * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86.
+         * In everywhere else, it uses scalar code.
+         *
+         * For 64->128-bit multiplies, even if the NEON was 100% optimal, it
+         * would still be slower than UMAAL (see XXH_mult64to128).
+         *
+         * Unfortunately, Clang doesn't handle the long multiplies properly and
+         * converts them to the nonexistent "vmulq_u64" intrinsic, which is then
+         * scalarized into an ugly mess of VMOV.32 instructions.
+         *
+         * This mess is difficult to avoid without turning autovectorization
+         * off completely, but they are usually relatively minor and/or not
+         * worth it to fix.
+         *
+         * This loop is the easiest to fix, as unlike XXH32, this pragma
+         * _actually works_ because it is a loop vectorization instead of an
+         * SLP vectorization.
+         */
+        #pragma clang loop vectorize(disable)
+#endif
+        for (i=8 ; i < nbRounds; i++) {
+            acc += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed);
+        }
+        /* last bytes */
+        acc += XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed);
+        return XXH3_avalanche(acc);
+    }
+}
+
+
+/* =======     Long Keys     ======= */
+
+#define XXH_STRIPE_LEN 64
+#define XXH_SECRET_CONSUME_RATE 8   /* nb of secret bytes consumed at each accumulation */
+#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64))
+
+#ifdef XXH_OLD_NAMES
+#  define STRIPE_LEN XXH_STRIPE_LEN
+#  define ACC_NB XXH_ACC_NB
+#endif
+
+XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64)
+{
+    if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64);
+    memcpy(dst, &v64, sizeof(v64));
+}
+
+/* Several intrinsic functions below are supposed to accept __int64 as argument,
+ * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ .
+ * However, several environments do not define __int64 type,
+ * requiring a workaround.
+ */
+#if !defined (__VMS) \
+  && (defined (__cplusplus) \
+  || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+    typedef int64_t xxh_i64;
+#else
+    /* the following type must have a width of 64-bit */
+    typedef long long xxh_i64;
+#endif
+
+/*
+ * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized.
+ *
+ * It is a hardened version of UMAC, based off of FARSH's implementation.
+ *
+ * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD
+ * implementations, and it is ridiculously fast.
+ *
+ * We harden it by mixing the original input to the accumulators as well as the product.
+ *
+ * This means that in the (relatively likely) case of a multiply by zero, the
+ * original input is preserved.
+ *
+ * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve
+ * cross-pollination, as otherwise the upper and lower halves would be
+ * essentially independent.
+ *
+ * This doesn't matter on 64-bit hashes since they all get merged together in
+ * the end, so we skip the extra step.
+ *
+ * Both XXH3_64bits and XXH3_128bits use this subroutine.
+ */
+
+#if (XXH_VECTOR == XXH_AVX512) || defined(XXH_X86DISPATCH)
+
+#ifndef XXH_TARGET_AVX512
+# define XXH_TARGET_AVX512  /* disable attribute target */
+#endif
+
+XXH_FORCE_INLINE XXH_TARGET_AVX512 void
+XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc,
+                     const void* XXH_RESTRICT input,
+                     const void* XXH_RESTRICT secret)
+{
+    XXH_ALIGN(64) __m512i* const xacc = (__m512i *) acc;
+    XXH_ASSERT((((size_t)acc) & 63) == 0);
+    XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
+
+    {
+        /* data_vec    = input[0]; */
+        __m512i const data_vec    = _mm512_loadu_si512   (input);
+        /* key_vec     = secret[0]; */
+        __m512i const key_vec     = _mm512_loadu_si512   (secret);
+        /* data_key    = data_vec ^ key_vec; */
+        __m512i const data_key    = _mm512_xor_si512     (data_vec, key_vec);
+        /* data_key_lo = data_key >> 32; */
+        __m512i const data_key_lo = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1));
+        /* product     = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
+        __m512i const product     = _mm512_mul_epu32     (data_key, data_key_lo);
+        /* xacc[0] += swap(data_vec); */
+        __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2));
+        __m512i const sum       = _mm512_add_epi64(*xacc, data_swap);
+        /* xacc[0] += product; */
+        *xacc = _mm512_add_epi64(product, sum);
+    }
+}
+
+/*
+ * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing.
+ *
+ * Multiplication isn't perfect, as explained by Google in HighwayHash:
+ *
+ *  // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to
+ *  // varying degrees. In descending order of goodness, bytes
+ *  // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32.
+ *  // As expected, the upper and lower bytes are much worse.
+ *
+ * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291
+ *
+ * Since our algorithm uses a pseudorandom secret to add some variance into the
+ * mix, we don't need to (or want to) mix as often or as much as HighwayHash does.
+ *
+ * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid
+ * extraction.
+ *
+ * Both XXH3_64bits and XXH3_128bits use this subroutine.
+ */
+
+XXH_FORCE_INLINE XXH_TARGET_AVX512 void
+XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+    XXH_ASSERT((((size_t)acc) & 63) == 0);
+    XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
+    {   XXH_ALIGN(64) __m512i* const xacc = (__m512i*) acc;
+        const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1);
+
+        /* xacc[0] ^= (xacc[0] >> 47) */
+        __m512i const acc_vec     = *xacc;
+        __m512i const shifted     = _mm512_srli_epi64    (acc_vec, 47);
+        __m512i const data_vec    = _mm512_xor_si512     (acc_vec, shifted);
+        /* xacc[0] ^= secret; */
+        __m512i const key_vec     = _mm512_loadu_si512   (secret);
+        __m512i const data_key    = _mm512_xor_si512     (data_vec, key_vec);
+
+        /* xacc[0] *= XXH_PRIME32_1; */
+        __m512i const data_key_hi = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1));
+        __m512i const prod_lo     = _mm512_mul_epu32     (data_key, prime32);
+        __m512i const prod_hi     = _mm512_mul_epu32     (data_key_hi, prime32);
+        *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32));
+    }
+}
+
+XXH_FORCE_INLINE XXH_TARGET_AVX512 void
+XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
+{
+    XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0);
+    XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64);
+    XXH_ASSERT(((size_t)customSecret & 63) == 0);
+    (void)(&XXH_writeLE64);
+    {   int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i);
+        __m512i const seed = _mm512_mask_set1_epi64(_mm512_set1_epi64((xxh_i64)seed64), 0xAA, -(xxh_i64)seed64);
+
+        XXH_ALIGN(64) const __m512i* const src  = (const __m512i*) XXH3_kSecret;
+        XXH_ALIGN(64)       __m512i* const dest = (      __m512i*) customSecret;
+        int i;
+        for (i=0; i < nbRounds; ++i) {
+            /* GCC has a bug, _mm512_stream_load_si512 accepts 'void*', not 'void const*',
+             * this will warn "discards ‘const’ qualifier". */
+            union {
+                XXH_ALIGN(64) const __m512i* cp;
+                XXH_ALIGN(64) void* p;
+            } remote_const_void;
+            remote_const_void.cp = src + i;
+            dest[i] = _mm512_add_epi64(_mm512_stream_load_si512(remote_const_void.p), seed);
+    }   }
+}
+
+#endif
+
+#if (XXH_VECTOR == XXH_AVX2) || defined(XXH_X86DISPATCH)
+
+#ifndef XXH_TARGET_AVX2
+# define XXH_TARGET_AVX2  /* disable attribute target */
+#endif
+
+XXH_FORCE_INLINE XXH_TARGET_AVX2 void
+XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc,
+                    const void* XXH_RESTRICT input,
+                    const void* XXH_RESTRICT secret)
+{
+    XXH_ASSERT((((size_t)acc) & 31) == 0);
+    {   XXH_ALIGN(32) __m256i* const xacc    =       (__m256i *) acc;
+        /* Unaligned. This is mainly for pointer arithmetic, and because
+         * _mm256_loadu_si256 requires  a const __m256i * pointer for some reason. */
+        const         __m256i* const xinput  = (const __m256i *) input;
+        /* Unaligned. This is mainly for pointer arithmetic, and because
+         * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
+        const         __m256i* const xsecret = (const __m256i *) secret;
+
+        size_t i;
+        for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
+            /* data_vec    = xinput[i]; */
+            __m256i const data_vec    = _mm256_loadu_si256    (xinput+i);
+            /* key_vec     = xsecret[i]; */
+            __m256i const key_vec     = _mm256_loadu_si256   (xsecret+i);
+            /* data_key    = data_vec ^ key_vec; */
+            __m256i const data_key    = _mm256_xor_si256     (data_vec, key_vec);
+            /* data_key_lo = data_key >> 32; */
+            __m256i const data_key_lo = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
+            /* product     = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
+            __m256i const product     = _mm256_mul_epu32     (data_key, data_key_lo);
+            /* xacc[i] += swap(data_vec); */
+            __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2));
+            __m256i const sum       = _mm256_add_epi64(xacc[i], data_swap);
+            /* xacc[i] += product; */
+            xacc[i] = _mm256_add_epi64(product, sum);
+    }   }
+}
+
+XXH_FORCE_INLINE XXH_TARGET_AVX2 void
+XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+    XXH_ASSERT((((size_t)acc) & 31) == 0);
+    {   XXH_ALIGN(32) __m256i* const xacc = (__m256i*) acc;
+        /* Unaligned. This is mainly for pointer arithmetic, and because
+         * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
+        const         __m256i* const xsecret = (const __m256i *) secret;
+        const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1);
+
+        size_t i;
+        for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
+            /* xacc[i] ^= (xacc[i] >> 47) */
+            __m256i const acc_vec     = xacc[i];
+            __m256i const shifted     = _mm256_srli_epi64    (acc_vec, 47);
+            __m256i const data_vec    = _mm256_xor_si256     (acc_vec, shifted);
+            /* xacc[i] ^= xsecret; */
+            __m256i const key_vec     = _mm256_loadu_si256   (xsecret+i);
+            __m256i const data_key    = _mm256_xor_si256     (data_vec, key_vec);
+
+            /* xacc[i] *= XXH_PRIME32_1; */
+            __m256i const data_key_hi = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
+            __m256i const prod_lo     = _mm256_mul_epu32     (data_key, prime32);
+            __m256i const prod_hi     = _mm256_mul_epu32     (data_key_hi, prime32);
+            xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32));
+        }
+    }
+}
+
+XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
+{
+    XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0);
+    XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6);
+    XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64);
+    (void)(&XXH_writeLE64);
+    XXH_PREFETCH(customSecret);
+    {   __m256i const seed = _mm256_set_epi64x(-(xxh_i64)seed64, (xxh_i64)seed64, -(xxh_i64)seed64, (xxh_i64)seed64);
+
+        XXH_ALIGN(64) const __m256i* const src  = (const __m256i*) XXH3_kSecret;
+        XXH_ALIGN(64)       __m256i*       dest = (      __m256i*) customSecret;
+
+#       if defined(__GNUC__) || defined(__clang__)
+        /*
+         * On GCC & Clang, marking 'dest' as modified will cause the compiler:
+         *   - do not extract the secret from sse registers in the internal loop
+         *   - use less common registers, and avoid pushing these reg into stack
+         * The asm hack causes Clang to assume that XXH3_kSecretPtr aliases with
+         * customSecret, and on aarch64, this prevented LDP from merging two
+         * loads together for free. Putting the loads together before the stores
+         * properly generates LDP.
+         */
+        __asm__("" : "+r" (dest));
+#       endif
+
+        /* GCC -O2 need unroll loop manually */
+        dest[0] = _mm256_add_epi64(_mm256_stream_load_si256(src+0), seed);
+        dest[1] = _mm256_add_epi64(_mm256_stream_load_si256(src+1), seed);
+        dest[2] = _mm256_add_epi64(_mm256_stream_load_si256(src+2), seed);
+        dest[3] = _mm256_add_epi64(_mm256_stream_load_si256(src+3), seed);
+        dest[4] = _mm256_add_epi64(_mm256_stream_load_si256(src+4), seed);
+        dest[5] = _mm256_add_epi64(_mm256_stream_load_si256(src+5), seed);
+    }
+}
+
+#endif
+
+#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH)
+
+#ifndef XXH_TARGET_SSE2
+# define XXH_TARGET_SSE2  /* disable attribute target */
+#endif
+
+XXH_FORCE_INLINE XXH_TARGET_SSE2 void
+XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc,
+                    const void* XXH_RESTRICT input,
+                    const void* XXH_RESTRICT secret)
+{
+    /* SSE2 is just a half-scale version of the AVX2 version. */
+    XXH_ASSERT((((size_t)acc) & 15) == 0);
+    {   XXH_ALIGN(16) __m128i* const xacc    =       (__m128i *) acc;
+        /* Unaligned. This is mainly for pointer arithmetic, and because
+         * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
+        const         __m128i* const xinput  = (const __m128i *) input;
+        /* Unaligned. This is mainly for pointer arithmetic, and because
+         * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
+        const         __m128i* const xsecret = (const __m128i *) secret;
+
+        size_t i;
+        for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
+            /* data_vec    = xinput[i]; */
+            __m128i const data_vec    = _mm_loadu_si128   (xinput+i);
+            /* key_vec     = xsecret[i]; */
+            __m128i const key_vec     = _mm_loadu_si128   (xsecret+i);
+            /* data_key    = data_vec ^ key_vec; */
+            __m128i const data_key    = _mm_xor_si128     (data_vec, key_vec);
+            /* data_key_lo = data_key >> 32; */
+            __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
+            /* product     = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
+            __m128i const product     = _mm_mul_epu32     (data_key, data_key_lo);
+            /* xacc[i] += swap(data_vec); */
+            __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2));
+            __m128i const sum       = _mm_add_epi64(xacc[i], data_swap);
+            /* xacc[i] += product; */
+            xacc[i] = _mm_add_epi64(product, sum);
+    }   }
+}
+
+XXH_FORCE_INLINE XXH_TARGET_SSE2 void
+XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+    XXH_ASSERT((((size_t)acc) & 15) == 0);
+    {   XXH_ALIGN(16) __m128i* const xacc = (__m128i*) acc;
+        /* Unaligned. This is mainly for pointer arithmetic, and because
+         * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
+        const         __m128i* const xsecret = (const __m128i *) secret;
+        const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1);
+
+        size_t i;
+        for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
+            /* xacc[i] ^= (xacc[i] >> 47) */
+            __m128i const acc_vec     = xacc[i];
+            __m128i const shifted     = _mm_srli_epi64    (acc_vec, 47);
+            __m128i const data_vec    = _mm_xor_si128     (acc_vec, shifted);
+            /* xacc[i] ^= xsecret[i]; */
+            __m128i const key_vec     = _mm_loadu_si128   (xsecret+i);
+            __m128i const data_key    = _mm_xor_si128     (data_vec, key_vec);
+
+            /* xacc[i] *= XXH_PRIME32_1; */
+            __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
+            __m128i const prod_lo     = _mm_mul_epu32     (data_key, prime32);
+            __m128i const prod_hi     = _mm_mul_epu32     (data_key_hi, prime32);
+            xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32));
+        }
+    }
+}
+
+XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
+{
+    XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
+    (void)(&XXH_writeLE64);
+    {   int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i);
+
+#       if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900
+        // MSVC 32bit mode does not support _mm_set_epi64x before 2015
+        XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, -(xxh_i64)seed64 };
+        __m128i const seed = _mm_load_si128((__m128i const*)seed64x2);
+#       else
+        __m128i const seed = _mm_set_epi64x(-(xxh_i64)seed64, (xxh_i64)seed64);
+#       endif
+        int i;
+
+        XXH_ALIGN(64)        const float* const src  = (float const*) XXH3_kSecret;
+        XXH_ALIGN(XXH_SEC_ALIGN) __m128i*       dest = (__m128i*) customSecret;
+#       if defined(__GNUC__) || defined(__clang__)
+        /*
+         * On GCC & Clang, marking 'dest' as modified will cause the compiler:
+         *   - do not extract the secret from sse registers in the internal loop
+         *   - use less common registers, and avoid pushing these reg into stack
+         */
+        __asm__("" : "+r" (dest));
+#       endif
+
+        for (i=0; i < nbRounds; ++i) {
+            dest[i] = _mm_add_epi64(_mm_castps_si128(_mm_load_ps(src+i*4)), seed);
+    }   }
+}
+
+#endif
+
+#if (XXH_VECTOR == XXH_NEON)
+
+XXH_FORCE_INLINE void
+XXH3_accumulate_512_neon( void* XXH_RESTRICT acc,
+                    const void* XXH_RESTRICT input,
+                    const void* XXH_RESTRICT secret)
+{
+    XXH_ASSERT((((size_t)acc) & 15) == 0);
+    {
+        XXH_ALIGN(16) uint64x2_t* const xacc = (uint64x2_t *) acc;
+        /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */
+        uint8_t const* const xinput = (const uint8_t *) input;
+        uint8_t const* const xsecret  = (const uint8_t *) secret;
+
+        size_t i;
+        for (i=0; i < XXH_STRIPE_LEN / sizeof(uint64x2_t); i++) {
+            /* data_vec = xinput[i]; */
+            uint8x16_t data_vec    = vld1q_u8(xinput  + (i * 16));
+            /* key_vec  = xsecret[i];  */
+            uint8x16_t key_vec     = vld1q_u8(xsecret + (i * 16));
+            uint64x2_t data_key;
+            uint32x2_t data_key_lo, data_key_hi;
+            /* xacc[i] += swap(data_vec); */
+            uint64x2_t const data64  = vreinterpretq_u64_u8(data_vec);
+            uint64x2_t const swapped = vextq_u64(data64, data64, 1);
+            xacc[i] = vaddq_u64 (xacc[i], swapped);
+            /* data_key = data_vec ^ key_vec; */
+            data_key = vreinterpretq_u64_u8(veorq_u8(data_vec, key_vec));
+            /* data_key_lo = (uint32x2_t) (data_key & 0xFFFFFFFF);
+             * data_key_hi = (uint32x2_t) (data_key >> 32);
+             * data_key = UNDEFINED; */
+            XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi);
+            /* xacc[i] += (uint64x2_t) data_key_lo * (uint64x2_t) data_key_hi; */
+            xacc[i] = vmlal_u32 (xacc[i], data_key_lo, data_key_hi);
+
+        }
+    }
+}
+
+XXH_FORCE_INLINE void
+XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+    XXH_ASSERT((((size_t)acc) & 15) == 0);
+
+    {   uint64x2_t* xacc       = (uint64x2_t*) acc;
+        uint8_t const* xsecret = (uint8_t const*) secret;
+        uint32x2_t prime       = vdup_n_u32 (XXH_PRIME32_1);
+
+        size_t i;
+        for (i=0; i < XXH_STRIPE_LEN/sizeof(uint64x2_t); i++) {
+            /* xacc[i] ^= (xacc[i] >> 47); */
+            uint64x2_t acc_vec  = xacc[i];
+            uint64x2_t shifted  = vshrq_n_u64 (acc_vec, 47);
+            uint64x2_t data_vec = veorq_u64   (acc_vec, shifted);
+
+            /* xacc[i] ^= xsecret[i]; */
+            uint8x16_t key_vec  = vld1q_u8(xsecret + (i * 16));
+            uint64x2_t data_key = veorq_u64(data_vec, vreinterpretq_u64_u8(key_vec));
+
+            /* xacc[i] *= XXH_PRIME32_1 */
+            uint32x2_t data_key_lo, data_key_hi;
+            /* data_key_lo = (uint32x2_t) (xacc[i] & 0xFFFFFFFF);
+             * data_key_hi = (uint32x2_t) (xacc[i] >> 32);
+             * xacc[i] = UNDEFINED; */
+            XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi);
+            {   /*
+                 * prod_hi = (data_key >> 32) * XXH_PRIME32_1;
+                 *
+                 * Avoid vmul_u32 + vshll_n_u32 since Clang 6 and 7 will
+                 * incorrectly "optimize" this:
+                 *   tmp     = vmul_u32(vmovn_u64(a), vmovn_u64(b));
+                 *   shifted = vshll_n_u32(tmp, 32);
+                 * to this:
+                 *   tmp     = "vmulq_u64"(a, b); // no such thing!
+                 *   shifted = vshlq_n_u64(tmp, 32);
+                 *
+                 * However, unlike SSE, Clang lacks a 64-bit multiply routine
+                 * for NEON, and it scalarizes two 64-bit multiplies instead.
+                 *
+                 * vmull_u32 has the same timing as vmul_u32, and it avoids
+                 * this bug completely.
+                 * See https://bugs.llvm.org/show_bug.cgi?id=39967
+                 */
+                uint64x2_t prod_hi = vmull_u32 (data_key_hi, prime);
+                /* xacc[i] = prod_hi << 32; */
+                xacc[i] = vshlq_n_u64(prod_hi, 32);
+                /* xacc[i] += (prod_hi & 0xFFFFFFFF) * XXH_PRIME32_1; */
+                xacc[i] = vmlal_u32(xacc[i], data_key_lo, prime);
+            }
+    }   }
+}
+
+#endif
+
+#if (XXH_VECTOR == XXH_VSX)
+
+XXH_FORCE_INLINE void
+XXH3_accumulate_512_vsx(  void* XXH_RESTRICT acc,
+                    const void* XXH_RESTRICT input,
+                    const void* XXH_RESTRICT secret)
+{
+          xxh_u64x2* const xacc     =       (xxh_u64x2*) acc;    /* presumed aligned */
+    xxh_u64x2 const* const xinput   = (xxh_u64x2 const*) input;   /* no alignment restriction */
+    xxh_u64x2 const* const xsecret  = (xxh_u64x2 const*) secret;    /* no alignment restriction */
+    xxh_u64x2 const v32 = { 32, 32 };
+    size_t i;
+    for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) {
+        /* data_vec = xinput[i]; */
+        xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + i);
+        /* key_vec = xsecret[i]; */
+        xxh_u64x2 const key_vec  = XXH_vec_loadu(xsecret + i);
+        xxh_u64x2 const data_key = data_vec ^ key_vec;
+        /* shuffled = (data_key << 32) | (data_key >> 32); */
+        xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32);
+        /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */
+        xxh_u64x2 const product  = XXH_vec_mulo((xxh_u32x4)data_key, shuffled);
+        xacc[i] += product;
+
+        /* swap high and low halves */
+#ifdef __s390x__
+        xacc[i] += vec_permi(data_vec, data_vec, 2);
+#else
+        xacc[i] += vec_xxpermdi(data_vec, data_vec, 2);
+#endif
+    }
+}
+
+XXH_FORCE_INLINE void
+XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+    XXH_ASSERT((((size_t)acc) & 15) == 0);
+
+    {         xxh_u64x2* const xacc    =       (xxh_u64x2*) acc;
+        const xxh_u64x2* const xsecret = (const xxh_u64x2*) secret;
+        /* constants */
+        xxh_u64x2 const v32  = { 32, 32 };
+        xxh_u64x2 const v47 = { 47, 47 };
+        xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 };
+        size_t i;
+        for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) {
+            /* xacc[i] ^= (xacc[i] >> 47); */
+            xxh_u64x2 const acc_vec  = xacc[i];
+            xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47);
+
+            /* xacc[i] ^= xsecret[i]; */
+            xxh_u64x2 const key_vec  = XXH_vec_loadu(xsecret + i);
+            xxh_u64x2 const data_key = data_vec ^ key_vec;
+
+            /* xacc[i] *= XXH_PRIME32_1 */
+            /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF);  */
+            xxh_u64x2 const prod_even  = XXH_vec_mule((xxh_u32x4)data_key, prime);
+            /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32);  */
+            xxh_u64x2 const prod_odd  = XXH_vec_mulo((xxh_u32x4)data_key, prime);
+            xacc[i] = prod_odd + (prod_even << v32);
+    }   }
+}
+
+#endif
+
+/* scalar variants - universal */
+
+XXH_FORCE_INLINE void
+XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc,
+                     const void* XXH_RESTRICT input,
+                     const void* XXH_RESTRICT secret)
+{
+    XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */
+    const xxh_u8* const xinput  = (const xxh_u8*) input;  /* no alignment restriction */
+    const xxh_u8* const xsecret = (const xxh_u8*) secret;   /* no alignment restriction */
+    size_t i;
+    XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0);
+    for (i=0; i < XXH_ACC_NB; i++) {
+        xxh_u64 const data_val = XXH_readLE64(xinput + 8*i);
+        xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + i*8);
+        xacc[i ^ 1] += data_val; /* swap adjacent lanes */
+        xacc[i] += XXH_mult32to64(data_key & 0xFFFFFFFF, data_key >> 32);
+    }
+}
+
+XXH_FORCE_INLINE void
+XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+    XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64* const xacc = (xxh_u64*) acc;   /* presumed aligned */
+    const xxh_u8* const xsecret = (const xxh_u8*) secret;   /* no alignment restriction */
+    size_t i;
+    XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0);
+    for (i=0; i < XXH_ACC_NB; i++) {
+        xxh_u64 const key64 = XXH_readLE64(xsecret + 8*i);
+        xxh_u64 acc64 = xacc[i];
+        acc64 = XXH_xorshift64(acc64, 47);
+        acc64 ^= key64;
+        acc64 *= XXH_PRIME32_1;
+        xacc[i] = acc64;
+    }
+}
+
+XXH_FORCE_INLINE void
+XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
+{
+    /*
+     * We need a separate pointer for the hack below,
+     * which requires a non-const pointer.
+     * Any decent compiler will optimize this out otherwise.
+     */
+    const xxh_u8* kSecretPtr = XXH3_kSecret;
+    XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
+
+#if defined(__clang__) && defined(__aarch64__)
+    /*
+     * UGLY HACK:
+     * Clang generates a bunch of MOV/MOVK pairs for aarch64, and they are
+     * placed sequentially, in order, at the top of the unrolled loop.
+     *
+     * While MOVK is great for generating constants (2 cycles for a 64-bit
+     * constant compared to 4 cycles for LDR), long MOVK chains stall the
+     * integer pipelines:
+     *   I   L   S
+     * MOVK
+     * MOVK
+     * MOVK
+     * MOVK
+     * ADD
+     * SUB      STR
+     *          STR
+     * By forcing loads from memory (as the asm line causes Clang to assume
+     * that XXH3_kSecretPtr has been changed), the pipelines are used more
+     * efficiently:
+     *   I   L   S
+     *      LDR
+     *  ADD LDR
+     *  SUB     STR
+     *          STR
+     * XXH3_64bits_withSeed, len == 256, Snapdragon 835
+     *   without hack: 2654.4 MB/s
+     *   with hack:    3202.9 MB/s
+     */
+    __asm__("" : "+r" (kSecretPtr));
+#endif
+    /*
+     * Note: in debug mode, this overrides the asm optimization
+     * and Clang will emit MOVK chains again.
+     */
+    XXH_ASSERT(kSecretPtr == XXH3_kSecret);
+
+    {   int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16;
+        int i;
+        for (i=0; i < nbRounds; i++) {
+            /*
+             * The asm hack causes Clang to assume that kSecretPtr aliases with
+             * customSecret, and on aarch64, this prevented LDP from merging two
+             * loads together for free. Putting the loads together before the stores
+             * properly generates LDP.
+             */
+            xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i)     + seed64;
+            xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64;
+            XXH_writeLE64((xxh_u8*)customSecret + 16*i,     lo);
+            XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi);
+    }   }
+}
+
+
+typedef void (*XXH3_f_accumulate_512)(void* XXH_RESTRICT, const void*, const void*);
+typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*);
+typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64);
+
+
+#if (XXH_VECTOR == XXH_AVX512)
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_avx512
+#define XXH3_scrambleAcc    XXH3_scrambleAcc_avx512
+#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512
+
+#elif (XXH_VECTOR == XXH_AVX2)
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_avx2
+#define XXH3_scrambleAcc    XXH3_scrambleAcc_avx2
+#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2
+
+#elif (XXH_VECTOR == XXH_SSE2)
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_sse2
+#define XXH3_scrambleAcc    XXH3_scrambleAcc_sse2
+#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2
+
+#elif (XXH_VECTOR == XXH_NEON)
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_neon
+#define XXH3_scrambleAcc    XXH3_scrambleAcc_neon
+#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
+
+#elif (XXH_VECTOR == XXH_VSX)
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_vsx
+#define XXH3_scrambleAcc    XXH3_scrambleAcc_vsx
+#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
+
+#else /* scalar */
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_scalar
+#define XXH3_scrambleAcc    XXH3_scrambleAcc_scalar
+#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
+
+#endif
+
+
+
+#ifndef XXH_PREFETCH_DIST
+#  ifdef __clang__
+#    define XXH_PREFETCH_DIST 320
+#  else
+#    if (XXH_VECTOR == XXH_AVX512)
+#      define XXH_PREFETCH_DIST 512
+#    else
+#      define XXH_PREFETCH_DIST 384
+#    endif
+#  endif  /* __clang__ */
+#endif  /* XXH_PREFETCH_DIST */
+
+/*
+ * XXH3_accumulate()
+ * Loops over XXH3_accumulate_512().
+ * Assumption: nbStripes will not overflow the secret size
+ */
+XXH_FORCE_INLINE void
+XXH3_accumulate(     xxh_u64* XXH_RESTRICT acc,
+                const xxh_u8* XXH_RESTRICT input,
+                const xxh_u8* XXH_RESTRICT secret,
+                      size_t nbStripes,
+                      XXH3_f_accumulate_512 f_acc512)
+{
+    size_t n;
+    for (n = 0; n < nbStripes; n++ ) {
+        const xxh_u8* const in = input + n*XXH_STRIPE_LEN;
+        XXH_PREFETCH(in + XXH_PREFETCH_DIST);
+        f_acc512(acc,
+                 in,
+                 secret + n*XXH_SECRET_CONSUME_RATE);
+    }
+}
+
+XXH_FORCE_INLINE void
+XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc,
+                      const xxh_u8* XXH_RESTRICT input, size_t len,
+                      const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+                            XXH3_f_accumulate_512 f_acc512,
+                            XXH3_f_scrambleAcc f_scramble)
+{
+    size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE;
+    size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock;
+    size_t const nb_blocks = (len - 1) / block_len;
+
+    size_t n;
+
+    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
+
+    for (n = 0; n < nb_blocks; n++) {
+        XXH3_accumulate(acc, input + n*block_len, secret, nbStripesPerBlock, f_acc512);
+        f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN);
+    }
+
+    /* last partial block */
+    XXH_ASSERT(len > XXH_STRIPE_LEN);
+    {   size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN;
+        XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE));
+        XXH3_accumulate(acc, input + nb_blocks*block_len, secret, nbStripes, f_acc512);
+
+        /* last stripe */
+        {   const xxh_u8* const p = input + len - XXH_STRIPE_LEN;
+#define XXH_SECRET_LASTACC_START 7  /* not aligned on 8, last secret is different from acc & scrambler */
+            f_acc512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START);
+    }   }
+}
+
+XXH_FORCE_INLINE xxh_u64
+XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret)
+{
+    return XXH3_mul128_fold64(
+               acc[0] ^ XXH_readLE64(secret),
+               acc[1] ^ XXH_readLE64(secret+8) );
+}
+
+static XXH64_hash_t
+XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start)
+{
+    xxh_u64 result64 = start;
+    size_t i = 0;
+
+    for (i = 0; i < 4; i++) {
+        result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i);
+#if defined(__clang__)                                /* Clang */ \
+    && (defined(__arm__) || defined(__thumb__))       /* ARMv7 */ \
+    && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */  \
+    && !defined(XXH_ENABLE_AUTOVECTORIZE)             /* Define to disable */
+        /*
+         * UGLY HACK:
+         * Prevent autovectorization on Clang ARMv7-a. Exact same problem as
+         * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b.
+         * XXH3_64bits, len == 256, Snapdragon 835:
+         *   without hack: 2063.7 MB/s
+         *   with hack:    2560.7 MB/s
+         */
+        __asm__("" : "+r" (result64));
+#endif
+    }
+
+    return XXH3_avalanche(result64);
+}
+
+#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \
+                        XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 }
+
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len,
+                           const void* XXH_RESTRICT secret, size_t secretSize,
+                           XXH3_f_accumulate_512 f_acc512,
+                           XXH3_f_scrambleAcc f_scramble)
+{
+    XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC;
+
+    XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc512, f_scramble);
+
+    /* converge into final hash */
+    XXH_STATIC_ASSERT(sizeof(acc) == 64);
+    /* do not align on 8, so that the secret is different from the accumulator */
+#define XXH_SECRET_MERGEACCS_START 11
+    XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
+    return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1);
+}
+
+/*
+ * It's important for performance that XXH3_hashLong is not inlined.
+ */
+XXH_NO_INLINE XXH64_hash_t
+XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len,
+                             XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
+{
+    (void)seed64;
+    return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate_512, XXH3_scrambleAcc);
+}
+
+/*
+ * It's important for performance that XXH3_hashLong is not inlined.
+ * Since the function is not inlined, the compiler may not be able to understand that,
+ * in some scenarios, its `secret` argument is actually a compile time constant.
+ * This variant enforces that the compiler can detect that,
+ * and uses this opportunity to streamline the generated code for better performance.
+ */
+XXH_NO_INLINE XXH64_hash_t
+XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len,
+                          XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
+{
+    (void)seed64; (void)secret; (void)secretLen;
+    return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate_512, XXH3_scrambleAcc);
+}
+
+/*
+ * XXH3_hashLong_64b_withSeed():
+ * Generate a custom key based on alteration of default XXH3_kSecret with the seed,
+ * and then use this key for long mode hashing.
+ *
+ * This operation is decently fast but nonetheless costs a little bit of time.
+ * Try to avoid it whenever possible (typically when seed==0).
+ *
+ * It's important for performance that XXH3_hashLong is not inlined. Not sure
+ * why (uop cache maybe?), but the difference is large and easily measurable.
+ */
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len,
+                                    XXH64_hash_t seed,
+                                    XXH3_f_accumulate_512 f_acc512,
+                                    XXH3_f_scrambleAcc f_scramble,
+                                    XXH3_f_initCustomSecret f_initSec)
+{
+    if (seed == 0)
+        return XXH3_hashLong_64b_internal(input, len,
+                                          XXH3_kSecret, sizeof(XXH3_kSecret),
+                                          f_acc512, f_scramble);
+    {   XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
+        f_initSec(secret, seed);
+        return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret),
+                                          f_acc512, f_scramble);
+    }
+}
+
+/*
+ * It's important for performance that XXH3_hashLong is not inlined.
+ */
+XXH_NO_INLINE XXH64_hash_t
+XXH3_hashLong_64b_withSeed(const void* input, size_t len,
+                           XXH64_hash_t seed, const xxh_u8* secret, size_t secretLen)
+{
+    (void)secret; (void)secretLen;
+    return XXH3_hashLong_64b_withSeed_internal(input, len, seed,
+                XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret);
+}
+
+
+typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t,
+                                          XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t);
+
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len,
+                     XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen,
+                     XXH3_hashLong64_f f_hashLong)
+{
+    XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN);
+    /*
+     * If an action is to be taken if `secretLen` condition is not respected,
+     * it should be done here.
+     * For now, it's a contract pre-condition.
+     * Adding a check and a branch here would cost performance at every hash.
+     * Also, note that function signature doesn't offer room to return an error.
+     */
+    if (len <= 16)
+        return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64);
+    if (len <= 128)
+        return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
+    if (len <= XXH3_MIDSIZE_MAX)
+        return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
+    return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen);
+}
+
+
+/* ===   Public entry point   === */
+
+XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* input, size_t len)
+{
+    return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default);
+}
+
+XXH_PUBLIC_API XXH64_hash_t
+XXH3_64bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize)
+{
+    return XXH3_64bits_internal(input, len, 0, secret, secretSize, XXH3_hashLong_64b_withSecret);
+}
+
+XXH_PUBLIC_API XXH64_hash_t
+XXH3_64bits_withSeed(const void* input, size_t len, XXH64_hash_t seed)
+{
+    return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed);
+}
+
+
+/* ===   XXH3 streaming   === */
+
+/*
+ * Malloc's a pointer that is always aligned to align.
+ *
+ * This must be freed with `XXH_alignedFree()`.
+ *
+ * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte
+ * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2
+ * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON.
+ *
+ * This underalignment previously caused a rather obvious crash which went
+ * completely unnoticed due to XXH3_createState() not actually being tested.
+ * Credit to RedSpah for noticing this bug.
+ *
+ * The alignment is done manually: Functions like posix_memalign or _mm_malloc
+ * are avoided: To maintain portability, we would have to write a fallback
+ * like this anyways, and besides, testing for the existence of library
+ * functions without relying on external build tools is impossible.
+ *
+ * The method is simple: Overallocate, manually align, and store the offset
+ * to the original behind the returned pointer.
+ *
+ * Align must be a power of 2 and 8 <= align <= 128.
+ */
+static void* XXH_alignedMalloc(size_t s, size_t align)
+{
+    XXH_ASSERT(align <= 128 && align >= 8); /* range check */
+    XXH_ASSERT((align & (align-1)) == 0);   /* power of 2 */
+    XXH_ASSERT(s != 0 && s < (s + align));  /* empty/overflow */
+    {   /* Overallocate to make room for manual realignment and an offset byte */
+        xxh_u8* base = (xxh_u8*)XXH_malloc(s + align);
+        if (base != NULL) {
+            /*
+             * Get the offset needed to align this pointer.
+             *
+             * Even if the returned pointer is aligned, there will always be
+             * at least one byte to store the offset to the original pointer.
+             */
+            size_t offset = align - ((size_t)base & (align - 1)); /* base % align */
+            /* Add the offset for the now-aligned pointer */
+            xxh_u8* ptr = base + offset;
+
+            XXH_ASSERT((size_t)ptr % align == 0);
+
+            /* Store the offset immediately before the returned pointer. */
+            ptr[-1] = (xxh_u8)offset;
+            return ptr;
+        }
+        return NULL;
+    }
+}
+/*
+ * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass
+ * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout.
+ */
+static void XXH_alignedFree(void* p)
+{
+    if (p != NULL) {
+        xxh_u8* ptr = (xxh_u8*)p;
+        /* Get the offset byte we added in XXH_malloc. */
+        xxh_u8 offset = ptr[-1];
+        /* Free the original malloc'd pointer */
+        xxh_u8* base = ptr - offset;
+        XXH_free(base);
+    }
+}
+XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void)
+{
+    XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64);
+    if (state==NULL) return NULL;
+    XXH3_INITSTATE(state);
+    return state;
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr)
+{
+    XXH_alignedFree(statePtr);
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API void
+XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state)
+{
+    memcpy(dst_state, src_state, sizeof(*dst_state));
+}
+
+static void
+XXH3_64bits_reset_internal(XXH3_state_t* statePtr,
+                           XXH64_hash_t seed,
+                           const void* secret, size_t secretSize)
+{
+    size_t const initStart = offsetof(XXH3_state_t, bufferedSize);
+    size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart;
+    XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart);
+    XXH_ASSERT(statePtr != NULL);
+    /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */
+    memset((char*)statePtr + initStart, 0, initLength);
+    statePtr->acc[0] = XXH_PRIME32_3;
+    statePtr->acc[1] = XXH_PRIME64_1;
+    statePtr->acc[2] = XXH_PRIME64_2;
+    statePtr->acc[3] = XXH_PRIME64_3;
+    statePtr->acc[4] = XXH_PRIME64_4;
+    statePtr->acc[5] = XXH_PRIME32_2;
+    statePtr->acc[6] = XXH_PRIME64_5;
+    statePtr->acc[7] = XXH_PRIME32_1;
+    statePtr->seed = seed;
+    statePtr->extSecret = (const unsigned char*)secret;
+    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
+    statePtr->secretLimit = secretSize - XXH_STRIPE_LEN;
+    statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE;
+}
+
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_reset(XXH3_state_t* statePtr)
+{
+    if (statePtr == NULL) return XXH_ERROR;
+    XXH3_64bits_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE);
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize)
+{
+    if (statePtr == NULL) return XXH_ERROR;
+    XXH3_64bits_reset_internal(statePtr, 0, secret, secretSize);
+    if (secret == NULL) return XXH_ERROR;
+    if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed)
+{
+    if (statePtr == NULL) return XXH_ERROR;
+    if (seed==0) return XXH3_64bits_reset(statePtr);
+    if (seed != statePtr->seed) XXH3_initCustomSecret(statePtr->customSecret, seed);
+    XXH3_64bits_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE);
+    return XXH_OK;
+}
+
+/* Note : when XXH3_consumeStripes() is invoked,
+ * there must be a guarantee that at least one more byte must be consumed from input
+ * so that the function can blindly consume all stripes using the "normal" secret segment */
+XXH_FORCE_INLINE void
+XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc,
+                    size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock,
+                    const xxh_u8* XXH_RESTRICT input, size_t nbStripes,
+                    const xxh_u8* XXH_RESTRICT secret, size_t secretLimit,
+                    XXH3_f_accumulate_512 f_acc512,
+                    XXH3_f_scrambleAcc f_scramble)
+{
+    XXH_ASSERT(nbStripes <= nbStripesPerBlock);  /* can handle max 1 scramble per invocation */
+    XXH_ASSERT(*nbStripesSoFarPtr < nbStripesPerBlock);
+    if (nbStripesPerBlock - *nbStripesSoFarPtr <= nbStripes) {
+        /* need a scrambling operation */
+        size_t const nbStripesToEndofBlock = nbStripesPerBlock - *nbStripesSoFarPtr;
+        size_t const nbStripesAfterBlock = nbStripes - nbStripesToEndofBlock;
+        XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripesToEndofBlock, f_acc512);
+        f_scramble(acc, secret + secretLimit);
+        XXH3_accumulate(acc, input + nbStripesToEndofBlock * XXH_STRIPE_LEN, secret, nbStripesAfterBlock, f_acc512);
+        *nbStripesSoFarPtr = nbStripesAfterBlock;
+    } else {
+        XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripes, f_acc512);
+        *nbStripesSoFarPtr += nbStripes;
+    }
+}
+
+/*
+ * Both XXH3_64bits_update and XXH3_128bits_update use this routine.
+ */
+XXH_FORCE_INLINE XXH_errorcode
+XXH3_update(XXH3_state_t* state,
+            const xxh_u8* input, size_t len,
+            XXH3_f_accumulate_512 f_acc512,
+            XXH3_f_scrambleAcc f_scramble)
+{
+    if (input==NULL)
+#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
+        return XXH_OK;
+#else
+        return XXH_ERROR;
+#endif
+
+    {   const xxh_u8* const bEnd = input + len;
+        const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
+
+        state->totalLen += len;
+
+        if (state->bufferedSize + len <= XXH3_INTERNALBUFFER_SIZE) {  /* fill in tmp buffer */
+            XXH_memcpy(state->buffer + state->bufferedSize, input, len);
+            state->bufferedSize += (XXH32_hash_t)len;
+            return XXH_OK;
+        }
+        /* total input is now > XXH3_INTERNALBUFFER_SIZE */
+
+        #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN)
+        XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0);   /* clean multiple */
+
+        /*
+         * Internal buffer is partially filled (always, except at beginning)
+         * Complete it, then consume it.
+         */
+        if (state->bufferedSize) {
+            size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize;
+            XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize);
+            input += loadSize;
+            XXH3_consumeStripes(state->acc,
+                               &state->nbStripesSoFar, state->nbStripesPerBlock,
+                                state->buffer, XXH3_INTERNALBUFFER_STRIPES,
+                                secret, state->secretLimit,
+                                f_acc512, f_scramble);
+            state->bufferedSize = 0;
+        }
+        XXH_ASSERT(input < bEnd);
+
+        /* Consume input by a multiple of internal buffer size */
+        if (input+XXH3_INTERNALBUFFER_SIZE < bEnd) {
+            const xxh_u8* const limit = bEnd - XXH3_INTERNALBUFFER_SIZE;
+            do {
+                XXH3_consumeStripes(state->acc,
+                                   &state->nbStripesSoFar, state->nbStripesPerBlock,
+                                    input, XXH3_INTERNALBUFFER_STRIPES,
+                                    secret, state->secretLimit,
+                                    f_acc512, f_scramble);
+                input += XXH3_INTERNALBUFFER_SIZE;
+            } while (input<limit);
+            /* for last partial stripe */
+            memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN);
+        }
+        XXH_ASSERT(input < bEnd);
+
+        /* Some remaining input (always) : buffer it */
+        XXH_memcpy(state->buffer, input, (size_t)(bEnd-input));
+        state->bufferedSize = (XXH32_hash_t)(bEnd-input);
+    }
+
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_update(XXH3_state_t* state, const void* input, size_t len)
+{
+    return XXH3_update(state, (const xxh_u8*)input, len,
+                       XXH3_accumulate_512, XXH3_scrambleAcc);
+}
+
+
+XXH_FORCE_INLINE void
+XXH3_digest_long (XXH64_hash_t* acc,
+                  const XXH3_state_t* state,
+                  const unsigned char* secret)
+{
+    /*
+     * Digest on a local copy. This way, the state remains unaltered, and it can
+     * continue ingesting more input afterwards.
+     */
+    memcpy(acc, state->acc, sizeof(state->acc));
+    if (state->bufferedSize >= XXH_STRIPE_LEN) {
+        size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN;
+        size_t nbStripesSoFar = state->nbStripesSoFar;
+        XXH3_consumeStripes(acc,
+                           &nbStripesSoFar, state->nbStripesPerBlock,
+                            state->buffer, nbStripes,
+                            secret, state->secretLimit,
+                            XXH3_accumulate_512, XXH3_scrambleAcc);
+        /* last stripe */
+        XXH3_accumulate_512(acc,
+                            state->buffer + state->bufferedSize - XXH_STRIPE_LEN,
+                            secret + state->secretLimit - XXH_SECRET_LASTACC_START);
+    } else {  /* bufferedSize < XXH_STRIPE_LEN */
+        xxh_u8 lastStripe[XXH_STRIPE_LEN];
+        size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize;
+        XXH_ASSERT(state->bufferedSize > 0);  /* there is always some input buffered */
+        memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize);
+        memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize);
+        XXH3_accumulate_512(acc,
+                            lastStripe,
+                            secret + state->secretLimit - XXH_SECRET_LASTACC_START);
+    }
+}
+
+XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* state)
+{
+    const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
+    if (state->totalLen > XXH3_MIDSIZE_MAX) {
+        XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
+        XXH3_digest_long(acc, state, secret);
+        return XXH3_mergeAccs(acc,
+                              secret + XXH_SECRET_MERGEACCS_START,
+                              (xxh_u64)state->totalLen * XXH_PRIME64_1);
+    }
+    /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */
+    if (state->seed)
+        return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
+    return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen),
+                                  secret, state->secretLimit + XXH_STRIPE_LEN);
+}
+
+
+#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x))
+
+XXH_PUBLIC_API void
+XXH3_generateSecret(void* secretBuffer, const void* customSeed, size_t customSeedSize)
+{
+    XXH_ASSERT(secretBuffer != NULL);
+    if (customSeedSize == 0) {
+        memcpy(secretBuffer, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE);
+        return;
+    }
+    XXH_ASSERT(customSeed != NULL);
+
+    {   size_t const segmentSize = sizeof(XXH128_hash_t);
+        size_t const nbSegments = XXH_SECRET_DEFAULT_SIZE / segmentSize;
+        XXH128_canonical_t scrambler;
+        XXH64_hash_t seeds[12];
+        size_t segnb;
+        XXH_ASSERT(nbSegments == 12);
+        XXH_ASSERT(segmentSize * nbSegments == XXH_SECRET_DEFAULT_SIZE); /* exact multiple */
+        XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0));
+
+        /*
+        * Copy customSeed to seeds[], truncating or repeating as necessary.
+        */
+        {   size_t toFill = XXH_MIN(customSeedSize, sizeof(seeds));
+            size_t filled = toFill;
+            memcpy(seeds, customSeed, toFill);
+            while (filled < sizeof(seeds)) {
+                toFill = XXH_MIN(filled, sizeof(seeds) - filled);
+                memcpy((char*)seeds + filled, seeds, toFill);
+                filled += toFill;
+        }   }
+
+        /* generate secret */
+        memcpy(secretBuffer, &scrambler, sizeof(scrambler));
+        for (segnb=1; segnb < nbSegments; segnb++) {
+            size_t const segmentStart = segnb * segmentSize;
+            XXH128_canonical_t segment;
+            XXH128_canonicalFromHash(&segment,
+                XXH128(&scrambler, sizeof(scrambler), XXH_readLE64(seeds + segnb) + segnb) );
+            memcpy((char*)secretBuffer + segmentStart, &segment, sizeof(segment));
+    }   }
+}
+
+
+/* ==========================================
+ * XXH3 128 bits (a.k.a XXH128)
+ * ==========================================
+ * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant,
+ * even without counting the significantly larger output size.
+ *
+ * For example, extra steps are taken to avoid the seed-dependent collisions
+ * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B).
+ *
+ * This strength naturally comes at the cost of some speed, especially on short
+ * lengths. Note that longer hashes are about as fast as the 64-bit version
+ * due to it using only a slight modification of the 64-bit loop.
+ *
+ * XXH128 is also more oriented towards 64-bit machines. It is still extremely
+ * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64).
+ */
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+    /* A doubled version of 1to3_64b with different constants. */
+    XXH_ASSERT(input != NULL);
+    XXH_ASSERT(1 <= len && len <= 3);
+    XXH_ASSERT(secret != NULL);
+    /*
+     * len = 1: combinedl = { input[0], 0x01, input[0], input[0] }
+     * len = 2: combinedl = { input[1], 0x02, input[0], input[1] }
+     * len = 3: combinedl = { input[2], 0x03, input[0], input[1] }
+     */
+    {   xxh_u8 const c1 = input[0];
+        xxh_u8 const c2 = input[len >> 1];
+        xxh_u8 const c3 = input[len - 1];
+        xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24)
+                                | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8);
+        xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13);
+        xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
+        xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed;
+        xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl;
+        xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph;
+        XXH128_hash_t h128;
+        h128.low64  = XXH64_avalanche(keyed_lo);
+        h128.high64 = XXH64_avalanche(keyed_hi);
+        return h128;
+    }
+}
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+    XXH_ASSERT(input != NULL);
+    XXH_ASSERT(secret != NULL);
+    XXH_ASSERT(4 <= len && len <= 8);
+    seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
+    {   xxh_u32 const input_lo = XXH_readLE32(input);
+        xxh_u32 const input_hi = XXH_readLE32(input + len - 4);
+        xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32);
+        xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed;
+        xxh_u64 const keyed = input_64 ^ bitflip;
+
+        /* Shift len to the left to ensure it is even, this avoids even multiplies. */
+        XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2));
+
+        m128.high64 += (m128.low64 << 1);
+        m128.low64  ^= (m128.high64 >> 3);
+
+        m128.low64   = XXH_xorshift64(m128.low64, 35);
+        m128.low64  *= 0x9FB21C651E98DF25ULL;
+        m128.low64   = XXH_xorshift64(m128.low64, 28);
+        m128.high64  = XXH3_avalanche(m128.high64);
+        return m128;
+    }
+}
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+    XXH_ASSERT(input != NULL);
+    XXH_ASSERT(secret != NULL);
+    XXH_ASSERT(9 <= len && len <= 16);
+    {   xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed;
+        xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed;
+        xxh_u64 const input_lo = XXH_readLE64(input);
+        xxh_u64       input_hi = XXH_readLE64(input + len - 8);
+        XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1);
+        /*
+         * Put len in the middle of m128 to ensure that the length gets mixed to
+         * both the low and high bits in the 128x64 multiply below.
+         */
+        m128.low64 += (xxh_u64)(len - 1) << 54;
+        input_hi   ^= bitfliph;
+        /*
+         * Add the high 32 bits of input_hi to the high 32 bits of m128, then
+         * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to
+         * the high 64 bits of m128.
+         *
+         * The best approach to this operation is different on 32-bit and 64-bit.
+         */
+        if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */
+            /*
+             * 32-bit optimized version, which is more readable.
+             *
+             * On 32-bit, it removes an ADC and delays a dependency between the two
+             * halves of m128.high64, but it generates an extra mask on 64-bit.
+             */
+            m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2);
+        } else {
+            /*
+             * 64-bit optimized (albeit more confusing) version.
+             *
+             * Uses some properties of addition and multiplication to remove the mask:
+             *
+             * Let:
+             *    a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF)
+             *    b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000)
+             *    c = XXH_PRIME32_2
+             *
+             *    a + (b * c)
+             * Inverse Property: x + y - x == y
+             *    a + (b * (1 + c - 1))
+             * Distributive Property: x * (y + z) == (x * y) + (x * z)
+             *    a + (b * 1) + (b * (c - 1))
+             * Identity Property: x * 1 == x
+             *    a + b + (b * (c - 1))
+             *
+             * Substitute a, b, and c:
+             *    input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1))
+             *
+             * Since input_hi.hi + input_hi.lo == input_hi, we get this:
+             *    input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1))
+             */
+            m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1);
+        }
+        /* m128 ^= XXH_swap64(m128 >> 64); */
+        m128.low64  ^= XXH_swap64(m128.high64);
+
+        {   /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */
+            XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2);
+            h128.high64 += m128.high64 * XXH_PRIME64_2;
+
+            h128.low64   = XXH3_avalanche(h128.low64);
+            h128.high64  = XXH3_avalanche(h128.high64);
+            return h128;
+    }   }
+}
+
+/*
+ * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN
+ */
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+    XXH_ASSERT(len <= 16);
+    {   if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed);
+        if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed);
+        if (len) return XXH3_len_1to3_128b(input, len, secret, seed);
+        {   XXH128_hash_t h128;
+            xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72);
+            xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88);
+            h128.low64 = XXH64_avalanche(seed ^ bitflipl);
+            h128.high64 = XXH64_avalanche( seed ^ bitfliph);
+            return h128;
+    }   }
+}
+
+/*
+ * A bit slower than XXH3_mix16B, but handles multiply by zero better.
+ */
+XXH_FORCE_INLINE XXH128_hash_t
+XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2,
+              const xxh_u8* secret, XXH64_hash_t seed)
+{
+    acc.low64  += XXH3_mix16B (input_1, secret+0, seed);
+    acc.low64  ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8);
+    acc.high64 += XXH3_mix16B (input_2, secret+16, seed);
+    acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8);
+    return acc;
+}
+
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
+                      const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+                      XXH64_hash_t seed)
+{
+    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
+    XXH_ASSERT(16 < len && len <= 128);
+
+    {   XXH128_hash_t acc;
+        acc.low64 = len * XXH_PRIME64_1;
+        acc.high64 = 0;
+        if (len > 32) {
+            if (len > 64) {
+                if (len > 96) {
+                    acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed);
+                }
+                acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed);
+            }
+            acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed);
+        }
+        acc = XXH128_mix32B(acc, input, input+len-16, secret, seed);
+        {   XXH128_hash_t h128;
+            h128.low64  = acc.low64 + acc.high64;
+            h128.high64 = (acc.low64    * XXH_PRIME64_1)
+                        + (acc.high64   * XXH_PRIME64_4)
+                        + ((len - seed) * XXH_PRIME64_2);
+            h128.low64  = XXH3_avalanche(h128.low64);
+            h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64);
+            return h128;
+        }
+    }
+}
+
+XXH_NO_INLINE XXH128_hash_t
+XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
+                       const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+                       XXH64_hash_t seed)
+{
+    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
+    XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
+
+    {   XXH128_hash_t acc;
+        int const nbRounds = (int)len / 32;
+        int i;
+        acc.low64 = len * XXH_PRIME64_1;
+        acc.high64 = 0;
+        for (i=0; i<4; i++) {
+            acc = XXH128_mix32B(acc,
+                                input  + (32 * i),
+                                input  + (32 * i) + 16,
+                                secret + (32 * i),
+                                seed);
+        }
+        acc.low64 = XXH3_avalanche(acc.low64);
+        acc.high64 = XXH3_avalanche(acc.high64);
+        XXH_ASSERT(nbRounds >= 4);
+        for (i=4 ; i < nbRounds; i++) {
+            acc = XXH128_mix32B(acc,
+                                input + (32 * i),
+                                input + (32 * i) + 16,
+                                secret + XXH3_MIDSIZE_STARTOFFSET + (32 * (i - 4)),
+                                seed);
+        }
+        /* last bytes */
+        acc = XXH128_mix32B(acc,
+                            input + len - 16,
+                            input + len - 32,
+                            secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16,
+                            0ULL - seed);
+
+        {   XXH128_hash_t h128;
+            h128.low64  = acc.low64 + acc.high64;
+            h128.high64 = (acc.low64    * XXH_PRIME64_1)
+                        + (acc.high64   * XXH_PRIME64_4)
+                        + ((len - seed) * XXH_PRIME64_2);
+            h128.low64  = XXH3_avalanche(h128.low64);
+            h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64);
+            return h128;
+        }
+    }
+}
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len,
+                            const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+                            XXH3_f_accumulate_512 f_acc512,
+                            XXH3_f_scrambleAcc f_scramble)
+{
+    XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC;
+
+    XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc512, f_scramble);
+
+    /* converge into final hash */
+    XXH_STATIC_ASSERT(sizeof(acc) == 64);
+    XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
+    {   XXH128_hash_t h128;
+        h128.low64  = XXH3_mergeAccs(acc,
+                                     secret + XXH_SECRET_MERGEACCS_START,
+                                     (xxh_u64)len * XXH_PRIME64_1);
+        h128.high64 = XXH3_mergeAccs(acc,
+                                     secret + secretSize
+                                            - sizeof(acc) - XXH_SECRET_MERGEACCS_START,
+                                     ~((xxh_u64)len * XXH_PRIME64_2));
+        return h128;
+    }
+}
+
+/*
+ * It's important for performance that XXH3_hashLong is not inlined.
+ */
+XXH_NO_INLINE XXH128_hash_t
+XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len,
+                           XXH64_hash_t seed64,
+                           const void* XXH_RESTRICT secret, size_t secretLen)
+{
+    (void)seed64; (void)secret; (void)secretLen;
+    return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret),
+                                       XXH3_accumulate_512, XXH3_scrambleAcc);
+}
+
+/*
+ * It's important for performance that XXH3_hashLong is not inlined.
+ */
+XXH_NO_INLINE XXH128_hash_t
+XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len,
+                              XXH64_hash_t seed64,
+                              const void* XXH_RESTRICT secret, size_t secretLen)
+{
+    (void)seed64;
+    return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen,
+                                       XXH3_accumulate_512, XXH3_scrambleAcc);
+}
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len,
+                                XXH64_hash_t seed64,
+                                XXH3_f_accumulate_512 f_acc512,
+                                XXH3_f_scrambleAcc f_scramble,
+                                XXH3_f_initCustomSecret f_initSec)
+{
+    if (seed64 == 0)
+        return XXH3_hashLong_128b_internal(input, len,
+                                           XXH3_kSecret, sizeof(XXH3_kSecret),
+                                           f_acc512, f_scramble);
+    {   XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
+        f_initSec(secret, seed64);
+        return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret),
+                                           f_acc512, f_scramble);
+    }
+}
+
+/*
+ * It's important for performance that XXH3_hashLong is not inlined.
+ */
+XXH_NO_INLINE XXH128_hash_t
+XXH3_hashLong_128b_withSeed(const void* input, size_t len,
+                            XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen)
+{
+    (void)secret; (void)secretLen;
+    return XXH3_hashLong_128b_withSeed_internal(input, len, seed64,
+                XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret);
+}
+
+typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t,
+                                            XXH64_hash_t, const void* XXH_RESTRICT, size_t);
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_128bits_internal(const void* input, size_t len,
+                      XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen,
+                      XXH3_hashLong128_f f_hl128)
+{
+    XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN);
+    /*
+     * If an action is to be taken if `secret` conditions are not respected,
+     * it should be done here.
+     * For now, it's a contract pre-condition.
+     * Adding a check and a branch here would cost performance at every hash.
+     */
+    if (len <= 16)
+        return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64);
+    if (len <= 128)
+        return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
+    if (len <= XXH3_MIDSIZE_MAX)
+        return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
+    return f_hl128(input, len, seed64, secret, secretLen);
+}
+
+
+/* ===   Public XXH128 API   === */
+
+XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* input, size_t len)
+{
+    return XXH3_128bits_internal(input, len, 0,
+                                 XXH3_kSecret, sizeof(XXH3_kSecret),
+                                 XXH3_hashLong_128b_default);
+}
+
+XXH_PUBLIC_API XXH128_hash_t
+XXH3_128bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize)
+{
+    return XXH3_128bits_internal(input, len, 0,
+                                 (const xxh_u8*)secret, secretSize,
+                                 XXH3_hashLong_128b_withSecret);
+}
+
+XXH_PUBLIC_API XXH128_hash_t
+XXH3_128bits_withSeed(const void* input, size_t len, XXH64_hash_t seed)
+{
+    return XXH3_128bits_internal(input, len, seed,
+                                 XXH3_kSecret, sizeof(XXH3_kSecret),
+                                 XXH3_hashLong_128b_withSeed);
+}
+
+XXH_PUBLIC_API XXH128_hash_t
+XXH128(const void* input, size_t len, XXH64_hash_t seed)
+{
+    return XXH3_128bits_withSeed(input, len, seed);
+}
+
+
+/* ===   XXH3 128-bit streaming   === */
+
+/*
+ * All the functions are actually the same as for 64-bit streaming variant.
+ * The only difference is the finalizatiom routine.
+ */
+
+static void
+XXH3_128bits_reset_internal(XXH3_state_t* statePtr,
+                            XXH64_hash_t seed,
+                            const void* secret, size_t secretSize)
+{
+    XXH3_64bits_reset_internal(statePtr, seed, secret, secretSize);
+}
+
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_reset(XXH3_state_t* statePtr)
+{
+    if (statePtr == NULL) return XXH_ERROR;
+    XXH3_128bits_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE);
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize)
+{
+    if (statePtr == NULL) return XXH_ERROR;
+    XXH3_128bits_reset_internal(statePtr, 0, secret, secretSize);
+    if (secret == NULL) return XXH_ERROR;
+    if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed)
+{
+    if (statePtr == NULL) return XXH_ERROR;
+    if (seed==0) return XXH3_128bits_reset(statePtr);
+    if (seed != statePtr->seed) XXH3_initCustomSecret(statePtr->customSecret, seed);
+    XXH3_128bits_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE);
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_update(XXH3_state_t* state, const void* input, size_t len)
+{
+    return XXH3_update(state, (const xxh_u8*)input, len,
+                       XXH3_accumulate_512, XXH3_scrambleAcc);
+}
+
+XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* state)
+{
+    const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
+    if (state->totalLen > XXH3_MIDSIZE_MAX) {
+        XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
+        XXH3_digest_long(acc, state, secret);
+        XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
+        {   XXH128_hash_t h128;
+            h128.low64  = XXH3_mergeAccs(acc,
+                                         secret + XXH_SECRET_MERGEACCS_START,
+                                         (xxh_u64)state->totalLen * XXH_PRIME64_1);
+            h128.high64 = XXH3_mergeAccs(acc,
+                                         secret + state->secretLimit + XXH_STRIPE_LEN
+                                                - sizeof(acc) - XXH_SECRET_MERGEACCS_START,
+                                         ~((xxh_u64)state->totalLen * XXH_PRIME64_2));
+            return h128;
+        }
+    }
+    /* len <= XXH3_MIDSIZE_MAX : short code */
+    if (state->seed)
+        return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
+    return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen),
+                                   secret, state->secretLimit + XXH_STRIPE_LEN);
+}
+
+/* 128-bit utility functions */
+
+#include <string.h>   /* memcmp, memcpy */
+
+/* return : 1 is equal, 0 if different */
+XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2)
+{
+    /* note : XXH128_hash_t is compact, it has no padding byte */
+    return !(memcmp(&h1, &h2, sizeof(h1)));
+}
+
+/* This prototype is compatible with stdlib's qsort().
+ * return : >0 if *h128_1  > *h128_2
+ *          <0 if *h128_1  < *h128_2
+ *          =0 if *h128_1 == *h128_2  */
+XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2)
+{
+    XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1;
+    XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2;
+    int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64);
+    /* note : bets that, in most cases, hash values are different */
+    if (hcmp) return hcmp;
+    return (h1.low64 > h2.low64) - (h2.low64 > h1.low64);
+}
+
+
+/*======   Canonical representation   ======*/
+XXH_PUBLIC_API void
+XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash)
+{
+    XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t));
+    if (XXH_CPU_LITTLE_ENDIAN) {
+        hash.high64 = XXH_swap64(hash.high64);
+        hash.low64  = XXH_swap64(hash.low64);
+    }
+    memcpy(dst, &hash.high64, sizeof(hash.high64));
+    memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64));
+}
+
+XXH_PUBLIC_API XXH128_hash_t
+XXH128_hashFromCanonical(const XXH128_canonical_t* src)
+{
+    XXH128_hash_t h;
+    h.high64 = XXH_readBE64(src);
+    h.low64  = XXH_readBE64(src->digest + 8);
+    return h;
+}
+
+/* Pop our optimization override from above */
+#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
+  && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
+  && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */
+#  pragma GCC pop_options
+#endif
+
+#endif  /* XXH_NO_LONG_LONG */
+
+
+#endif  /* XXH_IMPLEMENTATION */
+
+
+#if defined (__cplusplus)
+}
+#endif
diff --git a/cpp/src/arrow/visitor.cc b/cpp/src/arrow/visitor.cc
index 0a452d5c594..851785081c7 100644
--- a/cpp/src/arrow/visitor.cc
+++ b/cpp/src/arrow/visitor.cc
@@ -67,6 +67,7 @@ ARRAY_VISITOR_DEFAULT(SparseUnionArray)
 ARRAY_VISITOR_DEFAULT(DenseUnionArray)
 ARRAY_VISITOR_DEFAULT(DictionaryArray)
 ARRAY_VISITOR_DEFAULT(Decimal128Array)
+ARRAY_VISITOR_DEFAULT(Decimal256Array)
 ARRAY_VISITOR_DEFAULT(ExtensionArray)
 
 #undef ARRAY_VISITOR_DEFAULT
@@ -106,6 +107,7 @@ TYPE_VISITOR_DEFAULT(DayTimeIntervalType)
 TYPE_VISITOR_DEFAULT(MonthIntervalType)
 TYPE_VISITOR_DEFAULT(DurationType)
 TYPE_VISITOR_DEFAULT(Decimal128Type)
+TYPE_VISITOR_DEFAULT(Decimal256Type)
 TYPE_VISITOR_DEFAULT(ListType)
 TYPE_VISITOR_DEFAULT(LargeListType)
 TYPE_VISITOR_DEFAULT(MapType)
@@ -154,6 +156,7 @@ SCALAR_VISITOR_DEFAULT(DayTimeIntervalScalar)
 SCALAR_VISITOR_DEFAULT(MonthIntervalScalar)
 SCALAR_VISITOR_DEFAULT(DurationScalar)
 SCALAR_VISITOR_DEFAULT(Decimal128Scalar)
+SCALAR_VISITOR_DEFAULT(Decimal256Scalar)
 SCALAR_VISITOR_DEFAULT(ListScalar)
 SCALAR_VISITOR_DEFAULT(LargeListScalar)
 SCALAR_VISITOR_DEFAULT(MapScalar)
diff --git a/cpp/src/arrow/visitor.h b/cpp/src/arrow/visitor.h
index 7ab136c066f..0382e461199 100644
--- a/cpp/src/arrow/visitor.h
+++ b/cpp/src/arrow/visitor.h
@@ -54,6 +54,7 @@ class ARROW_EXPORT ArrayVisitor {
   virtual Status Visit(const MonthIntervalArray& array);
   virtual Status Visit(const DurationArray& array);
   virtual Status Visit(const Decimal128Array& array);
+  virtual Status Visit(const Decimal256Array& array);
   virtual Status Visit(const ListArray& array);
   virtual Status Visit(const LargeListArray& array);
   virtual Status Visit(const MapArray& array);
@@ -96,6 +97,7 @@ class ARROW_EXPORT TypeVisitor {
   virtual Status Visit(const DayTimeIntervalType& type);
   virtual Status Visit(const DurationType& type);
   virtual Status Visit(const Decimal128Type& type);
+  virtual Status Visit(const Decimal256Type& type);
   virtual Status Visit(const ListType& type);
   virtual Status Visit(const LargeListType& type);
   virtual Status Visit(const MapType& type);
@@ -138,6 +140,7 @@ class ARROW_EXPORT ScalarVisitor {
   virtual Status Visit(const MonthIntervalScalar& scalar);
   virtual Status Visit(const DurationScalar& scalar);
   virtual Status Visit(const Decimal128Scalar& scalar);
+  virtual Status Visit(const Decimal256Scalar& scalar);
   virtual Status Visit(const ListScalar& scalar);
   virtual Status Visit(const LargeListScalar& scalar);
   virtual Status Visit(const MapScalar& scalar);
diff --git a/cpp/src/arrow/visitor_inline.h b/cpp/src/arrow/visitor_inline.h
index bff97fcd9eb..45193f20413 100644
--- a/cpp/src/arrow/visitor_inline.h
+++ b/cpp/src/arrow/visitor_inline.h
@@ -68,6 +68,7 @@ namespace arrow {
   ACTION(MonthInterval);                        \
   ACTION(DayTimeInterval);                      \
   ACTION(Decimal128);                           \
+  ACTION(Decimal256);                           \
   ACTION(List);                                 \
   ACTION(LargeList);                            \
   ACTION(Map);                                  \
diff --git a/cpp/src/gandiva/CMakeLists.txt b/cpp/src/gandiva/CMakeLists.txt
index 85e8db60252..0ae5a193f53 100644
--- a/cpp/src/gandiva/CMakeLists.txt
+++ b/cpp/src/gandiva/CMakeLists.txt
@@ -106,7 +106,7 @@ endif()
 #     set(GANDIVA_SHARED_LINK_FLAGS "${GANDIVA_SHARED_LINK_FLAGS} /EXPORT:${SYMBOL}")
 #   endforeach()
 # endif()
-if(NOT APPLE AND NOT MSVC)
+if(NOT APPLE AND NOT MSVC_TOOLCHAIN)
   # Localize thirdparty symbols using a linker version script. This hides them
   # from the client application. The OS X linker does not support the
   # version-script option.
@@ -224,6 +224,7 @@ add_gandiva_test(internals-test
                  like_holder_test.cc
                  decimal_type_util_test.cc
                  random_generator_holder_test.cc
+                 gdv_function_stubs_test.cc
                  EXTRA_DEPENDENCIES
                  LLVM::LLVM_INTERFACE
                  EXTRA_INCLUDES
diff --git a/cpp/src/gandiva/engine.cc b/cpp/src/gandiva/engine.cc
index ad73751ecdc..a29a7e0ba4f 100644
--- a/cpp/src/gandiva/engine.cc
+++ b/cpp/src/gandiva/engine.cc
@@ -53,6 +53,7 @@
 #include <llvm/Linker/Linker.h>
 #include <llvm/MC/SubtargetFeature.h>
 #include <llvm/Support/DynamicLibrary.h>
+#include <llvm/Support/Host.h>
 #include <llvm/Support/TargetRegistry.h>
 #include <llvm/Support/TargetSelect.h>
 #include <llvm/Support/raw_ostream.h>
@@ -122,7 +123,7 @@ Status Engine::Make(const std::shared_ptr<Configuration>& conf,
   auto ctx = arrow::internal::make_unique<llvm::LLVMContext>();
   auto module = arrow::internal::make_unique<llvm::Module>("codegen", *ctx);
 
-  // Capture before moving, ExceutionEngine does not allow retrieving the
+  // Capture before moving, ExecutionEngine does not allow retrieving the
   // original Module.
   auto module_ptr = module.get();
 
diff --git a/cpp/src/gandiva/function_registry_arithmetic.cc b/cpp/src/gandiva/function_registry_arithmetic.cc
index 0f8ffd34bd8..2c3049a8b74 100644
--- a/cpp/src/gandiva/function_registry_arithmetic.cc
+++ b/cpp/src/gandiva/function_registry_arithmetic.cc
@@ -82,9 +82,21 @@ std::vector<NativeFunction> GetArithmeticFunctionRegistry() {
       BINARY_SYMMETRIC_SAFE_NULL_IF_NULL(bitwise_and, {}, int64),
       BINARY_SYMMETRIC_SAFE_NULL_IF_NULL(bitwise_or, {}, int32),
       BINARY_SYMMETRIC_SAFE_NULL_IF_NULL(bitwise_or, {}, int64),
+      BINARY_SYMMETRIC_SAFE_NULL_IF_NULL(bitwise_xor, {}, int32),
+      BINARY_SYMMETRIC_SAFE_NULL_IF_NULL(bitwise_xor, {}, int64),
       UNARY_SAFE_NULL_IF_NULL(bitwise_not, {}, int32, int32),
       UNARY_SAFE_NULL_IF_NULL(bitwise_not, {}, int64, int64),
 
+      // round functions
+      UNARY_SAFE_NULL_IF_NULL(round, {}, float32, float32),
+      UNARY_SAFE_NULL_IF_NULL(round, {}, float64, float64),
+      BINARY_GENERIC_SAFE_NULL_IF_NULL(round, {}, float32, int32, float32),
+      BINARY_GENERIC_SAFE_NULL_IF_NULL(round, {}, float64, int32, float64),
+      UNARY_SAFE_NULL_IF_NULL(round, {}, int32, int32),
+      UNARY_SAFE_NULL_IF_NULL(round, {}, int64, int64),
+      BINARY_GENERIC_SAFE_NULL_IF_NULL(round, {}, int32, int32, int32),
+      BINARY_GENERIC_SAFE_NULL_IF_NULL(round, {}, int64, int32, int64),
+
       // compare functions
       BINARY_RELATIONAL_BOOL_FN(equal, ({"eq", "same"})),
       BINARY_RELATIONAL_BOOL_FN(not_equal, {}),
diff --git a/cpp/src/gandiva/function_registry_datetime.cc b/cpp/src/gandiva/function_registry_datetime.cc
index 9100f3805d7..cd4ae004039 100644
--- a/cpp/src/gandiva/function_registry_datetime.cc
+++ b/cpp/src/gandiva/function_registry_datetime.cc
@@ -64,6 +64,9 @@ std::vector<NativeFunction> GetDateTimeFunctionRegistry() {
       NativeFunction("castTIMESTAMP", {}, DataTypeVector{date64()}, timestamp(),
                      kResultNullIfNull, "castTIMESTAMP_date64"),
 
+      NativeFunction("castTIMESTAMP", {}, DataTypeVector{int64()}, timestamp(),
+                     kResultNullIfNull, "castTIMESTAMP_int64"),
+
       NativeFunction("castDATE", {"to_date"}, DataTypeVector{timestamp()}, date64(),
                      kResultNullIfNull, "castDATE_timestamp"),
 
diff --git a/cpp/src/gandiva/function_registry_string.cc b/cpp/src/gandiva/function_registry_string.cc
index dd32c19ba17..ea3af5b45c9 100644
--- a/cpp/src/gandiva/function_registry_string.cc
+++ b/cpp/src/gandiva/function_registry_string.cc
@@ -16,6 +16,7 @@
 // under the License.
 
 #include "gandiva/function_registry_string.h"
+
 #include "gandiva/function_registry_common.h"
 
 namespace gandiva {
@@ -48,8 +49,7 @@ std::vector<NativeFunction> GetStringFunctionRegistry() {
 
       BINARY_UNSAFE_NULL_IF_NULL(locate, {"position"}, utf8, int32),
 
-      UNARY_OCTET_LEN_FN(octet_length, {}),
-      UNARY_OCTET_LEN_FN(bit_length, {}),
+      UNARY_OCTET_LEN_FN(octet_length, {}), UNARY_OCTET_LEN_FN(bit_length, {}),
 
       UNARY_UNSAFE_NULL_IF_NULL(char_length, {}, utf8, int32),
       UNARY_UNSAFE_NULL_IF_NULL(length, {}, utf8, int32),
@@ -62,17 +62,26 @@ std::vector<NativeFunction> GetStringFunctionRegistry() {
       UNARY_SAFE_NULL_NEVER_BOOL_FN(isnull, {}),
       UNARY_SAFE_NULL_NEVER_BOOL_FN(isnotnull, {}),
 
-      UNARY_UNSAFE_NULL_IF_NULL(castINT, {}, utf8, int32),
-      UNARY_UNSAFE_NULL_IF_NULL(castBIGINT, {}, utf8, int64),
-      UNARY_UNSAFE_NULL_IF_NULL(castFLOAT4, {}, utf8, float32),
-      UNARY_UNSAFE_NULL_IF_NULL(castFLOAT8, {}, utf8, float64),
-
       NativeFunction("upper", {}, DataTypeVector{utf8()}, utf8(), kResultNullIfNull,
                      "upper_utf8", NativeFunction::kNeedsContext),
 
       NativeFunction("lower", {}, DataTypeVector{utf8()}, utf8(), kResultNullIfNull,
                      "lower_utf8", NativeFunction::kNeedsContext),
 
+      NativeFunction("castINT", {}, DataTypeVector{utf8()}, int32(), kResultNullIfNull,
+                     "gdv_fn_castINT_utf8", NativeFunction::kNeedsContext),
+
+      NativeFunction("castBIGINT", {}, DataTypeVector{utf8()}, int64(), kResultNullIfNull,
+                     "gdv_fn_castBIGINT_utf8", NativeFunction::kNeedsContext),
+
+      NativeFunction("castFLOAT4", {}, DataTypeVector{utf8()}, float32(),
+                     kResultNullIfNull, "gdv_fn_castFLOAT4_utf8",
+                     NativeFunction::kNeedsContext),
+
+      NativeFunction("castFLOAT8", {}, DataTypeVector{utf8()}, float64(),
+                     kResultNullIfNull, "gdv_fn_castFLOAT8_utf8",
+                     NativeFunction::kNeedsContext),
+
       NativeFunction("castVARCHAR", {}, DataTypeVector{utf8(), int64()}, utf8(),
                      kResultNullIfNull, "castVARCHAR_utf8_int64",
                      NativeFunction::kNeedsContext),
@@ -106,11 +115,84 @@ std::vector<NativeFunction> GetStringFunctionRegistry() {
       NativeFunction("concatOperator", {}, DataTypeVector{utf8(), utf8()}, utf8(),
                      kResultNullIfNull, "concatOperator_utf8_utf8",
                      NativeFunction::kNeedsContext),
+      NativeFunction("concatOperator", {}, DataTypeVector{utf8(), utf8(), utf8()}, utf8(),
+                     kResultNullIfNull, "concatOperator_utf8_utf8_utf8",
+                     NativeFunction::kNeedsContext),
+      NativeFunction("concatOperator", {}, DataTypeVector{utf8(), utf8(), utf8(), utf8()},
+                     utf8(), kResultNullIfNull, "concatOperator_utf8_utf8_utf8_utf8",
+                     NativeFunction::kNeedsContext),
+      NativeFunction("concatOperator", {},
+                     DataTypeVector{utf8(), utf8(), utf8(), utf8(), utf8()}, utf8(),
+                     kResultNullIfNull, "concatOperator_utf8_utf8_utf8_utf8_utf8",
+                     NativeFunction::kNeedsContext),
+      NativeFunction("concatOperator", {},
+                     DataTypeVector{utf8(), utf8(), utf8(), utf8(), utf8(), utf8()},
+                     utf8(), kResultNullIfNull,
+                     "concatOperator_utf8_utf8_utf8_utf8_utf8_utf8",
+                     NativeFunction::kNeedsContext),
+      NativeFunction(
+          "concatOperator", {},
+          DataTypeVector{utf8(), utf8(), utf8(), utf8(), utf8(), utf8(), utf8()}, utf8(),
+          kResultNullIfNull, "concatOperator_utf8_utf8_utf8_utf8_utf8_utf8_utf8",
+          NativeFunction::kNeedsContext),
+      NativeFunction(
+          "concatOperator", {},
+          DataTypeVector{utf8(), utf8(), utf8(), utf8(), utf8(), utf8(), utf8(), utf8()},
+          utf8(), kResultNullIfNull,
+          "concatOperator_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8",
+          NativeFunction::kNeedsContext),
+      NativeFunction("concatOperator", {},
+                     DataTypeVector{utf8(), utf8(), utf8(), utf8(), utf8(), utf8(),
+                                    utf8(), utf8(), utf8()},
+                     utf8(), kResultNullIfNull,
+                     "concatOperator_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8",
+                     NativeFunction::kNeedsContext),
+      NativeFunction("concatOperator", {},
+                     DataTypeVector{utf8(), utf8(), utf8(), utf8(), utf8(), utf8(),
+                                    utf8(), utf8(), utf8(), utf8()},
+                     utf8(), kResultNullIfNull,
+                     "concatOperator_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8",
+                     NativeFunction::kNeedsContext),
 
       // concat treats null inputs as empty strings whereas concatOperator returns null if
       // one of the inputs is null
       NativeFunction("concat", {}, DataTypeVector{utf8(), utf8()}, utf8(),
                      kResultNullNever, "concat_utf8_utf8", NativeFunction::kNeedsContext),
+      NativeFunction("concat", {}, DataTypeVector{utf8(), utf8(), utf8()}, utf8(),
+                     kResultNullNever, "concat_utf8_utf8_utf8",
+                     NativeFunction::kNeedsContext),
+      NativeFunction("concat", {}, DataTypeVector{utf8(), utf8(), utf8(), utf8()}, utf8(),
+                     kResultNullNever, "concat_utf8_utf8_utf8_utf8",
+                     NativeFunction::kNeedsContext),
+      NativeFunction("concat", {}, DataTypeVector{utf8(), utf8(), utf8(), utf8(), utf8()},
+                     utf8(), kResultNullNever, "concat_utf8_utf8_utf8_utf8_utf8",
+                     NativeFunction::kNeedsContext),
+      NativeFunction("concat", {},
+                     DataTypeVector{utf8(), utf8(), utf8(), utf8(), utf8(), utf8()},
+                     utf8(), kResultNullNever, "concat_utf8_utf8_utf8_utf8_utf8_utf8",
+                     NativeFunction::kNeedsContext),
+      NativeFunction(
+          "concat", {},
+          DataTypeVector{utf8(), utf8(), utf8(), utf8(), utf8(), utf8(), utf8()}, utf8(),
+          kResultNullNever, "concat_utf8_utf8_utf8_utf8_utf8_utf8_utf8",
+          NativeFunction::kNeedsContext),
+      NativeFunction(
+          "concat", {},
+          DataTypeVector{utf8(), utf8(), utf8(), utf8(), utf8(), utf8(), utf8(), utf8()},
+          utf8(), kResultNullNever, "concat_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8",
+          NativeFunction::kNeedsContext),
+      NativeFunction("concat", {},
+                     DataTypeVector{utf8(), utf8(), utf8(), utf8(), utf8(), utf8(),
+                                    utf8(), utf8(), utf8()},
+                     utf8(), kResultNullNever,
+                     "concat_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8",
+                     NativeFunction::kNeedsContext),
+      NativeFunction("concat", {},
+                     DataTypeVector{utf8(), utf8(), utf8(), utf8(), utf8(), utf8(),
+                                    utf8(), utf8(), utf8(), utf8()},
+                     utf8(), kResultNullNever,
+                     "concat_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8",
+                     NativeFunction::kNeedsContext),
 
       NativeFunction("convert_fromUTF8", {"convert_fromutf8"}, DataTypeVector{binary()},
                      utf8(), kResultNullIfNull, "convert_fromUTF8_binary",
@@ -123,7 +205,13 @@ std::vector<NativeFunction> GetStringFunctionRegistry() {
       NativeFunction("replace", {}, DataTypeVector{utf8(), utf8(), utf8()}, utf8(),
                      kResultNullIfNull, "replace_utf8_utf8_utf8",
                      NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors),
-  };
+
+      NativeFunction("binary_string", {}, DataTypeVector{utf8()}, binary(),
+                     kResultNullIfNull, "binary_string", NativeFunction::kNeedsContext),
+
+      NativeFunction("split_part", {}, DataTypeVector{utf8(), utf8(), int32()}, utf8(),
+                     kResultNullIfNull, "split_part",
+                     NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors)};
 
   return string_fn_registry_;
 }
diff --git a/cpp/src/gandiva/gdv_function_stubs.cc b/cpp/src/gandiva/gdv_function_stubs.cc
index 042f5255641..71e6d728a28 100644
--- a/cpp/src/gandiva/gdv_function_stubs.cc
+++ b/cpp/src/gandiva/gdv_function_stubs.cc
@@ -20,6 +20,7 @@
 #include <string>
 #include <vector>
 
+#include "arrow/util/value_parsing.h"
 #include "gandiva/engine.h"
 #include "gandiva/exported_funcs.h"
 #include "gandiva/in_holder.h"
@@ -83,7 +84,7 @@ bool gdv_fn_in_expr_lookup_utf8(int64_t ptr, const char* data, int data_len,
   }
   gandiva::InHolder<std::string>* holder =
       reinterpret_cast<gandiva::InHolder<std::string>*>(ptr);
-  return holder->HasValue(std::string(data, data_len));
+  return holder->HasValue(arrow::util::string_view(data, data_len));
 }
 
 int32_t gdv_fn_populate_varlen_vector(int64_t context_ptr, int8_t* data_ptr,
@@ -140,6 +141,37 @@ char* gdv_fn_dec_to_string(int64_t context, int64_t x_high, uint64_t x_low,
   memcpy(ret, dec_str.data(), *dec_str_len);
   return ret;
 }
+
+#define CAST_NUMERIC_FROM_STRING(OUT_TYPE, ARROW_TYPE, TYPE_NAME)                    \
+  GANDIVA_EXPORT                                                                     \
+  OUT_TYPE gdv_fn_cast##TYPE_NAME##_utf8(int64_t context, const char* data,          \
+                                         int32_t len) {                              \
+    OUT_TYPE val = 0;                                                                \
+    /* trim leading and trailing spaces */                                           \
+    int32_t trimmed_len;                                                             \
+    int32_t start = 0, end = len - 1;                                                \
+    while (start <= end && data[start] == ' ') {                                     \
+      ++start;                                                                       \
+    }                                                                                \
+    while (end >= start && data[end] == ' ') {                                       \
+      --end;                                                                         \
+    }                                                                                \
+    trimmed_len = end - start + 1;                                                   \
+    const char* trimmed_data = data + start;                                         \
+    if (!arrow::internal::ParseValue<ARROW_TYPE>(trimmed_data, trimmed_len, &val)) { \
+      std::string err =                                                              \
+          "Failed to cast the string " + std::string(data, len) + " to " #OUT_TYPE;  \
+      gdv_fn_context_set_error_msg(context, err.c_str());                            \
+    }                                                                                \
+    return val;                                                                      \
+  }
+
+CAST_NUMERIC_FROM_STRING(int32_t, arrow::Int32Type, INT)
+CAST_NUMERIC_FROM_STRING(int64_t, arrow::Int64Type, BIGINT)
+CAST_NUMERIC_FROM_STRING(float, arrow::FloatType, FLOAT4)
+CAST_NUMERIC_FROM_STRING(double, arrow::DoubleType, FLOAT8)
+
+#undef CAST_NUMERIC_FROM_STRING
 }
 
 namespace gandiva {
@@ -252,6 +284,34 @@ void ExportedStubFunctions::AddMappings(Engine* engine) const {
   args = {types->i64_type(), types->i32_type(), types->i1_type()};
   engine->AddGlobalMappingForFunc("gdv_fn_random_with_seed", types->double_type(), args,
                                   reinterpret_cast<void*>(gdv_fn_random_with_seed));
+
+  args = {types->i64_type(),     // int64_t context_ptr
+          types->i8_ptr_type(),  // const char* data
+          types->i32_type()};    // int32_t lenr
+
+  engine->AddGlobalMappingForFunc("gdv_fn_castINT_utf8", types->i32_type(), args,
+                                  reinterpret_cast<void*>(gdv_fn_castINT_utf8));
+
+  args = {types->i64_type(),     // int64_t context_ptr
+          types->i8_ptr_type(),  // const char* data
+          types->i32_type()};    // int32_t lenr
+
+  engine->AddGlobalMappingForFunc("gdv_fn_castBIGINT_utf8", types->i64_type(), args,
+                                  reinterpret_cast<void*>(gdv_fn_castBIGINT_utf8));
+
+  args = {types->i64_type(),     // int64_t context_ptr
+          types->i8_ptr_type(),  // const char* data
+          types->i32_type()};    // int32_t lenr
+
+  engine->AddGlobalMappingForFunc("gdv_fn_castFLOAT4_utf8", types->float_type(), args,
+                                  reinterpret_cast<void*>(gdv_fn_castFLOAT4_utf8));
+
+  args = {types->i64_type(),     // int64_t context_ptr
+          types->i8_ptr_type(),  // const char* data
+          types->i32_type()};    // int32_t lenr
+
+  engine->AddGlobalMappingForFunc("gdv_fn_castFLOAT8_utf8", types->double_type(), args,
+                                  reinterpret_cast<void*>(gdv_fn_castFLOAT8_utf8));
 }
 
 }  // namespace gandiva
diff --git a/cpp/src/gandiva/gdv_function_stubs.h b/cpp/src/gandiva/gdv_function_stubs.h
index 4d66aa3e987..457f42511cc 100644
--- a/cpp/src/gandiva/gdv_function_stubs.h
+++ b/cpp/src/gandiva/gdv_function_stubs.h
@@ -19,6 +19,8 @@
 
 #include <cstdint>
 
+#include "gandiva/visibility.h"
+
 /// Stub functions that can be accessed from LLVM.
 extern "C" {
 
@@ -52,4 +54,16 @@ int32_t gdv_fn_dec_from_string(int64_t context, const char* in, int32_t in_lengt
 
 char* gdv_fn_dec_to_string(int64_t context, int64_t x_high, uint64_t x_low,
                            int32_t x_scale, int32_t* dec_str_len);
+
+GANDIVA_EXPORT
+int32_t gdv_fn_castINT_utf8(int64_t context, const char* data, int32_t data_len);
+
+GANDIVA_EXPORT
+int64_t gdv_fn_castBIGINT_utf8(int64_t context, const char* data, int32_t data_len);
+
+GANDIVA_EXPORT
+float gdv_fn_castFLOAT4_utf8(int64_t context, const char* data, int32_t data_len);
+
+GANDIVA_EXPORT
+double gdv_fn_castFLOAT8_utf8(int64_t context, const char* data, int32_t data_len);
 }
diff --git a/cpp/src/gandiva/gdv_function_stubs_test.cc b/cpp/src/gandiva/gdv_function_stubs_test.cc
new file mode 100644
index 00000000000..90ac1dfa540
--- /dev/null
+++ b/cpp/src/gandiva/gdv_function_stubs_test.cc
@@ -0,0 +1,163 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gandiva/gdv_function_stubs.h"
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "gandiva/execution_context.h"
+
+namespace gandiva {
+
+TEST(TestGdvFnStubs, TestCastINT) {
+  gandiva::ExecutionContext ctx;
+
+  int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
+
+  EXPECT_EQ(gdv_fn_castINT_utf8(ctx_ptr, "-45", 3), -45);
+  EXPECT_EQ(gdv_fn_castINT_utf8(ctx_ptr, "0", 1), 0);
+  EXPECT_EQ(gdv_fn_castINT_utf8(ctx_ptr, "2147483647", 10), 2147483647);
+  EXPECT_EQ(gdv_fn_castINT_utf8(ctx_ptr, "02147483647", 11), 2147483647);
+  EXPECT_EQ(gdv_fn_castINT_utf8(ctx_ptr, "-2147483648", 11), -2147483648LL);
+  EXPECT_EQ(gdv_fn_castINT_utf8(ctx_ptr, "-02147483648", 12), -2147483648LL);
+  EXPECT_EQ(gdv_fn_castINT_utf8(ctx_ptr, " 12 ", 4), 12);
+
+  gdv_fn_castINT_utf8(ctx_ptr, "2147483648", 10);
+  EXPECT_THAT(ctx.get_error(),
+              ::testing::HasSubstr("Failed to cast the string 2147483648 to int32"));
+  ctx.Reset();
+
+  gdv_fn_castINT_utf8(ctx_ptr, "-2147483649", 11);
+  EXPECT_THAT(ctx.get_error(),
+              ::testing::HasSubstr("Failed to cast the string -2147483649 to int32"));
+  ctx.Reset();
+
+  gdv_fn_castINT_utf8(ctx_ptr, "12.34", 5);
+  EXPECT_THAT(ctx.get_error(),
+              ::testing::HasSubstr("Failed to cast the string 12.34 to int32"));
+  ctx.Reset();
+
+  gdv_fn_castINT_utf8(ctx_ptr, "abc", 3);
+  EXPECT_THAT(ctx.get_error(),
+              ::testing::HasSubstr("Failed to cast the string abc to int32"));
+  ctx.Reset();
+
+  gdv_fn_castINT_utf8(ctx_ptr, "", 0);
+  EXPECT_THAT(ctx.get_error(),
+              ::testing::HasSubstr("Failed to cast the string  to int32"));
+  ctx.Reset();
+
+  gdv_fn_castINT_utf8(ctx_ptr, "-", 1);
+  EXPECT_THAT(ctx.get_error(),
+              ::testing::HasSubstr("Failed to cast the string - to int32"));
+  ctx.Reset();
+}
+
+TEST(TestGdvFnStubs, TestCastBIGINT) {
+  gandiva::ExecutionContext ctx;
+
+  int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
+
+  EXPECT_EQ(gdv_fn_castBIGINT_utf8(ctx_ptr, "-45", 3), -45);
+  EXPECT_EQ(gdv_fn_castBIGINT_utf8(ctx_ptr, "0", 1), 0);
+  EXPECT_EQ(gdv_fn_castBIGINT_utf8(ctx_ptr, "9223372036854775807", 19),
+            9223372036854775807LL);
+  EXPECT_EQ(gdv_fn_castBIGINT_utf8(ctx_ptr, "09223372036854775807", 20),
+            9223372036854775807LL);
+  EXPECT_EQ(gdv_fn_castBIGINT_utf8(ctx_ptr, "-9223372036854775808", 20),
+            -9223372036854775807LL - 1);
+  EXPECT_EQ(gdv_fn_castBIGINT_utf8(ctx_ptr, "-009223372036854775808", 22),
+            -9223372036854775807LL - 1);
+  EXPECT_EQ(gdv_fn_castBIGINT_utf8(ctx_ptr, " 12 ", 4), 12);
+
+  gdv_fn_castBIGINT_utf8(ctx_ptr, "9223372036854775808", 19);
+  EXPECT_THAT(
+      ctx.get_error(),
+      ::testing::HasSubstr("Failed to cast the string 9223372036854775808 to int64"));
+  ctx.Reset();
+
+  gdv_fn_castBIGINT_utf8(ctx_ptr, "-9223372036854775809", 20);
+  EXPECT_THAT(
+      ctx.get_error(),
+      ::testing::HasSubstr("Failed to cast the string -9223372036854775809 to int64"));
+  ctx.Reset();
+
+  gdv_fn_castBIGINT_utf8(ctx_ptr, "12.34", 5);
+  EXPECT_THAT(ctx.get_error(),
+              ::testing::HasSubstr("Failed to cast the string 12.34 to int64"));
+  ctx.Reset();
+
+  gdv_fn_castBIGINT_utf8(ctx_ptr, "abc", 3);
+  EXPECT_THAT(ctx.get_error(),
+              ::testing::HasSubstr("Failed to cast the string abc to int64"));
+  ctx.Reset();
+
+  gdv_fn_castBIGINT_utf8(ctx_ptr, "", 0);
+  EXPECT_THAT(ctx.get_error(),
+              ::testing::HasSubstr("Failed to cast the string  to int64"));
+  ctx.Reset();
+
+  gdv_fn_castBIGINT_utf8(ctx_ptr, "-", 1);
+  EXPECT_THAT(ctx.get_error(),
+              ::testing::HasSubstr("Failed to cast the string - to int64"));
+  ctx.Reset();
+}
+
+TEST(TestGdvFnStubs, TestCastFloat4) {
+  gandiva::ExecutionContext ctx;
+
+  int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
+
+  EXPECT_EQ(gdv_fn_castFLOAT4_utf8(ctx_ptr, "-45.34", 6), -45.34f);
+  EXPECT_EQ(gdv_fn_castFLOAT4_utf8(ctx_ptr, "0", 1), 0.0f);
+  EXPECT_EQ(gdv_fn_castFLOAT4_utf8(ctx_ptr, "5", 1), 5.0f);
+  EXPECT_EQ(gdv_fn_castFLOAT4_utf8(ctx_ptr, " 3.4 ", 5), 3.4f);
+
+  gdv_fn_castFLOAT4_utf8(ctx_ptr, "", 0);
+  EXPECT_THAT(ctx.get_error(),
+              ::testing::HasSubstr("Failed to cast the string  to float"));
+  ctx.Reset();
+
+  gdv_fn_castFLOAT4_utf8(ctx_ptr, "e", 1);
+  EXPECT_THAT(ctx.get_error(),
+              ::testing::HasSubstr("Failed to cast the string e to float"));
+  ctx.Reset();
+}
+
+TEST(TestGdvFnStubs, TestCastFloat8) {
+  gandiva::ExecutionContext ctx;
+
+  int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
+
+  EXPECT_EQ(gdv_fn_castFLOAT8_utf8(ctx_ptr, "-45.34", 6), -45.34);
+  EXPECT_EQ(gdv_fn_castFLOAT8_utf8(ctx_ptr, "0", 1), 0.0);
+  EXPECT_EQ(gdv_fn_castFLOAT8_utf8(ctx_ptr, "5", 1), 5.0);
+  EXPECT_EQ(gdv_fn_castFLOAT8_utf8(ctx_ptr, " 3.4 ", 5), 3.4);
+
+  gdv_fn_castFLOAT8_utf8(ctx_ptr, "", 0);
+  EXPECT_THAT(ctx.get_error(),
+              ::testing::HasSubstr("Failed to cast the string  to double"));
+  ctx.Reset();
+
+  gdv_fn_castFLOAT8_utf8(ctx_ptr, "e", 1);
+  EXPECT_THAT(ctx.get_error(),
+              ::testing::HasSubstr("Failed to cast the string e to double"));
+  ctx.Reset();
+}
+
+}  // namespace gandiva
diff --git a/cpp/src/gandiva/in_holder.h b/cpp/src/gandiva/in_holder.h
index 3e18c250038..034745b387f 100644
--- a/cpp/src/gandiva/in_holder.h
+++ b/cpp/src/gandiva/in_holder.h
@@ -20,6 +20,7 @@
 #include <string>
 #include <unordered_set>
 
+#include "arrow/util/hashing.h"
 #include "gandiva/arrow.h"
 #include "gandiva/gandiva_aliases.h"
 
@@ -42,4 +43,30 @@ class InHolder {
   std::unordered_set<Type> values_;
 };
 
+template <>
+class InHolder<std::string> {
+ public:
+  explicit InHolder(std::unordered_set<std::string> values) : values_(std::move(values)) {
+    values_lookup_.max_load_factor(0.25f);
+    for (const std::string& value : values_) {
+      values_lookup_.emplace(value);
+    }
+  }
+
+  bool HasValue(arrow::util::string_view value) const {
+    return values_lookup_.count(value) == 1;
+  }
+
+ private:
+  struct string_view_hash {
+   public:
+    std::size_t operator()(arrow::util::string_view v) const {
+      return arrow::internal::ComputeStringHash<0>(v.data(), v.length());
+    }
+  };
+
+  std::unordered_set<arrow::util::string_view, string_view_hash> values_lookup_;
+  const std::unordered_set<std::string> values_;
+};
+
 }  // namespace gandiva
diff --git a/cpp/src/gandiva/precompiled/arithmetic_ops.cc b/cpp/src/gandiva/precompiled/arithmetic_ops.cc
index 21bab714ab9..a173a60d6d0 100644
--- a/cpp/src/gandiva/precompiled/arithmetic_ops.cc
+++ b/cpp/src/gandiva/precompiled/arithmetic_ops.cc
@@ -71,6 +71,8 @@ BINARY_SYMMETRIC(bitwise_and, int32, &)
 BINARY_SYMMETRIC(bitwise_and, int64, &)
 BINARY_SYMMETRIC(bitwise_or, int32, |)
 BINARY_SYMMETRIC(bitwise_or, int64, |)
+BINARY_SYMMETRIC(bitwise_xor, int32, ^)
+BINARY_SYMMETRIC(bitwise_xor, int64, ^)
 
 #undef BINARY_SYMMETRIC
 
diff --git a/cpp/src/gandiva/precompiled/arithmetic_ops_test.cc b/cpp/src/gandiva/precompiled/arithmetic_ops_test.cc
index f8b9937a2c5..b3359ac7d6c 100644
--- a/cpp/src/gandiva/precompiled/arithmetic_ops_test.cc
+++ b/cpp/src/gandiva/precompiled/arithmetic_ops_test.cc
@@ -120,6 +120,15 @@ TEST(TestArithmeticOps, TestBitwiseOps) {
             0xFFFFFFFFFFFFFF7E);
   EXPECT_EQ(bitwise_or_int64_int64(0x6A5B1, 0x00000), 0x6A5B1);
 
+  // bitwise XOR
+  EXPECT_EQ(bitwise_xor_int32_int32(0x0147D, 0x17159), 0x16524);
+  EXPECT_EQ(bitwise_xor_int32_int32(0xFFFFFFCC, 0x00000297), 0XFFFFFD5B);
+  EXPECT_EQ(bitwise_xor_int32_int32(0x000, 0x285), 0x285);
+  EXPECT_EQ(bitwise_xor_int64_int64(0x563672F83, 0x0D9FCF85B), 0x5BA9BD7D8);
+  EXPECT_EQ(bitwise_xor_int64_int64(0xFFFFFFFFFFDA8F6A, 0xFFFFFFFFFFFF791C), 0X25F676);
+  EXPECT_EQ(bitwise_xor_int64_int64(0x6A5B1, 0x00000), 0x6A5B1);
+  EXPECT_EQ(bitwise_xor_int64_int64(0x6A5B1, 0x6A5B1), 0x00000);
+
   // bitwise NOT
   EXPECT_EQ(bitwise_not_int32(0x00017159), 0xFFFE8EA6);
   EXPECT_EQ(bitwise_not_int32(0xFFFFF226), 0x00000DD9);
diff --git a/cpp/src/gandiva/precompiled/extended_math_ops.cc b/cpp/src/gandiva/precompiled/extended_math_ops.cc
index 78a3993ccbd..62728305efe 100644
--- a/cpp/src/gandiva/precompiled/extended_math_ops.cc
+++ b/cpp/src/gandiva/precompiled/extended_math_ops.cc
@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include "arrow/util/logging.h"
 #include "gandiva/precompiled/decimal_ops.h"
 
 extern "C" {
@@ -111,6 +112,111 @@ LOG_WITH_BASE(float64, float64, float64)
 
 POWER(float64, float64, float64)
 
+FORCE_INLINE
+gdv_int32 round_int32(gdv_int32 num) { return num; }
+
+FORCE_INLINE
+gdv_int64 round_int64(gdv_int64 num) { return num; }
+
+// rounds the number to the nearest integer
+#define ROUND_DECIMAL(TYPE)                                                \
+  FORCE_INLINE                                                             \
+  gdv_##TYPE round_##TYPE(gdv_##TYPE num) {                                \
+    return static_cast<gdv_##TYPE>(trunc(num + ((num > 0) ? 0.5 : -0.5))); \
+  }
+
+ROUND_DECIMAL(float32)
+ROUND_DECIMAL(float64)
+
+// rounds the number to the given scale
+#define ROUND_DECIMAL_TO_SCALE(TYPE)                                        \
+  FORCE_INLINE                                                              \
+  gdv_##TYPE round_##TYPE##_int32(gdv_##TYPE number, gdv_int32 out_scale) { \
+    gdv_float64 scale_multiplier = get_scale_multiplier(out_scale);         \
+    return static_cast<gdv_##TYPE>(                                         \
+        trunc(number * scale_multiplier + ((number > 0) ? 0.5 : -0.5)) /    \
+        scale_multiplier);                                                  \
+  }
+
+ROUND_DECIMAL_TO_SCALE(float32)
+ROUND_DECIMAL_TO_SCALE(float64)
+
+FORCE_INLINE
+gdv_int32 round_int32_int32(gdv_int32 number, gdv_int32 precision) {
+  // for integers, there is nothing following the decimal point,
+  // so round() always returns the same number if precision >= 0
+  if (precision >= 0) {
+    return number;
+  }
+  gdv_int32 abs_precision = -precision;
+  // This is to ensure that there is no overflow while calculating 10^precision, 9 is
+  // the smallest N for which 10^N does not fit into 32 bits, so we can safely return 0
+  if (abs_precision > 9) {
+    return 0;
+  }
+  gdv_int32 num_sign = (number > 0) ? 1 : -1;
+  gdv_int32 abs_number = number * num_sign;
+  gdv_int32 power_of_10 = static_cast<gdv_int32>(get_power_of_10(abs_precision));
+  gdv_int32 remainder = abs_number % power_of_10;
+  abs_number -= remainder;
+  // if the fractional part of the quotient >= 0.5, round to next higher integer
+  if (remainder >= power_of_10 / 2) {
+    abs_number += power_of_10;
+  }
+  return abs_number * num_sign;
+}
+
+FORCE_INLINE
+gdv_int64 round_int64_int32(gdv_int64 number, gdv_int32 precision) {
+  // for long integers, there is nothing following the decimal point,
+  // so round() always returns the same number if precision >= 0
+  if (precision >= 0) {
+    return number;
+  }
+  gdv_int32 abs_precision = -precision;
+  // This is to ensure that there is no overflow while calculating 10^precision, 19 is
+  // the smallest N for which 10^N does not fit into 64 bits, so we can safely return 0
+  if (abs_precision > 18) {
+    return 0;
+  }
+  gdv_int32 num_sign = (number > 0) ? 1 : -1;
+  gdv_int64 abs_number = number * num_sign;
+  gdv_int64 power_of_10 = get_power_of_10(abs_precision);
+  gdv_int64 remainder = abs_number % power_of_10;
+  abs_number -= remainder;
+  // if the fractional part of the quotient >= 0.5, round to next higher integer
+  if (remainder >= power_of_10 / 2) {
+    abs_number += power_of_10;
+  }
+  return abs_number * num_sign;
+}
+
+FORCE_INLINE
+gdv_int64 get_power_of_10(gdv_int32 exp) {
+  DCHECK_GE(exp, 0);
+  DCHECK_LE(exp, 18);
+  static const gdv_int64 power_of_10[] = {1,
+                                          10,
+                                          100,
+                                          1000,
+                                          10000,
+                                          100000,
+                                          1000000,
+                                          10000000,
+                                          100000000,
+                                          1000000000,
+                                          10000000000,
+                                          100000000000,
+                                          1000000000000,
+                                          10000000000000,
+                                          100000000000000,
+                                          1000000000000000,
+                                          10000000000000000,
+                                          100000000000000000,
+                                          1000000000000000000};
+  return power_of_10[exp];
+}
+
 FORCE_INLINE
 gdv_int64 truncate_int64_int32(gdv_int64 in, gdv_int32 out_scale) {
   bool overflow = false;
@@ -125,4 +231,32 @@ gdv_int64 truncate_int64_int32(gdv_int64 in, gdv_int32 out_scale) {
       gandiva::BasicDecimalScalar128(decimal_with_outscale, 38, out_scale), &overflow);
 }
 
+FORCE_INLINE
+gdv_float64 get_scale_multiplier(gdv_int32 scale) {
+  static const gdv_float64 values[] = {1.0,
+                                       10.0,
+                                       100.0,
+                                       1000.0,
+                                       10000.0,
+                                       100000.0,
+                                       1000000.0,
+                                       10000000.0,
+                                       100000000.0,
+                                       1000000000.0,
+                                       10000000000.0,
+                                       100000000000.0,
+                                       1000000000000.0,
+                                       10000000000000.0,
+                                       100000000000000.0,
+                                       1000000000000000.0,
+                                       10000000000000000.0,
+                                       100000000000000000.0,
+                                       1000000000000000000.0,
+                                       10000000000000000000.0};
+  if (scale >= 0 && scale < 20) {
+    return values[scale];
+  }
+  return power_float64_float64(10.0, scale);
+}
+
 }  // extern "C"
diff --git a/cpp/src/gandiva/precompiled/extended_math_ops_test.cc b/cpp/src/gandiva/precompiled/extended_math_ops_test.cc
index 67ee6f5b312..9c4b107be8a 100644
--- a/cpp/src/gandiva/precompiled/extended_math_ops_test.cc
+++ b/cpp/src/gandiva/precompiled/extended_math_ops_test.cc
@@ -87,6 +87,52 @@ TEST(TestExtendedMathOps, TestLogWithBase) {
   EXPECT_EQ(context1.has_error(), false);
 }
 
+TEST(TestExtendedMathOps, TestRoundDecimal) {
+  EXPECT_FLOAT_EQ(round_float32(1234.245f), 1234);
+  EXPECT_FLOAT_EQ(round_float32(-11.7892f), -12);
+  EXPECT_FLOAT_EQ(round_float32(1.4999999f), 1);
+  EXPECT_FLOAT_EQ(round_float32_int32(1234.789f, 2), 1234.79f);
+  EXPECT_FLOAT_EQ(round_float32_int32(1234.12345f, -3), 1000);
+  EXPECT_FLOAT_EQ(round_float32_int32(-1234.4567f, 3), -1234.457f);
+  EXPECT_FLOAT_EQ(round_float32_int32(-1234.4567f, -3), -1000);
+  EXPECT_FLOAT_EQ(round_float32_int32(1234.4567f, 0), 1234);
+  EXPECT_FLOAT_EQ(round_float32_int32(1.5499999523162842f, 1), 1.5f);
+  EXPECT_FLOAT_EQ(round_float32_int32(static_cast<float>(1.55), 1), 1.5f);
+  EXPECT_FLOAT_EQ(round_float32_int32(static_cast<float>(9.134123), 2), 9.13f);
+  EXPECT_FLOAT_EQ(round_float32_int32(static_cast<float>(-1.923), 1), -1.9f);
+
+  VerifyFuzzyEquals(round_float64(1234.245), 1234);
+  VerifyFuzzyEquals(round_float64(-11.7892), -12);
+  VerifyFuzzyEquals(round_float64(1.4999999), 1);
+  VerifyFuzzyEquals(round_float64_int32(1234.789, 2), 1234.79);
+  VerifyFuzzyEquals(round_float64_int32(1234.12345, -3), 1000);
+  VerifyFuzzyEquals(round_float64_int32(-1234.4567, 3), -1234.457);
+  VerifyFuzzyEquals(round_float64_int32(-1234.4567, -3), -1000);
+  VerifyFuzzyEquals(round_float64_int32(1234.4567, 0), 1234);
+  VerifyFuzzyEquals(round_float64_int32((double)INT_MAX + 1, 0), (double)INT_MAX + 1);
+  VerifyFuzzyEquals(round_float64_int32((double)INT_MIN - 1, 0), (double)INT_MIN - 1);
+}
+
+TEST(TestExtendedMathOps, TestRound) {
+  EXPECT_EQ(round_int32(21134), 21134);
+  EXPECT_EQ(round_int32(-132422), -132422);
+  EXPECT_EQ(round_int32_int32(7589, -1), 7590);
+  EXPECT_EQ(round_int32_int32(8532, -2), 8500);
+  EXPECT_EQ(round_int32_int32(-8579, -1), -8580);
+  EXPECT_EQ(round_int32_int32(-8612, -2), -8600);
+  EXPECT_EQ(round_int32_int32(758, 2), 758);
+  EXPECT_EQ(round_int32_int32(8612, -5), 0);
+
+  EXPECT_EQ(round_int64(3453562312), 3453562312);
+  EXPECT_EQ(round_int64(-23453462343), -23453462343);
+  EXPECT_EQ(round_int64_int32(3453562312, -2), 3453562300);
+  EXPECT_EQ(round_int64_int32(3453562343, -5), 3453600000);
+  EXPECT_EQ(round_int64_int32(345353425343, 12), 345353425343);
+  EXPECT_EQ(round_int64_int32(-23453462343, -4), -23453460000);
+  EXPECT_EQ(round_int64_int32(-23453462343, -5), -23453500000);
+  EXPECT_EQ(round_int64_int32(345353425343, -12), 0);
+}
+
 TEST(TestExtendedMathOps, TestTruncate) {
   EXPECT_EQ(truncate_int64_int32(1234, 4), 1234);
   EXPECT_EQ(truncate_int64_int32(-1234, 4), -1234);
diff --git a/cpp/src/gandiva/precompiled/string_ops.cc b/cpp/src/gandiva/precompiled/string_ops.cc
index eded3db0dde..0432d6c761c 100644
--- a/cpp/src/gandiva/precompiled/string_ops.cc
+++ b/cpp/src/gandiva/precompiled/string_ops.cc
@@ -17,13 +17,13 @@
 
 // String functions
 #include "arrow/util/value_parsing.h"
-
 extern "C" {
 
 #include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+
 #include "./types.h"
 
 FORCE_INLINE
@@ -40,6 +40,19 @@ gdv_int32 bit_length_binary(const gdv_binary input, gdv_int32 length) {
   return length * 8;
 }
 
+FORCE_INLINE
+int match_string(const char* input, gdv_int32 input_len, gdv_int32 start_pos,
+                 const char* delim, gdv_int32 delim_len) {
+  for (int i = start_pos; i < input_len; i++) {
+    int left_chars = input_len - i;
+    if ((left_chars >= delim_len) && memcmp(input + i, delim, delim_len) == 0) {
+      return i + delim_len;
+    }
+  }
+
+  return -1;
+}
+
 FORCE_INLINE
 gdv_int32 mem_compare(const char* left, gdv_int32 left_len, const char* right,
                       gdv_int32 right_len) {
@@ -84,6 +97,16 @@ VAR_LEN_OP_TYPES(BINARY_RELATIONAL, greater_than_or_equal_to, >=)
   INNER(NAME, utf8)                \
   INNER(NAME, binary)
 
+FORCE_INLINE
+int to_binary_from_hex(char ch) {
+  if (ch >= 'A' && ch <= 'F') {
+    return 10 + (ch - 'A');
+  } else if (ch >= 'a' && ch <= 'f') {
+    return 10 + (ch - 'a');
+  }
+  return ch - '0';
+}
+
 FORCE_INLINE
 bool starts_with_utf8_utf8(const char* data, gdv_int32 data_len, const char* prefix,
                            gdv_int32 prefix_len) {
@@ -709,6 +732,490 @@ const char* concatOperator_utf8_utf8(gdv_int64 context, const char* left,
   return ret;
 }
 
+FORCE_INLINE
+const char* concat_utf8_utf8_utf8(gdv_int64 context, const char* in1, gdv_int32 in1_len,
+                                  bool in1_validity, const char* in2, gdv_int32 in2_len,
+                                  bool in2_validity, const char* in3, gdv_int32 in3_len,
+                                  bool in3_validity, gdv_int32* out_len) {
+  if (!in1_validity) {
+    in1_len = 0;
+  }
+  if (!in2_validity) {
+    in2_len = 0;
+  }
+  if (!in3_validity) {
+    in3_len = 0;
+  }
+  return concatOperator_utf8_utf8_utf8(context, in1, in1_len, in2, in2_len, in3, in3_len,
+                                       out_len);
+}
+
+FORCE_INLINE
+const char* concatOperator_utf8_utf8_utf8(gdv_int64 context, const char* in1,
+                                          gdv_int32 in1_len, const char* in2,
+                                          gdv_int32 in2_len, const char* in3,
+                                          gdv_int32 in3_len, gdv_int32* out_len) {
+  *out_len = in1_len + in2_len + in3_len;
+  if (*out_len <= 0) {
+    *out_len = 0;
+    return "";
+  }
+  char* ret = reinterpret_cast<char*>(gdv_fn_context_arena_malloc(context, *out_len));
+  if (ret == nullptr) {
+    gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string");
+    *out_len = 0;
+    return "";
+  }
+  memcpy(ret, in1, in1_len);
+  memcpy(ret + in1_len, in2, in2_len);
+  memcpy(ret + in1_len + in2_len, in3, in3_len);
+  return ret;
+}
+
+FORCE_INLINE
+const char* concat_utf8_utf8_utf8_utf8(gdv_int64 context, const char* in1,
+                                       gdv_int32 in1_len, bool in1_validity,
+                                       const char* in2, gdv_int32 in2_len,
+                                       bool in2_validity, const char* in3,
+                                       gdv_int32 in3_len, bool in3_validity,
+                                       const char* in4, gdv_int32 in4_len,
+                                       bool in4_validity, gdv_int32* out_len) {
+  if (!in1_validity) {
+    in1_len = 0;
+  }
+  if (!in2_validity) {
+    in2_len = 0;
+  }
+  if (!in3_validity) {
+    in3_len = 0;
+  }
+  if (!in4_validity) {
+    in4_len = 0;
+  }
+  return concatOperator_utf8_utf8_utf8_utf8(context, in1, in1_len, in2, in2_len, in3,
+                                            in3_len, in4, in4_len, out_len);
+}
+
+FORCE_INLINE
+const char* concatOperator_utf8_utf8_utf8_utf8(gdv_int64 context, const char* in1,
+                                               gdv_int32 in1_len, const char* in2,
+                                               gdv_int32 in2_len, const char* in3,
+                                               gdv_int32 in3_len, const char* in4,
+                                               gdv_int32 in4_len, gdv_int32* out_len) {
+  *out_len = in1_len + in2_len + in3_len + in4_len;
+  if (*out_len <= 0) {
+    *out_len = 0;
+    return "";
+  }
+  char* ret = reinterpret_cast<char*>(gdv_fn_context_arena_malloc(context, *out_len));
+  if (ret == nullptr) {
+    gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string");
+    *out_len = 0;
+    return "";
+  }
+  memcpy(ret, in1, in1_len);
+  memcpy(ret + in1_len, in2, in2_len);
+  memcpy(ret + in1_len + in2_len, in3, in3_len);
+  memcpy(ret + in1_len + in2_len + in3_len, in4, in4_len);
+  return ret;
+}
+
+FORCE_INLINE
+const char* concat_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, bool in1_validity,
+    const char* in2, gdv_int32 in2_len, bool in2_validity, const char* in3,
+    gdv_int32 in3_len, bool in3_validity, const char* in4, gdv_int32 in4_len,
+    bool in4_validity, const char* in5, gdv_int32 in5_len, bool in5_validity,
+    gdv_int32* out_len) {
+  if (!in1_validity) {
+    in1_len = 0;
+  }
+  if (!in2_validity) {
+    in2_len = 0;
+  }
+  if (!in3_validity) {
+    in3_len = 0;
+  }
+  if (!in4_validity) {
+    in4_len = 0;
+  }
+  if (!in5_validity) {
+    in5_len = 0;
+  }
+  return concatOperator_utf8_utf8_utf8_utf8_utf8(context, in1, in1_len, in2, in2_len, in3,
+                                                 in3_len, in4, in4_len, in5, in5_len,
+                                                 out_len);
+}
+
+FORCE_INLINE
+const char* concatOperator_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, const char* in2,
+    gdv_int32 in2_len, const char* in3, gdv_int32 in3_len, const char* in4,
+    gdv_int32 in4_len, const char* in5, gdv_int32 in5_len, gdv_int32* out_len) {
+  *out_len = in1_len + in2_len + in3_len + in4_len + in5_len;
+  if (*out_len <= 0) {
+    *out_len = 0;
+    return "";
+  }
+  char* ret = reinterpret_cast<char*>(gdv_fn_context_arena_malloc(context, *out_len));
+  if (ret == nullptr) {
+    gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string");
+    *out_len = 0;
+    return "";
+  }
+  memcpy(ret, in1, in1_len);
+  memcpy(ret + in1_len, in2, in2_len);
+  memcpy(ret + in1_len + in2_len, in3, in3_len);
+  memcpy(ret + in1_len + in2_len + in3_len, in4, in4_len);
+  memcpy(ret + in1_len + in2_len + in3_len + in4_len, in5, in5_len);
+  return ret;
+}
+
+FORCE_INLINE
+const char* concat_utf8_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, bool in1_validity,
+    const char* in2, gdv_int32 in2_len, bool in2_validity, const char* in3,
+    gdv_int32 in3_len, bool in3_validity, const char* in4, gdv_int32 in4_len,
+    bool in4_validity, const char* in5, gdv_int32 in5_len, bool in5_validity,
+    const char* in6, gdv_int32 in6_len, bool in6_validity, gdv_int32* out_len) {
+  if (!in1_validity) {
+    in1_len = 0;
+  }
+  if (!in2_validity) {
+    in2_len = 0;
+  }
+  if (!in3_validity) {
+    in3_len = 0;
+  }
+  if (!in4_validity) {
+    in4_len = 0;
+  }
+  if (!in5_validity) {
+    in5_len = 0;
+  }
+  if (!in6_validity) {
+    in6_len = 0;
+  }
+  return concatOperator_utf8_utf8_utf8_utf8_utf8_utf8(context, in1, in1_len, in2, in2_len,
+                                                      in3, in3_len, in4, in4_len, in5,
+                                                      in5_len, in6, in6_len, out_len);
+}
+
+FORCE_INLINE
+const char* concatOperator_utf8_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, const char* in2,
+    gdv_int32 in2_len, const char* in3, gdv_int32 in3_len, const char* in4,
+    gdv_int32 in4_len, const char* in5, gdv_int32 in5_len, const char* in6,
+    gdv_int32 in6_len, gdv_int32* out_len) {
+  *out_len = in1_len + in2_len + in3_len + in4_len + in5_len + in6_len;
+  if (*out_len <= 0) {
+    *out_len = 0;
+    return "";
+  }
+  char* ret = reinterpret_cast<char*>(gdv_fn_context_arena_malloc(context, *out_len));
+  if (ret == nullptr) {
+    gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string");
+    *out_len = 0;
+    return "";
+  }
+  memcpy(ret, in1, in1_len);
+  memcpy(ret + in1_len, in2, in2_len);
+  memcpy(ret + in1_len + in2_len, in3, in3_len);
+  memcpy(ret + in1_len + in2_len + in3_len, in4, in4_len);
+  memcpy(ret + in1_len + in2_len + in3_len + in4_len, in5, in5_len);
+  memcpy(ret + in1_len + in2_len + in3_len + in4_len + in5_len, in6, in6_len);
+  return ret;
+}
+
+FORCE_INLINE
+const char* concat_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, bool in1_validity,
+    const char* in2, gdv_int32 in2_len, bool in2_validity, const char* in3,
+    gdv_int32 in3_len, bool in3_validity, const char* in4, gdv_int32 in4_len,
+    bool in4_validity, const char* in5, gdv_int32 in5_len, bool in5_validity,
+    const char* in6, gdv_int32 in6_len, bool in6_validity, const char* in7,
+    gdv_int32 in7_len, bool in7_validity, gdv_int32* out_len) {
+  if (!in1_validity) {
+    in1_len = 0;
+  }
+  if (!in2_validity) {
+    in2_len = 0;
+  }
+  if (!in3_validity) {
+    in3_len = 0;
+  }
+  if (!in4_validity) {
+    in4_len = 0;
+  }
+  if (!in5_validity) {
+    in5_len = 0;
+  }
+  if (!in6_validity) {
+    in6_len = 0;
+  }
+  if (!in7_validity) {
+    in7_len = 0;
+  }
+  return concatOperator_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+      context, in1, in1_len, in2, in2_len, in3, in3_len, in4, in4_len, in5, in5_len, in6,
+      in6_len, in7, in7_len, out_len);
+}
+
+FORCE_INLINE
+const char* concatOperator_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, const char* in2,
+    gdv_int32 in2_len, const char* in3, gdv_int32 in3_len, const char* in4,
+    gdv_int32 in4_len, const char* in5, gdv_int32 in5_len, const char* in6,
+    gdv_int32 in6_len, const char* in7, gdv_int32 in7_len, gdv_int32* out_len) {
+  *out_len = in1_len + in2_len + in3_len + in4_len + in5_len + in6_len + in7_len;
+  if (*out_len <= 0) {
+    *out_len = 0;
+    return "";
+  }
+  char* ret = reinterpret_cast<char*>(gdv_fn_context_arena_malloc(context, *out_len));
+  if (ret == nullptr) {
+    gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string");
+    *out_len = 0;
+    return "";
+  }
+  memcpy(ret, in1, in1_len);
+  memcpy(ret + in1_len, in2, in2_len);
+  memcpy(ret + in1_len + in2_len, in3, in3_len);
+  memcpy(ret + in1_len + in2_len + in3_len, in4, in4_len);
+  memcpy(ret + in1_len + in2_len + in3_len + in4_len, in5, in5_len);
+  memcpy(ret + in1_len + in2_len + in3_len + in4_len + in5_len, in6, in6_len);
+  memcpy(ret + in1_len + in2_len + in3_len + in4_len + in5_len + in6_len, in7, in7_len);
+  return ret;
+}
+
+FORCE_INLINE
+const char* concat_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, bool in1_validity,
+    const char* in2, gdv_int32 in2_len, bool in2_validity, const char* in3,
+    gdv_int32 in3_len, bool in3_validity, const char* in4, gdv_int32 in4_len,
+    bool in4_validity, const char* in5, gdv_int32 in5_len, bool in5_validity,
+    const char* in6, gdv_int32 in6_len, bool in6_validity, const char* in7,
+    gdv_int32 in7_len, bool in7_validity, const char* in8, gdv_int32 in8_len,
+    bool in8_validity, gdv_int32* out_len) {
+  if (!in1_validity) {
+    in1_len = 0;
+  }
+  if (!in2_validity) {
+    in2_len = 0;
+  }
+  if (!in3_validity) {
+    in3_len = 0;
+  }
+  if (!in4_validity) {
+    in4_len = 0;
+  }
+  if (!in5_validity) {
+    in5_len = 0;
+  }
+  if (!in6_validity) {
+    in6_len = 0;
+  }
+  if (!in7_validity) {
+    in7_len = 0;
+  }
+  if (!in8_validity) {
+    in8_len = 0;
+  }
+  return concatOperator_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+      context, in1, in1_len, in2, in2_len, in3, in3_len, in4, in4_len, in5, in5_len, in6,
+      in6_len, in7, in7_len, in8, in8_len, out_len);
+}
+
+FORCE_INLINE
+const char* concatOperator_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, const char* in2,
+    gdv_int32 in2_len, const char* in3, gdv_int32 in3_len, const char* in4,
+    gdv_int32 in4_len, const char* in5, gdv_int32 in5_len, const char* in6,
+    gdv_int32 in6_len, const char* in7, gdv_int32 in7_len, const char* in8,
+    gdv_int32 in8_len, gdv_int32* out_len) {
+  *out_len =
+      in1_len + in2_len + in3_len + in4_len + in5_len + in6_len + in7_len + in8_len;
+  if (*out_len <= 0) {
+    *out_len = 0;
+    return "";
+  }
+  char* ret = reinterpret_cast<char*>(gdv_fn_context_arena_malloc(context, *out_len));
+  if (ret == nullptr) {
+    gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string");
+    *out_len = 0;
+    return "";
+  }
+  memcpy(ret, in1, in1_len);
+  memcpy(ret + in1_len, in2, in2_len);
+  memcpy(ret + in1_len + in2_len, in3, in3_len);
+  memcpy(ret + in1_len + in2_len + in3_len, in4, in4_len);
+  memcpy(ret + in1_len + in2_len + in3_len + in4_len, in5, in5_len);
+  memcpy(ret + in1_len + in2_len + in3_len + in4_len + in5_len, in6, in6_len);
+  memcpy(ret + in1_len + in2_len + in3_len + in4_len + in5_len + in6_len, in7, in7_len);
+  memcpy(ret + in1_len + in2_len + in3_len + in4_len + in5_len + in6_len + in7_len, in8,
+         in8_len);
+  return ret;
+}
+
+FORCE_INLINE
+const char* concat_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, bool in1_validity,
+    const char* in2, gdv_int32 in2_len, bool in2_validity, const char* in3,
+    gdv_int32 in3_len, bool in3_validity, const char* in4, gdv_int32 in4_len,
+    bool in4_validity, const char* in5, gdv_int32 in5_len, bool in5_validity,
+    const char* in6, gdv_int32 in6_len, bool in6_validity, const char* in7,
+    gdv_int32 in7_len, bool in7_validity, const char* in8, gdv_int32 in8_len,
+    bool in8_validity, const char* in9, gdv_int32 in9_len, bool in9_validity,
+    gdv_int32* out_len) {
+  if (!in1_validity) {
+    in1_len = 0;
+  }
+  if (!in2_validity) {
+    in2_len = 0;
+  }
+  if (!in3_validity) {
+    in3_len = 0;
+  }
+  if (!in4_validity) {
+    in4_len = 0;
+  }
+  if (!in5_validity) {
+    in5_len = 0;
+  }
+  if (!in6_validity) {
+    in6_len = 0;
+  }
+  if (!in7_validity) {
+    in7_len = 0;
+  }
+  if (!in8_validity) {
+    in8_len = 0;
+  }
+  if (!in9_validity) {
+    in9_len = 0;
+  }
+  return concatOperator_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+      context, in1, in1_len, in2, in2_len, in3, in3_len, in4, in4_len, in5, in5_len, in6,
+      in6_len, in7, in7_len, in8, in8_len, in9, in9_len, out_len);
+}
+
+FORCE_INLINE
+const char* concatOperator_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, const char* in2,
+    gdv_int32 in2_len, const char* in3, gdv_int32 in3_len, const char* in4,
+    gdv_int32 in4_len, const char* in5, gdv_int32 in5_len, const char* in6,
+    gdv_int32 in6_len, const char* in7, gdv_int32 in7_len, const char* in8,
+    gdv_int32 in8_len, const char* in9, gdv_int32 in9_len, gdv_int32* out_len) {
+  *out_len = in1_len + in2_len + in3_len + in4_len + in5_len + in6_len + in7_len +
+             in8_len + in9_len;
+  if (*out_len <= 0) {
+    *out_len = 0;
+    return "";
+  }
+  char* ret = reinterpret_cast<char*>(gdv_fn_context_arena_malloc(context, *out_len));
+  if (ret == nullptr) {
+    gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string");
+    *out_len = 0;
+    return "";
+  }
+  memcpy(ret, in1, in1_len);
+  memcpy(ret + in1_len, in2, in2_len);
+  memcpy(ret + in1_len + in2_len, in3, in3_len);
+  memcpy(ret + in1_len + in2_len + in3_len, in4, in4_len);
+  memcpy(ret + in1_len + in2_len + in3_len + in4_len, in5, in5_len);
+  memcpy(ret + in1_len + in2_len + in3_len + in4_len + in5_len, in6, in6_len);
+  memcpy(ret + in1_len + in2_len + in3_len + in4_len + in5_len + in6_len, in7, in7_len);
+  memcpy(ret + in1_len + in2_len + in3_len + in4_len + in5_len + in6_len + in7_len, in8,
+         in8_len);
+  memcpy(
+      ret + in1_len + in2_len + in3_len + in4_len + in5_len + in6_len + in7_len + in8_len,
+      in9, in9_len);
+  return ret;
+}
+
+FORCE_INLINE
+const char* concat_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, bool in1_validity,
+    const char* in2, gdv_int32 in2_len, bool in2_validity, const char* in3,
+    gdv_int32 in3_len, bool in3_validity, const char* in4, gdv_int32 in4_len,
+    bool in4_validity, const char* in5, gdv_int32 in5_len, bool in5_validity,
+    const char* in6, gdv_int32 in6_len, bool in6_validity, const char* in7,
+    gdv_int32 in7_len, bool in7_validity, const char* in8, gdv_int32 in8_len,
+    bool in8_validity, const char* in9, gdv_int32 in9_len, bool in9_validity,
+    const char* in10, gdv_int32 in10_len, bool in10_validity, gdv_int32* out_len) {
+  if (!in1_validity) {
+    in1_len = 0;
+  }
+  if (!in2_validity) {
+    in2_len = 0;
+  }
+  if (!in3_validity) {
+    in3_len = 0;
+  }
+  if (!in4_validity) {
+    in4_len = 0;
+  }
+  if (!in5_validity) {
+    in5_len = 0;
+  }
+  if (!in6_validity) {
+    in6_len = 0;
+  }
+  if (!in7_validity) {
+    in7_len = 0;
+  }
+  if (!in8_validity) {
+    in8_len = 0;
+  }
+  if (!in9_validity) {
+    in9_len = 0;
+  }
+  if (!in10_validity) {
+    in10_len = 0;
+  }
+  return concatOperator_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+      context, in1, in1_len, in2, in2_len, in3, in3_len, in4, in4_len, in5, in5_len, in6,
+      in6_len, in7, in7_len, in8, in8_len, in9, in9_len, in10, in10_len, out_len);
+}
+
+FORCE_INLINE
+const char* concatOperator_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, const char* in2,
+    gdv_int32 in2_len, const char* in3, gdv_int32 in3_len, const char* in4,
+    gdv_int32 in4_len, const char* in5, gdv_int32 in5_len, const char* in6,
+    gdv_int32 in6_len, const char* in7, gdv_int32 in7_len, const char* in8,
+    gdv_int32 in8_len, const char* in9, gdv_int32 in9_len, const char* in10,
+    gdv_int32 in10_len, gdv_int32* out_len) {
+  *out_len = in1_len + in2_len + in3_len + in4_len + in5_len + in6_len + in7_len +
+             in8_len + in9_len + in10_len;
+  if (*out_len <= 0) {
+    *out_len = 0;
+    return "";
+  }
+  char* ret = reinterpret_cast<char*>(gdv_fn_context_arena_malloc(context, *out_len));
+  if (ret == nullptr) {
+    gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string");
+    *out_len = 0;
+    return "";
+  }
+  memcpy(ret, in1, in1_len);
+  memcpy(ret + in1_len, in2, in2_len);
+  memcpy(ret + in1_len + in2_len, in3, in3_len);
+  memcpy(ret + in1_len + in2_len + in3_len, in4, in4_len);
+  memcpy(ret + in1_len + in2_len + in3_len + in4_len, in5, in5_len);
+  memcpy(ret + in1_len + in2_len + in3_len + in4_len + in5_len, in6, in6_len);
+  memcpy(ret + in1_len + in2_len + in3_len + in4_len + in5_len + in6_len, in7, in7_len);
+  memcpy(ret + in1_len + in2_len + in3_len + in4_len + in5_len + in6_len + in7_len, in8,
+         in8_len);
+  memcpy(
+      ret + in1_len + in2_len + in3_len + in4_len + in5_len + in6_len + in7_len + in8_len,
+      in9, in9_len);
+  memcpy(ret + in1_len + in2_len + in3_len + in4_len + in5_len + in6_len + in7_len +
+             in8_len + in9_len,
+         in10, in10_len);
+  return ret;
+}
+
 FORCE_INLINE
 const char* convert_fromUTF8_binary(gdv_int64 context, const char* bin_in, gdv_int32 len,
                                     gdv_int32* out_len) {
@@ -835,27 +1342,102 @@ const char* replace_utf8_utf8_utf8(gdv_int64 context, const char* text,
                                              out_len);
 }
 
-#define CAST_NUMERIC_FROM_STRING(OUT_TYPE, ARROW_TYPE, TYPE_NAME)                       \
-  FORCE_INLINE                                                                          \
-  gdv_##OUT_TYPE cast##TYPE_NAME##_utf8(int64_t context, const char* data,              \
-                                        int32_t len) {                                  \
-    gdv_##OUT_TYPE val = 0;                                                             \
-    int32_t trimmed_len;                                                                \
-    data = btrim_utf8(context, data, len, &trimmed_len);                                \
-    if (!arrow::internal::ParseValue<ARROW_TYPE>(data, trimmed_len, &val)) {            \
-      std::string err = "Failed to cast the string " + std::string(data, trimmed_len) + \
-                        " to " #OUT_TYPE;                                               \
-      gdv_fn_context_set_error_msg(context, err.c_str());                               \
-    }                                                                                   \
-    return val;                                                                         \
-  }
-
-CAST_NUMERIC_FROM_STRING(int32, arrow::Int32Type, INT)
-CAST_NUMERIC_FROM_STRING(int64, arrow::Int64Type, BIGINT)
-CAST_NUMERIC_FROM_STRING(float32, arrow::FloatType, FLOAT4)
-CAST_NUMERIC_FROM_STRING(float64, arrow::DoubleType, FLOAT8)
-
-#undef CAST_INT_FROM_STRING
-#undef CAST_FLOAT_FROM_STRING
+FORCE_INLINE
+const char* split_part(gdv_int64 context, const char* text, gdv_int32 text_len,
+                       const char* delimiter, gdv_int32 delim_len, gdv_int32 index,
+                       gdv_int32* out_len) {
+  *out_len = 0;
+  if (index < 1) {
+    char error_message[100];
+    snprintf(error_message, sizeof(error_message),
+             "Index in split_part must be positive, value provided was %d", index);
+    gdv_fn_context_set_error_msg(context, error_message);
+    return "";
+  }
+
+  if (delim_len == 0 || text_len == 0) {
+    // output will just be text if no delimiter is provided
+    *out_len = text_len;
+    return text;
+  }
+
+  int i = 0, match_no = 1;
+
+  while (i < text_len) {
+    // find the position where delimiter matched for the first time
+    int match_pos = match_string(text, text_len, i, delimiter, delim_len);
+    if (match_pos == -1 && match_no != index) {
+      // reached the end without finding a match.
+      return "";
+    } else {
+      // Found a match. If the match number is index then return this match
+      if (match_no == index) {
+        int end_pos = match_pos - delim_len;
+
+        if (match_pos == -1) {
+          // end position should be last position of the string as we have the last
+          // delimiter
+          end_pos = text_len;
+        }
+
+        *out_len = end_pos - i;
+        char* out_str =
+            reinterpret_cast<char*>(gdv_fn_context_arena_malloc(context, *out_len));
+        if (out_str == nullptr) {
+          gdv_fn_context_set_error_msg(context,
+                                       "Could not allocate memory for output string");
+          *out_len = 0;
+          return "";
+        }
+        memcpy(out_str, text + i, *out_len);
+        return out_str;
+      } else {
+        i = match_pos;
+        match_no++;
+      }
+    }
+  }
+
+  return "";
+}
+
+FORCE_INLINE
+const char* binary_string(gdv_int64 context, const char* text, gdv_int32 text_len,
+                          gdv_int32* out_len) {
+  gdv_binary ret =
+      reinterpret_cast<gdv_binary>(gdv_fn_context_arena_malloc(context, text_len));
+
+  if (ret == nullptr) {
+    gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string");
+    *out_len = 0;
+    return "";
+  }
+
+  if (text_len == 0) {
+    *out_len = 0;
+    return "";
+  }
+
+  // converting hex encoded string to normal string
+  int j = 0;
+  for (int i = 0; i < text_len; i++, j++) {
+    if (text[i] == '\\' && i + 3 < text_len &&
+        (text[i + 1] == 'x' || text[i + 1] == 'X')) {
+      char hd1 = text[i + 2];
+      char hd2 = text[i + 3];
+      if (isxdigit(hd1) && isxdigit(hd2)) {
+        // [a-fA-F0-9]
+        ret[j] = to_binary_from_hex(hd1) * 16 + to_binary_from_hex(hd2);
+        i += 3;
+      } else {
+        ret[j] = text[i];
+      }
+    } else {
+      ret[j] = text[i];
+    }
+  }
+  *out_len = j;
+  return ret;
+}
 
 }  // extern "C"
diff --git a/cpp/src/gandiva/precompiled/string_ops_test.cc b/cpp/src/gandiva/precompiled/string_ops_test.cc
index 88345d56cb8..b1836d877ab 100644
--- a/cpp/src/gandiva/precompiled/string_ops_test.cc
+++ b/cpp/src/gandiva/precompiled/string_ops_test.cc
@@ -17,6 +17,7 @@
 
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
+
 #include "gandiva/execution_context.h"
 #include "gandiva/precompiled/types.h"
 
@@ -356,7 +357,12 @@ TEST(TestStringOps, TestConcat) {
   uint64_t ctx_ptr = reinterpret_cast<gdv_int64>(&ctx);
   gdv_int32 out_len = 0;
 
-  const char* out_str = concatOperator_utf8_utf8(ctx_ptr, "asdf", 4, "jkl", 3, &out_len);
+  const char* out_str =
+      concat_utf8_utf8(ctx_ptr, "abcd", 4, true, "\npq", 3, false, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "abcd");
+  EXPECT_FALSE(ctx.has_error());
+
+  out_str = concatOperator_utf8_utf8(ctx_ptr, "asdf", 4, "jkl", 3, &out_len);
   EXPECT_EQ(std::string(out_str, out_len), "asdfjkl");
   EXPECT_FALSE(ctx.has_error());
 
@@ -375,6 +381,106 @@ TEST(TestStringOps, TestConcat) {
   out_str = concatOperator_utf8_utf8(ctx_ptr, "abcd\n", 5, "a", 1, &out_len);
   EXPECT_EQ(std::string(out_str, out_len), "abcd\na");
   EXPECT_FALSE(ctx.has_error());
+
+  out_str = concat_utf8_utf8_utf8(ctx_ptr, "abcd", 4, false, "\npq", 3, true, "ard", 3,
+                                  true, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "\npqard");
+  EXPECT_FALSE(ctx.has_error());
+
+  out_str =
+      concatOperator_utf8_utf8_utf8(ctx_ptr, "abcd\n", 5, "a", 1, "bcd", 3, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "abcd\nabcd");
+  EXPECT_FALSE(ctx.has_error());
+
+  out_str = concatOperator_utf8_utf8_utf8(ctx_ptr, "abcd", 4, "a", 1, "", 0, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "abcda");
+  EXPECT_FALSE(ctx.has_error());
+
+  out_str = concatOperator_utf8_utf8_utf8(ctx_ptr, "", 0, "a", 1, "pqrs", 4, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "apqrs");
+  EXPECT_FALSE(ctx.has_error());
+
+  out_str = concat_utf8_utf8_utf8_utf8(ctx_ptr, "abcd", 4, false, "\npq", 3, true, "ard",
+                                       3, true, "uvw", 3, false, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "\npqard");
+  EXPECT_FALSE(ctx.has_error());
+
+  out_str = concatOperator_utf8_utf8_utf8_utf8(ctx_ptr, "pqrs", 4, "", 0, "\nabc", 4, "y",
+                                               1, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "pqrs\nabcy");
+  EXPECT_FALSE(ctx.has_error());
+
+  out_str = concat_utf8_utf8_utf8_utf8_utf8(ctx_ptr, "abcd", 4, false, "\npq", 3, true,
+                                            "ard", 3, true, "uvw", 3, false, "abc\n", 4,
+                                            true, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "\npqardabc\n");
+  EXPECT_FALSE(ctx.has_error());
+
+  out_str = concatOperator_utf8_utf8_utf8_utf8_utf8(ctx_ptr, "pqrs", 4, "", 0, "\nabc", 4,
+                                                    "y", 1, "", 0, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "pqrs\nabcy");
+  EXPECT_FALSE(ctx.has_error());
+
+  out_str = concat_utf8_utf8_utf8_utf8_utf8_utf8(
+      ctx_ptr, "abcd", 4, false, "\npq", 3, true, "ard", 3, true, "uvw", 3, false,
+      "abc\n", 4, true, "sdfgs", 5, true, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "\npqardabc\nsdfgs");
+  EXPECT_FALSE(ctx.has_error());
+
+  out_str = concatOperator_utf8_utf8_utf8_utf8_utf8_utf8(
+      ctx_ptr, "pqrs", 4, "", 0, "\nabc", 4, "y", 1, "", 0, "\nbcd", 4, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "pqrs\nabcy\nbcd");
+  EXPECT_FALSE(ctx.has_error());
+
+  out_str = concat_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+      ctx_ptr, "abcd", 4, false, "\npq", 3, true, "ard", 3, true, "uvw", 3, false,
+      "abc\n", 4, true, "sdfgs", 5, true, "wfw", 3, false, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "\npqardabc\nsdfgs");
+  EXPECT_FALSE(ctx.has_error());
+
+  out_str = concatOperator_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+      ctx_ptr, "", 0, "pqrs", 4, "abc\n", 4, "y", 1, "", 0, "asdf", 4, "jkl", 3,
+      &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "pqrsabc\nyasdfjkl");
+  EXPECT_FALSE(ctx.has_error());
+
+  out_str = concat_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+      ctx_ptr, "abcd", 4, false, "\npq", 3, true, "ard", 3, true, "uvw", 3, false,
+      "abc\n", 4, true, "sdfgs", 5, true, "wfw", 3, false, "", 0, true, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "\npqardabc\nsdfgs");
+  EXPECT_FALSE(ctx.has_error());
+
+  out_str = concatOperator_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+      ctx_ptr, "", 0, "pqrs", 4, "abc\n", 4, "y", 1, "", 0, "asdf", 4, "jkl", 3, "", 0,
+      &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "pqrsabc\nyasdfjkl");
+  EXPECT_FALSE(ctx.has_error());
+
+  out_str = concat_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+      ctx_ptr, "abcd", 4, false, "\npq", 3, true, "ard", 3, true, "uvw", 3, false,
+      "abc\n", 4, true, "sdfgs", 5, true, "wfw", 3, false, "", 0, true, "qwert|n", 7,
+      true, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "\npqardabc\nsdfgsqwert|n");
+  EXPECT_FALSE(ctx.has_error());
+
+  out_str = concatOperator_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+      ctx_ptr, "", 0, "pqrs", 4, "abc\n", 4, "y", 1, "", 0, "asdf", 4, "jkl", 3, "", 0,
+      "sfl\n", 4, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "pqrsabc\nyasdfjklsfl\n");
+  EXPECT_FALSE(ctx.has_error());
+
+  out_str = concat_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+      ctx_ptr, "abcd", 4, false, "\npq", 3, true, "ard", 3, true, "uvw", 3, false,
+      "abc\n", 4, true, "sdfgs", 5, true, "wfw", 3, false, "", 0, true, "qwert|n", 7,
+      true, "ewfwe", 5, false, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "\npqardabc\nsdfgsqwert|n");
+  EXPECT_FALSE(ctx.has_error());
+
+  out_str = concatOperator_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+      ctx_ptr, "", 0, "pqrs", 4, "abc\n", 4, "y", 1, "", 0, "asdf", 4, "", 0, "jkl", 3,
+      "sfl\n", 4, "", 0, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "pqrsabc\nyasdfjklsfl\n");
+  EXPECT_FALSE(ctx.has_error());
 }
 
 TEST(TestStringOps, TestLower) {
@@ -808,138 +914,93 @@ TEST(TestStringOps, TestReplace) {
   ctx.Reset();
 }
 
-TEST(TestArithmeticOps, TestCastINT) {
+TEST(TestStringOps, TestBinaryString) {
   gandiva::ExecutionContext ctx;
+  uint64_t ctx_ptr = reinterpret_cast<gdv_int64>(&ctx);
+  gdv_int32 out_len = 0;
+  const char* out_str;
 
-  int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
+  out_str = binary_string(ctx_ptr, "TestString", 10, &out_len);
+  std::string output = std::string(out_str, out_len);
+  EXPECT_EQ(output, "TestString");
 
-  EXPECT_EQ(castINT_utf8(ctx_ptr, "-45", 3), -45);
-  EXPECT_EQ(castINT_utf8(ctx_ptr, "0", 1), 0);
-  EXPECT_EQ(castINT_utf8(ctx_ptr, "2147483647", 10), 2147483647);
-  EXPECT_EQ(castINT_utf8(ctx_ptr, "02147483647", 11), 2147483647);
-  EXPECT_EQ(castINT_utf8(ctx_ptr, "-2147483648", 11), -2147483648LL);
-  EXPECT_EQ(castINT_utf8(ctx_ptr, "-02147483648", 12), -2147483648LL);
-  EXPECT_EQ(castINT_utf8(ctx_ptr, " 12 ", 4), 12);
+  out_str = binary_string(ctx_ptr, "", 0, &out_len);
+  output = std::string(out_str, out_len);
+  EXPECT_EQ(output, "");
 
-  castINT_utf8(ctx_ptr, "2147483648", 10);
-  EXPECT_THAT(ctx.get_error(),
-              ::testing::HasSubstr("Failed to cast the string 2147483648 to int32"));
-  ctx.Reset();
+  out_str = binary_string(ctx_ptr, "T", 1, &out_len);
+  output = std::string(out_str, out_len);
+  EXPECT_EQ(output, "T");
 
-  castINT_utf8(ctx_ptr, "-2147483649", 11);
-  EXPECT_THAT(ctx.get_error(),
-              ::testing::HasSubstr("Failed to cast the string -2147483649 to int32"));
-  ctx.Reset();
+  out_str = binary_string(ctx_ptr, "\\x41\\x42\\x43", 12, &out_len);
+  output = std::string(out_str, out_len);
+  EXPECT_EQ(output, "ABC");
 
-  castINT_utf8(ctx_ptr, "12.34", 5);
-  EXPECT_THAT(ctx.get_error(),
-              ::testing::HasSubstr("Failed to cast the string 12.34 to int32"));
-  ctx.Reset();
+  out_str = binary_string(ctx_ptr, "\\x41", 4, &out_len);
+  output = std::string(out_str, out_len);
+  EXPECT_EQ(output, "A");
 
-  castINT_utf8(ctx_ptr, "abc", 3);
-  EXPECT_THAT(ctx.get_error(),
-              ::testing::HasSubstr("Failed to cast the string abc to int32"));
-  ctx.Reset();
+  out_str = binary_string(ctx_ptr, "\\x6d\\x6D", 8, &out_len);
+  output = std::string(out_str, out_len);
+  EXPECT_EQ(output, "mm");
 
-  castINT_utf8(ctx_ptr, "", 0);
-  EXPECT_THAT(ctx.get_error(),
-              ::testing::HasSubstr("Failed to cast the string  to int32"));
-  ctx.Reset();
+  out_str = binary_string(ctx_ptr, "\\x6f\\x6d", 8, &out_len);
+  output = std::string(out_str, out_len);
+  EXPECT_EQ(output, "om");
 
-  castINT_utf8(ctx_ptr, "-", 1);
-  EXPECT_THAT(ctx.get_error(),
-              ::testing::HasSubstr("Failed to cast the string - to int32"));
-  ctx.Reset();
+  out_str = binary_string(ctx_ptr, "\\x4f\\x4D", 8, &out_len);
+  output = std::string(out_str, out_len);
+  EXPECT_EQ(output, "OM");
 }
 
-TEST(TestArithmeticOps, TestCastBIGINT) {
+TEST(TestStringOps, TestSplitPart) {
   gandiva::ExecutionContext ctx;
+  uint64_t ctx_ptr = reinterpret_cast<gdv_int64>(&ctx);
+  gdv_int32 out_len = 0;
+  const char* out_str;
 
-  int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
-
-  EXPECT_EQ(castBIGINT_utf8(ctx_ptr, "-45", 3), -45);
-  EXPECT_EQ(castBIGINT_utf8(ctx_ptr, "0", 1), 0);
-  EXPECT_EQ(castBIGINT_utf8(ctx_ptr, "9223372036854775807", 19), 9223372036854775807LL);
-  EXPECT_EQ(castBIGINT_utf8(ctx_ptr, "09223372036854775807", 20), 9223372036854775807LL);
-  EXPECT_EQ(castBIGINT_utf8(ctx_ptr, "-9223372036854775808", 20),
-            -9223372036854775807LL - 1);
-  EXPECT_EQ(castBIGINT_utf8(ctx_ptr, "-009223372036854775808", 22),
-            -9223372036854775807LL - 1);
-  EXPECT_EQ(castBIGINT_utf8(ctx_ptr, " 12 ", 4), 12);
-
-  castBIGINT_utf8(ctx_ptr, "9223372036854775808", 19);
-  EXPECT_THAT(
-      ctx.get_error(),
-      ::testing::HasSubstr("Failed to cast the string 9223372036854775808 to int64"));
-  ctx.Reset();
-
-  castBIGINT_utf8(ctx_ptr, "-9223372036854775809", 20);
+  out_str = split_part(ctx_ptr, "A,B,C", 5, ",", 1, 0, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "");
   EXPECT_THAT(
       ctx.get_error(),
-      ::testing::HasSubstr("Failed to cast the string -9223372036854775809 to int64"));
-  ctx.Reset();
-
-  castBIGINT_utf8(ctx_ptr, "12.34", 5);
-  EXPECT_THAT(ctx.get_error(),
-              ::testing::HasSubstr("Failed to cast the string 12.34 to int64"));
-  ctx.Reset();
+      ::testing::HasSubstr("Index in split_part must be positive, value provided was 0"));
 
-  castBIGINT_utf8(ctx_ptr, "abc", 3);
-  EXPECT_THAT(ctx.get_error(),
-              ::testing::HasSubstr("Failed to cast the string abc to int64"));
-  ctx.Reset();
-
-  castBIGINT_utf8(ctx_ptr, "", 0);
-  EXPECT_THAT(ctx.get_error(),
-              ::testing::HasSubstr("Failed to cast the string  to int64"));
-  ctx.Reset();
+  out_str = split_part(ctx_ptr, "A,B,C", 5, ",", 1, 1, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "A");
 
-  castBIGINT_utf8(ctx_ptr, "-", 1);
-  EXPECT_THAT(ctx.get_error(),
-              ::testing::HasSubstr("Failed to cast the string - to int64"));
-  ctx.Reset();
-}
+  out_str = split_part(ctx_ptr, "A,B,C", 5, ",", 1, 2, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "B");
 
-TEST(TestArithmeticOps, TestCastFloat4) {
-  gandiva::ExecutionContext ctx;
+  out_str = split_part(ctx_ptr, "A,B,C", 5, ",", 1, 3, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "C");
 
-  int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
+  out_str = split_part(ctx_ptr, "abc~@~def~@~ghi", 15, "~@~", 3, 1, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "abc");
 
-  EXPECT_EQ(castFLOAT4_utf8(ctx_ptr, "-45.34", 6), -45.34f);
-  EXPECT_EQ(castFLOAT4_utf8(ctx_ptr, "0", 1), 0.0f);
-  EXPECT_EQ(castFLOAT4_utf8(ctx_ptr, "5", 1), 5.0f);
-  EXPECT_EQ(castFLOAT4_utf8(ctx_ptr, " 3.4 ", 5), 3.4f);
+  out_str = split_part(ctx_ptr, "abc~@~def~@~ghi", 15, "~@~", 3, 2, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "def");
 
-  castFLOAT4_utf8(ctx_ptr, "", 0);
-  EXPECT_THAT(ctx.get_error(),
-              ::testing::HasSubstr("Failed to cast the string  to float32"));
-  ctx.Reset();
+  out_str = split_part(ctx_ptr, "abc~@~def~@~ghi", 15, "~@~", 3, 3, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "ghi");
 
-  castFLOAT4_utf8(ctx_ptr, "e", 1);
-  EXPECT_THAT(ctx.get_error(),
-              ::testing::HasSubstr("Failed to cast the string e to float32"));
-  ctx.Reset();
-}
+  // Result must be empty when the index is > no of elements
+  out_str = split_part(ctx_ptr, "123|456|789", 11, "|", 1, 4, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "");
 
-TEST(TestParseStringHolder, TestCastFloat8) {
-  gandiva::ExecutionContext ctx;
+  out_str = split_part(ctx_ptr, "123|", 4, "|", 1, 1, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "123");
 
-  int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
+  out_str = split_part(ctx_ptr, "|123", 4, "|", 1, 1, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "");
 
-  EXPECT_EQ(castFLOAT8_utf8(ctx_ptr, "-45.34", 6), -45.34);
-  EXPECT_EQ(castFLOAT8_utf8(ctx_ptr, "0", 1), 0.0);
-  EXPECT_EQ(castFLOAT8_utf8(ctx_ptr, "5", 1), 5.0);
-  EXPECT_EQ(castFLOAT8_utf8(ctx_ptr, " 3.4 ", 5), 3.4);
+  out_str = split_part(ctx_ptr, "ç†ååçåå†", 18, "å", 2, 1, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "ç†");
 
-  castFLOAT8_utf8(ctx_ptr, "", 0);
-  EXPECT_THAT(ctx.get_error(),
-              ::testing::HasSubstr("Failed to cast the string  to float64"));
-  ctx.Reset();
+  out_str = split_part(ctx_ptr, "ç†ååçåå†", 18, "†åå", 6, 1, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "ç");
 
-  castFLOAT8_utf8(ctx_ptr, "e", 1);
-  EXPECT_THAT(ctx.get_error(),
-              ::testing::HasSubstr("Failed to cast the string e to float64"));
-  ctx.Reset();
+  out_str = split_part(ctx_ptr, "ç†ååçåå†", 18, "†", 3, 2, &out_len);
+  EXPECT_EQ(std::string(out_str, out_len), "ååçåå");
 }
 
 }  // namespace gandiva
diff --git a/cpp/src/gandiva/precompiled/time.cc b/cpp/src/gandiva/precompiled/time.cc
index 128c521433d..3d1a69a1973 100644
--- a/cpp/src/gandiva/precompiled/time.cc
+++ b/cpp/src/gandiva/precompiled/time.cc
@@ -699,6 +699,8 @@ gdv_timestamp castTIMESTAMP_utf8(int64_t context, const char* input, gdv_int32 l
 
 gdv_timestamp castTIMESTAMP_date64(gdv_date64 date_in_millis) { return date_in_millis; }
 
+gdv_timestamp castTIMESTAMP_int64(gdv_int64 in) { return in; }
+
 gdv_date64 castDATE_timestamp(gdv_timestamp timestamp_in_millis) {
   EpochTimePoint tp(timestamp_in_millis);
   return tp.ClearTimeOfDay().MillisSinceEpoch();
diff --git a/cpp/src/gandiva/precompiled/types.h b/cpp/src/gandiva/precompiled/types.h
index 25a8b1c4106..51f2e9c8238 100644
--- a/cpp/src/gandiva/precompiled/types.h
+++ b/cpp/src/gandiva/precompiled/types.h
@@ -132,6 +132,17 @@ gdv_int64 div_int64_int64(gdv_int64 context, gdv_int64 in1, gdv_int64 in2);
 gdv_float32 div_float32_float32(gdv_int64 context, gdv_float32 in1, gdv_float32 in2);
 gdv_float64 div_float64_float64(gdv_int64 context, gdv_float64 in1, gdv_float64 in2);
 
+gdv_float32 round_float32(gdv_float32);
+gdv_float64 round_float64(gdv_float64);
+gdv_float32 round_float32_int32(gdv_float32 number, gdv_int32 out_scale);
+gdv_float64 round_float64_int32(gdv_float64 number, gdv_int32 out_scale);
+gdv_float64 get_scale_multiplier(gdv_int32);
+gdv_int32 round_int32_int32(gdv_int32 number, gdv_int32 precision);
+gdv_int64 round_int64_int32(gdv_int64 number, gdv_int32 precision);
+gdv_int32 round_int32(gdv_int32);
+gdv_int64 round_int64(gdv_int64);
+gdv_int64 get_power_of_10(gdv_int32);
+
 gdv_float64 cbrt_int32(gdv_int32);
 gdv_float64 cbrt_int64(gdv_int64);
 gdv_float64 cbrt_float32(gdv_float32);
@@ -156,6 +167,8 @@ gdv_int32 bitwise_and_int32_int32(gdv_int32 in1, gdv_int32 in2);
 gdv_int64 bitwise_and_int64_int64(gdv_int64 in1, gdv_int64 in2);
 gdv_int32 bitwise_or_int32_int32(gdv_int32 in1, gdv_int32 in2);
 gdv_int64 bitwise_or_int64_int64(gdv_int64 in1, gdv_int64 in2);
+gdv_int32 bitwise_xor_int32_int32(gdv_int32 in1, gdv_int32 in2);
+gdv_int64 bitwise_xor_int64_int64(gdv_int64 in1, gdv_int64 in2);
 gdv_int32 bitwise_not_int32(gdv_int32);
 gdv_int64 bitwise_not_int64(gdv_int64);
 
@@ -185,6 +198,7 @@ gdv_date32 castDATE_int32(gdv_int32 date);
 gdv_timestamp castTIMESTAMP_utf8(int64_t execution_context, const char* input,
                                  gdv_int32 length);
 gdv_timestamp castTIMESTAMP_date64(gdv_date64);
+gdv_timestamp castTIMESTAMP_int64(gdv_int64);
 gdv_date64 castDATE_timestamp(gdv_timestamp);
 const char* castVARCHAR_timestamp_int64(int64_t, gdv_timestamp, gdv_int64, gdv_int32*);
 
@@ -195,9 +209,112 @@ const char* substr_utf8_int64_int64(gdv_int64 context, const char* input,
                                     gdv_int64 length, gdv_int32* out_len);
 const char* substr_utf8_int64(gdv_int64 context, const char* input, gdv_int32 in_len,
                               gdv_int64 offset64, gdv_int32* out_len);
+
+const char* concat_utf8_utf8(gdv_int64 context, const char* left, gdv_int32 left_len,
+                             bool left_validity, const char* right, gdv_int32 right_len,
+                             bool right_validity, gdv_int32* out_len);
+const char* concat_utf8_utf8_utf8(gdv_int64 context, const char* in1, gdv_int32 in1_len,
+                                  bool in1_validity, const char* in2, gdv_int32 in2_len,
+                                  bool in2_validity, const char* in3, gdv_int32 in3_len,
+                                  bool in3_validity, gdv_int32* out_len);
+const char* concat_utf8_utf8_utf8_utf8(gdv_int64 context, const char* in1,
+                                       gdv_int32 in1_len, bool in1_validity,
+                                       const char* in2, gdv_int32 in2_len,
+                                       bool in2_validity, const char* in3,
+                                       gdv_int32 in3_len, bool in3_validity,
+                                       const char* in4, gdv_int32 in4_len,
+                                       bool in4_validity, gdv_int32* out_len);
+const char* concat_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, bool in1_validity,
+    const char* in2, gdv_int32 in2_len, bool in2_validity, const char* in3,
+    gdv_int32 in3_len, bool in3_validity, const char* in4, gdv_int32 in4_len,
+    bool in4_validity, const char* in5, gdv_int32 in5_len, bool in5_validity,
+    gdv_int32* out_len);
+const char* concat_utf8_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, bool in1_validity,
+    const char* in2, gdv_int32 in2_len, bool in2_validity, const char* in3,
+    gdv_int32 in3_len, bool in3_validity, const char* in4, gdv_int32 in4_len,
+    bool in4_validity, const char* in5, gdv_int32 in5_len, bool in5_validity,
+    const char* in6, gdv_int32 in6_len, bool in6_validity, gdv_int32* out_len);
+const char* concat_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, bool in1_validity,
+    const char* in2, gdv_int32 in2_len, bool in2_validity, const char* in3,
+    gdv_int32 in3_len, bool in3_validity, const char* in4, gdv_int32 in4_len,
+    bool in4_validity, const char* in5, gdv_int32 in5_len, bool in5_validity,
+    const char* in6, gdv_int32 in6_len, bool in6_validity, const char* in7,
+    gdv_int32 in7_len, bool in7_validity, gdv_int32* out_len);
+const char* concat_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, bool in1_validity,
+    const char* in2, gdv_int32 in2_len, bool in2_validity, const char* in3,
+    gdv_int32 in3_len, bool in3_validity, const char* in4, gdv_int32 in4_len,
+    bool in4_validity, const char* in5, gdv_int32 in5_len, bool in5_validity,
+    const char* in6, gdv_int32 in6_len, bool in6_validity, const char* in7,
+    gdv_int32 in7_len, bool in7_validity, const char* in8, gdv_int32 in8_len,
+    bool in8_validity, gdv_int32* out_len);
+const char* concat_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, bool in1_validity,
+    const char* in2, gdv_int32 in2_len, bool in2_validity, const char* in3,
+    gdv_int32 in3_len, bool in3_validity, const char* in4, gdv_int32 in4_len,
+    bool in4_validity, const char* in5, gdv_int32 in5_len, bool in5_validity,
+    const char* in6, gdv_int32 in6_len, bool in6_validity, const char* in7,
+    gdv_int32 in7_len, bool in7_validity, const char* in8, gdv_int32 in8_len,
+    bool in8_validity, const char* in9, gdv_int32 in9_len, bool in9_validity,
+    gdv_int32* out_len);
+const char* concat_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, bool in1_validity,
+    const char* in2, gdv_int32 in2_len, bool in2_validity, const char* in3,
+    gdv_int32 in3_len, bool in3_validity, const char* in4, gdv_int32 in4_len,
+    bool in4_validity, const char* in5, gdv_int32 in5_len, bool in5_validity,
+    const char* in6, gdv_int32 in6_len, bool in6_validity, const char* in7,
+    gdv_int32 in7_len, bool in7_validity, const char* in8, gdv_int32 in8_len,
+    bool in8_validity, const char* in9, gdv_int32 in9_len, bool in9_validity,
+    const char* in10, gdv_int32 in10_len, bool in10_validity, gdv_int32* out_len);
+
 const char* concatOperator_utf8_utf8(gdv_int64 context, const char* left,
                                      gdv_int32 left_len, const char* right,
                                      gdv_int32 right_len, gdv_int32* out_len);
+const char* concatOperator_utf8_utf8_utf8(gdv_int64 context, const char* in1,
+                                          gdv_int32 in1_len, const char* in2,
+                                          gdv_int32 in2_len, const char* in3,
+                                          gdv_int32 in3_len, gdv_int32* out_len);
+const char* concatOperator_utf8_utf8_utf8_utf8(gdv_int64 context, const char* in1,
+                                               gdv_int32 in1_len, const char* in2,
+                                               gdv_int32 in2_len, const char* in3,
+                                               gdv_int32 in3_len, const char* in4,
+                                               gdv_int32 in4_len, gdv_int32* out_len);
+const char* concatOperator_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, const char* in2,
+    gdv_int32 in2_len, const char* in3, gdv_int32 in3_len, const char* in4,
+    gdv_int32 in4_len, const char* in5, gdv_int32 in5_len, gdv_int32* out_len);
+const char* concatOperator_utf8_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, const char* in2,
+    gdv_int32 in2_len, const char* in3, gdv_int32 in3_len, const char* in4,
+    gdv_int32 in4_len, const char* in5, gdv_int32 in5_len, const char* in6,
+    gdv_int32 in6_len, gdv_int32* out_len);
+const char* concatOperator_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, const char* in2,
+    gdv_int32 in2_len, const char* in3, gdv_int32 in3_len, const char* in4,
+    gdv_int32 in4_len, const char* in5, gdv_int32 in5_len, const char* in6,
+    gdv_int32 in6_len, const char* in7, gdv_int32 in7_len, gdv_int32* out_len);
+const char* concatOperator_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, const char* in2,
+    gdv_int32 in2_len, const char* in3, gdv_int32 in3_len, const char* in4,
+    gdv_int32 in4_len, const char* in5, gdv_int32 in5_len, const char* in6,
+    gdv_int32 in6_len, const char* in7, gdv_int32 in7_len, const char* in8,
+    gdv_int32 in8_len, gdv_int32* out_len);
+const char* concatOperator_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, const char* in2,
+    gdv_int32 in2_len, const char* in3, gdv_int32 in3_len, const char* in4,
+    gdv_int32 in4_len, const char* in5, gdv_int32 in5_len, const char* in6,
+    gdv_int32 in6_len, const char* in7, gdv_int32 in7_len, const char* in8,
+    gdv_int32 in8_len, const char* in9, gdv_int32 in9_len, gdv_int32* out_len);
+const char* concatOperator_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8_utf8(
+    gdv_int64 context, const char* in1, gdv_int32 in1_len, const char* in2,
+    gdv_int32 in2_len, const char* in3, gdv_int32 in3_len, const char* in4,
+    gdv_int32 in4_len, const char* in5, gdv_int32 in5_len, const char* in6,
+    gdv_int32 in6_len, const char* in7, gdv_int32 in7_len, const char* in8,
+    gdv_int32 in8_len, const char* in9, gdv_int32 in9_len, const char* in10,
+    gdv_int32 in10_len, gdv_int32* out_len);
 
 const char* castVARCHAR_utf8_int64(gdv_int64 context, const char* data,
                                    gdv_int32 data_len, int64_t out_len,
@@ -248,6 +365,10 @@ const char* replace_utf8_utf8_utf8(gdv_int64 context, const char* text,
                                    gdv_int32 from_str_len, const char* to_str,
                                    gdv_int32 to_str_len, gdv_int32* out_len);
 
+const char* split_part(gdv_int64 context, const char* text, gdv_int32 text_len,
+                       const char* splitter, gdv_int32 split_len, gdv_int32 index,
+                       gdv_int32* out_len);
+
 const char* castVARCHAR_int32_int64(int64_t context, int32_t value, int64_t len,
                                     int32_t* out_len);
 
@@ -260,6 +381,9 @@ const char* castVARCHAR_float32_int64(int64_t context, float value, int64_t len,
 const char* castVARCHAR_float64_int64(int64_t context, double value, int64_t len,
                                       int32_t* out_len);
 
+const char* binary_string(gdv_int64 context, const char* text, gdv_int32 text_len,
+                          gdv_int32* out_len);
+
 int32_t castINT_utf8(int64_t context, const char* data, int32_t len);
 
 int64_t castBIGINT_utf8(int64_t context, const char* data, int32_t len);
diff --git a/cpp/src/gandiva/random_generator_holder.h b/cpp/src/gandiva/random_generator_holder.h
index 8ae9f0c93e4..65b6607e878 100644
--- a/cpp/src/gandiva/random_generator_holder.h
+++ b/cpp/src/gandiva/random_generator_holder.h
@@ -21,6 +21,7 @@
 #include <random>
 
 #include "arrow/status.h"
+#include "arrow/util/io_util.h"
 
 #include "gandiva/function_holder.h"
 #include "gandiva/node.h"
@@ -46,8 +47,7 @@ class GANDIVA_EXPORT RandomGeneratorHolder : public FunctionHolder {
   }
 
   RandomGeneratorHolder() : distribution_(0, 1) {
-    std::random_device rd;
-    generator_.seed(rd());
+    generator_.seed(::arrow::internal::GetRandomSeed());
   }
 
   std::mt19937_64 generator_;
diff --git a/cpp/src/gandiva/tests/generate_data.h b/cpp/src/gandiva/tests/generate_data.h
index 519541f018a..9fb0e4eaef6 100644
--- a/cpp/src/gandiva/tests/generate_data.h
+++ b/cpp/src/gandiva/tests/generate_data.h
@@ -20,6 +20,7 @@
 #include <string>
 
 #include "arrow/util/decimal.h"
+#include "arrow/util/io_util.h"
 
 #pragma once
 
@@ -35,18 +36,13 @@ class DataGenerator {
 
 class Random {
  public:
-  explicit Random(uint32_t seed = 100) : seed_(seed) {}
+  Random() : gen_(::arrow::internal::GetRandomSeed()) {}
+  explicit Random(uint64_t seed) : gen_(seed) {}
 
-  // This is 3 times faster than random_device
-#ifndef _MSC_VER
-  int32_t next() { return rand_r(&seed_); }
-#else
-  int32_t next() { return random_dev_(); }
-#endif
+  int32_t next() { return gen_(); }
 
  private:
-  uint32_t seed_;
-  std::random_device random_dev_;
+  std::default_random_engine gen_;
 };
 
 class Int32DataGenerator : public DataGenerator<int32_t> {
diff --git a/cpp/src/jni/orc/jni_wrapper.cpp b/cpp/src/jni/orc/jni_wrapper.cpp
index a3419280e73..e6fc82312d5 100644
--- a/cpp/src/jni/orc/jni_wrapper.cpp
+++ b/cpp/src/jni/orc/jni_wrapper.cpp
@@ -226,7 +226,7 @@ Java_org_apache_arrow_adapter_orc_OrcStripeReaderJniWrapper_getSchema(JNIEnv* en
 
   auto schema = stripe_reader->schema();
 
-  auto maybe_buffer = arrow::ipc::SerializeSchema(*schema, nullptr);
+  auto maybe_buffer = arrow::ipc::SerializeSchema(*schema, arrow::default_memory_pool());
   if (!maybe_buffer.ok()) {
     return nullptr;
   }
diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt
index 481857a9205..03443b282f2 100644
--- a/cpp/src/parquet/CMakeLists.txt
+++ b/cpp/src/parquet/CMakeLists.txt
@@ -184,10 +184,12 @@ set(PARQUET_SRCS
     deprecated_io.cc
     encoding.cc
     encryption.cc
+    exception.cc
     file_reader.cc
     file_writer.cc
     internal_file_decryptor.cc
     internal_file_encryptor.cc
+    level_comparison.cc
     level_conversion.cc
     metadata.cc
     murmur3.cc
@@ -202,6 +204,27 @@ set(PARQUET_SRCS
     stream_writer.cc
     types.cc)
 
+if(ARROW_HAVE_RUNTIME_AVX2)
+  # AVX2 is used as a proxy for BMI2.
+  list(APPEND PARQUET_SRCS level_comparison_avx2.cc level_conversion_bmi2.cc)
+  set_source_files_properties(level_comparison_avx2.cc
+                              PROPERTIES
+                              SKIP_PRECOMPILE_HEADERS
+                              ON
+                              COMPILE_FLAGS
+                              "${ARROW_AVX2_FLAG}")
+  # WARNING: DO NOT BLINDLY COPY THIS CODE FOR OTHER BMI2 USE CASES.
+  # This code is always guarded by runtime dispatch which verifies
+  # BMI2 is present.  For a very small number of CPUs AVX2 does not
+  # imply BMI2.
+  set_source_files_properties(level_conversion_bmi2.cc
+                              PROPERTIES
+                              SKIP_PRECOMPILE_HEADERS
+                              ON
+                              COMPILE_FLAGS
+                              "${ARROW_AVX2_FLAG} -DARROW_HAVE_BMI2 -mbmi2")
+endif()
+
 if(PARQUET_REQUIRE_ENCRYPTION)
   set(PARQUET_SRCS ${PARQUET_SRCS} encryption_internal.cc)
 else()
@@ -233,7 +256,7 @@ if(NOT PARQUET_MINIMAL_DEPENDENCY)
 
 endif(NOT PARQUET_MINIMAL_DEPENDENCY)
 
-if(NOT APPLE AND NOT MSVC)
+if(NOT APPLE AND NOT MSVC_TOOLCHAIN)
   # Localize thirdparty symbols using a linker version script. This hides them
   # from the client application. The OS X linker does not support the
   # version-script option.
@@ -355,7 +378,12 @@ add_parquet_test(arrow-test
                  SOURCES
                  arrow/arrow_reader_writer_test.cc
                  arrow/arrow_schema_test.cc
+                 test_util.cc)
+
+add_parquet_test(arrow-internals-test
+                 SOURCES
                  arrow/path_internal_test.cc
+                 arrow/reconstruct_internal_test.cc
                  test_util.cc)
 
 if(PARQUET_REQUIRE_ENCRYPTION)
diff --git a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc
index 476d82f7fac..eacfad813cd 100644
--- a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc
+++ b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc
@@ -29,11 +29,15 @@
 #include <sstream>
 #include <vector>
 
-#include "arrow/api.h"
+#include "arrow/array/builder_binary.h"
+#include "arrow/array/builder_decimal.h"
+#include "arrow/array/builder_dict.h"
+#include "arrow/array/builder_primitive.h"
+#include "arrow/chunked_array.h"
 #include "arrow/compute/api.h"
-#include "arrow/pretty_print.h"
 #include "arrow/record_batch.h"
 #include "arrow/scalar.h"
+#include "arrow/table.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/testing/random.h"
 #include "arrow/testing/util.h"
@@ -55,6 +59,7 @@
 #include "parquet/test_util.h"
 
 using arrow::Array;
+using arrow::ArrayData;
 using arrow::ArrayVisitor;
 using arrow::Buffer;
 using arrow::ChunkedArray;
@@ -416,7 +421,6 @@ void DoSimpleRoundtrip(const std::shared_ptr<Table>& table, bool use_threads,
                               ::arrow::default_memory_pool(), &reader));
 
   reader->set_use_threads(use_threads);
-
   if (column_subset.size() > 0) {
     ASSERT_OK_NO_THROW(reader->ReadTable(column_subset, out));
   } else {
@@ -433,6 +437,7 @@ void CheckSimpleRoundtrip(const std::shared_ptr<Table>& table, int64_t row_group
       table, false /* use_threads */, row_group_size, {}, &result, arrow_properties));
   ::arrow::AssertSchemaEqual(*table->schema(), *result->schema(),
                              /*check_metadata=*/false);
+  ASSERT_OK(result->ValidateFull());
   ::arrow::AssertTablesEqual(*table, *result, false);
 }
 
@@ -687,7 +692,7 @@ TYPED_TEST(TestParquetIO, SingleColumnTableRequiredWrite) {
   ASSERT_NO_FATAL_FAILURE(this->ReaderFromSink(&reader));
   ASSERT_NO_FATAL_FAILURE(this->ReadTableFromFile(std::move(reader), &out));
   ASSERT_EQ(1, out->num_columns());
-  ASSERT_EQ(100, out->num_rows());
+  EXPECT_EQ(100, out->num_rows());
 
   std::shared_ptr<ChunkedArray> chunked_array = out->column(0);
   ASSERT_EQ(1, chunked_array->num_chunks());
@@ -1086,8 +1091,8 @@ TEST_F(TestUInt32ParquetIO, Parquet_1_0_Compatibility) {
   }
 
   std::vector<std::shared_ptr<Buffer>> buffers{values->null_bitmap(), int64_data};
-  auto arr_data = std::make_shared<::arrow::ArrayData>(::arrow::int64(), values->length(),
-                                                       buffers, values->null_count());
+  auto arr_data = std::make_shared<ArrayData>(::arrow::int64(), values->length(), buffers,
+                                              values->null_count());
   std::shared_ptr<Array> expected_values = MakeArray(arr_data);
   ASSERT_NE(expected_values, NULLPTR);
 
@@ -2361,8 +2366,40 @@ TEST(ArrowReadWrite, SingleColumnNullableStruct) {
       3);
 }
 
-// Disabled until implementation can be finished.
-TEST(TestArrowReadWrite, DISABLED_CanonicalNestedRoundTrip) {
+TEST(ArrowReadWrite, NestedRequiredField) {
+  auto int_field = ::arrow::field("int_array", ::arrow::int32(), /*nullable=*/false);
+  auto int_array = ::arrow::ArrayFromJSON(int_field->type(), "[0, 1, 2, 3, 4, 5, 7, 8]");
+  auto struct_field =
+      ::arrow::field("root", ::arrow::struct_({int_field}), /*nullable=*/true);
+  std::shared_ptr<Buffer> validity_bitmap;
+  ASSERT_OK_AND_ASSIGN(validity_bitmap, ::arrow::AllocateBitmap(8));
+  validity_bitmap->mutable_data()[0] = 0xCC;
+
+  auto struct_data = ArrayData::Make(struct_field->type(), /*length=*/8,
+                                     {validity_bitmap}, {int_array->data()});
+  CheckSimpleRoundtrip(::arrow::Table::Make(::arrow::schema({struct_field}),
+                                            {::arrow::MakeArray(struct_data)}),
+                       /*row_group_size=*/8);
+}
+
+TEST(ArrowReadWrite, NestedNullableField) {
+  auto int_field = ::arrow::field("int_array", ::arrow::int32());
+  auto int_array =
+      ::arrow::ArrayFromJSON(int_field->type(), "[0, null, 2, null, 4, 5, null, 8]");
+  auto struct_field =
+      ::arrow::field("root", ::arrow::struct_({int_field}), /*nullable=*/true);
+  std::shared_ptr<Buffer> validity_bitmap;
+  ASSERT_OK_AND_ASSIGN(validity_bitmap, ::arrow::AllocateBitmap(8));
+  validity_bitmap->mutable_data()[0] = 0xCC;
+
+  auto struct_data = ArrayData::Make(struct_field->type(), /*length=*/8,
+                                     {validity_bitmap}, {int_array->data()});
+  CheckSimpleRoundtrip(::arrow::Table::Make(::arrow::schema({struct_field}),
+                                            {::arrow::MakeArray(struct_data)}),
+                       /*row_group_size=*/8);
+}
+
+TEST(TestArrowReadWrite, CanonicalNestedRoundTrip) {
   auto doc_id = field("DocId", ::arrow::int64(), /*nullable=*/false);
   auto links = field(
       "Links",
@@ -2391,7 +2428,7 @@ TEST(TestArrowReadWrite, DISABLED_CanonicalNestedRoundTrip) {
   // string literals implemented properly
   auto name_array = ::arrow::ArrayFromJSON(
       name->type(),
-      "([[{\"Language\": [{\"Code\": \"en_us\", \"Country\":\"us\"},"
+      "[[{\"Language\": [{\"Code\": \"en_us\", \"Country\":\"us\"},"
       "{\"Code\": \"en_us\", \"Country\": null}],"
       "\"Url\": \"http://A\"},"
       "{\"Url\": \"http://B\"},"
@@ -2402,6 +2439,198 @@ TEST(TestArrowReadWrite, DISABLED_CanonicalNestedRoundTrip) {
   CheckSimpleRoundtrip(expected, 2);
 }
 
+TEST(ArrowReadWrite, ListOfStruct) {
+  using ::arrow::field;
+
+  auto type = ::arrow::list(::arrow::struct_(
+      {field("a", ::arrow::int16(), /*nullable=*/false), field("b", ::arrow::utf8())}));
+
+  const char* json = R"([
+      [{"a": 4, "b": "foo"}, {"a": 5}, {"a": 6, "b": "bar"}],
+      [null, {"a": 7}],
+      null,
+      []])";
+  auto array = ::arrow::ArrayFromJSON(type, json);
+  auto table = ::arrow::Table::Make(::arrow::schema({field("root", type)}), {array});
+  CheckSimpleRoundtrip(table, 2);
+}
+
+TEST(ArrowReadWrite, ListOfStructOfList1) {
+  using ::arrow::field;
+  using ::arrow::list;
+  using ::arrow::struct_;
+
+  auto type = list(struct_({field("a", ::arrow::int16(), /*nullable=*/false),
+                            field("b", list(::arrow::int64()))}));
+
+  const char* json = R"([
+      [{"a": 123, "b": [1, 2, null, 3]}, null],
+      null,
+      [],
+      [{"a": 456}, {"a": 789, "b": []}, {"a": 876, "b": [4, 5, 6]}]])";
+  auto array = ::arrow::ArrayFromJSON(type, json);
+  auto table = ::arrow::Table::Make(::arrow::schema({field("root", type)}), {array});
+  CheckSimpleRoundtrip(table, 2);
+}
+
+TEST(ArrowReadWrite, Map) {
+  using ::arrow::field;
+  using ::arrow::map;
+
+  auto type = map(::arrow::int16(), ::arrow::utf8());
+
+  const char* json = R"([
+      [[1, "a"], [2, "b"]],
+      [[3, "c"]],
+      [],
+      null,
+      [[4, "d"], [5, "e"], [6, "f"]]
+  ])";
+  auto array = ::arrow::ArrayFromJSON(type, json);
+  auto table = ::arrow::Table::Make(::arrow::schema({field("root", type)}), {array});
+  auto props_store_schema = ArrowWriterProperties::Builder().store_schema()->build();
+  CheckSimpleRoundtrip(table, 2, props_store_schema);
+}
+
+TEST(ArrowReadWrite, LargeList) {
+  using ::arrow::field;
+  using ::arrow::large_list;
+  using ::arrow::struct_;
+
+  auto type = large_list(::arrow::int16());
+
+  const char* json = R"([
+      [1, 2, 3],
+      [4, 5, 6],
+      [7, 8, 9]])";
+  auto array = ::arrow::ArrayFromJSON(type, json);
+  auto table = ::arrow::Table::Make(::arrow::schema({field("root", type)}), {array});
+  auto props_store_schema = ArrowWriterProperties::Builder().store_schema()->build();
+  CheckSimpleRoundtrip(table, 2, props_store_schema);
+}
+
+TEST(ArrowReadWrite, FixedSizeList) {
+  using ::arrow::field;
+  using ::arrow::fixed_size_list;
+  using ::arrow::struct_;
+
+  auto type = fixed_size_list(::arrow::int16(), /*size=*/3);
+
+  const char* json = R"([
+      [1, 2, 3],
+      [4, 5, 6],
+      [7, 8, 9]])";
+  auto array = ::arrow::ArrayFromJSON(type, json);
+  auto table = ::arrow::Table::Make(::arrow::schema({field("root", type)}), {array});
+  auto props_store_schema = ArrowWriterProperties::Builder().store_schema()->build();
+  CheckSimpleRoundtrip(table, 2, props_store_schema);
+}
+
+TEST(ArrowReadWrite, ListOfStructOfList2) {
+  using ::arrow::field;
+  using ::arrow::list;
+  using ::arrow::struct_;
+
+  auto type =
+      list(field("item",
+                 struct_({field("a", ::arrow::int16(), /*nullable=*/false),
+                          field("b", list(::arrow::int64()), /*nullable=*/false)}),
+                 /*nullable=*/false));
+
+  const char* json = R"([
+      [{"a": 123, "b": [1, 2, 3]}],
+      null,
+      [],
+      [{"a": 456, "b": []}, {"a": 789, "b": [null]}, {"a": 876, "b": [4, 5, 6]}]])";
+  auto array = ::arrow::ArrayFromJSON(type, json);
+  auto table = ::arrow::Table::Make(::arrow::schema({field("root", type)}), {array});
+  CheckSimpleRoundtrip(table, 2);
+}
+
+TEST(ArrowReadWrite, StructOfLists) {
+  using ::arrow::field;
+  using ::arrow::list;
+
+  auto type = ::arrow::struct_(
+      {field("a", list(::arrow::utf8()), /*nullable=*/false),
+       field("b", list(field("f", ::arrow::int64(), /*nullable=*/false)))});
+
+  const char* json = R"([
+      {"a": ["1", "2"], "b": []},
+      {"a": [], "b": [3, 4, 5]},
+      {"a": ["6"], "b": null},
+      {"a": [null, "7"], "b": [8]}])";
+  auto array = ::arrow::ArrayFromJSON(type, json);
+  auto table = ::arrow::Table::Make(::arrow::schema({field("root", type)}), {array});
+  CheckSimpleRoundtrip(table, 2);
+}
+
+TEST(ArrowReadWrite, ListOfStructOfLists1) {
+  using ::arrow::field;
+  using ::arrow::list;
+
+  auto type = list(::arrow::struct_(
+      {field("a", list(::arrow::utf8()), /*nullable=*/false),
+       field("b", list(field("f", ::arrow::int64(), /*nullable=*/false)))}));
+
+  const char* json = R"([
+      [{"a": ["1", "2"], "b": []}, null],
+      [],
+      null,
+      [null],
+      [{"a": [], "b": [3, 4, 5]}, {"a": ["6"], "b": null}],
+      [null, {"a": [null, "7"], "b": [8]}]])";
+  auto array = ::arrow::ArrayFromJSON(type, json);
+  auto table = ::arrow::Table::Make(::arrow::schema({field("root", type)}), {array});
+  CheckSimpleRoundtrip(table, 2);
+}
+
+TEST(ArrowReadWrite, ListOfStructOfLists2) {
+  using ::arrow::field;
+  using ::arrow::list;
+
+  auto type = list(
+      field("x",
+            ::arrow::struct_(
+                {field("a", list(::arrow::utf8()), /*nullable=*/false),
+                 field("b", list(field("f", ::arrow::int64(), /*nullable=*/false)))}),
+            /*nullable=*/false));
+
+  const char* json = R"([
+      [{"a": ["1", "2"], "b": []}],
+      [],
+      null,
+      [],
+      [{"a": [], "b": [3, 4, 5]}, {"a": ["6"], "b": null}],
+      [{"a": [null, "7"], "b": [8]}]])";
+  auto array = ::arrow::ArrayFromJSON(type, json);
+  auto table = ::arrow::Table::Make(::arrow::schema({field("root", type)}), {array});
+  CheckSimpleRoundtrip(table, 2);
+}
+
+TEST(ArrowReadWrite, ListOfStructOfLists3) {
+  using ::arrow::field;
+  using ::arrow::list;
+
+  auto type = list(field(
+      "x",
+      ::arrow::struct_({field("a", list(::arrow::utf8()), /*nullable=*/false),
+                        field("b", list(field("f", ::arrow::int64(), /*nullable=*/false)),
+                              /*nullable=*/false)}),
+      /*nullable=*/false));
+
+  const char* json = R"([
+      [{"a": ["1", "2"], "b": []}],
+      [],
+      null,
+      [],
+      [{"a": [], "b": [3, 4, 5]}, {"a": ["6"], "b": []}],
+      [{"a": [null, "7"], "b": [8]}]])";
+  auto array = ::arrow::ArrayFromJSON(type, json);
+  auto table = ::arrow::Table::Make(::arrow::schema({field("root", type)}), {array});
+  CheckSimpleRoundtrip(table, 2);
+}
+
 TEST(TestArrowReadWrite, DictionaryColumnChunkedWrite) {
   // This is a regression test for this:
   //
@@ -2464,6 +2693,39 @@ TEST(TestArrowReadWrite, DictionaryColumnChunkedWrite) {
   ::arrow::AssertTablesEqual(*expected_table, *result, false);
 }
 
+TEST(TestArrowReadWrite, NonUniqueDictionaryValues) {
+  // ARROW-10237
+  auto dict_with_dupes = ArrayFromJSON(::arrow::utf8(), R"(["a", "a", "b"])");
+  // test with all valid 4-long `indices`
+  for (int i = 0; i < 4 * 4 * 4 * 4; ++i) {
+    int j = i;
+    ASSERT_OK_AND_ASSIGN(
+        auto indices,
+        ArrayFromBuilderVisitor(::arrow::int32(), 4, [&](::arrow::Int32Builder* b) {
+          if (j % 4 < dict_with_dupes->length()) {
+            b->UnsafeAppend(j % 4);
+          } else {
+            b->UnsafeAppendNull();
+          }
+          j /= 4;
+        }));
+    ASSERT_OK_AND_ASSIGN(auto plain, ::arrow::compute::Take(*dict_with_dupes, *indices));
+    ASSERT_OK_AND_ASSIGN(auto encoded,
+                         ::arrow::DictionaryArray::FromArrays(indices, dict_with_dupes));
+
+    auto table = Table::Make(::arrow::schema({::arrow::field("d", encoded->type())}),
+                             ::arrow::ArrayVector{encoded});
+
+    ASSERT_OK(table->ValidateFull());
+
+    std::shared_ptr<Table> round_tripped;
+    ASSERT_NO_FATAL_FAILURE(DoSimpleRoundtrip(table, true, 20, {}, &round_tripped));
+
+    ASSERT_OK(round_tripped->ValidateFull());
+    ::arrow::AssertArraysEqual(*plain, *round_tripped->column(0)->chunk(0), true);
+  }
+}
+
 TEST(TestArrowWrite, CheckChunkSize) {
   const int num_columns = 2;
   const int num_rows = 128;
@@ -2810,12 +3072,6 @@ TEST_F(TestNestedSchemaRead, ReadTablePartial) {
   ASSERT_NO_FATAL_FAILURE(ValidateTableArrayTypes(*table));
 }
 
-TEST_F(TestNestedSchemaRead, StructAndListTogetherUnsupported) {
-  ASSERT_NO_FATAL_FAILURE(CreateSimpleNestedParquet(Repetition::REPEATED));
-  std::shared_ptr<Table> table;
-  ASSERT_RAISES(NotImplemented, reader_->ReadTable(&table));
-}
-
 TEST_P(TestNestedSchemaRead, DeepNestedSchemaRead) {
 #ifdef PARQUET_VALGRIND
   const int num_trees = 3;
@@ -2994,7 +3250,6 @@ TEST_P(TestArrowReaderAdHocSparkAndHvr, ReadDecimals) {
     ASSERT_OK(builder.Append(value));
   }
   ASSERT_OK(builder.Finish(&expected_array));
-
   AssertArraysEqual(*expected_array, *chunk);
 }
 
@@ -3266,8 +3521,6 @@ TEST(TestArrowWriteDictionaries, AutoReadAsDictionary) {
 }
 
 TEST(TestArrowWriteDictionaries, NestedSubfield) {
-  // ARROW-3246: Automatic decoding of dictionary subfields left as followup
-  // work
   auto offsets = ::arrow::ArrayFromJSON(::arrow::int32(), "[0, 0, 2, 3]");
   auto indices = ::arrow::ArrayFromJSON(::arrow::int32(), "[0, 0, 0]");
   auto dict = ::arrow::ArrayFromJSON(::arrow::utf8(), "[\"foo\"]");
@@ -3278,20 +3531,14 @@ TEST(TestArrowWriteDictionaries, NestedSubfield) {
   ASSERT_OK_AND_ASSIGN(auto values,
                        ::arrow::ListArray::FromArrays(*offsets, *dict_values));
 
-  auto dense_ty = ::arrow::list(::arrow::utf8());
-  auto dense_values =
-      ::arrow::ArrayFromJSON(dense_ty, "[[], [\"foo\", \"foo\"], [\"foo\"]]");
-
   auto table = MakeSimpleTable(values, /*nullable=*/true);
-  auto expected_table = MakeSimpleTable(dense_values, /*nullable=*/true);
 
   auto props_store_schema = ArrowWriterProperties::Builder().store_schema()->build();
   std::shared_ptr<Table> actual;
   DoRoundtrip(table, values->length(), &actual, default_writer_properties(),
               props_store_schema);
 
-  // The nested subfield is not automatically decoded to dictionary
-  ::arrow::AssertTablesEqual(*expected_table, *actual);
+  ::arrow::AssertTablesEqual(*table, *actual);
 }
 
 }  // namespace arrow
diff --git a/cpp/src/parquet/arrow/arrow_schema_test.cc b/cpp/src/parquet/arrow/arrow_schema_test.cc
index b61694bd56d..6b8fa591e91 100644
--- a/cpp/src/parquet/arrow/arrow_schema_test.cc
+++ b/cpp/src/parquet/arrow/arrow_schema_test.cc
@@ -18,6 +18,7 @@
 #include <memory>
 #include <vector>
 
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
 #include "parquet/arrow/reader.h"
@@ -27,8 +28,9 @@
 #include "parquet/schema.h"
 #include "parquet/test_util.h"
 
-#include "arrow/api.h"
+#include "arrow/array.h"
 #include "arrow/testing/gtest_util.h"
+#include "arrow/type.h"
 
 using arrow::ArrayFromVector;
 using arrow::Field;
@@ -38,10 +40,13 @@ using ParquetType = parquet::Type;
 using parquet::ConvertedType;
 using parquet::LogicalType;
 using parquet::Repetition;
+using parquet::internal::LevelInfo;
 using parquet::schema::GroupNode;
 using parquet::schema::NodePtr;
 using parquet::schema::PrimitiveNode;
 
+using ::testing::ElementsAre;
+
 namespace parquet {
 
 namespace arrow {
@@ -342,6 +347,44 @@ TEST_F(TestConvertParquetSchema, ParquetFlatDecimals) {
   ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(arrow_schema));
 }
 
+TEST_F(TestConvertParquetSchema, ParquetMaps) {
+  std::vector<NodePtr> parquet_fields;
+  std::vector<std::shared_ptr<Field>> arrow_fields;
+
+  // MAP encoding example taken from parquet-format/LogicalTypes.md
+
+  // Two column map.
+  {
+    auto key = PrimitiveNode::Make("key", Repetition::REQUIRED, ParquetType::BYTE_ARRAY,
+                                   ConvertedType::UTF8);
+    auto value = PrimitiveNode::Make("value", Repetition::OPTIONAL,
+                                     ParquetType::BYTE_ARRAY, ConvertedType::UTF8);
+
+    auto list = GroupNode::Make("key_value", Repetition::REPEATED, {key, value});
+    parquet_fields.push_back(
+        GroupNode::Make("my_map", Repetition::REQUIRED, {list}, LogicalType::Map()));
+    auto arrow_value = ::arrow::field("string", UTF8, /*nullable=*/true);
+    auto arrow_map = ::arrow::map(/*key=*/UTF8, arrow_value);
+    arrow_fields.push_back(::arrow::field("my_map", arrow_map, false));
+  }
+  // Single column map (i.e. set) gets converted to list of struct.
+  {
+    auto key = PrimitiveNode::Make("key", Repetition::REQUIRED, ParquetType::BYTE_ARRAY,
+                                   ConvertedType::UTF8);
+
+    auto list = GroupNode::Make("key_value", Repetition::REPEATED, {key});
+    parquet_fields.push_back(
+        GroupNode::Make("my_set", Repetition::REQUIRED, {list}, LogicalType::Map()));
+    auto arrow_list = ::arrow::list({::arrow::field("key", UTF8, /*nullable=*/false)});
+    arrow_fields.push_back(::arrow::field("my_set", arrow_list, false));
+  }
+
+  auto arrow_schema = ::arrow::schema(arrow_fields);
+  ASSERT_OK(ConvertSchema(parquet_fields));
+
+  ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(arrow_schema));
+}
+
 TEST_F(TestConvertParquetSchema, ParquetLists) {
   std::vector<NodePtr> parquet_fields;
   std::vector<std::shared_ptr<Field>> arrow_fields;
@@ -945,12 +988,13 @@ TEST_F(TestConvertArrowSchema, ParquetMaps) {
   std::vector<NodePtr> parquet_fields;
   std::vector<std::shared_ptr<Field>> arrow_fields;
 
-  //  optional group my_map (MAP) {
-  //      repeated group key_value {
-  //          required binary key (UTF8);
-  //          optional binary value (UTF8);
-  //	}
-  //  }
+  // // Map<String, String> (map and map values nullable)
+  // optional group my_map (MAP) {
+  //   repeated group key_value {
+  //     required binary key (UTF8);
+  //     optional binary value (UTF8);
+  //   }
+  // }
   {
     auto key = PrimitiveNode::Make("key", Repetition::REQUIRED, ParquetType::BYTE_ARRAY,
                                    ConvertedType::UTF8);
@@ -966,12 +1010,13 @@ TEST_F(TestConvertArrowSchema, ParquetMaps) {
     arrow_fields.push_back(::arrow::field("my_map", arrow_map, /*nullable=*/true));
   }
 
-  //  required group my_map (MAP) {
-  //      repeated group key_value {
-  //          required binary key (UTF8);
-  //          required binary value (UTF8);
-  //	}
-  //  }
+  // // Map<String, String> (non-nullable)
+  // required group my_map (MAP) {
+  //   repeated group key_value {
+  //     required binary key (UTF8);
+  //     required binary value (UTF8);
+  //   }
+  // }
   {
     auto key = PrimitiveNode::Make("key", Repetition::REQUIRED, ParquetType::BYTE_ARRAY,
                                    ConvertedType::UTF8);
@@ -998,7 +1043,7 @@ TEST_F(TestConvertArrowSchema, ParquetOtherLists) {
 
   // parquet_arrow will always generate 3-level LIST encodings
 
-  // // List<String> (list non-null, elements nullable)
+  // // LargeList<String> (list-like non-null, elements nullable)
   // required group my_list (LIST) {
   //   repeated group list {
   //     optional binary element (UTF8);
@@ -1014,6 +1059,12 @@ TEST_F(TestConvertArrowSchema, ParquetOtherLists) {
     auto arrow_list = ::arrow::large_list(arrow_element);
     arrow_fields.push_back(::arrow::field("my_list", arrow_list, false));
   }
+  // // FixedSizeList[10]<String> (list-like non-null, elements nullable)
+  // required group my_list (LIST) {
+  //   repeated group list {
+  //     optional binary element (UTF8);
+  //   }
+  // }
   {
     auto element = PrimitiveNode::Make("string", Repetition::OPTIONAL,
                                        ParquetType::BYTE_ARRAY, ConvertedType::UTF8);
@@ -1140,5 +1191,372 @@ TEST(TestFromParquetSchema, CorruptMetadata) {
   ASSERT_RAISES(IOError, FromParquetSchema(parquet_schema, props, &arrow_schema));
 }
 
+//
+// Test LevelInfo computation from a Parquet schema
+// (for Parquet -> Arrow reading).
+//
+
+::arrow::Result<std::deque<LevelInfo>> RootToTreeLeafLevels(
+    const SchemaManifest& manifest, int column_number) {
+  std::deque<LevelInfo> out;
+  const SchemaField* field;
+  RETURN_NOT_OK(manifest.GetColumnField(column_number, &field));
+  while (field != nullptr) {
+    out.push_front(field->level_info);
+    field = manifest.GetParent(field);
+  }
+  return out;
+}
+
+class TestLevels : public ::testing::Test {
+ public:
+  virtual void SetUp() {}
+
+  ::arrow::Status MaybeSetParquetSchema(const NodePtr& column) {
+    descriptor_.reset(new SchemaDescriptor());
+    manifest_.reset(new SchemaManifest());
+    descriptor_->Init(GroupNode::Make("root", Repetition::REQUIRED, {column}));
+    return SchemaManifest::Make(descriptor_.get(),
+                                std::shared_ptr<const ::arrow::KeyValueMetadata>(),
+                                ArrowReaderProperties(), manifest_.get());
+  }
+  void SetParquetSchema(const NodePtr& column) {
+    ASSERT_OK(MaybeSetParquetSchema(column));
+  }
+
+ protected:
+  std::unique_ptr<SchemaDescriptor> descriptor_;
+  std::unique_ptr<SchemaManifest> manifest_;
+};
+
+TEST_F(TestLevels, TestPrimitive) {
+  SetParquetSchema(
+      PrimitiveNode::Make("node_name", Repetition::REQUIRED, ParquetType::BOOLEAN));
+  ASSERT_OK_AND_ASSIGN(std::deque<LevelInfo> levels,
+                       RootToTreeLeafLevels(*manifest_, /*column_number=*/0));
+  EXPECT_THAT(levels, ElementsAre(LevelInfo{/*null_slot_usage=*/1,
+                                            /*def_level=*/0, /*rep_level=*/0,
+                                            /*ancestor_list_def_level*/ 0}));
+  SetParquetSchema(
+      PrimitiveNode::Make("node_name", Repetition::OPTIONAL, ParquetType::BOOLEAN));
+  ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_, /*column_number=*/0));
+  EXPECT_THAT(levels, ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1,
+                                            /*rep_level=*/0,
+                                            /*ancestor_list_def_level*/ 0}));
+
+  // Arrow schema: list(bool not null) not null
+  SetParquetSchema(
+      PrimitiveNode::Make("node_name", Repetition::REPEATED, ParquetType::BOOLEAN));
+  ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_, /*column_number=*/0));
+  EXPECT_THAT(
+      levels,
+      ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 0},  // List Field
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 1}));  //  primitive field
+}
+
+TEST_F(TestLevels, TestMaps) {
+  // Two column map.
+  auto key = PrimitiveNode::Make("key", Repetition::REQUIRED, ParquetType::BYTE_ARRAY,
+                                 ConvertedType::UTF8);
+  auto value = PrimitiveNode::Make("value", Repetition::OPTIONAL, ParquetType::BYTE_ARRAY,
+                                   ConvertedType::UTF8);
+
+  auto list = GroupNode::Make("key_value", Repetition::REPEATED, {key, value});
+  SetParquetSchema(
+      GroupNode::Make("my_map", Repetition::OPTIONAL, {list}, LogicalType::Map()));
+  ASSERT_OK_AND_ASSIGN(std::deque<LevelInfo> levels,
+                       RootToTreeLeafLevels(*manifest_, /*column_number=*/0));
+  EXPECT_THAT(
+      levels,
+      ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 0},
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 2},
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 2}));
+
+  ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_, /*column_number=*/1));
+  EXPECT_THAT(
+      levels,
+      ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 0},
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 2},
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/3, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 2}));
+
+  // single column map.
+  key = PrimitiveNode::Make("key", Repetition::REQUIRED, ParquetType::BYTE_ARRAY,
+                            ConvertedType::UTF8);
+
+  list = GroupNode::Make("key_value", Repetition::REPEATED, {key});
+  SetParquetSchema(
+      GroupNode::Make("my_set", Repetition::REQUIRED, {list}, LogicalType::Map()));
+
+  ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_, /*column_number=*/0));
+  EXPECT_THAT(
+      levels,
+      ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 0},
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 1}));
+}
+
+TEST_F(TestLevels, TestSimpleGroups) {
+  // Arrow schema: struct(child: struct(inner: boolean not null))
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {GroupNode::Make(
+          "child", Repetition::OPTIONAL,
+          {PrimitiveNode::Make("inner", Repetition::REQUIRED, ParquetType::BOOLEAN)})}));
+  ASSERT_OK_AND_ASSIGN(std::deque<LevelInfo> levels,
+                       RootToTreeLeafLevels(*manifest_, /*column_number=*/0));
+  EXPECT_THAT(
+      levels,
+      ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1, /*rep_level=*/0,
+                            /*ancestor_list_def_level*/ 0},
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2, /*rep_level=*/0,
+                            /*ancestor_list_def_level*/ 0},
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2, /*rep_level=*/0,
+                            /*ancestor_list_def_level*/ 0}));
+
+  // Arrow schema: struct(child: struct(inner: boolean ))
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {GroupNode::Make(
+          "child", Repetition::OPTIONAL,
+          {PrimitiveNode::Make("inner", Repetition::OPTIONAL, ParquetType::BOOLEAN)})}));
+  ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_, /*column_number=*/0));
+  EXPECT_THAT(
+      levels,
+      ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1, /*rep_level=*/0,
+                            /*ancestor_list_def_level*/ 0},
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2, /*rep_level=*/0,
+                            /*ancestor_list_def_level*/ 0},
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/3, /*rep_level=*/0,
+                            /*ancestor_list_def_level*/ 0}));
+
+  // Arrow schema: struct(child: struct(inner: boolean)) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {GroupNode::Make(
+          "child", Repetition::OPTIONAL,
+          {PrimitiveNode::Make("inner", Repetition::OPTIONAL, ParquetType::BOOLEAN)})}));
+  ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_, /*column_number=*/0));
+  EXPECT_THAT(
+      levels,
+      ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/0, /*rep_level=*/0,
+                            /*ancestor_list_def_level*/ 0},
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1, /*rep_level=*/0,
+                            /*ancestor_list_def_level*/ 0},
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2, /*rep_level=*/0,
+                            /*ancestor_list_def_level*/ 0}));
+}
+
+TEST_F(TestLevels, TestRepeatedGroups) {
+  // Arrow schema: list(bool)
+  SetParquetSchema(GroupNode::Make(
+      "child_list", Repetition::OPTIONAL,
+      {GroupNode::Make(
+          "list", Repetition::REPEATED,
+          {PrimitiveNode::Make("element", Repetition::OPTIONAL, ParquetType::BOOLEAN)})},
+      LogicalType::List()));
+
+  ASSERT_OK_AND_ASSIGN(std::deque<LevelInfo> levels,
+                       RootToTreeLeafLevels(*manifest_, /*column_number=*/0));
+  EXPECT_THAT(
+      levels,
+      ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 0},
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/3, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 2}));
+
+  // Arrow schema: list(bool) not null
+  SetParquetSchema(GroupNode::Make(
+      "child_list", Repetition::REQUIRED,
+      {GroupNode::Make(
+          "list", Repetition::REPEATED,
+          {PrimitiveNode::Make("element", Repetition::OPTIONAL, ParquetType::BOOLEAN)})},
+      LogicalType::List()));
+
+  ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_, /*column_number=*/0));
+  EXPECT_THAT(
+      levels,
+      ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 0},
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 1}));
+
+  // Arrow schema: list(bool not null)
+  SetParquetSchema(GroupNode::Make(
+      "child_list", Repetition::OPTIONAL,
+      {GroupNode::Make(
+          "list", Repetition::REPEATED,
+          {PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::BOOLEAN)})},
+      LogicalType::List()));
+
+  ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_, /*column_number=*/0));
+  EXPECT_THAT(
+      levels,
+      ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 0},
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 2}));
+
+  // Arrow schema: list(bool not null) not null
+  SetParquetSchema(GroupNode::Make(
+      "child_list", Repetition::REQUIRED,
+      {GroupNode::Make(
+          "list", Repetition::REPEATED,
+          {PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::BOOLEAN)})},
+      LogicalType::List()));
+
+  ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_, /*column_number=*/0));
+  EXPECT_THAT(
+      levels,
+      ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 0},
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 1}));
+
+  // Arrow schema: list(struct(child: struct(list(bool not null) not null)) non null) not
+  // null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REPEATED,
+      {GroupNode::Make(
+          "child", Repetition::OPTIONAL,
+          {PrimitiveNode::Make("inner", Repetition::REPEATED, ParquetType::BOOLEAN)})}));
+  ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_, /*column_number=*/0));
+  EXPECT_THAT(
+      levels,
+      ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 0},
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 1},
+
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 1},  // optional child struct
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/3, /*rep_level=*/2,
+                            /*ancestor_list_def_level*/ 1},  // repeated field
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/3, /*rep_level=*/2,
+                            /*ancestor_list_def_level*/ 3}));  // inner field
+
+  // Arrow schema: list(struct(child_list: list(struct(f0: bool f1: bool))) not null) not
+  // null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REPEATED,
+      {GroupNode::Make(
+          "child_list", Repetition::OPTIONAL,
+          {GroupNode::Make(
+              "list", Repetition::REPEATED,
+              {GroupNode::Make(
+                  "element", Repetition::OPTIONAL,
+                  {PrimitiveNode::Make("f0", Repetition::OPTIONAL, ParquetType::BOOLEAN),
+                   PrimitiveNode::Make("f1", Repetition::REQUIRED,
+                                       ParquetType::BOOLEAN)})})},
+          LogicalType::List())}));
+  ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_, /*column_number=*/0));
+  EXPECT_THAT(
+      levels,
+      ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 0},  // parent list
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 1},  // parent struct
+
+                  // Def_level=2 is handled together with def_level=3
+                  // When decoding.  Def_level=2 indicates present but empty
+                  // list.  def_level=3 indicates a present element in the
+                  // list.
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/3, /*rep_level=*/2,
+                            /*ancestor_list_def_level*/ 1},  // list field
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/4, /*rep_level=*/2,
+                            /*ancestor_list_def_level*/ 3},  // inner struct field
+
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/5, /*rep_level=*/2,
+                            /*ancestor_list_def_level*/ 3}));  // f0 bool field
+
+  ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_, /*column_number=*/1));
+  EXPECT_THAT(
+      levels,
+      ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 0},  // parent list
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 1},  // parent struct
+                  // Def_level=2 is handled together with def_level=3
+                  // When decoding.  Def_level=2 indicate present but empty
+                  // list.  def_level=3 indicates a present element in the
+                  // list.
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/3, /*rep_level=*/2,
+                            /*ancestor_list_def_level*/ 1},  // list field
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/4, /*rep_level=*/2,
+                            /*ancestor_list_def_level*/ 3},  // inner struct field
+
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/4, /*rep_level=*/2,
+                            /*ancestor_list_def_level*/ 3}));  // f1 bool field
+
+  // Arrow schema: list(struct(child_list: list(bool not null)) not null) not null
+  // Legacy 2-level encoding (required for backwards compatibility.  See
+  // https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#nested-types
+  // for definitions).
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REPEATED,
+      {GroupNode::Make(
+          "child_list", Repetition::OPTIONAL,
+          {PrimitiveNode::Make("bool", Repetition::REPEATED, ParquetType::BOOLEAN)},
+          LogicalType::List())}));
+
+  ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_, /*column_number=*/0));
+  EXPECT_THAT(
+      levels,
+      ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 0},  // parent list
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1, /*rep_level=*/1,
+                            /*ancestor_list_def_level*/ 1},  // parent struct
+
+                  // Def_level=2 is handled together with def_level=3
+                  // When decoding.  Def_level=2 indicate present but empty
+                  // list.  def_level=3 indicates a present element in the
+                  // list.
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/3, /*rep_level=*/2,
+                            /*ancestor_list_def_level*/ 1},  // list field
+                  LevelInfo{/*null_slot_usage=*/1, /*def_level=*/3, /*rep_level=*/2,
+                            /*ancestor_list_def_level*/ 3}));  // inner bool
+}
+
+TEST_F(TestLevels, ListErrors) {
+  {
+    ::arrow::Status error = MaybeSetParquetSchema(GroupNode::Make(
+        "child_list", Repetition::REPEATED,
+        {PrimitiveNode::Make("bool", Repetition::REPEATED, ParquetType::BOOLEAN)},
+        LogicalType::List()));
+    ASSERT_RAISES(Invalid, error);
+    std::string expected("LIST-annotated groups must not be repeated.");
+    EXPECT_EQ(error.message().substr(0, expected.size()), expected);
+  }
+  {
+    ::arrow::Status error = MaybeSetParquetSchema(GroupNode::Make(
+        "child_list", Repetition::OPTIONAL,
+        {PrimitiveNode::Make("f1", Repetition::REPEATED, ParquetType::BOOLEAN),
+         PrimitiveNode::Make("f2", Repetition::REPEATED, ParquetType::BOOLEAN)},
+        LogicalType::List()));
+    ASSERT_RAISES(Invalid, error);
+    std::string expected("LIST-annotated groups must have a single child.");
+    EXPECT_EQ(error.message().substr(0, expected.size()), expected);
+  }
+
+  {
+    ::arrow::Status error = MaybeSetParquetSchema(GroupNode::Make(
+        "child_list", Repetition::OPTIONAL,
+        {PrimitiveNode::Make("f1", Repetition::OPTIONAL, ParquetType::BOOLEAN)},
+        LogicalType::List()));
+    ASSERT_RAISES(Invalid, error);
+    std::string expected(
+        "Non-repeated nodes in a LIST-annotated group are not supported.");
+    EXPECT_EQ(error.message().substr(0, expected.size()), expected);
+  }
+}
+
 }  // namespace arrow
 }  // namespace parquet
diff --git a/cpp/src/parquet/arrow/generate_fuzz_corpus.cc b/cpp/src/parquet/arrow/generate_fuzz_corpus.cc
index f2a1b22b66b..ec5accdeea0 100644
--- a/cpp/src/parquet/arrow/generate_fuzz_corpus.cc
+++ b/cpp/src/parquet/arrow/generate_fuzz_corpus.cc
@@ -32,6 +32,7 @@
 #include "arrow/scalar.h"
 #include "arrow/table.h"
 #include "arrow/testing/random.h"
+#include "arrow/util/compression.h"
 #include "arrow/util/io_util.h"
 #include "arrow/util/key_value_metadata.h"
 #include "parquet/arrow/writer.h"
@@ -48,32 +49,94 @@ static constexpr int32_t kChunkSize = kBatchSize * 3 / 8;
 std::shared_ptr<WriterProperties> GetWriterProperties() {
   WriterProperties::Builder builder{};
   builder.disable_dictionary("no_dict");
+  builder.compression("compressed", Compression::BROTLI);
   return builder.build();
 }
 
 Result<std::shared_ptr<RecordBatch>> ExampleBatch1() {
+  constexpr double kNullProbability = 0.2;
+
   random::RandomArrayGenerator gen(42);
-  std::shared_ptr<Array> a, b, c, d, e, no_dict;
-  a = gen.Int16(kBatchSize, -10000, 10000, /*null_probability=*/0.2);
+  std::shared_ptr<Array> a, b, c, d, e, f, g, h, no_dict, compressed;
+  std::shared_ptr<Field> f_a, f_b, f_c, f_d, f_e, f_f, f_g, f_h, f_no_dict, f_compressed;
+
+  a = gen.Int16(kBatchSize, -10000, 10000, kNullProbability);
+  f_a = field("a", a->type());
+
   b = gen.Float64(kBatchSize, -1e10, 1e10, /*null_probability=*/0.0);
+  f_b = field("b", b->type());
+
   // A column of tiny strings that will hopefully trigger dict encoding
-  c = gen.String(kBatchSize, 0, 3, /*null_probability=*/0.2);
+  c = gen.String(kBatchSize, 0, 3, kNullProbability);
+  f_c = field("c", c->type());
+
+  // A column of lists
   {
-    auto values = gen.Int64(kBatchSize * 10, -10000, 10000, /*null_probability=*/0.2);
+    auto values = gen.Int64(kBatchSize * 10, -10000, 10000, kNullProbability);
     auto offsets = gen.Offsets(kBatchSize + 1, 0, static_cast<int32_t>(values->length()));
     ARROW_ASSIGN_OR_RAISE(d, ListArray::FromArrays(*offsets, *values));
   }
+  f_d = field("d", d->type());
+
   // A column of a repeated constant that will hopefully trigger RLE encoding
   ARROW_ASSIGN_OR_RAISE(e, MakeArrayFromScalar(Int16Scalar(42), kBatchSize));
-  // A non-dict-encoded column
-  no_dict = gen.String(kBatchSize, 0, 30, /*null_probability=*/0.2);
+  f_e = field("e", e->type());
+
+  // A column of lists of lists
+  {
+    auto inner_values = gen.Int64(kBatchSize * 9, -10000, 10000, kNullProbability);
+    auto inner_offsets =
+        gen.Offsets(kBatchSize * 3 + 1, 0, static_cast<int32_t>(inner_values->length()),
+                    kNullProbability);
+    ARROW_ASSIGN_OR_RAISE(auto inner_lists,
+                          ListArray::FromArrays(*inner_offsets, *inner_values));
+    auto offsets = gen.Offsets(
+        kBatchSize + 1, 0, static_cast<int32_t>(inner_lists->length()), kNullProbability);
+    ARROW_ASSIGN_OR_RAISE(f, ListArray::FromArrays(*offsets, *inner_lists));
+  }
+  f_f = field("f", f->type());
+
+  // A column of nested non-nullable structs
+  {
+    ARROW_ASSIGN_OR_RAISE(
+        auto inner_a,
+        StructArray::Make({a, b}, std::vector<std::string>{"inner1_aa", "inner1_ab"}));
+    ARROW_ASSIGN_OR_RAISE(
+        g, StructArray::Make({inner_a, c},
+                             {field("inner1_a", inner_a->type(), /*nullable=*/false),
+                              field("inner1_c", c->type())}));
+  }
+  f_g = field("g", g->type(), /*nullable=*/false);
+
+  // A column of nested nullable structs
+  {
+    auto null_bitmap = gen.NullBitmap(kBatchSize, kNullProbability);
+    ARROW_ASSIGN_OR_RAISE(
+        auto inner_a,
+        StructArray::Make({a, b}, std::vector<std::string>{"inner2_aa", "inner2_ab"},
+                          std::move(null_bitmap)));
+    null_bitmap = gen.NullBitmap(kBatchSize, kNullProbability);
+    ARROW_ASSIGN_OR_RAISE(
+        h,
+        StructArray::Make({inner_a, c}, std::vector<std::string>{"inner2_a", "inner2_c"},
+                          std::move(null_bitmap)));
+  }
+  f_h = field("h", h->type());
+
+  // A non-dict-encoded column (see GetWriterProperties)
+  no_dict = gen.String(kBatchSize, 0, 30, kNullProbability);
+  f_no_dict = field("no_dict", no_dict->type());
+
+  // A non-dict-encoded column (see GetWriterProperties)
+  compressed = gen.Int64(kBatchSize, -10, 10, kNullProbability);
+  f_compressed = field("compressed", compressed->type());
 
-  auto schema = ::arrow::schema(
-      {field("a", a->type()), field("b", b->type()), field("c", c->type()),
-       field("d", d->type()), field("e", e->type()), field("no_dict", no_dict->type())});
+  auto schema =
+      ::arrow::schema({f_a, f_b, f_c, f_d, f_e, f_f, f_g, f_h, f_compressed, f_no_dict});
   auto md = key_value_metadata({"key1", "key2"}, {"value1", ""});
   schema = schema->WithMetadata(md);
-  return RecordBatch::Make(schema, kBatchSize, {a, b, c, d, e, no_dict});
+  return RecordBatch::Make(schema, kBatchSize,
+                           {a, b, c, d, e, f, g, h, compressed, no_dict});
 }
 
 Result<std::vector<std::shared_ptr<RecordBatch>>> Batches() {
diff --git a/cpp/src/parquet/arrow/path_internal.cc b/cpp/src/parquet/arrow/path_internal.cc
index e2079b71f21..daa05a81c79 100644
--- a/cpp/src/parquet/arrow/path_internal.cc
+++ b/cpp/src/parquet/arrow/path_internal.cc
@@ -525,6 +525,7 @@ struct PathInfo {
   int16_t max_def_level = 0;
   int16_t max_rep_level = 0;
   bool has_dictionary = false;
+  bool leaf_is_nullable = false;
 };
 
 /// Contains logic for writing a single leaf node to parquet.
@@ -540,6 +541,7 @@ Status WritePath(ElementRange root_range, PathInfo* path_info,
   std::vector<ElementRange> stack(path_info->path.size());
   MultipathLevelBuilderResult builder_result;
   builder_result.leaf_array = path_info->primitive_array;
+  builder_result.leaf_is_nullable = path_info->leaf_is_nullable;
 
   if (path_info->max_def_level == 0) {
     // This case only occurs when there are no nullable or repeated
@@ -706,6 +708,7 @@ class PathBuilder {
   explicit PathBuilder(bool start_nullable) : nullable_in_parent_(start_nullable) {}
   template <typename T>
   void AddTerminalInfo(const T& array) {
+    info_.leaf_is_nullable = nullable_in_parent_;
     if (nullable_in_parent_) {
       info_.max_def_level++;
     }
@@ -812,13 +815,17 @@ class PathBuilder {
   Status Visit(const ::arrow::FixedSizeListArray& array) {
     MaybeAddNullable(array);
     int32_t list_size = array.list_type()->list_size();
-    if (list_size == 0) {
-      info_.max_def_level++;
-    }
+    // Technically we could encode fixed size lists with two level encodings
+    // but since we always use 3 level encoding we increment def levels as
+    // well.
+    info_.max_def_level++;
     info_.max_rep_level++;
     info_.path.push_back(FixedSizeListNode(FixedSizedRangeSelector{list_size},
                                            info_.max_rep_level, info_.max_def_level));
     nullable_in_parent_ = array.list_type()->value_field()->nullable();
+    if (array.offset() > 0) {
+      return VisitInline(*array.values()->Slice(array.value_offset(0)));
+    }
     return VisitInline(*array.values());
   }
 
diff --git a/cpp/src/parquet/arrow/path_internal.h b/cpp/src/parquet/arrow/path_internal.h
index 17c486dc7b2..c5b7fdfdac3 100644
--- a/cpp/src/parquet/arrow/path_internal.h
+++ b/cpp/src/parquet/arrow/path_internal.h
@@ -91,6 +91,9 @@ struct MultipathLevelBuilderResult {
   /// This allows for the parquet writing to determine which values ultimately
   /// needs to be written.
   std::vector<ElementRange> post_list_visited_elements;
+
+  /// Whether the leaf array is nullable.
+  bool leaf_is_nullable;
 };
 
 /// \brief Logic for being able to write out nesting (rep/def level) data that is
diff --git a/cpp/src/parquet/arrow/path_internal_test.cc b/cpp/src/parquet/arrow/path_internal_test.cc
index f122a08f668..065e9866e0b 100644
--- a/cpp/src/parquet/arrow/path_internal_test.cc
+++ b/cpp/src/parquet/arrow/path_internal_test.cc
@@ -461,6 +461,26 @@ TEST_F(MultipathLevelBuilderTest, TestStruct) {
       /*def_levels=*/std::vector<int16_t>({2, 2, 0}));
 }
 
+TEST_F(MultipathLevelBuilderTest, TestFixedSizeListNullableElements) {
+  auto entries = field("Entries", ::arrow::int64());
+  auto list_type = fixed_size_list(entries, 2);
+  auto array = ::arrow::ArrayFromJSON(list_type, R"([null, [2, 3], [4, 5], null])");
+
+  ASSERT_OK(
+      MultipathLevelBuilder::Write(*array, /*nullable=*/true, &context_, callback_));
+
+  ASSERT_THAT(results_, SizeIs(1));
+  results_[0].CheckLevels(/*def_levels=*/std::vector<int16_t>{0, 3, 3, 3, 3, 0},
+                          /*rep_levels=*/std::vector<int16_t>{0, 0, 1, 0, 1, 0});
+
+  // Null slots take up space in a fixed size list (they can in variable size
+  // lists as well) but the actual written values are only the "middle" elements
+  // in this case.
+  ASSERT_THAT(results_[0].post_list_elements, SizeIs(1));
+  EXPECT_THAT(results_[0].post_list_elements[0].start, Eq(2));
+  EXPECT_THAT(results_[0].post_list_elements[0].end, Eq(6));
+}
+
 TEST_F(MultipathLevelBuilderTest, TestFixedSizeList) {
   auto entries = field("Entries", ::arrow::int64(), /*nullable=*/false);
   auto list_type = fixed_size_list(entries, 2);
@@ -470,7 +490,7 @@ TEST_F(MultipathLevelBuilderTest, TestFixedSizeList) {
       MultipathLevelBuilder::Write(*array, /*nullable=*/true, &context_, callback_));
 
   ASSERT_THAT(results_, SizeIs(1));
-  results_[0].CheckLevels(/*def_levels=*/std::vector<int16_t>{0, 1, 1, 1, 1, 0},
+  results_[0].CheckLevels(/*def_levels=*/std::vector<int16_t>{0, 2, 2, 2, 2, 0},
                           /*rep_levels=*/std::vector<int16_t>{0, 0, 1, 0, 1, 0});
 
   // Null slots take up space in a fixed size list (they can in variable size
diff --git a/cpp/src/parquet/arrow/reader.cc b/cpp/src/parquet/arrow/reader.cc
index 1d02f09bc6a..0f4e218d72d 100644
--- a/cpp/src/parquet/arrow/reader.cc
+++ b/cpp/src/parquet/arrow/reader.cc
@@ -25,6 +25,7 @@
 
 #include "arrow/array.h"
 #include "arrow/buffer.h"
+#include "arrow/extension_type.h"
 #include "arrow/io/memory.h"
 #include "arrow/record_batch.h"
 #include "arrow/table.h"
@@ -32,8 +33,8 @@
 #include "arrow/util/iterator.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/make_unique.h"
+#include "arrow/util/parallel.h"
 #include "arrow/util/range.h"
-#include "arrow/util/thread_pool.h"
 #include "parquet/arrow/reader_internal.h"
 #include "parquet/column_reader.h"
 #include "parquet/exception.h"
@@ -43,9 +44,11 @@
 #include "parquet/schema.h"
 
 using arrow::Array;
+using arrow::ArrayData;
 using arrow::BooleanArray;
 using arrow::ChunkedArray;
 using arrow::DataType;
+using arrow::ExtensionType;
 using arrow::Field;
 using arrow::Future;
 using arrow::Int32Array;
@@ -57,40 +60,63 @@ using arrow::Status;
 using arrow::StructArray;
 using arrow::Table;
 using arrow::TimestampArray;
-using arrow::internal::Iota;
 
-using parquet::schema::GroupNode;
-using parquet::schema::Node;
-using parquet::schema::PrimitiveNode;
+using arrow::internal::checked_cast;
+using arrow::internal::Iota;
 
 // Help reduce verbosity
 using ParquetReader = parquet::ParquetFileReader;
 
 using parquet::internal::RecordReader;
 
-#define BEGIN_PARQUET_CATCH_EXCEPTIONS try {
-#define END_PARQUET_CATCH_EXCEPTIONS             \
-  }                                              \
-  catch (const ::parquet::ParquetException& e) { \
-    return ::arrow::Status::IOError(e.what());   \
-  }
-
 namespace parquet {
 namespace arrow {
+namespace {
+
+::arrow::Result<std::shared_ptr<ArrayData>> ChunksToSingle(const ChunkedArray& chunked) {
+  switch (chunked.num_chunks()) {
+    case 0: {
+      ARROW_ASSIGN_OR_RAISE(std::shared_ptr<Array> array,
+                            ::arrow::MakeArrayOfNull(chunked.type(), 0));
+      return array->data();
+    }
+    case 1:
+      return chunked.chunk(0)->data();
+    default:
+      // ARROW-3762(wesm): If item reader yields a chunked array, we reject as
+      // this is not yet implemented
+      return Status::NotImplemented(
+          "Nested data conversions not implemented for chunked array outputs");
+  }
+}
+
+}  // namespace
 
 class ColumnReaderImpl : public ColumnReader {
  public:
-  enum ReaderType { PRIMITIVE, LIST, STRUCT };
-
   virtual Status GetDefLevels(const int16_t** data, int64_t* length) = 0;
   virtual Status GetRepLevels(const int16_t** data, int64_t* length) = 0;
   virtual const std::shared_ptr<Field> field() = 0;
 
-  virtual const ColumnDescriptor* descr() const = 0;
+  ::arrow::Status NextBatch(int64_t batch_size,
+                            std::shared_ptr<::arrow::ChunkedArray>* out) final {
+    RETURN_NOT_OK(LoadBatch(batch_size));
+    RETURN_NOT_OK(BuildArray(batch_size, out));
+    for (int x = 0; x < (*out)->num_chunks(); x++) {
+      RETURN_NOT_OK((*out)->chunk(x)->Validate());
+    }
+    return Status::OK();
+  }
 
-  virtual ReaderType type() const = 0;
+  virtual ::arrow::Status LoadBatch(int64_t num_records) = 0;
+
+  virtual ::arrow::Status BuildArray(int64_t length_upper_bound,
+                                     std::shared_ptr<::arrow::ChunkedArray>* out) = 0;
+  virtual bool IsOrHasRepeatedChild() const = 0;
 };
 
+namespace {
+
 std::shared_ptr<std::unordered_set<int>> VectorToSharedSet(
     const std::vector<int>& values) {
   std::shared_ptr<std::unordered_set<int>> result(new std::unordered_set<int>());
@@ -98,6 +124,10 @@ std::shared_ptr<std::unordered_set<int>> VectorToSharedSet(
   return result;
 }
 
+// Forward declaration
+Status GetReader(const SchemaField& field, const std::shared_ptr<ReaderContext>& context,
+                 std::unique_ptr<ColumnReaderImpl>* out);
+
 // ----------------------------------------------------------------------
 // FileReaderImpl forward declaration
 
@@ -148,10 +178,10 @@ class FileReaderImpl : public FileReader {
   Status BoundsCheck(const std::vector<int>& row_groups,
                      const std::vector<int>& column_indices) {
     for (int i : row_groups) {
-      return BoundsCheckRowGroup(i);
+      RETURN_NOT_OK(BoundsCheckRowGroup(i));
     }
     for (int i : column_indices) {
-      return BoundsCheckColumn(i);
+      RETURN_NOT_OK(BoundsCheckColumn(i));
     }
     return Status::OK();
   }
@@ -369,34 +399,41 @@ class RowGroupReaderImpl : public RowGroupReader {
   int row_group_index_;
 };
 
+// ----------------------------------------------------------------------
+// Column reader implementations
+
 // Leaf reader is for primitive arrays and primitive children of nested arrays
 class LeafReader : public ColumnReaderImpl {
  public:
   LeafReader(std::shared_ptr<ReaderContext> ctx, std::shared_ptr<Field> field,
-             std::unique_ptr<FileColumnIterator> input)
+             std::unique_ptr<FileColumnIterator> input,
+             ::parquet::internal::LevelInfo leaf_info)
       : ctx_(std::move(ctx)),
         field_(std::move(field)),
         input_(std::move(input)),
         descr_(input_->descr()) {
     record_reader_ = RecordReader::Make(
-        descr_, ctx_->pool, field_->type()->id() == ::arrow::Type::DICTIONARY);
+        descr_, leaf_info, ctx_->pool, field_->type()->id() == ::arrow::Type::DICTIONARY);
     NextRowGroup();
   }
 
-  Status GetDefLevels(const int16_t** data, int64_t* length) override {
+  Status GetDefLevels(const int16_t** data, int64_t* length) final {
     *data = record_reader_->def_levels();
     *length = record_reader_->levels_position();
     return Status::OK();
   }
 
-  Status GetRepLevels(const int16_t** data, int64_t* length) override {
+  Status GetRepLevels(const int16_t** data, int64_t* length) final {
     *data = record_reader_->rep_levels();
     *length = record_reader_->levels_position();
     return Status::OK();
   }
 
-  Status NextBatch(int64_t records_to_read, std::shared_ptr<ChunkedArray>* out) override {
+  bool IsOrHasRepeatedChild() const final { return false; }
+
+  Status LoadBatch(int64_t records_to_read) final {
     BEGIN_PARQUET_CATCH_EXCEPTIONS
+    out_ = nullptr;
     record_reader_->Reset();
     // Pre-allocation gives much better performance for flat columns
     record_reader_->Reserve(records_to_read);
@@ -411,17 +448,21 @@ class LeafReader : public ColumnReaderImpl {
       }
     }
     RETURN_NOT_OK(TransferColumnData(record_reader_.get(), field_->type(), descr_,
-                                     ctx_->pool, out));
+                                     ctx_->pool, &out_));
     return Status::OK();
     END_PARQUET_CATCH_EXCEPTIONS
   }
 
-  const std::shared_ptr<Field> field() override { return field_; }
-  const ColumnDescriptor* descr() const override { return descr_; }
+  ::arrow::Status BuildArray(int64_t length_upper_bound,
+                             std::shared_ptr<::arrow::ChunkedArray>* out) final {
+    *out = out_;
+    return Status::OK();
+  }
 
-  ReaderType type() const override { return PRIMITIVE; }
+  const std::shared_ptr<Field> field() override { return field_; }
 
  private:
+  std::shared_ptr<ChunkedArray> out_;
   void NextRowGroup() {
     std::unique_ptr<PageReader> page_reader = input_->NextChunk();
     record_reader_->SetPageReader(std::move(page_reader));
@@ -434,16 +475,54 @@ class LeafReader : public ColumnReaderImpl {
   std::shared_ptr<RecordReader> record_reader_;
 };
 
-class NestedListReader : public ColumnReaderImpl {
+// Column reader for extension arrays
+class ExtensionReader : public ColumnReaderImpl {
  public:
-  NestedListReader(std::shared_ptr<ReaderContext> ctx, std::shared_ptr<Field> field,
-                   int16_t max_definition_level, int16_t max_repetition_level,
-                   std::unique_ptr<ColumnReaderImpl> item_reader)
+  ExtensionReader(std::shared_ptr<Field> field,
+                  std::unique_ptr<ColumnReaderImpl> storage_reader)
+      : field_(std::move(field)), storage_reader_(std::move(storage_reader)) {}
+
+  Status GetDefLevels(const int16_t** data, int64_t* length) override {
+    return storage_reader_->GetDefLevels(data, length);
+  }
+
+  Status GetRepLevels(const int16_t** data, int64_t* length) override {
+    return storage_reader_->GetRepLevels(data, length);
+  }
+
+  Status LoadBatch(int64_t number_of_records) final {
+    return storage_reader_->LoadBatch(number_of_records);
+  }
+
+  Status BuildArray(int64_t length_upper_bound,
+                    std::shared_ptr<ChunkedArray>* out) override {
+    std::shared_ptr<ChunkedArray> storage;
+    RETURN_NOT_OK(storage_reader_->BuildArray(length_upper_bound, &storage));
+    *out = ExtensionType::WrapArray(field_->type(), storage);
+    return Status::OK();
+  }
+
+  bool IsOrHasRepeatedChild() const final {
+    return storage_reader_->IsOrHasRepeatedChild();
+  }
+
+  const std::shared_ptr<Field> field() override { return field_; }
+
+ private:
+  std::shared_ptr<Field> field_;
+  std::unique_ptr<ColumnReaderImpl> storage_reader_;
+};
+
+template <typename IndexType>
+class ListReader : public ColumnReaderImpl {
+ public:
+  ListReader(std::shared_ptr<ReaderContext> ctx, std::shared_ptr<Field> field,
+             ::parquet::internal::LevelInfo level_info,
+             std::unique_ptr<ColumnReaderImpl> child_reader)
       : ctx_(std::move(ctx)),
         field_(std::move(field)),
-        max_definition_level_(max_definition_level),
-        max_repetition_level_(max_repetition_level),
-        item_reader_(std::move(item_reader)) {}
+        level_info_(level_info),
+        item_reader_(std::move(child_reader)) {}
 
   Status GetDefLevels(const int16_t** data, int64_t* length) override {
     return item_reader_->GetDefLevels(data, length);
@@ -453,227 +532,249 @@ class NestedListReader : public ColumnReaderImpl {
     return item_reader_->GetRepLevels(data, length);
   }
 
-  Status NextBatch(int64_t records_to_read, std::shared_ptr<ChunkedArray>* out) override {
-    if (item_reader_->type() == ColumnReaderImpl::STRUCT) {
-      return Status::Invalid("Mix of struct and list types not yet supported");
-    }
+  bool IsOrHasRepeatedChild() const final { return true; }
 
-    RETURN_NOT_OK(item_reader_->NextBatch(records_to_read, out));
+  Status LoadBatch(int64_t number_of_records) final {
+    return item_reader_->LoadBatch(number_of_records);
+  }
 
-    std::shared_ptr<Array> item_chunk;
-    switch ((*out)->num_chunks()) {
-      case 0: {
-        ARROW_ASSIGN_OR_RAISE(item_chunk, ::arrow::MakeArrayOfNull((*out)->type(), 0));
-        break;
-      }
-      case 1:
-        item_chunk = (*out)->chunk(0);
-        break;
-      default:
-        // ARROW-3762(wesm): If item reader yields a chunked array, we reject as
-        // this is not yet implemented
-        return Status::NotImplemented(
-            "Nested data conversions not implemented for chunked array outputs");
+  virtual ::arrow::Result<std::shared_ptr<ChunkedArray>> AssembleArray(
+      std::shared_ptr<ArrayData> data) {
+    if (field_->type()->id() == ::arrow::Type::MAP) {
+      // Error out if data is not map-compliant instead of aborting in MakeArray below
+      RETURN_NOT_OK(::arrow::MapArray::ValidateChildData(data->child_data));
     }
+    std::shared_ptr<Array> result = ::arrow::MakeArray(data);
+    return std::make_shared<ChunkedArray>(result);
+  }
 
+  Status BuildArray(int64_t length_upper_bound,
+                    std::shared_ptr<ChunkedArray>* out) override {
     const int16_t* def_levels;
     const int16_t* rep_levels;
     int64_t num_levels;
     RETURN_NOT_OK(item_reader_->GetDefLevels(&def_levels, &num_levels));
     RETURN_NOT_OK(item_reader_->GetRepLevels(&rep_levels, &num_levels));
-    std::shared_ptr<Array> result;
-    RETURN_NOT_OK(ReconstructNestedList(item_chunk, field_, max_definition_level_,
-                                        max_repetition_level_, def_levels, rep_levels,
-                                        num_levels, ctx_->pool, &result));
-    *out = std::make_shared<ChunkedArray>(result);
+
+    std::shared_ptr<ResizableBuffer> validity_buffer;
+    ::parquet::internal::ValidityBitmapInputOutput validity_io;
+    validity_io.values_read_upper_bound = length_upper_bound;
+    if (field_->nullable()) {
+      ARROW_ASSIGN_OR_RAISE(
+          validity_buffer,
+          AllocateResizableBuffer(BitUtil::BytesForBits(length_upper_bound), ctx_->pool));
+      validity_io.valid_bits = validity_buffer->mutable_data();
+    }
+    ARROW_ASSIGN_OR_RAISE(
+        std::shared_ptr<ResizableBuffer> offsets_buffer,
+        AllocateResizableBuffer(
+            sizeof(IndexType) * std::max(int64_t{1}, length_upper_bound + 1),
+            ctx_->pool));
+    // Ensure zero initialization in case we have reached a zero length list (and
+    // because first entry is always zero).
+    IndexType* offset_data = reinterpret_cast<IndexType*>(offsets_buffer->mutable_data());
+    offset_data[0] = 0;
+    BEGIN_PARQUET_CATCH_EXCEPTIONS
+    ::parquet::internal::DefRepLevelsToList(def_levels, rep_levels, num_levels,
+                                            level_info_, &validity_io, offset_data);
+    END_PARQUET_CATCH_EXCEPTIONS
+
+    RETURN_NOT_OK(item_reader_->BuildArray(offset_data[validity_io.values_read], out));
+
+    // Resize to actual number of elements returned.
+    RETURN_NOT_OK(
+        offsets_buffer->Resize((validity_io.values_read + 1) * sizeof(IndexType)));
+    if (validity_buffer != nullptr) {
+      RETURN_NOT_OK(
+          validity_buffer->Resize(BitUtil::BytesForBits(validity_io.values_read)));
+      validity_buffer->ZeroPadding();
+    }
+    ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ArrayData> item_chunk, ChunksToSingle(**out));
+
+    std::vector<std::shared_ptr<Buffer>> buffers{
+        validity_io.null_count > 0 ? validity_buffer : nullptr, offsets_buffer};
+    auto data = std::make_shared<ArrayData>(
+        field_->type(),
+        /*length=*/validity_io.values_read, std::move(buffers),
+        std::vector<std::shared_ptr<ArrayData>>{item_chunk}, validity_io.null_count);
+
+    ARROW_ASSIGN_OR_RAISE(*out, AssembleArray(std::move(data)));
     return Status::OK();
   }
 
   const std::shared_ptr<Field> field() override { return field_; }
 
-  const ColumnDescriptor* descr() const override { return nullptr; }
-
-  ReaderType type() const override { return LIST; }
-
  private:
   std::shared_ptr<ReaderContext> ctx_;
   std::shared_ptr<Field> field_;
-  int16_t max_definition_level_;
-  int16_t max_repetition_level_;
+  ::parquet::internal::LevelInfo level_info_;
   std::unique_ptr<ColumnReaderImpl> item_reader_;
 };
 
+class PARQUET_NO_EXPORT FixedSizeListReader : public ListReader<int32_t> {
+ public:
+  FixedSizeListReader(std::shared_ptr<ReaderContext> ctx, std::shared_ptr<Field> field,
+                      ::parquet::internal::LevelInfo level_info,
+                      std::unique_ptr<ColumnReaderImpl> child_reader)
+      : ListReader(std::move(ctx), std::move(field), level_info,
+                   std::move(child_reader)) {}
+  ::arrow::Result<std::shared_ptr<ChunkedArray>> AssembleArray(
+      std::shared_ptr<ArrayData> data) final {
+    DCHECK_EQ(data->buffers.size(), 2);
+    DCHECK_EQ(field()->type()->id(), ::arrow::Type::FIXED_SIZE_LIST);
+    const auto& type = checked_cast<::arrow::FixedSizeListType&>(*field()->type());
+    const int32_t* offsets = reinterpret_cast<const int32_t*>(data->buffers[1]->data());
+    for (int x = 1; x <= data->length; x++) {
+      int32_t size = offsets[x] - offsets[x - 1];
+      if (size != type.list_size()) {
+        return Status::Invalid("Expected all lists to be of size=", type.list_size(),
+                               " but index ", x, " had size=", size);
+      }
+    }
+    data->buffers.resize(1);
+    std::shared_ptr<Array> result = ::arrow::MakeArray(data);
+    return std::make_shared<ChunkedArray>(result);
+  }
+};
+
 class PARQUET_NO_EXPORT StructReader : public ColumnReaderImpl {
  public:
   explicit StructReader(std::shared_ptr<ReaderContext> ctx,
-                        const SchemaField& schema_field,
                         std::shared_ptr<Field> filtered_field,
-                        std::vector<std::unique_ptr<ColumnReaderImpl>>&& children)
+                        ::parquet::internal::LevelInfo level_info,
+                        std::vector<std::unique_ptr<ColumnReaderImpl>> children)
       : ctx_(std::move(ctx)),
-        schema_field_(schema_field),
         filtered_field_(std::move(filtered_field)),
-        struct_def_level_(schema_field.definition_level),
-        children_(std::move(children)) {}
+        level_info_(level_info),
+        children_(std::move(children)) {
+    // There could be a mix of children some might be repeated some might not be.
+    // If possible use one that isn't since that will be guaranteed to have the least
+    // number of levels to reconstruct a nullable bitmap.
+    auto result = std::find_if(children_.begin(), children_.end(),
+                               [](const std::unique_ptr<ColumnReaderImpl>& child) {
+                                 return !child->IsOrHasRepeatedChild();
+                               });
+    if (result != children_.end()) {
+      def_rep_level_child_ = result->get();
+      has_repeated_child_ = false;
+    } else if (!children_.empty()) {
+      def_rep_level_child_ = children_.front().get();
+      has_repeated_child_ = true;
+    }
+  }
+
+  bool IsOrHasRepeatedChild() const final { return has_repeated_child_; }
 
-  Status NextBatch(int64_t records_to_read, std::shared_ptr<ChunkedArray>* out) override;
+  Status LoadBatch(int64_t records_to_read) override {
+    for (const std::unique_ptr<ColumnReaderImpl>& reader : children_) {
+      RETURN_NOT_OK(reader->LoadBatch(records_to_read));
+    }
+    return Status::OK();
+  }
+  Status BuildArray(int64_t length_upper_bound,
+                    std::shared_ptr<ChunkedArray>* out) override;
   Status GetDefLevels(const int16_t** data, int64_t* length) override;
   Status GetRepLevels(const int16_t** data, int64_t* length) override;
   const std::shared_ptr<Field> field() override { return filtered_field_; }
-  const ColumnDescriptor* descr() const override { return nullptr; }
-  ReaderType type() const override { return STRUCT; }
 
  private:
-  std::shared_ptr<ReaderContext> ctx_;
-  SchemaField schema_field_;
-  std::shared_ptr<Field> filtered_field_;
-  int16_t struct_def_level_;
-  std::vector<std::unique_ptr<ColumnReaderImpl>> children_;
-  std::shared_ptr<ResizableBuffer> def_levels_buffer_;
-  Status DefLevelsToNullArray(std::shared_ptr<Buffer>* null_bitmap, int64_t* null_count);
+  const std::shared_ptr<ReaderContext> ctx_;
+  const std::shared_ptr<Field> filtered_field_;
+  const ::parquet::internal::LevelInfo level_info_;
+  const std::vector<std::unique_ptr<ColumnReaderImpl>> children_;
+  ColumnReaderImpl* def_rep_level_child_ = nullptr;
+  bool has_repeated_child_;
 };
 
-Status StructReader::DefLevelsToNullArray(std::shared_ptr<Buffer>* null_bitmap_out,
-                                          int64_t* null_count_out) {
-  auto null_count = 0;
-  const int16_t* def_levels_data;
-  int64_t def_levels_length;
-  RETURN_NOT_OK(GetDefLevels(&def_levels_data, &def_levels_length));
-  ARROW_ASSIGN_OR_RAISE(auto null_bitmap,
-                        AllocateEmptyBitmap(def_levels_length, ctx_->pool));
-  uint8_t* null_bitmap_ptr = null_bitmap->mutable_data();
-  for (int64_t i = 0; i < def_levels_length; i++) {
-    if (def_levels_data[i] < struct_def_level_) {
-      // Mark null
-      null_count += 1;
-    } else {
-      DCHECK_EQ(def_levels_data[i], struct_def_level_);
-      ::arrow::BitUtil::SetBit(null_bitmap_ptr, i);
-    }
+Status StructReader::GetDefLevels(const int16_t** data, int64_t* length) {
+  *data = nullptr;
+  if (children_.size() == 0) {
+    *length = 0;
+    return Status::Invalid("StructReader had no children");
   }
 
-  *null_count_out = null_count;
-  *null_bitmap_out = (null_count == 0) ? nullptr : null_bitmap;
+  // This method should only be called when this struct or one of its parents
+  // are optional/repeated or it has a repeated child.
+  // Meaning all children must have rep/def levels associated
+  // with them.
+  RETURN_NOT_OK(def_rep_level_child_->GetDefLevels(data, length));
   return Status::OK();
 }
 
-// TODO(itaiin): Consider caching the results of this calculation -
-//   note that this is only used once for each read for now
-Status StructReader::GetDefLevels(const int16_t** data, int64_t* length) {
+Status StructReader::GetRepLevels(const int16_t** data, int64_t* length) {
   *data = nullptr;
   if (children_.size() == 0) {
-    // Empty struct
     *length = 0;
-    return Status::OK();
+    return Status::Invalid("StructReader had no childre");
   }
 
-  // We have at least one child
-  const int16_t* child_def_levels;
-  int64_t child_length = 0;
-  bool found_nullable_child = false;
-  int16_t* result_levels = nullptr;
-
-  int child_index = 0;
-  while (child_index < static_cast<int>(children_.size())) {
-    if (!children_[child_index]->field()->nullable()) {
-      ++child_index;
-      continue;
-    }
-    RETURN_NOT_OK(children_[child_index]->GetDefLevels(&child_def_levels, &child_length));
-    auto size = child_length * sizeof(int16_t);
-    ARROW_ASSIGN_OR_RAISE(def_levels_buffer_, AllocateResizableBuffer(size, ctx_->pool));
-    // Initialize with the minimal def level
-    std::memset(def_levels_buffer_->mutable_data(), -1, size);
-    result_levels = reinterpret_cast<int16_t*>(def_levels_buffer_->mutable_data());
-    found_nullable_child = true;
-    break;
-  }
-
-  if (!found_nullable_child) {
-    *data = nullptr;
-    *length = 0;
-    return Status::OK();
-  }
+  // This method should only be called when this struct or one of its parents
+  // are optional/repeated or it has repeated child.
+  // Meaning all children must have rep/def levels associated
+  // with them.
+  RETURN_NOT_OK(def_rep_level_child_->GetRepLevels(data, length));
+  return Status::OK();
+}
 
-  // Look at the rest of the children
+Status StructReader::BuildArray(int64_t length_upper_bound,
+                                std::shared_ptr<ChunkedArray>* out) {
+  std::vector<std::shared_ptr<ArrayData>> children_array_data;
+  std::shared_ptr<ResizableBuffer> null_bitmap;
 
-  // When a struct is defined, all of its children def levels are at least at
-  // nesting level, and def level equals nesting level.
-  // When a struct is not defined, all of its children def levels are less than
-  // the nesting level, and the def level equals max(children def levels)
-  // All other possibilities are malformed definition data.
-  for (; child_index < static_cast<int>(children_.size()); ++child_index) {
-    // Child is non-nullable, and therefore has no definition levels
-    if (!children_[child_index]->field()->nullable()) {
-      continue;
-    }
+  ::parquet::internal::ValidityBitmapInputOutput validity_io;
+  validity_io.values_read_upper_bound = length_upper_bound;
+  // This simplifies accounting below.
+  validity_io.values_read = length_upper_bound;
 
-    auto& child = children_[child_index];
-    int64_t current_child_length;
-    RETURN_NOT_OK(child->GetDefLevels(&child_def_levels, &current_child_length));
-
-    if (child_length != current_child_length) {
-      std::stringstream ss;
-      ss << "Parquet struct decoding error. Expected to decode " << child_length
-         << " definition levels"
-         << " from child field \"" << child->field()->ToString() << "\" in parent \""
-         << this->field()->ToString() << "\" but was only able to decode "
-         << current_child_length;
-      return Status::IOError(ss.str());
-    }
-
-    DCHECK_EQ(child_length, current_child_length);
-    for (int64_t i = 0; i < child_length; i++) {
-      // Check that value is either uninitialized, or current
-      // and previous children def levels agree on the struct level
-      DCHECK((result_levels[i] == -1) || ((result_levels[i] >= struct_def_level_) ==
-                                          (child_def_levels[i] >= struct_def_level_)));
-      result_levels[i] =
-          std::max(result_levels[i], std::min(child_def_levels[i], struct_def_level_));
-    }
+  BEGIN_PARQUET_CATCH_EXCEPTIONS
+  const int16_t* def_levels;
+  const int16_t* rep_levels;
+  int64_t num_levels;
+
+  if (has_repeated_child_) {
+    ARROW_ASSIGN_OR_RAISE(
+        null_bitmap,
+        AllocateResizableBuffer(BitUtil::BytesForBits(length_upper_bound), ctx_->pool));
+    validity_io.valid_bits = null_bitmap->mutable_data();
+    RETURN_NOT_OK(GetDefLevels(&def_levels, &num_levels));
+    RETURN_NOT_OK(GetRepLevels(&rep_levels, &num_levels));
+    DefRepLevelsToBitmap(def_levels, rep_levels, num_levels, level_info_, &validity_io);
+  } else if (filtered_field_->nullable()) {
+    ARROW_ASSIGN_OR_RAISE(
+        null_bitmap,
+        AllocateResizableBuffer(BitUtil::BytesForBits(length_upper_bound), ctx_->pool));
+    validity_io.valid_bits = null_bitmap->mutable_data();
+    RETURN_NOT_OK(GetDefLevels(&def_levels, &num_levels));
+    DefLevelsToBitmap(def_levels, num_levels, level_info_, &validity_io);
   }
-  *data = reinterpret_cast<const int16_t*>(def_levels_buffer_->data());
-  *length = static_cast<int64_t>(child_length);
-  return Status::OK();
-}
-
-Status StructReader::GetRepLevels(const int16_t** data, int64_t* length) {
-  return Status::NotImplemented("GetRepLevels is not implemented for struct");
-}
 
-Status StructReader::NextBatch(int64_t records_to_read,
-                               std::shared_ptr<ChunkedArray>* out) {
-  std::vector<std::shared_ptr<Array>> children_arrays;
-  std::shared_ptr<Buffer> null_bitmap;
-  int64_t null_count;
+  // Ensure all values are initialized.
+  if (null_bitmap) {
+    RETURN_NOT_OK(null_bitmap->Resize(BitUtil::BytesForBits(validity_io.values_read)));
+    null_bitmap->ZeroPadding();
+  }
 
+  END_PARQUET_CATCH_EXCEPTIONS
   // Gather children arrays and def levels
   for (auto& child : children_) {
-    if (child->type() == ColumnReaderImpl::LIST) {
-      return Status::Invalid("Mix of struct and list types not yet supported");
-    }
-
     std::shared_ptr<ChunkedArray> field;
-    RETURN_NOT_OK(child->NextBatch(records_to_read, &field));
-
-    if (field->num_chunks() > 1) {
-      return Status::Invalid("Chunked field reads not yet supported with StructArray");
-    }
-    children_arrays.push_back(field->chunk(0));
+    RETURN_NOT_OK(child->BuildArray(validity_io.values_read, &field));
+    ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ArrayData> array_data, ChunksToSingle(*field));
+    children_array_data.push_back(std::move(array_data));
   }
 
-  RETURN_NOT_OK(DefLevelsToNullArray(&null_bitmap, &null_count));
-
-  int64_t struct_length = children_arrays[0]->length();
-  for (size_t i = 1; i < children_arrays.size(); ++i) {
-    if (children_arrays[i]->length() != struct_length) {
-      // TODO(wesm): This should really only occur if the Parquet file is
-      // malformed. Should this be a DCHECK?
-      return Status::Invalid("Struct children had different lengths");
-    }
+  if (!filtered_field_->nullable() && !has_repeated_child_) {
+    validity_io.values_read = children_array_data.front()->length;
   }
 
-  auto result = std::make_shared<StructArray>(field()->type(), struct_length,
-                                              children_arrays, null_bitmap, null_count);
+  std::vector<std::shared_ptr<Buffer>> buffers{validity_io.null_count > 0 ? null_bitmap
+                                                                          : nullptr};
+  auto data =
+      std::make_shared<ArrayData>(filtered_field_->type(),
+                                  /*length=*/validity_io.values_read, std::move(buffers),
+                                  std::move(children_array_data));
+  std::shared_ptr<Array> result = ::arrow::MakeArray(data);
+
   *out = std::make_shared<ChunkedArray>(result);
   return Status::OK();
 }
@@ -681,48 +782,66 @@ Status StructReader::NextBatch(int64_t records_to_read,
 // ----------------------------------------------------------------------
 // File reader implementation
 
-Status GetReader(const SchemaField& field, const std::shared_ptr<ReaderContext>& ctx,
+Status GetReader(const SchemaField& field, const std::shared_ptr<Field>& arrow_field,
+                 const std::shared_ptr<ReaderContext>& ctx,
                  std::unique_ptr<ColumnReaderImpl>* out) {
   BEGIN_PARQUET_CATCH_EXCEPTIONS
 
-  auto type_id = field.field->type()->id();
+  auto type_id = arrow_field->type()->id();
+
+  if (type_id == ::arrow::Type::EXTENSION) {
+    auto storage_field = arrow_field->WithType(
+        checked_cast<const ExtensionType&>(*arrow_field->type()).storage_type());
+    RETURN_NOT_OK(GetReader(field, storage_field, ctx, out));
+    out->reset(new ExtensionReader(arrow_field, std::move(*out)));
+    return Status::OK();
+  }
+
   if (field.children.size() == 0) {
     if (!field.is_leaf()) {
       return Status::Invalid("Parquet non-leaf node has no children");
     }
+    if (!ctx->IncludesLeaf(field.column_index)) {
+      *out = nullptr;
+      return Status::OK();
+    }
     std::unique_ptr<FileColumnIterator> input(
         ctx->iterator_factory(field.column_index, ctx->reader));
-    out->reset(new LeafReader(ctx, field.field, std::move(input)));
-  } else if (type_id == ::arrow::Type::LIST) {
-    // We can only read lists-of-lists or structs at the moment
-    auto list_field = field.field;
+    out->reset(new LeafReader(ctx, arrow_field, std::move(input), field.level_info));
+  } else if (type_id == ::arrow::Type::LIST || type_id == ::arrow::Type::MAP ||
+             type_id == ::arrow::Type::FIXED_SIZE_LIST ||
+             type_id == ::arrow::Type::LARGE_LIST) {
+    auto list_field = arrow_field;
     auto child = &field.children[0];
-    while (child->field->type()->id() == ::arrow::Type::LIST) {
-      child = &child->children[0];
-    }
-    if (child->field->type()->id() == ::arrow::Type::STRUCT) {
-      return Status::NotImplemented(
-          "Reading lists of structs from Parquet files "
-          "not yet supported: ",
-          field.field->ToString());
-    }
-    if (!ctx->IncludesLeaf(child->column_index)) {
+    std::unique_ptr<ColumnReaderImpl> child_reader;
+    RETURN_NOT_OK(GetReader(*child, ctx, &child_reader));
+    if (child_reader == nullptr) {
       *out = nullptr;
       return Status::OK();
     }
-    std::unique_ptr<ColumnReaderImpl> child_reader;
-    RETURN_NOT_OK(GetReader(*child, ctx, &child_reader));
-    // Use the max definition/repetition level of the leaf here
-    out->reset(new NestedListReader(ctx, list_field, child->definition_level,
-                                    child->repetition_level, std::move(child_reader)));
+    if (type_id == ::arrow::Type::LIST ||
+        type_id == ::arrow::Type::MAP) {  // Map can be reconstructed as list of structs.
+      if (type_id == ::arrow::Type::MAP &&
+          child_reader->field()->type()->num_fields() != 2) {
+        // This case applies if either key or value is filtered.
+        list_field = list_field->WithType(::arrow::list(child_reader->field()));
+      }
+      out->reset(new ListReader<int32_t>(ctx, list_field, field.level_info,
+                                         std::move(child_reader)));
+    } else if (type_id == ::arrow::Type::LARGE_LIST) {
+      out->reset(new ListReader<int64_t>(ctx, list_field, field.level_info,
+                                         std::move(child_reader)));
+
+    } else if (type_id == ::arrow::Type::FIXED_SIZE_LIST) {
+      out->reset(new FixedSizeListReader(ctx, list_field, field.level_info,
+                                         std::move(child_reader)));
+    } else {
+      return Status::UnknownError("Unknown list type: ", field.field->ToString());
+    }
   } else if (type_id == ::arrow::Type::STRUCT) {
     std::vector<std::shared_ptr<Field>> child_fields;
     std::vector<std::unique_ptr<ColumnReaderImpl>> child_readers;
     for (const auto& child : field.children) {
-      if (child.is_leaf() && !ctx->IncludesLeaf(child.column_index)) {
-        // Excluded leaf
-        continue;
-      }
       std::unique_ptr<ColumnReaderImpl> child_reader;
       RETURN_NOT_OK(GetReader(child, ctx, &child_reader));
       if (!child_reader) {
@@ -737,17 +856,25 @@ Status GetReader(const SchemaField& field, const std::shared_ptr<ReaderContext>&
       return Status::OK();
     }
     auto filtered_field =
-        ::arrow::field(field.field->name(), ::arrow::struct_(child_fields),
-                       field.field->nullable(), field.field->metadata());
-    out->reset(new StructReader(ctx, field, filtered_field, std::move(child_readers)));
+        ::arrow::field(arrow_field->name(), ::arrow::struct_(child_fields),
+                       arrow_field->nullable(), arrow_field->metadata());
+    out->reset(new StructReader(ctx, filtered_field, field.level_info,
+                                std::move(child_readers)));
   } else {
-    return Status::Invalid("Unsupported nested type: ", field.field->ToString());
+    return Status::Invalid("Unsupported nested type: ", arrow_field->ToString());
   }
   return Status::OK();
 
   END_PARQUET_CATCH_EXCEPTIONS
 }
 
+Status GetReader(const SchemaField& field, const std::shared_ptr<ReaderContext>& ctx,
+                 std::unique_ptr<ColumnReaderImpl>* out) {
+  return GetReader(field, field.field, ctx, out);
+}
+
+}  // namespace
+
 Status FileReaderImpl::GetRecordBatchReader(const std::vector<int>& row_groups,
                                             const std::vector<int>& column_indices,
                                             std::unique_ptr<RecordBatchReader>* out) {
@@ -789,6 +916,11 @@ Status FileReaderImpl::GetRecordBatchReader(const std::vector<int>& row_groups,
     return Status::OK();
   }
 
+  int64_t num_rows = 0;
+  for (int row_group : row_groups) {
+    num_rows += parquet_reader()->metadata()->RowGroup(row_group)->num_rows();
+  }
+
   using ::arrow::RecordBatchIterator;
 
   // NB: This lambda will be invoked outside the scope of this call to
@@ -796,11 +928,20 @@ Status FileReaderImpl::GetRecordBatchReader(const std::vector<int>& row_groups,
   // `this` is a non-owning pointer so we are relying on the parent FileReader outliving
   // this RecordBatchReader.
   ::arrow::Iterator<RecordBatchIterator> batches = ::arrow::MakeFunctionIterator(
-      [readers, batch_schema, this]() -> ::arrow::Result<RecordBatchIterator> {
+      [readers, batch_schema, num_rows,
+       this]() mutable -> ::arrow::Result<RecordBatchIterator> {
         ::arrow::ChunkedArrayVector columns(readers.size());
-        for (size_t i = 0; i < columns.size(); ++i) {
-          RETURN_NOT_OK(readers[i]->NextBatch(properties().batch_size(), &columns[i]));
-          if (columns[i] == nullptr || columns[i]->length() == 0) {
+
+        // don't reserve more rows than necessary
+        int64_t batch_size = std::min(properties().batch_size(), num_rows);
+        num_rows -= batch_size;
+
+        RETURN_NOT_OK(::arrow::internal::OptionalParallelFor(
+            reader_properties_.use_threads(), static_cast<int>(readers.size()),
+            [&](int i) { return readers[i]->NextBatch(batch_size, &columns[i]); }));
+
+        for (const auto& column : columns) {
+          if (column == nullptr || column->length() == 0) {
             return ::arrow::IterationTraits<RecordBatchIterator>::End();
           }
         }
@@ -852,27 +993,10 @@ Status FileReaderImpl::ReadRowGroups(const std::vector<int>& row_groups,
   RETURN_NOT_OK(GetFieldReaders(column_indices, row_groups, &readers, &result_schema));
 
   ::arrow::ChunkedArrayVector columns(readers.size());
-  auto ReadColumnFunc = [&](size_t i) {
-    return ReadColumn(static_cast<int>(i), row_groups, readers[i].get(), &columns[i]);
-  };
-
-  if (reader_properties_.use_threads()) {
-    std::vector<Future<Status>> futures(readers.size());
-    auto pool = ::arrow::internal::GetCpuThreadPool();
-    for (size_t i = 0; i < readers.size(); ++i) {
-      ARROW_ASSIGN_OR_RAISE(futures[i], pool->Submit(ReadColumnFunc, i));
-    }
-
-    Status final_status;
-    for (auto& fut : futures) {
-      final_status &= fut.status();
-    }
-    RETURN_NOT_OK(final_status);
-  } else {
-    for (size_t i = 0; i < readers.size(); ++i) {
-      RETURN_NOT_OK(ReadColumnFunc(i));
-    }
-  }
+  RETURN_NOT_OK(::arrow::internal::OptionalParallelFor(
+      reader_properties_.use_threads(), static_cast<int>(readers.size()), [&](int i) {
+        return ReadColumn(static_cast<int>(i), row_groups, readers[i].get(), &columns[i]);
+      }));
 
   int64_t num_rows = 0;
   if (!columns.empty()) {
diff --git a/cpp/src/parquet/arrow/reader_internal.cc b/cpp/src/parquet/arrow/reader_internal.cc
index 903cbabaae2..9dd3ac4f0ac 100644
--- a/cpp/src/parquet/arrow/reader_internal.cc
+++ b/cpp/src/parquet/arrow/reader_internal.cc
@@ -24,13 +24,10 @@
 #include <memory>
 #include <string>
 #include <type_traits>
-#include <unordered_map>
 #include <vector>
 
 #include "arrow/array.h"
-#include "arrow/builder.h"
 #include "arrow/datum.h"
-#include "arrow/extension_type.h"
 #include "arrow/io/memory.h"
 #include "arrow/ipc/reader.h"
 #include "arrow/ipc/writer.h"
@@ -646,7 +643,9 @@ static Status DecimalIntegerTransfer(RecordReader* reader, MemoryPool* pool,
 template <typename ParquetType>
 Status TransferDecimal(RecordReader* reader, MemoryPool* pool,
                        const std::shared_ptr<DataType>& type, Datum* out) {
-  DCHECK_EQ(type->id(), ::arrow::Type::DECIMAL);
+  if (type->id() != ::arrow::Type::DECIMAL128) {
+    return Status::NotImplemented("Only reading decimal128 types is currently supported");
+  }
 
   auto binary_reader = dynamic_cast<BinaryRecordReader*>(reader);
   DCHECK(binary_reader);
@@ -662,25 +661,6 @@ Status TransferDecimal(RecordReader* reader, MemoryPool* pool,
   return Status::OK();
 }
 
-Status TransferExtension(RecordReader* reader, std::shared_ptr<DataType> value_type,
-                         const ColumnDescriptor* descr, MemoryPool* pool, Datum* out) {
-  std::shared_ptr<ChunkedArray> result;
-  auto ext_type = std::static_pointer_cast<::arrow::ExtensionType>(value_type);
-  auto storage_type = ext_type->storage_type();
-  RETURN_NOT_OK(TransferColumnData(reader, storage_type, descr, pool, &result));
-
-  ::arrow::ArrayVector out_chunks(result->num_chunks());
-  for (int i = 0; i < result->num_chunks(); i++) {
-    auto chunk = result->chunk(i);
-    auto ext_data = chunk->data()->Copy();
-    ext_data->type = ext_type;
-    auto ext_result = ext_type->MakeArray(ext_data);
-    out_chunks[i] = ext_result;
-  }
-  *out = std::make_shared<ChunkedArray>(out_chunks);
-  return Status::OK();
-}
-
 #define TRANSFER_INT32(ENUM, ArrowType)                                              \
   case ::arrow::Type::ENUM: {                                                        \
     Status s = TransferInt<ArrowType, Int32Type>(reader, pool, value_type, &result); \
@@ -776,9 +756,6 @@ Status TransferColumnData(RecordReader* reader, std::shared_ptr<DataType> value_
           return Status::NotImplemented("TimeUnit not supported");
       }
     } break;
-    case ::arrow::Type::EXTENSION: {
-      RETURN_NOT_OK(TransferExtension(reader, value_type, descr, pool, &result));
-    } break;
     default:
       return Status::NotImplemented("No support for reading columns of type ",
                                     value_type->ToString());
@@ -795,118 +772,5 @@ Status TransferColumnData(RecordReader* reader, std::shared_ptr<DataType> value_
   return Status::OK();
 }
 
-Status ReconstructNestedList(const std::shared_ptr<Array>& arr,
-                             std::shared_ptr<Field> field, int16_t max_def_level,
-                             int16_t max_rep_level, const int16_t* def_levels,
-                             const int16_t* rep_levels, int64_t total_levels,
-                             ::arrow::MemoryPool* pool, std::shared_ptr<Array>* out) {
-  // Walk downwards to extract nullability
-  std::vector<std::string> item_names;
-  std::vector<bool> nullable;
-  std::vector<std::shared_ptr<const ::arrow::KeyValueMetadata>> field_metadata;
-  std::vector<std::shared_ptr<::arrow::Int32Builder>> offset_builders;
-  std::vector<std::shared_ptr<::arrow::BooleanBuilder>> valid_bits_builders;
-  nullable.push_back(field->nullable());
-  while (field->type()->num_fields() > 0) {
-    if (field->type()->num_fields() > 1) {
-      return Status::NotImplemented("Fields with more than one child are not supported.");
-    } else {
-      if (field->type()->id() != ::arrow::Type::LIST) {
-        return Status::NotImplemented("Currently only nesting with Lists is supported.");
-      }
-      field = field->type()->field(0);
-    }
-    item_names.push_back(field->name());
-    offset_builders.emplace_back(
-        std::make_shared<::arrow::Int32Builder>(::arrow::int32(), pool));
-    valid_bits_builders.emplace_back(
-        std::make_shared<::arrow::BooleanBuilder>(::arrow::boolean(), pool));
-    nullable.push_back(field->nullable());
-    field_metadata.push_back(field->metadata());
-  }
-
-  int64_t list_depth = offset_builders.size();
-  // This describes the minimal definition that describes a level that
-  // reflects a value in the primitive values array.
-  int16_t values_def_level = max_def_level;
-  if (nullable[nullable.size() - 1]) {
-    values_def_level--;
-  }
-
-  // The definition levels that are needed so that a list is declared
-  // as empty and not null.
-  std::vector<int16_t> empty_def_level(list_depth);
-  int def_level = 0;
-  for (int i = 0; i < list_depth; i++) {
-    if (nullable[i]) {
-      def_level++;
-    }
-    empty_def_level[i] = static_cast<int16_t>(def_level);
-    def_level++;
-  }
-
-  int32_t values_offset = 0;
-  std::vector<int64_t> null_counts(list_depth, 0);
-  for (int64_t i = 0; i < total_levels; i++) {
-    int16_t rep_level = rep_levels[i];
-    if (rep_level < max_rep_level) {
-      for (int64_t j = rep_level; j < list_depth; j++) {
-        if (j == (list_depth - 1)) {
-          RETURN_NOT_OK(offset_builders[j]->Append(values_offset));
-        } else {
-          RETURN_NOT_OK(offset_builders[j]->Append(
-              static_cast<int32_t>(offset_builders[j + 1]->length())));
-        }
-
-        if (((empty_def_level[j] - 1) == def_levels[i]) && (nullable[j])) {
-          RETURN_NOT_OK(valid_bits_builders[j]->Append(false));
-          null_counts[j]++;
-          break;
-        } else {
-          RETURN_NOT_OK(valid_bits_builders[j]->Append(true));
-          if (empty_def_level[j] == def_levels[i]) {
-            break;
-          }
-        }
-      }
-    }
-    if (def_levels[i] >= values_def_level) {
-      values_offset++;
-    }
-  }
-  // Add the final offset to all lists
-  for (int64_t j = 0; j < list_depth; j++) {
-    if (j == (list_depth - 1)) {
-      RETURN_NOT_OK(offset_builders[j]->Append(values_offset));
-    } else {
-      RETURN_NOT_OK(offset_builders[j]->Append(
-          static_cast<int32_t>(offset_builders[j + 1]->length())));
-    }
-  }
-
-  std::vector<std::shared_ptr<Buffer>> offsets;
-  std::vector<std::shared_ptr<Buffer>> valid_bits;
-  std::vector<int64_t> list_lengths;
-  for (int64_t j = 0; j < list_depth; j++) {
-    list_lengths.push_back(offset_builders[j]->length() - 1);
-    std::shared_ptr<Array> array;
-    RETURN_NOT_OK(offset_builders[j]->Finish(&array));
-    offsets.emplace_back(std::static_pointer_cast<Int32Array>(array)->values());
-    RETURN_NOT_OK(valid_bits_builders[j]->Finish(&array));
-    valid_bits.emplace_back(std::static_pointer_cast<BooleanArray>(array)->values());
-  }
-
-  *out = arr;
-
-  // TODO(wesm): Use passed-in field
-  for (int64_t j = list_depth - 1; j >= 0; j--) {
-    auto list_type = ::arrow::list(::arrow::field(item_names[j], (*out)->type(),
-                                                  nullable[j + 1], field_metadata[j]));
-    *out = std::make_shared<::arrow::ListArray>(list_type, list_lengths[j], offsets[j],
-                                                *out, valid_bits[j], null_counts[j]);
-  }
-  return Status::OK();
-}
-
 }  // namespace arrow
 }  // namespace parquet
diff --git a/cpp/src/parquet/arrow/reader_internal.h b/cpp/src/parquet/arrow/reader_internal.h
index 62eb166da01..ad0b781576f 100644
--- a/cpp/src/parquet/arrow/reader_internal.h
+++ b/cpp/src/parquet/arrow/reader_internal.h
@@ -22,7 +22,6 @@
 #include <deque>
 #include <functional>
 #include <memory>
-#include <unordered_map>
 #include <unordered_set>
 #include <utility>
 #include <vector>
@@ -104,13 +103,6 @@ Status TransferColumnData(::parquet::internal::RecordReader* reader,
                           const ColumnDescriptor* descr, ::arrow::MemoryPool* pool,
                           std::shared_ptr<::arrow::ChunkedArray>* out);
 
-Status ReconstructNestedList(const std::shared_ptr<::arrow::Array>& arr,
-                             std::shared_ptr<::arrow::Field> field, int16_t max_def_level,
-                             int16_t max_rep_level, const int16_t* def_levels,
-                             const int16_t* rep_levels, int64_t total_levels,
-                             ::arrow::MemoryPool* pool,
-                             std::shared_ptr<::arrow::Array>* out);
-
 struct ReaderContext {
   ParquetFileReader* reader;
   ::arrow::MemoryPool* pool;
@@ -126,8 +118,5 @@ struct ReaderContext {
   }
 };
 
-Status GetReader(const SchemaField& field, const std::shared_ptr<ReaderContext>& context,
-                 std::unique_ptr<ColumnReaderImpl>* out);
-
 }  // namespace arrow
 }  // namespace parquet
diff --git a/cpp/src/parquet/arrow/reader_writer_benchmark.cc b/cpp/src/parquet/arrow/reader_writer_benchmark.cc
index e499eed0b6c..dabe1dc3d1a 100644
--- a/cpp/src/parquet/arrow/reader_writer_benchmark.cc
+++ b/cpp/src/parquet/arrow/reader_writer_benchmark.cc
@@ -17,6 +17,7 @@
 
 #include "benchmark/benchmark.h"
 
+#include <array>
 #include <iostream>
 #include <random>
 
@@ -28,10 +29,17 @@
 #include "parquet/file_writer.h"
 #include "parquet/platform.h"
 
-#include "arrow/api.h"
+#include "arrow/array.h"
+#include "arrow/array/builder_primitive.h"
+#include "arrow/table.h"
+#include "arrow/testing/random.h"
+#include "arrow/util/bitmap_ops.h"
 #include "arrow/util/logging.h"
 
+using arrow::Array;
+using arrow::ArrayVector;
 using arrow::BooleanBuilder;
+using arrow::FieldVector;
 using arrow::NumericBuilder;
 
 #define EXIT_NOT_OK(s)                                        \
@@ -88,12 +96,11 @@ std::shared_ptr<ColumnDescriptor> MakeSchema(Repetition::type repetition) {
 }
 
 template <bool nullable, typename ParquetType>
-void SetBytesProcessed(::benchmark::State& state) {
-  int64_t bytes_processed =
-      state.iterations() * BENCHMARK_SIZE * sizeof(typename ParquetType::c_type);
-  if (nullable) {
-    bytes_processed += state.iterations() * BENCHMARK_SIZE * sizeof(int16_t);
-  }
+void SetBytesProcessed(::benchmark::State& state, int64_t num_values = BENCHMARK_SIZE) {
+  const int64_t items_processed = state.iterations() * num_values;
+  const int64_t bytes_processed = items_processed * sizeof(typename ParquetType::c_type);
+
+  state.SetItemsProcessed(bytes_processed);
   state.SetBytesProcessed(bytes_processed);
 }
 
@@ -101,13 +108,13 @@ constexpr int64_t kAlternatingOrNa = -1;
 
 template <typename T>
 std::vector<T> RandomVector(int64_t true_percentage, int64_t vector_size,
-                            const std::array<T, 2>& sample_values) {
-  std::vector<T> values(BENCHMARK_SIZE, {});
+                            const std::array<T, 2>& sample_values, int seed = 500) {
+  std::vector<T> values(vector_size, {});
   if (true_percentage == kAlternatingOrNa) {
     int n = {0};
     std::generate(values.begin(), values.end(), [&n] { return n++ % 2; });
   } else {
-    std::default_random_engine rng(500);
+    std::default_random_engine rng(seed);
     double true_probability = static_cast<double>(true_percentage) / 100.0;
     std::bernoulli_distribution dist(true_probability);
     std::generate(values.begin(), values.end(), [&] { return sample_values[dist(rng)]; });
@@ -127,7 +134,7 @@ std::shared_ptr<::arrow::Table> TableFromVector(
   if (nullable) {
     // Note true values select index 1 of sample_values
     auto valid_bytes = RandomVector<uint8_t>(/*true_percentage=*/null_percentage,
-                                             BENCHMARK_SIZE, /*sample_values=*/{1, 0});
+                                             vec.size(), /*sample_values=*/{1, 0});
     EXIT_NOT_OK(builder.AppendValues(vec.data(), vec.size(), valid_bytes.data()));
   } else {
     EXIT_NOT_OK(builder.AppendValues(vec.data(), vec.size(), nullptr));
@@ -146,8 +153,8 @@ std::shared_ptr<::arrow::Table> TableFromVector<BooleanType>(const std::vector<b
                                                              int64_t null_percentage) {
   BooleanBuilder builder;
   if (nullable) {
-    auto valid_bytes = RandomVector<bool>(/*true_percentage=*/null_percentage,
-                                          BENCHMARK_SIZE, {true, false});
+    auto valid_bytes = RandomVector<bool>(/*true_percentage=*/null_percentage, vec.size(),
+                                          {true, false});
     EXIT_NOT_OK(builder.AppendValues(vec, valid_bytes));
   } else {
     EXIT_NOT_OK(builder.AppendValues(vec));
@@ -197,18 +204,11 @@ struct Examples<bool> {
   static constexpr std::array<bool, 2> values() { return {false, true}; }
 };
 
-template <bool nullable, typename ParquetType>
-static void BM_ReadColumn(::benchmark::State& state) {
-  using T = typename ParquetType::c_type;
-
-  auto values = RandomVector<T>(/*percentage=*/state.range(1), BENCHMARK_SIZE,
-                                Examples<T>::values());
-
-  std::shared_ptr<::arrow::Table> table =
-      TableFromVector<ParquetType>(values, nullable, state.range(0));
+static void BenchmarkReadTable(::benchmark::State& state, const ::arrow::Table& table,
+                               int64_t num_values = -1, int64_t bytes_per_value = -1) {
   auto output = CreateOutputStream();
-  EXIT_NOT_OK(WriteTable(*table, ::arrow::default_memory_pool(), output, BENCHMARK_SIZE));
-
+  EXIT_NOT_OK(
+      WriteTable(table, ::arrow::default_memory_pool(), output, table.num_rows()));
   PARQUET_ASSIGN_OR_THROW(auto buffer, output->Finish());
 
   while (state.KeepRunning()) {
@@ -220,7 +220,43 @@ static void BM_ReadColumn(::benchmark::State& state) {
     std::shared_ptr<::arrow::Table> table;
     EXIT_NOT_OK(arrow_reader->ReadTable(&table));
   }
-  SetBytesProcessed<nullable, ParquetType>(state);
+
+  if (num_values == -1) {
+    num_values = table.num_rows();
+  }
+  state.SetItemsProcessed(num_values * state.iterations());
+  if (bytes_per_value != -1) {
+    state.SetBytesProcessed(num_values * state.iterations() * bytes_per_value);
+  }
+}
+
+static void BenchmarkReadArray(::benchmark::State& state,
+                               const std::shared_ptr<Array>& array, bool nullable,
+                               int64_t num_values = -1, int64_t bytes_per_value = -1) {
+  auto schema = ::arrow::schema({field("s", array->type(), nullable)});
+  auto table = ::arrow::Table::Make(schema, {array}, array->length());
+
+  EXIT_NOT_OK(table->Validate());
+
+  BenchmarkReadTable(state, *table, num_values, bytes_per_value);
+}
+
+//
+// Benchmark reading a primitive column
+//
+
+template <bool nullable, typename ParquetType>
+static void BM_ReadColumn(::benchmark::State& state) {
+  using T = typename ParquetType::c_type;
+
+  auto values = RandomVector<T>(/*percentage=*/state.range(1), BENCHMARK_SIZE,
+                                Examples<T>::values());
+
+  std::shared_ptr<::arrow::Table> table =
+      TableFromVector<ParquetType>(values, nullable, state.range(0));
+
+  BenchmarkReadTable(state, *table, table->num_rows(),
+                     sizeof(typename ParquetType::c_type));
 }
 
 // There are two parameters here that cover different data distributions.
@@ -280,6 +316,180 @@ BENCHMARK_TEMPLATE2(BM_ReadColumn, true, BooleanType)
     ->Args({kAlternatingOrNa, 1})
     ->Args({5, 10});
 
+//
+// Benchmark reading a nested column
+//
+
+const std::vector<int64_t> kNestedNullPercents = {0, 1, 50, 99};
+
+// XXX We can use ArgsProduct() starting from Benchmark 1.5.2
+static void NestedReadArguments(::benchmark::internal::Benchmark* b) {
+  for (const auto null_percentage : kNestedNullPercents) {
+    b->Arg(null_percentage);
+  }
+}
+
+static std::shared_ptr<Array> MakeStructArray(::arrow::random::RandomArrayGenerator* rng,
+                                              const ArrayVector& children,
+                                              double null_probability,
+                                              bool propagate_validity = false) {
+  ARROW_CHECK_GT(children.size(), 0);
+  const int64_t length = children[0]->length();
+
+  std::shared_ptr<::arrow::Buffer> null_bitmap;
+  if (null_probability > 0.0) {
+    null_bitmap = rng->NullBitmap(length, null_probability);
+    if (propagate_validity) {
+      // HACK: the Parquet writer currently doesn't allow non-empty list
+      // entries where a parent node is null (for instance, a struct-of-list
+      // where the outer struct is marked null but the inner list value is
+      // non-empty).
+      for (const auto& child : children) {
+        null_bitmap = *::arrow::internal::BitmapOr(
+            ::arrow::default_memory_pool(), null_bitmap->data(), 0,
+            child->null_bitmap_data(), 0, length, 0);
+      }
+    }
+  }
+  FieldVector fields(children.size());
+  char field_name = 'a';
+  for (size_t i = 0; i < children.size(); ++i) {
+    fields[i] = field(std::string{field_name++}, children[i]->type(),
+                      /*nullable=*/null_probability > 0.0);
+  }
+  return *::arrow::StructArray::Make(children, std::move(fields), null_bitmap);
+}
+
+// Make a (int32, int64) struct array
+static std::shared_ptr<Array> MakeStructArray(::arrow::random::RandomArrayGenerator* rng,
+                                              int64_t size, double null_probability) {
+  auto values1 = rng->Int32(size, -5, 5, null_probability);
+  auto values2 = rng->Int64(size, -12345678912345LL, 12345678912345LL, null_probability);
+  return MakeStructArray(rng, {values1, values2}, null_probability);
+}
+
+static void BM_ReadStructColumn(::benchmark::State& state) {
+  constexpr int64_t kNumValues = BENCHMARK_SIZE / 10;
+  const double null_probability = static_cast<double>(state.range(0)) / 100.0;
+  const bool nullable = (null_probability != 0.0);
+
+  ARROW_CHECK_GE(null_probability, 0.0);
+
+  const int64_t kBytesPerValue = sizeof(int32_t) + sizeof(int64_t);
+
+  ::arrow::random::RandomArrayGenerator rng(42);
+  auto array = MakeStructArray(&rng, kNumValues, null_probability);
+
+  BenchmarkReadArray(state, array, nullable, kNumValues, kBytesPerValue);
+}
+
+BENCHMARK(BM_ReadStructColumn)->Apply(NestedReadArguments);
+
+static void BM_ReadStructOfStructColumn(::benchmark::State& state) {
+  constexpr int64_t kNumValues = BENCHMARK_SIZE / 10;
+  const double null_probability = static_cast<double>(state.range(0)) / 100.0;
+  const bool nullable = (null_probability != 0.0);
+
+  ARROW_CHECK_GE(null_probability, 0.0);
+
+  const int64_t kBytesPerValue = 2 * (sizeof(int32_t) + sizeof(int64_t));
+
+  ::arrow::random::RandomArrayGenerator rng(42);
+  auto values1 = MakeStructArray(&rng, kNumValues, null_probability);
+  auto values2 = MakeStructArray(&rng, kNumValues, null_probability);
+  auto array = MakeStructArray(&rng, {values1, values2}, null_probability);
+
+  BenchmarkReadArray(state, array, nullable, kNumValues, kBytesPerValue);
+}
+
+BENCHMARK(BM_ReadStructOfStructColumn)->Apply(NestedReadArguments);
+
+static void BM_ReadStructOfListColumn(::benchmark::State& state) {
+  constexpr int64_t kNumValues = BENCHMARK_SIZE / 10;
+  const double null_probability = static_cast<double>(state.range(0)) / 100.0;
+  const bool nullable = (null_probability != 0.0);
+
+  ARROW_CHECK_GE(null_probability, 0.0);
+
+  ::arrow::random::RandomArrayGenerator rng(42);
+
+  const int64_t kBytesPerValue = sizeof(int32_t) + sizeof(int64_t);
+
+  auto values1 = rng.Int32(kNumValues, -5, 5, null_probability);
+  auto values2 =
+      rng.Int64(kNumValues, -12345678912345LL, 12345678912345LL, null_probability);
+  auto list1 = rng.List(*values1, kNumValues / 10, null_probability);
+  auto list2 = rng.List(*values2, kNumValues / 10, null_probability);
+  auto array = MakeStructArray(&rng, {list1, list2}, null_probability,
+                               /*propagate_validity =*/true);
+
+  BenchmarkReadArray(state, array, nullable, kNumValues, kBytesPerValue);
+}
+
+BENCHMARK(BM_ReadStructOfListColumn)->Apply(NestedReadArguments);
+
+static void BM_ReadListColumn(::benchmark::State& state) {
+  constexpr int64_t kNumValues = BENCHMARK_SIZE / 10;
+  const double null_probability = static_cast<double>(state.range(0)) / 100.0;
+  const bool nullable = (null_probability != 0.0);
+
+  ARROW_CHECK_GE(null_probability, 0.0);
+
+  ::arrow::random::RandomArrayGenerator rng(42);
+
+  auto values = rng.Int64(kNumValues, /*min=*/-5, /*max=*/5, null_probability);
+  const int64_t kBytesPerValue = sizeof(int64_t);
+
+  auto array = rng.List(*values, kNumValues / 10, null_probability);
+
+  BenchmarkReadArray(state, array, nullable, kNumValues, kBytesPerValue);
+}
+
+BENCHMARK(BM_ReadListColumn)->Apply(NestedReadArguments);
+
+static void BM_ReadListOfStructColumn(::benchmark::State& state) {
+  constexpr int64_t kNumValues = BENCHMARK_SIZE / 10;
+  const double null_probability = static_cast<double>(state.range(0)) / 100.0;
+  const bool nullable = (null_probability != 0.0);
+
+  ARROW_CHECK_GE(null_probability, 0.0);
+
+  ::arrow::random::RandomArrayGenerator rng(42);
+
+  auto values = MakeStructArray(&rng, kNumValues, null_probability);
+  const int64_t kBytesPerValue = sizeof(int32_t) + sizeof(int64_t);
+
+  auto array = rng.List(*values, kNumValues / 10, null_probability);
+
+  BenchmarkReadArray(state, array, nullable, kNumValues, kBytesPerValue);
+}
+
+BENCHMARK(BM_ReadListOfStructColumn)->Apply(NestedReadArguments);
+
+static void BM_ReadListOfListColumn(::benchmark::State& state) {
+  constexpr int64_t kNumValues = BENCHMARK_SIZE / 10;
+  const double null_probability = static_cast<double>(state.range(0)) / 100.0;
+  const bool nullable = (null_probability != 0.0);
+
+  ARROW_CHECK_GE(null_probability, 0.0);
+
+  ::arrow::random::RandomArrayGenerator rng(42);
+
+  auto values = rng.Int64(kNumValues, /*min=*/-5, /*max=*/5, null_probability);
+  const int64_t kBytesPerValue = sizeof(int64_t);
+
+  auto inner = rng.List(*values, kNumValues / 10, null_probability);
+  auto array = rng.List(*inner, kNumValues / 100, null_probability);
+
+  BenchmarkReadArray(state, array, nullable, kNumValues, kBytesPerValue);
+}
+
+BENCHMARK(BM_ReadListOfListColumn)->Apply(NestedReadArguments);
+
+//
+// Benchmark different ways of reading select row groups
+//
+
 static void BM_ReadIndividualRowGroups(::benchmark::State& state) {
   std::vector<int64_t> values(BENCHMARK_SIZE, 128);
   std::shared_ptr<::arrow::Table> table = TableFromVector<Int64Type>(values, true);
diff --git a/cpp/src/parquet/arrow/reconstruct_internal_test.cc b/cpp/src/parquet/arrow/reconstruct_internal_test.cc
new file mode 100644
index 00000000000..495b69f9eab
--- /dev/null
+++ b/cpp/src/parquet/arrow/reconstruct_internal_test.cc
@@ -0,0 +1,1639 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "parquet/arrow/path_internal.h"
+
+#include <algorithm>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "arrow/array/concatenate.h"
+#include "arrow/chunked_array.h"
+#include "arrow/io/memory.h"
+#include "arrow/result.h"
+#include "arrow/testing/gtest_util.h"
+#include "arrow/type_fwd.h"
+#include "arrow/util/checked_cast.h"
+#include "arrow/util/logging.h"
+
+#include "parquet/arrow/reader.h"
+#include "parquet/arrow/schema.h"
+#include "parquet/column_writer.h"
+#include "parquet/file_writer.h"
+#include "parquet/properties.h"
+
+using arrow::Array;
+using arrow::ArrayFromJSON;
+using arrow::AssertArraysEqual;
+using arrow::ChunkedArray;
+using arrow::DataType;
+using arrow::field;
+using arrow::int32;
+using arrow::int64;
+using arrow::list;
+using arrow::MemoryPool;
+using arrow::Result;
+using arrow::Status;
+using arrow::struct_;
+using arrow::internal::checked_cast;
+using arrow::internal::checked_pointer_cast;
+using arrow::io::BufferOutputStream;
+using arrow::io::BufferReader;
+
+using testing::ElementsAre;
+using testing::ElementsAreArray;
+using testing::Eq;
+using testing::NotNull;
+using testing::SizeIs;
+
+namespace parquet {
+namespace arrow {
+
+using parquet::schema::GroupNode;
+using parquet::schema::NodePtr;
+using parquet::schema::PrimitiveNode;
+
+using ParquetType = parquet::Type::type;
+template <ParquetType T>
+using ParquetTraits = parquet::type_traits<T>;
+
+using LevelVector = std::vector<int16_t>;
+// For readability
+using DefLevels = LevelVector;
+using RepLevels = LevelVector;
+using Int32Vector = std::vector<int32_t>;
+using Int64Vector = std::vector<int64_t>;
+
+// A Parquet file builder that allows writing values one leaf column at a time
+class FileBuilder {
+ public:
+  static Result<std::shared_ptr<FileBuilder>> Make(const NodePtr& group_node,
+                                                   int num_columns) {
+    auto self = std::make_shared<FileBuilder>();
+    RETURN_NOT_OK(self->Open(group_node, num_columns));
+    return self;
+  }
+
+  Result<std::shared_ptr<Buffer>> Finish() {
+    DCHECK_EQ(column_index_, num_columns_);
+    row_group_writer_->Close();
+    file_writer_->Close();
+    return stream_->Finish();
+  }
+
+  // Write a leaf (primitive) column
+  template <ParquetType TYPE, typename C_TYPE = typename ParquetTraits<TYPE>::value_type>
+  Status WriteColumn(const LevelVector& def_levels, const LevelVector& rep_levels,
+                     const std::vector<C_TYPE>& values) {
+    auto column_writer = row_group_writer_->NextColumn();
+    auto column_descr = column_writer->descr();
+    const int16_t max_def_level = column_descr->max_definition_level();
+    const int16_t max_rep_level = column_descr->max_repetition_level();
+    CheckTestedLevels(def_levels, max_def_level);
+    CheckTestedLevels(rep_levels, max_rep_level);
+
+    auto typed_writer =
+        checked_cast<TypedColumnWriter<PhysicalType<TYPE>>*>(column_writer);
+
+    const int64_t num_values = static_cast<int64_t>(
+        (max_def_level > 0) ? def_levels.size()
+                            : (max_rep_level > 0) ? rep_levels.size() : values.size());
+    const int64_t values_written = typed_writer->WriteBatch(
+        num_values, LevelPointerOrNull(def_levels, max_def_level),
+        LevelPointerOrNull(rep_levels, max_rep_level), values.data());
+    DCHECK_EQ(values_written, static_cast<int64_t>(values.size()));  // Sanity check
+
+    column_writer->Close();
+    ++column_index_;
+    return Status::OK();
+  }
+
+ protected:
+  Status Open(const NodePtr& group_node, int num_columns) {
+    ARROW_ASSIGN_OR_RAISE(stream_, BufferOutputStream::Create());
+    file_writer_ =
+        ParquetFileWriter::Open(stream_, checked_pointer_cast<GroupNode>(group_node));
+    row_group_writer_ = file_writer_->AppendRowGroup();
+    num_columns_ = num_columns;
+    column_index_ = 0;
+    return Status::OK();
+  }
+
+  void CheckTestedLevels(const LevelVector& levels, int16_t max_level) {
+    // Tests are expected to exercise all possible levels in [0, max_level]
+    if (!levels.empty()) {
+      const int16_t max_seen_level = *std::max_element(levels.begin(), levels.end());
+      DCHECK_EQ(max_seen_level, max_level);
+    }
+  }
+
+  const int16_t* LevelPointerOrNull(const LevelVector& levels, int16_t max_level) {
+    if (max_level > 0) {
+      DCHECK_GT(levels.size(), 0);
+      return levels.data();
+    } else {
+      DCHECK_EQ(levels.size(), 0);
+      return nullptr;
+    }
+  }
+
+  std::shared_ptr<BufferOutputStream> stream_;
+  std::unique_ptr<ParquetFileWriter> file_writer_;
+  RowGroupWriter* row_group_writer_;
+  int num_columns_;
+  int column_index_;
+};
+
+// A Parquet file tester that allows reading Arrow columns, corresponding to
+// children of the top-level group node.
+class FileTester {
+ public:
+  static Result<std::shared_ptr<FileTester>> Make(std::shared_ptr<Buffer> buffer,
+                                                  MemoryPool* pool) {
+    auto self = std::make_shared<FileTester>();
+    RETURN_NOT_OK(self->Open(buffer, pool));
+    return self;
+  }
+
+  Result<std::shared_ptr<Array>> ReadColumn(int column_index) {
+    std::shared_ptr<ChunkedArray> column;
+    RETURN_NOT_OK(file_reader_->ReadColumn(column_index, &column));
+    return ::arrow::Concatenate(column->chunks(), pool_);
+  }
+
+  void CheckColumn(int column_index, const Array& expected) {
+    ASSERT_OK_AND_ASSIGN(const auto actual, ReadColumn(column_index));
+    ASSERT_OK(actual->ValidateFull());
+    AssertArraysEqual(expected, *actual, /*verbose=*/true);
+  }
+
+ protected:
+  Status Open(std::shared_ptr<Buffer> buffer, MemoryPool* pool) {
+    pool_ = pool;
+    return OpenFile(std::make_shared<BufferReader>(buffer), pool_, &file_reader_);
+  }
+
+  MemoryPool* pool_;
+  std::unique_ptr<FileReader> file_reader_;
+};
+
+class TestReconstructColumn : public testing::Test {
+ public:
+  void SetUp() override { pool_ = ::arrow::default_memory_pool(); }
+
+  // Write the next leaf (primitive) column
+  template <ParquetType TYPE, typename C_TYPE = typename ParquetTraits<TYPE>::value_type>
+  Status WriteColumn(const LevelVector& def_levels, const LevelVector& rep_levels,
+                     const std::vector<C_TYPE>& values) {
+    if (!builder_) {
+      ARROW_ASSIGN_OR_RAISE(builder_,
+                            FileBuilder::Make(group_node_, descriptor_->num_columns()));
+    }
+    return builder_->WriteColumn<TYPE, C_TYPE>(def_levels, rep_levels, values);
+  }
+
+  template <typename C_TYPE>
+  Status WriteInt32Column(const LevelVector& def_levels, const LevelVector& rep_levels,
+                          const std::vector<C_TYPE>& values) {
+    return WriteColumn<ParquetType::INT32>(def_levels, rep_levels, values);
+  }
+
+  template <typename C_TYPE>
+  Status WriteInt64Column(const LevelVector& def_levels, const LevelVector& rep_levels,
+                          const std::vector<C_TYPE>& values) {
+    return WriteColumn<ParquetType::INT64>(def_levels, rep_levels, values);
+  }
+
+  // Read a Arrow column and check its values
+  void CheckColumn(int column_index, const Array& expected) {
+    if (!tester_) {
+      ASSERT_OK_AND_ASSIGN(auto buffer, builder_->Finish());
+      ASSERT_OK_AND_ASSIGN(tester_, FileTester::Make(buffer, pool_));
+    }
+    tester_->CheckColumn(column_index, expected);
+  }
+
+  void CheckColumn(const Array& expected) { CheckColumn(/*column_index=*/0, expected); }
+
+  // One-column shortcut
+  template <ParquetType TYPE, typename C_TYPE = typename ParquetTraits<TYPE>::value_type>
+  void AssertReconstruct(const Array& expected, const LevelVector& def_levels,
+                         const LevelVector& rep_levels,
+                         const std::vector<C_TYPE>& values) {
+    ASSERT_OK((WriteColumn<TYPE, C_TYPE>(def_levels, rep_levels, values)));
+    CheckColumn(/*column_index=*/0, expected);
+  }
+
+  ::arrow::Status MaybeSetParquetSchema(const NodePtr& column) {
+    descriptor_.reset(new SchemaDescriptor());
+    manifest_.reset(new SchemaManifest());
+    group_node_ = GroupNode::Make("root", Repetition::REQUIRED, {column});
+    descriptor_->Init(group_node_);
+    return SchemaManifest::Make(descriptor_.get(),
+                                std::shared_ptr<const ::arrow::KeyValueMetadata>(),
+                                ArrowReaderProperties(), manifest_.get());
+  }
+
+  void SetParquetSchema(const NodePtr& column) {
+    ASSERT_OK(MaybeSetParquetSchema(column));
+  }
+
+ protected:
+  MemoryPool* pool_;
+  NodePtr group_node_;
+  std::unique_ptr<SchemaDescriptor> descriptor_;
+  std::unique_ptr<SchemaManifest> manifest_;
+
+  std::shared_ptr<FileBuilder> builder_;
+  std::shared_ptr<FileTester> tester_;
+};
+
+static std::shared_ptr<DataType> OneFieldStruct(const std::string& name,
+                                                std::shared_ptr<DataType> type,
+                                                bool nullable = true) {
+  return struct_({field(name, type, nullable)});
+}
+
+static std::shared_ptr<DataType> List(std::shared_ptr<DataType> type,
+                                      bool nullable = true) {
+  // TODO should field name "element" (Parquet convention for List nodes)
+  // be changed to "item" (Arrow convention for List types)?
+  return list(field("element", type, nullable));
+}
+
+//
+// Primitive columns with no intermediate group node
+//
+
+TEST_F(TestReconstructColumn, PrimitiveOptional) {
+  SetParquetSchema(
+      PrimitiveNode::Make("node_name", Repetition::OPTIONAL, ParquetType::INT32));
+
+  LevelVector def_levels = {1, 0, 1, 1};
+  LevelVector rep_levels = {};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto expected = ArrayFromJSON(int32(), "[4, null, 5, 6]");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, PrimitiveRequired) {
+  SetParquetSchema(
+      PrimitiveNode::Make("node_name", Repetition::REQUIRED, ParquetType::INT32));
+
+  LevelVector def_levels = {};
+  LevelVector rep_levels = {};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto expected = ArrayFromJSON(int32(), "[4, 5, 6]");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, PrimitiveRepeated) {
+  // Arrow schema: list(int32 not null) not null
+  this->SetParquetSchema(
+      PrimitiveNode::Make("node_name", Repetition::REPEATED, ParquetType::INT32));
+
+  LevelVector def_levels = {0, 1, 1, 1};
+  LevelVector rep_levels = {0, 0, 1, 0};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto expected = ArrayFromJSON(list(field("node_name", int32(), /*nullable=*/false)),
+                                "[[], [4, 5], [6]]");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+//
+// Struct encodings (one field each)
+//
+
+TEST_F(TestReconstructColumn, NestedRequiredRequired) {
+  // Arrow schema: struct(a: int32 not null) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {PrimitiveNode::Make("a", Repetition::REQUIRED, ParquetType::INT32)}));
+
+  LevelVector def_levels = {};
+  LevelVector rep_levels = {};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto expected = ArrayFromJSON(OneFieldStruct("a", int32(), false),
+                                R"([{"a": 4}, {"a": 5}, {"a": 6}])");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, NestedOptionalRequired) {
+  // Arrow schema: struct(a: int32 not null)
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {PrimitiveNode::Make("a", Repetition::REQUIRED, ParquetType::INT32)}));
+
+  LevelVector def_levels = {0, 1, 1, 1};
+  LevelVector rep_levels = {};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto expected = ArrayFromJSON(OneFieldStruct("a", int32(), false),
+                                R"([null, {"a": 4}, {"a": 5}, {"a": 6}])");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, NestedRequiredOptional) {
+  // Arrow schema: struct(a: int32) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {PrimitiveNode::Make("a", Repetition::OPTIONAL, ParquetType::INT32)}));
+
+  LevelVector def_levels = {0, 1, 1, 1};
+  LevelVector rep_levels = {};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto expected = ArrayFromJSON(OneFieldStruct("a", int32()),
+                                R"([{"a": null}, {"a": 4}, {"a": 5}, {"a": 6}])");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, NestedOptionalOptional) {
+  // Arrow schema: struct(a: int32)
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {PrimitiveNode::Make("a", Repetition::OPTIONAL, ParquetType::INT32)}));
+
+  LevelVector def_levels = {0, 1, 2, 2};
+  LevelVector rep_levels = {};
+  std::vector<int32_t> values = {4, 5};
+
+  auto expected = ArrayFromJSON(OneFieldStruct("a", int32()),
+                                R"([null, {"a": null}, {"a": 4}, {"a": 5}])");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+//
+// Nested struct encodings (one field each)
+//
+
+TEST_F(TestReconstructColumn, NestedRequiredRequiredRequired) {
+  // Arrow schema: struct(a: struct(b: int32 not null) not null) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {GroupNode::Make(
+          "a", Repetition::REQUIRED,
+          {PrimitiveNode::Make("b", Repetition::REQUIRED, ParquetType::INT32)})}));
+
+  LevelVector def_levels = {};
+  LevelVector rep_levels = {};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto expected =
+      ArrayFromJSON(OneFieldStruct("a", OneFieldStruct("b", int32(), false), false),
+                    R"([{"a": {"b": 4}},
+                        {"a": {"b": 5}},
+                        {"a": {"b": 6}}
+                        ])");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, NestedRequiredOptionalRequired) {
+  // Arrow schema: struct(a: struct(b: int32 not null)) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {GroupNode::Make(
+          "a", Repetition::OPTIONAL,
+          {PrimitiveNode::Make("b", Repetition::REQUIRED, ParquetType::INT32)})}));
+
+  LevelVector def_levels = {1, 0, 1, 1};
+  LevelVector rep_levels = {};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto expected = ArrayFromJSON(OneFieldStruct("a", OneFieldStruct("b", int32(), false)),
+                                R"([{"a": {"b": 4}},
+                                    {"a": null},
+                                    {"a": {"b": 5}},
+                                    {"a": {"b": 6}}
+                                    ])");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, NestedOptionalRequiredOptional) {
+  // Arrow schema: struct(a: struct(b: int32) not null)
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {GroupNode::Make(
+          "a", Repetition::REQUIRED,
+          {PrimitiveNode::Make("b", Repetition::OPTIONAL, ParquetType::INT32)})}));
+
+  LevelVector def_levels = {1, 2, 0, 2, 2};
+  LevelVector rep_levels = {};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto expected = ArrayFromJSON(OneFieldStruct("a", OneFieldStruct("b", int32()), false),
+                                R"([{"a": {"b": null}},
+                                    {"a": {"b": 4}},
+                                    null,
+                                    {"a": {"b": 5}},
+                                    {"a": {"b": 6}}
+                                    ])");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, NestedOptionalOptionalOptional) {
+  // Arrow schema: struct(a: struct(b: int32) not null)
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {GroupNode::Make(
+          "a", Repetition::OPTIONAL,
+          {PrimitiveNode::Make("b", Repetition::OPTIONAL, ParquetType::INT32)})}));
+
+  LevelVector def_levels = {1, 2, 0, 3, 3, 3};
+  LevelVector rep_levels = {};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto expected = ArrayFromJSON(OneFieldStruct("a", OneFieldStruct("b", int32())),
+                                R"([{"a": null},
+                                    {"a": {"b": null}},
+                                    null,
+                                    {"a": {"b": 4}},
+                                    {"a": {"b": 5}},
+                                    {"a": {"b": 6}}
+                                    ])");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+//
+// Struct encodings (two fields)
+//
+
+TEST_F(TestReconstructColumn, NestedTwoFields1) {
+  // Arrow schema: struct(a: int32 not null, b: int64 not null) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {PrimitiveNode::Make("a", Repetition::REQUIRED, ParquetType::INT32),
+       PrimitiveNode::Make("b", Repetition::REQUIRED, ParquetType::INT64)}));
+
+  ASSERT_OK(WriteInt32Column(DefLevels{}, RepLevels{}, Int32Vector{4, 5, 6}));
+  ASSERT_OK(WriteInt64Column(DefLevels{}, RepLevels{}, Int64Vector{7, 8, 9}));
+
+  auto type = struct_(
+      {field("a", int32(), /*nullable=*/false), field("b", int64(), /*nullable=*/false)});
+  auto expected = ArrayFromJSON(type, R"([{"a": 4, "b": 7},
+                                          {"a": 5, "b": 8},
+                                          {"a": 6, "b": 9}])");
+
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+TEST_F(TestReconstructColumn, NestedTwoFields2) {
+  // Arrow schema: struct(a: int32 not null, b: int64) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {PrimitiveNode::Make("a", Repetition::REQUIRED, ParquetType::INT32),
+       PrimitiveNode::Make("b", Repetition::OPTIONAL, ParquetType::INT64)}));
+
+  ASSERT_OK(WriteInt32Column(DefLevels{}, RepLevels{}, Int32Vector{4, 5, 6}));
+  ASSERT_OK(WriteInt64Column(DefLevels{0, 1, 1}, RepLevels{}, Int64Vector{7, 8}));
+
+  auto type = struct_({field("a", int32(), /*nullable=*/false), field("b", int64())});
+  auto expected = ArrayFromJSON(type, R"([{"a": 4, "b": null},
+                                          {"a": 5, "b": 7},
+                                          {"a": 6, "b": 8}])");
+
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+TEST_F(TestReconstructColumn, NestedTwoFields3) {
+  // Arrow schema: struct(a: int32 not null, b: int64 not null)
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {PrimitiveNode::Make("a", Repetition::REQUIRED, ParquetType::INT32),
+       PrimitiveNode::Make("b", Repetition::REQUIRED, ParquetType::INT64)}));
+
+  ASSERT_OK(WriteInt32Column(DefLevels{0, 1, 1}, RepLevels{}, Int32Vector{4, 5}));
+  ASSERT_OK(WriteInt64Column(DefLevels{0, 1, 1}, RepLevels{}, Int64Vector{7, 8}));
+
+  auto type = struct_(
+      {field("a", int32(), /*nullable=*/false), field("b", int64(), /*nullable=*/false)});
+  auto expected = ArrayFromJSON(type, R"([null,
+                                         {"a": 4, "b": 7},
+                                         {"a": 5, "b": 8}])");
+
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+TEST_F(TestReconstructColumn, NestedTwoFields4) {
+  // Arrow schema: struct(a: int32, b: int64 not null)
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {PrimitiveNode::Make("a", Repetition::OPTIONAL, ParquetType::INT32),
+       PrimitiveNode::Make("b", Repetition::REQUIRED, ParquetType::INT64)}));
+
+  ASSERT_OK(WriteInt32Column(DefLevels{0, 1, 2}, RepLevels{}, Int32Vector{4}));
+  ASSERT_OK(WriteInt64Column(DefLevels{0, 1, 1}, RepLevels{}, Int64Vector{7, 8}));
+
+  auto type = struct_({field("a", int32()), field("b", int64(), /*nullable=*/false)});
+  auto expected = ArrayFromJSON(type, R"([null,
+                                         {"a": null, "b": 7},
+                                         {"a": 4, "b": 8}])");
+
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+TEST_F(TestReconstructColumn, NestedTwoFields5) {
+  // Arrow schema: struct(a: int32, b: int64)
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {PrimitiveNode::Make("a", Repetition::OPTIONAL, ParquetType::INT32),
+       PrimitiveNode::Make("b", Repetition::OPTIONAL, ParquetType::INT64)}));
+
+  ASSERT_OK(WriteInt32Column(DefLevels{0, 1, 2}, RepLevels{}, Int32Vector{4}));
+  ASSERT_OK(WriteInt64Column(DefLevels{0, 2, 1}, RepLevels{}, Int64Vector{7}));
+
+  auto type = struct_({field("a", int32()), field("b", int64())});
+  auto expected = ArrayFromJSON(type, R"([null,
+                                         {"a": null, "b": 7},
+                                         {"a": 4, "b": null}])");
+
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+//
+// Nested struct encodings (two fields)
+//
+
+TEST_F(TestReconstructColumn, NestedNestedTwoFields1) {
+  // Arrow schema: struct(a: struct(aa: int32 not null,
+  //                                ab: int64 not null) not null,
+  //                      b: int32 not null) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {GroupNode::Make(
+           "a", Repetition::REQUIRED,
+           {PrimitiveNode::Make("aa", Repetition::REQUIRED, ParquetType::INT32),
+            PrimitiveNode::Make("ab", Repetition::REQUIRED, ParquetType::INT64)}),
+       PrimitiveNode::Make("b", Repetition::REQUIRED, ParquetType::INT32)}));
+
+  // aa
+  ASSERT_OK(WriteInt32Column(DefLevels{}, RepLevels{}, Int32Vector{4, 5, 6}));
+  // ab
+  ASSERT_OK(WriteInt64Column(DefLevels{}, RepLevels{}, Int64Vector{7, 8, 9}));
+  // b
+  ASSERT_OK(WriteInt32Column(DefLevels{}, RepLevels{}, Int32Vector{10, 11, 12}));
+
+  auto type = struct_({field("a",
+                             struct_({field("aa", int32(), /*nullable=*/false),
+                                      field("ab", int64(), /*nullable=*/false)}),
+                             /*nullable=*/false),
+                       field("b", int32(), /*nullable=*/false)});
+  auto expected = ArrayFromJSON(type, R"([{"a": {"aa": 4, "ab": 7}, "b": 10},
+                                          {"a": {"aa": 5, "ab": 8}, "b": 11},
+                                          {"a": {"aa": 6, "ab": 9}, "b": 12}])");
+
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+TEST_F(TestReconstructColumn, NestedNestedTwoFields2) {
+  // Arrow schema: struct(a: struct(aa: int32,
+  //                                ab: int64 not null) not null,
+  //                      b: int32 not null) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {GroupNode::Make(
+           "a", Repetition::REQUIRED,
+           {PrimitiveNode::Make("aa", Repetition::OPTIONAL, ParquetType::INT32),
+            PrimitiveNode::Make("ab", Repetition::REQUIRED, ParquetType::INT64)}),
+       PrimitiveNode::Make("b", Repetition::REQUIRED, ParquetType::INT32)}));
+
+  // aa
+  ASSERT_OK(WriteInt32Column(DefLevels{1, 0, 1}, RepLevels{}, Int32Vector{4, 5}));
+  // ab
+  ASSERT_OK(WriteInt64Column(DefLevels{}, RepLevels{}, Int64Vector{7, 8, 9}));
+  // b
+  ASSERT_OK(WriteInt32Column(DefLevels{}, RepLevels{}, Int32Vector{10, 11, 12}));
+
+  auto type = struct_(
+      {field("a",
+             struct_({field("aa", int32()), field("ab", int64(), /*nullable=*/false)}),
+             /*nullable=*/false),
+       field("b", int32(), /*nullable=*/false)});
+  auto expected = ArrayFromJSON(type, R"([{"a": {"aa": 4, "ab": 7}, "b": 10},
+                                          {"a": {"aa": null, "ab": 8}, "b": 11},
+                                          {"a": {"aa": 5, "ab": 9}, "b": 12}])");
+
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+TEST_F(TestReconstructColumn, NestedNestedTwoFields3) {
+  // Arrow schema: struct(a: struct(aa: int32 not null,
+  //                                ab: int64) not null,
+  //                      b: int32) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {GroupNode::Make(
+           "a", Repetition::REQUIRED,
+           {PrimitiveNode::Make("aa", Repetition::REQUIRED, ParquetType::INT32),
+            PrimitiveNode::Make("ab", Repetition::OPTIONAL, ParquetType::INT64)}),
+       PrimitiveNode::Make("b", Repetition::OPTIONAL, ParquetType::INT32)}));
+
+  // aa
+  ASSERT_OK(WriteInt32Column(DefLevels{}, RepLevels{}, Int32Vector{4, 5, 6}));
+  // ab
+  ASSERT_OK(WriteInt64Column(DefLevels{0, 1, 1}, RepLevels{}, Int64Vector{7, 8}));
+  // b
+  ASSERT_OK(WriteInt32Column(DefLevels{1, 0, 1}, RepLevels{}, Int32Vector{10, 11}));
+
+  auto type = struct_(
+      {field("a",
+             struct_({field("aa", int32(), /*nullable=*/false), field("ab", int64())}),
+             /*nullable=*/false),
+       field("b", int32())});
+  auto expected = ArrayFromJSON(type, R"([{"a": {"aa": 4, "ab": null}, "b": 10},
+                                          {"a": {"aa": 5, "ab": 7}, "b": null},
+                                          {"a": {"aa": 6, "ab": 8}, "b": 11}])");
+
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+TEST_F(TestReconstructColumn, NestedNestedTwoFields4) {
+  // Arrow schema: struct(a: struct(aa: int32 not null,
+  //                                ab: int64),
+  //                      b: int32 not null) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {GroupNode::Make(
+           "a", Repetition::OPTIONAL,
+           {PrimitiveNode::Make("aa", Repetition::REQUIRED, ParquetType::INT32),
+            PrimitiveNode::Make("ab", Repetition::OPTIONAL, ParquetType::INT64)}),
+       PrimitiveNode::Make("b", Repetition::REQUIRED, ParquetType::INT32)}));
+
+  // aa
+  ASSERT_OK(WriteInt32Column(DefLevels{0, 1, 1}, RepLevels{}, Int32Vector{4, 5}));
+  // ab
+  ASSERT_OK(WriteInt64Column(DefLevels{0, 1, 2}, RepLevels{}, Int64Vector{7}));
+  // b
+  ASSERT_OK(WriteInt32Column(DefLevels{}, RepLevels{}, Int32Vector{10, 11, 12}));
+
+  auto type = struct_({field("a", struct_({field("aa", int32(), /*nullable=*/false),
+                                           field("ab", int64())})),
+                       field("b", int32(), /*nullable=*/false)});
+  auto expected = ArrayFromJSON(type, R"([{"a": null, "b": 10},
+                                          {"a": {"aa": 4, "ab": null}, "b": 11},
+                                          {"a": {"aa": 5, "ab": 7}, "b": 12}])");
+
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+TEST_F(TestReconstructColumn, NestedNestedTwoFields5) {
+  // Arrow schema: struct(a: struct(aa: int32 not null,
+  //                                ab: int64) not null,
+  //                      b: int32)
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {GroupNode::Make(
+           "a", Repetition::REQUIRED,
+           {PrimitiveNode::Make("aa", Repetition::REQUIRED, ParquetType::INT32),
+            PrimitiveNode::Make("ab", Repetition::OPTIONAL, ParquetType::INT64)}),
+       PrimitiveNode::Make("b", Repetition::OPTIONAL, ParquetType::INT32)}));
+
+  // aa
+  ASSERT_OK(WriteInt32Column(DefLevels{0, 1, 1}, RepLevels{}, Int32Vector{4, 5}));
+  // ab
+  ASSERT_OK(WriteInt64Column(DefLevels{0, 1, 2}, RepLevels{}, Int64Vector{7}));
+  // b
+  ASSERT_OK(WriteInt32Column(DefLevels{0, 2, 1}, RepLevels{}, Int32Vector{10}));
+
+  auto type = struct_(
+      {field("a",
+             struct_({field("aa", int32(), /*nullable=*/false), field("ab", int64())}),
+             /*nullable=*/false),
+       field("b", int32())});
+  auto expected = ArrayFromJSON(type, R"([null,
+                                          {"a": {"aa": 4, "ab": null}, "b": 10},
+                                          {"a": {"aa": 5, "ab": 7}, "b": null}])");
+
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+TEST_F(TestReconstructColumn, NestedNestedTwoFields6) {
+  // Arrow schema: struct(a: struct(aa: int32 not null,
+  //                                ab: int64),
+  //                      b: int32)
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {GroupNode::Make(
+           "a", Repetition::OPTIONAL,
+           {PrimitiveNode::Make("aa", Repetition::REQUIRED, ParquetType::INT32),
+            PrimitiveNode::Make("ab", Repetition::OPTIONAL, ParquetType::INT64)}),
+       PrimitiveNode::Make("b", Repetition::OPTIONAL, ParquetType::INT32)}));
+
+  // aa
+  ASSERT_OK(WriteInt32Column(DefLevels{0, 1, 2, 2}, RepLevels{}, Int32Vector{4, 5}));
+  // ab
+  ASSERT_OK(WriteInt64Column(DefLevels{0, 1, 2, 3}, RepLevels{}, Int64Vector{7}));
+  // b
+  ASSERT_OK(WriteInt32Column(DefLevels{0, 2, 1, 2}, RepLevels{}, Int32Vector{10, 11}));
+
+  auto type = struct_({field("a", struct_({field("aa", int32(), /*nullable=*/false),
+                                           field("ab", int64())})),
+                       field("b", int32())});
+  auto expected = ArrayFromJSON(type, R"([null,
+                                          {"a": null, "b": 10},
+                                          {"a": {"aa": 4, "ab": null}, "b": null},
+                                          {"a": {"aa": 5, "ab": 7}, "b": 11}])");
+
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+//
+// Three-level list encodings
+//
+
+TEST_F(TestReconstructColumn, ThreeLevelListRequiredRequired) {
+  // Arrow schema: list(int32 not null) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {GroupNode::Make(
+          "list", Repetition::REPEATED,
+          {PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::INT32)})},
+      LogicalType::List()));
+
+  LevelVector def_levels = {0, 1, 1, 1};
+  LevelVector rep_levels = {0, 0, 1, 0};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  // TODO should field name "element" (Parquet convention for List nodes)
+  // be changed to "item" (Arrow convention for List types)?
+  auto expected = ArrayFromJSON(List(int32(), /*nullable=*/false), "[[], [4, 5], [6]]");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, ThreeLevelListOptionalRequired) {
+  // Arrow schema: list(int32 not null)
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {GroupNode::Make(
+          "list", Repetition::REPEATED,
+          {PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::INT32)})},
+      LogicalType::List()));
+
+  LevelVector def_levels = {0, 1, 2, 2, 2};
+  LevelVector rep_levels = {0, 0, 0, 1, 0};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto expected =
+      ArrayFromJSON(List(int32(), /*nullable=*/false), "[null, [], [4, 5], [6]]");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, ThreeLevelListRequiredOptional) {
+  // Arrow schema: list(int32) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {GroupNode::Make(
+          "list", Repetition::REPEATED,
+          {PrimitiveNode::Make("element", Repetition::OPTIONAL, ParquetType::INT32)})},
+      LogicalType::List()));
+
+  LevelVector def_levels = {0, 1, 2, 2, 2};
+  LevelVector rep_levels = {0, 0, 1, 0, 1};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto expected = ArrayFromJSON(List(int32()), "[[], [null, 4], [5, 6]]");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, ThreeLevelListOptionalOptional) {
+  // Arrow schema: list(int32)
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {GroupNode::Make(
+          "list", Repetition::REPEATED,
+          {PrimitiveNode::Make("element", Repetition::OPTIONAL, ParquetType::INT32)})},
+      LogicalType::List()));
+
+  LevelVector def_levels = {0, 1, 2, 3, 3, 3};
+  LevelVector rep_levels = {0, 0, 0, 1, 0, 1};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto expected = ArrayFromJSON(List(int32()), "[null, [], [null, 4], [5, 6]]");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+//
+// Legacy list encodings
+//
+
+TEST_F(TestReconstructColumn, TwoLevelListRequired) {
+  // Arrow schema: list(int32 not null) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {PrimitiveNode::Make("element", Repetition::REPEATED, ParquetType::INT32)},
+      LogicalType::List()));
+
+  LevelVector def_levels = {0, 1, 1, 1};
+  LevelVector rep_levels = {0, 0, 1, 0};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  // TODO should field name "element" (Parquet convention for List nodes)
+  // be changed to "item" (Arrow convention for List types)?
+  auto expected = ArrayFromJSON(List(int32(), /*nullable=*/false), "[[], [4, 5], [6]]");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, TwoLevelListOptional) {
+  // Arrow schema: list(int32 not null)
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {PrimitiveNode::Make("element", Repetition::REPEATED, ParquetType::INT32)},
+      LogicalType::List()));
+
+  LevelVector def_levels = {0, 1, 2, 2, 2};
+  LevelVector rep_levels = {0, 0, 0, 1, 0};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto expected =
+      ArrayFromJSON(List(int32(), /*nullable=*/false), "[null, [], [4, 5], [6]]");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+//
+// List-in-struct
+//
+
+TEST_F(TestReconstructColumn, NestedList1) {
+  // Arrow schema: struct(a: list(int32 not null) not null) not null
+  SetParquetSchema(GroupNode::Make(
+      "a", Repetition::REQUIRED,
+      {GroupNode::Make(
+          "p", Repetition::REQUIRED,
+          {GroupNode::Make("list", Repetition::REPEATED,
+                           {PrimitiveNode::Make("element", Repetition::REQUIRED,
+                                                ParquetType::INT32)})},
+          LogicalType::List())}));
+
+  LevelVector def_levels = {0, 1, 1, 1};
+  LevelVector rep_levels = {0, 0, 1, 0};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto type = OneFieldStruct("p", List(int32(), /*nullable=*/false),
+                             /*nullable=*/false);
+  auto expected = ArrayFromJSON(type, R"([{"p": []},
+                                          {"p": [4, 5]},
+                                          {"p": [6]}])");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, NestedList2) {
+  // Arrow schema: struct(a: list(int32 not null) not null)
+  SetParquetSchema(GroupNode::Make(
+      "a", Repetition::OPTIONAL,
+      {GroupNode::Make(
+          "p", Repetition::REQUIRED,
+          {GroupNode::Make("list", Repetition::REPEATED,
+                           {PrimitiveNode::Make("element", Repetition::REQUIRED,
+                                                ParquetType::INT32)})},
+          LogicalType::List())}));
+
+  LevelVector def_levels = {0, 1, 2, 2, 2};
+  LevelVector rep_levels = {0, 0, 0, 1, 0};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto type = OneFieldStruct("p", List(int32(), /*nullable=*/false),
+                             /*nullable=*/false);
+  auto expected = ArrayFromJSON(type, R"([null,
+                                          {"p": []},
+                                          {"p": [4, 5]},
+                                          {"p": [6]}])");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, NestedList3) {
+  // Arrow schema: struct(a: list(int32 not null)) not null
+  SetParquetSchema(GroupNode::Make(
+      "a", Repetition::REQUIRED,  // column name (column a is a struct of)
+      {GroupNode::Make(
+          "p", Repetition::OPTIONAL,  // name in struct
+          {GroupNode::Make("list", Repetition::REPEATED,
+                           {PrimitiveNode::Make("element", Repetition::REQUIRED,
+                                                ParquetType::INT32)})},
+          LogicalType::List())}));
+
+  LevelVector def_levels = {0, 1, 2, 2, 2};
+  LevelVector rep_levels = {0, 0, 0, 1, 0};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto type = OneFieldStruct("p", List(int32(), /*nullable=*/false));
+  auto expected = ArrayFromJSON(type, R"([{"p": null},
+                                          {"p": []},
+                                          {"p": [4, 5]},
+                                          {"p": [6]}])");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, NestedList4) {
+  // Arrow schema: struct(a: list(int32 not null))
+  SetParquetSchema(GroupNode::Make(
+      "a", Repetition::OPTIONAL,
+      {GroupNode::Make(
+          "p", Repetition::OPTIONAL,
+          {GroupNode::Make("list", Repetition::REPEATED,
+                           {PrimitiveNode::Make("element", Repetition::REQUIRED,
+                                                ParquetType::INT32)})},
+          LogicalType::List())}));
+
+  LevelVector def_levels = {0, 1, 2, 3, 3, 3};
+  LevelVector rep_levels = {0, 0, 0, 0, 1, 0};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto type = OneFieldStruct("p", List(int32(), /*nullable=*/false));
+  auto expected = ArrayFromJSON(type, R"([null,
+                                          {"p": null},
+                                          {"p": []},
+                                          {"p": [4, 5]},
+                                          {"p": [6]}])");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, NestedList5) {
+  // Arrow schema: struct(a: list(int32) not null)
+  SetParquetSchema(GroupNode::Make(
+      "a", Repetition::OPTIONAL,
+      {GroupNode::Make(
+          "p", Repetition::REQUIRED,
+          {GroupNode::Make("list", Repetition::REPEATED,
+                           {PrimitiveNode::Make("element", Repetition::OPTIONAL,
+                                                ParquetType::INT32)})},
+          LogicalType::List())}));
+
+  LevelVector def_levels = {0, 1, 3, 2, 3, 3};
+  LevelVector rep_levels = {0, 0, 0, 1, 0, 1};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto type = OneFieldStruct("p", List(int32()), /*nullable=*/false);
+  auto expected = ArrayFromJSON(type, R"([null,
+                                          {"p": []},
+                                          {"p": [4, null]},
+                                          {"p": [5, 6]}])");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, NestedList6) {
+  // Arrow schema: struct(a: list(int32))
+  SetParquetSchema(GroupNode::Make(
+      "a", Repetition::OPTIONAL,
+      {GroupNode::Make(
+          "p", Repetition::OPTIONAL,
+          {GroupNode::Make("list", Repetition::REPEATED,
+                           {PrimitiveNode::Make("element", Repetition::OPTIONAL,
+                                                ParquetType::INT32)})},
+          LogicalType::List())}));
+
+  LevelVector def_levels = {0, 1, 2, 4, 3, 4, 4};
+  LevelVector rep_levels = {0, 0, 0, 0, 1, 0, 1};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto type = OneFieldStruct("p", List(int32()));
+  auto expected = ArrayFromJSON(type, R"([null,
+                                          {"p": null},
+                                          {"p": []},
+                                          {"p": [4, null]},
+                                          {"p": [5, 6]}])");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+//
+// Struct-in-list
+//
+
+TEST_F(TestReconstructColumn, ListNested1) {
+  // Arrow schema: list(struct(a: int32 not null) not null) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {GroupNode::Make("list", Repetition::REPEATED,
+                       {GroupNode::Make("element", Repetition::REQUIRED,
+                                        {PrimitiveNode::Make("a", Repetition::REQUIRED,
+                                                             ParquetType::INT32)})})},
+      LogicalType::List()));
+
+  LevelVector def_levels = {0, 1, 1, 1};
+  LevelVector rep_levels = {0, 0, 1, 0};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto type = List(OneFieldStruct("a", int32(), /*nullable=*/false),
+                   /*nullable=*/false);
+  auto expected = ArrayFromJSON(type,
+                                R"([[],
+                                    [{"a": 4}, {"a": 5}],
+                                    [{"a": 6}]])");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, ListNested2) {
+  // Arrow schema: list(struct(a: int32 not null) not null)
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {GroupNode::Make("list", Repetition::REPEATED,
+                       {GroupNode::Make("element", Repetition::REQUIRED,
+                                        {PrimitiveNode::Make("a", Repetition::REQUIRED,
+                                                             ParquetType::INT32)})})},
+      LogicalType::List()));
+
+  LevelVector def_levels = {0, 1, 2, 2, 2};
+  LevelVector rep_levels = {0, 0, 0, 1, 0};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto type = List(OneFieldStruct("a", int32(), /*nullable=*/false),
+                   /*nullable=*/false);
+  auto expected = ArrayFromJSON(type,
+                                R"([null,
+                                    [],
+                                    [{"a": 4}, {"a": 5}],
+                                    [{"a": 6}]])");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, ListNested3) {
+  // Arrow schema: list(struct(a: int32 not null)) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {GroupNode::Make("list", Repetition::REPEATED,
+                       {GroupNode::Make("element", Repetition::OPTIONAL,
+                                        {PrimitiveNode::Make("a", Repetition::REQUIRED,
+                                                             ParquetType::INT32)})})},
+      LogicalType::List()));
+
+  LevelVector def_levels = {0, 1, 2, 2, 2};
+  LevelVector rep_levels = {0, 0, 1, 1, 0};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto type = List(OneFieldStruct("a", int32(), /*nullable=*/false));
+  auto expected = ArrayFromJSON(type,
+                                R"([[],
+                                    [null, {"a": 4}, {"a": 5}],
+                                    [{"a": 6}]])");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, ListNested4) {
+  // Arrow schema: list(struct(a: int32 not null))
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {GroupNode::Make("list", Repetition::REPEATED,
+                       {GroupNode::Make("element", Repetition::OPTIONAL,
+                                        {PrimitiveNode::Make("a", Repetition::REQUIRED,
+                                                             ParquetType::INT32)})})},
+      LogicalType::List()));
+
+  LevelVector def_levels = {0, 1, 2, 3, 3, 3};
+  LevelVector rep_levels = {0, 0, 0, 1, 1, 0};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto type = List(OneFieldStruct("a", int32(), /*nullable=*/false));
+  auto expected = ArrayFromJSON(type,
+                                R"([null,
+                                    [],
+                                    [null, {"a": 4}, {"a": 5}],
+                                    [{"a": 6}]])");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, ListNested5) {
+  // Arrow schema: list(struct(a: int32) not null)
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {GroupNode::Make("list", Repetition::REPEATED,
+                       {GroupNode::Make("element", Repetition::REQUIRED,
+                                        {PrimitiveNode::Make("a", Repetition::OPTIONAL,
+                                                             ParquetType::INT32)})})},
+      LogicalType::List()));
+
+  LevelVector def_levels = {0, 1, 2, 3, 3, 3};
+  LevelVector rep_levels = {0, 0, 0, 1, 0, 1};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto type = List(OneFieldStruct("a", int32()),
+                   /*nullable=*/false);
+  auto expected = ArrayFromJSON(type,
+                                R"([null,
+                                    [],
+                                    [{"a": null}, {"a": 4}],
+                                    [{"a": 5}, {"a": 6}]])");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, ListNested6) {
+  // Arrow schema: list(struct(a: int32))
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {GroupNode::Make("list", Repetition::REPEATED,
+                       {GroupNode::Make("element", Repetition::OPTIONAL,
+                                        {PrimitiveNode::Make("a", Repetition::OPTIONAL,
+                                                             ParquetType::INT32)})})},
+      LogicalType::List()));
+
+  LevelVector def_levels = {0, 1, 2, 3, 4, 4, 4};
+  LevelVector rep_levels = {0, 0, 0, 1, 1, 0, 1};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto type = List(OneFieldStruct("a", int32()));
+  auto expected = ArrayFromJSON(type,
+                                R"([null,
+                                    [],
+                                    [null, {"a": null}, {"a": 4}],
+                                    [{"a": 5}, {"a": 6}]])");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+//
+// Struct (two fields)-in-list
+//
+
+TEST_F(TestReconstructColumn, ListNestedTwoFields1) {
+  // Arrow schema: list(struct(a: int32 not null,
+  //                           b: int64 not null) not null) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {GroupNode::Make(
+          "list", Repetition::REPEATED,
+          {GroupNode::Make(
+              "element", Repetition::REQUIRED,
+              {PrimitiveNode::Make("a", Repetition::REQUIRED, ParquetType::INT32),
+               PrimitiveNode::Make("b", Repetition::REQUIRED, ParquetType::INT64)})})},
+      LogicalType::List()));
+
+  // a
+  ASSERT_OK(WriteInt32Column(DefLevels{0, 1, 1, 1}, RepLevels{0, 0, 1, 0},
+                             Int32Vector{4, 5, 6}));
+  // b
+  ASSERT_OK(WriteInt64Column(DefLevels{0, 1, 1, 1}, RepLevels{0, 0, 1, 0},
+                             Int64Vector{7, 8, 9}));
+
+  auto type = List(struct_({field("a", int32(), /*nullable=*/false),
+                            field("b", int64(), /*nullable=*/false)}),
+                   /*nullable=*/false);
+  auto expected = ArrayFromJSON(type,
+                                R"([[],
+                                    [{"a": 4, "b": 7}, {"a": 5, "b": 8}],
+                                    [{"a": 6, "b": 9}]])");
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+TEST_F(TestReconstructColumn, ListNestedTwoFields2) {
+  // Arrow schema: list(struct(a: int32,
+  //                           b: int64 not null) not null) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {GroupNode::Make(
+          "list", Repetition::REPEATED,
+          {GroupNode::Make(
+              "element", Repetition::REQUIRED,
+              {PrimitiveNode::Make("a", Repetition::OPTIONAL, ParquetType::INT32),
+               PrimitiveNode::Make("b", Repetition::REQUIRED, ParquetType::INT64)})})},
+      LogicalType::List()));
+
+  // a
+  ASSERT_OK(
+      WriteInt32Column(DefLevels{0, 2, 1, 2}, RepLevels{0, 0, 1, 0}, Int32Vector{4, 5}));
+  // b
+  ASSERT_OK(WriteInt64Column(DefLevels{0, 1, 1, 1}, RepLevels{0, 0, 1, 0},
+                             Int64Vector{7, 8, 9}));
+
+  auto type =
+      List(struct_({field("a", int32()), field("b", int64(), /*nullable=*/false)}),
+           /*nullable=*/false);
+  auto expected = ArrayFromJSON(type,
+                                R"([[],
+                                    [{"a": 4, "b": 7}, {"a": null, "b": 8}],
+                                    [{"a": 5, "b": 9}]])");
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+TEST_F(TestReconstructColumn, ListNestedTwoFields3) {
+  // Arrow schema: list(struct(a: int32 not null,
+  //                           b: int64 not null)) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {GroupNode::Make(
+          "list", Repetition::REPEATED,
+          {GroupNode::Make(
+              "element", Repetition::OPTIONAL,
+              {PrimitiveNode::Make("a", Repetition::REQUIRED, ParquetType::INT32),
+               PrimitiveNode::Make("b", Repetition::REQUIRED, ParquetType::INT64)})})},
+      LogicalType::List()));
+
+  // a
+  ASSERT_OK(WriteInt32Column(DefLevels{0, 1, 2, 2, 2}, RepLevels{0, 0, 1, 1, 0},
+                             Int32Vector{4, 5, 6}));
+  // b
+  ASSERT_OK(WriteInt64Column(DefLevels{0, 1, 2, 2, 2}, RepLevels{0, 0, 1, 1, 0},
+                             Int64Vector{7, 8, 9}));
+
+  auto type = List(struct_({field("a", int32(), /*nullable=*/false),
+                            field("b", int64(), /*nullable=*/false)}));
+  auto expected = ArrayFromJSON(type,
+                                R"([[],
+                                    [null, {"a": 4, "b": 7}, {"a": 5, "b": 8}],
+                                    [{"a": 6, "b": 9}]])");
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+TEST_F(TestReconstructColumn, ListNestedTwoFields4) {
+  // Arrow schema: list(struct(a: int32,
+  //                           b: int64 not null) not null)
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {GroupNode::Make(
+          "list", Repetition::REPEATED,
+          {GroupNode::Make(
+              "element", Repetition::REQUIRED,
+              {PrimitiveNode::Make("a", Repetition::OPTIONAL, ParquetType::INT32),
+               PrimitiveNode::Make("b", Repetition::REQUIRED, ParquetType::INT64)})})},
+      LogicalType::List()));
+
+  // a
+  ASSERT_OK(WriteInt32Column(DefLevels{0, 1, 3, 2, 3}, RepLevels{0, 0, 0, 1, 0},
+                             Int32Vector{4, 5}));
+  // b
+  ASSERT_OK(WriteInt64Column(DefLevels{0, 1, 2, 2, 2}, RepLevels{0, 0, 0, 1, 0},
+                             Int64Vector{7, 8, 9}));
+
+  auto type =
+      List(struct_({field("a", int32()), field("b", int64(), /*nullable=*/false)}),
+           /*nullable=*/false);
+  auto expected = ArrayFromJSON(type,
+                                R"([null,
+                                    [],
+                                    [{"a": 4, "b": 7}, {"a": null, "b": 8}],
+                                    [{"a": 5, "b": 9}]])");
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+TEST_F(TestReconstructColumn, ListNestedTwoFields5) {
+  // Arrow schema: list(struct(a: int32,
+  //                           b: int64 not null))
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {GroupNode::Make(
+          "list", Repetition::REPEATED,
+          {GroupNode::Make(
+              "element", Repetition::OPTIONAL,
+              {PrimitiveNode::Make("a", Repetition::OPTIONAL, ParquetType::INT32),
+               PrimitiveNode::Make("b", Repetition::REQUIRED, ParquetType::INT64)})})},
+      LogicalType::List()));
+
+  // a
+  ASSERT_OK(WriteInt32Column(DefLevels{0, 1, 4, 2, 3}, RepLevels{0, 0, 0, 1, 0},
+                             Int32Vector{4}));
+  // b
+  ASSERT_OK(WriteInt64Column(DefLevels{0, 1, 3, 2, 3}, RepLevels{0, 0, 0, 1, 0},
+                             Int64Vector{7, 8}));
+
+  auto type =
+      List(struct_({field("a", int32()), field("b", int64(), /*nullable=*/false)}));
+  auto expected = ArrayFromJSON(type,
+                                R"([null,
+                                    [],
+                                    [{"a": 4, "b": 7}, null],
+                                    [{"a": null, "b": 8}]])");
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+TEST_F(TestReconstructColumn, ListNestedTwoFields6) {
+  // Arrow schema: list(struct(a: int32,
+  //                           b: int64))
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {GroupNode::Make(
+          "list", Repetition::REPEATED,
+          {GroupNode::Make(
+              "element", Repetition::OPTIONAL,
+              {PrimitiveNode::Make("a", Repetition::OPTIONAL, ParquetType::INT32),
+               PrimitiveNode::Make("b", Repetition::OPTIONAL, ParquetType::INT64)})})},
+      LogicalType::List()));
+
+  // a
+  ASSERT_OK(WriteInt32Column(DefLevels{0, 1, 4, 2, 3}, RepLevels{0, 0, 0, 1, 0},
+                             Int32Vector{4}));
+  // b
+  ASSERT_OK(WriteInt64Column(DefLevels{0, 1, 3, 2, 4}, RepLevels{0, 0, 0, 1, 0},
+                             Int64Vector{7}));
+
+  auto type = List(struct_({field("a", int32()), field("b", int64())}));
+  auto expected = ArrayFromJSON(type,
+                                R"([null,
+                                    [],
+                                    [{"a": 4, "b": null}, null],
+                                    [{"a": null, "b": 7}]])");
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+//
+// List-in-struct (two fields)
+//
+
+TEST_F(TestReconstructColumn, NestedTwoFieldsList1) {
+  // Arrow schema: struct(a: int64 not null,
+  //                      b: list(int32 not null) not null
+  //                     ) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {PrimitiveNode::Make("a", Repetition::REQUIRED, ParquetType::INT64),
+       GroupNode::Make(
+           "b", Repetition::REQUIRED,
+           {GroupNode::Make("list", Repetition::REPEATED,
+                            {PrimitiveNode::Make("element", Repetition::REQUIRED,
+                                                 ParquetType::INT32)})},
+           LogicalType::List())}));
+
+  // a
+  ASSERT_OK(WriteInt64Column(DefLevels{}, RepLevels{}, Int64Vector{4, 5, 6}));
+  // b
+  ASSERT_OK(WriteInt32Column(DefLevels{0, 1, 1, 1}, RepLevels{0, 0, 1, 0},
+                             Int32Vector{7, 8, 9}));
+
+  auto type =
+      struct_({field("a", int64(), /*nullable=*/false),
+               field("b", List(int32(), /*nullable=*/false), /*nullable=*/false)});
+  auto expected = ArrayFromJSON(type,
+                                R"([{"a": 4, "b": []},
+                                    {"a": 5, "b": [7, 8]},
+                                    {"a": 6, "b": [9]}])");
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+TEST_F(TestReconstructColumn, NestedTwoFieldsList2) {
+  // Arrow schema: struct(a: int64 not null,
+  //                      b: list(int32 not null)
+  //                     ) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {PrimitiveNode::Make("a", Repetition::REQUIRED, ParquetType::INT64),
+       GroupNode::Make(
+           "b", Repetition::OPTIONAL,
+           {GroupNode::Make("list", Repetition::REPEATED,
+                            {PrimitiveNode::Make("element", Repetition::REQUIRED,
+                                                 ParquetType::INT32)})},
+           LogicalType::List())}));
+
+  // a
+  ASSERT_OK(WriteInt64Column(DefLevels{}, RepLevels{}, Int64Vector{3, 4, 5, 6}));
+  // b
+  ASSERT_OK(WriteInt32Column(DefLevels{0, 1, 2, 2, 2}, RepLevels{0, 0, 0, 1, 0},
+                             Int32Vector{7, 8, 9}));
+
+  auto type = struct_({field("a", int64(), /*nullable=*/false),
+                       field("b", List(int32(), /*nullable=*/false))});
+  auto expected = ArrayFromJSON(type,
+                                R"([{"a": 3, "b": null},
+                                    {"a": 4, "b": []},
+                                    {"a": 5, "b": [7, 8]},
+                                    {"a": 6, "b": [9]}])");
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+TEST_F(TestReconstructColumn, NestedTwoFieldsList3) {
+  // Arrow schema: struct(a: int64,
+  //                      b: list(int32 not null)
+  //                     ) not null
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::REQUIRED,
+      {PrimitiveNode::Make("a", Repetition::OPTIONAL, ParquetType::INT64),
+       GroupNode::Make(
+           "b", Repetition::OPTIONAL,
+           {GroupNode::Make("list", Repetition::REPEATED,
+                            {PrimitiveNode::Make("element", Repetition::REQUIRED,
+                                                 ParquetType::INT32)})},
+           LogicalType::List())}));
+
+  // a
+  ASSERT_OK(WriteInt64Column(DefLevels{1, 1, 0, 1}, RepLevels{}, Int64Vector{4, 5, 6}));
+  // b
+  ASSERT_OK(WriteInt32Column(DefLevels{0, 1, 2, 2, 2}, RepLevels{0, 0, 0, 1, 0},
+                             Int32Vector{7, 8, 9}));
+
+  auto type =
+      struct_({field("a", int64()), field("b", List(int32(), /*nullable=*/false))});
+  auto expected = ArrayFromJSON(type,
+                                R"([{"a": 4, "b": null},
+                                    {"a": 5, "b": []},
+                                    {"a": null, "b": [7, 8]},
+                                    {"a": 6, "b": [9]}])");
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+TEST_F(TestReconstructColumn, NestedTwoFieldsList4) {
+  // Arrow schema: struct(a: int64,
+  //                      b: list(int32 not null)
+  //                     )
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {PrimitiveNode::Make("a", Repetition::OPTIONAL, ParquetType::INT64),
+       GroupNode::Make(
+           "b", Repetition::OPTIONAL,
+           {GroupNode::Make("list", Repetition::REPEATED,
+                            {PrimitiveNode::Make("element", Repetition::REQUIRED,
+                                                 ParquetType::INT32)})},
+           LogicalType::List())}));
+
+  // a
+  ASSERT_OK(
+      WriteInt64Column(DefLevels{0, 2, 2, 1, 2}, RepLevels{}, Int64Vector{4, 5, 6}));
+  // b
+  ASSERT_OK(WriteInt32Column(DefLevels{0, 1, 2, 3, 3, 3}, RepLevels{0, 0, 0, 0, 1, 0},
+                             Int32Vector{7, 8, 9}));
+
+  auto type =
+      struct_({field("a", int64()), field("b", List(int32(), /*nullable=*/false))});
+  auto expected = ArrayFromJSON(type,
+                                R"([null,
+                                    {"a": 4, "b": null},
+                                    {"a": 5, "b": []},
+                                    {"a": null, "b": [7, 8]},
+                                    {"a": 6, "b": [9]}])");
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+TEST_F(TestReconstructColumn, NestedTwoFieldsList5) {
+  // Arrow schema: struct(a: int64, b: list(int32))
+  SetParquetSchema(GroupNode::Make(
+      "parent", Repetition::OPTIONAL,
+      {PrimitiveNode::Make("a", Repetition::OPTIONAL, ParquetType::INT64),
+       GroupNode::Make(
+           "b", Repetition::OPTIONAL,
+           {GroupNode::Make("list", Repetition::REPEATED,
+                            {PrimitiveNode::Make("element", Repetition::OPTIONAL,
+                                                 ParquetType::INT32)})},
+           LogicalType::List())}));
+
+  // a
+  ASSERT_OK(
+      WriteInt64Column(DefLevels{0, 2, 2, 1, 2}, RepLevels{}, Int64Vector{4, 5, 6}));
+  // b
+  ASSERT_OK(WriteInt32Column(DefLevels{0, 1, 2, 4, 3, 4}, RepLevels{0, 0, 0, 0, 1, 0},
+                             Int32Vector{7, 8}));
+
+  auto type = struct_({field("a", int64()), field("b", List(int32()))});
+  auto expected = ArrayFromJSON(type,
+                                R"([null,
+                                    {"a": 4, "b": null},
+                                    {"a": 5, "b": []},
+                                    {"a": null, "b": [7, null]},
+                                    {"a": 6, "b": [8]}])");
+  CheckColumn(/*column_index=*/0, *expected);
+}
+
+//
+// List-in-list
+//
+
+TEST_F(TestReconstructColumn, ListList1) {
+  // Arrow schema: list(list(int32 not null) not null) not null
+  auto inner_list = GroupNode::Make(
+      "element", Repetition::REQUIRED,
+      {GroupNode::Make(
+          "list", Repetition::REPEATED,
+          {PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::INT32)})},
+      LogicalType::List());
+  SetParquetSchema(
+      GroupNode::Make("parent", Repetition::REQUIRED,
+                      {GroupNode::Make("list", Repetition::REPEATED, {inner_list})},
+                      LogicalType::List()));
+
+  LevelVector def_levels = {0, 1, 2, 2, 2};
+  LevelVector rep_levels = {0, 0, 1, 0, 2};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto type = List(List(int32(), /*nullable=*/false), /*nullable=*/false);
+  auto expected = ArrayFromJSON(type, "[[], [[], [4]], [[5, 6]]]");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, ListList2) {
+  // Arrow schema: list(list(int32 not null) not null)
+  auto inner_list = GroupNode::Make(
+      "element", Repetition::REQUIRED,
+      {GroupNode::Make(
+          "list", Repetition::REPEATED,
+          {PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::INT32)})},
+      LogicalType::List());
+  SetParquetSchema(
+      GroupNode::Make("parent", Repetition::OPTIONAL,
+                      {GroupNode::Make("list", Repetition::REPEATED, {inner_list})},
+                      LogicalType::List()));
+
+  LevelVector def_levels = {0, 1, 2, 3, 3, 3};
+  LevelVector rep_levels = {0, 0, 0, 1, 0, 2};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto type = List(List(int32(), /*nullable=*/false), /*nullable=*/false);
+  auto expected = ArrayFromJSON(type, "[null, [], [[], [4]], [[5, 6]]]");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, ListList3) {
+  // Arrow schema: list(list(int32 not null)) not null
+  auto inner_list = GroupNode::Make(
+      "element", Repetition::OPTIONAL,
+      {GroupNode::Make(
+          "list", Repetition::REPEATED,
+          {PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::INT32)})},
+      LogicalType::List());
+  SetParquetSchema(
+      GroupNode::Make("parent", Repetition::REQUIRED,
+                      {GroupNode::Make("list", Repetition::REPEATED, {inner_list})},
+                      LogicalType::List()));
+
+  LevelVector def_levels = {0, 1, 2, 3, 3, 3};
+  LevelVector rep_levels = {0, 0, 1, 0, 1, 2};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto type = List(List(int32(), /*nullable=*/false));
+  auto expected = ArrayFromJSON(type, "[[], [null, []], [[4], [5, 6]]]");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, ListList4) {
+  // Arrow schema: list(list(int32 not null))
+  auto inner_list = GroupNode::Make(
+      "element", Repetition::OPTIONAL,
+      {GroupNode::Make(
+          "list", Repetition::REPEATED,
+          {PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::INT32)})},
+      LogicalType::List());
+  SetParquetSchema(
+      GroupNode::Make("parent", Repetition::OPTIONAL,
+                      {GroupNode::Make("list", Repetition::REPEATED, {inner_list})},
+                      LogicalType::List()));
+
+  LevelVector def_levels = {0, 1, 2, 3, 4, 4, 4};
+  LevelVector rep_levels = {0, 0, 0, 1, 1, 0, 2};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto type = List(List(int32(), /*nullable=*/false));
+  auto expected = ArrayFromJSON(type, "[null, [], [null, [], [4]], [[5, 6]]]");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, ListList5) {
+  // Arrow schema: list(list(int32) not null)
+  auto inner_list = GroupNode::Make(
+      "element", Repetition::REQUIRED,
+      {GroupNode::Make(
+          "list", Repetition::REPEATED,
+          {PrimitiveNode::Make("element", Repetition::OPTIONAL, ParquetType::INT32)})},
+      LogicalType::List());
+  SetParquetSchema(
+      GroupNode::Make("parent", Repetition::OPTIONAL,
+                      {GroupNode::Make("list", Repetition::REPEATED, {inner_list})},
+                      LogicalType::List()));
+
+  LevelVector def_levels = {0, 1, 2, 4, 4, 3, 4};
+  LevelVector rep_levels = {0, 0, 0, 1, 0, 1, 2};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto type = List(List(int32()), /*nullable=*/false);
+  auto expected = ArrayFromJSON(type, "[null, [], [[], [4]], [[5], [null, 6]]]");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+TEST_F(TestReconstructColumn, ListList6) {
+  // Arrow schema: list(list(int32))
+  auto inner_list = GroupNode::Make(
+      "element", Repetition::OPTIONAL,
+      {GroupNode::Make(
+          "list", Repetition::REPEATED,
+          {PrimitiveNode::Make("element", Repetition::OPTIONAL, ParquetType::INT32)})},
+      LogicalType::List());
+  SetParquetSchema(
+      GroupNode::Make("parent", Repetition::OPTIONAL,
+                      {GroupNode::Make("list", Repetition::REPEATED, {inner_list})},
+                      LogicalType::List()));
+
+  LevelVector def_levels = {0, 1, 2, 3, 4, 5, 5, 5};
+  LevelVector rep_levels = {0, 0, 0, 1, 1, 2, 0, 2};
+  std::vector<int32_t> values = {4, 5, 6};
+
+  auto type = List(List(int32()));
+  auto expected = ArrayFromJSON(type, "[null, [], [null, [], [null, 4]], [[5, 6]]]");
+  AssertReconstruct<ParquetType::INT32>(*expected, def_levels, rep_levels, values);
+}
+
+// TODO legacy-list-in-struct etc.?
+
+}  // namespace arrow
+}  // namespace parquet
diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc
index ae9c61e44eb..555c7a85f1a 100644
--- a/cpp/src/parquet/arrow/schema.cc
+++ b/cpp/src/parquet/arrow/schema.cc
@@ -17,6 +17,7 @@
 
 #include "parquet/arrow/schema.h"
 
+#include <functional>
 #include <string>
 #include <vector>
 
@@ -27,12 +28,15 @@
 #include "arrow/util/base64.h"
 #include "arrow/util/checked_cast.h"
 #include "arrow/util/key_value_metadata.h"
+#include "arrow/util/logging.h"
+
 #include "parquet/arrow/schema_internal.h"
 #include "parquet/exception.h"
 #include "parquet/properties.h"
 #include "parquet/types.h"
 
 using arrow::Field;
+using arrow::FieldVector;
 using arrow::KeyValueMetadata;
 using arrow::Status;
 using arrow::internal::checked_cast;
@@ -51,6 +55,7 @@ using parquet::ConvertedType;
 using parquet::LogicalType;
 
 using parquet::internal::DecimalSize;
+using parquet::internal::LevelInfo;
 
 namespace parquet {
 
@@ -294,8 +299,7 @@ Status FieldToNode(const std::string& name, const std::shared_ptr<Field>& field,
     } break;
     case ArrowTypeId::DECIMAL: {
       type = ParquetType::FIXED_LEN_BYTE_ARRAY;
-      const auto& decimal_type =
-          static_cast<const ::arrow::Decimal128Type&>(*field->type());
+      const auto& decimal_type = static_cast<const ::arrow::DecimalType&>(*field->type());
       precision = decimal_type.precision();
       scale = decimal_type.scale();
       length = DecimalSize(precision);
@@ -410,21 +414,20 @@ ::arrow::Result<std::shared_ptr<ArrowType>> GetTypeForNode(
   return storage_type;
 }
 
-Status NodeToSchemaField(const Node& node, int16_t max_def_level, int16_t max_rep_level,
+Status NodeToSchemaField(const Node& node, LevelInfo current_levels,
                          SchemaTreeContext* ctx, const SchemaField* parent,
                          SchemaField* out);
 
-Status GroupToSchemaField(const GroupNode& node, int16_t max_def_level,
-                          int16_t max_rep_level, SchemaTreeContext* ctx,
-                          const SchemaField* parent, SchemaField* out);
+Status GroupToSchemaField(const GroupNode& node, LevelInfo current_levels,
+                          SchemaTreeContext* ctx, const SchemaField* parent,
+                          SchemaField* out);
 
 Status PopulateLeaf(int column_index, const std::shared_ptr<Field>& field,
-                    int16_t max_def_level, int16_t max_rep_level, SchemaTreeContext* ctx,
+                    LevelInfo current_levels, SchemaTreeContext* ctx,
                     const SchemaField* parent, SchemaField* out) {
   out->field = field;
   out->column_index = column_index;
-  out->definition_level = max_def_level;
-  out->repetition_level = max_rep_level;
+  out->level_info = current_levels;
   ctx->RecordLeaf(out);
   ctx->LinkParent(out, parent);
   return Status::OK();
@@ -442,31 +445,121 @@ std::shared_ptr<::arrow::KeyValueMetadata> FieldIdMetadata(int field_id) {
   return ::arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(field_id)});
 }
 
-Status GroupToStruct(const GroupNode& node, int16_t current_def_level,
-                     int16_t current_rep_level, SchemaTreeContext* ctx,
-                     const SchemaField* parent, SchemaField* out) {
+Status GroupToStruct(const GroupNode& node, LevelInfo current_levels,
+                     SchemaTreeContext* ctx, const SchemaField* parent,
+                     SchemaField* out) {
   std::vector<std::shared_ptr<Field>> arrow_fields;
   out->children.resize(node.field_count());
+  // All level increments for the node are expected to happen by callers.
+  // This is required because repeated elements need to have there own
+  // SchemaField.
+
   for (int i = 0; i < node.field_count(); i++) {
-    RETURN_NOT_OK(NodeToSchemaField(*node.field(i), current_def_level, current_rep_level,
-                                    ctx, out, &out->children[i]));
+    RETURN_NOT_OK(
+        NodeToSchemaField(*node.field(i), current_levels, ctx, out, &out->children[i]));
     arrow_fields.push_back(out->children[i].field);
   }
   auto struct_type = ::arrow::struct_(arrow_fields);
   out->field = ::arrow::field(node.name(), struct_type, node.is_optional(),
                               FieldIdMetadata(node.field_id()));
-  out->definition_level = current_def_level;
-  out->repetition_level = current_rep_level;
+  out->level_info = current_levels;
+  return Status::OK();
+}
+
+Status ListToSchemaField(const GroupNode& group, LevelInfo current_levels,
+                         SchemaTreeContext* ctx, const SchemaField* parent,
+                         SchemaField* out);
+
+Status MapToSchemaField(const GroupNode& group, LevelInfo current_levels,
+                        SchemaTreeContext* ctx, const SchemaField* parent,
+                        SchemaField* out) {
+  if (group.field_count() != 1) {
+    return Status::Invalid("MAP-annotated groups must have a single child.");
+  }
+  if (group.is_repeated()) {
+    return Status::Invalid("MAP-annotated groups must not be repeated.");
+  }
+
+  const Node& key_value_node = *group.field(0);
+
+  if (!key_value_node.is_repeated()) {
+    return Status::Invalid(
+        "Non-repeated key value in a MAP-annotated group are not supported.");
+  }
+
+  if (!key_value_node.is_group()) {
+    return Status::Invalid("Key-value node must be a group.");
+  }
+
+  const GroupNode& key_value = checked_cast<const GroupNode&>(key_value_node);
+  if (key_value.field_count() != 1 && key_value.field_count() != 2) {
+    return Status::Invalid("Key-value map node must have 1 or 2 child elements. Found: ",
+                           key_value.field_count());
+  }
+  const Node& key_node = *key_value.field(0);
+  if (!key_node.is_required()) {
+    return Status::Invalid("Map keys must be annotated as required.");
+  }
+  // Arrow doesn't support 1 column maps (i.e. Sets).  The options are to either
+  // make the values column nullable, or process the map as a list.  We choose the latter
+  // as it is simpler.
+  if (key_value.field_count() == 1) {
+    return ListToSchemaField(group, current_levels, ctx, parent, out);
+  }
+
+  current_levels.Increment(group);
+  int16_t repeated_ancestor_def_level = current_levels.IncrementRepeated();
+
+  out->children.resize(1);
+  SchemaField* key_value_field = &out->children[0];
+
+  key_value_field->children.resize(2);
+  SchemaField* key_field = &key_value_field->children[0];
+  SchemaField* value_field = &key_value_field->children[1];
+
+  ctx->LinkParent(out, parent);
+  ctx->LinkParent(key_value_field, out);
+  ctx->LinkParent(key_field, key_value_field);
+  ctx->LinkParent(value_field, key_value_field);
+
+  // required/optional group name=whatever {
+  //   repeated group name=key_values{
+  //     required TYPE key;
+  // required/optional TYPE value;
+  //   }
+  // }
+  //
+
+  RETURN_NOT_OK(NodeToSchemaField(*key_value.field(0), current_levels, ctx,
+                                  key_value_field, key_field));
+  RETURN_NOT_OK(NodeToSchemaField(*key_value.field(1), current_levels, ctx,
+                                  key_value_field, value_field));
+
+  key_value_field->field = ::arrow::field(
+      group.name(), ::arrow::struct_({key_field->field, value_field->field}),
+      /*nullable=*/false, FieldIdMetadata(key_value.field_id()));
+  key_value_field->level_info = current_levels;
+
+  out->field = ::arrow::field(group.name(),
+                              ::arrow::map(key_field->field->type(), value_field->field),
+                              group.is_optional(), FieldIdMetadata(group.field_id()));
+  out->level_info = current_levels;
+  // At this point current levels contains the def level for this list,
+  // we need to reset to the prior parent.
+  out->level_info.repeated_ancestor_def_level = repeated_ancestor_def_level;
   return Status::OK();
 }
 
-Status ListToSchemaField(const GroupNode& group, int16_t current_def_level,
-                         int16_t current_rep_level, SchemaTreeContext* ctx,
-                         const SchemaField* parent, SchemaField* out) {
+Status ListToSchemaField(const GroupNode& group, LevelInfo current_levels,
+                         SchemaTreeContext* ctx, const SchemaField* parent,
+                         SchemaField* out) {
   if (group.field_count() != 1) {
-    return Status::NotImplemented(
-        "Only LIST-annotated groups with a single child can be handled.");
+    return Status::Invalid("LIST-annotated groups must have a single child.");
+  }
+  if (group.is_repeated()) {
+    return Status::Invalid("LIST-annotated groups must not be repeated.");
   }
+  current_levels.Increment(group);
 
   out->children.resize(group.field_count());
   SchemaField* child_field = &out->children[0];
@@ -477,12 +570,11 @@ Status ListToSchemaField(const GroupNode& group, int16_t current_def_level,
   const Node& list_node = *group.field(0);
 
   if (!list_node.is_repeated()) {
-    return Status::NotImplemented(
+    return Status::Invalid(
         "Non-repeated nodes in a LIST-annotated group are not supported.");
   }
 
-  ++current_def_level;
-  ++current_rep_level;
+  int16_t repeated_ancestor_def_level = current_levels.IncrementRepeated();
   if (list_node.is_group()) {
     // Resolve 3-level encoding
     //
@@ -494,7 +586,7 @@ Status ListToSchemaField(const GroupNode& group, int16_t current_def_level,
     //
     // yields list<item: TYPE ?nullable> ?nullable
     //
-    // We distinguish the special base that we have
+    // We distinguish the special case that we have
     //
     // required/optional group name=whatever {
     //   repeated group name=array or $SOMETHING_tuple {
@@ -512,11 +604,10 @@ Status ListToSchemaField(const GroupNode& group, int16_t current_def_level,
     //   even for single child elements.
     if (list_group.field_count() == 1 && !HasStructListName(list_group)) {
       // List of primitive type
-      RETURN_NOT_OK(NodeToSchemaField(*list_group.field(0), current_def_level,
-                                      current_rep_level, ctx, out, child_field));
+      RETURN_NOT_OK(
+          NodeToSchemaField(*list_group.field(0), current_levels, ctx, out, child_field));
     } else {
-      RETURN_NOT_OK(GroupToStruct(list_group, current_def_level, current_rep_level, ctx,
-                                  out, child_field));
+      RETURN_NOT_OK(GroupToStruct(list_group, current_levels, ctx, out, child_field));
     }
   } else {
     // Two-level list encoding
@@ -530,22 +621,25 @@ Status ListToSchemaField(const GroupNode& group, int16_t current_def_level,
                     GetTypeForNode(column_index, primitive_node, ctx));
     auto item_field = ::arrow::field(list_node.name(), type, /*nullable=*/false,
                                      FieldIdMetadata(list_node.field_id()));
-    RETURN_NOT_OK(PopulateLeaf(column_index, item_field, current_def_level,
-                               current_rep_level, ctx, out, child_field));
+    RETURN_NOT_OK(
+        PopulateLeaf(column_index, item_field, current_levels, ctx, out, child_field));
   }
   out->field = ::arrow::field(group.name(), ::arrow::list(child_field->field),
                               group.is_optional(), FieldIdMetadata(group.field_id()));
-  out->definition_level = current_def_level;
-  out->repetition_level = current_rep_level;
+  out->level_info = current_levels;
+  // At this point current levels contains the def level for this list,
+  // we need to reset to the prior parent.
+  out->level_info.repeated_ancestor_def_level = repeated_ancestor_def_level;
   return Status::OK();
 }
 
-Status GroupToSchemaField(const GroupNode& node, int16_t current_def_level,
-                          int16_t current_rep_level, SchemaTreeContext* ctx,
-                          const SchemaField* parent, SchemaField* out) {
+Status GroupToSchemaField(const GroupNode& node, LevelInfo current_levels,
+                          SchemaTreeContext* ctx, const SchemaField* parent,
+                          SchemaField* out) {
   if (node.logical_type()->is_list()) {
-    return ListToSchemaField(node, current_def_level, current_rep_level, ctx, parent,
-                             out);
+    return ListToSchemaField(node, current_levels, ctx, parent, out);
+  } else if (node.logical_type()->is_map()) {
+    return MapToSchemaField(node, current_levels, ctx, parent, out);
   }
   std::shared_ptr<ArrowType> type;
   if (node.is_repeated()) {
@@ -556,39 +650,37 @@ Status GroupToSchemaField(const GroupNode& node, int16_t current_def_level,
     //   r/o TYPE[1] f1
     // }
     out->children.resize(1);
-    RETURN_NOT_OK(GroupToStruct(node, current_def_level, current_rep_level, ctx, out,
-                                &out->children[0]));
+
+    int16_t repeated_ancestor_def_level = current_levels.IncrementRepeated();
+    RETURN_NOT_OK(GroupToStruct(node, current_levels, ctx, out, &out->children[0]));
     out->field = ::arrow::field(node.name(), ::arrow::list(out->children[0].field),
-                                node.is_optional(), FieldIdMetadata(node.field_id()));
-    out->definition_level = current_def_level;
-    out->repetition_level = current_rep_level;
+                                /*nullable=*/false, FieldIdMetadata(node.field_id()));
+
+    ctx->LinkParent(&out->children[0], out);
+    out->level_info = current_levels;
+    // At this point current_levels contains this list as the def level, we need to
+    // use the previous ancenstor of thi slist.
+    out->level_info.repeated_ancestor_def_level = repeated_ancestor_def_level;
     return Status::OK();
   } else {
-    return GroupToStruct(node, current_def_level, current_rep_level, ctx, parent, out);
+    current_levels.Increment(node);
+    return GroupToStruct(node, current_levels, ctx, parent, out);
   }
 }
 
-Status NodeToSchemaField(const Node& node, int16_t current_def_level,
-                         int16_t current_rep_level, SchemaTreeContext* ctx,
-                         const SchemaField* parent, SchemaField* out) {
-  /// Workhorse function for converting a Parquet schema node to an Arrow
-  /// type. Handles different conventions for nested data
-  if (node.is_optional()) {
-    ++current_def_level;
-  } else if (node.is_repeated()) {
-    // Repeated fields add both a repetition and definition level. This is used
-    // to distinguish between an empty list and a list with an item in it.
-    ++current_rep_level;
-    ++current_def_level;
-  }
+Status NodeToSchemaField(const Node& node, LevelInfo current_levels,
+                         SchemaTreeContext* ctx, const SchemaField* parent,
+                         SchemaField* out) {
+  // Workhorse function for converting a Parquet schema node to an Arrow
+  // type. Handles different conventions for nested data.
 
   ctx->LinkParent(out, parent);
 
   // Now, walk the schema and create a ColumnDescriptor for each leaf node
   if (node.is_group()) {
     // A nested field, but we don't know what kind yet
-    return GroupToSchemaField(static_cast<const GroupNode&>(node), current_def_level,
-                              current_rep_level, ctx, parent, out);
+    return GroupToSchemaField(static_cast<const GroupNode&>(node), current_levels, ctx,
+                              parent, out);
   } else {
     // Either a normal flat primitive type, or a list type encoded with 1-level
     // list encoding. Note that the 3-level encoding is the form recommended by
@@ -606,23 +698,26 @@ Status NodeToSchemaField(const Node& node, int16_t current_def_level,
     if (node.is_repeated()) {
       // One-level list encoding, e.g.
       // a: repeated int32;
+      int16_t repeated_ancestor_def_level = current_levels.IncrementRepeated();
       out->children.resize(1);
       auto child_field = ::arrow::field(node.name(), type, /*nullable=*/false);
-      RETURN_NOT_OK(PopulateLeaf(column_index, child_field, current_def_level,
-                                 current_rep_level, ctx, out, &out->children[0]));
+      RETURN_NOT_OK(PopulateLeaf(column_index, child_field, current_levels, ctx, out,
+                                 &out->children[0]));
 
       out->field = ::arrow::field(node.name(), ::arrow::list(child_field),
                                   /*nullable=*/false, FieldIdMetadata(node.field_id()));
-      // Is this right?
-      out->definition_level = current_def_level;
-      out->repetition_level = current_rep_level;
+      out->level_info = current_levels;
+      // At this point current_levels has consider this list the ancestor so restore
+      // the actual ancenstor.
+      out->level_info.repeated_ancestor_def_level = repeated_ancestor_def_level;
       return Status::OK();
     } else {
+      current_levels.Increment(node);
       // A normal (required/optional) primitive node
       return PopulateLeaf(column_index,
                           ::arrow::field(node.name(), type, node.is_optional(),
                                          FieldIdMetadata(node.field_id())),
-                          current_def_level, current_rep_level, ctx, parent, out);
+                          current_levels, ctx, parent, out);
     }
   }
 }
@@ -675,49 +770,146 @@ Status GetOriginSchema(const std::shared_ptr<const KeyValueMetadata>& metadata,
 // Restore original Arrow field information that was serialized as Parquet metadata
 // but that is not necessarily present in the field reconstitued from Parquet data
 // (for example, Parquet timestamp types doesn't carry timezone information).
-Status ApplyOriginalMetadata(std::shared_ptr<Field> field, const Field& origin_field,
-                             std::shared_ptr<Field>* out) {
+
+Result<bool> ApplyOriginalMetadata(const Field& origin_field, SchemaField* inferred);
+
+std::function<std::shared_ptr<::arrow::DataType>(FieldVector)> GetNestedFactory(
+    const ArrowType& origin_type, const ArrowType& inferred_type) {
+  switch (inferred_type.id()) {
+    case ::arrow::Type::STRUCT:
+      if (origin_type.id() == ::arrow::Type::STRUCT) {
+        return ::arrow::struct_;
+      }
+      break;
+    case ::arrow::Type::LIST:
+      if (origin_type.id() == ::arrow::Type::LIST) {
+        return [](FieldVector fields) {
+          DCHECK_EQ(fields.size(), 1);
+          return ::arrow::list(std::move(fields[0]));
+        };
+      }
+      if (origin_type.id() == ::arrow::Type::LARGE_LIST) {
+        return [](FieldVector fields) {
+          DCHECK_EQ(fields.size(), 1);
+          return ::arrow::large_list(std::move(fields[0]));
+        };
+      }
+      if (origin_type.id() == ::arrow::Type::FIXED_SIZE_LIST) {
+        const auto list_size =
+            checked_cast<const ::arrow::FixedSizeListType&>(origin_type).list_size();
+        return [list_size](FieldVector fields) {
+          DCHECK_EQ(fields.size(), 1);
+          return ::arrow::fixed_size_list(std::move(fields[0]), list_size);
+        };
+      }
+      break;
+    default:
+      break;
+  }
+  return {};
+}
+
+Result<bool> ApplyOriginalStorageMetadata(const Field& origin_field,
+                                          SchemaField* inferred) {
+  bool modified = false;
+
   auto origin_type = origin_field.type();
-  if (field->type()->id() == ::arrow::Type::TIMESTAMP) {
+  auto inferred_type = inferred->field->type();
+
+  const int num_children = inferred_type->num_fields();
+
+  if (num_children > 0 && origin_type->num_fields() == num_children) {
+    DCHECK_EQ(static_cast<int>(inferred->children.size()), num_children);
+    const auto factory = GetNestedFactory(*origin_type, *inferred_type);
+    if (factory) {
+      // The type may be modified (e.g. LargeList) while the children stay the same
+      modified |= origin_type->id() != inferred_type->id();
+
+      // Apply original metadata recursively to children
+      for (int i = 0; i < inferred_type->num_fields(); ++i) {
+        ARROW_ASSIGN_OR_RAISE(
+            const bool child_modified,
+            ApplyOriginalMetadata(*origin_type->field(i), &inferred->children[i]));
+        modified |= child_modified;
+      }
+      if (modified) {
+        // Recreate this field using the modified child fields
+        ::arrow::FieldVector modified_children(inferred_type->num_fields());
+        for (int i = 0; i < inferred_type->num_fields(); ++i) {
+          modified_children[i] = inferred->children[i].field;
+        }
+        inferred->field =
+            inferred->field->WithType(factory(std::move(modified_children)));
+      }
+    }
+  }
+
+  if (origin_type->id() == ::arrow::Type::TIMESTAMP &&
+      inferred_type->id() == ::arrow::Type::TIMESTAMP) {
     // Restore time zone, if any
-    const auto& ts_type = static_cast<const ::arrow::TimestampType&>(*field->type());
-    const auto& ts_origin_type = static_cast<const ::arrow::TimestampType&>(*origin_type);
+    const auto& ts_type = checked_cast<const ::arrow::TimestampType&>(*inferred_type);
+    const auto& ts_origin_type =
+        checked_cast<const ::arrow::TimestampType&>(*origin_type);
 
     // If the unit is the same and the data is tz-aware, then set the original
     // time zone, since Parquet has no native storage for timezones
     if (ts_type.unit() == ts_origin_type.unit() && ts_type.timezone() == "UTC" &&
         ts_origin_type.timezone() != "") {
-      field = field->WithType(origin_type);
+      inferred->field = inferred->field->WithType(origin_type);
     }
+    modified = true;
   }
+
   if (origin_type->id() == ::arrow::Type::DICTIONARY &&
-      field->type()->id() != ::arrow::Type::DICTIONARY &&
-      IsDictionaryReadSupported(*field->type())) {
+      inferred_type->id() != ::arrow::Type::DICTIONARY &&
+      IsDictionaryReadSupported(*inferred_type)) {
+    // Direct dictionary reads are only suppored for a couple primitive types,
+    // so no need to recurse on value types.
     const auto& dict_origin_type =
-        static_cast<const ::arrow::DictionaryType&>(*origin_type);
-    field = field->WithType(
-        ::arrow::dictionary(::arrow::int32(), field->type(), dict_origin_type.ordered()));
-  }
-
-  if (origin_type->id() == ::arrow::Type::EXTENSION) {
-    // Restore extension type, if the storage type is as read from Parquet
-    const auto& ex_type = checked_cast<const ::arrow::ExtensionType&>(*origin_type);
-    if (ex_type.storage_type()->Equals(*field->type())) {
-      field = field->WithType(origin_type);
-    }
+        checked_cast<const ::arrow::DictionaryType&>(*origin_type);
+    inferred->field = inferred->field->WithType(
+        ::arrow::dictionary(::arrow::int32(), inferred_type, dict_origin_type.ordered()));
+    modified = true;
   }
 
   // Restore field metadata
   std::shared_ptr<const KeyValueMetadata> field_metadata = origin_field.metadata();
   if (field_metadata != nullptr) {
-    if (field->metadata()) {
+    if (inferred->field->metadata()) {
       // Prefer the metadata keys (like field_id) from the current metadata
-      field_metadata = field_metadata->Merge(*field->metadata());
+      field_metadata = field_metadata->Merge(*inferred->field->metadata());
     }
-    field = field->WithMetadata(field_metadata);
+    inferred->field = inferred->field->WithMetadata(field_metadata);
+    modified = true;
   }
-  *out = field;
-  return Status::OK();
+
+  return modified;
+}
+
+Result<bool> ApplyOriginalMetadata(const Field& origin_field, SchemaField* inferred) {
+  bool modified = false;
+
+  auto origin_type = origin_field.type();
+  auto inferred_type = inferred->field->type();
+
+  if (origin_type->id() == ::arrow::Type::EXTENSION) {
+    const auto& ex_type = checked_cast<const ::arrow::ExtensionType&>(*origin_type);
+    auto origin_storage_field = origin_field.WithType(ex_type.storage_type());
+
+    // Apply metadata recursively to storage type
+    RETURN_NOT_OK(ApplyOriginalStorageMetadata(*origin_storage_field, inferred));
+
+    // Restore extension type, if the storage type is the same as inferred
+    // from the Parquet type
+    if (ex_type.storage_type()->Equals(*inferred->field->type())) {
+      inferred->field = inferred->field->WithType(origin_type);
+    }
+    modified = true;
+  } else {
+    ARROW_ASSIGN_OR_RAISE(modified, ApplyOriginalStorageMetadata(origin_field, inferred));
+  }
+
+  return modified;
 }
 
 }  // namespace
@@ -790,9 +982,6 @@ Status SchemaManifest::Make(const SchemaDescriptor* schema,
                             const std::shared_ptr<const KeyValueMetadata>& metadata,
                             const ArrowReaderProperties& properties,
                             SchemaManifest* manifest) {
-  RETURN_NOT_OK(
-      GetOriginSchema(metadata, &manifest->schema_metadata, &manifest->origin_schema));
-
   SchemaTreeContext ctx;
   ctx.manifest = manifest;
   ctx.properties = properties;
@@ -800,9 +989,19 @@ Status SchemaManifest::Make(const SchemaDescriptor* schema,
   const GroupNode& schema_node = *schema->group_node();
   manifest->descr = schema;
   manifest->schema_fields.resize(schema_node.field_count());
+
+  // Try to deserialize original Arrow schema
+  RETURN_NOT_OK(
+      GetOriginSchema(metadata, &manifest->schema_metadata, &manifest->origin_schema));
+  // Ignore original schema if it's not compatible with the Parquet schema
+  if (manifest->origin_schema != nullptr &&
+      manifest->origin_schema->num_fields() != schema_node.field_count()) {
+    manifest->origin_schema = nullptr;
+  }
+
   for (int i = 0; i < static_cast<int>(schema_node.field_count()); ++i) {
     SchemaField* out_field = &manifest->schema_fields[i];
-    RETURN_NOT_OK(NodeToSchemaField(*schema_node.field(i), 0, 0, &ctx,
+    RETURN_NOT_OK(NodeToSchemaField(*schema_node.field(i), LevelInfo(), &ctx,
                                     /*parent=*/nullptr, out_field));
 
     // TODO(wesm): as follow up to ARROW-3246, we should really pass the origin
@@ -812,9 +1011,9 @@ Status SchemaManifest::Make(const SchemaDescriptor* schema,
     if (manifest->origin_schema == nullptr) {
       continue;
     }
+
     auto origin_field = manifest->origin_schema->field(i);
-    RETURN_NOT_OK(
-        ApplyOriginalMetadata(out_field->field, *origin_field, &out_field->field));
+    RETURN_NOT_OK(ApplyOriginalMetadata(*origin_field, out_field));
   }
   return Status::OK();
 }
diff --git a/cpp/src/parquet/arrow/schema.h b/cpp/src/parquet/arrow/schema.h
index c43bebb4ad1..dd60fde4342 100644
--- a/cpp/src/parquet/arrow/schema.h
+++ b/cpp/src/parquet/arrow/schema.h
@@ -28,6 +28,7 @@
 #include "arrow/type.h"
 #include "arrow/type_fwd.h"
 
+#include "parquet/level_conversion.h"
 #include "parquet/platform.h"
 #include "parquet/schema.h"
 
@@ -93,32 +94,7 @@ struct PARQUET_EXPORT SchemaField {
   // Only set for leaf nodes
   int column_index = -1;
 
-  // The definition level at which the value for the field
-  // is considered not null (definition levels greater than
-  // or equal to indicate this value indicate a not-null
-  // value for the field).
-  int16_t definition_level;
-  // The repetition level corresponding to this element
-  // or the closest repeated ancestor.  Any repetition
-  // level less than this indicates either a new list OR
-  // an empty list (which is determined in conjunction
-  // definition_level).
-  int16_t repetition_level;
-
-  bool IsStruct() const { return field->type()->id() == ::arrow::Type::STRUCT; }
-  bool IsRepeated() const {
-    // FixedSizeList will require special handling.
-    return field->type()->id() == ::arrow::Type::LIST ||
-           field->type()->id() == ::arrow::Type::LARGE_LIST ||
-           field->type()->id() == ::arrow::Type::MAP;
-  }
-
-  // The definition level indicating the level at which the closest
-  // repeated ancestor was not empty.  This is used to discrimate
-  // between a value less than |definition_level|
-  // being null or excluded entirely.
-  // TODO(ARROW-8493): Populate this value.
-  int16_t repeated_ancestor_definition_level = 0;
+  parquet::internal::LevelInfo level_info;
 
   bool is_leaf() const { return column_index != -1; }
 };
diff --git a/cpp/src/parquet/arrow/test_util.h b/cpp/src/parquet/arrow/test_util.h
index 04517c7d961..47989191f96 100644
--- a/cpp/src/parquet/arrow/test_util.h
+++ b/cpp/src/parquet/arrow/test_util.h
@@ -24,9 +24,13 @@
 #include <utility>
 #include <vector>
 
-#include "arrow/api.h"
+#include "arrow/array.h"
+#include "arrow/array/builder_binary.h"
+#include "arrow/array/builder_decimal.h"
+#include "arrow/array/builder_primitive.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/testing/random.h"
+#include "arrow/type_fwd.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/decimal.h"
 #include "parquet/column_reader.h"
@@ -253,6 +257,12 @@ ::arrow::enable_if_base_binary<ArrowType, Status> NullableArray(
       RETURN_NOT_OK(builder.AppendNull());
     } else {
       ::arrow::random_bytes(kBufferSize, seed + static_cast<uint32_t>(i), buffer);
+      if (ArrowType::is_utf8) {
+        // Trivially force data to be valid UTF8 by making it all ASCII
+        for (auto& byte : buffer) {
+          byte &= 0x7f;
+        }
+      }
       RETURN_NOT_OK(builder.Append(buffer, kBufferSize));
     }
   }
diff --git a/cpp/src/parquet/arrow/writer.cc b/cpp/src/parquet/arrow/writer.cc
index e151e748586..a1018d2c32c 100644
--- a/cpp/src/parquet/arrow/writer.cc
+++ b/cpp/src/parquet/arrow/writer.cc
@@ -31,9 +31,11 @@
 #include "arrow/table.h"
 #include "arrow/type.h"
 #include "arrow/util/base64.h"
+#include "arrow/util/checked_cast.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/make_unique.h"
 #include "arrow/visitor_inline.h"
+
 #include "parquet/arrow/path_internal.h"
 #include "parquet/arrow/reader_internal.h"
 #include "parquet/arrow/schema.h"
@@ -48,12 +50,12 @@ using arrow::BinaryArray;
 using arrow::BooleanArray;
 using arrow::ChunkedArray;
 using arrow::DataType;
-using arrow::Decimal128Array;
 using arrow::DictionaryArray;
+using arrow::ExtensionArray;
+using arrow::ExtensionType;
 using arrow::Field;
 using arrow::FixedSizeBinaryArray;
 using Int16BufferBuilder = arrow::TypedBufferBuilder<int16_t>;
-using arrow::ExtensionArray;
 using arrow::ListArray;
 using arrow::MemoryPool;
 using arrow::NumericArray;
@@ -63,6 +65,8 @@ using arrow::Status;
 using arrow::Table;
 using arrow::TimeUnit;
 
+using arrow::internal::checked_cast;
+
 using parquet::ParquetFileWriter;
 using parquet::ParquetVersion;
 using parquet::schema::GroupNode;
@@ -72,15 +76,18 @@ namespace arrow {
 
 namespace {
 
-int CalculateLeafCount(const DataType& type) {
-  if (type.num_fields() == 0) {
+int CalculateLeafCount(const DataType* type) {
+  if (type->id() == ::arrow::Type::EXTENSION) {
+    type = checked_cast<const ExtensionType&>(*type).storage_type().get();
+  }
+  if (type->num_fields() == 0) {
     // Primitive type.
     return 1;
   }
 
   int num_leaves = 0;
-  for (std::shared_ptr<Field> field : type.fields()) {
-    num_leaves += CalculateLeafCount(*(field->type()));
+  for (const auto& field : type->fields()) {
+    num_leaves += CalculateLeafCount(field->type().get());
   }
   return num_leaves;
 }
@@ -98,249 +105,6 @@ bool HasNullableRoot(const SchemaManifest& schema_manifest,
   return nullable;
 }
 
-class LevelBuilder {
- public:
-  explicit LevelBuilder(MemoryPool* pool, const SchemaField* schema_field,
-                        const SchemaManifest* schema_manifest)
-      : def_levels_(pool),
-        rep_levels_(pool),
-        schema_field_(schema_field),
-        schema_manifest_(schema_manifest) {}
-
-  Status VisitInline(const Array& array);
-
-  template <typename T>
-  ::arrow::enable_if_t<std::is_base_of<::arrow::FlatArray, T>::value, Status> Visit(
-      const T& array) {
-    array_offsets_.push_back(static_cast<int32_t>(array.offset()));
-    valid_bitmaps_.push_back(array.null_bitmap_data());
-    null_counts_.push_back(array.null_count());
-    values_array_ = std::make_shared<T>(array.data());
-    return Status::OK();
-  }
-
-  Status Visit(const DictionaryArray& array) {
-    // Only currently handle DictionaryArray where the dictionary is a
-    // primitive type
-    if (array.dict_type()->value_type()->num_fields() > 0) {
-      return Status::NotImplemented(
-          "Writing DictionaryArray with nested dictionary "
-          "type not yet supported");
-    }
-    array_offsets_.push_back(static_cast<int32_t>(array.offset()));
-    valid_bitmaps_.push_back(array.null_bitmap_data());
-    null_counts_.push_back(array.null_count());
-    values_array_ = std::make_shared<DictionaryArray>(array.data());
-    return Status::OK();
-  }
-
-  Status Visit(const ListArray& array) {
-    array_offsets_.push_back(static_cast<int32_t>(array.offset()));
-    valid_bitmaps_.push_back(array.null_bitmap_data());
-    null_counts_.push_back(array.null_count());
-    offsets_.push_back(array.raw_value_offsets());
-
-    // Min offset isn't always zero in the case of sliced Arrays.
-    min_offset_idx_ = array.value_offset(min_offset_idx_);
-    max_offset_idx_ = array.value_offset(max_offset_idx_);
-
-    return VisitInline(*array.values());
-  }
-
-  Status Visit(const ExtensionArray& array) { return VisitInline(*array.storage()); }
-
-#define NOT_IMPLEMENTED_VISIT(ArrowTypePrefix)                             \
-  Status Visit(const ::arrow::ArrowTypePrefix##Array& array) {             \
-    return Status::NotImplemented("Level generation for " #ArrowTypePrefix \
-                                  " not supported yet");                   \
-  }
-
-  // See ARROW-1644
-  NOT_IMPLEMENTED_VISIT(LargeList)
-  NOT_IMPLEMENTED_VISIT(Map)
-  NOT_IMPLEMENTED_VISIT(FixedSizeList)
-  NOT_IMPLEMENTED_VISIT(Struct)
-  NOT_IMPLEMENTED_VISIT(Union)
-
-#undef NOT_IMPLEMENTED_VISIT
-
-  Status ExtractNullability() {
-    // Walk upwards to extract nullability
-    const SchemaField* current_field = schema_field_;
-    while (current_field != nullptr) {
-      nullable_.push_front(current_field->field->nullable());
-      if (current_field->field->type()->num_fields() > 1) {
-        return Status::NotImplemented(
-            "Fields with more than one child are not supported.");
-      } else {
-        current_field = schema_manifest_->GetParent(current_field);
-      }
-    }
-    return Status::OK();
-  }
-
-  Status GenerateLevels(const Array& array, int64_t* values_offset, int64_t* num_values,
-                        int64_t* num_levels,
-                        const std::shared_ptr<ResizableBuffer>& def_levels_scratch,
-                        std::shared_ptr<Buffer>* def_levels_out,
-                        std::shared_ptr<Buffer>* rep_levels_out,
-                        std::shared_ptr<Array>* values_array) {
-    // Work downwards to extract bitmaps and offsets
-    min_offset_idx_ = 0;
-    max_offset_idx_ = array.length();
-    RETURN_NOT_OK(VisitInline(array));
-    *num_values = max_offset_idx_ - min_offset_idx_;
-    *values_offset = min_offset_idx_;
-    *values_array = values_array_;
-
-    RETURN_NOT_OK(ExtractNullability());
-
-    // Generate the levels.
-    if (nullable_.size() == 1) {
-      // We have a PrimitiveArray
-      *rep_levels_out = nullptr;
-      if (nullable_[0]) {
-        RETURN_NOT_OK(
-            def_levels_scratch->Resize(array.length() * sizeof(int16_t), false));
-        auto def_levels_ptr =
-            reinterpret_cast<int16_t*>(def_levels_scratch->mutable_data());
-        if (array.null_count() == 0) {
-          std::fill(def_levels_ptr, def_levels_ptr + array.length(), 1);
-        } else if (array.null_count() == array.length()) {
-          std::fill(def_levels_ptr, def_levels_ptr + array.length(), 0);
-        } else {
-          ::arrow::internal::BitmapReader valid_bits_reader(
-              array.null_bitmap_data(), array.offset(), array.length());
-          for (int i = 0; i < array.length(); i++) {
-            def_levels_ptr[i] = valid_bits_reader.IsSet() ? 1 : 0;
-            valid_bits_reader.Next();
-          }
-        }
-
-        *def_levels_out = def_levels_scratch;
-      } else {
-        *def_levels_out = nullptr;
-      }
-      *num_levels = array.length();
-    } else {
-      // Note it is hard to estimate memory consumption due to zero length
-      // arrays otherwise we would preallocate.  An upper boun on memory
-      // is the sum of the length of each list array  + number of elements
-      // but this might be too loose of an upper bound so we choose to use
-      // safe methods.
-      RETURN_NOT_OK(rep_levels_.Append(0));
-      RETURN_NOT_OK(HandleListEntries(0, 0, 0, array.length()));
-
-      RETURN_NOT_OK(def_levels_.Finish(def_levels_out));
-      RETURN_NOT_OK(rep_levels_.Finish(rep_levels_out));
-      *num_levels = (*rep_levels_out)->size() / sizeof(int16_t);
-    }
-
-    return Status::OK();
-  }
-
-  Status HandleList(int16_t def_level, int16_t rep_level, int64_t index) {
-    if (nullable_[rep_level]) {
-      if (null_counts_[rep_level] == 0 ||
-          BitUtil::GetBit(valid_bitmaps_[rep_level], index + array_offsets_[rep_level])) {
-        return HandleNonNullList(static_cast<int16_t>(def_level + 1), rep_level, index);
-      } else {
-        return def_levels_.Append(def_level);
-      }
-    } else {
-      return HandleNonNullList(def_level, rep_level, index);
-    }
-  }
-
-  Status HandleNonNullList(int16_t def_level, int16_t rep_level, int64_t index) {
-    const int32_t inner_offset = offsets_[rep_level][index];
-    const int32_t inner_length = offsets_[rep_level][index + 1] - inner_offset;
-    const int64_t recursion_level = rep_level + 1;
-    if (inner_length == 0) {
-      return def_levels_.Append(def_level);
-    }
-    if (recursion_level < static_cast<int64_t>(offsets_.size())) {
-      return HandleListEntries(static_cast<int16_t>(def_level + 1),
-                               static_cast<int16_t>(rep_level + 1), inner_offset,
-                               inner_length);
-    }
-    // We have reached the leaf: primitive list, handle remaining nullables
-    const bool nullable_level = nullable_[recursion_level];
-    const int64_t level_null_count = null_counts_[recursion_level];
-    const uint8_t* level_valid_bitmap = valid_bitmaps_[recursion_level];
-
-    if (inner_length >= 1) {
-      RETURN_NOT_OK(
-          rep_levels_.Append(inner_length - 1, static_cast<int16_t>(rep_level + 1)));
-    }
-
-    // Special case: this is a null array (all elements are null)
-    if (level_null_count && level_valid_bitmap == nullptr) {
-      return def_levels_.Append(inner_length, static_cast<int16_t>(def_level + 1));
-    }
-    for (int64_t i = 0; i < inner_length; i++) {
-      if (nullable_level &&
-          ((level_null_count == 0) ||
-           BitUtil::GetBit(level_valid_bitmap,
-                           inner_offset + i + array_offsets_[recursion_level]))) {
-        // Non-null element in a null level
-        RETURN_NOT_OK(def_levels_.Append(static_cast<int16_t>(def_level + 2)));
-      } else {
-        // This can be produced in two cases:
-        //  * elements are nullable and this one is null
-        //   (i.e. max_def_level = def_level + 2)
-        //  * elements are non-nullable (i.e. max_def_level = def_level + 1)
-        RETURN_NOT_OK(def_levels_.Append(static_cast<int16_t>(def_level + 1)));
-      }
-    }
-    return Status::OK();
-  }
-
-  Status HandleListEntries(int16_t def_level, int16_t rep_level, int64_t offset,
-                           int64_t length) {
-    for (int64_t i = 0; i < length; i++) {
-      if (i > 0) {
-        RETURN_NOT_OK(rep_levels_.Append(rep_level));
-      }
-      RETURN_NOT_OK(HandleList(def_level, rep_level, offset + i));
-    }
-    return Status::OK();
-  }
-
- private:
-  Int16BufferBuilder def_levels_;
-  Int16BufferBuilder rep_levels_;
-
-  const SchemaField* schema_field_;
-  const SchemaManifest* schema_manifest_;
-
-  std::vector<int64_t> null_counts_;
-  std::vector<const uint8_t*> valid_bitmaps_;
-  std::vector<const int32_t*> offsets_;
-  std::vector<int32_t> array_offsets_;
-  std::deque<bool> nullable_;
-
-  int64_t min_offset_idx_;
-  int64_t max_offset_idx_;
-  std::shared_ptr<Array> values_array_;
-};
-
-Status LevelBuilder::VisitInline(const Array& array) {
-  return VisitArrayInline(array, this);
-}
-
-Status GetLeafType(const ::arrow::DataType& type, ::arrow::Type::type* leaf_type) {
-  if (type.id() == ::arrow::Type::LIST || type.id() == ::arrow::Type::STRUCT) {
-    if (type.num_fields() != 1) {
-      return Status::Invalid("Nested column branch had multiple children: ", type);
-    }
-    return GetLeafType(*type.field(0)->type(), leaf_type);
-  } else {
-    *leaf_type = type.id();
-    return Status::OK();
-  }
-}
-
 // Manages writing nested parquet columns with support for all nested types
 // supported by parquet.
 class ArrowColumnWriterV2 {
@@ -379,13 +143,12 @@ class ArrowColumnWriterV2 {
 
               return column_writer->WriteArrow(result.def_levels, result.rep_levels,
                                                result.def_rep_level_count, *values_array,
-                                               ctx);
+                                               ctx, result.leaf_is_nullable);
             }));
       }
 
       PARQUET_CATCH_NOT_OK(column_writer->Close());
     }
-
     return Status::OK();
   }
 
@@ -406,7 +169,7 @@ class ArrowColumnWriterV2 {
     if (data.length() == 0) {
       return ::arrow::internal::make_unique<ArrowColumnWriterV2>(
           std::vector<std::unique_ptr<MultipathLevelBuilder>>{},
-          CalculateLeafCount(*data.type()), row_group_writer);
+          CalculateLeafCount(data.type().get()), row_group_writer);
     }
     while (chunk_index < data.num_chunks() && absolute_position < offset) {
       const int64_t chunk_length = data.chunk(chunk_index)->length();
@@ -427,7 +190,7 @@ class ArrowColumnWriterV2 {
 
     int64_t values_written = 0;
     std::vector<std::unique_ptr<MultipathLevelBuilder>> builders;
-    const int leaf_count = CalculateLeafCount(*data.type());
+    const int leaf_count = CalculateLeafCount(data.type().get());
     bool is_nullable = false;
     // The row_group_writer hasn't been advanced yet so add 1 to the current
     // which is the one this instance will start writing for.
@@ -488,104 +251,6 @@ class ArrowColumnWriterV2 {
   RowGroupWriter* row_group_writer_;
 };
 
-class ArrowColumnWriter {
- public:
-  ArrowColumnWriter(ArrowWriteContext* ctx, ColumnWriter* column_writer,
-                    const SchemaField* schema_field,
-                    const SchemaManifest* schema_manifest)
-      : ctx_(ctx),
-        writer_(column_writer),
-        schema_field_(schema_field),
-        schema_manifest_(schema_manifest) {}
-
-  Status Write(const Array& data) {
-    if (data.length() == 0) {
-      // Write nothing when length is 0
-      return Status::OK();
-    }
-
-    ::arrow::Type::type values_type;
-    RETURN_NOT_OK(GetLeafType(*data.type(), &values_type));
-
-    std::shared_ptr<Array> _values_array;
-    int64_t values_offset = 0;
-    int64_t num_levels = 0;
-    int64_t num_values = 0;
-    LevelBuilder level_builder(ctx_->memory_pool, schema_field_, schema_manifest_);
-    std::shared_ptr<Buffer> def_levels_buffer, rep_levels_buffer;
-    RETURN_NOT_OK(level_builder.GenerateLevels(
-        data, &values_offset, &num_values, &num_levels, ctx_->def_levels_buffer,
-        &def_levels_buffer, &rep_levels_buffer, &_values_array));
-    const int16_t* def_levels = nullptr;
-    if (def_levels_buffer) {
-      def_levels = reinterpret_cast<const int16_t*>(def_levels_buffer->data());
-    }
-    const int16_t* rep_levels = nullptr;
-    if (rep_levels_buffer) {
-      rep_levels = reinterpret_cast<const int16_t*>(rep_levels_buffer->data());
-    }
-    std::shared_ptr<Array> values_array = _values_array->Slice(values_offset, num_values);
-    return writer_->WriteArrow(def_levels, rep_levels, num_levels, *values_array, ctx_);
-  }
-
-  Status Write(const ChunkedArray& data, int64_t offset, const int64_t size) {
-    if (data.length() == 0) {
-      return Status::OK();
-    }
-
-    int64_t absolute_position = 0;
-    int chunk_index = 0;
-    int64_t chunk_offset = 0;
-    while (chunk_index < data.num_chunks() && absolute_position < offset) {
-      const int64_t chunk_length = data.chunk(chunk_index)->length();
-      if (absolute_position + chunk_length > offset) {
-        // Relative offset into the chunk to reach the desired start offset for
-        // writing
-        chunk_offset = offset - absolute_position;
-        break;
-      } else {
-        ++chunk_index;
-        absolute_position += chunk_length;
-      }
-    }
-
-    if (absolute_position >= data.length()) {
-      return Status::Invalid("Cannot write data at offset past end of chunked array");
-    }
-
-    int64_t values_written = 0;
-    while (values_written < size) {
-      const Array& chunk = *data.chunk(chunk_index);
-      const int64_t available_values = chunk.length() - chunk_offset;
-      const int64_t chunk_write_size = std::min(size - values_written, available_values);
-
-      // The chunk offset here will be 0 except for possibly the first chunk
-      // because of the advancing logic above
-      std::shared_ptr<Array> array_to_write = chunk.Slice(chunk_offset, chunk_write_size);
-      RETURN_NOT_OK(Write(*array_to_write));
-
-      if (chunk_write_size == available_values) {
-        chunk_offset = 0;
-        ++chunk_index;
-      }
-      values_written += chunk_write_size;
-    }
-
-    return Status::OK();
-  }
-
-  Status Close() {
-    PARQUET_CATCH_NOT_OK(writer_->Close());
-    return Status::OK();
-  }
-
- private:
-  ArrowWriteContext* ctx_;
-  ColumnWriter* writer_;
-  const SchemaField* schema_field_;
-  const SchemaManifest* schema_manifest_;
-};
-
 }  // namespace
 
 // ----------------------------------------------------------------------
@@ -637,19 +302,8 @@ class FileWriterImpl : public FileWriter {
 
   Status WriteColumnChunk(const std::shared_ptr<ChunkedArray>& data, int64_t offset,
                           int64_t size) override {
-    if (arrow_properties_->engine_version() == ArrowWriterProperties::V1) {
-      ColumnWriter* column_writer;
-      PARQUET_CATCH_NOT_OK(column_writer = row_group_writer_->NextColumn());
-
-      const SchemaField* schema_field = nullptr;
-      RETURN_NOT_OK(schema_manifest_.GetColumnField(row_group_writer_->current_column(),
-                                                    &schema_field));
-
-      ArrowColumnWriter arrow_writer(&column_write_context_, column_writer, schema_field,
-                                     &schema_manifest_);
-      RETURN_NOT_OK(arrow_writer.Write(*data, offset, size));
-      return arrow_writer.Close();
-    } else if (arrow_properties_->engine_version() == ArrowWriterProperties::V2) {
+    if (arrow_properties_->engine_version() == ArrowWriterProperties::V2 ||
+        arrow_properties_->engine_version() == ArrowWriterProperties::V1) {
       ARROW_ASSIGN_OR_RAISE(
           std::unique_ptr<ArrowColumnWriterV2> writer,
           ArrowColumnWriterV2::Make(*data, offset, size, schema_manifest_,
@@ -763,9 +417,8 @@ Status GetSchemaMetadata(const ::arrow::Schema& schema, ::arrow::MemoryPool* poo
     result = ::arrow::key_value_metadata({}, {});
   }
 
-  ::arrow::ipc::DictionaryMemo dict_memo;
   ARROW_ASSIGN_OR_RAISE(std::shared_ptr<Buffer> serialized,
-                        ::arrow::ipc::SerializeSchema(schema, &dict_memo, pool));
+                        ::arrow::ipc::SerializeSchema(schema, pool));
 
   // The serialized schema is not UTF-8, which is required for Thrift
   std::string schema_as_string = serialized->ToString();
diff --git a/cpp/src/parquet/arrow/writer.h b/cpp/src/parquet/arrow/writer.h
index dbd3d8ffb3f..60e373c664c 100644
--- a/cpp/src/parquet/arrow/writer.h
+++ b/cpp/src/parquet/arrow/writer.h
@@ -94,8 +94,6 @@ ::arrow::Status WriteMetaDataFile(const FileMetaData& file_metadata,
                                   ::arrow::io::OutputStream* sink);
 
 /// \brief Write a Table to Parquet.
-///
-/// The table shall only consist of columns of primitive type or of primitive lists.
 ::arrow::Status PARQUET_EXPORT
 WriteTable(const ::arrow::Table& table, MemoryPool* pool,
            std::shared_ptr<::arrow::io::OutputStream> sink, int64_t chunk_size,
diff --git a/cpp/src/parquet/column_reader.cc b/cpp/src/parquet/column_reader.cc
index 672b6e3708c..da3bc8d54f5 100644
--- a/cpp/src/parquet/column_reader.cc
+++ b/cpp/src/parquet/column_reader.cc
@@ -29,7 +29,9 @@
 #include <vector>
 
 #include "arrow/array.h"
-#include "arrow/builder.h"
+#include "arrow/array/builder_binary.h"
+#include "arrow/array/builder_dict.h"
+#include "arrow/array/builder_primitive.h"
 #include "arrow/chunked_array.h"
 #include "arrow/type.h"
 #include "arrow/util/bit_stream_utils.h"
@@ -42,6 +44,7 @@
 #include "parquet/encoding.h"
 #include "parquet/encryption_internal.h"
 #include "parquet/internal_file_decryptor.h"
+#include "parquet/level_comparison.h"
 #include "parquet/level_conversion.h"
 #include "parquet/properties.h"
 #include "parquet/statistics.h"
@@ -55,6 +58,25 @@ using arrow::internal::checked_cast;
 using arrow::internal::MultiplyWithOverflow;
 
 namespace parquet {
+namespace {
+inline bool HasSpacedValues(const ColumnDescriptor* descr) {
+  if (descr->max_repetition_level() > 0) {
+    // repeated+flat case
+    return !descr->schema_node()->is_required();
+  } else {
+    // non-repeated+nested case
+    // Find if a node forces nulls in the lowest level along the hierarchy
+    const schema::Node* node = descr->schema_node().get();
+    while (node) {
+      if (node->is_optional()) {
+        return true;
+      }
+      node = node->parent();
+    }
+    return false;
+  }
+}
+}  // namespace
 
 LevelDecoder::LevelDecoder() : num_values_remaining_(0) {}
 
@@ -63,6 +85,7 @@ LevelDecoder::~LevelDecoder() {}
 int LevelDecoder::SetData(Encoding::type encoding, int16_t max_level,
                           int num_buffered_values, const uint8_t* data,
                           int32_t data_size) {
+  max_level_ = max_level;
   int32_t num_bytes = 0;
   encoding_ = encoding;
   num_values_remaining_ = num_buffered_values;
@@ -110,6 +133,7 @@ int LevelDecoder::SetData(Encoding::type encoding, int16_t max_level,
 
 void LevelDecoder::SetDataV2(int32_t num_bytes, int16_t max_level,
                              int num_buffered_values, const uint8_t* data) {
+  max_level_ = max_level;
   // Repetition and definition levels always uses RLE encoding
   // in the DataPageV2 format.
   if (num_bytes < 0) {
@@ -135,6 +159,15 @@ int LevelDecoder::Decode(int batch_size, int16_t* levels) {
   } else {
     num_decoded = bit_packed_decoder_->GetBatch(bit_width_, levels, num_values);
   }
+  if (num_decoded > 0) {
+    internal::MinMax min_max = internal::FindMinMax(levels, num_decoded);
+    if (ARROW_PREDICT_FALSE(min_max.min < 0 || min_max.max > max_level_)) {
+      std::stringstream ss;
+      ss << "Malformed levels. min: " << min_max.min << " max: " << min_max.max
+         << " out of range.  Max Level: " << max_level_;
+      throw ParquetException(ss.str());
+    }
+  }
   num_values_remaining_ -= num_decoded;
   return num_decoded;
 }
@@ -190,7 +223,7 @@ class SerializedPageReader : public PageReader {
       InitDecryption();
     }
     max_page_header_size_ = kDefaultMaxPageHeaderSize;
-    decompressor_ = internal::GetReadCodec(codec);
+    decompressor_ = GetCodec(codec);
   }
 
   // Implement the PageReader interface
@@ -204,8 +237,9 @@ class SerializedPageReader : public PageReader {
 
   void InitDecryption();
 
-  std::shared_ptr<Buffer> DecompressPage(int compressed_len, int uncompressed_len,
-                                         const uint8_t* page_buffer);
+  std::shared_ptr<Buffer> DecompressIfNeeded(std::shared_ptr<Buffer> page_buffer,
+                                             int compressed_len, int uncompressed_len,
+                                             int levels_byte_len = 0);
 
   std::shared_ptr<ArrowInputStream> stream_;
 
@@ -321,10 +355,15 @@ std::shared_ptr<Page> SerializedPageReader::NextPage() {
 
     int compressed_len = current_page_header_.compressed_page_size;
     int uncompressed_len = current_page_header_.uncompressed_page_size;
+    if (compressed_len < 0 || uncompressed_len < 0) {
+      throw ParquetException("Invalid page header");
+    }
+
     if (crypto_ctx_.data_decryptor != nullptr) {
       UpdateDecryption(crypto_ctx_.data_decryptor, encryption::kDictionaryPage,
                        data_page_aad_);
     }
+
     // Read the compressed data page.
     PARQUET_ASSIGN_OR_THROW(auto page_buffer, stream_->Read(compressed_len));
     if (page_buffer->size() != compressed_len) {
@@ -343,10 +382,6 @@ std::shared_ptr<Page> SerializedPageReader::NextPage() {
 
       page_buffer = decryption_buffer_;
     }
-    // Uncompress it if we need to
-    if (decompressor_ != nullptr) {
-      page_buffer = DecompressPage(compressed_len, uncompressed_len, page_buffer->data());
-    }
 
     const PageType::type page_type = LoadEnumSafe(&current_page_header_.type);
 
@@ -360,6 +395,10 @@ std::shared_ptr<Page> SerializedPageReader::NextPage() {
         throw ParquetException("Invalid page header (negative number of values)");
       }
 
+      // Uncompress if needed
+      page_buffer =
+          DecompressIfNeeded(std::move(page_buffer), compressed_len, uncompressed_len);
+
       return std::make_shared<DictionaryPage>(page_buffer, dict_header.num_values,
                                               LoadEnumSafe(&dict_header.encoding),
                                               is_sorted);
@@ -373,6 +412,10 @@ std::shared_ptr<Page> SerializedPageReader::NextPage() {
       EncodedStatistics page_statistics = ExtractStatsFromHeader(header);
       seen_num_rows_ += header.num_values;
 
+      // Uncompress if needed
+      page_buffer =
+          DecompressIfNeeded(std::move(page_buffer), compressed_len, uncompressed_len);
+
       return std::make_shared<DataPageV1>(page_buffer, header.num_values,
                                           LoadEnumSafe(&header.encoding),
                                           LoadEnumSafe(&header.definition_level_encoding),
@@ -393,6 +436,15 @@ std::shared_ptr<Page> SerializedPageReader::NextPage() {
       EncodedStatistics page_statistics = ExtractStatsFromHeader(header);
       seen_num_rows_ += header.num_values;
 
+      // Uncompress if needed
+      int levels_byte_len;
+      if (AddWithOverflow(header.definition_levels_byte_length,
+                          header.repetition_levels_byte_length, &levels_byte_len)) {
+        throw ParquetException("Levels size too large (corrupt file?)");
+      }
+      page_buffer = DecompressIfNeeded(std::move(page_buffer), compressed_len,
+                                       uncompressed_len, levels_byte_len);
+
       return std::make_shared<DataPageV2>(
           page_buffer, header.num_values, header.num_nulls, header.num_rows,
           LoadEnumSafe(&header.encoding), header.definition_levels_byte_length,
@@ -407,34 +459,33 @@ std::shared_ptr<Page> SerializedPageReader::NextPage() {
   return std::shared_ptr<Page>(nullptr);
 }
 
-std::shared_ptr<Buffer> SerializedPageReader::DecompressPage(int compressed_len,
-                                                             int uncompressed_len,
-                                                             const uint8_t* page_buffer) {
+std::shared_ptr<Buffer> SerializedPageReader::DecompressIfNeeded(
+    std::shared_ptr<Buffer> page_buffer, int compressed_len, int uncompressed_len,
+    int levels_byte_len) {
+  if (decompressor_ == nullptr) {
+    return page_buffer;
+  }
+  if (compressed_len < levels_byte_len || uncompressed_len < levels_byte_len) {
+    throw ParquetException("Invalid page header");
+  }
+
   // Grow the uncompressed buffer if we need to.
   if (uncompressed_len > static_cast<int>(decompression_buffer_->size())) {
     PARQUET_THROW_NOT_OK(decompression_buffer_->Resize(uncompressed_len, false));
   }
 
-  if (current_page_header_.type != format::PageType::DATA_PAGE_V2) {
-    PARQUET_THROW_NOT_OK(
-        decompressor_->Decompress(compressed_len, page_buffer, uncompressed_len,
-                                  decompression_buffer_->mutable_data()));
-  } else {
-    // The levels are not compressed in V2 format
-    const auto& header = current_page_header_.data_page_header_v2;
-    int32_t levels_length =
-        header.repetition_levels_byte_length + header.definition_levels_byte_length;
+  if (levels_byte_len > 0) {
+    // First copy the levels as-is
     uint8_t* decompressed = decompression_buffer_->mutable_data();
-    memcpy(decompressed, page_buffer, levels_length);
-    decompressed += levels_length;
-    const uint8_t* compressed_values = page_buffer + levels_length;
-
-    // Decompress the values
-    PARQUET_THROW_NOT_OK(
-        decompressor_->Decompress(compressed_len - levels_length, compressed_values,
-                                  uncompressed_len - levels_length, decompressed));
+    memcpy(decompressed, page_buffer->data(), levels_byte_len);
   }
 
+  // Decompress the values
+  PARQUET_THROW_NOT_OK(decompressor_->Decompress(
+      compressed_len - levels_byte_len, page_buffer->data() + levels_byte_len,
+      uncompressed_len - levels_byte_len,
+      decompression_buffer_->mutable_data() + levels_byte_len));
+
   return decompression_buffer_;
 }
 
@@ -880,8 +931,7 @@ int64_t TypedColumnReaderImpl<DType>::ReadBatchSpaced(
       }
     }
 
-    const bool has_spaced_values = internal::HasSpacedValues(this->descr_);
-
+    const bool has_spaced_values = HasSpacedValues(this->descr_);
     int64_t null_count = 0;
     if (!has_spaced_values) {
       int values_to_read = 0;
@@ -896,9 +946,21 @@ int64_t TypedColumnReaderImpl<DType>::ReadBatchSpaced(
                                   /*bits_are_set=*/true);
       *values_read = total_values;
     } else {
-      internal::DefinitionLevelsToBitmap(def_levels, num_def_levels, this->max_def_level_,
-                                         this->max_rep_level_, values_read, &null_count,
-                                         valid_bits, valid_bits_offset);
+      internal::LevelInfo info;
+      info.repeated_ancestor_def_level = this->max_def_level_ - 1;
+      info.def_level = this->max_def_level_;
+      info.rep_level = this->max_rep_level_;
+      internal::ValidityBitmapInputOutput validity_io;
+      validity_io.values_read_upper_bound = num_def_levels;
+      validity_io.valid_bits = valid_bits;
+      validity_io.valid_bits_offset = valid_bits_offset;
+      validity_io.null_count = null_count;
+      validity_io.values_read = *values_read;
+
+      internal::DefLevelsToBitmap(def_levels, num_def_levels, info, &validity_io);
+      null_count = validity_io.null_count;
+      *values_read = validity_io.values_read;
+
       total_values =
           this->ReadValuesSpaced(*values_read, values, static_cast<int>(null_count),
                                  valid_bits, valid_bits_offset);
@@ -1008,8 +1070,10 @@ class TypedRecordReader : public ColumnReaderImplBase<DType>,
  public:
   using T = typename DType::c_type;
   using BASE = ColumnReaderImplBase<DType>;
-  TypedRecordReader(const ColumnDescriptor* descr, MemoryPool* pool) : BASE(descr, pool) {
-    nullable_values_ = internal::HasSpacedValues(descr);
+  TypedRecordReader(const ColumnDescriptor* descr, LevelInfo leaf_info, MemoryPool* pool)
+      : BASE(descr, pool) {
+    leaf_info_ = leaf_info;
+    nullable_values_ = leaf_info.HasNullableValues();
     at_record_start_ = true;
     records_read_ = 0;
     values_written_ = 0;
@@ -1128,7 +1192,7 @@ class TypedRecordReader : public ColumnReaderImplBase<DType>,
   }
 
   std::shared_ptr<ResizableBuffer> ReleaseIsValid() override {
-    if (nullable_values_) {
+    if (leaf_info_.HasNullableValues()) {
       auto result = valid_bits_;
       PARQUET_THROW_NOT_OK(result->Resize(BitUtil::BytesForBits(values_written_), true));
       valid_bits_ = AllocateBuffer(this->pool_);
@@ -1170,13 +1234,7 @@ class TypedRecordReader : public ColumnReaderImplBase<DType>,
             break;
           }
         }
-      } else if (ARROW_PREDICT_FALSE(rep_level > this->max_rep_level_)) {
-        std::stringstream ss;
-        ss << "Malformed repetition levels, " << rep_level << " exceeded maximum "
-           << this->max_rep_level_ << " indicated by schema";
-        throw ParquetException(ss.str());
       }
-
       // We have decided to consume the level at this position; therefore we
       // must advance until we find another record boundary
       at_record_start_ = false;
@@ -1184,11 +1242,6 @@ class TypedRecordReader : public ColumnReaderImplBase<DType>,
       const int16_t def_level = *def_levels++;
       if (def_level == this->max_def_level_) {
         ++values_to_read;
-      } else if (ARROW_PREDICT_FALSE(def_level > this->max_def_level_)) {
-        std::stringstream ss;
-        ss << "Malformed definition levels, " << def_level << " exceeded maximum "
-           << this->max_def_level_ << " indicated by schema";
-        throw ParquetException(ss.str());
       }
       ++levels_position_;
     }
@@ -1249,7 +1302,7 @@ class TypedRecordReader : public ColumnReaderImplBase<DType>,
       }
       values_capacity_ = new_values_capacity;
     }
-    if (nullable_values_) {
+    if (leaf_info_.HasNullableValues()) {
       int64_t valid_bytes_new = BitUtil::BytesForBits(values_capacity_);
       if (valid_bits_->size() < valid_bytes_new) {
         int64_t valid_bytes_old = BitUtil::BytesForBits(values_written_);
@@ -1344,20 +1397,24 @@ class TypedRecordReader : public ColumnReaderImplBase<DType>,
     }
 
     int64_t null_count = 0;
-    if (nullable_values_) {
-      int64_t values_with_nulls = 0;
-      DefinitionLevelsToBitmap(
-          def_levels() + start_levels_position, levels_position_ - start_levels_position,
-          this->max_def_level_, this->max_rep_level_, &values_with_nulls, &null_count,
-          valid_bits_->mutable_data(), values_written_);
-      values_to_read = values_with_nulls - null_count;
+    if (leaf_info_.HasNullableValues()) {
+      ValidityBitmapInputOutput validity_io;
+      validity_io.values_read_upper_bound = levels_position_ - start_levels_position;
+      validity_io.valid_bits = valid_bits_->mutable_data();
+      validity_io.valid_bits_offset = values_written_;
+
+      DefLevelsToBitmap(def_levels() + start_levels_position,
+                        levels_position_ - start_levels_position, leaf_info_,
+                        &validity_io);
+      values_to_read = validity_io.values_read - validity_io.null_count;
+      null_count = validity_io.null_count;
       DCHECK_GE(values_to_read, 0);
-      ReadValuesSpaced(values_with_nulls, null_count);
+      ReadValuesSpaced(validity_io.values_read, null_count);
     } else {
       DCHECK_GE(values_to_read, 0);
       ReadValuesDense(values_to_read);
     }
-    if (this->max_def_level_ > 0) {
+    if (this->leaf_info_.def_level > 0) {
       // Optional, repeated, or some mix thereof
       this->ConsumeBufferedValues(levels_position_ - start_levels_position);
     } else {
@@ -1415,13 +1472,15 @@ class TypedRecordReader : public ColumnReaderImplBase<DType>,
   T* ValuesHead() {
     return reinterpret_cast<T*>(values_->mutable_data()) + values_written_;
   }
+  LevelInfo leaf_info_;
 };
 
 class FLBARecordReader : public TypedRecordReader<FLBAType>,
                          virtual public BinaryRecordReader {
  public:
-  FLBARecordReader(const ColumnDescriptor* descr, ::arrow::MemoryPool* pool)
-      : TypedRecordReader<FLBAType>(descr, pool), builder_(nullptr) {
+  FLBARecordReader(const ColumnDescriptor* descr, LevelInfo leaf_info,
+                   ::arrow::MemoryPool* pool)
+      : TypedRecordReader<FLBAType>(descr, leaf_info, pool), builder_(nullptr) {
     DCHECK_EQ(descr_->physical_type(), Type::FIXED_LEN_BYTE_ARRAY);
     int byte_width = descr_->type_length();
     std::shared_ptr<::arrow::DataType> type = ::arrow::fixed_size_binary(byte_width);
@@ -1473,8 +1532,9 @@ class FLBARecordReader : public TypedRecordReader<FLBAType>,
 class ByteArrayChunkedRecordReader : public TypedRecordReader<ByteArrayType>,
                                      virtual public BinaryRecordReader {
  public:
-  ByteArrayChunkedRecordReader(const ColumnDescriptor* descr, ::arrow::MemoryPool* pool)
-      : TypedRecordReader<ByteArrayType>(descr, pool) {
+  ByteArrayChunkedRecordReader(const ColumnDescriptor* descr, LevelInfo leaf_info,
+                               ::arrow::MemoryPool* pool)
+      : TypedRecordReader<ByteArrayType>(descr, leaf_info, pool) {
     DCHECK_EQ(descr_->physical_type(), Type::BYTE_ARRAY);
     accumulator_.builder.reset(new ::arrow::BinaryBuilder(pool));
   }
@@ -1513,9 +1573,9 @@ class ByteArrayChunkedRecordReader : public TypedRecordReader<ByteArrayType>,
 class ByteArrayDictionaryRecordReader : public TypedRecordReader<ByteArrayType>,
                                         virtual public DictionaryRecordReader {
  public:
-  ByteArrayDictionaryRecordReader(const ColumnDescriptor* descr,
+  ByteArrayDictionaryRecordReader(const ColumnDescriptor* descr, LevelInfo leaf_info,
                                   ::arrow::MemoryPool* pool)
-      : TypedRecordReader<ByteArrayType>(descr, pool), builder_(pool) {
+      : TypedRecordReader<ByteArrayType>(descr, leaf_info, pool), builder_(pool) {
     this->read_dictionary_ = true;
   }
 
@@ -1602,35 +1662,36 @@ template <>
 void TypedRecordReader<FLBAType>::DebugPrintState() {}
 
 std::shared_ptr<RecordReader> MakeByteArrayRecordReader(const ColumnDescriptor* descr,
+                                                        LevelInfo leaf_info,
                                                         ::arrow::MemoryPool* pool,
                                                         bool read_dictionary) {
   if (read_dictionary) {
-    return std::make_shared<ByteArrayDictionaryRecordReader>(descr, pool);
+    return std::make_shared<ByteArrayDictionaryRecordReader>(descr, leaf_info, pool);
   } else {
-    return std::make_shared<ByteArrayChunkedRecordReader>(descr, pool);
+    return std::make_shared<ByteArrayChunkedRecordReader>(descr, leaf_info, pool);
   }
 }
 
 std::shared_ptr<RecordReader> RecordReader::Make(const ColumnDescriptor* descr,
-                                                 MemoryPool* pool,
+                                                 LevelInfo leaf_info, MemoryPool* pool,
                                                  const bool read_dictionary) {
   switch (descr->physical_type()) {
     case Type::BOOLEAN:
-      return std::make_shared<TypedRecordReader<BooleanType>>(descr, pool);
+      return std::make_shared<TypedRecordReader<BooleanType>>(descr, leaf_info, pool);
     case Type::INT32:
-      return std::make_shared<TypedRecordReader<Int32Type>>(descr, pool);
+      return std::make_shared<TypedRecordReader<Int32Type>>(descr, leaf_info, pool);
     case Type::INT64:
-      return std::make_shared<TypedRecordReader<Int64Type>>(descr, pool);
+      return std::make_shared<TypedRecordReader<Int64Type>>(descr, leaf_info, pool);
     case Type::INT96:
-      return std::make_shared<TypedRecordReader<Int96Type>>(descr, pool);
+      return std::make_shared<TypedRecordReader<Int96Type>>(descr, leaf_info, pool);
     case Type::FLOAT:
-      return std::make_shared<TypedRecordReader<FloatType>>(descr, pool);
+      return std::make_shared<TypedRecordReader<FloatType>>(descr, leaf_info, pool);
     case Type::DOUBLE:
-      return std::make_shared<TypedRecordReader<DoubleType>>(descr, pool);
+      return std::make_shared<TypedRecordReader<DoubleType>>(descr, leaf_info, pool);
     case Type::BYTE_ARRAY:
-      return MakeByteArrayRecordReader(descr, pool, read_dictionary);
+      return MakeByteArrayRecordReader(descr, leaf_info, pool, read_dictionary);
     case Type::FIXED_LEN_BYTE_ARRAY:
-      return std::make_shared<FLBARecordReader>(descr, pool);
+      return std::make_shared<FLBARecordReader>(descr, leaf_info, pool);
     default: {
       // PARQUET-1481: This can occur if the file is corrupt
       std::stringstream ss;
diff --git a/cpp/src/parquet/column_reader.h b/cpp/src/parquet/column_reader.h
index 7b5ee1b722a..60c44ffa6d2 100644
--- a/cpp/src/parquet/column_reader.h
+++ b/cpp/src/parquet/column_reader.h
@@ -23,6 +23,7 @@
 #include <vector>
 
 #include "parquet/exception.h"
+#include "parquet/level_conversion.h"
 #include "parquet/platform.h"
 #include "parquet/schema.h"
 #include "parquet/types.h"
@@ -75,6 +76,7 @@ class PARQUET_EXPORT LevelDecoder {
   Encoding::type encoding_;
   std::unique_ptr<::arrow::util::RleDecoder> rle_decoder_;
   std::unique_ptr<::arrow::BitUtil::BitReader> bit_packed_decoder_;
+  int16_t max_level_;
 };
 
 struct CryptoContext {
@@ -208,7 +210,7 @@ namespace internal {
 class RecordReader {
  public:
   static std::shared_ptr<RecordReader> Make(
-      const ColumnDescriptor* descr,
+      const ColumnDescriptor* descr, LevelInfo leaf_info,
       ::arrow::MemoryPool* pool = ::arrow::default_memory_pool(),
       const bool read_dictionary = false);
 
@@ -314,25 +316,6 @@ class DictionaryRecordReader : virtual public RecordReader {
   virtual std::shared_ptr<::arrow::ChunkedArray> GetResult() = 0;
 };
 
-// TODO(itaiin): another code path split to merge when the general case is done
-static inline bool HasSpacedValues(const ColumnDescriptor* descr) {
-  if (descr->max_repetition_level() > 0) {
-    // repeated+flat case
-    return !descr->schema_node()->is_required();
-  } else {
-    // non-repeated+nested case
-    // Find if a node forces nulls in the lowest level along the hierarchy
-    const schema::Node* node = descr->schema_node().get();
-    while (node) {
-      if (node->is_optional()) {
-        return true;
-      }
-      node = node->parent();
-    }
-    return false;
-  }
-}
-
 }  // namespace internal
 
 using BoolReader = TypedColumnReader<BooleanType>;
diff --git a/cpp/src/parquet/column_writer.cc b/cpp/src/parquet/column_writer.cc
index 6cb0baeb337..ed7058f82eb 100644
--- a/cpp/src/parquet/column_writer.cc
+++ b/cpp/src/parquet/column_writer.cc
@@ -41,6 +41,7 @@
 #include "parquet/encoding.h"
 #include "parquet/encryption_internal.h"
 #include "parquet/internal_file_encryptor.h"
+#include "parquet/level_conversion.h"
 #include "parquet/metadata.h"
 #include "parquet/platform.h"
 #include "parquet/properties.h"
@@ -49,6 +50,8 @@
 #include "parquet/thrift_internal.h"
 #include "parquet/types.h"
 
+using arrow::Array;
+using arrow::ArrayData;
 using arrow::Datum;
 using arrow::Status;
 using arrow::BitUtil::BitWriter;
@@ -59,6 +62,23 @@ namespace parquet {
 
 namespace {
 
+internal::LevelInfo ComputeLevelInfo(const ColumnDescriptor* descr) {
+  internal::LevelInfo level_info;
+  level_info.def_level = descr->max_definition_level();
+  level_info.rep_level = descr->max_repetition_level();
+
+  int16_t min_spaced_def_level = descr->max_definition_level();
+  const ::parquet::schema::Node* node = descr->schema_node().get();
+  while (node != nullptr && !node->is_repeated()) {
+    if (node->is_optional()) {
+      min_spaced_def_level--;
+    }
+    node = node->parent();
+  }
+  level_info.repeated_ancestor_def_level = min_spaced_def_level;
+  return level_info;
+}
+
 inline const int16_t* AddIfNotNull(const int16_t* base, int64_t offset) {
   if (base != nullptr) {
     return base + offset;
@@ -172,7 +192,7 @@ class SerializedPageWriter : public PageWriter {
     if (data_encryptor_ != nullptr || meta_encryptor_ != nullptr) {
       InitEncryption();
     }
-    compressor_ = internal::GetWriteCodec(codec, compression_level);
+    compressor_ = GetCodec(codec, compression_level);
     thrift_serializer_.reset(new ThriftSerializer);
   }
 
@@ -543,6 +563,7 @@ class ColumnWriterImpl {
                    Encoding::type encoding, const WriterProperties* properties)
       : metadata_(metadata),
         descr_(metadata->descr()),
+        level_info_(ComputeLevelInfo(metadata->descr())),
         pager_(std::move(pager)),
         has_dictionary_(use_dictionary),
         encoding_(encoding),
@@ -563,6 +584,7 @@ class ColumnWriterImpl {
         std::static_pointer_cast<ResizableBuffer>(AllocateBuffer(allocator_, 0));
     uncompressed_data_ =
         std::static_pointer_cast<ResizableBuffer>(AllocateBuffer(allocator_, 0));
+
     if (pager_->has_compressor()) {
       compressor_temp_buffer_ =
           std::static_pointer_cast<ResizableBuffer>(AllocateBuffer(allocator_, 0));
@@ -627,6 +649,9 @@ class ColumnWriterImpl {
 
   ColumnChunkMetaDataBuilder* metadata_;
   const ColumnDescriptor* descr_;
+  // scratch buffer if validity bits need to be recalculated.
+  std::shared_ptr<ResizableBuffer> bits_buffer_;
+  const internal::LevelInfo level_info_;
 
   std::unique_ptr<PageWriter> pager_;
 
@@ -957,17 +982,19 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, public TypedColumnWriter<
 
   int64_t Close() override { return ColumnWriterImpl::Close(); }
 
-  void WriteBatch(int64_t num_values, const int16_t* def_levels,
-                  const int16_t* rep_levels, const T* values) override {
+  int64_t WriteBatch(int64_t num_values, const int16_t* def_levels,
+                     const int16_t* rep_levels, const T* values) override {
     // We check for DataPage limits only after we have inserted the values. If a user
     // writes a large number of values, the DataPage size can be much above the limit.
     // The purpose of this chunking is to bound this. Even if a user writes large number
     // of values, the chunking will ensure the AddDataPage() is called at a reasonable
     // pagesize limit
     int64_t value_offset = 0;
+
     auto WriteChunk = [&](int64_t offset, int64_t batch_size) {
       int64_t values_to_write = WriteLevels(batch_size, AddIfNotNull(def_levels, offset),
                                             AddIfNotNull(rep_levels, offset));
+
       // PARQUET-780
       if (values_to_write > 0) {
         DCHECK_NE(nullptr, values);
@@ -981,6 +1008,7 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, public TypedColumnWriter<
       CheckDictionarySizeLimit();
     };
     DoInBatches(num_values, properties_->write_batch_size(), WriteChunk);
+    return value_offset;
   }
 
   void WriteBatchSpaced(int64_t num_values, const int16_t* def_levels,
@@ -991,11 +1019,21 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, public TypedColumnWriter<
     auto WriteChunk = [&](int64_t offset, int64_t batch_size) {
       int64_t batch_num_values = 0;
       int64_t batch_num_spaced_values = 0;
+      int64_t null_count;
+      MaybeCalculateValidityBits(AddIfNotNull(def_levels, offset), batch_size,
+                                 &batch_num_values, &batch_num_spaced_values,
+                                 &null_count);
+
       WriteLevelsSpaced(batch_size, AddIfNotNull(def_levels, offset),
-                        AddIfNotNull(rep_levels, offset), &batch_num_values,
-                        &batch_num_spaced_values);
-      WriteValuesSpaced(values + value_offset, batch_num_values, batch_num_spaced_values,
-                        valid_bits, valid_bits_offset + value_offset);
+                        AddIfNotNull(rep_levels, offset));
+      if (bits_buffer_ != nullptr) {
+        WriteValuesSpaced(values + value_offset, batch_num_values,
+                          batch_num_spaced_values, bits_buffer_->data(), /*offset=*/0);
+      } else {
+        WriteValuesSpaced(values + value_offset, batch_num_values,
+                          batch_num_spaced_values, valid_bits,
+                          valid_bits_offset + value_offset);
+      }
       CommitWriteAndCheckPageLimit(batch_size, batch_num_spaced_values);
       value_offset += batch_num_spaced_values;
 
@@ -1007,13 +1045,31 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, public TypedColumnWriter<
   }
 
   Status WriteArrow(const int16_t* def_levels, const int16_t* rep_levels,
-                    int64_t num_levels, const ::arrow::Array& array,
-                    ArrowWriteContext* ctx) override {
-    if (array.type()->id() == ::arrow::Type::DICTIONARY) {
-      return WriteArrowDictionary(def_levels, rep_levels, num_levels, array, ctx);
+                    int64_t num_levels, const ::arrow::Array& leaf_array,
+                    ArrowWriteContext* ctx, bool leaf_field_nullable) override {
+    BEGIN_PARQUET_CATCH_EXCEPTIONS
+    // Leaf nulls are canonical when there is only a single null element after a list
+    // and it is at the leaf.
+    bool single_nullable_element =
+        (level_info_.def_level == level_info_.repeated_ancestor_def_level + 1) &&
+        leaf_field_nullable;
+    bool maybe_parent_nulls = level_info_.HasNullableValues() && !single_nullable_element;
+    if (maybe_parent_nulls) {
+      ARROW_ASSIGN_OR_RAISE(
+          bits_buffer_,
+          arrow::AllocateResizableBuffer(
+              BitUtil::BytesForBits(properties_->write_batch_size()), ctx->memory_pool));
+      bits_buffer_->ZeroPadding();
+    }
+
+    if (leaf_array.type()->id() == ::arrow::Type::DICTIONARY) {
+      return WriteArrowDictionary(def_levels, rep_levels, num_levels, leaf_array, ctx,
+                                  maybe_parent_nulls);
     } else {
-      return WriteArrowDense(def_levels, rep_levels, num_levels, array, ctx);
+      return WriteArrowDense(def_levels, rep_levels, num_levels, leaf_array, ctx,
+                             maybe_parent_nulls);
     }
+    END_PARQUET_CATCH_EXCEPTIONS
   }
 
   int64_t EstimatedBufferedValueBytes() const override {
@@ -1030,11 +1086,11 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, public TypedColumnWriter<
   // plain encoding is circumvented
   Status WriteArrowDictionary(const int16_t* def_levels, const int16_t* rep_levels,
                               int64_t num_levels, const ::arrow::Array& array,
-                              ArrowWriteContext* context);
+                              ArrowWriteContext* context, bool maybe_parent_nulls);
 
   Status WriteArrowDense(const int16_t* def_levels, const int16_t* rep_levels,
                          int64_t num_levels, const ::arrow::Array& array,
-                         ArrowWriteContext* context);
+                         ArrowWriteContext* context, bool maybe_parent_nulls);
 
   void WriteDictionaryPage() override {
     // We have to dynamic cast here because of TypedEncoder<Type> as
@@ -1129,37 +1185,61 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, public TypedColumnWriter<
     return values_to_write;
   }
 
+  void MaybeCalculateValidityBits(const int16_t* def_levels, int64_t batch_size,
+                                  int64_t* out_values_to_write,
+                                  int64_t* out_spaced_values_to_write,
+                                  int64_t* null_count) {
+    if (bits_buffer_ == nullptr) {
+      if (!level_info_.HasNullableValues()) {
+        *out_values_to_write = batch_size;
+        *out_spaced_values_to_write = batch_size;
+        *null_count = 0;
+      } else {
+        for (int x = 0; x < batch_size; x++) {
+          *out_values_to_write += def_levels[x] == level_info_.def_level ? 1 : 0;
+          *out_spaced_values_to_write +=
+              def_levels[x] >= level_info_.repeated_ancestor_def_level ? 1 : 0;
+        }
+        *null_count = *out_values_to_write - *out_spaced_values_to_write;
+      }
+      return;
+    }
+    // Shrink to fit possible causes another allocation, and would only be necessary
+    // on the last batch.
+    int64_t new_bitmap_size = BitUtil::BytesForBits(batch_size);
+    if (new_bitmap_size != bits_buffer_->size()) {
+      PARQUET_THROW_NOT_OK(
+          bits_buffer_->Resize(new_bitmap_size, /*shrink_to_fit=*/false));
+      bits_buffer_->ZeroPadding();
+    }
+    internal::ValidityBitmapInputOutput io;
+    io.valid_bits = bits_buffer_->mutable_data();
+    io.values_read_upper_bound = batch_size;
+    internal::DefLevelsToBitmap(def_levels, batch_size, level_info_, &io);
+    *out_values_to_write = io.values_read - io.null_count;
+    *out_spaced_values_to_write = io.values_read;
+    *null_count = io.null_count;
+  }
+
+  std::shared_ptr<Array> MaybeReplaceValidity(std::shared_ptr<Array> array,
+                                              int64_t new_null_count) {
+    if (bits_buffer_ == nullptr) {
+      return array;
+    }
+    std::vector<std::shared_ptr<Buffer>> buffers = array->data()->buffers;
+    buffers[0] = bits_buffer_;
+    // Should be a leaf array.
+    DCHECK_EQ(array->num_fields(), 0);
+    return arrow::MakeArray(std::make_shared<ArrayData>(
+        array->type(), array->length(), std::move(buffers), new_null_count));
+  }
+
   void WriteLevelsSpaced(int64_t num_levels, const int16_t* def_levels,
-                         const int16_t* rep_levels, int64_t* out_values_to_write,
-                         int64_t* out_spaced_values_to_write) {
-    int64_t values_to_write = 0;
-    int64_t spaced_values_to_write = 0;
+                         const int16_t* rep_levels) {
     // If the field is required and non-repeated, there are no definition levels
     if (descr_->max_definition_level() > 0) {
-      // Minimal definition level for which spaced values are written
-      int16_t min_spaced_def_level = descr_->max_definition_level();
-      const ::parquet::schema::Node* node = descr_->schema_node().get();
-      while (node != nullptr && !node->is_repeated()) {
-        if (node->is_optional()) {
-          min_spaced_def_level--;
-        }
-        node = node->parent();
-      }
-      for (int64_t i = 0; i < num_levels; ++i) {
-        if (def_levels[i] == descr_->max_definition_level()) {
-          ++values_to_write;
-        }
-        if (def_levels[i] >= min_spaced_def_level) {
-          ++spaced_values_to_write;
-        }
-      }
       WriteDefinitionLevels(num_levels, def_levels);
-    } else {
-      // Required field, write all values
-      values_to_write = num_levels;
-      spaced_values_to_write = num_levels;
     }
-
     // Not present for non-repeated fields
     if (descr_->max_repetition_level() > 0) {
       // A row could include more than one value
@@ -1169,15 +1249,11 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, public TypedColumnWriter<
           rows_written_++;
         }
       }
-
       WriteRepetitionLevels(num_levels, rep_levels);
     } else {
       // Each value is exactly one row
       rows_written_ += static_cast<int>(num_levels);
     }
-
-    *out_values_to_write = values_to_write;
-    *out_spaced_values_to_write = spaced_values_to_write;
   }
 
   void CommitWriteAndCheckPageLimit(int64_t num_levels, int64_t num_values) {
@@ -1233,7 +1309,7 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, public TypedColumnWriter<
 
   void WriteValuesSpaced(const T* values, int64_t num_values, int64_t num_spaced_values,
                          const uint8_t* valid_bits, int64_t valid_bits_offset) {
-    if (descr_->schema_node()->is_optional()) {
+    if (num_values != num_spaced_values) {
       dynamic_cast<ValueEncoderType*>(current_encoder_.get())
           ->PutSpaced(values, static_cast<int>(num_spaced_values), valid_bits,
                       valid_bits_offset);
@@ -1250,11 +1326,9 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, public TypedColumnWriter<
 };
 
 template <typename DType>
-Status TypedColumnWriterImpl<DType>::WriteArrowDictionary(const int16_t* def_levels,
-                                                          const int16_t* rep_levels,
-                                                          int64_t num_levels,
-                                                          const ::arrow::Array& array,
-                                                          ArrowWriteContext* ctx) {
+Status TypedColumnWriterImpl<DType>::WriteArrowDictionary(
+    const int16_t* def_levels, const int16_t* rep_levels, int64_t num_levels,
+    const ::arrow::Array& array, ArrowWriteContext* ctx, bool maybe_parent_nulls) {
   // If this is the first time writing a DictionaryArray, then there's
   // a few possible paths to take:
   //
@@ -1274,7 +1348,8 @@ Status TypedColumnWriterImpl<DType>::WriteArrowDictionary(const int16_t* def_lev
     std::shared_ptr<::arrow::Array> dense_array;
     RETURN_NOT_OK(
         ConvertDictionaryToDense(array, properties_->memory_pool(), &dense_array));
-    return WriteArrowDense(def_levels, rep_levels, num_levels, *dense_array, ctx);
+    return WriteArrowDense(def_levels, rep_levels, num_levels, *dense_array, ctx,
+                           maybe_parent_nulls);
   };
 
   if (!IsDictionaryEncoding(current_encoder_->encoding()) ||
@@ -1297,10 +1372,18 @@ Status TypedColumnWriterImpl<DType>::WriteArrowDictionary(const int16_t* def_lev
   auto WriteIndicesChunk = [&](int64_t offset, int64_t batch_size) {
     int64_t batch_num_values = 0;
     int64_t batch_num_spaced_values = 0;
+    int64_t null_count = arrow::kUnknownNullCount;
+    // Bits is not null for nullable values.  At this point in the code we can't determine
+    // if the leaf array has the same null values as any parents it might have had so we
+    // need to recompute it from def levels.
+    MaybeCalculateValidityBits(AddIfNotNull(def_levels, offset), batch_size,
+                               &batch_num_values, &batch_num_spaced_values, &null_count);
     WriteLevelsSpaced(batch_size, AddIfNotNull(def_levels, offset),
-                      AddIfNotNull(rep_levels, offset), &batch_num_values,
-                      &batch_num_spaced_values);
-    dict_encoder->PutIndices(*indices->Slice(value_offset, batch_num_spaced_values));
+                      AddIfNotNull(rep_levels, offset));
+    std::shared_ptr<Array> writeable_indices =
+        indices->Slice(value_offset, batch_num_spaced_values);
+    writeable_indices = MaybeReplaceValidity(writeable_indices, null_count);
+    dict_encoder->PutIndices(*writeable_indices);
     CommitWriteAndCheckPageLimit(batch_size, batch_num_values);
     value_offset += batch_num_spaced_values;
   };
@@ -1310,6 +1393,14 @@ Status TypedColumnWriterImpl<DType>::WriteArrowDictionary(const int16_t* def_lev
     // It's a new dictionary. Call PutDictionary and keep track of it
     PARQUET_CATCH_NOT_OK(dict_encoder->PutDictionary(*dictionary));
 
+    // If there were duplicate value in the dictionary, the encoder's memo table
+    // will be out of sync with the indices in the Arrow array.
+    // The easiest solution for this uncommon case is to fallback to plain encoding.
+    if (dict_encoder->num_entries() != dictionary->length()) {
+      PARQUET_CATCH_NOT_OK(FallbackToPlainEncoding());
+      return WriteDense();
+    }
+
     // TODO(wesm): If some dictionary values are unobserved, then the
     // statistics will be inaccurate. Do we care enough to fix it?
     if (page_statistics_ != nullptr) {
@@ -1351,21 +1442,19 @@ struct SerializeFunctor {
 template <typename ParquetType, typename ArrowType>
 Status WriteArrowSerialize(const ::arrow::Array& array, int64_t num_levels,
                            const int16_t* def_levels, const int16_t* rep_levels,
-                           ArrowWriteContext* ctx,
-                           TypedColumnWriter<ParquetType>* writer) {
+                           ArrowWriteContext* ctx, TypedColumnWriter<ParquetType>* writer,
+                           bool maybe_parent_nulls) {
   using ParquetCType = typename ParquetType::c_type;
   using ArrayType = typename ::arrow::TypeTraits<ArrowType>::ArrayType;
 
   ParquetCType* buffer = nullptr;
   PARQUET_THROW_NOT_OK(ctx->GetScratchData<ParquetCType>(array.length(), &buffer));
 
-  bool no_nulls =
-      writer->descr()->schema_node()->is_required() || (array.null_count() == 0);
-
   SerializeFunctor<ParquetType, ArrowType> functor;
   RETURN_NOT_OK(functor.Serialize(checked_cast<const ArrayType&>(array), ctx, buffer));
-
-  if (no_nulls) {
+  bool no_nulls =
+      writer->descr()->schema_node()->is_required() || (array.null_count() == 0);
+  if (!maybe_parent_nulls && no_nulls) {
     PARQUET_CATCH_NOT_OK(writer->WriteBatch(num_levels, def_levels, rep_levels, buffer));
   } else {
     PARQUET_CATCH_NOT_OK(writer->WriteBatchSpaced(num_levels, def_levels, rep_levels,
@@ -1378,8 +1467,8 @@ Status WriteArrowSerialize(const ::arrow::Array& array, int64_t num_levels,
 template <typename ParquetType>
 Status WriteArrowZeroCopy(const ::arrow::Array& array, int64_t num_levels,
                           const int16_t* def_levels, const int16_t* rep_levels,
-                          ArrowWriteContext* ctx,
-                          TypedColumnWriter<ParquetType>* writer) {
+                          ArrowWriteContext* ctx, TypedColumnWriter<ParquetType>* writer,
+                          bool maybe_parent_nulls) {
   using T = typename ParquetType::c_type;
   const auto& data = static_cast<const ::arrow::PrimitiveArray&>(array);
   const T* values = nullptr;
@@ -1389,7 +1478,10 @@ Status WriteArrowZeroCopy(const ::arrow::Array& array, int64_t num_levels,
   } else {
     DCHECK_EQ(data.length(), 0);
   }
-  if (writer->descr()->schema_node()->is_required() || (data.null_count() == 0)) {
+  bool no_nulls =
+      writer->descr()->schema_node()->is_required() || (array.null_count() == 0);
+
+  if (!maybe_parent_nulls && no_nulls) {
     PARQUET_CATCH_NOT_OK(writer->WriteBatch(num_levels, def_levels, rep_levels, values));
   } else {
     PARQUET_CATCH_NOT_OK(writer->WriteBatchSpaced(num_levels, def_levels, rep_levels,
@@ -1402,12 +1494,12 @@ Status WriteArrowZeroCopy(const ::arrow::Array& array, int64_t num_levels,
 #define WRITE_SERIALIZE_CASE(ArrowEnum, ArrowType, ParquetType)  \
   case ::arrow::Type::ArrowEnum:                                 \
     return WriteArrowSerialize<ParquetType, ::arrow::ArrowType>( \
-        array, num_levels, def_levels, rep_levels, ctx, this);
+        array, num_levels, def_levels, rep_levels, ctx, this, maybe_parent_nulls);
 
 #define WRITE_ZERO_COPY_CASE(ArrowEnum, ArrowType, ParquetType)                       \
   case ::arrow::Type::ArrowEnum:                                                      \
     return WriteArrowZeroCopy<ParquetType>(array, num_levels, def_levels, rep_levels, \
-                                           ctx, this);
+                                           ctx, this, maybe_parent_nulls);
 
 #define ARROW_UNSUPPORTED()                                          \
   std::stringstream ss;                                              \
@@ -1429,16 +1521,14 @@ struct SerializeFunctor<BooleanType, ::arrow::BooleanType> {
 };
 
 template <>
-Status TypedColumnWriterImpl<BooleanType>::WriteArrowDense(const int16_t* def_levels,
-                                                           const int16_t* rep_levels,
-                                                           int64_t num_levels,
-                                                           const ::arrow::Array& array,
-                                                           ArrowWriteContext* ctx) {
+Status TypedColumnWriterImpl<BooleanType>::WriteArrowDense(
+    const int16_t* def_levels, const int16_t* rep_levels, int64_t num_levels,
+    const ::arrow::Array& array, ArrowWriteContext* ctx, bool maybe_parent_nulls) {
   if (array.type_id() != ::arrow::Type::BOOL) {
     ARROW_UNSUPPORTED();
   }
   return WriteArrowSerialize<BooleanType, ::arrow::BooleanType>(
-      array, num_levels, def_levels, rep_levels, ctx, this);
+      array, num_levels, def_levels, rep_levels, ctx, this, maybe_parent_nulls);
 }
 
 // ----------------------------------------------------------------------
@@ -1472,11 +1562,9 @@ struct SerializeFunctor<Int32Type, ::arrow::Time32Type> {
 };
 
 template <>
-Status TypedColumnWriterImpl<Int32Type>::WriteArrowDense(const int16_t* def_levels,
-                                                         const int16_t* rep_levels,
-                                                         int64_t num_levels,
-                                                         const ::arrow::Array& array,
-                                                         ArrowWriteContext* ctx) {
+Status TypedColumnWriterImpl<Int32Type>::WriteArrowDense(
+    const int16_t* def_levels, const int16_t* rep_levels, int64_t num_levels,
+    const ::arrow::Array& array, ArrowWriteContext* ctx, bool maybe_parent_nulls) {
   switch (array.type()->id()) {
     case ::arrow::Type::NA: {
       PARQUET_CATCH_NOT_OK(WriteBatch(num_levels, def_levels, rep_levels, nullptr));
@@ -1598,14 +1686,16 @@ struct SerializeFunctor<Int64Type, ::arrow::TimestampType> {
 
 Status WriteTimestamps(const ::arrow::Array& values, int64_t num_levels,
                        const int16_t* def_levels, const int16_t* rep_levels,
-                       ArrowWriteContext* ctx, TypedColumnWriter<Int64Type>* writer) {
+                       ArrowWriteContext* ctx, TypedColumnWriter<Int64Type>* writer,
+                       bool maybe_parent_nulls) {
   const auto& source_type = static_cast<const ::arrow::TimestampType&>(*values.type());
 
   auto WriteCoerce = [&](const ArrowWriterProperties* properties) {
     ArrowWriteContext temp_ctx = *ctx;
     temp_ctx.properties = properties;
     return WriteArrowSerialize<Int64Type, ::arrow::TimestampType>(
-        values, num_levels, def_levels, rep_levels, &temp_ctx, writer);
+        values, num_levels, def_levels, rep_levels, &temp_ctx, writer,
+        maybe_parent_nulls);
   };
 
   if (ctx->properties->coerce_timestamps_enabled()) {
@@ -1613,7 +1703,7 @@ Status WriteTimestamps(const ::arrow::Array& values, int64_t num_levels,
     if (source_type.unit() == ctx->properties->coerce_timestamps_unit()) {
       // No data conversion necessary
       return WriteArrowZeroCopy<Int64Type>(values, num_levels, def_levels, rep_levels,
-                                           ctx, writer);
+                                           ctx, writer, maybe_parent_nulls);
     } else {
       return WriteCoerce(ctx->properties);
     }
@@ -1638,19 +1728,18 @@ Status WriteTimestamps(const ::arrow::Array& values, int64_t num_levels,
   } else {
     // No data conversion necessary
     return WriteArrowZeroCopy<Int64Type>(values, num_levels, def_levels, rep_levels, ctx,
-                                         writer);
+                                         writer, maybe_parent_nulls);
   }
 }
 
 template <>
-Status TypedColumnWriterImpl<Int64Type>::WriteArrowDense(const int16_t* def_levels,
-                                                         const int16_t* rep_levels,
-                                                         int64_t num_levels,
-                                                         const ::arrow::Array& array,
-                                                         ArrowWriteContext* ctx) {
+Status TypedColumnWriterImpl<Int64Type>::WriteArrowDense(
+    const int16_t* def_levels, const int16_t* rep_levels, int64_t num_levels,
+    const ::arrow::Array& array, ArrowWriteContext* ctx, bool maybe_parent_nulls) {
   switch (array.type()->id()) {
     case ::arrow::Type::TIMESTAMP:
-      return WriteTimestamps(array, num_levels, def_levels, rep_levels, ctx, this);
+      return WriteTimestamps(array, num_levels, def_levels, rep_levels, ctx, this,
+                             maybe_parent_nulls);
       WRITE_ZERO_COPY_CASE(INT64, Int64Type, Int64Type)
       WRITE_SERIALIZE_CASE(UINT32, UInt32Type, Int64Type)
       WRITE_SERIALIZE_CASE(UINT64, UInt64Type, Int64Type)
@@ -1661,56 +1750,48 @@ Status TypedColumnWriterImpl<Int64Type>::WriteArrowDense(const int16_t* def_leve
 }
 
 template <>
-Status TypedColumnWriterImpl<Int96Type>::WriteArrowDense(const int16_t* def_levels,
-                                                         const int16_t* rep_levels,
-                                                         int64_t num_levels,
-                                                         const ::arrow::Array& array,
-                                                         ArrowWriteContext* ctx) {
+Status TypedColumnWriterImpl<Int96Type>::WriteArrowDense(
+    const int16_t* def_levels, const int16_t* rep_levels, int64_t num_levels,
+    const ::arrow::Array& array, ArrowWriteContext* ctx, bool maybe_parent_nulls) {
   if (array.type_id() != ::arrow::Type::TIMESTAMP) {
     ARROW_UNSUPPORTED();
   }
   return WriteArrowSerialize<Int96Type, ::arrow::TimestampType>(
-      array, num_levels, def_levels, rep_levels, ctx, this);
+      array, num_levels, def_levels, rep_levels, ctx, this, maybe_parent_nulls);
 }
 
 // ----------------------------------------------------------------------
 // Floating point types
 
 template <>
-Status TypedColumnWriterImpl<FloatType>::WriteArrowDense(const int16_t* def_levels,
-                                                         const int16_t* rep_levels,
-                                                         int64_t num_levels,
-                                                         const ::arrow::Array& array,
-                                                         ArrowWriteContext* ctx) {
+Status TypedColumnWriterImpl<FloatType>::WriteArrowDense(
+    const int16_t* def_levels, const int16_t* rep_levels, int64_t num_levels,
+    const ::arrow::Array& array, ArrowWriteContext* ctx, bool maybe_parent_nulls) {
   if (array.type_id() != ::arrow::Type::FLOAT) {
     ARROW_UNSUPPORTED();
   }
   return WriteArrowZeroCopy<FloatType>(array, num_levels, def_levels, rep_levels, ctx,
-                                       this);
+                                       this, maybe_parent_nulls);
 }
 
 template <>
-Status TypedColumnWriterImpl<DoubleType>::WriteArrowDense(const int16_t* def_levels,
-                                                          const int16_t* rep_levels,
-                                                          int64_t num_levels,
-                                                          const ::arrow::Array& array,
-                                                          ArrowWriteContext* ctx) {
+Status TypedColumnWriterImpl<DoubleType>::WriteArrowDense(
+    const int16_t* def_levels, const int16_t* rep_levels, int64_t num_levels,
+    const ::arrow::Array& array, ArrowWriteContext* ctx, bool maybe_parent_nulls) {
   if (array.type_id() != ::arrow::Type::DOUBLE) {
     ARROW_UNSUPPORTED();
   }
   return WriteArrowZeroCopy<DoubleType>(array, num_levels, def_levels, rep_levels, ctx,
-                                        this);
+                                        this, maybe_parent_nulls);
 }
 
 // ----------------------------------------------------------------------
 // Write Arrow to BYTE_ARRAY
 
 template <>
-Status TypedColumnWriterImpl<ByteArrayType>::WriteArrowDense(const int16_t* def_levels,
-                                                             const int16_t* rep_levels,
-                                                             int64_t num_levels,
-                                                             const ::arrow::Array& array,
-                                                             ArrowWriteContext* ctx) {
+Status TypedColumnWriterImpl<ByteArrayType>::WriteArrowDense(
+    const int16_t* def_levels, const int16_t* rep_levels, int64_t num_levels,
+    const ::arrow::Array& array, ArrowWriteContext* ctx, bool maybe_parent_nulls) {
   if (array.type()->id() != ::arrow::Type::BINARY &&
       array.type()->id() != ::arrow::Type::STRING) {
     ARROW_UNSUPPORTED();
@@ -1720,11 +1801,16 @@ Status TypedColumnWriterImpl<ByteArrayType>::WriteArrowDense(const int16_t* def_
   auto WriteChunk = [&](int64_t offset, int64_t batch_size) {
     int64_t batch_num_values = 0;
     int64_t batch_num_spaced_values = 0;
+    int64_t null_count = 0;
+
+    MaybeCalculateValidityBits(AddIfNotNull(def_levels, offset), batch_size,
+                               &batch_num_values, &batch_num_spaced_values, &null_count);
     WriteLevelsSpaced(batch_size, AddIfNotNull(def_levels, offset),
-                      AddIfNotNull(rep_levels, offset), &batch_num_values,
-                      &batch_num_spaced_values);
-    std::shared_ptr<::arrow::Array> data_slice =
+                      AddIfNotNull(rep_levels, offset));
+    std::shared_ptr<Array> data_slice =
         array.Slice(value_offset, batch_num_spaced_values);
+    data_slice = MaybeReplaceValidity(data_slice, null_count);
+
     current_encoder_->Put(*data_slice);
     if (page_statistics_ != nullptr) {
       page_statistics_->Update(*data_slice);
@@ -1774,7 +1860,8 @@ using ::arrow::internal::checked_pointer_cast;
 // Requires a custom serializer because decimal128 in parquet are in big-endian
 // format. Thus, a temporary local buffer is required.
 template <typename ParquetType, typename ArrowType>
-struct SerializeFunctor<ParquetType, ArrowType, ::arrow::enable_if_decimal<ArrowType>> {
+struct SerializeFunctor<ParquetType, ArrowType,
+                        ::arrow::enable_if_decimal128<ArrowType>> {
   Status Serialize(const ::arrow::Decimal128Array& array, ArrowWriteContext* ctx,
                    FLBA* out) {
     AllocateScratch(array, ctx);
@@ -1822,15 +1909,23 @@ struct SerializeFunctor<ParquetType, ArrowType, ::arrow::enable_if_decimal<Arrow
   int64_t* scratch;
 };
 
+template <typename ParquetType, typename ArrowType>
+struct SerializeFunctor<ParquetType, ArrowType,
+                        ::arrow::enable_if_decimal256<ArrowType>> {
+  Status Serialize(const ::arrow::Decimal256Array& array, ArrowWriteContext* ctx,
+                   FLBA* out) {
+    return Status::NotImplemented("Decimal256 serialization isn't implemented");
+  }
+};
+
 template <>
-Status TypedColumnWriterImpl<FLBAType>::WriteArrowDense(const int16_t* def_levels,
-                                                        const int16_t* rep_levels,
-                                                        int64_t num_levels,
-                                                        const ::arrow::Array& array,
-                                                        ArrowWriteContext* ctx) {
+Status TypedColumnWriterImpl<FLBAType>::WriteArrowDense(
+    const int16_t* def_levels, const int16_t* rep_levels, int64_t num_levels,
+    const ::arrow::Array& array, ArrowWriteContext* ctx, bool maybe_parent_nulls) {
   switch (array.type()->id()) {
     WRITE_SERIALIZE_CASE(FIXED_SIZE_BINARY, FixedSizeBinaryType, FLBAType)
-    WRITE_SERIALIZE_CASE(DECIMAL, Decimal128Type, FLBAType)
+    WRITE_SERIALIZE_CASE(DECIMAL128, Decimal128Type, FLBAType)
+    WRITE_SERIALIZE_CASE(DECIMAL256, Decimal256Type, FLBAType)
     default:
       break;
   }
diff --git a/cpp/src/parquet/column_writer.h b/cpp/src/parquet/column_writer.h
index f1cfb239c14..57f98533a72 100644
--- a/cpp/src/parquet/column_writer.h
+++ b/cpp/src/parquet/column_writer.h
@@ -141,10 +141,15 @@ class PARQUET_EXPORT ColumnWriter {
 
   /// \brief Write Apache Arrow columnar data directly to ColumnWriter. Returns
   /// error status if the array data type is not compatible with the concrete
-  /// writer type
+  /// writer type.
+  ///
+  /// leaf_array is always a primitive (possibly dictionary encoded type).
+  /// Leaf_field_nullable indicates whether the leaf array is considered nullable
+  /// according to its schema in a Table or its parent array.
   virtual ::arrow::Status WriteArrow(const int16_t* def_levels, const int16_t* rep_levels,
-                                     int64_t num_levels, const ::arrow::Array& array,
-                                     ArrowWriteContext* ctx) = 0;
+                                     int64_t num_levels, const ::arrow::Array& leaf_array,
+                                     ArrowWriteContext* ctx,
+                                     bool leaf_field_nullable) = 0;
 };
 
 // API to write values to a single column. This is the main client facing API.
@@ -155,8 +160,16 @@ class TypedColumnWriter : public ColumnWriter {
 
   // Write a batch of repetition levels, definition levels, and values to the
   // column.
-  virtual void WriteBatch(int64_t num_values, const int16_t* def_levels,
-                          const int16_t* rep_levels, const T* values) = 0;
+  // `num_values` is the number of logical leaf values.
+  // `def_levels` (resp. `rep_levels`) can be null if the column's max definition level
+  // (resp. max repetition level) is 0.
+  // If not null, each of `def_levels` and `rep_levels` must have at least
+  // `num_values`.
+  //
+  // The number of physical values written (taken from `values`) is returned.
+  // It can be smaller than `num_values` is there are some undefined values.
+  virtual int64_t WriteBatch(int64_t num_values, const int16_t* def_levels,
+                             const int16_t* rep_levels, const T* values) = 0;
 
   /// Write a batch of repetition levels, definition levels, and values to the
   /// column.
diff --git a/cpp/src/parquet/column_writer_test.cc b/cpp/src/parquet/column_writer_test.cc
index a92d4d28f7d..23554aa3dcd 100644
--- a/cpp/src/parquet/column_writer_test.cc
+++ b/cpp/src/parquet/column_writer_test.cc
@@ -488,15 +488,13 @@ TYPED_TEST(TestPrimitiveWriter, RequiredPlainWithStatsAndGzipCompression) {
 
 #ifdef ARROW_WITH_LZ4
 TYPED_TEST(TestPrimitiveWriter, RequiredPlainWithLz4Compression) {
-  ASSERT_THROW(this->TestRequiredWithSettings(Encoding::PLAIN, Compression::LZ4, false,
-                                              false, LARGE_SIZE),
-               ParquetException);
+  this->TestRequiredWithSettings(Encoding::PLAIN, Compression::LZ4, false, false,
+                                 LARGE_SIZE);
 }
 
 TYPED_TEST(TestPrimitiveWriter, RequiredPlainWithStatsAndLz4Compression) {
-  ASSERT_THROW(this->TestRequiredWithSettings(Encoding::PLAIN, Compression::LZ4, false,
-                                              true, LARGE_SIZE),
-               ParquetException);
+  this->TestRequiredWithSettings(Encoding::PLAIN, Compression::LZ4, false, true,
+                                 LARGE_SIZE);
 }
 #endif
 
diff --git a/cpp/src/parquet/encoding_test.cc b/cpp/src/parquet/encoding_test.cc
index 363f7e1ac41..6766c95dcdd 100644
--- a/cpp/src/parquet/encoding_test.cc
+++ b/cpp/src/parquet/encoding_test.cc
@@ -25,6 +25,7 @@
 #include <vector>
 
 #include "arrow/array.h"
+#include "arrow/array/builder_dict.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/testing/random.h"
 #include "arrow/testing/util.h"
diff --git a/cpp/src/parquet/encryption.h b/cpp/src/parquet/encryption.h
index 30389ba63c0..d29f72135a2 100644
--- a/cpp/src/parquet/encryption.h
+++ b/cpp/src/parquet/encryption.h
@@ -20,7 +20,6 @@
 #include <map>
 #include <memory>
 #include <string>
-#include <unordered_map>
 #include <utility>
 
 #include "parquet/exception.h"
diff --git a/cpp/src/parquet/encryption_read_configurations_test.cc b/cpp/src/parquet/encryption_read_configurations_test.cc
index a794a8cf2ad..eb95593d3f3 100644
--- a/cpp/src/parquet/encryption_read_configurations_test.cc
+++ b/cpp/src/parquet/encryption_read_configurations_test.cc
@@ -490,10 +490,10 @@ TEST_P(TestDecryptionConfiguration, TestDecryption) {
   // Decrypt parquet file that was generated in encryption-write-configurations-test.cc
   // test.
   std::string tmp_file_name = "tmp_" + std::string(param_file_name);
-  std::string file_name = data_file(tmp_file_name.c_str());
+  std::string file_name = temp_dir->path().ToString() + tmp_file_name;
   if (!fexists(file_name)) {
     std::stringstream ss;
-    ss << "File " << file_name << " is missing from parquet-testing repo.";
+    ss << "File " << file_name << " is missing from temporary dir.";
     throw ParquetTestException(ss.str());
   }
 
diff --git a/cpp/src/parquet/encryption_write_configurations_test.cc b/cpp/src/parquet/encryption_write_configurations_test.cc
index 99a920ae022..eb55f6bd8a5 100644
--- a/cpp/src/parquet/encryption_write_configurations_test.cc
+++ b/cpp/src/parquet/encryption_write_configurations_test.cc
@@ -32,7 +32,7 @@
 /*
  * This file contains unit-tests for writing encrypted Parquet files with
  * different encryption configurations.
- * The files are saved in parquet-testing/data folder and will be deleted after reading
+ * The files are saved in temporary folder and will be deleted after reading
  * them in encryption-read-configurations-test.cc test.
  *
  * A detailed description of the Parquet Modular Encryption specification can be found
@@ -65,12 +65,15 @@ namespace test {
 
 using FileClass = ::arrow::io::FileOutputStream;
 
+std::unique_ptr<TemporaryDir> temp_dir;
+
 class TestEncryptionConfiguration : public ::testing::Test {
  public:
   void SetUp() {
     // Setup the parquet schema
     schema_ = SetupEncryptionSchema();
   }
+  static void SetUpTestCase();
 
  protected:
   std::string path_to_double_field_ = "double_field";
@@ -87,7 +90,7 @@ class TestEncryptionConfiguration : public ::testing::Test {
   void EncryptFile(
       std::shared_ptr<parquet::FileEncryptionProperties> encryption_configurations,
       std::string file_name) {
-    std::string file = data_file(file_name.c_str());
+    std::string file = temp_dir->path().ToString() + file_name;
 
     WriterProperties::Builder prop_builder;
     prop_builder.compression(parquet::Compression::SNAPPY);
@@ -382,5 +385,9 @@ TEST_F(TestEncryptionConfiguration, EncryptTwoColumnsAndFooterUseAES_GCM_CTR) {
                         "tmp_encrypt_columns_and_footer_ctr.parquet.encrypted"));
 }
 
+// Set temp_dir before running the write/read tests. The encrypted files will
+// be written/read from this directory.
+void TestEncryptionConfiguration::SetUpTestCase() { temp_dir = *temp_data_dir(); }
+
 }  // namespace test
 }  // namespace parquet
diff --git a/cpp/src/parquet/exception.cc b/cpp/src/parquet/exception.cc
new file mode 100644
index 00000000000..c333957dd1d
--- /dev/null
+++ b/cpp/src/parquet/exception.cc
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "parquet/exception.h"
+
+namespace parquet {
+
+std::ostream& operator<<(std::ostream& os, const ParquetException& exception) {
+  os << exception.what();
+  return os;
+}
+
+}  // namespace parquet
diff --git a/cpp/src/parquet/exception.h b/cpp/src/parquet/exception.h
index 11efd8756f1..bfd1bfd9422 100644
--- a/cpp/src/parquet/exception.h
+++ b/cpp/src/parquet/exception.h
@@ -23,6 +23,7 @@
 #include <utility>
 
 #include "arrow/type_fwd.h"
+#include "arrow/util/string_builder.h"
 #include "parquet/platform.h"
 
 // PARQUET-1085
@@ -80,34 +81,42 @@ namespace parquet {
 class ParquetException : public std::exception {
  public:
   PARQUET_NORETURN static void EofException(const std::string& msg = "") {
-    std::stringstream ss;
-    ss << "Unexpected end of stream";
-    if (!msg.empty()) {
-      ss << ": " << msg;
+    static std::string prefix = "Unexpected end of stream";
+    if (msg.empty()) {
+      throw ParquetException(prefix);
     }
-    throw ParquetException(ss.str());
+    throw ParquetException(prefix, ": ", msg);
   }
 
   PARQUET_NORETURN static void NYI(const std::string& msg = "") {
-    std::stringstream ss;
-    ss << "Not yet implemented: " << msg << ".";
-    throw ParquetException(ss.str());
+    throw ParquetException("Not yet implemented: ", msg, ".");
   }
 
-  explicit ParquetException(const char* msg) : msg_(msg) {}
+  template <typename... Args>
+  explicit ParquetException(Args&&... args)
+      : msg_(::arrow::util::StringBuilder(std::forward<Args>(args)...)) {}
 
-  explicit ParquetException(const std::string& msg) : msg_(msg) {}
+  explicit ParquetException(std::string msg) : msg_(std::move(msg)) {}
 
-  explicit ParquetException(const char* msg, std::exception&) : msg_(msg) {}
+  explicit ParquetException(const char* msg, const std::exception&) : msg_(msg) {}
 
-  ~ParquetException() throw() override {}
+  ParquetException(const ParquetException&) = default;
+  ParquetException& operator=(const ParquetException&) = default;
+  ParquetException(ParquetException&&) = default;
+  ParquetException& operator=(ParquetException&&) = default;
 
-  const char* what() const throw() override { return msg_.c_str(); }
+  const char* what() const noexcept override { return msg_.c_str(); }
 
  private:
   std::string msg_;
 };
 
+// Support printing a ParquetException.
+// This is needed for clang-on-MSVC as there operator<< is not defined for
+// std::exception.
+PARQUET_EXPORT
+std::ostream& operator<<(std::ostream& os, const ParquetException& exception);
+
 class ParquetStatusException : public ParquetException {
  public:
   explicit ParquetStatusException(::arrow::Status status)
@@ -122,9 +131,17 @@ class ParquetStatusException : public ParquetException {
 // This class exists for the purpose of detecting an invalid or corrupted file.
 class ParquetInvalidOrCorruptedFileException : public ParquetStatusException {
  public:
-  template <typename... Args>
-  explicit ParquetInvalidOrCorruptedFileException(Args&&... args)
-      : ParquetStatusException(::arrow::Status::Invalid(std::forward<Args>(args)...)) {}
+  ParquetInvalidOrCorruptedFileException(const ParquetInvalidOrCorruptedFileException&) =
+      default;
+
+  template <typename Arg,
+            typename std::enable_if<
+                !std::is_base_of<ParquetInvalidOrCorruptedFileException, Arg>::value,
+                int>::type = 0,
+            typename... Args>
+  explicit ParquetInvalidOrCorruptedFileException(Arg arg, Args&&... args)
+      : ParquetStatusException(::arrow::Status::Invalid(std::forward<Arg>(arg),
+                                                        std::forward<Args>(args)...)) {}
 };
 
 template <typename StatusReturnBlock>
@@ -132,4 +149,11 @@ void ThrowNotOk(StatusReturnBlock&& b) {
   PARQUET_THROW_NOT_OK(b());
 }
 
+#define BEGIN_PARQUET_CATCH_EXCEPTIONS try {
+#define END_PARQUET_CATCH_EXCEPTIONS             \
+  }                                              \
+  catch (const ::parquet::ParquetException& e) { \
+    return ::arrow::Status::IOError(e.what());   \
+  }
+
 }  // namespace parquet
diff --git a/cpp/src/parquet/file_deserialize_test.cc b/cpp/src/parquet/file_deserialize_test.cc
index 1dd34922ae9..d0d333256fb 100644
--- a/cpp/src/parquet/file_deserialize_test.cc
+++ b/cpp/src/parquet/file_deserialize_test.cc
@@ -249,8 +249,10 @@ TEST_F(TestPageSerde, Compression) {
   codec_types.push_back(Compression::GZIP);
 #endif
 
-  // TODO: Add LZ4 compression type after PARQUET-1878 is complete.
-  // Testing for deserializing LZ4 is hard without writing enabled, so it is not included.
+#ifdef ARROW_WITH_LZ4
+  codec_types.push_back(Compression::LZ4);
+  codec_types.push_back(Compression::LZ4_HADOOP);
+#endif
 
 #ifdef ARROW_WITH_ZSTD
   codec_types.push_back(Compression::ZSTD);
@@ -259,7 +261,7 @@ TEST_F(TestPageSerde, Compression) {
   const int32_t num_rows = 32;  // dummy value
   data_page_header_.num_values = num_rows;
 
-  int num_pages = 10;
+  const int num_pages = 10;
 
   std::vector<std::vector<uint8_t>> faux_data;
   faux_data.resize(num_pages);
diff --git a/cpp/src/parquet/file_serialize_test.cc b/cpp/src/parquet/file_serialize_test.cc
index 72d7d6f742d..ad9e02e3655 100644
--- a/cpp/src/parquet/file_serialize_test.cc
+++ b/cpp/src/parquet/file_serialize_test.cc
@@ -57,6 +57,11 @@ class TestSerialize : public PrimitiveTypedTest<TestType> {
   int rows_per_batch_;
 
   void FileSerializeTest(Compression::type codec_type) {
+    FileSerializeTest(codec_type, codec_type);
+  }
+
+  void FileSerializeTest(Compression::type codec_type,
+                         Compression::type expected_codec_type) {
     auto sink = CreateOutputStream();
     auto gnode = std::static_pointer_cast<GroupNode>(this->node_);
 
@@ -123,7 +128,8 @@ class TestSerialize : public PrimitiveTypedTest<TestType> {
       ASSERT_EQ(num_columns_, rg_reader->metadata()->num_columns());
       ASSERT_EQ(rows_per_rowgroup_, rg_reader->metadata()->num_rows());
       // Check that the specified compression was actually used.
-      ASSERT_EQ(codec_type, rg_reader->metadata()->ColumnChunk(0)->compression());
+      ASSERT_EQ(expected_codec_type,
+                rg_reader->metadata()->ColumnChunk(0)->compression());
 
       int64_t values_read;
 
@@ -309,7 +315,12 @@ TYPED_TEST(TestSerialize, SmallFileGzip) {
 
 #ifdef ARROW_WITH_LZ4
 TYPED_TEST(TestSerialize, SmallFileLz4) {
-  ASSERT_THROW(this->FileSerializeTest(Compression::LZ4), ParquetException);
+  ASSERT_NO_FATAL_FAILURE(
+      this->FileSerializeTest(Compression::LZ4, Compression::LZ4_HADOOP));
+}
+
+TYPED_TEST(TestSerialize, SmallFileLz4Hadoop) {
+  ASSERT_NO_FATAL_FAILURE(this->FileSerializeTest(Compression::LZ4_HADOOP));
 }
 #endif
 
diff --git a/cpp/src/parquet/level_comparison.cc b/cpp/src/parquet/level_comparison.cc
new file mode 100644
index 00000000000..30614ae61fb
--- /dev/null
+++ b/cpp/src/parquet/level_comparison.cc
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "parquet/level_comparison.h"
+
+#define PARQUET_IMPL_NAMESPACE standard
+#include "parquet/level_comparison_inc.h"
+#undef PARQUET_IMPL_NAMESPACE
+
+#include <vector>
+
+#include "arrow/util/dispatch.h"
+
+namespace parquet {
+namespace internal {
+
+#if defined(ARROW_HAVE_RUNTIME_AVX2)
+MinMax FindMinMaxAvx2(const int16_t* levels, int64_t num_levels);
+uint64_t GreaterThanBitmapAvx2(const int16_t* levels, int64_t num_levels, int16_t rhs);
+#endif
+
+namespace {
+
+using ::arrow::internal::DispatchLevel;
+using ::arrow::internal::DynamicDispatch;
+
+// defined in level_comparison_avx2.cc
+
+struct GreaterThanDynamicFunction {
+  using FunctionType = decltype(&GreaterThanBitmap);
+
+  static std::vector<std::pair<DispatchLevel, FunctionType>> implementations() {
+    return {
+      { DispatchLevel::NONE, standard::GreaterThanBitmapImpl }
+#if defined(ARROW_HAVE_RUNTIME_AVX2)
+      , { DispatchLevel::AVX2, GreaterThanBitmapAvx2 }
+#endif
+    };
+  }
+};
+
+struct MinMaxDynamicFunction {
+  using FunctionType = decltype(&FindMinMax);
+
+  static std::vector<std::pair<DispatchLevel, FunctionType>> implementations() {
+    return {
+      { DispatchLevel::NONE, standard::FindMinMaxImpl }
+#if defined(ARROW_HAVE_RUNTIME_AVX2)
+      , { DispatchLevel::AVX2, FindMinMaxAvx2 }
+#endif
+    };
+  }
+};
+
+}  // namespace
+
+uint64_t GreaterThanBitmap(const int16_t* levels, int64_t num_levels, int16_t rhs) {
+  static DynamicDispatch<GreaterThanDynamicFunction> dispatch;
+  return dispatch.func(levels, num_levels, rhs);
+}
+
+MinMax FindMinMax(const int16_t* levels, int64_t num_levels) {
+  static DynamicDispatch<MinMaxDynamicFunction> dispatch;
+  return dispatch.func(levels, num_levels);
+}
+
+}  // namespace internal
+}  // namespace parquet
diff --git a/cpp/src/parquet/level_comparison.h b/cpp/src/parquet/level_comparison.h
new file mode 100644
index 00000000000..38e7ef8e2ec
--- /dev/null
+++ b/cpp/src/parquet/level_comparison.h
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include <algorithm>
+#include <cstdint>
+
+#include "parquet/platform.h"
+
+namespace parquet {
+namespace internal {
+
+/// Builds a  bitmap where each set bit indicates the corresponding level is greater
+/// than rhs.
+uint64_t PARQUET_EXPORT GreaterThanBitmap(const int16_t* levels, int64_t num_levels,
+                                          int16_t rhs);
+
+struct MinMax {
+  int16_t min;
+  int16_t max;
+};
+
+MinMax FindMinMax(const int16_t* levels, int64_t num_levels);
+
+}  // namespace internal
+}  // namespace parquet
diff --git a/cpp/src/parquet/level_comparison_avx2.cc b/cpp/src/parquet/level_comparison_avx2.cc
new file mode 100644
index 00000000000..b33eb2e2953
--- /dev/null
+++ b/cpp/src/parquet/level_comparison_avx2.cc
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#define PARQUET_IMPL_NAMESPACE avx2
+#include "parquet/level_comparison_inc.h"
+#undef PARQUET_IMPL_NAMESPACE
+
+namespace parquet {
+namespace internal {
+
+uint64_t GreaterThanBitmapAvx2(const int16_t* levels, int64_t num_levels, int16_t rhs) {
+  return avx2::GreaterThanBitmapImpl(levels, num_levels, rhs);
+}
+
+MinMax FindMinMaxAvx2(const int16_t* levels, int64_t num_levels) {
+  return avx2::FindMinMaxImpl(levels, num_levels);
+}
+
+}  // namespace internal
+}  // namespace parquet
diff --git a/cpp/src/parquet/level_comparison_inc.h b/cpp/src/parquet/level_comparison_inc.h
new file mode 100644
index 00000000000..f4cf7ab48e7
--- /dev/null
+++ b/cpp/src/parquet/level_comparison_inc.h
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include "arrow/util/bit_util.h"
+#include "parquet/level_comparison.h"
+
+// Used to make sure ODR rule isn't violated.
+#ifndef PARQUET_IMPL_NAMESPACE
+#error "PARQUET_IMPL_NAMESPACE must be defined"
+#endif
+namespace parquet {
+namespace internal {
+namespace PARQUET_IMPL_NAMESPACE {
+/// Builds a bitmap by applying predicate to the level vector provided.
+///
+/// \param[in] levels Rep or def level array.
+/// \param[in] num_levels The number of levels to process (must be [0, 64])
+/// \param[in] predicate The predicate to apply (must have the signature `bool
+/// predicate(int16_t)`.
+/// \returns The bitmap using least significant "bit" ordering.
+///
+template <typename Predicate>
+inline uint64_t LevelsToBitmap(const int16_t* levels, int64_t num_levels,
+                               Predicate predicate) {
+  // Both clang and GCC can vectorize this automatically with SSE4/AVX2.
+  uint64_t mask = 0;
+  for (int x = 0; x < num_levels; x++) {
+    mask |= static_cast<uint64_t>(predicate(levels[x]) ? 1 : 0) << x;
+  }
+  return ::arrow::BitUtil::ToLittleEndian(mask);
+}
+
+inline MinMax FindMinMaxImpl(const int16_t* levels, int64_t num_levels) {
+  MinMax out{std::numeric_limits<int16_t>::max(), std::numeric_limits<int16_t>::min()};
+  for (int x = 0; x < num_levels; x++) {
+    out.min = std::min(levels[x], out.min);
+    out.max = std::max(levels[x], out.max);
+  }
+  return out;
+}
+
+inline uint64_t GreaterThanBitmapImpl(const int16_t* levels, int64_t num_levels,
+                                      int16_t rhs) {
+  return LevelsToBitmap(levels, num_levels, [rhs](int16_t value) { return value > rhs; });
+}
+
+}  // namespace PARQUET_IMPL_NAMESPACE
+}  // namespace internal
+}  // namespace parquet
diff --git a/cpp/src/parquet/level_conversion.cc b/cpp/src/parquet/level_conversion.cc
index cfa5df1a7e0..fadd56e1de9 100644
--- a/cpp/src/parquet/level_conversion.cc
+++ b/cpp/src/parquet/level_conversion.cc
@@ -18,176 +18,163 @@
 
 #include <algorithm>
 #include <limits>
-#if defined(ARROW_HAVE_BMI2)
-#include <x86intrin.h>
-#endif
 
+#include "arrow/util/bit_run_reader.h"
 #include "arrow/util/bit_util.h"
+#include "arrow/util/cpu_info.h"
 #include "arrow/util/logging.h"
 #include "parquet/exception.h"
 
+#include "parquet/level_comparison.h"
+#define PARQUET_IMPL_NAMESPACE standard
+#include "parquet/level_conversion_inc.h"
+#undef PARQUET_IMPL_NAMESPACE
+
 namespace parquet {
 namespace internal {
 namespace {
-inline void CheckLevelRange(const int16_t* levels, int64_t num_levels,
-                            const int16_t max_expected_level) {
-  int16_t min_level = std::numeric_limits<int16_t>::max();
-  int16_t max_level = std::numeric_limits<int16_t>::min();
-  for (int x = 0; x < num_levels; x++) {
-    min_level = std::min(levels[x], min_level);
-    max_level = std::max(levels[x], max_level);
-  }
-  if (ARROW_PREDICT_FALSE(num_levels > 0 &&
-                          (min_level < 0 || max_level > max_expected_level))) {
-    throw ParquetException("definition level exceeds maximum");
+
+using ::arrow::internal::CpuInfo;
+
+template <typename OffsetType>
+void DefRepLevelsToListInfo(const int16_t* def_levels, const int16_t* rep_levels,
+                            int64_t num_def_levels, LevelInfo level_info,
+                            ValidityBitmapInputOutput* output, OffsetType* offsets) {
+  OffsetType* orig_pos = offsets;
+  std::unique_ptr<::arrow::internal::FirstTimeBitmapWriter> valid_bits_writer;
+  if (output->valid_bits) {
+    valid_bits_writer.reset(new ::arrow::internal::FirstTimeBitmapWriter(
+        output->valid_bits, output->valid_bits_offset, num_def_levels));
   }
-}
+  for (int x = 0; x < num_def_levels; x++) {
+    // Skip items that belong to empty or null ancestor lists and further nested lists.
+    if (def_levels[x] < level_info.repeated_ancestor_def_level ||
+        rep_levels[x] > level_info.rep_level) {
+      continue;
+    }
 
-#if !defined(ARROW_HAVE_AVX512)
-
-inline void DefinitionLevelsToBitmapScalar(
-    const int16_t* def_levels, int64_t num_def_levels, const int16_t max_definition_level,
-    const int16_t max_repetition_level, int64_t* values_read, int64_t* null_count,
-    uint8_t* valid_bits, int64_t valid_bits_offset) {
-  // We assume here that valid_bits is large enough to accommodate the
-  // additional definition levels and the ones that have already been written
-  ::arrow::internal::BitmapWriter valid_bits_writer(valid_bits, valid_bits_offset,
-                                                    num_def_levels);
-
-  // TODO(itaiin): As an interim solution we are splitting the code path here
-  // between repeated+flat column reads, and non-repeated+nested reads.
-  // Those paths need to be merged in the future
-  for (int i = 0; i < num_def_levels; ++i) {
-    if (def_levels[i] == max_definition_level) {
-      valid_bits_writer.Set();
-    } else if (max_repetition_level > 0) {
-      // repetition+flat case
-      if (def_levels[i] == (max_definition_level - 1)) {
-        valid_bits_writer.Clear();
-        *null_count += 1;
-      } else {
-        continue;
+    if (rep_levels[x] == level_info.rep_level) {
+      // A continuation of an existing list.
+      // offsets can be null for structs with repeated children (we don't need to know
+      // offsets until we get to the children).
+      if (offsets != nullptr) {
+        if (ARROW_PREDICT_FALSE(*offsets == std::numeric_limits<OffsetType>::max())) {
+          throw ParquetException("List index overflow.");
+        }
+        *offsets += 1;
       }
     } else {
-      // non-repeated+nested case
-      if (def_levels[i] < max_definition_level) {
-        valid_bits_writer.Clear();
-        *null_count += 1;
-      } else {
-        throw ParquetException("definition level exceeds maximum");
+      if (ARROW_PREDICT_FALSE(
+              (valid_bits_writer != nullptr &&
+               valid_bits_writer->position() >= output->values_read_upper_bound) ||
+              (offsets - orig_pos) >= output->values_read_upper_bound)) {
+        std::stringstream ss;
+        ss << "Definition levels exceeded upper bound: "
+           << output->values_read_upper_bound;
+        throw ParquetException(ss.str());
       }
-    }
 
-    valid_bits_writer.Next();
+      // current_rep < list rep_level i.e. start of a list (ancestor empty lists are
+      // filtered out above).
+      // offsets can be null for structs with repeated children (we don't need to know
+      // offsets until we get to the children).
+      if (offsets != nullptr) {
+        ++offsets;
+        // Use cumulative offsets because variable size lists are more common then
+        // fixed size lists so it should be cheaper to make these cumulative and
+        // subtract when validating fixed size lists.
+        *offsets = *(offsets - 1);
+        if (def_levels[x] >= level_info.def_level) {
+          if (ARROW_PREDICT_FALSE(*offsets == std::numeric_limits<OffsetType>::max())) {
+            throw ParquetException("List index overflow.");
+          }
+          *offsets += 1;
+        }
+      }
+
+      if (valid_bits_writer != nullptr) {
+        // the level_info def level for lists reflects element present level.
+        // the prior level distinguishes between empty lists.
+        if (def_levels[x] >= level_info.def_level - 1) {
+          valid_bits_writer->Set();
+        } else {
+          output->null_count++;
+          valid_bits_writer->Clear();
+        }
+        valid_bits_writer->Next();
+      }
+    }
+  }
+  if (valid_bits_writer != nullptr) {
+    valid_bits_writer->Finish();
+  }
+  if (offsets != nullptr) {
+    output->values_read = offsets - orig_pos;
+  } else if (valid_bits_writer != nullptr) {
+    output->values_read = valid_bits_writer->position();
+  }
+  if (output->null_count > 0 && level_info.null_slot_usage > 1) {
+    throw ParquetException(
+        "Null values with null_slot_usage > 1 not supported."
+        "(i.e. FixedSizeLists with null values are not supported)");
   }
-  valid_bits_writer.Finish();
-  *values_read = valid_bits_writer.position();
 }
+
+}  // namespace
+
+#if defined(ARROW_HAVE_RUNTIME_BMI2)
+// defined in level_conversion_bmi2.cc for dynamic dispatch.
+void DefLevelsToBitmapBmi2WithRepeatedParent(const int16_t* def_levels,
+                                             int64_t num_def_levels, LevelInfo level_info,
+                                             ValidityBitmapInputOutput* output);
 #endif
 
-template <bool has_repeated_parent>
-int64_t DefinitionLevelsBatchToBitmap(const int16_t* def_levels, const int64_t batch_size,
-                                      const int16_t required_definition_level,
-                                      ::arrow::internal::FirstTimeBitmapWriter* writer) {
-  CheckLevelRange(def_levels, batch_size, required_definition_level);
-  uint64_t defined_bitmap =
-      internal::GreaterThanBitmap(def_levels, batch_size, required_definition_level - 1);
-
-  DCHECK_LE(batch_size, 64);
-  if (has_repeated_parent) {
-#if defined(ARROW_HAVE_BMI2)
-    // This is currently a specialized code path assuming only (nested) lists
-    // present through the leaf (i.e. no structs). Upper level code only calls
-    // this method when the leaf-values are nullable (otherwise no spacing is needed),
-    // Because only nested lists exists it is sufficient to know that the field
-    // was either null or included it (i.e. definition level > max_definitation_level
-    // -2) If there where structs mixed in, we need to know the def_level of the
-    // repeated parent so we can check for def_level > "def level of repeated parent".
-    uint64_t present_bitmap = internal::GreaterThanBitmap(def_levels, batch_size,
-                                                          required_definition_level - 2);
-    uint64_t selected_bits = _pext_u64(defined_bitmap, present_bitmap);
-    writer->AppendWord(selected_bits, ::arrow::BitUtil::PopCount(present_bitmap));
-    return ::arrow::BitUtil::PopCount(selected_bits);
-#else
-    assert(false && "must not execute this without BMI2");
+void DefLevelsToBitmap(const int16_t* def_levels, int64_t num_def_levels,
+                       LevelInfo level_info, ValidityBitmapInputOutput* output) {
+  // It is simpler to rely on rep_level here until PARQUET-1899 is done and the code
+  // is deleted in a follow-up release.
+  if (level_info.rep_level > 0) {
+#if defined(ARROW_HAVE_RUNTIME_BMI2)
+    if (CpuInfo::GetInstance()->HasEfficientBmi2()) {
+      return DefLevelsToBitmapBmi2WithRepeatedParent(def_levels, num_def_levels,
+                                                     level_info, output);
+    }
 #endif
+    standard::DefLevelsToBitmapSimd</*has_repeated_parent=*/true>(
+        def_levels, num_def_levels, level_info, output);
   } else {
-    writer->AppendWord(defined_bitmap, batch_size);
-    return ::arrow::BitUtil::PopCount(defined_bitmap);
+    standard::DefLevelsToBitmapSimd</*has_repeated_parent=*/false>(
+        def_levels, num_def_levels, level_info, output);
   }
 }
 
-template <bool has_repeated_parent>
-void DefinitionLevelsToBitmapSimd(const int16_t* def_levels, int64_t num_def_levels,
-                                  const int16_t required_definition_level,
-                                  int64_t* values_read, int64_t* null_count,
-                                  uint8_t* valid_bits, int64_t valid_bits_offset) {
-  constexpr int64_t kBitMaskSize = 64;
-  ::arrow::internal::FirstTimeBitmapWriter writer(valid_bits,
-                                                  /*start_offset=*/valid_bits_offset,
-                                                  /*length=*/num_def_levels);
-  int64_t set_count = 0;
-  *values_read = 0;
-  while (num_def_levels > kBitMaskSize) {
-    set_count += DefinitionLevelsBatchToBitmap<has_repeated_parent>(
-        def_levels, kBitMaskSize, required_definition_level, &writer);
-    def_levels += kBitMaskSize;
-    num_def_levels -= kBitMaskSize;
-  }
-  set_count += DefinitionLevelsBatchToBitmap<has_repeated_parent>(
-      def_levels, num_def_levels, required_definition_level, &writer);
-
-  *values_read = writer.position();
-  *null_count += *values_read - set_count;
-  writer.Finish();
+uint64_t TestOnlyExtractBitsSoftware(uint64_t bitmap, uint64_t select_bitmap) {
+  return standard::ExtractBitsSoftware(bitmap, select_bitmap);
 }
 
-void DefinitionLevelsToBitmapLittleEndian(
-    const int16_t* def_levels, int64_t num_def_levels, const int16_t max_definition_level,
-    const int16_t max_repetition_level, int64_t* values_read, int64_t* null_count,
-    uint8_t* valid_bits, int64_t valid_bits_offset) {
-  if (max_repetition_level > 0) {
-// This is a short term hack to prevent using the pext BMI2 instructions
-// on non-intel platforms where performance is subpar.
-// In the medium term we will hopefully be able to runtime dispatch
-// to use this on intel only platforms that support pext.
-#if defined(ARROW_HAVE_AVX512)
-    // BMI2 is required for efficient bit extraction.
-    DefinitionLevelsToBitmapSimd</*has_repeated_parent=*/true>(
-        def_levels, num_def_levels, max_definition_level, values_read, null_count,
-        valid_bits, valid_bits_offset);
-#else
-    DefinitionLevelsToBitmapScalar(def_levels, num_def_levels, max_definition_level,
-                                   max_repetition_level, values_read, null_count,
-                                   valid_bits, valid_bits_offset);
-#endif  // ARROW_HAVE_BMI2
-
-  } else {
-    // No BMI2 intsturctions are used for non-repeated case.
-    DefinitionLevelsToBitmapSimd</*has_repeated_parent=*/false>(
-        def_levels, num_def_levels, max_definition_level, values_read, null_count,
-        valid_bits, valid_bits_offset);
-  }
+void DefRepLevelsToList(const int16_t* def_levels, const int16_t* rep_levels,
+                        int64_t num_def_levels, LevelInfo level_info,
+                        ValidityBitmapInputOutput* output, int32_t* offsets) {
+  DefRepLevelsToListInfo<int32_t>(def_levels, rep_levels, num_def_levels, level_info,
+                                  output, offsets);
 }
 
-}  // namespace
-
-void DefinitionLevelsToBitmap(const int16_t* def_levels, int64_t num_def_levels,
-                              const int16_t max_definition_level,
-                              const int16_t max_repetition_level, int64_t* values_read,
-                              int64_t* null_count, uint8_t* valid_bits,
-                              int64_t valid_bits_offset) {
-#if ARROW_LITTLE_ENDIAN
-  DefinitionLevelsToBitmapLittleEndian(def_levels, num_def_levels, max_definition_level,
-                                       max_repetition_level, values_read, null_count,
-                                       valid_bits, valid_bits_offset);
-
-#else
-  DefinitionLevelsToBitmapScalar(def_levels, num_def_levels, max_definition_level,
-                                 max_repetition_level, values_read, null_count,
-                                 valid_bits, valid_bits_offset);
+void DefRepLevelsToList(const int16_t* def_levels, const int16_t* rep_levels,
+                        int64_t num_def_levels, LevelInfo level_info,
+                        ValidityBitmapInputOutput* output, int64_t* offsets) {
+  DefRepLevelsToListInfo<int64_t>(def_levels, rep_levels, num_def_levels, level_info,
+                                  output, offsets);
+}
 
-#endif
+void DefRepLevelsToBitmap(const int16_t* def_levels, const int16_t* rep_levels,
+                          int64_t num_def_levels, LevelInfo level_info,
+                          ValidityBitmapInputOutput* output) {
+  // DefReplevelsToListInfo assumes it for the actual list method and this
+  // method is for parent structs, so we need to bump def and ref level.
+  level_info.rep_level += 1;
+  level_info.def_level += 1;
+  DefRepLevelsToListInfo<int32_t>(def_levels, rep_levels, num_def_levels, level_info,
+                                  output, /*offsets=*/nullptr);
 }
 
 }  // namespace internal
diff --git a/cpp/src/parquet/level_conversion.h b/cpp/src/parquet/level_conversion.h
index ff91ca4606e..d406724ce16 100644
--- a/cpp/src/parquet/level_conversion.h
+++ b/cpp/src/parquet/level_conversion.h
@@ -20,48 +20,179 @@
 #include <cstdint>
 
 #include "parquet/platform.h"
+#include "parquet/schema.h"
 
 namespace parquet {
 namespace internal {
 
-void PARQUET_EXPORT DefinitionLevelsToBitmap(
-    const int16_t* def_levels, int64_t num_def_levels, const int16_t max_definition_level,
-    const int16_t max_repetition_level, int64_t* values_read, int64_t* null_count,
-    uint8_t* valid_bits, int64_t valid_bits_offset);
-
-// These APIs are likely to be revised as part of ARROW-8494 to reduce duplicate code.
-// They currently represent minimal functionality for vectorized computation of definition
-// levels.
-
-#if defined(ARROW_LITTLE_ENDIAN)
-/// Builds a bitmap by applying predicate to the level vector provided.
-///
-/// \param[in] levels Rep or def level array.
-/// \param[in] num_levels The number of levels to process (must be [0, 64])
-/// \param[in] predicate The predicate to apply (must have the signature `bool
-/// predicate(int16_t)`.
-/// \returns The bitmap using least significant "bit" ordering.
-///
-/// N.B. Correct byte ordering is dependent on little-endian architectures.
-///
-template <typename Predicate>
-uint64_t LevelsToBitmap(const int16_t* levels, int64_t num_levels, Predicate predicate) {
-  // Both clang and GCC can vectorize this automatically with SSE4/AVX2.
-  uint64_t mask = 0;
-  for (int x = 0; x < num_levels; x++) {
-    mask |= static_cast<uint64_t>(predicate(levels[x]) ? 1 : 0) << x;
+struct PARQUET_EXPORT LevelInfo {
+  LevelInfo()
+      : null_slot_usage(1), def_level(0), rep_level(0), repeated_ancestor_def_level(0) {}
+  LevelInfo(int32_t null_slots, int32_t definition_level, int32_t repetition_level,
+            int32_t repeated_ancestor_definition_level)
+      : null_slot_usage(null_slots),
+        def_level(definition_level),
+        rep_level(repetition_level),
+        repeated_ancestor_def_level(repeated_ancestor_definition_level) {}
+
+  bool operator==(const LevelInfo& b) const {
+    return null_slot_usage == b.null_slot_usage && def_level == b.def_level &&
+           rep_level == b.rep_level &&
+           repeated_ancestor_def_level == b.repeated_ancestor_def_level;
+  }
+
+  bool HasNullableValues() const { return repeated_ancestor_def_level < def_level; }
+
+  // How many slots an undefined but present (i.e. null) element in
+  // parquet consumes when decoding to Arrow.
+  // "Slot" is used in the same context as the Arrow specification
+  // (i.e. a value holder).
+  // This is only ever >1 for descendents of FixedSizeList.
+  int32_t null_slot_usage = 1;
+
+  // The definition level at which the value for the field
+  // is considered not null (definition levels greater than
+  // or equal to this value indicate a not-null
+  // value for the field). For list fields definition levels
+  // greater than or equal to this field indicate a present,
+  // possibly null, child value.
+  int16_t def_level = 0;
+
+  // The repetition level corresponding to this element
+  // or the closest repeated ancestor.  Any repetition
+  // level less than this indicates either a new list OR
+  // an empty list (which is determined in conjunction
+  // with definition levels).
+  int16_t rep_level = 0;
+
+  // The definition level indicating the level at which the closest
+  // repeated ancestor is not empty.  This is used to discriminate
+  // between a value less than |def_level| being null or excluded entirely.
+  // For instance if we have an arrow schema like:
+  // list(struct(f0: int)).  Then then there are the following
+  // definition levels:
+  //   0 = null list
+  //   1 = present but empty list.
+  //   2 = a null value in the list
+  //   3 = a non null struct but null integer.
+  //   4 = a present integer.
+  // When reconstructing, the struct and integer arrays'
+  // repeated_ancestor_def_level would be 2.  Any
+  // def_level < 2 indicates that there isn't a corresponding
+  // child value in the list.
+  // i.e. [null, [], [null], [{f0: null}], [{f0: 1}]]
+  // has the def levels [0, 1, 2, 3, 4].  The actual
+  // struct array is only of length 3: [not-set, set, set] and
+  // the int array is also of length 3: [N/A, null, 1].
+  //
+  int16_t repeated_ancestor_def_level = 0;
+
+  /// Increments levels according to the cardinality of node.
+  void Increment(const schema::Node& node) {
+    if (node.is_repeated()) {
+      IncrementRepeated();
+      return;
+    }
+    if (node.is_optional()) {
+      IncrementOptional();
+      return;
+    }
+  }
+
+  /// Incremetns level for a optional node.
+  void IncrementOptional() { def_level++; }
+
+  /// Increments levels for the repeated node.  Returns
+  /// the previous ancestor_list_def_level.
+  int16_t IncrementRepeated() {
+    int16_t last_repeated_ancestor = repeated_ancestor_def_level;
+
+    // Repeated fields add both a repetition and definition level. This is used
+    // to distinguish between an empty list and a list with an item in it.
+    ++rep_level;
+    ++def_level;
+    // For levels >= repeated_ancenstor_def_level it indicates the list was
+    // non-null and had at least one element.  This is important
+    // for later decoding because we need to add a slot for these
+    // values.  for levels < current_def_level no slots are added
+    // to arrays.
+    repeated_ancestor_def_level = def_level;
+    return last_repeated_ancestor;
+  }
+
+  friend std::ostream& operator<<(std::ostream& os, const LevelInfo& levels) {
+    // This print method is to silence valgrind issues.  What's printed
+    // is not important because all asserts happen directly on
+    // members.
+    os << "{def=" << levels.def_level << ", rep=" << levels.rep_level
+       << ", repeated_ancestor_def=" << levels.repeated_ancestor_def_level;
+    if (levels.null_slot_usage > 1) {
+      os << ", null_slot_usage=" << levels.null_slot_usage;
+    }
+    os << "}";
+    return os;
   }
-  return mask;
-}
+};
+
+// Input/Output structure for reconstructed validity bitmaps.
+struct PARQUET_EXPORT ValidityBitmapInputOutput {
+  // Input only.
+  // The maximum number of values_read expected (actual
+  // values read must be less than or equal to this value.
+  // If this number is exceeded methods will throw a
+  // ParquetException. Exceeding this limit indicates
+  // either a corrupt or incorrectly written file.
+  int64_t values_read_upper_bound = 0;
+  // Output only. The number of values added to the encountered
+  // (this is logicallyt he count of the number of elements
+  // for an Arrow array).
+  int64_t values_read = 0;
+  // Input/Output. The number of nulls encountered.
+  int64_t null_count = 0;
+  // Output only. The validity bitmap to populate. May be be null only
+  // for DefRepLevelsToListInfo (if all that is needed is list offsets).
+  uint8_t* valid_bits = NULLPTR;
+  // Input only, offset into valid_bits to start at.
+  int64_t valid_bits_offset = 0;
+};
+
+//  Converts def_levels to validity bitmaps for non-list arrays and structs that have
+//  at least one member that is not a list and has no list descendents.
+//  For lists use DefRepLevelsToList and structs where all descendants contain
+//  a list use DefRepLevelsToBitmap.
+void PARQUET_EXPORT DefLevelsToBitmap(const int16_t* def_levels, int64_t num_def_levels,
+                                      LevelInfo level_info,
+                                      ValidityBitmapInputOutput* output);
+
+// Reconstructs a validity bitmap and list offsets for a list arrays based on
+// def/rep levels. The first element of offsets will not be modified if rep_levels
+// starts with a new list.  The first element of offsets will be used when calculating
+// the next offset.  See documentation onf DefLevelsToBitmap for when to use this
+// method vs the other ones in this file for reconstruction.
+//
+// Offsets must be sized to 1 + values_read_upper_bound.
+void PARQUET_EXPORT DefRepLevelsToList(const int16_t* def_levels,
+                                       const int16_t* rep_levels, int64_t num_def_levels,
+                                       LevelInfo level_info,
+                                       ValidityBitmapInputOutput* output,
+                                       int32_t* offsets);
+void PARQUET_EXPORT DefRepLevelsToList(const int16_t* def_levels,
+                                       const int16_t* rep_levels, int64_t num_def_levels,
+                                       LevelInfo level_info,
+                                       ValidityBitmapInputOutput* output,
+                                       int64_t* offsets);
 
-/// Builds a  bitmap where each set bit indicates the corresponding level is greater
-/// than rhs.
-static inline uint64_t GreaterThanBitmap(const int16_t* levels, int64_t num_levels,
-                                         int16_t rhs) {
-  return LevelsToBitmap(levels, num_levels, [rhs](int16_t value) { return value > rhs; });
-}
+// Reconstructs a validity bitmap for a struct every member is a list or has
+// a list descendant.  See documentation on DefLevelsToBitmap for when more
+// details on this method compared to the other ones defined above.
+void PARQUET_EXPORT DefRepLevelsToBitmap(const int16_t* def_levels,
+                                         const int16_t* rep_levels,
+                                         int64_t num_def_levels, LevelInfo level_info,
+                                         ValidityBitmapInputOutput* output);
 
-#endif
+// This is exposed to ensure we can properly test a software simulated pext function
+// (i.e. it isn't hidden by runtime dispatch).
+uint64_t PARQUET_EXPORT TestOnlyExtractBitsSoftware(uint64_t bitmap, uint64_t selection);
 
 }  // namespace internal
 }  // namespace parquet
diff --git a/cpp/src/parquet/level_conversion_benchmark.cc b/cpp/src/parquet/level_conversion_benchmark.cc
index 4f15838d339..f9e91c4820f 100644
--- a/cpp/src/parquet/level_conversion_benchmark.cc
+++ b/cpp/src/parquet/level_conversion_benchmark.cc
@@ -34,15 +34,17 @@ constexpr int16_t kHasRepeatedElements = 1;
 
 std::vector<uint8_t> RunDefinitionLevelsToBitmap(const std::vector<int16_t>& def_levels,
                                                  ::benchmark::State* state) {
-  int64_t values_read = 0;
-  int64_t null_count = 0;
   std::vector<uint8_t> bitmap(/*count=*/def_levels.size(), 0);
-  int rep = 0;
+  parquet::internal::LevelInfo info;
+  info.def_level = kHasRepeatedElements;
+  info.repeated_ancestor_def_level = kPresentDefLevel;
+  info.rep_level = 1;
+  parquet::internal::ValidityBitmapInputOutput validity_io;
+  validity_io.values_read_upper_bound = def_levels.size();
+  validity_io.valid_bits = bitmap.data();
   for (auto _ : *state) {
-    parquet::internal::DefinitionLevelsToBitmap(
-        def_levels.data(), def_levels.size(), /*max_definition_level=*/kPresentDefLevel,
-        /*max_repetition_level=*/kHasRepeatedElements, &values_read, &null_count,
-        bitmap.data(), /*valid_bits_offset=*/(rep++ % 8) * def_levels.size());
+    parquet::internal::DefLevelsToBitmap(def_levels.data(), def_levels.size(), info,
+                                         &validity_io);
   }
   state->SetBytesProcessed(int64_t(state->iterations()) * def_levels.size());
   return bitmap;
diff --git a/cpp/src/parquet/level_conversion_bmi2.cc b/cpp/src/parquet/level_conversion_bmi2.cc
new file mode 100644
index 00000000000..274d54e503c
--- /dev/null
+++ b/cpp/src/parquet/level_conversion_bmi2.cc
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include "parquet/level_conversion.h"
+
+#define PARQUET_IMPL_NAMESPACE bmi2
+#include "parquet/level_conversion_inc.h"
+#undef PARQUET_IMPL_NAMESPACE
+
+namespace parquet {
+namespace internal {
+void DefLevelsToBitmapBmi2WithRepeatedParent(const int16_t* def_levels,
+                                             int64_t num_def_levels, LevelInfo level_info,
+                                             ValidityBitmapInputOutput* output) {
+  bmi2::DefLevelsToBitmapSimd</*has_repeated_parent=*/true>(def_levels, num_def_levels,
+                                                            level_info, output);
+}
+
+}  // namespace internal
+}  // namespace parquet
diff --git a/cpp/src/parquet/level_conversion_inc.h b/cpp/src/parquet/level_conversion_inc.h
new file mode 100644
index 00000000000..ae9a7c7fec3
--- /dev/null
+++ b/cpp/src/parquet/level_conversion_inc.h
@@ -0,0 +1,356 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include "parquet/level_conversion.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <limits>
+
+#include "arrow/util/bit_run_reader.h"
+#include "arrow/util/bit_util.h"
+#include "arrow/util/logging.h"
+#include "arrow/util/simd.h"
+#include "parquet/exception.h"
+#include "parquet/level_comparison.h"
+
+namespace parquet {
+namespace internal {
+#ifndef PARQUET_IMPL_NAMESPACE
+#error "PARQUET_IMPL_NAMESPACE must be defined"
+#endif
+namespace PARQUET_IMPL_NAMESPACE {
+
+// clang-format off
+/* Python code to generate lookup table:
+
+kLookupBits = 5
+count = 0
+print('constexpr int kLookupBits = {};'.format(kLookupBits))
+print('constexpr uint8_t kPextTable[1 << kLookupBits][1 << kLookupBits] = {')
+print(' ', end = '')
+for mask in range(1 << kLookupBits):
+    for data in range(1 << kLookupBits):
+        bit_value = 0
+        bit_len = 0
+        for i in range(kLookupBits):
+            if mask & (1 << i):
+                bit_value |= (((data >> i) & 1) << bit_len)
+                bit_len += 1
+        out = '0x{:02X},'.format(bit_value)
+        count += 1
+        if count % (1 << kLookupBits) == 1:
+            print(' {')
+        if count % 8 == 1:
+            print('    ', end = '')
+        if count % 8 == 0:
+            print(out, end = '\n')
+        else:
+            print(out, end = ' ')
+        if count % (1 << kLookupBits) == 0:
+            print('  },', end = '')
+print('\n};')
+
+*/
+// clang-format on
+
+constexpr int kLookupBits = 5;
+constexpr uint8_t kPextTable[1 << kLookupBits][1 << kLookupBits] = {
+    {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    },
+    {
+        0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00,
+        0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01,
+        0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01,
+    },
+    {
+        0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01,
+        0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00,
+        0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01,
+    },
+    {
+        0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02,
+        0x03, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01,
+        0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03,
+    },
+    {
+        0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00,
+        0x00, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01,
+        0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01,
+    },
+    {
+        0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x00, 0x01, 0x00,
+        0x01, 0x02, 0x03, 0x02, 0x03, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03,
+        0x02, 0x03, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03,
+    },
+    {
+        0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x00, 0x00, 0x01,
+        0x01, 0x02, 0x02, 0x03, 0x03, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02,
+        0x03, 0x03, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03,
+    },
+    {
+        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02,
+        0x03, 0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05,
+        0x06, 0x07, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+    },
+    {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01,
+        0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    },
+    {
+        0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02,
+        0x03, 0x02, 0x03, 0x02, 0x03, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01,
+        0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x02, 0x03, 0x02, 0x03,
+    },
+    {
+        0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03,
+        0x03, 0x02, 0x02, 0x03, 0x03, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00,
+        0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x02, 0x02, 0x03, 0x03,
+    },
+    {
+        0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
+        0x07, 0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01,
+        0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x04, 0x05, 0x06, 0x07,
+    },
+    {
+        0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02,
+        0x02, 0x03, 0x03, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01,
+        0x01, 0x01, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03,
+    },
+    {
+        0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x04, 0x05, 0x04,
+        0x05, 0x06, 0x07, 0x06, 0x07, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03,
+        0x02, 0x03, 0x04, 0x05, 0x04, 0x05, 0x06, 0x07, 0x06, 0x07,
+    },
+    {
+        0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x04, 0x04, 0x05,
+        0x05, 0x06, 0x06, 0x07, 0x07, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02,
+        0x03, 0x03, 0x04, 0x04, 0x05, 0x05, 0x06, 0x06, 0x07, 0x07,
+    },
+    {
+        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A,
+        0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05,
+        0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
+    },
+    {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+        0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    },
+    {
+        0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00,
+        0x01, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x02, 0x03,
+        0x02, 0x03, 0x02, 0x03, 0x02, 0x03, 0x02, 0x03, 0x02, 0x03,
+    },
+    {
+        0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01,
+        0x01, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x02, 0x02,
+        0x03, 0x03, 0x02, 0x02, 0x03, 0x03, 0x02, 0x02, 0x03, 0x03,
+    },
+    {
+        0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02,
+        0x03, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x04, 0x05,
+        0x06, 0x07, 0x04, 0x05, 0x06, 0x07, 0x04, 0x05, 0x06, 0x07,
+    },
+    {
+        0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00,
+        0x00, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03,
+        0x03, 0x03, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03,
+    },
+    {
+        0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x00, 0x01, 0x00,
+        0x01, 0x02, 0x03, 0x02, 0x03, 0x04, 0x05, 0x04, 0x05, 0x06, 0x07,
+        0x06, 0x07, 0x04, 0x05, 0x04, 0x05, 0x06, 0x07, 0x06, 0x07,
+    },
+    {
+        0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x00, 0x00, 0x01,
+        0x01, 0x02, 0x02, 0x03, 0x03, 0x04, 0x04, 0x05, 0x05, 0x06, 0x06,
+        0x07, 0x07, 0x04, 0x04, 0x05, 0x05, 0x06, 0x06, 0x07, 0x07,
+    },
+    {
+        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02,
+        0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D,
+        0x0E, 0x0F, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
+    },
+    {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01,
+        0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
+        0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
+    },
+    {
+        0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02,
+        0x03, 0x02, 0x03, 0x02, 0x03, 0x04, 0x05, 0x04, 0x05, 0x04, 0x05,
+        0x04, 0x05, 0x06, 0x07, 0x06, 0x07, 0x06, 0x07, 0x06, 0x07,
+    },
+    {
+        0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03,
+        0x03, 0x02, 0x02, 0x03, 0x03, 0x04, 0x04, 0x05, 0x05, 0x04, 0x04,
+        0x05, 0x05, 0x06, 0x06, 0x07, 0x07, 0x06, 0x06, 0x07, 0x07,
+    },
+    {
+        0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
+        0x07, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x08, 0x09,
+        0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x0C, 0x0D, 0x0E, 0x0F,
+    },
+    {
+        0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02,
+        0x02, 0x03, 0x03, 0x03, 0x03, 0x04, 0x04, 0x04, 0x04, 0x05, 0x05,
+        0x05, 0x05, 0x06, 0x06, 0x06, 0x06, 0x07, 0x07, 0x07, 0x07,
+    },
+    {
+        0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x04, 0x05, 0x04,
+        0x05, 0x06, 0x07, 0x06, 0x07, 0x08, 0x09, 0x08, 0x09, 0x0A, 0x0B,
+        0x0A, 0x0B, 0x0C, 0x0D, 0x0C, 0x0D, 0x0E, 0x0F, 0x0E, 0x0F,
+    },
+    {
+        0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x04, 0x04, 0x05,
+        0x05, 0x06, 0x06, 0x07, 0x07, 0x08, 0x08, 0x09, 0x09, 0x0A, 0x0A,
+        0x0B, 0x0B, 0x0C, 0x0C, 0x0D, 0x0D, 0x0E, 0x0E, 0x0F, 0x0F,
+    },
+    {
+        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A,
+        0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15,
+        0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F,
+    },
+};
+
+inline uint64_t ExtractBitsSoftware(uint64_t bitmap, uint64_t select_bitmap) {
+  // A software emulation of _pext_u64
+
+  // These checks should be inline and are likely to be common cases.
+  if (select_bitmap == ~uint64_t{0}) {
+    return bitmap;
+  } else if (select_bitmap == 0) {
+    return 0;
+  }
+
+  // Fallback to lookup table method
+  uint64_t bit_value = 0;
+  int bit_len = 0;
+  constexpr uint8_t kLookupMask = (1U << kLookupBits) - 1;
+  while (select_bitmap != 0) {
+    const auto mask_len = ARROW_POPCOUNT32(select_bitmap & kLookupMask);
+    const uint64_t value = kPextTable[select_bitmap & kLookupMask][bitmap & kLookupMask];
+    bit_value |= (value << bit_len);
+    bit_len += mask_len;
+    bitmap >>= kLookupBits;
+    select_bitmap >>= kLookupBits;
+  }
+  return bit_value;
+}
+
+#ifdef ARROW_HAVE_BMI2
+
+// Use _pext_u64 on 64-bit builds, _pext_u32 on 32-bit builds,
+#if UINTPTR_MAX == 0xFFFFFFFF
+
+using extract_bitmap_t = uint32_t;
+inline extract_bitmap_t ExtractBits(extract_bitmap_t bitmap,
+                                    extract_bitmap_t select_bitmap) {
+  return _pext_u32(bitmap, select_bitmap);
+}
+
+#else
+
+using extract_bitmap_t = uint64_t;
+inline extract_bitmap_t ExtractBits(extract_bitmap_t bitmap,
+                                    extract_bitmap_t select_bitmap) {
+  return _pext_u64(bitmap, select_bitmap);
+}
+
+#endif
+
+#else  // !defined(ARROW_HAVE_BMI2)
+
+// Use 64-bit pext emulation when BMI2 isn't available.
+using extract_bitmap_t = uint64_t;
+inline extract_bitmap_t ExtractBits(extract_bitmap_t bitmap,
+                                    extract_bitmap_t select_bitmap) {
+  return ExtractBitsSoftware(bitmap, select_bitmap);
+}
+
+#endif
+
+static constexpr int64_t kExtractBitsSize = 8 * sizeof(extract_bitmap_t);
+
+template <bool has_repeated_parent>
+int64_t DefLevelsBatchToBitmap(const int16_t* def_levels, const int64_t batch_size,
+                               int64_t upper_bound_remaining, LevelInfo level_info,
+                               ::arrow::internal::FirstTimeBitmapWriter* writer) {
+  DCHECK_LE(batch_size, kExtractBitsSize);
+
+  // Greater than level_info.def_level - 1 implies >= the def_level
+  auto defined_bitmap = static_cast<extract_bitmap_t>(
+      internal::GreaterThanBitmap(def_levels, batch_size, level_info.def_level - 1));
+
+  if (has_repeated_parent) {
+    // Greater than level_info.repeated_ancestor_def_level - 1 implies >= the
+    // repeated_ancestor_def_level
+    auto present_bitmap = static_cast<extract_bitmap_t>(internal::GreaterThanBitmap(
+        def_levels, batch_size, level_info.repeated_ancestor_def_level - 1));
+    auto selected_bits = ExtractBits(defined_bitmap, present_bitmap);
+    int64_t selected_count = ::arrow::BitUtil::PopCount(present_bitmap);
+    if (ARROW_PREDICT_FALSE(selected_count > upper_bound_remaining)) {
+      throw ParquetException("Values read exceeded upper bound");
+    }
+    writer->AppendWord(selected_bits, selected_count);
+    return ::arrow::BitUtil::PopCount(selected_bits);
+  } else {
+    if (ARROW_PREDICT_FALSE(batch_size > upper_bound_remaining)) {
+      std::stringstream ss;
+      ss << "Values read exceeded upper bound";
+      throw ParquetException(ss.str());
+    }
+
+    writer->AppendWord(defined_bitmap, batch_size);
+    return ::arrow::BitUtil::PopCount(defined_bitmap);
+  }
+}
+
+template <bool has_repeated_parent>
+void DefLevelsToBitmapSimd(const int16_t* def_levels, int64_t num_def_levels,
+                           LevelInfo level_info, ValidityBitmapInputOutput* output) {
+  ::arrow::internal::FirstTimeBitmapWriter writer(
+      output->valid_bits,
+      /*start_offset=*/output->valid_bits_offset,
+      /*length=*/num_def_levels);
+  int64_t set_count = 0;
+  output->values_read = 0;
+  int64_t values_read_remaining = output->values_read_upper_bound;
+  while (num_def_levels > kExtractBitsSize) {
+    set_count += DefLevelsBatchToBitmap<has_repeated_parent>(
+        def_levels, kExtractBitsSize, values_read_remaining, level_info, &writer);
+    def_levels += kExtractBitsSize;
+    num_def_levels -= kExtractBitsSize;
+    values_read_remaining = output->values_read_upper_bound - writer.position();
+  }
+  set_count += DefLevelsBatchToBitmap<has_repeated_parent>(
+      def_levels, num_def_levels, values_read_remaining, level_info, &writer);
+
+  output->values_read = writer.position();
+  output->null_count += output->values_read - set_count;
+  writer.Finish();
+}
+
+}  // namespace PARQUET_IMPL_NAMESPACE
+}  // namespace internal
+}  // namespace parquet
diff --git a/cpp/src/parquet/level_conversion_test.cc b/cpp/src/parquet/level_conversion_test.cc
index d4f3719289d..a3036758a0a 100644
--- a/cpp/src/parquet/level_conversion_test.cc
+++ b/cpp/src/parquet/level_conversion_test.cc
@@ -17,18 +17,24 @@
 
 #include "parquet/level_conversion.h"
 
+#include "parquet/level_comparison.h"
+#include "parquet/test_util.h"
+
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 
 #include <string>
 #include <vector>
 
+#include "arrow/testing/gtest_compat.h"
 #include "arrow/util/bit_util.h"
 #include "arrow/util/bitmap.h"
+#include "arrow/util/ubsan.h"
 
 namespace parquet {
 namespace internal {
 
+using ::arrow::internal::Bitmap;
 using ::testing::ElementsAreArray;
 
 std::string BitmapToString(const uint8_t* bitmap, int64_t bit_count) {
@@ -39,53 +45,55 @@ std::string BitmapToString(const std::vector<uint8_t>& bitmap, int64_t bit_count
   return BitmapToString(bitmap.data(), bit_count);
 }
 
-TEST(TestColumnReader, DefinitionLevelsToBitmap) {
+TEST(TestColumnReader, DefLevelsToBitmap) {
   // Bugs in this function were exposed in ARROW-3930
   std::vector<int16_t> def_levels = {3, 3, 3, 2, 3, 3, 3, 3, 3};
 
   std::vector<uint8_t> valid_bits(2, 0);
 
-  const int max_def_level = 3;
-  const int max_rep_level = 1;
+  LevelInfo level_info;
+  level_info.def_level = 3;
+  level_info.rep_level = 1;
+
+  ValidityBitmapInputOutput io;
+  io.values_read_upper_bound = def_levels.size();
+  io.values_read = -1;
+  io.valid_bits = valid_bits.data();
 
-  int64_t values_read = -1;
-  int64_t null_count = 0;
-  internal::DefinitionLevelsToBitmap(def_levels.data(), 9, max_def_level, max_rep_level,
-                                     &values_read, &null_count, valid_bits.data(),
-                                     0 /* valid_bits_offset */);
-  ASSERT_EQ(9, values_read);
-  ASSERT_EQ(1, null_count);
+  internal::DefLevelsToBitmap(def_levels.data(), 9, level_info, &io);
+  ASSERT_EQ(9, io.values_read);
+  ASSERT_EQ(1, io.null_count);
 
   // Call again with 0 definition levels, make sure that valid_bits is unmodified
   const uint8_t current_byte = valid_bits[1];
-  null_count = 0;
-  internal::DefinitionLevelsToBitmap(def_levels.data(), 0, max_def_level, max_rep_level,
-                                     &values_read, &null_count, valid_bits.data(),
-                                     9 /* valid_bits_offset */);
-  ASSERT_EQ(0, values_read);
-  ASSERT_EQ(0, null_count);
+  io.null_count = 0;
+  internal::DefLevelsToBitmap(def_levels.data(), 0, level_info, &io);
+
+  ASSERT_EQ(0, io.values_read);
+  ASSERT_EQ(0, io.null_count);
   ASSERT_EQ(current_byte, valid_bits[1]);
 }
 
-TEST(TestColumnReader, DefinitionLevelsToBitmapPowerOfTwo) {
+TEST(TestColumnReader, DefLevelsToBitmapPowerOfTwo) {
   // PARQUET-1623: Invalid memory access when decoding a valid bits vector that has a
   // length equal to a power of two and also using a non-zero valid_bits_offset.  This
   // should not fail when run with ASAN or valgrind.
   std::vector<int16_t> def_levels = {3, 3, 3, 2, 3, 3, 3, 3};
   std::vector<uint8_t> valid_bits(1, 0);
 
-  const int max_def_level = 3;
-  const int max_rep_level = 1;
+  LevelInfo level_info;
+  level_info.rep_level = 1;
+  level_info.def_level = 3;
 
-  int64_t values_read = -1;
-  int64_t null_count = 0;
+  ValidityBitmapInputOutput io;
+  io.values_read_upper_bound = def_levels.size();
+  io.values_read = -1;
+  io.valid_bits = valid_bits.data();
 
   // Read the latter half of the validity bitmap
-  internal::DefinitionLevelsToBitmap(def_levels.data() + 4, 4, max_def_level,
-                                     max_rep_level, &values_read, &null_count,
-                                     valid_bits.data(), 4 /* valid_bits_offset */);
-  ASSERT_EQ(4, values_read);
-  ASSERT_EQ(0, null_count);
+  internal::DefLevelsToBitmap(def_levels.data() + 4, 4, level_info, &io);
+  ASSERT_EQ(4, io.values_read);
+  ASSERT_EQ(0, io.null_count);
 }
 
 #if defined(ARROW_LITTLE_ENDIAN)
@@ -106,24 +114,247 @@ TEST(GreaterThanBitmap, GeneratesExpectedBitmasks) {
 }
 #endif
 
-TEST(DefinitionLevelsToBitmap, WithRepetitionLevelFiltersOutEmptyListValues) {
+TEST(DefLevelsToBitmap, WithRepetitionLevelFiltersOutEmptyListValues) {
   std::vector<uint8_t> validity_bitmap(/*count*/ 8, 0);
-  int64_t null_count = 5;
-  int64_t values_read = 1;
 
+  ValidityBitmapInputOutput io;
+  io.values_read_upper_bound = 64;
+  io.values_read = 1;
+  io.null_count = 5;
+  io.valid_bits = validity_bitmap.data();
+  io.valid_bits_offset = 1;
+
+  LevelInfo level_info;
+  level_info.repeated_ancestor_def_level = 1;
+  level_info.def_level = 2;
+  level_info.rep_level = 1;
   // All zeros should be ignored, ones should be unset in the bitmp and 2 should be set.
   std::vector<int16_t> def_levels = {0, 0, 0, 2, 2, 1, 0, 2};
-  DefinitionLevelsToBitmap(
-      def_levels.data(), def_levels.size(), /*max_definition_level=*/2,
-      /*max_repetition_level=*/1, &values_read, &null_count, validity_bitmap.data(),
-      /*valid_bits_offset=*/1);
+  DefLevelsToBitmap(def_levels.data(), def_levels.size(), level_info, &io);
 
   EXPECT_EQ(BitmapToString(validity_bitmap, /*bit_count=*/8), "01101000");
   for (size_t x = 1; x < validity_bitmap.size(); x++) {
     EXPECT_EQ(validity_bitmap[x], 0) << "index: " << x;
   }
-  EXPECT_EQ(null_count, /*5 + 1 =*/6);
-  EXPECT_EQ(values_read, 4);  // value should get overwritten.
+  EXPECT_EQ(io.null_count, /*5 + 1 =*/6);
+  EXPECT_EQ(io.values_read, 4);  // value should get overwritten.
+}
+
+struct MultiLevelTestData {
+ public:
+  std::vector<int16_t> def_levels;
+  std::vector<int16_t> rep_levels;
+};
+
+MultiLevelTestData TriplyNestedList() {
+  // Triply nested list values borrow from write_path
+  // [null, [[1 , null, 3], []], []],
+  // [[[]], [[], [1, 2]], null, [[3]]],
+  // null,
+  // []
+  return MultiLevelTestData{
+      /*def_levels=*/std::vector<int16_t>{2, 7, 6, 7, 5, 3,  // first row
+                                          5, 5, 7, 7, 2, 7,  // second row
+                                          0,                 // third row
+                                          1},
+      /*rep_levels=*/std::vector<int16_t>{0, 1, 3, 3, 2, 1,  // first row
+                                          0, 1, 2, 3, 1, 1,  // second row
+                                          0, 0}};
+}
+
+template <typename ConverterType>
+class NestedListTest : public testing::Test {
+ public:
+  void InitForLength(int length) {
+    this->validity_bits_.clear();
+    this->validity_bits_.insert(this->validity_bits_.end(), length, 0);
+    validity_io_.valid_bits = validity_bits_.data();
+    validity_io_.values_read_upper_bound = length;
+    offsets_.clear();
+    offsets_.insert(offsets_.end(), length + 1, 0);
+  }
+
+  typename ConverterType::OffsetsType* Run(const MultiLevelTestData& test_data,
+                                           LevelInfo level_info) {
+    return this->converter_.ComputeListInfo(test_data, level_info, &validity_io_,
+                                            offsets_.data());
+  }
+
+  ConverterType converter_;
+  ValidityBitmapInputOutput validity_io_;
+  std::vector<uint8_t> validity_bits_;
+  std::vector<typename ConverterType::OffsetsType> offsets_;
+};
+
+template <typename IndexType>
+struct RepDefLevelConverter {
+  using OffsetsType = IndexType;
+  OffsetsType* ComputeListInfo(const MultiLevelTestData& test_data, LevelInfo level_info,
+                               ValidityBitmapInputOutput* output, IndexType* offsets) {
+    DefRepLevelsToList(test_data.def_levels.data(), test_data.rep_levels.data(),
+                       test_data.def_levels.size(), level_info, output, offsets);
+    return offsets + output->values_read;
+  }
+};
+
+using ConverterTypes =
+    ::testing::Types<RepDefLevelConverter</*list_length_type=*/int32_t>,
+                     RepDefLevelConverter</*list_length_type=*/int64_t>>;
+TYPED_TEST_SUITE(NestedListTest, ConverterTypes);
+
+TYPED_TEST(NestedListTest, OuterMostTest) {
+  // [null, [[1 , null, 3], []], []],
+  // [[[]], [[], [1, 2]], null, [[3]]],
+  // null,
+  // []
+  // -> 4 outer most lists (len(3), len(4), null, len(0))
+  LevelInfo level_info;
+  level_info.rep_level = 1;
+  level_info.def_level = 2;
+
+  this->InitForLength(4);
+  typename TypeParam::OffsetsType* next_position =
+      this->Run(TriplyNestedList(), level_info);
+
+  EXPECT_EQ(next_position, this->offsets_.data() + 4);
+  EXPECT_THAT(this->offsets_, testing::ElementsAre(0, 3, 7, 7, 7));
+
+  EXPECT_EQ(this->validity_io_.values_read, 4);
+  EXPECT_EQ(this->validity_io_.null_count, 1);
+  EXPECT_EQ(BitmapToString(this->validity_io_.valid_bits, /*length=*/4), "1101");
+}
+
+TYPED_TEST(NestedListTest, MiddleListTest) {
+  // [null, [[1 , null, 3], []], []],
+  // [[[]], [[], [1, 2]], null, [[3]]],
+  // null,
+  // []
+  // -> middle lists (null, len(2), len(0),
+  //                  len(1), len(2), null, len(1),
+  //                  N/A,
+  //                  N/A
+  LevelInfo level_info;
+  level_info.rep_level = 2;
+  level_info.def_level = 4;
+  level_info.repeated_ancestor_def_level = 2;
+
+  this->InitForLength(7);
+  typename TypeParam::OffsetsType* next_position =
+      this->Run(TriplyNestedList(), level_info);
+
+  EXPECT_EQ(next_position, this->offsets_.data() + 7);
+  EXPECT_THAT(this->offsets_, testing::ElementsAre(0, 0, 2, 2, 3, 5, 5, 6));
+
+  EXPECT_EQ(this->validity_io_.values_read, 7);
+  EXPECT_EQ(this->validity_io_.null_count, 2);
+  EXPECT_EQ(BitmapToString(this->validity_io_.valid_bits, /*length=*/7), "0111101");
+}
+
+TYPED_TEST(NestedListTest, InnerMostListTest) {
+  // [null, [[1, null, 3], []], []],
+  // [[[]], [[], [1, 2]], null, [[3]]],
+  // null,
+  // []
+  // -> 6 inner lists (N/A, [len(3), len(0)], N/A
+  //                        len(0), [len(0), len(2)], N/A, len(1),
+  //                        N/A,
+  //                        N/A
+  LevelInfo level_info;
+  level_info.rep_level = 3;
+  level_info.def_level = 6;
+  level_info.repeated_ancestor_def_level = 4;
+
+  this->InitForLength(6);
+  typename TypeParam::OffsetsType* next_position =
+      this->Run(TriplyNestedList(), level_info);
+
+  EXPECT_EQ(next_position, this->offsets_.data() + 6);
+  EXPECT_THAT(this->offsets_, testing::ElementsAre(0, 3, 3, 3, 3, 5, 6));
+
+  EXPECT_EQ(this->validity_io_.values_read, 6);
+  EXPECT_EQ(this->validity_io_.null_count, 0);
+  EXPECT_EQ(BitmapToString(this->validity_io_.valid_bits, /*length=*/6), "111111");
+}
+
+TYPED_TEST(NestedListTest, SimpleLongList) {
+  LevelInfo level_info;
+  level_info.rep_level = 1;
+  level_info.def_level = 2;
+  level_info.repeated_ancestor_def_level = 0;
+
+  MultiLevelTestData test_data;
+  // No empty lists.
+  test_data.def_levels = std::vector<int16_t>(65 * 9, 2);
+  for (int x = 0; x < 65; x++) {
+    test_data.rep_levels.push_back(0);
+    test_data.rep_levels.insert(test_data.rep_levels.end(), 8,
+                                /*rep_level=*/1);
+  }
+
+  std::vector<typename TypeParam::OffsetsType> expected_offsets(66, 0);
+  for (size_t x = 1; x < expected_offsets.size(); x++) {
+    expected_offsets[x] = static_cast<typename TypeParam::OffsetsType>(x) * 9;
+  }
+  this->InitForLength(65);
+  typename TypeParam::OffsetsType* next_position = this->Run(test_data, level_info);
+
+  EXPECT_EQ(next_position, this->offsets_.data() + 65);
+  EXPECT_THAT(this->offsets_, testing::ElementsAreArray(expected_offsets));
+
+  EXPECT_EQ(this->validity_io_.values_read, 65);
+  EXPECT_EQ(this->validity_io_.null_count, 0);
+  EXPECT_EQ(BitmapToString(this->validity_io_.valid_bits, /*length=*/65),
+            "11111111 "
+            "11111111 "
+            "11111111 "
+            "11111111 "
+            "11111111 "
+            "11111111 "
+            "11111111 "
+            "11111111 "
+            "1");
+}
+
+TYPED_TEST(NestedListTest, TestOverflow) {
+  LevelInfo level_info;
+  level_info.rep_level = 1;
+  level_info.def_level = 2;
+  level_info.repeated_ancestor_def_level = 0;
+
+  MultiLevelTestData test_data;
+  test_data.def_levels = std::vector<int16_t>{2};
+  test_data.rep_levels = std::vector<int16_t>{0};
+
+  this->InitForLength(2);
+  // Offsets is populated as the cumulative sum of all elements,
+  // so populating the offsets[0] with max-value impacts the
+  // other values populated.
+  this->offsets_[0] = std::numeric_limits<typename TypeParam::OffsetsType>::max();
+  this->offsets_[1] = std::numeric_limits<typename TypeParam::OffsetsType>::max();
+  ASSERT_THROW(this->Run(test_data, level_info), ParquetException);
+
+  ASSERT_THROW(this->Run(test_data, level_info), ParquetException);
+
+  // Same thing should happen if the list already existed.
+  test_data.rep_levels = std::vector<int16_t>{1};
+  ASSERT_THROW(this->Run(test_data, level_info), ParquetException);
+
+  // Should be OK because it shouldn't increment.
+  test_data.def_levels = std::vector<int16_t>{0};
+  test_data.rep_levels = std::vector<int16_t>{0};
+  this->Run(test_data, level_info);
+}
+
+TEST(TestOnlyExtractBitsSoftware, BasicTest) {
+  auto check = [](uint64_t bitmap, uint64_t selection, uint64_t expected) -> void {
+    EXPECT_EQ(TestOnlyExtractBitsSoftware(bitmap, selection), expected);
+  };
+  check(0xFF, 0, 0);
+  check(0xFF, ~uint64_t{0}, 0xFF);
+  check(0xFF00FF, 0xAAAA, 0x000F);
+  check(0xFF0AFF, 0xAFAA, 0x00AF);
+  check(0xFFAAFF, 0xAFAA, 0x03AF);
+  check(0xFECBDA9876543210ULL, 0xF00FF00FF00FF00FULL, 0xFBD87430ULL);
 }
 
 }  // namespace internal
diff --git a/cpp/src/parquet/metadata.cc b/cpp/src/parquet/metadata.cc
index 9d21c62cae1..5407922ffa9 100644
--- a/cpp/src/parquet/metadata.cc
+++ b/cpp/src/parquet/metadata.cc
@@ -17,9 +17,8 @@
 
 #include "parquet/metadata.h"
 
-#include <inttypes.h>
-
 #include <algorithm>
+#include <cinttypes>
 #include <ostream>
 #include <string>
 #include <utility>
@@ -159,8 +158,6 @@ class ColumnCryptoMetaData::ColumnCryptoMetaDataImpl {
   explicit ColumnCryptoMetaDataImpl(const format::ColumnCryptoMetaData* crypto_metadata)
       : crypto_metadata_(crypto_metadata) {}
 
-  ~ColumnCryptoMetaDataImpl() {}
-
   bool encrypted_with_footer_key() const {
     return crypto_metadata_->__isset.ENCRYPTION_WITH_FOOTER_KEY;
   }
@@ -188,7 +185,7 @@ ColumnCryptoMetaData::ColumnCryptoMetaData(const uint8_t* metadata)
     : impl_(new ColumnCryptoMetaDataImpl(
           reinterpret_cast<const format::ColumnCryptoMetaData*>(metadata))) {}
 
-ColumnCryptoMetaData::~ColumnCryptoMetaData() {}
+ColumnCryptoMetaData::~ColumnCryptoMetaData() = default;
 
 std::shared_ptr<schema::ColumnPath> ColumnCryptoMetaData::path_in_schema() const {
   return impl_->path_in_schema();
@@ -203,11 +200,11 @@ const std::string& ColumnCryptoMetaData::key_metadata() const {
 // ColumnChunk metadata
 class ColumnChunkMetaData::ColumnChunkMetaDataImpl {
  public:
-  explicit ColumnChunkMetaDataImpl(
-      const format::ColumnChunk* column, const ColumnDescriptor* descr,
-      int16_t row_group_ordinal, int16_t column_ordinal,
-      const ApplicationVersion* writer_version,
-      std::shared_ptr<InternalFileDecryptor> file_decryptor = nullptr)
+  explicit ColumnChunkMetaDataImpl(const format::ColumnChunk* column,
+                                   const ColumnDescriptor* descr,
+                                   int16_t row_group_ordinal, int16_t column_ordinal,
+                                   const ApplicationVersion* writer_version,
+                                   std::shared_ptr<InternalFileDecryptor> file_decryptor)
       : column_(column), descr_(descr), writer_version_(writer_version) {
     column_metadata_ = &column->meta_data;
     if (column->__isset.crypto_metadata) {  // column metadata is encrypted
@@ -222,10 +219,10 @@ class ColumnChunkMetaData::ColumnChunkMetaDataImpl {
 
           std::string aad_column_metadata = encryption::CreateModuleAad(
               file_decryptor->file_aad(), encryption::kColumnMetaData, row_group_ordinal,
-              column_ordinal, (int16_t)-1);
+              column_ordinal, static_cast<int16_t>(-1));
           auto decryptor = file_decryptor->GetColumnMetaDecryptor(
               path->ToDotString(), key_metadata, aad_column_metadata);
-          uint32_t len = static_cast<uint32_t>(column->encrypted_column_metadata.size());
+          auto len = static_cast<uint32_t>(column->encrypted_column_metadata.size());
           DeserializeThriftMsg(
               reinterpret_cast<const uint8_t*>(column->encrypted_column_metadata.c_str()),
               &len, &decrypted_metadata_, decryptor);
@@ -247,6 +244,11 @@ class ColumnChunkMetaData::ColumnChunkMetaDataImpl {
     }
     possible_stats_ = nullptr;
   }
+
+  bool Equals(const ColumnChunkMetaDataImpl& other) const {
+    return *column_metadata_ == *other.column_metadata_;
+  }
+
   // column chunk
   inline int64_t file_offset() const { return column_->file_offset; }
   inline const std::string& file_path() const { return column_->file_path; }
@@ -340,18 +342,20 @@ std::unique_ptr<ColumnChunkMetaData> ColumnChunkMetaData::Make(
     int16_t column_ordinal, std::shared_ptr<InternalFileDecryptor> file_decryptor) {
   return std::unique_ptr<ColumnChunkMetaData>(
       new ColumnChunkMetaData(metadata, descr, row_group_ordinal, column_ordinal,
-                              writer_version, file_decryptor));
+                              writer_version, std::move(file_decryptor)));
 }
 
 ColumnChunkMetaData::ColumnChunkMetaData(
     const void* metadata, const ColumnDescriptor* descr, int16_t row_group_ordinal,
     int16_t column_ordinal, const ApplicationVersion* writer_version,
     std::shared_ptr<InternalFileDecryptor> file_decryptor)
-    : impl_{std::unique_ptr<ColumnChunkMetaDataImpl>(new ColumnChunkMetaDataImpl(
+    : impl_{new ColumnChunkMetaDataImpl(
           reinterpret_cast<const format::ColumnChunk*>(metadata), descr,
-          row_group_ordinal, column_ordinal, writer_version, file_decryptor))} {}
+          row_group_ordinal, column_ordinal, writer_version, std::move(file_decryptor))} {
+}
+
+ColumnChunkMetaData::~ColumnChunkMetaData() = default;
 
-ColumnChunkMetaData::~ColumnChunkMetaData() {}
 // column chunk
 int64_t ColumnChunkMetaData::file_offset() const { return impl_->file_offset(); }
 
@@ -417,6 +421,10 @@ std::unique_ptr<ColumnCryptoMetaData> ColumnChunkMetaData::crypto_metadata() con
   return impl_->crypto_metadata();
 }
 
+bool ColumnChunkMetaData::Equals(const ColumnChunkMetaData& other) const {
+  return impl_->Equals(*other.impl_);
+}
+
 // row-group metadata
 class RowGroupMetaData::RowGroupMetaDataImpl {
  public:
@@ -427,7 +435,11 @@ class RowGroupMetaData::RowGroupMetaDataImpl {
       : row_group_(row_group),
         schema_(schema),
         writer_version_(writer_version),
-        file_decryptor_(file_decryptor) {}
+        file_decryptor_(std::move(file_decryptor)) {}
+
+  bool Equals(const RowGroupMetaDataImpl& other) const {
+    return *row_group_ == *other.row_group_;
+  }
 
   inline int num_columns() const { return static_cast<int>(row_group_->columns.size()); }
 
@@ -444,15 +456,13 @@ class RowGroupMetaData::RowGroupMetaDataImpl {
   inline const SchemaDescriptor* schema() const { return schema_; }
 
   std::unique_ptr<ColumnChunkMetaData> ColumnChunk(int i) {
-    if (!(i < num_columns())) {
-      std::stringstream ss;
-      ss << "The file only has " << num_columns()
-         << " columns, requested metadata for column: " << i;
-      throw ParquetException(ss.str());
+    if (i < num_columns()) {
+      return ColumnChunkMetaData::Make(&row_group_->columns[i], schema_->Column(i),
+                                       writer_version_, row_group_->ordinal,
+                                       static_cast<int16_t>(i), file_decryptor_);
     }
-    return ColumnChunkMetaData::Make(&row_group_->columns[i], schema_->Column(i),
-                                     writer_version_, row_group_->ordinal, (int16_t)i,
-                                     file_decryptor_);
+    throw ParquetException("The file only has ", num_columns(),
+                           " columns, requested metadata for column: ", i);
   }
 
  private:
@@ -467,17 +477,21 @@ std::unique_ptr<RowGroupMetaData> RowGroupMetaData::Make(
     const ApplicationVersion* writer_version,
     std::shared_ptr<InternalFileDecryptor> file_decryptor) {
   return std::unique_ptr<RowGroupMetaData>(
-      new RowGroupMetaData(metadata, schema, writer_version, file_decryptor));
+      new RowGroupMetaData(metadata, schema, writer_version, std::move(file_decryptor)));
 }
 
 RowGroupMetaData::RowGroupMetaData(const void* metadata, const SchemaDescriptor* schema,
                                    const ApplicationVersion* writer_version,
                                    std::shared_ptr<InternalFileDecryptor> file_decryptor)
-    : impl_{std::unique_ptr<RowGroupMetaDataImpl>(
-          new RowGroupMetaDataImpl(reinterpret_cast<const format::RowGroup*>(metadata),
-                                   schema, writer_version, file_decryptor))} {}
+    : impl_{new RowGroupMetaDataImpl(reinterpret_cast<const format::RowGroup*>(metadata),
+                                     schema, writer_version, std::move(file_decryptor))} {
+}
+
+RowGroupMetaData::~RowGroupMetaData() = default;
 
-RowGroupMetaData::~RowGroupMetaData() {}
+bool RowGroupMetaData::Equals(const RowGroupMetaData& other) const {
+  return impl_->Equals(*other.impl_);
+}
 
 int RowGroupMetaData::num_columns() const { return impl_->num_columns(); }
 
@@ -485,6 +499,8 @@ int64_t RowGroupMetaData::num_rows() const { return impl_->num_rows(); }
 
 int64_t RowGroupMetaData::total_byte_size() const { return impl_->total_byte_size(); }
 
+int64_t RowGroupMetaData::file_offset() const { return impl_->file_offset(); }
+
 const SchemaDescriptor* RowGroupMetaData::schema() const { return impl_->schema(); }
 
 std::unique_ptr<ColumnChunkMetaData> RowGroupMetaData::ColumnChunk(int i) const {
@@ -504,12 +520,12 @@ bool RowGroupMetaData::can_decompress() const {
 // file metadata
 class FileMetaData::FileMetaDataImpl {
  public:
-  FileMetaDataImpl() : metadata_len_(0) {}
+  FileMetaDataImpl() = default;
 
   explicit FileMetaDataImpl(
       const void* metadata, uint32_t* metadata_len,
       std::shared_ptr<InternalFileDecryptor> file_decryptor = nullptr)
-      : metadata_len_(0), file_decryptor_(file_decryptor) {
+      : file_decryptor_(file_decryptor) {
     metadata_.reset(new format::FileMetaData);
 
     auto footer_decryptor =
@@ -542,9 +558,9 @@ class FileMetaData::FileMetaDataImpl {
     serializer.SerializeToBuffer(metadata_.get(), &serialized_len, &serialized_data);
 
     // encrypt with nonce
-    uint8_t* nonce = const_cast<uint8_t*>(reinterpret_cast<const uint8_t*>(signature));
-    uint8_t* tag = const_cast<uint8_t*>(reinterpret_cast<const uint8_t*>(signature)) +
-                   encryption::kNonceLength;
+    auto nonce = const_cast<uint8_t*>(reinterpret_cast<const uint8_t*>(signature));
+    auto tag = const_cast<uint8_t*>(reinterpret_cast<const uint8_t*>(signature)) +
+               encryption::kNonceLength;
 
     std::string key = file_decryptor_->GetFooterKey();
     std::string aad = encryption::CreateFooterAad(file_decryptor_->file_aad());
@@ -632,6 +648,10 @@ class FileMetaData::FileMetaDataImpl {
                                   file_decryptor_);
   }
 
+  bool Equals(const FileMetaDataImpl& other) const {
+    return *metadata_ == *other.metadata_;
+  }
+
   const SchemaDescriptor* schema() const { return &schema_; }
 
   const std::shared_ptr<const KeyValueMetadata>& key_value_metadata() const {
@@ -664,13 +684,52 @@ class FileMetaData::FileMetaDataImpl {
     }
   }
 
+  std::shared_ptr<FileMetaData> Subset(const std::vector<int>& row_groups) {
+    for (int i : row_groups) {
+      if (i < num_row_groups()) continue;
+
+      throw ParquetException(
+          "The file only has ", num_row_groups(),
+          " row groups, but requested a subset including row group: ", i);
+    }
+
+    std::shared_ptr<FileMetaData> out(new FileMetaData());
+    out->impl_.reset(new FileMetaDataImpl());
+    out->impl_->metadata_.reset(new format::FileMetaData());
+
+    auto metadata = out->impl_->metadata_.get();
+    metadata->version = metadata_->version;
+    metadata->schema = metadata_->schema;
+
+    metadata->row_groups.resize(row_groups.size());
+    int i = 0;
+    for (int selected_index : row_groups) {
+      metadata->num_rows += row_group(selected_index).num_rows;
+      metadata->row_groups[i++] = row_group(selected_index);
+    }
+
+    metadata->key_value_metadata = metadata_->key_value_metadata;
+    metadata->created_by = metadata_->created_by;
+    metadata->column_orders = metadata_->column_orders;
+    metadata->encryption_algorithm = metadata_->encryption_algorithm;
+    metadata->footer_signing_key_metadata = metadata_->footer_signing_key_metadata;
+    metadata->__isset = metadata_->__isset;
+
+    out->impl_->schema_ = schema_;
+    out->impl_->writer_version_ = writer_version_;
+    out->impl_->key_value_metadata_ = key_value_metadata_;
+    out->impl_->file_decryptor_ = file_decryptor_;
+
+    return out;
+  }
+
   void set_file_decryptor(std::shared_ptr<InternalFileDecryptor> file_decryptor) {
     file_decryptor_ = file_decryptor;
   }
 
  private:
   friend FileMetaDataBuilder;
-  uint32_t metadata_len_;
+  uint32_t metadata_len_ = 0;
   std::unique_ptr<format::FileMetaData> metadata_;
   SchemaDescriptor schema_;
   ApplicationVersion writer_version_;
@@ -731,7 +790,11 @@ FileMetaData::FileMetaData(const void* metadata, uint32_t* metadata_len,
 FileMetaData::FileMetaData()
     : impl_{std::unique_ptr<FileMetaDataImpl>(new FileMetaDataImpl())} {}
 
-FileMetaData::~FileMetaData() {}
+FileMetaData::~FileMetaData() = default;
+
+bool FileMetaData::Equals(const FileMetaData& other) const {
+  return impl_->Equals(*other.impl_);
+}
 
 std::unique_ptr<RowGroupMetaData> FileMetaData::RowGroup(int i) const {
   return impl_->RowGroup(i);
@@ -809,6 +872,11 @@ void FileMetaData::AppendRowGroups(const FileMetaData& other) {
   impl_->AppendRowGroups(other.impl_);
 }
 
+std::shared_ptr<FileMetaData> FileMetaData::Subset(
+    const std::vector<int>& row_groups) const {
+  return impl_->Subset(row_groups);
+}
+
 void FileMetaData::WriteTo(::arrow::io::OutputStream* dst,
                            const std::shared_ptr<Encryptor>& encryptor) const {
   return impl_->WriteTo(dst, encryptor);
@@ -816,7 +884,7 @@ void FileMetaData::WriteTo(::arrow::io::OutputStream* dst,
 
 class FileCryptoMetaData::FileCryptoMetaDataImpl {
  public:
-  FileCryptoMetaDataImpl() {}
+  FileCryptoMetaDataImpl() = default;
 
   explicit FileCryptoMetaDataImpl(const uint8_t* metadata, uint32_t* metadata_len) {
     metadata_.reset(new format::FileCryptoMetaData);
@@ -824,8 +892,6 @@ class FileCryptoMetaData::FileCryptoMetaDataImpl {
     metadata_len_ = *metadata_len;
   }
 
-  ~FileCryptoMetaDataImpl() {}
-
   EncryptionAlgorithm encryption_algorithm() {
     return FromThrift(metadata_->encryption_algorithm);
   }
@@ -861,7 +927,7 @@ FileCryptoMetaData::FileCryptoMetaData(const uint8_t* serialized_metadata,
 
 FileCryptoMetaData::FileCryptoMetaData() : impl_(new FileCryptoMetaDataImpl()) {}
 
-FileCryptoMetaData::~FileCryptoMetaData() {}
+FileCryptoMetaData::~FileCryptoMetaData() = default;
 
 void FileCryptoMetaData::WriteTo(::arrow::io::OutputStream* dst) const {
   impl_->WriteTo(dst);
@@ -876,13 +942,14 @@ std::string FileMetaData::SerializeToString() const {
   return metadata_buffer->ToString();
 }
 
-ApplicationVersion::ApplicationVersion(const std::string& application, int major,
-                                       int minor, int patch)
-    : application_(application), version{major, minor, patch, "", "", ""} {}
+ApplicationVersion::ApplicationVersion(std::string application, int major, int minor,
+                                       int patch)
+    : application_(std::move(application)), version{major, minor, patch, "", "", ""} {}
 
 ApplicationVersion::ApplicationVersion(const std::string& created_by) {
-  regex app_regex{ApplicationVersion::APPLICATION_FORMAT};
-  regex ver_regex{ApplicationVersion::VERSION_FORMAT};
+  // Use singletons to compile only once (ARROW-9863)
+  static regex app_regex{ApplicationVersion::APPLICATION_FORMAT};
+  static regex ver_regex{ApplicationVersion::VERSION_FORMAT};
   smatch app_matches;
   smatch ver_matches;
 
@@ -1176,7 +1243,7 @@ ColumnChunkMetaDataBuilder::ColumnChunkMetaDataBuilder(
               std::move(props), column,
               reinterpret_cast<format::ColumnChunk*>(contents)))} {}
 
-ColumnChunkMetaDataBuilder::~ColumnChunkMetaDataBuilder() {}
+ColumnChunkMetaDataBuilder::~ColumnChunkMetaDataBuilder() = default;
 
 const void* ColumnChunkMetaDataBuilder::contents() const { return impl_->contents(); }
 
@@ -1294,10 +1361,9 @@ std::unique_ptr<RowGroupMetaDataBuilder> RowGroupMetaDataBuilder::Make(
 RowGroupMetaDataBuilder::RowGroupMetaDataBuilder(std::shared_ptr<WriterProperties> props,
                                                  const SchemaDescriptor* schema_,
                                                  void* contents)
-    : impl_{std::unique_ptr<RowGroupMetaDataBuilderImpl>(
-          new RowGroupMetaDataBuilderImpl(std::move(props), schema_, contents))} {}
+    : impl_{new RowGroupMetaDataBuilderImpl(std::move(props), schema_, contents)} {}
 
-RowGroupMetaDataBuilder::~RowGroupMetaDataBuilder() {}
+RowGroupMetaDataBuilder::~RowGroupMetaDataBuilder() = default;
 
 ColumnChunkMetaDataBuilder* RowGroupMetaDataBuilder::NextColumnChunk() {
   return impl_->NextColumnChunk();
@@ -1325,10 +1391,10 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl {
   explicit FileMetaDataBuilderImpl(
       const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props,
       std::shared_ptr<const KeyValueMetadata> key_value_metadata)
-      : properties_(std::move(props)),
+      : metadata_(new format::FileMetaData()),
+        properties_(std::move(props)),
         schema_(schema),
         key_value_metadata_(std::move(key_value_metadata)) {
-    metadata_.reset(new format::FileMetaData());
     if (properties_->file_encryption_properties() != nullptr &&
         properties_->file_encryption_properties()->encrypted_footer()) {
       crypto_metadata_.reset(new format::FileCryptoMetaData());
@@ -1395,8 +1461,9 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl {
       EncryptionAlgorithm algo = file_encryption_properties->algorithm();
       signing_algorithm.aad.aad_file_unique = algo.aad.aad_file_unique;
       signing_algorithm.aad.supply_aad_prefix = algo.aad.supply_aad_prefix;
-      if (!algo.aad.supply_aad_prefix)
+      if (!algo.aad.supply_aad_prefix) {
         signing_algorithm.aad.aad_prefix = algo.aad.aad_prefix;
+      }
       signing_algorithm.algorithm = ParquetCipher::AES_GCM_V1;
 
       metadata_->__set_encryption_algorithm(ToThrift(signing_algorithm));
@@ -1463,7 +1530,7 @@ FileMetaDataBuilder::FileMetaDataBuilder(
     : impl_{std::unique_ptr<FileMetaDataBuilderImpl>(new FileMetaDataBuilderImpl(
           schema, std::move(props), std::move(key_value_metadata)))} {}
 
-FileMetaDataBuilder::~FileMetaDataBuilder() {}
+FileMetaDataBuilder::~FileMetaDataBuilder() = default;
 
 RowGroupMetaDataBuilder* FileMetaDataBuilder::AppendRowGroup() {
   return impl_->AppendRowGroup();
diff --git a/cpp/src/parquet/metadata.h b/cpp/src/parquet/metadata.h
index 8c26461dc66..b487cf445cd 100644
--- a/cpp/src/parquet/metadata.h
+++ b/cpp/src/parquet/metadata.h
@@ -89,9 +89,9 @@ class PARQUET_EXPORT ApplicationVersion {
     std::string build_info;
   } version;
 
-  ApplicationVersion() {}
+  ApplicationVersion() = default;
   explicit ApplicationVersion(const std::string& created_by);
-  ApplicationVersion(const std::string& application, int major, int minor, int patch);
+  ApplicationVersion(std::string application, int major, int minor, int patch);
 
   // Returns true if version is strictly less than other_version
   bool VersionLt(const ApplicationVersion& other_version) const;
@@ -109,6 +109,8 @@ class PARQUET_EXPORT ColumnCryptoMetaData {
   static std::unique_ptr<ColumnCryptoMetaData> Make(const uint8_t* metadata);
   ~ColumnCryptoMetaData();
 
+  bool Equals(const ColumnCryptoMetaData& other) const;
+
   std::shared_ptr<schema::ColumnPath> path_in_schema() const;
   bool encrypted_with_footer_key() const;
   const std::string& key_metadata() const;
@@ -139,6 +141,8 @@ class PARQUET_EXPORT ColumnChunkMetaData {
 
   ~ColumnChunkMetaData();
 
+  bool Equals(const ColumnChunkMetaData& other) const;
+
   // column chunk
   int64_t file_offset() const;
 
@@ -190,6 +194,8 @@ class PARQUET_EXPORT RowGroupMetaData {
 
   ~RowGroupMetaData();
 
+  bool Equals(const RowGroupMetaData& other) const;
+
   /// \brief The number of columns in this row group. The order must match the
   /// parent's column ordering.
   int num_columns() const;
@@ -210,6 +216,13 @@ class PARQUET_EXPORT RowGroupMetaData {
 
   /// \brief Total byte size of all the uncompressed column data in this row group.
   int64_t total_byte_size() const;
+
+  /// \brief Byte offset from beginning of file to first page (data or
+  /// dictionary) in this row group
+  ///
+  /// The file_offset field that this method exposes is optional. This method
+  /// will return 0 if that field is not set to a meaningful value.
+  int64_t file_offset() const;
   // Return const-pointer to make it clear that this object is not to be copied
   const SchemaDescriptor* schema() const;
   // Indicate if all of the RowGroup's ColumnChunks can be decompressed.
@@ -237,6 +250,8 @@ class PARQUET_EXPORT FileMetaData {
 
   ~FileMetaData();
 
+  bool Equals(const FileMetaData& other) const;
+
   /// \brief The number of top-level columns in the schema.
   ///
   /// Parquet thrift definition requires that nested schema elements are
@@ -327,6 +342,10 @@ class PARQUET_EXPORT FileMetaData {
   /// \throws ParquetException if schemas are not equal.
   void AppendRowGroups(const FileMetaData& other);
 
+  /// \brief Return a FileMetaData containing a subset of the row groups in this
+  /// FileMetaData.
+  std::shared_ptr<FileMetaData> Subset(const std::vector<int>& row_groups) const;
+
  private:
   friend FileMetaDataBuilder;
   friend class SerializedFile;
diff --git a/cpp/src/parquet/metadata_test.cc b/cpp/src/parquet/metadata_test.cc
index 4255540b651..1fda8cdd46d 100644
--- a/cpp/src/parquet/metadata_test.cc
+++ b/cpp/src/parquet/metadata_test.cc
@@ -203,6 +203,7 @@ TEST(Metadata, TestBuildAccess) {
     f_accessors[loop_index]->set_file_path("/foo/bar/bar.parquet");
     ASSERT_EQ("/foo/bar/bar.parquet", rg2_column1->file_path());
   }
+
   // Test AppendRowGroups
   auto f_accessor_2 = GenerateTableMetaData(schema, props, nrows, stats_int, stats_float);
   f_accessor->AppendRowGroups(*f_accessor_2);
@@ -212,6 +213,14 @@ TEST(Metadata, TestBuildAccess) {
   ASSERT_EQ(ParquetVersion::PARQUET_2_0, f_accessor->version());
   ASSERT_EQ(DEFAULT_CREATED_BY, f_accessor->created_by());
   ASSERT_EQ(3, f_accessor->num_schema_elements());
+
+  // Test Subset
+  auto f_accessor_1 = f_accessor->Subset({2, 3});
+  ASSERT_TRUE(f_accessor_1->Equals(*f_accessor_2));
+
+  f_accessor_1 = f_accessor_2->Subset({0});
+  f_accessor_1->AppendRowGroups(*f_accessor->Subset({0}));
+  ASSERT_TRUE(f_accessor_1->Equals(*f_accessor->Subset({2, 0})));
 }
 
 TEST(Metadata, TestV1Version) {
diff --git a/cpp/src/parquet/printer.cc b/cpp/src/parquet/printer.cc
index a667a4857ec..224a19dda6b 100644
--- a/cpp/src/parquet/printer.cc
+++ b/cpp/src/parquet/printer.cc
@@ -25,6 +25,7 @@
 #include <vector>
 
 #include "arrow/util/key_value_metadata.h"
+#include "arrow/util/string.h"
 
 #include "parquet/column_scanner.h"
 #include "parquet/exception.h"
@@ -121,7 +122,9 @@ void ParquetFilePrinter::DebugPrint(std::ostream& stream, std::list<int> selecte
         stream << "  Statistics Not Set";
       }
       stream << std::endl
-             << "  Compression: " << Codec::GetCodecAsString(column_chunk->compression())
+             << "  Compression: "
+             << arrow::internal::AsciiToUpper(
+                    Codec::GetCodecAsString(column_chunk->compression()))
              << ", Encodings:";
       for (auto encoding : column_chunk->encodings()) {
         stream << " " << EncodingToString(encoding);
@@ -256,7 +259,8 @@ void ParquetFilePrinter::JSONPrint(std::ostream& stream, std::list<int> selected
         stream << "\"False\",";
       }
       stream << "\n           \"Compression\": \""
-             << Codec::GetCodecAsString(column_chunk->compression())
+             << arrow::internal::AsciiToUpper(
+                    Codec::GetCodecAsString(column_chunk->compression()))
              << "\", \"Encodings\": \"";
       for (auto encoding : column_chunk->encodings()) {
         stream << EncodingToString(encoding) << " ";
diff --git a/cpp/src/parquet/reader_test.cc b/cpp/src/parquet/reader_test.cc
index a271075eaf9..5fda439e4a1 100644
--- a/cpp/src/parquet/reader_test.cc
+++ b/cpp/src/parquet/reader_test.cc
@@ -28,6 +28,7 @@
 #include "arrow/io/file.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/testing/random.h"
+#include "arrow/util/checked_cast.h"
 
 #include "parquet/column_reader.h"
 #include "parquet/column_scanner.h"
@@ -38,8 +39,9 @@
 #include "parquet/printer.h"
 #include "parquet/test_util.h"
 
-namespace parquet {
+using arrow::internal::checked_pointer_cast;
 
+namespace parquet {
 using schema::GroupNode;
 using schema::PrimitiveNode;
 
@@ -58,6 +60,31 @@ std::string nation_dict_truncated_data_page() {
   return data_file("nation.dict-malformed.parquet");
 }
 
+// Compressed using custom Hadoop LZ4 format (block LZ4 format + custom header)
+std::string hadoop_lz4_compressed() { return data_file("hadoop_lz4_compressed.parquet"); }
+
+// Compressed using block LZ4 format
+std::string non_hadoop_lz4_compressed() {
+  return data_file("non_hadoop_lz4_compressed.parquet");
+}
+
+// TODO: Assert on definition and repetition levels
+template <typename DType, typename ValueType>
+void AssertColumnValues(std::shared_ptr<TypedColumnReader<DType>> col, int64_t batch_size,
+                        int64_t expected_levels_read,
+                        std::vector<ValueType>& expected_values,
+                        int64_t expected_values_read) {
+  std::vector<ValueType> values(batch_size);
+  int64_t values_read;
+
+  auto levels_read =
+      col->ReadBatch(batch_size, nullptr, nullptr, values.data(), &values_read);
+  ASSERT_EQ(expected_levels_read, levels_read);
+
+  ASSERT_EQ(expected_values, values);
+  ASSERT_EQ(expected_values_read, values_read);
+}
+
 class TestAllTypesPlain : public ::testing::Test {
  public:
   void SetUp() { reader_ = ParquetFileReader::OpenFile(alltypes_plain()); }
@@ -487,4 +514,45 @@ TEST(TestFileReader, BufferedReads) {
   }
 }
 
+class TestCodec : public ::testing::TestWithParam<std::string> {
+ protected:
+  const std::string& GetDataFile() { return GetParam(); }
+};
+
+TEST_P(TestCodec, FileMetadataAndValues) {
+  std::unique_ptr<ParquetFileReader> reader_ = ParquetFileReader::OpenFile(GetDataFile());
+  std::shared_ptr<RowGroupReader> group = reader_->RowGroup(0);
+
+  // This file only has 4 rows
+  ASSERT_EQ(4, reader_->metadata()->num_rows());
+  // This file only has 3 columns
+  ASSERT_EQ(3, reader_->metadata()->num_columns());
+  // This file only has 1 row group
+  ASSERT_EQ(1, reader_->metadata()->num_row_groups());
+  // This row group must have 4 rows
+  ASSERT_EQ(4, group->metadata()->num_rows());
+
+  // column 0, c0
+  auto col0 = checked_pointer_cast<Int64Reader>(group->Column(0));
+  std::vector<int64_t> expected_values = {1593604800, 1593604800, 1593604801, 1593604801};
+  AssertColumnValues(col0, 4, 4, expected_values, 4);
+
+  // column 1, c1
+  std::vector<ByteArray> expected_byte_arrays = {ByteArray("abc"), ByteArray("def"),
+                                                 ByteArray("abc"), ByteArray("def")};
+  auto col1 = checked_pointer_cast<ByteArrayReader>(group->Column(1));
+  AssertColumnValues(col1, 4, 4, expected_byte_arrays, 4);
+
+  // column 2, v11
+  std::vector<double> expected_double_values = {42.0, 7.7, 42.125, 7.7};
+  auto col2 = checked_pointer_cast<DoubleReader>(group->Column(2));
+  AssertColumnValues(col2, 4, 4, expected_double_values, 4);
+}
+
+#ifdef ARROW_WITH_LZ4
+INSTANTIATE_TEST_SUITE_P(Lz4CodecTests, TestCodec,
+                         ::testing::Values(hadoop_lz4_compressed(),
+                                           non_hadoop_lz4_compressed()));
+#endif
+
 }  // namespace parquet
diff --git a/cpp/src/parquet/statistics.cc b/cpp/src/parquet/statistics.cc
index babf34ac304..bb970367f20 100644
--- a/cpp/src/parquet/statistics.cc
+++ b/cpp/src/parquet/statistics.cc
@@ -223,11 +223,11 @@ CleanStatistic(std::pair<T, T> min_max) {
 
   // Ignore if one of the value is nan.
   if (std::isnan(min) || std::isnan(max)) {
-    return nonstd::nullopt;
+    return arrow::util::nullopt;
   }
 
   if (min == std::numeric_limits<T>::max() && max == std::numeric_limits<T>::lowest()) {
-    return nonstd::nullopt;
+    return arrow::util::nullopt;
   }
 
   T zero{};
@@ -245,7 +245,7 @@ CleanStatistic(std::pair<T, T> min_max) {
 
 optional<std::pair<FLBA, FLBA>> CleanStatistic(std::pair<FLBA, FLBA> min_max) {
   if (min_max.first.ptr == nullptr || min_max.second.ptr == nullptr) {
-    return nonstd::nullopt;
+    return arrow::util::nullopt;
   }
   return min_max;
 }
@@ -253,7 +253,7 @@ optional<std::pair<FLBA, FLBA>> CleanStatistic(std::pair<FLBA, FLBA> min_max) {
 optional<std::pair<ByteArray, ByteArray>> CleanStatistic(
     std::pair<ByteArray, ByteArray> min_max) {
   if (min_max.first.ptr == nullptr || min_max.second.ptr == nullptr) {
-    return nonstd::nullopt;
+    return arrow::util::nullopt;
   }
   return min_max;
 }
@@ -467,6 +467,20 @@ class TypedStatisticsImpl : public TypedStatistics<DType> {
 
   bool HasMinMax() const override { return has_min_max_; }
 
+  bool Equals(const Statistics& raw_other) const override {
+    if (physical_type() != raw_other.physical_type()) return false;
+
+    const auto& other = checked_cast<const TypedStatisticsImpl&>(raw_other);
+
+    if (has_min_max_ != other.has_min_max_) return false;
+
+    return (has_min_max_ && MinMaxEqual(other)) && null_count() == other.null_count() &&
+           distinct_count() == other.distinct_count() &&
+           num_values() == other.num_values();
+  }
+
+  bool MinMaxEqual(const TypedStatisticsImpl& other) const;
+
   void Reset() override {
     ResetCounts();
     has_min_max_ = false;
@@ -584,6 +598,20 @@ class TypedStatisticsImpl : public TypedStatistics<DType> {
   }
 };
 
+template <>
+inline bool TypedStatisticsImpl<FLBAType>::MinMaxEqual(
+    const TypedStatisticsImpl<FLBAType>& other) const {
+  uint32_t len = descr_->type_length();
+  return std::memcmp(min_.ptr, other.min_.ptr, len) == 0 &&
+         std::memcmp(max_.ptr, other.max_.ptr, len) == 0;
+}
+
+template <typename DType>
+bool TypedStatisticsImpl<DType>::MinMaxEqual(
+    const TypedStatisticsImpl<DType>& other) const {
+  return min_ != other.min_ && max_ != other.max_;
+}
+
 template <>
 inline void TypedStatisticsImpl<FLBAType>::Copy(const FLBA& src, FLBA* dst,
                                                 ResizableBuffer* buffer) {
diff --git a/cpp/src/parquet/statistics.h b/cpp/src/parquet/statistics.h
index 5c7ffba4cdc..48621e52906 100644
--- a/cpp/src/parquet/statistics.h
+++ b/cpp/src/parquet/statistics.h
@@ -244,6 +244,9 @@ class PARQUET_EXPORT Statistics {
   /// \brief The full type descriptor from the column schema
   virtual const ColumnDescriptor* descr() const = 0;
 
+  /// \brief Check two Statistics for equality
+  virtual bool Equals(const Statistics& other) const = 0;
+
  protected:
   static std::shared_ptr<Statistics> Make(Type::type physical_type, const void* min,
                                           const void* max, int64_t num_values,
diff --git a/cpp/src/parquet/stream_reader_test.cc b/cpp/src/parquet/stream_reader_test.cc
index 3dc17fd9e93..f6db4dd1969 100644
--- a/cpp/src/parquet/stream_reader_test.cc
+++ b/cpp/src/parquet/stream_reader_test.cc
@@ -25,7 +25,7 @@
 #include <memory>
 #include <utility>
 
-#include "arrow/io/api.h"
+#include "arrow/io/file.h"
 #include "parquet/exception.h"
 #include "parquet/test_util.h"
 
diff --git a/cpp/src/parquet/stream_writer_test.cc b/cpp/src/parquet/stream_writer_test.cc
index 12c1abbbacc..62e714c4ec2 100644
--- a/cpp/src/parquet/stream_writer_test.cc
+++ b/cpp/src/parquet/stream_writer_test.cc
@@ -23,7 +23,7 @@
 #include <memory>
 #include <utility>
 
-#include "arrow/io/api.h"
+#include "arrow/io/file.h"
 #include "parquet/exception.h"
 
 namespace parquet {
diff --git a/cpp/src/parquet/test_encryption_util.h b/cpp/src/parquet/test_encryption_util.h
index e430246eb27..39911e62d7a 100644
--- a/cpp/src/parquet/test_encryption_util.h
+++ b/cpp/src/parquet/test_encryption_util.h
@@ -32,7 +32,9 @@
 
 #include <gtest/gtest.h>
 
+#include "arrow/status.h"
 #include "arrow/testing/util.h"
+#include "arrow/util/io_util.h"
 
 #include "parquet/column_page.h"
 #include "parquet/column_reader.h"
@@ -44,6 +46,8 @@
 namespace parquet {
 namespace test {
 
+using arrow::internal::TemporaryDir;
+
 using parquet::ConvertedType;
 using parquet::Repetition;
 using parquet::Type;
@@ -65,5 +69,14 @@ inline std::string data_file(const char* file) {
   return ss.str();
 }
 
+// A temporary directory that contains the encrypted files generated in the tests.
+extern std::unique_ptr<TemporaryDir> temp_dir;
+
+inline arrow::Result<std::unique_ptr<TemporaryDir>> temp_data_dir() {
+  arrow::Result<std::unique_ptr<TemporaryDir>> dir;
+  ARROW_ASSIGN_OR_RAISE(dir, TemporaryDir::Make("parquet-encryption-test-"));
+  return dir;
+}
+
 }  // namespace test
 }  // namespace parquet
diff --git a/cpp/src/parquet/test_util.h b/cpp/src/parquet/test_util.h
index 6ffe2f41b00..737ecc2fdde 100644
--- a/cpp/src/parquet/test_util.h
+++ b/cpp/src/parquet/test_util.h
@@ -22,7 +22,6 @@
 #pragma once
 
 #include <algorithm>
-#include <chrono>
 #include <limits>
 #include <memory>
 #include <random>
diff --git a/cpp/src/parquet/thrift_internal.h b/cpp/src/parquet/thrift_internal.h
index e1959e79438..561d6d15a48 100644
--- a/cpp/src/parquet/thrift_internal.h
+++ b/cpp/src/parquet/thrift_internal.h
@@ -101,8 +101,7 @@ static inline Compression::type FromThriftUnsafe(format::CompressionCodec::type
     case format::CompressionCodec::BROTLI:
       return Compression::BROTLI;
     case format::CompressionCodec::LZ4:
-      // ARROW-9424: Existing files use LZ4_RAW but this may need to change
-      return Compression::LZ4;
+      return Compression::LZ4_HADOOP;
     case format::CompressionCodec::ZSTD:
       return Compression::ZSTD;
     default:
@@ -280,7 +279,9 @@ static inline format::CompressionCodec::type ToThrift(Compression::type type) {
       return format::CompressionCodec::LZO;
     case Compression::BROTLI:
       return format::CompressionCodec::BROTLI;
+    // For compatibility with existing source code
     case Compression::LZ4:
+    case Compression::LZ4_HADOOP:
       return format::CompressionCodec::LZ4;
     case Compression::ZSTD:
       return format::CompressionCodec::ZSTD;
diff --git a/cpp/src/parquet/types.cc b/cpp/src/parquet/types.cc
index 1b4bb4acdd5..b4a9ceecc7c 100644
--- a/cpp/src/parquet/types.cc
+++ b/cpp/src/parquet/types.cc
@@ -30,7 +30,7 @@
 
 #include "generated/parquet_types.h"
 
-using ::arrow::internal::checked_cast;
+using arrow::internal::checked_cast;
 using arrow::util::Codec;
 
 namespace parquet {
@@ -43,25 +43,19 @@ bool IsCodecSupported(Compression::type codec) {
     case Compression::BROTLI:
     case Compression::ZSTD:
     case Compression::LZ4:
+    case Compression::LZ4_HADOOP:
       return true;
     default:
       return false;
   }
 }
 
-namespace internal {
+std::unique_ptr<Codec> GetCodec(Compression::type codec) {
+  return GetCodec(codec, Codec::UseDefaultCompressionLevel());
+}
 
-std::unique_ptr<Codec> GetCodec(Compression::type codec, int compression_level,
-                                bool for_writing) {
+std::unique_ptr<Codec> GetCodec(Compression::type codec, int compression_level) {
   std::unique_ptr<Codec> result;
-  if (for_writing && (codec == Compression::LZ4 || codec == Compression::LZ4_FRAME)) {
-    throw ParquetException(
-        "Per ARROW-9424, writing files with LZ4 compression has been "
-        "disabled until implementation issues have been resolved. "
-        "It is recommended to read any existing files and rewrite them "
-        "using a different compression.");
-  }
-
   if (!IsCodecSupported(codec)) {
     std::stringstream ss;
     ss << "Codec type " << Codec::GetCodecAsString(codec)
@@ -69,28 +63,15 @@ std::unique_ptr<Codec> GetCodec(Compression::type codec, int compression_level,
     throw ParquetException(ss.str());
   }
 
+  if (codec == Compression::LZ4) {
+    // For compatibility with existing source code
+    codec = Compression::LZ4_HADOOP;
+  }
+
   PARQUET_ASSIGN_OR_THROW(result, Codec::Create(codec, compression_level));
   return result;
 }
 
-std::unique_ptr<Codec> GetReadCodec(Compression::type codec) {
-  return GetCodec(codec, Codec::UseDefaultCompressionLevel(), /*for_writing=*/false);
-}
-
-std::unique_ptr<Codec> GetWriteCodec(Compression::type codec, int compression_level) {
-  return GetCodec(codec, compression_level, /*for_writing=*/true);
-}
-
-}  // namespace internal
-
-std::unique_ptr<Codec> GetCodec(Compression::type codec, int compression_level) {
-  return internal::GetCodec(codec, compression_level, /*for_writing=*/false);
-}
-
-std::unique_ptr<Codec> GetCodec(Compression::type codec) {
-  return GetCodec(codec, Codec::UseDefaultCompressionLevel());
-}
-
 std::string FormatStatValue(Type::type parquet_type, ::arrow::util::string_view val) {
   std::stringstream result;
 
diff --git a/cpp/src/parquet/types.h b/cpp/src/parquet/types.h
index 950f0383b6d..51fb13b2332 100644
--- a/cpp/src/parquet/types.h
+++ b/cpp/src/parquet/types.h
@@ -464,15 +464,6 @@ struct Encoding {
 PARQUET_EXPORT
 bool IsCodecSupported(Compression::type codec);
 
-namespace internal {
-
-// ARROW-9424: Separate functions for reading and writing so we can disable LZ4
-// on writing
-std::unique_ptr<Codec> GetReadCodec(Compression::type codec);
-std::unique_ptr<Codec> GetWriteCodec(Compression::type codec, int compression_level);
-
-}  // namespace internal
-
 PARQUET_EXPORT
 std::unique_ptr<Codec> GetCodec(Compression::type codec);
 
diff --git a/cpp/src/plasma/CMakeLists.txt b/cpp/src/plasma/CMakeLists.txt
index 43b3f033c83..8c8523a37d5 100644
--- a/cpp/src/plasma/CMakeLists.txt
+++ b/cpp/src/plasma/CMakeLists.txt
@@ -56,12 +56,12 @@ set(PLASMA_LINK_LIBS arrow_shared)
 set(PLASMA_STATIC_LINK_LIBS arrow_static)
 
 if(ARROW_CUDA)
-  set(PLASMA_LINK_LIBS ${PLASMA_LINK_LIBS} arrow_cuda_shared)
-  set(PLASMA_STATIC_LINK_LIBS arrow_cuda_static ${PLASMA_STATIC_LINK_LIBS})
+  list(INSERT PLASMA_LINK_LIBS 0 arrow_cuda_shared)
+  list(INSERT PLASMA_STATIC_LINK_LIBS 0 arrow_cuda_static)
   add_definitions(-DPLASMA_CUDA)
 endif()
 
-if(NOT APPLE AND NOT MSVC)
+if(NOT APPLE AND NOT MSVC_TOOLCHAIN)
   # Localize thirdparty symbols using a linker version script. This hides them
   # from the client application. The OS X linker does not support the
   # version-script option.
diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
index 1c2ec6812a8..5072d29334c 100644
--- a/cpp/src/plasma/client.cc
+++ b/cpp/src/plasma/client.cc
@@ -64,7 +64,6 @@ using arrow::cuda::CudaDeviceManager;
 #endif
 
 #define XXH_INLINE_ALL 1
-#define XXH_NAMESPACE plasma_client_
 #include "arrow/vendored/xxhash.h"
 
 #define XXH64_DEFAULT_SEED 0
diff --git a/cpp/src/plasma/protocol.cc b/cpp/src/plasma/protocol.cc
index 69d53e7432d..735636cda9f 100644
--- a/cpp/src/plasma/protocol.cc
+++ b/cpp/src/plasma/protocol.cc
@@ -581,7 +581,7 @@ Status ReadListReply(const uint8_t* data, size_t size, ObjectTable* objects) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<fb::PlasmaListReply>(data);
   DCHECK(VerifyFlatbuffer(message, data, size));
-  for (auto const& object : *message->objects()) {
+  for (auto const object : *message->objects()) {
     ObjectID object_id = ObjectID::from_binary(object->object_id()->str());
     auto entry = std::unique_ptr<ObjectTableEntry>(new ObjectTableEntry());
     entry->data_size = object->data_size();
diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
index b12e8426751..032a12fcfac 100644
--- a/cpp/src/plasma/store.cc
+++ b/cpp/src/plasma/store.cc
@@ -497,7 +497,7 @@ void PlasmaStore::ProcessGetRequest(Client* client,
   }
 
   if (!evicted_ids.empty()) {
-    unsigned char digest[kDigestSize];
+    unsigned char digest[kDigestSize] = {};
     std::vector<std::shared_ptr<Buffer>> buffers;
     for (size_t i = 0; i < evicted_ids.size(); ++i) {
       ARROW_CHECK(evicted_entries[i]->pointer != nullptr);
diff --git a/cpp/submodules/parquet-testing b/cpp/submodules/parquet-testing
index 46c9e977f58..d914f9d2894 160000
--- a/cpp/submodules/parquet-testing
+++ b/cpp/submodules/parquet-testing
@@ -1 +1 @@
-Subproject commit 46c9e977f58f6c5ef1b81f782f3746b3656e5a8c
+Subproject commit d914f9d289488c7db1759d7a88a4a1b8f062c7dd
diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt
index becfcc4381f..545943d30c6 100644
--- a/cpp/thirdparty/versions.txt
+++ b/cpp/thirdparty/versions.txt
@@ -24,7 +24,10 @@
 # format).
 
 ARROW_ABSL_BUILD_VERSION=2eba343b51e0923cd3fb919a6abd6120590fc059
-ARROW_AWSSDK_BUILD_VERSION=1.7.160
+ARROW_AWSSDK_BUILD_VERSION=1.8.57
+ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.1.5
+ARROW_AWS_C_COMMON_BUILD_VERSION=v0.4.59
+ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.1.5
 ARROW_BOOST_BUILD_VERSION=1.71.0
 ARROW_BROTLI_BUILD_VERSION=v1.0.7
 ARROW_BZIP2_BUILD_VERSION=1.0.8
@@ -36,7 +39,7 @@ ARROW_GRPC_BUILD_VERSION=v1.29.1
 ARROW_GTEST_BUILD_VERSION=1.8.1
 ARROW_JEMALLOC_BUILD_VERSION=5.2.1
 ARROW_LZ4_BUILD_VERSION=v1.9.2
-ARROW_MIMALLOC_BUILD_VERSION=v1.6.3
+ARROW_MIMALLOC_BUILD_VERSION=v1.6.4
 ARROW_ORC_BUILD_VERSION=1.6.2
 ARROW_PROTOBUF_BUILD_VERSION=v3.12.1
 # Because of https://github.com/Tencent/rapidjson/pull/1323, we require
diff --git a/csharp/Directory.Build.props b/csharp/Directory.Build.props
index c2f0502f300..f6d8d43f250 100644
--- a/csharp/Directory.Build.props
+++ b/csharp/Directory.Build.props
@@ -29,7 +29,7 @@
     <Product>Apache Arrow library</Product>
     <Copyright>Copyright 2016-2019 The Apache Software Foundation</Copyright>
     <Company>The Apache Software Foundation</Company>
-    <Version>2.0.0-SNAPSHOT</Version>
+    <Version>3.0.0-SNAPSHOT</Version>
   </PropertyGroup>
 
   <PropertyGroup>
diff --git a/csharp/README.md b/csharp/README.md
index c6901a1e4e2..29c6086379b 100644
--- a/csharp/README.md
+++ b/csharp/README.md
@@ -94,6 +94,7 @@ This implementation is under development and may not be suitable for use in prod
 - Time64
 - Binary (fixed-length)
 - List
+- Struct
 
 ### Type Metadata
 
@@ -119,7 +120,6 @@ This implementation is under development and may not be suitable for use in prod
     - Tensor
     - Table
 - Arrays
-    - Struct
     - Union
         - Dense
         - Sparse
diff --git a/csharp/src/Apache.Arrow/Apache.Arrow.csproj b/csharp/src/Apache.Arrow/Apache.Arrow.csproj
index a7d2b30772e..62574029f97 100644
--- a/csharp/src/Apache.Arrow/Apache.Arrow.csproj
+++ b/csharp/src/Apache.Arrow/Apache.Arrow.csproj
@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFrameworks>netstandard1.3;netcoreapp2.1</TargetFrameworks>
@@ -37,5 +37,6 @@
   </ItemGroup>
   <ItemGroup Condition="'$(TargetFramework)' == 'netcoreapp2.1'">
     <Compile Remove="Extensions\StreamExtensions.netstandard.cs" />
+    <Compile Remove="Extensions\TupleExtensions.netstandard.cs" />
   </ItemGroup>
 </Project>
diff --git a/csharp/src/Apache.Arrow/Arrays/Date32Array.cs b/csharp/src/Apache.Arrow/Arrays/Date32Array.cs
index bb07c9d3b16..35c0065e119 100644
--- a/csharp/src/Apache.Arrow/Arrays/Date32Array.cs
+++ b/csharp/src/Apache.Arrow/Arrays/Date32Array.cs
@@ -18,13 +18,20 @@
 
 namespace Apache.Arrow
 {
+    /// <summary>
+    /// The <see cref="Date32Array"/> class holds an array of dates in the <c>Date32</c> format, where each date is
+    /// stored as the number of days since the dawn of (UNIX) time.
+    /// </summary>
     public class Date32Array : PrimitiveArray<int>
     {
-        private const long MillisecondsPerDay = 86400000;
+        private static readonly DateTime _epochDate = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Unspecified);
 
-        public class Builder : PrimitiveArrayBuilder<DateTimeOffset, int, Date32Array, Builder>
+        /// <summary>
+        /// The <see cref="Builder"/> class can be used to fluently build <see cref="Date32Array"/> objects.
+        /// </summary>
+        public class Builder : DateArrayBuilder<int, Date32Array, Builder>
         {
-            internal class DateBuilder : PrimitiveArrayBuilder<int, Date32Array, DateBuilder>
+            private class DateBuilder : PrimitiveArrayBuilder<int, Date32Array, DateBuilder>
             {
                 protected override Date32Array Build(
                     ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer,
@@ -32,11 +39,23 @@ protected override Date32Array Build(
                     new Date32Array(valueBuffer, nullBitmapBuffer, length, nullCount, offset);
             }
 
+            /// <summary>
+            /// Construct a new instance of the <see cref="Builder"/> class.
+            /// </summary>
             public Builder() : base(new DateBuilder()) { }
 
-            protected override int ConvertTo(DateTimeOffset value)
+            protected override int Convert(DateTime dateTime)
             {
-                return (int)(value.ToUnixTimeMilliseconds() / MillisecondsPerDay);
+                return (int)(dateTime.Date - _epochDate).TotalDays;
+            }
+
+            protected override int Convert(DateTimeOffset dateTimeOffset)
+            {
+                // The internal value stored for a DateTimeOffset can be thought of as the number of 24-hour "blocks"
+                // of time that have elapsed since the UNIX epoch.  This is the same as converting it to UTC first and
+                // then taking the date element from that.  It is not the same as what would result from looking at the
+                // DateTimeOffset.Date property.
+                return (int)(dateTimeOffset.UtcDateTime.Date - _epochDate).TotalDays;
             }
         }
 
@@ -55,16 +74,39 @@ public Date32Array(ArrayData data)
 
         public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor);
 
-        public DateTimeOffset? GetDate(int index)
+        [Obsolete("Use `GetDateTimeOffset()` instead")]
+        public DateTimeOffset? GetDate(int index) => GetDateTimeOffset(index);
+
+        /// <summary>
+        /// Get the date at the specified index in the form of a <see cref="DateTime"/> object.
+        /// </summary>
+        /// <remarks>
+        /// The <see cref="DateTime.Kind"/> property of the returned object is set to
+        /// <see cref="DateTimeKind.Unspecified"/>.
+        /// </remarks>
+        /// <param name="index">Index at which to get the date.</param>
+        /// <returns>Returns a <see cref="DateTime"/> object, or <c>null</c> if there is no object at that index.
+        /// </returns>
+        public DateTime? GetDateTime(int index)
         {
             int? value = GetValue(index);
+            return value.HasValue
+                ? _epochDate.AddDays(value.Value)
+                : default(DateTime?);
+        }
 
-            if (!value.HasValue)
-            {
-                return default;
-            }
-
-            return DateTimeOffset.FromUnixTimeMilliseconds(value.Value * MillisecondsPerDay);
+        /// <summary>
+        /// Get the date at the specified index in the form of a <see cref="DateTimeOffset"/> object.
+        /// </summary>
+        /// <param name="index">Index at which to get the date.</param>
+        /// <returns>Returns a <see cref="DateTimeOffset"/> object, or <c>null</c> if there is no object at that index.
+        /// </returns>
+        public DateTimeOffset? GetDateTimeOffset(int index)
+        {
+            int? value = GetValue(index);
+            return value.HasValue
+                ? new DateTimeOffset(_epochDate.AddDays(value.Value), TimeSpan.Zero)
+                : default(DateTimeOffset?);
         }
     }
 }
diff --git a/csharp/src/Apache.Arrow/Arrays/Date64Array.cs b/csharp/src/Apache.Arrow/Arrays/Date64Array.cs
index 763986dcc31..cf977b2e496 100644
--- a/csharp/src/Apache.Arrow/Arrays/Date64Array.cs
+++ b/csharp/src/Apache.Arrow/Arrays/Date64Array.cs
@@ -15,13 +15,18 @@
 
 using Apache.Arrow.Types;
 using System;
-using System.Collections.Generic;
-using Apache.Arrow.Memory;
 
 namespace Apache.Arrow
 {
+    /// <summary>
+    /// The <see cref="Date64Array"/> class holds an array of dates in the <c>Date64</c> format, where each date is
+    /// stored as the number of milliseconds since the dawn of (UNIX) time, excluding leap seconds, in multiples of
+    /// 86400000.
+    /// </summary>
     public class Date64Array: PrimitiveArray<long>
     {
+        private const long MillisecondsPerDay = 86400000;
+
         public Date64Array(
             ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer,
             int length, int nullCount, int offset)
@@ -29,25 +34,44 @@ public Date64Array(
                 new[] { nullBitmapBuffer, valueBuffer }))
         { }
 
-        public class Builder : PrimitiveArrayBuilder<DateTimeOffset, long, Date64Array, Builder>
+        /// <summary>
+        /// The <see cref="Builder"/> class can be used to fluently build <see cref="Date64Array"/> objects.
+        /// </summary>
+        public class Builder : DateArrayBuilder<long, Date64Array, Builder>
         {
-            internal class DateBuilder: PrimitiveArrayBuilder<long, Date64Array, DateBuilder>
+            private class DateBuilder: PrimitiveArrayBuilder<long, Date64Array, DateBuilder>
             {
                 protected override Date64Array Build(
                     ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer,
                     int length, int nullCount, int offset) =>
                     new Date64Array(valueBuffer, nullBitmapBuffer, length, nullCount, offset);
-            } 
+            }
 
+            /// <summary>
+            /// Construct a new instance of the <see cref="Builder"/> class.
+            /// </summary>
             public Builder() : base(new DateBuilder()) { }
 
-            protected override long ConvertTo(DateTimeOffset value)
+            protected override long Convert(DateTime dateTime)
+            {
+                var dateTimeOffset = new DateTimeOffset(
+                    DateTime.SpecifyKind(dateTime.Date, DateTimeKind.Unspecified),
+                    TimeSpan.Zero);
+                return dateTimeOffset.ToUnixTimeMilliseconds();
+            }
+
+            protected override long Convert(DateTimeOffset dateTimeOffset)
             {
-                return value.ToUnixTimeMilliseconds();
+                // The internal value stored for a DateTimeOffset can be thought of as the number of milliseconds,
+                // in multiples of 86400000, that have passed since the UNIX epoch.  It is not the same as what would
+                // result from encoding the date from the DateTimeOffset.Date property.
+                long millis = dateTimeOffset.ToUnixTimeMilliseconds();
+                long days = millis / MillisecondsPerDay;
+                return (millis < 0 ? days - 1 : days) * MillisecondsPerDay;
             }
         }
 
-        public Date64Array(ArrayData data) 
+        public Date64Array(ArrayData data)
             : base(data)
         {
             data.EnsureDataType(ArrowTypeId.Date64);
@@ -55,16 +79,39 @@ public Date64Array(ArrayData data)
 
         public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor);
 
-        public DateTimeOffset? GetDate(int index)
+        [Obsolete("Use `GetDateTimeOffset()` instead")]
+        public DateTimeOffset? GetDate(int index) => GetDateTimeOffset(index);
+
+        /// <summary>
+        /// Get the date at the specified index in the form of a <see cref="DateTime"/> object.
+        /// </summary>
+        /// <remarks>
+        /// The <see cref="DateTime.Kind"/> property of the returned object is set to
+        /// <see cref="DateTimeKind.Unspecified"/>.
+        /// </remarks>
+        /// <param name="index">Index at which to get the date.</param>
+        /// <returns>Returns a <see cref="DateTime"/> object, or <c>null</c> if there is no object at that index.
+        /// </returns>
+        public DateTime? GetDateTime(int index)
         {
             long? value = GetValue(index);
+            return value.HasValue
+                ? DateTimeOffset.FromUnixTimeMilliseconds(value.Value).Date
+                : default(DateTime?);
+        }
 
-            if (!value.HasValue)
-            {
-                return default;
-            }
-
-            return DateTimeOffset.FromUnixTimeMilliseconds(value.Value);
+        /// <summary>
+        /// Get the date at the specified index in the form of a <see cref="DateTimeOffset"/> object.
+        /// </summary>
+        /// <param name="index">Index at which to get the date.</param>
+        /// <returns>Returns a <see cref="DateTimeOffset"/> object, or <c>null</c> if there is no object at that index.
+        /// </returns>
+        public DateTimeOffset? GetDateTimeOffset(int index)
+        {
+            long? value = GetValue(index);
+            return value.HasValue
+                ? DateTimeOffset.FromUnixTimeMilliseconds(value.Value)
+                : default(DateTimeOffset?);
         }
     }
 }
diff --git a/csharp/src/Apache.Arrow/Arrays/DateArrayBuilder.cs b/csharp/src/Apache.Arrow/Arrays/DateArrayBuilder.cs
new file mode 100644
index 00000000000..4e69f6fe3e7
--- /dev/null
+++ b/csharp/src/Apache.Arrow/Arrays/DateArrayBuilder.cs
@@ -0,0 +1,209 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Apache.Arrow
+{
+    /// <summary>
+    /// The <see cref="DateArrayBuilder{TUnderlying,TArray,TBuilder}"/> class is an abstract array builder that can
+    /// accept dates in the form of <see cref="DateTime"/> or <see cref="DateTimeOffset"/> and convert to some
+    /// underlying date representation.
+    /// </summary>
+    public abstract class DateArrayBuilder<TUnderlying, TArray, TBuilder> :
+        DelegatingArrayBuilder<TUnderlying, TArray, TBuilder>,
+        IArrowArrayBuilder<DateTime, TArray, TBuilder>,
+        IArrowArrayBuilder<DateTimeOffset, TArray, TBuilder>
+        where TArray : IArrowArray
+        where TBuilder : class, IArrowArrayBuilder<TArray>
+    {
+        /// <summary>
+        /// Construct a new instance of the <see cref="DateArrayBuilder{TUnderlying,TArray,TBuilder}"/> class.
+        /// </summary>
+        /// <param name="innerBuilder">Inner builder that will produce arrays of type <typeparamref name="TArray"/>.
+        /// </param>
+        protected DateArrayBuilder(IArrowArrayBuilder<TUnderlying, TArray, IArrowArrayBuilder<TArray>> innerBuilder)
+            : base(innerBuilder)
+        { }
+
+        /// <summary>
+        /// Append a date in the form of a <see cref="DateTime"/> object to the array.
+        /// </summary>
+        /// <remarks>
+        /// The value of <see cref="DateTime.Kind"/> on the input does not have any effect on the behaviour of this
+        /// method.
+        /// </remarks>
+        /// <param name="value">Date to add.</param>
+        /// <returns>Returns the builder (for fluent-style composition).</returns>
+        public TBuilder Append(DateTime value)
+        {
+            InnerBuilder.Append(Convert(value));
+            return this as TBuilder;
+        }
+
+        /// <summary>
+        /// Append a date from a <see cref="DateTimeOffset"/> object to the array.
+        /// </summary>
+        /// <remarks>
+        /// Note that to convert the supplied <paramref name="value"/> parameter to a date, it is first converted to
+        /// UTC and the date then taken from the UTC date/time.  Depending on the value of its
+        /// <see cref="DateTimeOffset.Offset"/> property, this may not necessarily be the same as the date obtained by
+        /// calling its <see cref="DateTimeOffset.Date"/> property.
+        /// </remarks>
+        /// <param name="value">Date to add.</param>
+        /// <returns>Returns the builder (for fluent-style composition).</returns>
+        public TBuilder Append(DateTimeOffset value)
+        {
+            InnerBuilder.Append(Convert(value));
+            return this as TBuilder;
+        }
+
+        /// <summary>
+        /// Append a span of dates in the form of <see cref="DateTime"/> objects to the array.
+        /// </summary>
+        /// <remarks>
+        /// The value of <see cref="DateTime.Kind"/> on any of the inputs does not have any effect on the behaviour of
+        /// this method.
+        /// </remarks>
+        /// <param name="span">Span of dates to add.</param>
+        /// <returns>Returns the builder (for fluent-style composition).</returns>
+        public TBuilder Append(ReadOnlySpan<DateTime> span)
+        {
+            InnerBuilder.Reserve(span.Length);
+            foreach (var item in span)
+            {
+                InnerBuilder.Append(Convert(item));
+            }
+
+            return this as TBuilder;
+        }
+
+        /// <summary>
+        /// Append a span of dates in the form of <see cref="DateTimeOffset"/> objects to the array.
+        /// </summary>
+        /// <remarks>
+        /// Note that to convert the <see cref="DateTimeOffset"/> objects in the <paramref name="span"/> parameter to
+        /// dates, they are first converted to UTC and the date then taken from the UTC date/times.  Depending on the
+        /// value of each <see cref="DateTimeOffset.Offset"/> property, this may not necessarily be the same as the
+        /// date obtained by calling the <see cref="DateTimeOffset.Date"/> property.
+        /// </remarks>
+        /// <param name="span">Span of dates to add.</param>
+        /// <returns>Returns the builder (for fluent-style composition).</returns>
+        public TBuilder Append(ReadOnlySpan<DateTimeOffset> span)
+        {
+            InnerBuilder.Reserve(span.Length);
+            foreach (var item in span)
+            {
+                InnerBuilder.Append(Convert(item));
+            }
+
+            return this as TBuilder;
+        }
+
+        /// <summary>
+        /// Append a null date to the array.
+        /// </summary>
+        /// <returns>Returns the builder (for fluent-style composition).</returns>
+        public TBuilder AppendNull()
+        {
+            InnerBuilder.AppendNull();
+            return this as TBuilder;
+        }
+
+        /// <summary>
+        /// Append a collection of dates in the form of <see cref="DateTime"/> objects to the array.
+        /// </summary>
+        /// <remarks>
+        /// The value of <see cref="DateTime.Kind"/> on any of the inputs does not have any effect on the behaviour of
+        /// this method.
+        /// </remarks>
+        /// <param name="values">Collection of dates to add.</param>
+        /// <returns>Returns the builder (for fluent-style composition).</returns>
+        public TBuilder AppendRange(IEnumerable<DateTime> values)
+        {
+            InnerBuilder.AppendRange(values.Select(Convert));
+            return this as TBuilder;
+        }
+
+        /// <summary>
+        /// Append a collection of dates in the form of <see cref="DateTimeOffset"/> objects to the array.
+        /// </summary>
+        /// <remarks>
+        /// Note that to convert the <see cref="DateTimeOffset"/> objects in the <paramref name="values"/> parameter to
+        /// dates, they are first converted to UTC and the date then taken from the UTC date/times.  Depending on the
+        /// value of each <see cref="DateTimeOffset.Offset"/> property, this may not necessarily be the same as the
+        /// date obtained by calling the <see cref="DateTimeOffset.Date"/> property.
+        /// </remarks>
+        /// <param name="values">Collection of dates to add.</param>
+        /// <returns>Returns the builder (for fluent-style composition).</returns>
+        public TBuilder AppendRange(IEnumerable<DateTimeOffset> values)
+        {
+            InnerBuilder.AppendRange(values.Select(Convert));
+            return this as TBuilder;
+        }
+
+        /// <summary>
+        /// Set the value of a date in the form of a <see cref="DateTime"/> object at the specified index.
+        /// </summary>
+        /// <remarks>
+        /// The value of <see cref="DateTime.Kind"/> on the input does not have any effect on the behaviour of this
+        /// method.
+        /// </remarks>
+        /// <param name="index">Index at which to set value.</param>
+        /// <param name="value">Date to set.</param>
+        /// <returns>Returns the builder (for fluent-style composition).</returns>
+        public TBuilder Set(int index, DateTime value)
+        {
+            InnerBuilder.Set(index, Convert(value));
+            return this as TBuilder;
+        }
+
+        /// <summary>
+        /// Set the value of a date in the form of a <see cref="DateTimeOffset"/> object at the specified index.
+        /// </summary>
+        /// <remarks>
+        /// Note that to convert the supplied <paramref name="value"/> parameter to a date, it is first converted to
+        /// UTC and the date then taken from the UTC date/time.  Depending on the value of its
+        /// <see cref="DateTimeOffset.Offset"/> property, this may not necessarily be the same as the date obtained by
+        /// calling its <see cref="DateTimeOffset.Date"/> property.
+        /// </remarks>
+        /// <param name="index">Index at which to set value.</param>
+        /// <param name="value">Date to set.</param>
+        /// <returns>Returns the builder (for fluent-style composition).</returns>
+        public TBuilder Set(int index, DateTimeOffset value)
+        {
+            InnerBuilder.Set(index, Convert(value));
+            return this as TBuilder;
+        }
+
+        /// <summary>
+        /// Swap the values of the dates at the specified indices.
+        /// </summary>
+        /// <param name="i">First index.</param>
+        /// <param name="j">Second index.</param>
+        /// <returns>Returns the builder (for fluent-style composition).</returns>
+        public TBuilder Swap(int i, int j)
+        {
+            InnerBuilder.Swap(i, j);
+            return this as TBuilder;
+        }
+
+        protected abstract TUnderlying Convert(DateTime dateTime);
+
+        protected abstract TUnderlying Convert(DateTimeOffset dateTimeOffset);
+    }
+}
diff --git a/csharp/src/Apache.Arrow/Arrays/DelegatingArrayBuilder.cs b/csharp/src/Apache.Arrow/Arrays/DelegatingArrayBuilder.cs
new file mode 100644
index 00000000000..f2ab3ee13cb
--- /dev/null
+++ b/csharp/src/Apache.Arrow/Arrays/DelegatingArrayBuilder.cs
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using Apache.Arrow.Memory;
+
+namespace Apache.Arrow
+{
+    /// <summary>
+    /// The <see cref="DelegatingArrayBuilder{T,TArray,TBuilder}"/> class can be used as the base for any array builder
+    /// that needs to delegate most of its functionality to an inner array builder.
+    /// </summary>
+    /// <remarks>
+    /// The typical use case is when an array builder may accept a number of different types as input, but which are
+    /// all internally converted to a single type for assembly into an array.
+    /// </remarks>
+    /// <typeparam name="T">Type of item accepted by inner array builder.</typeparam>
+    /// <typeparam name="TArray">Type of array produced by this (and the inner) builder.</typeparam>
+    /// <typeparam name="TBuilder">Type of builder (see Curiously-Recurring Template Pattern).</typeparam>
+    public abstract class DelegatingArrayBuilder<T, TArray, TBuilder> : IArrowArrayBuilder<TArray, TBuilder>
+        where TArray : IArrowArray
+        where TBuilder : class, IArrowArrayBuilder<TArray>
+    {
+        /// <summary>
+        /// Gets the inner array builder.
+        /// </summary>
+        protected IArrowArrayBuilder<T, TArray, IArrowArrayBuilder<TArray>> InnerBuilder { get; }
+
+        /// <summary>
+        /// Gets the number of items added to the array so far.
+        /// </summary>
+        public int Length => InnerBuilder.Length;
+
+        /// <summary>
+        /// Construct a new instance of the <see cref="DelegatingArrayBuilder{T,TArray,TBuilder}"/> class.
+        /// </summary>
+        /// <param name="innerBuilder">Inner array builder.</param>
+        protected DelegatingArrayBuilder(IArrowArrayBuilder<T, TArray, IArrowArrayBuilder<TArray>> innerBuilder)
+        {
+            InnerBuilder = innerBuilder ?? throw new ArgumentNullException(nameof(innerBuilder));
+        }
+
+        /// <summary>
+        /// Build an Arrow Array from the appended contents so far.
+        /// </summary>
+        /// <param name="allocator">Optional memory allocator.</param>
+        /// <returns>Returns the built array.</returns>
+        public TArray Build(MemoryAllocator allocator = default) => InnerBuilder.Build(allocator);
+
+        /// <summary>
+        /// Reserve a given number of items' additional capacity.
+        /// </summary>
+        /// <param name="additionalCapacity">Number of items of required additional capacity.</param>
+        /// <returns>Returns the builder (for fluent-style composition).</returns>
+        public TBuilder Reserve(int additionalCapacity)
+        {
+            InnerBuilder.Reserve(additionalCapacity);
+            return this as TBuilder;
+        }
+
+        /// <summary>
+        /// Resize the array to a given size.
+        /// </summary>
+        /// <remarks>
+        /// Note that if the required capacity is larger than the current length of the populated array so far,
+        /// the array's contents in the new, expanded region are undefined.
+        /// </remarks>
+        /// <remarks>
+        /// Note that if the required capacity is smaller than the current length of the populated array so far,
+        /// the array will be truncated and items at the end of the array will be lost.
+        /// </remarks>
+        /// <param name="capacity">Number of items of required capacity.</param>
+        /// <returns>Returns the builder (for fluent-style composition).</returns>
+        public TBuilder Resize(int capacity)
+        {
+            InnerBuilder.Resize(capacity);
+            return this as TBuilder;
+        }
+
+        /// <summary>
+        /// Clear all contents appended so far.
+        /// </summary>
+        /// <returns>Returns the builder (for fluent-style composition).</returns>
+        public TBuilder Clear()
+        {
+            InnerBuilder.Clear();
+            return this as TBuilder;
+        }
+    }
+}
diff --git a/csharp/src/Apache.Arrow/Arrays/StructArray.cs b/csharp/src/Apache.Arrow/Arrays/StructArray.cs
index a9dc5bc2023..31aea9b4113 100644
--- a/csharp/src/Apache.Arrow/Arrays/StructArray.cs
+++ b/csharp/src/Apache.Arrow/Arrays/StructArray.cs
@@ -16,33 +16,44 @@
 using Apache.Arrow.Types;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 
 namespace Apache.Arrow
 {
     public class StructArray : Array
     {
-        private readonly List<Array> _fields;
+        private IReadOnlyList<IArrowArray> _fields;
 
-        public IEnumerable<Array> Fields => _fields;
+        public IReadOnlyList<IArrowArray> Fields =>
+            LazyInitializer.EnsureInitialized(ref _fields, () => InitializeFields());
 
         public StructArray(
             IArrowType dataType, int length,
-            IEnumerable<Array> children,
+            IEnumerable<IArrowArray> children,
             ArrowBuffer nullBitmapBuffer, int nullCount = 0, int offset = 0)
-        : this(new ArrayData(
-            dataType, length, nullCount, offset, new[] { nullBitmapBuffer }, 
-            children.Select(child => child.Data)))
-        { }
+            : this(new ArrayData(
+                dataType, length, nullCount, offset, new[] { nullBitmapBuffer },
+                children.Select(child => child.Data)))
+        {
+            _fields = children.ToArray();
+        }
 
         public StructArray(ArrayData data)
             : base(data)
         {
             data.EnsureDataType(ArrowTypeId.Struct);
-
-            _fields = new List<Array>();
         }
 
         public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor);
 
+        private IReadOnlyList<IArrowArray> InitializeFields()
+        {
+            IArrowArray[] result = new IArrowArray[Data.Children.Length];
+            for (int i = 0; i < Data.Children.Length; i++)
+            {
+                result[i] = ArrowArrayFactory.BuildArray(Data.Children[i]);
+            }
+            return result;
+        }
     }
 }
diff --git a/csharp/src/Apache.Arrow/Extensions/StreamExtensions.netcoreapp2.1.cs b/csharp/src/Apache.Arrow/Extensions/StreamExtensions.netcoreapp2.1.cs
index f51dc53b097..efcacdc844d 100644
--- a/csharp/src/Apache.Arrow/Extensions/StreamExtensions.netcoreapp2.1.cs
+++ b/csharp/src/Apache.Arrow/Extensions/StreamExtensions.netcoreapp2.1.cs
@@ -25,5 +25,10 @@ public static int Read(this Stream stream, Memory<byte> buffer)
         {
             return stream.Read(buffer.Span);
         }
+
+        public static void Write(this Stream stream, ReadOnlyMemory<byte> buffer)
+        {
+            stream.Write(buffer.Span);
+        }
     }
 }
diff --git a/csharp/src/Apache.Arrow/Extensions/StreamExtensions.netstandard.cs b/csharp/src/Apache.Arrow/Extensions/StreamExtensions.netstandard.cs
index ce23bd1eb7b..b983be0fd0d 100644
--- a/csharp/src/Apache.Arrow/Extensions/StreamExtensions.netstandard.cs
+++ b/csharp/src/Apache.Arrow/Extensions/StreamExtensions.netstandard.cs
@@ -74,6 +74,27 @@ async ValueTask<int> FinishReadAsync(Task<int> readTask, byte[] localBuffer, Mem
             }
         }
 
+        public static void Write(this Stream stream, ReadOnlyMemory<byte> buffer)
+        {
+            if (MemoryMarshal.TryGetArray(buffer, out ArraySegment<byte> array))
+            {
+                stream.Write(array.Array, array.Offset, array.Count);
+            }
+            else
+            {
+                byte[] sharedBuffer = ArrayPool<byte>.Shared.Rent(buffer.Length);
+                try
+                {
+                    buffer.Span.CopyTo(sharedBuffer);
+                    stream.Write(sharedBuffer, 0, buffer.Length);
+                }
+                finally
+                {
+                    ArrayPool<byte>.Shared.Return(sharedBuffer);
+                }
+            }
+        }
+
         public static ValueTask WriteAsync(this Stream stream, ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default)
         {
             if (MemoryMarshal.TryGetArray(buffer, out ArraySegment<byte> array))
diff --git a/csharp/src/Apache.Arrow/Extensions/TupleExtensions.netstandard.cs b/csharp/src/Apache.Arrow/Extensions/TupleExtensions.netstandard.cs
new file mode 100644
index 00000000000..fe42075f14f
--- /dev/null
+++ b/csharp/src/Apache.Arrow/Extensions/TupleExtensions.netstandard.cs
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+
+namespace Apache.Arrow
+{
+    // Helpers to Deconstruct Tuples on netstandard
+    internal static partial class TupleExtensions
+    {
+        public static void Deconstruct<T1, T2>(this Tuple<T1, T2> value, out T1 item1, out T2 item2)
+        {
+            item1 = value.Item1;
+            item2 = value.Item2;
+        }
+    }
+}
diff --git a/csharp/src/Apache.Arrow/Ipc/ArrowFileWriter.cs b/csharp/src/Apache.Arrow/Ipc/ArrowFileWriter.cs
index 756ebfa3c35..a1da2925f34 100644
--- a/csharp/src/Apache.Arrow/Ipc/ArrowFileWriter.cs
+++ b/csharp/src/Apache.Arrow/Ipc/ArrowFileWriter.cs
@@ -61,6 +61,19 @@ public ArrowFileWriter(Stream stream, Schema schema, bool leaveOpen, IpcOptions
             RecordBatchBlocks = new List<Block>();
         }
 
+        public override void WriteRecordBatch(RecordBatch recordBatch)
+        {
+            // TODO: Compare record batch schema
+
+            if (!HasWrittenHeader)
+            {
+                WriteHeader();
+                HasWrittenHeader = true;
+            }
+
+            WriteRecordBatchInternal(recordBatch);
+        }
+
         public override async Task WriteRecordBatchAsync(RecordBatch recordBatch, CancellationToken cancellationToken = default)
         {
             // TODO: Compare record batch schema
@@ -104,6 +117,13 @@ private protected override void FinishedWritingRecordBatch(long bodyLength, long
             _currentRecordBatchOffset = -1;
         }
 
+        private protected override void WriteEndInternal()
+        {
+            base.WriteEndInternal();
+
+            WriteFooter(Schema);
+        }
+
         private protected override async ValueTask WriteEndInternalAsync(CancellationToken cancellationToken)
         {
             await base.WriteEndInternalAsync(cancellationToken);
@@ -111,6 +131,14 @@ private protected override async ValueTask WriteEndInternalAsync(CancellationTok
             await WriteFooterAsync(Schema, cancellationToken);
         }
 
+        private void WriteHeader()
+        {
+            // Write magic number and empty padding up to the 8-byte boundary
+
+            WriteMagic();
+            WritePadding(CalculatePadding(ArrowFileConstants.Magic.Length));
+        }
+
         private async Task WriteHeaderAsync(CancellationToken cancellationToken)
         {
             // Write magic number and empty padding up to the 8-byte boundary
@@ -120,6 +148,64 @@ await WritePaddingAsync(CalculatePadding(ArrowFileConstants.Magic.Length))
                 .ConfigureAwait(false);
         }
 
+        private void WriteFooter(Schema schema)
+        {
+            Builder.Clear();
+
+            long offset = BaseStream.Position;
+
+            // Serialize the schema
+
+            FlatBuffers.Offset<Flatbuf.Schema> schemaOffset = SerializeSchema(schema);
+
+            // Serialize all record batches
+
+            Flatbuf.Footer.StartRecordBatchesVector(Builder, RecordBatchBlocks.Count);
+
+            foreach (Block recordBatch in RecordBatchBlocks)
+            {
+                Flatbuf.Block.CreateBlock(
+                    Builder, recordBatch.Offset, recordBatch.MetadataLength, recordBatch.BodyLength);
+            }
+
+            FlatBuffers.VectorOffset recordBatchesVectorOffset = Builder.EndVector();
+
+            // Serialize all dictionaries
+            // NOTE: Currently unsupported.
+
+            Flatbuf.Footer.StartDictionariesVector(Builder, 0);
+
+            FlatBuffers.VectorOffset dictionaryBatchesOffset = Builder.EndVector();
+
+            // Serialize and write the footer flatbuffer
+
+            FlatBuffers.Offset<Flatbuf.Footer> footerOffset = Flatbuf.Footer.CreateFooter(Builder, CurrentMetadataVersion,
+                schemaOffset, dictionaryBatchesOffset, recordBatchesVectorOffset);
+
+            Builder.Finish(footerOffset.Value);
+
+            WriteFlatBuffer();
+
+            // Write footer length
+
+            Buffers.RentReturn(4, (buffer) =>
+            {
+                int footerLength;
+                checked
+                {
+                    footerLength = (int)(BaseStream.Position - offset);
+                }
+
+                BinaryPrimitives.WriteInt32LittleEndian(buffer.Span, footerLength);
+
+                BaseStream.Write(buffer);
+            });
+
+            // Write magic
+
+            WriteMagic();
+        }
+
         private async Task WriteFooterAsync(Schema schema, CancellationToken cancellationToken)
         {
             Builder.Clear();
@@ -182,6 +268,11 @@ await Buffers.RentReturnAsync(4, async (buffer) =>
             await WriteMagicAsync(cancellationToken).ConfigureAwait(false);
         }
 
+        private void WriteMagic()
+        {
+            BaseStream.Write(ArrowFileConstants.Magic);
+        }
+
         private ValueTask WriteMagicAsync(CancellationToken cancellationToken)
         {
             return BaseStream.WriteAsync(ArrowFileConstants.Magic, cancellationToken);
diff --git a/csharp/src/Apache.Arrow/Ipc/ArrowReaderImplementation.cs b/csharp/src/Apache.Arrow/Ipc/ArrowReaderImplementation.cs
index 4c3fdd0fcac..a28bc5724ff 100644
--- a/csharp/src/Apache.Arrow/Ipc/ArrowReaderImplementation.cs
+++ b/csharp/src/Apache.Arrow/Ipc/ArrowReaderImplementation.cs
@@ -138,7 +138,6 @@ private List<IArrowArray> BuildArrays(
             return arrays;
         }
 
-
         private ArrayData LoadPrimitiveField(
             ref RecordBatchEnumerator recordBatchEnumerator,
             Field field,
@@ -147,10 +146,10 @@ private ArrayData LoadPrimitiveField(
         {
 
             ArrowBuffer nullArrowBuffer = BuildArrowBuffer(bodyData, recordBatchEnumerator.CurrentBuffer);
-            recordBatchEnumerator.MoveNextBuffer();
-            ArrowBuffer valueArrowBuffer = BuildArrowBuffer(bodyData, recordBatchEnumerator.CurrentBuffer);
-            recordBatchEnumerator.MoveNextBuffer();
-
+            if (!recordBatchEnumerator.MoveNextBuffer())
+            {
+                throw new Exception("Unable to move to the next buffer.");
+            }
 
             int fieldLength = (int)fieldNode.Length;
             int fieldNullCount = (int)fieldNode.NullCount;
@@ -165,13 +164,24 @@ private ArrayData LoadPrimitiveField(
                 throw new InvalidDataException("Null count length must be >= 0"); // TODO:Localize exception message
             }
 
-            ArrowBuffer[] arrowBuff = new[] { nullArrowBuffer, valueArrowBuffer };
+            ArrowBuffer[] arrowBuff;
+            if (field.DataType.TypeId == ArrowTypeId.Struct)
+            {
+                arrowBuff = new[] { nullArrowBuffer };
+            }
+            else
+            {
+                ArrowBuffer valueArrowBuffer = BuildArrowBuffer(bodyData, recordBatchEnumerator.CurrentBuffer);
+                recordBatchEnumerator.MoveNextBuffer();
+
+                arrowBuff = new[] { nullArrowBuffer, valueArrowBuffer };
+            }
+
             ArrayData[] children = GetChildren(ref recordBatchEnumerator, field, bodyData);
 
             return new ArrayData(field.DataType, fieldLength, fieldNullCount, 0, arrowBuff, children);
         }
 
-
         private ArrayData LoadVariableField(
             ref RecordBatchEnumerator recordBatchEnumerator,
             Field field,
@@ -180,9 +190,15 @@ private ArrayData LoadVariableField(
         {
 
             ArrowBuffer nullArrowBuffer = BuildArrowBuffer(bodyData, recordBatchEnumerator.CurrentBuffer);
-            recordBatchEnumerator.MoveNextBuffer();
+            if (!recordBatchEnumerator.MoveNextBuffer())
+            {
+                throw new Exception("Unable to move to the next buffer.");
+            }
             ArrowBuffer offsetArrowBuffer = BuildArrowBuffer(bodyData, recordBatchEnumerator.CurrentBuffer);
-            recordBatchEnumerator.MoveNextBuffer();
+            if (!recordBatchEnumerator.MoveNextBuffer())
+            {
+                throw new Exception("Unable to move to the next buffer.");
+            }
             ArrowBuffer valueArrowBuffer = BuildArrowBuffer(bodyData, recordBatchEnumerator.CurrentBuffer);
             recordBatchEnumerator.MoveNextBuffer();
 
@@ -212,14 +228,14 @@ private ArrayData[] GetChildren(
         {
             if (!(field.DataType is NestedType type)) return null;
 
-            int childrenCount = type.Children.Count;
+            int childrenCount = type.Fields.Count;
             var children = new ArrayData[childrenCount];
             for (int index = 0; index < childrenCount; index++)
             {
                 recordBatchEnumerator.MoveNextNode();
                 Flatbuf.FieldNode childFieldNode = recordBatchEnumerator.CurrentNode;
 
-                Field childField = type.Children[index];
+                Field childField = type.Fields[index];
                 ArrayData child = childField.DataType.IsFixedPrimitive()
                     ? LoadPrimitiveField(ref recordBatchEnumerator, childField, in childFieldNode, bodyData)
                     : LoadVariableField(ref recordBatchEnumerator, childField, in childFieldNode, bodyData);
diff --git a/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs b/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs
index cb2b920d61c..a7daf65db2a 100644
--- a/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs
+++ b/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs
@@ -44,7 +44,8 @@ internal class ArrowRecordBatchFlatBufferBuilder :
             IArrowArrayVisitor<Date64Array>,
             IArrowArrayVisitor<ListArray>,
             IArrowArrayVisitor<StringArray>,
-            IArrowArrayVisitor<BinaryArray>
+            IArrowArrayVisitor<BinaryArray>,
+            IArrowArrayVisitor<StructArray>
         {
             public readonly struct Buffer
             {
@@ -102,6 +103,16 @@ public void Visit(BinaryArray array)
                 _buffers.Add(CreateBuffer(array.ValueBuffer));
             }
 
+            public void Visit(StructArray array)
+            {
+                _buffers.Add(CreateBuffer(array.NullBitmapBuffer));
+
+                for (int i = 0; i < array.Fields.Count; i++)
+                {
+                    array.Fields[i].Accept(this);
+                }
+            }
+
             private void CreateBuffers(BooleanArray array)
             {
                 _buffers.Add(CreateBuffer(array.NullBitmapBuffer));
@@ -204,7 +215,7 @@ private void CountSelfAndChildrenNodes(IArrowType type, ref int count)
         {
             if (type is NestedType nestedType)
             {
-                foreach (Field childField in nestedType.Children)
+                foreach (Field childField in nestedType.Fields)
                 {
                     CountSelfAndChildrenNodes(childField.DataType, ref count);
                 }
@@ -212,53 +223,79 @@ private void CountSelfAndChildrenNodes(IArrowType type, ref int count)
             count++;
         }
 
-        private protected async Task WriteRecordBatchInternalAsync(RecordBatch recordBatch,
-            CancellationToken cancellationToken = default)
+        private protected void WriteRecordBatchInternal(RecordBatch recordBatch)
         {
             // TODO: Truncate buffers with extraneous padding / unused capacity
 
             if (!HasWrittenSchema)
             {
-                await WriteSchemaAsync(Schema, cancellationToken).ConfigureAwait(false);
+                WriteSchema(Schema);
                 HasWrittenSchema = true;
             }
 
-            Builder.Clear();
+            (ArrowRecordBatchFlatBufferBuilder recordBatchBuilder, VectorOffset fieldNodesVectorOffset) =
+                PreparingWritingRecordBatch(recordBatch);
 
-            // Serialize field nodes
+            VectorOffset buffersVectorOffset = Builder.EndVector();
 
-            int fieldCount = Schema.Fields.Count;
+            // Serialize record batch
 
-            Flatbuf.RecordBatch.StartNodesVector(Builder, CountAllNodes());
+            StartingWritingRecordBatch();
 
-            // flatbuffer struct vectors have to be created in reverse order
-            for (int i = fieldCount - 1; i >= 0; i--)
-            {
-                CreateSelfAndChildrenFieldNodes(recordBatch.Column(i).Data);
-            }
+            Offset<Flatbuf.RecordBatch> recordBatchOffset = Flatbuf.RecordBatch.CreateRecordBatch(Builder, recordBatch.Length,
+                fieldNodesVectorOffset,
+                buffersVectorOffset);
 
-            VectorOffset fieldNodesVectorOffset = Builder.EndVector();
+            long metadataLength = WriteMessage(Flatbuf.MessageHeader.RecordBatch,
+                recordBatchOffset, recordBatchBuilder.TotalLength);
 
-            // Serialize buffers
+            // Write buffer data
 
-            var recordBatchBuilder = new ArrowRecordBatchFlatBufferBuilder();
-            for (int i = 0; i < fieldCount; i++)
+            IReadOnlyList<ArrowRecordBatchFlatBufferBuilder.Buffer> buffers = recordBatchBuilder.Buffers;
+
+            long bodyLength = 0;
+
+            for (int i = 0; i < buffers.Count; i++)
             {
-                IArrowArray fieldArray = recordBatch.Column(i);
-                fieldArray.Accept(recordBatchBuilder);
+                ArrowBuffer buffer = buffers[i].DataBuffer;
+                if (buffer.IsEmpty)
+                    continue;
+
+                WriteBuffer(buffer);
+
+                int paddedLength = checked((int)BitUtility.RoundUpToMultipleOf8(buffer.Length));
+                int padding = paddedLength - buffer.Length;
+                if (padding > 0)
+                {
+                    WritePadding(padding);
+                }
+
+                bodyLength += paddedLength;
             }
 
-            IReadOnlyList<ArrowRecordBatchFlatBufferBuilder.Buffer> buffers = recordBatchBuilder.Buffers;
+            // Write padding so the record batch message body length is a multiple of 8 bytes
 
-            Flatbuf.RecordBatch.StartBuffersVector(Builder, buffers.Count);
+            int bodyPaddingLength = CalculatePadding(bodyLength);
 
-            // flatbuffer struct vectors have to be created in reverse order
-            for (int i = buffers.Count - 1; i >= 0; i--)
+            WritePadding(bodyPaddingLength);
+
+            FinishedWritingRecordBatch(bodyLength + bodyPaddingLength, metadataLength);
+        }
+
+        private protected async Task WriteRecordBatchInternalAsync(RecordBatch recordBatch,
+            CancellationToken cancellationToken = default)
+        {
+            // TODO: Truncate buffers with extraneous padding / unused capacity
+
+            if (!HasWrittenSchema)
             {
-                Flatbuf.Buffer.CreateBuffer(Builder,
-                    buffers[i].Offset, buffers[i].DataBuffer.Length);
+                await WriteSchemaAsync(Schema, cancellationToken).ConfigureAwait(false);
+                HasWrittenSchema = true;
             }
 
+            (ArrowRecordBatchFlatBufferBuilder recordBatchBuilder, VectorOffset fieldNodesVectorOffset) =
+                PreparingWritingRecordBatch(recordBatch);
+
             VectorOffset buffersVectorOffset = Builder.EndVector();
 
             // Serialize record batch
@@ -275,6 +312,8 @@ private protected async Task WriteRecordBatchInternalAsync(RecordBatch recordBat
 
             // Write buffer data
 
+            IReadOnlyList<ArrowRecordBatchFlatBufferBuilder.Buffer> buffers = recordBatchBuilder.Buffers;
+
             long bodyLength = 0;
 
             for (int i = 0; i < buffers.Count; i++)
@@ -304,6 +343,52 @@ private protected async Task WriteRecordBatchInternalAsync(RecordBatch recordBat
             FinishedWritingRecordBatch(bodyLength + bodyPaddingLength, metadataLength);
         }
 
+        private Tuple<ArrowRecordBatchFlatBufferBuilder, VectorOffset> PreparingWritingRecordBatch(RecordBatch recordBatch)
+        {
+            Builder.Clear();
+
+            // Serialize field nodes
+
+            int fieldCount = Schema.Fields.Count;
+
+            Flatbuf.RecordBatch.StartNodesVector(Builder, CountAllNodes());
+
+            // flatbuffer struct vectors have to be created in reverse order
+            for (int i = fieldCount - 1; i >= 0; i--)
+            {
+                CreateSelfAndChildrenFieldNodes(recordBatch.Column(i).Data);
+            }
+
+            VectorOffset fieldNodesVectorOffset = Builder.EndVector();
+
+            // Serialize buffers
+
+            var recordBatchBuilder = new ArrowRecordBatchFlatBufferBuilder();
+            for (int i = 0; i < fieldCount; i++)
+            {
+                IArrowArray fieldArray = recordBatch.Column(i);
+                fieldArray.Accept(recordBatchBuilder);
+            }
+
+            IReadOnlyList<ArrowRecordBatchFlatBufferBuilder.Buffer> buffers = recordBatchBuilder.Buffers;
+
+            Flatbuf.RecordBatch.StartBuffersVector(Builder, buffers.Count);
+
+            // flatbuffer struct vectors have to be created in reverse order
+            for (int i = buffers.Count - 1; i >= 0; i--)
+            {
+                Flatbuf.Buffer.CreateBuffer(Builder,
+                    buffers[i].Offset, buffers[i].DataBuffer.Length);
+            }
+
+            return Tuple.Create(recordBatchBuilder, fieldNodesVectorOffset);
+        }
+
+        private protected virtual void WriteEndInternal()
+        {
+            WriteIpcMessageLength(length: 0);
+        }
+
         private protected virtual ValueTask WriteEndInternalAsync(CancellationToken cancellationToken)
         {
             return WriteIpcMessageLengthAsync(length: 0, cancellationToken);
@@ -317,11 +402,25 @@ private protected virtual void FinishedWritingRecordBatch(long bodyLength, long
         {
         }
 
+        public virtual void WriteRecordBatch(RecordBatch recordBatch)
+        {
+            WriteRecordBatchInternal(recordBatch);
+        }
+
         public virtual Task WriteRecordBatchAsync(RecordBatch recordBatch, CancellationToken cancellationToken = default)
         {
             return WriteRecordBatchInternalAsync(recordBatch, cancellationToken);
         }
 
+        public void WriteEnd()
+        {
+            if (!HasWrittenEnd)
+            {
+                WriteEndInternal();
+                HasWrittenEnd = true;
+            }
+        }
+
         public async Task WriteEndAsync(CancellationToken cancellationToken = default)
         {
             if (!HasWrittenEnd)
@@ -331,6 +430,11 @@ public async Task WriteEndAsync(CancellationToken cancellationToken = default)
             }
         }
 
+        private void WriteBuffer(ArrowBuffer arrowBuffer)
+        {
+            BaseStream.Write(arrowBuffer.Memory);
+        }
+
         private ValueTask WriteBufferAsync(ArrowBuffer arrowBuffer, CancellationToken cancellationToken = default)
         {
             return BaseStream.WriteAsync(arrowBuffer.Memory, cancellationToken);
@@ -374,12 +478,12 @@ private ValueTask WriteBufferAsync(ArrowBuffer arrowBuffer, CancellationToken ca
                 return System.Array.Empty<Offset<Flatbuf.Field>>();
             }
 
-            int childrenCount = type.Children.Count;
+            int childrenCount = type.Fields.Count;
             var children = new Offset<Flatbuf.Field>[childrenCount];
 
             for (int i = 0; i < childrenCount; i++)
             {
-                Field childField = type.Children[i];
+                Field childField = type.Fields[i];
                 StringOffset childFieldNameOffset = Builder.CreateString(childField.Name);
                 ArrowTypeFlatbufferBuilder.FieldType childFieldType = _fieldTypeBuilder.BuildFieldType(childField);
                 VectorOffset childFieldChildrenVectorOffset = Builder.CreateVectorOfTables(GetChildrenFieldOffsets(childField));
@@ -391,6 +495,21 @@ private ValueTask WriteBufferAsync(ArrowBuffer arrowBuffer, CancellationToken ca
             return children;
         }
 
+        private Offset<Flatbuf.Schema> WriteSchema(Schema schema)
+        {
+            Builder.Clear();
+
+            // Build schema
+
+            Offset<Flatbuf.Schema> schemaOffset = SerializeSchema(schema);
+
+            // Build message
+
+            WriteMessage(Flatbuf.MessageHeader.Schema, schemaOffset, 0);
+
+            return schemaOffset;
+        }
+
         private async ValueTask<Offset<Flatbuf.Schema>> WriteSchemaAsync(Schema schema, CancellationToken cancellationToken)
         {
             Builder.Clear();
@@ -407,6 +526,36 @@ await WriteMessageAsync(Flatbuf.MessageHeader.Schema, schemaOffset, 0, cancellat
             return schemaOffset;
         }
 
+        /// <summary>
+        /// Writes the message to the <see cref="BaseStream"/>.
+        /// </summary>
+        /// <returns>
+        /// The number of bytes written to the stream.
+        /// </returns>
+        private long WriteMessage<T>(
+            Flatbuf.MessageHeader headerType, Offset<T> headerOffset, int bodyLength)
+            where T : struct
+        {
+            Offset<Flatbuf.Message> messageOffset = Flatbuf.Message.CreateMessage(
+                Builder, CurrentMetadataVersion, headerType, headerOffset.Value,
+                bodyLength);
+
+            Builder.Finish(messageOffset.Value);
+
+            ReadOnlyMemory<byte> messageData = Builder.DataBuffer.ToReadOnlyMemory(Builder.DataBuffer.Position, Builder.Offset);
+            int messagePaddingLength = CalculatePadding(_options.SizeOfIpcLength + messageData.Length);
+
+            WriteIpcMessageLength(messageData.Length + messagePaddingLength);
+
+            BaseStream.Write(messageData);
+            WritePadding(messagePaddingLength);
+
+            checked
+            {
+                return _options.SizeOfIpcLength + messageData.Length + messagePaddingLength;
+            }
+        }
+
         /// <summary>
         /// Writes the message to the <see cref="BaseStream"/>.
         /// </summary>
@@ -439,6 +588,13 @@ await WriteIpcMessageLengthAsync(messageData.Length + messagePaddingLength, canc
             }
         }
 
+        private protected void WriteFlatBuffer()
+        {
+            ReadOnlyMemory<byte> segment = Builder.DataBuffer.ToReadOnlyMemory(Builder.DataBuffer.Position, Builder.Offset);
+
+            BaseStream.Write(segment);
+        }
+
         private protected async ValueTask WriteFlatBufferAsync(CancellationToken cancellationToken = default)
         {
             ReadOnlyMemory<byte> segment = Builder.DataBuffer.ToReadOnlyMemory(Builder.DataBuffer.Position, Builder.Offset);
@@ -446,6 +602,23 @@ private protected async ValueTask WriteFlatBufferAsync(CancellationToken cancell
             await BaseStream.WriteAsync(segment, cancellationToken).ConfigureAwait(false);
         }
 
+        private void WriteIpcMessageLength(int length)
+        {
+            Buffers.RentReturn(_options.SizeOfIpcLength, (buffer) =>
+            {
+                Memory<byte> currentBufferPosition = buffer;
+                if (!_options.WriteLegacyIpcFormat)
+                {
+                    BinaryPrimitives.WriteInt32LittleEndian(
+                        currentBufferPosition.Span, MessageSerializer.IpcContinuationToken);
+                    currentBufferPosition = currentBufferPosition.Slice(sizeof(int));
+                }
+
+                BinaryPrimitives.WriteInt32LittleEndian(currentBufferPosition.Span, length);
+                BaseStream.Write(buffer);
+            });
+        }
+
         private async ValueTask WriteIpcMessageLengthAsync(int length, CancellationToken cancellationToken)
         {
             await Buffers.RentReturnAsync(_options.SizeOfIpcLength, async (buffer) =>
@@ -472,6 +645,14 @@ protected int CalculatePadding(long offset, int alignment = 8)
             }
         }
 
+        private protected void WritePadding(int length)
+        {
+            if (length > 0)
+            {
+                BaseStream.Write(s_padding.AsMemory(0, Math.Min(s_padding.Length, length)));
+            }
+        }
+
         private protected ValueTask WritePaddingAsync(int length)
         {
             if (length > 0)
diff --git a/csharp/src/Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs b/csharp/src/Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs
index e4afee3c8b6..b331e89b115 100644
--- a/csharp/src/Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs
+++ b/csharp/src/Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs
@@ -30,7 +30,7 @@ public struct FieldType
             public readonly int Offset;
 
             public static FieldType Build<T>(Flatbuf.Type type, Offset<T> offset)
-                where T: struct =>
+                where T : struct =>
                 new FieldType(type, offset.Value);
 
             public FieldType(Flatbuf.Type type, int offset)
@@ -40,7 +40,7 @@ public FieldType(Flatbuf.Type type, int offset)
             }
         }
 
-        class TypeVisitor : 
+        class TypeVisitor :
             IArrowTypeVisitor<BooleanType>,
             IArrowTypeVisitor<Int8Type>,
             IArrowTypeVisitor<Int16Type>,
@@ -60,7 +60,8 @@ class TypeVisitor :
             IArrowTypeVisitor<BinaryType>,
             IArrowTypeVisitor<TimestampType>,
             IArrowTypeVisitor<ListType>,
-            IArrowTypeVisitor<UnionType>
+            IArrowTypeVisitor<UnionType>,
+            IArrowTypeVisitor<StructType>
         {
             private FlatBufferBuilder Builder { get; }
 
@@ -100,7 +101,7 @@ public void Visit(ListType type)
             {
                 Flatbuf.List.StartList(Builder);
                 Result = FieldType.Build(
-                    Flatbuf.Type.List, 
+                    Flatbuf.Type.List,
                     Flatbuf.List.EndList(Builder));
             }
 
@@ -118,14 +119,14 @@ public void Visit(StringType type)
             }
 
             public void Visit(TimestampType type)
-            {  
+            {
                 StringOffset timezoneStringOffset = default;
 
                 if (!string.IsNullOrWhiteSpace(type.Timezone))
                     timezoneStringOffset = Builder.CreateString(type.Timezone);
 
                 Result = FieldType.Build(
-                    Flatbuf.Type.Timestamp, 
+                    Flatbuf.Type.Timestamp,
                     Flatbuf.Timestamp.CreateTimestamp(Builder, ToFlatBuffer(type.Unit), timezoneStringOffset));
             }
 
@@ -171,6 +172,12 @@ public void Visit(Time64Type type)
                     Flatbuf.Time.CreateTime(Builder, ToFlatBuffer(type.Unit), 64));
             }
 
+            public void Visit(StructType type)
+            {
+                Flatbuf.Struct_.StartStruct_(Builder);
+                Result = FieldType.Build(Flatbuf.Type.Struct_, Flatbuf.Struct_.EndStruct_(Builder));
+            }
+
             private void CreateIntType(NumberType type)
             {
                 Result = FieldType.Build(
diff --git a/csharp/src/Apache.Arrow/Ipc/MessageSerializer.cs b/csharp/src/Apache.Arrow/Ipc/MessageSerializer.cs
index 997cc635cd5..e86336abe4e 100644
--- a/csharp/src/Apache.Arrow/Ipc/MessageSerializer.cs
+++ b/csharp/src/Apache.Arrow/Ipc/MessageSerializer.cs
@@ -14,6 +14,7 @@
 // limitations under the License.
 
 using System;
+using System.Diagnostics;
 using System.IO;
 
 namespace Apache.Arrow.Ipc
@@ -58,15 +59,28 @@ internal static Schema GetSchema(Flatbuf.Schema schema)
             {
                 Flatbuf.Field field = schema.Fields(i).GetValueOrDefault();
 
-                schemaBuilder.Field(
-                    new Field(field.Name, GetFieldArrowType(field), field.Nullable));
+                schemaBuilder.Field(FieldFromFlatbuffer(field));
             }
 
             return schemaBuilder.Build();
         }
 
+        private static Field FieldFromFlatbuffer(Flatbuf.Field flatbufField)
+        {
+            Field[] childFields = null;
+            if (flatbufField.ChildrenLength > 0)
+            {
+                childFields = new Field[flatbufField.ChildrenLength];
+                for (int i = 0; i < flatbufField.ChildrenLength; i++)
+                {
+                    Flatbuf.Field? childFlatbufField = flatbufField.Children(i);
+                    childFields[i] = FieldFromFlatbuffer(childFlatbufField.Value);
+                }
+            }
+            return new Field(flatbufField.Name, GetFieldArrowType(flatbufField, childFields), flatbufField.Nullable);
+        }
 
-        private static Types.IArrowType GetFieldArrowType(Flatbuf.Field field)
+        private static Types.IArrowType GetFieldArrowType(Flatbuf.Field field, Field[] childFields = null)
         {
             switch (field.TypeType)
             {
@@ -126,11 +140,14 @@ private static Types.IArrowType GetFieldArrowType(Flatbuf.Field field)
                 case Flatbuf.Type.Binary:
                     return Types.BinaryType.Default;
                 case Flatbuf.Type.List:
-                    if (field.ChildrenLength != 1)
+                    if (childFields == null || childFields.Length != 1)
                     {
-                        throw new InvalidDataException($"List type must have only one child.");
+                        throw new InvalidDataException($"List type must have exactly one child.");
                     }
-                    return new Types.ListType(GetFieldArrowType(field.Children(0).GetValueOrDefault()));
+                    return new Types.ListType(childFields[0]);
+                case Flatbuf.Type.Struct_:
+                    Debug.Assert(childFields != null);
+                    return new Types.StructType(childFields);
                 default:
                     throw new InvalidDataException($"Arrow primitive '{field.TypeType}' is unsupported.");
             }
diff --git a/csharp/src/Apache.Arrow/Types/ListType.cs b/csharp/src/Apache.Arrow/Types/ListType.cs
index ec702cfdd18..a006c2282dd 100644
--- a/csharp/src/Apache.Arrow/Types/ListType.cs
+++ b/csharp/src/Apache.Arrow/Types/ListType.cs
@@ -22,9 +22,9 @@ public sealed class ListType : NestedType
         public override ArrowTypeId TypeId => ArrowTypeId.List;
         public override string Name => "list";
 
-        public Field ValueField => Children[0];
+        public Field ValueField => Fields[0];
 
-        public IArrowType ValueDataType => Children[0].DataType;
+        public IArrowType ValueDataType => Fields[0].DataType;
 
         public ListType(Field valueField)
            : base(valueField) { }
diff --git a/csharp/src/Apache.Arrow/Types/NestedType.cs b/csharp/src/Apache.Arrow/Types/NestedType.cs
index cdb25675808..da6b0140aa5 100644
--- a/csharp/src/Apache.Arrow/Types/NestedType.cs
+++ b/csharp/src/Apache.Arrow/Types/NestedType.cs
@@ -20,24 +20,27 @@ namespace Apache.Arrow.Types
 {
     public abstract class NestedType : ArrowType
     {
-        public IReadOnlyList<Field> Children { get; }
+        [Obsolete("Use `Fields` instead")]
+        public IReadOnlyList<Field> Children => Fields;
 
-        protected NestedType(IReadOnlyList<Field> children)
+        public IReadOnlyList<Field> Fields { get; }
+
+        protected NestedType(IReadOnlyList<Field> fields)
         {
-            if (children == null || children.Count == 0)
+            if (fields == null || fields.Count == 0)
             {
-                throw new ArgumentNullException(nameof(children));
+                throw new ArgumentNullException(nameof(fields));
             }
-            Children = children;
+            Fields = fields;
         }
 
-        protected NestedType(Field child)
+        protected NestedType(Field field)
         {
-            if (child == null)
+            if (field == null)
             {
-                throw new ArgumentNullException(nameof(child));
+                throw new ArgumentNullException(nameof(field));
             }
-            Children = new List<Field> { child };
+            Fields = new Field[] { field };
         }
     }
 }
diff --git a/csharp/src/Apache.Arrow/Types/StructType.cs b/csharp/src/Apache.Arrow/Types/StructType.cs
index fb074c10130..79e83db165c 100644
--- a/csharp/src/Apache.Arrow/Types/StructType.cs
+++ b/csharp/src/Apache.Arrow/Types/StructType.cs
@@ -19,19 +19,13 @@
 
 namespace Apache.Arrow.Types
 {
-    public sealed class StructType : ArrowType
+    public sealed class StructType : NestedType
     {
-        private readonly List<Field> _fields;
-
         public override ArrowTypeId TypeId => ArrowTypeId.Struct;
         public override string Name => "struct";
 
-        public IEnumerable<Field> Fields => _fields;
-
-        public StructType(IEnumerable<Field> fields)
-        {
-            _fields = fields?.ToList();
-        }
+        public StructType(IReadOnlyList<Field> fields) : base(fields)
+        { }
 
         public Field GetFieldByName(string name,
             IEqualityComparer<string> comparer = default)
@@ -51,8 +45,15 @@ public int GetFieldIndex(string name,
 
             // TODO: Consider caching field index if this method is in hot path.
 
-            return _fields.FindIndex(
-                field => comparer.Equals(field.Name, name));
+            for (int i = 0; i < Fields.Count; i++)
+            {
+                if (comparer.Equals(Fields[i].Name, name))
+                {
+                    return i;
+                }
+            }
+
+            return -1;
         }
 
         public override void Accept(IArrowTypeVisitor visitor) => Accept(this, visitor);
diff --git a/csharp/test/Apache.Arrow.Tests/ArrayTypeComparer.cs b/csharp/test/Apache.Arrow.Tests/ArrayTypeComparer.cs
new file mode 100644
index 00000000000..f75111b66d0
--- /dev/null
+++ b/csharp/test/Apache.Arrow.Tests/ArrayTypeComparer.cs
@@ -0,0 +1,121 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System.Diagnostics;
+using Apache.Arrow.Types;
+using Xunit;
+
+namespace Apache.Arrow.Tests
+{
+    public class ArrayTypeComparer :
+        IArrowTypeVisitor<TimestampType>,
+        IArrowTypeVisitor<Date32Type>,
+        IArrowTypeVisitor<Date64Type>,
+        IArrowTypeVisitor<Time32Type>,
+        IArrowTypeVisitor<Time64Type>,
+        IArrowTypeVisitor<FixedSizeBinaryType>,
+        IArrowTypeVisitor<ListType>,
+        IArrowTypeVisitor<StructType>
+    {
+        private readonly IArrowType _expectedType;
+
+        public ArrayTypeComparer(IArrowType expectedType)
+        {
+            Debug.Assert(expectedType != null);
+            _expectedType = expectedType;
+        }
+
+        public void Visit(TimestampType actualType)
+        {
+            Assert.IsAssignableFrom<TimestampType>(_expectedType);
+
+            var expectedType = (TimestampType)_expectedType;
+
+            Assert.Equal(expectedType.Timezone, actualType.Timezone);
+            Assert.Equal(expectedType.Unit, actualType.Unit);
+        }
+
+        public void Visit(Date32Type actualType)
+        {
+            Assert.IsAssignableFrom<Date32Type>(_expectedType);
+            var expectedType = (Date32Type)_expectedType;
+
+            Assert.Equal(expectedType.Unit, actualType.Unit);
+        }
+
+        public void Visit(Date64Type actualType)
+        {
+            Assert.IsAssignableFrom<Date64Type>(_expectedType);
+            var expectedType = (Date64Type)_expectedType;
+
+            Assert.Equal(expectedType.Unit, actualType.Unit);
+        }
+
+        public void Visit(Time32Type actualType)
+        {
+            Assert.IsAssignableFrom<Time32Type>(_expectedType);
+            var expectedType = (Time32Type)_expectedType;
+
+            Assert.Equal(expectedType.Unit, actualType.Unit);
+        }
+
+        public void Visit(Time64Type actualType)
+        {
+            Assert.IsAssignableFrom<Time64Type>(_expectedType);
+            var expectedType = (Time64Type)_expectedType;
+
+            Assert.Equal(expectedType.Unit, actualType.Unit);
+        }
+
+        public void Visit(FixedSizeBinaryType actualType)
+        {
+            Assert.IsAssignableFrom<FixedSizeBinaryType>(_expectedType);
+            var expectedType = (FixedSizeBinaryType)_expectedType;
+
+            Assert.Equal(expectedType.ByteWidth, actualType.ByteWidth);
+        }
+
+        public void Visit(ListType actualType)
+        {
+            Assert.IsAssignableFrom<ListType>(_expectedType);
+            var expectedType = (ListType)_expectedType;
+
+            CompareNested(expectedType, actualType);
+        }
+
+        public void Visit(StructType actualType)
+        {
+            Assert.IsAssignableFrom<StructType>(_expectedType);
+            var expectedType = (StructType)_expectedType;
+
+            CompareNested(expectedType, actualType);
+        }
+
+        private static void CompareNested(NestedType expectedType, NestedType actualType)
+        {
+            Assert.Equal(expectedType.Fields.Count, actualType.Fields.Count);
+
+            for (int i = 0; i < expectedType.Fields.Count; i++)
+            {
+                FieldComparer.Compare(expectedType.Fields[i], actualType.Fields[i]);
+            }
+        }
+
+        public void Visit(IArrowType actualType)
+        {
+            Assert.IsAssignableFrom(actualType.GetType(), _expectedType);
+        }
+    }
+}
diff --git a/csharp/test/Apache.Arrow.Tests/ArrowArrayTests.cs b/csharp/test/Apache.Arrow.Tests/ArrowArrayTests.cs
index 32290fe3872..18d405613bb 100644
--- a/csharp/test/Apache.Arrow.Tests/ArrowArrayTests.cs
+++ b/csharp/test/Apache.Arrow.Tests/ArrowArrayTests.cs
@@ -204,7 +204,7 @@ public void Visit(Date32Array array)
                 Assert.IsAssignableFrom<Date32Array>(_baseArray);
                 var baseArray = (Date32Array)_baseArray;
 
-                Assert.Equal(baseArray.GetDate(array.Offset), array.GetDate(0));
+                Assert.Equal(baseArray.GetDateTimeOffset(array.Offset), array.GetDateTimeOffset(0));
             }
 
             public void Visit(Date64Array array)
@@ -213,7 +213,7 @@ public void Visit(Date64Array array)
                 Assert.IsAssignableFrom<Date64Array>(_baseArray);
                 var baseArray = (Date64Array)_baseArray;
 
-                Assert.Equal(baseArray.GetDate(array.Offset), array.GetDate(0));
+                Assert.Equal(baseArray.GetDateTimeOffset(array.Offset), array.GetDateTimeOffset(0));
             }
 
             public void Visit(FloatArray array) => ValidateArrays(array);
diff --git a/csharp/test/Apache.Arrow.Tests/ArrowFileWriterTests.cs b/csharp/test/Apache.Arrow.Tests/ArrowFileWriterTests.cs
index a8d6f3f95f9..49ea6f9429b 100644
--- a/csharp/test/Apache.Arrow.Tests/ArrowFileWriterTests.cs
+++ b/csharp/test/Apache.Arrow.Tests/ArrowFileWriterTests.cs
@@ -61,6 +61,34 @@ public async Task WritesFooterAlignedMulitpleOf8()
         {
             RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100);
 
+            var stream = new MemoryStream();
+            var writer = new ArrowFileWriter(
+                stream,
+                originalBatch.Schema,
+                leaveOpen: true,
+                // use WriteLegacyIpcFormat, which only uses a 4-byte length prefix
+                // which causes the length prefix to not be 8-byte aligned by default
+                new IpcOptions() { WriteLegacyIpcFormat = true });
+
+            writer.WriteRecordBatch(originalBatch);
+            writer.WriteEnd();
+
+            stream.Position = 0;
+
+            await ValidateRecordBatchFile(stream, originalBatch);
+        }
+
+        /// <summary>
+        /// Tests that writing an arrow file will always align the Block lengths
+        /// to 8 bytes. There are asserts in both the reader and writer which will fail
+        /// if this isn't the case.
+        /// </summary>
+        /// <returns></returns>
+        [Fact]
+        public async Task WritesFooterAlignedMulitpleOf8Async()
+        {
+            RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100);
+
             var stream = new MemoryStream();
             var writer = new ArrowFileWriter(
                 stream,
@@ -75,11 +103,16 @@ public async Task WritesFooterAlignedMulitpleOf8()
 
             stream.Position = 0;
 
+            await ValidateRecordBatchFile(stream, originalBatch);
+        }
+
+        private async Task ValidateRecordBatchFile(Stream stream, RecordBatch recordBatch)
+        {
             var reader = new ArrowFileReader(stream);
             int count = await reader.RecordBatchCountAsync();
             Assert.Equal(1, count);
             RecordBatch readBatch = await reader.ReadRecordBatchAsync(0);
-            ArrowReaderVerifier.CompareBatches(originalBatch, readBatch);
+            ArrowReaderVerifier.CompareBatches(recordBatch, readBatch);
         }
     }
 }
diff --git a/csharp/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs b/csharp/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs
index 82bd53a47fb..b1c4653e908 100644
--- a/csharp/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs
+++ b/csharp/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs
@@ -16,7 +16,6 @@
 using Apache.Arrow.Ipc;
 using Apache.Arrow.Types;
 using System;
-using System.Diagnostics;
 using System.Linq;
 using System.Threading.Tasks;
 using Xunit;
@@ -47,7 +46,7 @@ public static async Task VerifyReaderAsync(ArrowStreamReader reader, RecordBatch
 
         public static void CompareBatches(RecordBatch expectedBatch, RecordBatch actualBatch)
         {
-            Assert.True(SchemaComparer.Equals(expectedBatch.Schema, actualBatch.Schema));
+            SchemaComparer.Compare(expectedBatch.Schema, actualBatch.Schema);
             Assert.Equal(expectedBatch.Length, actualBatch.Length);
             Assert.Equal(expectedBatch.ColumnCount, actualBatch.ColumnCount);
 
@@ -60,78 +59,6 @@ public static void CompareBatches(RecordBatch expectedBatch, RecordBatch actualB
             }
         }
 
-        private class ArrayTypeComparer :
-            IArrowTypeVisitor<TimestampType>,
-            IArrowTypeVisitor<Date32Type>,
-            IArrowTypeVisitor<Date64Type>,
-            IArrowTypeVisitor<Time32Type>,
-            IArrowTypeVisitor<Time64Type>,
-            IArrowTypeVisitor<FixedSizeBinaryType>
-        {
-            private readonly IArrowType _expectedType;
-
-            public ArrayTypeComparer(IArrowType expectedType)
-            {
-                Debug.Assert(expectedType != null);
-                _expectedType = expectedType;
-            }
-
-            public void Visit(TimestampType actualType)
-            {
-                Assert.IsAssignableFrom<TimestampType>(_expectedType);
-
-                var expectedType = (TimestampType) _expectedType;
-                
-                Assert.Equal(expectedType.Timezone, actualType.Timezone);
-                Assert.Equal(expectedType.Unit, actualType.Unit);
-            }
-
-            public void Visit(Date32Type actualType)
-            {
-                Assert.IsAssignableFrom<Date32Type>(_expectedType);
-                var expectedType = (Date32Type)_expectedType;
-
-                Assert.Equal(expectedType.Unit, actualType.Unit);
-            }
-
-            public void Visit(Date64Type actualType)
-            {
-                Assert.IsAssignableFrom<Date64Type>(_expectedType);
-                var expectedType = (Date64Type)_expectedType;
-
-                Assert.Equal(expectedType.Unit, actualType.Unit);
-            }
-
-            public void Visit(Time32Type actualType)
-            {
-                Assert.IsAssignableFrom<Time32Type>(_expectedType);
-                var expectedType = (Time32Type)_expectedType;
-
-                Assert.Equal(expectedType.Unit, actualType.Unit);
-            }
-
-            public void Visit(Time64Type actualType)
-            {
-                Assert.IsAssignableFrom<Time64Type>(_expectedType);
-                var expectedType = (Time64Type)_expectedType;
-
-                Assert.Equal(expectedType.Unit, actualType.Unit);
-            }
-
-            public void Visit(FixedSizeBinaryType actualType)
-            {
-                Assert.IsAssignableFrom<FixedSizeBinaryType>(_expectedType);
-                var expectedType = (FixedSizeBinaryType)_expectedType;
-
-                Assert.Equal(expectedType.ByteWidth, actualType.ByteWidth);
-            }
-
-            public void Visit(IArrowType actualType)
-            {
-                Assert.IsAssignableFrom(actualType.GetType(), _expectedType);
-            }
-        }
-
         private class ArrayComparer :
             IArrowArrayVisitor<Int8Array>,
             IArrowArrayVisitor<Int16Array>,
@@ -149,7 +76,8 @@ private class ArrayComparer :
             IArrowArrayVisitor<Date64Array>,
             IArrowArrayVisitor<ListArray>,
             IArrowArrayVisitor<StringArray>,
-            IArrowArrayVisitor<BinaryArray>
+            IArrowArrayVisitor<BinaryArray>,
+            IArrowArrayVisitor<StructArray>
         {
             private readonly IArrowArray _expectedArray;
             private readonly ArrayTypeComparer _arrayTypeComparer;
@@ -179,18 +107,36 @@ public ArrayComparer(IArrowArray expectedArray)
             public void Visit(StringArray array) => CompareBinaryArrays<StringArray>(array);
 
             public void Visit(BinaryArray array) => CompareBinaryArrays<BinaryArray>(array);
+
+            public void Visit(StructArray array)
+            {
+                Assert.IsAssignableFrom<StructArray>(_expectedArray);
+                StructArray expectedArray = (StructArray)_expectedArray;
+
+                Assert.Equal(expectedArray.Length, array.Length);
+                Assert.Equal(expectedArray.NullCount, array.NullCount);
+                Assert.Equal(expectedArray.Offset, array.Offset);
+                Assert.Equal(expectedArray.Data.Children.Length, array.Data.Children.Length);
+                Assert.Equal(expectedArray.Fields.Count, array.Fields.Count);
+
+                for (int i = 0; i < array.Fields.Count; i++)
+                {
+                    array.Fields[i].Accept(new ArrayComparer(expectedArray.Fields[i]));
+                }
+            }
+
             public void Visit(FixedSizeBinaryType array) => throw new NotImplementedException();
             public void Visit(IArrowArray array) => throw new NotImplementedException();
 
             private void CompareBinaryArrays<T>(BinaryArray actualArray)
-                where T: IArrowArray
+                where T : IArrowArray
             {
                 Assert.IsAssignableFrom<T>(_expectedArray);
                 Assert.IsAssignableFrom<T>(actualArray);
 
                 var expectedArray = (BinaryArray)_expectedArray;
 
-                _arrayTypeComparer.Visit(actualArray.Data.DataType);
+                actualArray.Data.DataType.Accept(_arrayTypeComparer);
 
                 Assert.Equal(expectedArray.Length, actualArray.Length);
                 Assert.Equal(expectedArray.NullCount, actualArray.NullCount);
@@ -207,7 +153,7 @@ private void CompareArrays<T>(PrimitiveArray<T> actualArray)
                 Assert.IsAssignableFrom<PrimitiveArray<T>>(_expectedArray);
                 PrimitiveArray<T> expectedArray = (PrimitiveArray<T>)_expectedArray;
 
-                _arrayTypeComparer.Visit(actualArray.Data.DataType);
+                actualArray.Data.DataType.Accept(_arrayTypeComparer);
 
                 Assert.Equal(expectedArray.Length, actualArray.Length);
                 Assert.Equal(expectedArray.NullCount, actualArray.NullCount);
@@ -222,7 +168,7 @@ private void CompareArrays(BooleanArray actualArray)
                 Assert.IsAssignableFrom<BooleanArray>(_expectedArray);
                 BooleanArray expectedArray = (BooleanArray)_expectedArray;
 
-                _arrayTypeComparer.Visit(actualArray.Data.DataType);
+                actualArray.Data.DataType.Accept(_arrayTypeComparer);
 
                 Assert.Equal(expectedArray.Length, actualArray.Length);
                 Assert.Equal(expectedArray.NullCount, actualArray.NullCount);
@@ -239,7 +185,7 @@ private void CompareArrays(ListArray actualArray)
                 Assert.IsAssignableFrom<ListArray>(_expectedArray);
                 ListArray expectedArray = (ListArray)_expectedArray;
 
-                _arrayTypeComparer.Visit(actualArray.Data.DataType);
+                actualArray.Data.DataType.Accept(_arrayTypeComparer);
 
                 Assert.Equal(expectedArray.Length, actualArray.Length);
                 Assert.Equal(expectedArray.NullCount, actualArray.NullCount);
diff --git a/csharp/test/Apache.Arrow.Tests/ArrowStreamWriterTests.cs b/csharp/test/Apache.Arrow.Tests/ArrowStreamWriterTests.cs
index 3387289e97b..b65d7353186 100644
--- a/csharp/test/Apache.Arrow.Tests/ArrowStreamWriterTests.cs
+++ b/csharp/test/Apache.Arrow.Tests/ArrowStreamWriterTests.cs
@@ -56,7 +56,41 @@ public void Ctor_LeaveOpenTrue_StreamValidOnDispose()
         }
 
         [Fact]
-        public async Task CanWriteToNetworkStream()
+        public void CanWriteToNetworkStream()
+        {
+            RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100);
+
+            const int port = 32153;
+            TcpListener listener = new TcpListener(IPAddress.Loopback, port);
+            listener.Start();
+
+            using (TcpClient sender = new TcpClient())
+            {
+                sender.Connect(IPAddress.Loopback, port);
+                NetworkStream stream = sender.GetStream();
+
+                using (var writer = new ArrowStreamWriter(stream, originalBatch.Schema))
+                {
+                    writer.WriteRecordBatch(originalBatch);
+                    writer.WriteEnd();
+
+                    stream.Flush();
+                }
+            }
+
+            using (TcpClient receiver = listener.AcceptTcpClient())
+            {
+                NetworkStream stream = receiver.GetStream();
+                using (var reader = new ArrowStreamReader(stream))
+                {
+                    RecordBatch newBatch = reader.ReadNextRecordBatch();
+                    ArrowReaderVerifier.CompareBatches(originalBatch, newBatch);
+                }
+            }
+        }
+
+        [Fact]
+        public async Task CanWriteToNetworkStreamAsync()
         {
             RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100);
 
@@ -90,15 +124,23 @@ public async Task CanWriteToNetworkStream()
         }
 
         [Fact]
-        public async Task WriteEmptyBatch()
+        public void WriteEmptyBatch()
+        {
+            RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 0);
+
+            TestRoundTripRecordBatch(originalBatch);
+        }
+
+        [Fact]
+        public async Task WriteEmptyBatchAsync()
         {
             RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 0);
 
-            await TestRoundTripRecordBatch(originalBatch);
+            await TestRoundTripRecordBatchAsync(originalBatch);
         }
 
         [Fact]
-        public async Task WriteBatchWithNulls()
+        public void WriteBatchWithNulls()
         {
             RecordBatch originalBatch = new RecordBatch.Builder()
                 .Append("Column1", false, col => col.Int32(array => array.AppendRange(Enumerable.Range(0, 10))))
@@ -122,10 +164,59 @@ public async Task WriteBatchWithNulls()
                     offset: 0))
                 .Build();
 
-            await TestRoundTripRecordBatch(originalBatch);
+            TestRoundTripRecordBatch(originalBatch);
         }
 
-        private static async Task TestRoundTripRecordBatch(RecordBatch originalBatch, IpcOptions options = null)
+        [Fact]
+        public async Task WriteBatchWithNullsAsync()
+        {
+            RecordBatch originalBatch = new RecordBatch.Builder()
+                .Append("Column1", false, col => col.Int32(array => array.AppendRange(Enumerable.Range(0, 10))))
+                .Append("Column2", true, new Int32Array(
+                    valueBuffer: new ArrowBuffer.Builder<int>().AppendRange(Enumerable.Range(0, 10)).Build(),
+                    nullBitmapBuffer: new ArrowBuffer.Builder<byte>().Append(0xfd).Append(0xff).Build(),
+                    length: 10,
+                    nullCount: 2,
+                    offset: 0))
+                .Append("Column3", true, new Int32Array(
+                    valueBuffer: new ArrowBuffer.Builder<int>().AppendRange(Enumerable.Range(0, 10)).Build(),
+                    nullBitmapBuffer: new ArrowBuffer.Builder<byte>().Append(0x00).Append(0x00).Build(),
+                    length: 10,
+                    nullCount: 10,
+                    offset: 0))
+                .Append("NullableBooleanColumn", true, new BooleanArray(
+                    valueBuffer: new ArrowBuffer.Builder<byte>().Append(0xfd).Append(0xff).Build(),
+                    nullBitmapBuffer: new ArrowBuffer.Builder<byte>().Append(0xed).Append(0xff).Build(),
+                    length: 10,
+                    nullCount: 3,
+                    offset: 0))
+                .Build();
+
+            await TestRoundTripRecordBatchAsync(originalBatch);
+        }
+
+        private static void TestRoundTripRecordBatch(RecordBatch originalBatch, IpcOptions options = null)
+        {
+            using (MemoryStream stream = new MemoryStream())
+            {
+                using (var writer = new ArrowStreamWriter(stream, originalBatch.Schema, leaveOpen: true, options))
+                {
+                    writer.WriteRecordBatch(originalBatch);
+                    writer.WriteEnd();
+                }
+
+                stream.Position = 0;
+
+                using (var reader = new ArrowStreamReader(stream))
+                {
+                    RecordBatch newBatch = reader.ReadNextRecordBatch();
+                    ArrowReaderVerifier.CompareBatches(originalBatch, newBatch);
+                }
+            }
+        }
+
+
+        private static async Task TestRoundTripRecordBatchAsync(RecordBatch originalBatch, IpcOptions options = null)
         {
             using (MemoryStream stream = new MemoryStream())
             {
@@ -146,7 +237,7 @@ private static async Task TestRoundTripRecordBatch(RecordBatch originalBatch, Ip
         }
 
         [Fact]
-        public async Task WriteBatchWithCorrectPadding()
+        public void WriteBatchWithCorrectPadding()
         {
             byte value1 = 0x04;
             byte value2 = 0x14;
@@ -172,7 +263,73 @@ public async Task WriteBatchWithCorrectPadding()
                 },
                 length: 1);
 
-            await TestRoundTripRecordBatch(batch);
+            TestRoundTripRecordBatch(batch);
+
+            using (MemoryStream stream = new MemoryStream())
+            {
+                using (var writer = new ArrowStreamWriter(stream, batch.Schema, leaveOpen: true))
+                {
+                    writer.WriteRecordBatch(batch);
+                    writer.WriteEnd();
+                }
+
+                byte[] writtenBytes = stream.ToArray();
+
+                // ensure that the data buffers at the end are 8-byte aligned
+                Assert.Equal(value1, writtenBytes[writtenBytes.Length - 24]);
+                Assert.Equal(value1, writtenBytes[writtenBytes.Length - 23]);
+                for (int i = 22; i > 16; i--)
+                {
+                    Assert.Equal(0, writtenBytes[writtenBytes.Length - i]);
+                }
+
+                Assert.Equal(value2, writtenBytes[writtenBytes.Length - 16]);
+                Assert.Equal(value2, writtenBytes[writtenBytes.Length - 15]);
+                for (int i = 14; i > 8; i--)
+                {
+                    Assert.Equal(0, writtenBytes[writtenBytes.Length - i]);
+                }
+
+                // verify the EOS is written correctly
+                for (int i = 8; i > 4; i--)
+                {
+                    Assert.Equal(0xFF, writtenBytes[writtenBytes.Length - i]);
+                }
+                for (int i = 4; i > 0; i--)
+                {
+                    Assert.Equal(0x00, writtenBytes[writtenBytes.Length - i]);
+                }
+            }
+        }
+
+        [Fact]
+        public async Task WriteBatchWithCorrectPaddingAsync()
+        {
+            byte value1 = 0x04;
+            byte value2 = 0x14;
+            var batch = new RecordBatch(
+                new Schema.Builder()
+                    .Field(f => f.Name("age").DataType(Int32Type.Default))
+                    .Field(f => f.Name("characterCount").DataType(Int32Type.Default))
+                    .Build(),
+                new IArrowArray[]
+                {
+                    new Int32Array(
+                        new ArrowBuffer(new byte[] { value1, value1, 0x00, 0x00 }),
+                        ArrowBuffer.Empty,
+                        length: 1,
+                        nullCount: 0,
+                        offset: 0),
+                    new Int32Array(
+                        new ArrowBuffer(new byte[] { value2, value2, 0x00, 0x00 }),
+                        ArrowBuffer.Empty,
+                        length: 1,
+                        nullCount: 0,
+                        offset: 0)
+                },
+                length: 1);
+
+            await TestRoundTripRecordBatchAsync(batch);
 
             using (MemoryStream stream = new MemoryStream())
             {
@@ -212,16 +369,64 @@ public async Task WriteBatchWithCorrectPadding()
         }
 
         [Fact]
-        public async Task LegacyIpcFormatRoundTrips()
+        public void LegacyIpcFormatRoundTrips()
+        {
+            RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100);
+            TestRoundTripRecordBatch(originalBatch, new IpcOptions() { WriteLegacyIpcFormat = true });
+        }
+
+
+        [Fact]
+        public async Task LegacyIpcFormatRoundTripsAsync()
         {
             RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100);
-            await TestRoundTripRecordBatch(originalBatch, new IpcOptions() { WriteLegacyIpcFormat = true });
+            await TestRoundTripRecordBatchAsync(originalBatch, new IpcOptions() { WriteLegacyIpcFormat = true });
+        }
+
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public void WriteLegacyIpcFormat(bool writeLegacyIpcFormat)
+        {
+            RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100);
+            var options = new IpcOptions() { WriteLegacyIpcFormat = writeLegacyIpcFormat };
+
+            using (MemoryStream stream = new MemoryStream())
+            {
+                using (var writer = new ArrowStreamWriter(stream, originalBatch.Schema, leaveOpen: true, options))
+                {
+                    writer.WriteRecordBatch(originalBatch);
+                    writer.WriteEnd();
+                }
+
+                stream.Position = 0;
+
+                // ensure the continuation is written correctly
+                byte[] buffer = stream.ToArray();
+                int messageLength = BinaryPrimitives.ReadInt32LittleEndian(buffer);
+                int endOfBuffer1 = BinaryPrimitives.ReadInt32LittleEndian(buffer.AsSpan(buffer.Length - 8));
+                int endOfBuffer2 = BinaryPrimitives.ReadInt32LittleEndian(buffer.AsSpan(buffer.Length - 4));
+                if (writeLegacyIpcFormat)
+                {
+                    // the legacy IPC format doesn't have a continuation token at the start
+                    Assert.NotEqual(-1, messageLength);
+                    Assert.NotEqual(-1, endOfBuffer1);
+                }
+                else
+                {
+                    // the latest IPC format has a continuation token at the start
+                    Assert.Equal(-1, messageLength);
+                    Assert.Equal(-1, endOfBuffer1);
+                }
+
+                Assert.Equal(0, endOfBuffer2);
+            }
         }
 
         [Theory]
         [InlineData(true)]
         [InlineData(false)]
-        public async Task WriteLegacyIpcFormat(bool writeLegacyIpcFormat)
+        public async Task WriteLegacyIpcFormatAsync(bool writeLegacyIpcFormat)
         {
             RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100);
             var options = new IpcOptions() { WriteLegacyIpcFormat = writeLegacyIpcFormat };
diff --git a/csharp/test/Apache.Arrow.Tests/Date32ArrayTests.cs b/csharp/test/Apache.Arrow.Tests/Date32ArrayTests.cs
index f056b1195d9..0d6aad96e5d 100644
--- a/csharp/test/Apache.Arrow.Tests/Date32ArrayTests.cs
+++ b/csharp/test/Apache.Arrow.Tests/Date32ArrayTests.cs
@@ -14,34 +14,111 @@
 // limitations under the License.
 
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using Xunit;
 
 namespace Apache.Arrow.Tests
 {
     public class Date32ArrayTests
     {
-        public class Set
+        public static IEnumerable<object[]> GetDatesData() =>
+            TestDateAndTimeData.ExampleDates.Select(d => new object[] { d });
+
+        public static IEnumerable<object[]> GetDateTimesData() =>
+            TestDateAndTimeData.ExampleDateTimes.Select(dt => new object[] { dt });
+
+        public static IEnumerable<object[]> GetDateTimeOffsetsData() =>
+            TestDateAndTimeData.ExampleDateTimeOffsets.Select(dto => new object[] { dto });
+
+        public class AppendNull
         {
             [Fact]
-            public void SetAndGet()
+            public void AppendThenGetGivesNull()
+            {
+                // Arrange
+                var builder = new Date32Array.Builder();
+
+                // Act
+                builder = builder.AppendNull();
+
+                // Assert
+                var array = builder.Build();
+                Assert.Equal(1, array.Length);
+                Assert.Null(array.GetDateTime(0));
+                Assert.Null(array.GetDateTimeOffset(0));
+                Assert.Null(array.GetValue(0));
+            }
+        }
+
+        public class AppendDateTime
+        {
+            [Theory]
+            [MemberData(nameof(GetDatesData), MemberType = typeof(Date32ArrayTests))]
+            public void AppendDateGivesSameDate(DateTime date)
+            {
+                // Arrange
+                var builder = new Date32Array.Builder();
+                var expectedDateTime = date;
+                var expectedDateTimeOffset =
+                    new DateTimeOffset(DateTime.SpecifyKind(date, DateTimeKind.Unspecified), TimeSpan.Zero);
+                int expectedValue = (int)date.Subtract(new DateTime(1970, 1, 1)).TotalDays;
+
+                // Act
+                builder = builder.Append(date);
+
+                // Assert
+                var array = builder.Build();
+                Assert.Equal(1, array.Length);
+                Assert.Equal(expectedDateTime, array.GetDateTime(0));
+                Assert.Equal(expectedDateTimeOffset, array.GetDateTimeOffset(0));
+                Assert.Equal(expectedValue, array.GetValue(0));
+            }
+
+            [Theory]
+            [MemberData(nameof(GetDateTimesData), MemberType = typeof(Date32ArrayTests))]
+            public void AppendWithTimeGivesSameWithTimeIgnored(DateTime dateTime)
+            {
+                // Arrange
+                var builder = new Date32Array.Builder();
+                var expectedDateTime = dateTime.Date;
+                var expectedDateTimeOffset =
+                    new DateTimeOffset(DateTime.SpecifyKind(dateTime.Date, DateTimeKind.Unspecified), TimeSpan.Zero);
+                int expectedValue = (int)dateTime.Date.Subtract(new DateTime(1970, 1, 1)).TotalDays;
+
+                // Act
+                builder = builder.Append(dateTime);
+
+                // Assert
+                var array = builder.Build();
+                Assert.Equal(1, array.Length);
+                Assert.Equal(expectedDateTime, array.GetDateTime(0));
+                Assert.Equal(expectedDateTimeOffset, array.GetDateTimeOffset(0));
+                Assert.Equal(expectedValue, array.GetValue(0));
+            }
+        }
+
+        public class AppendDateTimeOffset
+        {
+            [Theory]
+            [MemberData(nameof(GetDateTimeOffsetsData), MemberType = typeof(Date32ArrayTests))]
+            public void AppendGivesUtcDate(DateTimeOffset dateTimeOffset)
             {
-                var now = DateTimeOffset.UtcNow;
-
-                // throw away the time portion of the date time
-                var expected = new DateTime(now.Year,
-					    now.Month,
-					    now.Day,
-					    0,
-					    0,
-					    0,
-					    DateTimeKind.Utc);
-
-                var array = new Date32Array.Builder()
-                    .Resize(1)
-                    .Set(0, expected)
-                    .Build();
-
-                Assert.Equal(expected, array.GetDate(0).Value.Date);
+                // Arrange
+                var builder = new Date32Array.Builder();
+                var expectedDateTime = dateTimeOffset.UtcDateTime.Date;
+                var expectedDateTimeOffset = new DateTimeOffset(dateTimeOffset.UtcDateTime.Date, TimeSpan.Zero);
+                int expectedValue = (int)dateTimeOffset.UtcDateTime.Date.Subtract(new DateTime(1970, 1, 1)).TotalDays;
+
+                // Act
+                builder = builder.Append(dateTimeOffset);
+
+                // Assert
+                var array = builder.Build();
+                Assert.Equal(1, array.Length);
+                Assert.Equal(expectedDateTime, array.GetDateTime(0));
+                Assert.Equal(expectedDateTimeOffset, array.GetDateTimeOffset(0));
+                Assert.Equal(expectedValue, array.GetValue(0));
             }
         }
     }
diff --git a/csharp/test/Apache.Arrow.Tests/Date64ArrayTests.cs b/csharp/test/Apache.Arrow.Tests/Date64ArrayTests.cs
new file mode 100644
index 00000000000..65cffc84e55
--- /dev/null
+++ b/csharp/test/Apache.Arrow.Tests/Date64ArrayTests.cs
@@ -0,0 +1,133 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Xunit;
+
+namespace Apache.Arrow.Tests
+{
+    public class Date64ArrayTests
+    {
+        private const long MillisecondsPerDay = 86400000;
+
+        public static IEnumerable<object[]> GetDatesData() =>
+            TestDateAndTimeData.ExampleDates.Select(d => new object[] { d });
+
+        public static IEnumerable<object[]> GetDateTimesData() =>
+            TestDateAndTimeData.ExampleDateTimes.Select(dt => new object[] { dt });
+
+        public static IEnumerable<object[]> GetDateTimeOffsetsData() =>
+            TestDateAndTimeData.ExampleDateTimeOffsets.Select(dto => new object[] { dto });
+
+        public class AppendNull
+        {
+            [Fact]
+            public void AppendThenGetGivesNull()
+            {
+                // Arrange
+                var builder = new Date64Array.Builder();
+
+                // Act
+                builder = builder.AppendNull();
+
+                // Assert
+                var array = builder.Build();
+                Assert.Equal(1, array.Length);
+                Assert.Null(array.GetDateTime(0));
+                Assert.Null(array.GetDateTimeOffset(0));
+                Assert.Null(array.GetValue(0));
+            }
+        }
+
+        public class AppendDateTime
+        {
+            [Theory]
+            [MemberData(nameof(GetDatesData), MemberType = typeof(Date64ArrayTests))]
+            public void AppendDateGivesSameDate(DateTime date)
+            {
+                // Arrange
+                var builder = new Date64Array.Builder();
+                var expectedDateTime = date;
+                var expectedDateTimeOffset =
+                    new DateTimeOffset(DateTime.SpecifyKind(date, DateTimeKind.Unspecified), TimeSpan.Zero);
+                long expectedValue = (long)date.Subtract(new DateTime(1970, 1, 1)).TotalDays * MillisecondsPerDay;
+
+                // Act
+                builder = builder.Append(date);
+
+                // Assert
+                var array = builder.Build();
+                Assert.Equal(1, array.Length);
+                Assert.Equal(expectedDateTime, array.GetDateTime(0));
+                Assert.Equal(expectedDateTimeOffset, array.GetDateTimeOffset(0));
+                Assert.Equal(expectedValue, array.GetValue(0));
+                Assert.Equal(0, array.GetValue(0).Value % MillisecondsPerDay);
+            }
+
+            [Theory]
+            [MemberData(nameof(GetDateTimesData), MemberType = typeof(Date64ArrayTests))]
+            public void AppendWithTimeGivesSameWithTimeIgnored(DateTime dateTime)
+            {
+                // Arrange
+                var builder = new Date64Array.Builder();
+                var expectedDateTime = dateTime.Date;
+                var expectedDateTimeOffset =
+                    new DateTimeOffset(DateTime.SpecifyKind(dateTime.Date, DateTimeKind.Unspecified), TimeSpan.Zero);
+                long expectedValue =
+                    (long)dateTime.Date.Subtract(new DateTime(1970, 1, 1)).TotalDays * MillisecondsPerDay;
+
+                // Act
+                builder = builder.Append(dateTime);
+
+                // Assert
+                var array = builder.Build();
+                Assert.Equal(1, array.Length);
+                Assert.Equal(expectedDateTime, array.GetDateTime(0));
+                Assert.Equal(expectedDateTimeOffset, array.GetDateTimeOffset(0));
+                Assert.Equal(expectedValue, array.GetValue(0));
+                Assert.Equal(0, array.GetValue(0).Value % MillisecondsPerDay);
+            }
+        }
+
+        public class AppendDateTimeOffset
+        {
+            [Theory]
+            [MemberData(nameof(GetDateTimeOffsetsData), MemberType = typeof(Date64ArrayTests))]
+            public void AppendGivesUtcDate(DateTimeOffset dateTimeOffset)
+            {
+                // Arrange
+                var builder = new Date64Array.Builder();
+                var expectedDateTime = dateTimeOffset.UtcDateTime.Date;
+                var expectedDateTimeOffset = new DateTimeOffset(dateTimeOffset.UtcDateTime.Date, TimeSpan.Zero);
+                long expectedValue =
+                    (long)dateTimeOffset.UtcDateTime.Date.Subtract(new DateTime(1970, 1, 1)).TotalDays *
+                    MillisecondsPerDay;
+
+                // Act
+                builder = builder.Append(dateTimeOffset);
+
+                // Assert
+                var array = builder.Build();
+                Assert.Equal(1, array.Length);
+                Assert.Equal(expectedDateTime, array.GetDateTime(0));
+                Assert.Equal(expectedDateTimeOffset, array.GetDateTimeOffset(0));
+                Assert.Equal(expectedValue, array.GetValue(0));
+                Assert.Equal(0, array.GetValue(0).Value % MillisecondsPerDay);
+            }
+        }
+    }
+}
diff --git a/csharp/test/Apache.Arrow.Tests/FieldComparer.cs b/csharp/test/Apache.Arrow.Tests/FieldComparer.cs
index 97a303fe663..fac5e05c593 100644
--- a/csharp/test/Apache.Arrow.Tests/FieldComparer.cs
+++ b/csharp/test/Apache.Arrow.Tests/FieldComparer.cs
@@ -14,29 +14,31 @@
 // limitations under the License.
 
 using System.Linq;
+using Xunit;
 
 namespace Apache.Arrow.Tests
 {
     public class FieldComparer
     {
-        public static bool Equals(Field f1, Field f2)
+        public static void Compare(Field expected, Field actual)
         {
-            if (ReferenceEquals(f1, f2))
+            if (ReferenceEquals(expected, actual))
             {
-                return true;
+                return;
             }
-            if (f2 != null && f1 != null && f1.Name == f2.Name && f1.IsNullable == f2.IsNullable &&
-                f1.DataType.TypeId == f2.DataType.TypeId && f1.HasMetadata == f2.HasMetadata)
+
+            Assert.Equal(expected.Name, actual.Name);
+            Assert.Equal(expected.IsNullable, actual.IsNullable);
+
+            Assert.Equal(expected.HasMetadata, actual.HasMetadata);
+            if (expected.HasMetadata)
             {
-                if (f1.HasMetadata && f2.HasMetadata)
-                {
-                    return f1.Metadata.Keys.Count() == f2.Metadata.Keys.Count() &&
-                           f1.Metadata.Keys.All(k => f2.Metadata.ContainsKey(k) && f1.Metadata[k] == f2.Metadata[k]) &&
-                           f2.Metadata.Keys.All(k => f1.Metadata.ContainsKey(k) && f2.Metadata[k] == f1.Metadata[k]);
-                }
-                return true;
+                Assert.Equal(expected.Metadata.Keys.Count(), actual.Metadata.Keys.Count());
+                Assert.True(expected.Metadata.Keys.All(k => actual.Metadata.ContainsKey(k) && expected.Metadata[k] == actual.Metadata[k]));
+                Assert.True(actual.Metadata.Keys.All(k => expected.Metadata.ContainsKey(k) && actual.Metadata[k] == expected.Metadata[k]));
             }
-            return false;
+
+            actual.DataType.Accept(new ArrayTypeComparer(expected.DataType));
         }
     }
 }
diff --git a/csharp/test/Apache.Arrow.Tests/SchemaBuilderTests.cs b/csharp/test/Apache.Arrow.Tests/SchemaBuilderTests.cs
index 61dee626bec..6ddbcd204d7 100644
--- a/csharp/test/Apache.Arrow.Tests/SchemaBuilderTests.cs
+++ b/csharp/test/Apache.Arrow.Tests/SchemaBuilderTests.cs
@@ -18,6 +18,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using Xunit;
+using Xunit.Sdk;
 
 namespace Apache.Arrow.Tests
 {
@@ -123,10 +124,10 @@ public void MetadataConstruction()
                 Assert.True(metadata0.Keys.SequenceEqual(schema0.Metadata.Keys) && metadata0.Values.SequenceEqual(schema0.Metadata.Values));
                 Assert.True(metadata1.Keys.SequenceEqual(schema1.Metadata.Keys) && metadata1.Values.SequenceEqual(schema1.Metadata.Values));
                 Assert.True(metadata0.Keys.SequenceEqual(schema2.Metadata.Keys) && metadata0.Values.SequenceEqual(schema2.Metadata.Values));
-                Assert.True(SchemaComparer.Equals(schema0, schema2));
-                Assert.False(SchemaComparer.Equals(schema0, schema1));
-                Assert.False(SchemaComparer.Equals(schema2, schema1));
-                Assert.False(SchemaComparer.Equals(schema2, schema3));
+                SchemaComparer.Compare(schema0, schema2);
+                Assert.Throws<EqualException>(() => SchemaComparer.Compare(schema0, schema1));
+                Assert.Throws<EqualException>(() => SchemaComparer.Compare(schema2, schema1));
+                Assert.Throws<EqualException>(() => SchemaComparer.Compare(schema2, schema3));
             }
 
             [Theory]
diff --git a/csharp/test/Apache.Arrow.Tests/SchemaComparer.cs b/csharp/test/Apache.Arrow.Tests/SchemaComparer.cs
index 8d865a85290..d1fc6363958 100644
--- a/csharp/test/Apache.Arrow.Tests/SchemaComparer.cs
+++ b/csharp/test/Apache.Arrow.Tests/SchemaComparer.cs
@@ -14,35 +14,33 @@
 // limitations under the License.
 
 using System.Linq;
+using Xunit;
 
 namespace Apache.Arrow.Tests
 {
     public class SchemaComparer
     {
-        public static bool Equals(Schema s1, Schema s2)
+        public static void Compare(Schema expected, Schema actual)
         {
-            if (ReferenceEquals(s1, s2))
+            if (ReferenceEquals(expected, actual))
             {
-                return true;
-            }
-            if (s2 == null || s1 == null || s1.HasMetadata != s2.HasMetadata || s1.Fields.Count != s2.Fields.Count)
-            {
-                return false;
+                return;
             }
 
-            if (!s1.Fields.Keys.All(k => s2.Fields.ContainsKey(k) && FieldComparer.Equals(s1.Fields[k], s2.Fields[k])) ||
-                !s2.Fields.Keys.All(k => s1.Fields.ContainsKey(k) && FieldComparer.Equals(s2.Fields[k], s1.Fields[k])))
+            Assert.Equal(expected.HasMetadata, actual.HasMetadata);
+            if (expected.HasMetadata)
             {
-                return false;
+                Assert.Equal(expected.Metadata.Keys.Count(), actual.Metadata.Keys.Count());
+                Assert.True(expected.Metadata.Keys.All(k => actual.Metadata.ContainsKey(k) && expected.Metadata[k] == actual.Metadata[k]));
+                Assert.True(actual.Metadata.Keys.All(k => expected.Metadata.ContainsKey(k) && actual.Metadata[k] == expected.Metadata[k]));
             }
 
-            if (s1.HasMetadata && s2.HasMetadata)
+            Assert.Equal(expected.Fields.Count, actual.Fields.Count);
+            Assert.True(expected.Fields.Keys.All(k => actual.Fields.ContainsKey(k)));
+            foreach (string name in expected.Fields.Keys)
             {
-                return s1.Metadata.Keys.Count() == s2.Metadata.Keys.Count() &&
-                       s1.Metadata.Keys.All(k => s2.Metadata.ContainsKey(k) && s1.Metadata[k] == s2.Metadata[k]) &&
-                       s2.Metadata.Keys.All(k => s1.Metadata.ContainsKey(k) && s2.Metadata[k] == s1.Metadata[k]);
+                FieldComparer.Compare(expected.Fields[name], actual.Fields[name]);
             }
-            return true;
         }
     }
 }
diff --git a/csharp/test/Apache.Arrow.Tests/StructArrayTests.cs b/csharp/test/Apache.Arrow.Tests/StructArrayTests.cs
new file mode 100644
index 00000000000..e2d0fa85137
--- /dev/null
+++ b/csharp/test/Apache.Arrow.Tests/StructArrayTests.cs
@@ -0,0 +1,144 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using Apache.Arrow.Ipc;
+using Apache.Arrow.Types;
+using System.Collections.Generic;
+using System.IO;
+using Xunit;
+
+namespace Apache.Arrow.Tests
+{
+    public class StructArrayTests
+    {
+        [Fact]
+        public void TestStructArray()
+        {
+            // The following can be improved with a Builder class for StructArray.
+            List<Field> fields = new List<Field>();
+            Field.Builder fieldBuilder = new Field.Builder();
+            fields.Add(fieldBuilder.Name("Strings").DataType(StringType.Default).Nullable(true).Build());
+            fieldBuilder = new Field.Builder();
+            fields.Add(fieldBuilder.Name("Ints").DataType(Int32Type.Default).Nullable(true).Build());
+            StructType structType = new StructType(fields);
+
+            StringArray.Builder stringBuilder = new StringArray.Builder();
+            StringArray stringArray = stringBuilder.Append("joe").AppendNull().AppendNull().Append("mark").Build();
+            Int32Array.Builder intBuilder = new Int32Array.Builder();
+            Int32Array intArray = intBuilder.Append(1).Append(2).AppendNull().Append(4).Build();
+            List<Array> arrays = new List<Array>();
+            arrays.Add(stringArray);
+            arrays.Add(intArray);
+
+            ArrowBuffer.BitmapBuilder nullBitmap = new ArrowBuffer.BitmapBuilder();
+            var nullBitmapBuffer = nullBitmap.Append(true).Append(true).Append(false).Append(true).Build();
+            StructArray structs = new StructArray(structType, 4, arrays, nullBitmapBuffer, 1);
+
+            Assert.Equal(4, structs.Length);
+            Assert.Equal(1, structs.NullCount);
+            ArrayData[] childArrays = structs.Data.Children; // Data for StringArray and Int32Array
+            Assert.Equal(2, childArrays.Length);
+            for (int i = 0; i < childArrays.Length; i++)
+            {
+                ArrayData arrayData = childArrays[i];
+                Assert.Null(arrayData.Children);
+                if (i == 0)
+                {
+                    Assert.Equal(ArrowTypeId.String, arrayData.DataType.TypeId);
+                    Array array = new StringArray(arrayData);
+                    StringArray structStringArray = array as StringArray;
+                    Assert.NotNull(structStringArray);
+                    Assert.Equal(structs.Length, structStringArray.Length);
+                    Assert.Equal(stringArray.Length, structStringArray.Length);
+                    Assert.Equal(stringArray.NullCount, structStringArray.NullCount);
+                    for (int j = 0; j < stringArray.Length; j++)
+                    {
+                        Assert.Equal(stringArray.GetString(j), structStringArray.GetString(j));
+                    }
+                }
+                if (i == 1)
+                {
+                    Assert.Equal(ArrowTypeId.Int32, arrayData.DataType.TypeId);
+                    Array array = new Int32Array(arrayData);
+                    Int32Array structIntArray = array as Int32Array;
+                    Assert.NotNull(structIntArray);
+                    Assert.Equal(structs.Length, structIntArray.Length);
+                    Assert.Equal(intArray.Length, structIntArray.Length);
+                    Assert.Equal(intArray.NullCount, structIntArray.NullCount);
+                    for (int j = 0; j < intArray.Length; j++)
+                    {
+                        Assert.Equal(intArray.GetValue(j), structIntArray.GetValue(j));
+                    }
+                }
+            }
+        }
+
+        [Fact]
+        public void TestListOfStructArray()
+        {
+            Schema.Builder builder = new Schema.Builder();
+            Field structField = new Field(
+                "struct",
+                new StructType(
+                    new[]
+                    {
+                        new Field("name", StringType.Default, nullable: false),
+                        new Field("age", Int64Type.Default, nullable: false),
+                    }),
+                nullable: false);
+
+            Field listField = new Field("listOfStructs", new ListType(structField), nullable: false);
+            builder.Field(listField);
+            Schema schema = builder.Build();
+
+            StringArray stringArray = new StringArray.Builder()
+                .Append("joe").AppendNull().AppendNull().Append("mark").Append("abe").Append("phil").Build();
+            Int64Array intArray = new Int64Array.Builder()
+                .Append(1).Append(2).AppendNull().Append(4).Append(10).Append(55).Build();
+
+            ArrowBuffer nullBitmapBuffer = new ArrowBuffer.BitmapBuilder()
+                .Append(true).Append(true).Append(false).Append(true).Append(true).Append(true).Build();
+
+            StructArray structs = new StructArray(structField.DataType, 6, new IArrowArray[] { stringArray, intArray }, nullBitmapBuffer, nullCount: 1);
+
+            ArrowBuffer offsetsBuffer = new ArrowBuffer.Builder<int>()
+                .Append(0).Append(2).Append(5).Append(6).Build();
+            ListArray listArray = new ListArray(listField.DataType, 3, offsetsBuffer, structs, ArrowBuffer.Empty);
+
+            RecordBatch batch = new RecordBatch(schema, new[] { listArray }, 3);
+            TestRoundTripRecordBatch(batch);
+        }
+
+        private static void TestRoundTripRecordBatch(RecordBatch originalBatch)
+        {
+            using (MemoryStream stream = new MemoryStream())
+            {
+                using (var writer = new ArrowStreamWriter(stream, originalBatch.Schema, leaveOpen: true))
+                {
+                    writer.WriteRecordBatch(originalBatch);
+                    writer.WriteEnd();
+                }
+
+                stream.Position = 0;
+
+                using (var reader = new ArrowStreamReader(stream))
+                {
+                    RecordBatch newBatch = reader.ReadNextRecordBatch();
+                    ArrowReaderVerifier.CompareBatches(originalBatch, newBatch);
+                }
+            }
+        }
+    }
+}
diff --git a/csharp/test/Apache.Arrow.Tests/TestData.cs b/csharp/test/Apache.Arrow.Tests/TestData.cs
index ba581e6669f..0066f49db51 100644
--- a/csharp/test/Apache.Arrow.Tests/TestData.cs
+++ b/csharp/test/Apache.Arrow.Tests/TestData.cs
@@ -47,6 +47,7 @@ public static RecordBatch CreateSampleRecordBatch(int length, int columnSetCount
                 builder.Field(CreateField(Date64Type.Default, i));
                 builder.Field(CreateField(TimestampType.Default, i));
                 builder.Field(CreateField(StringType.Default, i));
+                builder.Field(CreateField(new StructType(new List<Field> { CreateField(StringType.Default, i), CreateField(Int32Type.Default, i) }), i));
                 //builder.Field(CreateField(new FixedSizeBinaryType(16), i));
                 //builder.Field(CreateField(new DecimalType(19, 2)));
                 //builder.Field(CreateField(HalfFloatType.Default));
@@ -104,7 +105,8 @@ private class ArrayCreator :
             IArrowTypeVisitor<DoubleType>,
             IArrowTypeVisitor<TimestampType>,
             IArrowTypeVisitor<StringType>,
-            IArrowTypeVisitor<ListType>
+            IArrowTypeVisitor<ListType>,
+            IArrowTypeVisitor<StructType>
         {
             private int Length { get; }
             public IArrowArray Array { get; private set; }
@@ -201,6 +203,34 @@ public void Visit(ListType type)
 
             }
 
+            public void Visit(StructType type)
+            {
+                StringArray.Builder stringBuilder = new StringArray.Builder();
+                for (int i = 0; i < Length; i++)
+                {
+                    stringBuilder.Append(i.ToString());
+                }
+                StringArray stringArray = stringBuilder.Build();
+                Int32Array.Builder intBuilder = new Int32Array.Builder();
+                for (int i = 0; i < Length; i++)
+                {
+                    intBuilder.Append(i);
+                }
+                Int32Array intArray = intBuilder.Build();
+
+                List<Array> arrays = new List<Array>();
+                arrays.Add(stringArray);
+                arrays.Add(intArray);
+
+                ArrowBuffer.BitmapBuilder nullBitmap = new ArrowBuffer.BitmapBuilder();
+                for (int i = 0; i < Length; i++)
+                {
+                    nullBitmap.Append(true);
+                }
+                
+                Array = new StructArray(type, Length, arrays, nullBitmap.Build());
+            }
+
             private void GenerateArray<T, TArray, TArrayBuilder>(IArrowArrayBuilder<T, TArray, TArrayBuilder> builder, Func<int, T> generator)
                 where TArrayBuilder : IArrowArrayBuilder<T, TArray, TArrayBuilder>
                 where TArray : IArrowArray
diff --git a/csharp/test/Apache.Arrow.Tests/TestDateAndTimeData.cs b/csharp/test/Apache.Arrow.Tests/TestDateAndTimeData.cs
new file mode 100644
index 00000000000..1f2eae45b03
--- /dev/null
+++ b/csharp/test/Apache.Arrow.Tests/TestDateAndTimeData.cs
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Apache.Arrow.Tests
+{
+    /// <summary>
+    /// The <see cref="TestDateAndTimeData"/> class holds example dates and times useful for testing.
+    /// </summary>
+    internal static class TestDateAndTimeData
+    {
+        private static readonly DateTime _earliestDate = new DateTime(1, 1, 1);
+        private static readonly DateTime _latestDate = new DateTime(9999, 12, 31);
+
+        private static readonly DateTime[] _exampleDates =
+        {
+            _earliestDate, new DateTime(1969, 12, 31), new DateTime(1970, 1, 1), new DateTime(1970, 1, 2),
+            new DateTime(1972, 6, 30), new DateTime(2015, 6, 30), new DateTime(2016, 12, 31), new DateTime(2020, 2, 29),
+            new DateTime(2020, 7, 1), _latestDate,
+        };
+
+        private static readonly TimeSpan[] _exampleTimes =
+        {
+            new TimeSpan(0, 0, 1), new TimeSpan(12, 0, 0), new TimeSpan(23, 59, 59),
+        };
+
+        private static readonly DateTimeKind[] _exampleKinds =
+        {
+            DateTimeKind.Local, DateTimeKind.Unspecified, DateTimeKind.Utc,
+        };
+
+        private static readonly TimeSpan[] _exampleOffsets =
+        {
+            TimeSpan.FromHours(-2),
+            TimeSpan.Zero,
+            TimeSpan.FromHours(2),
+        };
+
+        /// <summary>
+        /// Gets a collection of example dates (i.e. with a zero time component), of all different kinds.
+        /// </summary>
+        public static IEnumerable<DateTime> ExampleDates =>
+            from date in _exampleDates
+            from kind in _exampleKinds
+            select DateTime.SpecifyKind(date, kind);
+
+        /// <summary>
+        /// Gets a collection of example date/times, of all different kinds.
+        /// </summary>
+        public static IEnumerable<DateTime> ExampleDateTimes =>
+            from date in _exampleDates
+            from time in _exampleTimes
+            from kind in _exampleKinds
+            select DateTime.SpecifyKind(date.Add(time), kind);
+
+        /// <summary>
+        /// Gets a collection of example date time offsets.
+        /// </summary>
+        /// <returns></returns>
+        public static IEnumerable<DateTimeOffset> ExampleDateTimeOffsets =>
+            from date in _exampleDates
+            from time in _exampleTimes
+            from offset in _exampleOffsets
+            where !(date == _earliestDate && offset.Ticks > 0)
+            where !(date == _latestDate && offset.Ticks < 0)
+            select new DateTimeOffset(date.Add(time), offset);
+    }
+}
diff --git a/csharp/test/Apache.Arrow.Tests/TypeTests.cs b/csharp/test/Apache.Arrow.Tests/TypeTests.cs
index ff622089b32..c279d698479 100644
--- a/csharp/test/Apache.Arrow.Tests/TypeTests.cs
+++ b/csharp/test/Apache.Arrow.Tests/TypeTests.cs
@@ -18,6 +18,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using Xunit;
+using Xunit.Sdk;
 
 namespace Apache.Arrow.Tests
 {
@@ -45,9 +46,9 @@ public void Equality()
             Field f0_other = new Field.Builder().Name("f0").DataType(Int32Type.Default).Build();
             Field f0_with_meta = new Field.Builder().Name("f0").DataType(Int32Type.Default).Nullable(true).Metadata("a", "1").Metadata("b", "2").Build();
 
-            Assert.True(FieldComparer.Equals(f0_nullable, f0_other));
-            Assert.False(FieldComparer.Equals(f0_nullable, f0_nonnullable));
-            Assert.False(FieldComparer.Equals(f0_nullable, f0_with_meta));
+            FieldComparer.Compare(f0_nullable, f0_other);
+            Assert.Throws<EqualException>(() => FieldComparer.Compare(f0_nullable, f0_nonnullable));
+            Assert.Throws<EqualException>(() => FieldComparer.Compare(f0_nullable, f0_with_meta));
         }
 
         [Fact]
@@ -58,7 +59,7 @@ public void TestMetadataConstruction()
             Field f0_nullable = new Field.Builder().Name("f0").DataType(Int32Type.Default).Metadata(metadata).Build();
             Field f1_nullable = new Field.Builder().Name("f0").DataType(Int32Type.Default).Metadata(metadata1).Build();
             Assert.True(metadata.Keys.SequenceEqual(f0_nullable.Metadata.Keys) && metadata.Values.SequenceEqual(f0_nullable.Metadata.Values));
-            Assert.True(FieldComparer.Equals(f0_nullable, f1_nullable));
+            FieldComparer.Compare(f0_nullable, f1_nullable);
         }
 
         [Fact]
@@ -73,9 +74,9 @@ public void TestStructBasics()
             StructType struct_type = new StructType(fields);
 
             var structFields = struct_type.Fields;
-            Assert.True(FieldComparer.Equals(structFields.ElementAt(0), f0_nullable));
-            Assert.True(FieldComparer.Equals(structFields.ElementAt(1), f1_nullable));
-            Assert.True(FieldComparer.Equals(structFields.ElementAt(2), f2_nullable));
+            FieldComparer.Compare(structFields.ElementAt(0), f0_nullable);
+            FieldComparer.Compare(structFields.ElementAt(1), f1_nullable);
+            FieldComparer.Compare(structFields.ElementAt(2), f2_nullable);
         }
 
         [Fact]
@@ -89,13 +90,29 @@ public void TestStructGetFieldByName()
             List<Field> fields = new List<Field>() { f0_nullable, f1_nullable, f2_nullable };
             StructType struct_type = new StructType(fields);
 
-            var structFields = struct_type.Fields;
-            Assert.True(FieldComparer.Equals(struct_type.GetFieldByName("f0"), f0_nullable));
-            Assert.True(FieldComparer.Equals(struct_type.GetFieldByName("f1"), f1_nullable));
-            Assert.True(FieldComparer.Equals(struct_type.GetFieldByName("f2"), f2_nullable));
+            FieldComparer.Compare(struct_type.GetFieldByName("f0"), f0_nullable);
+            FieldComparer.Compare(struct_type.GetFieldByName("f1"), f1_nullable);
+            FieldComparer.Compare(struct_type.GetFieldByName("f2"), f2_nullable);
             Assert.True(struct_type.GetFieldByName("not_found") == null);
         }
 
+        [Fact]
+        public void TestStructGetFieldIndex()
+        {
+            Field f0_nullable = new Field.Builder().Name("f0").DataType(Int32Type.Default).Build();
+            Field f1_nullable = new Field.Builder().Name("f1").DataType(StringType.Default).Build();
+            Field f2_nullable = new Field.Builder().Name("f2").DataType(UInt8Type.Default).Build();
+
+            StructType struct_type = new StructType(new[] { f0_nullable, f1_nullable, f2_nullable });
+
+            Assert.Equal(0, struct_type.GetFieldIndex("f0"));
+            Assert.Equal(1, struct_type.GetFieldIndex("f1"));
+            Assert.Equal(2, struct_type.GetFieldIndex("F2", StringComparer.OrdinalIgnoreCase));
+            Assert.Equal(-1, struct_type.GetFieldIndex("F2"));
+            Assert.Equal(-1, struct_type.GetFieldIndex("F2", StringComparer.Ordinal));
+            Assert.Equal(-1, struct_type.GetFieldIndex("not_found"));
+        }
+
         [Fact]
         public void TestListTypeConstructor()
         {
@@ -103,7 +120,7 @@ public void TestListTypeConstructor()
             var stringType1 = new ListType(stringField);
             var stringType2 = new ListType(StringType.Default);
 
-            Assert.True(FieldComparer.Equals(stringType1.ValueField, stringType2.ValueField));
+            FieldComparer.Compare(stringType1.ValueField, stringType2.ValueField);
             Assert.Equal(stringType1.ValueDataType.TypeId, stringType2.ValueDataType.TypeId);
         }
 
diff --git a/dev/README.md b/dev/README.md
index a09fcba146e..364bab5d3b5 100644
--- a/dev/README.md
+++ b/dev/README.md
@@ -24,9 +24,13 @@ testing, or committing to Arrow.
 
 Merging a pull request requires being a committer on the project. In addition
 you need to have linked your GitHub and ASF accounts on
-https://gitbox.apache.org/setup/ to be able to push to GitHub as the main
+https://gitbox.apache.org/setup/ to be able to push to GitHub as the mainshe
 remote.
 
+NOTE: It may take some time (a few hours) between when you complete
+the setup at GitBox, and when your github account will be added as a
+committer.
+
 ## How to merge a Pull request
 
 ```
diff --git a/dev/archery/archery/bot.py b/dev/archery/archery/bot.py
index b4ea24c1f3b..d222d1ef377 100644
--- a/dev/archery/archery/bot.py
+++ b/dev/archery/archery/bot.py
@@ -122,7 +122,12 @@ class CrossbowCommentFormatter:
             title='Appveyor',
             url='https://ci.appveyor.com/project/{repo}/history',
             badge='https://img.shields.io/appveyor/ci/{repo}/{branch}.svg'
-        )
+        ),
+        'drone': _markdown_badge.format(
+            title='Drone',
+            url='https://cloud.drone.io/{repo}',
+            badge='https://img.shields.io/drone/build/{repo}/{branch}.svg'
+        ),
     }
 
     def __init__(self, crossbow_repo):
@@ -248,13 +253,15 @@ def crossbow(obj, crossbow):
 
 
 @crossbow.command()
-@click.argument('task', nargs=-1, required=False)
-@click.option('--group', '-g', multiple=True,
+@click.argument('tasks', nargs=-1, required=False)
+@click.option('--group', '-g', 'groups', multiple=True,
               help='Submit task groups as defined in tests.yml')
+@click.option('--param', '-p', 'params', multiple=True,
+              help='Additional task parameters for rendering the CI templates')
 @click.option('--dry-run/--push', default=False,
               help='Just display the new changelog, don\'t write it')
 @click.pass_obj
-def submit(obj, task, group, dry_run):
+def submit(obj, tasks, groups, params, dry_run):
     """Submit crossbow testing tasks.
 
     See groups defined in arrow/dev/tasks/tests.yml
@@ -268,9 +275,11 @@ def submit(obj, task, group, dry_run):
     if dry_run:
         args.append('--dry-run')
 
-    for g in group:
+    for p in params:
+        args.extend(['-p', p])
+    for g in groups:
         args.extend(['-g', g])
-    for t in task:
+    for t in tasks:
         args.append(t)
 
     # pygithub pull request object
diff --git a/dev/archery/archery/cli.py b/dev/archery/archery/cli.py
index d8c944b5f00..e167d7ac65d 100644
--- a/dev/archery/archery/cli.py
+++ b/dev/archery/archery/cli.py
@@ -937,19 +937,33 @@ def release_changelog_regenerate(obj):
 
 
 @release.command('cherry-pick')
+@click.argument('version')
+@click.option('--dry-run/--execute', default=True,
+              help="Display the git commands instead of executing them.")
+@click.option('--recreate/--continue', default=True,
+              help="Recreate the maintenance branch or only apply unapplied "
+                   "patches.")
 @click.pass_obj
-def release_cherry_pick(obj):
-    """Cherry pick commits."""
-    from .release import PatchRelease
-
-    release = obj['release']
-    if not isinstance(release, PatchRelease):
-        raise click.UsageError('Cherry-pick command only supported for patch '
-                               'releases')
-
-    commands = release.generate_update_branch_commands()
-    for cmd in commands:
-        click.echo(cmd)
+def release_cherry_pick(obj, version, dry_run, recreate):
+    """
+    Cherry pick commits.
+    """
+    from .release import Release, MinorRelease, PatchRelease
+
+    release = Release.from_jira(version, jira=obj['jira'], repo=obj['repo'])
+    if not isinstance(release, (MinorRelease, PatchRelease)):
+        raise click.UsageError('Cherry-pick command only supported for minor '
+                               'and patch releases')
+
+    if not dry_run:
+        release.cherry_pick_commits(recreate_branch=recreate)
+        click.echo('Executed the following commands:\n')
+
+    click.echo(
+        'git checkout {} -b {}'.format(release.previous.tag, release.branch)
+    )
+    for commit in release.commits_to_pick():
+        click.echo('git cherry-pick {}'.format(commit.hexsha))
 
 
 if __name__ == "__main__":
diff --git a/dev/archery/archery/integration/datagen.py b/dev/archery/archery/integration/datagen.py
index 39c4b43f1d3..3d50381f0d3 100644
--- a/dev/archery/archery/integration/datagen.py
+++ b/dev/archery/archery/integration/datagen.py
@@ -400,14 +400,15 @@ def generate_column(self, size, name=None):
 
 DECIMAL_PRECISION_TO_VALUE = {
     key: (1 << (8 * i - 1)) - 1 for i, key in enumerate(
-        [1, 3, 5, 7, 10, 12, 15, 17, 19, 22, 24, 27, 29, 32, 34, 36],
+        [1, 3, 5, 7, 10, 12, 15, 17, 19, 22, 24, 27, 29, 32, 34, 36,
+         40, 42, 44, 50, 60, 70],
         start=1,
     )
 }
 
 
 def decimal_range_from_precision(precision):
-    assert 1 <= precision <= 38
+    assert 1 <= precision <= 76
     try:
         max_value = DECIMAL_PRECISION_TO_VALUE[precision]
     except KeyError:
@@ -417,7 +418,7 @@ def decimal_range_from_precision(precision):
 
 
 class DecimalField(PrimitiveField):
-    def __init__(self, name, precision, scale, bit_width=128, *,
+    def __init__(self, name, precision, scale, bit_width, *,
                  nullable=True, metadata=None):
         super().__init__(name, nullable=True,
                          metadata=metadata)
@@ -434,6 +435,7 @@ def _get_type(self):
             ('name', 'decimal'),
             ('precision', self.precision),
             ('scale', self.scale),
+            ('bitWidth', self.bit_width),
         ])
 
     def generate_column(self, size, name=None):
@@ -448,7 +450,7 @@ def generate_column(self, size, name=None):
 
 class DecimalColumn(PrimitiveColumn):
 
-    def __init__(self, name, count, is_valid, values, bit_width=128):
+    def __init__(self, name, count, is_valid, values, bit_width):
         super().__init__(name, count, is_valid, values)
         self.bit_width = bit_width
 
@@ -1272,17 +1274,33 @@ def generate_null_trivial_case(batch_sizes):
     return _generate_file('null_trivial', fields, batch_sizes)
 
 
-def generate_decimal_case():
+def generate_decimal128_case():
     fields = [
-        DecimalField(name='f{}'.format(i), precision=precision, scale=2)
+        DecimalField(name='f{}'.format(i), precision=precision, scale=2,
+                     bit_width=128)
         for i, precision in enumerate(range(3, 39))
     ]
 
     possible_batch_sizes = 7, 10
     batch_sizes = [possible_batch_sizes[i % 2] for i in range(len(fields))]
+    # 'decimal' is the original name for the test, and it must match
+    # provide "gold" files that test backwards compatibility, so they
+    # can be appropriately skipped.
     return _generate_file('decimal', fields, batch_sizes)
 
 
+def generate_decimal256_case():
+    fields = [
+        DecimalField(name='f{}'.format(i), precision=precision, scale=5,
+                     bit_width=256)
+        for i, precision in enumerate(range(37, 70))
+    ]
+
+    possible_batch_sizes = 7, 10
+    batch_sizes = [possible_batch_sizes[i % 2] for i in range(len(fields))]
+    return _generate_file('decimal256', fields, batch_sizes)
+
+
 def generate_datetime_case():
     fields = [
         DateField('f0', DateField.DAY),
@@ -1485,7 +1503,7 @@ def generate_extension_case():
                           dictionaries=[dict0])
 
 
-def get_generated_json_files(tempdir=None, flight=False):
+def get_generated_json_files(tempdir=None):
     tempdir = tempdir or tempfile.mkdtemp(prefix='arrow-integration-')
 
     def _temp_path():
@@ -1493,33 +1511,32 @@ def _temp_path():
 
     file_objs = [
         generate_primitive_case([], name='primitive_no_batches'),
-        generate_primitive_case([17, 20], name='primitive')
-        .skip_category('Rust'),
-        generate_primitive_case([0, 0, 0], name='primitive_zerolength')
-        .skip_category('Rust'),
+        generate_primitive_case([17, 20], name='primitive'),
+        generate_primitive_case([0, 0, 0], name='primitive_zerolength'),
 
         generate_primitive_large_offsets_case([17, 20])
         .skip_category('Go')
-        .skip_category('JS')
-        .skip_category('Rust'),
+        .skip_category('JS'),
 
         generate_null_case([10, 0])
-        .skip_category('Rust')
         .skip_category('JS')   # TODO(ARROW-7900)
         .skip_category('Go'),  # TODO(ARROW-7901)
 
         generate_null_trivial_case([0, 0])
-        .skip_category('Rust')
         .skip_category('JS')   # TODO(ARROW-7900)
         .skip_category('Go'),  # TODO(ARROW-7901)
 
-        generate_decimal_case()
+        generate_decimal128_case()
         .skip_category('Go')  # TODO(ARROW-7948): Decimal + Go
         .skip_category('Rust'),
 
-        generate_datetime_case()
+        generate_decimal256_case()
+        .skip_category('Go')  # TODO(ARROW-7948): Decimal + Go
+        .skip_category('JS')
         .skip_category('Rust'),
 
+        generate_datetime_case(),
+
         generate_interval_case()
         .skip_category('JS')  # TODO(ARROW-5239): Intervals + JS
         .skip_category('Rust'),
@@ -1583,11 +1600,6 @@ def _temp_path():
         .skip_category('Rust'),
     ]
 
-    if flight:
-        file_objs.append(generate_primitive_case([24 * 1024],
-                                                 name='large_batch')
-                         .skip_category('Rust'))
-
     generated_paths = []
     for file_obj in file_objs:
         out_path = os.path.join(tempdir, 'generated_' +
diff --git a/dev/archery/archery/integration/runner.py b/dev/archery/archery/integration/runner.py
index 3deb8d32af8..c1d7a697ab0 100644
--- a/dev/archery/archery/integration/runner.py
+++ b/dev/archery/archery/integration/runner.py
@@ -337,10 +337,7 @@ def run_all_tests(with_cpp=True, with_java=True, with_js=True,
         testers.append(RustTester(**kwargs))
 
     static_json_files = get_static_json_files()
-    generated_json_files = datagen.get_generated_json_files(
-        tempdir=tempdir,
-        flight=run_flight
-    )
+    generated_json_files = datagen.get_generated_json_files(tempdir=tempdir)
     json_files = static_json_files + generated_json_files
 
     # Additional integration test cases for Arrow Flight.
diff --git a/dev/archery/archery/release.py b/dev/archery/archery/release.py
index cdbf372aea4..a6b9ecc4225 100644
--- a/dev/archery/archery/release.py
+++ b/dev/archery/archery/release.py
@@ -19,6 +19,7 @@
 import functools
 import os
 import re
+import pathlib
 import shelve
 import warnings
 
@@ -34,22 +35,39 @@ def cached_property(fn):
     return property(functools.lru_cache(maxsize=1)(fn))
 
 
-class JiraVersion(SemVer):
+class Version(SemVer):
 
     __slots__ = SemVer.__slots__ + ('released', 'release_date')
 
-    def __init__(self, original_jira_version):
-        super().__init__(**SemVer.parse(original_jira_version.name).to_dict())
-        self.released = original_jira_version.released
-        self.release_date = getattr(original_jira_version, 'releaseDate', None)
+    def __init__(self, version_string, released=False, release_date=None):
+        semver = SemVer.parse(version_string)
+        super().__init__(**semver.to_dict())
+        self.released = released
+        self.release_date = release_date
 
+    @classmethod
+    def from_jira(cls, jira_version):
+        return cls(
+            version_string=jira_version.name,
+            released=jira_version.released,
+            release_date=getattr(jira_version, 'releaseDate', None)
+        )
+
+
+class Issue:
 
-class JiraIssue:
+    def __init__(self, key, type, summary):
+        self.key = key
+        self.type = type
+        self.summary = summary
 
-    def __init__(self, original_jira_issue):
-        self.key = original_jira_issue.key
-        self.type = original_jira_issue.fields.issuetype.name
-        self.summary = original_jira_issue.fields.summary
+    @classmethod
+    def from_jira(cls, jira_issue):
+        return cls(
+            key=jira_issue.key,
+            type=jira_issue.fields.issuetype.name,
+            summary=jira_issue.fields.summary
+        )
 
     @property
     def project(self):
@@ -62,37 +80,34 @@ def number(self):
 
 class Jira(JIRA):
 
-    def __init__(self, user=None, password=None):
+    def __init__(self, user=None, password=None,
+                 url='https://issues.apache.org/jira'):
         user = user or os.environ.get('APACHE_JIRA_USER')
         password = password or os.environ.get('APACHE_JIRA_PASSWORD')
-        super().__init__(
-            {'server': 'https://issues.apache.org/jira'},
-            basic_auth=(user, password)
-        )
+        super().__init__(url, basic_auth=(user, password))
 
-    def arrow_version(self, version_string):
+    def project_version(self, version_string, project='ARROW'):
         # query version from jira to populated with additional metadata
-        versions = self.arrow_versions()
-        # JiraVersion instances are comparable with strings
-        return versions[versions.index(version_string)]
+        versions = {str(v): v for v in self.project_versions(project)}
+        return versions[version_string]
 
-    def arrow_versions(self):
+    def project_versions(self, project):
         versions = []
-        for v in self.project_versions('ARROW'):
+        for v in super().project_versions(project):
             try:
-                versions.append(JiraVersion(v))
+                versions.append(Version.from_jira(v))
             except ValueError:
                 # ignore invalid semantic versions like JS-0.4.0
                 continue
         return sorted(versions, reverse=True)
 
     def issue(self, key):
-        return JiraIssue(super().issue(key))
+        return Issue.from_jira(super().issue(key))
 
-    def arrow_issues(self, version):
-        query = "project=ARROW AND fixVersion={}".format(version)
-        issues = self.search_issues(query, maxResults=False)
-        return list(map(JiraIssue, issues))
+    def project_issues(self, version, project='ARROW'):
+        query = "project={} AND fixVersion={}".format(project, version)
+        issues = super().search_issues(query, maxResults=False)
+        return list(map(Issue.from_jira, issues))
 
 
 class CachedJira:
@@ -143,6 +158,19 @@ def __str__(self):
         out += self.summary
         return out
 
+    def __eq__(self, other):
+        return (
+            self.summary == other.summary and
+            self.project == other.project and
+            self.issue == other.issue and
+            self.components == other.components
+        )
+
+    def __hash__(self):
+        return hash(
+            (self.summary, self.project, self.issue, tuple(self.components))
+        )
+
     @classmethod
     def parse(cls, headline):
         matches = _TITLE_REGEX.match(headline)
@@ -229,23 +257,41 @@ def __repr__(self):
         return "<{} {!r} {}>".format(self.__class__.__name__,
                                      str(self.version), status)
 
-    @classmethod
-    def from_jira(cls, version, jira=None, repo=None):
-        jira = jira or Jira()
+    @staticmethod
+    def from_jira(version, jira=None, repo=None):
+        if jira is None:
+            jira = Jira()
+        elif isinstance(jira, str):
+            jira = Jira(jira)
+        elif not isinstance(jira, (Jira, CachedJira)):
+            raise TypeError("`jira` argument must be a server url or a valid "
+                            "Jira instance")
 
         if repo is None:
             arrow = ArrowSources.find()
             repo = Repo(arrow.path)
-        else:
+        elif isinstance(repo, (str, pathlib.Path)):
             repo = Repo(repo)
+        elif not isinstance(repo, Repo):
+            raise TypeError("`repo` argument must be a path or a valid Repo "
+                            "instance")
 
         if isinstance(version, str):
-            version = jira.arrow_version(version)
-        elif not isinstance(version, JiraVersion):
+            version = jira.project_version(version, project='ARROW')
+        elif not isinstance(version, Version):
             raise TypeError(version)
 
         # decide the type of the release based on the version number
-        klass = Release if version.patch == 0 else PatchRelease
+        if version.patch == 0:
+            if version.minor == 0:
+                klass = MajorRelease
+            elif version.major == 0:
+                # handle minor releases before 1.0 as major releases
+                klass = MajorRelease
+            else:
+                klass = MinorRelease
+        else:
+            klass = PatchRelease
 
         # prevent instantiating release object directly
         obj = klass.__new__(klass)
@@ -265,28 +311,46 @@ def tag(self):
 
     @property
     def branch(self):
-        # TODO(kszucs): add apache remote
-        return "master"
+        raise NotImplementedError()
+
+    @property
+    def siblings(self):
+        """
+        Releases to consider when calculating previous and next releases.
+        """
+        raise NotImplementedError()
 
     @cached_property
     def previous(self):
         # select all non-patch releases
-        versions = [v for v in self.jira.arrow_versions() if v.patch == 0]
-        position = versions.index(self.version) + 1
-        if position == len(versions):
+        position = self.siblings.index(self.version)
+        try:
+            previous = self.siblings[position + 1]
+        except IndexError:
             # first release doesn't have a previous one
             return None
-        previous = versions[position]
-        return Release.from_jira(previous)
+        else:
+            return Release.from_jira(previous, jira=self.jira, repo=self.repo)
+
+    @cached_property
+    def next(self):
+        # select all non-patch releases
+        position = self.siblings.index(self.version)
+        if position <= 0:
+            raise ValueError("There is no upcoming release set in JIRA after "
+                             "version {}".format(self.version))
+        upcoming = self.siblings[position - 1]
+        return Release.from_jira(upcoming, jira=self.jira, repo=self.repo)
 
     @cached_property
     def issues(self):
-        return {i.key: i for i in self.jira.arrow_issues(self.version)}
+        issues = self.jira.project_issues(self.version, project='ARROW')
+        return {i.key: i for i in issues}
 
     @cached_property
     def commits(self):
         """
-        All commits applied between two versions on the master branch.
+        All commits applied between two versions.
         """
         if self.previous is None:
             # first release
@@ -370,52 +434,99 @@ def changelog(self):
         return JiraChangelog(release=self, categories=categories)
 
 
-class PatchRelease(Release):
+class MaintenanceMixin:
+    """
+    Utility methods for cherry-picking commits from the main branch.
+    """
+
+    def commits_to_pick(self, exclude_already_applied=True):
+        # collect commits applied on the main branch since the root of the
+        # maintenance branch (the previous major release)
+        if self.version.major == 0:
+            # treat minor releases as major releases preceeding 1.0.0 release
+            commit_range = "apache-arrow-0.{}.0..master".format(
+                self.version.minor - 1
+            )
+        else:
+            commit_range = "apache-arrow-{}.0.0..master".format(
+                self.version.major
+            )
+
+        # keeping the original order of the commits helps to minimize the merge
+        # conflicts during cherry-picks
+        commits = map(Commit, self.repo.iter_commits(commit_range))
+
+        # exclude patches that have been already applied to the maintenance
+        # branch, we cannot identify patches based on sha because it changes
+        # after the cherry pick so use commit title instead
+        if exclude_already_applied:
+            already_applied = {c.title for c in self.commits}
+        else:
+            already_applied = set()
+
+        # iterate over the commits applied on the main branch and filter out
+        # the ones that are included in the jira release
+        patches_to_pick = [c for c in commits if
+                           c.issue in self.issues and
+                           c.title not in already_applied]
+
+        return reversed(patches_to_pick)
+
+    def cherry_pick_commits(self, recreate_branch=True):
+        if recreate_branch:
+            # delete, create and checkout the maintenance branch based off of
+            # the previous tag
+            if self.branch in self.repo.branches:
+                self.repo.git.branch('-D', self.branch)
+            self.repo.git.checkout(self.previous.tag, b=self.branch)
+        else:
+            # just checkout the already existing maintenance branch
+            self.repo.git.checkout(self.branch)
+
+        # cherry pick the commits based on the jira tickets
+        for commit in self.commits_to_pick():
+            self.repo.git.cherry_pick(commit.hexsha)
+
+
+class MajorRelease(Release):
 
     @property
     def branch(self):
-        # TODO(kszucs): add apache remote
-        return "maint-{}.{}.x".format(self.version.major, self.version.minor)
+        return "master"
 
     @cached_property
-    def previous(self):
-        # select all releases under this minor
-        versions = [v for v in self.jira.arrow_versions()
-                    if v.minor == self.version.minor]
-        previous = versions[versions.index(self.version) + 1]
-        return Release.from_jira(previous)
-
-    def generate_update_branch_commands(self):
-        # cherry pick not yet cherry picked commits on top of the maintenance
-        # branch
-        try:
-            target = self.repo.branches[self.branch]
-        except IndexError:
-            # maintenance branch doesn't exist yet, so create one based off of
-            # the previous git tag
-            target = self.repo.create_head(self.branch, self.previous.tag)
-
-        # collect commits applied on master since the root of the maintenance
-        # branch (the minor release of this patch release)
-        commit_range = "apache-arrow-{}.{}.0..master".format(
-            self.version.major, self.version.minor
-        )
-        commits = list(map(Commit, self.repo.iter_commits(commit_range)))
-
-        # iterate over commits applied on master and keep the original order of
-        # the commits to minimize the merge conflicts during cherry-picks
-        patch_commits = [c for c in commits if c.issue in self.issues]
-
-        commands = [
-            'git checkout -b {} {}'.format(target, self.previous.tag)
-        ]
-        for c in reversed(patch_commits):
-            commands.append(
-                'git cherry-pick {}  # {}'.format(c.hexsha, c.title)
-            )
+    def siblings(self):
+        """
+        Filter only the major releases.
+        """
+        # handle minor releases before 1.0 as major releases
+        return [v for v in self.jira.project_versions('ARROW')
+                if v.patch == 0 and (v.major == 0 or v.minor == 0)]
+
 
-        return commands
+class MinorRelease(Release, MaintenanceMixin):
 
-    # TODO(kszucs): update_branch method which tries to cherry pick to a
-    # temporary branch and if the patches apply cleanly then update the maint
-    # reference
+    @property
+    def branch(self):
+        return "maint-{}.x.x".format(self.version.major)
+
+    @cached_property
+    def siblings(self):
+        """
+        Filter the major and minor releases.
+        """
+        return [v for v in self.jira.project_versions('ARROW') if v.patch == 0]
+
+
+class PatchRelease(Release, MaintenanceMixin):
+
+    @property
+    def branch(self):
+        return "maint-{}.{}.x".format(self.version.major, self.version.minor)
+
+    @cached_property
+    def siblings(self):
+        """
+        No filtering, consider all releases.
+        """
+        return self.jira.project_versions('ARROW')
diff --git a/dev/archery/archery/testing.py b/dev/archery/archery/testing.py
index 9b9c619c42b..a773e156032 100644
--- a/dev/archery/archery/testing.py
+++ b/dev/archery/archery/testing.py
@@ -21,6 +21,19 @@
 import re
 
 
+class DotDict(dict):
+
+    def __getattr__(self, key):
+        try:
+            item = self[key]
+        except KeyError:
+            raise AttributeError(key)
+        if isinstance(item, dict):
+            return DotDict(item)
+        else:
+            return item
+
+
 class PartialEnv(dict):
 
     def __eq__(self, other):
diff --git a/dev/archery/archery/tests/test_docker.py b/dev/archery/archery/tests/test_docker.py
index 73e7f661200..2ed0c9dd94c 100644
--- a/dev/archery/archery/tests/test_docker.py
+++ b/dev/archery/archery/tests/test_docker.py
@@ -122,7 +122,6 @@
     - conda-python:
       - conda-python-pandas
       - conda-python-dask
-    - conda-r
   - ubuntu-cpp:
     - ubuntu-cpp-cmake32
     - ubuntu-c-glib:
@@ -138,8 +137,6 @@
     image: dummy
   conda-python-dask:
     image: dummy
-  conda-r:
-    image: dummy
   ubuntu-cpp:
     image: dummy
   ubuntu-cpp-cmake32:
@@ -529,7 +526,6 @@ def test_listing_images(arrow_compose_path):
         'conda-python',
         'conda-python-dask',
         'conda-python-pandas',
-        'conda-r',
         'ubuntu-c-glib',
         'ubuntu-cpp',
         'ubuntu-cpp-cmake32',
diff --git a/dev/archery/archery/tests/test_release.py b/dev/archery/archery/tests/test_release.py
new file mode 100644
index 00000000000..ed7732df480
--- /dev/null
+++ b/dev/archery/archery/tests/test_release.py
@@ -0,0 +1,333 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pytest
+
+from archery.release import (
+    Release, MajorRelease, MinorRelease, PatchRelease,
+    Jira, Version, Issue, CommitTitle, Commit
+)
+from archery.testing import DotDict
+
+
+# subset of issues per revision
+_issues = {
+    "1.0.1": [
+        Issue("ARROW-9684", type="Bug", summary="[C++] Title"),
+        Issue("ARROW-9667", type="New Feature", summary="[Crossbow] Title"),
+        Issue("ARROW-9659", type="Bug", summary="[C++] Title"),
+        Issue("ARROW-9644", type="Bug", summary="[C++][Dataset] Title"),
+        Issue("ARROW-9643", type="Bug", summary="[C++] Title"),
+        Issue("ARROW-9609", type="Bug", summary="[C++] Title"),
+        Issue("ARROW-9606", type="Bug", summary="[C++][Dataset] Title")
+    ],
+    "1.0.0": [
+        Issue("ARROW-300", type="New Feature", summary="[Format] Title"),
+        Issue("ARROW-4427", type="Task", summary="[Doc] Title"),
+        Issue("ARROW-5035", type="Improvement", summary="[C#] Title"),
+        Issue("ARROW-8473", type="Bug", summary="[Rust] Title"),
+        Issue("ARROW-8472", type="Bug", summary="[Go][Integration] Title"),
+        Issue("ARROW-8471", type="Bug", summary="[C++][Integration] Title"),
+        Issue("ARROW-8974", type="Improvement", summary="[C++] Title"),
+        Issue("ARROW-8973", type="New Feature", summary="[Java] Title")
+    ],
+    "0.17.1": [
+        Issue("ARROW-8684", type="Bug", summary="[Python] Title"),
+        Issue("ARROW-8657", type="Bug", summary="[C++][Parquet] Title"),
+        Issue("ARROW-8641", type="Bug", summary="[Python] Title"),
+        Issue("ARROW-8609", type="Bug", summary="[C++] Title"),
+    ],
+    "0.17.0": [
+        Issue("ARROW-2882", type="New Feature", summary="[C++][Python] Title"),
+        Issue("ARROW-2587", type="Bug", summary="[Python] Title"),
+        Issue("ARROW-2447", type="Improvement", summary="[C++] Title"),
+        Issue("ARROW-2255", type="Bug", summary="[Integration] Title"),
+        Issue("ARROW-1907", type="Bug", summary="[C++/Python] Title"),
+        Issue("ARROW-1636", type="New Feature", summary="[Format] Title")
+    ]
+}
+
+
+class FakeJira(Jira):
+
+    def __init__(self):
+        pass
+
+    def project_versions(self, project='ARROW'):
+        return [
+            Version("3.0.0", released=False),
+            Version("2.0.0", released=False),
+            Version("1.1.0", released=False),
+            Version("1.0.1", released=False),
+            Version("1.0.0", released=True),
+            Version("0.17.1", released=True),
+            Version("0.17.0", released=True),
+            Version("0.16.0", released=True),
+            Version("0.15.2", released=True),
+            Version("0.15.1", released=True),
+            Version("0.15.0", released=True),
+        ]
+
+    def project_issues(self, version, project='ARROW'):
+        return _issues[str(version)]
+
+
+@pytest.fixture
+def fake_jira():
+    return FakeJira()
+
+
+def test_version(fake_jira):
+    v = Version("1.2.5")
+    assert str(v) == "1.2.5"
+    assert v.major == 1
+    assert v.minor == 2
+    assert v.patch == 5
+    assert v.released is False
+    assert v.release_date is None
+
+    v = Version("1.0.0", released=True, release_date="2020-01-01")
+    assert str(v) == "1.0.0"
+    assert v.major == 1
+    assert v.minor == 0
+    assert v.patch == 0
+    assert v.released is True
+    assert v.release_date == "2020-01-01"
+
+
+def test_issue(fake_jira):
+    i = Issue("ARROW-1234", type='Bug', summary="title")
+    assert i.key == "ARROW-1234"
+    assert i.type == "Bug"
+    assert i.summary == "title"
+    assert i.project == "ARROW"
+    assert i.number == 1234
+
+    i = Issue("PARQUET-1111", type='Improvement', summary="another title")
+    assert i.key == "PARQUET-1111"
+    assert i.type == "Improvement"
+    assert i.summary == "another title"
+    assert i.project == "PARQUET"
+    assert i.number == 1111
+
+    fake_jira_issue = DotDict({
+        'key': 'ARROW-2222',
+        'fields': {
+            'issuetype': {
+                'name': 'Feature'
+            },
+            'summary': 'Issue title'
+        }
+    })
+    i = Issue.from_jira(fake_jira_issue)
+    assert i.key == "ARROW-2222"
+    assert i.type == "Feature"
+    assert i.summary == "Issue title"
+    assert i.project == "ARROW"
+    assert i.number == 2222
+
+
+def test_commit_title():
+    t = CommitTitle.parse(
+        "ARROW-9598: [C++][Parquet] Fix writing nullable structs"
+    )
+    assert t.project == "ARROW"
+    assert t.issue == "ARROW-9598"
+    assert t.components == ["C++", "Parquet"]
+    assert t.summary == "Fix writing nullable structs"
+
+    t = CommitTitle.parse(
+        "ARROW-8002: [C++][Dataset][R] Support partitioned dataset writing"
+    )
+    assert t.project == "ARROW"
+    assert t.issue == "ARROW-8002"
+    assert t.components == ["C++", "Dataset", "R"]
+    assert t.summary == "Support partitioned dataset writing"
+
+    t = CommitTitle.parse(
+        "ARROW-9600: [Rust][Arrow] pin older version of proc-macro2 during "
+        "build"
+    )
+    assert t.project == "ARROW"
+    assert t.issue == "ARROW-9600"
+    assert t.components == ["Rust", "Arrow"]
+    assert t.summary == "pin older version of proc-macro2 during build"
+
+    t = CommitTitle.parse("[Release] Update versions for 1.0.0")
+    assert t.project is None
+    assert t.issue is None
+    assert t.components == ["Release"]
+    assert t.summary == "Update versions for 1.0.0"
+
+    t = CommitTitle.parse("[Python][Doc] Fix rst role dataset.rst (#7725)")
+    assert t.project is None
+    assert t.issue is None
+    assert t.components == ["Python", "Doc"]
+    assert t.summary == "Fix rst role dataset.rst (#7725)"
+
+    t = CommitTitle.parse(
+        "PARQUET-1882: [C++] Buffered Reads should allow for 0 length"
+    )
+    assert t.project == 'PARQUET'
+    assert t.issue == 'PARQUET-1882'
+    assert t.components == ["C++"]
+    assert t.summary == "Buffered Reads should allow for 0 length"
+
+    t = CommitTitle.parse(
+        "ARROW-9340 [R] Use CRAN version of decor package "
+        "\nsomething else\n"
+        "\nwhich should be truncated"
+    )
+    assert t.project == 'ARROW'
+    assert t.issue == 'ARROW-9340'
+    assert t.components == ["R"]
+    assert t.summary == "Use CRAN version of decor package "
+
+
+def test_release_basics(fake_jira):
+    r = Release.from_jira("1.0.0", jira=fake_jira)
+    assert isinstance(r, MajorRelease)
+    assert r.is_released is True
+    assert r.branch == 'master'
+    assert r.tag == 'apache-arrow-1.0.0'
+
+    r = Release.from_jira("1.1.0", jira=fake_jira)
+    assert isinstance(r, MinorRelease)
+    assert r.is_released is False
+    assert r.branch == 'maint-1.x.x'
+    assert r.tag == 'apache-arrow-1.1.0'
+
+    # minor releases before 1.0 are treated as major releases
+    r = Release.from_jira("0.17.0", jira=fake_jira)
+    assert isinstance(r, MajorRelease)
+    assert r.is_released is True
+    assert r.branch == 'master'
+    assert r.tag == 'apache-arrow-0.17.0'
+
+    r = Release.from_jira("0.17.1", jira=fake_jira)
+    assert isinstance(r, PatchRelease)
+    assert r.is_released is True
+    assert r.branch == 'maint-0.17.x'
+    assert r.tag == 'apache-arrow-0.17.1'
+
+
+def test_previous_and_next_release(fake_jira):
+    r = Release.from_jira("3.0.0", jira=fake_jira)
+    assert isinstance(r.previous, MajorRelease)
+    assert r.previous.version == Version("2.0.0")
+    with pytest.raises(ValueError, match="There is no upcoming release set"):
+        assert r.next
+
+    r = Release.from_jira("2.0.0", jira=fake_jira)
+    assert isinstance(r.previous, MajorRelease)
+    assert isinstance(r.next, MajorRelease)
+    assert r.previous.version == Version("1.0.0")
+    assert r.next.version == Version("3.0.0")
+
+    r = Release.from_jira("1.1.0", jira=fake_jira)
+    assert isinstance(r.previous, MajorRelease)
+    assert isinstance(r.next, MajorRelease)
+    assert r.previous.version == Version("1.0.0")
+    assert r.next.version == Version("2.0.0")
+
+    r = Release.from_jira("1.0.0", jira=fake_jira)
+    assert isinstance(r.next, MajorRelease)
+    assert isinstance(r.previous, MajorRelease)
+    assert r.previous.version == Version("0.17.0")
+    assert r.next.version == Version("2.0.0")
+
+    r = Release.from_jira("0.17.0", jira=fake_jira)
+    assert isinstance(r.previous, MajorRelease)
+    assert r.previous.version == Version("0.16.0")
+
+    r = Release.from_jira("0.15.2", jira=fake_jira)
+    assert isinstance(r.previous, PatchRelease)
+    assert isinstance(r.next, MajorRelease)
+    assert r.previous.version == Version("0.15.1")
+    assert r.next.version == Version("0.16.0")
+
+    r = Release.from_jira("0.15.1", jira=fake_jira)
+    assert isinstance(r.previous, MajorRelease)
+    assert isinstance(r.next, PatchRelease)
+    assert r.previous.version == Version("0.15.0")
+    assert r.next.version == Version("0.15.2")
+
+
+def test_release_issues(fake_jira):
+    # major release issues
+    r = Release.from_jira("1.0.0", jira=fake_jira)
+    assert r.issues.keys() == set([
+        "ARROW-300",
+        "ARROW-4427",
+        "ARROW-5035",
+        "ARROW-8473",
+        "ARROW-8472",
+        "ARROW-8471",
+        "ARROW-8974",
+        "ARROW-8973"
+    ])
+    # minor release issues
+    r = Release.from_jira("0.17.0", jira=fake_jira)
+    assert r.issues.keys() == set([
+        "ARROW-2882",
+        "ARROW-2587",
+        "ARROW-2447",
+        "ARROW-2255",
+        "ARROW-1907",
+        "ARROW-1636",
+    ])
+    # patch release issues
+    r = Release.from_jira("1.0.1", jira=fake_jira)
+    assert r.issues.keys() == set([
+        "ARROW-9684",
+        "ARROW-9667",
+        "ARROW-9659",
+        "ARROW-9644",
+        "ARROW-9643",
+        "ARROW-9609",
+        "ARROW-9606"
+    ])
+
+
+@pytest.mark.parametrize(('version', 'ncommits'), [
+    ("1.0.0", 771),
+    ("0.17.1", 27),
+    ("0.17.0", 569),
+    ("0.15.1", 41)
+])
+def test_release_commits(fake_jira, version, ncommits):
+    r = Release.from_jira(version, jira=fake_jira)
+    assert len(r.commits) == ncommits
+    for c in r.commits:
+        assert isinstance(c, Commit)
+        assert isinstance(c.title, CommitTitle)
+        assert c.url.endswith(c.hexsha)
+
+
+def test_maintenance_patch_selection(fake_jira):
+    r = Release.from_jira("0.17.1", jira=fake_jira)
+
+    shas_to_pick = [
+        c.hexsha for c in r.commits_to_pick(exclude_already_applied=False)
+    ]
+    expected = [
+        '8939b4bd446ee406d5225c79d563a27d30fd7d6d',
+        'bcef6c95a324417e85e0140f9745d342cd8784b3',
+        '6002ec388840de5622e39af85abdc57a2cccc9b2',
+        '9123dadfd123bca7af4eaa9455f5b0d1ca8b929d',
+    ]
+    assert shas_to_pick == expected
diff --git a/dev/release/00-prepare-test.rb b/dev/release/00-prepare-test.rb
index e6841aa85a8..a611c0a8662 100644
--- a/dev/release/00-prepare-test.rb
+++ b/dev/release/00-prepare-test.rb
@@ -317,6 +317,8 @@ def test_version_pre_tag
                      hunks: [
                        ["-version = \"#{@snapshot_version}\"",
                         "+version = \"#{@release_version}\""],
+                       ["-arrow = { path = \"../arrow\", version = \"#{@snapshot_version}\", optional = true }",
+                        "+arrow = { path = \"../arrow\", version = \"#{@release_version}\", optional = true }"],
                        ["-arrow = { path = \"../arrow\", version = \"#{@snapshot_version}\" }",
                         "+arrow = { path = \"../arrow\", version = \"#{@release_version}\" }"]
                      ],
@@ -330,6 +332,35 @@ def test_version_pre_tag
                         "+See [crate documentation](https://docs.rs/crate/parquet/#{@release_version}) on available API."],
                      ],
                    },
+                   {
+                     path: "rust/parquet_derive/Cargo.toml",
+                     hunks: [
+                       ["-version = \"#{@snapshot_version}\"",
+                        "+version = \"#{@release_version}\""],
+                       ["-parquet = { path = \"../parquet\", version = \"#{@snapshot_version}\" }",
+                        "+parquet = { path = \"../parquet\", version = \"#{@release_version}\" }"],
+                     ],
+                   },
+                   {
+                     path: "rust/parquet_derive/README.md",
+                     hunks: [
+                       ["-parquet = \"#{@snapshot_version}\"",
+                        "-parquet_derive = \"#{@snapshot_version}\"",
+                        "+parquet = \"#{@release_version}\"",
+                        "+parquet_derive = \"#{@release_version}\""],
+                     ],
+                   },
+                   {
+                     path: "rust/parquet_derive_test/Cargo.toml",
+                     hunks: [
+                       ["-version = \"#{@snapshot_version}\"",
+                        "+version = \"#{@release_version}\"",
+                        "-parquet = { path = \"../parquet\", version = \"#{@snapshot_version}\" }",
+                        "-parquet_derive = { path = \"../parquet_derive\", version = \"#{@snapshot_version}\" }",
+                        "+parquet = { path = \"../parquet\", version = \"#{@release_version}\" }",
+                        "+parquet_derive = { path = \"../parquet_derive\", version = \"#{@release_version}\" }"],
+                     ],
+                   },
                  ],
                  parse_patch(git("log", "-n", "1", "-p")))
   end
@@ -524,6 +555,8 @@ def test_version_post_tag
                      hunks: [
                        ["-version = \"#{@release_version}\"",
                         "+version = \"#{@next_snapshot_version}\""],
+                       ["-arrow = { path = \"../arrow\", version = \"#{@release_version}\", optional = true }",
+                        "+arrow = { path = \"../arrow\", version = \"#{@next_snapshot_version}\", optional = true }"],
                        ["-arrow = { path = \"../arrow\", version = \"#{@release_version}\" }",
                         "+arrow = { path = \"../arrow\", version = \"#{@next_snapshot_version}\" }"]
                      ],
@@ -537,6 +570,35 @@ def test_version_post_tag
                         "+See [crate documentation](https://docs.rs/crate/parquet/#{@next_snapshot_version}) on available API."],
                      ],
                    },
+                   {
+                     path: "rust/parquet_derive/Cargo.toml",
+                     hunks: [
+                       ["-version = \"#{@release_version}\"",
+                        "+version = \"#{@next_snapshot_version}\""],
+                       ["-parquet = { path = \"../parquet\", version = \"#{@release_version}\" }",
+                        "+parquet = { path = \"../parquet\", version = \"#{@next_snapshot_version}\" }"],
+                     ],
+                   },
+                   {
+                     path: "rust/parquet_derive/README.md",
+                     hunks: [
+                       ["-parquet = \"#{@release_version}\"",
+                        "-parquet_derive = \"#{@release_version}\"",
+                        "+parquet = \"#{@next_snapshot_version}\"",
+                        "+parquet_derive = \"#{@next_snapshot_version}\""],
+                     ],
+                   },
+                   {
+                     path: "rust/parquet_derive_test/Cargo.toml",
+                     hunks: [
+                       ["-version = \"#{@release_version}\"",
+                        "+version = \"#{@next_snapshot_version}\"",
+                        "-parquet = { path = \"../parquet\", version = \"#{@release_version}\" }",
+                        "-parquet_derive = { path = \"../parquet_derive\", version = \"#{@release_version}\" }",
+                        "+parquet = { path = \"../parquet\", version = \"#{@next_snapshot_version}\" }",
+                        "+parquet_derive = { path = \"../parquet_derive\", version = \"#{@next_snapshot_version}\" }"],
+                     ],
+                   },
                  ],
                  parse_patch(git("log", "-n", "1", "-p")))
   end
diff --git a/dev/release/00-prepare.sh b/dev/release/00-prepare.sh
index 1678c0228da..35a0d35333a 100755
--- a/dev/release/00-prepare.sh
+++ b/dev/release/00-prepare.sh
@@ -148,32 +148,20 @@ update_versions() {
   cd "${SOURCE_DIR}/../../rust"
   sed -i.bak -E \
     -e "s/^version = \".+\"/version = \"${version}\"/g" \
-    -e "s/^(arrow = .* version = )\".*\"(( .*)|(, features = .*))$/\\1\"${version}\"\\2/g" \
+    -e "s/^(arrow = .* version = )\".*\"(( .*)|(, features = .*)|(, optional = .*))$/\\1\"${version}\"\\2/g" \
     -e "s/^(arrow-flight = .* version = )\".+\"( .*)/\\1\"${version}\"\\2/g" \
     -e "s/^(parquet = .* version = )\".*\"(( .*)|(, features = .*))$/\\1\"${version}\"\\2/g" \
+    -e "s/^(parquet_derive = .* version = )\".*\"(( .*)|(, features = .*))$/\\1\"${version}\"\\2/g" \
     */Cargo.toml
   rm -f */Cargo.toml.bak
   git add */Cargo.toml
 
-  # Update version number for parquet README
-  sed -i.bak -E -e \
-      "s/^parquet = \".+\"/parquet = \"${version}\"/g" \
-      parquet/README.md
-  sed -i.bak -E -e \
-      "s/docs.rs\/crate\/parquet\/.+\)/docs.rs\/crate\/parquet\/${version}\)/g" \
-      parquet/README.md
-  rm -f parquet/README.md.bak
-  git add parquet/README.md
-
-  # Update version number for datafusion README
-  sed -i.bak -E -e \
-      "s/^datafusion = \".+\"/datafusion = \"${version}\"/g" \
-      datafusion/README.md
-  sed -i.bak -E -e \
-      "s/docs.rs\/crate\/datafusion\/.+\)/docs.rs\/crate\/datafusion\/${version}\)/g" \
-      datafusion/README.md
-  rm -f datafusion/README.md.bak
-  git add datafusion/README.md
+  sed -i.bak -E \
+    -e "s/^([^ ]+) = \".+\"/\\1 = \"${version}\"/g" \
+    -e "s,docs\.rs/crate/([^/]+)/[^)]+,docs.rs/crate/\\1/${version},g" \
+    */README.md
+  rm -f */README.md.bak
+  git add */README.md
   cd -
 }
 
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
index 94ae48e992a..2ba1bc98d08 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -65,42 +65,42 @@ dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-glib-dev.ins
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-glib-doc.doc-base
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-glib-doc.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-glib-doc.links
-dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-glib200.install
+dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-glib300.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-cuda-dev.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-cuda-glib-dev.install
-dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-cuda-glib200.install
-dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-cuda200.install
+dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-cuda-glib300.install
+dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-cuda300.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dataset-dev.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dataset-glib-dev.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dataset-glib-doc.doc-base
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dataset-glib-doc.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dataset-glib-doc.links
-dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dataset-glib200.install
-dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dataset200.install
+dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dataset-glib300.install
+dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dataset300.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-python-dev.install
-dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-python200.install
-dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow200.install
+dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-python300.install
+dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow300.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva-dev.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva-glib-dev.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva-glib-doc.doc-base
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva-glib-doc.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva-glib-doc.links
-dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva-glib200.install
-dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva200.install
+dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva-glib300.install
+dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva300.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libparquet-dev.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libparquet-glib-dev.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libparquet-glib-doc.doc-base
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libparquet-glib-doc.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libparquet-glib-doc.links
-dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libparquet-glib200.install
-dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libparquet200.install
+dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libparquet-glib300.install
+dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libparquet300.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libplasma-dev.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libplasma-glib-dev.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libplasma-glib-doc.doc-base
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libplasma-glib-doc.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libplasma-glib-doc.links
-dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libplasma-glib200.install
-dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libplasma200.install
+dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libplasma-glib300.install
+dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libplasma300.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/patches/series
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/plasma-store-server.install
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/rules
@@ -119,46 +119,46 @@ dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-dev.install
 dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.doc-base
 dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.install
 dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.links
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib200.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib300.install
 dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-dev.install
 dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib-dev.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib200.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda200.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib300.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda300.install
 dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-dev.install
 dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-dev.install
 dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.doc-base
 dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.install
 dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.links
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib200.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset200.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib300.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset300.install
 dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-dev.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight200.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight300.install
 dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-dev.install
 dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight-dev.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight200.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-python200.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow200.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight300.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-python300.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow300.install
 dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install
 dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-dev.install
 dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.doc-base
 dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.install
 dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.links
-dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib200.install
-dev/tasks/linux-packages/apache-arrow/debian/libgandiva200.install
+dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib300.install
+dev/tasks/linux-packages/apache-arrow/debian/libgandiva300.install
 dev/tasks/linux-packages/apache-arrow/debian/libparquet-dev.install
 dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-dev.install
 dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.doc-base
 dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.install
 dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.links
-dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib200.install
-dev/tasks/linux-packages/apache-arrow/debian/libparquet200.install
+dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib300.install
+dev/tasks/linux-packages/apache-arrow/debian/libparquet300.install
 dev/tasks/linux-packages/apache-arrow/debian/libplasma-dev.install
 dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-dev.install
 dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.doc-base
 dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.install
 dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.links
-dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib200.install
-dev/tasks/linux-packages/apache-arrow/debian/libplasma200.install
+dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib300.install
+dev/tasks/linux-packages/apache-arrow/debian/libplasma300.install
 dev/tasks/linux-packages/apache-arrow/debian/patches/series
 dev/tasks/linux-packages/apache-arrow/debian/plasma-store-server.install
 dev/tasks/linux-packages/apache-arrow/debian/rules
@@ -166,8 +166,11 @@ dev/tasks/linux-packages/apache-arrow/debian/source/format
 dev/tasks/linux-packages/apache-arrow/debian/watch
 dev/tasks/conda-recipes/*
 docs/requirements.txt
+go/arrow/flight/Flight_grpc.pb.go
 go/arrow/go.sum
 go/arrow/Gopkg.lock
+go/arrow/flight/Flight.pb.go
+go/arrow/flight/Flight_grpc.pb.go
 go/arrow/internal/cpu/*
 go/arrow/type_string.go
 go/*.tmpldata
@@ -248,8 +251,19 @@ r/man/*.Rd
 r/cran-comments.md
 r/vignettes/*.Rmd
 r/tests/testthat/test-*.txt
+r/inst/include/cpp11.hpp
+r/inst/include/cpp11/*.hpp
 .gitattributes
 ruby/red-arrow/.yardopts
 rust/arrow/test/data/*.csv
 rust/rust-toolchain
 rust/arrow-flight/src/arrow.flight.protocol.rs
+julia/Arrow/Project.toml
+julia/Arrow/README.md
+julia/Arrow/docs/Manifest.toml
+julia/Arrow/docs/Project.toml
+julia/Arrow/docs/make.jl
+julia/Arrow/docs/mkdocs.yml
+julia/Arrow/docs/src/index.md
+julia/Arrow/docs/src/manual.md
+julia/Arrow/docs/src/reference.md
\ No newline at end of file
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index e44692e59cb..0cb9e1b8b11 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -213,7 +213,6 @@ setup_tempdir() {
   fi
 }
 
-
 setup_miniconda() {
   # Setup short-lived miniconda for Python and integration tests
   if [ "$(uname)" == "Darwin" ]; then
@@ -230,16 +229,18 @@ setup_miniconda() {
     bash miniconda.sh -b -p $MINICONDA
     rm -f miniconda.sh
   fi
+  echo "Installed miniconda at ${MINICONDA}"
 
   . $MINICONDA/etc/profile.d/conda.sh
 
   conda create -n arrow-test -y -q -c conda-forge \
-        python=3.6 \
-        nomkl \
-        numpy \
-        pandas \
-        cython
+    python=3.6 \
+    nomkl \
+    numpy \
+    pandas \
+    cython
   conda activate arrow-test
+  echo "Using conda environment ${CONDA_PREFIX}"
 }
 
 # Build and test Java (Requires newer Maven -- I used 3.3.9)
@@ -374,7 +375,7 @@ test_python() {
   fi
 
   python setup.py build_ext --inplace
-  py.test pyarrow -v --pdb
+  pytest pyarrow -v --pdb
 
   popd
 }
@@ -399,7 +400,7 @@ test_glib() {
   export GI_TYPELIB_PATH=$ARROW_HOME/lib/girepository-1.0:$GI_TYPELIB_PATH
 
   if ! bundle --version; then
-    gem install bundler
+    gem install --no-document bundler
   fi
 
   bundle install --path vendor/bundle
@@ -610,18 +611,26 @@ test_binary_distribution() {
 }
 
 check_python_imports() {
-  local py_arch=$1
-
-  python -c "import pyarrow.parquet"
-  python -c "import pyarrow.plasma"
-  python -c "import pyarrow.fs"
-
-  if [[ "$py_arch" =~ ^3 ]]; then
-    # Flight, Gandiva and Dataset are only available for py3
-    python -c "import pyarrow.dataset"
-    python -c "import pyarrow.flight"
-    python -c "import pyarrow.gandiva"
-  fi
+   python << IMPORT_TESTS
+import platform
+
+import pyarrow
+import pyarrow.parquet
+import pyarrow.plasma
+import pyarrow.fs
+import pyarrow._hdfs
+import pyarrow.dataset
+import pyarrow.flight
+
+if platform.system() == "Darwin":
+    macos_version = tuple(map(int, platform.mac_ver()[0].split('.')))
+    check_s3fs = macos_version >= (10, 13)
+else:
+    check_s3fs = True
+
+if check_s3fs:
+    import pyarrow._s3fs
+IMPORT_TESTS
 }
 
 test_linux_wheels() {
@@ -637,10 +646,11 @@ test_linux_wheels() {
     for ml_spec in ${manylinuxes}; do
       # check the mandatory and optional imports
       pip install python-rc/${VERSION}-rc${RC_NUMBER}/pyarrow-${VERSION}-cp${py_arch//[mu.]/}-cp${py_arch//./}-manylinux${ml_spec}_x86_64.whl
-      check_python_imports py_arch
+      check_python_imports
 
       # install test requirements and execute the tests
       pip install -r ${ARROW_DIR}/python/requirements-test.txt
+      python -c 'import pyarrow; pyarrow.create_library_symlinks()'
       pytest --pyargs pyarrow
     done
 
@@ -657,22 +667,13 @@ test_macos_wheels() {
     conda activate ${env}
     pip install -U pip
 
-    macos_suffix=macosx
-    case "${py_arch}" in
-    *m)
-      macos_suffix="${macos_suffix}_10_9_intel"
-      ;;
-    *)
-      macos_suffix="${macos_suffix}_10_9_x86_64"
-      ;;
-    esac
-
     # check the mandatory and optional imports
-    pip install python-rc/${VERSION}-rc${RC_NUMBER}/pyarrow-${VERSION}-cp${py_arch//[m.]/}-cp${py_arch//./}-${macos_suffix}.whl
-    check_python_imports py_arch
+    pip install --find-links python-rc/${VERSION}-rc${RC_NUMBER} pyarrow==${VERSION}
+    check_python_imports
 
     # install test requirements and execute the tests
     pip install -r ${ARROW_DIR}/python/requirements-test.txt
+    python -c 'import pyarrow; pyarrow.create_library_symlinks()'
     pytest --pyargs pyarrow
 
     conda deactivate
@@ -780,15 +781,16 @@ cd ${ARROW_TMPDIR}
 
 if [ ${NEED_MINICONDA} -gt 0 ]; then
   setup_miniconda
-  echo "Using miniconda environment ${MINICONDA}"
 fi
 
 if [ "${ARTIFACT}" == "source" ]; then
   dist_name="apache-arrow-${VERSION}"
   if [ ${TEST_SOURCE} -gt 0 ]; then
     import_gpg_keys
-    fetch_archive ${dist_name}
-    tar xf ${dist_name}.tar.gz
+    if [ ! -d "${dist_name}" ]; then
+      fetch_archive ${dist_name}
+      tar xf ${dist_name}.tar.gz
+    fi
   else
     mkdir -p ${dist_name}
     if [ ! -f ${TEST_ARCHIVE} ]; then
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.6.____cpython.yaml
index 58c8c905c04..47122636a58 100644
--- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.6.____cpython.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.6.____cpython.yaml
@@ -1,13 +1,15 @@
 BUILD:
 - aarch64-conda_cos7-linux-gnu
+aws_sdk_cpp:
+- 1.8.54
 boost_cpp:
-- '1.72'
+- 1.74.0
 bzip2:
 - '1'
 c_compiler:
 - gcc
 c_compiler_version:
-- '7'
+- '7.5'
 cdt_arch:
 - aarch64
 cdt_name:
@@ -16,10 +18,12 @@ channel_sources:
 - conda-forge
 channel_targets:
 - conda-forge main
+cuda_compiler_version:
+- None
 cxx_compiler:
 - gxx
 cxx_compiler_version:
-- '7'
+- '7.5'
 docker_image:
 - condaforge/linux-anvil-aarch64
 gflags:
@@ -29,9 +33,13 @@ glog:
 grpc_cpp:
 - '1.30'
 libprotobuf:
-- '3.12'
+- '3.13'
 lz4_c:
 - 1.9.2
+numpy:
+- '1.16'
+orc:
+- 1.6.4
 pin_run_as_build:
   boost-cpp:
     max_pin: x.x.x
@@ -47,9 +55,11 @@ pin_run_as_build:
 python:
 - 3.6.* *_cpython
 re2:
-- 2020.07.06
+- 2020.08.01
 snappy:
 - '1'
+target_platform:
+- linux-aarch64
 zlib:
 - '1.2'
 zstd:
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.7.____cpython.yaml
index 01f6f27f810..1ce4521a92a 100644
--- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.7.____cpython.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.7.____cpython.yaml
@@ -1,13 +1,15 @@
 BUILD:
 - aarch64-conda_cos7-linux-gnu
+aws_sdk_cpp:
+- 1.8.54
 boost_cpp:
-- '1.72'
+- 1.74.0
 bzip2:
 - '1'
 c_compiler:
 - gcc
 c_compiler_version:
-- '7'
+- '7.5'
 cdt_arch:
 - aarch64
 cdt_name:
@@ -16,10 +18,12 @@ channel_sources:
 - conda-forge
 channel_targets:
 - conda-forge main
+cuda_compiler_version:
+- None
 cxx_compiler:
 - gxx
 cxx_compiler_version:
-- '7'
+- '7.5'
 docker_image:
 - condaforge/linux-anvil-aarch64
 gflags:
@@ -29,9 +33,13 @@ glog:
 grpc_cpp:
 - '1.30'
 libprotobuf:
-- '3.12'
+- '3.13'
 lz4_c:
 - 1.9.2
+numpy:
+- '1.16'
+orc:
+- 1.6.4
 pin_run_as_build:
   boost-cpp:
     max_pin: x.x.x
@@ -47,9 +55,11 @@ pin_run_as_build:
 python:
 - 3.7.* *_cpython
 re2:
-- 2020.07.06
+- 2020.08.01
 snappy:
 - '1'
+target_platform:
+- linux-aarch64
 zlib:
 - '1.2'
 zstd:
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.8.____cpython.yaml
index fb352edcf5a..9bf11fd5215 100644
--- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.8.____cpython.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.8.____cpython.yaml
@@ -1,13 +1,15 @@
 BUILD:
 - aarch64-conda_cos7-linux-gnu
+aws_sdk_cpp:
+- 1.8.54
 boost_cpp:
-- '1.72'
+- 1.74.0
 bzip2:
 - '1'
 c_compiler:
 - gcc
 c_compiler_version:
-- '7'
+- '7.5'
 cdt_arch:
 - aarch64
 cdt_name:
@@ -16,10 +18,12 @@ channel_sources:
 - conda-forge
 channel_targets:
 - conda-forge main
+cuda_compiler_version:
+- None
 cxx_compiler:
 - gxx
 cxx_compiler_version:
-- '7'
+- '7.5'
 docker_image:
 - condaforge/linux-anvil-aarch64
 gflags:
@@ -29,9 +33,13 @@ glog:
 grpc_cpp:
 - '1.30'
 libprotobuf:
-- '3.12'
+- '3.13'
 lz4_c:
 - 1.9.2
+numpy:
+- '1.16'
+orc:
+- 1.6.4
 pin_run_as_build:
   boost-cpp:
     max_pin: x.x.x
@@ -47,9 +55,11 @@ pin_run_as_build:
 python:
 - 3.8.* *_cpython
 re2:
-- 2020.07.06
+- 2020.08.01
 snappy:
 - '1'
+target_platform:
+- linux-aarch64
 zlib:
 - '1.2'
 zstd:
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.6.____cpython.yaml
index d11f56d1eae..05234c57e34 100644
--- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.6.____cpython.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.6.____cpython.yaml
@@ -1,5 +1,7 @@
+aws_sdk_cpp:
+- 1.8.54
 boost_cpp:
-- '1.72'
+- 1.74.0
 bzip2:
 - '1'
 c_compiler:
@@ -27,9 +29,13 @@ glog:
 grpc_cpp:
 - '1.30'
 libprotobuf:
-- '3.12'
+- '3.13'
 lz4_c:
 - 1.9.2
+numpy:
+- '1.16'
+orc:
+- 1.6.4
 pin_run_as_build:
   boost-cpp:
     max_pin: x.x.x
@@ -45,9 +51,11 @@ pin_run_as_build:
 python:
 - 3.6.* *_cpython
 re2:
-- 2020.07.06
+- 2020.08.01
 snappy:
 - '1'
+target_platform:
+- linux-64
 zip_keys:
 - - cuda_compiler_version
   - docker_image
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.7.____cpython.yaml
index 820fe15b1cf..e4ab58abee6 100644
--- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.7.____cpython.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.7.____cpython.yaml
@@ -1,5 +1,7 @@
+aws_sdk_cpp:
+- 1.8.54
 boost_cpp:
-- '1.72'
+- 1.74.0
 bzip2:
 - '1'
 c_compiler:
@@ -27,9 +29,13 @@ glog:
 grpc_cpp:
 - '1.30'
 libprotobuf:
-- '3.12'
+- '3.13'
 lz4_c:
 - 1.9.2
+numpy:
+- '1.16'
+orc:
+- 1.6.4
 pin_run_as_build:
   boost-cpp:
     max_pin: x.x.x
@@ -45,9 +51,11 @@ pin_run_as_build:
 python:
 - 3.7.* *_cpython
 re2:
-- 2020.07.06
+- 2020.08.01
 snappy:
 - '1'
+target_platform:
+- linux-64
 zip_keys:
 - - cuda_compiler_version
   - docker_image
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.8.____cpython.yaml
index f2e543bb0e6..e15e3bf7623 100644
--- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.8.____cpython.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.8.____cpython.yaml
@@ -1,5 +1,7 @@
+aws_sdk_cpp:
+- 1.8.54
 boost_cpp:
-- '1.72'
+- 1.74.0
 bzip2:
 - '1'
 c_compiler:
@@ -27,9 +29,13 @@ glog:
 grpc_cpp:
 - '1.30'
 libprotobuf:
-- '3.12'
+- '3.13'
 lz4_c:
 - 1.9.2
+numpy:
+- '1.16'
+orc:
+- 1.6.4
 pin_run_as_build:
   boost-cpp:
     max_pin: x.x.x
@@ -45,9 +51,11 @@ pin_run_as_build:
 python:
 - 3.8.* *_cpython
 re2:
-- 2020.07.06
+- 2020.08.01
 snappy:
 - '1'
+target_platform:
+- linux-64
 zip_keys:
 - - cuda_compiler_version
   - docker_image
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.6.____cpython.yaml
index fc4fd6e8b98..e2952d8375d 100644
--- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.6.____cpython.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.6.____cpython.yaml
@@ -1,5 +1,7 @@
+aws_sdk_cpp:
+- 1.8.54
 boost_cpp:
-- '1.72'
+- 1.74.0
 bzip2:
 - '1'
 c_compiler:
@@ -27,9 +29,13 @@ glog:
 grpc_cpp:
 - '1.30'
 libprotobuf:
-- '3.12'
+- '3.13'
 lz4_c:
 - 1.9.2
+numpy:
+- '1.16'
+orc:
+- 1.6.4
 pin_run_as_build:
   boost-cpp:
     max_pin: x.x.x
@@ -45,9 +51,11 @@ pin_run_as_build:
 python:
 - 3.6.* *_cpython
 re2:
-- 2020.07.06
+- 2020.08.01
 snappy:
 - '1'
+target_platform:
+- linux-64
 zip_keys:
 - - cuda_compiler_version
   - docker_image
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.7.____cpython.yaml
index 55aafb78bd8..0da1784bdd6 100644
--- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.7.____cpython.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.7.____cpython.yaml
@@ -1,5 +1,7 @@
+aws_sdk_cpp:
+- 1.8.54
 boost_cpp:
-- '1.72'
+- 1.74.0
 bzip2:
 - '1'
 c_compiler:
@@ -27,9 +29,13 @@ glog:
 grpc_cpp:
 - '1.30'
 libprotobuf:
-- '3.12'
+- '3.13'
 lz4_c:
 - 1.9.2
+numpy:
+- '1.16'
+orc:
+- 1.6.4
 pin_run_as_build:
   boost-cpp:
     max_pin: x.x.x
@@ -45,9 +51,11 @@ pin_run_as_build:
 python:
 - 3.7.* *_cpython
 re2:
-- 2020.07.06
+- 2020.08.01
 snappy:
 - '1'
+target_platform:
+- linux-64
 zip_keys:
 - - cuda_compiler_version
   - docker_image
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.8.____cpython.yaml
index b6b5a825cd3..8fc3382d8a9 100644
--- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.8.____cpython.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.8.____cpython.yaml
@@ -1,5 +1,7 @@
+aws_sdk_cpp:
+- 1.8.54
 boost_cpp:
-- '1.72'
+- 1.74.0
 bzip2:
 - '1'
 c_compiler:
@@ -27,9 +29,13 @@ glog:
 grpc_cpp:
 - '1.30'
 libprotobuf:
-- '3.12'
+- '3.13'
 lz4_c:
 - 1.9.2
+numpy:
+- '1.16'
+orc:
+- 1.6.4
 pin_run_as_build:
   boost-cpp:
     max_pin: x.x.x
@@ -45,9 +51,11 @@ pin_run_as_build:
 python:
 - 3.8.* *_cpython
 re2:
-- 2020.07.06
+- 2020.08.01
 snappy:
 - '1'
+target_platform:
+- linux-64
 zip_keys:
 - - cuda_compiler_version
   - docker_image
diff --git a/dev/tasks/conda-recipes/.ci_support/osx_python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_python3.6.____cpython.yaml
index 04f0059177c..e4d163f526c 100644
--- a/dev/tasks/conda-recipes/.ci_support/osx_python3.6.____cpython.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/osx_python3.6.____cpython.yaml
@@ -1,7 +1,9 @@
 MACOSX_DEPLOYMENT_TARGET:
 - '10.9'
+aws_sdk_cpp:
+- 1.8.54
 boost_cpp:
-- '1.72'
+- 1.74.0
 bzip2:
 - '1'
 c_compiler:
@@ -12,6 +14,8 @@ channel_sources:
 - conda-forge,defaults
 channel_targets:
 - conda-forge main
+cuda_compiler_version:
+- None
 cxx_compiler:
 - clangxx
 cxx_compiler_version:
@@ -23,13 +27,15 @@ glog:
 grpc_cpp:
 - '1.30'
 libprotobuf:
-- '3.12'
+- '3.13'
 lz4_c:
 - 1.9.2
 macos_machine:
 - x86_64-apple-darwin13.4.0
-macos_min_version:
-- '10.9'
+numpy:
+- '1.16'
+orc:
+- 1.6.4
 pin_run_as_build:
   boost-cpp:
     max_pin: x.x.x
@@ -45,9 +51,11 @@ pin_run_as_build:
 python:
 - 3.6.* *_cpython
 re2:
-- 2020.07.06
+- 2020.08.01
 snappy:
 - '1'
+target_platform:
+- osx-64
 zlib:
 - '1.2'
 zstd:
diff --git a/dev/tasks/conda-recipes/.ci_support/osx_python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_python3.7.____cpython.yaml
index 584be4ed280..3a805335574 100644
--- a/dev/tasks/conda-recipes/.ci_support/osx_python3.7.____cpython.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/osx_python3.7.____cpython.yaml
@@ -1,7 +1,9 @@
 MACOSX_DEPLOYMENT_TARGET:
 - '10.9'
+aws_sdk_cpp:
+- 1.8.54
 boost_cpp:
-- '1.72'
+- 1.74.0
 bzip2:
 - '1'
 c_compiler:
@@ -12,6 +14,8 @@ channel_sources:
 - conda-forge,defaults
 channel_targets:
 - conda-forge main
+cuda_compiler_version:
+- None
 cxx_compiler:
 - clangxx
 cxx_compiler_version:
@@ -23,13 +27,15 @@ glog:
 grpc_cpp:
 - '1.30'
 libprotobuf:
-- '3.12'
+- '3.13'
 lz4_c:
 - 1.9.2
 macos_machine:
 - x86_64-apple-darwin13.4.0
-macos_min_version:
-- '10.9'
+numpy:
+- '1.16'
+orc:
+- 1.6.4
 pin_run_as_build:
   boost-cpp:
     max_pin: x.x.x
@@ -45,9 +51,11 @@ pin_run_as_build:
 python:
 - 3.7.* *_cpython
 re2:
-- 2020.07.06
+- 2020.08.01
 snappy:
 - '1'
+target_platform:
+- osx-64
 zlib:
 - '1.2'
 zstd:
diff --git a/dev/tasks/conda-recipes/.ci_support/osx_python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_python3.8.____cpython.yaml
index bff87169d1f..9e7aa65b8df 100644
--- a/dev/tasks/conda-recipes/.ci_support/osx_python3.8.____cpython.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/osx_python3.8.____cpython.yaml
@@ -1,7 +1,9 @@
 MACOSX_DEPLOYMENT_TARGET:
 - '10.9'
+aws_sdk_cpp:
+- 1.8.54
 boost_cpp:
-- '1.72'
+- 1.74.0
 bzip2:
 - '1'
 c_compiler:
@@ -12,6 +14,8 @@ channel_sources:
 - conda-forge,defaults
 channel_targets:
 - conda-forge main
+cuda_compiler_version:
+- None
 cxx_compiler:
 - clangxx
 cxx_compiler_version:
@@ -23,13 +27,15 @@ glog:
 grpc_cpp:
 - '1.30'
 libprotobuf:
-- '3.12'
+- '3.13'
 lz4_c:
 - 1.9.2
 macos_machine:
 - x86_64-apple-darwin13.4.0
-macos_min_version:
-- '10.9'
+numpy:
+- '1.16'
+orc:
+- 1.6.4
 pin_run_as_build:
   boost-cpp:
     max_pin: x.x.x
@@ -45,9 +51,11 @@ pin_run_as_build:
 python:
 - 3.8.* *_cpython
 re2:
-- 2020.07.06
+- 2020.08.01
 snappy:
 - '1'
+target_platform:
+- osx-64
 zlib:
 - '1.2'
 zstd:
diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base3.6.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base3.6.yaml
new file mode 100644
index 00000000000..ac945ce72d3
--- /dev/null
+++ b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base3.6.yaml
@@ -0,0 +1,22 @@
+c_compiler:
+- gcc
+c_compiler_version:
+- '7'
+channel_sources:
+- conda-forge,defaults
+channel_targets:
+- conda-forge main
+cxx_compiler:
+- gxx
+cxx_compiler_version:
+- '7'
+docker_image:
+- condaforge/linux-anvil-comp7
+pin_run_as_build:
+  r-base:
+    min_pin: x.x
+    max_pin: x.x
+r_base:
+- '3.6'
+target_platform:
+- linux-64
diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.0.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.0.yaml
new file mode 100644
index 00000000000..51d26f834cc
--- /dev/null
+++ b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.0.yaml
@@ -0,0 +1,22 @@
+c_compiler:
+- gcc
+c_compiler_version:
+- '7'
+channel_sources:
+- conda-forge,defaults
+channel_targets:
+- conda-forge main
+cxx_compiler:
+- gxx
+cxx_compiler_version:
+- '7'
+docker_image:
+- condaforge/linux-anvil-comp7
+pin_run_as_build:
+  r-base:
+    min_pin: x.x
+    max_pin: x.x
+r_base:
+- '4.0'
+target_platform:
+- linux-64
diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base3.6.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base3.6.yaml
new file mode 100644
index 00000000000..e3c5b898be6
--- /dev/null
+++ b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base3.6.yaml
@@ -0,0 +1,26 @@
+MACOSX_DEPLOYMENT_TARGET:
+- '10.9'
+c_compiler:
+- clang
+c_compiler_version:
+- '10'
+channel_sources:
+- conda-forge,defaults
+channel_targets:
+- conda-forge main
+cxx_compiler:
+- clangxx
+cxx_compiler_version:
+- '10'
+macos_machine:
+- x86_64-apple-darwin13.4.0
+macos_min_version:
+- '10.9'
+pin_run_as_build:
+  r-base:
+    min_pin: x.x
+    max_pin: x.x
+r_base:
+- '3.6'
+target_platform:
+- osx-64
diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.0.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.0.yaml
new file mode 100644
index 00000000000..8343a284b97
--- /dev/null
+++ b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.0.yaml
@@ -0,0 +1,26 @@
+MACOSX_DEPLOYMENT_TARGET:
+- '10.9'
+c_compiler:
+- clang
+c_compiler_version:
+- '10'
+channel_sources:
+- conda-forge,defaults
+channel_targets:
+- conda-forge main
+cxx_compiler:
+- clangxx
+cxx_compiler_version:
+- '10'
+macos_machine:
+- x86_64-apple-darwin13.4.0
+macos_min_version:
+- '10.9'
+pin_run_as_build:
+  r-base:
+    min_pin: x.x
+    max_pin: x.x
+r_base:
+- '4.0'
+target_platform:
+- osx-64
diff --git a/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base3.6.yaml b/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base3.6.yaml
new file mode 100644
index 00000000000..3fb7f88499a
--- /dev/null
+++ b/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base3.6.yaml
@@ -0,0 +1,12 @@
+channel_sources:
+- conda-forge,defaults
+channel_targets:
+- conda-forge main
+pin_run_as_build:
+  r-base:
+    min_pin: x.x
+    max_pin: x.x
+r_base:
+- '3.6'
+target_platform:
+- win-64
diff --git a/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.0.yaml b/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.0.yaml
new file mode 100644
index 00000000000..02c2a70756d
--- /dev/null
+++ b/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.0.yaml
@@ -0,0 +1,12 @@
+channel_sources:
+- conda-forge,defaults
+channel_targets:
+- conda-forge main
+pin_run_as_build:
+  r-base:
+    min_pin: x.x
+    max_pin: x.x
+r_base:
+- '4.0'
+target_platform:
+- win-64
diff --git a/dev/tasks/conda-recipes/.ci_support/win_python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_python3.6.____cpython.yaml
index 2f6ae18f6a2..bce682a27af 100644
--- a/dev/tasks/conda-recipes/.ci_support/win_python3.6.____cpython.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/win_python3.6.____cpython.yaml
@@ -1,5 +1,7 @@
+aws_sdk_cpp:
+- 1.8.54
 boost_cpp:
-- '1.72'
+- 1.74.0
 bzip2:
 - '1'
 c_compiler:
@@ -8,6 +10,8 @@ channel_sources:
 - conda-forge,defaults
 channel_targets:
 - conda-forge main
+cuda_compiler_version:
+- None
 cxx_compiler:
 - vs2017
 gflags:
@@ -17,9 +21,11 @@ glog:
 grpc_cpp:
 - '1.30'
 libprotobuf:
-- '3.12'
+- '3.13'
 lz4_c:
 - 1.9.2
+numpy:
+- '1.16'
 pin_run_as_build:
   boost-cpp:
     max_pin: x.x.x
@@ -35,9 +41,11 @@ pin_run_as_build:
 python:
 - 3.6.* *_cpython
 re2:
-- 2020.07.06
+- 2020.08.01
 snappy:
 - '1'
+target_platform:
+- win-64
 zlib:
 - '1.2'
 zstd:
diff --git a/dev/tasks/conda-recipes/.ci_support/win_python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_python3.7.____cpython.yaml
index dd81b08cedb..8b74b5d79a7 100644
--- a/dev/tasks/conda-recipes/.ci_support/win_python3.7.____cpython.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/win_python3.7.____cpython.yaml
@@ -1,5 +1,7 @@
+aws_sdk_cpp:
+- 1.8.54
 boost_cpp:
-- '1.72'
+- 1.74.0
 bzip2:
 - '1'
 c_compiler:
@@ -8,6 +10,8 @@ channel_sources:
 - conda-forge,defaults
 channel_targets:
 - conda-forge main
+cuda_compiler_version:
+- None
 cxx_compiler:
 - vs2017
 gflags:
@@ -17,9 +21,11 @@ glog:
 grpc_cpp:
 - '1.30'
 libprotobuf:
-- '3.12'
+- '3.13'
 lz4_c:
 - 1.9.2
+numpy:
+- '1.16'
 pin_run_as_build:
   boost-cpp:
     max_pin: x.x.x
@@ -35,9 +41,11 @@ pin_run_as_build:
 python:
 - 3.7.* *_cpython
 re2:
-- 2020.07.06
+- 2020.08.01
 snappy:
 - '1'
+target_platform:
+- win-64
 zlib:
 - '1.2'
 zstd:
diff --git a/dev/tasks/conda-recipes/.ci_support/win_python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_python3.8.____cpython.yaml
index 877040a3ed2..32cbf60e952 100644
--- a/dev/tasks/conda-recipes/.ci_support/win_python3.8.____cpython.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/win_python3.8.____cpython.yaml
@@ -1,5 +1,7 @@
+aws_sdk_cpp:
+- 1.8.54
 boost_cpp:
-- '1.72'
+- 1.74.0
 bzip2:
 - '1'
 c_compiler:
@@ -8,6 +10,8 @@ channel_sources:
 - conda-forge,defaults
 channel_targets:
 - conda-forge main
+cuda_compiler_version:
+- None
 cxx_compiler:
 - vs2017
 gflags:
@@ -17,9 +21,11 @@ glog:
 grpc_cpp:
 - '1.30'
 libprotobuf:
-- '3.12'
+- '3.13'
 lz4_c:
 - 1.9.2
+numpy:
+- '1.16'
 pin_run_as_build:
   boost-cpp:
     max_pin: x.x.x
@@ -35,9 +41,11 @@ pin_run_as_build:
 python:
 - 3.8.* *_cpython
 re2:
-- 2020.07.06
+- 2020.08.01
 snappy:
 - '1'
+target_platform:
+- win-64
 zlib:
 - '1.2'
 zstd:
diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat b/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat
index c3d729dd44e..cc2ed71fe3e 100644
--- a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat
+++ b/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat
@@ -2,11 +2,12 @@ mkdir "%SRC_DIR%"\cpp\build
 pushd "%SRC_DIR%"\cpp\build
 
 cmake -G "Ninja" ^
+      -DBUILD_SHARED_LIBS=ON ^
       -DCMAKE_INSTALL_PREFIX="%LIBRARY_PREFIX%" ^
       -DARROW_DEPENDENCY_SOURCE=SYSTEM ^
       -DARROW_PACKAGE_PREFIX="%LIBRARY_PREFIX%" ^
       -DLLVM_TOOLS_BINARY_DIR="%LIBRARY_BIN%" ^
-      -DPYTHON_EXECUTABLE="%PYTHON%" ^
+      -DPython3_EXECUTABLE="%PYTHON%" ^
       -DARROW_WITH_BZ2:BOOL=ON ^
       -DARROW_WITH_ZLIB:BOOL=ON ^
       -DARROW_WITH_ZSTD:BOOL=ON ^
diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat b/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat
index 61d455da419..8f3357748df 100644
--- a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat
+++ b/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat
@@ -32,3 +32,7 @@ SET PYARROW_CMAKE_GENERATOR=Ninja
                    --record=record.txt
 if errorlevel 1 exit 1
 popd
+
+if [%PKG_NAME%] == [pyarrow] (
+    rd /s /q %SP_DIR%\pyarrow\tests
+)
diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh
index 7be499fb576..e15fc92808c 100755
--- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh
+++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh
@@ -35,6 +35,7 @@ else
 fi
 
 cmake \
+    -DBUILD_SHARED_LIBS=ON \
     -DARROW_BOOST_USE_SHARED=ON \
     -DARROW_BUILD_BENCHMARKS=OFF \
     -DARROW_BUILD_STATIC=OFF \
@@ -67,6 +68,7 @@ cmake \
     -DCMAKE_INSTALL_PREFIX=$PREFIX \
     -DCMAKE_RANLIB=${RANLIB} \
     -DLLVM_TOOLS_BINARY_DIR=$PREFIX/bin \
+    -DCMAKE_UNITY_BUILD=ON \
     -GNinja \
     ${EXTRA_CMAKE_ARGS} \
     ..
diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh
index 337fa91824c..d0fb55de580 100755
--- a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh
+++ b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh
@@ -8,6 +8,8 @@ export ARROW_HOME=$PREFIX
 export PARQUET_HOME=$PREFIX
 export SETUPTOOLS_SCM_PRETEND_VERSION=$PKG_VERSION
 export PYARROW_BUILD_TYPE=release
+export PYARROW_BUNDLE_ARROW_CPP_HEADERS=0
+export PYARROW_BUNDLE_PLASMA_EXECUTABLE=0
 export PYARROW_WITH_DATASET=1
 export PYARROW_WITH_FLIGHT=1
 export PYARROW_WITH_GANDIVA=1
@@ -16,6 +18,7 @@ export PYARROW_WITH_ORC=1
 export PYARROW_WITH_PARQUET=1
 export PYARROW_WITH_PLASMA=1
 export PYARROW_WITH_S3=1
+export PYARROW_CMAKE_GENERATOR=Ninja
 BUILD_EXT_FLAGS=""
 
 # Enable CUDA support
@@ -38,3 +41,7 @@ $PYTHON setup.py \
         build_ext \
         install --single-version-externally-managed \
                 --record=record.txt
+
+if [[ "$PKG_NAME" == "pyarrow" ]]; then
+    rm -r ${SP_DIR}/pyarrow/tests
+fi
diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml
index cc256b6ad87..272ed8c5382 100644
--- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml
+++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml
@@ -1,9 +1,8 @@
 {% set version = ARROW_VERSION %}
 {% set number = "0" %}
 {% set cuda_enabled = cuda_compiler_version is not undefined and cuda_compiler_version == '9.2' %}
-{% set build_ext_version = "1.0.0" %}
+{% set build_ext_version = "1.0.1" %}
 {% set build_ext = "cuda" if cuda_enabled else "cpu" %}
-{% set build_string = "py{}h{}_{}_{}".format(CONDA_PY, PKG_HASH, number, build_ext) %}
 {% set proc_build_number = "1" %}
 
 package:
@@ -14,7 +13,7 @@ source:
   path: ../../../../
 
 build:
-  number: {{ number }}
+  number: 0
   skip: true  # [cuda_compiler_version not in (undefined, "None", "9.2")]
   run_exports:
     - {{ pin_subpackage("arrow-cpp", max_pin="x.x.x") }}
@@ -40,13 +39,14 @@ outputs:
     script: bld-arrow.bat   # [win]
     version: {{ version }}
     build:
-      number: {{ number }}
-      string: "{{ build_string }}"
+      string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }}
       skip: true  # [cuda_compiler_version not in (undefined, "None", "9.2")]
       run_exports:
         - {{ pin_subpackage("arrow-cpp", max_pin="x.x.x") }}
       ignore_run_exports:
         - cudatoolkit
+        # Gandiva only needs headers
+        - boost-cpp
       track_features:
         {{ "- arrow-cuda" if cuda_enabled else "" }}
     requirements:
@@ -68,14 +68,11 @@ outputs:
         - glog
         - grpc-cpp
         - libprotobuf
+        - clangdev 11
+        - llvmdev 11
         - libutf8proc
-        - clangdev 10.* # [not win]
-        - llvmdev 10.* # [not win]
-        # llvmdev 9.* or later require Visual Studio 2017
-        - clangdev 8.* # [win]
-        - llvmdev 8.* # [win]
         - lz4-c
-        - numpy 1.16.*
+        - numpy
         - orc  # [unix]
         - python
         - rapidjson
@@ -87,7 +84,6 @@ outputs:
       run:
         - {{ pin_compatible('numpy', lower_bound='1.16') }}
         - aws-sdk-cpp
-        - boost-cpp
         - brotli
         - c-ares
         - gflags
@@ -165,8 +161,7 @@ outputs:
     script: bld-pyarrow.bat   # [win]
     version: {{ version }}
     build:
-      number: {{ number }}
-      string: "{{ build_string }}"
+      string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }}
       skip: true  # [cuda_compiler_version not in (undefined, "None", "9.2")]
       ignore_run_exports:
         - cudatoolkit
@@ -183,7 +178,6 @@ outputs:
         - {{ compiler("cuda") }}  # [cuda_compiler_version not in (undefined, "None")]
       host:
         - {{ pin_subpackage('arrow-cpp', exact=True) }}
-        - boost-cpp
         - cython
         - numpy 1.16.*
         - python
@@ -193,13 +187,9 @@ outputs:
       run:
         - {{ pin_subpackage('arrow-cpp', exact=True) }}
         - {{ pin_compatible('numpy', lower_bound='1.16') }}
-        - boost-cpp
-        - glog
-        - pandas
         # empty parquet-cpp metapackage, force old versions to be uninstalled
         - parquet-cpp 1.5.1.*
         - python
-        - six
       run_constrained:
         - arrow-cpp-proc * {{ build_ext }}
         - cudatoolkit >=9.2  # [cuda_compiler_version not in (undefined, "None")]
@@ -226,6 +216,57 @@ outputs:
         # We can only test importing cuda package but cannot run when a
         # CUDA device is not available, for instance, when building from CI.
         - pyarrow.cuda     # [cuda_compiler_version not in (undefined, "None")]
+      commands:
+        - test ! -f ${SP_DIR}/pyarrow/plasma-store-server       # [unix]
+        - test ! -f ${SP_DIR}/pyarrow/tests/test_array.py       # [unix]
+        - if exist %SP_DIR%/pyarrow/tests/test_array.py exit 1  # [win]
+  - name: pyarrow-tests
+    script: build-pyarrow.sh  # [not win]
+    script: bld-pyarrow.bat   # [win]
+    version: {{ version }}
+    build:
+      string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }}
+      skip: true  # [cuda_compiler_version not in (undefined, "None", "9.2")]
+      ignore_run_exports:
+        - cudatoolkit
+      track_features:
+        {{ "- arrow-cuda" if cuda_enabled else "" }}
+    requirements:
+      build:
+        - cmake 3.16.*
+        - ninja
+        - make  # [unix]
+        - {{ compiler('c') }}
+        - {{ compiler('cxx') }}
+        # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda
+        - {{ compiler("cuda") }}  # [cuda_compiler_version not in (undefined, "None")]
+      host:
+        - {{ pin_subpackage('arrow-cpp', exact=True) }}
+        - {{ pin_subpackage('pyarrow', exact=True) }}
+        - cython
+        - numpy 1.16.*
+        - python
+        - setuptools
+        - setuptools_scm
+        - six
+      run:
+        - {{ pin_subpackage('pyarrow', exact=True) }}
+        - python
+      run_constrained:
+        - arrow-cpp-proc * {{ build_ext }}
+        - cudatoolkit >=9.2  # [cuda_compiler_version not in (undefined, "None")]
+
+    about:
+      home: http://github.com/apache/arrow
+      license: Apache-2.0
+      license_file:
+        - LICENSE.txt
+      summary: Python test files for Apache Arrow
+
+    test:
+      commands:
+        - test -f ${SP_DIR}/pyarrow/tests/test_array.py             # [unix]
+        - if not exist %SP_DIR%/pyarrow/tests/test_array.py exit 1  # [win]
 
 about:
   home: http://github.com/apache/arrow
diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml
index 05bee14424c..3ab9504dbb6 100755
--- a/dev/tasks/conda-recipes/azure.linux.yml
+++ b/dev/tasks/conda-recipes/azure.linux.yml
@@ -6,6 +6,7 @@ jobs:
 
   variables:
     CONFIG: {{ config }}
+    R_CONFIG: {{ r_config|default("") }}
     ARROW_VERSION: {{ arrow.no_rc_version }}
     UPLOAD_PACKAGES: False
 
diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml
index feeabc14f1f..3251754d2dc 100755
--- a/dev/tasks/conda-recipes/azure.osx.yml
+++ b/dev/tasks/conda-recipes/azure.osx.yml
@@ -5,6 +5,7 @@ jobs:
   timeoutInMinutes: 360
   variables:
     CONFIG: {{ config }}
+    R_CONFIG: {{ r_config|default("") }}
     ARROW_VERSION: {{ arrow.no_rc_version }}
     UPLOAD_PACKAGES: False
   steps:
@@ -23,8 +24,8 @@ jobs:
 
   - script: |
       source activate base
-      conda install -n base -c conda-forge --quiet --yes conda-forge-ci-setup=2 conda-build
-    displayName: 'Add conda-forge-ci-setup=2'
+      conda install -n base -c conda-forge --quiet --yes conda-forge-ci-setup=3 conda-build
+    displayName: 'Add conda-forge-ci-setup=3'
 
   - script: |
       git clone --no-checkout {{ arrow.remote }} arrow
@@ -65,6 +66,12 @@ jobs:
         -m ./.ci_support/${CONFIG}.yaml \
         --clobber-file ./.ci_support/clobber_${CONFIG}.yaml \
         --output-folder ./build_artifacts
+
+      if [ ! -z "${R_CONFIG}" ]; then
+        conda build r-arrow \
+          -m ./.ci_support/r/${R_CONFIG}.yaml \
+          --output-folder ./build_artifacts
+      fi
     workingDirectory: arrow/dev/tasks/conda-recipes
     displayName: Build recipes
 
diff --git a/dev/tasks/conda-recipes/azure.win.yml b/dev/tasks/conda-recipes/azure.win.yml
index f7f3708ca21..d4322746eac 100755
--- a/dev/tasks/conda-recipes/azure.win.yml
+++ b/dev/tasks/conda-recipes/azure.win.yml
@@ -5,6 +5,7 @@ jobs:
   timeoutInMinutes: 360
   variables:
     CONFIG: {{ config }}
+    R_CONFIG: {{ r_config|default("") }}
     ARROW_VERSION: {{ arrow.no_rc_version }}
     CONDA_BLD_PATH: D:\\bld\\
     UPLOAD_PACKAGES: False
@@ -57,6 +58,7 @@ jobs:
         call activate base
         run_conda_forge_build_setup
       displayName: conda-forge build setup
+      workingDirectory: arrow\dev\tasks\conda-recipes
 
     - script: |
         conda.exe build arrow-cpp parquet-cpp -m .ci_support\%CONFIG%.yaml
@@ -66,6 +68,14 @@ jobs:
         PYTHONUNBUFFERED: 1
       condition: not(contains(variables['CONFIG'], 'vs2008'))
 
+    - script: |
+        conda.exe build r-arrow -m .ci_support\r\%R_CONFIG%.yaml
+      displayName: Build recipe
+      workingDirectory: arrow\dev\tasks\conda-recipes
+      env:
+        PYTHONUNBUFFERED: 1
+      condition: contains(variables['R_CONFIG'], 'win')
+
     # Using github release tries to find a common ancestor between the
     # currently pushed tag and the latest tag of the github repository
     # (don't know why).
diff --git a/dev/tasks/conda-recipes/build_steps.sh b/dev/tasks/conda-recipes/build_steps.sh
index a18f97290b9..a19a3124eae 100755
--- a/dev/tasks/conda-recipes/build_steps.sh
+++ b/dev/tasks/conda-recipes/build_steps.sh
@@ -40,4 +40,12 @@ conda build \
     --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" \
     --output-folder "${output_dir}"
 
+if [ ! -z "${R_CONFIG}" ]; then
+  conda build \
+      "${FEEDSTOCK_ROOT}/r-arrow" \
+      -m "${CI_SUPPORT}/r/${R_CONFIG}.yaml" \
+      --output-folder "${output_dir}"
+fi
+
+
 touch "${output_dir}/conda-forge-build-done-${CONFIG}"
diff --git a/dev/tasks/conda-recipes/drone-steps.sh b/dev/tasks/conda-recipes/drone-steps.sh
new file mode 100755
index 00000000000..71564108b2f
--- /dev/null
+++ b/dev/tasks/conda-recipes/drone-steps.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -ex
+
+OUTPUT_DIR=$1
+QUEUE_REMOTE_URL=$2
+TASK_BRANCH=$3
+TASK_TAG=$4
+UPLOAD_TO_ANACONDA=$5
+
+conda install -y mamba
+$FEEDSTOCK_ROOT/build_steps.sh ${OUTPUT_DIR}
+
+# Upload as Github release
+mamba install -y click github3.py jinja2 jira pygit2 ruamel.yaml setuptools_scm toolz anaconda-client shyaml -c conda-forge
+pushd $DRONE_WORKSPACE
+python arrow/dev/tasks/crossbow.py \
+  --queue-path . \
+  --queue-remote ${QUEUE_REMOTE_URL} \
+  upload-artifacts \
+  --sha ${TASK_BRANCH} \
+  --tag ${TASK_TAG} \
+  --pattern "${OUTPUT_DIR}/linux-aarch64/*.tar.bz2"
+
+if [[ "${UPLOAD_TO_ANACONDA}" == "1" ]]; then
+  anaconda -t ${CROSSBOW_ANACONDA_TOKEN} upload --force build_artifacts/linux-aarch64/*.tar.bz2
+fi
diff --git a/dev/tasks/conda-recipes/drone.yml b/dev/tasks/conda-recipes/drone.yml
new file mode 100644
index 00000000000..a461c79b9d4
--- /dev/null
+++ b/dev/tasks/conda-recipes/drone.yml
@@ -0,0 +1,43 @@
+---
+kind: pipeline
+name: {{ config }}
+
+platform:
+  os: linux
+  arch: arm64
+
+# Omit double builds with crossbow
+trigger:
+  event:
+    - push
+
+steps:
+- name: Install and build
+  image: condaforge/linux-anvil-aarch64
+  environment:
+    CONFIG: {{ config }}
+    UPLOAD_PACKAGES: False
+    ARROW_VERSION: {{ arrow.no_rc_version }}
+    PLATFORM: linux-aarch64
+    BINSTAR_TOKEN:
+      from_secret: BINSTAR_TOKEN
+    FEEDSTOCK_TOKEN:
+      from_secret: FEEDSTOCK_TOKEN
+    STAGING_BINSTAR_TOKEN:
+      from_secret: STAGING_BINSTAR_TOKEN
+    CROSSBOW_GITHUB_TOKEN:
+      from_secret: CROSSBOW_GITHUB_TOKEN
+    CROSSBOW_ANACONDA_TOKEN:
+      from_secret: CROSSBOW_ANACONDA_TOKEN
+  commands:
+    - export RECIPE_ROOT="$FEEDSTOCK_ROOT/arrow-cpp"
+    - export CI=drone
+    - export GIT_BRANCH="{{ arrow.branch }}"
+    - export FEEDSTOCK_NAME=arrow-cpp
+    - export FEEDSTOCK_ROOT="$DRONE_WORKSPACE/arrow/dev/tasks/conda-recipes"
+    - sed -i '$ichown -R conda:conda "$FEEDSTOCK_ROOT"' /opt/docker/bin/entrypoint
+    - yum install -y git
+    - git clone --no-checkout {{ arrow.remote }} arrow
+    - pushd arrow && git fetch -t {{ arrow.remote }} {{ arrow.branch }} && git checkout FETCH_HEAD && git submodule update --init --recursive && popd
+    - mkdir -p $(pwd)/build_artifacts && chmod a+rwx $(pwd)/build_artifacts
+    - /opt/docker/bin/entrypoint $FEEDSTOCK_ROOT/drone-steps.sh $(pwd)/build_artifacts {{ queue_remote_url }} {{ task.branch }} {{ task.tag }} {% if arrow.branch == 'master' %}1{% else %}0{% endif %}
diff --git a/dev/tasks/conda-recipes/r-arrow/bld.bat b/dev/tasks/conda-recipes/r-arrow/bld.bat
new file mode 100644
index 00000000000..a193ddc0a77
--- /dev/null
+++ b/dev/tasks/conda-recipes/r-arrow/bld.bat
@@ -0,0 +1,9 @@
+bash %RECIPE_DIR%/build_win.sh
+IF %ERRORLEVEL% NEQ 0 exit 1
+cp %RECIPE_DIR%/configure.win r
+IF %ERRORLEVEL% NEQ 0 exit 1
+cp %RECIPE_DIR%/install.libs.R r/src
+IF %ERRORLEVEL% NEQ 0 exit 1
+set "MAKEFLAGS=-j%CPU_COUNT%"
+"%R%" CMD INSTALL --build r
+IF %ERRORLEVEL% NEQ 0 exit 1
diff --git a/dev/tasks/conda-recipes/r-arrow/build.sh b/dev/tasks/conda-recipes/r-arrow/build.sh
new file mode 100644
index 00000000000..0a6c7961fc6
--- /dev/null
+++ b/dev/tasks/conda-recipes/r-arrow/build.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+export DISABLE_AUTOBREW=1
+$R CMD INSTALL --build r/.
diff --git a/dev/tasks/conda-recipes/r-arrow/build_win.sh b/dev/tasks/conda-recipes/r-arrow/build_win.sh
new file mode 100755
index 00000000000..88e0462f663
--- /dev/null
+++ b/dev/tasks/conda-recipes/r-arrow/build_win.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -exuo pipefail
+
+
+# Rename arrow.dll to lib_arrow.dll to avoid conflicts with the arrow-cpp arrow.dll
+sed -i -e 's/void R_init_arrow/__declspec(dllexport) void R_init_lib_arrow/g' r/src/arrowExports.cpp
+sed -i -e 's/useDynLib(arrow/useDynLib(lib_arrow/g' r/NAMESPACE
diff --git a/dev/tasks/conda-recipes/r-arrow/configure.win b/dev/tasks/conda-recipes/r-arrow/configure.win
new file mode 100755
index 00000000000..3a6ea90f27e
--- /dev/null
+++ b/dev/tasks/conda-recipes/r-arrow/configure.win
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+echo "PKG_CPPFLAGS=-DNDEBUG -I\"${LIBRARY_PREFIX}/include\" -I\"${PREFIX}/include\" -DARROW_R_WITH_ARROW -DARROW_R_WITH_S3" > src/Makevars.win
+echo "PKG_CXXFLAGS=\$(CXX_VISIBILITY)" >> src/Makevars.win
+echo 'CXX_STD=CXX11' >> src/Makevars.win
+echo "PKG_LIBS=-L\"${LIBRARY_PREFIX}/lib\" -larrow_dataset -lparquet -larrow" >> src/Makevars.win
diff --git a/dev/tasks/conda-recipes/r-arrow/install.libs.R b/dev/tasks/conda-recipes/r-arrow/install.libs.R
new file mode 100644
index 00000000000..005bbe16b99
--- /dev/null
+++ b/dev/tasks/conda-recipes/r-arrow/install.libs.R
@@ -0,0 +1,5 @@
+src_dir <- file.path(R_PACKAGE_SOURCE, "src", fsep = "/")
+dest_dir <- file.path(R_PACKAGE_DIR, paste0("libs", R_ARCH), fsep="/")
+
+dir.create(file.path(R_PACKAGE_DIR, paste0("libs", R_ARCH), fsep="/"), recursive = TRUE, showWarnings = FALSE)
+file.copy(file.path(src_dir, "arrow.dll", fsep = "/"), file.path(dest_dir, "lib_arrow.dll", fsep = "/"))
diff --git a/dev/tasks/conda-recipes/r-arrow/meta.yaml b/dev/tasks/conda-recipes/r-arrow/meta.yaml
new file mode 100644
index 00000000000..5f0643bef37
--- /dev/null
+++ b/dev/tasks/conda-recipes/r-arrow/meta.yaml
@@ -0,0 +1,66 @@
+{% set version = ARROW_VERSION %}
+{% set posix = 'm2-' if win else '' %}
+{% set native = 'm2w64-' if win else '' %}
+
+package:
+  name: r-arrow
+  version: {{ version|replace("-", "_") }}
+
+source:
+  path: ../../../../
+
+build:
+  merge_build_host: true  # [win]
+  number: 0
+  rpaths:
+    - lib/R/lib/
+    - lib/
+
+requirements:
+  build:
+    - {{ compiler('c') }}        # [not win]
+    - {{ compiler('cxx') }}      # [not win]
+    - {{ compiler('r_clang') }}  # [win]
+    - pkg-config                 # [not win]
+    - {{ posix }}make
+    - {{ posix }}sed         # [win]
+    - {{ posix }}coreutils   # [win]
+    - {{ posix }}filesystem  # [win]
+    - {{ posix }}zip         # [win]
+  host:
+    # Needs to be here, otherwise merge_build_host runs into issues
+    - pkg-config  # [win]
+    - r-base
+    - arrow-cpp {{ version }}
+    - r-cpp11
+    - r-r6
+    - r-assertthat
+    - r-bit64
+    - r-purrr
+    - r-rlang
+    - r-tidyselect
+  run:
+    - r-base
+    - r-r6
+    - r-assertthat
+    - r-bit64
+    - r-purrr
+    - r-rlang
+    - r-tidyselect
+
+test:
+  commands:
+    - $R -e "library('arrow')"           # [not win]
+    - "\"%R%\" -e \"library('arrow'); data(mtcars); write_parquet(mtcars, 'test.parquet')\""  # [win]
+
+about:
+  home: https://github.com/apache/arrow
+  license: Apache-2.0
+  license_file: LICENSE.txt
+  summary: R Integration to 'Apache' 'Arrow'.
+  license_family: APACHE
+
+extra:
+  recipe-maintainers:
+    - conda-forge/r
+    - conda-forge/arrow-cpp
diff --git a/dev/tasks/conda-recipes/run_docker_build.sh b/dev/tasks/conda-recipes/run_docker_build.sh
index d728d07e162..8a900379487 100755
--- a/dev/tasks/conda-recipes/run_docker_build.sh
+++ b/dev/tasks/conda-recipes/run_docker_build.sh
@@ -61,6 +61,7 @@ docker run ${DOCKER_RUN_ARGS} \
            -v "${build_dir}":/build:rw \
            -e FEEDSTOCK_ROOT="/arrow/dev/tasks/conda-recipes" \
            -e CONFIG \
+           -e R_CONFIG \
            -e HOST_USER_ID \
            -e UPLOAD_PACKAGES \
            -e ARROW_VERSION \
diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index cb14de6415b..a68794c3ac1 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -582,7 +582,8 @@ def put(self, job, prefix='build'):
             # adding CI's name to the end of the branch in order to use skip
             # patterns on travis and circleci
             task.branch = '{}-{}-{}'.format(job.branch, task.ci, task_name)
-            files = task.render_files(arrow=job.target,
+            files = task.render_files(**job.params,
+                                      arrow=job.target,
                                       queue_remote_url=self.remote_url)
             branch = self.create_branch(task.branch, files=files)
             self.create_tag(task.tag, branch.target)
@@ -691,7 +692,14 @@ class Task(Serializable):
     """
 
     def __init__(self, ci, template, artifacts=None, params=None):
-        assert ci in {'circle', 'travis', 'appveyor', 'azure', 'github'}
+        assert ci in {
+            'circle',
+            'travis',
+            'appveyor',
+            'azure',
+            'github',
+            'drone',
+        }
         self.ci = ci
         self.template = template
         self.artifacts = artifacts or []
@@ -702,12 +710,12 @@ def __init__(self, ci, template, artifacts=None, params=None):
         self._status = None  # status cache
         self._assets = None  # assets cache
 
-    def render_files(self, **extra_params):
+    def render_files(self, **params):
         from jinja2 import Template, StrictUndefined
         from jinja2.exceptions import TemplateError
 
         path = CWD / self.template
-        params = toolz.merge(self.params, extra_params)
+        params = toolz.merge(self.params, params)
         template = Template(path.read_text(), undefined=StrictUndefined)
         try:
             rendered = template.render(task=self, **params)
@@ -733,6 +741,7 @@ def filename(self):
             'appveyor': 'appveyor.yml',
             'azure': 'azure-pipelines.yml',
             'github': '.github/workflows/crossbow.yml',
+            'drone': '.drone.yml',
         }
         return config_files[self.ci]
 
@@ -863,15 +872,21 @@ def uploaded_assets(self):
 class Job(Serializable):
     """Describes multiple tasks against a single target repository"""
 
-    def __init__(self, target, tasks):
+    def __init__(self, target, tasks, params=None):
         if not tasks:
             raise ValueError('no tasks were provided for the job')
         if not all(isinstance(task, Task) for task in tasks.values()):
             raise ValueError('each `tasks` mus be an instance of Task')
         if not isinstance(target, Target):
             raise ValueError('`target` must be an instance of Target')
+        if not isinstance(target, Target):
+            raise ValueError('`target` must be an instance of Target')
+        if not isinstance(params, dict):
+            raise ValueError('`params` must be an instance of dict')
+
         self.target = target
         self.tasks = tasks
+        self.params = params or {}  # additional parameters for the tasks
         self.branch = None  # filled after adding to a queue
         self._queue = None  # set by the queue object after put or get
 
@@ -903,7 +918,7 @@ def date(self):
         return self.queue.date_of(self)
 
     @classmethod
-    def from_config(cls, config, target, tasks=None, groups=None):
+    def from_config(cls, config, target, tasks=None, groups=None, params=None):
         """
         Intantiate a job from based on a config.
 
@@ -915,9 +930,11 @@ def from_config(cls, config, target, tasks=None, groups=None):
             Describes target repository and revision the builds run against.
         tasks : Optional[List[str]], default None
             List of glob patterns for matching task names.
-        groups : tasks : Optional[List[str]], default None
+        groups : Optional[List[str]], default None
             List of exact group names matching predefined task sets in the
             config.
+        params : Optional[Dict[str, str]], default None
+            Additional rendering parameters for the task templates.
 
         Returns
         -------
@@ -940,7 +957,7 @@ def from_config(cls, config, target, tasks=None, groups=None):
             artifacts = [fn.format(**versions) for fn in artifacts]
             tasks[task_name] = Task(artifacts=artifacts, **task)
 
-        return cls(target=target, tasks=tasks)
+        return cls(target=target, tasks=tasks, params=params)
 
     def is_finished(self):
         for task in self.tasks.values():
@@ -1400,6 +1417,8 @@ def check_config(config_path):
 @click.argument('tasks', nargs=-1, required=False)
 @click.option('--group', '-g', 'groups', multiple=True,
               help='Submit task groups as defined in task.yml')
+@click.option('--param', '-p', 'params', multiple=True,
+              help='Additional task parameters for rendering the CI templates')
 @click.option('--job-prefix', default='build',
               help='Arbitrary prefix for branch names, e.g. nightly')
 @click.option('--config-path', '-c',
@@ -1421,7 +1440,7 @@ def check_config(config_path):
               help='Just display the rendered CI configurations without '
                    'submitting them')
 @click.pass_obj
-def submit(obj, tasks, groups, job_prefix, config_path, arrow_version,
+def submit(obj, tasks, groups, params, job_prefix, config_path, arrow_version,
            arrow_remote, arrow_branch, arrow_sha, dry_run):
     output = obj['output']
     queue, arrow = obj['queue'], obj['arrow']
@@ -1440,9 +1459,12 @@ def submit(obj, tasks, groups, job_prefix, config_path, arrow_version,
     target = Target.from_repo(arrow, remote=arrow_remote, branch=arrow_branch,
                               head=arrow_sha, version=arrow_version)
 
+    # parse additional job parameters
+    params = dict([p.split("=") for p in params])
+
     # instantiate the job object
     job = Job.from_config(config=config, target=target, tasks=tasks,
-                          groups=groups)
+                          groups=groups, params=params)
 
     if dry_run:
         yaml.dump(job, output)
diff --git a/dev/tasks/gandiva-jars/build-cpp-linux.sh b/dev/tasks/gandiva-jars/build-cpp-linux.sh
index 24402fc4945..20f9d32682c 100755
--- a/dev/tasks/gandiva-jars/build-cpp-linux.sh
+++ b/dev/tasks/gandiva-jars/build-cpp-linux.sh
@@ -64,7 +64,7 @@ cmake -DCMAKE_BUILD_TYPE=Release \
     -DBOOST_ROOT=/arrow_boost_dist \
     -GNinja /arrow/cpp
 ninja install
-ninja test
+CTEST_OUTPUT_ON_FAILURE=1 ninja test
 popd
 
 
diff --git a/dev/tasks/homebrew-formulae/apache-arrow.rb b/dev/tasks/homebrew-formulae/apache-arrow.rb
index 31f0def48ac..888e2a357a8 100644
--- a/dev/tasks/homebrew-formulae/apache-arrow.rb
+++ b/dev/tasks/homebrew-formulae/apache-arrow.rb
@@ -1,7 +1,7 @@
 class ApacheArrow < Formula
   desc "Columnar in-memory analytics layer designed to accelerate big data"
   homepage "https://arrow.apache.org/"
-  url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-2.0.0-SNAPSHOT/apache-arrow-2.0.0-SNAPSHOT.tar.gz"
+  url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-3.0.0-SNAPSHOT/apache-arrow-3.0.0-SNAPSHOT.tar.gz"
   sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28"
   license "Apache-2.0"
   head "https://github.com/apache/arrow.git"
@@ -16,7 +16,7 @@ class ApacheArrow < Formula
   depends_on "numpy"
   depends_on "openssl@1.1"
   depends_on "protobuf"
-  depends_on "python@3.8"
+  depends_on "python@3.9"
   depends_on "rapidjson"
   depends_on "snappy"
   depends_on "thrift"
@@ -24,6 +24,8 @@ class ApacheArrow < Formula
 
   def install
     ENV.cxx11
+    # link against system libc++ instead of llvm provided libc++
+    ENV.remove "HOMEBREW_LIBRARY_PATHS", Formula["llvm"].opt_lib
     args = %W[
       -DARROW_FLIGHT=ON
       -DARROW_GANDIVA=ON
@@ -40,7 +42,7 @@ def install
       -DARROW_WITH_SNAPPY=ON
       -DARROW_WITH_BROTLI=ON
       -DARROW_INSTALL_NAME_RPATH=OFF
-      -DPYTHON_EXECUTABLE=#{Formula["python@3.8"].bin/"python3"}
+      -DPython3_EXECUTABLE=#{Formula["python@3.9"].bin/"python3"}
     ]
     # Re-enable -DARROW_S3=ON and add back aws-sdk-cpp to depends_on in ARROW-6437
 
diff --git a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
index 725847acd9c..07bb1e4d6d3 100644
--- a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
+++ b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
@@ -19,7 +19,7 @@
 class ApacheArrow < Formula
   desc "Columnar in-memory analytics layer designed to accelerate big data"
   homepage "https://arrow.apache.org/"
-  url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-1.0.0.9000/apache-arrow-1.0.0.9000.tar.gz"
+  url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-2.0.0.9000/apache-arrow-2.0.0.9000.tar.gz"
   sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28"
   head "https://github.com/apache/arrow.git"
 
@@ -32,9 +32,11 @@ class ApacheArrow < Formula
   # NOTE: if you add something here, be sure to add to PKG_LIBS in r/tools/autobrew
   depends_on "boost" => :build
   depends_on "cmake" => :build
+  depends_on "aws-sdk-cpp"
   depends_on "lz4"
   depends_on "snappy"
   depends_on "thrift"
+  depends_on "zstd"
 
   def install
     ENV.cxx11
@@ -50,14 +52,15 @@ def install
       -DARROW_JEMALLOC=ON
       -DARROW_USE_GLOG=OFF
       -DARROW_PYTHON=OFF
-      -DARROW_S3=OFF
+      -DARROW_S3=ON
       -DARROW_WITH_LZ4=ON
       -DARROW_WITH_ZLIB=ON
       -DARROW_WITH_SNAPPY=ON
       -DARROW_WITH_UTF8PROC=OFF
+      -DARROW_WITH_ZSTD=ON
       -DARROW_BUILD_UTILITIES=ON
       -DARROW_VERBOSE_THIRDPARTY_BUILD=ON
-      -DCMAKE_UNITY_BUILD=ON
+      -DCMAKE_UNITY_BUILD=OFF
       -DPARQUET_BUILD_EXECUTABLES=ON
       -DLZ4_HOME=#{Formula["lz4"].prefix}
       -DTHRIFT_HOME=#{Formula["thrift"].prefix}
diff --git a/dev/tasks/homebrew-formulae/travis.osx.r.yml b/dev/tasks/homebrew-formulae/travis.osx.r.yml
index 9442caeb298..4340708512e 100644
--- a/dev/tasks/homebrew-formulae/travis.osx.r.yml
+++ b/dev/tasks/homebrew-formulae/travis.osx.r.yml
@@ -45,10 +45,11 @@ before_install:
 - sed -i.bak -E -e 's@https://github.com/apache/arrow.git"$@{{ arrow.remote }}.git", :revision => "{{ arrow.head }}"@' tools/apache-arrow.rb && rm -f tools/apache-arrow.rb.bak
 # Sometimes crossbow gives a remote URL with .git and sometimes not. Make sure there's only one
 - sed -i.bak -E -e 's@.git.git@.git@' tools/apache-arrow.rb && rm -f tools/apache-arrow.rb.bak
+# Get minio for S3 testing
+- brew install minio
 script:
-- Rscript -e 'install.packages("rcmdcheck")'
+- Rscript -e 'install.packages(c("rcmdcheck", "sys"))'
 # Note that this is not --as-cran. CRAN doesn't do macOS checks --as-cran
-- travis_wait Rscript -e "rcmdcheck::rcmdcheck(build_args = '--no-build-vignettes', args = c('--no-manual', '--ignore-vignettes', '--run-donttest'), error_on = 'warning', check_dir = 'check')"
-after_failure:
-# If there's a build failure, it's probably in this log
-- cat arrow.Rcheck/00install.out
+- travis_wait Rscript -e "minio_dir <- tempfile(); dir.create(minio_dir); pid <- sys::exec_background('minio', c('server', minio_dir)); on.exit(tools::pskill(pid)); rcmdcheck::rcmdcheck(build_args = '--no-build-vignettes', args = c('--no-manual', '--ignore-vignettes', '--run-donttest'), error_on = 'warning', check_dir = 'check')"
+# If there's a build failure, it's probably in this log. Let's print it regardless though
+- cat check/arrow.Rcheck/00install.out
diff --git a/dev/tasks/homebrew-formulae/travis.osx.yml b/dev/tasks/homebrew-formulae/travis.osx.yml
index b49dbd6cb1c..8fdc02690a6 100644
--- a/dev/tasks/homebrew-formulae/travis.osx.yml
+++ b/dev/tasks/homebrew-formulae/travis.osx.yml
@@ -42,7 +42,8 @@ before_script:
 - brew unlink python@2
 - brew config
 - brew doctor || true
-- brew audit $ARROW_FORMULA
+- cp $ARROW_FORMULA $(brew --repository homebrew/core)/Formula/apache-arrow.rb
 script:
-- brew install -v --HEAD $ARROW_FORMULA
-- brew test $ARROW_FORMULA
+- brew install -v --HEAD apache-arrow
+- brew test apache-arrow
+- brew audit --strict apache-arrow
diff --git a/dev/tasks/linux-packages/apache-arrow-archive-keyring/debian/changelog b/dev/tasks/linux-packages/apache-arrow-archive-keyring/debian/changelog
index 798db15b629..b88bb1df038 100644
--- a/dev/tasks/linux-packages/apache-arrow-archive-keyring/debian/changelog
+++ b/dev/tasks/linux-packages/apache-arrow-archive-keyring/debian/changelog
@@ -1,3 +1,9 @@
+apache-arrow-archive-keyring (2.0.0-1) unstable; urgency=low
+
+  * New upstream release.
+
+ -- Krisztián Szűcs <szucs.krisztian@gmail.com>  Mon, 12 Oct 2020 23:38:01 -0000
+
 apache-arrow-archive-keyring (1.0.0-1) unstable; urgency=low
 
   * New upstream release.
diff --git a/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in b/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in
index 1dbfc0c5440..299b78079ca 100644
--- a/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in
+++ b/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in
@@ -94,6 +94,9 @@ else
 fi
 
 %changelog
+* Mon Oct 12 2020 Krisztián Szűcs <szucs.krisztian@gmail.com> - 2.0.0-1
+- New upstream release.
+
 * Mon Jul 20 2020 Krisztián Szűcs <szucs.krisztian@gmail.com> - 1.0.0-1
 - New upstream release.
 
diff --git a/dev/tasks/linux-packages/apache-arrow/Rakefile b/dev/tasks/linux-packages/apache-arrow/Rakefile
index ba07b71774c..32ebeb9b431 100644
--- a/dev/tasks/linux-packages/apache-arrow/Rakefile
+++ b/dev/tasks/linux-packages/apache-arrow/Rakefile
@@ -82,20 +82,6 @@ class ApacheArrowPackageTask < PackageTask
          "--output", @full_archive_name)
     end
   end
-
-  def detect_llvm_version
-    detect_env("LLVM")
-  end
-
-  def docker_image(os, architecture)
-    "#{super}-llvm-#{detect_llvm_version}"
-  end
-
-  def docker_build_options(os, architecture)
-    [
-      "--build-arg", "LLVM=#{detect_llvm_version}",
-    ]
-  end
 end
 
 task = ApacheArrowPackageTask.new
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-buster/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/debian-buster/Dockerfile
index fdb57005ca2..c6268f69f83 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/debian-buster/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/debian-buster/Dockerfile
@@ -35,19 +35,9 @@ RUN \
     /etc/apt/sources.list.d/backports.list
 
 ARG DEBUG
-ARG LLVM
 RUN \
   quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \
   apt update ${quiet} && \
-  apt install -y -V ${quiet} \
-    apt-transport-https \
-    ca-certificates \
-    gnupg \
-    wget && \
-  wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
-  echo "deb https://apt.llvm.org/buster/ llvm-toolchain-buster-${LLVM} main" > \
-    /etc/apt/sources.list.d/llvm.list && \
-  apt update ${quiet} && \
   apt install -y -V ${quiet} \
     build-essential \
     cmake \
@@ -61,6 +51,7 @@ RUN \
     libbrotli-dev \
     libbz2-dev \
     libc-ares-dev \
+    libcurl4-openssl-dev \
     libgirepository1.0-dev \
     libglib2.0-doc \
     libgmock-dev \
@@ -82,11 +73,9 @@ RUN \
     rapidjson-dev \
     tzdata \
     zlib1g-dev && \
-  if [ "$(dpkg --print-architecture)" != "arm64" ]; then \
-    apt install -y -V ${quiet} \
-      clang-${LLVM} \
-      llvm-${LLVM}-dev; \
-  fi && \
+  apt install -y -V -t buster-backports ${quiet} \
+    clang-8 \
+    llvm-8-dev && \
   if apt list | grep '^nvidia-cuda-toolkit/'; then \
     apt install -y -V ${quiet} nvidia-cuda-toolkit; \
   fi && \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-stretch/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/debian-stretch/Dockerfile
index 606b2c9fe70..fd0c9bc6218 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/debian-stretch/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/debian-stretch/Dockerfile
@@ -37,21 +37,12 @@ RUN \
     /etc/apt/sources.list.d/backports-sloppy.list
 
 ARG DEBUG
-ARG LLVM
 RUN \
   quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \
   apt update ${quiet} && \
-  apt install -y -V ${quiet} \
-    apt-transport-https \
-    ca-certificates \
-    gnupg \
-    wget && \
-  wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
-  echo "deb https://apt.llvm.org/stretch/ llvm-toolchain-stretch-${LLVM} main" > \
-    /etc/apt/sources.list.d/llvm.list && \
-  apt update ${quiet} && \
   apt install -y -V ${quiet} \
     build-essential \
+    clang-7 \
     cmake \
     devscripts \
     fakeroot \
@@ -63,6 +54,7 @@ RUN \
     libbrotli-dev \
     libbz2-dev \
     libc-ares-dev \
+    libcurl4-openssl-dev \
     libgirepository1.0-dev \
     libglib2.0-doc \
     libgoogle-glog-dev \
@@ -72,6 +64,7 @@ RUN \
     libssl-dev \
     libutf8proc-dev \
     libzstd-dev \
+    llvm-7-dev \
     lsb-release \
     ninja-build \
     pkg-config \
@@ -89,11 +82,6 @@ RUN \
     libgmock-dev \
     libgtest-dev \
     rapidjson-dev && \
-  if [ "$(dpkg --print-architecture)" != "arm64" ]; then \
-    apt install -y -V ${quiet} \
-      clang-${LLVM} \
-      llvm-${LLVM}-dev; \
-  fi && \
   if apt list | grep '^nvidia-cuda-toolkit/'; then \
     apt install -y -V -t stretch-backports ${quiet} nvidia-cuda-toolkit; \
   fi && \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile
index 606db04a27c..f0596ced9d5 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile
@@ -29,21 +29,12 @@ RUN \
     /etc/apt/apt.conf.d/disable-install-recommends
 
 ARG DEBUG
-ARG LLVM
 RUN \
   quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \
   apt update ${quiet} && \
-  apt install -y -V ${quiet} \
-    apt-transport-https \
-    ca-certificates \
-    gnupg \
-    wget && \
-  wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
-  echo "deb https://apt.llvm.org/bionic/ llvm-toolchain-bionic-${LLVM} main" > \
-    /etc/apt/sources.list.d/llvm.list && \
-  apt update ${quiet} && \
   apt install -y -V ${quiet} \
     build-essential \
+    clang-10 \
     cmake \
     devscripts \
     fakeroot \
@@ -54,6 +45,7 @@ RUN \
     libboost-system-dev \
     libbrotli-dev \
     libbz2-dev \
+    libcurl4-openssl-dev \
     libgirepository1.0-dev \
     libglib2.0-doc \
     libgoogle-glog-dev \
@@ -64,6 +56,7 @@ RUN \
     libssl-dev \
     libutf8proc-dev \
     libzstd-dev \
+    llvm-10-dev \
     lsb-release \
     ninja-build \
     pkg-config \
@@ -75,11 +68,6 @@ RUN \
     rapidjson-dev \
     tzdata \
     zlib1g-dev && \
-  if [ "$(dpkg --print-architecture)" != "arm64" ]; then \
-    apt install -y -V ${quiet} \
-      clang-${LLVM} \
-      llvm-${LLVM}-dev; \
-  fi && \
   if apt list | grep '^nvidia-cuda-toolkit/'; then \
     apt install -y -V ${quiet} nvidia-cuda-toolkit; \
   fi && \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile
index e9468d4109c..898e64a6d55 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile
@@ -29,12 +29,12 @@ RUN \
     /etc/apt/apt.conf.d/disable-install-recommends
 
 ARG DEBUG
-ARG LLVM
 RUN \
   quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \
   apt update ${quiet} && \
   apt install -y -V ${quiet} \
     build-essential \
+    clang-10 \
     cmake \
     debhelper \
     devscripts \
@@ -45,6 +45,7 @@ RUN \
     libboost-system-dev \
     libbrotli-dev \
     libbz2-dev \
+    libcurl4-openssl-dev \
     libgirepository1.0-dev \
     libglib2.0-doc \
     libgmock-dev \
@@ -57,6 +58,7 @@ RUN \
     libthrift-dev \
     libutf8proc-dev \
     libzstd-dev \
+    llvm-10-dev \
     lsb-release \
     ninja-build \
     pkg-config \
@@ -67,11 +69,6 @@ RUN \
     rapidjson-dev \
     tzdata \
     zlib1g-dev && \
-  if [ "$(dpkg --print-architecture)" != "arm64" ]; then \
-    apt install -y -V ${quiet} \
-      clang-${LLVM} \
-      llvm-${LLVM}-dev; \
-  fi && \
   if apt list | grep '^nvidia-cuda-toolkit/'; then \
     apt install -y -V ${quiet} nvidia-cuda-toolkit; \
   fi && \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-xenial/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-xenial/Dockerfile
index 79da9f59068..9e38399864f 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-xenial/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-xenial/Dockerfile
@@ -29,25 +29,13 @@ RUN \
     /etc/apt/apt.conf.d/disable-install-recommends
 
 ARG DEBUG
-# LLVM 10 or later requires C++ 14 but g++-5's C++ 14 support is limited.
-# cpp/src/arrow/vendored/datetime/date.h doesn't work.
-# ARG LLVM
-ENV LLVM=8
 RUN \
   quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \
   apt update ${quiet} && \
-  apt install -y -V ${quiet} \
-    apt-transport-https \
-    ca-certificates \
-    gnupg \
-    wget && \
-  wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
-  echo "deb https://apt.llvm.org/xenial/ llvm-toolchain-xenial-${LLVM} main" > \
-    /etc/apt/sources.list.d/llvm.list && \
-  apt update ${quiet} && \
   apt install -y -V ${quiet} \
     autoconf-archive \
     build-essential \
+    clang-8 \
     cmake \
     devscripts \
     dh-autoreconf \
@@ -59,6 +47,7 @@ RUN \
     libboost-system-dev \
     libbrotli-dev \
     libbz2-dev \
+    libcurl4-openssl-dev \
     libgirepository1.0-dev \
     libglib2.0-doc \
     libgoogle-glog-dev \
@@ -70,17 +59,13 @@ RUN \
     libssl-dev \
     libutf8proc-dev \
     libzstd1-dev \
+    llvm-8-dev \
     lsb-release \
     pkg-config \
     protobuf-compiler \
     python3-dev \
     python3-numpy \
     zlib1g-dev && \
-  if [ "$(dpkg --print-architecture)" != "arm64" ]; then \
-    apt install -y -V ${quiet} \
-      clang-${LLVM} \
-      llvm-${LLVM}-dev; \
-  fi && \
   if apt list | grep '^nvidia-cuda-toolkit/'; then \
     apt install -y -V ${quiet} nvidia-cuda-toolkit; \
   fi && \
diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/changelog b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/changelog
index 9cd03f215ab..ce434a6fe79 100644
--- a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/changelog
+++ b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/changelog
@@ -1,3 +1,9 @@
+apache-arrow (2.0.0-1) unstable; urgency=low
+
+  * New upstream release.
+
+ -- Krisztián Szűcs <szucs.krisztian@gmail.com>  Mon, 12 Oct 2020 23:38:01 -0000
+
 apache-arrow (1.0.0-1) unstable; urgency=low
 
   * New upstream release.
diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/control b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/control
index 5cf476d9bcd..11a6e244b6b 100644
--- a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/control
+++ b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/control
@@ -14,6 +14,7 @@ Build-Depends:
   libboost-regex-dev,
   libboost-system-dev,
   libbrotli-dev,
+  libcurl4-openssl-dev,
   libgirepository1.0-dev,
   libgoogle-glog-dev,
   liblz4-dev,
@@ -34,7 +35,7 @@ Build-Depends-Indep: libglib2.0-doc
 Standards-Version: 3.9.7
 Homepage: https://arrow.apache.org/
 
-Package: libarrow200
+Package: libarrow300
 Section: libs
 Architecture: any
 Multi-Arch: same
@@ -46,7 +47,7 @@ Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ library files.
 
-Package: libarrow-cuda200
+Package: libarrow-cuda300
 Section: libs
 Architecture: i386 amd64
 Multi-Arch: same
@@ -54,12 +55,12 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow200 (= ${binary:Version})
+  libarrow300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ library files for CUDA support.
 
-Package: libarrow-dataset200
+Package: libarrow-dataset300
 Section: libs
 Architecture: any
 Multi-Arch: same
@@ -67,12 +68,12 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow200 (= ${binary:Version})
+  libarrow300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ library files for dataset module.
 
-Package: libarrow-python200
+Package: libarrow-python300
 Section: libs
 Architecture: any
 Multi-Arch: same
@@ -80,7 +81,7 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow200 (= ${binary:Version}),
+  libarrow300 (= ${binary:Version}),
   python3,
   python3-numpy
 Description: Apache Arrow is a data processing library for analysis
@@ -93,7 +94,7 @@ Architecture: any
 Multi-Arch: same
 Depends:
   ${misc:Depends},
-  libarrow200 (= ${binary:Version})
+  libarrow300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ header files.
@@ -105,7 +106,7 @@ Multi-Arch: same
 Depends:
   ${misc:Depends},
   libarrow-dev (= ${binary:Version}),
-  libarrow-cuda200 (= ${binary:Version})
+  libarrow-cuda300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ header files for CUDA support.
@@ -117,7 +118,7 @@ Multi-Arch: same
 Depends:
   ${misc:Depends},
   libarrow-dev (= ${binary:Version}),
-  libarrow-dataset200 (= ${binary:Version})
+  libarrow-dataset300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ header files for dataset module.
@@ -129,20 +130,20 @@ Multi-Arch: same
 Depends:
   ${misc:Depends},
   libarrow-dev (= ${binary:Version}),
-  libarrow-python200 (= ${binary:Version})
+  libarrow-python300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ header files for Python support.
 
-Package: libgandiva200
+Package: libgandiva300
 Section: libs
-Architecture: i386 amd64
+Architecture: any
 Multi-Arch: same
 Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow200 (= ${binary:Version})
+  libarrow300 (= ${binary:Version})
 Description: Gandiva is a toolset for compiling and evaluating expressions
  on Arrow Data.
  .
@@ -150,18 +151,18 @@ Description: Gandiva is a toolset for compiling and evaluating expressions
 
 Package: libgandiva-dev
 Section: libdevel
-Architecture: i386 amd64
+Architecture: any
 Multi-Arch: same
 Depends:
   ${misc:Depends},
   libarrow-dev (= ${binary:Version}),
-  libgandiva200 (= ${binary:Version})
+  libgandiva300 (= ${binary:Version})
 Description: Gandiva is a toolset for compiling and evaluating expressions
  on Arrow Data.
  .
  This package provides C++ header files.
 
-Package: libplasma200
+Package: libplasma300
 Section: libs
 Architecture: i386 amd64
 Multi-Arch: same
@@ -169,7 +170,7 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow-cuda200 (= ${binary:Version})
+  libarrow-cuda300 (= ${binary:Version})
 Description: Plasma is an in-memory object store and cache for big data.
  .
  This package provides C++ library files to connect plasma-store-server.
@@ -181,7 +182,7 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libplasma200 (= ${binary:Version})
+  libplasma300 (= ${binary:Version})
 Description: Plasma is an in-memory object store and cache for big data.
  .
  This package provides plasma-store-server.
@@ -193,12 +194,12 @@ Multi-Arch: same
 Depends:
   ${misc:Depends},
   libarrow-cuda-dev (= ${binary:Version}),
-  libplasma200 (= ${binary:Version})
+  libplasma300 (= ${binary:Version})
 Description: Plasma is an in-memory object store and cache for big data.
  .
  This package provides C++ header files.
 
-Package: libparquet200
+Package: libparquet300
 Section: libs
 Architecture: any
 Multi-Arch: same
@@ -217,12 +218,12 @@ Multi-Arch: same
 Depends:
   ${misc:Depends},
   libarrow-dev (= ${binary:Version}),
-  libparquet200 (= ${binary:Version})
+  libparquet300 (= ${binary:Version})
 Description: Apache Parquet is a columnar storage format
  .
  This package provides C++ header files.
 
-Package: libarrow-glib200
+Package: libarrow-glib300
 Section: libs
 Architecture: any
 Multi-Arch: same
@@ -230,7 +231,7 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow200 (= ${binary:Version})
+  libarrow300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides GLib based library files.
@@ -254,7 +255,7 @@ Depends:
   ${misc:Depends},
   libglib2.0-dev,
   libarrow-dev (= ${binary:Version}),
-  libarrow-glib200 (= ${binary:Version}),
+  libarrow-glib300 (= ${binary:Version}),
   gir1.2-arrow-1.0 (= ${binary:Version})
 Suggests: libarrow-glib-doc
 Description: Apache Arrow is a data processing library for analysis
@@ -272,7 +273,7 @@ Description: Apache Arrow is a data processing library for analysis
  .
  This package provides documentations.
 
-Package: libarrow-cuda-glib200
+Package: libarrow-cuda-glib300
 Section: libs
 Architecture: i386 amd64
 Multi-Arch: same
@@ -280,8 +281,8 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow-glib200 (= ${binary:Version}),
-  libarrow-cuda200 (= ${binary:Version})
+  libarrow-glib300 (= ${binary:Version}),
+  libarrow-cuda300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides GLib based library files for CUDA support.
@@ -305,13 +306,13 @@ Depends:
   ${misc:Depends},
   libarrow-cuda-dev (= ${binary:Version}),
   libarrow-glib-dev (= ${binary:Version}),
-  libarrow-cuda-glib200 (= ${binary:Version}),
+  libarrow-cuda-glib300 (= ${binary:Version}),
   gir1.2-arrow-cuda-1.0 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides GLib based header files for CUDA support.
 
-Package: libarrow-dataset-glib200
+Package: libarrow-dataset-glib300
 Section: libs
 Architecture: any
 Multi-Arch: same
@@ -319,8 +320,8 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow-glib200 (= ${binary:Version}),
-  libarrow-dataset200 (= ${binary:Version})
+  libarrow-glib300 (= ${binary:Version}),
+  libarrow-dataset300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides GLib based library files for dataset module.
@@ -344,7 +345,7 @@ Depends:
   ${misc:Depends},
   libarrow-dataset-dev (= ${binary:Version}),
   libarrow-glib-dev (= ${binary:Version}),
-  libarrow-dataset-glib200 (= ${binary:Version}),
+  libarrow-dataset-glib300 (= ${binary:Version}),
   gir1.2-arrow-dataset-1.0 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
@@ -361,16 +362,16 @@ Description: Apache Arrow is a data processing library for analysis
  .
  This package provides documentations for dataset module.
 
-Package: libgandiva-glib200
+Package: libgandiva-glib300
 Section: libs
-Architecture: i386 amd64
+Architecture: any
 Multi-Arch: same
 Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow-glib200 (= ${binary:Version}),
-  libgandiva200 (= ${binary:Version})
+  libarrow-glib300 (= ${binary:Version}),
+  libgandiva300 (= ${binary:Version})
 Description: Gandiva is a toolset for compiling and evaluating expressions
  on Arrow Data.
  .
@@ -378,7 +379,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions
 
 Package: gir1.2-gandiva-1.0
 Section: introspection
-Architecture: i386 amd64
+Architecture: any
 Multi-Arch: same
 Depends:
   ${gir:Depends},
@@ -390,13 +391,13 @@ Description: Gandiva is a toolset for compiling and evaluating expressions
 
 Package: libgandiva-glib-dev
 Section: libdevel
-Architecture: i386 amd64
+Architecture: any
 Multi-Arch: same
 Depends:
   ${misc:Depends},
   libgandiva-dev (= ${binary:Version}),
   libarrow-glib-dev (= ${binary:Version}),
-  libgandiva-glib200 (= ${binary:Version}),
+  libgandiva-glib300 (= ${binary:Version}),
   gir1.2-gandiva-1.0 (= ${binary:Version})
 Description: Gandiva is a toolset for compiling and evaluating expressions
  on Arrow Data.
@@ -405,7 +406,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions
 
 Package: libgandiva-glib-doc
 Section: doc
-Architecture: i386 amd64
+Architecture: any
 Multi-Arch: foreign
 Depends:
   ${misc:Depends}
@@ -415,7 +416,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions
  .
  This package provides documentations.
 
-Package: libplasma-glib200
+Package: libplasma-glib300
 Section: libs
 Architecture: i386 amd64
 Multi-Arch: same
@@ -423,8 +424,8 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow-cuda-glib200 (= ${binary:Version}),
-  libplasma200 (= ${binary:Version})
+  libarrow-cuda-glib300 (= ${binary:Version}),
+  libplasma300 (= ${binary:Version})
 Description: Plasma is an in-memory object store and cache for big data.
  .
  This package provides GLib based library files to connect plasma-store-server.
@@ -448,7 +449,7 @@ Depends:
   ${misc:Depends},
   libplasma-dev (= ${binary:Version}),
   libarrow-cuda-glib-dev (= ${binary:Version}),
-  libplasma-glib200 (= ${binary:Version}),
+  libplasma-glib300 (= ${binary:Version}),
   gir1.2-plasma-1.0 (= ${binary:Version})
 Description: Plasma is an in-memory object store and cache for big data.
  .
@@ -465,7 +466,7 @@ Description: Plasma is an in-memory object store and cache for big data.
  .
  This package provides documentations.
 
-Package: libparquet-glib200
+Package: libparquet-glib300
 Section: libs
 Architecture: any
 Multi-Arch: same
@@ -473,8 +474,8 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow-glib200 (= ${binary:Version}),
-  libparquet200 (= ${binary:Version})
+  libarrow-glib300 (= ${binary:Version}),
+  libparquet300 (= ${binary:Version})
 Description: Apache Parquet is a columnar storage format
  .
  This package provides GLib based library files.
@@ -498,7 +499,7 @@ Depends:
   ${misc:Depends},
   libarrow-glib-dev (= ${binary:Version}),
   libparquet-dev (= ${binary:Version}),
-  libparquet-glib200 (= ${binary:Version}),
+  libparquet-glib300 (= ${binary:Version}),
   gir1.2-parquet-1.0 (= ${binary:Version})
 Suggests: libparquet-glib-doc
 Description: Apache Parquet is a columnar storage format
diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-cuda-glib200.install b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-cuda-glib300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-cuda-glib200.install
rename to dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-cuda-glib300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-cuda200.install b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-cuda300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-cuda200.install
rename to dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-cuda300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dataset-glib200.install b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dataset-glib300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dataset-glib200.install
rename to dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dataset-glib300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dataset200.install b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dataset300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dataset200.install
rename to dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dataset300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dev.install b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dev.install
index 549cecef388..84da58caee4 100644
--- a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dev.install
+++ b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dev.install
@@ -5,6 +5,7 @@ usr/lib/*/cmake/arrow/ArrowTargets*.cmake
 usr/lib/*/cmake/arrow/FindArrow.cmake
 usr/lib/*/cmake/arrow/FindBrotli.cmake
 usr/lib/*/cmake/arrow/FindLz4.cmake
+usr/lib/*/cmake/arrow/FindRE2.cmake
 usr/lib/*/cmake/arrow/FindSnappy.cmake
 usr/lib/*/cmake/arrow/Findutf8proc.cmake
 usr/lib/*/cmake/arrow/Findzstd.cmake
diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-glib200.install b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-glib300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-glib200.install
rename to dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-glib300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-python200.install b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-python300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-python200.install
rename to dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-python300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow200.install b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow200.install
rename to dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva-dev.install b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva-dev.install
index 7dabb79d0dd..26e7e76fb38 100644
--- a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva-dev.install
+++ b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva-dev.install
@@ -2,7 +2,6 @@ usr/include/gandiva/
 usr/lib/*/cmake/arrow/GandivaConfig*.cmake
 usr/lib/*/cmake/arrow/GandivaTargets*.cmake
 usr/lib/*/cmake/arrow/FindGandiva.cmake
-usr/lib/*/cmake/arrow/FindRE2.cmake
 usr/lib/*/libgandiva.a
 usr/lib/*/libgandiva.so
 usr/lib/*/pkgconfig/gandiva.pc
diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva-glib200.install b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva-glib300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva-glib200.install
rename to dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva-glib300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva200.install b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva200.install
rename to dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libparquet-glib200.install b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libparquet-glib300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libparquet-glib200.install
rename to dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libparquet-glib300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libparquet200.install b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libparquet300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libparquet200.install
rename to dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libparquet300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libplasma-glib200.install b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libplasma-glib300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libplasma-glib200.install
rename to dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libplasma-glib300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libplasma200.install b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libplasma300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libplasma200.install
rename to dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libplasma300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/rules b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/rules
index 3badd9c2338..e9f8fde676d 100755
--- a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/rules
+++ b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/rules
@@ -42,6 +42,7 @@ override_dh_auto_configure:
 	  -DARROW_PARQUET=ON					\
 	  -DARROW_PLASMA=$${ARROW_PLASMA}			\
 	  -DARROW_PYTHON=ON					\
+	  -DARROW_S3=ON						\
 	  -DARROW_WITH_BROTLI=ON				\
 	  -DARROW_WITH_BZ2=ON					\
 	  -DARROW_WITH_LZ4=ON					\
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/changelog b/dev/tasks/linux-packages/apache-arrow/debian/changelog
index 9cd03f215ab..ce434a6fe79 100644
--- a/dev/tasks/linux-packages/apache-arrow/debian/changelog
+++ b/dev/tasks/linux-packages/apache-arrow/debian/changelog
@@ -1,3 +1,9 @@
+apache-arrow (2.0.0-1) unstable; urgency=low
+
+  * New upstream release.
+
+ -- Krisztián Szűcs <szucs.krisztian@gmail.com>  Mon, 12 Oct 2020 23:38:01 -0000
+
 apache-arrow (1.0.0-1) unstable; urgency=low
 
   * New upstream release.
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/control b/dev/tasks/linux-packages/apache-arrow/debian/control
index bbf732a781f..f2624026a26 100644
--- a/dev/tasks/linux-packages/apache-arrow/debian/control
+++ b/dev/tasks/linux-packages/apache-arrow/debian/control
@@ -13,6 +13,7 @@ Build-Depends:
   libboost-system-dev,
   libbrotli-dev,
   libbz2-dev,
+  libcurl4-openssl-dev,
   libgirepository1.0-dev,
   libgoogle-glog-dev,
   libgtest-dev,
@@ -33,7 +34,7 @@ Build-Depends-Indep: libglib2.0-doc
 Standards-Version: 3.9.8
 Homepage: https://arrow.apache.org/
 
-Package: libarrow200
+Package: libarrow300
 Section: libs
 Architecture: any
 Multi-Arch: same
@@ -45,7 +46,7 @@ Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ library files.
 
-Package: libarrow-cuda200
+Package: libarrow-cuda300
 Section: libs
 Architecture: i386 amd64
 Multi-Arch: same
@@ -53,12 +54,12 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow200 (= ${binary:Version})
+  libarrow300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ library files for CUDA support.
 
-Package: libarrow-dataset200
+Package: libarrow-dataset300
 Section: libs
 Architecture: any
 Multi-Arch: same
@@ -66,13 +67,13 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow200 (= ${binary:Version}),
-  libparquet200 (= ${binary:Version})
+  libarrow300 (= ${binary:Version}),
+  libparquet300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ library files for Dataset module.
 
-Package: libarrow-flight200
+Package: libarrow-flight300
 Section: libs
 Architecture: any
 Multi-Arch: same
@@ -80,12 +81,12 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow200 (= ${binary:Version})
+  libarrow300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ library files for Flight RPC system.
 
-Package: libarrow-python200
+Package: libarrow-python300
 Section: libs
 Architecture: any
 Multi-Arch: same
@@ -93,14 +94,14 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow200 (= ${binary:Version}),
+  libarrow300 (= ${binary:Version}),
   python3,
   python3-numpy
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ library files for Python support.
 
-Package: libarrow-python-flight200
+Package: libarrow-python-flight300
 Section: libs
 Architecture: any
 Multi-Arch: same
@@ -108,8 +109,8 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow-flight200 (= ${binary:Version}),
-  libarrow-python200 (= ${binary:Version})
+  libarrow-flight300 (= ${binary:Version}),
+  libarrow-python300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ library files for Flight and Python support.
@@ -120,7 +121,7 @@ Architecture: any
 Multi-Arch: same
 Depends:
   ${misc:Depends},
-  libarrow200 (= ${binary:Version})
+  libarrow300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ header files.
@@ -132,7 +133,7 @@ Multi-Arch: same
 Depends:
   ${misc:Depends},
   libarrow-dev (= ${binary:Version}),
-  libarrow-cuda200 (= ${binary:Version})
+  libarrow-cuda300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ header files for CUDA support.
@@ -144,7 +145,7 @@ Multi-Arch: same
 Depends:
   ${misc:Depends},
   libarrow-dev (= ${binary:Version}),
-  libarrow-dataset200 (= ${binary:Version}),
+  libarrow-dataset300 (= ${binary:Version}),
   libparquet-dev (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
@@ -157,7 +158,7 @@ Multi-Arch: same
 Depends:
   ${misc:Depends},
   libarrow-dev (= ${binary:Version}),
-  libarrow-flight200 (= ${binary:Version})
+  libarrow-flight300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ header files for Flight RPC system.
@@ -169,7 +170,7 @@ Multi-Arch: same
 Depends:
   ${misc:Depends},
   libarrow-dev (= ${binary:Version}),
-  libarrow-python200 (= ${binary:Version})
+  libarrow-python300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ header files for Python support.
@@ -182,20 +183,20 @@ Depends:
   ${misc:Depends},
   libarrow-flight-dev (= ${binary:Version}),
   libarrow-python-dev (= ${binary:Version}),
-  libarrow-python-flight200 (= ${binary:Version})
+  libarrow-python-flight300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ header files for Flight and Python support.
 
-Package: libgandiva200
+Package: libgandiva300
 Section: libs
-Architecture: i386 amd64
+Architecture: any
 Multi-Arch: same
 Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow200 (= ${binary:Version})
+  libarrow300 (= ${binary:Version})
 Description: Gandiva is a toolset for compiling and evaluating expressions
  on Arrow Data.
  .
@@ -203,18 +204,18 @@ Description: Gandiva is a toolset for compiling and evaluating expressions
 
 Package: libgandiva-dev
 Section: libdevel
-Architecture: i386 amd64
+Architecture: any
 Multi-Arch: same
 Depends:
   ${misc:Depends},
   libarrow-dev (= ${binary:Version}),
-  libgandiva200 (= ${binary:Version})
+  libgandiva300 (= ${binary:Version})
 Description: Gandiva is a toolset for compiling and evaluating expressions
  on Arrow Data.
  .
  This package provides C++ header files.
 
-Package: libplasma200
+Package: libplasma300
 Section: libs
 Architecture: i386 amd64
 Multi-Arch: same
@@ -222,7 +223,7 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow-cuda200 (= ${binary:Version})
+  libarrow-cuda300 (= ${binary:Version})
 Description: Plasma is an in-memory object store and cache for big data.
  .
  This package provides C++ library files to connect plasma-store-server.
@@ -234,7 +235,7 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libplasma200 (= ${binary:Version})
+  libplasma300 (= ${binary:Version})
 Description: Plasma is an in-memory object store and cache for big data.
  .
  This package provides plasma-store-server.
@@ -246,12 +247,12 @@ Multi-Arch: same
 Depends:
   ${misc:Depends},
   libarrow-cuda-dev (= ${binary:Version}),
-  libplasma200 (= ${binary:Version})
+  libplasma300 (= ${binary:Version})
 Description: Plasma is an in-memory object store and cache for big data.
  .
  This package provides C++ header files.
 
-Package: libparquet200
+Package: libparquet300
 Section: libs
 Architecture: any
 Multi-Arch: same
@@ -270,12 +271,12 @@ Multi-Arch: same
 Depends:
   ${misc:Depends},
   libarrow-dev (= ${binary:Version}),
-  libparquet200 (= ${binary:Version})
+  libparquet300 (= ${binary:Version})
 Description: Apache Parquet is a columnar storage format
  .
  This package provides C++ header files.
 
-Package: libarrow-glib200
+Package: libarrow-glib300
 Section: libs
 Architecture: any
 Multi-Arch: same
@@ -283,7 +284,7 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow200 (= ${binary:Version})
+  libarrow300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides GLib based library files.
@@ -307,7 +308,7 @@ Depends:
   ${misc:Depends},
   libglib2.0-dev,
   libarrow-dev (= ${binary:Version}),
-  libarrow-glib200 (= ${binary:Version}),
+  libarrow-glib300 (= ${binary:Version}),
   gir1.2-arrow-1.0 (= ${binary:Version})
 Suggests: libarrow-glib-doc
 Description: Apache Arrow is a data processing library for analysis
@@ -325,7 +326,7 @@ Description: Apache Arrow is a data processing library for analysis
  .
  This package provides documentations.
 
-Package: libarrow-cuda-glib200
+Package: libarrow-cuda-glib300
 Section: libs
 Architecture: i386 amd64
 Multi-Arch: same
@@ -333,8 +334,8 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow-glib200 (= ${binary:Version}),
-  libarrow-cuda200 (= ${binary:Version})
+  libarrow-glib300 (= ${binary:Version}),
+  libarrow-cuda300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides GLib based library files for CUDA support.
@@ -358,13 +359,13 @@ Depends:
   ${misc:Depends},
   libarrow-cuda-dev (= ${binary:Version}),
   libarrow-glib-dev (= ${binary:Version}),
-  libarrow-cuda-glib200 (= ${binary:Version}),
+  libarrow-cuda-glib300 (= ${binary:Version}),
   gir1.2-arrow-cuda-1.0 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides GLib based header files for CUDA support.
 
-Package: libarrow-dataset-glib200
+Package: libarrow-dataset-glib300
 Section: libs
 Architecture: any
 Multi-Arch: same
@@ -372,8 +373,8 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow-glib200 (= ${binary:Version}),
-  libarrow-dataset200 (= ${binary:Version})
+  libarrow-glib300 (= ${binary:Version}),
+  libarrow-dataset300 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
  This package provides GLib based library files for dataset module.
@@ -397,7 +398,7 @@ Depends:
   ${misc:Depends},
   libarrow-dataset-dev (= ${binary:Version}),
   libarrow-glib-dev (= ${binary:Version}),
-  libarrow-dataset-glib200 (= ${binary:Version}),
+  libarrow-dataset-glib300 (= ${binary:Version}),
   gir1.2-arrow-dataset-1.0 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
@@ -414,16 +415,16 @@ Description: Apache Arrow is a data processing library for analysis
  .
  This package provides documentations for dataset module.
 
-Package: libgandiva-glib200
+Package: libgandiva-glib300
 Section: libs
-Architecture: i386 amd64
+Architecture: any
 Multi-Arch: same
 Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow-glib200 (= ${binary:Version}),
-  libgandiva200 (= ${binary:Version})
+  libarrow-glib300 (= ${binary:Version}),
+  libgandiva300 (= ${binary:Version})
 Description: Gandiva is a toolset for compiling and evaluating expressions
  on Arrow Data.
  .
@@ -431,7 +432,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions
 
 Package: gir1.2-gandiva-1.0
 Section: introspection
-Architecture: i386 amd64
+Architecture: any
 Multi-Arch: same
 Depends:
   ${gir:Depends},
@@ -443,13 +444,13 @@ Description: Gandiva is a toolset for compiling and evaluating expressions
 
 Package: libgandiva-glib-dev
 Section: libdevel
-Architecture: i386 amd64
+Architecture: any
 Multi-Arch: same
 Depends:
   ${misc:Depends},
   libgandiva-dev (= ${binary:Version}),
   libarrow-glib-dev (= ${binary:Version}),
-  libgandiva-glib200 (= ${binary:Version}),
+  libgandiva-glib300 (= ${binary:Version}),
   gir1.2-gandiva-1.0 (= ${binary:Version})
 Description: Gandiva is a toolset for compiling and evaluating expressions
  on Arrow Data.
@@ -458,7 +459,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions
 
 Package: libgandiva-glib-doc
 Section: doc
-Architecture: i386 amd64
+Architecture: any
 Multi-Arch: foreign
 Depends:
   ${misc:Depends}
@@ -468,7 +469,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions
  .
  This package provides documentations.
 
-Package: libplasma-glib200
+Package: libplasma-glib300
 Section: libs
 Architecture: i386 amd64
 Multi-Arch: same
@@ -476,8 +477,8 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow-cuda-glib200 (= ${binary:Version}),
-  libplasma200 (= ${binary:Version})
+  libarrow-cuda-glib300 (= ${binary:Version}),
+  libplasma300 (= ${binary:Version})
 Description: Plasma is an in-memory object store and cache for big data.
  .
  This package provides GLib based library files to connect plasma-store-server.
@@ -501,7 +502,7 @@ Depends:
   ${misc:Depends},
   libplasma-dev (= ${binary:Version}),
   libarrow-cuda-glib-dev (= ${binary:Version}),
-  libplasma-glib200 (= ${binary:Version}),
+  libplasma-glib300 (= ${binary:Version}),
   gir1.2-plasma-1.0 (= ${binary:Version})
 Description: Plasma is an in-memory object store and cache for big data.
  .
@@ -518,7 +519,7 @@ Description: Plasma is an in-memory object store and cache for big data.
  .
  This package provides documentations.
 
-Package: libparquet-glib200
+Package: libparquet-glib300
 Section: libs
 Architecture: any
 Multi-Arch: same
@@ -526,8 +527,8 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow-glib200 (= ${binary:Version}),
-  libparquet200 (= ${binary:Version})
+  libarrow-glib300 (= ${binary:Version}),
+  libparquet300 (= ${binary:Version})
 Description: Apache Parquet is a columnar storage format
  .
  This package provides GLib based library files.
@@ -551,7 +552,7 @@ Depends:
   ${misc:Depends},
   libarrow-glib-dev (= ${binary:Version}),
   libparquet-dev (= ${binary:Version}),
-  libparquet-glib200 (= ${binary:Version}),
+  libparquet-glib300 (= ${binary:Version}),
   gir1.2-parquet-1.0 (= ${binary:Version})
 Suggests: libparquet-glib-doc
 Description: Apache Parquet is a columnar storage format
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib200.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib200.install
rename to dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda200.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda200.install
rename to dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib200.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib200.install
rename to dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset200.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset200.install
rename to dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install
index 549cecef388..84da58caee4 100644
--- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install
+++ b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install
@@ -5,6 +5,7 @@ usr/lib/*/cmake/arrow/ArrowTargets*.cmake
 usr/lib/*/cmake/arrow/FindArrow.cmake
 usr/lib/*/cmake/arrow/FindBrotli.cmake
 usr/lib/*/cmake/arrow/FindLz4.cmake
+usr/lib/*/cmake/arrow/FindRE2.cmake
 usr/lib/*/cmake/arrow/FindSnappy.cmake
 usr/lib/*/cmake/arrow/Findutf8proc.cmake
 usr/lib/*/cmake/arrow/Findzstd.cmake
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight200.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight200.install
rename to dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib200.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib200.install
rename to dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight200.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight200.install
rename to dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python200.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian/libarrow-python200.install
rename to dev/tasks/linux-packages/apache-arrow/debian/libarrow-python300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow200.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian/libarrow200.install
rename to dev/tasks/linux-packages/apache-arrow/debian/libarrow300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install
index 7dabb79d0dd..26e7e76fb38 100644
--- a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install
+++ b/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install
@@ -2,7 +2,6 @@ usr/include/gandiva/
 usr/lib/*/cmake/arrow/GandivaConfig*.cmake
 usr/lib/*/cmake/arrow/GandivaTargets*.cmake
 usr/lib/*/cmake/arrow/FindGandiva.cmake
-usr/lib/*/cmake/arrow/FindRE2.cmake
 usr/lib/*/libgandiva.a
 usr/lib/*/libgandiva.so
 usr/lib/*/pkgconfig/gandiva.pc
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib200.install b/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib200.install
rename to dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva200.install b/dev/tasks/linux-packages/apache-arrow/debian/libgandiva300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian/libgandiva200.install
rename to dev/tasks/linux-packages/apache-arrow/debian/libgandiva300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib200.install b/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib200.install
rename to dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libparquet200.install b/dev/tasks/linux-packages/apache-arrow/debian/libparquet300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian/libparquet200.install
rename to dev/tasks/linux-packages/apache-arrow/debian/libparquet300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib200.install b/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib200.install
rename to dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libplasma200.install b/dev/tasks/linux-packages/apache-arrow/debian/libplasma300.install
similarity index 100%
rename from dev/tasks/linux-packages/apache-arrow/debian/libplasma200.install
rename to dev/tasks/linux-packages/apache-arrow/debian/libplasma300.install
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/rules b/dev/tasks/linux-packages/apache-arrow/debian/rules
index 41d93c7a4f1..fc20c96547b 100755
--- a/dev/tasks/linux-packages/apache-arrow/debian/rules
+++ b/dev/tasks/linux-packages/apache-arrow/debian/rules
@@ -39,6 +39,7 @@ override_dh_auto_configure:
 	  -DARROW_PARQUET=ON					\
 	  -DARROW_PLASMA=$${ARROW_PLASMA}			\
 	  -DARROW_PYTHON=ON					\
+	  -DARROW_S3=ON						\
 	  -DARROW_WITH_BROTLI=ON				\
 	  -DARROW_WITH_BZ2=ON					\
 	  -DARROW_WITH_LZ4=ON					\
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
index 6af80dab06b..4bf50c1bd7a 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
+++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
@@ -40,6 +40,10 @@
 %define use_gandiva (%{_centos_ver} >= 8 && %{_arch} != "aarch64")
 %define use_ninja (%{_centos_ver} >= 8)
 %define use_parquet (%{_centos_ver} >= 7)
+# TODO: Enable this. This works on local but is fragile on GitHub Actions and
+# Travis CI.
+# %define use_s3 (%{_centos_ver} >= 8)
+%define use_s3 0
 
 %define use_glib (%{_centos_ver} >= 7)
 %define use_meson (%{_centos_ver} >= 8)
@@ -67,6 +71,9 @@ BuildRequires:	brotli-devel
 %endif
 BuildRequires:	bzip2-devel
 BuildRequires:	cmake%{cmake_version}
+%if %{use_s3}
+BuildRequires:	curl-devel
+%endif
 BuildRequires:	gcc-c++
 %if %{_centos_ver} >= 7
 BuildRequires:	gflags-devel
@@ -137,7 +144,9 @@ cd cpp/build
 %endif
   -DARROW_PLASMA=ON \
   -DARROW_PYTHON=ON \
-  -DARROW_VERBOSE_THIRDPARTY_BUILD=ON \
+%if %{use_s3}
+  -DARROW_S3=ON \
+%endif
   -DARROW_WITH_BROTLI=ON \
   -DARROW_WITH_BZ2=ON \
   -DARROW_WITH_LZ4=ON \
@@ -789,6 +798,9 @@ Documentation for Apache Parquet GLib.
 %endif
 
 %changelog
+* Mon Oct 12 2020 Krisztián Szűcs <szucs.krisztian@gmail.com> - 2.0.0-1
+- New upstream release.
+
 * Mon Jul 20 2020 Krisztián Szűcs <szucs.krisztian@gmail.com> - 1.0.0-1
 - New upstream release.
 
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-8/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-8/Dockerfile
index 19e947200ff..947b1a59af9 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/centos-8/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-8/Dockerfile
@@ -32,6 +32,7 @@ RUN \
     bzip2-devel \
     clang \
     cmake \
+    curl-devel \
     flex \
     gcc-c++ \
     gflags-devel \
diff --git a/dev/tasks/linux-packages/github.linux.amd64.yml b/dev/tasks/linux-packages/github.linux.amd64.yml
index 58931aac7da..97ec576a31c 100644
--- a/dev/tasks/linux-packages/github.linux.amd64.yml
+++ b/dev/tasks/linux-packages/github.linux.amd64.yml
@@ -27,7 +27,7 @@ on:
 jobs:
   package:
     name: Package
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - name: Set up Ruby
         uses: actions/setup-ruby@master
@@ -74,7 +74,9 @@ jobs:
       # The tag upload took 43 minutes because of this scan, so use an
       # alternative upload script.
       - name: Set up Python
-        uses: actions/setup-python@master
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
       - name: Set up Crossbow
         run: |
           pip install \
diff --git a/dev/tasks/linux-packages/package-task.rb b/dev/tasks/linux-packages/package-task.rb
index 1762e9fd39b..7fa7af10b3e 100644
--- a/dev/tasks/linux-packages/package-task.rb
+++ b/dev/tasks/linux-packages/package-task.rb
@@ -54,7 +54,7 @@ def initialize(package, version, release_time, options={})
       @rpm_version = @version
       @rpm_release = "1"
     end
-    @deb_release = "1"
+    @deb_release = ENV["DEB_RELEASE"] || "1"
   end
 
   def define
diff --git a/dev/tasks/nuget-packages/github.linux.yml b/dev/tasks/nuget-packages/github.linux.yml
index c8143e432c1..95c4464355e 100644
--- a/dev/tasks/nuget-packages/github.linux.yml
+++ b/dev/tasks/nuget-packages/github.linux.yml
@@ -40,7 +40,9 @@ jobs:
           git -C arrow checkout FETCH_HEAD
           git -C arrow submodule update --init --recursive
       - name: Setup Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
       - name: Setup Archery
         run: pip install -e arrow/dev/archery[docker]
       - name: Prepare version
diff --git a/dev/tasks/python-wheels/osx-build.sh b/dev/tasks/python-wheels/osx-build.sh
index 91292905303..e029932ecf6 100755
--- a/dev/tasks/python-wheels/osx-build.sh
+++ b/dev/tasks/python-wheels/osx-build.sh
@@ -63,17 +63,19 @@ function build_wheel {
           -DARROW_BUILD_TESTS=OFF \
           -DARROW_DATASET=ON \
           -DARROW_DEPENDENCY_SOURCE=BUNDLED \
-          -DARROW_HDFS=ON \
           -DARROW_FLIGHT=ON \
           -DARROW_GANDIVA=OFF \
           -DARROW_GRPC_USE_SHARED=OFF \
+          -DARROW_HDFS=ON \
           -DARROW_JEMALLOC=ON \
+          -DARROW_OPENSSL_USE_SHARED=OFF \
           -DARROW_ORC=OFF \
           -DARROW_PARQUET=ON \
           -DARROW_PLASMA=ON \
           -DARROW_PROTOBUF_USE_SHARED=OFF \
           -DARROW_PYTHON=ON \
           -DARROW_RPATH_ORIGIN=ON \
+          -DARROW_S3=${ARROW_S3} \
           -DARROW_VERBOSE_THIRDPARTY_BUILD=ON \
           -DARROW_WITH_BROTLI=ON \
           -DARROW_WITH_BZ2=ON \
@@ -84,10 +86,7 @@ function build_wheel {
           -DBOOST_SOURCE=SYSTEM \
           -DCMAKE_BUILD_TYPE=Release \
           -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
-          -DgRPC_SOURCE=SYSTEM \
-          -DLLVM_SOURCE=SYSTEM \
           -DMAKE=make \
-          -DOPENSSL_USE_STATIC_LIBS=ON \
           -DProtobuf_SOURCE=SYSTEM \
           ..
     make -j$(sysctl -n hw.logicalcpu)
@@ -102,18 +101,19 @@ function build_wheel {
     unset ARROW_HOME
     unset PARQUET_HOME
 
+    export PYARROW_BUILD_TYPE='release'
+    export PYARROW_BUNDLE_ARROW_CPP=1
+    export PYARROW_INSTALL_TESTS=1
     export PYARROW_WITH_DATASET=1
     export PYARROW_WITH_FLIGHT=1
+    export PYARROW_WITH_GANDIVA=0
     export PYARROW_WITH_HDFS=1
-    export PYARROW_WITH_PLASMA=1
-    export PYARROW_WITH_PARQUET=1
-    export PYARROW_WITH_ORC=0
     export PYARROW_WITH_JEMALLOC=1
+    export PYARROW_WITH_ORC=0
+    export PYARROW_WITH_PARQUET=1
     export PYARROW_WITH_PLASMA=1
-    export PYARROW_WITH_GANDIVA=0
-    export PYARROW_BUNDLE_ARROW_CPP=1
-    export PYARROW_BUILD_TYPE='release'
-    export PYARROW_INSTALL_TESTS=1
+    export PYARROW_WITH_PLASMA=1
+    export PYARROW_WITH_S3=${ARROW_S3}
     export SETUPTOOLS_SCM_PRETEND_VERSION=$PYARROW_VERSION
     pushd python
     python setup.py build_ext bdist_wheel
@@ -160,4 +160,7 @@ import pyarrow._hdfs
 import pyarrow.dataset
 import pyarrow.flight
 "
+    if [ "$ARROW_S3" = "ON" ]; then
+        python -c "import pyarrow._s3fs"
+    fi
 }
diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml
index f2a62b9443a..99c667a6450 100644
--- a/dev/tasks/python-wheels/travis.osx.yml
+++ b/dev/tasks/python-wheels/travis.osx.yml
@@ -22,30 +22,19 @@ addons:
   homebrew:
     packages:
       - automake
+      - bison
       - boost
-      - brotli
-      - c-ares
       - ccache
       - cmake
       - flatbuffers
+      - flex
       - git
-      - glog
-      - grpc
-      - lz4
-      - ninja
-      - numpy
       - openssl@1.1
       - protobuf
       - python@3.8
-      - rapidjson
-      - re2
-      - snappy
       - thrift
       - wget
-      - zstd
-      - bison
-      - flex
-    update: true
+    update: false
 
 # don't build twice
 if: tag IS blank
@@ -57,12 +46,12 @@ env:
     - PYARROW_VERSION={{ arrow.no_rc_version }}
     - PYARROW_BUILD_VERBOSE=1
     - MB_PYTHON_VERSION={{ python_version }}
-    - MACOSX_DEPLOYMENT_TARGET="10.9"
-    - HOMEBREW_NO_AUTO_UPDATE=1
+    - MACOSX_DEPLOYMENT_TARGET={{ macos_deployment_target }}
+    - ARROW_S3={{ arrow_s3 }}
 
 before_install:
   - git clone https://github.com/matthew-brett/multibuild
-  - git -C multibuild checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa
+  - git -C multibuild checkout 8882150df6529658700b66bec124dfb77eefca26
 
   - git clone --no-checkout {{ arrow.remote }} arrow
   - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index a064a420a4e..9acf6b239db 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -133,11 +133,24 @@ tasks:
     ci: azure
     template: conda-recipes/azure.clean.yml
 
+  # Important notes on the conda setup here:
+  #
+  # * On conda-forge the `pyarrow` and `arrow-cpp` packages are built in
+  #   the same feedstock as the dependency matrix is the same for them as
+  #   Python and the OS are the main dimension. The R package `r-arrow` is
+  #   an independent feedstock as it doesn't have the Python but the
+  #   R dimension. To limit the number of CI jobs, we are building `r-arrow`
+  #   for R 3.6 with the Python 3.6 jobs and for R 4.0 with the Python 3.7 jobs.
+  # * The files in `dev/tasks/conda-recipes/.ci_support/` are automatically
+  #   generated and to be synced regularly from the feedstock. We have no way
+  #   yet to generate them inside the arrow repository automatically.
+
   conda-linux-gcc-py36-cpu:
     ci: azure
     template: conda-recipes/azure.linux.yml
     params:
       config: linux_cuda_compiler_versionNonepython3.6.____cpython
+      r_config: linux_64_r_base3.6
     artifacts:
       - arrow-cpp-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2
       - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2
@@ -147,6 +160,7 @@ tasks:
     template: conda-recipes/azure.linux.yml
     params:
       config: linux_cuda_compiler_versionNonepython3.7.____cpython
+      r_config: linux_64_r_base4.0
     artifacts:
       - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2
       - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2
@@ -187,6 +201,33 @@ tasks:
       - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.tar.bz2
       - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.tar.bz2
 
+  conda-linux-gcc-py36-aarch64:
+    ci: drone
+    template: conda-recipes/drone.yml
+    params:
+      config: linux_aarch64_python3.6.____cpython
+    artifacts:
+      - arrow-cpp-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2
+      - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2
+
+  conda-linux-gcc-py37-aarch64:
+    ci: drone
+    template: conda-recipes/drone.yml
+    params:
+      config: linux_aarch64_python3.7.____cpython
+    artifacts:
+      - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2
+      - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2
+
+  conda-linux-gcc-py38-aarch64:
+    ci: drone
+    template: conda-recipes/drone.yml
+    params:
+      config: linux_aarch64_python3.8.____cpython
+    artifacts:
+      - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2
+      - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2
+
   ############################## Conda OSX ####################################
 
   conda-osx-clang-py36:
@@ -194,6 +235,7 @@ tasks:
     template: conda-recipes/azure.osx.yml
     params:
       config: osx_python3.6.____cpython
+      r_config: osx_64_r_base3.6
     artifacts:
       - arrow-cpp-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2
       - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2
@@ -203,6 +245,7 @@ tasks:
     template: conda-recipes/azure.osx.yml
     params:
       config: osx_python3.7.____cpython
+      r_config: osx_64_r_base4.0
     artifacts:
       - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2
       - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2
@@ -223,6 +266,7 @@ tasks:
     template: conda-recipes/azure.win.yml
     params:
       config: win_python3.6.____cpython
+      r_config: win_64_r_base3.6
     artifacts:
       - arrow-cpp-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2
       - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2
@@ -232,6 +276,7 @@ tasks:
     template: conda-recipes/azure.win.yml
     params:
       config: win_python3.7.____cpython
+      r_config: win_64_r_base4.0
     artifacts:
       - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2
       - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2
@@ -417,38 +462,88 @@ tasks:
 
   ############################## Wheel OSX ####################################
 
-  wheel-osx-cp35m:
+  wheel-osx-mavericks-cp35m:
     ci: travis
     template: python-wheels/travis.osx.yml
     params:
       python_version: 3.5
+      macos_deployment_target: 10.9
+      arrow_s3: "OFF"
     artifacts:
       - pyarrow-{no_rc_version}-cp35-cp35m-macosx_10_9_intel.whl
 
-  wheel-osx-cp36m:
+  wheel-osx-mavericks-cp36m:
     ci: travis
     template: python-wheels/travis.osx.yml
     params:
       python_version: 3.6
+      macos_deployment_target: 10.9
+      arrow_s3: "OFF"
     artifacts:
-      - pyarrow-{no_rc_version}-cp36-cp36m-macosx_10_9_intel.whl
+      - pyarrow-{no_rc_version}-cp36-cp36m-macosx_10_9_x86_64.whl
 
-  wheel-osx-cp37m:
+  wheel-osx-mavericks-cp37m:
     ci: travis
     template: python-wheels/travis.osx.yml
     params:
       python_version: 3.7
+      macos_deployment_target: 10.9
+      arrow_s3: "OFF"
     artifacts:
-      - pyarrow-{no_rc_version}-cp37-cp37m-macosx_10_9_intel.whl
+      - pyarrow-{no_rc_version}-cp37-cp37m-macosx_10_9_x86_64.whl
 
-  wheel-osx-cp38:
+  wheel-osx-mavericks-cp38:
     ci: travis
     template: python-wheels/travis.osx.yml
     params:
       python_version: 3.8
+      macos_deployment_target: 10.9
+      arrow_s3: "OFF"
     artifacts:
       - pyarrow-{no_rc_version}-cp38-cp38-macosx_10_9_x86_64.whl
 
+  # enable S3 support from macOS 10.13 so we don't need to bundle curl, crypt and ssl
+
+  wheel-osx-high-sierra-cp35m:
+    ci: travis
+    template: python-wheels/travis.osx.yml
+    params:
+      python_version: 3.5
+      macos_deployment_target: 10.13
+      arrow_s3: "ON"
+    artifacts:
+      - pyarrow-{no_rc_version}-cp35-cp35m-macosx_10_13_intel.whl
+
+  wheel-osx-high-sierra-cp36m:
+    ci: travis
+    template: python-wheels/travis.osx.yml
+    params:
+      python_version: 3.6
+      macos_deployment_target: 10.13
+      arrow_s3: "ON"
+    artifacts:
+      - pyarrow-{no_rc_version}-cp36-cp36m-macosx_10_13_x86_64.whl
+
+  wheel-osx-high-sierra-cp37m:
+    ci: travis
+    template: python-wheels/travis.osx.yml
+    params:
+      python_version: 3.7
+      macos_deployment_target: 10.13
+      arrow_s3: "ON"
+    artifacts:
+      - pyarrow-{no_rc_version}-cp37-cp37m-macosx_10_13_x86_64.whl
+
+  wheel-osx-high-sierra-cp38:
+    ci: travis
+    template: python-wheels/travis.osx.yml
+    params:
+      python_version: 3.8
+      macos_deployment_target: 10.13
+      arrow_s3: "ON"
+    artifacts:
+      - pyarrow-{no_rc_version}-cp38-cp38-macosx_10_13_x86_64.whl
+
   ############################## Wheel Windows ################################
 
   wheel-win-cp35m:
@@ -521,50 +616,50 @@ tasks:
       - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-glib200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-glib200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-glib300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-glib300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-cuda-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-cuda-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-cuda-glib200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-cuda-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-cuda200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-cuda200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-cuda-glib300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-cuda-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-cuda300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-cuda300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-dataset-glib200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-dataset-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-dataset200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-dataset200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-dataset-glib300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-dataset-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-dataset300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-dataset300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-flight200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-flight200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-flight300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-flight300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-python200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-python200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-python300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-python300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow300_{no_rc_version}-1_[a-z0-9]+.deb
       - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libgandiva-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libgandiva-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libgandiva-glib200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libgandiva-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libgandiva200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libgandiva200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libgandiva-glib300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libgandiva-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libgandiva300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libgandiva300_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libparquet-glib200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libparquet-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libparquet200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libparquet200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libparquet-glib300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libparquet-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libparquet300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libparquet300_{no_rc_version}-1_[a-z0-9]+.deb
       - libplasma-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libplasma-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libplasma-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libplasma-glib200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libplasma-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libplasma200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libplasma200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libplasma-glib300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libplasma-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libplasma300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libplasma300_{no_rc_version}-1_[a-z0-9]+.deb
       - plasma-store-server-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
       - plasma-store-server_{no_rc_version}-1_[a-z0-9]+.deb
 
@@ -590,29 +685,29 @@ tasks:
       - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-dataset-glib200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-dataset-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-dataset200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-dataset200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-dataset-glib300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-dataset-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-dataset300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-dataset300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-flight200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-flight200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-flight300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-flight300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-glib200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-glib200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-glib300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-glib300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-python200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-python200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-python300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-python300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow300_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libparquet-glib200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libparquet-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libparquet200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libparquet200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libparquet-glib300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libparquet-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libparquet300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libparquet300_{no_rc_version}-1_[a-z0-9]+.deb
 
   debian-buster-amd64:
     ci: github
@@ -642,50 +737,50 @@ tasks:
       - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-glib200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-glib200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-glib300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-glib300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-cuda-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-cuda-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-cuda-glib200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-cuda-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-cuda200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-cuda200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-cuda-glib300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-cuda-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-cuda300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-cuda300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-dataset-glib200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-dataset-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-dataset200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-dataset200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-dataset-glib300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-dataset-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-dataset300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-dataset300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-flight200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-flight200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-flight300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-flight300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-python200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-python200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-python300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-python300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow300_{no_rc_version}-1_[a-z0-9]+.deb
       - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libgandiva-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libgandiva-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libgandiva-glib200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libgandiva-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libgandiva200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libgandiva200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libgandiva-glib300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libgandiva-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libgandiva300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libgandiva300_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libparquet-glib200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libparquet-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libparquet200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libparquet200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libparquet-glib300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libparquet-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libparquet300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libparquet300_{no_rc_version}-1_[a-z0-9]+.deb
       - libplasma-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libplasma-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libplasma-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libplasma-glib200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libplasma-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libplasma200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libplasma200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libplasma-glib300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libplasma-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libplasma300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libplasma300_{no_rc_version}-1_[a-z0-9]+.deb
       - plasma-store-server-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
       - plasma-store-server_{no_rc_version}-1_[a-z0-9]+.deb
 
@@ -711,29 +806,29 @@ tasks:
       - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-dataset-glib200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-dataset-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-dataset200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-dataset200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-dataset-glib300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-dataset-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-dataset300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-dataset300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-flight200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-flight200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-flight300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-flight300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-glib200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-glib200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-glib300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-glib300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-python200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow-python200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libarrow200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-python300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-python300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow300_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libparquet-glib200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libparquet-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libparquet200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
-      - libparquet200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libparquet-glib300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libparquet-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libparquet300-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libparquet300_{no_rc_version}-1_[a-z0-9]+.deb
 
   ubuntu-xenial-amd64:
     ci: github
@@ -763,34 +858,34 @@ tasks:
       - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-glib200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-glib300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-cuda-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-cuda-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-cuda-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-cuda200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-cuda-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-cuda300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-dataset-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-dataset200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-dataset-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-dataset300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-python200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-python300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow300_{no_rc_version}-1_[a-z0-9]+.deb
       - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libgandiva-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libgandiva-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libgandiva-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libgandiva200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libgandiva-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libgandiva300_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libparquet-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libparquet200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libparquet-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libparquet300_{no_rc_version}-1_[a-z0-9]+.deb
       - libplasma-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libplasma-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libplasma-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libplasma-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libplasma200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libplasma-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libplasma300_{no_rc_version}-1_[a-z0-9]+.deb
       - plasma-store-server_{no_rc_version}-1_[a-z0-9]+.deb
 
   ubuntu-xenial-arm64:
@@ -814,20 +909,20 @@ tasks:
       - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-glib200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-glib300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-dataset-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-dataset200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-dataset-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-dataset300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-python200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-python300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow300_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libparquet-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libparquet200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libparquet-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libparquet300_{no_rc_version}-1_[a-z0-9]+.deb
 
   ubuntu-bionic-amd64:
     ci: github
@@ -857,36 +952,36 @@ tasks:
       - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-glib200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-glib300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-cuda-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-cuda-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-cuda-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-cuda200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-cuda-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-cuda300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-dataset-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-dataset200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-dataset-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-dataset300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-flight200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-flight300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-python200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-python300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow300_{no_rc_version}-1_[a-z0-9]+.deb
       - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libgandiva-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libgandiva-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libgandiva-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libgandiva200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libgandiva-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libgandiva300_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libparquet-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libparquet200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libparquet-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libparquet300_{no_rc_version}-1_[a-z0-9]+.deb
       - libplasma-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libplasma-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libplasma-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libplasma-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libplasma200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libplasma-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libplasma300_{no_rc_version}-1_[a-z0-9]+.deb
       - plasma-store-server-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
       - plasma-store-server_{no_rc_version}-1_[a-z0-9]+.deb
 
@@ -911,22 +1006,22 @@ tasks:
       - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-glib200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-glib300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-dataset-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-dataset200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-dataset-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-dataset300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-flight200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-flight300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-python200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-python300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow300_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libparquet-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libparquet200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libparquet-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libparquet300_{no_rc_version}-1_[a-z0-9]+.deb
 
   ubuntu-focal-amd64:
     ci: github
@@ -956,36 +1051,36 @@ tasks:
       - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-glib200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-glib300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-cuda-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-cuda-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-cuda-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-cuda200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-cuda-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-cuda300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-dataset-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-dataset200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-dataset-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-dataset300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-flight200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-flight300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-python200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-python300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow300_{no_rc_version}-1_[a-z0-9]+.deb
       - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libgandiva-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libgandiva-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libgandiva-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libgandiva200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libgandiva-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libgandiva300_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libparquet-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libparquet200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libparquet-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libparquet300_{no_rc_version}-1_[a-z0-9]+.deb
       - libplasma-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libplasma-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libplasma-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libplasma-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libplasma200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libplasma-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libplasma300_{no_rc_version}-1_[a-z0-9]+.deb
       - plasma-store-server-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
       - plasma-store-server_{no_rc_version}-1_[a-z0-9]+.deb
 
@@ -1010,22 +1105,22 @@ tasks:
       - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-glib200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-glib300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-dataset-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-dataset200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-dataset-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-dataset300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-flight200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-flight300_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow-python200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libarrow200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-python300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow300_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
-      - libparquet-glib200_{no_rc_version}-1_[a-z0-9]+.deb
-      - libparquet200_{no_rc_version}-1_[a-z0-9]+.deb
+      - libparquet-glib300_{no_rc_version}-1_[a-z0-9]+.deb
+      - libparquet300_{no_rc_version}-1_[a-z0-9]+.deb
 
   centos-6-amd64:
     ci: github
@@ -1281,179 +1376,208 @@ tasks:
 
   verify-rc-binaries-binary:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.linux.yml
     params:
-      os: "ubuntu"
+      env:
+        TEST_DEFAULT: 0
+        TEST_BINARY: 1
       artifact: "binaries"
-      flag: "TEST_BINARY=1"
 
   verify-rc-binaries-apt:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.linux.yml
     params:
-      os: "ubuntu"
+      env:
+        TEST_DEFAULT: 0
+        TEST_APT: 1
       artifact: "binaries"
-      flag: "TEST_APT=1"
 
   verify-rc-binaries-yum:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.linux.yml
     params:
-      os: "ubuntu"
+      env:
+        TEST_DEFAULT: 0
+        TEST_YUM: 1
       artifact: "binaries"
-      flag: "TEST_YUM=1"
 
   verify-rc-wheels-linux:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.linux.yml
     params:
-      os: "ubuntu"
+      env:
+        TEST_DEFAULT: 0
       artifact: "wheels"
-      flag: ""
 
   verify-rc-wheels-macos:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.osx.yml
     params:
-      os: "macOS"
+      env:
+        TEST_DEFAULT: 0
       artifact: "wheels"
-      flag: ""
 
   verify-rc-source-macos-java:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.osx.yml
     params:
-      os: "macOS"
+      env:
+        TEST_DEFAULT: 0
+        TEST_JAVA: 1
       artifact: "source"
-      flag: "TEST_JAVA=1"
 
   verify-rc-source-macos-csharp:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.osx.yml
     params:
-      os: "macOS"
+      env:
+        TEST_DEFAULT: 0
+        TEST_CSHARP: 1
       artifact: "source"
-      flag: "TEST_CSHARP=1"
 
   verify-rc-source-macos-ruby:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.osx.yml
     params:
-      os: "macOS"
+      env:
+        TEST_DEFAULT: 0
+        TEST_RUBY: 1
       artifact: "source"
-      flag: "TEST_RUBY=1"
 
   verify-rc-source-macos-python:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.osx.yml
     params:
-      os: "macOS"
+      env:
+        TEST_DEFAULT: 0
+        TEST_PYTHON: 1
+        # https://stackoverflow.com/questions/56083725/macos-build-issues-lstdc-not-found-while-building-python-package
+        MACOSX_DEPLOYMENT_TARGET: "10.15"
       artifact: "source"
-      flag: "TEST_PYTHON=1"
 
   verify-rc-source-macos-js:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.osx.yml
     params:
-      os: "macOS"
+      env:
+        INSTALL_NODE: 0
+        TEST_DEFAULT: 0
+        TEST_JS: 1
       artifact: "source"
-      flag: "TEST_JS=1"
 
   verify-rc-source-macos-go:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.osx.yml
     params:
-      os: "macOS"
+      env:
+        TEST_DEFAULT: 0
+        TEST_GO: 1
       artifact: "source"
-      flag: "TEST_GO=1"
 
   verify-rc-source-macos-rust:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.osx.yml
     params:
-      os: "macOS"
+      env:
+        TEST_DEFAULT: 0
+        TEST_RUST: 1
       artifact: "source"
-      flag: "TEST_RUST=1"
 
   verify-rc-source-macos-integration:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.osx.yml
     params:
-      os: "macOS"
+      env:
+        INSTALL_NODE: 0
+        TEST_DEFAULT: 0
+        TEST_INTEGRATION: 1
       artifact: "source"
-      flag: "TEST_INTEGRATION=1"
 
   verify-rc-source-linux-java:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.linux.yml
     params:
-      os: "ubuntu"
+      env:
+        TEST_DEFAULT: 0
+        TEST_JAVA: 1
       artifact: "source"
-      flag: "TEST_JAVA=1"
 
   verify-rc-source-linux-csharp:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.linux.yml
     params:
-      os: "ubuntu"
+      env:
+        TEST_DEFAULT: 0
+        TEST_CSHARP: 1
       artifact: "source"
-      flag: "TEST_CSHARP=1"
 
   verify-rc-source-linux-ruby:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.linux.yml
     params:
-      os: "ubuntu"
+      env:
+        TEST_DEFAULT: 0
+        TEST_RUBY: 1
       artifact: "source"
-      flag: "TEST_RUBY=1"
 
   verify-rc-source-linux-python:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.linux.yml
     params:
-      os: "ubuntu"
+      env:
+        TEST_DEFAULT: 0
+        TEST_PYTHON: 1
       artifact: "source"
-      flag: "TEST_PYTHON=1"
 
   verify-rc-source-linux-js:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.linux.yml
     params:
-      os: "ubuntu"
+      env:
+        INSTALL_NODE: 0
+        TEST_DEFAULT: 0
+        TEST_JS: 1
       artifact: "source"
-      flag: "TEST_JS=1"
 
   verify-rc-source-linux-go:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.linux.yml
     params:
-      os: "ubuntu"
+      env:
+        TEST_DEFAULT: 0
+        TEST_GO: 1
       artifact: "source"
-      flag: "TEST_GO=1"
 
   verify-rc-source-linux-rust:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.linux.yml
     params:
-      os: "ubuntu"
+      env:
+        TEST_DEFAULT: 0
+        TEST_RUST: 1
       artifact: "source"
-      flag: "TEST_RUST=1"
 
   verify-rc-source-linux-integration:
     ci: github
-    template: verify-rc/github.nix.yml
+    template: verify-rc/github.linux.yml
     params:
-      os: "ubuntu"
+      env:
+        INSTALL_NODE: 0
+        TEST_DEFAULT: 0
+        TEST_INTEGRATION: 1
       artifact: "source"
-      flag: "TEST_INTEGRATION=1"
 
   verify-rc-source-windows:
     ci: github
-    template: verify-rc/github.windows.source.yml
+    template: verify-rc/github.win.yml
+    params:
+      script: "verify-release-candidate.bat"
 
   verify-rc-wheels-windows:
     ci: github
-    template: verify-rc/github.windows.wheels.yml
+    template: verify-rc/github.win.yml
+    params:
+      script: "verify-release-candidate-wheels.bat"
 
   ############################## Docker tests #################################
 
@@ -1493,12 +1617,12 @@ tasks:
         UBUNTU: 18.04
       run: ubuntu-cpp
 
-  test-fedora-32-cpp:
+  test-fedora-33-cpp:
     ci: circle
     template: docker-tests/circle.linux.yml
     params:
       env:
-        FEDORA: 32
+        FEDORA: 33
       run: fedora-cpp
 
   test-ubuntu-18.04-cpp-release:
@@ -1585,6 +1709,18 @@ tasks:
         PYTHON: 3.8
       run: conda-python
 
+  test-conda-python-3.8-hypothesis:
+    ci: github
+    template: docker-tests/github.linux.yml
+    params:
+      env:
+        HYPOTHESIS_PROFILE: ci
+        PYARROW_TEST_HYPOTHESIS: ON
+        PYTHON: 3.8
+        # limit to execute hypothesis tests only
+        PYTEST_ARGS: "-m hypothesis"
+      run: conda-python-pandas
+
   test-debian-10-python-3:
     ci: azure
     template: docker-tests/azure.linux.yml
@@ -1601,12 +1737,12 @@ tasks:
         UBUNTU: 18.04
       run: ubuntu-python
 
-  test-fedora-32-python-3:
+  test-fedora-33-python-3:
     ci: azure
     template: docker-tests/azure.linux.yml
     params:
       env:
-        FEDORA: 32
+        FEDORA: 33
       run: fedora-python
 
   test-r-linux-as-cran:
@@ -1678,14 +1814,6 @@ tasks:
       r_tag: 3.6-opensuse42
       not_cran: "TRUE"
 
-  test-conda-r-4.0:
-    ci: github
-    template: docker-tests/github.linux.yml
-    params:
-      env:
-        R: 4.0
-      run: conda-r
-
   test-ubuntu-18.04-r-sanitizer:
     ci: azure
     template: docker-tests/azure.linux.yml
diff --git a/dev/tasks/verify-rc/github.linux.yml b/dev/tasks/verify-rc/github.linux.yml
new file mode 100644
index 00000000000..49d937ac6fa
--- /dev/null
+++ b/dev/tasks/verify-rc/github.linux.yml
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE: must set "Crossbow" as name to have the badge links working in the
+# github comment reports!
+name: Crossbow
+
+on:
+  push:
+    branches:
+      - "*-github-*"
+
+jobs:
+  verify:
+    name: "Verify release candidate Ubuntu {{ artifact }}"
+    runs-on: ubuntu-latest
+    {%- if env is defined %}
+    env:
+    {%- for key, value in env.items() %}
+      {{ key }}: {{ value }}
+    {%- endfor %}
+    {%- endif %}
+    steps:
+      - name: Checkout Arrow
+        run: |
+          git clone --no-checkout {{ arrow.remote }} arrow
+          git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
+          git -C arrow checkout FETCH_HEAD
+          git -C arrow submodule update --init --recursive
+      - name: Fetch Submodules and Tags
+        shell: bash
+        run: cd arrow && ci/scripts/util_checkout.sh
+      - name: Install System Dependencies
+        run: |
+          # TODO: don't require removing newer llvms
+          sudo apt-get --purge remove -y llvm-9 clang-9
+          sudo apt-get install -y \
+            wget curl libboost-all-dev jq \
+            autoconf-archive gtk-doc-tools libgirepository1.0-dev flex bison
+
+          if [ "$TEST_JAVA" = "1" ]; then
+            # Maven
+            MAVEN_VERSION=3.6.3
+            wget https://downloads.apache.org/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.zip
+            unzip apache-maven-$MAVEN_VERSION-bin.zip
+            mkdir -p $HOME/java
+            mv apache-maven-$MAVEN_VERSION $HOME/java
+            export PATH=$HOME/java/apache-maven-$MAVEN_VERSION/bin:$PATH
+          fi
+
+          if [ "$TEST_RUBY" = "1" ]; then
+            ruby --version
+            sudo gem install bundler
+          fi
+      - uses: actions/setup-node@v2-beta
+        with:
+          node-version: '14'
+      - name: Run verification
+        shell: bash
+        run: |
+          arrow/dev/release/verify-release-candidate.sh \
+            {{ artifact }} \
+            {{ release|default("1.0.0") }} {{ rc|default("0") }}
diff --git a/dev/tasks/verify-rc/github.nix.yml b/dev/tasks/verify-rc/github.nix.yml
deleted file mode 100644
index 8482cdc97ca..00000000000
--- a/dev/tasks/verify-rc/github.nix.yml
+++ /dev/null
@@ -1,82 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# NOTE: must set "Crossbow" as name to have the badge links working in the
-# github comment reports!
-name: Crossbow
-
-on:
-  push:
-    branches:
-      - "*-github-*"
-
-jobs:
-  verify:
-    name: "Verify release candidate {{ os }} {{ artifact }} {{ flag }}"
-    runs-on: {{ os }}-latest
-    steps:
-      - name: Checkout Arrow
-        run: |
-          git clone --no-checkout {{ arrow.remote }} arrow
-          git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
-          git -C arrow checkout FETCH_HEAD
-          git -C arrow submodule update --init --recursive
-      - name: Free Up Disk Space
-        shell: bash
-        run: arrow/ci/scripts/util_cleanup.sh
-      - name: Fetch Submodules and Tags
-        shell: bash
-        run: cd arrow && ci/scripts/util_checkout.sh
-      - name: Run verification
-        shell: bash
-        env:
-          INSTALL_NODE: 0
-        run: |
-          set -e
-
-          {{ flag }}
-          if [ $(uname) = "Darwin" ]; then
-            brew update
-            brew bundle --file=arrow/cpp/Brewfile
-            brew bundle --file=arrow/c_glib/Brewfile
-            if [ "$TEST_PYTHON" = "1" ]; then
-              # https://stackoverflow.com/questions/56083725/macos-build-issues-lstdc-not-found-while-building-python-package
-              export MACOSX_DEPLOYMENT_TARGET=10.9
-            fi
-          else
-            # TODO: don't require removing newer llvms
-            sudo apt-get --purge remove -y llvm-9 clang-9
-            sudo apt-get install -y \
-              wget curl libboost-all-dev jq \
-              autoconf-archive gtk-doc-tools libgirepository1.0-dev flex bison
-            if [ "$TEST_JAVA" = "1" ]; then
-              # Maven
-              MAVEN_VERSION=3.6.3
-              wget https://downloads.apache.org/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.zip
-              unzip apache-maven-$MAVEN_VERSION-bin.zip
-              mkdir -p $HOME/java
-              mv apache-maven-$MAVEN_VERSION $HOME/java
-              export PATH=$HOME/java/apache-maven-$MAVEN_VERSION/bin:$PATH
-            fi
-            if [ "$TEST_RUBY" = "1" ]; then
-              ruby --version
-              sudo gem install bundler
-            fi
-          fi
-          # TODO: put version and rc number in some separate file?
-          # If you edit the versions, be sure to edit the other workflow files in this directory too
-          TEST_DEFAULT=0 {{ flag }} arrow/dev/release/verify-release-candidate.sh {{ artifact }} 0.17.0 0
diff --git a/dev/tasks/verify-rc/github.windows.wheels.yml b/dev/tasks/verify-rc/github.osx.yml
similarity index 67%
rename from dev/tasks/verify-rc/github.windows.wheels.yml
rename to dev/tasks/verify-rc/github.osx.yml
index 082c2aa04ca..a0f6fc4af4e 100644
--- a/dev/tasks/verify-rc/github.windows.wheels.yml
+++ b/dev/tasks/verify-rc/github.osx.yml
@@ -26,8 +26,14 @@ on:
 
 jobs:
   verify:
-    name: "Verify release candidate Windows wheels"
-    runs-on: windows-latest
+    name: "Verify release candidate macOS {{ artifact }}"
+    runs-on: macos-latest
+    {%- if env is defined %}
+    env:
+    {%- for key, value in env.items() %}
+      {{ key }}: {{ value }}
+    {%- endfor %}
+    {%- endif %}
     steps:
       - name: Checkout Arrow
         run: |
@@ -38,11 +44,18 @@ jobs:
       - name: Fetch Submodules and Tags
         shell: bash
         run: cd arrow && ci/scripts/util_checkout.sh
-      - uses: s-weigand/setup-conda@v1
+      - name: Install System Dependencies
+        shell: bash
+        run: |
+          brew update
+          brew bundle --file=arrow/cpp/Brewfile
+          brew bundle --file=arrow/c_glib/Brewfile
+      - uses: actions/setup-node@v2-beta
+        with:
+          node-version: '14'
       - name: Run verification
-        shell: cmd
+        shell: bash
         run: |
-          choco install wget
-          cd arrow
-          # If you edit the versions, be sure to edit the other workflow files in this directory too
-          dev/release/verify-release-candidate-wheels.bat 0.17.0 0
+          arrow/dev/release/verify-release-candidate.sh \
+            {{ artifact }} \
+            {{ release|default("1.0.0") }} {{ rc|default("0") }}
diff --git a/dev/tasks/verify-rc/github.windows.source.yml b/dev/tasks/verify-rc/github.win.yml
similarity index 83%
rename from dev/tasks/verify-rc/github.windows.source.yml
rename to dev/tasks/verify-rc/github.win.yml
index d236bb0a2a5..fbe0ee26812 100644
--- a/dev/tasks/verify-rc/github.windows.source.yml
+++ b/dev/tasks/verify-rc/github.win.yml
@@ -27,7 +27,13 @@ on:
 jobs:
   verify:
     name: "Verify release candidate Windows source"
-    runs-on: windows-latest
+    runs-on: windows-2016
+    {%- if env is defined %}
+    env:
+    {%- for key, value in env.items() %}
+      {{ key }}: {{ value }}
+    {%- endfor %}
+    {%- endif %}
     steps:
       - name: Checkout Arrow
         run: |
@@ -39,11 +45,12 @@ jobs:
         shell: bash
         run: cd arrow && ci/scripts/util_checkout.sh
       - uses: s-weigand/setup-conda@v1
-      - name: Run verification
-        shell: cmd
+      - name: Install System Dependencies
         run: |
           choco install boost-msvc-14.1
           choco install wget
+      - name: Run verification
+        shell: cmd
+        run: |
           cd arrow
-          # If you edit the versions, be sure to edit the other workflow files in this directory too
-          dev/release/verify-release-candidate.bat 0.17.0 0
+          dev/release/{{ script }} {{ release|default("1.0.0") }} {{ rc|default("0") }}
diff --git a/docker-compose.yml b/docker-compose.yml
index 86cffad8128..195fc0104d2 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -94,7 +94,6 @@ x-hierarchy:
       - conda-python-turbodbc
       - conda-python-kartothek
       - conda-python-spark
-    - conda-r
   - debian-cpp:
     - debian-c-glib:
       - debian-ruby
@@ -197,6 +196,7 @@ services:
       ARROW_FLIGHT: "OFF"
       ARROW_GANDIVA: "OFF"
       ARROW_JEMALLOC: "OFF"
+      ARROW_RUNTIME_SIMD_LEVEL: "AVX2"  # AVX512 not supported by Valgrind (ARROW-9851)
       ARROW_S3: "OFF"
       ARROW_TEST_MEMCHECK: "ON"
       ARROW_USE_LD_GOLD: "ON"
@@ -247,6 +247,7 @@ services:
       cache_from:
         - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
       args:
+        arch: ${ARCH}
         base: "${ARCH}/ubuntu:${UBUNTU}"
         clang_tools: ${CLANG_TOOLS}
         llvm: ${LLVM}
@@ -276,6 +277,7 @@ services:
       cache_from:
         - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-cpp
       args:
+        arch: ${ARCH}
         base: nvidia/cuda:${CUDA}-devel-ubuntu${UBUNTU}
         clang_tools: ${CLANG_TOOLS}
         llvm: ${LLVM}
@@ -329,7 +331,7 @@ services:
     #   docker-compose run --rm fedora-cpp
     # Parameters:
     #   ARCH: amd64, arm64v8, ...
-    #   FEDORA: 32
+    #   FEDORA: 33
     image: ${REPO}:${ARCH}-fedora-${FEDORA}-cpp
     build:
       context: .
@@ -569,7 +571,7 @@ services:
     #   docker-compose run --rm fedora-python
     # Parameters:
     #   ARCH: amd64, arm64v8, ...
-    #   FEDORA: 32
+    #   FEDORA: 33
     image: ${REPO}:${ARCH}-fedora-${FEDORA}-python-3
     build:
       context: .
@@ -821,30 +823,6 @@ services:
 
   ################################## R ########################################
 
-  conda-r:
-    # Usage:
-    #   docker-compose build conda-cpp
-    #   docker-compose build conda-r
-    #   docker-compose run conda-r
-    image: ${REPO}:${ARCH}-conda-r-${R}
-    build:
-      context: .
-      dockerfile: ci/docker/conda-r.dockerfile
-      cache_from:
-        - ${REPO}:${ARCH}-conda-r-${R}
-      args:
-        repo: ${REPO}
-        arch: ${ARCH}
-        r: ${R}
-    shm_size: *shm-size
-    environment:
-      <<: *ccache
-      NOT_CRAN: 'true'
-    volumes: *conda-volumes
-    command:
-      ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
-        /arrow/ci/scripts/r_test.sh /arrow"]
-
   ubuntu-r:
     # Usage:
     #   docker-compose build ubuntu-cpp
@@ -857,6 +835,7 @@ services:
       cache_from:
         - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-r-${R}
       args:
+        arch: ${ARCH}
         r: ${R}
         base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
     shm_size: *shm-size
@@ -887,18 +866,18 @@ services:
         - ${REPO}:r-${R_ORG}-${R_IMAGE}-${R_TAG}
       args:
         base: ${R_ORG}/${R_IMAGE}:${R_TAG}
+        r_dev: ${ARROW_R_DEV}
     shm_size: *shm-size
     environment:
       LIBARROW_DOWNLOAD: "false"
       ARROW_HOME: "/arrow"
+      ARROW_R_DEV: ${ARROW_R_DEV}
       # To test for CRAN release, delete ^^ these two env vars so we download the Apache release
       ARROW_USE_PKG_CONFIG: "false"
     volumes:
       - .:/arrow:delegated
     command: >
-      /bin/bash -c "
-        export ARROW_R_DEV=${ARROW_R_DEV} &&
-        /arrow/ci/scripts/r_test.sh /arrow"
+      /bin/bash -c "/arrow/ci/scripts/r_test.sh /arrow"
 
   ubuntu-r-sanitizer:
     # Only 18.04 and amd64 supported
diff --git a/docs/source/cpp/api.rst b/docs/source/cpp/api.rst
index 59d221012d3..626b388b664 100644
--- a/docs/source/cpp/api.rst
+++ b/docs/source/cpp/api.rst
@@ -29,6 +29,7 @@ API Reference
    api/scalar
    api/builder
    api/table
+   api/c_abi
    api/compute
    api/tensor
    api/utilities
diff --git a/docs/source/cpp/api/c_abi.rst b/docs/source/cpp/api/c_abi.rst
new file mode 100644
index 00000000000..4e451c3eca6
--- /dev/null
+++ b/docs/source/cpp/api/c_abi.rst
@@ -0,0 +1,48 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+============
+C Interfaces
+============
+
+.. seealso::
+   The :ref:`C data interface <c-data-interface>` and
+   :ref:`C stream interface <c-stream-interface>` specifications.
+
+ABI Structures
+==============
+
+.. doxygenstruct:: ArrowSchema
+   :project: arrow_cpp
+
+.. doxygenstruct:: ArrowArray
+   :project: arrow_cpp
+
+.. doxygenstruct:: ArrowArrayStream
+   :project: arrow_cpp
+
+C Data Interface
+================
+
+.. doxygengroup:: c-data-interface
+   :content-only:
+
+C Stream Interface
+==================
+
+.. doxygengroup:: c-stream-interface
+   :content-only:
diff --git a/docs/source/cpp/compute.rst b/docs/source/cpp/compute.rst
index 15eae53fb1d..c2b901ff8a0 100644
--- a/docs/source/cpp/compute.rst
+++ b/docs/source/cpp/compute.rst
@@ -31,8 +31,8 @@ The generic Compute API
 Functions and function registry
 -------------------------------
 
-Functions represent logical compute operations over inputs of possibly
-varying types.  Internally, a function is implemented by one or several
+Functions represent compute operations over inputs of possibly varying 
+types.  Internally, a function is implemented by one or several
 "kernels", depending on the concrete input types (for example, a function
 adding values from two inputs can have different kernels depending on
 whether the inputs are integral or floating-point).
@@ -83,7 +83,7 @@ Some functions accept or require an options structure that determines the
 exact semantics of the function::
 
    MinMaxOptions options;
-   options.null_handling = MinMaxOptions::OUTPUT_NULL;
+   options.null_handling = MinMaxOptions::EMIT_NULL;
 
    std::shared_ptr<arrow::Array> array = ...;
    arrow::Datum min_max_datum;
@@ -101,6 +101,8 @@ exact semantics of the function::
    :doc:`Compute API reference <api/compute>`
 
 
+.. _compute-function-list:
+
 Available functions
 ===================
 
@@ -140,8 +142,12 @@ Aggregations
 +--------------------------+------------+--------------------+-----------------------+--------------------------------------------+
 | mode                     | Unary      | Numeric            | Scalar Struct  (2)    |                                            |
 +--------------------------+------------+--------------------+-----------------------+--------------------------------------------+
+| stddev                   | Unary      | Numeric            | Scalar Float64        | :struct:`VarianceOptions`                  |
++--------------------------+------------+--------------------+-----------------------+--------------------------------------------+
 | sum                      | Unary      | Numeric            | Scalar Numeric (3)    |                                            |
 +--------------------------+------------+--------------------+-----------------------+--------------------------------------------+
+| variance                 | Unary      | Numeric            | Scalar Float64        | :struct:`VarianceOptions`                  |
++--------------------------+------------+--------------------+-----------------------+--------------------------------------------+
 
 Notes:
 
@@ -189,6 +195,10 @@ an ``Invalid`` :class:`Status` when overflow is detected.
 +--------------------------+------------+--------------------+---------------------+
 | add_checked              | Binary     | Numeric            | Numeric             |
 +--------------------------+------------+--------------------+---------------------+
+| divide                   | Binary     | Numeric            | Numeric             |
++--------------------------+------------+--------------------+---------------------+
+| divide_checked           | Binary     | Numeric            | Numeric             |
++--------------------------+------------+--------------------+---------------------+
 | multiply                 | Binary     | Numeric            | Numeric             |
 +--------------------------+------------+--------------------+---------------------+
 | multiply_checked         | Binary     | Numeric            | Numeric             |
@@ -385,6 +395,37 @@ Containment tests
 * \(3) Output is true iff the corresponding input element is equal to one
   of the elements in :member:`SetLookupOptions::value_set`.
 
+
+String splitting
+~~~~~~~~~~~~~~~~
+
+These functions split strings into lists of strings.  All kernels can optionally
+be configured with a ``max_splits`` and a ``reverse`` parameter, where
+``max_splits == -1`` means no limit (the default).  When ``reverse`` is true,
+the splitting is done starting from the end of the string; this is only relevant
+when a positive ``max_splits`` is given.
+
++--------------------------+------------+-------------------------+-------------------+----------------------------------+---------+
+| Function name            | Arity      | Input types             | Output type       | Options class                    | Notes   |
++==========================+============+=========================+===================+==================================+=========+
+| split_pattern            | Unary      | String-like             | List-like         | :struct:`SplitPatternOptions`    | \(1)    |
++--------------------------+------------+-------------------------+-------------------+----------------------------------+---------+
+| utf8_split_whitespace    | Unary      | String-like             | List-like         | :struct:`SplitOptions`           | \(2)    |
++--------------------------+------------+-------------------------+-------------------+----------------------------------+---------+
+| ascii_split_whitespace   | Unary      | String-like             | List-like         | :struct:`SplitOptions`           | \(3)    |
++--------------------------+------------+-------------------------+-------------------+----------------------------------+---------+
+
+* \(1) The string is split when an exact pattern is found (the pattern itself
+  is not included in the output).
+
+* \(2) A non-zero length sequence of Unicode defined whitespace codepoints
+  is seen as separator.
+
+* \(3) A non-zero length sequence of ASCII defined whitespace bytes
+  (``'\t'``, ``'\n'``, ``'\v'``, ``'\f'``, ``'\r'``  and ``' '``) is seen
+  as separator.
+
+
 Structural transforms
 ~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/docs/source/cpp/csv.rst b/docs/source/cpp/csv.rst
index 50a5cdb8956..4bb608fc570 100644
--- a/docs/source/cpp/csv.rst
+++ b/docs/source/cpp/csv.rst
@@ -126,10 +126,14 @@ can be chosen from the following list:
 * Float32 and Float64
 * Decimal128
 * Boolean
+* Date32 and Date64
 * Timestamp
 * Binary and Large Binary
 * String and Large String (with optional UTF8 input validation)
 * Fixed-Size Binary
+* Dictionary with index type Int32 and value type one of the following:
+  Binary, String, LargeBinary, LargeString,  Int32, UInt32, Int64, UInt64,
+  Float32, Float64, Decimal128
 
 Other data types do not support conversion from CSV values and will error out.
 
diff --git a/docs/source/developers/archery.rst b/docs/source/developers/archery.rst
index 012dffb0491..7508ba1bf07 100644
--- a/docs/source/developers/archery.rst
+++ b/docs/source/developers/archery.rst
@@ -27,7 +27,7 @@ Installation
 ------------
 
 Archery requires Python 3.5 or later. It is recommended to install archery in
-*editable* mode with the ``-e`` flag to automatically update the intallation
+*editable* mode with the ``-e`` flag to automatically update the installation
 when pulling the Arrow repository.
 
 .. code:: bash
diff --git a/docs/source/developers/contributing.rst b/docs/source/developers/contributing.rst
index 38e3f484da9..1eeeafe07fe 100644
--- a/docs/source/developers/contributing.rst
+++ b/docs/source/developers/contributing.rst
@@ -304,3 +304,51 @@ to your branch, which they sometimes do to help move a pull request along.
 In addition, the GitHub PR "suggestion" feature can also add commits to
 your branch, so it is possible that your local copy of your branch is missing
 some additions.
+
+Guidance for specific features
+==============================
+
+From time to time the community has discussions on specific types of features
+and improvements that they expect to support.  This section outlines decisions
+that have been made in this regard.
+
+Endianness
+++++++++++
+
+The Arrow format allows setting endianness.  Due to the popularity of
+little endian architectures most of implementation assume little endian by
+default. There has been some  effort to support big endian platforms as well.
+Based on a `mailing-list discussion
+<https://mail-archives.apache.org/mod_mbox/arrow-dev/202009.mbox/%3cCAK7Z5T--HHhr9Dy43PYhD6m-XoU4qoGwQVLwZsG-kOxXjPTyZA@mail.gmail.com%3e>`__,
+the requirements for a new platform are:
+
+1. A robust (non-flaky, returning results in a reasonable time) Continuous
+   Integration setup.
+2. Benchmarks for performance critical parts of the code to demonstrate
+   no regression.
+
+Furthermore, for big-endian support, there are two levels that an
+implementation can support:
+
+1. Native endianness (all Arrow communication happens with processes of the
+   same endianness).  This includes ancillary functionality such as reading
+   and writing various file formats, such as Parquet.
+2. Cross endian support (implementations will do byte reordering when
+   appropriate for :ref:`IPC <format-ipc>` and :ref:`Flight <flight-rpc>`
+   messages).
+
+The decision on what level to support is based on maintainers' preferences for
+complexity and technical risk.  In general all implementations should be open
+to native endianness support (provided the CI and performance requirements
+are met).  Cross endianness support is a question for individual maintainers.
+
+The current implementations aiming for cross endian support are:
+
+1. C++
+
+Implementations that do not intend to implement cross endian support:
+
+1. Java
+
+For other libraries, a discussion to gather consensus on the mailing-list
+should be had before submitting PRs.
diff --git a/docs/source/developers/cpp/conventions.rst b/docs/source/developers/cpp/conventions.rst
index 2b27bbd447d..9db15fbcf91 100644
--- a/docs/source/developers/cpp/conventions.rst
+++ b/docs/source/developers/cpp/conventions.rst
@@ -15,6 +15,8 @@
 .. specific language governing permissions and limitations
 .. under the License.
 
+.. highlight:: cpp
+
 ===========
 Conventions
 ===========
@@ -34,6 +36,25 @@ C++ header files use the ``.h`` extension. Any header file name not
 containing ``internal`` is considered to be a public header, and will be
 automatically installed by the build.
 
+Comments and Docstrings
+=======================
+
+Regular comments start with ``//``.
+
+Doxygen docstrings start with ``///``, and Doxygen directives start with ``\``,
+like this::
+
+   /// \brief Allocate a fixed size mutable buffer from a memory pool, zero its padding.
+   ///
+   /// \param[in] size size of buffer to allocate
+   /// \param[in] pool a memory pool
+   ARROW_EXPORT
+   Result<std::unique_ptr<Buffer>> AllocateBuffer(const int64_t size,
+                                                  MemoryPool* pool = NULLPTR);
+
+The summary line of a docstring uses the infinitive, not the indicative
+(for example, "Allocate a buffer" rather than "Allocates a buffer").
+
 Memory Pools
 ============
 
diff --git a/docs/source/developers/docker.rst b/docs/source/developers/docker.rst
index cdf77a754be..eaabad90df1 100644
--- a/docs/source/developers/docker.rst
+++ b/docs/source/developers/docker.rst
@@ -123,7 +123,7 @@ can be useful to skip the build phases:
 
     # if the second run tries the build the image again and none of the files
     # referenced in the relevant dockerfile have changed, then it indicates a
-    # cache miss caused by the issue desribed above
+    # cache miss caused by the issue described above
     archery docker run conda-python
 
     # since the image is properly built with the first command, there is no
diff --git a/docs/source/developers/python.rst b/docs/source/developers/python.rst
index 11df444a035..d1fe086cb15 100644
--- a/docs/source/developers/python.rst
+++ b/docs/source/developers/python.rst
@@ -32,6 +32,11 @@ We follow a similar PEP8-like coding style to the `pandas project
 <https://github.com/pandas-dev/pandas>`_.  To check style issues, use the
 :ref:`Archery <archery>` subcommand ``lint``:
 
+.. code-block:: shell
+
+   pip install -e arrow/dev/archery
+   pip install -r arrow/dev/archery/requirements-lint.txt
+
 .. code-block:: shell
 
    archery lint --python
@@ -148,7 +153,7 @@ Pull in the test data and setup the environment variables:
    git submodule update
    export PARQUET_TEST_DATA="${PWD}/cpp/submodules/parquet-testing/data"
    export ARROW_TEST_DATA="${PWD}/testing/data"
-
+   popd
 
 Using Conda
 ~~~~~~~~~~~
@@ -305,7 +310,7 @@ libraries are needed for Parquet support.
 If multiple versions of Python are installed in your environment, you may have
 to pass additional parameters to cmake so that it can find the right
 executable, headers and libraries.  For example, specifying
-``-DPYTHON_EXECUTABLE=$VIRTUAL_ENV/bin/python`` (assuming that you're in
+``-DPython3_EXECUTABLE=$VIRTUAL_ENV/bin/python`` (assuming that you're in
 virtualenv) enables cmake to choose the python executable which you are using.
 
 .. note::
@@ -323,6 +328,12 @@ virtualenv) enables cmake to choose the python executable which you are using.
    :ref:`here <cpp-build-dependency-management>`)
    to explicitly tell CMake not to use conda.
 
+.. note::
+
+   With older versions of ``cmake`` (<3.15) you might need to pass ``-DPYTHON_EXECUTABLE``
+   instead of ``-DPython3_EXECUTABLE``. See `cmake documentation <https://cmake.org/cmake/help/latest/module/FindPython3.html#artifacts-specification>`
+   for more details.
+
 For any other C++ build challenges, see :ref:`cpp-development`.
 
 Now, build pyarrow:
diff --git a/docs/source/format/CDataInterface.rst b/docs/source/format/CDataInterface.rst
index 768dc471144..e4ba0095e49 100644
--- a/docs/source/format/CDataInterface.rst
+++ b/docs/source/format/CDataInterface.rst
@@ -132,21 +132,23 @@ strings:
 | ``g``           | float64                  |            |
 +-----------------+--------------------------+------------+
 
-+-----------------+---------------------------------------+------------+
-| Format string   | Arrow data type                       | Notes      |
-+=================+=======================================+============+
-| ``z``           | binary                                |            |
-+-----------------+---------------------------------------+------------+
-| ``Z``           | large binary                          |            |
-+-----------------+---------------------------------------+------------+
-| ``u``           | utf-8 string                          |            |
-+-----------------+---------------------------------------+------------+
-| ``U``           | large utf-8 string                    |            |
-+-----------------+---------------------------------------+------------+
-| ``d:19,10``     | decimal128 [precision 19, scale 10]   |            |
-+-----------------+---------------------------------------+------------+
-| ``w:42``        | fixed-width binary [42 bytes]         |            |
-+-----------------+---------------------------------------+------------+
++-----------------+---------------------------------------------------+------------+
+| Format string   | Arrow data type                                   | Notes      |
++=================+===================================================+============+
+| ``z``           | binary                                            |            |
++-----------------+---------------------------------------------------+------------+
+| ``Z``           | large binary                                      |            |
++-----------------+---------------------------------------------------+------------+
+| ``u``           | utf-8 string                                      |            |
++-----------------+---------------------------------------------------+------------+
+| ``U``           | large utf-8 string                                |            |
++-----------------+---------------------------------------------------+------------+
+| ``d:19,10``     | decimal128 [precision 19, scale 10]               |            |
++-----------------+---------------------------------------------------+------------+
+| ``d:19,10,NNN`` | decimal bitwidth = NNN [precision 19, scale 10]   |            |
++-----------------+---------------------------------------------------+------------+
+| ``w:42``        | fixed-width binary [42 bytes]                     |            |
++-----------------+---------------------------------------------------+------------+
 
 Temporal types have multi-character format strings starting with ``t``:
 
@@ -535,6 +537,8 @@ Therefore, the consumer MUST not try to interfere with the producer's
 handling of these members' lifetime.  The only way the consumer influences
 data lifetime is by calling the base structure's ``release`` callback.
 
+.. _c-data-interface-released:
+
 Released structure
 ''''''''''''''''''
 
@@ -682,7 +686,7 @@ release callback is trivial.
    void export_int32_type(struct ArrowSchema* schema) {
       *schema = (struct ArrowSchema) {
          // Type description
-         .format = "l",
+         .format = "i",
          .name = "",
          .metadata = NULL,
          .flags = 0,
diff --git a/docs/source/format/CStreamInterface.rst b/docs/source/format/CStreamInterface.rst
new file mode 100644
index 00000000000..b8ccce35592
--- /dev/null
+++ b/docs/source/format/CStreamInterface.rst
@@ -0,0 +1,218 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. highlight:: c
+
+.. _c-stream-interface:
+
+============================
+The Arrow C stream interface
+============================
+
+.. warning::
+   This interface is experimental and may evolve based on feedback from
+   early users.  ABI stability is not guaranteed yet.  Feel free to
+   `contact us <https://arrow.apache.org/community/>`__.
+
+The C stream interface builds on the structures defined in the
+:ref:`C data interface <c-data-interface>` and combines them into a higher-level
+specification so as to ease the communication of streaming data within a single
+process.
+
+Semantics
+=========
+
+An Arrow C stream exposes a streaming source of data chunks, each with the
+same schema.  Chunks are obtained by calling a blocking pull-style iteration
+function.
+
+Structure definition
+====================
+
+The C stream interface is defined by a single ``struct`` definition::
+
+   struct ArrowArrayStream {
+     // Callbacks providing stream functionality
+     int (*get_schema)(struct ArrowArrayStream*, struct ArrowSchema* out);
+     int (*get_next)(struct ArrowArrayStream*, struct ArrowArray* out);
+     const char* (*get_last_error)(struct ArrowArrayStream*);
+
+     // Release callback
+     void (*release)(struct ArrowArrayStream*);
+
+     // Opaque producer-specific data
+     void* private_data;
+   };
+
+The ArrowArrayStream structure
+------------------------------
+
+The ``ArrowArrayStream`` provides the required callbacks to interact with a
+streaming source of Arrow arrays.  It has the following fields:
+
+.. c:member:: int (*ArrowArrayStream.get_schema)(struct ArrowArrayStream*, struct ArrowSchema* out)
+
+   *Mandatory.*  This callback allows the consumer to query the schema of
+   the chunks of data in the stream.  The schema is the same for all
+   data chunks.
+
+   This callback must NOT be called on a released ``ArrowArrayStream``.
+
+   *Return value:* 0 on success, a non-zero
+   :ref:`error code <c-stream-interface-error-codes>` otherwise.
+
+.. c:member:: int (*ArrowArrayStream.get_next)(struct ArrowArrayStream*, struct ArrowArray* out)
+
+   *Mandatory.*  This callback allows the consumer to get the next chunk
+   of data in the stream.
+
+   This callback must NOT be called on a released ``ArrowArrayStream``.
+
+   *Return value:* 0 on success, a non-zero
+   :ref:`error code <c-stream-interface-error-codes>` otherwise.
+
+   On success, the consumer must check whether the ``ArrowArray`` is
+   marked :ref:`released <c-data-interface-released>`.  If the
+   ``ArrowArray`` is released, then the end of stream has been reached.
+   Otherwise, the ``ArrowArray`` contains a valid data chunk.
+
+.. c:member:: const char* (*ArrowArrayStream.get_last_error)(struct ArrowArrayStream*)
+
+   *Mandatory.*  This callback allows the consumer to get a textual description
+   of the last error.
+
+   This callback must ONLY be called if the last operation on the
+   ``ArrowArrayStream`` returned an error.  It must NOT be called on a
+   released ``ArrowArrayStream``.
+
+   *Return value:* a pointer to a NULL-terminated character string (UTF8-encoded).
+   NULL can also be returned if no detailed description is available.
+
+   The returned pointer is only guaranteed to be valid until the next call of
+   one of the stream's callbacks.  The character string it points to should
+   be copied to consumer-managed storage if it is intended to survive longer.
+
+.. c:member:: void (*ArrowArrayStream.release)(struct ArrowArrayStream*)
+
+   *Mandatory.*  A pointer to a producer-provided release callback.
+
+.. c:member:: void* ArrowArrayStream.private_data
+
+   *Optional.*  An opaque pointer to producer-provided private data.
+
+   Consumers MUST not process this member.  Lifetime of this member
+   is handled by the producer, and especially by the release callback.
+
+
+.. _c-stream-interface-error-codes:
+
+Error codes
+-----------
+
+The ``get_schema`` and ``get_next`` callbacks may return an error under the form
+of a non-zero integer code.  Such error codes should be interpreted like
+``errno`` numbers (as defined by the local platform).  Note that the symbolic
+forms of these constants are stable from platform to platform, but their numeric
+values are platform-specific.
+
+In particular, it is recommended to recognize the following values:
+
+* ``EINVAL``: for a parameter or input validation error
+* ``ENOMEM``: for a memory allocation failure (out of memory)
+* ``EIO``: for a generic input/output error
+
+.. seealso::
+   `Standard POSIX error codes <https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/errno.h.html>`__.
+
+   `Error codes recognized by the Windows C runtime library
+   <https://docs.microsoft.com/en-us/cpp/c-runtime-library/errno-doserrno-sys-errlist-and-sys-nerr>`__.
+
+Result lifetimes
+----------------
+
+The data returned by the ``get_schema`` and ``get_next`` callbacks must be
+released independently.  Their lifetimes are not tied to that of the
+``ArrowArrayStream``.
+
+Stream lifetime
+---------------
+
+Lifetime of the C stream is managed using a release callback with similar
+usage as in the :ref:`C data interface <c-data-interface-released>`.
+
+
+C consumer example
+==================
+
+Let's say a particular database provides the following C API to execute
+a SQL query and return the result set as a Arrow C stream::
+
+   void MyDB_Query(const char* query, struct ArrowArrayStream* result_set);
+
+Then a consumer could use the following code to iterate over the results::
+
+   static void handle_error(int errcode, struct ArrowArrayStream* stream) {
+      // Print stream error
+      const char* errdesc = stream->get_last_error(stream);
+      if (errdesc != NULL) {
+         fputs(errdesc, stderr);
+      } else {
+         fputs(strerror(errcode), stderr);
+      }
+      // Release stream and abort
+      stream->release(stream),
+      exit(1);
+   }
+
+   void run_query() {
+      struct ArrowArrayStream stream;
+      struct ArrowSchema schema;
+      struct ArrowArray chunk;
+      int errcode;
+
+      MyDB_Query("SELECT * FROM my_table", &stream);
+
+      // Query result set schema
+      errcode = stream.get_schema(&stream, &schema);
+      if (errcode != 0) {
+         handle_error(errcode, &stream);
+      }
+
+      int64_t num_rows = 0;
+
+      // Iterate over results: loop until error or end of stream
+      while ((errcode = stream.get_next(&stream, &chunk) == 0) &&
+             chunk.release != NULL) {
+         // Do something with chunk...
+         fprintf(stderr, "Result chunk: got %lld rows\n", chunk.length);
+         num_rows += chunk.length;
+
+         // Release chunk
+         chunk.release(&chunk);
+      }
+
+      // Was it an error?
+      if (errcode != 0) {
+         handle_error(errcode, &stream);
+      }
+
+      fprintf(stderr, "Result stream ended: total %lld rows\n", num_rows);
+
+      // Release schema and stream
+      schema.release(&schema);
+      stream.release(&stream);
+   }
diff --git a/docs/source/format/Columnar.rst b/docs/source/format/Columnar.rst
index f51c6aaf633..84e3013adde 100644
--- a/docs/source/format/Columnar.rst
+++ b/docs/source/format/Columnar.rst
@@ -787,6 +787,8 @@ layouts depending on the particular realization of the type.
 We do not go into detail about the logical types definitions in this
 document as we consider `Schema.fbs`_ to be authoritative.
 
+.. _format-ipc:
+
 Serialization and Interprocess Communication (IPC)
 ==================================================
 
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 2d95e22f16a..cfcf8653982 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -43,6 +43,7 @@ such topics as:
    format/Flight
    format/Integration
    format/CDataInterface
+   format/CStreamInterface
    format/Other
 
 .. _toc.usage:
diff --git a/docs/source/java/vector.rst b/docs/source/java/vector.rst
index d4e644c7723..f114c050c81 100644
--- a/docs/source/java/vector.rst
+++ b/docs/source/java/vector.rst
@@ -120,7 +120,7 @@ Some points to note about the steps above:
 
 * For fixed width vectors (e.g. IntVector), we can set values at different indices in arbitrary orders.
   For variable width vectors (e.g. VarCharVector), however, we must set values in non-decreasing order of the
-  indices. Otherwise, the values after the set positiion will become invalid. For example, suppose we use the
+  indices. Otherwise, the values after the set position will become invalid. For example, suppose we use the
   following statements to populate a variable width vector:
 
 .. code-block:: Java
diff --git a/docs/source/python/api.rst b/docs/source/python/api.rst
index 5c4d6074d62..12cf4e06802 100644
--- a/docs/source/python/api.rst
+++ b/docs/source/python/api.rst
@@ -27,6 +27,7 @@ API Reference
    api/datatypes
    api/arrays
    api/memory
+   api/compute
    api/files
    api/tables
    api/ipc
diff --git a/docs/source/python/api/compute.rst b/docs/source/python/api/compute.rst
new file mode 100644
index 00000000000..2ade10291ef
--- /dev/null
+++ b/docs/source/python/api/compute.rst
@@ -0,0 +1,206 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. _api.compute:
+.. currentmodule:: pyarrow.compute
+
+Compute Functions
+=================
+
+Aggregations
+------------
+
+.. autosummary::
+   :toctree: ../generated/
+
+   count
+   mean
+   min_max
+   mode
+   stddev
+   sum
+   variance
+
+Arithmetic Functions
+--------------------
+
+By default these functions do not detect overflow. Each function is also
+available in an overflow-checking variant, suffixed ``_checked``, which 
+throws an ``ArrowInvalid`` exception when overflow is detected.
+
+.. autosummary::
+   :toctree: ../generated/
+
+   add
+   add_checked
+   divide
+   divide_checked
+   multiply
+   multiply_checked
+   subtract
+   subtract_checked
+
+Comparisons
+-----------
+
+These functions expect two inputs of the same type. If one of the inputs is `null`
+they return ``null``.
+
+.. autosummary::
+   :toctree: ../generated/
+
+   equal
+   greater
+   greater_equal
+   less
+   less_equal
+   not_equal
+
+Logical Functions
+-----------------
+
+These functions normally emit a null when one of the inputs is null. However, Kleene
+logic variants are provided (suffixed ``_kleene``). See User Guide for details.
+
+.. autosummary::
+   :toctree: ../generated/
+
+   and_
+   and_kleene
+   invert
+   or_
+   or_kleene
+   xor
+
+String Predicates
+-----------------
+
+In these functions an empty string emits false in the output. For ASCII 
+variants (prefixed ``ascii_``) a string element with non-ASCII characters
+emits false in the output.
+
+The first set of functions emit true if the input contains only 
+characters of a given class.
+
+.. autosummary::
+   :toctree: ../generated/
+
+   ascii_is_alnum
+   ascii_is_alpha
+   ascii_is_decimal
+   ascii_is_lower
+   ascii_is_printable
+   ascii_is_space
+   ascii_is_upper
+   utf8_is_alnum
+   utf8_is_alpha
+   utf8_is_decimal
+   utf8_is_digit
+   utf8_is_lower
+   utf8_is_numeric
+   utf8_is_printable
+   utf8_is_space
+   utf8_is_upper
+
+The second set of functions also consider the order of characters
+in the string element.
+
+.. autosummary::
+   :toctree: ../generated/
+
+   ascii_is_title
+   utf8_is_title
+
+The third set of functions examines string elements on 
+a byte-by-byte basis.
+
+.. autosummary::
+   :toctree: ../generated/
+
+   string_is_ascii
+
+String Transforms
+-----------------
+
+.. autosummary::
+   :toctree: ../generated/
+
+   ascii_lower
+   ascii_upper
+   utf8_lower
+   utf8_upper
+
+Containment tests
+-----------------
+
+.. autosummary::
+   :toctree: ../generated/
+
+   index_in
+   is_in
+   match_substring
+
+Conversions
+-----------
+
+.. autosummary::
+   :toctree: ../generated/
+
+   cast
+   strptime
+
+Selections
+----------
+
+.. autosummary::
+   :toctree: ../generated/
+
+   filter
+   take
+
+Associative transforms
+----------------------
+
+.. autosummary::
+   :toctree: ../generated/
+
+   dictionary_encode
+   unique
+   value_counts
+
+Sorts and partitions
+--------------------
+
+.. autosummary::
+   :toctree: ../generated/
+
+   partition_nth_indices
+   sort_indices
+
+Structural Transforms
+---------------------
+
+.. autosummary::
+   :toctree: ../generated/
+
+   binary_length
+   fill_null
+   is_null
+   is_valid
+   list_value_length
+   list_flatten
+   list_parent_indices
diff --git a/docs/source/python/api/dataset.rst b/docs/source/python/api/dataset.rst
index c011917b4ed..90d5bbf340a 100644
--- a/docs/source/python/api/dataset.rst
+++ b/docs/source/python/api/dataset.rst
@@ -34,6 +34,7 @@ Factory functions
    :toctree: ../generated/
 
    dataset
+   parquet_dataset
    partitioning
    field
    scalar
diff --git a/docs/source/python/api/filesystems.rst b/docs/source/python/api/filesystems.rst
index 8023a8ad09b..3e2ac29eeb0 100644
--- a/docs/source/python/api/filesystems.rst
+++ b/docs/source/python/api/filesystems.rst
@@ -28,7 +28,7 @@ Interface
 .. autosummary::
    :toctree: ../generated/
 
-   FileStats
+   FileInfo
    FileSelector
    FileSystem
 
@@ -41,3 +41,13 @@ Concrete Subclasses
    LocalFileSystem
    S3FileSystem
    HadoopFileSystem
+   SubTreeFileSystem
+
+To define filesystems with behavior implemented in Python:
+
+.. autosummary::
+   :toctree: ../generated/
+
+   PyFileSystem
+   FileSystemHandler
+   FSSpecHandler
diff --git a/docs/source/python/api/ipc.rst b/docs/source/python/api/ipc.rst
index f4c0a17dba9..cc3ccfe40bc 100644
--- a/docs/source/python/api/ipc.rst
+++ b/docs/source/python/api/ipc.rst
@@ -48,6 +48,13 @@ Inter-Process Communication
 Serialization
 -------------
 
+.. warning::
+
+   The serialization functionality is deprecated in pyarrow 2.0, and will
+   be removed in a future version. Use the standard library ``pickle`` or
+   the IPC functionality of pyarrow (see :ref:`ipc`).
+
+
 .. autosummary::
    :toctree: ../generated/
 
diff --git a/docs/source/python/compute.rst b/docs/source/python/compute.rst
new file mode 100644
index 00000000000..51126d97c82
--- /dev/null
+++ b/docs/source/python/compute.rst
@@ -0,0 +1,55 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. currentmodule:: pyarrow.compute
+.. _compute:
+
+=================
+Compute Functions
+=================
+
+Arrow supports logical compute operations over inputs of possibly 
+varying types.  Many compute functions support both array (chunked or not) 
+and scalar inputs, but some will mandate either.  For example, 
+the ``fill_null`` function requires its second input to be a scalar, 
+while ``sort_indices`` requires its first and only input to
+be an array.
+
+Below are a few simple examples:
+
+   >>> import pyarrow as pa
+   >>> import pyarrow.compute as pc
+   >>> a = pa.array([1, 1, 2, 3])
+   >>> pc.sum(a)
+   <pyarrow.Int64Scalar: 7>
+   >>> b = pa.array([4, 1, 2, 8])
+   >>> pc.equal(a, b)
+   <pyarrow.lib.BooleanArray object at 0x7f686e4eef30>
+   [
+     false,
+     true,
+     true,
+     false
+   ]   
+   >>> x, y = pa.scalar(7.8), pa.scalar(9.3)
+   >>> pc.multiply(x, y)
+   <pyarrow.DoubleScalar: 72.54>
+
+
+.. seealso::
+
+   :ref:`Available compute functions (C++ documentation) <compute-function-list>`.
diff --git a/docs/source/python/dataset.rst b/docs/source/python/dataset.rst
index 6c07ad98e25..e77c6113f7d 100644
--- a/docs/source/python/dataset.rst
+++ b/docs/source/python/dataset.rst
@@ -46,7 +46,7 @@ For those familiar with the existing :class:`pyarrow.parquet.ParquetDataset` for
 reading Parquet datasets: ``pyarrow.dataset``'s goal is similar but not specific
 to the Parquet format and not tied to Python: the same datasets API is exposed
 in the R bindings or Arrow. In addition ``pyarrow.dataset`` boasts improved
-perfomance and new features (e.g. filtering within files rather than only on
+performance and new features (e.g. filtering within files rather than only on
 partition keys).
 
 
@@ -343,6 +343,41 @@ useful for testing or benchmarking.
                          partitioning=["year", "month"])
 
 
+Working with Parquet Datasets
+-----------------------------
+
+While the Datasets API provides a unified interface to different file formats,
+some specific methods exist for Parquet Datasets.
+
+Some processing frameworks such as Dask (optionally) use a ``_metadata`` file
+with partitioned datasets which includes information about the schema and the
+row group metadata of the full dataset. Using such file can give a more
+efficient creation of a parquet Dataset, since it does not need to infer the
+schema and crawl the directories for all Parquet files (this is especially the
+case for filesystems where accessing files is expensive). The
+:func:`parquet_dataset` function allows to create a Dataset from a partitioned
+dataset with a ``_metadata`` file:
+
+.. code-block:: python
+
+    dataset = ds.parquet_dataset("/path/to/dir/_metadata")
+
+By default, the constructed :class:`Dataset` object for Parquet datasets maps
+each fragment to a single Parquet file. If you want fragments mapping to each
+row group of a Parquet file, you can use the ``split_by_row_group()`` method of
+the fragments:
+
+.. code-block:: python
+
+    fragments = list(dataset.get_fragments())
+    fragments[0].split_by_row_group()
+
+This method returns a list of new Fragments mapping to each row group of
+the original Fragment (Parquet file). Both ``get_fragments()`` and
+``split_by_row_group()`` accept an optional filter expression to get a
+filtered list of fragments.
+
+
 Manual specification of the Dataset
 -----------------------------------
 
diff --git a/docs/source/python/filesystems.rst b/docs/source/python/filesystems.rst
index a3e71f1a2db..184bd481fb0 100644
--- a/docs/source/python/filesystems.rst
+++ b/docs/source/python/filesystems.rst
@@ -31,41 +31,178 @@ storage is exposed.  Data paths are represented as *abstract paths*, which
 are ``/``-separated, even on Windows, and shouldn't include special path
 components such as ``.`` and ``..``.  Symbolic links, if supported by the
 underlying storage, are automatically dereferenced.  Only basic
-:class:`metadata <FileStats>` about file entries, such as the file size
+:class:`metadata <FileInfo>` about file entries, such as the file size
 and modification time, is made available.
 
-Types
+The core interface is represented by the base class :class:`FileSystem`.
+Concrete subclasses are available for various kinds of storage, such as local
+filesystem access (:class:`LocalFileSystem`), HDFS (:class:`HadoopFileSystem`)
+and Amazon S3-compatible storage (:class:`S3FileSystem`).
+
+
+Usage
 -----
 
-The core interface is represented by the base class :class:`FileSystem`.
-Concrete subclasses are available for various kinds of storage:
-:class:`local filesystem access <LocalFileSystem>`,
-:class:`HDFS <HadoopFileSystem>` and
-:class:`Amazon S3-compatible storage <S3FileSystem>`.
+A FileSystem object can be created with one of the constructors (and check the
+respective constructor for its options)::
+
+   >>> from pyarrow import fs
+   >>> local = fs.LocalFileSystem()
+
+or alternatively inferred from a URI::
+
+   >>> s3, path = fs.FileSystem.from_uri("s3://my-bucket")
+   >>> s3
+   <pyarrow._s3fs.S3FileSystem at 0x7f6760cbf4f0>
+   >>> path
+   'my-bucket'
+
+
+Reading and writing files
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Several of the IO-related functions in PyArrow accept either a URI (and infer
+the filesystem) or an explicit ``filesystem`` argument to specify the filesystem
+to read or write from. For example, the :meth:`pyarrow.parquet.read_table`
+function can be used in the following ways::
+
+   # using a URI -> filesystem is inferred
+   pq.read_table("s3://my-bucket/data.parquet")
+   # using a path and filesystem
+   s3 = fs.S3FileSystem(..)
+   pq.read_table("my-bucket/data.parquet", filesystem=s3)
+
+The filesystem interface further allows to open files for reading (input) or
+writing (output) directly, which can be combined with functions that work with
+file-like objects. For example::
+
+   local = fs.LocalFileSystem()
+
+   with local.open_output_stream("test.arrow") as file:
+      with pa.RecordBatchFileWriter(file, table.schema) as writer:
+         writer.write_table(table)
+
+
+Listing files
+~~~~~~~~~~~~~
+
+Inspecting the directories and files on a filesystem can be done with the
+:meth:`FileSystem.get_file_info` method. To list the contents of a directory,
+use the :class:`FileSelector` object to specify the selection::
+
+   >>> local.get_file_info(fs.FileSelector("dataset/", recursive=True))
+   [<FileInfo for 'dataset/part=B': type=FileType.Directory>,
+    <FileInfo for 'dataset/part=B/data0.parquet': type=FileType.File, size=1564>,
+    <FileInfo for 'dataset/part=A': type=FileType.Directory>,
+    <FileInfo for 'dataset/part=A/data0.parquet': type=FileType.File, size=1564>]
+
+This returns a list of :class:`FileInfo` objects, containing information about
+the type (file or directory), the size, the date last modified, etc.
+
+You can also get this information for a single explicit path (or list of
+paths)::
+
+   >>> local.get_file_info('test.arrow')
+   <FileInfo for 'test.arrow': type=FileType.File, size=3250>
+
+   >>> local.get_file_info('non_existent')
+   <FileInfo for 'non_existent': type=FileType.NotFound>
+
+S3
+--
 
-Example
--------
+The :class:`S3FileSystem` constructor has several options to configure the S3
+connection (e.g. credentials, the region, an endpoint override, etc). In
+addition, the constructor will also inspect configured S3 credentials as
+supported by AWS (for example the ``AWS_ACCESS_KEY_ID`` and
+``AWS_SECRET_ACCESS_KEY`` environment variables).
 
-Assuming your S3 credentials are correctly configured (for example by setting
-the ``AWS_ACCESS_KEY_ID`` and ``AWS_SECRET_ACCESS_KEY`` environment variables),
-here is how you can read contents from a S3 bucket::
+Example how you can read contents from a S3 bucket::
 
    >>> from pyarrow import fs
    >>> s3 = fs.S3FileSystem(region='eu-west-3')
 
    # List all contents in a bucket, recursively
-   >>> s3.get_target_stats(fs.FileSelector('my-test-bucket', recursive=True))
-   [<FileStats for 'my-test-bucket/File1': type=FileType.File, size=10>,
-    <FileStats for 'my-test-bucket/File5': type=FileType.File, size=10>,
-    <FileStats for 'my-test-bucket/Dir1': type=FileType.Directory>,
-    <FileStats for 'my-test-bucket/Dir2': type=FileType.Directory>,
-    <FileStats for 'my-test-bucket/EmptyDir': type=FileType.Directory>,
-    <FileStats for 'my-test-bucket/Dir1/File2': type=FileType.File, size=11>,
-    <FileStats for 'my-test-bucket/Dir1/Subdir': type=FileType.Directory>,
-    <FileStats for 'my-test-bucket/Dir2/Subdir': type=FileType.Directory>,
-    <FileStats for 'my-test-bucket/Dir2/Subdir/File3': type=FileType.File, size=10>]
+   >>> s3.get_file_info(fs.FileSelector('my-test-bucket', recursive=True))
+   [<FileInfo for 'my-test-bucket/File1': type=FileType.File, size=10>,
+    <FileInfo for 'my-test-bucket/File5': type=FileType.File, size=10>,
+    <FileInfo for 'my-test-bucket/Dir1': type=FileType.Directory>,
+    <FileInfo for 'my-test-bucket/Dir2': type=FileType.Directory>,
+    <FileInfo for 'my-test-bucket/EmptyDir': type=FileType.Directory>,
+    <FileInfo for 'my-test-bucket/Dir1/File2': type=FileType.File, size=11>,
+    <FileInfo for 'my-test-bucket/Dir1/Subdir': type=FileType.Directory>,
+    <FileInfo for 'my-test-bucket/Dir2/Subdir': type=FileType.Directory>,
+    <FileInfo for 'my-test-bucket/Dir2/Subdir/File3': type=FileType.File, size=10>]
 
    # Open a file for reading and download its contents
    >>> f = s3.open_input_stream('my-test-bucket/Dir1/File2')
    >>> f.readall()
    b'some data'
+
+.. seealso::
+
+   See the `AWS docs <https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html>`__
+   for the different ways to configure the AWS credentials.
+
+
+Hadoop File System (HDFS)
+-------------------------
+
+PyArrow comes with bindings to the Hadoop File System (based on C++ bindings
+using ``libhdfs``, a JNI-based interface to the Java Hadoop client). You connect
+using the :class:`HadoopFileSystem` constructor::
+
+.. code-block:: python
+
+   from pyarrow import fs
+   hdfs = fs.HadoopFileSystem(host, port, user=user, kerb_ticket=ticket_cache_path)
+
+The ``libhdfs`` library is loaded **at runtime** (rather than at link / library
+load time, since the library may not be in your LD_LIBRARY_PATH), and relies on
+some environment variables.
+
+* ``HADOOP_HOME``: the root of your installed Hadoop distribution. Often has
+  `lib/native/libhdfs.so`.
+
+* ``JAVA_HOME``: the location of your Java SDK installation.
+
+* ``ARROW_LIBHDFS_DIR`` (optional): explicit location of ``libhdfs.so`` if it is
+  installed somewhere other than ``$HADOOP_HOME/lib/native``.
+
+* ``CLASSPATH``: must contain the Hadoop jars. You can set these using:
+
+  .. code-block:: shell
+
+      export CLASSPATH=`$HADOOP_HOME/bin/hdfs classpath --glob`
+
+  If ``CLASSPATH`` is not set, then it will be set automatically if the
+  ``hadoop`` executable is in your system path, or if ``HADOOP_HOME`` is set.
+
+
+Using fsspec-compatible filesystems
+-----------------------------------
+
+The filesystems mentioned above are natively supported by Arrow C++ / PyArrow.
+The Python ecosystem, however, also has several filesystem packages. Those
+packages following the
+`fsspec <https://filesystem-spec.readthedocs.io/en/latest/>`__ interface can be
+used in PyArrow as well.
+
+Functions accepting a filesystem object will also accept an fsspec subclass.
+For example::
+
+   # creating an fsspec-based filesystem object for Google Cloud Storage
+   import gcsfs
+   fs = gcsfs.GCSFileSystem(project='my-google-project')
+
+   # using this to read a partitioned dataset
+   import pyarrow.dataset as ds
+   ds.dataset("data/", filesystem=fs)
+
+Under the hood, the fsspec filesystem object is wrapped into a python-based
+PyArrow filesystem (:class:`PyFileSystem`) using :class:`FSSpecHandler`.
+You can also manually do this to get an object with the PyArrow FileSystem
+interface::
+
+   from pyarrow.fs import PyFileSystem, FSSpecHandler
+   pa_fs = PyFileSystem(FSSpecHandler(fs))
diff --git a/docs/source/python/filesystems_deprecated.rst b/docs/source/python/filesystems_deprecated.rst
index 51a07d5efa6..04887e97738 100644
--- a/docs/source/python/filesystems_deprecated.rst
+++ b/docs/source/python/filesystems_deprecated.rst
@@ -18,9 +18,9 @@
 Filesystem Interface (legacy)
 =============================
 
-.. note::
-   This section documents the deprecated filesystem layer.  It is highly
-   recommended to use the :ref:`new filesystem layer <filesystem>` instead.
+.. warning::
+   This section documents the deprecated filesystem layer.  You should
+   use the :ref:`new filesystem layer <filesystem>` instead.
 
 .. _hdfs:
 
diff --git a/docs/source/python/index.rst b/docs/source/python/index.rst
index d4daf4029ac..cc7383044e0 100644
--- a/docs/source/python/index.rst
+++ b/docs/source/python/index.rst
@@ -36,6 +36,7 @@ files into Arrow structures.
    install
    memory
    data
+   compute
    ipc
    filesystems
    filesystems_deprecated
diff --git a/docs/source/python/ipc.rst b/docs/source/python/ipc.rst
index b7a032d1342..1be8ff62ce5 100644
--- a/docs/source/python/ipc.rst
+++ b/docs/source/python/ipc.rst
@@ -157,17 +157,25 @@ DataFrame output:
 Arbitrary Object Serialization
 ------------------------------
 
+.. warning::
+
+   The custom serialization functionality is deprecated in pyarrow 2.0, and
+   will be removed in a future version.
+
+   While the serialization functions in this section utilize the Arrow stream
+   protocol internally, they do not produce data that is compatible with the
+   above ``ipc.open_file`` and ``ipc.open_stream`` functions.
+
+   For arbitrary objects, you can use the standard library ``pickle``
+   functionality instead. For pyarrow objects, you can use the IPC
+   serialization format through the ``pyarrow.ipc`` module, as explained
+   above.
+
 In ``pyarrow`` we are able to serialize and deserialize many kinds of Python
 objects. While not a complete replacement for the ``pickle`` module, these
 functions can be significantly faster, particular when dealing with collections
 of NumPy arrays.
 
-.. warning::
-
-   While the functions in this section utilize the Arrow stream protocol
-   internally, they do not produce data that is compatible with the above
-   ``ipc.open_file`` and ``ipc.open_stream`` functions.
-
 As an example, consider a dictionary containing NumPy arrays:
 
 .. ipython:: python
@@ -322,21 +330,3 @@ An object can be reconstructed from its component-based representation using
 
 ``deserialize_components`` is also available as a method on
 ``SerializationContext`` objects.
-
-Serializing pandas Objects
---------------------------
-
-The default serialization context has optimized handling of pandas
-objects like ``DataFrame`` and ``Series``. Combined with component-based
-serialization above, this enables zero-copy transport of pandas DataFrame
-objects not containing any Python objects:
-
-.. ipython:: python
-
-   import pandas as pd
-   df = pd.DataFrame({'a': [1, 2, 3, 4, 5]})
-   context = pa.default_serialization_context()
-   serialized_df = context.serialize(df)
-   df_components = serialized_df.to_components()
-   original_df = context.deserialize_components(df_components)
-   original_df
diff --git a/docs/source/python/json.rst b/docs/source/python/json.rst
index e4abbff4d13..99ecbc19a12 100644
--- a/docs/source/python/json.rst
+++ b/docs/source/python/json.rst
@@ -21,10 +21,10 @@
 Reading JSON files
 ==================
 
-Arrow supports reading columnar data from JSON files.  In this context, a
-JSON file consists of multiple JSON objects, one per line, representing
-individual data rows.  For example, this file represents two rows of data
-with four columns "a", "b", "c", "d":
+Arrow supports reading columnar data from line-delimited JSON files. 
+In this context, a JSON file consists of multiple JSON objects, one per line,
+representing individual data rows.  For example, this file represents
+two rows of data with four columns "a", "b", "c", "d":
 
 .. code-block:: json
 
@@ -38,6 +38,9 @@ The features currently offered are the following:
   such as ``my_data.json.gz``)
 * sophisticated type inference (see below)
 
+.. note::
+   Currently only the line-delimited JSON format is supported.
+
 
 Usage
 -----
diff --git a/docs/source/status.rst b/docs/source/status.rst
index 9dcc06ddab6..d43c3bc80ca 100644
--- a/docs/source/status.rst
+++ b/docs/source/status.rst
@@ -75,7 +75,7 @@ Data Types
 +-------------------+-------+-------+-------+------------+-------+-------+
 | Large List        | ✓     | ✓     |       |            |       |       |
 +-------------------+-------+-------+-------+------------+-------+-------+
-| Struct            | ✓     | ✓     | ✓     | ✓          |       |       |
+| Struct            | ✓     | ✓     | ✓     | ✓          |  ✓    |       |
 +-------------------+-------+-------+-------+------------+-------+-------+
 | Map               | ✓     | ✓     |       | ✓          |       |       |
 +-------------------+-------+-------+-------+------------+-------+-------+
diff --git a/format/Flight.proto b/format/Flight.proto
index 71ae7ca9d56..7b0f591a237 100644
--- a/format/Flight.proto
+++ b/format/Flight.proto
@@ -19,6 +19,8 @@
 syntax = "proto3";
 
 option java_package = "org.apache.arrow.flight.impl";
+option go_package = "github.com/apache/arrow/go/flight;flight";
+
 package arrow.flight.protocol;
 
 /*
diff --git a/format/Schema.fbs b/format/Schema.fbs
index fd886f698f5..3b37e5d85f3 100644
--- a/format/Schema.fbs
+++ b/format/Schema.fbs
@@ -173,8 +173,8 @@ table Bool {
 }
 
 /// Exact decimal value represented as an integer value in two's
-/// complement. Currently only 128-bit (16-byte) integers are used but this may
-/// be expanded in the future. The representation uses the endianness indicated
+/// complement. Currently only 128-bit (16-byte) and 256-bit (32-byte) integers
+/// are used. The representation uses the endianness indicated
 /// in the Schema.
 table Decimal {
   /// Total number of decimal digits
@@ -183,10 +183,8 @@ table Decimal {
   /// Number of digits after the decimal point "."
   scale: int;
 
-  /// Number of bits per value. The only accepted width right now is 128 but
-  /// this field exists for forward compatibility so that other bit widths may
-  /// be supported in future format versions. We use bitWidth for consistency
-  /// with Int::bitWidth.
+  /// Number of bits per value. The only accepted widths are 128 and 256.
+  /// We use bitWidth for consistency with Int::bitWidth.
   bitWidth: int = 128;
 }
 
diff --git a/go/arrow/flight/Flight.pb.go b/go/arrow/flight/Flight.pb.go
new file mode 100644
index 00000000000..75c6c2c3c05
--- /dev/null
+++ b/go/arrow/flight/Flight.pb.go
@@ -0,0 +1,1473 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+// <p>
+// http://www.apache.org/licenses/LICENSE-2.0
+// <p>
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.9.1
+// source: Flight.proto
+
+package flight
+
+import (
+	proto "github.com/golang/protobuf/proto"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
+
+//
+// Describes what type of descriptor is defined.
+type FlightDescriptor_DescriptorType int32
+
+const (
+	// Protobuf pattern, not used.
+	FlightDescriptor_UNKNOWN FlightDescriptor_DescriptorType = 0
+	//
+	// A named path that identifies a dataset. A path is composed of a string
+	// or list of strings describing a particular dataset. This is conceptually
+	//  similar to a path inside a filesystem.
+	FlightDescriptor_PATH FlightDescriptor_DescriptorType = 1
+	//
+	// An opaque command to generate a dataset.
+	FlightDescriptor_CMD FlightDescriptor_DescriptorType = 2
+)
+
+// Enum value maps for FlightDescriptor_DescriptorType.
+var (
+	FlightDescriptor_DescriptorType_name = map[int32]string{
+		0: "UNKNOWN",
+		1: "PATH",
+		2: "CMD",
+	}
+	FlightDescriptor_DescriptorType_value = map[string]int32{
+		"UNKNOWN": 0,
+		"PATH":    1,
+		"CMD":     2,
+	}
+)
+
+func (x FlightDescriptor_DescriptorType) Enum() *FlightDescriptor_DescriptorType {
+	p := new(FlightDescriptor_DescriptorType)
+	*p = x
+	return p
+}
+
+func (x FlightDescriptor_DescriptorType) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (FlightDescriptor_DescriptorType) Descriptor() protoreflect.EnumDescriptor {
+	return file_Flight_proto_enumTypes[0].Descriptor()
+}
+
+func (FlightDescriptor_DescriptorType) Type() protoreflect.EnumType {
+	return &file_Flight_proto_enumTypes[0]
+}
+
+func (x FlightDescriptor_DescriptorType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use FlightDescriptor_DescriptorType.Descriptor instead.
+func (FlightDescriptor_DescriptorType) EnumDescriptor() ([]byte, []int) {
+	return file_Flight_proto_rawDescGZIP(), []int{9, 0}
+}
+
+//
+// The request that a client provides to a server on handshake.
+type HandshakeRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	//
+	// A defined protocol version
+	ProtocolVersion uint64 `protobuf:"varint,1,opt,name=protocol_version,json=protocolVersion,proto3" json:"protocol_version,omitempty"`
+	//
+	// Arbitrary auth/handshake info.
+	Payload []byte `protobuf:"bytes,2,opt,name=payload,proto3" json:"payload,omitempty"`
+}
+
+func (x *HandshakeRequest) Reset() {
+	*x = HandshakeRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_Flight_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *HandshakeRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*HandshakeRequest) ProtoMessage() {}
+
+func (x *HandshakeRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_Flight_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use HandshakeRequest.ProtoReflect.Descriptor instead.
+func (*HandshakeRequest) Descriptor() ([]byte, []int) {
+	return file_Flight_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *HandshakeRequest) GetProtocolVersion() uint64 {
+	if x != nil {
+		return x.ProtocolVersion
+	}
+	return 0
+}
+
+func (x *HandshakeRequest) GetPayload() []byte {
+	if x != nil {
+		return x.Payload
+	}
+	return nil
+}
+
+type HandshakeResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	//
+	// A defined protocol version
+	ProtocolVersion uint64 `protobuf:"varint,1,opt,name=protocol_version,json=protocolVersion,proto3" json:"protocol_version,omitempty"`
+	//
+	// Arbitrary auth/handshake info.
+	Payload []byte `protobuf:"bytes,2,opt,name=payload,proto3" json:"payload,omitempty"`
+}
+
+func (x *HandshakeResponse) Reset() {
+	*x = HandshakeResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_Flight_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *HandshakeResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*HandshakeResponse) ProtoMessage() {}
+
+func (x *HandshakeResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_Flight_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use HandshakeResponse.ProtoReflect.Descriptor instead.
+func (*HandshakeResponse) Descriptor() ([]byte, []int) {
+	return file_Flight_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *HandshakeResponse) GetProtocolVersion() uint64 {
+	if x != nil {
+		return x.ProtocolVersion
+	}
+	return 0
+}
+
+func (x *HandshakeResponse) GetPayload() []byte {
+	if x != nil {
+		return x.Payload
+	}
+	return nil
+}
+
+//
+// A message for doing simple auth.
+type BasicAuth struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Username string `protobuf:"bytes,2,opt,name=username,proto3" json:"username,omitempty"`
+	Password string `protobuf:"bytes,3,opt,name=password,proto3" json:"password,omitempty"`
+}
+
+func (x *BasicAuth) Reset() {
+	*x = BasicAuth{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_Flight_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *BasicAuth) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*BasicAuth) ProtoMessage() {}
+
+func (x *BasicAuth) ProtoReflect() protoreflect.Message {
+	mi := &file_Flight_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use BasicAuth.ProtoReflect.Descriptor instead.
+func (*BasicAuth) Descriptor() ([]byte, []int) {
+	return file_Flight_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *BasicAuth) GetUsername() string {
+	if x != nil {
+		return x.Username
+	}
+	return ""
+}
+
+func (x *BasicAuth) GetPassword() string {
+	if x != nil {
+		return x.Password
+	}
+	return ""
+}
+
+type Empty struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *Empty) Reset() {
+	*x = Empty{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_Flight_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Empty) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Empty) ProtoMessage() {}
+
+func (x *Empty) ProtoReflect() protoreflect.Message {
+	mi := &file_Flight_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Empty.ProtoReflect.Descriptor instead.
+func (*Empty) Descriptor() ([]byte, []int) {
+	return file_Flight_proto_rawDescGZIP(), []int{3}
+}
+
+//
+// Describes an available action, including both the name used for execution
+// along with a short description of the purpose of the action.
+type ActionType struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Type        string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"`
+	Description string `protobuf:"bytes,2,opt,name=description,proto3" json:"description,omitempty"`
+}
+
+func (x *ActionType) Reset() {
+	*x = ActionType{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_Flight_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ActionType) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ActionType) ProtoMessage() {}
+
+func (x *ActionType) ProtoReflect() protoreflect.Message {
+	mi := &file_Flight_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ActionType.ProtoReflect.Descriptor instead.
+func (*ActionType) Descriptor() ([]byte, []int) {
+	return file_Flight_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *ActionType) GetType() string {
+	if x != nil {
+		return x.Type
+	}
+	return ""
+}
+
+func (x *ActionType) GetDescription() string {
+	if x != nil {
+		return x.Description
+	}
+	return ""
+}
+
+//
+// A service specific expression that can be used to return a limited set
+// of available Arrow Flight streams.
+type Criteria struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Expression []byte `protobuf:"bytes,1,opt,name=expression,proto3" json:"expression,omitempty"`
+}
+
+func (x *Criteria) Reset() {
+	*x = Criteria{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_Flight_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Criteria) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Criteria) ProtoMessage() {}
+
+func (x *Criteria) ProtoReflect() protoreflect.Message {
+	mi := &file_Flight_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Criteria.ProtoReflect.Descriptor instead.
+func (*Criteria) Descriptor() ([]byte, []int) {
+	return file_Flight_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *Criteria) GetExpression() []byte {
+	if x != nil {
+		return x.Expression
+	}
+	return nil
+}
+
+//
+// An opaque action specific for the service.
+type Action struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"`
+	Body []byte `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"`
+}
+
+func (x *Action) Reset() {
+	*x = Action{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_Flight_proto_msgTypes[6]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Action) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Action) ProtoMessage() {}
+
+func (x *Action) ProtoReflect() protoreflect.Message {
+	mi := &file_Flight_proto_msgTypes[6]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Action.ProtoReflect.Descriptor instead.
+func (*Action) Descriptor() ([]byte, []int) {
+	return file_Flight_proto_rawDescGZIP(), []int{6}
+}
+
+func (x *Action) GetType() string {
+	if x != nil {
+		return x.Type
+	}
+	return ""
+}
+
+func (x *Action) GetBody() []byte {
+	if x != nil {
+		return x.Body
+	}
+	return nil
+}
+
+//
+// An opaque result returned after executing an action.
+type Result struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Body []byte `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"`
+}
+
+func (x *Result) Reset() {
+	*x = Result{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_Flight_proto_msgTypes[7]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Result) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Result) ProtoMessage() {}
+
+func (x *Result) ProtoReflect() protoreflect.Message {
+	mi := &file_Flight_proto_msgTypes[7]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Result.ProtoReflect.Descriptor instead.
+func (*Result) Descriptor() ([]byte, []int) {
+	return file_Flight_proto_rawDescGZIP(), []int{7}
+}
+
+func (x *Result) GetBody() []byte {
+	if x != nil {
+		return x.Body
+	}
+	return nil
+}
+
+//
+// Wrap the result of a getSchema call
+type SchemaResult struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// schema of the dataset as described in Schema.fbs::Schema.
+	Schema []byte `protobuf:"bytes,1,opt,name=schema,proto3" json:"schema,omitempty"`
+}
+
+func (x *SchemaResult) Reset() {
+	*x = SchemaResult{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_Flight_proto_msgTypes[8]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *SchemaResult) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SchemaResult) ProtoMessage() {}
+
+func (x *SchemaResult) ProtoReflect() protoreflect.Message {
+	mi := &file_Flight_proto_msgTypes[8]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SchemaResult.ProtoReflect.Descriptor instead.
+func (*SchemaResult) Descriptor() ([]byte, []int) {
+	return file_Flight_proto_rawDescGZIP(), []int{8}
+}
+
+func (x *SchemaResult) GetSchema() []byte {
+	if x != nil {
+		return x.Schema
+	}
+	return nil
+}
+
+//
+// The name or tag for a Flight. May be used as a way to retrieve or generate
+// a flight or be used to expose a set of previously defined flights.
+type FlightDescriptor struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Type FlightDescriptor_DescriptorType `protobuf:"varint,1,opt,name=type,proto3,enum=arrow.flight.protocol.FlightDescriptor_DescriptorType" json:"type,omitempty"`
+	//
+	// Opaque value used to express a command. Should only be defined when
+	// type = CMD.
+	Cmd []byte `protobuf:"bytes,2,opt,name=cmd,proto3" json:"cmd,omitempty"`
+	//
+	// List of strings identifying a particular dataset. Should only be defined
+	// when type = PATH.
+	Path []string `protobuf:"bytes,3,rep,name=path,proto3" json:"path,omitempty"`
+}
+
+func (x *FlightDescriptor) Reset() {
+	*x = FlightDescriptor{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_Flight_proto_msgTypes[9]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *FlightDescriptor) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*FlightDescriptor) ProtoMessage() {}
+
+func (x *FlightDescriptor) ProtoReflect() protoreflect.Message {
+	mi := &file_Flight_proto_msgTypes[9]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use FlightDescriptor.ProtoReflect.Descriptor instead.
+func (*FlightDescriptor) Descriptor() ([]byte, []int) {
+	return file_Flight_proto_rawDescGZIP(), []int{9}
+}
+
+func (x *FlightDescriptor) GetType() FlightDescriptor_DescriptorType {
+	if x != nil {
+		return x.Type
+	}
+	return FlightDescriptor_UNKNOWN
+}
+
+func (x *FlightDescriptor) GetCmd() []byte {
+	if x != nil {
+		return x.Cmd
+	}
+	return nil
+}
+
+func (x *FlightDescriptor) GetPath() []string {
+	if x != nil {
+		return x.Path
+	}
+	return nil
+}
+
+//
+// The access coordinates for retrieval of a dataset. With a FlightInfo, a
+// consumer is able to determine how to retrieve a dataset.
+type FlightInfo struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// schema of the dataset as described in Schema.fbs::Schema.
+	Schema []byte `protobuf:"bytes,1,opt,name=schema,proto3" json:"schema,omitempty"`
+	//
+	// The descriptor associated with this info.
+	FlightDescriptor *FlightDescriptor `protobuf:"bytes,2,opt,name=flight_descriptor,json=flightDescriptor,proto3" json:"flight_descriptor,omitempty"`
+	//
+	// A list of endpoints associated with the flight. To consume the whole
+	// flight, all endpoints must be consumed.
+	Endpoint []*FlightEndpoint `protobuf:"bytes,3,rep,name=endpoint,proto3" json:"endpoint,omitempty"`
+	// Set these to -1 if unknown.
+	TotalRecords int64 `protobuf:"varint,4,opt,name=total_records,json=totalRecords,proto3" json:"total_records,omitempty"`
+	TotalBytes   int64 `protobuf:"varint,5,opt,name=total_bytes,json=totalBytes,proto3" json:"total_bytes,omitempty"`
+}
+
+func (x *FlightInfo) Reset() {
+	*x = FlightInfo{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_Flight_proto_msgTypes[10]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *FlightInfo) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*FlightInfo) ProtoMessage() {}
+
+func (x *FlightInfo) ProtoReflect() protoreflect.Message {
+	mi := &file_Flight_proto_msgTypes[10]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use FlightInfo.ProtoReflect.Descriptor instead.
+func (*FlightInfo) Descriptor() ([]byte, []int) {
+	return file_Flight_proto_rawDescGZIP(), []int{10}
+}
+
+func (x *FlightInfo) GetSchema() []byte {
+	if x != nil {
+		return x.Schema
+	}
+	return nil
+}
+
+func (x *FlightInfo) GetFlightDescriptor() *FlightDescriptor {
+	if x != nil {
+		return x.FlightDescriptor
+	}
+	return nil
+}
+
+func (x *FlightInfo) GetEndpoint() []*FlightEndpoint {
+	if x != nil {
+		return x.Endpoint
+	}
+	return nil
+}
+
+func (x *FlightInfo) GetTotalRecords() int64 {
+	if x != nil {
+		return x.TotalRecords
+	}
+	return 0
+}
+
+func (x *FlightInfo) GetTotalBytes() int64 {
+	if x != nil {
+		return x.TotalBytes
+	}
+	return 0
+}
+
+//
+// A particular stream or split associated with a flight.
+type FlightEndpoint struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	//
+	// Token used to retrieve this stream.
+	Ticket *Ticket `protobuf:"bytes,1,opt,name=ticket,proto3" json:"ticket,omitempty"`
+	//
+	// A list of URIs where this ticket can be redeemed. If the list is
+	// empty, the expectation is that the ticket can only be redeemed on the
+	// current service where the ticket was generated.
+	Location []*Location `protobuf:"bytes,2,rep,name=location,proto3" json:"location,omitempty"`
+}
+
+func (x *FlightEndpoint) Reset() {
+	*x = FlightEndpoint{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_Flight_proto_msgTypes[11]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *FlightEndpoint) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*FlightEndpoint) ProtoMessage() {}
+
+func (x *FlightEndpoint) ProtoReflect() protoreflect.Message {
+	mi := &file_Flight_proto_msgTypes[11]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use FlightEndpoint.ProtoReflect.Descriptor instead.
+func (*FlightEndpoint) Descriptor() ([]byte, []int) {
+	return file_Flight_proto_rawDescGZIP(), []int{11}
+}
+
+func (x *FlightEndpoint) GetTicket() *Ticket {
+	if x != nil {
+		return x.Ticket
+	}
+	return nil
+}
+
+func (x *FlightEndpoint) GetLocation() []*Location {
+	if x != nil {
+		return x.Location
+	}
+	return nil
+}
+
+//
+// A location where a Flight service will accept retrieval of a particular
+// stream given a ticket.
+type Location struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Uri string `protobuf:"bytes,1,opt,name=uri,proto3" json:"uri,omitempty"`
+}
+
+func (x *Location) Reset() {
+	*x = Location{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_Flight_proto_msgTypes[12]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Location) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Location) ProtoMessage() {}
+
+func (x *Location) ProtoReflect() protoreflect.Message {
+	mi := &file_Flight_proto_msgTypes[12]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Location.ProtoReflect.Descriptor instead.
+func (*Location) Descriptor() ([]byte, []int) {
+	return file_Flight_proto_rawDescGZIP(), []int{12}
+}
+
+func (x *Location) GetUri() string {
+	if x != nil {
+		return x.Uri
+	}
+	return ""
+}
+
+//
+// An opaque identifier that the service can use to retrieve a particular
+// portion of a stream.
+type Ticket struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Ticket []byte `protobuf:"bytes,1,opt,name=ticket,proto3" json:"ticket,omitempty"`
+}
+
+func (x *Ticket) Reset() {
+	*x = Ticket{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_Flight_proto_msgTypes[13]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Ticket) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Ticket) ProtoMessage() {}
+
+func (x *Ticket) ProtoReflect() protoreflect.Message {
+	mi := &file_Flight_proto_msgTypes[13]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Ticket.ProtoReflect.Descriptor instead.
+func (*Ticket) Descriptor() ([]byte, []int) {
+	return file_Flight_proto_rawDescGZIP(), []int{13}
+}
+
+func (x *Ticket) GetTicket() []byte {
+	if x != nil {
+		return x.Ticket
+	}
+	return nil
+}
+
+//
+// A batch of Arrow data as part of a stream of batches.
+type FlightData struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	//
+	// The descriptor of the data. This is only relevant when a client is
+	// starting a new DoPut stream.
+	FlightDescriptor *FlightDescriptor `protobuf:"bytes,1,opt,name=flight_descriptor,json=flightDescriptor,proto3" json:"flight_descriptor,omitempty"`
+	//
+	// Header for message data as described in Message.fbs::Message.
+	DataHeader []byte `protobuf:"bytes,2,opt,name=data_header,json=dataHeader,proto3" json:"data_header,omitempty"`
+	//
+	// Application-defined metadata.
+	AppMetadata []byte `protobuf:"bytes,3,opt,name=app_metadata,json=appMetadata,proto3" json:"app_metadata,omitempty"`
+	//
+	// The actual batch of Arrow data. Preferably handled with minimal-copies
+	// coming last in the definition to help with sidecar patterns (it is
+	// expected that some implementations will fetch this field off the wire
+	// with specialized code to avoid extra memory copies).
+	DataBody []byte `protobuf:"bytes,1000,opt,name=data_body,json=dataBody,proto3" json:"data_body,omitempty"`
+}
+
+func (x *FlightData) Reset() {
+	*x = FlightData{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_Flight_proto_msgTypes[14]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *FlightData) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*FlightData) ProtoMessage() {}
+
+func (x *FlightData) ProtoReflect() protoreflect.Message {
+	mi := &file_Flight_proto_msgTypes[14]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use FlightData.ProtoReflect.Descriptor instead.
+func (*FlightData) Descriptor() ([]byte, []int) {
+	return file_Flight_proto_rawDescGZIP(), []int{14}
+}
+
+func (x *FlightData) GetFlightDescriptor() *FlightDescriptor {
+	if x != nil {
+		return x.FlightDescriptor
+	}
+	return nil
+}
+
+func (x *FlightData) GetDataHeader() []byte {
+	if x != nil {
+		return x.DataHeader
+	}
+	return nil
+}
+
+func (x *FlightData) GetAppMetadata() []byte {
+	if x != nil {
+		return x.AppMetadata
+	}
+	return nil
+}
+
+func (x *FlightData) GetDataBody() []byte {
+	if x != nil {
+		return x.DataBody
+	}
+	return nil
+}
+
+//*
+// The response message associated with the submission of a DoPut.
+type PutResult struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	AppMetadata []byte `protobuf:"bytes,1,opt,name=app_metadata,json=appMetadata,proto3" json:"app_metadata,omitempty"`
+}
+
+func (x *PutResult) Reset() {
+	*x = PutResult{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_Flight_proto_msgTypes[15]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *PutResult) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*PutResult) ProtoMessage() {}
+
+func (x *PutResult) ProtoReflect() protoreflect.Message {
+	mi := &file_Flight_proto_msgTypes[15]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use PutResult.ProtoReflect.Descriptor instead.
+func (*PutResult) Descriptor() ([]byte, []int) {
+	return file_Flight_proto_rawDescGZIP(), []int{15}
+}
+
+func (x *PutResult) GetAppMetadata() []byte {
+	if x != nil {
+		return x.AppMetadata
+	}
+	return nil
+}
+
+var File_Flight_proto protoreflect.FileDescriptor
+
+var file_Flight_proto_rawDesc = []byte{
+	0x0a, 0x0c, 0x46, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x15,
+	0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x22, 0x57, 0x0a, 0x10, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61,
+	0x6b, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x04, 0x52, 0x0f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x65, 0x72,
+	0x73, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x58,
+	0x0a, 0x11, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x12, 0x29, 0x0a, 0x10, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x5f,
+	0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x18,
+	0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52,
+	0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x43, 0x0a, 0x09, 0x42, 0x61, 0x73, 0x69,
+	0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d,
+	0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d,
+	0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x03, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x22, 0x07, 0x0a,
+	0x05, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x42, 0x0a, 0x0a, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e,
+	0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63,
+	0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64,
+	0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x2a, 0x0a, 0x08, 0x43, 0x72,
+	0x69, 0x74, 0x65, 0x72, 0x69, 0x61, 0x12, 0x1e, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73,
+	0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x65, 0x78, 0x70, 0x72,
+	0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x30, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e,
+	0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04,
+	0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x02, 0x20, 0x01,
+	0x28, 0x0c, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x22, 0x1c, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75,
+	0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c,
+	0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x22, 0x26, 0x0a, 0x0c, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61,
+	0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x22, 0xb6,
+	0x01, 0x0a, 0x10, 0x46, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70,
+	0x74, 0x6f, 0x72, 0x12, 0x4a, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x0e, 0x32, 0x36, 0x2e, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x69, 0x67, 0x68, 0x74,
+	0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72,
+	0x69, 0x70, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12,
+	0x10, 0x0a, 0x03, 0x63, 0x6d, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x6d,
+	0x64, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52,
+	0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x30, 0x0a, 0x0e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70,
+	0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f,
+	0x57, 0x4e, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x50, 0x41, 0x54, 0x48, 0x10, 0x01, 0x12, 0x07,
+	0x0a, 0x03, 0x43, 0x4d, 0x44, 0x10, 0x02, 0x22, 0x83, 0x02, 0x0a, 0x0a, 0x46, 0x6c, 0x69, 0x67,
+	0x68, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x54,
+	0x0a, 0x11, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70,
+	0x74, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x61, 0x72, 0x72, 0x6f,
+	0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f,
+	0x6c, 0x2e, 0x46, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74,
+	0x6f, 0x72, 0x52, 0x10, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69,
+	0x70, 0x74, 0x6f, 0x72, 0x12, 0x41, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74,
+	0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e, 0x66,
+	0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x46,
+	0x6c, 0x69, 0x67, 0x68, 0x74, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x08, 0x65,
+	0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x74, 0x6f, 0x74, 0x61, 0x6c,
+	0x5f, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c,
+	0x74, 0x6f, 0x74, 0x61, 0x6c, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x12, 0x1f, 0x0a, 0x0b,
+	0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28,
+	0x03, 0x52, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0x84, 0x01,
+	0x0a, 0x0e, 0x46, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74,
+	0x12, 0x35, 0x0a, 0x06, 0x74, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b,
+	0x32, 0x1d, 0x2e, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x52,
+	0x06, 0x74, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x3b, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74,
+	0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x61, 0x72, 0x72, 0x6f,
+	0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f,
+	0x6c, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x22, 0x1c, 0x0a, 0x08, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e,
+	0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x69, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75,
+	0x72, 0x69, 0x22, 0x20, 0x0a, 0x06, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06,
+	0x74, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x74, 0x69,
+	0x63, 0x6b, 0x65, 0x74, 0x22, 0xc4, 0x01, 0x0a, 0x0a, 0x46, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x44,
+	0x61, 0x74, 0x61, 0x12, 0x54, 0x0a, 0x11, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x5f, 0x64, 0x65,
+	0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27,
+	0x2e, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x44, 0x65, 0x73,
+	0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x10, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x44,
+	0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x61, 0x74,
+	0x61, 0x5f, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a,
+	0x64, 0x61, 0x74, 0x61, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x21, 0x0a, 0x0c, 0x61, 0x70,
+	0x70, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c,
+	0x52, 0x0b, 0x61, 0x70, 0x70, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x1c, 0x0a,
+	0x09, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x62, 0x6f, 0x64, 0x79, 0x18, 0xe8, 0x07, 0x20, 0x01, 0x28,
+	0x0c, 0x52, 0x08, 0x64, 0x61, 0x74, 0x61, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x2e, 0x0a, 0x09, 0x50,
+	0x75, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x61, 0x70, 0x70, 0x5f,
+	0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b,
+	0x61, 0x70, 0x70, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x32, 0xa7, 0x06, 0x0a, 0x0d,
+	0x46, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x64, 0x0a,
+	0x09, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x27, 0x2e, 0x61, 0x72, 0x72,
+	0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63,
+	0x6f, 0x6c, 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x52, 0x65, 0x71, 0x75,
+	0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67,
+	0x68, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x48, 0x61, 0x6e, 0x64,
+	0x73, 0x68, 0x61, 0x6b, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28,
+	0x01, 0x30, 0x01, 0x12, 0x55, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x6c, 0x69, 0x67, 0x68,
+	0x74, 0x73, 0x12, 0x1f, 0x2e, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67, 0x68,
+	0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x43, 0x72, 0x69, 0x74, 0x65,
+	0x72, 0x69, 0x61, 0x1a, 0x21, 0x2e, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67,
+	0x68, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x69, 0x67,
+	0x68, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0x00, 0x30, 0x01, 0x12, 0x5d, 0x0a, 0x0d, 0x47, 0x65,
+	0x74, 0x46, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x27, 0x2e, 0x61, 0x72,
+	0x72, 0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x63, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69,
+	0x70, 0x74, 0x6f, 0x72, 0x1a, 0x21, 0x2e, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69,
+	0x67, 0x68, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x69,
+	0x67, 0x68, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x09, 0x47, 0x65, 0x74,
+	0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x27, 0x2e, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e, 0x66,
+	0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x46,
+	0x6c, 0x69, 0x67, 0x68, 0x74, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x1a,
+	0x23, 0x2e, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65,
+	0x73, 0x75, 0x6c, 0x74, 0x22, 0x00, 0x12, 0x4d, 0x0a, 0x05, 0x44, 0x6f, 0x47, 0x65, 0x74, 0x12,
+	0x1d, 0x2e, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x1a, 0x21,
+	0x2e, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x44, 0x61, 0x74,
+	0x61, 0x22, 0x00, 0x30, 0x01, 0x12, 0x52, 0x0a, 0x05, 0x44, 0x6f, 0x50, 0x75, 0x74, 0x12, 0x21,
+	0x2e, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x44, 0x61, 0x74,
+	0x61, 0x1a, 0x20, 0x2e, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73,
+	0x75, 0x6c, 0x74, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x58, 0x0a, 0x0a, 0x44, 0x6f, 0x45,
+	0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x21, 0x2e, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e,
+	0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e,
+	0x46, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x44, 0x61, 0x74, 0x61, 0x1a, 0x21, 0x2e, 0x61, 0x72, 0x72,
+	0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63,
+	0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x44, 0x61, 0x74, 0x61, 0x22, 0x00, 0x28,
+	0x01, 0x30, 0x01, 0x12, 0x4c, 0x0a, 0x08, 0x44, 0x6f, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12,
+	0x1d, 0x2e, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x1d,
+	0x2e, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, 0x00, 0x30,
+	0x01, 0x12, 0x52, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73,
+	0x12, 0x1c, 0x2e, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x21,
+	0x2e, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70,
+	0x65, 0x22, 0x00, 0x30, 0x01, 0x42, 0x48, 0x0a, 0x1c, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61,
+	0x63, 0x68, 0x65, 0x2e, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2e, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74,
+	0x2e, 0x69, 0x6d, 0x70, 0x6c, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
+	0x6d, 0x2f, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2f, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x2f, 0x67,
+	0x6f, 0x2f, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x3b, 0x66, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x62,
+	0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_Flight_proto_rawDescOnce sync.Once
+	file_Flight_proto_rawDescData = file_Flight_proto_rawDesc
+)
+
+func file_Flight_proto_rawDescGZIP() []byte {
+	file_Flight_proto_rawDescOnce.Do(func() {
+		file_Flight_proto_rawDescData = protoimpl.X.CompressGZIP(file_Flight_proto_rawDescData)
+	})
+	return file_Flight_proto_rawDescData
+}
+
+var file_Flight_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
+var file_Flight_proto_msgTypes = make([]protoimpl.MessageInfo, 16)
+var file_Flight_proto_goTypes = []interface{}{
+	(FlightDescriptor_DescriptorType)(0), // 0: arrow.flight.protocol.FlightDescriptor.DescriptorType
+	(*HandshakeRequest)(nil),             // 1: arrow.flight.protocol.HandshakeRequest
+	(*HandshakeResponse)(nil),            // 2: arrow.flight.protocol.HandshakeResponse
+	(*BasicAuth)(nil),                    // 3: arrow.flight.protocol.BasicAuth
+	(*Empty)(nil),                        // 4: arrow.flight.protocol.Empty
+	(*ActionType)(nil),                   // 5: arrow.flight.protocol.ActionType
+	(*Criteria)(nil),                     // 6: arrow.flight.protocol.Criteria
+	(*Action)(nil),                       // 7: arrow.flight.protocol.Action
+	(*Result)(nil),                       // 8: arrow.flight.protocol.Result
+	(*SchemaResult)(nil),                 // 9: arrow.flight.protocol.SchemaResult
+	(*FlightDescriptor)(nil),             // 10: arrow.flight.protocol.FlightDescriptor
+	(*FlightInfo)(nil),                   // 11: arrow.flight.protocol.FlightInfo
+	(*FlightEndpoint)(nil),               // 12: arrow.flight.protocol.FlightEndpoint
+	(*Location)(nil),                     // 13: arrow.flight.protocol.Location
+	(*Ticket)(nil),                       // 14: arrow.flight.protocol.Ticket
+	(*FlightData)(nil),                   // 15: arrow.flight.protocol.FlightData
+	(*PutResult)(nil),                    // 16: arrow.flight.protocol.PutResult
+}
+var file_Flight_proto_depIdxs = []int32{
+	0,  // 0: arrow.flight.protocol.FlightDescriptor.type:type_name -> arrow.flight.protocol.FlightDescriptor.DescriptorType
+	10, // 1: arrow.flight.protocol.FlightInfo.flight_descriptor:type_name -> arrow.flight.protocol.FlightDescriptor
+	12, // 2: arrow.flight.protocol.FlightInfo.endpoint:type_name -> arrow.flight.protocol.FlightEndpoint
+	14, // 3: arrow.flight.protocol.FlightEndpoint.ticket:type_name -> arrow.flight.protocol.Ticket
+	13, // 4: arrow.flight.protocol.FlightEndpoint.location:type_name -> arrow.flight.protocol.Location
+	10, // 5: arrow.flight.protocol.FlightData.flight_descriptor:type_name -> arrow.flight.protocol.FlightDescriptor
+	1,  // 6: arrow.flight.protocol.FlightService.Handshake:input_type -> arrow.flight.protocol.HandshakeRequest
+	6,  // 7: arrow.flight.protocol.FlightService.ListFlights:input_type -> arrow.flight.protocol.Criteria
+	10, // 8: arrow.flight.protocol.FlightService.GetFlightInfo:input_type -> arrow.flight.protocol.FlightDescriptor
+	10, // 9: arrow.flight.protocol.FlightService.GetSchema:input_type -> arrow.flight.protocol.FlightDescriptor
+	14, // 10: arrow.flight.protocol.FlightService.DoGet:input_type -> arrow.flight.protocol.Ticket
+	15, // 11: arrow.flight.protocol.FlightService.DoPut:input_type -> arrow.flight.protocol.FlightData
+	15, // 12: arrow.flight.protocol.FlightService.DoExchange:input_type -> arrow.flight.protocol.FlightData
+	7,  // 13: arrow.flight.protocol.FlightService.DoAction:input_type -> arrow.flight.protocol.Action
+	4,  // 14: arrow.flight.protocol.FlightService.ListActions:input_type -> arrow.flight.protocol.Empty
+	2,  // 15: arrow.flight.protocol.FlightService.Handshake:output_type -> arrow.flight.protocol.HandshakeResponse
+	11, // 16: arrow.flight.protocol.FlightService.ListFlights:output_type -> arrow.flight.protocol.FlightInfo
+	11, // 17: arrow.flight.protocol.FlightService.GetFlightInfo:output_type -> arrow.flight.protocol.FlightInfo
+	9,  // 18: arrow.flight.protocol.FlightService.GetSchema:output_type -> arrow.flight.protocol.SchemaResult
+	15, // 19: arrow.flight.protocol.FlightService.DoGet:output_type -> arrow.flight.protocol.FlightData
+	16, // 20: arrow.flight.protocol.FlightService.DoPut:output_type -> arrow.flight.protocol.PutResult
+	15, // 21: arrow.flight.protocol.FlightService.DoExchange:output_type -> arrow.flight.protocol.FlightData
+	8,  // 22: arrow.flight.protocol.FlightService.DoAction:output_type -> arrow.flight.protocol.Result
+	5,  // 23: arrow.flight.protocol.FlightService.ListActions:output_type -> arrow.flight.protocol.ActionType
+	15, // [15:24] is the sub-list for method output_type
+	6,  // [6:15] is the sub-list for method input_type
+	6,  // [6:6] is the sub-list for extension type_name
+	6,  // [6:6] is the sub-list for extension extendee
+	0,  // [0:6] is the sub-list for field type_name
+}
+
+func init() { file_Flight_proto_init() }
+func file_Flight_proto_init() {
+	if File_Flight_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_Flight_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*HandshakeRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_Flight_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*HandshakeResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_Flight_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*BasicAuth); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_Flight_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Empty); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_Flight_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ActionType); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_Flight_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Criteria); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_Flight_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Action); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_Flight_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Result); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_Flight_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SchemaResult); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_Flight_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*FlightDescriptor); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_Flight_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*FlightInfo); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_Flight_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*FlightEndpoint); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_Flight_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Location); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_Flight_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Ticket); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_Flight_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*FlightData); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_Flight_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*PutResult); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_Flight_proto_rawDesc,
+			NumEnums:      1,
+			NumMessages:   16,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_Flight_proto_goTypes,
+		DependencyIndexes: file_Flight_proto_depIdxs,
+		EnumInfos:         file_Flight_proto_enumTypes,
+		MessageInfos:      file_Flight_proto_msgTypes,
+	}.Build()
+	File_Flight_proto = out.File
+	file_Flight_proto_rawDesc = nil
+	file_Flight_proto_goTypes = nil
+	file_Flight_proto_depIdxs = nil
+}
diff --git a/go/arrow/flight/Flight_grpc.pb.go b/go/arrow/flight/Flight_grpc.pb.go
new file mode 100644
index 00000000000..c2b98d5613e
--- /dev/null
+++ b/go/arrow/flight/Flight_grpc.pb.go
@@ -0,0 +1,877 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package flight
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion7
+
+// FlightServiceClient is the client API for FlightService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type FlightServiceClient interface {
+	//
+	// Handshake between client and server. Depending on the server, the
+	// handshake may be required to determine the token that should be used for
+	// future operations. Both request and response are streams to allow multiple
+	// round-trips depending on auth mechanism.
+	Handshake(ctx context.Context, opts ...grpc.CallOption) (FlightService_HandshakeClient, error)
+	//
+	// Get a list of available streams given a particular criteria. Most flight
+	// services will expose one or more streams that are readily available for
+	// retrieval. This api allows listing the streams available for
+	// consumption. A user can also provide a criteria. The criteria can limit
+	// the subset of streams that can be listed via this interface. Each flight
+	// service allows its own definition of how to consume criteria.
+	ListFlights(ctx context.Context, in *Criteria, opts ...grpc.CallOption) (FlightService_ListFlightsClient, error)
+	//
+	// For a given FlightDescriptor, get information about how the flight can be
+	// consumed. This is a useful interface if the consumer of the interface
+	// already can identify the specific flight to consume. This interface can
+	// also allow a consumer to generate a flight stream through a specified
+	// descriptor. For example, a flight descriptor might be something that
+	// includes a SQL statement or a Pickled Python operation that will be
+	// executed. In those cases, the descriptor will not be previously available
+	// within the list of available streams provided by ListFlights but will be
+	// available for consumption for the duration defined by the specific flight
+	// service.
+	GetFlightInfo(ctx context.Context, in *FlightDescriptor, opts ...grpc.CallOption) (*FlightInfo, error)
+	//
+	// For a given FlightDescriptor, get the Schema as described in Schema.fbs::Schema
+	// This is used when a consumer needs the Schema of flight stream. Similar to
+	// GetFlightInfo this interface may generate a new flight that was not previously
+	// available in ListFlights.
+	GetSchema(ctx context.Context, in *FlightDescriptor, opts ...grpc.CallOption) (*SchemaResult, error)
+	//
+	// Retrieve a single stream associated with a particular descriptor
+	// associated with the referenced ticket. A Flight can be composed of one or
+	// more streams where each stream can be retrieved using a separate opaque
+	// ticket that the flight service uses for managing a collection of streams.
+	DoGet(ctx context.Context, in *Ticket, opts ...grpc.CallOption) (FlightService_DoGetClient, error)
+	//
+	// Push a stream to the flight service associated with a particular
+	// flight stream. This allows a client of a flight service to upload a stream
+	// of data. Depending on the particular flight service, a client consumer
+	// could be allowed to upload a single stream per descriptor or an unlimited
+	// number. In the latter, the service might implement a 'seal' action that
+	// can be applied to a descriptor once all streams are uploaded.
+	DoPut(ctx context.Context, opts ...grpc.CallOption) (FlightService_DoPutClient, error)
+	//
+	// Open a bidirectional data channel for a given descriptor. This
+	// allows clients to send and receive arbitrary Arrow data and
+	// application-specific metadata in a single logical stream. In
+	// contrast to DoGet/DoPut, this is more suited for clients
+	// offloading computation (rather than storage) to a Flight service.
+	DoExchange(ctx context.Context, opts ...grpc.CallOption) (FlightService_DoExchangeClient, error)
+	//
+	// Flight services can support an arbitrary number of simple actions in
+	// addition to the possible ListFlights, GetFlightInfo, DoGet, DoPut
+	// operations that are potentially available. DoAction allows a flight client
+	// to do a specific action against a flight service. An action includes
+	// opaque request and response objects that are specific to the type action
+	// being undertaken.
+	DoAction(ctx context.Context, in *Action, opts ...grpc.CallOption) (FlightService_DoActionClient, error)
+	//
+	// A flight service exposes all of the available action types that it has
+	// along with descriptions. This allows different flight consumers to
+	// understand the capabilities of the flight service.
+	ListActions(ctx context.Context, in *Empty, opts ...grpc.CallOption) (FlightService_ListActionsClient, error)
+}
+
+type flightServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewFlightServiceClient(cc grpc.ClientConnInterface) FlightServiceClient {
+	return &flightServiceClient{cc}
+}
+
+var flightServiceHandshakeStreamDesc = &grpc.StreamDesc{
+	StreamName:    "Handshake",
+	ServerStreams: true,
+	ClientStreams: true,
+}
+
+func (c *flightServiceClient) Handshake(ctx context.Context, opts ...grpc.CallOption) (FlightService_HandshakeClient, error) {
+	stream, err := c.cc.NewStream(ctx, flightServiceHandshakeStreamDesc, "/arrow.flight.protocol.FlightService/Handshake", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &flightServiceHandshakeClient{stream}
+	return x, nil
+}
+
+type FlightService_HandshakeClient interface {
+	Send(*HandshakeRequest) error
+	Recv() (*HandshakeResponse, error)
+	grpc.ClientStream
+}
+
+type flightServiceHandshakeClient struct {
+	grpc.ClientStream
+}
+
+func (x *flightServiceHandshakeClient) Send(m *HandshakeRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *flightServiceHandshakeClient) Recv() (*HandshakeResponse, error) {
+	m := new(HandshakeResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var flightServiceListFlightsStreamDesc = &grpc.StreamDesc{
+	StreamName:    "ListFlights",
+	ServerStreams: true,
+}
+
+func (c *flightServiceClient) ListFlights(ctx context.Context, in *Criteria, opts ...grpc.CallOption) (FlightService_ListFlightsClient, error) {
+	stream, err := c.cc.NewStream(ctx, flightServiceListFlightsStreamDesc, "/arrow.flight.protocol.FlightService/ListFlights", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &flightServiceListFlightsClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type FlightService_ListFlightsClient interface {
+	Recv() (*FlightInfo, error)
+	grpc.ClientStream
+}
+
+type flightServiceListFlightsClient struct {
+	grpc.ClientStream
+}
+
+func (x *flightServiceListFlightsClient) Recv() (*FlightInfo, error) {
+	m := new(FlightInfo)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var flightServiceGetFlightInfoStreamDesc = &grpc.StreamDesc{
+	StreamName: "GetFlightInfo",
+}
+
+func (c *flightServiceClient) GetFlightInfo(ctx context.Context, in *FlightDescriptor, opts ...grpc.CallOption) (*FlightInfo, error) {
+	out := new(FlightInfo)
+	err := c.cc.Invoke(ctx, "/arrow.flight.protocol.FlightService/GetFlightInfo", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+var flightServiceGetSchemaStreamDesc = &grpc.StreamDesc{
+	StreamName: "GetSchema",
+}
+
+func (c *flightServiceClient) GetSchema(ctx context.Context, in *FlightDescriptor, opts ...grpc.CallOption) (*SchemaResult, error) {
+	out := new(SchemaResult)
+	err := c.cc.Invoke(ctx, "/arrow.flight.protocol.FlightService/GetSchema", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+var flightServiceDoGetStreamDesc = &grpc.StreamDesc{
+	StreamName:    "DoGet",
+	ServerStreams: true,
+}
+
+func (c *flightServiceClient) DoGet(ctx context.Context, in *Ticket, opts ...grpc.CallOption) (FlightService_DoGetClient, error) {
+	stream, err := c.cc.NewStream(ctx, flightServiceDoGetStreamDesc, "/arrow.flight.protocol.FlightService/DoGet", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &flightServiceDoGetClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type FlightService_DoGetClient interface {
+	Recv() (*FlightData, error)
+	grpc.ClientStream
+}
+
+type flightServiceDoGetClient struct {
+	grpc.ClientStream
+}
+
+func (x *flightServiceDoGetClient) Recv() (*FlightData, error) {
+	m := new(FlightData)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var flightServiceDoPutStreamDesc = &grpc.StreamDesc{
+	StreamName:    "DoPut",
+	ServerStreams: true,
+	ClientStreams: true,
+}
+
+func (c *flightServiceClient) DoPut(ctx context.Context, opts ...grpc.CallOption) (FlightService_DoPutClient, error) {
+	stream, err := c.cc.NewStream(ctx, flightServiceDoPutStreamDesc, "/arrow.flight.protocol.FlightService/DoPut", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &flightServiceDoPutClient{stream}
+	return x, nil
+}
+
+type FlightService_DoPutClient interface {
+	Send(*FlightData) error
+	Recv() (*PutResult, error)
+	grpc.ClientStream
+}
+
+type flightServiceDoPutClient struct {
+	grpc.ClientStream
+}
+
+func (x *flightServiceDoPutClient) Send(m *FlightData) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *flightServiceDoPutClient) Recv() (*PutResult, error) {
+	m := new(PutResult)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var flightServiceDoExchangeStreamDesc = &grpc.StreamDesc{
+	StreamName:    "DoExchange",
+	ServerStreams: true,
+	ClientStreams: true,
+}
+
+func (c *flightServiceClient) DoExchange(ctx context.Context, opts ...grpc.CallOption) (FlightService_DoExchangeClient, error) {
+	stream, err := c.cc.NewStream(ctx, flightServiceDoExchangeStreamDesc, "/arrow.flight.protocol.FlightService/DoExchange", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &flightServiceDoExchangeClient{stream}
+	return x, nil
+}
+
+type FlightService_DoExchangeClient interface {
+	Send(*FlightData) error
+	Recv() (*FlightData, error)
+	grpc.ClientStream
+}
+
+type flightServiceDoExchangeClient struct {
+	grpc.ClientStream
+}
+
+func (x *flightServiceDoExchangeClient) Send(m *FlightData) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *flightServiceDoExchangeClient) Recv() (*FlightData, error) {
+	m := new(FlightData)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var flightServiceDoActionStreamDesc = &grpc.StreamDesc{
+	StreamName:    "DoAction",
+	ServerStreams: true,
+}
+
+func (c *flightServiceClient) DoAction(ctx context.Context, in *Action, opts ...grpc.CallOption) (FlightService_DoActionClient, error) {
+	stream, err := c.cc.NewStream(ctx, flightServiceDoActionStreamDesc, "/arrow.flight.protocol.FlightService/DoAction", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &flightServiceDoActionClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type FlightService_DoActionClient interface {
+	Recv() (*Result, error)
+	grpc.ClientStream
+}
+
+type flightServiceDoActionClient struct {
+	grpc.ClientStream
+}
+
+func (x *flightServiceDoActionClient) Recv() (*Result, error) {
+	m := new(Result)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var flightServiceListActionsStreamDesc = &grpc.StreamDesc{
+	StreamName:    "ListActions",
+	ServerStreams: true,
+}
+
+func (c *flightServiceClient) ListActions(ctx context.Context, in *Empty, opts ...grpc.CallOption) (FlightService_ListActionsClient, error) {
+	stream, err := c.cc.NewStream(ctx, flightServiceListActionsStreamDesc, "/arrow.flight.protocol.FlightService/ListActions", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &flightServiceListActionsClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type FlightService_ListActionsClient interface {
+	Recv() (*ActionType, error)
+	grpc.ClientStream
+}
+
+type flightServiceListActionsClient struct {
+	grpc.ClientStream
+}
+
+func (x *flightServiceListActionsClient) Recv() (*ActionType, error) {
+	m := new(ActionType)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// FlightServiceService is the service API for FlightService service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterFlightServiceService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type FlightServiceService struct {
+	//
+	// Handshake between client and server. Depending on the server, the
+	// handshake may be required to determine the token that should be used for
+	// future operations. Both request and response are streams to allow multiple
+	// round-trips depending on auth mechanism.
+	Handshake func(FlightService_HandshakeServer) error
+	//
+	// Get a list of available streams given a particular criteria. Most flight
+	// services will expose one or more streams that are readily available for
+	// retrieval. This api allows listing the streams available for
+	// consumption. A user can also provide a criteria. The criteria can limit
+	// the subset of streams that can be listed via this interface. Each flight
+	// service allows its own definition of how to consume criteria.
+	ListFlights func(*Criteria, FlightService_ListFlightsServer) error
+	//
+	// For a given FlightDescriptor, get information about how the flight can be
+	// consumed. This is a useful interface if the consumer of the interface
+	// already can identify the specific flight to consume. This interface can
+	// also allow a consumer to generate a flight stream through a specified
+	// descriptor. For example, a flight descriptor might be something that
+	// includes a SQL statement or a Pickled Python operation that will be
+	// executed. In those cases, the descriptor will not be previously available
+	// within the list of available streams provided by ListFlights but will be
+	// available for consumption for the duration defined by the specific flight
+	// service.
+	GetFlightInfo func(context.Context, *FlightDescriptor) (*FlightInfo, error)
+	//
+	// For a given FlightDescriptor, get the Schema as described in Schema.fbs::Schema
+	// This is used when a consumer needs the Schema of flight stream. Similar to
+	// GetFlightInfo this interface may generate a new flight that was not previously
+	// available in ListFlights.
+	GetSchema func(context.Context, *FlightDescriptor) (*SchemaResult, error)
+	//
+	// Retrieve a single stream associated with a particular descriptor
+	// associated with the referenced ticket. A Flight can be composed of one or
+	// more streams where each stream can be retrieved using a separate opaque
+	// ticket that the flight service uses for managing a collection of streams.
+	DoGet func(*Ticket, FlightService_DoGetServer) error
+	//
+	// Push a stream to the flight service associated with a particular
+	// flight stream. This allows a client of a flight service to upload a stream
+	// of data. Depending on the particular flight service, a client consumer
+	// could be allowed to upload a single stream per descriptor or an unlimited
+	// number. In the latter, the service might implement a 'seal' action that
+	// can be applied to a descriptor once all streams are uploaded.
+	DoPut func(FlightService_DoPutServer) error
+	//
+	// Open a bidirectional data channel for a given descriptor. This
+	// allows clients to send and receive arbitrary Arrow data and
+	// application-specific metadata in a single logical stream. In
+	// contrast to DoGet/DoPut, this is more suited for clients
+	// offloading computation (rather than storage) to a Flight service.
+	DoExchange func(FlightService_DoExchangeServer) error
+	//
+	// Flight services can support an arbitrary number of simple actions in
+	// addition to the possible ListFlights, GetFlightInfo, DoGet, DoPut
+	// operations that are potentially available. DoAction allows a flight client
+	// to do a specific action against a flight service. An action includes
+	// opaque request and response objects that are specific to the type action
+	// being undertaken.
+	DoAction func(*Action, FlightService_DoActionServer) error
+	//
+	// A flight service exposes all of the available action types that it has
+	// along with descriptions. This allows different flight consumers to
+	// understand the capabilities of the flight service.
+	ListActions func(*Empty, FlightService_ListActionsServer) error
+}
+
+func (s *FlightServiceService) handshake(_ interface{}, stream grpc.ServerStream) error {
+	return s.Handshake(&flightServiceHandshakeServer{stream})
+}
+func (s *FlightServiceService) listFlights(_ interface{}, stream grpc.ServerStream) error {
+	m := new(Criteria)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return s.ListFlights(m, &flightServiceListFlightsServer{stream})
+}
+func (s *FlightServiceService) getFlightInfo(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(FlightDescriptor)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return s.GetFlightInfo(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     s,
+		FullMethod: "/arrow.flight.protocol.FlightService/GetFlightInfo",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return s.GetFlightInfo(ctx, req.(*FlightDescriptor))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+func (s *FlightServiceService) getSchema(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(FlightDescriptor)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return s.GetSchema(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     s,
+		FullMethod: "/arrow.flight.protocol.FlightService/GetSchema",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return s.GetSchema(ctx, req.(*FlightDescriptor))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+func (s *FlightServiceService) doGet(_ interface{}, stream grpc.ServerStream) error {
+	m := new(Ticket)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return s.DoGet(m, &flightServiceDoGetServer{stream})
+}
+func (s *FlightServiceService) doPut(_ interface{}, stream grpc.ServerStream) error {
+	return s.DoPut(&flightServiceDoPutServer{stream})
+}
+func (s *FlightServiceService) doExchange(_ interface{}, stream grpc.ServerStream) error {
+	return s.DoExchange(&flightServiceDoExchangeServer{stream})
+}
+func (s *FlightServiceService) doAction(_ interface{}, stream grpc.ServerStream) error {
+	m := new(Action)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return s.DoAction(m, &flightServiceDoActionServer{stream})
+}
+func (s *FlightServiceService) listActions(_ interface{}, stream grpc.ServerStream) error {
+	m := new(Empty)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return s.ListActions(m, &flightServiceListActionsServer{stream})
+}
+
+type FlightService_HandshakeServer interface {
+	Send(*HandshakeResponse) error
+	Recv() (*HandshakeRequest, error)
+	grpc.ServerStream
+}
+
+type flightServiceHandshakeServer struct {
+	grpc.ServerStream
+}
+
+func (x *flightServiceHandshakeServer) Send(m *HandshakeResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *flightServiceHandshakeServer) Recv() (*HandshakeRequest, error) {
+	m := new(HandshakeRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+type FlightService_ListFlightsServer interface {
+	Send(*FlightInfo) error
+	grpc.ServerStream
+}
+
+type flightServiceListFlightsServer struct {
+	grpc.ServerStream
+}
+
+func (x *flightServiceListFlightsServer) Send(m *FlightInfo) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+type FlightService_DoGetServer interface {
+	Send(*FlightData) error
+	grpc.ServerStream
+}
+
+type flightServiceDoGetServer struct {
+	grpc.ServerStream
+}
+
+func (x *flightServiceDoGetServer) Send(m *FlightData) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+type FlightService_DoPutServer interface {
+	Send(*PutResult) error
+	Recv() (*FlightData, error)
+	grpc.ServerStream
+}
+
+type flightServiceDoPutServer struct {
+	grpc.ServerStream
+}
+
+func (x *flightServiceDoPutServer) Send(m *PutResult) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *flightServiceDoPutServer) Recv() (*FlightData, error) {
+	m := new(FlightData)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+type FlightService_DoExchangeServer interface {
+	Send(*FlightData) error
+	Recv() (*FlightData, error)
+	grpc.ServerStream
+}
+
+type flightServiceDoExchangeServer struct {
+	grpc.ServerStream
+}
+
+func (x *flightServiceDoExchangeServer) Send(m *FlightData) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *flightServiceDoExchangeServer) Recv() (*FlightData, error) {
+	m := new(FlightData)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+type FlightService_DoActionServer interface {
+	Send(*Result) error
+	grpc.ServerStream
+}
+
+type flightServiceDoActionServer struct {
+	grpc.ServerStream
+}
+
+func (x *flightServiceDoActionServer) Send(m *Result) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+type FlightService_ListActionsServer interface {
+	Send(*ActionType) error
+	grpc.ServerStream
+}
+
+type flightServiceListActionsServer struct {
+	grpc.ServerStream
+}
+
+func (x *flightServiceListActionsServer) Send(m *ActionType) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+// RegisterFlightServiceService registers a service implementation with a gRPC server.
+func RegisterFlightServiceService(s grpc.ServiceRegistrar, srv *FlightServiceService) {
+	srvCopy := *srv
+	if srvCopy.Handshake == nil {
+		srvCopy.Handshake = func(FlightService_HandshakeServer) error {
+			return status.Errorf(codes.Unimplemented, "method Handshake not implemented")
+		}
+	}
+	if srvCopy.ListFlights == nil {
+		srvCopy.ListFlights = func(*Criteria, FlightService_ListFlightsServer) error {
+			return status.Errorf(codes.Unimplemented, "method ListFlights not implemented")
+		}
+	}
+	if srvCopy.GetFlightInfo == nil {
+		srvCopy.GetFlightInfo = func(context.Context, *FlightDescriptor) (*FlightInfo, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method GetFlightInfo not implemented")
+		}
+	}
+	if srvCopy.GetSchema == nil {
+		srvCopy.GetSchema = func(context.Context, *FlightDescriptor) (*SchemaResult, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method GetSchema not implemented")
+		}
+	}
+	if srvCopy.DoGet == nil {
+		srvCopy.DoGet = func(*Ticket, FlightService_DoGetServer) error {
+			return status.Errorf(codes.Unimplemented, "method DoGet not implemented")
+		}
+	}
+	if srvCopy.DoPut == nil {
+		srvCopy.DoPut = func(FlightService_DoPutServer) error {
+			return status.Errorf(codes.Unimplemented, "method DoPut not implemented")
+		}
+	}
+	if srvCopy.DoExchange == nil {
+		srvCopy.DoExchange = func(FlightService_DoExchangeServer) error {
+			return status.Errorf(codes.Unimplemented, "method DoExchange not implemented")
+		}
+	}
+	if srvCopy.DoAction == nil {
+		srvCopy.DoAction = func(*Action, FlightService_DoActionServer) error {
+			return status.Errorf(codes.Unimplemented, "method DoAction not implemented")
+		}
+	}
+	if srvCopy.ListActions == nil {
+		srvCopy.ListActions = func(*Empty, FlightService_ListActionsServer) error {
+			return status.Errorf(codes.Unimplemented, "method ListActions not implemented")
+		}
+	}
+	sd := grpc.ServiceDesc{
+		ServiceName: "arrow.flight.protocol.FlightService",
+		Methods: []grpc.MethodDesc{
+			{
+				MethodName: "GetFlightInfo",
+				Handler:    srvCopy.getFlightInfo,
+			},
+			{
+				MethodName: "GetSchema",
+				Handler:    srvCopy.getSchema,
+			},
+		},
+		Streams: []grpc.StreamDesc{
+			{
+				StreamName:    "Handshake",
+				Handler:       srvCopy.handshake,
+				ServerStreams: true,
+				ClientStreams: true,
+			},
+			{
+				StreamName:    "ListFlights",
+				Handler:       srvCopy.listFlights,
+				ServerStreams: true,
+			},
+			{
+				StreamName:    "DoGet",
+				Handler:       srvCopy.doGet,
+				ServerStreams: true,
+			},
+			{
+				StreamName:    "DoPut",
+				Handler:       srvCopy.doPut,
+				ServerStreams: true,
+				ClientStreams: true,
+			},
+			{
+				StreamName:    "DoExchange",
+				Handler:       srvCopy.doExchange,
+				ServerStreams: true,
+				ClientStreams: true,
+			},
+			{
+				StreamName:    "DoAction",
+				Handler:       srvCopy.doAction,
+				ServerStreams: true,
+			},
+			{
+				StreamName:    "ListActions",
+				Handler:       srvCopy.listActions,
+				ServerStreams: true,
+			},
+		},
+		Metadata: "Flight.proto",
+	}
+
+	s.RegisterService(&sd, nil)
+}
+
+// NewFlightServiceService creates a new FlightServiceService containing the
+// implemented methods of the FlightService service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewFlightServiceService(s interface{}) *FlightServiceService {
+	ns := &FlightServiceService{}
+	if h, ok := s.(interface {
+		Handshake(FlightService_HandshakeServer) error
+	}); ok {
+		ns.Handshake = h.Handshake
+	}
+	if h, ok := s.(interface {
+		ListFlights(*Criteria, FlightService_ListFlightsServer) error
+	}); ok {
+		ns.ListFlights = h.ListFlights
+	}
+	if h, ok := s.(interface {
+		GetFlightInfo(context.Context, *FlightDescriptor) (*FlightInfo, error)
+	}); ok {
+		ns.GetFlightInfo = h.GetFlightInfo
+	}
+	if h, ok := s.(interface {
+		GetSchema(context.Context, *FlightDescriptor) (*SchemaResult, error)
+	}); ok {
+		ns.GetSchema = h.GetSchema
+	}
+	if h, ok := s.(interface {
+		DoGet(*Ticket, FlightService_DoGetServer) error
+	}); ok {
+		ns.DoGet = h.DoGet
+	}
+	if h, ok := s.(interface {
+		DoPut(FlightService_DoPutServer) error
+	}); ok {
+		ns.DoPut = h.DoPut
+	}
+	if h, ok := s.(interface {
+		DoExchange(FlightService_DoExchangeServer) error
+	}); ok {
+		ns.DoExchange = h.DoExchange
+	}
+	if h, ok := s.(interface {
+		DoAction(*Action, FlightService_DoActionServer) error
+	}); ok {
+		ns.DoAction = h.DoAction
+	}
+	if h, ok := s.(interface {
+		ListActions(*Empty, FlightService_ListActionsServer) error
+	}); ok {
+		ns.ListActions = h.ListActions
+	}
+	return ns
+}
+
+// UnstableFlightServiceService is the service API for FlightService service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableFlightServiceService interface {
+	//
+	// Handshake between client and server. Depending on the server, the
+	// handshake may be required to determine the token that should be used for
+	// future operations. Both request and response are streams to allow multiple
+	// round-trips depending on auth mechanism.
+	Handshake(FlightService_HandshakeServer) error
+	//
+	// Get a list of available streams given a particular criteria. Most flight
+	// services will expose one or more streams that are readily available for
+	// retrieval. This api allows listing the streams available for
+	// consumption. A user can also provide a criteria. The criteria can limit
+	// the subset of streams that can be listed via this interface. Each flight
+	// service allows its own definition of how to consume criteria.
+	ListFlights(*Criteria, FlightService_ListFlightsServer) error
+	//
+	// For a given FlightDescriptor, get information about how the flight can be
+	// consumed. This is a useful interface if the consumer of the interface
+	// already can identify the specific flight to consume. This interface can
+	// also allow a consumer to generate a flight stream through a specified
+	// descriptor. For example, a flight descriptor might be something that
+	// includes a SQL statement or a Pickled Python operation that will be
+	// executed. In those cases, the descriptor will not be previously available
+	// within the list of available streams provided by ListFlights but will be
+	// available for consumption for the duration defined by the specific flight
+	// service.
+	GetFlightInfo(context.Context, *FlightDescriptor) (*FlightInfo, error)
+	//
+	// For a given FlightDescriptor, get the Schema as described in Schema.fbs::Schema
+	// This is used when a consumer needs the Schema of flight stream. Similar to
+	// GetFlightInfo this interface may generate a new flight that was not previously
+	// available in ListFlights.
+	GetSchema(context.Context, *FlightDescriptor) (*SchemaResult, error)
+	//
+	// Retrieve a single stream associated with a particular descriptor
+	// associated with the referenced ticket. A Flight can be composed of one or
+	// more streams where each stream can be retrieved using a separate opaque
+	// ticket that the flight service uses for managing a collection of streams.
+	DoGet(*Ticket, FlightService_DoGetServer) error
+	//
+	// Push a stream to the flight service associated with a particular
+	// flight stream. This allows a client of a flight service to upload a stream
+	// of data. Depending on the particular flight service, a client consumer
+	// could be allowed to upload a single stream per descriptor or an unlimited
+	// number. In the latter, the service might implement a 'seal' action that
+	// can be applied to a descriptor once all streams are uploaded.
+	DoPut(FlightService_DoPutServer) error
+	//
+	// Open a bidirectional data channel for a given descriptor. This
+	// allows clients to send and receive arbitrary Arrow data and
+	// application-specific metadata in a single logical stream. In
+	// contrast to DoGet/DoPut, this is more suited for clients
+	// offloading computation (rather than storage) to a Flight service.
+	DoExchange(FlightService_DoExchangeServer) error
+	//
+	// Flight services can support an arbitrary number of simple actions in
+	// addition to the possible ListFlights, GetFlightInfo, DoGet, DoPut
+	// operations that are potentially available. DoAction allows a flight client
+	// to do a specific action against a flight service. An action includes
+	// opaque request and response objects that are specific to the type action
+	// being undertaken.
+	DoAction(*Action, FlightService_DoActionServer) error
+	//
+	// A flight service exposes all of the available action types that it has
+	// along with descriptions. This allows different flight consumers to
+	// understand the capabilities of the flight service.
+	ListActions(*Empty, FlightService_ListActionsServer) error
+}
diff --git a/go/arrow/flight/client.go b/go/arrow/flight/client.go
new file mode 100644
index 00000000000..7fe1f5d96a8
--- /dev/null
+++ b/go/arrow/flight/client.go
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package flight
+
+import (
+	"context"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// Client is an interface wrapped around the generated FlightServiceClient which is
+// generated by grpc protobuf definitions. This interface provides a useful hiding
+// of the authentication handshake via calling Authenticate and using the
+// ClientAuthHandler rather than manually having to implement the grpc communication
+// and sending of the auth token.
+type Client interface {
+	// Authenticate uses the ClientAuthHandler that was used when creating the client
+	// in order to use the Handshake endpoints of the service.
+	Authenticate(context.Context, ...grpc.CallOption) error
+	Close() error
+	// join the interface from the FlightServiceClient instead of re-defining all
+	// the endpoints here.
+	FlightServiceClient
+}
+
+type client struct {
+	conn        *grpc.ClientConn
+	authHandler ClientAuthHandler
+
+	FlightServiceClient
+}
+
+// NewFlightClient takes in the address of the grpc server and an auth handler for the
+// application-level handshake. If using TLS or other grpc configurations they can still
+// be passed via the grpc.DialOption list just as if connecting manually without this
+// helper function.
+//
+// Alternatively, a grpc client can be constructed as normal without this helper as the
+// grpc generated client code is still exported. This exists to add utility and helpers
+// around the authentication and passing the token with requests.
+func NewFlightClient(addr string, auth ClientAuthHandler, opts ...grpc.DialOption) (Client, error) {
+	if auth != nil {
+		opts = append([]grpc.DialOption{
+			grpc.WithChainStreamInterceptor(createClientAuthStreamInterceptor(auth)),
+			grpc.WithChainUnaryInterceptor(createClientAuthUnaryInterceptor(auth)),
+		}, opts...)
+	}
+
+	conn, err := grpc.Dial(addr, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return &client{conn: conn, FlightServiceClient: NewFlightServiceClient(conn), authHandler: auth}, nil
+}
+
+func (c *client) Authenticate(ctx context.Context, opts ...grpc.CallOption) error {
+	if c.authHandler == nil {
+		return status.Error(codes.NotFound, "cannot authenticate without an auth-handler")
+	}
+
+	stream, err := c.FlightServiceClient.Handshake(ctx, opts...)
+	if err != nil {
+		return err
+	}
+
+	return c.authHandler.Authenticate(ctx, &clientAuthConn{stream})
+}
+
+func (c *client) Close() error {
+	c.FlightServiceClient = nil
+	return c.conn.Close()
+}
diff --git a/go/arrow/flight/client_auth.go b/go/arrow/flight/client_auth.go
new file mode 100644
index 00000000000..5f7c151abf7
--- /dev/null
+++ b/go/arrow/flight/client_auth.go
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package flight
+
+import (
+	"context"
+	"strings"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
+)
+
+// ClientAuthHandler defines an interface for the Flight client to perform
+// the authentication handshake. The token that is retrieved from GetToken
+// will be sent as part of the context metadata in subsequent requests after
+// authentication is performed using the key "auth-token-bin".
+type ClientAuthHandler interface {
+	Authenticate(context.Context, AuthConn) error
+	GetToken(context.Context) (string, error)
+}
+
+type clientAuthConn struct {
+	stream FlightService_HandshakeClient
+}
+
+func (a *clientAuthConn) Read() ([]byte, error) {
+	in, err := a.stream.Recv()
+	if err != nil {
+		return nil, err
+	}
+
+	return in.Payload, nil
+}
+
+func (a *clientAuthConn) Send(b []byte) error {
+	return a.stream.Send(&HandshakeRequest{Payload: b})
+}
+
+func createClientAuthUnaryInterceptor(auth ClientAuthHandler) grpc.UnaryClientInterceptor {
+	if auth == nil {
+		return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+			return invoker(ctx, method, req, reply, cc, opts...)
+		}
+	}
+
+	return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+		tok, err := auth.GetToken(ctx)
+		if err != nil {
+			return status.Errorf(codes.Unauthenticated, "error retrieving token: %s", err)
+		}
+
+		return invoker(metadata.NewOutgoingContext(ctx, metadata.Pairs(grpcAuthHeader, tok)), method, req, reply, cc, opts...)
+	}
+}
+
+func createClientAuthStreamInterceptor(auth ClientAuthHandler) grpc.StreamClientInterceptor {
+	if auth == nil {
+		return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
+			return streamer(ctx, desc, cc, method, opts...)
+		}
+	}
+
+	return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
+		if strings.HasSuffix(method, "/Handshake") {
+			return streamer(ctx, desc, cc, method, opts...)
+		}
+
+		tok, err := auth.GetToken(ctx)
+		if err != nil {
+			return nil, status.Errorf(codes.Unauthenticated, "error retrieving token: %s", err)
+		}
+
+		return streamer(metadata.NewOutgoingContext(ctx, metadata.Pairs(grpcAuthHeader, tok)), desc, cc, method, opts...)
+	}
+}
diff --git a/go/arrow/flight/example_flight_server_test.go b/go/arrow/flight/example_flight_server_test.go
new file mode 100644
index 00000000000..01f291fda11
--- /dev/null
+++ b/go/arrow/flight/example_flight_server_test.go
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package flight_test
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log"
+
+	"github.com/apache/arrow/go/arrow/flight"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+type serverAuth struct{}
+
+func (sa *serverAuth) Authenticate(c flight.AuthConn) error {
+	in, err := c.Read()
+	if err == io.EOF {
+		return status.Error(codes.Unauthenticated, "no auth info provided")
+	}
+
+	if err != nil {
+		return status.Error(codes.FailedPrecondition, "error reading auth handshake")
+	}
+
+	// do something with in....
+	fmt.Println(string(in))
+
+	// send auth token back
+	return c.Send([]byte("foobar"))
+}
+
+func (sa *serverAuth) IsValid(token string) (interface{}, error) {
+	if token == "foobar" {
+		return "foo", nil
+	}
+	return "", status.Error(codes.PermissionDenied, "invalid auth token")
+}
+
+func Example_server() {
+	server := flight.NewFlightServer(&serverAuth{})
+	server.Init("localhost:0")
+	server.RegisterFlightService(&flight.FlightServiceService{})
+
+	go server.Serve()
+	defer server.Shutdown()
+
+	conn, err := grpc.Dial(server.Addr().String(), grpc.WithInsecure())
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer conn.Close()
+
+	client := flight.NewFlightServiceClient(conn)
+	stream, err := client.Handshake(context.Background())
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// ignore error handling here for brevity
+	stream.Send(&flight.HandshakeRequest{Payload: []byte("baz")})
+
+	resp, _ := stream.Recv()
+	fmt.Println(string(resp.Payload))
+
+	// Output:
+	// baz
+	// foobar
+}
diff --git a/go/arrow/flight/flight_test.go b/go/arrow/flight/flight_test.go
new file mode 100644
index 00000000000..d771445ce06
--- /dev/null
+++ b/go/arrow/flight/flight_test.go
@@ -0,0 +1,313 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package flight_test
+
+import (
+	"context"
+	"errors"
+	"io"
+	"testing"
+
+	"github.com/apache/arrow/go/arrow/array"
+	"github.com/apache/arrow/go/arrow/flight"
+	"github.com/apache/arrow/go/arrow/internal/arrdata"
+	"github.com/apache/arrow/go/arrow/ipc"
+	"github.com/apache/arrow/go/arrow/memory"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+type flightServer struct {
+	mem memory.Allocator
+}
+
+func (f *flightServer) getmem() memory.Allocator {
+	if f.mem == nil {
+		f.mem = memory.NewGoAllocator()
+	}
+
+	return f.mem
+}
+
+func (f *flightServer) ListFlights(c *flight.Criteria, fs flight.FlightService_ListFlightsServer) error {
+	expr := string(c.GetExpression())
+
+	auth := ""
+	authVal := flight.AuthFromContext(fs.Context())
+	if authVal != nil {
+		auth = authVal.(string)
+	}
+
+	for _, name := range arrdata.RecordNames {
+		if expr != "" && expr != name {
+			continue
+		}
+
+		recs := arrdata.Records[name]
+		totalRows := int64(0)
+		for _, r := range recs {
+			totalRows += r.NumRows()
+		}
+
+		fs.Send(&flight.FlightInfo{
+			Schema: ipc.FlightInfoSchemaBytes(recs[0].Schema(), f.getmem()),
+			FlightDescriptor: &flight.FlightDescriptor{
+				Type: flight.FlightDescriptor_PATH,
+				Path: []string{name, auth},
+			},
+			TotalRecords: totalRows,
+			TotalBytes:   -1,
+		})
+	}
+
+	return nil
+}
+
+func (f *flightServer) GetSchema(_ context.Context, in *flight.FlightDescriptor) (*flight.SchemaResult, error) {
+	if in == nil {
+		return nil, status.Error(codes.InvalidArgument, "invalid flight descriptor")
+	}
+
+	recs, ok := arrdata.Records[in.Path[0]]
+	if !ok {
+		return nil, status.Error(codes.NotFound, "flight not found")
+	}
+
+	return &flight.SchemaResult{Schema: ipc.FlightInfoSchemaBytes(recs[0].Schema(), f.getmem())}, nil
+}
+
+func (f *flightServer) DoGet(tkt *flight.Ticket, fs flight.FlightService_DoGetServer) error {
+	recs := arrdata.Records[string(tkt.GetTicket())]
+
+	w := ipc.NewFlightDataWriter(fs, ipc.WithSchema(recs[0].Schema()))
+	for _, r := range recs {
+		w.Write(r)
+	}
+
+	return nil
+}
+
+type servAuth struct{}
+
+func (a *servAuth) Authenticate(c flight.AuthConn) error {
+	tok, err := c.Read()
+	if err == io.EOF {
+		return nil
+	}
+
+	if string(tok) != "foobar" {
+		return errors.New("novalid")
+	}
+
+	if err != nil {
+		return err
+	}
+
+	return c.Send([]byte("baz"))
+}
+
+func (a *servAuth) IsValid(token string) (interface{}, error) {
+	if token == "baz" {
+		return "bar", nil
+	}
+	return "", errors.New("novalid")
+}
+
+type ctxauth struct{}
+
+type clientAuth struct{}
+
+func (a *clientAuth) Authenticate(ctx context.Context, c flight.AuthConn) error {
+	if err := c.Send(ctx.Value(ctxauth{}).([]byte)); err != nil {
+		return err
+	}
+
+	_, err := c.Read()
+	return err
+}
+
+func (a *clientAuth) GetToken(ctx context.Context) (string, error) {
+	return ctx.Value(ctxauth{}).(string), nil
+}
+
+func TestListFlights(t *testing.T) {
+	s := flight.NewFlightServer(nil)
+	s.Init("localhost:0")
+	f := &flightServer{}
+	s.RegisterFlightService(&flight.FlightServiceService{
+		ListFlights: f.ListFlights,
+	})
+
+	go s.Serve()
+	defer s.Shutdown()
+
+	client, err := flight.NewFlightClient(s.Addr().String(), nil, grpc.WithInsecure())
+	if err != nil {
+		t.Error(err)
+	}
+	defer client.Close()
+
+	flightStream, err := client.ListFlights(context.Background(), &flight.Criteria{})
+	if err != nil {
+		t.Error(err)
+	}
+
+	for {
+		info, err := flightStream.Recv()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			t.Error(err)
+		}
+
+		fname := info.GetFlightDescriptor().GetPath()[0]
+		recs, ok := arrdata.Records[fname]
+		if !ok {
+			t.Fatalf("got unknown flight info: %s", fname)
+		}
+
+		sc, err := ipc.SchemaFromFlightInfo(info.GetSchema())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !recs[0].Schema().Equal(sc) {
+			t.Fatalf("flight info schema transfer failed: \ngot = %#v\nwant = %#v\n", sc, recs[0].Schema())
+		}
+
+		var total int64 = 0
+		for _, r := range recs {
+			total += r.NumRows()
+		}
+
+		if info.TotalRecords != total {
+			t.Fatalf("got wrong number of total records: got = %d, wanted = %d", info.TotalRecords, total)
+		}
+	}
+}
+
+func TestGetSchema(t *testing.T) {
+	s := flight.NewFlightServer(nil)
+	s.Init("localhost:0")
+	f := &flightServer{}
+	s.RegisterFlightService(&flight.FlightServiceService{
+		GetSchema: f.GetSchema,
+	})
+
+	go s.Serve()
+	defer s.Shutdown()
+
+	client, err := flight.NewFlightClient(s.Addr().String(), nil, grpc.WithInsecure())
+	if err != nil {
+		t.Error(err)
+	}
+	defer client.Close()
+
+	for name, testrecs := range arrdata.Records {
+		t.Run("flight get schema: "+name, func(t *testing.T) {
+			res, err := client.GetSchema(context.Background(), &flight.FlightDescriptor{Path: []string{name}})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			schema, err := ipc.SchemaFromFlightInfo(res.GetSchema())
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if !testrecs[0].Schema().Equal(schema) {
+				t.Fatalf("schema not match: \ngot = %#v\nwant = %#v\n", schema, testrecs[0].Schema())
+			}
+		})
+	}
+}
+
+func TestServer(t *testing.T) {
+	f := &flightServer{}
+	service := &flight.FlightServiceService{
+		ListFlights: f.ListFlights,
+		DoGet:       f.DoGet,
+	}
+
+	s := flight.NewFlightServer(&servAuth{})
+	s.Init("localhost:0")
+	s.RegisterFlightService(service)
+
+	go s.Serve()
+	defer s.Shutdown()
+
+	client, err := flight.NewFlightClient(s.Addr().String(), &clientAuth{}, grpc.WithInsecure())
+	if err != nil {
+		t.Error(err)
+	}
+	defer client.Close()
+
+	err = client.Authenticate(context.WithValue(context.Background(), ctxauth{}, []byte("foobar")))
+	if err != nil {
+		t.Error(err)
+	}
+
+	ctx := context.WithValue(context.Background(), ctxauth{}, "baz")
+
+	fistream, err := client.ListFlights(ctx, &flight.Criteria{Expression: []byte("decimal128")})
+	if err != nil {
+		t.Error(err)
+	}
+
+	fi, err := fistream.Recv()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(fi.FlightDescriptor.GetPath()) != 2 || fi.FlightDescriptor.GetPath()[1] != "bar" {
+		t.Fatalf("path should have auth info: want %s got %s", "bar", fi.FlightDescriptor.GetPath()[1])
+	}
+
+	fdata, err := client.DoGet(ctx, &flight.Ticket{Ticket: []byte("decimal128")})
+	if err != nil {
+		t.Error(err)
+	}
+
+	r, err := ipc.NewFlightDataReader(fdata)
+	if err != nil {
+		t.Error(err)
+	}
+
+	expected := arrdata.Records["decimal128"]
+	idx := 0
+	var numRows int64 = 0
+	for {
+		rec, err := r.Read()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			t.Error(err)
+		}
+
+		numRows += rec.NumRows()
+		if !array.RecordEqual(expected[idx], rec) {
+			t.Errorf("flight data stream records don't match: \ngot = %#v\nwant = %#v", rec, expected[idx])
+		}
+		idx++
+	}
+
+	if numRows != fi.TotalRecords {
+		t.Fatalf("got %d, want %d", numRows, fi.TotalRecords)
+	}
+}
diff --git a/go/arrow/flight/gen.go b/go/arrow/flight/gen.go
new file mode 100644
index 00000000000..8a4b8744098
--- /dev/null
+++ b/go/arrow/flight/gen.go
@@ -0,0 +1,19 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package flight // import "github.com/apache/arrow/go/arrow/flight"
+
+//go:generate protoc -I../../../format --go_out=. --go-grpc_out=. --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative Flight.proto
diff --git a/go/arrow/flight/server.go b/go/arrow/flight/server.go
new file mode 100644
index 00000000000..90492a7b3a3
--- /dev/null
+++ b/go/arrow/flight/server.go
@@ -0,0 +1,118 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package flight
+
+import (
+	"net"
+	"os"
+	"os/signal"
+
+	"google.golang.org/grpc"
+)
+
+// Server is an interface for hiding some of the grpc specifics to make
+// it slightly easier to manage a flight service, slightly modeled after
+// the C++ implementation
+type Server interface {
+	// Init takes in the address to bind to and creates the listener
+	Init(addr string) error
+	// Addr will return the address that was bound to for the service to listen on
+	Addr() net.Addr
+	// SetShutdownOnSignals sets notifications on the given signals to call GracefulStop
+	// on the grpc service if any of those signals are received
+	SetShutdownOnSignals(sig ...os.Signal)
+	// Serve blocks until accepting a connection fails with a fatal error. It will return
+	// a non-nil error unless it stopped due to calling Shutdown or receiving one of the
+	// signals set in SetShutdownOnSignals
+	Serve() error
+	// Shutdown will call GracefulStop on the grpc server so that it stops accepting connections
+	// and will wait until current methods complete
+	Shutdown()
+	// RegisterFlightService sets up the handler for the Flight Endpoints as per
+	// normal Grpc setups
+	RegisterFlightService(*FlightServiceService)
+}
+
+type server struct {
+	lis        net.Listener
+	sigChannel <-chan os.Signal
+	done       chan bool
+
+	authHandler ServerAuthHandler
+	server      *grpc.Server
+}
+
+// NewFlightServer takes in an auth handler for managing the handshake authentication
+// and any grpc Server options desired, such as TLS certs and so on which will just
+// be passed through to the underlying grpc server.
+//
+// Alternatively, a grpc server can be created normally without this helper as the
+// grpc server generated code is still being exported. This only exists to allow
+// the utility of the helpers
+func NewFlightServer(auth ServerAuthHandler, opt ...grpc.ServerOption) Server {
+	if auth != nil {
+		opt = append([]grpc.ServerOption{
+			grpc.ChainStreamInterceptor(createServerAuthStreamInterceptor(auth)),
+			grpc.ChainUnaryInterceptor(createServerAuthUnaryInterceptor(auth)),
+		}, opt...)
+	}
+
+	return &server{
+		authHandler: auth,
+		server:      grpc.NewServer(opt...),
+	}
+}
+
+func (s *server) Init(addr string) (err error) {
+	s.lis, err = net.Listen("tcp", addr)
+	return
+}
+
+func (s *server) Addr() net.Addr {
+	return s.lis.Addr()
+}
+
+func (s *server) SetShutdownOnSignals(sig ...os.Signal) {
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, sig...)
+	s.sigChannel = c
+}
+
+func (s *server) Serve() error {
+	s.done = make(chan bool)
+	go func() {
+		select {
+		case <-s.sigChannel:
+			s.server.GracefulStop()
+		case <-s.done:
+		}
+	}()
+	err := s.server.Serve(s.lis)
+	close(s.done)
+	return err
+}
+
+func (s *server) RegisterFlightService(svc *FlightServiceService) {
+	if svc.Handshake == nil {
+		svc.Handshake = s.handshake
+	}
+	RegisterFlightServiceService(s.server, svc)
+}
+
+func (s *server) Shutdown() {
+	s.server.GracefulStop()
+}
diff --git a/go/arrow/flight/server_auth.go b/go/arrow/flight/server_auth.go
new file mode 100644
index 00000000000..99e6f8e3c1e
--- /dev/null
+++ b/go/arrow/flight/server_auth.go
@@ -0,0 +1,145 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package flight
+
+import (
+	"context"
+	"strings"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
+)
+
+const grpcAuthHeader = "auth-token-bin"
+
+// AuthConn wraps the stream from grpc for handshakes to simplify handling
+// handshake request and response from the flight.proto forwarding just the
+// payloads and errors instead of having to deal with the handshake request
+// and response protos directly
+type AuthConn interface {
+	Read() ([]byte, error)
+	Send([]byte) error
+}
+
+type serverAuthConn struct {
+	stream FlightService_HandshakeServer
+}
+
+func (a *serverAuthConn) Read() ([]byte, error) {
+	in, err := a.stream.Recv()
+	if err != nil {
+		return nil, err
+	}
+
+	return in.Payload, nil
+}
+
+func (a *serverAuthConn) Send(b []byte) error {
+	return a.stream.Send(&HandshakeResponse{Payload: b})
+}
+
+// ServerAuthHandler defines an interface for the server to perform the handshake.
+// The token is expected to be sent as part of the context metadata in subsequent
+// requests with a key of "auth-token-bin" which will then call IsValid to validate
+type ServerAuthHandler interface {
+	Authenticate(AuthConn) error
+	IsValid(token string) (interface{}, error)
+}
+
+type authCtxKey struct{}
+
+type authWrappedStream struct {
+	grpc.ServerStream
+	ctx context.Context
+}
+
+func (a *authWrappedStream) Context() context.Context { return a.ctx }
+
+// AuthFromContext will return back whatever object was returned from `IsValid` for a
+// given request context allowing handlers to retrieve identifying information
+// for the current request for use.
+func AuthFromContext(ctx context.Context) interface{} {
+	return ctx.Value(authCtxKey{})
+}
+
+func createServerAuthUnaryInterceptor(auth ServerAuthHandler) grpc.UnaryServerInterceptor {
+	if auth == nil {
+		return func(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
+			return handler(ctx, req)
+		}
+	}
+
+	return func(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
+		var authTok string
+		md, ok := metadata.FromIncomingContext(ctx)
+		if ok {
+			vals := md.Get(grpcAuthHeader)
+			if len(vals) > 0 {
+				authTok = vals[0]
+			}
+		}
+
+		peerIdentity, err := auth.IsValid(authTok)
+		if err != nil {
+			return nil, status.Errorf(codes.PermissionDenied, "auth-error: %s", err)
+		}
+
+		return handler(context.WithValue(ctx, authCtxKey{}, peerIdentity), req)
+	}
+}
+
+func createServerAuthStreamInterceptor(auth ServerAuthHandler) grpc.StreamServerInterceptor {
+	if auth == nil {
+		return func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+			return handler(srv, stream)
+		}
+	}
+
+	return func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+		if strings.HasSuffix(info.FullMethod, "/Handshake") {
+			return handler(srv, stream)
+		}
+
+		var authTok string
+		md, ok := metadata.FromIncomingContext(stream.Context())
+		if ok {
+			vals := md.Get(grpcAuthHeader)
+			if len(vals) > 0 {
+				authTok = vals[0]
+			}
+		}
+
+		peerIdentity, err := auth.IsValid(authTok)
+		if err != nil {
+			return status.Errorf(codes.Unauthenticated, "auth-error: %s", err)
+		}
+
+		stream = &authWrappedStream{ServerStream: stream, ctx: context.WithValue(stream.Context(), authCtxKey{}, peerIdentity)}
+		return handler(srv, stream)
+	}
+}
+
+// our implementation of handshake using the authhandler
+func (s *server) handshake(stream FlightService_HandshakeServer) error {
+	if s.authHandler == nil {
+		return nil
+	}
+
+	return s.authHandler.Authenticate(&serverAuthConn{stream})
+}
diff --git a/go/arrow/go.mod b/go/arrow/go.mod
index a83f7cc2fdb..ca760281449 100644
--- a/go/arrow/go.mod
+++ b/go/arrow/go.mod
@@ -20,8 +20,16 @@ go 1.12
 
 require (
 	github.com/davecgh/go-spew v1.1.0 // indirect
+	github.com/golang/protobuf v1.4.2
 	github.com/google/flatbuffers v1.11.0
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/stretchr/testify v1.2.0
+	golang.org/x/net v0.0.0-20200904194848-62affa334b73 // indirect
+	golang.org/x/sys v0.0.0-20200909081042-eff7692f9009 // indirect
+	golang.org/x/text v0.3.3 // indirect
 	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
+	google.golang.org/genproto v0.0.0-20200911024640-645f7a48b24f // indirect
+	google.golang.org/grpc v1.32.0
+	google.golang.org/grpc/cmd/protoc-gen-go-grpc v0.0.0-20200910201057-6591123024b3 // indirect
+	google.golang.org/protobuf v1.25.0
 )
diff --git a/go/arrow/go.sum b/go/arrow/go.sum
index 25e38a1bb4a..f56e738d4c3 100644
--- a/go/arrow/go.sum
+++ b/go/arrow/go.sum
@@ -1,10 +1,104 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1 h1:ZFgWrT+bLgsYPirOnRfKLYJLvssAegOj/hgyMFdJZe0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/google/flatbuffers v1.11.0 h1:O7CEyB8Cb3/DmtxODGtLHcEvpr81Jm5qLg/hsHnxA2A=
 github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/stretchr/testify v1.2.0 h1:LThGCOvhuJic9Gyd1VBCkhyUXmO8vKaBFvBsJ2k03rg=
 github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
+golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200909081042-eff7692f9009 h1:W0lCpv29Hv0UaM1LXb9QlBHLNP8UFfcKjblhVCWftOM=
+golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200911024640-645f7a48b24f h1:Yv4xsIx7HZOoyUGSJ2ksDyWE2qIBXROsZKt2ny3hCGM=
+google.golang.org/genproto v0.0.0-20200911024640-645f7a48b24f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.32.0 h1:zWTV+LMdc3kaiJMSTOFz2UgSBgx8RNQoTGiZu3fR9S0=
+google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc/cmd/protoc-gen-go-grpc v0.0.0-20200910201057-6591123024b3 h1:MZjUhWVLZHiPPNKvwdt31HZVHrASfgk1ScV3vVTKbDo=
+google.golang.org/grpc/cmd/protoc-gen-go-grpc v0.0.0-20200910201057-6591123024b3/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/go/arrow/internal/cpu/cpu_s390x.go b/go/arrow/internal/cpu/cpu_s390x.go
new file mode 100644
index 00000000000..3c1a2964243
--- /dev/null
+++ b/go/arrow/internal/cpu/cpu_s390x.go
@@ -0,0 +1,7 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cpu
+
+const CacheLineSize = 256
diff --git a/go/arrow/ipc/cmd/arrow-cat/main.go b/go/arrow/ipc/cmd/arrow-cat/main.go
index 527bceb9752..8e7f19bf1ba 100644
--- a/go/arrow/ipc/cmd/arrow-cat/main.go
+++ b/go/arrow/ipc/cmd/arrow-cat/main.go
@@ -95,7 +95,6 @@ func processStream(w io.Writer, rin io.Reader) error {
 			}
 			return err
 		}
-		defer r.Release()
 
 		n := 0
 		for r.Next() {
@@ -106,6 +105,7 @@ func processStream(w io.Writer, rin io.Reader) error {
 				fmt.Fprintf(w, "  col[%d] %q: %v\n", i, rec.ColumnName(i), col)
 			}
 		}
+		r.Release()
 	}
 	return nil
 }
@@ -158,11 +158,11 @@ func processFile(w io.Writer, fname string) error {
 		if err != nil {
 			return err
 		}
-		defer rec.Release()
 
 		for i, col := range rec.Columns() {
 			fmt.Fprintf(w, "  col[%d] %q: %v\n", i, rec.ColumnName(i), col)
 		}
+		rec.Release()
 	}
 
 	return nil
diff --git a/go/arrow/ipc/cmd/arrow-ls/main.go b/go/arrow/ipc/cmd/arrow-ls/main.go
index 36ab63b5017..f731cbf63d6 100644
--- a/go/arrow/ipc/cmd/arrow-ls/main.go
+++ b/go/arrow/ipc/cmd/arrow-ls/main.go
@@ -94,7 +94,6 @@ func processStream(w io.Writer, rin io.Reader) error {
 			}
 			return err
 		}
-		defer r.Release()
 
 		fmt.Fprintf(w, "%v\n", r.Schema())
 
@@ -103,6 +102,7 @@ func processStream(w io.Writer, rin io.Reader) error {
 			nrecs++
 		}
 		fmt.Fprintf(w, "records: %d\n", nrecs)
+		r.Release()
 	}
 	return nil
 }
diff --git a/go/arrow/ipc/file_reader.go b/go/arrow/ipc/file_reader.go
index b7509d0c479..961803b33ef 100644
--- a/go/arrow/ipc/file_reader.go
+++ b/go/arrow/ipc/file_reader.go
@@ -152,9 +152,9 @@ func (f *FileReader) readSchema() error {
 		if err != nil {
 			return err
 		}
-		defer msg.Release()
 
 		id, dict, err := readDictionary(msg.meta, f.fields, f.r)
+		msg.Release()
 		if err != nil {
 			return xerrors.Errorf("arrow/ipc: could not read dictionary %d from file: %w", i, err)
 		}
diff --git a/go/arrow/ipc/flight_data_reader.go b/go/arrow/ipc/flight_data_reader.go
new file mode 100644
index 00000000000..462286f7ca6
--- /dev/null
+++ b/go/arrow/ipc/flight_data_reader.go
@@ -0,0 +1,210 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ipc
+
+import (
+	"bytes"
+	"io"
+	"sync/atomic"
+
+	"github.com/apache/arrow/go/arrow"
+	"github.com/apache/arrow/go/arrow/array"
+	"github.com/apache/arrow/go/arrow/arrio"
+	"github.com/apache/arrow/go/arrow/flight"
+	"github.com/apache/arrow/go/arrow/internal/debug"
+	"github.com/apache/arrow/go/arrow/internal/flatbuf"
+	"github.com/apache/arrow/go/arrow/memory"
+	"golang.org/x/xerrors"
+)
+
+// FlightDataStreamReader wraps a grpc stream for receiving FlightData objects
+type FlightDataStreamReader interface {
+	Recv() (*flight.FlightData, error)
+}
+
+// FlightDataReader reads records from a stream of messages
+type FlightDataReader struct {
+	r      FlightDataStreamReader
+	schema *arrow.Schema
+
+	refCount int64
+	rec      array.Record
+	err      error
+
+	types dictTypeMap
+	memo  dictMemo
+
+	mem memory.Allocator
+
+	done bool
+}
+
+// NewFlightDataReader returns a reader that will produce records from a flight data stream
+//
+// implementation is generally based on the ipc.Reader, expecting the first message to be the
+// schema with the subsequent messages being the record batches.
+func NewFlightDataReader(r FlightDataStreamReader, opts ...Option) (*FlightDataReader, error) {
+	cfg := newConfig(opts...)
+
+	rr := &FlightDataReader{
+		r:   r,
+		mem: cfg.alloc,
+	}
+
+	msg, err := rr.nextMessage()
+	if err != nil {
+		return nil, xerrors.Errorf("arrow/ipc: could not read message schema: %w", err)
+	}
+
+	if msg.Type() != MessageSchema {
+		return nil, xerrors.Errorf("arrow/ipc: invalid message type (got=%v, want=%v)", msg.Type(), MessageSchema)
+	}
+
+	// FIXME(sbinet) refactor msg-header handling (a la ipc.Reader.readSchema)
+	var schemaFB flatbuf.Schema
+	initFB(&schemaFB, msg.msg.Header)
+
+	rr.types, err = dictTypesFromFB(&schemaFB)
+	if err != nil {
+		return nil, xerrors.Errorf("arrow/ipc: could not read dictionary types from message schema: %w", err)
+	}
+
+	// TODO(sbinet): see ipc.Reader.readSchema
+	for range rr.types {
+		panic("not implemented") // ReadNextDictionary
+	}
+
+	rr.schema, err = schemaFromFB(&schemaFB, &rr.memo)
+	if err != nil {
+		return nil, xerrors.Errorf("arrow/ipc: could not decode schema from message schema: %w", err)
+	}
+
+	if cfg.schema != nil && !cfg.schema.Equal(rr.schema) {
+		return nil, errInconsistentSchema
+	}
+
+	return rr, nil
+}
+
+func (f *FlightDataReader) nextMessage() (*Message, error) {
+	fd, err := f.r.Recv()
+	if err != nil {
+		return nil, err
+	}
+
+	return NewMessage(memory.NewBufferBytes(fd.DataHeader), memory.NewBufferBytes(fd.DataBody)), nil
+}
+
+func (f *FlightDataReader) next() bool {
+	var msg *Message
+	msg, f.err = f.nextMessage()
+	if f.err != nil {
+		f.done = true
+		if f.err == io.EOF {
+			f.err = nil
+		}
+		return false
+	}
+
+	if got, want := msg.Type(), MessageRecordBatch; got != want {
+		f.err = xerrors.Errorf("arrow/ipc: invalid message type (got=%v, want=%v)", got, want)
+		return false
+	}
+
+	f.rec = newRecord(f.schema, msg.meta, bytes.NewReader(msg.body.Bytes()))
+	return true
+}
+
+// Record returns the current record that has been extracted from the stream.
+// It is valid until the next call to Next or Read
+func (f *FlightDataReader) Record() array.Record {
+	return f.rec
+}
+
+// Next returns whether a record was able to be extracted from the stream or not.
+func (f *FlightDataReader) Next() bool {
+	if f.rec != nil {
+		f.rec.Release()
+		f.rec = nil
+	}
+
+	if f.err != nil || f.done {
+		return false
+	}
+
+	return f.next()
+}
+
+// Read reads the current record from the flight stream and an error, if any.
+// When we reach the end of the flight stream it will return (nil, io.EOF).
+// Also calls release on the previous existing record if any.
+func (f *FlightDataReader) Read() (array.Record, error) {
+	if f.rec != nil {
+		f.rec.Release()
+		f.rec = nil
+	}
+
+	if !f.next() {
+		if f.done {
+			return nil, io.EOF
+		}
+		return nil, f.err
+	}
+
+	return f.rec, nil
+}
+
+// Retain increases the refcount by 1.
+// Retain can be called by multiple goroutines simultaneously.
+func (f *FlightDataReader) Retain() {
+	atomic.AddInt64(&f.refCount, 1)
+}
+
+// Release decreases the refcount by 1.
+// When the refcount is 0 the memory is freed.
+// Release may be called simultaneously from multiple goroutines.
+func (f *FlightDataReader) Release() {
+	debug.Assert(atomic.LoadInt64(&f.refCount) > 0, "too many releases")
+
+	if atomic.AddInt64(&f.refCount, -1) == 0 {
+		if f.rec != nil {
+			f.rec.Release()
+			f.rec = nil
+		}
+		if f.r != nil {
+			f.r = nil
+		}
+	}
+}
+
+// Err returns the last error encounted during the iteration of the stream.
+func (f *FlightDataReader) Err() error { return f.err }
+
+// Schema returns the schema of the underlying records as described by the
+// first message received.
+func (f *FlightDataReader) Schema() *arrow.Schema { return f.schema }
+
+func SchemaFromFlightInfo(b []byte) (*arrow.Schema, error) {
+	fb := flatbuf.GetRootAsSchema(b, 0)
+	dict := newMemo()
+	return schemaFromFB(fb, &dict)
+}
+
+var (
+	_ array.RecordReader = (*FlightDataReader)(nil)
+	_ arrio.Reader       = (*FlightDataReader)(nil)
+)
diff --git a/go/arrow/ipc/flight_data_writer.go b/go/arrow/ipc/flight_data_writer.go
new file mode 100644
index 00000000000..7a5bc27bb6e
--- /dev/null
+++ b/go/arrow/ipc/flight_data_writer.go
@@ -0,0 +1,150 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ipc
+
+import (
+	"bytes"
+
+	"github.com/apache/arrow/go/arrow"
+	"github.com/apache/arrow/go/arrow/array"
+	"github.com/apache/arrow/go/arrow/arrio"
+	"github.com/apache/arrow/go/arrow/bitutil"
+	"github.com/apache/arrow/go/arrow/flight"
+	"github.com/apache/arrow/go/arrow/memory"
+	flatbuffers "github.com/google/flatbuffers/go"
+	"golang.org/x/xerrors"
+)
+
+// FlightDataStreamWriter wraps a grpc stream for sending FlightData
+type FlightDataStreamWriter interface {
+	Send(*flight.FlightData) error
+}
+
+// FlightDataWriter is a stream writer for writing with Flight RPC
+type FlightDataWriter struct {
+	w   FlightDataStreamWriter
+	fd  flight.FlightData
+	buf bytes.Buffer
+
+	mem     memory.Allocator
+	started bool
+	schema  *arrow.Schema
+}
+
+// NewFlightDataWriter returns a writer for writing array Records to a flight data stream.
+func NewFlightDataWriter(w FlightDataStreamWriter, opts ...Option) *FlightDataWriter {
+	cfg := newConfig(opts...)
+	return &FlightDataWriter{
+		w:      w,
+		mem:    cfg.alloc,
+		schema: cfg.schema,
+	}
+}
+
+func (w *FlightDataWriter) start() error {
+	w.started = true
+
+	ps := payloadsFromSchema(w.schema, w.mem, nil)
+	defer ps.Release()
+
+	for i := range ps {
+		if err := w.writePayload(&ps[i]); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (w *FlightDataWriter) Close() (err error) {
+	if !w.started {
+		err = w.start()
+	}
+
+	return err
+}
+
+// Write the provided record to the underlying stream
+func (w *FlightDataWriter) Write(rec array.Record) error {
+	if !w.started {
+		err := w.start()
+		if err != nil {
+			return err
+		}
+	}
+
+	schema := rec.Schema()
+	if schema == nil || !schema.Equal(w.schema) {
+		return errInconsistentSchema
+	}
+
+	const allow64b = true
+	var (
+		data = payload{}
+		enc  = newRecordEncoder(w.mem, 0, kMaxNestingDepth, allow64b)
+	)
+	defer data.Release()
+
+	if err := enc.Encode(&data, rec); err != nil {
+		return xerrors.Errorf("arrow/ipc: could not encode record to payload: %w", err)
+	}
+
+	return w.writePayload(&data)
+}
+
+func (w *FlightDataWriter) writePayload(data *payload) (err error) {
+	w.fd.DataHeader = data.meta.Bytes()
+	tmp := &w.buf
+	tmp.Reset()
+
+	for _, bufs := range data.body {
+		if bufs == nil {
+			continue
+		}
+
+		size := int64(bufs.Len())
+		padding := bitutil.CeilByte64(size) - size
+		if size > 0 {
+			_, err = tmp.Write(bufs.Bytes())
+			if err != nil {
+				return xerrors.Errorf("arrow/ipc: could not write payload message body: %w", err)
+			}
+		}
+
+		if padding > 0 {
+			_, err = tmp.Write(paddingBytes[:padding])
+			if err != nil {
+				return xerrors.Errorf("arrow/ipc: could not write payload message padding: %w", err)
+			}
+		}
+	}
+
+	w.fd.DataBody = tmp.Bytes()
+	return w.w.Send(&w.fd)
+}
+
+func FlightInfoSchemaBytes(schema *arrow.Schema, mem memory.Allocator) []byte {
+	dict := newMemo()
+	b := flatbuffers.NewBuilder(1024)
+	offset := schemaToFB(b, schema, &dict)
+	b.Finish(offset)
+	return b.FinishedBytes()
+}
+
+var (
+	_ arrio.Writer = (*FlightDataWriter)(nil)
+)
diff --git a/go/arrow/math/float64_avx2_amd64.go b/go/arrow/math/float64_avx2_amd64.go
index 9301b328497..172a23a69a9 100644
--- a/go/arrow/math/float64_avx2_amd64.go
+++ b/go/arrow/math/float64_avx2_amd64.go
@@ -27,13 +27,13 @@ import (
 )
 
 //go:noescape
-func _sum_float64_avx2(buf, len, res unsafe.Pointer)
+func _sum_float64_avx2(buf unsafe.Pointer, len uintptr, res unsafe.Pointer)
 
 func sum_float64_avx2(a *array.Float64) float64 {
 	buf := a.Float64Values()
 	var (
 		p1  = unsafe.Pointer(&buf[0])
-		p2  = unsafe.Pointer(uintptr(len(buf)))
+		p2  = uintptr(len(buf))
 		res float64
 	)
 	_sum_float64_avx2(p1, p2, unsafe.Pointer(&res))
diff --git a/go/arrow/math/float64_sse4_amd64.go b/go/arrow/math/float64_sse4_amd64.go
index 30a79f48a41..d641b839e96 100644
--- a/go/arrow/math/float64_sse4_amd64.go
+++ b/go/arrow/math/float64_sse4_amd64.go
@@ -27,13 +27,13 @@ import (
 )
 
 //go:noescape
-func _sum_float64_sse4(buf, len, res unsafe.Pointer)
+func _sum_float64_sse4(buf unsafe.Pointer, len uintptr, res unsafe.Pointer)
 
 func sum_float64_sse4(a *array.Float64) float64 {
 	buf := a.Float64Values()
 	var (
 		p1  = unsafe.Pointer(&buf[0])
-		p2  = unsafe.Pointer(uintptr(len(buf)))
+		p2  = uintptr(len(buf))
 		res float64
 	)
 	_sum_float64_sse4(p1, p2, unsafe.Pointer(&res))
diff --git a/go/arrow/math/int64_avx2_amd64.go b/go/arrow/math/int64_avx2_amd64.go
index 55e03cd6311..e22fbd53092 100644
--- a/go/arrow/math/int64_avx2_amd64.go
+++ b/go/arrow/math/int64_avx2_amd64.go
@@ -27,13 +27,13 @@ import (
 )
 
 //go:noescape
-func _sum_int64_avx2(buf, len, res unsafe.Pointer)
+func _sum_int64_avx2(buf unsafe.Pointer, len uintptr, res unsafe.Pointer)
 
 func sum_int64_avx2(a *array.Int64) int64 {
 	buf := a.Int64Values()
 	var (
 		p1  = unsafe.Pointer(&buf[0])
-		p2  = unsafe.Pointer(uintptr(len(buf)))
+		p2  = uintptr(len(buf))
 		res int64
 	)
 	_sum_int64_avx2(p1, p2, unsafe.Pointer(&res))
diff --git a/go/arrow/math/int64_sse4_amd64.go b/go/arrow/math/int64_sse4_amd64.go
index c13c8de1442..28201705293 100644
--- a/go/arrow/math/int64_sse4_amd64.go
+++ b/go/arrow/math/int64_sse4_amd64.go
@@ -27,13 +27,13 @@ import (
 )
 
 //go:noescape
-func _sum_int64_sse4(buf, len, res unsafe.Pointer)
+func _sum_int64_sse4(buf unsafe.Pointer, len uintptr, res unsafe.Pointer)
 
 func sum_int64_sse4(a *array.Int64) int64 {
 	buf := a.Int64Values()
 	var (
 		p1  = unsafe.Pointer(&buf[0])
-		p2  = unsafe.Pointer(uintptr(len(buf)))
+		p2  = uintptr(len(buf))
 		res int64
 	)
 	_sum_int64_sse4(p1, p2, unsafe.Pointer(&res))
diff --git a/go/arrow/math/type_simd_amd64.go.tmpl b/go/arrow/math/type_simd_amd64.go.tmpl
index b572290cc5b..5fde1f8f5d7 100644
--- a/go/arrow/math/type_simd_amd64.go.tmpl
+++ b/go/arrow/math/type_simd_amd64.go.tmpl
@@ -27,13 +27,13 @@ import (
 {{$name := printf "%s_%s" .In.Type .D.arch}}
 {{with .In}}
 //go:noescape
-func _sum_{{$name}}(buf, len, res unsafe.Pointer)
+func _sum_{{$name}}(buf unsafe.Pointer, len uintptr, res unsafe.Pointer)
 
 func sum_{{$name}}(a *array.{{.Name}}) {{.Type}} {
 	buf := a.{{.Name}}Values()
 	var (
 		p1  = unsafe.Pointer(&buf[0])
-		p2  = unsafe.Pointer(uintptr(len(buf)))
+		p2  = uintptr(len(buf))
 		res {{.Type}}
 	)
 	_sum_{{$name}}(p1, p2, unsafe.Pointer(&res))
diff --git a/go/arrow/math/uint64_avx2_amd64.go b/go/arrow/math/uint64_avx2_amd64.go
index ff0500e7c8a..51a388447c1 100644
--- a/go/arrow/math/uint64_avx2_amd64.go
+++ b/go/arrow/math/uint64_avx2_amd64.go
@@ -27,13 +27,13 @@ import (
 )
 
 //go:noescape
-func _sum_uint64_avx2(buf, len, res unsafe.Pointer)
+func _sum_uint64_avx2(buf unsafe.Pointer, len uintptr, res unsafe.Pointer)
 
 func sum_uint64_avx2(a *array.Uint64) uint64 {
 	buf := a.Uint64Values()
 	var (
 		p1  = unsafe.Pointer(&buf[0])
-		p2  = unsafe.Pointer(uintptr(len(buf)))
+		p2  = uintptr(len(buf))
 		res uint64
 	)
 	_sum_uint64_avx2(p1, p2, unsafe.Pointer(&res))
diff --git a/go/arrow/math/uint64_sse4_amd64.go b/go/arrow/math/uint64_sse4_amd64.go
index 428bbe69fb6..901dad63026 100644
--- a/go/arrow/math/uint64_sse4_amd64.go
+++ b/go/arrow/math/uint64_sse4_amd64.go
@@ -27,13 +27,13 @@ import (
 )
 
 //go:noescape
-func _sum_uint64_sse4(buf, len, res unsafe.Pointer)
+func _sum_uint64_sse4(buf unsafe.Pointer, len uintptr, res unsafe.Pointer)
 
 func sum_uint64_sse4(a *array.Uint64) uint64 {
 	buf := a.Uint64Values()
 	var (
 		p1  = unsafe.Pointer(&buf[0])
-		p2  = unsafe.Pointer(uintptr(len(buf)))
+		p2  = uintptr(len(buf))
 		res uint64
 	)
 	_sum_uint64_sse4(p1, p2, unsafe.Pointer(&res))
diff --git a/java/adapter/avro/pom.xml b/java/adapter/avro/pom.xml
index a73e17722b7..f2d24dc6a97 100644
--- a/java/adapter/avro/pom.xml
+++ b/java/adapter/avro/pom.xml
@@ -16,7 +16,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>2.0.0-SNAPSHOT</version>
+    <version>3.0.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowUtils.java b/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowUtils.java
index 29e44dad3a5..80293c8b85c 100644
--- a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowUtils.java
+++ b/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowUtils.java
@@ -298,7 +298,7 @@ private static ArrowType createDecimalArrowType(LogicalTypes.Decimal logicalType
     Preconditions.checkArgument(scale <= precision,
         "Invalid decimal scale: %s (greater than precision: %s)", scale, precision);
 
-    return new ArrowType.Decimal(precision, scale);
+    return new ArrowType.Decimal(precision, scale, 128);
 
   }
 
diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml
index 54475eb8203..a954a3be4e6 100644
--- a/java/adapter/jdbc/pom.xml
+++ b/java/adapter/jdbc/pom.xml
@@ -16,7 +16,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>2.0.0-SNAPSHOT</version>
+        <version>3.0.0-SNAPSHOT</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 
diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java
index e534d2060c5..f64f178c6f4 100644
--- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java
+++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java
@@ -251,7 +251,7 @@ public static ArrowType getArrowTypeForJdbcField(JdbcFieldInfo fieldInfo, Calend
       case Types.DECIMAL:
         int precision = fieldInfo.getPrecision();
         int scale = fieldInfo.getScale();
-        return new ArrowType.Decimal(precision, scale);
+        return new ArrowType.Decimal(precision, scale, 128);
       case Types.REAL:
       case Types.FLOAT:
         return new ArrowType.FloatingPoint(SINGLE);
diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java
index c431bc794f3..b9b83daccc2 100644
--- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java
+++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java
@@ -20,8 +20,6 @@
 import java.sql.Date;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
 import java.util.Calendar;
 import java.util.concurrent.TimeUnit;
 
@@ -30,22 +28,10 @@
 
 /**
  * Consumer which consume date type values from {@link ResultSet}.
- * Write the data to {@link org.apache.arrow.vector.DateMilliVector}.
+ * Write the data to {@link org.apache.arrow.vector.DateDayVector}.
  */
 public class DateConsumer {
 
-  public static final int MAX_DAY;
-
-  static {
-    SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
-    try {
-      java.util.Date date = dateFormat.parse("9999-12-31");
-      MAX_DAY = (int) TimeUnit.MILLISECONDS.toDays(date.getTime());
-    } catch (ParseException e) {
-      throw new IllegalArgumentException("Failed to parse max day", e);
-    }
-  }
-
   /**
    * Creates a consumer for {@link DateMilliVector}.
    */
@@ -85,13 +71,9 @@ public void consume(ResultSet resultSet) throws SQLException {
       Date date = calendar == null ? resultSet.getDate(columnIndexInResultSet) :
           resultSet.getDate(columnIndexInResultSet, calendar);
       if (!resultSet.wasNull()) {
-        int day = (int) TimeUnit.MILLISECONDS.toDays(date.getTime());
-        if (day < 0 || day > MAX_DAY) {
-          throw new IllegalArgumentException("Day overflow: " + day);
-        }
         // for fixed width vectors, we have allocated enough memory proactively,
         // so there is no need to call the setSafe method here.
-        vector.set(currentIndex, day);
+        vector.set(currentIndex, Math.toIntExact(TimeUnit.MILLISECONDS.toDays(date.getTime())));
       }
       currentIndex++;
     }
@@ -123,13 +105,9 @@ public NonNullableDateConsumer(DateDayVector vector, int index, Calendar calenda
     public void consume(ResultSet resultSet) throws SQLException {
       Date date = calendar == null ? resultSet.getDate(columnIndexInResultSet) :
           resultSet.getDate(columnIndexInResultSet, calendar);
-      int day = (int) TimeUnit.MILLISECONDS.toDays(date.getTime());
-      if (day < 0 || day > MAX_DAY) {
-        throw new IllegalArgumentException("Day overflow: " + day);
-      }
       // for fixed width vectors, we have allocated enough memory proactively,
       // so there is no need to call the setSafe method here.
-      vector.set(currentIndex, day);
+      vector.set(currentIndex, Math.toIntExact(TimeUnit.MILLISECONDS.toDays(date.getTime())));
       currentIndex++;
     }
   }
diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_date_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_date_h2.yml
index da33d9d47cd..bca886ceb4e 100644
--- a/java/adapter/jdbc/src/test/resources/h2/test1_date_h2.yml
+++ b/java/adapter/jdbc/src/test/resources/h2/test1_date_h2.yml
@@ -28,6 +28,7 @@ data:
   - 'INSERT INTO table1 VALUES (''2018-02-12'');'
   - 'INSERT INTO table1 VALUES (''2018-02-12'');'
   - 'INSERT INTO table1 VALUES (''2018-02-12'');'
+  - 'INSERT INTO table1 VALUES (''1969-01-01'');'
 
 query: 'select date_field10 from table1;'
 
@@ -44,3 +45,4 @@ values:
  - '17574'
  - '17574'
  - '17574'
+ - '-365'
diff --git a/java/adapter/orc/pom.xml b/java/adapter/orc/pom.xml
index 456c25a714b..160a1fe2da5 100644
--- a/java/adapter/orc/pom.xml
+++ b/java/adapter/orc/pom.xml
@@ -87,7 +87,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>2.0.0-SNAPSHOT</version>
+        <version>3.0.0-SNAPSHOT</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 
diff --git a/java/algorithm/pom.xml b/java/algorithm/pom.xml
index 57e0d3798ef..430849c8e10 100644
--- a/java/algorithm/pom.xml
+++ b/java/algorithm/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>2.0.0-SNAPSHOT</version>
+    <version>3.0.0-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-algorithm</artifactId>
   <name>Arrow Algorithms</name>
diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/deduplicate/DeduplicationUtils.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/deduplicate/DeduplicationUtils.java
index 01bbaeb0e7e..8811e43d3d0 100644
--- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/deduplicate/DeduplicationUtils.java
+++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/deduplicate/DeduplicationUtils.java
@@ -18,13 +18,13 @@
 package org.apache.arrow.algorithm.deduplicate;
 
 import org.apache.arrow.memory.ArrowBuf;
-import org.apache.arrow.util.DataSizeRoundingUtil;
 import org.apache.arrow.util.Preconditions;
 import org.apache.arrow.vector.BitVectorHelper;
 import org.apache.arrow.vector.IntVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.compare.Range;
 import org.apache.arrow.vector.compare.RangeEqualsVisitor;
+import org.apache.arrow.vector.util.DataSizeRoundingUtil;
 
 /**
  * Utilities for vector deduplication.
diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/deduplicate/VectorRunDeduplicator.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/deduplicate/VectorRunDeduplicator.java
index 8eb990b31bc..5ef03cbe4a7 100644
--- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/deduplicate/VectorRunDeduplicator.java
+++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/deduplicate/VectorRunDeduplicator.java
@@ -19,11 +19,11 @@
 
 import org.apache.arrow.memory.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.util.DataSizeRoundingUtil;
 import org.apache.arrow.util.Preconditions;
 import org.apache.arrow.vector.BitVectorHelper;
 import org.apache.arrow.vector.IntVector;
 import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.util.DataSizeRoundingUtil;
 
 /**
  * Remove adjacent equal elements from a vector.
diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthInPlaceVectorSorter.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthInPlaceVectorSorter.java
index b254ae56bb9..aaa7ba117c3 100644
--- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthInPlaceVectorSorter.java
+++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthInPlaceVectorSorter.java
@@ -141,6 +141,8 @@ void choosePivot(int low, int high) {
       pivotBuffer.copyFrom(medianIdx, 0, vec);
       vec.copyFrom(low, medianIdx, vec);
       vec.copyFrom(0, low, pivotBuffer);
+    } else {
+      pivotBuffer.copyFrom(low, 0, vec);
     }
 
     comparator.attachVectors(vec, pivotBuffer);
diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthOutOfPlaceVectorSorter.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthOutOfPlaceVectorSorter.java
index 97a3df47579..4f6c7665718 100644
--- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthOutOfPlaceVectorSorter.java
+++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthOutOfPlaceVectorSorter.java
@@ -18,6 +18,7 @@
 package org.apache.arrow.algorithm.sort;
 
 import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.util.Preconditions;
 import org.apache.arrow.vector.BaseFixedWidthVector;
 import org.apache.arrow.vector.BitVectorHelper;
 import org.apache.arrow.vector.IntVector;
@@ -44,6 +45,17 @@ public void sortOutOfPlace(V srcVector, V dstVector, VectorValueComparator<V> co
     ArrowBuf dstValidityBuffer = dstVector.getValidityBuffer();
     ArrowBuf dstValueBuffer = dstVector.getDataBuffer();
 
+    // check buffer size
+    Preconditions.checkArgument(dstValidityBuffer.capacity() * 8 >= srcVector.getValueCount(),
+        "Not enough capacity for the validity buffer of the dst vector. " +
+            "Expected capacity %s, actual capacity %s",
+        (srcVector.getValueCount() + 7) / 8, dstValidityBuffer.capacity());
+    Preconditions.checkArgument(
+        dstValueBuffer.capacity() >= srcVector.getValueCount() * srcVector.getTypeWidth(),
+        "Not enough capacity for the data buffer of the dst vector. " +
+            "Expected capacity %s, actual capacity %s",
+        srcVector.getValueCount() * srcVector.getTypeWidth(), dstValueBuffer.capacity());
+
     // sort value indices
     try (IntVector sortedIndices = new IntVector("", srcVector.getAllocator())) {
       sortedIndices.allocateNew(srcVector.getValueCount());
diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/VariableWidthOutOfPlaceVectorSorter.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/VariableWidthOutOfPlaceVectorSorter.java
index dbb097bb53d..62003752ec7 100644
--- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/VariableWidthOutOfPlaceVectorSorter.java
+++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/VariableWidthOutOfPlaceVectorSorter.java
@@ -18,6 +18,7 @@
 package org.apache.arrow.algorithm.sort;
 
 import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.util.Preconditions;
 import org.apache.arrow.vector.BaseVariableWidthVector;
 import org.apache.arrow.vector.BitVectorHelper;
 import org.apache.arrow.vector.IntVector;
@@ -45,6 +46,22 @@ public void sortOutOfPlace(V srcVector, V dstVector, VectorValueComparator<V> co
     ArrowBuf dstValueBuffer = dstVector.getDataBuffer();
     ArrowBuf dstOffsetBuffer = dstVector.getOffsetBuffer();
 
+    // check buffer size
+    Preconditions.checkArgument(dstValidityBuffer.capacity() * 8 >= srcVector.getValueCount(),
+        "Not enough capacity for the validity buffer of the dst vector. " +
+            "Expected capacity %s, actual capacity %s",
+        (srcVector.getValueCount() + 7) / 8, dstValidityBuffer.capacity());
+    Preconditions.checkArgument(
+        dstOffsetBuffer.capacity() >= (srcVector.getValueCount() + 1) * BaseVariableWidthVector.OFFSET_WIDTH,
+        "Not enough capacity for the offset buffer of the dst vector. " +
+            "Expected capacity %s, actual capacity %s",
+        (srcVector.getValueCount() + 1) * BaseVariableWidthVector.OFFSET_WIDTH, dstOffsetBuffer.capacity());
+    long dataSize = srcVector.getOffsetBuffer().getInt(
+        srcVector.getValueCount() * BaseVariableWidthVector.OFFSET_WIDTH);
+    Preconditions.checkArgument(
+        dstValueBuffer.capacity() >= dataSize, "No enough capacity for the data buffer of the dst vector. " +
+            "Expected capacity %s, actual capacity %s", dataSize, dstValueBuffer.capacity());
+
     // sort value indices
     try (IntVector sortedIndices = new IntVector("", srcVector.getAllocator())) {
       sortedIndices.allocateNew(srcVector.getValueCount());
diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/deduplicate/TestDeduplicationUtils.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/deduplicate/TestDeduplicationUtils.java
index 5068fb82716..def83fba7b7 100644
--- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/deduplicate/TestDeduplicationUtils.java
+++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/deduplicate/TestDeduplicationUtils.java
@@ -23,10 +23,10 @@
 import org.apache.arrow.memory.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.util.DataSizeRoundingUtil;
 import org.apache.arrow.vector.BitVectorHelper;
 import org.apache.arrow.vector.IntVector;
 import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.util.DataSizeRoundingUtil;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthInPlaceVectorSorter.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthInPlaceVectorSorter.java
index 4a8a33bfc7a..91ef52017df 100644
--- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthInPlaceVectorSorter.java
+++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthInPlaceVectorSorter.java
@@ -19,6 +19,9 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+
+import java.util.stream.IntStream;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
@@ -209,4 +212,29 @@ public void testChoosePivotAllPermutes() {
       }
     }
   }
+
+  @Test
+  public void testSortInt2() {
+    try (IntVector vector = new IntVector("vector", allocator)) {
+      ValueVectorDataPopulator.setVector(vector,
+          0, 1, 2, 3, 4, 5, 30, 31, 32, 33,
+          34, 35, 60, 61, 62, 63, 64, 65, 6, 7,
+          8, 9, 10, 11, 36, 37, 38, 39, 40, 41,
+          66, 67, 68, 69, 70, 71);
+
+      FixedWidthInPlaceVectorSorter sorter = new FixedWidthInPlaceVectorSorter();
+      VectorValueComparator<IntVector> comparator = DefaultVectorComparators.createDefaultComparator(vector);
+
+      sorter.sortInPlace(vector, comparator);
+
+      int[] actual = new int[vector.getValueCount()];
+      IntStream.range(0, vector.getValueCount()).forEach(
+          i -> actual[i] = vector.get(i));
+
+      assertArrayEquals(
+          new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
+              11, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39,
+              40, 41, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71}, actual);
+    }
+  }
 }
diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthOutOfPlaceVectorSorter.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthOutOfPlaceVectorSorter.java
index 7c2d28f1916..e3701f1d8ee 100644
--- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthOutOfPlaceVectorSorter.java
+++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthOutOfPlaceVectorSorter.java
@@ -18,6 +18,9 @@
 package org.apache.arrow.algorithm.sort;
 
 import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+
+import java.util.stream.IntStream;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
@@ -27,6 +30,7 @@
 import org.apache.arrow.vector.IntVector;
 import org.apache.arrow.vector.SmallIntVector;
 import org.apache.arrow.vector.TinyIntVector;
+import org.apache.arrow.vector.testing.ValueVectorDataPopulator;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -326,4 +330,36 @@ public void testSortDouble() {
       sortedVec.close();
     }
   }
+
+  @Test
+  public void testSortInt2() {
+    try (IntVector vec = new IntVector("", allocator)) {
+      ValueVectorDataPopulator.setVector(vec,
+          0, 1, 2, 3, 4, 5, 30, 31, 32, 33,
+          34, 35, 60, 61, 62, 63, 64, 65, 6, 7,
+          8, 9, 10, 11, 36, 37, 38, 39, 40, 41,
+          66, 67, 68, 69, 70, 71);
+
+      // sort the vector
+      FixedWidthOutOfPlaceVectorSorter sorter = new FixedWidthOutOfPlaceVectorSorter();
+      VectorValueComparator<IntVector> comparator = DefaultVectorComparators.createDefaultComparator(vec);
+
+      try (IntVector sortedVec = (IntVector) vec.getField().getFieldType().createNewSingleVector("", allocator, null)) {
+        sortedVec.allocateNew(vec.getValueCount());
+        sortedVec.setValueCount(vec.getValueCount());
+
+        sorter.sortOutOfPlace(vec, sortedVec, comparator);
+
+        // verify results
+        int[] actual = new int[sortedVec.getValueCount()];
+        IntStream.range(0, sortedVec.getValueCount()).forEach(
+            i -> actual[i] = sortedVec.get(i));
+
+        assertArrayEquals(
+            new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
+                11, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39,
+                40, 41, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71}, actual);
+      }
+    }
+  }
 }
diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthSorting.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthSorting.java
new file mode 100644
index 00000000000..ba2a341bf44
--- /dev/null
+++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthSorting.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.algorithm.sort;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Function;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.BaseFixedWidthVector;
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.Float4Vector;
+import org.apache.arrow.vector.Float8Vector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.SmallIntVector;
+import org.apache.arrow.vector.TinyIntVector;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+/**
+ * Test sorting fixed width vectors with random data.
+ */
+@RunWith(Parameterized.class)
+public class TestFixedWidthSorting<V extends BaseFixedWidthVector, U extends Comparable<U>> {
+
+  static final int[] VECTOR_LENGTHS = new int[] {2, 5, 10, 50, 100, 1000, 3000};
+
+  static final double[] NULL_FRACTIONS = {0, 0.1, 0.3, 0.5, 0.7, 0.9, 1};
+
+  private final int length;
+
+  private final double nullFraction;
+
+  private final boolean inPlace;
+
+  private final Function<BufferAllocator, V> vectorGenerator;
+
+  private final TestSortingUtil.DataGenerator<V, U> dataGenerator;
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void prepare() {
+    allocator = new RootAllocator(Integer.MAX_VALUE);
+  }
+
+  @After
+  public void shutdown() {
+    allocator.close();
+  }
+
+  public TestFixedWidthSorting(
+      int length, double nullFraction, boolean inPlace, String desc,
+      Function<BufferAllocator, V> vectorGenerator, TestSortingUtil.DataGenerator<V, U> dataGenerator) {
+    this.length = length;
+    this.nullFraction = nullFraction;
+    this.inPlace = inPlace;
+    this.vectorGenerator = vectorGenerator;
+    this.dataGenerator = dataGenerator;
+  }
+
+  @Test
+  public void testSort() {
+    if (inPlace) {
+      sortInPlace();
+    } else {
+      sortOutOfPlace();
+    }
+  }
+
+  void sortInPlace() {
+    try (V vector = vectorGenerator.apply(allocator)) {
+      U[] array = dataGenerator.populate(vector, length, nullFraction);
+      TestSortingUtil.sortArray(array);
+
+      FixedWidthInPlaceVectorSorter sorter = new FixedWidthInPlaceVectorSorter();
+      VectorValueComparator<V> comparator = DefaultVectorComparators.createDefaultComparator(vector);
+
+      sorter.sortInPlace(vector, comparator);
+
+      TestSortingUtil.verifyResults(vector, array);
+    }
+  }
+
+  void sortOutOfPlace() {
+    try (V vector = vectorGenerator.apply(allocator)) {
+      U[] array = dataGenerator.populate(vector, length, nullFraction);
+      TestSortingUtil.sortArray(array);
+
+      // sort the vector
+      FixedWidthOutOfPlaceVectorSorter sorter = new FixedWidthOutOfPlaceVectorSorter();
+      VectorValueComparator<V> comparator = DefaultVectorComparators.createDefaultComparator(vector);
+
+      try (V sortedVec = (V) vector.getField().getFieldType().createNewSingleVector("", allocator, null)) {
+        sortedVec.allocateNew(vector.getValueCount());
+        sortedVec.setValueCount(vector.getValueCount());
+
+        sorter.sortOutOfPlace(vector, sortedVec, comparator);
+
+        // verify results
+        TestSortingUtil.verifyResults(sortedVec, array);
+      }
+    }
+  }
+
+  @Parameterized.Parameters(name = "length = {0}, null fraction = {1}, in place = {2}, vector = {3}")
+  public static Collection<Object[]> getParameters() {
+    List<Object[]> params = new ArrayList<>();
+    for (int length : VECTOR_LENGTHS) {
+      for (double nullFrac : NULL_FRACTIONS) {
+        for (boolean inPlace : new boolean[] {true, false}) {
+          params.add(new Object[] {
+              length, nullFrac, inPlace, "TinyIntVector",
+              (Function<BufferAllocator, TinyIntVector>) (allocator -> new TinyIntVector("vector", allocator)),
+              TestSortingUtil.TINY_INT_GENERATOR
+          });
+
+          params.add(new Object[] {
+              length, nullFrac, inPlace, "SmallIntVector",
+              (Function<BufferAllocator, SmallIntVector>) (allocator -> new SmallIntVector("vector", allocator)),
+              TestSortingUtil.SMALL_INT_GENERATOR
+          });
+
+          params.add(new Object[] {
+              length, nullFrac, inPlace, "IntVector",
+              (Function<BufferAllocator, IntVector>) (allocator -> new IntVector("vector", allocator)),
+              TestSortingUtil.INT_GENERATOR
+          });
+
+          params.add(new Object[] {
+              length, nullFrac, inPlace, "BigIntVector",
+              (Function<BufferAllocator, BigIntVector>) (allocator -> new BigIntVector("vector", allocator)),
+              TestSortingUtil.LONG_GENERATOR
+          });
+
+          params.add(new Object[] {
+              length, nullFrac, inPlace, "Float4Vector",
+              (Function<BufferAllocator, Float4Vector>) (allocator -> new Float4Vector("vector", allocator)),
+              TestSortingUtil.FLOAT_GENERATOR
+          });
+
+          params.add(new Object[] {
+              length, nullFrac, inPlace, "Float8Vector",
+              (Function<BufferAllocator, Float8Vector>) (allocator -> new Float8Vector("vector", allocator)),
+              TestSortingUtil.DOUBLE_GENERATOR
+          });
+        }
+      }
+    }
+    return params;
+  }
+}
diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestSortingUtil.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestSortingUtil.java
new file mode 100644
index 00000000000..ea86551061d
--- /dev/null
+++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestSortingUtil.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.algorithm.sort;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.lang.reflect.Array;
+import java.util.Arrays;
+import java.util.Random;
+import java.util.function.BiConsumer;
+import java.util.function.Supplier;
+
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.Float4Vector;
+import org.apache.arrow.vector.Float8Vector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.SmallIntVector;
+import org.apache.arrow.vector.TinyIntVector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.testing.RandomDataGenerator;
+import org.apache.arrow.vector.testing.ValueVectorDataPopulator;
+
+/**
+ * Utilities for sorting related utilities.
+ */
+public class TestSortingUtil {
+
+  static final Random random = new Random(0);
+
+  static final DataGenerator<TinyIntVector, Byte> TINY_INT_GENERATOR = new DataGenerator<>(
+      RandomDataGenerator.TINY_INT_GENERATOR,
+      (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), Byte.class);
+
+  static final DataGenerator<SmallIntVector, Short> SMALL_INT_GENERATOR = new DataGenerator<>(
+      RandomDataGenerator.SMALL_INT_GENERATOR,
+      (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), Short.class);
+
+  static final DataGenerator<IntVector, Integer> INT_GENERATOR = new DataGenerator<>(
+      RandomDataGenerator.INT_GENERATOR,
+      (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), Integer.class);
+
+  static final DataGenerator<BigIntVector, Long> LONG_GENERATOR = new DataGenerator<>(
+      RandomDataGenerator.LONG_GENERATOR,
+      (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), Long.class);
+
+  static final DataGenerator<Float4Vector, Float> FLOAT_GENERATOR = new DataGenerator<>(
+      RandomDataGenerator.FLOAT_GENERATOR,
+      (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), Float.class);
+
+  static final DataGenerator<Float8Vector, Double> DOUBLE_GENERATOR = new DataGenerator<>(
+      RandomDataGenerator.DOUBLE_GENERATOR,
+      (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), Double.class);
+
+  static final DataGenerator<VarCharVector, String> STRING_GENERATOR = new DataGenerator<>(
+      () -> {
+        int strLength = random.nextInt(20) + 1;
+        return generateRandomString(strLength);
+      },
+      (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), String.class);
+
+  private TestSortingUtil() {
+  }
+
+  /**
+   * Verify that a vector is equal to an array.
+   */
+  public static <V extends ValueVector, U> void verifyResults(V vector, U[] expected) {
+    assertEquals(vector.getValueCount(), expected.length);
+    for (int i = 0; i < expected.length; i++) {
+      assertEquals(vector.getObject(i), expected[i]);
+    }
+  }
+
+  /**
+   * Sort an array with null values come first.
+   */
+  public static <U extends Comparable<U>> void sortArray(U[] array) {
+    Arrays.sort(array, (a, b) -> {
+      if (a == null || b == null) {
+        if (a == null && b == null) {
+          return 0;
+        }
+
+        // exactly one is null
+        if (a == null) {
+          return -1;
+        } else {
+          return 1;
+        }
+      }
+      return a.compareTo(b);
+    });
+  }
+
+  /**
+   * Generate a string with alphabetic characters only.
+   */
+  static String generateRandomString(int length) {
+    byte[] str = new byte[length];
+    final int lower = 'a';
+    final int upper = 'z';
+
+    for (int i = 0; i < length; i++) {
+      // make r non-negative
+      int r = random.nextInt() & Integer.MAX_VALUE;
+      str[i] = (byte) (r % (upper - lower + 1) + lower);
+    }
+
+    return new String(str);
+  }
+
+  /**
+   * Utility to generate data for testing.
+   * @param <V> vector type.
+   * @param <U> data element type.
+   */
+  static class DataGenerator<V extends ValueVector, U extends Comparable<U>> {
+
+    final Supplier<U> dataGenerator;
+
+    final BiConsumer<V, U[]> vectorPopulator;
+
+    final Class<U> clazz;
+
+    DataGenerator(
+        Supplier<U> dataGenerator, BiConsumer<V, U[]> vectorPopulator, Class<U> clazz) {
+      this.dataGenerator = dataGenerator;
+      this.vectorPopulator = vectorPopulator;
+      this.clazz = clazz;
+    }
+
+    /**
+     * Populate the vector according to the specified parameters.
+     * @param vector the vector to populate.
+     * @param length vector length.
+     * @param nullFraction the fraction of null values.
+     * @return An array with the same data as the vector.
+     */
+    U[] populate(V vector, int length, double nullFraction) {
+      U[] array = (U[]) Array.newInstance(clazz, length);
+      for (int i = 0; i < length; i++) {
+        double r = Math.random();
+        U value = r < nullFraction ? null : dataGenerator.get();
+        array[i] = value;
+      }
+      vectorPopulator.accept(vector, array);
+      return array;
+    }
+  }
+}
diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestVariableWidthSorting.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestVariableWidthSorting.java
new file mode 100644
index 00000000000..068fe8b69a8
--- /dev/null
+++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestVariableWidthSorting.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.algorithm.sort;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.function.Function;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.BaseVariableWidthVector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.util.Text;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+/**
+ * Test sorting variable width vectors with random data.
+ */
+@RunWith(Parameterized.class)
+public class TestVariableWidthSorting<V extends BaseVariableWidthVector, U extends Comparable<U>> {
+
+  static final int[] VECTOR_LENGTHS = new int[] {2, 5, 10, 50, 100, 1000, 3000};
+
+  static final double[] NULL_FRACTIONS = {0, 0.1, 0.3, 0.5, 0.7, 0.9, 1};
+
+  private final int length;
+
+  private final double nullFraction;
+
+  private final Function<BufferAllocator, V> vectorGenerator;
+
+  private final TestSortingUtil.DataGenerator<V, U> dataGenerator;
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void prepare() {
+    allocator = new RootAllocator(Integer.MAX_VALUE);
+  }
+
+  @After
+  public void shutdown() {
+    allocator.close();
+  }
+
+  public TestVariableWidthSorting(
+      int length, double nullFraction, String desc,
+      Function<BufferAllocator, V> vectorGenerator, TestSortingUtil.DataGenerator<V, U> dataGenerator) {
+    this.length = length;
+    this.nullFraction = nullFraction;
+    this.vectorGenerator = vectorGenerator;
+    this.dataGenerator = dataGenerator;
+  }
+
+  @Test
+  public void testSort() {
+    sortOutOfPlace();
+  }
+
+  void sortOutOfPlace() {
+    try (V vector = vectorGenerator.apply(allocator)) {
+      U[] array = dataGenerator.populate(vector, length, nullFraction);
+      Arrays.sort(array, (Comparator<? super U>) new StringComparator());
+
+      // sort the vector
+      VariableWidthOutOfPlaceVectorSorter sorter = new VariableWidthOutOfPlaceVectorSorter();
+      VectorValueComparator<V> comparator = DefaultVectorComparators.createDefaultComparator(vector);
+
+      try (V sortedVec = (V) vector.getField().getFieldType().createNewSingleVector("", allocator, null)) {
+        int dataSize = vector.getOffsetBuffer().getInt(vector.getValueCount() * 4);
+        sortedVec.allocateNew(dataSize, vector.getValueCount());
+        sortedVec.setValueCount(vector.getValueCount());
+
+        sorter.sortOutOfPlace(vector, sortedVec, comparator);
+
+        // verify results
+        verifyResults(sortedVec, (String[]) array);
+      }
+    }
+  }
+
+  @Parameterized.Parameters(name = "length = {0}, null fraction = {1}, vector = {2}")
+  public static Collection<Object[]> getParameters() {
+    List<Object[]> params = new ArrayList<>();
+    for (int length : VECTOR_LENGTHS) {
+      for (double nullFrac : NULL_FRACTIONS) {
+        params.add(new Object[]{
+            length, nullFrac, "VarCharVector",
+            (Function<BufferAllocator, VarCharVector>) (allocator -> new VarCharVector("vector", allocator)),
+            TestSortingUtil.STRING_GENERATOR
+        });
+      }
+    }
+    return params;
+  }
+
+  /**
+   * Verify results as byte arrays.
+   */
+  public static <V extends ValueVector> void verifyResults(V vector, String[] expected) {
+    assertEquals(vector.getValueCount(), expected.length);
+    for (int i = 0; i < expected.length; i++) {
+      if (expected[i] == null) {
+        assertTrue(vector.isNull(i));
+      } else {
+        assertArrayEquals(((Text) vector.getObject(i)).getBytes(), expected[i].getBytes());
+      }
+    }
+  }
+
+  /**
+   * String comparator with the same behavior as that of
+   * {@link DefaultVectorComparators.VariableWidthComparator}.
+   */
+  static class StringComparator implements Comparator<String> {
+
+    @Override
+    public int compare(String str1, String str2) {
+      if (str1 == null || str2 == null) {
+        if (str1 == null && str2 == null) {
+          return 0;
+        }
+
+        return str1 == null ? -1 : 1;
+      }
+
+      byte[] bytes1 = str1.getBytes();
+      byte[] bytes2 = str2.getBytes();
+
+      for (int i = 0; i < bytes1.length && i < bytes2.length; i++) {
+        if (bytes1[i] != bytes2[i]) {
+          return (bytes1[i] & 0xff) < (bytes2[i] & 0xff) ? -1 : 1;
+        }
+      }
+      return bytes1.length - bytes2.length;
+    }
+  }
+}
diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml
index 292dc1c9287..7fd3f89e881 100644
--- a/java/flight/flight-core/pom.xml
+++ b/java/flight/flight-core/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>2.0.0-SNAPSHOT</version>
+    <version>3.0.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ArrowMessage.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ArrowMessage.java
index 917d8435e06..19cde94a636 100644
--- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ArrowMessage.java
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ArrowMessage.java
@@ -35,6 +35,8 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.util.AutoCloseables;
 import org.apache.arrow.util.Preconditions;
+import org.apache.arrow.vector.compression.NoCompressionCodec;
+import org.apache.arrow.vector.ipc.message.ArrowBodyCompression;
 import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
 import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
 import org.apache.arrow.vector.ipc.message.IpcOption;
@@ -124,6 +126,7 @@ public static HeaderType getHeader(byte b) {
   private final MessageMetadataResult message;
   private final ArrowBuf appMetadata;
   private final List<ArrowBuf> bufs;
+  private final ArrowBodyCompression bodyCompression;
 
   public ArrowMessage(FlightDescriptor descriptor, Schema schema, IpcOption option) {
     this.writeOption = option;
@@ -133,6 +136,7 @@ public ArrowMessage(FlightDescriptor descriptor, Schema schema, IpcOption option
     bufs = ImmutableList.of();
     this.descriptor = descriptor;
     this.appMetadata = null;
+    this.bodyCompression = NoCompressionCodec.DEFAULT_BODY_COMPRESSION;
   }
 
   /**
@@ -147,6 +151,7 @@ public ArrowMessage(ArrowRecordBatch batch, ArrowBuf appMetadata, IpcOption opti
     this.bufs = ImmutableList.copyOf(batch.getBuffers());
     this.descriptor = null;
     this.appMetadata = appMetadata;
+    this.bodyCompression = batch.getBodyCompression();
   }
 
   public ArrowMessage(ArrowDictionaryBatch batch, IpcOption option) {
@@ -159,6 +164,7 @@ public ArrowMessage(ArrowDictionaryBatch batch, IpcOption option) {
     this.bufs = ImmutableList.copyOf(batch.getDictionary().getBuffers());
     this.descriptor = null;
     this.appMetadata = null;
+    this.bodyCompression = batch.getDictionary().getBodyCompression();
   }
 
   /**
@@ -172,6 +178,7 @@ public ArrowMessage(ArrowBuf appMetadata) {
     this.bufs = ImmutableList.of();
     this.descriptor = null;
     this.appMetadata = appMetadata;
+    this.bodyCompression = NoCompressionCodec.DEFAULT_BODY_COMPRESSION;
   }
 
   public ArrowMessage(FlightDescriptor descriptor) {
@@ -181,6 +188,7 @@ public ArrowMessage(FlightDescriptor descriptor) {
     this.bufs = ImmutableList.of();
     this.descriptor = descriptor;
     this.appMetadata = null;
+    this.bodyCompression = NoCompressionCodec.DEFAULT_BODY_COMPRESSION;
   }
 
   private ArrowMessage(FlightDescriptor descriptor, MessageMetadataResult message, ArrowBuf appMetadata,
@@ -194,6 +202,7 @@ private ArrowMessage(FlightDescriptor descriptor, MessageMetadataResult message,
     this.descriptor = descriptor;
     this.appMetadata = appMetadata;
     this.bufs = buf == null ? ImmutableList.of() : ImmutableList.of(buf);
+    this.bodyCompression = NoCompressionCodec.DEFAULT_BODY_COMPRESSION;
   }
 
   public MessageMetadataResult asSchemaMessage() {
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/BackpressureStrategy.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/BackpressureStrategy.java
new file mode 100644
index 00000000000..de34643a758
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/BackpressureStrategy.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+import org.apache.arrow.vector.VectorSchemaRoot;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Helper interface to dynamically handle backpressure when implementing FlightProducers.
+ * This must only be used in FlightProducer implementations that are non-blocking.
+ */
+public interface BackpressureStrategy {
+  /**
+   * The state of the client after a call to waitForListener.
+   */
+  enum WaitResult {
+    /**
+     * Listener is ready.
+     */
+    READY,
+
+    /**
+     * Listener was cancelled by the client.
+     */
+    CANCELLED,
+
+    /**
+     * Timed out waiting for the listener to change state.
+     */
+    TIMEOUT,
+
+    /**
+     * Indicates that the wait was interrupted for a reason
+     * unrelated to the listener itself.
+     */
+    OTHER
+  }
+
+  /**
+   * Set up operations to work against the given listener.
+   *
+   * This must be called exactly once and before any calls to {@link #waitForListener(long)} and
+   * {@link OutboundStreamListener#start(VectorSchemaRoot)}
+   * @param listener The listener this strategy applies to.
+   */
+  void register(FlightProducer.ServerStreamListener listener);
+
+  /**
+   * Waits for the listener to be ready or cancelled up to the given timeout.
+   *
+   * @param timeout The timeout in milliseconds. Infinite if timeout is <= 0.
+   * @return The result of the wait.
+   */
+  WaitResult waitForListener(long timeout);
+
+  /**
+   * A back pressure strategy that uses callbacks to notify when the client is ready or cancelled.
+   */
+  class CallbackBackpressureStrategy implements BackpressureStrategy {
+    private final Object lock = new Object();
+    private FlightProducer.ServerStreamListener listener;
+
+    @Override
+    public void register(FlightProducer.ServerStreamListener listener) {
+      this.listener = listener;
+      listener.setOnReadyHandler(this::onReady);
+      listener.setOnCancelHandler(this::onCancel);
+    }
+
+    @Override
+    public WaitResult waitForListener(long timeout) {
+      Preconditions.checkNotNull(listener);
+      long remainingTimeout = timeout;
+      final long startTime = System.currentTimeMillis();
+      synchronized (lock) {
+        while (!listener.isReady() && !listener.isCancelled()) {
+          try {
+            lock.wait(remainingTimeout);
+            if (timeout != 0) { // If timeout was zero explicitly, we should never report timeout.
+              remainingTimeout = startTime + timeout - System.currentTimeMillis();
+              if (remainingTimeout <= 0) {
+                return WaitResult.TIMEOUT;
+              }
+            }
+            if (!shouldContinueWaiting(listener, remainingTimeout)) {
+              return WaitResult.OTHER;
+            }
+          } catch (InterruptedException ex) {
+            Thread.currentThread().interrupt();
+            return WaitResult.OTHER;
+          }
+        }
+
+        if (listener.isReady()) {
+          return WaitResult.READY;
+        } else if (listener.isCancelled()) {
+          return WaitResult.CANCELLED;
+        } else if (System.currentTimeMillis() > startTime + timeout) {
+          return WaitResult.TIMEOUT;
+        }
+        throw new RuntimeException("Invalid state when waiting for listener.");
+      }
+    }
+
+    /**
+     * Interrupt waiting on the listener to change state.
+     *
+     * This method can be used in conjunction with
+     * {@link #shouldContinueWaiting(FlightProducer.ServerStreamListener, long)} to allow FlightProducers to
+     * terminate streams internally and notify clients.
+     */
+    public void interruptWait() {
+      synchronized (lock) {
+        lock.notifyAll();
+      }
+    }
+
+    /**
+     * Callback function to run to check if the listener should continue
+     * to be waited on if it leaves the waiting state without being cancelled,
+     * ready, or timed out.
+     *
+     * This method should be used to determine if the wait on the listener was interrupted explicitly using a
+     * call to {@link #interruptWait()} or if it was woken up due to a spurious wake.
+     */
+    protected boolean shouldContinueWaiting(FlightProducer.ServerStreamListener listener, long remainingTimeout) {
+      return true;
+    }
+
+    /**
+     * Callback to execute when the listener becomes ready.
+     */
+    protected void readyCallback() {
+    }
+
+    /**
+     * Callback to execute when the listener is cancelled.
+     */
+    protected void cancelCallback() {
+    }
+
+    private void onReady() {
+      synchronized (lock) {
+        readyCallback();
+        lock.notifyAll();
+      }
+    }
+
+    private void onCancel() {
+      synchronized (lock) {
+        cancelCallback();
+        lock.notifyAll();
+      }
+    }
+  }
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/CallOptions.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/CallOptions.java
index aaf5691fae3..bbb4edef9c0 100644
--- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/CallOptions.java
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/CallOptions.java
@@ -56,7 +56,7 @@ public <T extends AbstractStub<T>> T wrapStub(T stub) {
   /**
    * CallOptions specific to GRPC stubs.
    */
-  interface GrpcCallOption extends CallOption {
+  public interface GrpcCallOption extends CallOption {
     <T extends AbstractStub<T>> T wrapStub(T stub);
   }
 }
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightCallHeaders.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightCallHeaders.java
new file mode 100644
index 00000000000..dd26d190872
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightCallHeaders.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+import java.util.Collection;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+
+import io.grpc.Metadata;
+
+/**
+ * An implementation of the Flight headers interface for headers.
+ */
+public class FlightCallHeaders implements CallHeaders {
+  private final Multimap<String, Object> keysAndValues;
+
+  public FlightCallHeaders() {
+    this.keysAndValues = ArrayListMultimap.create();
+  }
+
+  @Override
+  public String get(String key) {
+    final Collection<Object> values = this.keysAndValues.get(key);
+    if (values.isEmpty()) {
+      return null;
+    }
+
+    if (key.endsWith(Metadata.BINARY_HEADER_SUFFIX)) {
+      return new String((byte[]) Iterables.get(values, 0));
+    }
+
+    return (String) Iterables.get(values, 0);
+  }
+
+  @Override
+  public byte[] getByte(String key) {
+    final Collection<Object> values = this.keysAndValues.get(key);
+    if (values.isEmpty()) {
+      return null;
+    }
+
+    if (key.endsWith(Metadata.BINARY_HEADER_SUFFIX)) {
+      return (byte[]) Iterables.get(values, 0);
+    }
+
+    return ((String) Iterables.get(values, 0)).getBytes();
+  }
+
+  @Override
+  public Iterable<String> getAll(String key) {
+    if (key.endsWith(Metadata.BINARY_HEADER_SUFFIX)) {
+      return this.keysAndValues.get(key).stream().map(o -> new String((byte[]) o)).collect(Collectors.toList());
+    }
+    return (Collection<String>) (Collection<?>) this.keysAndValues.get(key);
+  }
+
+  @Override
+  public Iterable<byte[]> getAllByte(String key) {
+    if (key.endsWith(Metadata.BINARY_HEADER_SUFFIX)) {
+      return (Collection<byte[]>) (Collection<?>) this.keysAndValues.get(key);
+    }
+    return this.keysAndValues.get(key).stream().map(o -> ((String) o).getBytes()).collect(Collectors.toList());
+  }
+
+  @Override
+  public void insert(String key, String value) {
+    this.keysAndValues.put(key, value);
+  }
+
+  @Override
+  public void insert(String key, byte[] value) {
+    Preconditions.checkArgument(key.endsWith("-bin"), "Binary header is named %s. It must end with %s", key, "-bin");
+    Preconditions.checkArgument(key.length() > "-bin".length(), "empty key name");
+
+    this.keysAndValues.put(key, value);
+  }
+
+  @Override
+  public Set<String> keys() {
+    return this.keysAndValues.keySet();
+  }
+
+  @Override
+  public boolean containsKey(String key) {
+    return this.keysAndValues.containsKey(key);
+  }
+
+  public String toString() {
+    return this.keysAndValues.toString();
+  }
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightClient.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightClient.java
index f477fa01c29..762b37859b9 100644
--- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightClient.java
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightClient.java
@@ -22,6 +22,7 @@
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Optional;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.function.BooleanSupplier;
@@ -33,7 +34,12 @@
 import org.apache.arrow.flight.auth.ClientAuthHandler;
 import org.apache.arrow.flight.auth.ClientAuthInterceptor;
 import org.apache.arrow.flight.auth.ClientAuthWrapper;
+import org.apache.arrow.flight.auth2.BasicAuthCredentialWriter;
+import org.apache.arrow.flight.auth2.ClientBearerHeaderHandler;
+import org.apache.arrow.flight.auth2.ClientHandshakeWrapper;
+import org.apache.arrow.flight.auth2.ClientIncomingAuthHeaderMiddleware;
 import org.apache.arrow.flight.grpc.ClientInterceptorAdapter;
+import org.apache.arrow.flight.grpc.CredentialCallOption;
 import org.apache.arrow.flight.grpc.StatusUtils;
 import org.apache.arrow.flight.impl.Flight;
 import org.apache.arrow.flight.impl.Flight.Empty;
@@ -62,6 +68,7 @@
 import io.netty.channel.EventLoopGroup;
 import io.netty.channel.ServerChannel;
 import io.netty.handler.ssl.SslContextBuilder;
+import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
 
 /**
  * Client for Flight services.
@@ -79,6 +86,7 @@ public class FlightClient implements AutoCloseable {
   private final MethodDescriptor<Flight.Ticket, ArrowMessage> doGetDescriptor;
   private final MethodDescriptor<ArrowMessage, Flight.PutResult> doPutDescriptor;
   private final MethodDescriptor<ArrowMessage, ArrowMessage> doExchangeDescriptor;
+  private final List<FlightClientMiddleware.Factory> middleware;
 
   /**
    * Create a Flight client from an allocator and a gRPC channel.
@@ -87,6 +95,7 @@ public class FlightClient implements AutoCloseable {
       List<FlightClientMiddleware.Factory> middleware) {
     this.allocator = incomingAllocator.newChildAllocator("flight-client", 0, Long.MAX_VALUE);
     this.channel = channel;
+    this.middleware = middleware;
 
     final ClientInterceptor[] interceptors;
     interceptors = new ClientInterceptor[]{authInterceptor, new ClientInterceptorAdapter(middleware)};
@@ -175,6 +184,32 @@ public void authenticate(ClientAuthHandler handler, CallOption... options) {
     authInterceptor.setAuthHandler(handler);
   }
 
+  /**
+   * Authenticates with a username and password.
+   *
+   * @param username the username.
+   * @param password the password.
+   * @return a CredentialCallOption containing a bearer token if the server emitted one, or
+   *     empty if no bearer token was returned. This can be used in subsequent API calls.
+   */
+  public Optional<CredentialCallOption> authenticateBasicToken(String username, String password) {
+    final ClientIncomingAuthHeaderMiddleware.Factory clientAuthMiddleware =
+            new ClientIncomingAuthHeaderMiddleware.Factory(new ClientBearerHeaderHandler());
+    middleware.add(clientAuthMiddleware);
+    handshake(new CredentialCallOption(new BasicAuthCredentialWriter(username, password)));
+
+    return Optional.ofNullable(clientAuthMiddleware.getCredentialCallOption());
+  }
+
+  /**
+   * Executes the handshake against the Flight service.
+   *
+   * @param options RPC-layer hints for this call.
+   */
+  public void handshake(CallOption... options) {
+    ClientHandshakeWrapper.doClientHandshake(CallOptions.wrapStub(asyncStub, options));
+  }
+
   /**
    * Create or append a descriptor with another stream.
    *
@@ -321,7 +356,7 @@ public ExchangeReaderWriter doExchange(FlightDescriptor descriptor, CallOption..
       final ClientCallStreamObserver<ArrowMessage> observer = (ClientCallStreamObserver<ArrowMessage>)
               ClientCalls.asyncBidiStreamingCall(call, stream.asObserver());
       final ClientStreamListener writer = new PutObserver(
-          descriptor, observer, stream.completed::isDone,
+          descriptor, observer, stream.cancelled::isDone,
           () -> {
             try {
               stream.completed.get();
@@ -380,6 +415,9 @@ public void close() throws Exception {
     }
   }
 
+  /**
+   * A stream observer for Flight.PutResult
+   */
   private static class SetStreamObserver implements StreamObserver<Flight.PutResult> {
     private final BufferAllocator allocator;
     private final StreamListener<PutResult> listener;
@@ -523,7 +561,6 @@ public static Builder builder(BufferAllocator allocator, Location location) {
    * A builder for Flight clients.
    */
   public static final class Builder {
-
     private BufferAllocator allocator;
     private Location location;
     private boolean forceTls = false;
@@ -533,6 +570,7 @@ public static final class Builder {
     private InputStream clientKey = null;
     private String overrideHostname = null;
     private List<FlightClientMiddleware.Factory> middleware = new ArrayList<>();
+    private boolean verifyServer = true;
 
     private Builder() {
     }
@@ -592,6 +630,11 @@ public Builder intercept(FlightClientMiddleware.Factory factory) {
       return this;
     }
 
+    public Builder verifyServer(boolean verifyServer) {
+      this.verifyServer = verifyServer;
+      return this;
+    }
+
     /**
      * Create the client from this builder.
      */
@@ -637,19 +680,29 @@ public FlightClient build() {
       if (this.forceTls || LocationSchemes.GRPC_TLS.equals(location.getUri().getScheme())) {
         builder.useTransportSecurity();
 
-        if (this.trustedCertificates != null || this.clientCertificate != null || this.clientKey != null) {
-          final SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient();
+        final boolean hasTrustedCerts = this.trustedCertificates != null;
+        final boolean hasKeyCertPair = this.clientCertificate != null && this.clientKey != null;
+        if (!this.verifyServer && (hasTrustedCerts || hasKeyCertPair)) {
+          throw new IllegalArgumentException("FlightClient has been configured to disable server verification, " +
+              "but certificate options have been specified.");
+        }
+
+        final SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient();
+
+        if (!this.verifyServer) {
+          sslContextBuilder.trustManager(InsecureTrustManagerFactory.INSTANCE);
+        } else if (this.trustedCertificates != null || this.clientCertificate != null || this.clientKey != null) {
           if (this.trustedCertificates != null) {
             sslContextBuilder.trustManager(this.trustedCertificates);
           }
           if (this.clientCertificate != null && this.clientKey != null) {
             sslContextBuilder.keyManager(this.clientCertificate, this.clientKey);
           }
-          try {
-            builder.sslContext(sslContextBuilder.build());
-          } catch (SSLException e) {
-            throw new RuntimeException(e);
-          }
+        }
+        try {
+          builder.sslContext(sslContextBuilder.build());
+        } catch (SSLException e) {
+          throw new RuntimeException(e);
         }
 
         if (this.overrideHostname != null) {
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightConstants.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightConstants.java
index 2ea8cc7e344..2d039c9d24e 100644
--- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightConstants.java
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightConstants.java
@@ -24,4 +24,6 @@ public interface FlightConstants {
 
   String SERVICE = "arrow.flight.protocol.FlightService";
 
+  FlightServerMiddleware.Key<ServerHeaderMiddleware> HEADER_KEY =
+      FlightServerMiddleware.Key.of("org.apache.arrow.flight.ServerHeaderMiddleware");
 }
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightRuntimeException.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightRuntimeException.java
index 9d32269c4c1..76d3349a2c3 100644
--- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightRuntimeException.java
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightRuntimeException.java
@@ -41,6 +41,6 @@ public CallStatus status() {
   @Override
   public String toString() {
     String s = getClass().getName();
-    return String.format("%s: %s", s, status);
+    return String.format("%s: %s: %s", s, status.code(), status.description());
   }
 }
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightServer.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightServer.java
index 3c8b7aee23c..d59480bfb0a 100644
--- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightServer.java
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightServer.java
@@ -36,6 +36,9 @@
 
 import org.apache.arrow.flight.auth.ServerAuthHandler;
 import org.apache.arrow.flight.auth.ServerAuthInterceptor;
+import org.apache.arrow.flight.auth2.Auth2Constants;
+import org.apache.arrow.flight.auth2.CallHeaderAuthenticator;
+import org.apache.arrow.flight.auth2.ServerCallHeaderAuthMiddleware;
 import org.apache.arrow.flight.grpc.ServerInterceptorAdapter;
 import org.apache.arrow.flight.grpc.ServerInterceptorAdapter.KeyFactory;
 import org.apache.arrow.memory.BufferAllocator;
@@ -164,6 +167,7 @@ public static final class Builder {
     private FlightProducer producer;
     private final Map<String, Object> builderOptions;
     private ServerAuthHandler authHandler = ServerAuthHandler.NO_OP;
+    private CallHeaderAuthenticator headerAuthenticator = CallHeaderAuthenticator.NO_OP;
     private ExecutorService executor = null;
     private int maxInboundMessageSize = MAX_GRPC_MESSAGE_SIZE;
     private InputStream certChain;
@@ -187,6 +191,14 @@ public static final class Builder {
 
     /** Create the server for this builder. */
     public FlightServer build() {
+      // Add the auth middleware if applicable.
+      if (headerAuthenticator != CallHeaderAuthenticator.NO_OP) {
+        this.middleware(FlightServerMiddleware.Key.of(Auth2Constants.AUTHORIZATION_HEADER),
+            new ServerCallHeaderAuthMiddleware.Factory(headerAuthenticator));
+      }
+
+      this.middleware(FlightConstants.HEADER_KEY, new ServerHeaderMiddleware.Factory());
+
       final NettyServerBuilder builder;
       switch (location.getUri().getScheme()) {
         case LocationSchemes.GRPC_DOMAIN_SOCKET: {
@@ -334,6 +346,14 @@ public Builder authHandler(ServerAuthHandler authHandler) {
       return this;
     }
 
+    /**
+     * Set the header-based authentication mechanism.
+     */
+    public Builder headerAuthenticator(CallHeaderAuthenticator headerAuthenticator) {
+      this.headerAuthenticator = headerAuthenticator;
+      return this;
+    }
+
     /**
      * Provide a transport-specific option. Not guaranteed to have any effect.
      */
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightServerMiddleware.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightServerMiddleware.java
index dc152e6e085..9bc8bbfe739 100644
--- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightServerMiddleware.java
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightServerMiddleware.java
@@ -44,10 +44,11 @@ interface Factory<T extends FlightServerMiddleware> {
      *
      * @param info Details about the call.
      * @param incomingHeaders A mutable set of request headers.
+     * @param context Context about the current request.
      *
      * @throws FlightRuntimeException if the middleware wants to reject the call with the given status
      */
-    T onCallStarted(CallInfo info, CallHeaders incomingHeaders);
+    T onCallStarted(CallInfo info, CallHeaders incomingHeaders, RequestContext context);
   }
 
   /**
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightService.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightService.java
index 30c7d309877..4fb0dea2cba 100644
--- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightService.java
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightService.java
@@ -28,17 +28,20 @@
 import org.apache.arrow.flight.auth.AuthConstants;
 import org.apache.arrow.flight.auth.ServerAuthHandler;
 import org.apache.arrow.flight.auth.ServerAuthWrapper;
+import org.apache.arrow.flight.auth2.Auth2Constants;
 import org.apache.arrow.flight.grpc.ContextPropagatingExecutorService;
+import org.apache.arrow.flight.grpc.RequestContextAdapter;
 import org.apache.arrow.flight.grpc.ServerInterceptorAdapter;
 import org.apache.arrow.flight.grpc.StatusUtils;
 import org.apache.arrow.flight.impl.Flight;
 import org.apache.arrow.flight.impl.FlightServiceGrpc.FlightServiceImplBase;
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.VectorUnloader;
+import org.apache.arrow.util.AutoCloseables;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import io.grpc.Status;
+import com.google.common.base.Strings;
+
 import io.grpc.stub.ServerCallStreamObserver;
 import io.grpc.stub.StreamObserver;
 
@@ -64,11 +67,25 @@ class FlightService extends FlightServiceImplBase {
   }
 
   private CallContext makeContext(ServerCallStreamObserver<?> responseObserver) {
-    return new CallContext(AuthConstants.PEER_IDENTITY_KEY.get(), responseObserver::isCancelled);
+    // Try to get the peer identity from middleware first (using the auth2 interfaces).
+    final RequestContext context = RequestContextAdapter.REQUEST_CONTEXT_KEY.get();
+    String peerIdentity = null;
+    if (context != null) {
+      peerIdentity = context.get(Auth2Constants.PEER_IDENTITY_KEY);
+    }
+
+    if (Strings.isNullOrEmpty(peerIdentity)) {
+      // Try the legacy auth interface, which defaults to empty string.
+      peerIdentity = AuthConstants.PEER_IDENTITY_KEY.get();
+    }
+
+    return new CallContext(peerIdentity, responseObserver::isCancelled);
   }
 
   @Override
   public StreamObserver<Flight.HandshakeRequest> handshake(StreamObserver<Flight.HandshakeResponse> responseObserver) {
+    // This method is not meaningful with the auth2 interfaces. Authentication would already
+    // have happened by header/middleware with the auth2 classes.
     return ServerAuthWrapper.wrapHandshake(authHandler, responseObserver, executors);
   }
 
@@ -88,6 +105,7 @@ public void listFlights(Flight.Criteria criteria, StreamObserver<Flight.FlightIn
   public void doGetCustom(Flight.Ticket ticket, StreamObserver<ArrowMessage> responseObserverSimple) {
     final ServerCallStreamObserver<ArrowMessage> responseObserver =
         (ServerCallStreamObserver<ArrowMessage>) responseObserverSimple;
+
     final GetListener listener = new GetListener(responseObserver, this::handleExceptionWithMiddleware);
     try {
       producer.getStream(makeContext(responseObserver), new Ticket(ticket), listener);
@@ -127,8 +145,7 @@ private static class GetListener extends OutboundStreamListenerImpl implements S
     private ServerCallStreamObserver<ArrowMessage> responseObserver;
     private final Consumer<Throwable> errorHandler;
     private Runnable onCancelHandler = null;
-    // null until stream started
-    private volatile VectorUnloader unloader;
+    private Runnable onReadyHandler = null;
     private boolean completed;
 
     public GetListener(ServerCallStreamObserver<ArrowMessage> responseObserver, Consumer<Throwable> errorHandler) {
@@ -137,6 +154,7 @@ public GetListener(ServerCallStreamObserver<ArrowMessage> responseObserver, Cons
       this.completed = false;
       this.responseObserver = responseObserver;
       this.responseObserver.setOnCancelHandler(this::onCancel);
+      this.responseObserver.setOnReadyHandler(this::onReady);
       this.responseObserver.disableAutoInboundFlowControl();
     }
 
@@ -147,11 +165,22 @@ private void onCancel() {
       }
     }
 
+    private void onReady() {
+      if (onReadyHandler != null) {
+        onReadyHandler.run();
+      }
+    }
+
     @Override
     public void setOnCancelHandler(Runnable handler) {
       this.onCancelHandler = handler;
     }
 
+    @Override
+    public void setOnReadyHandler(Runnable handler) {
+      this.onReadyHandler = handler;
+    }
+
     @Override
     public boolean isCancelled() {
       return responseObserver.isCancelled();
@@ -189,24 +218,22 @@ public StreamObserver<ArrowMessage> doPutCustom(final StreamObserver<Flight.PutR
     responseObserver.disableAutoInboundFlowControl();
     responseObserver.request(1);
 
-    final FlightStream fs = new FlightStream(allocator, PENDING_REQUESTS, (String message, Throwable cause) -> {
-      responseObserver.onError(Status.CANCELLED.withCause(cause).withDescription(message).asException());
-    }, responseObserver::request);
+    final StreamPipe<PutResult, Flight.PutResult> ackStream = StreamPipe
+        .wrap(responseObserver, PutResult::toProtocol, this::handleExceptionWithMiddleware);
+    final FlightStream fs = new FlightStream(
+        allocator,
+        PENDING_REQUESTS,
+        /* server-upload streams are not cancellable */null,
+        responseObserver::request);
+    // When the ackStream is completed, the FlightStream will be closed with it
+    ackStream.setAutoCloseable(fs);
     final StreamObserver<ArrowMessage> observer = fs.asObserver();
     executors.submit(() -> {
-      final StreamPipe<PutResult, Flight.PutResult> ackStream = StreamPipe
-          .wrap(responseObserver, PutResult::toProtocol, this::handleExceptionWithMiddleware);
       try {
         producer.acceptPut(makeContext(responseObserver), fs, ackStream).run();
       } catch (Exception ex) {
         ackStream.onError(ex);
       } finally {
-        // Close this stream before telling gRPC that the call is complete. That way we don't race with server shutdown.
-        try {
-          fs.close();
-        } catch (Exception e) {
-          handleExceptionWithMiddleware(e);
-        }
         // ARROW-6136: Close the stream if and only if acceptPut hasn't closed it itself
         // We don't do this for other streams since the implementation may be asynchronous
         ackStream.ensureCompleted();
@@ -236,7 +263,7 @@ public void getFlightInfo(Flight.FlightDescriptor request, StreamObserver<Flight
    */
   private void handleExceptionWithMiddleware(Throwable t) {
     final Map<Key<?>, FlightServerMiddleware> middleware = ServerInterceptorAdapter.SERVER_MIDDLEWARE_KEY.get();
-    if (middleware == null) {
+    if (middleware == null || middleware.isEmpty()) {
       logger.error("Uncaught exception in Flight method body", t);
       return;
     }
@@ -258,14 +285,14 @@ public void getSchema(Flight.FlightDescriptor request, StreamObserver<Flight.Sch
 
   /** Ensures that other resources are cleaned up when the service finishes its call.  */
   private static class ExchangeListener extends GetListener {
-    private final AutoCloseable resource;
+
+    private AutoCloseable resource;
     private boolean closed = false;
     private Runnable onCancelHandler = null;
 
-    public ExchangeListener(ServerCallStreamObserver<ArrowMessage> responseObserver, Consumer<Throwable> errorHandler,
-                            AutoCloseable resource) {
+    public ExchangeListener(ServerCallStreamObserver<ArrowMessage> responseObserver, Consumer<Throwable> errorHandler) {
       super(responseObserver, errorHandler);
-      this.resource = resource;
+      this.resource = null;
       super.setOnCancelHandler(() -> {
         try {
           if (onCancelHandler != null) {
@@ -285,7 +312,7 @@ private void cleanup() {
       }
       closed = true;
       try {
-        this.resource.close();
+        AutoCloseables.close(resource);
       } catch (Exception e) {
         throw CallStatus.INTERNAL
             .withCause(e)
@@ -321,19 +348,16 @@ public void setOnCancelHandler(Runnable handler) {
   public StreamObserver<ArrowMessage> doExchangeCustom(StreamObserver<ArrowMessage> responseObserverSimple) {
     final ServerCallStreamObserver<ArrowMessage> responseObserver =
         (ServerCallStreamObserver<ArrowMessage>) responseObserverSimple;
-    final FlightStream fs = new FlightStream(allocator, PENDING_REQUESTS, (String message, Throwable cause) -> {
-      responseObserver.onError(Status.CANCELLED.withCause(cause).withDescription(message).asException());
-    }, responseObserver::request);
-    // When service completes the call, this cleans up the FlightStream
     final ExchangeListener listener = new ExchangeListener(
         responseObserver,
-        this::handleExceptionWithMiddleware,
-        () -> {
-          // Force the stream to "complete" so it will close without incident. At this point, we don't care since
-          // we are about to end the call. (Normally it will raise an error.)
-          fs.completed.complete(null);
-          fs.close();
-        });
+        this::handleExceptionWithMiddleware);
+    final FlightStream fs = new FlightStream(
+        allocator,
+        PENDING_REQUESTS,
+        /* server-upload streams are not cancellable */null,
+        responseObserver::request);
+    // When service completes the call, this cleans up the FlightStream
+    listener.resource = fs;
     responseObserver.disableAutoInboundFlowControl();
     responseObserver.request(1);
     final StreamObserver<ArrowMessage> observer = fs.asObserver();
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightStream.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightStream.java
index 0e832174106..5ac22c06646 100644
--- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightStream.java
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightStream.java
@@ -23,6 +23,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -54,7 +55,6 @@
  * An adaptor between protobuf streams and flight data streams.
  */
 public class FlightStream implements AutoCloseable {
-
   // Use AutoCloseable sentinel objects to simplify logic in #close
   private final AutoCloseable DONE = () -> {
   };
@@ -68,7 +68,13 @@ public class FlightStream implements AutoCloseable {
   private final SettableFuture<FlightDescriptor> descriptor = SettableFuture.create();
   private final int pendingTarget;
   private final Requestor requestor;
+  // The completion flags.
+  // This flag is only updated as the user iterates through the data, i.e. it tracks whether the user has read all the
+  // data and closed the stream
   final CompletableFuture<Void> completed;
+  // This flag is immediately updated when gRPC signals that the server has ended the call. This is used to make sure
+  // we don't block forever trying to write to a server that has rejected a call.
+  final CompletableFuture<Void> cancelled;
 
   private volatile int pending = 1;
   private volatile VectorSchemaRoot fulfilledRoot;
@@ -84,16 +90,19 @@ public class FlightStream implements AutoCloseable {
    *
    * @param allocator  The allocator to use for creating/reallocating buffers for Vectors.
    * @param pendingTarget Target number of messages to receive.
-   * @param cancellable Only provided for streams from server to client, used to cancel mid-stream requests.
+   * @param cancellable Used to cancel mid-stream requests.
    * @param requestor A callback to determine how many pending items there are.
    */
   public FlightStream(BufferAllocator allocator, int pendingTarget, Cancellable cancellable, Requestor requestor) {
+    Objects.requireNonNull(allocator);
+    Objects.requireNonNull(requestor);
     this.allocator = allocator;
     this.pendingTarget = pendingTarget;
     this.cancellable = cancellable;
     this.requestor = requestor;
     this.dictionaries = new DictionaryProvider.MapDictionaryProvider();
     this.completed = new CompletableFuture<>();
+    this.cancelled = new CompletableFuture<>();
   }
 
   /**
@@ -158,29 +167,52 @@ public FlightDescriptor getDescriptor() {
   /**
    * Closes the stream (freeing any existing resources).
    *
-   * <p>If the stream isn't complete and is cancellable, this method will cancel the stream first.</p>
+   * <p>If the stream isn't complete and is cancellable, this method will cancel and drain the stream first.
    */
   public void close() throws Exception {
     final List<AutoCloseable> closeables = new ArrayList<>();
-    // cancellation can throw, but we still want to clean up resources, so make it an AutoCloseable too
-    closeables.add(() -> {
-      if (!completed.isDone() && cancellable != null) {
-        cancel("Stream closed before end.", /* no exception to report */ null);
+    Throwable suppressor = null;
+    if (cancellable != null) {
+      // Client-side stream. Cancel the call, to help ensure gRPC doesn't deliver a message after close() ends.
+      // On the server side, we can't rely on draining the stream , because this gRPC bug means the completion callback
+      // may never run https://github.com/grpc/grpc-java/issues/5882
+      try {
+        synchronized (cancellable) {
+          if (!cancelled.isDone()) {
+            // Only cancel if the call is not done on the gRPC side
+            cancellable.cancel("Stream closed before end", /* no exception to report */null);
+          }
+        }
+        // Drain the stream without the lock (as next() implicitly needs the lock)
+        while (next()) { }
+      } catch (FlightRuntimeException e) {
+        suppressor = e;
       }
-    });
-    if (fulfilledRoot != null) {
-      closeables.add(fulfilledRoot);
     }
-    closeables.add(applicationMetadata);
-    closeables.addAll(queue);
-    if (dictionaries != null) {
-      dictionaries.getDictionaryIds().forEach(id -> closeables.add(dictionaries.lookup(id).getVector()));
+    // Perform these operations under a lock. This way the observer can't enqueue new messages while we're in the
+    // middle of cleanup. This should only be a concern for server-side streams since client-side streams are drained
+    // by the lambda above.
+    synchronized (completed) {
+      try {
+        if (fulfilledRoot != null) {
+          closeables.add(fulfilledRoot);
+        }
+        closeables.add(applicationMetadata);
+        closeables.addAll(queue);
+        if (dictionaries != null) {
+          dictionaries.getDictionaryIds().forEach(id -> closeables.add(dictionaries.lookup(id).getVector()));
+        }
+        if (suppressor != null) {
+          AutoCloseables.close(suppressor, closeables);
+        } else {
+          AutoCloseables.close(closeables);
+        }
+      } finally {
+        // The value of this CompletableFuture is meaningless, only whether it's completed (or has an exception)
+        // No-op if already complete
+        completed.complete(null);
+      }
     }
-
-    AutoCloseables.close(closeables);
-    // Other code ignores the value of this CompletableFuture, only whether it's completed (or has an exception)
-    // No-op if already complete; do this after the check in the AutoCloseable lambda above
-    completed.complete(null);
   }
 
   /**
@@ -337,8 +369,22 @@ private class Observer implements StreamObserver<ArrowMessage> {
       super();
     }
 
+    /** Helper to add an item to the queue under the appropriate lock. */
+    private void enqueue(AutoCloseable message) {
+      synchronized (completed) {
+        if (completed.isDone()) {
+          // The stream is already closed (RPC ended), discard the message
+          AutoCloseables.closeNoChecked(message);
+        } else {
+          queue.add(message);
+        }
+      }
+    }
+
     @Override
     public void onNext(ArrowMessage msg) {
+      // Operations here have to be under a lock so that we don't add a message to the queue while in the middle of
+      // close().
       requestOutstanding();
       switch (msg.getMessageType()) {
         case NONE: {
@@ -347,7 +393,7 @@ public void onNext(ArrowMessage msg) {
             descriptor.set(new FlightDescriptor(msg.getDescriptor()));
           }
           if (msg.getApplicationMetadata() != null) {
-            queue.add(msg);
+            enqueue(msg);
           }
           break;
         }
@@ -367,29 +413,31 @@ public void onNext(ArrowMessage msg) {
           try {
             MetadataV4UnionChecker.checkRead(schema, metadataVersion);
           } catch (IOException e) {
-            queue.add(DONE_EX);
             ex = e;
+            enqueue(DONE_EX);
             break;
           }
 
-          fulfilledRoot = VectorSchemaRoot.create(schema, allocator);
-          loader = new VectorLoader(fulfilledRoot);
-          if (msg.getDescriptor() != null) {
-            descriptor.set(new FlightDescriptor(msg.getDescriptor()));
+          synchronized (completed) {
+            if (!completed.isDone()) {
+              fulfilledRoot = VectorSchemaRoot.create(schema, allocator);
+              loader = new VectorLoader(fulfilledRoot);
+              if (msg.getDescriptor() != null) {
+                descriptor.set(new FlightDescriptor(msg.getDescriptor()));
+              }
+              root.set(fulfilledRoot);
+            }
           }
-          root.set(fulfilledRoot);
           break;
         }
         case RECORD_BATCH:
-          queue.add(msg);
-          break;
         case DICTIONARY_BATCH:
-          queue.add(msg);
+          enqueue(msg);
           break;
         case TENSOR:
         default:
-          queue.add(DONE_EX);
           ex = new UnsupportedOperationException("Unable to handle message of type: " + msg.getMessageType());
+          enqueue(DONE_EX);
       }
     }
 
@@ -397,12 +445,14 @@ public void onNext(ArrowMessage msg) {
     public void onError(Throwable t) {
       ex = StatusUtils.fromThrowable(t);
       queue.add(DONE_EX);
+      cancelled.complete(null);
       root.setException(ex);
     }
 
     @Override
     public void onCompleted() {
       // Depends on gRPC calling onNext and onCompleted non-concurrently
+      cancelled.complete(null);
       queue.add(DONE);
     }
   }
@@ -410,17 +460,16 @@ public void onCompleted() {
   /**
    * Cancels sending the stream to a client.
    *
-   * @throws UnsupportedOperationException on a stream being uploaded from the client.
+   * <p>Callers should drain the stream (with {@link #next()}) to ensure all messages sent before cancellation are
+   * received and to wait for the underlying transport to acknowledge cancellation.
    */
   public void cancel(String message, Throwable exception) {
-    completed.completeExceptionally(
-        CallStatus.CANCELLED.withDescription(message).withCause(exception).toRuntimeException());
-    if (cancellable != null) {
-      cancellable.cancel(message, exception);
-    } else {
+    if (cancellable == null) {
       throw new UnsupportedOperationException("Streams cannot be cancelled that are produced by client. " +
           "Instead, server should reject incoming messages.");
     }
+    cancellable.cancel(message, exception);
+    // Do not mark the stream as completed, as gRPC may still be delivering messages.
   }
 
   StreamObserver<ArrowMessage> asObserver() {
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/HeaderCallOption.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/HeaderCallOption.java
new file mode 100644
index 00000000000..e2fad1a402d
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/HeaderCallOption.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+import io.grpc.Metadata;
+import io.grpc.stub.AbstractStub;
+import io.grpc.stub.MetadataUtils;
+
+/**
+ * Method option for supplying headers to method calls.
+ */
+public class HeaderCallOption implements CallOptions.GrpcCallOption {
+  private final Metadata propertiesMetadata = new Metadata();
+
+  /**
+   * Header property constructor.
+   *
+   * @param headers the headers that should be sent across. If a header is a string, it should only be valid ASCII
+   *                characters. Binary headers should end in "-bin".
+   */
+  public HeaderCallOption(CallHeaders headers) {
+    for (String key : headers.keys()) {
+      if (key.endsWith(Metadata.BINARY_HEADER_SUFFIX)) {
+        final Metadata.Key<byte[]> metaKey = Metadata.Key.of(key, Metadata.BINARY_BYTE_MARSHALLER);
+        headers.getAllByte(key).forEach(v -> propertiesMetadata.put(metaKey, v));
+      } else {
+        final Metadata.Key<String> metaKey = Metadata.Key.of(key, Metadata.ASCII_STRING_MARSHALLER);
+        headers.getAll(key).forEach(v -> propertiesMetadata.put(metaKey, v));
+      }
+    }
+  }
+
+  @Override
+  public <T extends AbstractStub<T>> T wrapStub(T stub) {
+    return MetadataUtils.attachHeaders(stub, propertiesMetadata);
+  }
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/OutboundStreamListener.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/OutboundStreamListener.java
index f77155f5300..f578d67984f 100644
--- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/OutboundStreamListener.java
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/OutboundStreamListener.java
@@ -36,6 +36,19 @@ public interface OutboundStreamListener {
    */
   boolean isReady();
 
+  /**
+   * Set a callback for when the listener is ready for new calls to putNext(), i.e. {@link #isReady()}
+   * has become true.
+   *
+   * <p>Note that this callback may only be called some time after {@link #isReady()} becomes true, and may never
+   * be called if all executor threads on the server are busy, or the RPC method body is implemented in a blocking
+   * fashion. Note that isReady() must still be checked after the callback is run as it may have been run
+   * spuriously.
+   */
+  default void setOnReadyHandler(Runnable handler) {
+    throw new UnsupportedOperationException("Not yet implemented.");
+  }
+
   /**
    * Start sending data, using the schema of the given {@link VectorSchemaRoot}.
    *
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/OutboundStreamListenerImpl.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/OutboundStreamListenerImpl.java
index 40ac5fdb35e..345c11908fe 100644
--- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/OutboundStreamListenerImpl.java
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/OutboundStreamListenerImpl.java
@@ -48,6 +48,11 @@ public boolean isReady() {
     return responseObserver.isReady();
   }
 
+  @Override
+  public void setOnReadyHandler(Runnable handler) {
+    responseObserver.setOnReadyHandler(handler);
+  }
+
   @Override
   public void start(VectorSchemaRoot root, DictionaryProvider dictionaries, IpcOption option) {
     this.option = option;
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/RequestContext.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/RequestContext.java
new file mode 100644
index 00000000000..5117d05c2d2
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/RequestContext.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+import java.util.Set;
+
+/**
+ * Tracks variables about the current request.
+ */
+public interface RequestContext {
+  /**
+   * Register a variable and a value.
+   * @param key the variable name.
+   * @param value the value.
+   */
+  void put(String key, String value);
+
+  /**
+   * Retrieve a registered variable.
+   * @param key the variable name.
+   * @return the value, or null if not found.
+   */
+  String get(String key);
+
+  /**
+   * Retrieves the keys that have been registered to this context.
+   * @return the keys used in this context.
+   */
+  Set<String> keySet();
+
+  /**
+   * Deletes a registered variable.
+   * @return the value associated with the deleted variable, or null if the key doesn't exist.
+   */
+  String remove(String key);
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerHeaderMiddleware.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerHeaderMiddleware.java
new file mode 100644
index 00000000000..527c3128c65
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerHeaderMiddleware.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+/**
+ * Middleware that's used to extract and pass headers to the server during requests.
+ */
+public class ServerHeaderMiddleware implements FlightServerMiddleware {
+  /**
+   * Factory for accessing ServerHeaderMiddleware.
+   */
+  public static class Factory implements FlightServerMiddleware.Factory<ServerHeaderMiddleware> {
+    /**
+     * Construct a factory for receiving call headers.
+     */
+    public Factory() {
+    }
+
+    @Override
+    public ServerHeaderMiddleware onCallStarted(CallInfo callInfo, CallHeaders incomingHeaders,
+                                                RequestContext context) {
+      return new ServerHeaderMiddleware(incomingHeaders);
+    }
+  }
+
+  private final CallHeaders headers;
+
+  private ServerHeaderMiddleware(CallHeaders incomingHeaders) {
+    this.headers = incomingHeaders;
+  }
+
+  /**
+   * Retrieve the headers for this call.
+   */
+  public CallHeaders headers() {
+    return headers;
+  }
+
+  @Override
+  public void onBeforeSendingHeaders(CallHeaders outgoingHeaders) {
+  }
+
+  @Override
+  public void onCallCompleted(CallStatus status) {
+  }
+
+  @Override
+  public void onCallErrored(Throwable err) {
+  }
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/StreamPipe.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/StreamPipe.java
index 35bc228d654..d506914d588 100644
--- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/StreamPipe.java
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/StreamPipe.java
@@ -22,6 +22,7 @@
 
 import org.apache.arrow.flight.FlightProducer.StreamListener;
 import org.apache.arrow.flight.grpc.StatusUtils;
+import org.apache.arrow.util.AutoCloseables;
 
 import io.grpc.stub.StreamObserver;
 
@@ -33,9 +34,10 @@
  */
 class StreamPipe<FROM, TO> implements StreamListener<FROM> {
 
-  private StreamObserver<TO> delegate;
-  private Function<FROM, TO> mapFunction;
+  private final StreamObserver<TO> delegate;
+  private final Function<FROM, TO> mapFunction;
   private final Consumer<Throwable> errorHandler;
+  private AutoCloseable resource;
   private boolean closed = false;
 
   /**
@@ -58,6 +60,12 @@ public StreamPipe(StreamObserver<TO> delegate, Function<FROM, TO> func, Consumer
     this.delegate = delegate;
     this.mapFunction = func;
     this.errorHandler = errorHandler;
+    this.resource = null;
+  }
+
+  /** Set an AutoCloseable resource to be cleaned up when the gRPC observer is to be completed. */
+  void setAutoCloseable(AutoCloseable ac) {
+    resource = ac;
   }
 
   @Override
@@ -71,9 +79,15 @@ public void onError(Throwable t) {
       errorHandler.accept(t);
       return;
     }
-    // Set closed to true in case onError throws, so that we don't try to close again
-    closed = true;
-    delegate.onError(StatusUtils.toGrpcException(t));
+    try {
+      AutoCloseables.close(resource);
+    } catch (Exception e) {
+      errorHandler.accept(e);
+    } finally {
+      // Set closed to true in case onError throws, so that we don't try to close again
+      closed = true;
+      delegate.onError(StatusUtils.toGrpcException(t));
+    }
   }
 
   @Override
@@ -82,9 +96,15 @@ public void onCompleted() {
       errorHandler.accept(new IllegalStateException("Tried to complete already-completed call"));
       return;
     }
-    // Set closed to true in case onCompleted throws, so that we don't try to close again
-    closed = true;
-    delegate.onCompleted();
+    try {
+      AutoCloseables.close(resource);
+    } catch (Exception e) {
+      errorHandler.accept(e);
+    } finally {
+      // Set closed to true in case onCompleted throws, so that we don't try to close again
+      closed = true;
+      delegate.onCompleted();
+    }
   }
 
   /**
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth/BasicServerAuthHandler.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth/BasicServerAuthHandler.java
index 9ac80118184..34e3efc0d11 100644
--- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth/BasicServerAuthHandler.java
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth/BasicServerAuthHandler.java
@@ -71,5 +71,4 @@ public boolean authenticate(ServerAuthSender outgoing, Iterator<byte[]> incoming
   public Optional<String> isValid(byte[] token) {
     return authValidator.isValid(token);
   }
-
 }
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth/ServerAuthWrapper.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth/ServerAuthWrapper.java
index b110ef98d18..ad1a36a935f 100644
--- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth/ServerAuthWrapper.java
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth/ServerAuthWrapper.java
@@ -103,7 +103,7 @@ public byte[] next() {
           }
           return bytes;
         }
-          throw new IllegalStateException("Requesting more messages than client sent.");
+        throw new IllegalStateException("Requesting more messages than client sent.");
       }
 
       @Override
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/Auth2Constants.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/Auth2Constants.java
new file mode 100644
index 00000000000..624d7d5ff7c
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/Auth2Constants.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.auth2;
+
+/**
+ * Constants used in authorization of flight connections.
+ */
+public final class Auth2Constants {
+  public static final String PEER_IDENTITY_KEY = "arrow-flight-peer-identity";
+  public static final String BEARER_PREFIX = "Bearer ";
+  public static final String BASIC_PREFIX = "Basic ";
+  public static final String AUTHORIZATION_HEADER = "Authorization";
+
+  private Auth2Constants() {
+  }
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/AuthUtilities.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/AuthUtilities.java
new file mode 100644
index 00000000000..c73b7cf1a3a
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/AuthUtilities.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.auth2;
+
+import org.apache.arrow.flight.CallHeaders;
+
+/**
+ * Utility class for completing the auth process.
+ */
+public final class AuthUtilities {
+  private AuthUtilities() {
+
+  }
+
+  /**
+   * Helper method for retrieving a value from the Authorization header.
+   *
+   * @param headers     The headers to inspect.
+   * @param valuePrefix The prefix within the value portion of the header to extract away.
+   * @return The header value.
+   */
+  public static String getValueFromAuthHeader(CallHeaders headers, String valuePrefix) {
+    final String authHeaderValue = headers.get(Auth2Constants.AUTHORIZATION_HEADER);
+    if (authHeaderValue != null) {
+      if (authHeaderValue.regionMatches(true, 0, valuePrefix, 0, valuePrefix.length())) {
+        return authHeaderValue.substring(valuePrefix.length());
+      }
+    }
+    return null;
+  }
+
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/BasicAuthCredentialWriter.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/BasicAuthCredentialWriter.java
new file mode 100644
index 00000000000..698287e880b
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/BasicAuthCredentialWriter.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.auth2;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.function.Consumer;
+
+import org.apache.arrow.flight.CallHeaders;
+
+/**
+ * Client credentials that use a username and password.
+ */
+public final class BasicAuthCredentialWriter implements Consumer<CallHeaders> {
+
+  private final String name;
+  private final String password;
+
+  public BasicAuthCredentialWriter(String name, String password) {
+    this.name = name;
+    this.password = password;
+  }
+
+  @Override
+  public void accept(CallHeaders outputHeaders) {
+    outputHeaders.insert(Auth2Constants.AUTHORIZATION_HEADER, Auth2Constants.BASIC_PREFIX +
+        Base64.getEncoder().encodeToString(String.format("%s:%s", name, password).getBytes(StandardCharsets.UTF_8)));
+  }
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/BasicCallHeaderAuthenticator.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/BasicCallHeaderAuthenticator.java
new file mode 100644
index 00000000000..fff7b46909a
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/BasicCallHeaderAuthenticator.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.auth2;
+
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+import org.apache.arrow.flight.CallHeaders;
+import org.apache.arrow.flight.CallStatus;
+import org.apache.arrow.flight.FlightRuntimeException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A ServerAuthHandler for username/password authentication.
+ */
+public class BasicCallHeaderAuthenticator implements CallHeaderAuthenticator {
+
+  private static final Logger logger = LoggerFactory.getLogger(BasicCallHeaderAuthenticator.class);
+
+  private final CredentialValidator authValidator;
+
+  public BasicCallHeaderAuthenticator(CredentialValidator authValidator) {
+    this.authValidator = authValidator;
+  }
+
+  @Override
+  public AuthResult authenticate(CallHeaders incomingHeaders) {
+    try {
+      final String authEncoded = AuthUtilities.getValueFromAuthHeader(
+          incomingHeaders, Auth2Constants.BASIC_PREFIX);
+      if (authEncoded == null) {
+        throw CallStatus.UNAUTHENTICATED.toRuntimeException();
+      }
+      // The value has the format Base64(<username>:<password>)
+      final String authDecoded = new String(Base64.getDecoder().decode(authEncoded), StandardCharsets.UTF_8);
+      final int colonPos = authDecoded.indexOf(':');
+      if (colonPos == -1) {
+        throw CallStatus.UNAUTHENTICATED.toRuntimeException();
+      }
+
+      final String user = authDecoded.substring(0, colonPos);
+      final String password = authDecoded.substring(colonPos + 1);
+      return authValidator.validate(user, password);
+    } catch (UnsupportedEncodingException ex) {
+      // Note: Intentionally discarding the exception cause when reporting back to the client for security purposes.
+      logger.error("Authentication failed due to missing encoding.", ex);
+      throw CallStatus.INTERNAL.toRuntimeException();
+    } catch (FlightRuntimeException ex) {
+      throw ex;
+    } catch (Exception ex) {
+      // Note: Intentionally discarding the exception cause when reporting back to the client for security purposes.
+      logger.error("Authentication failed.", ex);
+      throw CallStatus.UNAUTHENTICATED.toRuntimeException();
+    }
+  }
+
+  /**
+   * Interface that this handler delegates to for validating the incoming headers.
+   */
+  public interface CredentialValidator {
+    /**
+     * Validate the supplied credentials (username/password) and return the peer identity.
+     *
+     * @param username The username to validate.
+     * @param password The password to validate.
+     * @return The peer identity if the supplied credentials are valid.
+     * @throws Exception If the supplied credentials are not valid.
+     */
+    AuthResult validate(String username, String password) throws Exception;
+  }
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/BearerCredentialWriter.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/BearerCredentialWriter.java
new file mode 100644
index 00000000000..715ee502b68
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/BearerCredentialWriter.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.auth2;
+
+import java.util.function.Consumer;
+
+import org.apache.arrow.flight.CallHeaders;
+
+/**
+ * Client credentials that use a bearer token.
+ */
+public final class BearerCredentialWriter implements Consumer<CallHeaders> {
+
+  private final String bearer;
+
+  public BearerCredentialWriter(String bearer) {
+    this.bearer = bearer;
+  }
+
+  @Override
+  public void accept(CallHeaders outputHeaders) {
+    outputHeaders.insert(Auth2Constants.AUTHORIZATION_HEADER, Auth2Constants.BEARER_PREFIX + bearer);
+  }
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/BearerTokenAuthenticator.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/BearerTokenAuthenticator.java
new file mode 100644
index 00000000000..2006e0a2b12
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/BearerTokenAuthenticator.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.auth2;
+
+import org.apache.arrow.flight.CallHeaders;
+
+/**
+ * Partial implementation of {@link CallHeaderAuthenticator} for bearer-token based authentication.
+ */
+public abstract class BearerTokenAuthenticator implements CallHeaderAuthenticator {
+
+  final CallHeaderAuthenticator initialAuthenticator;
+
+  public BearerTokenAuthenticator(CallHeaderAuthenticator initialAuthenticator) {
+    this.initialAuthenticator = initialAuthenticator;
+  }
+
+  @Override
+  public AuthResult authenticate(CallHeaders incomingHeaders) {
+    // Check if headers contain a bearer token and if so, validate the token.
+    final String bearerToken =
+        AuthUtilities.getValueFromAuthHeader(incomingHeaders, Auth2Constants.BEARER_PREFIX);
+    if (bearerToken != null) {
+      return validateBearer(bearerToken);
+    }
+
+    // Delegate to the basic auth handler to do the validation.
+    final CallHeaderAuthenticator.AuthResult result = initialAuthenticator.authenticate(incomingHeaders);
+    return getAuthResultWithBearerToken(result);
+  }
+
+  /**
+   * Callback to run when the initial authenticator succeeds.
+   * @param authResult A successful initial authentication result.
+   * @return an alternate AuthResult based on the original AuthResult that will write a bearer token to output headers.
+   */
+  protected abstract AuthResult getAuthResultWithBearerToken(AuthResult authResult);
+
+  /**
+   * Validate the bearer token.
+   * @param bearerToken The bearer token to validate.
+   * @return A successful AuthResult if validation succeeded.
+   * @throws Exception If the token validation fails.
+   */
+  protected abstract AuthResult validateBearer(String bearerToken);
+
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/CallHeaderAuthenticator.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/CallHeaderAuthenticator.java
new file mode 100644
index 00000000000..87e60f1fa81
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/CallHeaderAuthenticator.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.auth2;
+
+import org.apache.arrow.flight.CallHeaders;
+import org.apache.arrow.flight.FlightRuntimeException;
+
+/**
+ * Interface for Server side authentication handlers.
+ *
+ * A CallHeaderAuthenticator is used by {@link ServerCallHeaderAuthMiddleware} to validate headers sent by a Flight
+ * client for authentication purposes. The headers validated do not necessarily have to be Authorization headers.
+ *
+ * The workflow is that the FlightServer will intercept headers on a request, validate the headers, and
+ * either send back an UNAUTHENTICATED error, or succeed and potentially send back additional headers to the client.
+ *
+ * Implementations of CallHeaderAuthenticator should take care not to provide leak confidential details (such as
+ * indicating if usernames are valid or not) for security reasons when reporting errors back to clients.
+ *
+ * Example CallHeaderAuthenticators provided include:
+ * The {@link BasicCallHeaderAuthenticator} will authenticate basic HTTP credentials.
+ *
+ * The {@link BearerTokenAuthenticator} will authenticate basic HTTP credentials initially, then also send back a
+ * bearer token that the client can use for subsequent requests. The {@link GeneratedBearerTokenAuthenticator} will
+ * provide internally generated bearer tokens and maintain a cache of them.
+ */
+public interface CallHeaderAuthenticator {
+
+  /**
+   * Encapsulates the result of the {@link CallHeaderAuthenticator} analysis of headers.
+   *
+   * This includes the identity of the incoming user and any outbound headers to send as a response to the client.
+   */
+  interface AuthResult {
+    /**
+     * The peer identity that was determined by the handshake process based on the
+     * authentication credentials supplied by the client.
+     *
+     * @return The peer identity.
+     */
+    String getPeerIdentity();
+
+    /**
+     * Appends a header to the outgoing call headers.
+     * @param outgoingHeaders The outgoing headers.
+     */
+    default void appendToOutgoingHeaders(CallHeaders outgoingHeaders) {
+
+    }
+  }
+
+  /**
+   * Validate the auth headers sent by the client.
+   *
+   * @param incomingHeaders The incoming headers to authenticate.
+   * @return an auth result containing a peer identity and optionally a bearer token.
+   * @throws FlightRuntimeException with CallStatus.UNAUTHENTICATED if credentials were not supplied
+   *     or if credentials were supplied but were not valid.
+   */
+  AuthResult authenticate(CallHeaders incomingHeaders);
+
+  /**
+   * An auth handler that does nothing.
+   */
+  CallHeaderAuthenticator NO_OP = new CallHeaderAuthenticator() {
+    @Override
+    public AuthResult authenticate(CallHeaders incomingHeaders) {
+      return () -> "";
+    }
+  };
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/ClientBearerHeaderHandler.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/ClientBearerHeaderHandler.java
new file mode 100644
index 00000000000..45bdb6d959f
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/ClientBearerHeaderHandler.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.auth2;
+
+import org.apache.arrow.flight.CallHeaders;
+import org.apache.arrow.flight.grpc.CredentialCallOption;
+
+/**
+ * A client header handler that parses the incoming headers for a bearer token.
+ */
+public class ClientBearerHeaderHandler implements ClientHeaderHandler {
+
+  @Override
+  public CredentialCallOption getCredentialCallOptionFromIncomingHeaders(CallHeaders incomingHeaders) {
+    final String bearerValue = AuthUtilities.getValueFromAuthHeader(incomingHeaders, Auth2Constants.BEARER_PREFIX);
+    if (bearerValue != null) {
+      return new CredentialCallOption(new BearerCredentialWriter(bearerValue));
+    }
+    return null;
+  }
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/ClientHandshakeWrapper.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/ClientHandshakeWrapper.java
new file mode 100644
index 00000000000..16a5142509d
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/ClientHandshakeWrapper.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.auth2;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.arrow.flight.CallStatus;
+import org.apache.arrow.flight.FlightRuntimeException;
+import org.apache.arrow.flight.grpc.StatusUtils;
+import org.apache.arrow.flight.impl.Flight.HandshakeRequest;
+import org.apache.arrow.flight.impl.Flight.HandshakeResponse;
+import org.apache.arrow.flight.impl.FlightServiceGrpc.FlightServiceStub;
+
+import io.grpc.StatusRuntimeException;
+import io.grpc.stub.StreamObserver;
+
+/**
+ * Utility class for executing a handshake with a FlightServer.
+ */
+public class ClientHandshakeWrapper {
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ClientHandshakeWrapper.class);
+
+  /**
+   * Do handshake for a client.  The stub will be authenticated after this method returns.
+   *
+   * @param stub The service stub.
+   */
+  public static void doClientHandshake(FlightServiceStub stub) {
+    final HandshakeObserver observer = new HandshakeObserver();
+    try {
+      observer.requestObserver = stub.handshake(observer);
+      observer.requestObserver.onNext(HandshakeRequest.newBuilder().build());
+      observer.requestObserver.onCompleted();
+      try {
+        if (!observer.completed.get()) {
+          // TODO: ARROW-5681
+          throw CallStatus.UNAUTHENTICATED.toRuntimeException();
+        }
+      } catch (InterruptedException ex) {
+        Thread.currentThread().interrupt();
+        throw ex;
+      } catch (ExecutionException ex) {
+        final FlightRuntimeException wrappedException = StatusUtils.fromThrowable(ex.getCause());
+        logger.error("Failed on completing future", wrappedException);
+        throw wrappedException;
+      }
+    } catch (StatusRuntimeException sre) {
+      logger.error("Failed with SREe", sre);
+      throw StatusUtils.fromGrpcRuntimeException(sre);
+    } catch (Throwable ex) {
+      logger.error("Failed with unknown", ex);
+      if (ex instanceof FlightRuntimeException) {
+        throw (FlightRuntimeException) ex;
+      }
+      throw StatusUtils.fromThrowable(ex);
+    }
+  }
+
+  private static class HandshakeObserver implements StreamObserver<HandshakeResponse> {
+
+    private volatile StreamObserver<HandshakeRequest> requestObserver;
+    private final CompletableFuture<Boolean> completed;
+
+    public HandshakeObserver() {
+      super();
+      completed = new CompletableFuture<>();
+    }
+
+    @Override
+    public void onNext(HandshakeResponse value) {
+    }
+
+    @Override
+    public void onError(Throwable t) {
+      completed.completeExceptionally(t);
+    }
+
+    @Override
+    public void onCompleted() {
+      completed.complete(true);
+    }
+  }
+
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/ClientHeaderHandler.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/ClientHeaderHandler.java
new file mode 100644
index 00000000000..514189f9b13
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/ClientHeaderHandler.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.auth2;
+
+import org.apache.arrow.flight.CallHeaders;
+import org.apache.arrow.flight.grpc.CredentialCallOption;
+
+/**
+ * Interface for client side header parsing and conversion to CredentialCallOption.
+ */
+public interface ClientHeaderHandler {
+  /**
+   * Parses the incoming headers and converts them into a CredentialCallOption.
+   * @param incomingHeaders Incoming headers to parse.
+   * @return An instance of CredentialCallOption.
+   */
+  CredentialCallOption getCredentialCallOptionFromIncomingHeaders(CallHeaders incomingHeaders);
+  
+  /**
+   * An client header handler that does nothing.
+   */
+  ClientHeaderHandler NO_OP = new ClientHeaderHandler() {
+    @Override
+    public CredentialCallOption getCredentialCallOptionFromIncomingHeaders(CallHeaders incomingHeaders) {
+      return null;
+    }
+  };
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/ClientIncomingAuthHeaderMiddleware.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/ClientIncomingAuthHeaderMiddleware.java
new file mode 100644
index 00000000000..be5f3f54d32
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/ClientIncomingAuthHeaderMiddleware.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.auth2;
+
+import org.apache.arrow.flight.CallHeaders;
+import org.apache.arrow.flight.CallInfo;
+import org.apache.arrow.flight.CallStatus;
+import org.apache.arrow.flight.FlightClientMiddleware;
+import org.apache.arrow.flight.grpc.CredentialCallOption;
+
+/**
+ * Middleware for capturing bearer tokens sent back from the Flight server.
+ */
+public class ClientIncomingAuthHeaderMiddleware implements FlightClientMiddleware {
+  private final Factory factory;
+
+  /**
+   * Factory used within FlightClient.
+   */
+  public static class Factory implements FlightClientMiddleware.Factory {
+    private final ClientHeaderHandler headerHandler;
+    private CredentialCallOption credentialCallOption;
+
+    /**
+     * Construct a factory with the given header handler.
+     * @param headerHandler The header handler that will be used for handling incoming headers from the flight server.
+     */
+    public Factory(ClientHeaderHandler headerHandler) {
+      this.headerHandler = headerHandler;
+    }
+
+    @Override
+    public FlightClientMiddleware onCallStarted(CallInfo info) {
+      return new ClientIncomingAuthHeaderMiddleware(this);
+    }
+
+    void setCredentialCallOption(CredentialCallOption callOption) {
+      this.credentialCallOption = callOption;
+    }
+
+    public CredentialCallOption getCredentialCallOption() {
+      return credentialCallOption;
+    }
+  }
+
+  private ClientIncomingAuthHeaderMiddleware(Factory factory) {
+    this.factory = factory;
+  }
+
+  @Override
+  public void onBeforeSendingHeaders(CallHeaders outgoingHeaders) {
+  }
+
+  @Override
+  public void onHeadersReceived(CallHeaders incomingHeaders) {
+    factory.setCredentialCallOption(
+            factory.headerHandler.getCredentialCallOptionFromIncomingHeaders(incomingHeaders));
+  }
+
+  @Override
+  public void onCallCompleted(CallStatus status) {
+  }
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/GeneratedBearerTokenAuthenticator.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/GeneratedBearerTokenAuthenticator.java
new file mode 100644
index 00000000000..8b312b6b77a
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/GeneratedBearerTokenAuthenticator.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.auth2;
+
+import java.nio.ByteBuffer;
+import java.util.Base64;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.arrow.flight.CallHeaders;
+import org.apache.arrow.flight.CallStatus;
+import org.apache.arrow.flight.grpc.MetadataAdapter;
+
+import com.google.common.base.Strings;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+
+import io.grpc.Metadata;
+
+/**
+ * Generates and caches bearer tokens from user credentials.
+ */
+public class GeneratedBearerTokenAuthenticator extends BearerTokenAuthenticator {
+  private final Cache<String, String> bearerToIdentityCache;
+
+  /**
+   * Generate bearer tokens for the given basic call authenticator.
+   * @param authenticator The authenticator to initial validate inputs with.
+   */
+  public GeneratedBearerTokenAuthenticator(CallHeaderAuthenticator authenticator) {
+    this(authenticator, CacheBuilder.newBuilder().expireAfterAccess(2, TimeUnit.HOURS));
+  }
+
+  /**
+   * Generate bearer tokens for the given basic call authenticator.
+   * @param authenticator The authenticator to initial validate inputs with.
+   * @param timeoutMinutes The time before tokens expire after being accessed.
+   */
+  public GeneratedBearerTokenAuthenticator(CallHeaderAuthenticator authenticator, int timeoutMinutes) {
+    this(authenticator, CacheBuilder.newBuilder().expireAfterAccess(timeoutMinutes, TimeUnit.MINUTES));
+  }
+
+  /**
+   * Generate bearer tokens for the given basic call authenticator.
+   * @param authenticator The authenticator to initial validate inputs with.
+   * @param cacheBuilder The configuration of the cache of bearer tokens.
+   */
+  public GeneratedBearerTokenAuthenticator(CallHeaderAuthenticator authenticator,
+      CacheBuilder<Object, Object> cacheBuilder) {
+    super(authenticator);
+    bearerToIdentityCache = cacheBuilder.build();
+  }
+
+  @Override
+  protected AuthResult validateBearer(String bearerToken) {
+    final String peerIdentity = bearerToIdentityCache.getIfPresent(bearerToken);
+    if (peerIdentity == null) {
+      throw CallStatus.UNAUTHENTICATED.toRuntimeException();
+    }
+
+    return new AuthResult() {
+      @Override
+      public String getPeerIdentity() {
+        return peerIdentity;
+      }
+
+      @Override
+      public void appendToOutgoingHeaders(CallHeaders outgoingHeaders) {
+        if (null == AuthUtilities.getValueFromAuthHeader(outgoingHeaders, Auth2Constants.BEARER_PREFIX)) {
+          outgoingHeaders.insert(Auth2Constants.AUTHORIZATION_HEADER, Auth2Constants.BEARER_PREFIX + bearerToken);
+        }
+      }
+    };
+  }
+
+  @Override
+  protected AuthResult getAuthResultWithBearerToken(AuthResult authResult) {
+    // We generate a dummy header and call appendToOutgoingHeaders with it.
+    // We then inspect the dummy header and parse the bearer token if present in the header
+    // and generate a new bearer token if a bearer token is not present in the header.
+    final CallHeaders dummyHeaders = new MetadataAdapter(new Metadata());
+    authResult.appendToOutgoingHeaders(dummyHeaders);
+    String bearerToken =
+            AuthUtilities.getValueFromAuthHeader(dummyHeaders, Auth2Constants.BEARER_PREFIX);
+    final AuthResult authResultWithBearerToken;
+    if (Strings.isNullOrEmpty(bearerToken)) {
+      // Generate a new bearer token and return an AuthResult that can write it.
+      final UUID uuid = UUID.randomUUID();
+      final ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[16]);
+      byteBuffer.putLong(uuid.getMostSignificantBits());
+      byteBuffer.putLong(uuid.getLeastSignificantBits());
+      final String newToken = Base64.getEncoder().encodeToString(byteBuffer.array());
+      bearerToken = newToken;
+      authResultWithBearerToken = new AuthResult() {
+        @Override
+        public String getPeerIdentity() {
+          return authResult.getPeerIdentity();
+        }
+
+        @Override
+        public void appendToOutgoingHeaders(CallHeaders outgoingHeaders) {
+          authResult.appendToOutgoingHeaders(outgoingHeaders);
+          outgoingHeaders.insert(Auth2Constants.AUTHORIZATION_HEADER, Auth2Constants.BEARER_PREFIX + newToken);
+        }
+      };
+    } else {
+      // Use the bearer token supplied by the original auth result.
+      authResultWithBearerToken = authResult;
+    }
+    bearerToIdentityCache.put(bearerToken, authResult.getPeerIdentity());
+    return authResultWithBearerToken;
+  }
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/ServerCallHeaderAuthMiddleware.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/ServerCallHeaderAuthMiddleware.java
new file mode 100644
index 00000000000..9bfa7381844
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/ServerCallHeaderAuthMiddleware.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.auth2;
+
+import static org.apache.arrow.flight.auth2.CallHeaderAuthenticator.AuthResult;
+
+import org.apache.arrow.flight.CallHeaders;
+import org.apache.arrow.flight.CallInfo;
+import org.apache.arrow.flight.CallStatus;
+import org.apache.arrow.flight.FlightServerMiddleware;
+import org.apache.arrow.flight.RequestContext;
+
+/**
+ * Middleware that's used to validate credentials during the handshake and verify
+ * the bearer token in subsequent requests.
+ */
+public class ServerCallHeaderAuthMiddleware implements FlightServerMiddleware {
+  /**
+   * Factory for accessing ServerAuthMiddleware.
+   */
+  public static class Factory implements FlightServerMiddleware.Factory<ServerCallHeaderAuthMiddleware> {
+    private final CallHeaderAuthenticator authHandler;
+
+    /**
+     * Construct a factory with the given auth handler.
+     * @param authHandler The auth handler what will be used for authenticating requests.
+     */
+    public Factory(CallHeaderAuthenticator authHandler) {
+      this.authHandler = authHandler;
+    }
+
+    @Override
+    public ServerCallHeaderAuthMiddleware onCallStarted(CallInfo callInfo, CallHeaders incomingHeaders,
+                                                        RequestContext context) {
+      final AuthResult result = authHandler.authenticate(incomingHeaders);
+      context.put(Auth2Constants.PEER_IDENTITY_KEY, result.getPeerIdentity());
+      return new ServerCallHeaderAuthMiddleware(result);
+    }
+  }
+
+  private final AuthResult authResult;
+
+  public ServerCallHeaderAuthMiddleware(AuthResult authResult) {
+    this.authResult = authResult;
+  }
+
+  @Override
+  public void onBeforeSendingHeaders(CallHeaders outgoingHeaders) {
+    authResult.appendToOutgoingHeaders(outgoingHeaders);
+  }
+
+  @Override
+  public void onCallCompleted(CallStatus status) {
+  }
+
+  @Override
+  public void onCallErrored(Throwable err) {
+  }
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/client/ClientCookieMiddleware.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/client/ClientCookieMiddleware.java
new file mode 100644
index 00000000000..56f24e1019c
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/client/ClientCookieMiddleware.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.client;
+
+import java.net.HttpCookie;
+import java.util.List;
+import java.util.Locale;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.stream.Collectors;
+
+import org.apache.arrow.flight.CallHeaders;
+import org.apache.arrow.flight.CallInfo;
+import org.apache.arrow.flight.CallStatus;
+import org.apache.arrow.flight.FlightClientMiddleware;
+import org.apache.arrow.util.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A client middleware for receiving and sending cookie information.
+ * Note that this class will not persist permanent cookies beyond the lifetime
+ * of this session.
+ *
+ * This middleware will automatically remove cookies that have expired.
+ * <b>Note</b>: Negative max-age values currently do not get marked as expired due to
+ * a JDK issue. Use max-age=0 to explicitly remove an existing cookie.
+ */
+public class ClientCookieMiddleware implements FlightClientMiddleware {
+  private static final Logger LOGGER = LoggerFactory.getLogger(ClientCookieMiddleware.class);
+
+  private static final String SET_COOKIE_HEADER = "Set-Cookie";
+  private static final String COOKIE_HEADER = "Cookie";
+
+  private final Factory factory;
+
+  @VisibleForTesting
+  ClientCookieMiddleware(Factory factory) {
+    this.factory = factory;
+  }
+
+  /**
+   * Factory used within FlightClient.
+   */
+  public static class Factory implements FlightClientMiddleware.Factory {
+    // Use a map to track the most recent version of a cookie from the server.
+    // Note that cookie names are case-sensitive (but header names aren't).
+    private ConcurrentMap<String, HttpCookie> cookies = new ConcurrentHashMap<>();
+
+    @Override
+    public ClientCookieMiddleware onCallStarted(CallInfo info) {
+      return new ClientCookieMiddleware(this);
+    }
+
+    private void updateCookies(Iterable<String> newCookieHeaderValues) {
+      // Note: Intentionally overwrite existing cookie values.
+      // A cookie defined once will continue to be used in all subsequent
+      // requests on the client instance. The server can send the same cookie again
+      // with a different value and the client will use the new value in future requests.
+      // The server can also update a cookie to have an Expiry in the past or negative age
+      // to signal that the client should stop using the cookie immediately.
+      newCookieHeaderValues.forEach(headerValue -> {
+        try {
+          final List<HttpCookie> parsedCookies = HttpCookie.parse(headerValue);
+          parsedCookies.forEach(parsedCookie -> {
+            final String cookieNameLc = parsedCookie.getName().toLowerCase(Locale.ENGLISH);
+            if (parsedCookie.hasExpired()) {
+              cookies.remove(cookieNameLc);
+            } else {
+              cookies.put(parsedCookie.getName().toLowerCase(Locale.ENGLISH), parsedCookie);
+            }
+          });
+        } catch (IllegalArgumentException ex) {
+          LOGGER.warn("Skipping incorrectly formatted Set-Cookie header with value '{}'.", headerValue);
+        }
+      });
+    }
+  }
+
+  @Override
+  public void onBeforeSendingHeaders(CallHeaders outgoingHeaders) {
+    final String cookieValue = getValidCookiesAsString();
+    if (!cookieValue.isEmpty()) {
+      outgoingHeaders.insert(COOKIE_HEADER, cookieValue);
+    }
+  }
+
+  @Override
+  public void onHeadersReceived(CallHeaders incomingHeaders) {
+    final Iterable<String> setCookieHeaders = incomingHeaders.getAll(SET_COOKIE_HEADER);
+    if (setCookieHeaders != null) {
+      factory.updateCookies(setCookieHeaders);
+    }
+  }
+
+  @Override
+  public void onCallCompleted(CallStatus status) {
+
+  }
+
+  /**
+   * Discards expired cookies and returns the valid cookies as a String delimited by ';'.
+   */
+  @VisibleForTesting
+  String getValidCookiesAsString() {
+    // Discard expired cookies.
+    factory.cookies.entrySet().removeIf(cookieEntry -> cookieEntry.getValue().hasExpired());
+
+    // Cookie header value format:
+    // [<cookie-name1>=<cookie-value1>; <cookie-name2>=<cookie-value2; ...]
+    return factory.cookies.entrySet().stream()
+        .map(cookie -> cookie.getValue().toString())
+        .collect(Collectors.joining("; "));
+  }
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/example/integration/MiddlewareScenario.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/example/integration/MiddlewareScenario.java
index 70ad20d60b5..c710ce98b56 100644
--- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/example/integration/MiddlewareScenario.java
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/example/integration/MiddlewareScenario.java
@@ -34,6 +34,7 @@
 import org.apache.arrow.flight.FlightServerMiddleware;
 import org.apache.arrow.flight.Location;
 import org.apache.arrow.flight.NoOpFlightProducer;
+import org.apache.arrow.flight.RequestContext;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.types.pojo.Schema;
 
@@ -122,7 +123,8 @@ public void onCallErrored(Throwable err) {
     static class Factory implements FlightServerMiddleware.Factory<InjectingServerMiddleware> {
 
       @Override
-      public InjectingServerMiddleware onCallStarted(CallInfo info, CallHeaders incomingHeaders) {
+      public InjectingServerMiddleware onCallStarted(CallInfo info, CallHeaders incomingHeaders,
+          RequestContext context) {
         String incoming = incomingHeaders.get(HEADER);
         return new InjectingServerMiddleware(incoming == null ? "" : incoming);
       }
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/CallCredentialAdapter.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/CallCredentialAdapter.java
new file mode 100644
index 00000000000..285ddb9ba6e
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/CallCredentialAdapter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.grpc;
+
+import java.util.concurrent.Executor;
+import java.util.function.Consumer;
+
+import org.apache.arrow.flight.CallHeaders;
+
+import io.grpc.CallCredentials;
+import io.grpc.Metadata;
+
+/**
+ * Adapter class to utilize a CredentialWriter to implement Grpc CallCredentials.
+ */
+public class CallCredentialAdapter extends CallCredentials {
+
+  private final Consumer<CallHeaders> credentialWriter;
+
+  public CallCredentialAdapter(Consumer<CallHeaders> credentialWriter) {
+    this.credentialWriter = credentialWriter;
+  }
+
+  @Override
+  public void applyRequestMetadata(RequestInfo requestInfo, Executor executor, MetadataApplier metadataApplier) {
+    executor.execute(() ->
+    {
+      final Metadata headers = new Metadata();
+      credentialWriter.accept(new MetadataAdapter(headers));
+      metadataApplier.apply(headers);
+    });
+  }
+
+  @Override
+  public void thisUsesUnstableApi() {
+    // Mandatory to override this to acknowledge that CallCredentials is Experimental.
+  }
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/CredentialCallOption.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/CredentialCallOption.java
new file mode 100644
index 00000000000..3bde7a83511
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/CredentialCallOption.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.grpc;
+
+import java.util.function.Consumer;
+
+import org.apache.arrow.flight.CallHeaders;
+import org.apache.arrow.flight.CallOptions;
+
+import io.grpc.stub.AbstractStub;
+
+/**
+ * Method option for supplying credentials to method calls.
+ */
+public class CredentialCallOption implements CallOptions.GrpcCallOption {
+  private final Consumer<CallHeaders> credentialWriter;
+
+  public CredentialCallOption(Consumer<CallHeaders> credentialWriter) {
+    this.credentialWriter = credentialWriter;
+  }
+
+  @Override
+  public <T extends AbstractStub<T>> T wrapStub(T stub) {
+    return stub.withCallCredentials(new CallCredentialAdapter(credentialWriter));
+  }
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/MetadataAdapter.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/MetadataAdapter.java
index d1b0f60a807..4327f0ca85b 100644
--- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/MetadataAdapter.java
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/MetadataAdapter.java
@@ -32,11 +32,11 @@
  *
  * <p>This allows us to present the headers (metadata) from gRPC without copying to/from our own object.
  */
-class MetadataAdapter implements CallHeaders {
+public class MetadataAdapter implements CallHeaders {
 
   private final Metadata metadata;
 
-  MetadataAdapter(Metadata metadata) {
+  public MetadataAdapter(Metadata metadata) {
     this.metadata = metadata;
   }
 
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/RequestContextAdapter.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/RequestContextAdapter.java
new file mode 100644
index 00000000000..9be4d12b90e
--- /dev/null
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/RequestContextAdapter.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.grpc;
+
+import java.util.HashMap;
+import java.util.Set;
+
+import org.apache.arrow.flight.RequestContext;
+
+import io.grpc.Context;
+
+
+/**
+ * Adapter for holding key value pairs.
+ */
+public class RequestContextAdapter implements RequestContext {
+  public static final Context.Key<RequestContext> REQUEST_CONTEXT_KEY =
+          Context.key("arrow-flight-request-context");
+  private final HashMap<String, String> map = new HashMap<>();
+
+  @Override
+  public void put(String key, String value) {
+    if (map.putIfAbsent(key, value) != null) {
+      throw new IllegalArgumentException("Duplicate write to a RequestContext at key " + key + " not allowed.");
+    }
+  }
+
+  @Override
+  public String get(String key) {
+    return map.get(key);
+  }
+
+  @Override
+  public Set<String> keySet() {
+    return map.keySet();
+  }
+
+  @Override
+  public String remove(String key) {
+    return map.remove(key);
+  }
+}
diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/ServerInterceptorAdapter.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/ServerInterceptorAdapter.java
index 9e7dc9d5832..ddf43ff846b 100644
--- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/ServerInterceptorAdapter.java
+++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/ServerInterceptorAdapter.java
@@ -87,10 +87,11 @@ public <ReqT, RespT> Listener<ReqT> interceptCall(ServerCall<ReqT, RespT> call,
     // Use LinkedHashMap to preserve insertion order
     final Map<FlightServerMiddleware.Key<?>, FlightServerMiddleware> middlewareMap = new LinkedHashMap<>();
     final MetadataAdapter headerAdapter = new MetadataAdapter(headers);
+    final RequestContextAdapter requestContextAdapter = new RequestContextAdapter();
     for (final KeyFactory<?> factory : factories) {
       final FlightServerMiddleware m;
       try {
-        m = factory.factory.onCallStarted(info, headerAdapter);
+        m = factory.factory.onCallStarted(info, headerAdapter, requestContextAdapter);
       } catch (FlightRuntimeException e) {
         // Cancel call
         call.close(StatusUtils.toGrpcStatus(e.status()), new Metadata());
@@ -101,8 +102,9 @@ public <ReqT, RespT> Listener<ReqT> interceptCall(ServerCall<ReqT, RespT> call,
     }
 
     // Inject the middleware into the context so RPC method implementations can communicate with middleware instances
-    final Context contextWithMiddleware = Context.current().withValue(SERVER_MIDDLEWARE_KEY,
-        Collections.unmodifiableMap(middlewareMap));
+    final Context contextWithMiddlewareAndRequestsOptions = Context.current()
+        .withValue(SERVER_MIDDLEWARE_KEY, Collections.unmodifiableMap(middlewareMap))
+        .withValue(RequestContextAdapter.REQUEST_CONTEXT_KEY, requestContextAdapter);
 
     final SimpleForwardingServerCall<ReqT, RespT> forwardingServerCall = new SimpleForwardingServerCall<ReqT, RespT>(
         call) {
@@ -137,6 +139,7 @@ public void close(Status status, Metadata trailers) {
         middleware.forEach(m -> m.onCallCompleted(flightStatus));
       }
     };
-    return Contexts.interceptCall(contextWithMiddleware, forwardingServerCall, headers, next);
+    return Contexts.interceptCall(contextWithMiddlewareAndRequestsOptions, forwardingServerCall, headers, next);
+
   }
 }
diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestBackPressure.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestBackPressure.java
index d0e26e13d7f..1a71c363e17 100644
--- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestBackPressure.java
+++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestBackPressure.java
@@ -17,7 +17,10 @@
 
 package org.apache.arrow.flight;
 
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
 
 import org.apache.arrow.flight.perf.PerformanceTestServer;
 import org.apache.arrow.flight.perf.TestPerf;
@@ -43,10 +46,47 @@ public class TestBackPressure {
   @Ignore
   @Test
   public void ensureIndependentSteams() throws Exception {
+    ensureIndependentSteams((b) -> (location -> new PerformanceTestServer(b, location)));
+  }
+
+  /**
+   * Make sure that failing to consume one stream doesn't block other streams.
+   */
+  @Ignore
+  @Test
+  public void ensureIndependentSteamsWithCallbacks() throws Exception {
+    ensureIndependentSteams((b) -> (location -> new PerformanceTestServer(b, location,
+        new BackpressureStrategy.CallbackBackpressureStrategy(), true)));
+  }
+
+  /**
+   * Test to make sure stream doesn't go faster than the consumer is consuming.
+   */
+  @Ignore
+  @Test
+  public void ensureWaitUntilProceed() throws Exception {
+    ensureWaitUntilProceed(new PollingBackpressureStrategy(), false);
+  }
+
+  /**
+   * Test to make sure stream doesn't go faster than the consumer is consuming using a callback-based
+   * backpressure strategy.
+   */
+  @Ignore
+  @Test
+  public void ensureWaitUntilProceedWithCallbacks() throws Exception {
+    ensureWaitUntilProceed(new RecordingCallbackBackpressureStrategy(), true);
+  }
+
+  /**
+   * Make sure that failing to consume one stream doesn't block other streams.
+   */
+  private static void ensureIndependentSteams(Function<BufferAllocator, Function<Location, PerformanceTestServer>>
+                                                  serverConstructor) throws Exception {
     try (
         final BufferAllocator a = new RootAllocator(Long.MAX_VALUE);
         final PerformanceTestServer server = FlightTestUtil.getStartedServer(
-            (location) -> (new PerformanceTestServer(a, location)));
+            (location) -> (serverConstructor.apply(a).apply(location)));
         final FlightClient client = FlightClient.builder(a, server.getLocation()).build()
     ) {
       try (FlightStream fs1 = client.getStream(client.getInfo(
@@ -73,45 +113,46 @@ public void ensureIndependentSteams() throws Exception {
   /**
    * Make sure that a stream doesn't go faster than the consumer is consuming.
    */
-  @Ignore
-  @Test
-  public void ensureWaitUntilProceed() throws Exception {
+  private static void ensureWaitUntilProceed(SleepTimeRecordingBackpressureStrategy bpStrategy, boolean isNonBlocking)
+      throws Exception {
     // request some values.
     final long wait = 3000;
     final long epsilon = 1000;
 
-    AtomicLong sleepTime = new AtomicLong(0);
     try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
 
       final FlightProducer producer = new NoOpFlightProducer() {
 
         @Override
-        public void getStream(CallContext context, Ticket ticket,
-            ServerStreamListener listener) {
-          int batches = 0;
-          final Schema pojoSchema = new Schema(ImmutableList.of(Field.nullable("a", MinorType.BIGINT.getType())));
-          try (VectorSchemaRoot root = VectorSchemaRoot.create(pojoSchema, allocator)) {
-            listener.start(root);
-            while (true) {
-              while (!listener.isReady()) {
-                try {
-                  Thread.sleep(1);
-                  sleepTime.addAndGet(1L);
-                } catch (InterruptedException ignore) {
+        public void getStream(CallContext context, Ticket ticket, ServerStreamListener listener) {
+          bpStrategy.register(listener);
+          final Runnable loadData = () -> {
+            int batches = 0;
+            final Schema pojoSchema = new Schema(ImmutableList.of(Field.nullable("a", MinorType.BIGINT.getType())));
+            try (VectorSchemaRoot root = VectorSchemaRoot.create(pojoSchema, allocator)) {
+              listener.start(root);
+              while (true) {
+                bpStrategy.waitForListener(0);
+                if (batches > 100) {
+                  root.clear();
+                  listener.completed();
+                  return;
                 }
-              }
 
-              if (batches > 100) {
-                root.clear();
-                listener.completed();
-                return;
+                root.allocateNew();
+                root.setRowCount(4095);
+                listener.putNext();
+                batches++;
               }
-
-              root.allocateNew();
-              root.setRowCount(4095);
-              listener.putNext();
-              batches++;
             }
+          };
+
+          if (!isNonBlocking) {
+            loadData.run();
+          } else {
+            final ExecutorService service = Executors.newSingleThreadExecutor();
+            service.submit(loadData);
+            service.shutdown();
           }
         }
       };
@@ -137,8 +178,8 @@ public void getStream(CallContext context, Ticket ticket,
         }
         long expected = wait - epsilon;
         Assert.assertTrue(
-            String.format("Expected a sleep of at least %dms but only slept for %d", expected, sleepTime.get()),
-            sleepTime.get() > expected);
+            String.format("Expected a sleep of at least %dms but only slept for %d", expected,
+                bpStrategy.getSleepTime()), bpStrategy.getSleepTime() > expected);
 
       }
     }
@@ -158,4 +199,64 @@ private static void consume(FlightStream stream, int batches) {
       batches--;
     }
   }
+
+  private interface SleepTimeRecordingBackpressureStrategy extends BackpressureStrategy {
+    /**
+     * Returns the total time spent waiting on the listener to be ready.
+     * @return the total time spent waiting on the listener to be ready.
+     */
+    long getSleepTime();
+  }
+
+  /**
+   * Implementation of a backpressure strategy that polls on isReady and records amount of time spent in Thread.sleep().
+   */
+  private static class PollingBackpressureStrategy implements SleepTimeRecordingBackpressureStrategy {
+    private final AtomicLong sleepTime = new AtomicLong(0);
+    private FlightProducer.ServerStreamListener listener;
+
+    @Override
+    public long getSleepTime() {
+      return sleepTime.get();
+    }
+
+    @Override
+    public void register(FlightProducer.ServerStreamListener listener) {
+      this.listener = listener;
+    }
+
+    @Override
+    public WaitResult waitForListener(long timeout) {
+      while (!listener.isReady()) {
+        try {
+          Thread.sleep(1);
+          sleepTime.addAndGet(1L);
+        } catch (InterruptedException ignore) {
+        }
+      }
+      return WaitResult.READY;
+    }
+  }
+
+  /**
+   * Implementation of a backpressure strategy that uses callbacks to detect changes in client readiness state
+   * and records spent time waiting.
+   */
+  private static class RecordingCallbackBackpressureStrategy extends BackpressureStrategy.CallbackBackpressureStrategy
+      implements SleepTimeRecordingBackpressureStrategy {
+    private final AtomicLong sleepTime = new AtomicLong(0);
+
+    @Override
+    public long getSleepTime() {
+      return sleepTime.get();
+    }
+
+    @Override
+    public WaitResult waitForListener(long timeout) {
+      final long startTime = System.currentTimeMillis();
+      final WaitResult result = super.waitForListener(timeout);
+      sleepTime.addAndGet(System.currentTimeMillis() - startTime);
+      return result;
+    }
+  }
 }
diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestBasicOperation.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestBasicOperation.java
index 8242bc0a390..bae658230b5 100644
--- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestBasicOperation.java
+++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestBasicOperation.java
@@ -20,10 +20,12 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
@@ -33,6 +35,8 @@
 import org.apache.arrow.flight.impl.Flight.FlightDescriptor.DescriptorType;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.IntVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.types.pojo.ArrowType;
@@ -244,6 +248,50 @@ public void getStream() throws Exception {
     });
   }
 
+  /** Ensure the client is configured to accept large messages. */
+  @Test
+  public void getStreamLargeBatch() throws Exception {
+    test(c -> {
+      try (final FlightStream stream = c.getStream(new Ticket(Producer.TICKET_LARGE_BATCH))) {
+        Assert.assertEquals(128, stream.getRoot().getFieldVectors().size());
+        Assert.assertTrue(stream.next());
+        Assert.assertEquals(65536, stream.getRoot().getRowCount());
+        Assert.assertTrue(stream.next());
+        Assert.assertEquals(65536, stream.getRoot().getRowCount());
+        Assert.assertFalse(stream.next());
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    });
+  }
+
+  /** Ensure the server is configured to accept large messages. */
+  @Test
+  public void startPutLargeBatch() throws Exception {
+    try (final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE)) {
+      final List<FieldVector> vectors = new ArrayList<>();
+      for (int col = 0; col < 128; col++) {
+        final BigIntVector vector = new BigIntVector("f" + col, allocator);
+        for (int row = 0; row < 65536; row++) {
+          vector.setSafe(row, row);
+        }
+        vectors.add(vector);
+      }
+      test(c -> {
+        try (final VectorSchemaRoot root = new VectorSchemaRoot(vectors)) {
+          root.setRowCount(65536);
+          final ClientStreamListener stream = c.startPut(FlightDescriptor.path(""), root, new SyncPutListener());
+          stream.putNext();
+          stream.putNext();
+          stream.completed();
+          stream.getResult();
+        } catch (Exception e) {
+          throw new RuntimeException(e);
+        }
+      });
+    }
+  }
+
   private void test(Consumer<FlightClient> consumer) throws Exception {
     test((c, a) -> {
       consumer.accept(c);
@@ -273,6 +321,7 @@ private void test(BiConsumer<FlightClient, BufferAllocator> consumer) throws Exc
    * An example FlightProducer for test purposes.
    */
   public static class Producer implements FlightProducer, AutoCloseable {
+    static final byte[] TICKET_LARGE_BATCH = "large-batch".getBytes(StandardCharsets.UTF_8);
 
     private final BufferAllocator allocator;
 
@@ -313,8 +362,11 @@ public Runnable acceptPut(CallContext context, FlightStream flightStream, Stream
     }
 
     @Override
-    public void getStream(CallContext context, Ticket ticket,
-        ServerStreamListener listener) {
+    public void getStream(CallContext context, Ticket ticket, ServerStreamListener listener) {
+      if (Arrays.equals(TICKET_LARGE_BATCH, ticket.getBytes())) {
+        getLargeBatch(listener);
+        return;
+      }
       final int size = 10;
 
       IntVector iv = new IntVector("c1", allocator);
@@ -343,6 +395,24 @@ public void getStream(CallContext context, Ticket ticket,
       listener.completed();
     }
 
+    private void getLargeBatch(ServerStreamListener listener) {
+      final List<FieldVector> vectors = new ArrayList<>();
+      for (int col = 0; col < 128; col++) {
+        final BigIntVector vector = new BigIntVector("f" + col, allocator);
+        for (int row = 0; row < 65536; row++) {
+          vector.setSafe(row, row);
+        }
+        vectors.add(vector);
+      }
+      try (final VectorSchemaRoot root = new VectorSchemaRoot(vectors)) {
+        root.setRowCount(65536);
+        listener.start(root);
+        listener.putNext();
+        listener.putNext();
+        listener.completed();
+      }
+    }
+
     @Override
     public void close() throws Exception {
       allocator.close();
diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestCallOptions.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestCallOptions.java
index 3acb9473006..45e3e496092 100644
--- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestCallOptions.java
+++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestCallOptions.java
@@ -30,6 +30,8 @@
 import org.junit.Ignore;
 import org.junit.Test;
 
+import io.grpc.Metadata;
+
 public class TestCallOptions {
 
   @Test
@@ -64,10 +66,63 @@ public void underTimeout() {
     });
   }
 
+  @Test
+  public void singleProperty() {
+    final FlightCallHeaders headers = new FlightCallHeaders();
+    headers.insert("key", "value");
+    testHeaders(headers);
+  }
+
+  @Test
+  public void multipleProperties() {
+    final FlightCallHeaders headers = new FlightCallHeaders();
+    headers.insert("key", "value");
+    headers.insert("key2", "value2");
+    testHeaders(headers);
+  }
+
+  @Test
+  public void binaryProperties() {
+    final FlightCallHeaders headers = new FlightCallHeaders();
+    headers.insert("key-bin", "value".getBytes());
+    headers.insert("key3-bin", "ëfßæ".getBytes());
+    testHeaders(headers);
+  }
+
+  @Test
+  public void mixedProperties() {
+    final FlightCallHeaders headers = new FlightCallHeaders();
+    headers.insert("key", "value");
+    headers.insert("key3-bin", "ëfßæ".getBytes());
+    testHeaders(headers);
+  }
+
+  private void testHeaders(CallHeaders headers) {
+    try (
+        BufferAllocator a = new RootAllocator(Long.MAX_VALUE);
+        HeaderProducer producer = new HeaderProducer();
+        FlightServer s =
+            FlightTestUtil.getStartedServer((location) -> FlightServer.builder(a, location, producer).build());
+        FlightClient client = FlightClient.builder(a, s.getLocation()).build()) {
+      client.doAction(new Action(""), new HeaderCallOption(headers)).hasNext();
+
+      final CallHeaders incomingHeaders = producer.headers();
+      for (String key : headers.keys()) {
+        if (key.endsWith(Metadata.BINARY_HEADER_SUFFIX)) {
+          Assert.assertArrayEquals(headers.getByte(key), incomingHeaders.getByte(key));
+        } else {
+          Assert.assertEquals(headers.get(key), incomingHeaders.get(key));
+        }
+      }
+    } catch (InterruptedException | IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
   void test(Consumer<FlightClient> testFn) {
     try (
         BufferAllocator a = new RootAllocator(Long.MAX_VALUE);
-        Producer producer = new Producer(a);
+        Producer producer = new Producer();
         FlightServer s =
             FlightTestUtil.getStartedServer((location) -> FlightServer.builder(a, location, producer).build());
         FlightClient client = FlightClient.builder(a, s.getLocation()).build()) {
@@ -77,12 +132,27 @@ void test(Consumer<FlightClient> testFn) {
     }
   }
 
-  static class Producer extends NoOpFlightProducer implements AutoCloseable {
+  static class HeaderProducer extends NoOpFlightProducer implements AutoCloseable {
+    CallHeaders headers;
 
-    private final BufferAllocator allocator;
+    @Override
+    public void close() {
+    }
+
+    public CallHeaders headers() {
+      return headers;
+    }
+
+    @Override
+    public void doAction(CallContext context, Action action, StreamListener<Result> listener) {
+      this.headers = context.getMiddleware(FlightConstants.HEADER_KEY).headers();
+      listener.onCompleted();
+    }
+  }
+
+  static class Producer extends NoOpFlightProducer implements AutoCloseable {
 
-    Producer(BufferAllocator allocator) {
-      this.allocator = allocator;
+    Producer() {
     }
 
     @Override
diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestClientMiddleware.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestClientMiddleware.java
index 742bd9bcc21..ccfc9f2d177 100644
--- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestClientMiddleware.java
+++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestClientMiddleware.java
@@ -157,7 +157,7 @@ public void onCallErrored(Throwable err) {
     static class Factory implements FlightServerMiddleware.Factory<ServerSpanInjector> {
 
       @Override
-      public ServerSpanInjector onCallStarted(CallInfo info, CallHeaders incomingHeaders) {
+      public ServerSpanInjector onCallStarted(CallInfo info, CallHeaders incomingHeaders, RequestContext context) {
         return new ServerSpanInjector(incomingHeaders.get("x-span"));
       }
     }
@@ -256,7 +256,8 @@ public FlightClientMiddleware onCallStarted(CallInfo info) {
   static class MultiHeaderServerMiddlewareFactory implements
       FlightServerMiddleware.Factory<MultiHeaderServerMiddleware> {
     @Override
-    public MultiHeaderServerMiddleware onCallStarted(CallInfo info, CallHeaders incomingHeaders) {
+    public MultiHeaderServerMiddleware onCallStarted(CallInfo info, CallHeaders incomingHeaders,
+        RequestContext context) {
       // Echo the headers back to the client. Copy values out of CallHeaders since the underlying gRPC metadata
       // object isn't safe to use after this function returns.
       Map<String, List<byte[]>> binaryHeaders = new HashMap<>();
diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestDoExchange.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestDoExchange.java
index 7aa95f747ef..b7e7a20bece 100644
--- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestDoExchange.java
+++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestDoExchange.java
@@ -22,6 +22,7 @@
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
@@ -52,6 +53,7 @@ public class TestDoExchange {
   static byte[] EXCHANGE_ECHO = "echo".getBytes(StandardCharsets.UTF_8);
   static byte[] EXCHANGE_METADATA_ONLY = "only-metadata".getBytes(StandardCharsets.UTF_8);
   static byte[] EXCHANGE_TRANSFORM = "transform".getBytes(StandardCharsets.UTF_8);
+  static byte[] EXCHANGE_CANCEL = "cancel".getBytes(StandardCharsets.UTF_8);
 
   private BufferAllocator allocator;
   private FlightServer server;
@@ -117,7 +119,7 @@ public void testDoExchangeDoGet() throws Exception {
           value++;
         }
       }
-      assertEquals(10, value);
+      assertEquals(100, value);
     }
   }
 
@@ -247,7 +249,83 @@ public void testTransform() throws Exception {
     }
   }
 
+  /** Have the server immediately cancel; ensure the client doesn't hang. */
+  @Test
+  public void testServerCancel() throws Exception {
+    try (final FlightClient.ExchangeReaderWriter stream =
+             client.doExchange(FlightDescriptor.command(EXCHANGE_CANCEL))) {
+      final FlightStream reader = stream.getReader();
+      final FlightClient.ClientStreamListener writer = stream.getWriter();
+
+      final FlightRuntimeException fre = assertThrows(FlightRuntimeException.class, reader::next);
+      assertEquals(FlightStatusCode.CANCELLED, fre.status().code());
+      assertEquals("expected", fre.status().description());
+
+      // Before, this would hang forever, because the writer checks if the stream is ready and not cancelled.
+      // However, the cancellation flag (was) only updated by reading, and the stream is never ready once the call ends.
+      // The test looks weird since normally, an application shouldn't try to write after the read fails. However,
+      // an application that isn't reading data wouldn't notice, and would instead get stuck on the write.
+      // Here, we read first to avoid a race condition in the test itself.
+      writer.putMetadata(allocator.getEmpty());
+    }
+  }
+
+  /** Have the server immediately cancel; ensure the server cleans up the FlightStream. */
+  @Test
+  public void testServerCancelLeak() throws Exception {
+    try (final FlightClient.ExchangeReaderWriter stream =
+             client.doExchange(FlightDescriptor.command(EXCHANGE_CANCEL))) {
+      final FlightStream reader = stream.getReader();
+      final FlightClient.ClientStreamListener writer = stream.getWriter();
+      try (final VectorSchemaRoot root = VectorSchemaRoot.create(Producer.SCHEMA, allocator)) {
+        writer.start(root);
+        final IntVector ints = (IntVector) root.getVector("a");
+        for (int i = 0; i < 128; i++) {
+          for (int row = 0; row < 1024; row++) {
+            ints.setSafe(row, row);
+          }
+          root.setRowCount(1024);
+          writer.putNext();
+        }
+      }
+
+      final FlightRuntimeException fre = assertThrows(FlightRuntimeException.class, reader::next);
+      assertEquals(FlightStatusCode.CANCELLED, fre.status().code());
+      assertEquals("expected", fre.status().description());
+    }
+  }
+
+  /** Have the client cancel without reading; ensure memory is not leaked. */
+  @Test
+  public void testClientCancel() throws Exception {
+    try (final FlightClient.ExchangeReaderWriter stream =
+             client.doExchange(FlightDescriptor.command(EXCHANGE_DO_GET))) {
+      final FlightStream reader = stream.getReader();
+      reader.cancel("", null);
+      // Cancel should be idempotent
+      reader.cancel("", null);
+    }
+  }
+
+  /** Have the client close the stream without reading; ensure memory is not leaked. */
+  @Test
+  public void testClientClose() throws Exception {
+    try (final FlightClient.ExchangeReaderWriter stream =
+             client.doExchange(FlightDescriptor.command(EXCHANGE_DO_GET))) {
+      assertEquals(Producer.SCHEMA, stream.getReader().getSchema());
+    }
+    // Intentionally leak the allocator in this test. gRPC has a bug where it does not wait for all calls to complete
+    // when shutting down the server, so this test will fail otherwise because it closes the allocator while the
+    // server-side call still has memory allocated.
+    // TODO(ARROW-9586): fix this once we track outstanding RPCs outside of gRPC.
+    // https://stackoverflow.com/questions/46716024/
+    allocator = null;
+    client = null;
+  }
+
   static class Producer extends NoOpFlightProducer {
+    static final Schema SCHEMA = new Schema(
+        Collections.singletonList(Field.nullable("a", new ArrowType.Int(32, true))));
     private final BufferAllocator allocator;
 
     Producer(BufferAllocator allocator) {
@@ -266,6 +344,8 @@ public void doExchange(CallContext context, FlightStream reader, ServerStreamLis
         echo(context, reader, writer);
       } else if (Arrays.equals(reader.getDescriptor().getCommand(), EXCHANGE_TRANSFORM)) {
         transform(context, reader, writer);
+      } else if (Arrays.equals(reader.getDescriptor().getCommand(), EXCHANGE_CANCEL)) {
+        cancel(context, reader, writer);
       } else {
         writer.error(CallStatus.UNIMPLEMENTED.withDescription("Command not implemented").toRuntimeException());
       }
@@ -273,13 +353,12 @@ public void doExchange(CallContext context, FlightStream reader, ServerStreamLis
 
     /** Emulate DoGet. */
     private void doGet(CallContext context, FlightStream reader, ServerStreamListener writer) {
-      final Schema schema = new Schema(Collections.singletonList(Field.nullable("a", new ArrowType.Int(32, true))));
-      try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
+      try (VectorSchemaRoot root = VectorSchemaRoot.create(SCHEMA, allocator)) {
         writer.start(root);
         root.allocateNew();
         IntVector iv = (IntVector) root.getVector("a");
 
-        for (int i = 0; i < 10; i += 2) {
+        for (int i = 0; i < 100; i += 2) {
           iv.set(0, i);
           iv.set(1, i + 1);
           root.setRowCount(2);
@@ -391,5 +470,10 @@ private void transform(CallContext context, FlightStream reader, ServerStreamLis
       writer.putMetadata(count);
       writer.completed();
     }
+
+    /** Immediately cancel the call. */
+    private void cancel(CallContext context, FlightStream reader, ServerStreamListener writer) {
+      writer.error(CallStatus.CANCELLED.withDescription("expected").toRuntimeException());
+    }
   }
 }
diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestFlightService.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestFlightService.java
new file mode 100644
index 00000000000..65ef12a8acf
--- /dev/null
+++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestFlightService.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+import static org.junit.jupiter.api.Assertions.fail;
+
+import org.apache.arrow.flight.impl.Flight;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.util.AutoCloseables;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import io.grpc.stub.ServerCallStreamObserver;
+
+public class TestFlightService {
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void setup() {
+    allocator = new RootAllocator(Long.MAX_VALUE);
+  }
+
+  @After
+  public void cleanup() throws Exception {
+    AutoCloseables.close(allocator);
+  }
+
+  @Test
+  public void testFlightServiceWithNoAuthHandlerOrInterceptors() {
+    // This test is for ARROW-10491. There was a bug where FlightService would try to access the RequestContext,
+    // but the RequestContext was getting set to null because no interceptors were active to initialize it
+    // when using FlightService directly rather than starting up a FlightServer.
+
+    // Arrange
+    final FlightProducer producer = new NoOpFlightProducer() {
+      @Override
+      public void getStream(CallContext context, Ticket ticket,
+                            ServerStreamListener listener) {
+        listener.completed();
+      }
+    };
+
+    // This response observer notifies that the test failed if onError() is called.
+    final ServerCallStreamObserver<ArrowMessage> observer = new ServerCallStreamObserver<ArrowMessage>() {
+      @Override
+      public boolean isCancelled() {
+        return false;
+      }
+
+      @Override
+      public void setOnCancelHandler(Runnable runnable) {
+
+      }
+
+      @Override
+      public void setCompression(String s) {
+
+      }
+
+      @Override
+      public boolean isReady() {
+        return false;
+      }
+
+      @Override
+      public void setOnReadyHandler(Runnable runnable) {
+
+      }
+
+      @Override
+      public void disableAutoInboundFlowControl() {
+
+      }
+
+      @Override
+      public void request(int i) {
+
+      }
+
+      @Override
+      public void setMessageCompression(boolean b) {
+
+      }
+
+      @Override
+      public void onNext(ArrowMessage arrowMessage) {
+
+      }
+
+      @Override
+      public void onError(Throwable throwable) {
+        fail(throwable);
+      }
+
+      @Override
+      public void onCompleted() {
+
+      }
+    };
+    final FlightService flightService = new FlightService(allocator, producer, null, null);
+
+    // Act
+    flightService.doGetCustom(Flight.Ticket.newBuilder().build(), observer);
+
+    // fail() would have been called if an error happened during doGetCustom(), so this test passed.
+  }
+}
diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestLeak.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestLeak.java
index dc729c49656..6e28704997f 100644
--- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestLeak.java
+++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestLeak.java
@@ -173,7 +173,7 @@ public Runnable acceptPut(CallContext context,
         FlightStream flightStream, StreamListener<PutResult> ackStream) {
       return () -> {
         flightStream.getRoot();
-        flightStream.cancel("CANCELLED", null);
+        ackStream.onError(CallStatus.CANCELLED.withDescription("CANCELLED").toRuntimeException());
         callFinished.countDown();
         ackStream.onCompleted();
       };
diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestServerMiddleware.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestServerMiddleware.java
index 345820a09bc..1f3e35ca38d 100644
--- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestServerMiddleware.java
+++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestServerMiddleware.java
@@ -217,7 +217,7 @@ static class Factory implements FlightServerMiddleware.Factory<ErrorRecorder> {
       ErrorRecorder instance = new ErrorRecorder();
 
       @Override
-      public ErrorRecorder onCallStarted(CallInfo info, CallHeaders incomingHeaders) {
+      public ErrorRecorder onCallStarted(CallInfo info, CallHeaders incomingHeaders, RequestContext context) {
         return instance;
       }
     }
diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestTls.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestTls.java
index 0ad1eaeb963..c5cd871e2be 100644
--- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestTls.java
+++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestTls.java
@@ -85,6 +85,22 @@ public void rejectHostname() {
     });
   }
 
+  /**
+   * Test a basic request over TLS.
+   */
+  @Test
+  public void connectTlsDisableServerVerification() {
+    test((builder) -> {
+      try (final FlightClient client = builder.verifyServer(false).build()) {
+        final Iterator<Result> responses = client.doAction(new Action("hello-world"));
+        final byte[] response = responses.next().getBody();
+        Assert.assertEquals("Hello, world!", new String(response, StandardCharsets.UTF_8));
+        Assert.assertFalse(responses.hasNext());
+      } catch (InterruptedException e) {
+        throw new RuntimeException(e);
+      }
+    });
+  }
 
   void test(Consumer<Builder> testFn) {
     final FlightTestUtil.CertKeyPair certKey = FlightTestUtil.exampleTlsCerts().get(0);
diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/auth/TestBasicAuth.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/auth/TestBasicAuth.java
index d05910da9fd..c18f5709b54 100644
--- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/auth/TestBasicAuth.java
+++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/auth/TestBasicAuth.java
@@ -137,7 +137,7 @@ public void getStream(CallContext context, Ticket ticket, ServerStreamListener l
               return;
             }
             final Schema pojoSchema = new Schema(ImmutableList.of(Field.nullable("a",
-                    Types.MinorType.BIGINT.getType())));
+                Types.MinorType.BIGINT.getType())));
             try (VectorSchemaRoot root = VectorSchemaRoot.create(pojoSchema, allocator)) {
               listener.start(root);
               root.allocateNew();
diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/auth2/TestBasicAuth2.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/auth2/TestBasicAuth2.java
new file mode 100644
index 00000000000..9bec32f1b72
--- /dev/null
+++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/auth2/TestBasicAuth2.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.auth2;
+
+import java.io.IOException;
+
+import org.apache.arrow.flight.CallStatus;
+import org.apache.arrow.flight.Criteria;
+import org.apache.arrow.flight.FlightClient;
+import org.apache.arrow.flight.FlightInfo;
+import org.apache.arrow.flight.FlightProducer;
+import org.apache.arrow.flight.FlightServer;
+import org.apache.arrow.flight.FlightStatusCode;
+import org.apache.arrow.flight.FlightStream;
+import org.apache.arrow.flight.FlightTestUtil;
+import org.apache.arrow.flight.NoOpFlightProducer;
+import org.apache.arrow.flight.Ticket;
+import org.apache.arrow.flight.grpc.CredentialCallOption;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.util.AutoCloseables;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+
+public class TestBasicAuth2 {
+
+  private static final String USERNAME_1 = "flight1";
+  private static final String USERNAME_2 = "flight2";
+  private static final String NO_USERNAME = "";
+  private static final String PASSWORD_1 = "woohoo1";
+  private static final String PASSWORD_2 = "woohoo2";
+  private BufferAllocator allocator;
+  private FlightServer server;
+  private FlightClient client;
+  private FlightClient client2;
+
+  @Before
+  public void setup() throws Exception {
+    allocator = new RootAllocator(Long.MAX_VALUE);
+    startServerAndClient();
+  }
+
+  private FlightProducer getFlightProducer() {
+    return new NoOpFlightProducer() {
+      @Override
+      public void listFlights(CallContext context, Criteria criteria,
+          StreamListener<FlightInfo> listener) {
+        if (!context.peerIdentity().equals(USERNAME_1) && !context.peerIdentity().equals(USERNAME_2)) {
+          listener.onError(new IllegalArgumentException("Invalid username"));
+          return;
+        }
+        listener.onCompleted();
+      }
+
+      @Override
+      public void getStream(CallContext context, Ticket ticket, ServerStreamListener listener) {
+        if (!context.peerIdentity().equals(USERNAME_1) && !context.peerIdentity().equals(USERNAME_2)) {
+          listener.error(new IllegalArgumentException("Invalid username"));
+          return;
+        }
+        final Schema pojoSchema = new Schema(ImmutableList.of(Field.nullable("a",
+                Types.MinorType.BIGINT.getType())));
+        try (VectorSchemaRoot root = VectorSchemaRoot.create(pojoSchema, allocator)) {
+          listener.start(root);
+          root.allocateNew();
+          root.setRowCount(4095);
+          listener.putNext();
+          listener.completed();
+        }
+      }
+    };
+  }
+
+  private void startServerAndClient() throws IOException {
+    final FlightProducer flightProducer = getFlightProducer();
+    this.server = FlightTestUtil.getStartedServer((location) -> FlightServer
+        .builder(allocator, location, flightProducer)
+        .headerAuthenticator(new GeneratedBearerTokenAuthenticator(
+            new BasicCallHeaderAuthenticator(this::validate)))
+        .build());
+
+    this.client = FlightClient.builder(allocator, server.getLocation())
+        .build();
+  }
+
+  @After
+  public void shutdown() throws Exception {
+    AutoCloseables.close(client, client2, server, allocator);
+    client = null;
+    client2 = null;
+    server = null;
+    allocator = null;
+  }
+
+  private void startClient2() throws IOException {
+    client2 = FlightClient.builder(allocator, server.getLocation())
+        .build();
+  }
+
+  private CallHeaderAuthenticator.AuthResult validate(String username, String password) {
+    if (Strings.isNullOrEmpty(username)) {
+      throw CallStatus.UNAUTHENTICATED.withDescription("Credentials not supplied.").toRuntimeException();
+    }
+    final String identity;
+    if (USERNAME_1.equals(username) && PASSWORD_1.equals(password)) {
+      identity = USERNAME_1;
+    } else if (USERNAME_2.equals(username) && PASSWORD_2.equals(password)) {
+      identity = USERNAME_2;
+    } else {
+      throw CallStatus.UNAUTHENTICATED.withDescription("Username or password is invalid.").toRuntimeException();
+    }
+    return () -> identity;
+  }
+
+  @Test
+  public void validAuthWithBearerAuthServer() throws IOException {
+    testValidAuth(client);
+  }
+
+  @Test
+  public void validAuthWithMultipleClientsWithSameCredentialsWithBearerAuthServer() throws IOException {
+    startClient2();
+    testValidAuthWithMultipleClientsWithSameCredentials(client, client2);
+  }
+
+  @Test
+  public void validAuthWithMultipleClientsWithDifferentCredentialsWithBearerAuthServer() throws IOException {
+    startClient2();
+    testValidAuthWithMultipleClientsWithDifferentCredentials(client, client2);
+  }
+
+  // ARROW-7722: this test occasionally leaks memory
+  @Ignore
+  @Test
+  public void asyncCall() throws Exception {
+    final CredentialCallOption bearerToken = client
+        .authenticateBasicToken(USERNAME_1, PASSWORD_1).get();
+    client.listFlights(Criteria.ALL, bearerToken);
+    try (final FlightStream s = client.getStream(new Ticket(new byte[1]))) {
+      while (s.next()) {
+        Assert.assertEquals(4095, s.getRoot().getRowCount());
+      }
+    }
+  }
+
+  @Test
+  public void invalidAuthWithBearerAuthServer() throws IOException {
+    testInvalidAuth(client);
+  }
+
+  @Test
+  public void didntAuthWithBearerAuthServer() throws IOException {
+    didntAuth(client);
+  }
+
+  private void testValidAuth(FlightClient client) {
+    final CredentialCallOption bearerToken = client
+            .authenticateBasicToken(USERNAME_1, PASSWORD_1).get();
+    Assert.assertTrue(ImmutableList.copyOf(client
+            .listFlights(Criteria.ALL, bearerToken))
+            .isEmpty());
+  }
+
+  private void testValidAuthWithMultipleClientsWithSameCredentials(
+          FlightClient client1, FlightClient client2) {
+    final CredentialCallOption bearerToken1 = client1
+            .authenticateBasicToken(USERNAME_1, PASSWORD_1).get();
+    final CredentialCallOption bearerToken2 = client2
+            .authenticateBasicToken(USERNAME_1, PASSWORD_1).get();
+    Assert.assertTrue(ImmutableList.copyOf(client1
+            .listFlights(Criteria.ALL, bearerToken1))
+            .isEmpty());
+    Assert.assertTrue(ImmutableList.copyOf(client2
+            .listFlights(Criteria.ALL, bearerToken2))
+            .isEmpty());
+  }
+
+  private void testValidAuthWithMultipleClientsWithDifferentCredentials(
+          FlightClient client1, FlightClient client2) {
+    final CredentialCallOption bearerToken1 = client1
+            .authenticateBasicToken(USERNAME_1, PASSWORD_1).get();
+    final CredentialCallOption bearerToken2 = client2
+            .authenticateBasicToken(USERNAME_2, PASSWORD_2).get();
+    Assert.assertTrue(ImmutableList.copyOf(client1
+            .listFlights(Criteria.ALL, bearerToken1))
+            .isEmpty());
+    Assert.assertTrue(ImmutableList.copyOf(client2
+            .listFlights(Criteria.ALL, bearerToken2))
+            .isEmpty());
+  }
+
+  private void testInvalidAuth(FlightClient client) {
+    FlightTestUtil.assertCode(FlightStatusCode.UNAUTHENTICATED, () ->
+            client.authenticateBasicToken(USERNAME_1, "WRONG"));
+
+    FlightTestUtil.assertCode(FlightStatusCode.UNAUTHENTICATED, () ->
+            client.authenticateBasicToken(NO_USERNAME, PASSWORD_1));
+
+    FlightTestUtil.assertCode(FlightStatusCode.UNAUTHENTICATED, () ->
+            client.listFlights(Criteria.ALL).forEach(action -> Assert.fail()));
+  }
+
+  private void didntAuth(FlightClient client) {
+    FlightTestUtil.assertCode(FlightStatusCode.UNAUTHENTICATED, () ->
+            client.listFlights(Criteria.ALL).forEach(action -> Assert.fail()));
+  }
+}
diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/client/TestCookieHandling.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/client/TestCookieHandling.java
new file mode 100644
index 00000000000..f205f9a3b63
--- /dev/null
+++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/client/TestCookieHandling.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.client;
+
+import java.io.IOException;
+
+import org.apache.arrow.flight.CallHeaders;
+import org.apache.arrow.flight.CallInfo;
+import org.apache.arrow.flight.CallStatus;
+import org.apache.arrow.flight.Criteria;
+import org.apache.arrow.flight.ErrorFlightMetadata;
+import org.apache.arrow.flight.FlightClient;
+import org.apache.arrow.flight.FlightInfo;
+import org.apache.arrow.flight.FlightMethod;
+import org.apache.arrow.flight.FlightProducer;
+import org.apache.arrow.flight.FlightServer;
+import org.apache.arrow.flight.FlightServerMiddleware;
+import org.apache.arrow.flight.FlightTestUtil;
+import org.apache.arrow.flight.NoOpFlightProducer;
+import org.apache.arrow.flight.RequestContext;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.util.AutoCloseables;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * Tests for correct handling of cookies from the FlightClient using {@link ClientCookieMiddleware}.
+ */
+public class TestCookieHandling {
+  private static final String SET_COOKIE_HEADER = "Set-Cookie";
+  private static final String COOKIE_HEADER = "Cookie";
+  private BufferAllocator allocator;
+  private FlightServer server;
+  private FlightClient client;
+
+  private ClientCookieMiddlewareTestFactory testFactory = new ClientCookieMiddlewareTestFactory();
+  private ClientCookieMiddleware cookieMiddleware = new ClientCookieMiddleware(testFactory);
+
+  @Before
+  public void setup() throws Exception {
+    allocator = new RootAllocator(Long.MAX_VALUE);
+    startServerAndClient();
+  }
+
+  @After
+  public void cleanup() throws Exception {
+    testFactory = new ClientCookieMiddlewareTestFactory();
+    cookieMiddleware = testFactory.onCallStarted(new CallInfo(FlightMethod.DO_ACTION));
+    AutoCloseables.close(client, server, allocator);
+    client = null;
+    server = null;
+    allocator = null;
+  }
+
+  @Test
+  public void basicCookie() {
+    CallHeaders headersToSend = new ErrorFlightMetadata();
+    headersToSend.insert(SET_COOKIE_HEADER, "k=v");
+    cookieMiddleware = testFactory.onCallStarted(new CallInfo(FlightMethod.DO_ACTION));
+    cookieMiddleware.onHeadersReceived(headersToSend);
+    Assert.assertEquals("k=v", cookieMiddleware.getValidCookiesAsString());
+  }
+
+  @Test
+  public void cookieStaysAfterMultipleRequests() {
+    CallHeaders headersToSend = new ErrorFlightMetadata();
+    headersToSend.insert(SET_COOKIE_HEADER, "k=v");
+    cookieMiddleware = testFactory.onCallStarted(new CallInfo(FlightMethod.DO_ACTION));
+    cookieMiddleware.onHeadersReceived(headersToSend);
+    Assert.assertEquals("k=v", cookieMiddleware.getValidCookiesAsString());
+
+    headersToSend = new ErrorFlightMetadata();
+    cookieMiddleware = testFactory.onCallStarted(new CallInfo(FlightMethod.DO_ACTION));
+    cookieMiddleware.onHeadersReceived(headersToSend);
+    Assert.assertEquals("k=v", cookieMiddleware.getValidCookiesAsString());
+
+    headersToSend = new ErrorFlightMetadata();
+    cookieMiddleware = testFactory.onCallStarted(new CallInfo(FlightMethod.DO_ACTION));
+    cookieMiddleware.onHeadersReceived(headersToSend);
+    Assert.assertEquals("k=v", cookieMiddleware.getValidCookiesAsString());
+  }
+
+  @Ignore
+  @Test
+  public void cookieAutoExpires() {
+    CallHeaders headersToSend = new ErrorFlightMetadata();
+    headersToSend.insert(SET_COOKIE_HEADER, "k=v; Max-Age=2");
+    cookieMiddleware = testFactory.onCallStarted(new CallInfo(FlightMethod.DO_ACTION));
+    cookieMiddleware.onHeadersReceived(headersToSend);
+    // Note: using max-age changes cookie version from 0->1, which quotes values.
+    Assert.assertEquals("k=\"v\"", cookieMiddleware.getValidCookiesAsString());
+
+    headersToSend = new ErrorFlightMetadata();
+    cookieMiddleware = testFactory.onCallStarted(new CallInfo(FlightMethod.DO_ACTION));
+    cookieMiddleware.onHeadersReceived(headersToSend);
+    Assert.assertEquals("k=\"v\"", cookieMiddleware.getValidCookiesAsString());
+
+    try {
+      Thread.sleep(5000);
+    } catch (InterruptedException ignored) {
+    }
+
+    // Verify that the k cookie was discarded because it expired.
+    Assert.assertTrue(cookieMiddleware.getValidCookiesAsString().isEmpty());
+  }
+
+  @Test
+  public void cookieExplicitlyExpires() {
+    CallHeaders headersToSend = new ErrorFlightMetadata();
+    headersToSend.insert(SET_COOKIE_HEADER, "k=v; Max-Age=2");
+    cookieMiddleware = testFactory.onCallStarted(new CallInfo(FlightMethod.DO_ACTION));
+    cookieMiddleware.onHeadersReceived(headersToSend);
+    // Note: using max-age changes cookie version from 0->1, which quotes values.
+    Assert.assertEquals("k=\"v\"", cookieMiddleware.getValidCookiesAsString());
+
+    // Note: The JDK treats Max-Age < 0 as not expired and treats 0 as expired.
+    // This violates the RFC, which states that less than zero and zero should both be expired.
+    headersToSend = new ErrorFlightMetadata();
+    headersToSend.insert(SET_COOKIE_HEADER, "k=v; Max-Age=0");
+    cookieMiddleware = testFactory.onCallStarted(new CallInfo(FlightMethod.DO_ACTION));
+    cookieMiddleware.onHeadersReceived(headersToSend);
+
+    // Verify that the k cookie was discarded because the server told the client it is expired.
+    Assert.assertTrue(cookieMiddleware.getValidCookiesAsString().isEmpty());
+  }
+
+  @Ignore
+  @Test
+  public void cookieExplicitlyExpiresWithMaxAgeMinusOne() {
+    CallHeaders headersToSend = new ErrorFlightMetadata();
+    headersToSend.insert(SET_COOKIE_HEADER, "k=v; Max-Age=2");
+    cookieMiddleware = testFactory.onCallStarted(new CallInfo(FlightMethod.DO_ACTION));
+    cookieMiddleware.onHeadersReceived(headersToSend);
+    // Note: using max-age changes cookie version from 0->1, which quotes values.
+    Assert.assertEquals("k=\"v\"", cookieMiddleware.getValidCookiesAsString());
+
+    headersToSend = new ErrorFlightMetadata();
+
+    // The Java HttpCookie class has a bug where it uses a -1 maxAge to indicate
+    // a persistent cookie, when the RFC spec says this should mean the cookie expires immediately.
+    headersToSend.insert(SET_COOKIE_HEADER, "k=v; Max-Age=-1");
+    cookieMiddleware = testFactory.onCallStarted(new CallInfo(FlightMethod.DO_ACTION));
+    cookieMiddleware.onHeadersReceived(headersToSend);
+
+    // Verify that the k cookie was discarded because the server told the client it is expired.
+    Assert.assertTrue(cookieMiddleware.getValidCookiesAsString().isEmpty());
+  }
+
+  @Test
+  public void changeCookieValue() {
+    CallHeaders headersToSend = new ErrorFlightMetadata();
+    headersToSend.insert(SET_COOKIE_HEADER, "k=v");
+    cookieMiddleware.onHeadersReceived(headersToSend);
+    Assert.assertEquals("k=v", cookieMiddleware.getValidCookiesAsString());
+
+    headersToSend = new ErrorFlightMetadata();
+    headersToSend.insert(SET_COOKIE_HEADER, "k=v2");
+    cookieMiddleware.onHeadersReceived(headersToSend);
+    Assert.assertEquals("k=v2", cookieMiddleware.getValidCookiesAsString());
+  }
+
+  @Test
+  public void multipleCookiesWithSetCookie() {
+    CallHeaders headersToSend = new ErrorFlightMetadata();
+    headersToSend.insert(SET_COOKIE_HEADER, "firstKey=firstVal");
+    headersToSend.insert(SET_COOKIE_HEADER, "secondKey=secondVal");
+    cookieMiddleware.onHeadersReceived(headersToSend);
+    Assert.assertEquals("firstKey=firstVal; secondKey=secondVal", cookieMiddleware.getValidCookiesAsString());
+  }
+
+  @Test
+  public void cookieStaysAfterMultipleRequestsEndToEnd() {
+    client.handshake();
+    Assert.assertEquals("k=v", testFactory.clientCookieMiddleware.getValidCookiesAsString());
+    client.handshake();
+    Assert.assertEquals("k=v", testFactory.clientCookieMiddleware.getValidCookiesAsString());
+    client.listFlights(Criteria.ALL);
+    Assert.assertEquals("k=v", testFactory.clientCookieMiddleware.getValidCookiesAsString());
+  }
+
+  /**
+   * A server middleware component that injects SET_COOKIE_HEADER into the outgoing headers.
+   */
+  static class SetCookieHeaderInjector implements FlightServerMiddleware {
+    private final Factory factory;
+
+    public SetCookieHeaderInjector(Factory factory) {
+      this.factory = factory;
+    }
+
+    @Override
+    public void onBeforeSendingHeaders(CallHeaders outgoingHeaders) {
+      if (!factory.receivedCookieHeader) {
+        outgoingHeaders.insert(SET_COOKIE_HEADER, "k=v");
+      }
+    }
+
+    @Override
+    public void onCallCompleted(CallStatus status) {
+
+    }
+
+    @Override
+    public void onCallErrored(Throwable err) {
+
+    }
+
+    static class Factory implements FlightServerMiddleware.Factory<SetCookieHeaderInjector> {
+      private boolean receivedCookieHeader = false;
+
+      @Override
+      public SetCookieHeaderInjector onCallStarted(CallInfo info, CallHeaders incomingHeaders,
+                                                   RequestContext context) {
+        receivedCookieHeader = null != incomingHeaders.get(COOKIE_HEADER);
+        return new SetCookieHeaderInjector(this);
+      }
+    }
+  }
+
+  public static class ClientCookieMiddlewareTestFactory extends ClientCookieMiddleware.Factory {
+
+    private ClientCookieMiddleware clientCookieMiddleware;
+
+    @Override
+    public ClientCookieMiddleware onCallStarted(CallInfo info) {
+      this.clientCookieMiddleware = new ClientCookieMiddleware(this);
+      return this.clientCookieMiddleware;
+    }
+  }
+
+  private void startServerAndClient() throws IOException {
+    final FlightProducer flightProducer = new NoOpFlightProducer() {
+      public void listFlights(CallContext context, Criteria criteria,
+                              StreamListener<FlightInfo> listener) {
+        listener.onCompleted();
+      }
+    };
+
+    this.server = FlightTestUtil.getStartedServer((location) -> FlightServer
+            .builder(allocator, location, flightProducer)
+            .middleware(FlightServerMiddleware.Key.of("test"), new SetCookieHeaderInjector.Factory())
+            .build());
+
+    this.client = FlightClient.builder(allocator, server.getLocation())
+            .intercept(testFactory)
+            .build();
+  }
+}
diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/perf/PerformanceTestServer.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/perf/PerformanceTestServer.java
index 7e13eaca3f8..4200bbec5be 100644
--- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/perf/PerformanceTestServer.java
+++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/perf/PerformanceTestServer.java
@@ -21,10 +21,14 @@
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
+import org.apache.arrow.flight.BackpressureStrategy;
 import org.apache.arrow.flight.FlightDescriptor;
 import org.apache.arrow.flight.FlightEndpoint;
 import org.apache.arrow.flight.FlightInfo;
+import org.apache.arrow.flight.FlightProducer;
 import org.apache.arrow.flight.FlightServer;
 import org.apache.arrow.flight.Location;
 import org.apache.arrow.flight.NoOpFlightProducer;
@@ -51,12 +55,34 @@ public class PerformanceTestServer implements AutoCloseable {
   private final Location location;
   private final BufferAllocator allocator;
   private final PerfProducer producer;
+  private final boolean isNonBlocking;
 
   public PerformanceTestServer(BufferAllocator incomingAllocator, Location location) {
+    this(incomingAllocator, location, new BackpressureStrategy() {
+      private FlightProducer.ServerStreamListener listener;
+
+      @Override
+      public void register(FlightProducer.ServerStreamListener listener) {
+        this.listener = listener;
+      }
+
+      @Override
+      public WaitResult waitForListener(long timeout) {
+        while (!listener.isReady()) {
+          // busy wait
+        }
+        return WaitResult.READY;
+      }
+    }, false);
+  }
+
+  public PerformanceTestServer(BufferAllocator incomingAllocator, Location location, BackpressureStrategy bpStrategy,
+                               boolean isNonBlocking) {
     this.allocator = incomingAllocator.newChildAllocator("perf-server", 0, Long.MAX_VALUE);
     this.location = location;
-    this.producer = new PerfProducer();
+    this.producer = new PerfProducer(bpStrategy);
     this.flightServer = FlightServer.builder(this.allocator, location, producer).build();
+    this.isNonBlocking = isNonBlocking;
   }
 
   public Location getLocation() {
@@ -73,68 +99,81 @@ public void close() throws Exception {
   }
 
   private final class PerfProducer extends NoOpFlightProducer {
+    private final BackpressureStrategy bpStrategy;
+
+    private PerfProducer(BackpressureStrategy bpStrategy) {
+      this.bpStrategy = bpStrategy;
+    }
 
     @Override
     public void getStream(CallContext context, Ticket ticket,
         ServerStreamListener listener) {
-      VectorSchemaRoot root = null;
-      try {
-        Token token = Token.parseFrom(ticket.getBytes());
-        Perf perf = token.getDefinition();
-        Schema schema = Schema.deserialize(ByteBuffer.wrap(perf.getSchema().toByteArray()));
-        root = VectorSchemaRoot.create(schema, allocator);
-        BigIntVector a = (BigIntVector) root.getVector("a");
-        BigIntVector b = (BigIntVector) root.getVector("b");
-        BigIntVector c = (BigIntVector) root.getVector("c");
-        BigIntVector d = (BigIntVector) root.getVector("d");
-        listener.start(root);
-        root.allocateNew();
-
-        int current = 0;
-        long i = token.getStart();
-        while (i < token.getEnd()) {
-          if (listener.isCancelled()) {
-            root.clear();
-            return;
-          }
-
-          if (TestPerf.VALIDATE) {
-            a.setSafe(current, i);
-          }
-
-          i++;
-          current++;
-          if (i % perf.getRecordsPerBatch() == 0) {
-            root.setRowCount(current);
-
-            while (!listener.isReady()) {
-              //Thread.sleep(0, nanos);
-            }
-
+      bpStrategy.register(listener);
+      final Runnable loadData = () -> {
+        VectorSchemaRoot root = null;
+        try {
+          Token token = Token.parseFrom(ticket.getBytes());
+          Perf perf = token.getDefinition();
+          Schema schema = Schema.deserialize(ByteBuffer.wrap(perf.getSchema().toByteArray()));
+          root = VectorSchemaRoot.create(schema, allocator);
+          BigIntVector a = (BigIntVector) root.getVector("a");
+          BigIntVector b = (BigIntVector) root.getVector("b");
+          BigIntVector c = (BigIntVector) root.getVector("c");
+          BigIntVector d = (BigIntVector) root.getVector("d");
+          listener.start(root);
+          root.allocateNew();
+
+          int current = 0;
+          long i = token.getStart();
+          while (i < token.getEnd()) {
             if (listener.isCancelled()) {
               root.clear();
               return;
             }
-            listener.putNext();
-            current = 0;
-            root.allocateNew();
+
+            if (TestPerf.VALIDATE) {
+              a.setSafe(current, i);
+            }
+
+            i++;
+            current++;
+            if (i % perf.getRecordsPerBatch() == 0) {
+              root.setRowCount(current);
+
+              bpStrategy.waitForListener(0);
+              if (listener.isCancelled()) {
+                root.clear();
+                return;
+              }
+              listener.putNext();
+              current = 0;
+              root.allocateNew();
+            }
           }
-        }
 
-        // send last partial batch.
-        if (current != 0) {
-          root.setRowCount(current);
-          listener.putNext();
-        }
-        listener.completed();
-      } catch (InvalidProtocolBufferException e) {
-        throw new RuntimeException(e);
-      } finally {
-        try {
-          AutoCloseables.close(root);
-        } catch (Exception e) {
+          // send last partial batch.
+          if (current != 0) {
+            root.setRowCount(current);
+            listener.putNext();
+          }
+          listener.completed();
+        } catch (InvalidProtocolBufferException e) {
           throw new RuntimeException(e);
+        } finally {
+          try {
+            AutoCloseables.close(root);
+          } catch (Exception e) {
+            throw new RuntimeException(e);
+          }
         }
+      };
+
+      if (!isNonBlocking) {
+        loadData.run();
+      } else {
+        final ExecutorService service = Executors.newSingleThreadExecutor();
+        service.submit(loadData);
+        service.shutdown();
       }
     }
 
diff --git a/java/flight/flight-grpc/pom.xml b/java/flight/flight-grpc/pom.xml
index 35eab05186b..d296c4927ac 100644
--- a/java/flight/flight-grpc/pom.xml
+++ b/java/flight/flight-grpc/pom.xml
@@ -13,7 +13,7 @@
   <parent>
     <artifactId>arrow-java-root</artifactId>
     <groupId>org.apache.arrow</groupId>
-    <version>2.0.0-SNAPSHOT</version>
+    <version>3.0.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>
diff --git a/java/flight/flight-grpc/src/main/java/org/apache/arrow/flight/FlightGrpcUtils.java b/java/flight/flight-grpc/src/main/java/org/apache/arrow/flight/FlightGrpcUtils.java
index 2abb71c0a1b..eb5e492b4cd 100644
--- a/java/flight/flight-grpc/src/main/java/org/apache/arrow/flight/FlightGrpcUtils.java
+++ b/java/flight/flight-grpc/src/main/java/org/apache/arrow/flight/FlightGrpcUtils.java
@@ -19,17 +19,111 @@
 
 import java.util.Collections;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.arrow.flight.auth.ServerAuthHandler;
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.util.VisibleForTesting;
 
 import io.grpc.BindableService;
+import io.grpc.CallOptions;
+import io.grpc.ClientCall;
+import io.grpc.ConnectivityState;
 import io.grpc.ManagedChannel;
+import io.grpc.MethodDescriptor;
 
 /**
  * Exposes Flight GRPC service & client.
  */
 public class FlightGrpcUtils {
+  /**
+   * Proxy class for ManagedChannel that makes closure a no-op.
+   */
+  @VisibleForTesting
+  static class NonClosingProxyManagedChannel extends ManagedChannel {
+    private final ManagedChannel channel;
+    private boolean isShutdown;
+
+    NonClosingProxyManagedChannel(ManagedChannel channel) {
+      this.channel = channel;
+      this.isShutdown = channel.isShutdown();
+    }
+
+    @Override
+    public ManagedChannel shutdown() {
+      isShutdown = true;
+      return this;
+    }
+
+    @Override
+    public boolean isShutdown() {
+      if (this.channel.isShutdown()) {
+        // If the underlying channel is shut down, ensure we're updated to match.
+        shutdown();
+      }
+      return isShutdown;
+    }
+
+    @Override
+    public boolean isTerminated() {
+      return this.isShutdown();
+    }
+
+    @Override
+    public ManagedChannel shutdownNow() {
+      return shutdown();
+    }
+
+    @Override
+    public boolean awaitTermination(long l, TimeUnit timeUnit) {
+      // Don't actually await termination, since it'll be a no-op, so simply return whether or not
+      // the channel has been shut down already.
+      return this.isShutdown();
+    }
+
+    @Override
+    public <RequestT, ResponseT> ClientCall<RequestT, ResponseT> newCall(
+        MethodDescriptor<RequestT, ResponseT> methodDescriptor, CallOptions callOptions) {
+      if (this.isShutdown()) {
+        throw new IllegalStateException("Channel has been shut down.");
+      }
+
+      return this.channel.newCall(methodDescriptor, callOptions);
+    }
+
+    @Override
+    public String authority() {
+      return this.channel.authority();
+    }
+
+    @Override
+    public ConnectivityState getState(boolean requestConnection) {
+      if (this.isShutdown()) {
+        return ConnectivityState.SHUTDOWN;
+      }
+
+      return this.channel.getState(requestConnection);
+    }
+
+    @Override
+    public void notifyWhenStateChanged(ConnectivityState source, Runnable callback) {
+      // The proxy has no insight into the underlying channel state changes, so we'll have to leak the abstraction
+      // a bit here and simply pass to the underlying channel, even though it will never transition to shutdown via
+      // the proxy. This should be fine, since it's mainly targeted at the FlightClient and there's no getter for
+      // the channel.
+      this.channel.notifyWhenStateChanged(source, callback);
+    }
+
+    @Override
+    public void resetConnectBackoff() {
+      this.channel.resetConnectBackoff();
+    }
+
+    @Override
+    public void enterIdle() {
+      this.channel.enterIdle();
+    }
+  }
 
   private FlightGrpcUtils() {}
 
@@ -49,10 +143,19 @@ public static BindableService createFlightService(BufferAllocator allocator, Fli
   /**
    * Creates a Flight client.
    * @param incomingAllocator  Memory allocator
-   * @param channel provides a connection to a gRPC server
-   * @return FlightClient
+   * @param channel provides a connection to a gRPC server.
    */
   public static FlightClient createFlightClient(BufferAllocator incomingAllocator, ManagedChannel channel) {
     return new FlightClient(incomingAllocator, channel, Collections.emptyList());
   }
+
+  /**
+   * Creates a Flight client.
+   * @param incomingAllocator  Memory allocator
+   * @param channel provides a connection to a gRPC server. Will not be closed on closure of the returned FlightClient.
+   */
+  public static FlightClient createFlightClientWithSharedChannel(
+      BufferAllocator incomingAllocator, ManagedChannel channel) {
+    return new FlightClient(incomingAllocator, new NonClosingProxyManagedChannel(channel), Collections.emptyList());
+  }
 }
diff --git a/java/flight/flight-grpc/src/test/java/org/apache/arrow/flight/TestFlightGrpcUtils.java b/java/flight/flight-grpc/src/test/java/org/apache/arrow/flight/TestFlightGrpcUtils.java
index b27df95f768..142a0f93734 100644
--- a/java/flight/flight-grpc/src/test/java/org/apache/arrow/flight/TestFlightGrpcUtils.java
+++ b/java/flight/flight-grpc/src/test/java/org/apache/arrow/flight/TestFlightGrpcUtils.java
@@ -26,12 +26,15 @@
 import org.apache.arrow.flight.auth.ServerAuthHandler;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
+import org.junit.After;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 
 import com.google.protobuf.Empty;
 
 import io.grpc.BindableService;
+import io.grpc.ConnectivityState;
 import io.grpc.ManagedChannel;
 import io.grpc.Server;
 import io.grpc.inprocess.InProcessChannelBuilder;
@@ -42,32 +45,42 @@
  * Unit test which adds 2 services to same server end point.
  */
 public class TestFlightGrpcUtils {
+  private Server server;
+  private BufferAllocator allocator;
+  private String serverName;
 
-  /**
-   * This test checks if multiple gRPC services can be added to the same
-   * server endpoint and if they can be used by different clients via the same channel.
-   * @throws IOException If server fails to start.
-   */
-  @Test
-  public void testMultipleGrpcServices() throws IOException {
-
+  @Before
+  public void setup() throws IOException {
     //Defines flight service
-    final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+    allocator = new RootAllocator(Integer.MAX_VALUE);
     final NoOpFlightProducer producer = new NoOpFlightProducer();
     final ServerAuthHandler authHandler = ServerAuthHandler.NO_OP;
     final ExecutorService exec = Executors.newCachedThreadPool();
     final BindableService flightBindingService = FlightGrpcUtils.createFlightService(allocator, producer,
-            authHandler, exec);
+        authHandler, exec);
 
     //initializes server with 2 services - FlightBindingService & TestService
-    final String serverName = InProcessServerBuilder.generateName();
-    final Server server = InProcessServerBuilder.forName(serverName)
-            .directExecutor()
-            .addService(flightBindingService)
-            .addService(new TestServiceAdapter())
-            .build();
+    serverName = InProcessServerBuilder.generateName();
+    server = InProcessServerBuilder.forName(serverName)
+        .directExecutor()
+        .addService(flightBindingService)
+        .addService(new TestServiceAdapter())
+        .build();
     server.start();
+  }
 
+  @After
+  public void cleanup() {
+    server.shutdownNow();
+  }
+
+  /**
+   * This test checks if multiple gRPC services can be added to the same
+   * server endpoint and if they can be used by different clients via the same channel.
+   * @throws IOException If server fails to start.
+   */
+  @Test
+  public void testMultipleGrpcServices() throws IOException {
     //Initializes channel so that multiple clients can communicate with server
     final ManagedChannel managedChannel = InProcessChannelBuilder.forName(serverName)
             .directExecutor()
@@ -85,6 +98,81 @@ public void testMultipleGrpcServices() throws IOException {
     Assert.assertEquals(Empty.newBuilder().build(), blockingStub.test(Empty.newBuilder().build()));
   }
 
+  @Test
+  public void testShutdown() throws IOException, InterruptedException {
+    //Initializes channel so that multiple clients can communicate with server
+    final ManagedChannel managedChannel = InProcessChannelBuilder.forName(serverName)
+        .directExecutor()
+        .build();
+
+    //Defines flight client and calls service method. Since we use a NoOpFlightProducer we expect the service
+    //to throw a RunTimeException
+    final FlightClient flightClient = FlightGrpcUtils.createFlightClientWithSharedChannel(allocator, managedChannel);
+
+    // Should be a no-op.
+    flightClient.close();
+    Assert.assertFalse(managedChannel.isShutdown());
+    Assert.assertFalse(managedChannel.isTerminated());
+    Assert.assertEquals(ConnectivityState.IDLE, managedChannel.getState(false));
+    managedChannel.shutdownNow();
+  }
+
+  @Test
+  public void testProxyChannel() throws IOException, InterruptedException {
+    //Initializes channel so that multiple clients can communicate with server
+    final ManagedChannel managedChannel = InProcessChannelBuilder.forName(serverName)
+        .directExecutor()
+        .build();
+
+    final FlightGrpcUtils.NonClosingProxyManagedChannel proxyChannel =
+        new FlightGrpcUtils.NonClosingProxyManagedChannel(managedChannel);
+    Assert.assertFalse(proxyChannel.isShutdown());
+    Assert.assertFalse(proxyChannel.isTerminated());
+    proxyChannel.shutdown();
+    Assert.assertTrue(proxyChannel.isShutdown());
+    Assert.assertTrue(proxyChannel.isTerminated());
+    Assert.assertEquals(ConnectivityState.SHUTDOWN, proxyChannel.getState(false));
+    try {
+      proxyChannel.newCall(null, null);
+      Assert.fail();
+    } catch (IllegalStateException e) {
+      // This is expected, since the proxy channel is shut down.
+    }
+
+    Assert.assertFalse(managedChannel.isShutdown());
+    Assert.assertFalse(managedChannel.isTerminated());
+    Assert.assertEquals(ConnectivityState.IDLE, managedChannel.getState(false));
+
+    managedChannel.shutdownNow();
+  }
+
+  @Test
+  public void testProxyChannelWithClosedChannel() throws IOException, InterruptedException {
+    //Initializes channel so that multiple clients can communicate with server
+    final ManagedChannel managedChannel = InProcessChannelBuilder.forName(serverName)
+        .directExecutor()
+        .build();
+
+    final FlightGrpcUtils.NonClosingProxyManagedChannel proxyChannel =
+        new FlightGrpcUtils.NonClosingProxyManagedChannel(managedChannel);
+    Assert.assertFalse(proxyChannel.isShutdown());
+    Assert.assertFalse(proxyChannel.isTerminated());
+    managedChannel.shutdownNow();
+    Assert.assertTrue(proxyChannel.isShutdown());
+    Assert.assertTrue(proxyChannel.isTerminated());
+    Assert.assertEquals(ConnectivityState.SHUTDOWN, proxyChannel.getState(false));
+    try {
+      proxyChannel.newCall(null, null);
+      Assert.fail();
+    } catch (IllegalStateException e) {
+      // This is expected, since the proxy channel is shut down.
+    }
+
+    Assert.assertTrue(managedChannel.isShutdown());
+    Assert.assertTrue(managedChannel.isTerminated());
+    Assert.assertEquals(ConnectivityState.SHUTDOWN, managedChannel.getState(false));
+  }
+
   /**
    * Private class used for testing purposes that overrides service behavior.
    */
diff --git a/java/format/pom.xml b/java/format/pom.xml
index 9dd895c9355..48ac1b15520 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>2.0.0-SNAPSHOT</version>
+  <version>3.0.0-SNAPSHOT</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/gandiva/pom.xml b/java/gandiva/pom.xml
index abdd2abfe21..0e64487a77a 100644
--- a/java/gandiva/pom.xml
+++ b/java/gandiva/pom.xml
@@ -14,7 +14,7 @@
     <parent>
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-java-root</artifactId>
-      <version>2.0.0-SNAPSHOT</version>
+      <version>3.0.0-SNAPSHOT</version>
     </parent>
 
     <groupId>org.apache.arrow.gandiva</groupId>
diff --git a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/DecimalTypeUtil.java b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/DecimalTypeUtil.java
index f6b76f98188..e0c072cfbe5 100644
--- a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/DecimalTypeUtil.java
+++ b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/DecimalTypeUtil.java
@@ -87,7 +87,7 @@ private static Decimal adjustScaleIfNeeded(int precision, int scale) {
       precision = MAX_PRECISION;
       scale = Math.max(scale - delta, minScale);
     }
-    return new Decimal(precision, scale);
+    return new Decimal(precision, scale, 128);
   }
 
 }
diff --git a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/ExpressionRegistry.java b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/ExpressionRegistry.java
index 6b2610ff3d2..0155af08234 100644
--- a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/ExpressionRegistry.java
+++ b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/ExpressionRegistry.java
@@ -175,7 +175,7 @@ private static ArrowType getArrowType(ExtGandivaType type) {
       case GandivaType.NONE_VALUE:
         return new ArrowType.Null();
       case GandivaType.DECIMAL_VALUE:
-        return new ArrowType.Decimal(0, 0);
+        return new ArrowType.Decimal(0, 0, 128);
       case GandivaType.INTERVAL_VALUE:
         return new ArrowType.Interval(mapArrowIntervalUnit(type.getIntervalType()));
       case GandivaType.FIXED_SIZE_BINARY_VALUE:
diff --git a/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/DecimalTypeUtilTest.java b/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/DecimalTypeUtilTest.java
index 96bffe3f285..fe51c09e33d 100644
--- a/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/DecimalTypeUtilTest.java
+++ b/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/DecimalTypeUtilTest.java
@@ -83,7 +83,7 @@ public void testOutputTypesForMod() {
   }
 
   private ArrowType.Decimal getDecimal(int precision, int scale) {
-    return new ArrowType.Decimal(precision, scale);
+    return new ArrowType.Decimal(precision, scale, 128);
   }
 
 }
diff --git a/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorDecimalTest.java b/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorDecimalTest.java
index f3de03b66f1..28a57c9f8a4 100644
--- a/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorDecimalTest.java
+++ b/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorDecimalTest.java
@@ -56,7 +56,7 @@ public class ProjectorDecimalTest extends org.apache.arrow.gandiva.evaluator.Bas
   public void test_add() throws GandivaException {
     int precision = 38;
     int scale = 8;
-    ArrowType.Decimal decimal = new ArrowType.Decimal(precision, scale);
+    ArrowType.Decimal decimal = new ArrowType.Decimal(precision, scale, 128);
     Field a = Field.nullable("a", decimal);
     Field b = Field.nullable("b", decimal);
     List<Field> args = Lists.newArrayList(a, b);
@@ -115,8 +115,8 @@ public void test_add() throws GandivaException {
   public void test_add_literal() throws GandivaException {
     int precision = 2;
     int scale = 0;
-    ArrowType.Decimal decimal = new ArrowType.Decimal(precision, scale);
-    ArrowType.Decimal literalType = new ArrowType.Decimal(2, 1);
+    ArrowType.Decimal decimal = new ArrowType.Decimal(precision, scale, 128);
+    ArrowType.Decimal literalType = new ArrowType.Decimal(2, 1, 128);
     Field a = Field.nullable("a", decimal);
 
     ArrowType.Decimal outputType = DecimalTypeUtil.getResultTypeForOperation(DecimalTypeUtil
@@ -169,7 +169,7 @@ public void test_add_literal() throws GandivaException {
   public void test_multiply() throws GandivaException {
     int precision = 38;
     int scale = 8;
-    ArrowType.Decimal decimal = new ArrowType.Decimal(precision, scale);
+    ArrowType.Decimal decimal = new ArrowType.Decimal(precision, scale, 128);
     Field a = Field.nullable("a", decimal);
     Field b = Field.nullable("b", decimal);
     List<Field> args = Lists.newArrayList(a, b);
@@ -226,8 +226,8 @@ public void test_multiply() throws GandivaException {
 
   @Test
   public void testCompare() throws GandivaException {
-    Decimal aType = new Decimal(38, 3);
-    Decimal bType = new Decimal(38, 2);
+    Decimal aType = new Decimal(38, 3, 128);
+    Decimal bType = new Decimal(38, 2, 128);
     Field a = Field.nullable("a", aType);
     Field b = Field.nullable("b", bType);
     List<Field> args = Lists.newArrayList(a, b);
@@ -315,9 +315,9 @@ public void testCompare() throws GandivaException {
 
   @Test
   public void testRound() throws GandivaException {
-    Decimal aType = new Decimal(38, 2);
-    Decimal aWithScaleZero = new Decimal(38, 0);
-    Decimal aWithScaleOne = new Decimal(38, 1);
+    Decimal aType = new Decimal(38, 2, 128);
+    Decimal aWithScaleZero = new Decimal(38, 0, 128);
+    Decimal aWithScaleOne = new Decimal(38, 1, 128);
     Field a = Field.nullable("a", aType);
     List<Field> args = Lists.newArrayList(a);
 
@@ -419,8 +419,8 @@ public void testRound() throws GandivaException {
 
   @Test
   public void testCastToDecimal() throws GandivaException {
-    Decimal decimalType = new Decimal(38, 2);
-    Decimal decimalWithScaleOne = new Decimal(38, 1);
+    Decimal decimalType = new Decimal(38, 2, 128);
+    Decimal decimalWithScaleOne = new Decimal(38, 1, 128);
     Field dec = Field.nullable("dec", decimalType);
     Field int64f = Field.nullable("int64", int64);
     Field doublef = Field.nullable("float64", float64);
@@ -517,7 +517,7 @@ public void testCastToDecimal() throws GandivaException {
 
   @Test
   public void testCastToLong() throws GandivaException {
-    Decimal decimalType = new Decimal(38, 2);
+    Decimal decimalType = new Decimal(38, 2, 128);
     Field dec = Field.nullable("dec", decimalType);
 
     Schema schema = new Schema(Lists.newArrayList(dec));
@@ -575,7 +575,7 @@ public void testCastToLong() throws GandivaException {
 
   @Test
   public void testCastToDouble() throws GandivaException {
-    Decimal decimalType = new Decimal(38, 2);
+    Decimal decimalType = new Decimal(38, 2, 128);
     Field dec = Field.nullable("dec", decimalType);
 
     Schema schema = new Schema(Lists.newArrayList(dec));
@@ -633,7 +633,7 @@ public void testCastToDouble() throws GandivaException {
 
   @Test
   public void testCastToString() throws GandivaException {
-    Decimal decimalType = new Decimal(38, 2);
+    Decimal decimalType = new Decimal(38, 2, 128);
     Field dec = Field.nullable("dec", decimalType);
     Field str = Field.nullable("str", new ArrowType.Utf8());
     TreeNode field = TreeBuilder.makeField(dec);
@@ -695,7 +695,7 @@ public void testCastToString() throws GandivaException {
 
   @Test
   public void testCastStringToDecimal() throws GandivaException {
-    Decimal decimalType = new Decimal(4, 2);
+    Decimal decimalType = new Decimal(4, 2, 128);
     Field dec = Field.nullable("dec", decimalType);
 
     Field str = Field.nullable("str", new ArrowType.Utf8());
@@ -761,7 +761,7 @@ public void testInvalidDecimal() throws GandivaException {
     exception.expect(IllegalArgumentException.class);
     exception.expectMessage("Gandiva only supports decimals of upto 38 precision. Input precision" +
             " : 0");
-    Decimal decimalType = new Decimal(0, 0);
+    Decimal decimalType = new Decimal(0, 0, 128);
     Field int64f = Field.nullable("int64", int64);
 
     Schema schema = new Schema(Lists.newArrayList(int64f));
@@ -780,7 +780,7 @@ public void testInvalidDecimalGt38() throws GandivaException {
     exception.expect(IllegalArgumentException.class);
     exception.expectMessage("Gandiva only supports decimals of upto 38 precision. Input precision" +
             " : 42");
-    Decimal decimalType = new Decimal(42, 0);
+    Decimal decimalType = new Decimal(42, 0, 128);
     Field int64f = Field.nullable("int64", int64);
 
     Schema schema = new Schema(Lists.newArrayList(int64f));
diff --git a/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java b/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java
index 753cdf6a10a..85ac83b42da 100644
--- a/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java
+++ b/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java
@@ -1741,4 +1741,191 @@ public void testCaseInsensitiveFunctions() throws Exception {
     releaseValueVectors(output);
   }
 
+  @Test
+  public void testCastInt() throws Exception {
+    Field inField = Field.nullable("input", new ArrowType.Utf8());
+    TreeNode inNode = TreeBuilder.makeField(inField);
+    TreeNode castINTFn = TreeBuilder.makeFunction("castINT", Lists.newArrayList(inNode),
+        int32);
+    Field resultField = Field.nullable("result", int32);
+    List<ExpressionTree> exprs =
+        Lists.newArrayList(
+            TreeBuilder.makeExpression(castINTFn, resultField));
+    Schema schema = new Schema(Lists.newArrayList(inField));
+    Projector eval = Projector.make(schema, exprs);
+    int numRows = 5;
+    byte[] validity = new byte[] {(byte) 255};
+    String[] values =
+        new String[] {
+            "0", "123", "-123", "-1", "1"
+        };
+    int[] expValues =
+        new int[] {
+            0, 123, -123, -1, 1
+        };
+    ArrowBuf bufValidity = buf(validity);
+    List<ArrowBuf> bufData = stringBufs(values);
+    ArrowFieldNode fieldNode = new ArrowFieldNode(numRows, 0);
+    ArrowRecordBatch batch =
+        new ArrowRecordBatch(
+            numRows,
+            Lists.newArrayList(fieldNode),
+            Lists.newArrayList(bufValidity, bufData.get(0), bufData.get(1)));
+    List<ValueVector> output = new ArrayList<>();
+    for (int i = 0; i < exprs.size(); i++) {
+      IntVector intVector = new IntVector(EMPTY_SCHEMA_PATH, allocator);
+      intVector.allocateNew(numRows);
+      output.add(intVector);
+    }
+    eval.evaluate(batch, output);
+    eval.close();
+    for (ValueVector valueVector : output) {
+      IntVector intVector = (IntVector) valueVector;
+      for (int j = 0; j < numRows; j++) {
+        assertFalse(intVector.isNull(j));
+        assertTrue(expValues[j] == intVector.get(j));
+      }
+    }
+    releaseRecordBatch(batch);
+    releaseValueVectors(output);
+  }
+
+  @Test(expected = GandivaException.class)
+  public void testCastIntInvalidValue() throws Exception {
+    Field inField = Field.nullable("input", new ArrowType.Utf8());
+    TreeNode inNode = TreeBuilder.makeField(inField);
+    TreeNode castINTFn = TreeBuilder.makeFunction("castINT", Lists.newArrayList(inNode),
+        int32);
+    Field resultField = Field.nullable("result", int32);
+    List<ExpressionTree> exprs =
+        Lists.newArrayList(
+            TreeBuilder.makeExpression(castINTFn, resultField));
+    Schema schema = new Schema(Lists.newArrayList(inField));
+    Projector eval = Projector.make(schema, exprs);
+    int numRows = 1;
+    byte[] validity = new byte[] {(byte) 255};
+    String[] values =
+        new String[] {
+            "abc"
+        };
+    ArrowBuf bufValidity = buf(validity);
+    List<ArrowBuf> bufData = stringBufs(values);
+    ArrowFieldNode fieldNode = new ArrowFieldNode(numRows, 0);
+    ArrowRecordBatch batch =
+        new ArrowRecordBatch(
+            numRows,
+            Lists.newArrayList(fieldNode),
+            Lists.newArrayList(bufValidity, bufData.get(0), bufData.get(1)));
+    List<ValueVector> output = new ArrayList<>();
+    for (int i = 0; i < exprs.size(); i++) {
+      IntVector intVector = new IntVector(EMPTY_SCHEMA_PATH, allocator);
+      intVector.allocateNew(numRows);
+      output.add(intVector);
+    }
+    try {
+      eval.evaluate(batch, output);
+    } finally {
+      eval.close();
+      releaseRecordBatch(batch);
+      releaseValueVectors(output);
+    }
+  }
+
+  @Test
+  public void testCastFloat() throws Exception {
+    Field inField = Field.nullable("input", new ArrowType.Utf8());
+    TreeNode inNode = TreeBuilder.makeField(inField);
+    TreeNode castFLOAT8Fn = TreeBuilder.makeFunction("castFLOAT8", Lists.newArrayList(inNode),
+        float64);
+    Field resultField = Field.nullable("result", float64);
+    List<ExpressionTree> exprs =
+        Lists.newArrayList(
+            TreeBuilder.makeExpression(castFLOAT8Fn, resultField));
+    Schema schema = new Schema(Lists.newArrayList(inField));
+    Projector eval = Projector.make(schema, exprs);
+    int numRows = 5;
+    byte[] validity = new byte[] {(byte) 255};
+    String[] values =
+        new String[] {
+            "2.3",
+            "-11.11",
+            "0",
+            "111",
+            "12345.67"
+        };
+    double[] expValues =
+        new double[] {
+            2.3, -11.11, 0, 111, 12345.67
+        };
+    ArrowBuf bufValidity = buf(validity);
+    List<ArrowBuf> bufData = stringBufs(values);
+    ArrowFieldNode fieldNode = new ArrowFieldNode(numRows, 0);
+    ArrowRecordBatch batch =
+        new ArrowRecordBatch(
+            numRows,
+            Lists.newArrayList(fieldNode),
+            Lists.newArrayList(bufValidity, bufData.get(0), bufData.get(1)));
+    List<ValueVector> output = new ArrayList<>();
+    for (int i = 0; i < exprs.size(); i++) {
+      Float8Vector float8Vector = new Float8Vector(EMPTY_SCHEMA_PATH, allocator);
+      float8Vector.allocateNew(numRows);
+      output.add(float8Vector);
+    }
+    eval.evaluate(batch, output);
+    eval.close();
+    for (ValueVector valueVector : output) {
+      Float8Vector float8Vector = (Float8Vector) valueVector;
+      for (int j = 0; j < numRows; j++) {
+        assertFalse(float8Vector.isNull(j));
+        assertTrue(expValues[j] == float8Vector.get(j));
+      }
+    }
+    releaseRecordBatch(batch);
+    releaseValueVectors(output);
+  }
+
+  @Test(expected = GandivaException.class)
+  public void testCastFloatInvalidValue() throws Exception {
+    Field inField = Field.nullable("input", new ArrowType.Utf8());
+    TreeNode inNode = TreeBuilder.makeField(inField);
+    TreeNode castFLOAT8Fn = TreeBuilder.makeFunction("castFLOAT8", Lists.newArrayList(inNode),
+        float64);
+    Field resultField = Field.nullable("result", float64);
+    List<ExpressionTree> exprs =
+        Lists.newArrayList(
+            TreeBuilder.makeExpression(castFLOAT8Fn, resultField));
+    Schema schema = new Schema(Lists.newArrayList(inField));
+    Projector eval = Projector.make(schema, exprs);
+    int numRows = 5;
+    byte[] validity = new byte[] {(byte) 255};
+    String[] values =
+        new String[] {
+            "2.3",
+            "-11.11",
+            "abc",
+            "111",
+            "12345.67"
+        };
+    ArrowBuf bufValidity = buf(validity);
+    List<ArrowBuf> bufData = stringBufs(values);
+    ArrowFieldNode fieldNode = new ArrowFieldNode(numRows, 0);
+    ArrowRecordBatch batch =
+        new ArrowRecordBatch(
+            numRows,
+            Lists.newArrayList(fieldNode),
+            Lists.newArrayList(bufValidity, bufData.get(0), bufData.get(1)));
+    List<ValueVector> output = new ArrayList<>();
+    for (int i = 0; i < exprs.size(); i++) {
+      Float8Vector float8Vector = new Float8Vector(EMPTY_SCHEMA_PATH, allocator);
+      float8Vector.allocateNew(numRows);
+      output.add(float8Vector);
+    }
+    try {
+      eval.evaluate(batch, output);
+    } finally {
+      eval.close();
+      releaseRecordBatch(batch);
+      releaseValueVectors(output);
+    }
+  }
 }
diff --git a/java/memory/memory-core/pom.xml b/java/memory/memory-core/pom.xml
index 34666ec3254..dc893fe3fac 100644
--- a/java/memory/memory-core/pom.xml
+++ b/java/memory/memory-core/pom.xml
@@ -13,7 +13,7 @@
   <parent>
     <artifactId>arrow-memory</artifactId>
     <groupId>org.apache.arrow</groupId>
-    <version>2.0.0-SNAPSHOT</version>
+    <version>3.0.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 
diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/Accountant.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/Accountant.java
index da93511b4f2..42dac7b8c60 100644
--- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/Accountant.java
+++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/Accountant.java
@@ -140,7 +140,7 @@ private void updatePeak() {
    * @param size to increase
    * @return Whether the allocation fit within limits.
    */
-  boolean forceAllocate(long size) {
+  public boolean forceAllocate(long size) {
     final AllocationOutcome.Status outcome = allocate(size, true, true, null);
     return outcome.isOk();
   }
@@ -220,7 +220,6 @@ public void releaseBytes(long size) {
       final long actualToReleaseToParent = Math.min(size, possibleAmountToReleaseToParent);
       parent.releaseBytes(actualToReleaseToParent);
     }
-
   }
 
   public boolean isOverLimit() {
diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationManager.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationManager.java
index c61d041097e..9c7cfa9d90d 100644
--- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationManager.java
+++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationManager.java
@@ -47,11 +47,11 @@ public abstract class AllocationManager {
 
   private static final AtomicLong MANAGER_ID_GENERATOR = new AtomicLong(0);
 
-  private final RootAllocator root;
+  private final BufferAllocator root;
   private final long allocatorManagerId = MANAGER_ID_GENERATOR.incrementAndGet();
   // ARROW-1627 Trying to minimize memory overhead caused by previously used IdentityHashMap
   // see JIRA for details
-  private final LowCostIdentityHashMap<BaseAllocator, BufferLedger> map = new LowCostIdentityHashMap<>();
+  private final LowCostIdentityHashMap<BufferAllocator, BufferLedger> map = new LowCostIdentityHashMap<>();
   private final long amCreationTime = System.nanoTime();
 
   // The ReferenceManager created at the time of creation of this AllocationManager
@@ -60,11 +60,11 @@ public abstract class AllocationManager {
   private volatile BufferLedger owningLedger;
   private volatile long amDestructionTime = 0;
 
-  protected AllocationManager(BaseAllocator accountingAllocator) {
+  protected AllocationManager(BufferAllocator accountingAllocator) {
     Preconditions.checkNotNull(accountingAllocator);
     accountingAllocator.assertOpen();
 
-    this.root = accountingAllocator.root;
+    this.root = accountingAllocator.getRoot();
 
     // we do a no retain association since our creator will want to retrieve the newly created
     // ledger and will create a reference count at that point
@@ -87,13 +87,13 @@ void setOwningLedger(final BufferLedger ledger) {
    * @return The reference manager (new or existing) that associates the underlying
    *         buffer to this new ledger.
    */
-  BufferLedger associate(final BaseAllocator allocator) {
+  BufferLedger associate(final BufferAllocator allocator) {
     return associate(allocator, true);
   }
 
-  private BufferLedger associate(final BaseAllocator allocator, final boolean retain) {
+  private BufferLedger associate(final BufferAllocator allocator, final boolean retain) {
     allocator.assertOpen();
-    Preconditions.checkState(root == allocator.root,
+    Preconditions.checkState(root == allocator.getRoot(),
           "A buffer can only be associated between two allocators that share the same root");
 
     synchronized (this) {
@@ -118,9 +118,11 @@ private BufferLedger associate(final BaseAllocator allocator, final boolean reta
       Preconditions.checkState(oldLedger == null,
           "Detected inconsistent state: A reference manager already exists for this allocator");
 
-      // needed for debugging only: keep a pointer to reference manager inside allocator
-      // to dump state, verify allocator state etc
-      allocator.associateLedger(ledger);
+      if (allocator instanceof BaseAllocator) {
+        // needed for debugging only: keep a pointer to reference manager inside allocator
+        // to dump state, verify allocator state etc
+        ((BaseAllocator) allocator).associateLedger(ledger);
+      }
       return ledger;
     }
   }
@@ -133,7 +135,7 @@ private BufferLedger associate(final BaseAllocator allocator, final boolean reta
    * calling ReferenceManager drops to 0.
    */
   void release(final BufferLedger ledger) {
-    final BaseAllocator allocator = (BaseAllocator) ledger.getAllocator();
+    final BufferAllocator allocator = ledger.getAllocator();
     allocator.assertOpen();
 
     // remove the <BaseAllocator, BufferLedger> mapping for the allocator
@@ -142,9 +144,12 @@ void release(final BufferLedger ledger) {
         "Expecting a mapping for allocator and reference manager");
     final BufferLedger oldLedger = map.remove(allocator);
 
-    // needed for debug only: tell the allocator that AllocationManager is removing a
-    // reference manager associated with this particular allocator
-    ((BaseAllocator) oldLedger.getAllocator()).dissociateLedger(oldLedger);
+    BufferAllocator oldAllocator = oldLedger.getAllocator();
+    if (oldAllocator instanceof BaseAllocator) {
+      // needed for debug only: tell the allocator that AllocationManager is removing a
+      // reference manager associated with this particular allocator
+      ((BaseAllocator) oldAllocator).dissociateLedger(oldLedger);
+    }
 
     if (oldLedger == owningLedger) {
       // the release call was made by the owning reference manager
@@ -152,10 +157,10 @@ void release(final BufferLedger ledger) {
         // the only <allocator, reference manager> mapping was for the owner
         // which now has been removed, it implies we can safely destroy the
         // underlying memory chunk as it is no longer being referenced
-        ((BaseAllocator) oldLedger.getAllocator()).releaseBytes(getSize());
+        oldAllocator.releaseBytes(getSize());
         // free the memory chunk associated with the allocation manager
         release0();
-        ((BaseAllocator) oldLedger.getAllocator()).getListener().onRelease(getSize());
+        oldAllocator.getListener().onRelease(getSize());
         amDestructionTime = System.nanoTime();
         owningLedger = null;
       } else {
@@ -209,7 +214,7 @@ public interface Factory {
      * @param size Size (in bytes) of memory managed by the AllocationManager
      * @return The created AllocationManager used by this allocator
      */
-    AllocationManager create(BaseAllocator accountingAllocator, long size);
+    AllocationManager create(BufferAllocator accountingAllocator, long size);
 
     ArrowBuf empty();
   }
diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java
index e81a92f3c31..ea5e29f7253 100644
--- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java
+++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java
@@ -163,7 +163,7 @@ public synchronized ArrowBuf capacity(long newCapacity) {
    * Returns the byte order of elements in this buffer.
    */
   public ByteOrder order() {
-    return ByteOrder.LITTLE_ENDIAN;
+    return ByteOrder.nativeOrder();
   }
 
   /**
@@ -846,7 +846,7 @@ public void setBytes(long index, ByteBuffer src) {
         src.position(src.position() + length);
       } else {
         final ByteOrder originalByteOrder = src.order();
-        src.order(ByteOrder.LITTLE_ENDIAN);
+        src.order(order());
         try {
           // copy word at a time
           while (length - 128 >= LONG_SIZE) {
diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BaseAllocator.java
index 81f664985d5..246b2212e26 100644
--- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BaseAllocator.java
+++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BaseAllocator.java
@@ -61,10 +61,10 @@ abstract class BaseAllocator extends Accountant implements BufferAllocator {
   public static final Config DEFAULT_CONFIG = ImmutableConfig.builder().build();
 
   // Package exposed for sharing between AllocatorManger and BaseAllocator objects
-  final String name;
-  final RootAllocator root;
+  private final String name;
+  private final RootAllocator root;
   private final Object DEBUG_LOCK = DEBUG ? new Object() : null;
-  final AllocationListener listener;
+  private final AllocationListener listener;
   private final BaseAllocator parentAllocator;
   private final Map<BaseAllocator, Object> childAllocators;
   private final ArrowBuf empty;
@@ -124,7 +124,8 @@ protected BaseAllocator(
     this.roundingPolicy = config.getRoundingPolicy();
   }
 
-  AllocationListener getListener() {
+  @Override
+  public AllocationListener getListener() {
     return listener;
   }
 
@@ -314,6 +315,11 @@ private AllocationManager newAllocationManager(BaseAllocator accountingAllocator
     return allocationManagerFactory.create(accountingAllocator, size);
   }
 
+  @Override
+  public BufferAllocator getRoot() {
+    return root;
+  }
+
   @Override
   public BufferAllocator newChildAllocator(
       final String name,
@@ -343,7 +349,7 @@ public BufferAllocator newChildAllocator(
       synchronized (DEBUG_LOCK) {
         childAllocators.put(childAllocator, childAllocator);
         historicalLog.recordEvent("allocator[%s] created new child allocator[%s]", name,
-            childAllocator.name);
+            childAllocator.getName());
       }
     } else {
       childAllocators.put(childAllocator, childAllocator);
diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferAllocator.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferAllocator.java
index aa1f856c591..8fbf6f7b073 100644
--- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferAllocator.java
+++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferAllocator.java
@@ -49,6 +49,14 @@ public interface BufferAllocator extends AutoCloseable {
    */
   ArrowBuf buffer(long size, BufferManager manager);
 
+  /**
+   * Get the root allocator of this allocator. If this allocator is already a root, return
+   * this directly.
+   *
+   * @return The root allocator
+   */
+  BufferAllocator getRoot();
+
   /**
    * Create a new child allocator.
    *
@@ -126,6 +134,30 @@ BufferAllocator newChildAllocator(
    */
   long getHeadroom();
 
+  /**
+   * Forcibly allocate bytes. Returns whether the allocation fit within limits.
+   *
+   * @param size to increase
+   * @return Whether the allocation fit within limits.
+   */
+  boolean forceAllocate(long size);
+
+
+  /**
+   * Release bytes from this allocator.
+   *
+   * @param size to release
+   */
+  void releaseBytes(long size);
+
+  /**
+   * Returns the allocation listener used by this allocator.
+   *
+   * @return the {@link AllocationListener} instance. Or {@link AllocationListener#NOOP} by default if no listener
+   *         is configured when this allocator was created.
+   */
+  AllocationListener getListener();
+
   /**
    * Returns the parent allocator.
    *
diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java
index 9fa4de71d8d..48b3e183d5a 100644
--- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java
+++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java
@@ -31,7 +31,7 @@
  * ArrowBufs managed by this reference manager share a common
  * fate (same reference count).
  */
-public class BufferLedger implements ValueWithKeyIncluded<BaseAllocator>, ReferenceManager {
+public class BufferLedger implements ValueWithKeyIncluded<BufferAllocator>, ReferenceManager {
   private final IdentityHashMap<ArrowBuf, Object> buffers =
           BaseAllocator.DEBUG ? new IdentityHashMap<>() : null;
   private static final AtomicLong LEDGER_ID_GENERATOR = new AtomicLong(0);
@@ -41,14 +41,14 @@ public class BufferLedger implements ValueWithKeyIncluded<BaseAllocator>, Refere
   // manage request for retain
   // correctly
   private final long lCreationTime = System.nanoTime();
-  private final BaseAllocator allocator;
+  private final BufferAllocator allocator;
   private final AllocationManager allocationManager;
   private final HistoricalLog historicalLog =
       BaseAllocator.DEBUG ? new HistoricalLog(BaseAllocator.DEBUG_LOG_LENGTH,
         "BufferLedger[%d]", 1) : null;
   private volatile long lDestructionTime = 0;
 
-  BufferLedger(final BaseAllocator allocator, final AllocationManager allocationManager) {
+  BufferLedger(final BufferAllocator allocator, final AllocationManager allocationManager) {
     this.allocator = allocator;
     this.allocationManager = allocationManager;
   }
@@ -57,7 +57,7 @@ boolean isOwningLedger() {
     return this == allocationManager.getOwningLedger();
   }
 
-  public BaseAllocator getKey() {
+  public BufferAllocator getKey() {
     return allocator;
   }
 
@@ -238,7 +238,7 @@ public ArrowBuf deriveBuffer(final ArrowBuf sourceBuffer, long index, long lengt
               "ArrowBuf(BufferLedger, BufferAllocator[%s], " +
                       "UnsafeDirectLittleEndian[identityHashCode == " +
                       "%d](%s)) => ledger hc == %d",
-              allocator.name, System.identityHashCode(derivedBuf), derivedBuf.toString(),
+              allocator.getName(), System.identityHashCode(derivedBuf), derivedBuf.toString(),
               System.identityHashCode(this));
 
       synchronized (buffers) {
@@ -275,7 +275,7 @@ ArrowBuf newArrowBuf(final long length, final BufferManager manager) {
       historicalLog.recordEvent(
           "ArrowBuf(BufferLedger, BufferAllocator[%s], " +
           "UnsafeDirectLittleEndian[identityHashCode == " + "%d](%s)) => ledger hc == %d",
-          allocator.name, System.identityHashCode(buf), buf.toString(),
+          allocator.getName(), System.identityHashCode(buf), buf.toString(),
           System.identityHashCode(this));
 
       synchronized (buffers) {
@@ -317,7 +317,7 @@ public ArrowBuf retain(final ArrowBuf srcBuffer, BufferAllocator target) {
     // alternatively, if there was already a mapping for <buffer allocator, ref manager> in
     // allocation manager, the ref count of the new buffer will be targetrefmanager.refcount() + 1
     // and this will be true for all the existing buffers currently managed by targetrefmanager
-    final BufferLedger targetRefManager = allocationManager.associate((BaseAllocator) target);
+    final BufferLedger targetRefManager = allocationManager.associate(target);
     // create a new ArrowBuf to associate with new allocator and target ref manager
     final long targetBufLength = srcBuffer.capacity();
     ArrowBuf targetArrowBuf = targetRefManager.deriveBuffer(srcBuffer, 0, targetBufLength);
@@ -336,8 +336,8 @@ public ArrowBuf retain(final ArrowBuf srcBuffer, BufferAllocator target) {
   boolean transferBalance(final ReferenceManager targetReferenceManager) {
     Preconditions.checkArgument(targetReferenceManager != null,
         "Expecting valid target reference manager");
-    final BaseAllocator targetAllocator = (BaseAllocator) targetReferenceManager.getAllocator();
-    Preconditions.checkArgument(allocator.root == targetAllocator.root,
+    final BufferAllocator targetAllocator = targetReferenceManager.getAllocator();
+    Preconditions.checkArgument(allocator.getRoot() == targetAllocator.getRoot(),
         "You can only transfer between two allocators that share the same root.");
 
     allocator.assertOpen();
@@ -411,7 +411,7 @@ public TransferResult transferOwnership(final ArrowBuf srcBuffer, final BufferAl
     // alternatively, if there was already a mapping for <buffer allocator, ref manager> in
     // allocation manager, the ref count of the new buffer will be targetrefmanager.refcount() + 1
     // and this will be true for all the existing buffers currently managed by targetrefmanager
-    final BufferLedger targetRefManager = allocationManager.associate((BaseAllocator) target);
+    final BufferLedger targetRefManager = allocationManager.associate(target);
     // create a new ArrowBuf to associate with new allocator and target ref manager
     final long targetBufLength = srcBuffer.capacity();
     final ArrowBuf targetArrowBuf = targetRefManager.deriveBuffer(srcBuffer, 0, targetBufLength);
@@ -486,7 +486,7 @@ void print(StringBuilder sb, int indent, BaseAllocator.Verbosity verbosity) {
       .append("ledger[")
       .append(ledgerId)
       .append("] allocator: ")
-      .append(allocator.name)
+      .append(allocator.getName())
       .append("), isOwning: ")
       .append(", size: ")
       .append(", references: ")
diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ByteFunctionHelpers.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ByteFunctionHelpers.java
index df571289d87..9579245ca70 100644
--- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ByteFunctionHelpers.java
+++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ByteFunctionHelpers.java
@@ -17,6 +17,8 @@
 
 package org.apache.arrow.memory.util;
 
+import java.nio.ByteOrder;
+
 import org.apache.arrow.memory.ArrowBuf;
 import org.apache.arrow.memory.BoundsChecking;
 import org.apache.arrow.memory.util.hash.ArrowBufHasher;
@@ -28,6 +30,8 @@
 public class ByteFunctionHelpers {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ByteFunctionHelpers.class);
 
+  private static final boolean LITTLE_ENDIAN = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
+
   private ByteFunctionHelpers() {}
 
   /**
@@ -152,7 +156,11 @@ private static int memcmp(
         long leftLong = MemoryUtil.UNSAFE.getLong(lPos);
         long rightLong = MemoryUtil.UNSAFE.getLong(rPos);
         if (leftLong != rightLong) {
-          return unsignedLongCompare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong));
+          if (LITTLE_ENDIAN) {
+            return unsignedLongCompare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong));
+          } else {
+            return unsignedLongCompare(leftLong, rightLong);
+          }
         }
         lPos += 8;
         rPos += 8;
@@ -164,7 +172,11 @@ private static int memcmp(
       long leftLong = MemoryUtil.UNSAFE.getLong(lPos);
       long rightLong = MemoryUtil.UNSAFE.getLong(rPos);
       if (leftLong != rightLong) {
-        return unsignedLongCompare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong));
+        if (LITTLE_ENDIAN) {
+          return unsignedLongCompare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong));
+        } else {
+          return unsignedLongCompare(leftLong, rightLong);
+        }
       }
       lPos += 8;
       rPos += 8;
@@ -175,7 +187,11 @@ private static int memcmp(
       int leftInt = MemoryUtil.UNSAFE.getInt(lPos);
       int rightInt = MemoryUtil.UNSAFE.getInt(rPos);
       if (leftInt != rightInt) {
-        return unsignedIntCompare(Integer.reverseBytes(leftInt), Integer.reverseBytes(rightInt));
+        if (LITTLE_ENDIAN) {
+          return unsignedIntCompare(Integer.reverseBytes(leftInt), Integer.reverseBytes(rightInt));
+        } else {
+          return unsignedIntCompare(leftInt, rightInt);
+        }
       }
       lPos += 4;
       rPos += 4;
@@ -259,7 +275,11 @@ private static int memcmp(
       long leftLong = MemoryUtil.UNSAFE.getLong(lPos);
       long rightLong = MemoryUtil.UNSAFE.getLong(right, MemoryUtil.BYTE_ARRAY_BASE_OFFSET + rPos);
       if (leftLong != rightLong) {
-        return unsignedLongCompare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong));
+        if (LITTLE_ENDIAN) {
+          return unsignedLongCompare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong));
+        } else {
+          return unsignedLongCompare(leftLong, rightLong);
+        }
       }
       lPos += 8;
       rPos += 8;
@@ -270,7 +290,11 @@ private static int memcmp(
       int leftInt = MemoryUtil.UNSAFE.getInt(lPos);
       int rightInt = MemoryUtil.UNSAFE.getInt(right, MemoryUtil.BYTE_ARRAY_BASE_OFFSET + rPos);
       if (leftInt != rightInt) {
-        return unsignedIntCompare(Integer.reverseBytes(leftInt), Integer.reverseBytes(rightInt));
+        if (LITTLE_ENDIAN) {
+          return unsignedIntCompare(Integer.reverseBytes(leftInt), Integer.reverseBytes(rightInt));
+        } else {
+          return unsignedIntCompare(leftInt, rightInt);
+        }
       }
       lPos += 4;
       rPos += 4;
diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/LargeMemoryUtil.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/LargeMemoryUtil.java
index f60a986bd50..db63bbd14ba 100644
--- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/LargeMemoryUtil.java
+++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/LargeMemoryUtil.java
@@ -18,7 +18,6 @@
 package org.apache.arrow.memory.util;
 
 import org.apache.arrow.memory.BoundsChecking;
-import org.apache.arrow.util.Preconditions;
 
 /** Contains utilities for dealing with a 64-bit address base. */
 public final class LargeMemoryUtil {
@@ -31,8 +30,7 @@ private LargeMemoryUtil() {}
    */
   public static int checkedCastToInt(long length) {
     if (BoundsChecking.BOUNDS_CHECKING_ENABLED) {
-      Preconditions.checkArgument(length <= Integer.MAX_VALUE || length >= Integer.MIN_VALUE,
-          "Can't cast long to int: %s", length);
+      return Math.toIntExact(length);
     }
     return (int) length;
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/util/AutoCloseables.java b/java/memory/memory-core/src/main/java/org/apache/arrow/util/AutoCloseables.java
similarity index 100%
rename from java/vector/src/main/java/org/apache/arrow/util/AutoCloseables.java
rename to java/memory/memory-core/src/main/java/org/apache/arrow/util/AutoCloseables.java
diff --git a/java/vector/src/main/java/org/apache/arrow/util/Collections2.java b/java/memory/memory-core/src/main/java/org/apache/arrow/util/Collections2.java
similarity index 100%
rename from java/vector/src/main/java/org/apache/arrow/util/Collections2.java
rename to java/memory/memory-core/src/main/java/org/apache/arrow/util/Collections2.java
diff --git a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java
index e4553104715..bfe496532b1 100644
--- a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java
+++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java
@@ -34,7 +34,7 @@ public class DefaultAllocationManagerFactory implements AllocationManager.Factor
       MemoryUtil.UNSAFE.allocateMemory(0));
 
   @Override
-  public AllocationManager create(BaseAllocator accountingAllocator, long size) {
+  public AllocationManager create(BufferAllocator accountingAllocator, long size) {
     return new AllocationManager(accountingAllocator) {
       private final long allocatedSize = size;
       private final long address = MemoryUtil.UNSAFE.allocateMemory(size);
diff --git a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/TestLargeMemoryUtil.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/TestLargeMemoryUtil.java
new file mode 100755
index 00000000000..952fcb5f051
--- /dev/null
+++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/TestLargeMemoryUtil.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.memory.util;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.URLClassLoader;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
+
+public class TestLargeMemoryUtil {
+
+  /**
+   * Get a copy of the current class loader.
+   * @return the newly created class loader.
+   */
+  private ClassLoader copyClassLoader() {
+    ClassLoader curClassLoader = this.getClass().getClassLoader();
+    if (curClassLoader instanceof URLClassLoader) {
+      // for Java 1.8
+      return new URLClassLoader(((URLClassLoader) curClassLoader).getURLs(), null);
+    }
+
+    // for Java 1.9 and Java 11.
+    return null;
+  }
+
+  /**
+   * Use the checkedCastToInt method from the current classloader.
+   * @param classLoader the class loader from which to call the method.
+   * @return the return value of the method.
+   */
+  private int checkedCastToInt(ClassLoader classLoader, long value) throws Exception {
+    Class<?> clazz = classLoader.loadClass("org.apache.arrow.memory.util.LargeMemoryUtil");
+    Method method = clazz.getMethod("checkedCastToInt", long.class);
+    return (int) method.invoke(null, value);
+  }
+
+  private void checkExpectedOverflow(ClassLoader classLoader, long value) {
+    InvocationTargetException ex = Assertions.assertThrows(InvocationTargetException.class, () -> {
+      checkedCastToInt(classLoader, value);
+    });
+    Assert.assertTrue(ex.getCause() instanceof ArithmeticException);
+    Assert.assertEquals("integer overflow", ex.getCause().getMessage());
+  }
+
+  @Test
+  public void testEnableLargeMemoryUtilCheck() throws Exception {
+    String savedNewProperty = System.getProperty("arrow.enable_unsafe_memory_access");
+    System.setProperty("arrow.enable_unsafe_memory_access", "false");
+    try {
+      ClassLoader classLoader = copyClassLoader();
+      if (classLoader != null) {
+        Assert.assertEquals(Integer.MAX_VALUE, checkedCastToInt(classLoader, Integer.MAX_VALUE));
+        checkExpectedOverflow(classLoader, Integer.MAX_VALUE + 1L);
+        checkExpectedOverflow(classLoader, Integer.MIN_VALUE - 1L);
+      }
+    } finally {
+      // restore system property
+      if (savedNewProperty != null) {
+        System.setProperty("arrow.enable_unsafe_memory_access", savedNewProperty);
+      } else {
+        System.clearProperty("arrow.enable_unsafe_memory_access");
+      }
+    }
+  }
+
+  @Test
+  public void testDisabledLargeMemoryUtilCheck() throws Exception {
+    String savedNewProperty = System.getProperty("arrow.enable_unsafe_memory_access");
+    System.setProperty("arrow.enable_unsafe_memory_access", "true");
+    try {
+      ClassLoader classLoader = copyClassLoader();
+      if (classLoader != null) {
+        Assert.assertEquals(Integer.MAX_VALUE, checkedCastToInt(classLoader, Integer.MAX_VALUE));
+        Assert.assertEquals(Integer.MIN_VALUE, checkedCastToInt(classLoader, Integer.MAX_VALUE + 1L));
+        Assert.assertEquals(Integer.MAX_VALUE, checkedCastToInt(classLoader, Integer.MIN_VALUE - 1L));
+      }
+    } finally {
+      // restore system property
+      if (savedNewProperty != null) {
+        System.setProperty("arrow.enable_unsafe_memory_access", savedNewProperty);
+      } else {
+        System.clearProperty("arrow.enable_unsafe_memory_access");
+      }
+    }
+  }
+}
diff --git a/java/memory/memory-netty/pom.xml b/java/memory/memory-netty/pom.xml
index 5c375af5241..673567ca3df 100644
--- a/java/memory/memory-netty/pom.xml
+++ b/java/memory/memory-netty/pom.xml
@@ -13,7 +13,7 @@
   <parent>
     <artifactId>arrow-memory</artifactId>
     <groupId>org.apache.arrow</groupId>
-    <version>2.0.0-SNAPSHOT</version>
+    <version>3.0.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 
diff --git a/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java b/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java
index 15651a38e4a..10cfb5c1648 100644
--- a/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java
+++ b/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java
@@ -26,7 +26,7 @@ public class DefaultAllocationManagerFactory implements AllocationManager.Factor
   public static final AllocationManager.Factory FACTORY = NettyAllocationManager.FACTORY;
 
   @Override
-  public AllocationManager create(BaseAllocator accountingAllocator, long size) {
+  public AllocationManager create(BufferAllocator accountingAllocator, long size) {
     return FACTORY.create(accountingAllocator, size);
   }
 
diff --git a/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java b/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java
index 45bd5d91347..20004778307 100644
--- a/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java
+++ b/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java
@@ -30,7 +30,7 @@ public class NettyAllocationManager extends AllocationManager {
   public static final AllocationManager.Factory FACTORY = new AllocationManager.Factory() {
 
     @Override
-    public AllocationManager create(BaseAllocator accountingAllocator, long size) {
+    public AllocationManager create(BufferAllocator accountingAllocator, long size) {
       return new NettyAllocationManager(accountingAllocator, size);
     }
 
@@ -65,7 +65,7 @@ public ArrowBuf empty() {
    */
   private final int allocationCutOffValue;
 
-  NettyAllocationManager(BaseAllocator accountingAllocator, long requestedSize, int allocationCutOffValue) {
+  NettyAllocationManager(BufferAllocator accountingAllocator, long requestedSize, int allocationCutOffValue) {
     super(accountingAllocator);
     this.allocationCutOffValue = allocationCutOffValue;
 
@@ -80,7 +80,7 @@ public ArrowBuf empty() {
     }
   }
 
-  NettyAllocationManager(BaseAllocator accountingAllocator, long requestedSize) {
+  NettyAllocationManager(BufferAllocator accountingAllocator, long requestedSize) {
     this(accountingAllocator, requestedSize, DEFAULT_ALLOCATION_CUTOFF_VALUE);
   }
 
diff --git a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
index a42e272a42e..ef49e41785f 100644
--- a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
+++ b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
@@ -393,7 +393,7 @@ private BaseAllocator createAllocatorWithCustomizedAllocationManager() {
         .maxAllocation(MAX_ALLOCATION)
         .allocationManagerFactory(new AllocationManager.Factory() {
           @Override
-          public AllocationManager create(BaseAllocator accountingAllocator, long requestedSize) {
+          public AllocationManager create(BufferAllocator accountingAllocator, long requestedSize) {
             return new AllocationManager(accountingAllocator) {
               private final Unsafe unsafe = getUnsafe();
               private final long address = unsafe.allocateMemory(requestedSize);
diff --git a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestEndianness.java b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestEndianness.java
index 6f0da8728ce..dcaeb24889e 100644
--- a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestEndianness.java
+++ b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestEndianness.java
@@ -19,6 +19,8 @@
 
 import static org.junit.Assert.assertEquals;
 
+import java.nio.ByteOrder;
+
 import org.junit.Test;
 
 import io.netty.buffer.ByteBuf;
@@ -27,14 +29,21 @@
 public class TestEndianness {
 
   @Test
-  public void testLittleEndian() {
+  public void testNativeEndian() {
     final BufferAllocator a = new RootAllocator(10000);
     final ByteBuf b = NettyArrowBuf.unwrapBuffer(a.buffer(4));
     b.setInt(0, 35);
-    assertEquals(b.getByte(0), 35);
-    assertEquals(b.getByte(1), 0);
-    assertEquals(b.getByte(2), 0);
-    assertEquals(b.getByte(3), 0);
+    if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) {
+      assertEquals(b.getByte(0), 35);
+      assertEquals(b.getByte(1), 0);
+      assertEquals(b.getByte(2), 0);
+      assertEquals(b.getByte(3), 0);
+    } else {
+      assertEquals(b.getByte(0), 0);
+      assertEquals(b.getByte(1), 0);
+      assertEquals(b.getByte(2), 0);
+      assertEquals(b.getByte(3), 35);
+    }
     b.release();
     a.close();
   }
diff --git a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestNettyAllocationManager.java b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestNettyAllocationManager.java
index f386ea66b2a..1b64cd73363 100644
--- a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestNettyAllocationManager.java
+++ b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestNettyAllocationManager.java
@@ -35,7 +35,7 @@ private BaseAllocator createCustomizedAllocator() {
     return new RootAllocator(BaseAllocator.configBuilder()
         .allocationManagerFactory(new AllocationManager.Factory() {
           @Override
-          public AllocationManager create(BaseAllocator accountingAllocator, long size) {
+          public AllocationManager create(BufferAllocator accountingAllocator, long size) {
             return new NettyAllocationManager(accountingAllocator, size, CUSTOMIZED_ALLOCATION_CUTOFF_VALUE);
           }
 
diff --git a/java/memory/memory-unsafe/pom.xml b/java/memory/memory-unsafe/pom.xml
index 5f90c9a150b..1c94ddf5766 100644
--- a/java/memory/memory-unsafe/pom.xml
+++ b/java/memory/memory-unsafe/pom.xml
@@ -13,7 +13,7 @@
   <parent>
     <artifactId>arrow-memory</artifactId>
     <groupId>org.apache.arrow</groupId>
-    <version>2.0.0-SNAPSHOT</version>
+    <version>3.0.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 
diff --git a/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java b/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java
index 3963c1875d0..720c3d02d23 100644
--- a/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java
+++ b/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java
@@ -26,7 +26,7 @@ public class DefaultAllocationManagerFactory implements AllocationManager.Factor
   public static final AllocationManager.Factory FACTORY = UnsafeAllocationManager.FACTORY;
 
   @Override
-  public AllocationManager create(BaseAllocator accountingAllocator, long size) {
+  public AllocationManager create(BufferAllocator accountingAllocator, long size) {
     return FACTORY.create(accountingAllocator, size);
   }
 
diff --git a/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java b/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java
index f9756539c55..b10aba3598d 100644
--- a/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java
+++ b/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java
@@ -32,7 +32,7 @@ public final class UnsafeAllocationManager extends AllocationManager {
 
   public static final AllocationManager.Factory FACTORY = new Factory() {
     @Override
-    public AllocationManager create(BaseAllocator accountingAllocator, long size) {
+    public AllocationManager create(BufferAllocator accountingAllocator, long size) {
       return new UnsafeAllocationManager(accountingAllocator, size);
     }
 
@@ -46,7 +46,7 @@ public ArrowBuf empty() {
 
   private final long allocatedAddress;
 
-  UnsafeAllocationManager(BaseAllocator accountingAllocator, long requestedSize) {
+  UnsafeAllocationManager(BufferAllocator accountingAllocator, long requestedSize) {
     super(accountingAllocator);
     allocatedAddress = MemoryUtil.UNSAFE.allocateMemory(requestedSize);
     allocatedSize = requestedSize;
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 93b2af44923..f8db2d38629 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>2.0.0-SNAPSHOT</version>
+    <version>3.0.0-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/performance/pom.xml b/java/performance/pom.xml
index 42b0b35f58d..695890b6fe6 100644
--- a/java/performance/pom.xml
+++ b/java/performance/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <artifactId>arrow-java-root</artifactId>
         <groupId>org.apache.arrow</groupId>
-        <version>2.0.0-SNAPSHOT</version>
+        <version>3.0.0-SNAPSHOT</version>
     </parent>
     <artifactId>arrow-performance</artifactId>
     <packaging>jar</packaging>
@@ -86,7 +86,7 @@
         <dependency>
             <groupId>org.apache.arrow</groupId>
             <artifactId>arrow-algorithm</artifactId>
-            <version>2.0.0-SNAPSHOT</version>
+            <version>3.0.0-SNAPSHOT</version>
             <scope>test</scope>
         </dependency>
     </dependencies>
@@ -96,6 +96,11 @@
         <jmh.version>1.21</jmh.version>
         <javac.target>1.8</javac.target>
         <uberjar.name>benchmarks</uberjar.name>
+        <skip.perf.benchmarks>true</skip.perf.benchmarks>
+        <benchmark.filter>.*</benchmark.filter>
+        <benchmark.forks>1</benchmark.forks>
+        <benchmark.warmups>5</benchmark.warmups>
+        <benchmark.runs>5</benchmark.runs>
     </properties>
 
     <build>
@@ -140,6 +145,37 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>exec-maven-plugin</artifactId>
+                <version>1.6.0</version>
+                <executions>
+                    <execution>
+                        <id>run-java-benchmarks</id>
+                        <phase>integration-test</phase>
+                        <goals>
+                            <goal>exec</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <skip>${skip.perf.benchmarks}</skip>
+                    <classpathScope>test</classpathScope>
+                    <executable>java</executable>
+                    <arguments>
+                        <argument>-classpath</argument>
+                        <classpath />
+                        <argument>org.openjdk.jmh.Main</argument>
+                        <argument>${benchmark.filter}</argument>
+                        <argument>-f</argument>
+                        <argument>${benchmark.forks}</argument>
+                        <argument>-wi</argument>
+                        <argument>${benchmark.warmups}</argument>
+                        <argument>-i</argument>
+                        <argument>${benchmark.runs}</argument>
+                    </arguments>
+                </configuration>
+            </plugin>
         </plugins>
         <pluginManagement>
             <plugins>
diff --git a/java/plasma/pom.xml b/java/plasma/pom.xml
index aab9e5163fc..570b7a4ca15 100644
--- a/java/plasma/pom.xml
+++ b/java/plasma/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>2.0.0-SNAPSHOT</version>
+        <version>3.0.0-SNAPSHOT</version>
     </parent>
     <artifactId>arrow-plasma</artifactId>
     <name>Arrow Plasma Client</name>
diff --git a/java/pom.xml b/java/pom.xml
index 192e548d7b5..81b5b0613ab 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>2.0.0-SNAPSHOT</version>
+  <version>3.0.0-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -38,7 +38,7 @@
     <dep.hadoop.version>2.7.1</dep.hadoop.version>
     <dep.fbs.version>1.9.0</dep.fbs.version>
     <dep.flatc.version>1.9.0</dep.flatc.version>
-    <dep.avro.version>1.9.2</dep.avro.version>
+    <dep.avro.version>1.10.0</dep.avro.version>
     <arrow.vector.classifier />
     <forkCount>2</forkCount>
     <checkstyle.failOnViolation>true</checkstyle.failOnViolation>
@@ -49,7 +49,7 @@
     <connection>scm:git:https://github.com/apache/arrow.git</connection>
     <developerConnection>scm:git:https://github.com/apache/arrow.git</developerConnection>
     <url>https://github.com/apache/arrow</url>
-    <tag>apache-arrow-0.10.0</tag>
+    <tag>apache-arrow-2.0.0</tag>
   </scm>
 
   <mailingLists>
@@ -370,16 +370,7 @@
           <artifactId>maven-compiler-plugin</artifactId>
           <version>3.6.2</version>
           <configuration>
-            <compilerArgs>
-              <arg>-XDcompilePolicy=simple</arg>
-              <arg>-Xplugin:ErrorProne</arg>
-            </compilerArgs>
             <annotationProcessorPaths>
-              <path>
-                <groupId>com.google.errorprone</groupId>
-                <artifactId>error_prone_core</artifactId>
-                <version>2.4.0</version>
-              </path>
               <path>
                 <groupId>org.immutables</groupId>
                 <artifactId>value</artifactId>
@@ -717,11 +708,49 @@
       </properties>
     </profile>
 
-    <!-- using github.com/google/error-prone-javac is required when running on JDK 8 -->
     <profile>
-      <id>jdk8</id>
+      <id>error-prone</id>
+      <!-- 
+           Do not activate Error Prone while running with Eclipse/M2E as it causes incompatibilities
+           with other annotation processors.
+           See https://github.com/jbosstools/m2e-apt/issues/62 for details
+      -->
+      <activation>
+        <property>
+           <name>!m2e.version</name>
+        </property>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-compiler-plugin</artifactId>
+            <configuration>
+              <compilerArgs combine.children="append">
+                <arg>-XDcompilePolicy=simple</arg>
+                <arg>-Xplugin:ErrorProne</arg>
+              </compilerArgs>
+              <annotationProcessorPaths combine.children="append">
+                <path>
+                  <groupId>com.google.errorprone</groupId>
+                  <artifactId>error_prone_core</artifactId>
+                  <version>2.4.0</version>
+                </path>
+              </annotationProcessorPaths>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
+    <profile>
+      <id>error-prone-jdk8</id>
+      <!-- using github.com/google/error-prone-javac is required when running on JDK 8 -->
       <activation>
         <jdk>1.8</jdk>
+        <property>
+           <name>!m2e.version</name>
+        </property>
       </activation>
       <build>
         <plugins>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index f48c510d9da..a3a88bf26c1 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>2.0.0-SNAPSHOT</version>
+        <version>3.0.0-SNAPSHOT</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 03234d70c37..3c74609f277 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>2.0.0-SNAPSHOT</version>
+    <version>3.0.0-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>
diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd
index 4d2a540f572..3cf9a968791 100644
--- a/java/vector/src/main/codegen/data/ArrowTypes.tdd
+++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd
@@ -92,7 +92,7 @@
     },
     {
       name: "Decimal",
-      fields: [{name: "precision", type: int}, {name: "scale", type: int}],
+      fields: [{name: "precision", type: int}, {name: "scale", type: int}, {name: "bitWidth", type: int}],
       complex: false
     },
     {
diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
index b9e052941ed..574b065662e 100644
--- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
+++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
@@ -113,6 +113,22 @@
         { class: "IntervalDay", millisecondsOffset: 4, friendlyType: "Duration", fields: [ {name: "days", type:"int"}, {name: "milliseconds", type:"int"}] }
       ]
     },
+    {
+      major: "Fixed",
+      width: 32,
+      javaType: "ArrowBuf",
+      boxedType: "ArrowBuf",
+
+      minor: [
+        {
+          class: "Decimal256",
+          maxPrecisionDigits: 76, nDecimalDigits: 4, friendlyType: "BigDecimal",
+          typeParams: [ {name: "scale", type: "int"}, { name: "precision", type: "int"}],
+          arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Decimal",
+          fields: [{name: "start", type: "long"}, {name: "buffer", type: "ArrowBuf"}]
+        }
+      ]
+    },
     {
       major: "Fixed",
       width: 16,
@@ -125,10 +141,11 @@
           maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal",
           typeParams: [ {name: "scale", type: "int"}, { name: "precision", type: "int"}],
           arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Decimal",
-          fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}]
+          fields: [{name: "start", type: "long"}, {name: "buffer", type: "ArrowBuf"}]
         }
       ]
     },
+
     {
       major: "Fixed",
       width: -1,
diff --git a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
index 4f6d5ea1aee..bce842d5911 100644
--- a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
@@ -76,20 +76,20 @@ public void write(${name}Holder holder) {
     fail("${name}");
   }
 
-  <#if minor.class == "Decimal">
+  <#if minor.class?starts_with("Decimal")>
   public void write${minor.class}(${friendlyType} value) {
     fail("${name}");
   }
 
-  public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list><#if minor.class == "Decimal">, ArrowType arrowType</#if>) {
+  public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>, ArrowType arrowType) {
     fail("${name}");
   }
 
-  public void writeBigEndianBytesToDecimal(byte[] value) {
+  public void writeBigEndianBytesTo${minor.class}(byte[] value) {
     fail("${name}");
   }
 
-  public void writeBigEndianBytesToDecimal(byte[] value, ArrowType arrowType) {
+  public void writeBigEndianBytesTo${minor.class}(byte[] value, ArrowType arrowType) {
     fail("${name}");
   }
   </#if>
diff --git a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
index 5566c808258..6b14dbf2a57 100644
--- a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
@@ -75,7 +75,7 @@ public void endList() {
 
   <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
     <#assign fields = minor.fields!type.fields />
-  <#if minor.class != "Decimal">
+  <#if minor.class != "Decimal" && minor.class != "Decimal256">
   @Override
   public void write(${name}Holder holder) {
     getWriter(MinorType.${name?upper_case}).write(holder);
@@ -85,7 +85,7 @@ public void write(${name}Holder holder) {
     getWriter(MinorType.${name?upper_case}).write${minor.class}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
   }
 
-  <#else>
+  <#elseif minor.class == "Decimal">
   @Override
   public void write(DecimalHolder holder) {
     getWriter(MinorType.DECIMAL).write(holder);
@@ -106,6 +106,28 @@ public void writeBigEndianBytesToDecimal(byte[] value, ArrowType arrowType) {
   public void writeBigEndianBytesToDecimal(byte[] value) {
     getWriter(MinorType.DECIMAL).writeBigEndianBytesToDecimal(value);
   }
+  <#elseif minor.class == "Decimal256">
+  @Override
+  public void write(Decimal256Holder holder) {
+    getWriter(MinorType.DECIMAL256).write(holder);
+  }
+
+  public void writeDecimal256(long start, ArrowBuf buffer, ArrowType arrowType) {
+    getWriter(MinorType.DECIMAL256).writeDecimal256(start, buffer, arrowType);
+  }
+
+  public void writeDecimal256(long start, ArrowBuf buffer) {
+    getWriter(MinorType.DECIMAL256).writeDecimal256(start, buffer);
+  }
+  public void writeBigEndianBytesToDecimal256(byte[] value, ArrowType arrowType) {
+    getWriter(MinorType.DECIMAL256).writeBigEndianBytesToDecimal256(value, arrowType);
+  }
+
+  public void writeBigEndianBytesToDecimal256(byte[] value) {
+    getWriter(MinorType.DECIMAL256).writeBigEndianBytesToDecimal256(value);
+  }
+
+
   </#if>
 
   </#list></#list>
diff --git a/java/vector/src/main/codegen/templates/ArrowType.java b/java/vector/src/main/codegen/templates/ArrowType.java
index 77894af2365..b08d4ad0afa 100644
--- a/java/vector/src/main/codegen/templates/ArrowType.java
+++ b/java/vector/src/main/codegen/templates/ArrowType.java
@@ -165,7 +165,34 @@ public static class ${name} extends <#if type.complex>ComplexType<#else>Primitiv
     ${fieldType} ${field.name};
     </#list>
 
+
+    <#if type.name == "Decimal">
+    // Needed to support golden file integration tests.
     @JsonCreator
+    public static Decimal createDecimal(
+      @JsonProperty("precision") int precision,
+      @JsonProperty("scale") int scale,
+      @JsonProperty("bitWidth") Integer bitWidth) {
+
+      return new Decimal(precision, scale, bitWidth == null ? 128 : bitWidth);
+    }
+
+    /**
+     * Construct Decimal with 128 bits.
+     * 
+     * This is kept mainly for the sake of backward compatibility.
+     * Please use {@link org.apache.arrow.vector.types.pojo.ArrowType.Decimal#Decimal(int, int, int)} instead.
+     *
+     * @deprecated This API will be removed in a future release.
+     */
+    @Deprecated
+    public Decimal(int precision, int scale) {
+      this(precision, scale, 128);
+    }
+
+    <#else>
+    @JsonCreator
+    </#if>
     public ${type.name}(
     <#list type.fields as field>
     <#assign fieldType = field.valueType!field.type>
@@ -327,9 +354,8 @@ public static org.apache.arrow.vector.types.pojo.ArrowType getTypeForField(org.a
       </#if>
       </#list>
       <#if type.name == "Decimal">
-      int bitWidth = ${nameLower}Type.bitWidth();
-      if (bitWidth != defaultDecimalBitWidth) {
-        throw new IllegalArgumentException("Library only supports 128-bit decimal values");
+      if (bitWidth != defaultDecimalBitWidth && bitWidth != 256) {
+        throw new IllegalArgumentException("Library only supports 128-bit and 256-bit decimal values");
       }
       </#if>
       return new ${name}(<#list type.fields as field><#if field.valueType??>${field.valueType}.fromFlatbufID(${field.name})<#else>${field.name}</#if><#if field_has_next>, </#if></#list>);
diff --git a/java/vector/src/main/codegen/templates/ComplexCopier.java b/java/vector/src/main/codegen/templates/ComplexCopier.java
index 1189e8e04e0..39a84041e7e 100644
--- a/java/vector/src/main/codegen/templates/ComplexCopier.java
+++ b/java/vector/src/main/codegen/templates/ComplexCopier.java
@@ -124,7 +124,7 @@ private static void writeValue(FieldReader reader, FieldWriter writer) {
           Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder();
           reader.read(${uncappedName}Holder);
           if (${uncappedName}Holder.isSet == 1) {
-            writer.write${name}(<#list fields as field>${uncappedName}Holder.${field.name}<#if field_has_next>, </#if></#list><#if minor.class == "Decimal">, new ArrowType.Decimal(decimalHolder.precision, decimalHolder.scale)</#if>);
+            writer.write${name}(<#list fields as field>${uncappedName}Holder.${field.name}<#if field_has_next>, </#if></#list><#if minor.class?starts_with("Decimal")>, new ArrowType.Decimal(${uncappedName}Holder.precision, ${uncappedName}Holder.scale, ${name}Holder.WIDTH * 8)</#if>);
           }
         } else {
           writer.writeNull();
@@ -145,7 +145,7 @@ private static FieldWriter getStructWriterForReader(FieldReader reader, StructWr
     case ${name?upper_case}:
       return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}</#if>(name);
     </#if>
-    <#if minor.class == "Decimal">
+    <#if minor.class?starts_with("Decimal")>
     case ${name?upper_case}:
       if (reader.getField().getType() instanceof ArrowType.Decimal) {
         ArrowType.Decimal type = (ArrowType.Decimal) reader.getField().getType();
@@ -154,6 +154,7 @@ private static FieldWriter getStructWriterForReader(FieldReader reader, StructWr
         return (FieldWriter) writer.${uncappedName}(name);
       }
     </#if>
+    
     </#list></#list>
     case STRUCT:
       return (FieldWriter) writer.struct(name);
diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java
index ab99ac38dcd..0381e5559e4 100644
--- a/java/vector/src/main/codegen/templates/ComplexWriters.java
+++ b/java/vector/src/main/codegen/templates/ComplexWriters.java
@@ -99,7 +99,7 @@ public void setPosition(int idx) {
 
   <#else>
 
-  <#if minor.class != "Decimal">
+  <#if !minor.class?starts_with("Decimal")>
   public void write(${minor.class}Holder h) {
     vector.setSafe(idx(), h);
     vector.setValueCount(idx()+1);
@@ -123,15 +123,15 @@ public void write(Nullable${minor.class}Holder h) {
   }
   </#if>
 
-  <#if minor.class == "Decimal">
+  <#if minor.class?starts_with("Decimal")>
 
-  public void write(DecimalHolder h){
+  public void write(${minor.class}Holder h){
     DecimalUtility.checkPrecisionAndScale(h.precision, h.scale, vector.getPrecision(), vector.getScale());
     vector.setSafe(idx(), h);
     vector.setValueCount(idx() + 1);
   }
 
-  public void write(NullableDecimalHolder h){
+  public void write(Nullable${minor.class}Holder h){
     if (h.isSet == 1) {
       DecimalUtility.checkPrecisionAndScale(h.precision, h.scale, vector.getPrecision(), vector.getScale());
     }
@@ -139,37 +139,38 @@ public void write(NullableDecimalHolder h){
     vector.setValueCount(idx() + 1);
   }
 
-  public void writeDecimal(int start, ArrowBuf buffer){
+  public void write${minor.class}(long start, ArrowBuf buffer){
     vector.setSafe(idx(), 1, start, buffer);
     vector.setValueCount(idx() + 1);
   }
 
-  public void writeDecimal(int start, ArrowBuf buffer, ArrowType arrowType){
+  public void write${minor.class}(long start, ArrowBuf buffer, ArrowType arrowType){
     DecimalUtility.checkPrecisionAndScale(((ArrowType.Decimal) arrowType).getPrecision(),
       ((ArrowType.Decimal) arrowType).getScale(), vector.getPrecision(), vector.getScale());
     vector.setSafe(idx(), 1, start, buffer);
     vector.setValueCount(idx() + 1);
   }
 
-  public void writeDecimal(BigDecimal value){
+  public void write${minor.class}(BigDecimal value){
     // vector.setSafe already does precision and scale checking
     vector.setSafe(idx(), value);
     vector.setValueCount(idx() + 1);
   }
 
-  public void writeBigEndianBytesToDecimal(byte[] value, ArrowType arrowType){
+  public void writeBigEndianBytesTo${minor.class}(byte[] value, ArrowType arrowType){
     DecimalUtility.checkPrecisionAndScale(((ArrowType.Decimal) arrowType).getPrecision(),
         ((ArrowType.Decimal) arrowType).getScale(), vector.getPrecision(), vector.getScale());
     vector.setBigEndianSafe(idx(), value);
     vector.setValueCount(idx() + 1);
   }
 
-  public void writeBigEndianBytesToDecimal(byte[] value){
+  public void writeBigEndianBytesTo${minor.class}(byte[] value){
     vector.setBigEndianSafe(idx(), value);
     vector.setValueCount(idx() + 1);
   }
   </#if>
 
+  
   public void writeNull() {
     vector.setNull(idx());
     vector.setValueCount(idx()+1);
@@ -190,18 +191,18 @@ public void writeNull() {
 public interface ${eName}Writer extends BaseWriter {
   public void write(${minor.class}Holder h);
 
-  <#if minor.class == "Decimal">@Deprecated</#if>
+  <#if minor.class?starts_with("Decimal")>@Deprecated</#if>
   public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>);
-<#if minor.class == "Decimal">
+<#if minor.class?starts_with("Decimal")>
 
   public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>, ArrowType arrowType);
 
   public void write${minor.class}(${friendlyType} value);
 
-  public void writeBigEndianBytesToDecimal(byte[] value, ArrowType arrowType);
+  public void writeBigEndianBytesTo${minor.class}(byte[] value, ArrowType arrowType);
 
   @Deprecated
-  public void writeBigEndianBytesToDecimal(byte[] value);
+  public void writeBigEndianBytesTo${minor.class}(byte[] value);
 </#if>
 }
 
diff --git a/java/vector/src/main/codegen/templates/DenseUnionReader.java b/java/vector/src/main/codegen/templates/DenseUnionReader.java
index 51bd7d172de..f7e161ac86f 100644
--- a/java/vector/src/main/codegen/templates/DenseUnionReader.java
+++ b/java/vector/src/main/codegen/templates/DenseUnionReader.java
@@ -92,7 +92,7 @@ private FieldReader getReaderForIndex(int index) {
       <#list type.minor as minor>
         <#assign name = minor.class?cap_first />
         <#assign uncappedName = name?uncap_first/>
-        <#if !minor.typeParams?? || minor.class == "Decimal">
+        <#if !minor.typeParams?? || minor.class?starts_with("Decimal")>
       case ${name?upper_case}:
       reader = (FieldReader) get${name}(typeId);
       break;
@@ -165,7 +165,7 @@ public int size() {
       <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
       <#assign safeType=friendlyType />
       <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
-      <#if !minor.typeParams?? || minor.class == "Decimal">
+      <#if !minor.typeParams?? || minor.class?starts_with("Decimal")>
 
   private ${name}ReaderImpl get${name}(byte typeId) {
     ${name}ReaderImpl reader = (${name}ReaderImpl) readers[typeId];
diff --git a/java/vector/src/main/codegen/templates/DenseUnionVector.java b/java/vector/src/main/codegen/templates/DenseUnionVector.java
index e67445f1bb1..de4d60d553b 100644
--- a/java/vector/src/main/codegen/templates/DenseUnionVector.java
+++ b/java/vector/src/main/codegen/templates/DenseUnionVector.java
@@ -19,7 +19,6 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.ReferenceManager;
 import org.apache.arrow.memory.util.CommonUtil;
-import org.apache.arrow.util.DataSizeRoundingUtil;
 import org.apache.arrow.util.Preconditions;
 import org.apache.arrow.vector.BaseValueVector;
 import org.apache.arrow.vector.BitVectorHelper;
@@ -37,6 +36,7 @@
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.DataSizeRoundingUtil;
 import org.apache.arrow.vector.util.TransferPair;
 
 import java.util.Arrays;
@@ -63,7 +63,6 @@
 import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.BaseValueVector;
 import org.apache.arrow.vector.util.OversizedAllocationException;
-import org.apache.arrow.util.DataSizeRoundingUtil;
 import org.apache.arrow.util.Preconditions;
 
 import static org.apache.arrow.vector.types.UnionMode.Dense;
@@ -122,7 +121,7 @@ public class DenseUnionVector implements FieldVector {
 
   private final FieldType fieldType;
 
-  private static final byte TYPE_WIDTH = 1;
+  public static final byte TYPE_WIDTH = 1;
   public static final byte OFFSET_WIDTH = 4;
 
   private static final FieldType INTERNAL_STRUCT_TYPE = new FieldType(/*nullable*/ false,
@@ -306,13 +305,13 @@ public StructVector getStruct(byte typeId) {
       <#assign fields = minor.fields!type.fields />
       <#assign uncappedName = name?uncap_first/>
       <#assign lowerCaseName = name?lower_case/>
-      <#if !minor.typeParams?? || minor.class == "Decimal">
+      <#if !minor.typeParams?? || minor.class?starts_with("Decimal")>
 
-  public ${name}Vector get${name}Vector(byte typeId<#if minor.class == "Decimal">, ArrowType arrowType</#if>) {
+  public ${name}Vector get${name}Vector(byte typeId<#if minor.class?starts_with("Decimal")>, ArrowType arrowType</#if>) {
     ValueVector vector = typeId < 0 ? null : childVectors[typeId];
     if (vector == null) {
       int vectorCount = internalStruct.size();
-      vector = addOrGet(typeId, MinorType.${name?upper_case}<#if minor.class == "Decimal">, arrowType</#if>, ${name}Vector.class);
+      vector = addOrGet(typeId, MinorType.${name?upper_case}<#if minor.class?starts_with("Decimal")>, arrowType</#if>, ${name}Vector.class);
       childVectors[typeId] = vector;
       if (internalStruct.size() > vectorCount) {
         vector.allocateNew();
@@ -810,7 +809,7 @@ public void setSafe(int index, DenseUnionHolder holder) {
           <#assign name = minor.class?cap_first />
           <#assign fields = minor.fields!type.fields />
           <#assign uncappedName = name?uncap_first/>
-          <#if !minor.typeParams?? || minor.class == "Decimal">
+          <#if !minor.typeParams?? || minor.class?starts_with("Decimal")>
       case ${name?upper_case}:
       Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder();
       reader.read(${uncappedName}Holder);
@@ -834,13 +833,13 @@ public void setSafe(int index, DenseUnionHolder holder) {
         <#assign name = minor.class?cap_first />
         <#assign fields = minor.fields!type.fields />
         <#assign uncappedName = name?uncap_first/>
-        <#if !minor.typeParams?? || minor.class == "Decimal">
+        <#if !minor.typeParams?? || minor.class?starts_with("Decimal")>
   public void setSafe(int index, Nullable${name}Holder holder) {
     while (index >= getOffsetBufferValueCapacity()) {
       reallocOffsetBuffer();
     }
     byte typeId = getTypeId(index);
-    ${name}Vector vector = get${name}Vector(typeId<#if minor.class == "Decimal">, new ArrowType.Decimal(holder.precision, holder.scale)</#if>);
+    ${name}Vector vector = get${name}Vector(typeId<#if minor.class?starts_with("Decimal")>, new ArrowType.Decimal(holder.precision, holder.scale, holder.WIDTH * 8)</#if>);
     int offset = vector.getValueCount();
     vector.setValueCount(offset + 1);
     vector.setSafe(offset, holder);
diff --git a/java/vector/src/main/codegen/templates/DenseUnionWriter.java b/java/vector/src/main/codegen/templates/DenseUnionWriter.java
index ee6f614c8f8..769b84268af 100644
--- a/java/vector/src/main/codegen/templates/DenseUnionWriter.java
+++ b/java/vector/src/main/codegen/templates/DenseUnionWriter.java
@@ -123,7 +123,7 @@ BaseWriter getWriter(byte typeId) {
         <#assign name = minor.class?cap_first />
         <#assign fields = minor.fields!type.fields />
         <#assign uncappedName = name?uncap_first/>
-        <#if !minor.typeParams?? || minor.class == "Decimal">
+        <#if !minor.typeParams?? || minor.class?starts_with("Decimal")>
       case ${name?upper_case}:
       return get${name}Writer(typeId);
         </#if>
@@ -138,7 +138,7 @@ BaseWriter getWriter(byte typeId) {
       <#assign name = minor.class?cap_first />
       <#assign fields = minor.fields!type.fields />
       <#assign uncappedName = name?uncap_first/>
-      <#if !minor.typeParams?? || minor.class == "Decimal">
+      <#if !minor.typeParams?? || minor.class?starts_with("Decimal")>
 
   private ${name}Writer get${name}Writer(byte typeId) {
     ${name}Writer writer = (${name}Writer) writers[typeId];
@@ -159,10 +159,10 @@ public void write(${name}Holder holder) {
     throw new UnsupportedOperationException();
   }
 
-  public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>, byte typeId<#if minor.class == "Decimal">, ArrowType arrowType</#if>) {
+  public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>, byte typeId<#if minor.class?starts_with("Decimal")>, ArrowType arrowType</#if>) {
     data.setTypeId(idx(), typeId);
     get${name}Writer(typeId).setPosition(data.getOffset(idx()));
-    get${name}Writer(typeId).write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list><#if minor.class == "Decimal">, arrowType</#if>);
+    get${name}Writer(typeId).write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list><#if minor.class?starts_with("Decimal")>, arrowType</#if>);
   }
       </#if>
     </#list>
@@ -208,7 +208,7 @@ public StructWriter struct(String name) {
   <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
   <#assign upperName = minor.class?upper_case />
   <#assign capName = minor.class?cap_first />
-  <#if !minor.typeParams?? || minor.class == "Decimal" >
+  <#if !minor.typeParams?? || minor.class?starts_with("Decimal") >
   @Override
   public ${capName}Writer ${lowerName}(String name) {
     byte typeId = data.getTypeId(idx());
@@ -225,7 +225,7 @@ public StructWriter struct(String name) {
     return getListWriter(typeId).${lowerName}();
   }
   </#if>
-  <#if minor.class == "Decimal">
+  <#if minor.class?starts_with("Decimal")>
   public ${capName}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) {
     byte typeId = data.getTypeId(idx());
     data.setTypeId(idx(), typeId);
diff --git a/java/vector/src/main/codegen/templates/HolderReaderImpl.java b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
index fa7e83a9f8b..e41c7db2f2a 100644
--- a/java/vector/src/main/codegen/templates/HolderReaderImpl.java
+++ b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
@@ -129,6 +129,11 @@ public void read(Nullable${name}Holder h) {
     holder.buffer.getBytes(holder.start, bytes, 0, ${type.width});
     ${friendlyType} value = new BigDecimal(new BigInteger(bytes), holder.scale);
     return value;
+  <#elseif minor.class == "Decimal256">
+    byte[] bytes = new byte[${type.width}];
+    holder.buffer.getBytes(holder.start, bytes, 0, ${type.width});
+    ${friendlyType} value = new BigDecimal(new BigInteger(bytes), holder.scale);
+    return value;
   <#elseif minor.class == "FixedSizeBinary">
     byte[] value = new byte [holder.byteWidth];
     holder.buffer.getBytes(0, value, 0, holder.byteWidth);
diff --git a/java/vector/src/main/codegen/templates/StructWriters.java b/java/vector/src/main/codegen/templates/StructWriters.java
index 7df22179eec..b908d1058fb 100644
--- a/java/vector/src/main/codegen/templates/StructWriters.java
+++ b/java/vector/src/main/codegen/templates/StructWriters.java
@@ -255,7 +255,7 @@ public void end() {
                 <#assign constructorParams = constructorParams + [ typeParam.name ] />
               </#list>
             </#if>    
-            new ${minor.arrowType}(${constructorParams?join(", ")})
+            new ${minor.arrowType}(${constructorParams?join(", ")}<#if minor.class?starts_with("Decimal")>, ${vectName}Vector.TYPE_WIDTH * 8</#if>)
           <#else>
             MinorType.${upperName}.getType()
           </#if>
@@ -274,7 +274,7 @@ public void end() {
     } else {
       if (writer instanceof PromotableWriter) {
         // ensure writers are initialized
-        ((PromotableWriter)writer).getWriter(MinorType.${upperName}<#if minor.class == "Decimal">, new ${minor.arrowType}(precision, scale)</#if>);
+        ((PromotableWriter)writer).getWriter(MinorType.${upperName}<#if minor.class?starts_with("Decimal")>, new ${minor.arrowType}(precision, scale, ${vectName}Vector.TYPE_WIDTH * 8)</#if>);
       }
     }
     return writer;
diff --git a/java/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java b/java/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java
index 0574dcf572d..f04b4db3208 100644
--- a/java/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java
@@ -16,9 +16,12 @@
  */
 
 import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.vector.complex.writer.Decimal256Writer;
 import org.apache.arrow.vector.complex.writer.DecimalWriter;
+import org.apache.arrow.vector.holders.Decimal256Holder;
 import org.apache.arrow.vector.holders.DecimalHolder;
 
+
 import java.lang.UnsupportedOperationException;
 import java.math.BigDecimal;
 
@@ -127,6 +130,22 @@ public DecimalWriter decimal(String name) {
     return writer.decimal(name);
   }
 
+
+  @Override
+  public Decimal256Writer decimal256() {
+    return this;
+  }
+
+  @Override
+  public Decimal256Writer decimal256(String name, int scale, int precision) {
+    return writer.decimal256(name, scale, precision);
+  }
+
+  @Override
+  public Decimal256Writer decimal256(String name) {
+    return writer.decimal256(name);
+  }
+
   @Override
   public StructWriter struct() {
     inStruct = true;
@@ -180,6 +199,16 @@ public void write(DecimalHolder holder) {
     writer.write(holder);
     writer.setPosition(writer.idx() + 1);
   }
+ 
+  @Override
+  public void write(Decimal256Holder holder) {
+    if (writer.idx() >= (idx() + 1) * listSize) {
+      throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize));
+    }
+    writer.write(holder);
+    writer.setPosition(writer.idx() + 1);
+  }
+
 
   @Override
   public void writeNull() {
@@ -189,7 +218,7 @@ public void writeNull() {
     writer.writeNull();
   }
 
-  public void writeDecimal(int start, ArrowBuf buffer, ArrowType arrowType) {
+  public void writeDecimal(long start, ArrowBuf buffer, ArrowType arrowType) {
     if (writer.idx() >= (idx() + 1) * listSize) {
       throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize));
     }
@@ -213,6 +242,31 @@ public void writeBigEndianBytesToDecimal(byte[] value, ArrowType arrowType) {
     writer.setPosition(writer.idx() + 1);
   }
 
+  public void writeDecimal256(long start, ArrowBuf buffer, ArrowType arrowType) {
+    if (writer.idx() >= (idx() + 1) * listSize) {
+      throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize));
+    }
+    writer.writeDecimal256(start, buffer, arrowType);
+    writer.setPosition(writer.idx() + 1);
+  }
+
+  public void writeDecimal256(BigDecimal value) {
+    if (writer.idx() >= (idx() + 1) * listSize) {
+      throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize));
+    }
+    writer.writeDecimal256(value);
+    writer.setPosition(writer.idx() + 1);
+  }
+
+  public void writeBigEndianBytesToDecimal256(byte[] value, ArrowType arrowType) {
+    if (writer.idx() >= (idx() + 1) * listSize) {
+      throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize));
+    }
+    writer.writeBigEndianBytesToDecimal256(value, arrowType);
+    writer.setPosition(writer.idx() + 1);
+  }
+
+
   <#list vv.types as type>
     <#list type.minor as minor>
       <#assign name = minor.class?cap_first />
diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java
index a2664436acc..155895d8932 100644
--- a/java/vector/src/main/codegen/templates/UnionListWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionListWriter.java
@@ -16,9 +16,12 @@
  */
 
 import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.vector.complex.writer.Decimal256Writer;
 import org.apache.arrow.vector.complex.writer.DecimalWriter;
+import org.apache.arrow.vector.holders.Decimal256Holder;
 import org.apache.arrow.vector.holders.DecimalHolder;
 
+
 import java.lang.UnsupportedOperationException;
 import java.math.BigDecimal;
 
@@ -133,6 +136,22 @@ public DecimalWriter decimal(String name) {
     return writer.decimal(name);
   }
 
+  @Override
+  public Decimal256Writer decimal256() {
+    return this;
+  }
+
+  @Override
+  public Decimal256Writer decimal256(String name, int scale, int precision) {
+    return writer.decimal256(name, scale, precision);
+  }
+
+  @Override
+  public Decimal256Writer decimal256(String name) {
+    return writer.decimal256(name);
+  }
+
+
   @Override
   public StructWriter struct() {
     inStruct = true;
@@ -199,17 +218,23 @@ public void write(DecimalHolder holder) {
     writer.setPosition(writer.idx()+1);
   }
 
+  @Override
+  public void write(Decimal256Holder holder) {
+    writer.write(holder);
+    writer.setPosition(writer.idx()+1);
+  }
+
   @Override
   public void writeNull() {
     writer.writeNull();
   }
 
-  public void writeDecimal(int start, ArrowBuf buffer, ArrowType arrowType) {
+  public void writeDecimal(long start, ArrowBuf buffer, ArrowType arrowType) {
     writer.writeDecimal(start, buffer, arrowType);
     writer.setPosition(writer.idx()+1);
   }
 
-  public void writeDecimal(int start, ArrowBuf buffer) {
+  public void writeDecimal(long start, ArrowBuf buffer) {
     writer.writeDecimal(start, buffer);
     writer.setPosition(writer.idx()+1);
   }
@@ -224,6 +249,27 @@ public void writeBigEndianBytesToDecimal(byte[] value, ArrowType arrowType){
     writer.setPosition(writer.idx() + 1);
   }
 
+  public void writeDecimal256(long start, ArrowBuf buffer, ArrowType arrowType) {
+    writer.writeDecimal256(start, buffer, arrowType);
+    writer.setPosition(writer.idx()+1);
+  }
+
+  public void writeDecimal256(long start, ArrowBuf buffer) {
+    writer.writeDecimal256(start, buffer);
+    writer.setPosition(writer.idx()+1);
+  }
+
+  public void writeDecimal256(BigDecimal value) {
+    writer.writeDecimal256(value);
+    writer.setPosition(writer.idx()+1);
+  }
+
+  public void writeBigEndianBytesToDecimal256(byte[] value, ArrowType arrowType){
+    writer.writeBigEndianBytesToDecimal256(value, arrowType);
+    writer.setPosition(writer.idx() + 1);
+  }
+
+
   <#list vv.types as type>
     <#list type.minor as minor>
       <#assign name = minor.class?cap_first />
diff --git a/java/vector/src/main/codegen/templates/UnionMapWriter.java b/java/vector/src/main/codegen/templates/UnionMapWriter.java
index 01b371329a9..cec73c45f5c 100644
--- a/java/vector/src/main/codegen/templates/UnionMapWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionMapWriter.java
@@ -16,7 +16,9 @@
  */
 
 import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.vector.complex.writer.Decimal256Writer;
 import org.apache.arrow.vector.complex.writer.DecimalWriter;
+import org.apache.arrow.vector.holders.Decimal256Holder;
 import org.apache.arrow.vector.holders.DecimalHolder;
 
 import java.lang.UnsupportedOperationException;
@@ -169,6 +171,19 @@ public DecimalWriter decimal() {
     }
   }
 
+  @Override
+  public Decimal256Writer decimal256() {
+    switch (mode) {
+      case KEY:
+        return entryWriter.decimal256(MapVector.KEY_NAME);
+      case VALUE:
+        return entryWriter.decimal256(MapVector.VALUE_NAME);
+      default:
+        return this;
+    }
+  }
+
+
   @Override
   public StructWriter struct() {
     switch (mode) {
diff --git a/java/vector/src/main/codegen/templates/UnionReader.java b/java/vector/src/main/codegen/templates/UnionReader.java
index 20fdb41d4af..6ed03fa2117 100644
--- a/java/vector/src/main/codegen/templates/UnionReader.java
+++ b/java/vector/src/main/codegen/templates/UnionReader.java
@@ -34,7 +34,7 @@
 @SuppressWarnings("unused")
 public class UnionReader extends AbstractFieldReader {
 
-  private BaseReader[] readers = new BaseReader[43];
+  private BaseReader[] readers = new BaseReader[44];
   public UnionVector data;
   
   public UnionReader(UnionVector data) {
@@ -45,7 +45,7 @@ public MinorType getMinorType() {
     return TYPES[data.getTypeValue(idx())];
   }
 
-  private static MinorType[] TYPES = new MinorType[43];
+  private static MinorType[] TYPES = new MinorType[44];
 
   static {
     for (MinorType minorType : MinorType.values()) {
@@ -88,7 +88,7 @@ private FieldReader getReaderForIndex(int index) {
       <#list type.minor as minor>
         <#assign name = minor.class?cap_first />
         <#assign uncappedName = name?uncap_first/>
-        <#if !minor.typeParams?? || minor.class == "Decimal">
+        <#if !minor.typeParams?? || minor.class?starts_with("Decimal")>
     case ${name?upper_case}:
       return (FieldReader) get${name}();
         </#if>
@@ -157,7 +157,7 @@ public int size() {
       <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
       <#assign safeType=friendlyType />
       <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
-      <#if !minor.typeParams?? || minor.class == "Decimal" >
+      <#if !minor.typeParams?? || minor.class?starts_with("Decimal") >
 
   private ${name}ReaderImpl ${uncappedName}Reader;
 
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 26d1be8df44..f33f44bbc60 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -20,7 +20,6 @@
 import org.apache.arrow.memory.ReferenceManager;
 import org.apache.arrow.memory.util.CommonUtil;
 import org.apache.arrow.memory.util.hash.ArrowBufHasher;
-import org.apache.arrow.util.DataSizeRoundingUtil;
 import org.apache.arrow.util.Preconditions;
 import org.apache.arrow.vector.BaseValueVector;
 import org.apache.arrow.vector.BitVectorHelper;
@@ -37,6 +36,7 @@
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.DataSizeRoundingUtil;
 
 <@pp.dropOutputFile />
 <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/UnionVector.java" />
@@ -55,7 +55,6 @@
 import org.apache.arrow.vector.compare.VectorVisitor;
 import org.apache.arrow.vector.complex.impl.ComplexCopier;
 import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.util.DataSizeRoundingUtil;
 import org.apache.arrow.vector.util.ValueVectorUtility;
 import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.memory.util.ArrowBufPointer;
@@ -108,7 +107,7 @@ public class UnionVector implements FieldVector {
   private final FieldType fieldType;
   private final Field[] typeIds = new Field[Byte.MAX_VALUE + 1];
 
-  private static final byte TYPE_WIDTH = 1;
+  public static final byte TYPE_WIDTH = 1;
   private static final FieldType INTERNAL_STRUCT_TYPE = new FieldType(false /*nullable*/,
       ArrowType.Struct.INSTANCE, null /*dictionary*/, null /*metadata*/);
 
@@ -273,18 +272,18 @@ public StructVector getStruct() {
       <#assign fields = minor.fields!type.fields />
       <#assign uncappedName = name?uncap_first/>
       <#assign lowerCaseName = name?lower_case/>
-      <#if !minor.typeParams?? || minor.class == "Decimal" >
+      <#if !minor.typeParams?? || minor.class?starts_with("Decimal") >
 
   private ${name}Vector ${uncappedName}Vector;
 
-  public ${name}Vector get${name}Vector(<#if minor.class == "Decimal"> ArrowType arrowType</#if>) {
-    return get${name}Vector(null<#if minor.class == "Decimal">, arrowType</#if>);
+  public ${name}Vector get${name}Vector(<#if minor.class?starts_with("Decimal")> ArrowType arrowType</#if>) {
+    return get${name}Vector(null<#if minor.class?starts_with("Decimal")>, arrowType</#if>);
   }
 
-  public ${name}Vector get${name}Vector(String name<#if minor.class == "Decimal">, ArrowType arrowType</#if>) {
+  public ${name}Vector get${name}Vector(String name<#if minor.class?starts_with("Decimal")>, ArrowType arrowType</#if>) {
     if (${uncappedName}Vector == null) {
       int vectorCount = internalStruct.size();
-      ${uncappedName}Vector = addOrGet(name, MinorType.${name?upper_case},<#if minor.class == "Decimal"> arrowType,</#if> ${name}Vector.class);
+      ${uncappedName}Vector = addOrGet(name, MinorType.${name?upper_case},<#if minor.class?starts_with("Decimal")> arrowType,</#if> ${name}Vector.class);
       if (internalStruct.size() > vectorCount) {
         ${uncappedName}Vector.allocateNew();
         if (callBack != null) {
@@ -294,10 +293,10 @@ public StructVector getStruct() {
     }
     return ${uncappedName}Vector;
   }
-  <#if minor.class == "Decimal">
+  <#if minor.class?starts_with("Decimal")>
   public ${name}Vector get${name}Vector() {
     if (${uncappedName}Vector == null) {
-      throw new IllegalArgumentException("No Decimal Vector present. Provide ArrowType argument to create a new vector");
+      throw new IllegalArgumentException("No ${uncappedName} present. Provide ArrowType argument to create a new vector");
     }
     return ${uncappedName}Vector;
   }
@@ -638,9 +637,9 @@ public ValueVector getVectorByType(int typeId, ArrowType arrowType) {
           <#assign name = minor.class?cap_first />
           <#assign fields = minor.fields!type.fields />
           <#assign uncappedName = name?uncap_first/>
-          <#if !minor.typeParams?? || minor.class == "Decimal" >
+          <#if !minor.typeParams?? || minor.class?starts_with("Decimal") >
         case ${name?upper_case}:
-        return get${name}Vector(name<#if minor.class == "Decimal">, arrowType</#if>);
+        return get${name}Vector(name<#if minor.class?starts_with("Decimal")>, arrowType</#if>);
           </#if>
         </#list>
       </#list>
@@ -723,11 +722,11 @@ public void setSafe(int index, UnionHolder holder, ArrowType arrowType) {
           <#assign name = minor.class?cap_first />
           <#assign fields = minor.fields!type.fields />
           <#assign uncappedName = name?uncap_first/>
-          <#if !minor.typeParams?? || minor.class == "Decimal" >
+          <#if !minor.typeParams?? || minor.class?starts_with("Decimal") >
       case ${name?upper_case}:
         Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder();
         reader.read(${uncappedName}Holder);
-        setSafe(index, ${uncappedName}Holder<#if minor.class == "Decimal">, arrowType</#if>);
+        setSafe(index, ${uncappedName}Holder<#if minor.class?starts_with("Decimal")>, arrowType</#if>);
         break;
           </#if>
         </#list>
@@ -749,10 +748,10 @@ public void setSafe(int index, UnionHolder holder, ArrowType arrowType) {
         <#assign name = minor.class?cap_first />
         <#assign fields = minor.fields!type.fields />
         <#assign uncappedName = name?uncap_first/>
-        <#if !minor.typeParams?? || minor.class == "Decimal" >
-    public void setSafe(int index, Nullable${name}Holder holder<#if minor.class == "Decimal">, ArrowType arrowType</#if>) {
+        <#if !minor.typeParams?? || minor.class?starts_with("Decimal") >
+    public void setSafe(int index, Nullable${name}Holder holder<#if minor.class?starts_with("Decimal")>, ArrowType arrowType</#if>) {
       setType(index, MinorType.${name?upper_case});
-      get${name}Vector(null<#if minor.class == "Decimal">, arrowType</#if>).setSafe(index, holder);
+      get${name}Vector(null<#if minor.class?starts_with("Decimal")>, arrowType</#if>).setSafe(index, holder);
     }
 
         </#if>
diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java
index 6f2b2e1bf0e..59322d42fde 100644
--- a/java/vector/src/main/codegen/templates/UnionWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionWriter.java
@@ -125,9 +125,9 @@ BaseWriter getWriter(MinorType minorType, ArrowType arrowType) {
         <#assign name = minor.class?cap_first />
         <#assign fields = minor.fields!type.fields />
         <#assign uncappedName = name?uncap_first/>
-        <#if !minor.typeParams?? || minor.class == "Decimal">
+        <#if !minor.typeParams?? || minor.class?starts_with("Decimal")>
     case ${name?upper_case}:
-      return get${name}Writer(<#if minor.class == "Decimal" >arrowType</#if>);
+      return get${name}Writer(<#if minor.class?starts_with("Decimal") >arrowType</#if>);
         </#if>
       </#list>
     </#list>
@@ -141,49 +141,49 @@ BaseWriter getWriter(MinorType minorType, ArrowType arrowType) {
       <#assign fields = minor.fields!type.fields />
       <#assign uncappedName = name?uncap_first/>
       <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
-      <#if !minor.typeParams?? || minor.class == "Decimal" >
+      <#if !minor.typeParams?? || minor.class?starts_with("Decimal") >
 
   private ${name}Writer ${name?uncap_first}Writer;
 
-  private ${name}Writer get${name}Writer(<#if minor.class == "Decimal">ArrowType arrowType</#if>) {
+  private ${name}Writer get${name}Writer(<#if minor.class?starts_with("Decimal")>ArrowType arrowType</#if>) {
     if (${uncappedName}Writer == null) {
-      ${uncappedName}Writer = new ${name}WriterImpl(data.get${name}Vector(<#if minor.class == "Decimal">arrowType</#if>));
+      ${uncappedName}Writer = new ${name}WriterImpl(data.get${name}Vector(<#if minor.class?starts_with("Decimal")>arrowType</#if>));
       ${uncappedName}Writer.setPosition(idx());
       writers.add(${uncappedName}Writer);
     }
     return ${uncappedName}Writer;
   }
 
-  public ${name}Writer as${name}(<#if minor.class == "Decimal">ArrowType arrowType</#if>) {
+  public ${name}Writer as${name}(<#if minor.class?starts_with("Decimal")>ArrowType arrowType</#if>) {
     data.setType(idx(), MinorType.${name?upper_case});
-    return get${name}Writer(<#if minor.class == "Decimal">arrowType</#if>);
+    return get${name}Writer(<#if minor.class?starts_with("Decimal")>arrowType</#if>);
   }
 
   @Override
   public void write(${name}Holder holder) {
     data.setType(idx(), MinorType.${name?upper_case});
-    <#if minor.class == "Decimal">ArrowType arrowType = new ArrowType.Decimal(holder.precision, holder.scale);</#if>
-    get${name}Writer(<#if minor.class == "Decimal">arrowType</#if>).setPosition(idx());
-    get${name}Writer(<#if minor.class == "Decimal">arrowType</#if>).write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, </#if></#list><#if minor.class == "Decimal">, arrowType</#if>);
+    <#if minor.class?starts_with("Decimal")>ArrowType arrowType = new ArrowType.Decimal(holder.precision, holder.scale, ${name}Holder.WIDTH * 8);</#if>
+    get${name}Writer(<#if minor.class?starts_with("Decimal")>arrowType</#if>).setPosition(idx());
+    get${name}Writer(<#if minor.class?starts_with("Decimal")>arrowType</#if>).write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, </#if></#list><#if minor.class?starts_with("Decimal")>, arrowType</#if>);
   }
 
-  public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list><#if minor.class == "Decimal">, ArrowType arrowType</#if>) {
+  public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list><#if minor.class?starts_with("Decimal")>, ArrowType arrowType</#if>) {
     data.setType(idx(), MinorType.${name?upper_case});
-    get${name}Writer(<#if minor.class == "Decimal">arrowType</#if>).setPosition(idx());
-    get${name}Writer(<#if minor.class == "Decimal">arrowType</#if>).write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list><#if minor.class == "Decimal">, arrowType</#if>);
+    get${name}Writer(<#if minor.class?starts_with("Decimal")>arrowType</#if>).setPosition(idx());
+    get${name}Writer(<#if minor.class?starts_with("Decimal")>arrowType</#if>).write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list><#if minor.class?starts_with("Decimal")>, arrowType</#if>);
   }
-  <#if minor.class == "Decimal">
-  public void write${minor.class}(${friendlyType} value) {
-    data.setType(idx(), MinorType.DECIMAL);
-    ArrowType arrowType = new ArrowType.Decimal(value.precision(), value.scale());
-    getDecimalWriter(arrowType).setPosition(idx());
-    getDecimalWriter(arrowType).writeDecimal(value);
+  <#if minor.class?starts_with("Decimal")>
+  public void write${name}(${friendlyType} value) {
+    data.setType(idx(), MinorType.${name?upper_case});
+    ArrowType arrowType = new ArrowType.Decimal(value.precision(), value.scale(), ${name}Vector.TYPE_WIDTH * 8);
+    get${name}Writer(arrowType).setPosition(idx());
+    get${name}Writer(arrowType).write${name}(value);
   }
 
-  public void writeBigEndianBytesToDecimal(byte[] value, ArrowType arrowType) {
-    data.setType(idx(), MinorType.DECIMAL);
-    getDecimalWriter(arrowType).setPosition(idx());
-    getDecimalWriter(arrowType).writeBigEndianBytesToDecimal(value, arrowType);
+  public void writeBigEndianBytesTo${name}(byte[] value, ArrowType arrowType) {
+    data.setType(idx(), MinorType.${name?upper_case});
+    get${name}Writer(arrowType).setPosition(idx());
+    get${name}Writer(arrowType).writeBigEndianBytesTo${name}(value, arrowType);
   }
   </#if>
       </#if>
@@ -226,7 +226,7 @@ public StructWriter struct(String name) {
   <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
   <#assign upperName = minor.class?upper_case />
   <#assign capName = minor.class?cap_first />
-  <#if !minor.typeParams?? || minor.class == "Decimal" >
+  <#if !minor.typeParams?? || minor.class?starts_with("Decimal") >
   @Override
   public ${capName}Writer ${lowerName}(String name) {
     data.setType(idx(), MinorType.STRUCT);
@@ -241,7 +241,7 @@ public StructWriter struct(String name) {
     return getListWriter().${lowerName}();
   }
   </#if>
-  <#if minor.class == "Decimal">
+  <#if minor.class?starts_with("Decimal")>
   @Override
   public ${capName}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) {
     data.setType(idx(), MinorType.STRUCT);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
index ee47f6dd812..ea31a4f7846 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
@@ -49,6 +49,7 @@ public abstract class BaseFixedWidthVector extends BaseValueVector
   private final int typeWidth;
 
   protected int lastValueCapacity;
+  protected int actualValueCapacity;
 
   protected final Field field;
   private int allocationMonitor;
@@ -72,6 +73,7 @@ public BaseFixedWidthVector(Field field, final BufferAllocator allocator, final
     validityBuffer = allocator.getEmpty();
     valueBuffer = allocator.getEmpty();
     lastValueCapacity = INITIAL_VALUE_ALLOCATION;
+    refreshValueCapacity();
   }
 
 
@@ -182,14 +184,21 @@ public void setInitialCapacity(int valueCount) {
    */
   @Override
   public int getValueCapacity() {
-    return Math.min(getValueBufferValueCapacity(), getValidityBufferValueCapacity());
+    return actualValueCapacity;
   }
 
-  private int getValueBufferValueCapacity() {
+  /**
+   * Call this if you change the capacity of valueBuffer or validityBuffer.
+   */
+  protected void refreshValueCapacity() {
+    actualValueCapacity = Math.min(getValueBufferValueCapacity(), getValidityBufferValueCapacity());
+  }
+
+  protected int getValueBufferValueCapacity() {
     return capAtMaxInt(valueBuffer.capacity() / typeWidth);
   }
 
-  private int getValidityBufferValueCapacity() {
+  protected int getValidityBufferValueCapacity() {
     return capAtMaxInt(validityBuffer.capacity() * 8);
   }
 
@@ -238,6 +247,7 @@ public void clear() {
     valueCount = 0;
     validityBuffer = releaseBuffer(validityBuffer);
     valueBuffer = releaseBuffer(valueBuffer);
+    refreshValueCapacity();
   }
 
   /* used to step down the memory allocation */
@@ -331,6 +341,7 @@ private void allocateBytes(int valueCount) {
     validityBuffer = buffers.getValidityBuf();
     zeroVector();
 
+    refreshValueCapacity();
     lastValueCapacity = getValueCapacity();
   }
 
@@ -343,6 +354,7 @@ private void allocateBytes(int valueCount) {
   private void allocateValidityBuffer(final int validityBufferSize) {
     validityBuffer = allocator.buffer(validityBufferSize);
     validityBuffer.readerIndex(0);
+    refreshValueCapacity();
   }
 
   /**
@@ -441,6 +453,7 @@ public void reAlloc() {
     validityBuffer.getReferenceManager().release();
     validityBuffer = newValidityBuffer;
 
+    refreshValueCapacity();
     lastValueCapacity = getValueCapacity();
   }
 
@@ -492,6 +505,7 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers
     validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator);
     valueBuffer.getReferenceManager().release();
     valueBuffer = dataBuffer.getReferenceManager().retain(dataBuffer, allocator);
+    refreshValueCapacity();
 
     valueCount = fieldNode.getLength();
   }
@@ -572,6 +586,7 @@ public void transferTo(BaseFixedWidthVector target) {
     target.validityBuffer = transferBuffer(validityBuffer, target.allocator);
     target.valueBuffer = transferBuffer(valueBuffer, target.allocator);
     target.valueCount = valueCount;
+    target.refreshValueCapacity();
     clear();
   }
 
@@ -602,6 +617,7 @@ private void splitAndTransferValueBuffer(int startIndex, int length,
     final int sliceLength = length * typeWidth;
     final ArrowBuf slicedBuffer = valueBuffer.slice(startPoint, sliceLength);
     target.valueBuffer = transferBuffer(slicedBuffer, target.allocator);
+    target.refreshValueCapacity();
   }
 
   /**
@@ -623,6 +639,7 @@ private void splitAndTransferValidityBuffer(int startIndex, int length,
         }
         target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget);
         target.validityBuffer.getReferenceManager().retain(1);
+        target.refreshValueCapacity();
       } else {
         /* Copy data
          * When the first bit starts from the middle of a byte (offset != 0),
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
index 516030a08b7..0af9461c525 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
@@ -24,8 +24,8 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.ReferenceManager;
 import org.apache.arrow.memory.util.CommonUtil;
-import org.apache.arrow.util.DataSizeRoundingUtil;
 import org.apache.arrow.util.Preconditions;
+import org.apache.arrow.vector.util.DataSizeRoundingUtil;
 import org.apache.arrow.vector.util.TransferPair;
 import org.apache.arrow.vector.util.ValueVectorUtility;
 import org.slf4j.Logger;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index 14d357f6c9d..3bcfd983e64 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -119,14 +119,9 @@ public void setInitialCapacity(int valueCount) {
     lastValueCapacity = valueCount;
   }
 
-  /**
-   * Get the current value capacity for the vector.
-   *
-   * @return number of elements that vector can hold.
-   */
   @Override
-  public int getValueCapacity() {
-    return capAtMaxInt(validityBuffer.capacity() * 8);
+  protected int getValueBufferValueCapacity() {
+    return capAtMaxInt(valueBuffer.capacity() * 8);
   }
 
   /**
@@ -171,6 +166,7 @@ public void splitAndTransferTo(int startIndex, int length, BaseFixedWidthVector
             validityBuffer, target.validityBuffer);
     target.valueBuffer = splitAndTransferBuffer(startIndex, length, target,
             valueBuffer, target.valueBuffer);
+    target.refreshValueCapacity();
 
     target.setValueCount(length);
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java
index d731b02d360..ec73382a0ef 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java
@@ -25,8 +25,8 @@
 import org.apache.arrow.memory.ArrowBuf;
 import org.apache.arrow.memory.BoundsChecking;
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.util.DataSizeRoundingUtil;
 import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
+import org.apache.arrow.vector.util.DataSizeRoundingUtil;
 
 import io.netty.util.internal.PlatformDependent;
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java
index 0bd64c06eab..09c874e3980 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java
@@ -51,6 +51,7 @@ public String getName() {
   private static final BufferLayout LARGE_OFFSET_BUFFER = new BufferLayout(BufferType.OFFSET, 64);
   private static final BufferLayout TYPE_BUFFER = new BufferLayout(BufferType.TYPE, 32);
   private static final BufferLayout BIT_BUFFER = new BufferLayout(BufferType.DATA, 1);
+  private static final BufferLayout VALUES_256 = new BufferLayout(BufferType.DATA, 256);
   private static final BufferLayout VALUES_128 = new BufferLayout(BufferType.DATA, 128);
   private static final BufferLayout VALUES_64 = new BufferLayout(BufferType.DATA, 64);
   private static final BufferLayout VALUES_32 = new BufferLayout(BufferType.DATA, 32);
@@ -85,8 +86,10 @@ public static BufferLayout dataBuffer(int typeBitWidth) {
         return VALUES_64;
       case 128:
         return VALUES_128;
+      case 256:
+        return VALUES_256;
       default:
-        throw new IllegalArgumentException("only 8, 16, 32, 64, or 128 bits supported");
+        throw new IllegalArgumentException("only 8, 16, 32, 64, 128, or 256 bits supported");
     }
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java
new file mode 100644
index 00000000000..c5fef82d052
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java
@@ -0,0 +1,584 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import static org.apache.arrow.vector.NullCheckingForGet.NULL_CHECKING_ENABLED;
+
+import java.math.BigDecimal;
+import java.nio.ByteOrder;
+
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.Decimal256ReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.Decimal256Holder;
+import org.apache.arrow.vector.holders.NullableDecimal256Holder;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.DecimalUtility;
+import org.apache.arrow.vector.util.TransferPair;
+
+import io.netty.util.internal.PlatformDependent;
+
+/**
+ * Decimal256Vector implements a fixed width vector (32 bytes) of
+ * decimal values which could be null. A validity buffer (bit vector) is
+ * maintained to track which elements in the vector are null.
+ */
+public final class Decimal256Vector extends BaseFixedWidthVector {
+  public static final byte TYPE_WIDTH = 32;
+  private static final boolean LITTLE_ENDIAN = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
+  private final FieldReader reader;
+
+  private final int precision;
+  private final int scale;
+
+  /**
+   * Instantiate a Decimal256Vector. This doesn't allocate any memory for
+   * the data in vector.
+   *
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public Decimal256Vector(String name, BufferAllocator allocator,
+                               int precision, int scale) {
+    this(name, FieldType.nullable(new ArrowType.Decimal(precision, scale, /*bitWidth=*/TYPE_WIDTH * 8)), allocator);
+  }
+
+  /**
+   * Instantiate a Decimal256Vector. This doesn't allocate any memory for
+   * the data in vector.
+   *
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public Decimal256Vector(String name, FieldType fieldType, BufferAllocator allocator) {
+    this(new Field(name, fieldType, null), allocator);
+  }
+
+  /**
+   * Instantiate a Decimal256Vector. This doesn't allocate any memory for
+   * the data in vector.
+   *
+   * @param field field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public Decimal256Vector(Field field, BufferAllocator allocator) {
+    super(field, allocator, TYPE_WIDTH);
+    ArrowType.Decimal arrowType = (ArrowType.Decimal) field.getFieldType().getType();
+    reader = new Decimal256ReaderImpl(Decimal256Vector.this);
+    this.precision = arrowType.getPrecision();
+    this.scale = arrowType.getScale();
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector.
+   *
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   *
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public MinorType getMinorType() {
+    return MinorType.DECIMAL256;
+  }
+
+
+  /*----------------------------------------------------------------*
+   |                                                                |
+   |          vector value retrieval methods                        |
+   |                                                                |
+   *----------------------------------------------------------------*/
+
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public ArrowBuf get(int index) throws IllegalStateException {
+    if (NULL_CHECKING_ENABLED && isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return valueBuffer.slice((long) index * TYPE_WIDTH, TYPE_WIDTH);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableDecimal256Holder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.buffer = valueBuffer;
+    holder.precision = precision;
+    holder.scale = scale;
+    holder.start = ((long) index) * TYPE_WIDTH;
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public BigDecimal getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
+      return DecimalUtility.getBigDecimalFromArrowBuf(valueBuffer, index, scale, TYPE_WIDTH);
+    }
+  }
+
+  /**
+   * Return precision for the decimal value.
+   */
+  public int getPrecision() {
+    return precision;
+  }
+
+  /**
+   * Return scale for the decimal value.
+   */
+  public int getScale() {
+    return scale;
+  }
+
+
+  /*----------------------------------------------------------------*
+   |                                                                |
+   |          vector value setter methods                           |
+   |                                                                |
+   *----------------------------------------------------------------*/
+
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index    position of element
+   * @param buffer   ArrowBuf containing decimal value.
+   */
+  public void set(int index, ArrowBuf buffer) {
+    BitVectorHelper.setBit(validityBuffer, index);
+    valueBuffer.setBytes((long) index * TYPE_WIDTH, buffer, 0, TYPE_WIDTH);
+  }
+
+  /**
+   * Set the decimal element at given index to the provided array of bytes.
+   * Decimal256 is now implemented as Native Endian. This API allows the user
+   * to pass a decimal value in the form of byte array in BE byte order.
+   *
+   * <p>Consumers of Arrow code can use this API instead of first swapping
+   * the source bytes (doing a write and read) and then finally writing to
+   * ArrowBuf of decimal vector.
+   *
+   * <p>This method takes care of adding the necessary padding if the length
+   * of byte array is less then 32 (length of decimal type).
+   *
+   * @param index position of element
+   * @param value array of bytes containing decimal in big endian byte order.
+   */
+  public void setBigEndian(int index, byte[] value) {
+    BitVectorHelper.setBit(validityBuffer, index);
+    final int length = value.length;
+
+    // do the bound check.
+    valueBuffer.checkBytes((long) index * TYPE_WIDTH, (long) (index + 1) * TYPE_WIDTH);
+
+    long outAddress = valueBuffer.memoryAddress() + (long) index * TYPE_WIDTH;
+    if (length == 0) {
+      PlatformDependent.setMemory(outAddress, Decimal256Vector.TYPE_WIDTH, (byte) 0);
+      return;
+    }
+    if (LITTLE_ENDIAN) {
+      // swap bytes to convert BE to LE
+      for (int byteIdx = 0; byteIdx < length; ++byteIdx) {
+        PlatformDependent.putByte(outAddress + byteIdx, value[length - 1 - byteIdx]);
+      }
+
+      if (length == TYPE_WIDTH) {
+        return;
+      }
+
+      if (length < TYPE_WIDTH) {
+        // sign extend
+        final byte pad = (byte) (value[0] < 0 ? 0xFF : 0x00);
+        PlatformDependent.setMemory(outAddress + length, Decimal256Vector.TYPE_WIDTH - length, pad);
+        return;
+      }
+    } else {
+      if (length <= TYPE_WIDTH) {
+        // copy data from value to outAddress
+        PlatformDependent.copyMemory(value, 0, outAddress + Decimal256Vector.TYPE_WIDTH - length, length);
+        // sign extend
+        final byte pad = (byte) (value[0] < 0 ? 0xFF : 0x00);
+        PlatformDependent.setMemory(outAddress, Decimal256Vector.TYPE_WIDTH - length, pad);
+        return;
+      }
+    }
+    throw new IllegalArgumentException(
+        "Invalid decimal value length. Valid length in [1 - 32], got " + length);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index    position of element
+   * @param start    start index of data in the buffer
+   * @param buffer   ArrowBuf containing decimal value.
+   */
+  public void set(int index, long start, ArrowBuf buffer) {
+    BitVectorHelper.setBit(validityBuffer, index);
+    valueBuffer.setBytes((long) index * TYPE_WIDTH, buffer, start, TYPE_WIDTH);
+  }
+
+  /**
+   * Sets the element at given index using the buffer whose size maybe <= 32 bytes.
+   * @param index index to write the decimal to
+   * @param start start of value in the buffer
+   * @param buffer contains the decimal in native endian bytes
+   * @param length length of the value in the buffer
+   */
+  public void setSafe(int index, long start, ArrowBuf buffer, int length) {
+    handleSafe(index);
+    BitVectorHelper.setBit(validityBuffer, index);
+
+    // do the bound checks.
+    buffer.checkBytes(start, start + length);
+    valueBuffer.checkBytes((long) index * TYPE_WIDTH, (long) (index + 1) * TYPE_WIDTH);
+
+    long inAddress = buffer.memoryAddress() + start;
+    long outAddress = valueBuffer.memoryAddress() + (long) index * TYPE_WIDTH;
+    if (LITTLE_ENDIAN) {
+      PlatformDependent.copyMemory(inAddress, outAddress, length);
+      // sign extend
+      if (length < TYPE_WIDTH) {
+        byte msb = PlatformDependent.getByte(inAddress + length - 1);
+        final byte pad = (byte) (msb < 0 ? 0xFF : 0x00);
+        PlatformDependent.setMemory(outAddress + length, Decimal256Vector.TYPE_WIDTH - length, pad);
+      }
+    } else {
+      PlatformDependent.copyMemory(inAddress, outAddress + Decimal256Vector.TYPE_WIDTH - length, length);
+      // sign extend
+      if (length < TYPE_WIDTH) {
+        byte msb = PlatformDependent.getByte(inAddress);
+        final byte pad = (byte) (msb < 0 ? 0xFF : 0x00);
+        PlatformDependent.setMemory(outAddress, Decimal256Vector.TYPE_WIDTH - length, pad);
+      }
+    }
+  }
+
+
+  /**
+   * Sets the element at given index using the buffer whose size maybe <= 32 bytes.
+   * @param index index to write the decimal to
+   * @param start start of value in the buffer
+   * @param buffer contains the decimal in big endian bytes
+   * @param length length of the value in the buffer
+   */
+  public void setBigEndianSafe(int index, long start, ArrowBuf buffer, int length) {
+    handleSafe(index);
+    BitVectorHelper.setBit(validityBuffer, index);
+
+    // do the bound checks.
+    buffer.checkBytes(start, start + length);
+    valueBuffer.checkBytes((long) index * TYPE_WIDTH, (long) (index + 1) * TYPE_WIDTH);
+
+    // not using buffer.getByte() to avoid boundary checks for every byte.
+    long inAddress = buffer.memoryAddress() + start;
+    long outAddress = valueBuffer.memoryAddress() + (long) index * TYPE_WIDTH;
+    if (LITTLE_ENDIAN) {
+      // swap bytes to convert BE to LE
+      for (int byteIdx = 0; byteIdx < length; ++byteIdx) {
+        byte val = PlatformDependent.getByte((inAddress + length - 1) - byteIdx);
+        PlatformDependent.putByte(outAddress + byteIdx, val);
+      }
+      // sign extend
+      if (length < 32) {
+        byte msb = PlatformDependent.getByte(inAddress);
+        final byte pad = (byte) (msb < 0 ? 0xFF : 0x00);
+        PlatformDependent.setMemory(outAddress + length, Decimal256Vector.TYPE_WIDTH - length, pad);
+      }
+    } else {
+      PlatformDependent.copyMemory(inAddress, outAddress + Decimal256Vector.TYPE_WIDTH - length, length);
+      // sign extend
+      if (length < TYPE_WIDTH) {
+        byte msb = PlatformDependent.getByte(inAddress);
+        final byte pad = (byte) (msb < 0 ? 0xFF : 0x00);
+        PlatformDependent.setMemory(outAddress, Decimal256Vector.TYPE_WIDTH - length, pad);
+      }
+    }
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   BigDecimal containing decimal value.
+   */
+  public void set(int index, BigDecimal value) {
+    BitVectorHelper.setBit(validityBuffer, index);
+    DecimalUtility.checkPrecisionAndScale(value, precision, scale);
+    DecimalUtility.writeBigDecimalToArrowBuf(value, valueBuffer, index, TYPE_WIDTH);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   long value.
+   */
+  public void set(int index, long value) {
+    BitVectorHelper.setBit(validityBuffer, index);
+    DecimalUtility.writeLongToArrowBuf(value, valueBuffer, index, TYPE_WIDTH);
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableDecimal256Holder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
+      BitVectorHelper.setBit(validityBuffer, index);
+      valueBuffer.setBytes((long) index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH);
+    } else {
+      BitVectorHelper.unsetBit(validityBuffer, index);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, Decimal256Holder holder) {
+    BitVectorHelper.setBit(validityBuffer, index);
+    valueBuffer.setBytes((long) index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #set(int, ArrowBuf)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param buffer  ArrowBuf containing decimal value.
+   */
+  public void setSafe(int index, ArrowBuf buffer) {
+    handleSafe(index);
+    set(index, buffer);
+  }
+
+  /**
+   * Same as {@link #setBigEndian(int, byte[])} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   */
+  public void setBigEndianSafe(int index, byte[] value) {
+    handleSafe(index);
+    setBigEndian(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, int, ArrowBuf)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index    position of element
+   * @param start    start index of data in the buffer
+   * @param buffer   ArrowBuf containing decimal value.
+   */
+  public void setSafe(int index, long start, ArrowBuf buffer) {
+    handleSafe(index);
+    set(index, start, buffer);
+  }
+
+  /**
+   * Same as {@link #set(int, BigDecimal)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   BigDecimal containing decimal value.
+   */
+  public void setSafe(int index, BigDecimal value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, long)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   long value.
+   */
+  public void setSafe(int index, long value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableDecimalHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableDecimal256Holder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, Decimal256Holder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, Decimal256Holder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   *
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param start start position of the value in the buffer
+   * @param buffer buffer containing the value to be stored in the vector
+   */
+  public void set(int index, int isSet, long start, ArrowBuf buffer) {
+    if (isSet > 0) {
+      set(index, start, buffer);
+    } else {
+      BitVectorHelper.unsetBit(validityBuffer, index);
+    }
+  }
+
+  /**
+   * Same as {@link #setSafe(int, int, int, ArrowBuf)} except that it handles
+   * the case when the position of new value is beyond the current value
+   * capacity of the vector.
+   *
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param start start position of the value in the buffer
+   * @param buffer buffer containing the value to be stored in the vector
+   */
+  public void setSafe(int index, int isSet, long start, ArrowBuf buffer) {
+    handleSafe(index);
+    set(index, isSet, start, buffer);
+  }
+
+  /*----------------------------------------------------------------*
+   |                                                                |
+   |                      vector transfer                           |
+   |                                                                |
+   *----------------------------------------------------------------*/
+
+
+  /**
+   * Construct a TransferPair comprising of this and a target vector of
+   * the same type.
+   *
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   *
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((Decimal256Vector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    Decimal256Vector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new Decimal256Vector(ref, allocator, Decimal256Vector.this.precision,
+              Decimal256Vector.this.scale);
+    }
+
+    public TransferImpl(Decimal256Vector to) {
+      this.to = to;
+    }
+
+    @Override
+    public Decimal256Vector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, Decimal256Vector.this);
+    }
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
index 554e174dc2b..472c2db7986 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
@@ -20,6 +20,7 @@
 import static org.apache.arrow.vector.NullCheckingForGet.NULL_CHECKING_ENABLED;
 
 import java.math.BigDecimal;
+import java.nio.ByteOrder;
 
 import org.apache.arrow.memory.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
@@ -43,6 +44,7 @@
  */
 public final class DecimalVector extends BaseFixedWidthVector {
   public static final byte TYPE_WIDTH = 16;
+  private static final boolean LITTLE_ENDIAN = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
   private final FieldReader reader;
 
   private final int precision;
@@ -57,7 +59,7 @@ public final class DecimalVector extends BaseFixedWidthVector {
    */
   public DecimalVector(String name, BufferAllocator allocator,
                                int precision, int scale) {
-    this(name, FieldType.nullable(new ArrowType.Decimal(precision, scale)), allocator);
+    this(name, FieldType.nullable(new ArrowType.Decimal(precision, scale, TYPE_WIDTH * 8)), allocator);
   }
 
   /**
@@ -158,7 +160,7 @@ public BigDecimal getObject(int index) {
     if (isSet(index) == 0) {
       return null;
     } else {
-      return DecimalUtility.getBigDecimalFromArrowBuf(valueBuffer, index, scale);
+      return DecimalUtility.getBigDecimalFromArrowBuf(valueBuffer, index, scale, TYPE_WIDTH);
     }
   }
 
@@ -197,7 +199,7 @@ public void set(int index, ArrowBuf buffer) {
 
   /**
    * Set the decimal element at given index to the provided array of bytes.
-   * Decimal is now implemented as Little Endian. This API allows the user
+   * Decimal is now implemented as Native Endian. This API allows the user
    * to pass a decimal value in the form of byte array in BE byte order.
    *
    * <p>Consumers of Arrow code can use this API instead of first swapping
@@ -218,25 +220,38 @@ public void setBigEndian(int index, byte[] value) {
     valueBuffer.checkBytes((long) index * TYPE_WIDTH, (long) (index + 1) * TYPE_WIDTH);
 
     long outAddress = valueBuffer.memoryAddress() + (long) index * TYPE_WIDTH;
-    // swap bytes to convert BE to LE
-    for (int byteIdx = 0; byteIdx < length; ++byteIdx) {
-      PlatformDependent.putByte(outAddress + byteIdx, value[length - 1 - byteIdx]);
-    }
-
-    if (length == TYPE_WIDTH) {
-      return;
-    }
-
     if (length == 0) {
       PlatformDependent.setMemory(outAddress, DecimalVector.TYPE_WIDTH, (byte) 0);
-    } else if (length < TYPE_WIDTH) {
-      // sign extend
-      final byte pad = (byte) (value[0] < 0 ? 0xFF : 0x00);
-      PlatformDependent.setMemory(outAddress + length, DecimalVector.TYPE_WIDTH - length, pad);
+      return;
+    }
+    if (LITTLE_ENDIAN) {
+      // swap bytes to convert BE to LE
+      for (int byteIdx = 0; byteIdx < length; ++byteIdx) {
+        PlatformDependent.putByte(outAddress + byteIdx, value[length - 1 - byteIdx]);
+      }
+
+      if (length == TYPE_WIDTH) {
+        return;
+      }
+
+      if (length < TYPE_WIDTH) {
+        // sign extend
+        final byte pad = (byte) (value[0] < 0 ? 0xFF : 0x00);
+        PlatformDependent.setMemory(outAddress + length, DecimalVector.TYPE_WIDTH - length, pad);
+        return;
+      }
     } else {
-      throw new IllegalArgumentException(
-          "Invalid decimal value length. Valid length in [1 - 16], got " + length);
+      if (length <= TYPE_WIDTH) {
+        // copy data from value to outAddress
+        PlatformDependent.copyMemory(value, 0, outAddress + DecimalVector.TYPE_WIDTH - length, length);
+        // sign extend
+        final byte pad = (byte) (value[0] < 0 ? 0xFF : 0x00);
+        PlatformDependent.setMemory(outAddress, DecimalVector.TYPE_WIDTH - length, pad);
+        return;
+      }
     }
+    throw new IllegalArgumentException(
+        "Invalid decimal value length. Valid length in [1 - 16], got " + length);
   }
 
   /**
@@ -246,7 +261,7 @@ public void setBigEndian(int index, byte[] value) {
    * @param start    start index of data in the buffer
    * @param buffer   ArrowBuf containing decimal value.
    */
-  public void set(int index, int start, ArrowBuf buffer) {
+  public void set(int index, long start, ArrowBuf buffer) {
     BitVectorHelper.setBit(validityBuffer, index);
     valueBuffer.setBytes((long) index * TYPE_WIDTH, buffer, start, TYPE_WIDTH);
   }
@@ -255,10 +270,10 @@ public void set(int index, int start, ArrowBuf buffer) {
    * Sets the element at given index using the buffer whose size maybe <= 16 bytes.
    * @param index index to write the decimal to
    * @param start start of value in the buffer
-   * @param buffer contains the decimal in little endian bytes
+   * @param buffer contains the decimal in native endian bytes
    * @param length length of the value in the buffer
    */
-  public void setSafe(int index, int start, ArrowBuf buffer, int length) {
+  public void setSafe(int index, long start, ArrowBuf buffer, int length) {
     handleSafe(index);
     BitVectorHelper.setBit(validityBuffer, index);
 
@@ -268,12 +283,22 @@ public void setSafe(int index, int start, ArrowBuf buffer, int length) {
 
     long inAddress = buffer.memoryAddress() + start;
     long outAddress = valueBuffer.memoryAddress() + (long) index * TYPE_WIDTH;
-    PlatformDependent.copyMemory(inAddress, outAddress, length);
-    // sign extend
-    if (length < 16) {
-      byte msb = PlatformDependent.getByte(inAddress + length - 1);
-      final byte pad = (byte) (msb < 0 ? 0xFF : 0x00);
-      PlatformDependent.setMemory(outAddress + length, DecimalVector.TYPE_WIDTH - length, pad);
+    if (LITTLE_ENDIAN) {
+      PlatformDependent.copyMemory(inAddress, outAddress, length);
+      // sign extend
+      if (length < TYPE_WIDTH) {
+        byte msb = PlatformDependent.getByte(inAddress + length - 1);
+        final byte pad = (byte) (msb < 0 ? 0xFF : 0x00);
+        PlatformDependent.setMemory(outAddress + length, DecimalVector.TYPE_WIDTH - length, pad);
+      }
+    } else {
+      PlatformDependent.copyMemory(inAddress, outAddress + DecimalVector.TYPE_WIDTH - length, length);
+      // sign extend
+      if (length < TYPE_WIDTH) {
+        byte msb = PlatformDependent.getByte(inAddress);
+        final byte pad = (byte) (msb < 0 ? 0xFF : 0x00);
+        PlatformDependent.setMemory(outAddress, DecimalVector.TYPE_WIDTH - length, pad);
+      }
     }
   }
 
@@ -285,7 +310,7 @@ public void setSafe(int index, int start, ArrowBuf buffer, int length) {
    * @param buffer contains the decimal in big endian bytes
    * @param length length of the value in the buffer
    */
-  public void setBigEndianSafe(int index, int start, ArrowBuf buffer, int length) {
+  public void setBigEndianSafe(int index, long start, ArrowBuf buffer, int length) {
     handleSafe(index);
     BitVectorHelper.setBit(validityBuffer, index);
 
@@ -296,16 +321,26 @@ public void setBigEndianSafe(int index, int start, ArrowBuf buffer, int length)
     // not using buffer.getByte() to avoid boundary checks for every byte.
     long inAddress = buffer.memoryAddress() + start;
     long outAddress = valueBuffer.memoryAddress() + (long) index * TYPE_WIDTH;
-    // swap bytes to convert BE to LE
-    for (int byteIdx = 0; byteIdx < length; ++byteIdx) {
-      byte val = PlatformDependent.getByte((inAddress + length - 1) - byteIdx);
-      PlatformDependent.putByte(outAddress + byteIdx, val);
-    }
-    // sign extend
-    if (length < 16) {
-      byte msb = PlatformDependent.getByte(inAddress);
-      final byte pad = (byte) (msb < 0 ? 0xFF : 0x00);
-      PlatformDependent.setMemory(outAddress + length, DecimalVector.TYPE_WIDTH - length, pad);
+    if (LITTLE_ENDIAN) {
+      // swap bytes to convert BE to LE
+      for (int byteIdx = 0; byteIdx < length; ++byteIdx) {
+        byte val = PlatformDependent.getByte((inAddress + length - 1) - byteIdx);
+        PlatformDependent.putByte(outAddress + byteIdx, val);
+      }
+      // sign extend
+      if (length < TYPE_WIDTH) {
+        byte msb = PlatformDependent.getByte(inAddress);
+        final byte pad = (byte) (msb < 0 ? 0xFF : 0x00);
+        PlatformDependent.setMemory(outAddress + length, DecimalVector.TYPE_WIDTH - length, pad);
+      }
+    } else {
+      PlatformDependent.copyMemory(inAddress, outAddress + DecimalVector.TYPE_WIDTH - length, length);
+      // sign extend
+      if (length < TYPE_WIDTH) {
+        byte msb = PlatformDependent.getByte(inAddress);
+        final byte pad = (byte) (msb < 0 ? 0xFF : 0x00);
+        PlatformDependent.setMemory(outAddress, DecimalVector.TYPE_WIDTH - length, pad);
+      }
     }
   }
 
@@ -318,7 +353,7 @@ public void setBigEndianSafe(int index, int start, ArrowBuf buffer, int length)
   public void set(int index, BigDecimal value) {
     BitVectorHelper.setBit(validityBuffer, index);
     DecimalUtility.checkPrecisionAndScale(value, precision, scale);
-    DecimalUtility.writeBigDecimalToArrowBuf(value, valueBuffer, index);
+    DecimalUtility.writeBigDecimalToArrowBuf(value, valueBuffer, index, TYPE_WIDTH);
   }
 
   /**
@@ -329,7 +364,7 @@ public void set(int index, BigDecimal value) {
    */
   public void set(int index, long value) {
     BitVectorHelper.setBit(validityBuffer, index);
-    DecimalUtility.writeLongToArrowBuf(value, valueBuffer, index);
+    DecimalUtility.writeLongToArrowBuf(value, valueBuffer, index, TYPE_WIDTH);
   }
 
   /**
@@ -394,7 +429,7 @@ public void setBigEndianSafe(int index, byte[] value) {
    * @param start    start index of data in the buffer
    * @param buffer   ArrowBuf containing decimal value.
    */
-  public void setSafe(int index, int start, ArrowBuf buffer) {
+  public void setSafe(int index, long start, ArrowBuf buffer) {
     handleSafe(index);
     set(index, start, buffer);
   }
@@ -460,7 +495,7 @@ public void setSafe(int index, DecimalHolder holder) {
    * @param start start position of the value in the buffer
    * @param buffer buffer containing the value to be stored in the vector
    */
-  public void set(int index, int isSet, int start, ArrowBuf buffer) {
+  public void set(int index, int isSet, long start, ArrowBuf buffer) {
     if (isSet > 0) {
       set(index, start, buffer);
     } else {
@@ -478,7 +513,7 @@ public void set(int index, int isSet, int start, ArrowBuf buffer) {
    * @param start start position of the value in the buffer
    * @param buffer buffer containing the value to be stored in the vector
    */
-  public void setSafe(int index, int isSet, int start, ArrowBuf buffer) {
+  public void setSafe(int index, int isSet, long start, ArrowBuf buffer) {
     handleSafe(index);
     set(index, isSet, start, buffer);
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java
index 942ccaf7883..9f65c5693ff 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java
@@ -44,6 +44,7 @@
 public final class DurationVector extends BaseFixedWidthVector {
   private static final byte TYPE_WIDTH = 8;
   private final FieldReader reader;
+
   private final TimeUnit unit;
 
   /**
@@ -193,6 +194,13 @@ private StringBuilder getAsStringBuilderHelper(int index) {
     return new StringBuilder(getObject(index).toString());
   }
 
+  /**
+   * Gets the time unit of the duration.
+   */
+  public TimeUnit getUnit() {
+    return unit;
+  }
+
   /*----------------------------------------------------------------*
    |                                                                |
    |          vector value setter methods                           |
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java
index 501ca98c0a4..1004ce1a74a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java
@@ -155,7 +155,7 @@ public TypeLayout visit(FloatingPoint type) {
 
       @Override
       public TypeLayout visit(Decimal type) {
-        return newFixedWidthTypeLayout(BufferLayout.dataBuffer(128));
+        return newFixedWidthTypeLayout(BufferLayout.dataBuffer(type.getBitWidth()));
       }
 
       @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/UInt1Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt1Vector.java
index f91d1713995..4a2e5b12950 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/UInt1Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt1Vector.java
@@ -36,6 +36,16 @@
  * maintained to track which elements in the vector are null.
  */
 public final class UInt1Vector extends BaseFixedWidthVector implements BaseIntVector {
+  /**
+   * The mask to use when promoting the unsigned byte value to an integer.
+   */
+  public static final int PROMOTION_MASK = 0xFF;
+
+  /**
+   * The maximum 8-bit unsigned integer.
+   */
+  public static final byte MAX_UINT1 = (byte) 0XFF;
+
   private static final byte TYPE_WIDTH = 1;
   private final FieldReader reader;
 
@@ -83,7 +93,7 @@ public MinorType getMinorType() {
    */
   public static short getNoOverflow(final ArrowBuf buffer, final int index) {
     byte b = buffer.getByte(index * TYPE_WIDTH);
-    return (short) (0xFF & b);
+    return (short) (PROMOTION_MASK & b);
   }
 
 
@@ -315,7 +325,7 @@ public void setUnsafeWithPossibleTruncate(int index, long value) {
 
   @Override
   public long getValueAsLong(int index) {
-    return this.get(index);
+    return this.get(index) & PROMOTION_MASK;
   }
 
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java
index de024ffbcad..660194bce1f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java
@@ -36,6 +36,12 @@
  * maintained to track which elements in the vector are null.
  */
 public final class UInt2Vector extends BaseFixedWidthVector implements BaseIntVector {
+
+  /**
+   * The maximum 16-bit unsigned integer.
+   */
+  public static final char MAX_UINT2 = (char) 0XFFFF;
+
   private static final byte TYPE_WIDTH = 2;
   private final FieldReader reader;
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java
index d74947873a8..8166dfd6778 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java
@@ -36,6 +36,17 @@
  * maintained to track which elements in the vector are null.
  */
 public final class UInt4Vector extends BaseFixedWidthVector implements BaseIntVector {
+
+  /**
+   * The mask to use when promoting the unsigned int value to a long int.
+   */
+  public static final long PROMOTION_MASK = 0x00000000FFFFFFFFL;
+
+  /**
+   * The maximum 32-bit unsigned integer.
+   */
+  public static final int MAX_UINT4 = 0XFFFFFFFF;
+
   private static final byte TYPE_WIDTH = 4;
   private final FieldReader reader;
 
@@ -83,7 +94,7 @@ public MinorType getMinorType() {
    */
   public static long getNoOverflow(final ArrowBuf buffer, final int index) {
     long l = buffer.getInt((long) index * TYPE_WIDTH);
-    return (0x00000000FFFFFFFFL) & l;
+    return PROMOTION_MASK & l;
   }
 
   /**
@@ -286,7 +297,7 @@ public void setUnsafeWithPossibleTruncate(int index, long value) {
 
   @Override
   public long getValueAsLong(int index) {
-    return this.get(index);
+    return this.get(index) & PROMOTION_MASK;
   }
 
   private class TransferImpl implements TransferPair {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java
index c04af54e819..48f8c50dde5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java
@@ -38,6 +38,12 @@
  * maintained to track which elements in the vector are null.
  */
 public final class UInt8Vector extends BaseFixedWidthVector implements BaseIntVector {
+
+  /**
+   * The maximum 64-bit unsigned long integer.
+   */
+  public static final long MAX_UINT8 = 0XFFFFFFFFFFFFFFFFL;
+
   private static final byte TYPE_WIDTH = 8;
   private final FieldReader reader;
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
index 2720219767a..3a4b00de5ef 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
@@ -25,6 +25,8 @@
 
 import org.apache.arrow.memory.ArrowBuf;
 import org.apache.arrow.util.Collections2;
+import org.apache.arrow.vector.compression.CompressionCodec;
+import org.apache.arrow.vector.compression.CompressionUtil;
 import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
 import org.apache.arrow.vector.types.pojo.Field;
@@ -54,8 +56,9 @@ public VectorLoader(VectorSchemaRoot root) {
   public void load(ArrowRecordBatch recordBatch) {
     Iterator<ArrowBuf> buffers = recordBatch.getBuffers().iterator();
     Iterator<ArrowFieldNode> nodes = recordBatch.getNodes().iterator();
+    CompressionCodec codec = CompressionUtil.createCodec(recordBatch.getBodyCompression().getCodec());
     for (FieldVector fieldVector : root.getFieldVectors()) {
-      loadBuffers(fieldVector, fieldVector.getField(), buffers, nodes);
+      loadBuffers(fieldVector, fieldVector.getField(), buffers, nodes, codec);
     }
     root.setRowCount(recordBatch.getLength());
     if (nodes.hasNext() || buffers.hasNext()) {
@@ -68,13 +71,15 @@ private void loadBuffers(
       FieldVector vector,
       Field field,
       Iterator<ArrowBuf> buffers,
-      Iterator<ArrowFieldNode> nodes) {
+      Iterator<ArrowFieldNode> nodes,
+      CompressionCodec codec) {
     checkArgument(nodes.hasNext(), "no more field nodes for for field %s and vector %s", field, vector);
     ArrowFieldNode fieldNode = nodes.next();
     int bufferLayoutCount = TypeLayout.getTypeBufferCount(field.getType());
     List<ArrowBuf> ownBuffers = new ArrayList<>(bufferLayoutCount);
     for (int j = 0; j < bufferLayoutCount; j++) {
-      ownBuffers.add(buffers.next());
+      ArrowBuf nextBuf = buffers.next();
+      ownBuffers.add(codec.decompress(vector.getAllocator(), nextBuf));
     }
     try {
       vector.loadFieldBuffers(fieldNode, ownBuffers);
@@ -91,7 +96,7 @@ private void loadBuffers(
       for (int i = 0; i < childrenFromFields.size(); i++) {
         Field child = children.get(i);
         FieldVector fieldVector = childrenFromFields.get(i);
-        loadBuffers(fieldVector, child, buffers, nodes);
+        loadBuffers(fieldVector, child, buffers, nodes, codec);
       }
     }
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
index 62d063b0fd8..e2cbf3ec1d8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
@@ -21,6 +21,9 @@
 import java.util.List;
 
 import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.vector.compression.CompressionCodec;
+import org.apache.arrow.vector.compression.CompressionUtil;
+import org.apache.arrow.vector.compression.NoCompressionCodec;
 import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
 
@@ -32,13 +35,14 @@ public class VectorUnloader {
 
   private final VectorSchemaRoot root;
   private final boolean includeNullCount;
+  private final CompressionCodec codec;
   private final boolean alignBuffers;
 
   /**
    * Constructs a new instance of the given set of vectors.
    */
   public VectorUnloader(VectorSchemaRoot root) {
-    this(root, true, true);
+    this(root, true, NoCompressionCodec.INSTANCE, true);
   }
 
   /**
@@ -48,9 +52,24 @@ public VectorUnloader(VectorSchemaRoot root) {
    * @param includeNullCount Controls whether null count is copied to the {@link ArrowRecordBatch}
    * @param alignBuffers Controls if buffers get aligned to 8-byte boundaries.
    */
-  public VectorUnloader(VectorSchemaRoot root, boolean includeNullCount, boolean alignBuffers) {
+  public VectorUnloader(
+      VectorSchemaRoot root, boolean includeNullCount, boolean alignBuffers) {
+    this(root, includeNullCount, NoCompressionCodec.INSTANCE, alignBuffers);
+  }
+
+  /**
+   * Constructs a new instance.
+   *
+   * @param root  The set of vectors to serialize to an {@link ArrowRecordBatch}.
+   * @param includeNullCount Controls whether null count is copied to the {@link ArrowRecordBatch}
+   * @param codec the codec for compressing data. If it is null, then no compression is needed.
+   * @param alignBuffers Controls if buffers get aligned to 8-byte boundaries.
+   */
+  public VectorUnloader(
+      VectorSchemaRoot root, boolean includeNullCount, CompressionCodec codec, boolean alignBuffers) {
     this.root = root;
     this.includeNullCount = includeNullCount;
+    this.codec = codec;
     this.alignBuffers = alignBuffers;
   }
 
@@ -64,7 +83,8 @@ public ArrowRecordBatch getRecordBatch() {
     for (FieldVector vector : root.getFieldVectors()) {
       appendNodes(vector, nodes, buffers);
     }
-    return new ArrowRecordBatch(root.getRowCount(), nodes, buffers, alignBuffers);
+    return new ArrowRecordBatch(
+        root.getRowCount(), nodes, buffers, CompressionUtil.createBodyCompression(codec), alignBuffers);
   }
 
   private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
@@ -76,7 +96,9 @@ private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<Ar
           "wrong number of buffers for field %s in vector %s. found: %s",
           vector.getField(), vector.getClass().getSimpleName(), fieldBuffers));
     }
-    buffers.addAll(fieldBuffers);
+    for (ArrowBuf buf : fieldBuffers) {
+      buffers.add(codec.compress(vector.getAllocator(), buf));
+    }
     for (FieldVector child : vector.getChildrenFromFields()) {
       appendNodes(child, nodes, buffers);
     }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index 6f40836e06b..b87281dbc14 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -30,6 +30,7 @@
 import org.apache.arrow.vector.complex.StructVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.writer.FieldWriter;
+import org.apache.arrow.vector.holders.Decimal256Holder;
 import org.apache.arrow.vector.holders.DecimalHolder;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType;
@@ -54,6 +55,7 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
   private final NullableStructWriterFactory nullableStructWriterFactory;
   private int position;
   private static final int MAX_DECIMAL_PRECISION = 38;
+  private static final int MAX_DECIMAL256_PRECISION = 76;
 
   private enum State {
     UNTYPED, SINGLE, UNION
@@ -316,26 +318,54 @@ private FieldWriter promoteToUnion() {
 
   @Override
   public void write(DecimalHolder holder) {
-    getWriter(MinorType.DECIMAL, new ArrowType.Decimal(MAX_DECIMAL_PRECISION, holder.scale)).write(holder);
+    getWriter(MinorType.DECIMAL,
+              new ArrowType.Decimal(MAX_DECIMAL_PRECISION, holder.scale, /*bitWidth=*/128)).write(holder);
   }
 
   @Override
-  public void writeDecimal(int start, ArrowBuf buffer, ArrowType arrowType) {
+  public void writeDecimal(long start, ArrowBuf buffer, ArrowType arrowType) {
     getWriter(MinorType.DECIMAL, new ArrowType.Decimal(MAX_DECIMAL_PRECISION,
-        ((ArrowType.Decimal) arrowType).getScale())).writeDecimal(start, buffer, arrowType);
+        ((ArrowType.Decimal) arrowType).getScale(), /*bitWidth=*/128)).writeDecimal(start, buffer, arrowType);
   }
 
   @Override
   public void writeDecimal(BigDecimal value) {
-    getWriter(MinorType.DECIMAL, new ArrowType.Decimal(MAX_DECIMAL_PRECISION, value.scale())).writeDecimal(value);
+    getWriter(MinorType.DECIMAL,
+      new ArrowType.Decimal(MAX_DECIMAL_PRECISION, value.scale(), /*bitWidth=*/128)).writeDecimal(value);
   }
 
   @Override
   public void writeBigEndianBytesToDecimal(byte[] value, ArrowType arrowType) {
     getWriter(MinorType.DECIMAL, new ArrowType.Decimal(MAX_DECIMAL_PRECISION,
-        ((ArrowType.Decimal) arrowType).getScale())).writeBigEndianBytesToDecimal(value, arrowType);
+        ((ArrowType.Decimal) arrowType).getScale(), /*bitWidth=*/128)).writeBigEndianBytesToDecimal(value, arrowType);
   }
 
+  @Override
+  public void write(Decimal256Holder holder) {
+    getWriter(MinorType.DECIMAL256,
+              new ArrowType.Decimal(MAX_DECIMAL256_PRECISION, holder.scale, /*bitWidth=*/256)).write(holder);
+  }
+
+  @Override
+  public void writeDecimal256(long start, ArrowBuf buffer, ArrowType arrowType) {
+    getWriter(MinorType.DECIMAL256, new ArrowType.Decimal(MAX_DECIMAL256_PRECISION,
+        ((ArrowType.Decimal) arrowType).getScale(), /*bitWidth=*/256)).writeDecimal256(start, buffer, arrowType);
+  }
+
+  @Override
+  public void writeDecimal256(BigDecimal value) {
+    getWriter(MinorType.DECIMAL256,
+        new ArrowType.Decimal(MAX_DECIMAL256_PRECISION, value.scale(), /*bitWidth=*/256)).writeDecimal256(value);
+  }
+
+  @Override
+  public void writeBigEndianBytesToDecimal256(byte[] value, ArrowType arrowType) {
+    getWriter(MinorType.DECIMAL256, new ArrowType.Decimal(MAX_DECIMAL256_PRECISION,
+        ((ArrowType.Decimal) arrowType).getScale(),
+          /*bitWidth=*/256)).writeBigEndianBytesToDecimal256(value, arrowType);
+  }
+
+ 
   @Override
   public void allocate() {
     getWriter().allocate();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compression/CompressionCodec.java b/java/vector/src/main/java/org/apache/arrow/vector/compression/CompressionCodec.java
new file mode 100644
index 00000000000..ce2dd73aab5
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/compression/CompressionCodec.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.compression;
+
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+
+/**
+ * The codec for compression/decompression.
+ */
+public interface CompressionCodec {
+
+  /**
+   * Compress a buffer.
+   * @param allocator the allocator for allocating memory for compressed buffer.
+   * @param unCompressedBuffer the buffer to compress.
+   *                           Implementation of this method should take care of releasing this buffer.
+   * @return the compressed buffer.
+   */
+  ArrowBuf compress(BufferAllocator allocator, ArrowBuf unCompressedBuffer);
+
+  /**
+   * Decompress a buffer.
+   * @param allocator the allocator for allocating memory for decompressed buffer.
+   * @param compressedBuffer the buffer to be decompressed.
+   *                         Implementation of this method should take care of releasing this buffer.
+   * @return the decompressed buffer.
+   */
+  ArrowBuf decompress(BufferAllocator allocator, ArrowBuf compressedBuffer);
+
+  /**
+   * Gets the name of the codec.
+   * @return the name of the codec.
+   */
+  String getCodecName();
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compression/CompressionUtil.java b/java/vector/src/main/java/org/apache/arrow/vector/compression/CompressionUtil.java
new file mode 100644
index 00000000000..464f3aa8e9c
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/compression/CompressionUtil.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.compression;
+
+import org.apache.arrow.flatbuf.BodyCompressionMethod;
+import org.apache.arrow.flatbuf.CompressionType;
+import org.apache.arrow.vector.ipc.message.ArrowBodyCompression;
+
+/**
+ * Utilities for data compression/decompression.
+ */
+public class CompressionUtil {
+
+  private CompressionUtil() {
+  }
+
+  /**
+   * Creates the {@link ArrowBodyCompression} object, given the {@link CompressionCodec}.
+   * The implementation of this method should depend on the values of {@link CompressionType#names}.
+   */
+  public static ArrowBodyCompression createBodyCompression(CompressionCodec codec) {
+    switch (codec.getCodecName()) {
+      case "default":
+        return NoCompressionCodec.DEFAULT_BODY_COMPRESSION;
+      case "LZ4_FRAME":
+        return new ArrowBodyCompression(CompressionType.LZ4_FRAME, BodyCompressionMethod.BUFFER);
+      case "ZSTD":
+        return new ArrowBodyCompression(CompressionType.ZSTD, BodyCompressionMethod.BUFFER);
+      default:
+        throw new IllegalArgumentException("Unknown codec: " + codec.getCodecName());
+    }
+  }
+
+  /**
+   * Creates the {@link CompressionCodec} given the compression type.
+   */
+  public static CompressionCodec createCodec(byte compressionType) {
+    switch (compressionType) {
+      case NoCompressionCodec.COMPRESSION_TYPE:
+        return NoCompressionCodec.INSTANCE;
+      default:
+        throw new IllegalArgumentException("Compression type not supported: " + compressionType);
+    }
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compression/NoCompressionCodec.java b/java/vector/src/main/java/org/apache/arrow/vector/compression/NoCompressionCodec.java
new file mode 100644
index 00000000000..72273de7630
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/compression/NoCompressionCodec.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.compression;
+
+import org.apache.arrow.flatbuf.BodyCompressionMethod;
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.ipc.message.ArrowBodyCompression;
+
+/**
+ * The default compression codec that does no compression.
+ */
+public class NoCompressionCodec implements CompressionCodec {
+
+  public static final NoCompressionCodec INSTANCE = new NoCompressionCodec();
+
+  public static final byte COMPRESSION_TYPE = -1;
+
+  public static final ArrowBodyCompression DEFAULT_BODY_COMPRESSION =
+      new ArrowBodyCompression(COMPRESSION_TYPE, BodyCompressionMethod.BUFFER);
+
+  private NoCompressionCodec() {
+  }
+
+  @Override
+  public ArrowBuf compress(BufferAllocator allocator, ArrowBuf unCompressedBuffer) {
+    return unCompressedBuffer;
+  }
+
+  @Override
+  public ArrowBuf decompress(BufferAllocator allocator, ArrowBuf compressedBuffer) {
+    return compressedBuffer;
+  }
+
+  @Override
+  public String getCodecName() {
+    return "default";
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
index 13935ef4f10..40f83c04fe5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
@@ -45,6 +45,7 @@
 import org.apache.arrow.vector.BigIntVector;
 import org.apache.arrow.vector.BitVectorHelper;
 import org.apache.arrow.vector.BufferLayout.BufferType;
+import org.apache.arrow.vector.Decimal256Vector;
 import org.apache.arrow.vector.DecimalVector;
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.Float4Vector;
@@ -438,7 +439,7 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException
         for (int i = 0; i < count; i++) {
           parser.nextToken();
           BigDecimal decimalValue = new BigDecimal(parser.readValueAs(String.class));
-          DecimalUtility.writeBigDecimalToArrowBuf(decimalValue, buf, i);
+          DecimalUtility.writeBigDecimalToArrowBuf(decimalValue, buf, i, DecimalVector.TYPE_WIDTH);
         }
 
         buf.writerIndex(size);
@@ -446,6 +447,24 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException
       }
     };
 
+    BufferReader DECIMAL256 = new BufferReader() {
+      @Override
+      protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
+        final int size = count * Decimal256Vector.TYPE_WIDTH;
+        ArrowBuf buf = allocator.buffer(size);
+
+        for (int i = 0; i < count; i++) {
+          parser.nextToken();
+          BigDecimal decimalValue = new BigDecimal(parser.readValueAs(String.class));
+          DecimalUtility.writeBigDecimalToArrowBuf(decimalValue, buf, i, Decimal256Vector.TYPE_WIDTH);
+        }
+
+        buf.writerIndex(size);
+        return buf;
+      }
+    };
+
+
     BufferReader FIXEDSIZEBINARY = new BufferReader() {
       @Override
       protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
@@ -615,6 +634,9 @@ private ArrowBuf readIntoBuffer(BufferAllocator allocator, BufferType bufferType
         case DECIMAL:
           reader = helper.DECIMAL;
           break;
+        case DECIMAL256:
+          reader = helper.DECIMAL256;
+          break;
         case FIXEDSIZEBINARY:
           reader = helper.FIXEDSIZEBINARY;
           break;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
index e210b002890..f2854c95c30 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
@@ -36,6 +36,7 @@
 import org.apache.arrow.vector.BufferLayout.BufferType;
 import org.apache.arrow.vector.DateDayVector;
 import org.apache.arrow.vector.DateMilliVector;
+import org.apache.arrow.vector.Decimal256Vector;
 import org.apache.arrow.vector.DecimalVector;
 import org.apache.arrow.vector.DurationVector;
 import org.apache.arrow.vector.FieldVector;
@@ -377,11 +378,21 @@ private void writeValueToGenerator(
         }
         case DECIMAL: {
           int scale = ((DecimalVector) vector).getScale();
-          BigDecimal decimalValue = DecimalUtility.getBigDecimalFromArrowBuf(buffer, index, scale);
+          BigDecimal decimalValue = DecimalUtility.getBigDecimalFromArrowBuf(buffer, index, scale, 
+                                                                             DecimalVector.TYPE_WIDTH);
           // We write the unscaled value, because the scale is stored in the type metadata.
           generator.writeString(decimalValue.unscaledValue().toString());
           break;
         }
+        case DECIMAL256: {
+          int scale = ((Decimal256Vector) vector).getScale();
+          BigDecimal decimalValue = DecimalUtility.getBigDecimalFromArrowBuf(buffer, index, scale, 
+                                                                             Decimal256Vector.TYPE_WIDTH);
+          // We write the unscaled value, because the scale is stored in the type metadata.
+          generator.writeString(decimalValue.unscaledValue().toString());
+          break;
+        }
+
         default:
           throw new UnsupportedOperationException("minor type: " + vector.getMinorType());
       }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBodyCompression.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBodyCompression.java
new file mode 100644
index 00000000000..5370ddfa09d
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBodyCompression.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.ipc.message;
+
+import org.apache.arrow.flatbuf.BodyCompression;
+
+import com.google.flatbuffers.FlatBufferBuilder;
+
+/**
+ * Compression information about data written to a channel.
+ */
+public class ArrowBodyCompression implements FBSerializable {
+
+  private final byte codec;
+
+  private final byte method;
+
+  public ArrowBodyCompression(byte codec, byte method) {
+    this.codec = codec;
+    this.method = method;
+  }
+
+  @Override
+  public int writeTo(FlatBufferBuilder builder) {
+    return BodyCompression.createBodyCompression(builder, codec, method);
+  }
+
+  public byte getCodec() {
+    return codec;
+  }
+
+  public byte getMethod() {
+    return method;
+  }
+
+  @Override
+  public String toString() {
+    return "ArrowBodyCompression [codec=" + codec + ", method=" + method + "]";
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.java
index a07daaa581c..6fa2fb1a72d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.java
@@ -25,7 +25,9 @@
 import org.apache.arrow.flatbuf.RecordBatch;
 import org.apache.arrow.memory.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.util.DataSizeRoundingUtil;
+import org.apache.arrow.util.Preconditions;
+import org.apache.arrow.vector.compression.NoCompressionCodec;
+import org.apache.arrow.vector.util.DataSizeRoundingUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -50,12 +52,21 @@ public class ArrowRecordBatch implements ArrowMessage {
 
   private final List<ArrowBuf> buffers;
 
+  private final ArrowBodyCompression bodyCompression;
+
   private final List<ArrowBuffer> buffersLayout;
 
   private boolean closed = false;
 
-  public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
-    this(length, nodes, buffers, true);
+  public ArrowRecordBatch(
+      int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
+    this(length, nodes, buffers, NoCompressionCodec.DEFAULT_BODY_COMPRESSION, true);
+  }
+
+  public ArrowRecordBatch(
+      int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers,
+      ArrowBodyCompression bodyCompression) {
+    this(length, nodes, buffers, bodyCompression, true);
   }
 
   /**
@@ -64,12 +75,17 @@ public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> b
    * @param length  how many rows in this batch
    * @param nodes   field level info
    * @param buffers will be retained until this recordBatch is closed
+   * @param bodyCompression compression info.
    */
-  public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers, boolean alignBuffers) {
+  public ArrowRecordBatch(
+      int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers,
+      ArrowBodyCompression bodyCompression, boolean alignBuffers) {
     super();
     this.length = length;
     this.nodes = nodes;
     this.buffers = buffers;
+    Preconditions.checkArgument(bodyCompression != null, "body compression cannot be null");
+    this.bodyCompression = bodyCompression;
     List<ArrowBuffer> arrowBuffers = new ArrayList<>(buffers.size());
     long offset = 0;
     for (ArrowBuf arrowBuf : buffers) {
@@ -91,10 +107,14 @@ public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> b
   // this constructor is different from the public ones in that the reference manager's
   // <code>retain</code> method is not called, so the first <code>dummy</code> parameter is used
   // to distinguish this from the public constructor.
-  private ArrowRecordBatch(boolean dummy, int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
+  private ArrowRecordBatch(
+      boolean dummy, int length, List<ArrowFieldNode> nodes,
+      List<ArrowBuf> buffers, ArrowBodyCompression bodyCompression) {
     this.length = length;
     this.nodes = nodes;
     this.buffers = buffers;
+    Preconditions.checkArgument(bodyCompression != null, "body compression cannot be null");
+    this.bodyCompression = bodyCompression;
     this.closed = false;
     List<ArrowBuffer> arrowBuffers = new ArrayList<>();
     long offset = 0;
@@ -114,6 +134,10 @@ public int getLength() {
     return length;
   }
 
+  public ArrowBodyCompression getBodyCompression() {
+    return bodyCompression;
+  }
+
   /**
    * Get the nodes in this record batch.
    *
@@ -151,7 +175,7 @@ public ArrowRecordBatch cloneWithTransfer(final BufferAllocator allocator) {
             .writerIndex(buf.writerIndex()))
         .collect(Collectors.toList());
     close();
-    return new ArrowRecordBatch(false, length, nodes, newBufs);
+    return new ArrowRecordBatch(false, length, nodes, newBufs, bodyCompression);
   }
 
   /**
@@ -169,10 +193,17 @@ public int writeTo(FlatBufferBuilder builder) {
     int nodesOffset = FBSerializables.writeAllStructsToVector(builder, nodes);
     RecordBatch.startBuffersVector(builder, buffers.size());
     int buffersOffset = FBSerializables.writeAllStructsToVector(builder, buffersLayout);
+    int compressOffset = 0;
+    if (bodyCompression != null && bodyCompression != NoCompressionCodec.DEFAULT_BODY_COMPRESSION) {
+      compressOffset = bodyCompression.writeTo(builder);
+    }
     RecordBatch.startRecordBatch(builder);
     RecordBatch.addLength(builder, length);
     RecordBatch.addNodes(builder, nodesOffset);
     RecordBatch.addBuffers(builder, buffersOffset);
+    if (bodyCompression != null && bodyCompression != NoCompressionCodec.DEFAULT_BODY_COMPRESSION) {
+      RecordBatch.addCompression(builder, compressOffset);
+    }
     return RecordBatch.endRecordBatch(builder);
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java
index 8679088b1f4..b2d1425270f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java
@@ -34,6 +34,7 @@
 import org.apache.arrow.memory.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.util.Preconditions;
+import org.apache.arrow.vector.compression.NoCompressionCodec;
 import org.apache.arrow.vector.ipc.ReadChannel;
 import org.apache.arrow.vector.ipc.WriteChannel;
 import org.apache.arrow.vector.types.pojo.Schema;
@@ -61,10 +62,10 @@ public class MessageSerializer {
   public static final int IPC_CONTINUATION_TOKEN = -1;
 
   /**
-   * Convert an array of 4 bytes to a little endian i32 value.
+   * Convert an array of 4 bytes in little-endian to an native-endian i32 value.
    *
-   * @param bytes byte array with minimum length of 4
-   * @return converted little endian 32-bit integer
+   * @param bytes byte array with minimum length of 4 in little-endian
+   * @return converted an native-endian 32-bit integer
    */
   public static int bytesToInt(byte[] bytes) {
     return ((bytes[3] & 255) << 24) +
@@ -74,7 +75,7 @@ public static int bytesToInt(byte[] bytes) {
   }
 
   /**
-   * Convert an integer to a 4 byte array.
+   * Convert an integer to a little endian 4 byte array.
    *
    * @param value integer value input
    * @param bytes existing byte array with minimum length of 4 to contain the conversion output
@@ -87,7 +88,7 @@ public static void intToBytes(int value, byte[] bytes) {
   }
 
   /**
-   * Convert a long to a 8 byte array.
+   * Convert a long to a little-endian 8 byte array.
    *
    * @param value long value input
    * @param bytes existing byte array with minimum length of 8 to contain the conversion output
@@ -425,11 +426,16 @@ public static ArrowRecordBatch deserializeRecordBatch(RecordBatch recordBatchFB,
       ArrowBuf vectorBuffer = body.slice(bufferFB.offset(), bufferFB.length());
       buffers.add(vectorBuffer);
     }
+
+    ArrowBodyCompression bodyCompression = recordBatchFB.compression() == null ?
+        NoCompressionCodec.DEFAULT_BODY_COMPRESSION
+        : new ArrowBodyCompression(recordBatchFB.compression().codec(), recordBatchFB.compression().method());
+
     if ((int) recordBatchFB.length() != recordBatchFB.length()) {
       throw new IOException("Cannot currently deserialize record batches with more than INT_MAX records.");
     }
     ArrowRecordBatch arrowRecordBatch =
-        new ArrowRecordBatch(checkedCastToInt(recordBatchFB.length()), nodes, buffers);
+        new ArrowRecordBatch(checkedCastToInt(recordBatchFB.length()), nodes, buffers, bodyCompression);
     body.getReferenceManager().release();
     return arrowRecordBatch;
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 886478ce403..e93bc695917 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -27,6 +27,7 @@
 import org.apache.arrow.vector.BitVector;
 import org.apache.arrow.vector.DateDayVector;
 import org.apache.arrow.vector.DateMilliVector;
+import org.apache.arrow.vector.Decimal256Vector;
 import org.apache.arrow.vector.DecimalVector;
 import org.apache.arrow.vector.DurationVector;
 import org.apache.arrow.vector.ExtensionTypeVector;
@@ -72,6 +73,7 @@
 import org.apache.arrow.vector.complex.impl.BitWriterImpl;
 import org.apache.arrow.vector.complex.impl.DateDayWriterImpl;
 import org.apache.arrow.vector.complex.impl.DateMilliWriterImpl;
+import org.apache.arrow.vector.complex.impl.Decimal256WriterImpl;
 import org.apache.arrow.vector.complex.impl.DecimalWriterImpl;
 import org.apache.arrow.vector.complex.impl.DenseUnionWriter;
 import org.apache.arrow.vector.complex.impl.DurationWriterImpl;
@@ -528,6 +530,20 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new DecimalWriterImpl((DecimalVector) vector);
       }
     },
+    DECIMAL256(null) {
+      @Override
+      public FieldVector getNewVector(
+          Field field,
+          BufferAllocator allocator,
+          CallBack schemaChangeCallback) {
+        return new Decimal256Vector(field, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new Decimal256WriterImpl((Decimal256Vector) vector);
+      }
+    },
     FIXEDSIZEBINARY(null) {
       @Override
       public FieldVector getNewVector(
@@ -899,6 +915,9 @@ public MinorType visit(Bool type) {
 
       @Override
       public MinorType visit(Decimal type) {
+        if (type.getBitWidth() == 256) {
+          return MinorType.DECIMAL256;
+        }
         return MinorType.DECIMAL;
       }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/util/DataSizeRoundingUtil.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DataSizeRoundingUtil.java
similarity index 98%
rename from java/vector/src/main/java/org/apache/arrow/util/DataSizeRoundingUtil.java
rename to java/vector/src/main/java/org/apache/arrow/vector/util/DataSizeRoundingUtil.java
index 2946503cb64..3af2c983740 100644
--- a/java/vector/src/main/java/org/apache/arrow/util/DataSizeRoundingUtil.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DataSizeRoundingUtil.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.util;
+package org.apache.arrow.vector.util;
 
 /**
  * Utilities for rounding data size.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
index 711fa3b9cbf..f778bcb209f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
@@ -20,6 +20,7 @@
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 
 import org.apache.arrow.memory.ArrowBuf;
 
@@ -31,27 +32,31 @@
 public class DecimalUtility {
   private DecimalUtility() {}
 
-  public static final int DECIMAL_BYTE_LENGTH = 16;
-  public static final byte [] zeroes = new byte[] {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-  public static final byte [] minus_one = new byte[] {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1};
+  public static final byte [] zeroes = new byte[] {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                                                   0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+  public static final byte [] minus_one = new byte[] {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+                                                      -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1};
+  private static final boolean LITTLE_ENDIAN = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
 
   /**
    * Read an ArrowType.Decimal at the given value index in the ArrowBuf and convert to a BigDecimal
    * with the given scale.
    */
-  public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int index, int scale) {
-    byte[] value = new byte[DECIMAL_BYTE_LENGTH];
+  public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int index, int scale, int byteWidth) {
+    byte[] value = new byte[byteWidth];
     byte temp;
-    final int startIndex = index * DECIMAL_BYTE_LENGTH;
-
-    // Decimal stored as little endian, need to swap bytes to make BigDecimal
-    bytebuf.getBytes(startIndex, value, 0, DECIMAL_BYTE_LENGTH);
-    int stop = DECIMAL_BYTE_LENGTH / 2;
-    for (int i = 0, j; i < stop; i++) {
-      temp = value[i];
-      j = (DECIMAL_BYTE_LENGTH - 1) - i;
-      value[i] = value[j];
-      value[j] = temp;
+    final long startIndex = (long) index * byteWidth;
+
+    bytebuf.getBytes(startIndex, value, 0, byteWidth);
+    if (LITTLE_ENDIAN) {
+      // Decimal stored as native endian, need to swap bytes to make BigDecimal if native endian is LE
+      int stop = byteWidth / 2;
+      for (int i = 0, j; i < stop; i++) {
+        temp = value[i];
+        j = (byteWidth - 1) - i;
+        value[i] = value[j];
+        value[j] = temp;
+      }
     }
     BigInteger unscaledValue = new BigInteger(value);
     return new BigDecimal(unscaledValue, scale);
@@ -61,8 +66,8 @@ public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int index,
    * Read an ArrowType.Decimal from the ByteBuffer and convert to a BigDecimal with the given
    * scale.
    */
-  public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int scale) {
-    byte[] value = new byte[DECIMAL_BYTE_LENGTH];
+  public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int scale, int byteWidth) {
+    byte[] value = new byte[byteWidth];
     bytebuf.get(value);
     BigInteger unscaledValue = new BigInteger(value);
     return new BigDecimal(unscaledValue, scale);
@@ -72,10 +77,10 @@ public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int sca
    * Read an ArrowType.Decimal from the ArrowBuf at the given value index and return it as a byte
    * array.
    */
-  public static byte[] getByteArrayFromArrowBuf(ArrowBuf bytebuf, int index) {
-    final byte[] value = new byte[DECIMAL_BYTE_LENGTH];
-    final long startIndex = (long) index * DECIMAL_BYTE_LENGTH;
-    bytebuf.getBytes(startIndex, value, 0, DECIMAL_BYTE_LENGTH);
+  public static byte[] getByteArrayFromArrowBuf(ArrowBuf bytebuf, int index, int byteWidth) {
+    final byte[] value = new byte[byteWidth];
+    final long startIndex = (long) index * byteWidth;
+    bytebuf.getBytes(startIndex, value, 0, byteWidth);
     return value;
   }
 
@@ -119,19 +124,33 @@ public static boolean checkPrecisionAndScale(int decimalPrecision, int decimalSc
    * UnsupportedOperationException if the decimal size is greater than the Decimal vector byte
    * width.
    */
-  public static void writeBigDecimalToArrowBuf(BigDecimal value, ArrowBuf bytebuf, int index) {
+  public static void writeBigDecimalToArrowBuf(BigDecimal value, ArrowBuf bytebuf, int index, int byteWidth) {
     final byte[] bytes = value.unscaledValue().toByteArray();
-    writeByteArrayToArrowBufHelper(bytes, bytebuf, index);
+    writeByteArrayToArrowBufHelper(bytes, bytebuf, index, byteWidth);
   }
 
   /**
    * Write the given long to the ArrowBuf at the given value index.
+   * This routine extends the original sign bit to a new upper area in 128-bit or 256-bit.
    */
-  public static void writeLongToArrowBuf(long value, ArrowBuf bytebuf, int index) {
-    final long addressOfValue = bytebuf.memoryAddress() + (long) index * DECIMAL_BYTE_LENGTH;
-    PlatformDependent.putLong(addressOfValue, value);
+  public static void writeLongToArrowBuf(long value, ArrowBuf bytebuf, int index, int byteWidth) {
+    if (byteWidth != 16 && byteWidth != 32) {
+      throw new UnsupportedOperationException("DecimalUtility.writeLongToArrowBuf() currently supports " +
+          "128-bit or 256-bit width data");
+    }
+    final long addressOfValue = bytebuf.memoryAddress() + (long) index * byteWidth;
     final long padValue = Long.signum(value) == -1 ? -1L : 0L;
-    PlatformDependent.putLong(addressOfValue + Long.BYTES, padValue);
+    if (LITTLE_ENDIAN) {
+      PlatformDependent.putLong(addressOfValue, value);
+      for (int i = 1; i <= (byteWidth - 8) / 8; i++) {
+        PlatformDependent.putLong(addressOfValue + Long.BYTES * i, padValue);
+      }
+    } else {
+      for (int i = 0; i < (byteWidth - 8) / 8; i++) {
+        PlatformDependent.putLong(addressOfValue + Long.BYTES * i, padValue);
+      }
+      PlatformDependent.putLong(addressOfValue + Long.BYTES * (byteWidth - 8) / 8, value);
+    }
   }
 
   /**
@@ -139,25 +158,31 @@ public static void writeLongToArrowBuf(long value, ArrowBuf bytebuf, int index)
    * UnsupportedOperationException if the decimal size is greater than the Decimal vector byte
    * width.
    */
-  public static void writeByteArrayToArrowBuf(byte[] bytes, ArrowBuf bytebuf, int index) {
-    writeByteArrayToArrowBufHelper(bytes, bytebuf, index);
+  public static void writeByteArrayToArrowBuf(byte[] bytes, ArrowBuf bytebuf, int index, int byteWidth) {
+    writeByteArrayToArrowBufHelper(bytes, bytebuf, index, byteWidth);
   }
 
-  private static void writeByteArrayToArrowBufHelper(byte[] bytes, ArrowBuf bytebuf, int index) {
-    final long startIndex = (long) index * DECIMAL_BYTE_LENGTH;
-    if (bytes.length > DECIMAL_BYTE_LENGTH) {
-      throw new UnsupportedOperationException("Decimal size greater than 16 bytes");
+  private static void writeByteArrayToArrowBufHelper(byte[] bytes, ArrowBuf bytebuf, int index, int byteWidth) {
+    final long startIndex = (long) index * byteWidth;
+    if (bytes.length > byteWidth) {
+      throw new UnsupportedOperationException("Decimal size greater than " + byteWidth + " bytes: " + bytes.length);
     }
 
-    // Decimal stored as little endian, need to swap data bytes before writing to ArrowBuf
-    byte[] bytesLE = new byte[bytes.length];
-    for (int i = 0; i < bytes.length; i++) {
-      bytesLE[i] = bytes[bytes.length - 1 - i];
+    byte [] padBytes = bytes[0] < 0 ? minus_one : zeroes;
+    if (LITTLE_ENDIAN) {
+      // Decimal stored as native-endian, need to swap data bytes before writing to ArrowBuf if LE
+      byte[] bytesLE = new byte[bytes.length];
+      for (int i = 0; i < bytes.length; i++) {
+        bytesLE[i] = bytes[bytes.length - 1 - i];
+      }
+
+      // Write LE data
+      bytebuf.setBytes(startIndex, bytesLE, 0, bytes.length);
+      bytebuf.setBytes(startIndex + bytes.length, padBytes, 0, byteWidth - bytes.length);
+    } else {
+      // Write BE data
+      bytebuf.setBytes(startIndex + byteWidth - bytes.length, bytes, 0, bytes.length);
+      bytebuf.setBytes(startIndex, padBytes, 0, byteWidth - bytes.length);
     }
-
-    // Write LE data
-    byte [] padByes = bytes[0] < 0 ? minus_one : zeroes;
-    bytebuf.setBytes(startIndex, bytesLE, 0, bytes.length);
-    bytebuf.setBytes(startIndex + bytes.length, padByes, 0, DECIMAL_BYTE_LENGTH - bytes.length);
   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DictionaryUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DictionaryUtility.java
index 345fa592241..9592f3975ab 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/DictionaryUtility.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DictionaryUtility.java
@@ -49,16 +49,13 @@ public static Field toMessageFormat(Field field, DictionaryProvider provider, Se
       return field;
     }
     DictionaryEncoding encoding = field.getDictionary();
-    List<Field> children = field.getChildren();
+    List<Field> children;
 
-    List<Field> updatedChildren = new ArrayList<>(children.size());
-    for (Field child : children) {
-      updatedChildren.add(toMessageFormat(child, provider, dictionaryIdsUsed));
-    }
 
     ArrowType type;
     if (encoding == null) {
       type = field.getType();
+      children = field.getChildren();
     } else {
       long id = encoding.getId();
       Dictionary dictionary = provider.lookup(id);
@@ -66,10 +63,16 @@ public static Field toMessageFormat(Field field, DictionaryProvider provider, Se
         throw new IllegalArgumentException("Could not find dictionary with ID " + id);
       }
       type = dictionary.getVectorType();
+      children = dictionary.getVector().getField().getChildren();
 
       dictionaryIdsUsed.add(id);
     }
 
+    final List<Field> updatedChildren = new ArrayList<>(children.size());
+    for (Field child : children) {
+      updatedChildren.add(toMessageFormat(child, provider, dictionaryIdsUsed));
+    }
+
     return new Field(field.getName(), new FieldType(field.isNullable(), type, encoding, field.getMetadata()),
       updatedChildren);
   }
@@ -115,8 +118,10 @@ public static Field toMemoryFormat(Field field, BufferAllocator allocator, Map<L
     }
 
     ArrowType type;
+    List<Field> fieldChildren = null;
     if (encoding == null) {
       type = field.getType();
+      fieldChildren = updatedChildren;
     } else {
       // re-type the field for in-memory format
       type = encoding.getIndexType();
@@ -127,13 +132,14 @@ public static Field toMemoryFormat(Field field, BufferAllocator allocator, Map<L
       if (!dictionaries.containsKey(encoding.getId())) {
         // create a new dictionary vector for the values
         String dictName = "DICT" + encoding.getId();
-        Field dictionaryField = new Field(dictName, new FieldType(false, field.getType(), null, null), children);
+        Field dictionaryField = new Field(dictName,
+            new FieldType(field.isNullable(), field.getType(), null, null), updatedChildren);
         FieldVector dictionaryVector = dictionaryField.createVector(allocator);
         dictionaries.put(encoding.getId(), new Dictionary(dictionaryVector, encoding));
       }
     }
 
     return new Field(field.getName(), new FieldType(field.isNullable(), type, encoding, field.getMetadata()),
-      updatedChildren);
+      fieldChildren);
   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/ValueVectorUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/ValueVectorUtility.java
index d9c1173c11d..60553b4e342 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/ValueVectorUtility.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/ValueVectorUtility.java
@@ -17,22 +17,24 @@
 
 package org.apache.arrow.vector.util;
 
-import org.apache.arrow.memory.ArrowBuf;
+import static org.apache.arrow.vector.validate.ValidateUtil.validateOrThrow;
+
 import org.apache.arrow.util.Preconditions;
 import org.apache.arrow.vector.BaseFixedWidthVector;
-import org.apache.arrow.vector.BufferLayout;
-import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.TypeLayout;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.types.pojo.ArrowType;
-import org.apache.arrow.vector.validate.ValidateVectorVisitor;
+import org.apache.arrow.vector.validate.ValidateVectorBufferVisitor;
+import org.apache.arrow.vector.validate.ValidateVectorDataVisitor;
+import org.apache.arrow.vector.validate.ValidateVectorTypeVisitor;
 
 /**
  * Utility methods for {@link ValueVector}.
  */
 public class ValueVectorUtility {
 
+  private ValueVectorUtility() {
+  }
+
   /**
    * Get the toString() representation of vector suitable for debugging.
    * Note since vectors may have millions of values, this method only show max 20 values.
@@ -92,44 +94,57 @@ public static String getToString(ValueVector vector, int start, int end) {
   }
 
   /**
-   * Validate field vector.
+   * Utility to validate vector in O(1) time.
    */
-  public static void validate(FieldVector vector) {
+  public static void validate(ValueVector vector) {
     Preconditions.checkNotNull(vector);
 
-    ArrowType arrowType = vector.getField().getType();
-    int typeBufferCount = TypeLayout.getTypeBufferCount(arrowType);
-    TypeLayout typeLayout = TypeLayout.getTypeLayout(arrowType);
+    ValidateVectorTypeVisitor typeVisitor = new ValidateVectorTypeVisitor();
+    vector.accept(typeVisitor, null);
 
-    if (vector.getValueCount() < 0) {
-      throw new IllegalArgumentException("vector valueCount is negative");
-    }
+    ValidateVectorBufferVisitor bufferVisitor = new ValidateVectorBufferVisitor();
+    vector.accept(bufferVisitor, null);
+  }
 
-    if (vector.getFieldBuffers().size() != typeBufferCount) {
-      throw new IllegalArgumentException(String.format("Expected %s buffers in vector of type %s, got %s",
-          typeBufferCount, vector.getField().getType().toString(), vector.getBufferSize()));
-    }
+  /**
+   * Utility to validate vector in O(n) time, where n is the value count.
+   */
+  public static void validateFull(ValueVector vector) {
+    validate(vector);
 
-    for (int i = 0; i < typeBufferCount; i++) {
-      ArrowBuf buffer = vector.getFieldBuffers().get(i);
-      BufferLayout bufferLayout = typeLayout.getBufferLayouts().get(i);
-      if (buffer == null) {
-        continue;
-      }
-      int minBufferSize = vector.getValueCount() * bufferLayout.getTypeBitWidth();
+    ValidateVectorDataVisitor dataVisitor = new ValidateVectorDataVisitor();
+    vector.accept(dataVisitor, null);
+  }
 
-      if (buffer.capacity() < minBufferSize / 8) {
-        throw new IllegalArgumentException(String.format("Buffer #%s too small in vector of type %s" +
-                "and valueCount %s : expected at least %s byte(s), got %s",
-            i, vector.getField().getType().toString(),
-            vector.getValueCount(), minBufferSize, buffer.capacity()));
-      }
+  /**
+   * Utility to validate vector schema root in O(1) time.
+   */
+  public static void validate(VectorSchemaRoot root) {
+    Preconditions.checkNotNull(root);
+    int valueCount = root.getRowCount();
+    validateOrThrow(valueCount >= 0, "The row count of vector schema root %s is negative.", valueCount);
+    for (ValueVector childVec : root.getFieldVectors()) {
+      validateOrThrow(valueCount == childVec.getValueCount(),
+          "Child vector and vector schema root have different value counts. " +
+              "Child vector value count %s, vector schema root value count %s", childVec.getValueCount(), valueCount);
+      validate(childVec);
     }
-
-    ValidateVectorVisitor visitor = new ValidateVectorVisitor();
-    vector.accept(visitor, null);
   }
 
+  /**
+   * Utility to validate vector in O(n) time, where n is the value count.
+   */
+  public static void validateFull(VectorSchemaRoot root) {
+    Preconditions.checkNotNull(root);
+    int valueCount = root.getRowCount();
+    validateOrThrow(valueCount >= 0, "The row count of vector schema root %s is negative.", valueCount);
+    for (ValueVector childVec : root.getFieldVectors()) {
+      validateOrThrow(valueCount == childVec.getValueCount(),
+          "Child vector and vector schema root have different value counts. " +
+              "Child vector value count %s, vector schema root value count %s", childVec.getValueCount(), valueCount);
+      validateFull(childVec);
+    }
+  }
 
   /**
    * Pre allocate memory for BaseFixedWidthVector.
@@ -154,5 +169,4 @@ public static void ensureCapacity(VectorSchemaRoot root, int targetCapacity) {
       }
     }
   }
-
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateUtil.java b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateUtil.java
new file mode 100644
index 00000000000..e1b60e92616
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateUtil.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.validate;
+
+/**
+ * Utilities for vector validation.
+ */
+public class ValidateUtil {
+
+  private ValidateUtil() {
+  }
+
+  /**
+   * Validate the expression.
+   * @param expression the expression to validate.
+   * @param errorMessage the error message.
+   * @throws ValidateException if the expression evaluates to false.
+   */
+  public static void validateOrThrow(boolean expression, String errorMessage) {
+    if (!expression) {
+      throw new ValidateException(errorMessage);
+    }
+  }
+
+  /**
+   * Validate the expression.
+   * @param expression the expression to validate.
+   * @param errorMessage the error message template.
+   * @param args the error message arguments.
+   * @throws ValidateException if the expression evaluates to false.
+   */
+  public static void validateOrThrow(boolean expression, String errorMessage, Object... args) {
+    if (!expression) {
+      throw new ValidateException(String.format(errorMessage, args));
+    }
+  }
+
+  /**
+   * A exception that is thrown when the vector validation fails.
+   */
+  public static class ValidateException extends RuntimeException {
+    public ValidateException(String message) {
+      super(message);
+    }
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java
new file mode 100644
index 00000000000..46064c37bdd
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.validate;
+
+import static org.apache.arrow.vector.validate.ValidateUtil.validateOrThrow;
+
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.vector.BaseFixedWidthVector;
+import org.apache.arrow.vector.BaseLargeVariableWidthVector;
+import org.apache.arrow.vector.BaseVariableWidthVector;
+import org.apache.arrow.vector.BitVector;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.NullVector;
+import org.apache.arrow.vector.TypeLayout;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.compare.VectorVisitor;
+import org.apache.arrow.vector.complex.DenseUnionVector;
+import org.apache.arrow.vector.complex.FixedSizeListVector;
+import org.apache.arrow.vector.complex.LargeListVector;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.NonNullableStructVector;
+import org.apache.arrow.vector.complex.UnionVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+
+/**
+ * Visitor to validate vector buffers.
+ */
+public class ValidateVectorBufferVisitor implements VectorVisitor<Void, Void> {
+
+  private void validateVectorCommon(ValueVector vector) {
+    ArrowType arrowType = vector.getField().getType();
+    validateOrThrow(vector.getValueCount() >= 0,
+        "Vector valueCount %s is negative.", vector.getValueCapacity());
+
+    if (vector instanceof FieldVector) {
+      FieldVector fieldVector = (FieldVector) vector;
+      int typeBufferCount = TypeLayout.getTypeBufferCount(arrowType);
+      validateOrThrow(fieldVector.getFieldBuffers().size() == typeBufferCount,
+          "Expected %s buffers in vector of type %s, got %s.",
+              typeBufferCount, vector.getField().getType().toString(), fieldVector.getFieldBuffers().size());
+    }
+  }
+
+  private void validateValidityBuffer(ValueVector vector, int valueCount) {
+    ArrowBuf validityBuffer = vector.getValidityBuffer();
+    validateOrThrow(validityBuffer != null, "The validity buffer is null.");
+    validateOrThrow(validityBuffer.capacity() * 8 >= valueCount,
+        "Not enough capacity for the validity buffer. Minimum capacity %s, actual capacity %s.",
+        (valueCount + 7) / 8, validityBuffer.capacity());
+  }
+
+  private void validateOffsetBuffer(ValueVector vector, long minCapacity) {
+    ArrowBuf offsetBuffer = vector.getOffsetBuffer();
+    validateOrThrow(offsetBuffer != null, "The offset buffer is null.");
+    validateOrThrow(offsetBuffer.capacity() >= minCapacity,
+        "Not enough capacity for the offset buffer. Minimum capacity %s, actual capacity %s.",
+        minCapacity, offsetBuffer.capacity());
+  }
+
+  private void validateFixedWidthDataBuffer(ValueVector vector, int valueCount, int bitWidth) {
+    ArrowBuf dataBuffer = vector.getDataBuffer();
+    validateOrThrow(dataBuffer != null, "The fixed width data buffer is null.");
+    validateOrThrow((long) bitWidth * valueCount <= dataBuffer.capacity() * 8L,
+        "Not enough capacity for fixed width data buffer. Minimum capacity %s, actual capacity %s.",
+        ((long) bitWidth * valueCount + 7L) / 8L, dataBuffer.capacity());
+  }
+
+  private void validateDataBuffer(ValueVector vector, long minCapacity) {
+    ArrowBuf dataBuffer = vector.getDataBuffer();
+    validateOrThrow(dataBuffer != null, "The data buffer is null.");
+    validateOrThrow(dataBuffer.capacity() >= minCapacity,
+        "Not enough capacity for data buffer. Minimum capacity %s, actual capacity %s.",
+        minCapacity, dataBuffer.capacity());
+  }
+
+  private void validateTypeBuffer(ArrowBuf typeBuf, long minCapacity) {
+    validateOrThrow(typeBuf != null, "The type buffer is null.");
+    validateOrThrow(typeBuf.capacity() >= minCapacity,
+        "Not enough capacity for type buffer. Minimum capacity %s, actual capacity %s.",
+        minCapacity, typeBuf.capacity());
+  }
+
+  @Override
+  public Void visit(BaseFixedWidthVector vector, Void value) {
+    int bitWidth = (vector instanceof BitVector) ? 1 : vector.getTypeWidth() * 8;
+    int valueCount = vector.getValueCount();
+    validateVectorCommon(vector);
+    validateValidityBuffer(vector, valueCount);
+    validateFixedWidthDataBuffer(vector, valueCount, bitWidth);
+    return null;
+  }
+
+  @Override
+  public Void visit(BaseVariableWidthVector vector, Void value) {
+    int valueCount = vector.getValueCount();
+    validateVectorCommon(vector);
+    validateValidityBuffer(vector, valueCount);
+    long minOffsetCapacity = valueCount == 0 ? 0L : (long) (valueCount + 1) * BaseVariableWidthVector.OFFSET_WIDTH;
+    validateOffsetBuffer(vector, minOffsetCapacity);
+    int lastOffset = valueCount == 0 ? 0 :
+        vector.getOffsetBuffer().getInt(valueCount * BaseVariableWidthVector.OFFSET_WIDTH);
+    validateDataBuffer(vector, lastOffset);
+    return null;
+  }
+
+  @Override
+  public Void visit(BaseLargeVariableWidthVector vector, Void value) {
+    int valueCount = vector.getValueCount();
+    validateVectorCommon(vector);
+    validateValidityBuffer(vector, valueCount);
+    long minOffsetCapacity = valueCount == 0 ? 0L
+        : (long) (valueCount + 1) * BaseLargeVariableWidthVector.OFFSET_WIDTH;
+    validateOffsetBuffer(vector, minOffsetCapacity);
+    long lastOffset = valueCount == 0 ? 0L :
+        vector.getOffsetBuffer().getLong((long) valueCount * BaseLargeVariableWidthVector.OFFSET_WIDTH);
+    validateDataBuffer(vector, lastOffset);
+    return null;
+  }
+
+  @Override
+  public Void visit(ListVector vector, Void value) {
+    int valueCount = vector.getValueCount();
+    validateVectorCommon(vector);
+    validateValidityBuffer(vector, valueCount);
+    long minOffsetCapacity = valueCount == 0 ? 0L : (long) (valueCount + 1) * ListVector.OFFSET_WIDTH;
+    validateOffsetBuffer(vector, minOffsetCapacity);
+
+    FieldVector dataVector = vector.getDataVector();
+    int lastOffset = valueCount == 0 ? 0 :
+        vector.getOffsetBuffer().getInt(valueCount * BaseVariableWidthVector.OFFSET_WIDTH);
+    int dataVectorLength = dataVector == null ? 0 : dataVector.getValueCount();
+    validateOrThrow(dataVectorLength >= lastOffset,
+        "Inner vector does not contain enough elements. Minimum element count %s, actual element count %s",
+        lastOffset + 1, dataVectorLength);
+
+    if (dataVector != null) {
+      dataVector.accept(this, null);
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(FixedSizeListVector vector, Void value) {
+    int valueCount = vector.getValueCount();
+    validateVectorCommon(vector);
+    validateValidityBuffer(vector, valueCount);
+    FieldVector dataVector = vector.getDataVector();
+    int dataVectorLength = dataVector == null ? 0 : dataVector.getValueCount();
+    validateOrThrow(dataVectorLength >= valueCount * vector.getListSize(),
+        "Inner vector does not contain enough elements. Minimum element count %s, actual element count %s.",
+        valueCount * vector.getListSize(), dataVectorLength);
+    if (dataVector != null) {
+      dataVector.accept(this, null);
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(LargeListVector vector, Void value) {
+    int valueCount = vector.getValueCount();
+    validateVectorCommon(vector);
+    validateValidityBuffer(vector, valueCount);
+    long minOffsetCapacity = valueCount == 0 ? 0L : (long) (valueCount + 1) * LargeListVector.OFFSET_WIDTH;
+    validateOffsetBuffer(vector, minOffsetCapacity);
+
+    FieldVector dataVector = vector.getDataVector();
+    long lastOffset = valueCount == 0 ? 0 :
+        vector.getOffsetBuffer().getLong(valueCount * BaseLargeVariableWidthVector.OFFSET_WIDTH);
+    int dataVectorLength = dataVector == null ? 0 : dataVector.getValueCount();
+    validateOrThrow(dataVectorLength >= lastOffset,
+        "Inner vector does not contain enough elements. Minimum element count %s, actual element count %s",
+        lastOffset + 1, dataVectorLength);
+
+    if (dataVector != null) {
+      dataVector.accept(this, null);
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(NonNullableStructVector vector, Void value) {
+    int valueCount = vector.getValueCount();
+    validateVectorCommon(vector);
+    validateValidityBuffer(vector, valueCount);
+    for (ValueVector subVector : vector.getChildrenFromFields()) {
+      validateOrThrow(valueCount == subVector.getValueCount(),
+          "Struct vector length not equal to child vector length. Struct vector length %s, child vector length %s",
+          valueCount, subVector.getValueCount());
+      subVector.accept(this, null);
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(UnionVector vector, Void value) {
+    int valueCount = vector.getValueCount();
+    validateVectorCommon(vector);
+    validateTypeBuffer(vector.getTypeBuffer(), valueCount * UnionVector.TYPE_WIDTH);
+    for (ValueVector subVector : vector.getChildrenFromFields()) {
+      validateOrThrow(valueCount == subVector.getValueCount(),
+          "Union vector length not equal to child vector length. Union vector length %s, child vector length %s",
+          valueCount, subVector.getValueCount());
+      subVector.accept(this, null);
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(DenseUnionVector vector, Void value) {
+    int valueCount = vector.getValueCount();
+    validateVectorCommon(vector);
+    validateOffsetBuffer(vector, (long) valueCount * DenseUnionVector.OFFSET_WIDTH);
+    validateTypeBuffer(vector.getTypeBuffer(), valueCount * DenseUnionVector.TYPE_WIDTH);
+    for (ValueVector subVector : vector.getChildrenFromFields()) {
+      subVector.accept(this, null);
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(NullVector vector, Void value) {
+    return null;
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java
new file mode 100644
index 00000000000..23a0beeb51f
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.validate;
+
+import static org.apache.arrow.vector.validate.ValidateUtil.validateOrThrow;
+
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.vector.BaseFixedWidthVector;
+import org.apache.arrow.vector.BaseLargeVariableWidthVector;
+import org.apache.arrow.vector.BaseVariableWidthVector;
+import org.apache.arrow.vector.NullVector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.compare.VectorVisitor;
+import org.apache.arrow.vector.complex.DenseUnionVector;
+import org.apache.arrow.vector.complex.FixedSizeListVector;
+import org.apache.arrow.vector.complex.LargeListVector;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.NonNullableStructVector;
+import org.apache.arrow.vector.complex.UnionVector;
+
+/**
+ * Utility for validating vector data.
+ */
+public class ValidateVectorDataVisitor implements VectorVisitor<Void, Void> {
+
+  private void validateOffsetBuffer(ValueVector vector, int valueCount) {
+    if (valueCount == 0) {
+      return;
+    }
+    ArrowBuf offsetBuffer = vector.getOffsetBuffer();
+
+    // verify that the values in the offset buffer is non-decreasing
+    int prevValue = offsetBuffer.getInt(0);
+    for (int i = 1; i <= valueCount; i++) {
+      int curValue = offsetBuffer.getInt(i * 4);
+      validateOrThrow(curValue >= 0, "The value at position %s of the offset buffer is negative: %s.", i, curValue);
+      validateOrThrow(curValue >= prevValue,
+          "The values in positions %s and %s of the offset buffer are decreasing: %s, %s.",
+          i - 1, i, prevValue, curValue);
+      prevValue = curValue;
+    }
+  }
+
+  private void validateLargeOffsetBuffer(ValueVector vector, int valueCount) {
+    if (valueCount == 0) {
+      return;
+    }
+    ArrowBuf offsetBuffer = vector.getOffsetBuffer();
+
+    // verify that the values in the large offset buffer is non-decreasing
+    long prevValue = offsetBuffer.getLong(0);
+    for (int i = 1; i <= valueCount; i++) {
+      long curValue = offsetBuffer.getLong((long) i * 8);
+      validateOrThrow(curValue >= 0L, "The value at position %s of the large offset buffer is negative: %s.",
+          i, curValue);
+      validateOrThrow(curValue >= prevValue,
+          "The values in positions %s and %s of the large offset buffer are decreasing: %s, %s.",
+          i - 1, i, prevValue, curValue);
+      prevValue = curValue;
+    }
+  }
+
+  private void validateTypeBuffer(ArrowBuf typeBuf, int valueCount) {
+    for (int i = 0; i < valueCount; i++) {
+      validateOrThrow(typeBuf.getByte(i) >= 0, "The type id at position %s is negative: %s.",
+          i, typeBuf.getByte(i));
+    }
+  }
+
+  @Override
+  public Void visit(BaseFixedWidthVector vector, Void value) {
+    return null;
+  }
+
+  @Override
+  public Void visit(BaseVariableWidthVector vector, Void value) {
+    validateOffsetBuffer(vector, vector.getValueCount());
+    return null;
+  }
+
+  @Override
+  public Void visit(BaseLargeVariableWidthVector vector, Void value) {
+    validateLargeOffsetBuffer(vector, vector.getValueCount());
+    return null;
+  }
+
+  @Override
+  public Void visit(ListVector vector, Void value) {
+    validateOffsetBuffer(vector, vector.getValueCount());
+    ValueVector innerVector = vector.getDataVector();
+    if (innerVector != null) {
+      innerVector.accept(this, null);
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(FixedSizeListVector vector, Void value) {
+    validateOffsetBuffer(vector, vector.getValueCount());
+    ValueVector innerVector = vector.getDataVector();
+    if (innerVector != null) {
+      innerVector.accept(this, null);
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(LargeListVector vector, Void value) {
+    validateLargeOffsetBuffer(vector, vector.getValueCount());
+    ValueVector innerVector = vector.getDataVector();
+    if (innerVector != null) {
+      innerVector.accept(this, null);
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(NonNullableStructVector vector, Void value) {
+    for (ValueVector subVector : vector.getChildrenFromFields()) {
+      subVector.accept(this, null);
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(UnionVector vector, Void value) {
+    validateTypeBuffer(vector.getTypeBuffer(), vector.getValueCount());
+    for (ValueVector subVector : vector.getChildrenFromFields()) {
+      subVector.accept(this, null);
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(DenseUnionVector vector, Void value) {
+    validateTypeBuffer(vector.getTypeBuffer(), vector.getValueCount());
+
+    // validate offset buffer
+    for (int i = 0; i < vector.getValueCount(); i++) {
+      int offset = vector.getOffset(i);
+      byte typeId = vector.getTypeId(i);
+      ValueVector subVector = vector.getVectorByType(typeId);
+      validateOrThrow(offset < subVector.getValueCount(),
+          "Dense union vector offset exceeds sub-vector boundary. Vector offset %s, sub vector size %s",
+          offset, subVector.getValueCount());
+    }
+
+    for (ValueVector subVector : vector.getChildrenFromFields()) {
+      subVector.accept(this, null);
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(NullVector vector, Void value) {
+    return null;
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java
new file mode 100644
index 00000000000..de00c6d8ff3
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java
@@ -0,0 +1,356 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.validate;
+
+import static org.apache.arrow.vector.validate.ValidateUtil.validateOrThrow;
+
+import org.apache.arrow.vector.BaseFixedWidthVector;
+import org.apache.arrow.vector.BaseLargeVariableWidthVector;
+import org.apache.arrow.vector.BaseVariableWidthVector;
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.BitVector;
+import org.apache.arrow.vector.DateDayVector;
+import org.apache.arrow.vector.DateMilliVector;
+import org.apache.arrow.vector.Decimal256Vector;
+import org.apache.arrow.vector.DecimalVector;
+import org.apache.arrow.vector.DurationVector;
+import org.apache.arrow.vector.FixedSizeBinaryVector;
+import org.apache.arrow.vector.Float4Vector;
+import org.apache.arrow.vector.Float8Vector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.IntervalDayVector;
+import org.apache.arrow.vector.IntervalYearVector;
+import org.apache.arrow.vector.LargeVarBinaryVector;
+import org.apache.arrow.vector.LargeVarCharVector;
+import org.apache.arrow.vector.NullVector;
+import org.apache.arrow.vector.SmallIntVector;
+import org.apache.arrow.vector.TimeMicroVector;
+import org.apache.arrow.vector.TimeMilliVector;
+import org.apache.arrow.vector.TimeNanoVector;
+import org.apache.arrow.vector.TimeSecVector;
+import org.apache.arrow.vector.TimeStampMicroTZVector;
+import org.apache.arrow.vector.TimeStampMicroVector;
+import org.apache.arrow.vector.TimeStampMilliTZVector;
+import org.apache.arrow.vector.TimeStampMilliVector;
+import org.apache.arrow.vector.TimeStampNanoTZVector;
+import org.apache.arrow.vector.TimeStampNanoVector;
+import org.apache.arrow.vector.TimeStampSecTZVector;
+import org.apache.arrow.vector.TimeStampSecVector;
+import org.apache.arrow.vector.TinyIntVector;
+import org.apache.arrow.vector.UInt1Vector;
+import org.apache.arrow.vector.UInt2Vector;
+import org.apache.arrow.vector.UInt4Vector;
+import org.apache.arrow.vector.UInt8Vector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VarBinaryVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.compare.VectorVisitor;
+import org.apache.arrow.vector.complex.DenseUnionVector;
+import org.apache.arrow.vector.complex.FixedSizeListVector;
+import org.apache.arrow.vector.complex.LargeListVector;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.NonNullableStructVector;
+import org.apache.arrow.vector.complex.UnionVector;
+import org.apache.arrow.vector.types.DateUnit;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
+import org.apache.arrow.vector.types.IntervalUnit;
+import org.apache.arrow.vector.types.TimeUnit;
+import org.apache.arrow.vector.types.UnionMode;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+
+/**
+ * Utility to validate vector type information.
+ */
+public class ValidateVectorTypeVisitor implements VectorVisitor<Void, Void> {
+
+  private void validateVectorCommon(ValueVector vector, Class<? extends ArrowType> expectedArrowType) {
+    validateOrThrow(vector.getField() != null, "Vector field is empty.");
+    validateOrThrow(vector.getField().getFieldType() != null, "Vector field type is empty.");
+    ArrowType arrowType = vector.getField().getFieldType().getType();
+    validateOrThrow(arrowType != null, "Vector arrow type is empty.");
+    validateOrThrow(expectedArrowType == arrowType.getClass(),
+        "Incorrect arrow type for " + vector.getClass() + " : " + arrowType.toString());
+  }
+
+  private void validateIntVector(ValueVector vector, int expectedWidth, boolean expectedSigned) {
+    validateOrThrow(vector.getField().getFieldType().getType() instanceof ArrowType.Int,
+        "Vector %s is not an integer vector.", vector.getClass());
+    ArrowType.Int intType = (ArrowType.Int) vector.getField().getFieldType().getType();
+    validateOrThrow(intType.getIsSigned() == expectedSigned,
+        "Expecting bit width %s, actual width %s.", expectedWidth, intType.getBitWidth());
+    validateOrThrow(intType.getBitWidth() == expectedWidth, "Expecting bit width %s, actual bit width %s.",
+        expectedWidth, intType.getBitWidth());
+  }
+
+  private void validateFloatingPointVector(ValueVector vector, FloatingPointPrecision expectedPrecision) {
+    validateOrThrow(vector.getField().getFieldType().getType() instanceof ArrowType.FloatingPoint,
+        "Vector %s is not a floating point vector.", vector.getClass());
+    ArrowType.FloatingPoint floatType = (ArrowType.FloatingPoint) vector.getField().getFieldType().getType();
+    validateOrThrow(floatType.getPrecision() == expectedPrecision, "Expecting precision %s, actual precision %s.",
+        expectedPrecision, floatType.getPrecision());
+  }
+
+  private void validateDateVector(ValueVector vector, DateUnit expectedDateUnit) {
+    validateOrThrow(vector.getField().getFieldType().getType() instanceof ArrowType.Date,
+        "Vector %s is not a date vector", vector.getClass());
+    ArrowType.Date dateType = (ArrowType.Date) vector.getField().getFieldType().getType();
+    validateOrThrow(dateType.getUnit() == expectedDateUnit,
+        "Expecting date unit %s, actual date unit %s.", expectedDateUnit, dateType.getUnit());
+  }
+
+  private void validateTimeVector(ValueVector vector, TimeUnit expectedTimeUnit, int expectedBitWidth) {
+    validateOrThrow(vector.getField().getFieldType().getType() instanceof ArrowType.Time,
+        "Vector %s is not a time vector.", vector.getClass());
+    ArrowType.Time timeType = (ArrowType.Time) vector.getField().getFieldType().getType();
+    validateOrThrow(timeType.getUnit() == expectedTimeUnit,
+        "Expecting time unit %s, actual time unit %s.", expectedTimeUnit, timeType.getUnit());
+    validateOrThrow(timeType.getBitWidth() == expectedBitWidth,
+        "Expecting bit width %s, actual bit width %s.", expectedBitWidth, timeType.getBitWidth());
+  }
+
+  private void validateIntervalVector(ValueVector vector, IntervalUnit expectedIntervalUnit) {
+    validateOrThrow(vector.getField().getFieldType().getType() instanceof ArrowType.Interval,
+        "Vector %s is not an interval vector.", vector.getClass());
+    ArrowType.Interval intervalType = (ArrowType.Interval) vector.getField().getFieldType().getType();
+    validateOrThrow(intervalType.getUnit() == expectedIntervalUnit,
+        "Expecting interval unit %s, actual date unit %s.", expectedIntervalUnit, intervalType.getUnit());
+  }
+
+  private void validateTimeStampVector(ValueVector vector, TimeUnit expectedTimeUnit, boolean expectTZ) {
+    validateOrThrow(vector.getField().getFieldType().getType() instanceof ArrowType.Timestamp,
+        "Vector %s is not a time stamp vector.", vector.getClass());
+    ArrowType.Timestamp timestampType = (ArrowType.Timestamp) vector.getField().getFieldType().getType();
+    validateOrThrow(timestampType.getUnit() == expectedTimeUnit,
+        "Expecting time stamp unit %s, actual time stamp unit %s.", expectedTimeUnit, timestampType.getUnit());
+    if (expectTZ) {
+      validateOrThrow(timestampType.getTimezone() != null, "The time zone should not be null");
+    } else {
+      validateOrThrow(timestampType.getTimezone() == null, "The time zone should be null");
+    }
+  }
+
+  @Override
+  public Void visit(BaseFixedWidthVector vector, Void value) {
+    if (vector instanceof TinyIntVector) {
+      validateVectorCommon(vector, ArrowType.Int.class);
+      validateIntVector(vector, 8, true);
+    } else if (vector instanceof SmallIntVector) {
+      validateVectorCommon(vector, ArrowType.Int.class);
+      validateIntVector(vector, 16, true);
+    } else if (vector instanceof IntVector) {
+      validateVectorCommon(vector, ArrowType.Int.class);
+      validateIntVector(vector, 32, true);
+    } else if (vector instanceof BigIntVector) {
+      validateVectorCommon(vector, ArrowType.Int.class);
+      validateIntVector(vector, 64, true);
+    } else if (vector instanceof UInt1Vector) {
+      validateVectorCommon(vector, ArrowType.Int.class);
+      validateIntVector(vector, 8, false);
+    } else if (vector instanceof UInt2Vector) {
+      validateVectorCommon(vector, ArrowType.Int.class);
+      validateIntVector(vector, 16, false);
+    } else if (vector instanceof UInt4Vector) {
+      validateVectorCommon(vector, ArrowType.Int.class);
+      validateIntVector(vector, 32, false);
+    } else if (vector instanceof UInt8Vector) {
+      validateVectorCommon(vector, ArrowType.Int.class);
+      validateIntVector(vector, 64, false);
+    } else if (vector instanceof BitVector) {
+      validateVectorCommon(vector, ArrowType.Bool.class);
+    } else if (vector instanceof DecimalVector || vector instanceof Decimal256Vector) {
+      validateVectorCommon(vector, ArrowType.Decimal.class);
+      ArrowType.Decimal arrowType = (ArrowType.Decimal) vector.getField().getType();
+      validateOrThrow(arrowType.getScale() > 0, "The scale of decimal %s is not positive.", arrowType.getScale());
+      validateOrThrow(arrowType.getPrecision() > 0, "The precision of decimal %S is not positive.",
+          arrowType.getPrecision());
+    } else if (vector instanceof DateDayVector) {
+      validateVectorCommon(vector, ArrowType.Date.class);
+      validateDateVector(vector, DateUnit.DAY);
+    } else if (vector instanceof DateMilliVector) {
+      validateVectorCommon(vector, ArrowType.Date.class);
+      validateDateVector(vector, DateUnit.MILLISECOND);
+    } else if (vector instanceof DurationVector) {
+      validateVectorCommon(vector, ArrowType.Duration.class);
+      ArrowType.Duration arrowType = (ArrowType.Duration) vector.getField().getType();
+      validateOrThrow(((DurationVector) vector).getUnit() == arrowType.getUnit(),
+          "Different duration time unit for vector and arrow type. Vector time unit %s, type time unit %s.",
+          ((DurationVector) vector).getUnit(), arrowType.getUnit());
+    } else if (vector instanceof Float4Vector) {
+      validateVectorCommon(vector, ArrowType.FloatingPoint.class);
+      validateFloatingPointVector(vector, FloatingPointPrecision.SINGLE);
+    } else if (vector instanceof Float8Vector) {
+      validateVectorCommon(vector, ArrowType.FloatingPoint.class);
+      validateFloatingPointVector(vector, FloatingPointPrecision.DOUBLE);
+    } else if (vector instanceof IntervalDayVector) {
+      validateVectorCommon(vector, ArrowType.Interval.class);
+      validateIntervalVector(vector, IntervalUnit.DAY_TIME);
+    } else if (vector instanceof IntervalYearVector) {
+      validateVectorCommon(vector, ArrowType.Interval.class);
+      validateIntervalVector(vector, IntervalUnit.YEAR_MONTH);
+    } else if (vector instanceof TimeMicroVector) {
+      validateVectorCommon(vector, ArrowType.Time.class);
+      validateTimeVector(vector, TimeUnit.MICROSECOND, 64);
+    } else if (vector instanceof TimeMilliVector) {
+      validateVectorCommon(vector, ArrowType.Time.class);
+      validateTimeVector(vector, TimeUnit.MILLISECOND, 32);
+    } else if (vector instanceof TimeNanoVector) {
+      validateVectorCommon(vector, ArrowType.Time.class);
+      validateTimeVector(vector, TimeUnit.NANOSECOND, 64);
+    } else if (vector instanceof TimeSecVector) {
+      validateVectorCommon(vector, ArrowType.Time.class);
+      validateTimeVector(vector, TimeUnit.SECOND, 32);
+    } else if (vector instanceof TimeStampMicroTZVector) {
+      validateVectorCommon(vector, ArrowType.Timestamp.class);
+      validateTimeStampVector(vector, TimeUnit.MICROSECOND, true);
+    } else if (vector instanceof TimeStampMicroVector) {
+      validateVectorCommon(vector, ArrowType.Timestamp.class);
+      validateTimeStampVector(vector, TimeUnit.MICROSECOND, false);
+    } else if (vector instanceof TimeStampMilliTZVector) {
+      validateVectorCommon(vector, ArrowType.Timestamp.class);
+      validateTimeStampVector(vector, TimeUnit.MILLISECOND, true);
+    } else if (vector instanceof TimeStampMilliVector) {
+      validateVectorCommon(vector, ArrowType.Timestamp.class);
+      validateTimeStampVector(vector, TimeUnit.MILLISECOND, false);
+    } else if (vector instanceof TimeStampNanoTZVector) {
+      validateVectorCommon(vector, ArrowType.Timestamp.class);
+      validateTimeStampVector(vector, TimeUnit.NANOSECOND, true);
+    } else if (vector instanceof TimeStampNanoVector) {
+      validateVectorCommon(vector, ArrowType.Timestamp.class);
+      validateTimeStampVector(vector, TimeUnit.NANOSECOND, false);
+    } else if (vector instanceof TimeStampSecTZVector) {
+      validateVectorCommon(vector, ArrowType.Timestamp.class);
+      validateTimeStampVector(vector, TimeUnit.SECOND, true);
+    } else if (vector instanceof TimeStampSecVector) {
+      validateVectorCommon(vector, ArrowType.Timestamp.class);
+      validateTimeStampVector(vector, TimeUnit.SECOND, false);
+    } else if (vector instanceof FixedSizeBinaryVector) {
+      validateVectorCommon(vector, ArrowType.FixedSizeBinary.class);
+      ArrowType.FixedSizeBinary arrowType = (ArrowType.FixedSizeBinary) vector.getField().getType();
+      validateOrThrow(arrowType.getByteWidth() > 0, "The byte width of a FixedSizeBinaryVector %s is not positive.",
+          arrowType.getByteWidth());
+      validateOrThrow(arrowType.getByteWidth() == vector.getTypeWidth(),
+          "Type width mismatch for FixedSizeBinaryVector. Vector type width %s, arrow type type width %s.",
+          vector.getTypeWidth(), arrowType.getByteWidth());
+    } else {
+      throw new IllegalArgumentException("Unknown type for fixed width vector " + vector.getClass());
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(BaseVariableWidthVector vector, Void value) {
+    if (vector instanceof VarCharVector) {
+      validateVectorCommon(vector, ArrowType.Utf8.class);
+    } else if (vector instanceof VarBinaryVector) {
+      validateVectorCommon(vector, ArrowType.Binary.class);
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(BaseLargeVariableWidthVector vector, Void value) {
+    if (vector instanceof LargeVarCharVector) {
+      validateVectorCommon(vector, ArrowType.LargeUtf8.class);
+    } else if (vector instanceof LargeVarBinaryVector) {
+      validateVectorCommon(vector, ArrowType.LargeBinary.class);
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(ListVector vector, Void value) {
+    validateVectorCommon(vector, ArrowType.List.class);
+    ValueVector innerVector = vector.getDataVector();
+    if (innerVector != null) {
+      innerVector.accept(this, null);
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(FixedSizeListVector vector, Void value) {
+    validateVectorCommon(vector, ArrowType.FixedSizeList.class);
+    ArrowType.FixedSizeList arrowType = (ArrowType.FixedSizeList) vector.getField().getType();
+    validateOrThrow(arrowType.getListSize() == vector.getListSize(),
+        "Inconsistent list size for FixedSizeListVector. Vector list size %s, arrow type list size %s.",
+        vector.getListSize(), arrowType.getListSize());
+    validateOrThrow(arrowType.getListSize() > 0, "The list size %s is not positive.", arrowType.getListSize());
+    ValueVector innerVector = vector.getDataVector();
+    if (innerVector != null) {
+      innerVector.accept(this, null);
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(LargeListVector vector, Void value) {
+    validateVectorCommon(vector, ArrowType.LargeList.class);
+    ValueVector innerVector = vector.getDataVector();
+    if (innerVector != null) {
+      innerVector.accept(this, null);
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(NonNullableStructVector vector, Void value) {
+    validateVectorCommon(vector, ArrowType.Struct.class);
+    validateOrThrow(vector.getField().getChildren().size() == vector.getChildrenFromFields().size(),
+        "Child field count and child vector count mismatch. Vector child count %s, field child count %s",
+        vector.getChildrenFromFields().size(), vector.getField().getChildren().size());
+    for (int i = 0; i < vector.getChildrenFromFields().size(); i++) {
+      ValueVector subVector = vector.getChildByOrdinal(i);
+      FieldType subType = vector.getField().getChildren().get(i).getFieldType();
+
+      validateOrThrow(subType.equals(subVector.getField().getFieldType()),
+          "Struct vector's field type not equal to the child vector's field type. " +
+              "Struct field type %s, sub-vector field type %s", subType, subVector.getField().getFieldType());
+      subVector.accept(this, null);
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(UnionVector vector, Void value) {
+    validateVectorCommon(vector, ArrowType.Union.class);
+    ArrowType.Union arrowType = (ArrowType.Union) vector.getField().getType();
+    validateOrThrow(arrowType.getMode() == UnionMode.Sparse, "The union mode of UnionVector must be sparse");
+    for (ValueVector subVector : vector.getChildrenFromFields()) {
+      subVector.accept(this, null);
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(DenseUnionVector vector, Void value) {
+    validateVectorCommon(vector, ArrowType.Union.class);
+    ArrowType.Union arrowType = (ArrowType.Union) vector.getField().getType();
+    validateOrThrow(arrowType.getMode() == UnionMode.Dense, "The union mode of DenseUnionVector must be dense");
+    for (ValueVector subVector : vector.getChildrenFromFields()) {
+      subVector.accept(this, null);
+    }
+    return null;
+  }
+
+  @Override
+  public Void visit(NullVector vector, Void value) {
+    validateVectorCommon(vector, ArrowType.Null.class);
+    return null;
+  }
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ITTestLargeVector.java b/java/vector/src/test/java/org/apache/arrow/vector/ITTestLargeVector.java
index 8b824d6a291..19648dc9e13 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ITTestLargeVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ITTestLargeVector.java
@@ -21,9 +21,12 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
+import java.math.BigDecimal;
+
 import org.apache.arrow.memory.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.holders.NullableDecimalHolder;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -114,7 +117,7 @@ public void testLargeDecimalVector() {
     final int vecLength = (int) (bufSize / DecimalVector.TYPE_WIDTH);
 
     try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
-         DecimalVector largeVec = new DecimalVector("vec", allocator, 38, 16)) {
+         DecimalVector largeVec = new DecimalVector("vec", allocator, 38, 0)) {
       largeVec.allocateNew(vecLength);
 
       logger.trace("Successfully allocated a vector with capacity {}", vecLength);
@@ -139,6 +142,22 @@ public void testLargeDecimalVector() {
         }
       }
       logger.trace("Successfully read {} values", vecLength);
+
+      // try setting values with a large offset in the buffer
+      largeVec.set(vecLength - 1, 12345L);
+      assertEquals(12345L, largeVec.getObject(vecLength - 1).longValue());
+
+      NullableDecimalHolder holder = new NullableDecimalHolder();
+      holder.buffer = largeVec.valueBuffer;
+      holder.isSet = 1;
+      holder.start = (long) (vecLength - 1) * largeVec.getTypeWidth();
+      assertTrue(holder.start > Integer.MAX_VALUE);
+      largeVec.set(0, holder);
+
+      BigDecimal decimal = largeVec.getObject(0);
+      assertEquals(12345L, decimal.longValue());
+
+      logger.trace("Successfully setting values from large offsets");
     }
     logger.trace("Successfully released the large vector.");
   }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java
new file mode 100644
index 00000000000..82c912cef2f
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java
@@ -0,0 +1,357 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestDecimal256Vector {
+
+  private static long[] intValues;
+
+  static {
+    intValues = new long[60];
+    for (int i = 0; i < intValues.length / 2; i++) {
+      intValues[i] = 1 << i + 1;
+      intValues[2 * i] = -1 * (1 << i + 1);
+    }
+  }
+
+  private int scale = 3;
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100);
+  }
+
+  @After
+  public void terminate() throws Exception {
+    allocator.close();
+  }
+
+  @Test
+  public void testValuesWriteRead() {
+    try (Decimal256Vector decimalVector = TestUtils.newVector(Decimal256Vector.class, "decimal",
+        new ArrowType.Decimal(10, scale, 256), allocator);) {
+
+      try (Decimal256Vector oldConstructor = new Decimal256Vector("decimal", allocator, 10, scale);) {
+        assertEquals(decimalVector.getField().getType(), oldConstructor.getField().getType());
+      }
+
+      decimalVector.allocateNew();
+      BigDecimal[] values = new BigDecimal[intValues.length];
+      for (int i = 0; i < intValues.length; i++) {
+        BigDecimal decimal = new BigDecimal(BigInteger.valueOf(intValues[i]), scale);
+        values[i] = decimal;
+        decimalVector.setSafe(i, decimal);
+      }
+
+      decimalVector.setValueCount(intValues.length);
+
+      for (int i = 0; i < intValues.length; i++) {
+        BigDecimal value = decimalVector.getObject(i);
+        assertEquals("unexpected data at index: " + i, values[i], value);
+      }
+    }
+  }
+
+  @Test
+  public void testDecimal256DifferentScaleAndPrecision() {
+    try (Decimal256Vector decimalVector = TestUtils.newVector(Decimal256Vector.class, "decimal",
+        new ArrowType.Decimal(4, 2, 256), allocator)) {
+      decimalVector.allocateNew();
+
+      // test Decimal256 with different scale
+      {
+        BigDecimal decimal = new BigDecimal(BigInteger.valueOf(0), 3);
+        UnsupportedOperationException ue =
+            assertThrows(UnsupportedOperationException.class, () -> decimalVector.setSafe(0, decimal));
+        assertEquals("BigDecimal scale must equal that in the Arrow vector: 3 != 2", ue.getMessage());
+      }
+
+      // test BigDecimal with larger precision than initialized
+      {
+        BigDecimal decimal = new BigDecimal(BigInteger.valueOf(12345), 2);
+        UnsupportedOperationException ue =
+            assertThrows(UnsupportedOperationException.class, () -> decimalVector.setSafe(0, decimal));
+        assertEquals("BigDecimal precision can not be greater than that in the Arrow vector: 5 > 4", ue.getMessage());
+      }
+    }
+  }
+
+  @Test
+  public void testWriteBigEndian() {
+    try (Decimal256Vector decimalVector = TestUtils.newVector(Decimal256Vector.class, "decimal",
+        new ArrowType.Decimal(38, 18, 256), allocator);) {
+      decimalVector.allocateNew();
+      BigDecimal decimal1 = new BigDecimal("123456789.000000000000000000");
+      BigDecimal decimal2 = new BigDecimal("11.123456789123456789");
+      BigDecimal decimal3 = new BigDecimal("1.000000000000000000");
+      BigDecimal decimal4 = new BigDecimal("0.111111111000000000");
+      BigDecimal decimal5 = new BigDecimal("987654321.123456789000000000");
+      BigDecimal decimal6 = new BigDecimal("222222222222.222222222000000000");
+      BigDecimal decimal7 = new BigDecimal("7777777777777.666666667000000000");
+      BigDecimal decimal8 = new BigDecimal("1212121212.343434343000000000");
+
+      byte[] decimalValue1 = decimal1.unscaledValue().toByteArray();
+      byte[] decimalValue2 = decimal2.unscaledValue().toByteArray();
+      byte[] decimalValue3 = decimal3.unscaledValue().toByteArray();
+      byte[] decimalValue4 = decimal4.unscaledValue().toByteArray();
+      byte[] decimalValue5 = decimal5.unscaledValue().toByteArray();
+      byte[] decimalValue6 = decimal6.unscaledValue().toByteArray();
+      byte[] decimalValue7 = decimal7.unscaledValue().toByteArray();
+      byte[] decimalValue8 = decimal8.unscaledValue().toByteArray();
+
+      decimalVector.setBigEndian(0, decimalValue1);
+      decimalVector.setBigEndian(1, decimalValue2);
+      decimalVector.setBigEndian(2, decimalValue3);
+      decimalVector.setBigEndian(3, decimalValue4);
+      decimalVector.setBigEndian(4, decimalValue5);
+      decimalVector.setBigEndian(5, decimalValue6);
+      decimalVector.setBigEndian(6, decimalValue7);
+      decimalVector.setBigEndian(7, decimalValue8);
+
+      decimalVector.setValueCount(8);
+      assertEquals(8, decimalVector.getValueCount());
+      assertEquals(decimal1, decimalVector.getObject(0));
+      assertEquals(decimal2, decimalVector.getObject(1));
+      assertEquals(decimal3, decimalVector.getObject(2));
+      assertEquals(decimal4, decimalVector.getObject(3));
+      assertEquals(decimal5, decimalVector.getObject(4));
+      assertEquals(decimal6, decimalVector.getObject(5));
+      assertEquals(decimal7, decimalVector.getObject(6));
+      assertEquals(decimal8, decimalVector.getObject(7));
+    }
+  }
+
+  @Test
+  public void testLongReadWrite() {
+    try (Decimal256Vector decimalVector = TestUtils.newVector(Decimal256Vector.class, "decimal",
+            new ArrowType.Decimal(38, 0, 256), allocator)) {
+      decimalVector.allocateNew();
+
+      long[] longValues = {0L, -2L, Long.MAX_VALUE, Long.MIN_VALUE, 187L};
+
+      for (int i = 0; i < longValues.length; ++i) {
+        decimalVector.set(i, longValues[i]);
+      }
+
+      decimalVector.setValueCount(longValues.length);
+
+      for (int i = 0; i < longValues.length; ++i) {
+        assertEquals(new BigDecimal(longValues[i]), decimalVector.getObject(i));
+      }
+    }
+  }
+
+
+  @Test
+  public void testBigDecimalReadWrite() {
+    try (Decimal256Vector decimalVector = TestUtils.newVector(Decimal256Vector.class, "decimal",
+        new ArrowType.Decimal(38, 9, 256), allocator);) {
+      decimalVector.allocateNew();
+      BigDecimal decimal1 = new BigDecimal("123456789.000000000");
+      BigDecimal decimal2 = new BigDecimal("11.123456789");
+      BigDecimal decimal3 = new BigDecimal("1.000000000");
+      BigDecimal decimal4 = new BigDecimal("-0.111111111");
+      BigDecimal decimal5 = new BigDecimal("-987654321.123456789");
+      BigDecimal decimal6 = new BigDecimal("-222222222222.222222222");
+      BigDecimal decimal7 = new BigDecimal("7777777777777.666666667");
+      BigDecimal decimal8 = new BigDecimal("1212121212.343434343");
+
+      decimalVector.set(0, decimal1);
+      decimalVector.set(1, decimal2);
+      decimalVector.set(2, decimal3);
+      decimalVector.set(3, decimal4);
+      decimalVector.set(4, decimal5);
+      decimalVector.set(5, decimal6);
+      decimalVector.set(6, decimal7);
+      decimalVector.set(7, decimal8);
+
+      decimalVector.setValueCount(8);
+      assertEquals(8, decimalVector.getValueCount());
+      assertEquals(decimal1, decimalVector.getObject(0));
+      assertEquals(decimal2, decimalVector.getObject(1));
+      assertEquals(decimal3, decimalVector.getObject(2));
+      assertEquals(decimal4, decimalVector.getObject(3));
+      assertEquals(decimal5, decimalVector.getObject(4));
+      assertEquals(decimal6, decimalVector.getObject(5));
+      assertEquals(decimal7, decimalVector.getObject(6));
+      assertEquals(decimal8, decimalVector.getObject(7));
+    }
+  }
+
+  /**
+   * Test {@link Decimal256Vector#setBigEndian(int, byte[])} which takes BE layout input and stores in native-endian
+   * (NE) layout.
+   * Cases to cover: input byte array in different lengths in range [1-16] and negative values.
+   */
+  @Test
+  public void decimalBE2NE() {
+    try (Decimal256Vector decimalVector = TestUtils.newVector(Decimal256Vector.class, "decimal",
+        new ArrowType.Decimal(23, 2, 256), allocator)) {
+      decimalVector.allocateNew();
+
+      BigInteger[] testBigInts = new BigInteger[] {
+          new BigInteger("0"),
+          new BigInteger("-1"),
+          new BigInteger("23"),
+          new BigInteger("234234"),
+          new BigInteger("-234234234"),
+          new BigInteger("234234234234"),
+          new BigInteger("-56345345345345"),
+          new BigInteger("2982346298346289346293467923465345634500"), // converts to 16+ byte array
+          new BigInteger("-389457298347598237459832459823434653600"), // converts to 16+ byte array
+          new BigInteger("-345345"),
+          new BigInteger("754533")
+      };
+
+      int insertionIdx = 0;
+      insertionIdx++; // insert a null
+      for (BigInteger val : testBigInts) {
+        decimalVector.setBigEndian(insertionIdx++, val.toByteArray());
+      }
+      insertionIdx++; // insert a null
+      // insert a zero length buffer
+      decimalVector.setBigEndian(insertionIdx++, new byte[0]);
+
+      // Try inserting a buffer larger than 33 bytes and expect a failure
+      final int insertionIdxCapture = insertionIdx;
+      IllegalArgumentException ex = assertThrows(IllegalArgumentException.class,
+          () -> decimalVector.setBigEndian(insertionIdxCapture, new byte[33]));
+      assertTrue(ex.getMessage().equals("Invalid decimal value length. Valid length in [1 - 32], got 33"));
+      decimalVector.setValueCount(insertionIdx);
+
+      // retrieve values and check if they are correct
+      int outputIdx = 0;
+      assertTrue(decimalVector.isNull(outputIdx++));
+      for (BigInteger expected : testBigInts) {
+        final BigDecimal actual = decimalVector.getObject(outputIdx++);
+        assertEquals(expected, actual.unscaledValue());
+      }
+      assertTrue(decimalVector.isNull(outputIdx++));
+      assertEquals(BigInteger.valueOf(0), decimalVector.getObject(outputIdx).unscaledValue());
+    }
+  }
+
+  @Test
+  public void setUsingArrowBufOfLEInts() {
+    try (Decimal256Vector decimalVector = TestUtils.newVector(Decimal256Vector.class, "decimal",
+            new ArrowType.Decimal(5, 2, 256), allocator);
+         ArrowBuf buf = allocator.buffer(8);) {
+      decimalVector.allocateNew();
+
+      // add a positive value equivalent to 705.32
+      int val = 70532;
+      buf.setInt(0, val);
+      decimalVector.setSafe(0, 0, buf, 4);
+
+      // add a -ve value equivalent to -705.32
+      val = -70532;
+      buf.setInt(4, val);
+      decimalVector.setSafe(1, 4, buf, 4);
+
+      decimalVector.setValueCount(2);
+
+      BigDecimal [] expectedValues = new BigDecimal[] {BigDecimal.valueOf(705.32), BigDecimal
+              .valueOf(-705.32)};
+      for (int i = 0; i < 2; i ++) {
+        BigDecimal value = decimalVector.getObject(i);
+        assertEquals(expectedValues[i], value);
+      }
+    }
+
+  }
+
+  @Test
+  public void setUsingArrowLongLEBytes() {
+    try (Decimal256Vector decimalVector = TestUtils.newVector(Decimal256Vector.class, "decimal",
+            new ArrowType.Decimal(18, 0, 256), allocator);
+         ArrowBuf buf = allocator.buffer(16);) {
+      decimalVector.allocateNew();
+
+      long val = Long.MAX_VALUE;
+      buf.setLong(0, val);
+      decimalVector.setSafe(0, 0, buf, 8);
+
+      val = Long.MIN_VALUE;
+      buf.setLong(8, val);
+      decimalVector.setSafe(1, 8, buf, 8);
+
+      decimalVector.setValueCount(2);
+
+      BigDecimal [] expectedValues = new BigDecimal[] {BigDecimal.valueOf(Long.MAX_VALUE), BigDecimal
+              .valueOf(Long.MIN_VALUE)};
+      for (int i = 0; i < 2; i ++) {
+        BigDecimal value = decimalVector.getObject(i);
+        assertEquals(expectedValues[i], value);
+      }
+    }
+  }
+
+  @Test
+  public void setUsingArrowBufOfBEBytes() {
+    try (Decimal256Vector decimalVector = TestUtils.newVector(Decimal256Vector.class, "decimal",
+            new ArrowType.Decimal(5, 2, 256), allocator);
+         ArrowBuf buf = allocator.buffer(9);) {
+      BigDecimal [] expectedValues = new BigDecimal[] {BigDecimal.valueOf(705.32), BigDecimal
+              .valueOf(-705.32), BigDecimal.valueOf(705.32)};
+      verifyWritingArrowBufWithBigEndianBytes(decimalVector, buf, expectedValues, 3);
+    }
+
+    try (Decimal256Vector decimalVector = TestUtils.newVector(Decimal256Vector.class, "decimal",
+            new ArrowType.Decimal(43, 2, 256), allocator);
+         ArrowBuf buf = allocator.buffer(45);) {
+      BigDecimal[] expectedValues = new BigDecimal[] {new BigDecimal("29823462983462893462934679234653450000000.63"),
+                                                      new BigDecimal("-2982346298346289346293467923465345.63"),
+                                                      new BigDecimal("2982346298346289346293467923465345.63")};
+      verifyWritingArrowBufWithBigEndianBytes(decimalVector, buf, expectedValues, 15);
+    }
+  }
+
+  private void verifyWritingArrowBufWithBigEndianBytes(Decimal256Vector decimalVector,
+                                                       ArrowBuf buf, BigDecimal[] expectedValues,
+                                                       int length) {
+    decimalVector.allocateNew();
+    for (int i = 0; i < expectedValues.length; i++) {
+      byte[] bigEndianBytes = expectedValues[i].unscaledValue().toByteArray();
+      buf.setBytes(length * i , bigEndianBytes, 0 , bigEndianBytes.length);
+      decimalVector.setBigEndianSafe(i, length * i, buf, bigEndianBytes.length);
+    }
+
+    decimalVector.setValueCount(3);
+
+    for (int i = 0; i < expectedValues.length; i ++) {
+      BigDecimal value = decimalVector.getObject(i);
+      assertEquals(expectedValues[i], value);
+    }
+  }
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
index 28d799bb1ca..c7e3e436e40 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
@@ -60,7 +60,7 @@ public void terminate() throws Exception {
   @Test
   public void testValuesWriteRead() {
     try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal",
-        new ArrowType.Decimal(10, scale), allocator);) {
+        new ArrowType.Decimal(10, scale, 128), allocator);) {
 
       try (DecimalVector oldConstructor = new DecimalVector("decimal", allocator, 10, scale);) {
         assertEquals(decimalVector.getField().getType(), oldConstructor.getField().getType());
@@ -86,7 +86,7 @@ public void testValuesWriteRead() {
   @Test
   public void testBigDecimalDifferentScaleAndPrecision() {
     try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal",
-        new ArrowType.Decimal(4, 2), allocator);) {
+        new ArrowType.Decimal(4, 2, 128), allocator);) {
       decimalVector.allocateNew();
 
       // test BigDecimal with different scale
@@ -116,7 +116,7 @@ public void testBigDecimalDifferentScaleAndPrecision() {
   @Test
   public void testWriteBigEndian() {
     try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal",
-        new ArrowType.Decimal(38, 9), allocator);) {
+        new ArrowType.Decimal(38, 9, 128), allocator);) {
       decimalVector.allocateNew();
       BigDecimal decimal1 = new BigDecimal("123456789.000000000");
       BigDecimal decimal2 = new BigDecimal("11.123456789");
@@ -161,7 +161,7 @@ public void testWriteBigEndian() {
   @Test
   public void testLongReadWrite() {
     try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal",
-            new ArrowType.Decimal(38, 0), allocator)) {
+            new ArrowType.Decimal(38, 0, 128), allocator)) {
       decimalVector.allocateNew();
 
       long[] longValues = {0L, -2L, Long.MAX_VALUE, Long.MIN_VALUE, 187L};
@@ -182,7 +182,7 @@ public void testLongReadWrite() {
   @Test
   public void testBigDecimalReadWrite() {
     try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal",
-        new ArrowType.Decimal(38, 9), allocator);) {
+        new ArrowType.Decimal(38, 9, 128), allocator);) {
       decimalVector.allocateNew();
       BigDecimal decimal1 = new BigDecimal("123456789.000000000");
       BigDecimal decimal2 = new BigDecimal("11.123456789");
@@ -216,13 +216,14 @@ public void testBigDecimalReadWrite() {
   }
 
   /**
-   * Test {@link DecimalVector#setBigEndian(int, byte[])} which takes BE layout input and stores in LE layout.
+   * Test {@link DecimalVector#setBigEndian(int, byte[])} which takes BE layout input and stores in native-endian (NE)
+   * layout.
    * Cases to cover: input byte array in different lengths in range [1-16] and negative values.
    */
   @Test
-  public void decimalBE2LE() {
+  public void decimalBE2NE() {
     try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal",
-        new ArrowType.Decimal(21, 2), allocator)) {
+        new ArrowType.Decimal(21, 2, 128), allocator)) {
       decimalVector.allocateNew();
 
       BigInteger[] testBigInts = new BigInteger[] {
@@ -270,9 +271,9 @@ public void decimalBE2LE() {
   }
 
   @Test
-  public void setUsingArrowBufOfLEInts() {
+  public void setUsingArrowBufOfInts() {
     try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal",
-            new ArrowType.Decimal(5, 2), allocator);
+            new ArrowType.Decimal(5, 2, 128), allocator);
          ArrowBuf buf = allocator.buffer(8);) {
       decimalVector.allocateNew();
 
@@ -299,9 +300,9 @@ public void setUsingArrowBufOfLEInts() {
   }
 
   @Test
-  public void setUsingArrowLongLEBytes() {
+  public void setUsingArrowLongBytes() {
     try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal",
-            new ArrowType.Decimal(18, 0), allocator);
+            new ArrowType.Decimal(18, 0, 128), allocator);
          ArrowBuf buf = allocator.buffer(16);) {
       decimalVector.allocateNew();
 
@@ -327,7 +328,7 @@ public void setUsingArrowLongLEBytes() {
   @Test
   public void setUsingArrowBufOfBEBytes() {
     try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal",
-            new ArrowType.Decimal(5, 2), allocator);
+            new ArrowType.Decimal(5, 2, 128), allocator);
          ArrowBuf buf = allocator.buffer(9);) {
       BigDecimal [] expectedValues = new BigDecimal[] {BigDecimal.valueOf(705.32), BigDecimal
               .valueOf(-705.32), BigDecimal.valueOf(705.32)};
@@ -335,7 +336,7 @@ public void setUsingArrowBufOfBEBytes() {
     }
 
     try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal",
-            new ArrowType.Decimal(36, 2), allocator);
+            new ArrowType.Decimal(36, 2, 128), allocator);
          ArrowBuf buf = allocator.buffer(45);) {
       BigDecimal[] expectedValues = new BigDecimal[] {new BigDecimal("2982346298346289346293467923465345.63"),
                                                       new BigDecimal("-2982346298346289346293467923465345.63"),
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
index 812ee389d01..e8fc444d14a 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
@@ -20,6 +20,7 @@
 import static org.apache.arrow.vector.TestUtils.newVarBinaryVector;
 import static org.apache.arrow.vector.TestUtils.newVarCharVector;
 import static org.apache.arrow.vector.testing.ValueVectorDataPopulator.setVector;
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -29,6 +30,7 @@
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.function.ToIntBiFunction;
 
 import org.apache.arrow.memory.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
@@ -878,6 +880,108 @@ public void testEncodeStructSubFieldWithCertainColumns() {
     }
   }
 
+  private void testDictionary(Dictionary dictionary, ToIntBiFunction<ValueVector, Integer> valGetter) {
+    try (VarCharVector vector = new VarCharVector("vector", allocator)) {
+      setVector(vector, "1", "3", "5", "7", "9");
+      try (ValueVector encodedVector = DictionaryEncoder.encode(vector, dictionary)) {
+
+        // verify encoded result
+        assertEquals(vector.getValueCount(), encodedVector.getValueCount());
+        assertEquals(valGetter.applyAsInt(encodedVector, 0), 1);
+        assertEquals(valGetter.applyAsInt(encodedVector, 1), 3);
+        assertEquals(valGetter.applyAsInt(encodedVector, 2), 5);
+        assertEquals(valGetter.applyAsInt(encodedVector, 3), 7);
+        assertEquals(valGetter.applyAsInt(encodedVector, 4), 9);
+
+        try (ValueVector decodedVector = DictionaryEncoder.decode(encodedVector, dictionary)) {
+          assertTrue(decodedVector instanceof VarCharVector);
+          assertEquals(vector.getValueCount(), decodedVector.getValueCount());
+          assertArrayEquals("1".getBytes(), ((VarCharVector) decodedVector).get(0));
+          assertArrayEquals("3".getBytes(), ((VarCharVector) decodedVector).get(1));
+          assertArrayEquals("5".getBytes(), ((VarCharVector) decodedVector).get(2));
+          assertArrayEquals("7".getBytes(), ((VarCharVector) decodedVector).get(3));
+          assertArrayEquals("9".getBytes(), ((VarCharVector) decodedVector).get(4));
+        }
+      }
+    }
+  }
+
+  @Test
+  public void testDictionaryUInt1() {
+    try (VarCharVector dictionaryVector = new VarCharVector("dict vector", allocator)) {
+      setVector(dictionaryVector, "0", "1", "2", "3", "4", "5", "6", "7", "8", "9");
+      Dictionary dictionary1 = new Dictionary(dictionaryVector,
+          new DictionaryEncoding(/*id=*/10L, /*ordered=*/false,
+              /*indexType=*/new ArrowType.Int(/*bitWidth*/8, /*isSigned*/false)));
+      testDictionary(dictionary1, (vector, index) -> ((UInt1Vector) vector).get(index));
+    }
+  }
+
+  @Test
+  public void testDictionaryUInt2() {
+    try (VarCharVector dictionaryVector = new VarCharVector("dict vector", allocator)) {
+      setVector(dictionaryVector, "0", "1", "2", "3", "4", "5", "6", "7", "8", "9");
+      Dictionary dictionary2 = new Dictionary(dictionaryVector,
+          new DictionaryEncoding(/*id=*/20L, /*ordered=*/false,
+              /*indexType=*/new ArrowType.Int(/*indexType=*/16, /*isSigned*/false)));
+      testDictionary(dictionary2, (vector, index) -> ((UInt2Vector) vector).get(index));
+    }
+  }
+
+  @Test
+  public void testDictionaryUInt4() {
+    try (VarCharVector dictionaryVector = new VarCharVector("dict vector", allocator)) {
+      setVector(dictionaryVector, "0", "1", "2", "3", "4", "5", "6", "7", "8", "9");
+      Dictionary dictionary4 = new Dictionary(dictionaryVector,
+          new DictionaryEncoding(/*id=*/30L, /*ordered=*/false,
+              /*indexType=*/new ArrowType.Int(/*indexType=*/32, /*isSigned*/false)));
+      testDictionary(dictionary4, (vector, index) -> ((UInt4Vector) vector).get(index));
+    }
+  }
+
+  @Test
+  public void testDictionaryUInt8() {
+    try (VarCharVector dictionaryVector = new VarCharVector("dict vector", allocator)) {
+      setVector(dictionaryVector, "0", "1", "2", "3", "4", "5", "6", "7", "8", "9");
+      Dictionary dictionary8 = new Dictionary(dictionaryVector,
+              new DictionaryEncoding(/*id=*/40L, /*ordered=*/false,
+                  /*indexType=*/new ArrowType.Int(/*indexType=*/64, /*isSigned*/false)));
+      testDictionary(dictionary8, (vector, index) -> (int) ((UInt8Vector) vector).get(index));
+    }
+  }
+
+  @Test
+  public void testDictionaryUIntOverflow() {
+    // the size is within the range of UInt1, but outside the range of TinyInt.
+    final int vecLength = 256;
+    try (VarCharVector dictionaryVector = new VarCharVector("dict vector", allocator)) {
+      dictionaryVector.allocateNew(vecLength * 3, vecLength);
+      for (int i = 0; i < vecLength; i++) {
+        dictionaryVector.set(i, String.valueOf(i).getBytes());
+      }
+      dictionaryVector.setValueCount(vecLength);
+
+      Dictionary dictionary = new Dictionary(dictionaryVector,
+          new DictionaryEncoding(/*id=*/10L, /*ordered=*/false,
+              /*indexType=*/new ArrowType.Int(/*indexType=*/8, /*isSigned*/false)));
+
+      try (VarCharVector vector = new VarCharVector("vector", allocator)) {
+        setVector(vector, "255");
+        try (UInt1Vector encodedVector = (UInt1Vector) DictionaryEncoder.encode(vector, dictionary)) {
+
+          // verify encoded result
+          assertEquals(1, encodedVector.getValueCount());
+          assertEquals(255, encodedVector.getValueAsLong(0));
+
+          try (VarCharVector decodedVector = (VarCharVector) DictionaryEncoder.decode(encodedVector, dictionary)) {
+            assertEquals(1, decodedVector.getValueCount());
+            assertArrayEquals("255".getBytes(), decodedVector.get(0));
+          }
+        }
+      }
+    }
+  }
+
   private int[] convertListToIntArray(JsonStringArrayList list) {
     int[] values = new int[list.size()];
     for (int i = 0; i < list.size(); i++) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java
index 9a731fe099d..fc2a78597f3 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java
@@ -149,7 +149,7 @@ public void testSetLastSetUsage() throws Exception {
       dataVector.setSafe(0, 1, 10);
       dataVector.setSafe(1, 1, 11);
       dataVector.setSafe(2, 1, 12);
-      offsetBuffer.setInt((index + 1) * LargeListVector.OFFSET_WIDTH, 3);
+      offsetBuffer.setLong((index + 1) * LargeListVector.OFFSET_WIDTH, 3);
 
       index += 1;
 
@@ -157,7 +157,7 @@ public void testSetLastSetUsage() throws Exception {
       BitVectorHelper.setBit(validityBuffer, index);
       dataVector.setSafe(3, 1, 13);
       dataVector.setSafe(4, 1, 14);
-      offsetBuffer.setInt((index + 1) * LargeListVector.OFFSET_WIDTH, 5);
+      offsetBuffer.setLong((index + 1) * LargeListVector.OFFSET_WIDTH, 5);
 
       index += 1;
 
@@ -166,7 +166,7 @@ public void testSetLastSetUsage() throws Exception {
       dataVector.setSafe(5, 1, 15);
       dataVector.setSafe(6, 1, 16);
       dataVector.setSafe(7, 1, 17);
-      offsetBuffer.setInt((index + 1) * LargeListVector.OFFSET_WIDTH, 8);
+      offsetBuffer.setLong((index + 1) * LargeListVector.OFFSET_WIDTH, 8);
 
       /* check current lastSet */
       assertEquals(-1L, listVector.getLastSet());
@@ -217,7 +217,7 @@ public void testSetLastSetUsage() throws Exception {
       assertEquals(0.8D, listVector.getDensity(), 0);
 
       index = 0;
-      offset = offsetBuffer.getInt(index * LargeListVector.OFFSET_WIDTH);
+      offset = (int) offsetBuffer.getLong(index * LargeListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(0), Integer.toString(offset));
 
       Object actual = dataVector.getObject(offset);
@@ -230,7 +230,7 @@ public void testSetLastSetUsage() throws Exception {
       assertEquals(new Long(12), (Long) actual);
 
       index++;
-      offset = offsetBuffer.getInt(index * LargeListVector.OFFSET_WIDTH);
+      offset = (int) offsetBuffer.getLong(index * LargeListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(3), Integer.toString(offset));
 
       actual = dataVector.getObject(offset);
@@ -240,7 +240,7 @@ public void testSetLastSetUsage() throws Exception {
       assertEquals(new Long(14), (Long) actual);
 
       index++;
-      offset = offsetBuffer.getInt(index * LargeListVector.OFFSET_WIDTH);
+      offset = (int) offsetBuffer.getLong(index * LargeListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(5), Integer.toString(offset));
 
       actual = dataVector.getObject(offset);
@@ -253,7 +253,7 @@ public void testSetLastSetUsage() throws Exception {
       assertEquals(new Long(17), (Long) actual);
 
       index++;
-      offset = offsetBuffer.getInt(index * LargeListVector.OFFSET_WIDTH);
+      offset = (int) offsetBuffer.getLong(index * LargeListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(8), Integer.toString(offset));
 
       actual = dataVector.getObject(offset);
@@ -327,7 +327,7 @@ public void testSplitAndTransfer() throws Exception {
 
       /* index 0 */
       assertFalse(listVector.isNull(index));
-      offset = offsetBuffer.getInt(index * LargeListVector.OFFSET_WIDTH);
+      offset = (int) offsetBuffer.getLong(index * LargeListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(0), Integer.toString(offset));
 
       actual = dataVector.getObject(offset);
@@ -342,7 +342,7 @@ public void testSplitAndTransfer() throws Exception {
       /* index 1 */
       index++;
       assertFalse(listVector.isNull(index));
-      offset = offsetBuffer.getInt(index * LargeListVector.OFFSET_WIDTH);
+      offset = (int) offsetBuffer.getLong(index * LargeListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(3), Integer.toString(offset));
 
       actual = dataVector.getObject(offset);
@@ -354,7 +354,7 @@ public void testSplitAndTransfer() throws Exception {
       /* index 2 */
       index++;
       assertFalse(listVector.isNull(index));
-      offset = offsetBuffer.getInt(index * LargeListVector.OFFSET_WIDTH);
+      offset = (int) offsetBuffer.getLong(index * LargeListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(5), Integer.toString(offset));
 
       actual = dataVector.getObject(offset);
@@ -372,7 +372,7 @@ public void testSplitAndTransfer() throws Exception {
       /* index 3 */
       index++;
       assertFalse(listVector.isNull(index));
-      offset = offsetBuffer.getInt(index * LargeListVector.OFFSET_WIDTH);
+      offset = (int) offsetBuffer.getLong(index * LargeListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(9), Integer.toString(offset));
 
       actual = dataVector.getObject(offset);
@@ -381,7 +381,7 @@ public void testSplitAndTransfer() throws Exception {
       /* index 4 */
       index++;
       assertFalse(listVector.isNull(index));
-      offset = offsetBuffer.getInt(index * LargeListVector.OFFSET_WIDTH);
+      offset = (int) offsetBuffer.getLong(index * LargeListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(10), Integer.toString(offset));
 
       actual = dataVector.getObject(offset);
@@ -399,7 +399,7 @@ public void testSplitAndTransfer() throws Exception {
       /* index 5 */
       index++;
       assertTrue(listVector.isNull(index));
-      offset = offsetBuffer.getInt(index * LargeListVector.OFFSET_WIDTH);
+      offset = (int) offsetBuffer.getLong(index * LargeListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(14), Integer.toString(offset));
 
       /* do split and transfer */
@@ -428,16 +428,16 @@ public void testSplitAndTransfer() throws Exception {
           BigIntVector dataVector1 = (BigIntVector) toVector.getDataVector();
 
           for (int i = 0; i < splitLength; i++) {
-            dataLength1 = offsetBuffer.getInt((start + i + 1) * LargeListVector.OFFSET_WIDTH) -
-                    offsetBuffer.getInt((start + i) * LargeListVector.OFFSET_WIDTH);
-            dataLength2 = toOffsetBuffer.getInt((i + 1) * LargeListVector.OFFSET_WIDTH) -
-                    toOffsetBuffer.getInt(i * LargeListVector.OFFSET_WIDTH);
+            dataLength1 = (int) offsetBuffer.getLong((start + i + 1) * LargeListVector.OFFSET_WIDTH) -
+                    (int) offsetBuffer.getLong((start + i) * LargeListVector.OFFSET_WIDTH);
+            dataLength2 = (int) toOffsetBuffer.getLong((i + 1) * LargeListVector.OFFSET_WIDTH) -
+                    (int) toOffsetBuffer.getLong(i * LargeListVector.OFFSET_WIDTH);
 
             assertEquals("Different data lengths at index: " + i + " and start: " + start,
                     dataLength1, dataLength2);
 
-            offset1 = offsetBuffer.getInt((start + i) * LargeListVector.OFFSET_WIDTH);
-            offset2 = toOffsetBuffer.getInt(i * LargeListVector.OFFSET_WIDTH);
+            offset1 = (int) offsetBuffer.getLong((start + i) * LargeListVector.OFFSET_WIDTH);
+            offset2 = (int) toOffsetBuffer.getLong(i * LargeListVector.OFFSET_WIDTH);
 
             for (int j = 0; j < dataLength1; j++) {
               assertEquals("Different data at indexes: " + offset1 + " and " + offset2,
@@ -560,9 +560,9 @@ public void testNestedLargeListVector() throws Exception {
       final ArrowBuf offsetBuffer = listVector.getOffsetBuffer();
 
       /* listVector has 2 lists at index 0 and 3 lists at index 1 */
-      assertEquals(0, offsetBuffer.getInt(0 * LargeListVector.OFFSET_WIDTH));
-      assertEquals(2, offsetBuffer.getInt(1 * LargeListVector.OFFSET_WIDTH));
-      assertEquals(5, offsetBuffer.getInt(2 * LargeListVector.OFFSET_WIDTH));
+      assertEquals(0, offsetBuffer.getLong(0 * LargeListVector.OFFSET_WIDTH));
+      assertEquals(2, offsetBuffer.getLong(1 * LargeListVector.OFFSET_WIDTH));
+      assertEquals(5, offsetBuffer.getLong(2 * LargeListVector.OFFSET_WIDTH));
     }
   }
 
@@ -688,9 +688,9 @@ public void testNestedLargeListVector2() throws Exception {
       final ArrowBuf offsetBuffer = listVector.getOffsetBuffer();
 
       /* listVector has 2 lists at index 0 and 3 lists at index 1 */
-      assertEquals(0, offsetBuffer.getInt(0 * LargeListVector.OFFSET_WIDTH));
-      assertEquals(2, offsetBuffer.getInt(1 * LargeListVector.OFFSET_WIDTH));
-      assertEquals(4, offsetBuffer.getInt(2 * LargeListVector.OFFSET_WIDTH));
+      assertEquals(0, offsetBuffer.getLong(0 * LargeListVector.OFFSET_WIDTH));
+      assertEquals(2, offsetBuffer.getLong(1 * LargeListVector.OFFSET_WIDTH));
+      assertEquals(4, offsetBuffer.getLong(2 * LargeListVector.OFFSET_WIDTH));
     }
   }
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestTypeLayout.java b/java/vector/src/test/java/org/apache/arrow/vector/TestTypeLayout.java
index 18175276737..97930f433d3 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestTypeLayout.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestTypeLayout.java
@@ -61,9 +61,13 @@ public void testTypeBufferCount() {
     type = new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE);
     assertEquals(TypeLayout.getTypeBufferCount(type), TypeLayout.getTypeLayout(type).getBufferLayouts().size());
 
-    type = new ArrowType.Decimal(10, 10);
+    type = new ArrowType.Decimal(10, 10, 128);
     assertEquals(TypeLayout.getTypeBufferCount(type), TypeLayout.getTypeLayout(type).getBufferLayouts().size());
 
+    type = new ArrowType.Decimal(10, 10, 256);
+    assertEquals(TypeLayout.getTypeBufferCount(type), TypeLayout.getTypeLayout(type).getBufferLayouts().size());
+
+
     type = new ArrowType.FixedSizeBinary(5);
     assertEquals(TypeLayout.getTypeBufferCount(type), TypeLayout.getTypeLayout(type).getBufferLayouts().size());
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index 0cd9e801800..0be5e1e963c 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -2977,4 +2977,47 @@ public void testEmptyBufBehavior() {
       assertEquals(0, vector.getOffsetBuffer().capacity());
     }
   }
+
+  @Test
+  public void testSetGetUInt1() {
+    try (UInt1Vector vector = new UInt1Vector("vector", allocator)) {
+      vector.allocateNew(2);
+
+      vector.setWithPossibleTruncate(0, UInt1Vector.MAX_UINT1);
+      vector.setUnsafeWithPossibleTruncate(1, UInt1Vector.MAX_UINT1);
+      vector.setValueCount(2);
+
+      assertEquals(UInt1Vector.MAX_UINT1 & UInt1Vector.PROMOTION_MASK, vector.getValueAsLong(0));
+      assertEquals(UInt1Vector.MAX_UINT1 & UInt1Vector.PROMOTION_MASK, vector.getValueAsLong(1));
+    }
+  }
+
+  @Test
+  public void testSetGetUInt2() {
+    try (UInt2Vector vector = new UInt2Vector("vector", allocator)) {
+      vector.allocateNew(2);
+
+      vector.setWithPossibleTruncate(0, UInt2Vector.MAX_UINT2);
+      vector.setUnsafeWithPossibleTruncate(1, UInt2Vector.MAX_UINT2);
+      vector.setValueCount(2);
+
+      assertEquals(UInt2Vector.MAX_UINT2, vector.getValueAsLong(0));
+      assertEquals(UInt2Vector.MAX_UINT2, vector.getValueAsLong(1));
+    }
+  }
+
+  @Test
+  public void testSetGetUInt4() {
+    try (UInt4Vector vector = new UInt4Vector("vector", allocator)) {
+      vector.allocateNew(2);
+
+      vector.setWithPossibleTruncate(0, UInt4Vector.MAX_UINT4);
+      vector.setUnsafeWithPossibleTruncate(1, UInt4Vector.MAX_UINT4);
+      vector.setValueCount(2);
+
+      long expected = UInt4Vector.MAX_UINT4 & UInt4Vector.PROMOTION_MASK;
+      assertEquals(expected, vector.getValueAsLong(0));
+      assertEquals(expected, vector.getValueAsLong(1));
+    }
+  }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorAlloc.java
index 089f1f84ff8..b9e7c8661a7 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorAlloc.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorAlloc.java
@@ -72,7 +72,7 @@ public void testVectorAllocWithField() {
         field("UTF8", MinorType.VARCHAR.getType()),
         field("VARBINARY", MinorType.VARBINARY.getType()),
         field("BIT", MinorType.BIT.getType()),
-        field("DECIMAL", new Decimal(38, 5)),
+        field("DECIMAL", new Decimal(38, 5, 128)),
         field("FIXEDSIZEBINARY", new FixedSizeBinary(50)),
         field("DATEDAY", MinorType.DATEDAY.getType()),
         field("DATEMILLI", MinorType.DATEMILLI.getType()),
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
index e8a51f9d7f2..2d3f82d9956 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
@@ -23,7 +23,6 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.util.DataSizeRoundingUtil;
 import org.apache.arrow.vector.complex.DenseUnionVector;
 import org.apache.arrow.vector.complex.FixedSizeListVector;
 import org.apache.arrow.vector.complex.ListVector;
@@ -37,6 +36,7 @@
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.DataSizeRoundingUtil;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java
index a0f35052634..0c560abe23d 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java
@@ -24,6 +24,7 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.DecimalVector;
 import org.apache.arrow.vector.compare.VectorEqualsVisitor;
 import org.apache.arrow.vector.complex.FixedSizeListVector;
 import org.apache.arrow.vector.complex.ListVector;
@@ -522,12 +523,12 @@ public void testMapWithListValue() throws Exception {
   public void testCopyFixedSizedListOfDecimalsVector() {
     try (FixedSizeListVector from = FixedSizeListVector.empty("v", 4, allocator);
          FixedSizeListVector to = FixedSizeListVector.empty("v", 4, allocator)) {
-      from.addOrGetVector(FieldType.nullable(new ArrowType.Decimal(3, 0)));
-      to.addOrGetVector(FieldType.nullable(new ArrowType.Decimal(3, 0)));
+      from.addOrGetVector(FieldType.nullable(new ArrowType.Decimal(3, 0, 128)));
+      to.addOrGetVector(FieldType.nullable(new ArrowType.Decimal(3, 0, 128)));
 
       DecimalHolder holder = new DecimalHolder();
-      holder.buffer = allocator.buffer(DecimalUtility.DECIMAL_BYTE_LENGTH);
-      ArrowType arrowType = new ArrowType.Decimal(3, 0);
+      holder.buffer = allocator.buffer(DecimalVector.TYPE_WIDTH);
+      ArrowType arrowType = new ArrowType.Decimal(3, 0, 128);
 
       // populate from vector
       UnionFixedSizeListWriter writer = from.getWriter();
@@ -535,13 +536,13 @@ public void testCopyFixedSizedListOfDecimalsVector() {
         writer.startList();
         writer.decimal().writeDecimal(BigDecimal.valueOf(i));
 
-        DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(i * 2), holder.buffer, 0);
+        DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(i * 2), holder.buffer, 0, DecimalVector.TYPE_WIDTH);
         holder.start = 0;
         holder.scale = 0;
         holder.precision = 3;
         writer.decimal().write(holder);
 
-        DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(i * 3), holder.buffer, 0);
+        DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(i * 3), holder.buffer, 0, DecimalVector.TYPE_WIDTH);
         writer.decimal().writeDecimal(0, holder.buffer, arrowType);
 
         writer.decimal().writeBigEndianBytesToDecimal(BigDecimal.valueOf(i * 4).unscaledValue().toByteArray(),
@@ -582,7 +583,7 @@ public void testCopyUnionListWithDecimal() {
         listWriter.decimal().writeDecimal(BigDecimal.valueOf(i * 2));
         listWriter.integer().writeInt(i);
         listWriter.decimal().writeBigEndianBytesToDecimal(BigDecimal.valueOf(i * 3).unscaledValue().toByteArray(),
-            new ArrowType.Decimal(3, 0));
+            new ArrowType.Decimal(3, 0, 128));
 
         listWriter.endList();
       }
@@ -623,7 +624,7 @@ public void testCopyStructVector() {
         innerStructWriter.integer("innerint").writeInt(i * 3);
         innerStructWriter.decimal("innerdec", 0, 38).writeDecimal(BigDecimal.valueOf(i * 4));
         innerStructWriter.decimal("innerdec", 0, 38).writeBigEndianBytesToDecimal(BigDecimal.valueOf(i * 4)
-            .unscaledValue().toByteArray(), new ArrowType.Decimal(3, 0));
+            .unscaledValue().toByteArray(), new ArrowType.Decimal(3, 0, 128));
         innerStructWriter.end();
         structWriter.end();
       }
@@ -649,8 +650,8 @@ public void testCopyStructVector() {
   public void testCopyDecimalVectorWrongScale() {
     try (FixedSizeListVector from = FixedSizeListVector.empty("v", 3, allocator);
          FixedSizeListVector to = FixedSizeListVector.empty("v", 3, allocator)) {
-      from.addOrGetVector(FieldType.nullable(new ArrowType.Decimal(3, 2)));
-      to.addOrGetVector(FieldType.nullable(new ArrowType.Decimal(3, 1)));
+      from.addOrGetVector(FieldType.nullable(new ArrowType.Decimal(3, 2, 128)));
+      to.addOrGetVector(FieldType.nullable(new ArrowType.Decimal(3, 1, 128)));
 
       // populate from vector
       UnionFixedSizeListWriter writer = from.getWriter();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index 769a94f50f2..c378f6d53f0 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -30,6 +30,7 @@
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.util.AutoCloseables;
 import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.DecimalVector;
 import org.apache.arrow.vector.Float4Vector;
 import org.apache.arrow.vector.Float8Vector;
 import org.apache.arrow.vector.IntVector;
@@ -310,21 +311,21 @@ public void listDecimalType() {
       listVector.allocateNew();
       UnionListWriter listWriter = new UnionListWriter(listVector);
       DecimalHolder holder = new DecimalHolder();
-      holder.buffer = allocator.buffer(DecimalUtility.DECIMAL_BYTE_LENGTH);
-      ArrowType arrowType = new ArrowType.Decimal(10, 0);
+      holder.buffer = allocator.buffer(DecimalVector.TYPE_WIDTH);
+      ArrowType arrowType = new ArrowType.Decimal(10, 0, 128);
       for (int i = 0; i < COUNT; i++) {
         listWriter.startList();
         for (int j = 0; j < i % 7; j++) {
           if (j % 4 == 0) {
             listWriter.writeDecimal(new BigDecimal(j));
           } else if (j % 4 == 1) {
-            DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(j), holder.buffer, 0);
+            DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(j), holder.buffer, 0, DecimalVector.TYPE_WIDTH);
             holder.start = 0;
             holder.scale = 0;
             holder.precision = 10;
             listWriter.write(holder);
           } else if (j % 4 == 2) {
-            DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(j), holder.buffer, 0);
+            DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(j), holder.buffer, 0, DecimalVector.TYPE_WIDTH);
             listWriter.writeDecimal(0, holder.buffer, arrowType);
           } else {
             byte[] value = BigDecimal.valueOf(j).unscaledValue().toByteArray();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/MessageSerializerTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/MessageSerializerTest.java
index a26a0ac62ad..d4e97c2605e 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/MessageSerializerTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/MessageSerializerTest.java
@@ -84,7 +84,7 @@ public void testWriteMessageBufferAligned() throws IOException {
     WriteChannel out = new WriteChannel(Channels.newChannel(outputStream));
 
     // This is not a valid Arrow Message, only to test writing and alignment
-    ByteBuffer buffer = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN);
+    ByteBuffer buffer = ByteBuffer.allocate(8).order(ByteOrder.nativeOrder());
     buffer.putInt(1);
     buffer.putInt(2);
     buffer.flip();
@@ -100,19 +100,25 @@ public void testWriteMessageBufferAligned() throws IOException {
 
     ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray());
     ReadChannel in = new ReadChannel(Channels.newChannel(inputStream));
-    ByteBuffer result = ByteBuffer.allocate(32).order(ByteOrder.LITTLE_ENDIAN);
+    ByteBuffer result = ByteBuffer.allocate(32).order(ByteOrder.nativeOrder());
     in.readFully(result);
     result.rewind();
 
     // First message continuation, size, and 2 int values
     assertEquals(MessageSerializer.IPC_CONTINUATION_TOKEN, result.getInt());
+    // mesage length is represented in little endian
+    result.order(ByteOrder.LITTLE_ENDIAN);
     assertEquals(8, result.getInt());
+    result.order(ByteOrder.nativeOrder());
     assertEquals(1, result.getInt());
     assertEquals(2, result.getInt());
 
     // Second message continuation, size, 1 int value and 4 bytes padding
     assertEquals(MessageSerializer.IPC_CONTINUATION_TOKEN, result.getInt());
+    // mesage length is represented in little endian
+    result.order(ByteOrder.LITTLE_ENDIAN);
     assertEquals(8, result.getInt());
+    result.order(ByteOrder.nativeOrder());
     assertEquals(3, result.getInt());
     assertEquals(0, result.getInt());
   }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java
index 15a19ed62d6..84e26a3414a 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java
@@ -21,10 +21,12 @@
 import static java.util.Arrays.asList;
 import static org.apache.arrow.memory.util.LargeMemoryUtil.checkedCastToInt;
 import static org.apache.arrow.vector.TestUtils.newVarCharVector;
+import static org.apache.arrow.vector.TestUtils.newVector;
 import static org.apache.arrow.vector.testing.ValueVectorDataPopulator.setVector;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
 import java.io.ByteArrayInputStream;
@@ -41,6 +43,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.function.BiFunction;
 import java.util.stream.Collectors;
 
 import org.apache.arrow.flatbuf.FieldNode;
@@ -55,11 +58,16 @@
 import org.apache.arrow.vector.IntVector;
 import org.apache.arrow.vector.NullVector;
 import org.apache.arrow.vector.TestUtils;
+import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.VectorLoader;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.VectorUnloader;
+import org.apache.arrow.vector.compare.Range;
+import org.apache.arrow.vector.compare.RangeEqualsVisitor;
+import org.apache.arrow.vector.compare.TypeEqualsVisitor;
 import org.apache.arrow.vector.compare.VectorEqualsVisitor;
+import org.apache.arrow.vector.complex.StructVector;
 import org.apache.arrow.vector.dictionary.Dictionary;
 import org.apache.arrow.vector.dictionary.DictionaryEncoder;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
@@ -69,6 +77,7 @@
 import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
 import org.apache.arrow.vector.ipc.message.IpcOption;
 import org.apache.arrow.vector.ipc.message.MessageSerializer;
+import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
@@ -87,10 +96,12 @@ public class TestArrowReaderWriter {
   private VarCharVector dictionaryVector1;
   private VarCharVector dictionaryVector2;
   private VarCharVector dictionaryVector3;
+  private StructVector dictionaryVector4;
 
   private Dictionary dictionary1;
   private Dictionary dictionary2;
   private Dictionary dictionary3;
+  private Dictionary dictionary4;
 
   private Schema schema;
   private Schema encodedSchema;
@@ -119,6 +130,12 @@ public void init() {
         "aa".getBytes(StandardCharsets.UTF_8),
         "bb".getBytes(StandardCharsets.UTF_8),
         "cc".getBytes(StandardCharsets.UTF_8));
+    
+    dictionaryVector4 = newVector(StructVector.class, "D4", MinorType.STRUCT, allocator);
+    final Map<String, List<Integer>> dictionaryValues4 = new HashMap<>();
+    dictionaryValues4.put("a", Arrays.asList(1, 2, 3));
+    dictionaryValues4.put("b", Arrays.asList(4, 5, 6));
+    setVector(dictionaryVector4, dictionaryValues4);
 
     dictionary1 = new Dictionary(dictionaryVector1,
         new DictionaryEncoding(/*id=*/1L, /*ordered=*/false, /*indexType=*/null));
@@ -126,6 +143,8 @@ public void init() {
         new DictionaryEncoding(/*id=*/2L, /*ordered=*/false, /*indexType=*/null));
     dictionary3 = new Dictionary(dictionaryVector3,
         new DictionaryEncoding(/*id=*/1L, /*ordered=*/false, /*indexType=*/null));
+    dictionary4 = new Dictionary(dictionaryVector4,
+        new DictionaryEncoding(/*id=*/3L, /*ordered=*/false, /*indexType=*/null));
   }
 
   @After
@@ -133,6 +152,7 @@ public void terminate() throws Exception {
     dictionaryVector1.close();
     dictionaryVector2.close();
     dictionaryVector3.close();
+    dictionaryVector4.close();
     allocator.close();
   }
 
@@ -305,6 +325,82 @@ public void testWriteReadWithDictionaries() throws IOException {
     }
   }
 
+  @Test
+  public void testWriteReadWithStructDictionaries() throws IOException {
+    DictionaryProvider.MapDictionaryProvider provider =
+        new DictionaryProvider.MapDictionaryProvider();
+    provider.put(dictionary4);
+
+    try (final StructVector vector =
+        newVector(StructVector.class, "D4", MinorType.STRUCT, allocator)) {
+      final Map<String, List<Integer>> values = new HashMap<>();
+      // Index: 0, 2, 1, 2, 1, 0, 0
+      values.put("a", Arrays.asList(1, 3, 2, 3, 2, 1, 1));
+      values.put("b", Arrays.asList(4, 6, 5, 6, 5, 4, 4));
+      setVector(vector, values);
+      FieldVector encodedVector = (FieldVector) DictionaryEncoder.encode(vector, dictionary4);
+
+      List<Field> fields = Arrays.asList(encodedVector.getField());
+      List<FieldVector> vectors = Collections2.asImmutableList(encodedVector);
+      try (
+          VectorSchemaRoot root =
+              new VectorSchemaRoot(fields, vectors, encodedVector.getValueCount());
+          ByteArrayOutputStream out = new ByteArrayOutputStream();
+          ArrowFileWriter writer = new ArrowFileWriter(root, provider, newChannel(out));) {
+
+        writer.start();
+        writer.writeBatch();
+        writer.end();
+
+        try (
+            SeekableReadChannel channel = new SeekableReadChannel(
+                new ByteArrayReadableSeekableByteChannel(out.toByteArray()));
+            ArrowFileReader reader = new ArrowFileReader(channel, allocator)) {
+          final VectorSchemaRoot readRoot = reader.getVectorSchemaRoot();
+          final Schema readSchema = readRoot.getSchema();
+          assertEquals(root.getSchema(), readSchema);
+          assertEquals(1, reader.getDictionaryBlocks().size());
+          assertEquals(1, reader.getRecordBlocks().size());
+
+          reader.loadNextBatch();
+          assertEquals(1, readRoot.getFieldVectors().size());
+          assertEquals(1, reader.getDictionaryVectors().size());
+
+          // Read the encoded vector and check it
+          final FieldVector readEncoded = readRoot.getVector(0);
+          assertEquals(encodedVector.getValueCount(), readEncoded.getValueCount());
+          assertTrue(new RangeEqualsVisitor(encodedVector, readEncoded)
+              .rangeEquals(new Range(0, 0, encodedVector.getValueCount())));
+
+          // Read the dictionary
+          final Map<Long, Dictionary> readDictionaryMap = reader.getDictionaryVectors();
+          final Dictionary readDictionary =
+              readDictionaryMap.get(readEncoded.getField().getDictionary().getId());
+          assertNotNull(readDictionary);
+
+          // Assert the dictionary vector is correct
+          final FieldVector readDictionaryVector = readDictionary.getVector();
+          assertEquals(dictionaryVector4.getValueCount(), readDictionaryVector.getValueCount());
+          final BiFunction<ValueVector, ValueVector, Boolean> typeComparatorIgnoreName =
+              (v1, v2) -> new TypeEqualsVisitor(v1, false, true).equals(v2);
+          assertTrue("Dictionary vectors are not equal",
+              new RangeEqualsVisitor(dictionaryVector4, readDictionaryVector,
+                  typeComparatorIgnoreName)
+                      .rangeEquals(new Range(0, 0, dictionaryVector4.getValueCount())));
+
+          // Assert the decoded vector is correct
+          try (final ValueVector readVector =
+              DictionaryEncoder.decode(readEncoded, readDictionary)) {
+            assertEquals(vector.getValueCount(), readVector.getValueCount());
+            assertTrue("Decoded vectors are not equal",
+                new RangeEqualsVisitor(vector, readVector, typeComparatorIgnoreName)
+                    .rangeEquals(new Range(0, 0, vector.getValueCount())));
+          }
+        }
+      }
+    }
+  }
+
   @Test
   public void testEmptyStreamInFileIPC() throws IOException {
 
@@ -716,7 +812,7 @@ public void testLegacyIpcBackwardsCompatibility() throws Exception {
 
   @Test
   public void testChannelReadFully() throws IOException {
-    final ByteBuffer buf = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN);
+    final ByteBuffer buf = ByteBuffer.allocate(4).order(ByteOrder.nativeOrder());
     buf.putInt(200);
     buf.rewind();
 
@@ -737,7 +833,7 @@ public void testChannelReadFully() throws IOException {
 
   @Test
   public void testChannelReadFullyEos() throws IOException {
-    final ByteBuffer buf = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN);
+    final ByteBuffer buf = ByteBuffer.allocate(4).order(ByteOrder.nativeOrder());
     buf.putInt(10);
     buf.rewind();
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestUIntDictionaryRoundTrip.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestUIntDictionaryRoundTrip.java
new file mode 100644
index 00000000000..6aa7a0c6df5
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestUIntDictionaryRoundTrip.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.ipc;
+
+import static org.apache.arrow.vector.testing.ValueVectorDataPopulator.setVector;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.channels.Channels;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+import java.util.function.ToIntBiFunction;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.UInt1Vector;
+import org.apache.arrow.vector.UInt2Vector;
+import org.apache.arrow.vector.UInt4Vector;
+import org.apache.arrow.vector.UInt8Vector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.dictionary.Dictionary;
+import org.apache.arrow.vector.dictionary.DictionaryProvider;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+/**
+ * Test the round-trip of dictionary encoding,
+ * with unsigned integer as indices.
+ */
+@RunWith(Parameterized.class)
+public class TestUIntDictionaryRoundTrip {
+
+  private final boolean streamMode;
+
+  public TestUIntDictionaryRoundTrip(boolean streamMode) {
+    this.streamMode = streamMode;
+  }
+
+  private BufferAllocator allocator;
+
+  private DictionaryProvider.MapDictionaryProvider dictionaryProvider;
+
+  @Before
+  public void init() {
+    allocator = new RootAllocator(Long.MAX_VALUE);
+    dictionaryProvider = new DictionaryProvider.MapDictionaryProvider();
+  }
+
+  @After
+  public void terminate() throws Exception {
+    allocator.close();
+  }
+
+  private byte[] writeData(FieldVector encodedVector) throws IOException {
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    VectorSchemaRoot root =
+        new VectorSchemaRoot(
+            Arrays.asList(encodedVector.getField()), Arrays.asList(encodedVector), encodedVector.getValueCount());
+    try (ArrowWriter writer = streamMode ?
+        new ArrowStreamWriter(root, dictionaryProvider, out) :
+        new ArrowFileWriter(root, dictionaryProvider, Channels.newChannel(out))) {
+      writer.start();
+      writer.writeBatch();
+      writer.end();
+
+      return out.toByteArray();
+    }
+  }
+
+  private void readData(
+      byte[] data,
+      Field expectedField,
+      ToIntBiFunction<ValueVector, Integer> valGetter,
+      long dictionaryID,
+      int[] expectedIndices,
+      String[] expectedDictItems) throws IOException {
+    try (ArrowReader reader = streamMode ?
+             new ArrowStreamReader(new ByteArrayInputStream(data), allocator) :
+             new ArrowFileReader(new SeekableReadChannel(new ByteArrayReadableSeekableByteChannel(data)), allocator)) {
+
+      // verify schema
+      Schema readSchema = reader.getVectorSchemaRoot().getSchema();
+      assertEquals(1, readSchema.getFields().size());
+      assertEquals(expectedField, readSchema.getFields().get(0));
+
+      // verify vector schema root
+      assertTrue(reader.loadNextBatch());
+      VectorSchemaRoot root = reader.getVectorSchemaRoot();
+
+      assertEquals(1, root.getFieldVectors().size());
+      ValueVector encodedVector = root.getVector(0);
+      assertEquals(expectedIndices.length, encodedVector.getValueCount());
+
+      for (int i = 0; i < expectedIndices.length; i++) {
+        assertEquals(expectedIndices[i], valGetter.applyAsInt(encodedVector, i));
+      }
+
+      // verify dictionary
+      Map<Long, Dictionary> dictVectors = reader.getDictionaryVectors();
+      assertEquals(1, dictVectors.size());
+      Dictionary dictionary = dictVectors.get(dictionaryID);
+      assertNotNull(dictionary);
+
+      assertTrue(dictionary.getVector() instanceof VarCharVector);
+      VarCharVector dictVector = (VarCharVector) dictionary.getVector();
+      assertEquals(expectedDictItems.length, dictVector.getValueCount());
+      for (int i = 0; i < dictVector.getValueCount(); i++) {
+        assertArrayEquals(expectedDictItems[i].getBytes(), dictVector.get(i));
+      }
+    }
+  }
+
+  private ValueVector createEncodedVector(int bitWidth, VarCharVector dictionaryVector) {
+    final DictionaryEncoding dictionaryEncoding =
+        new DictionaryEncoding(bitWidth, false, new ArrowType.Int(bitWidth, false));
+    Dictionary dictionary = new Dictionary(dictionaryVector, dictionaryEncoding);
+    dictionaryProvider.put(dictionary);
+
+    final FieldType type =
+        new FieldType(true, dictionaryEncoding.getIndexType(), dictionaryEncoding, null);
+    final Field field = new Field("encoded", type, null);
+    return field.createVector(allocator);
+  }
+
+  @Test
+  public void testUInt1RoundTrip() throws IOException {
+    final int vectorLength = UInt1Vector.MAX_UINT1 & UInt1Vector.PROMOTION_MASK;
+    try (VarCharVector dictionaryVector = new VarCharVector("dictionary", allocator);
+         UInt1Vector encodedVector1 = (UInt1Vector) createEncodedVector(8, dictionaryVector)) {
+      int[] indices = new int[vectorLength];
+      String[] dictionaryItems = new String[vectorLength];
+      for (int i = 0; i < vectorLength; i++) {
+        encodedVector1.setSafe(i, (byte) i);
+        indices[i] = i;
+        dictionaryItems[i] = String.valueOf(i);
+      }
+      encodedVector1.setValueCount(vectorLength);
+      setVector(dictionaryVector, dictionaryItems);
+      byte[] data = writeData(encodedVector1);
+      readData(
+          data, encodedVector1.getField(), (vector, index) -> (int) ((UInt1Vector) vector).getValueAsLong(index),
+          8L, indices, dictionaryItems);
+    }
+  }
+
+  @Test
+  public void testUInt2RoundTrip() throws IOException {
+    try (VarCharVector dictionaryVector = new VarCharVector("dictionary", allocator);
+        UInt2Vector encodedVector2 = (UInt2Vector) createEncodedVector(16, dictionaryVector)) {
+      int[] indices = new int[]{1, 3, 5, 7, 9, UInt2Vector.MAX_UINT2};
+      String[] dictItems = new String[UInt2Vector.MAX_UINT2];
+      for (int i = 0; i < UInt2Vector.MAX_UINT2; i++) {
+        dictItems[i] = String.valueOf(i);
+      }
+
+      setVector(encodedVector2, (char) 1, (char) 3, (char) 5, (char) 7, (char) 9, UInt2Vector.MAX_UINT2);
+      setVector(dictionaryVector, dictItems);
+
+      byte[] data = writeData(encodedVector2);
+      readData(data, encodedVector2.getField(), (vector, index) -> (int) ((UInt2Vector) vector).getValueAsLong(index),
+          16L, indices, dictItems);
+    }
+  }
+
+  @Test
+  public void testUInt4RoundTrip() throws IOException {
+    final int dictLength = 10;
+    try (VarCharVector dictionaryVector = new VarCharVector("dictionary", allocator);
+        UInt4Vector encodedVector4 = (UInt4Vector) createEncodedVector(32, dictionaryVector)) {
+      int[] indices = new int[]{1, 3, 5, 7, 9};
+      String[] dictItems = new String[dictLength];
+      for (int i = 0; i < dictLength; i++) {
+        dictItems[i] = String.valueOf(i);
+      }
+
+      setVector(encodedVector4, 1, 3, 5, 7, 9);
+      setVector(dictionaryVector, dictItems);
+
+      setVector(encodedVector4, 1, 3, 5, 7, 9);
+      byte[] data = writeData(encodedVector4);
+      readData(data, encodedVector4.getField(), (vector, index) -> (int) ((UInt4Vector) vector).getValueAsLong(index),
+          32L, indices, dictItems);
+    }
+  }
+
+  @Test
+  public void testUInt8RoundTrip() throws IOException {
+    final int dictLength = 10;
+    try (VarCharVector dictionaryVector = new VarCharVector("dictionary", allocator);
+        UInt8Vector encodedVector8 = (UInt8Vector) createEncodedVector(64, dictionaryVector)) {
+      int[] indices = new int[]{1, 3, 5, 7, 9};
+      String[] dictItems = new String[dictLength];
+      for (int i = 0; i < dictLength; i++) {
+        dictItems[i] = String.valueOf(i);
+      }
+
+      setVector(encodedVector8, 1L, 3L, 5L, 7L, 9L);
+      setVector(dictionaryVector, dictItems);
+
+      byte[] data = writeData(encodedVector8);
+      readData(data, encodedVector8.getField(), (vector, index) -> (int) ((UInt8Vector) vector).getValueAsLong(index),
+          64L, indices, dictItems);
+    }
+  }
+
+  @Parameterized.Parameters(name = "stream mode = {0}")
+  public static Collection<Object[]> getRepeat() {
+    return Arrays.asList(
+        new Object[]{true},
+        new Object[]{false}
+    );
+  }
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/testing/RandomDataGenerator.java b/java/vector/src/test/java/org/apache/arrow/vector/testing/RandomDataGenerator.java
new file mode 100644
index 00000000000..4b1094d288b
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/testing/RandomDataGenerator.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.testing;
+
+import java.util.Random;
+import java.util.function.Supplier;
+
+/**
+ * Utility for generating random data.
+ */
+public class RandomDataGenerator {
+
+  static final Random random = new Random(0);
+
+  public static final Supplier<Byte> TINY_INT_GENERATOR = () -> (byte) random.nextInt();
+
+  public static final Supplier<Short> SMALL_INT_GENERATOR = () -> (short) random.nextInt();
+
+  public static final Supplier<Integer> INT_GENERATOR = () -> random.nextInt();
+
+  public static final Supplier<Long> LONG_GENERATOR = () -> random.nextLong();
+
+  public static final Supplier<Float> FLOAT_GENERATOR = () -> random.nextFloat();
+
+  public static final Supplier<Double> DOUBLE_GENERATOR = () -> random.nextDouble();
+
+  private RandomDataGenerator() {
+  }
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java b/java/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java
index 9975682605f..15d6a5cf993 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java
@@ -21,6 +21,8 @@
 
 import java.nio.charset.StandardCharsets;
 import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
 
 import org.apache.arrow.vector.BigIntVector;
 import org.apache.arrow.vector.BitVector;
@@ -58,9 +60,12 @@
 import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.complex.BaseRepeatedValueVector;
 import org.apache.arrow.vector.complex.FixedSizeListVector;
+import org.apache.arrow.vector.complex.LargeListVector;
 import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.StructVector;
 import org.apache.arrow.vector.holders.IntervalDayHolder;
 import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.FieldType;
 
 /**
@@ -608,6 +613,37 @@ public static void setVector(ListVector vector, List<Integer>... values) {
     vector.setValueCount(values.length);
   }
 
+  /**
+   * Populate values for {@link LargeListVector}.
+   */
+  public static void setVector(LargeListVector vector, List<Integer>... values) {
+    vector.allocateNewSafe();
+    Types.MinorType type = Types.MinorType.INT;
+    vector.addOrGetVector(FieldType.nullable(type.getType()));
+
+    IntVector dataVector = (IntVector) vector.getDataVector();
+    dataVector.allocateNew();
+
+    // set underlying vectors
+    int curPos = 0;
+    vector.getOffsetBuffer().setLong(0, curPos);
+    for (int i = 0; i < values.length; i++) {
+      if (values[i] == null) {
+        BitVectorHelper.unsetBit(vector.getValidityBuffer(), i);
+      } else {
+        BitVectorHelper.setBit(vector.getValidityBuffer(), i);
+        for (int value : values[i]) {
+          dataVector.setSafe(curPos, value);
+          curPos += 1;
+        }
+      }
+      vector.getOffsetBuffer().setLong((long) (i + 1) * LargeListVector.OFFSET_WIDTH, curPos);
+    }
+    dataVector.setValueCount(curPos);
+    vector.setLastSet(values.length - 1);
+    vector.setValueCount(values.length);
+  }
+
   /**
    * Populate values for {@link FixedSizeListVector}.
    */
@@ -641,4 +677,32 @@ public static void setVector(FixedSizeListVector vector, List<Integer>... values
     dataVector.setValueCount(curPos);
     vector.setValueCount(values.length);
   }
+
+  /**
+   * Populate values for {@link StructVector}.
+   */
+  public static void setVector(StructVector vector, Map<String, List<Integer>> values) {
+    vector.allocateNewSafe();
+
+    int valueCount = 0;
+    for (final Entry<String, List<Integer>> entry : values.entrySet()) {
+      // Add the child
+      final IntVector child = vector.addOrGet(entry.getKey(),
+          FieldType.nullable(MinorType.INT.getType()), IntVector.class);
+
+      // Write the values to the child
+      child.allocateNew();
+      final List<Integer> v = entry.getValue();
+      for (int i = 0; i < v.size(); i++) {
+        if (v.get(i) != null) {
+          child.set(i, v.get(i));
+          vector.setIndexDefined(i);
+        } else {
+          child.setNull(i);
+        }
+      }
+      valueCount = Math.max(valueCount, v.size());
+    }
+    vector.setValueCount(valueCount);
+  }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
index 3ca8d0af6a6..3d93407ac64 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
@@ -90,7 +90,7 @@ public void testAll() throws IOException {
         field("g", new Utf8()),
         field("h", new Binary()),
         field("i", new Bool()),
-        field("j", new Decimal(5, 5)),
+        field("j", new Decimal(5, 5, 128)),
         field("k", new Date(DateUnit.DAY)),
         field("l", new Date(DateUnit.MILLISECOND)),
         field("m", new Time(TimeUnit.SECOND, 32)),
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/DecimalUtilityTest.java b/java/vector/src/test/java/org/apache/arrow/vector/util/DecimalUtilityTest.java
index 667e9624ed8..804092ed94a 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/util/DecimalUtilityTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/util/DecimalUtilityTest.java
@@ -27,76 +27,100 @@
 import org.junit.Test;
 
 public class DecimalUtilityTest {
-  private static final BigInteger MAX_BIG_INT = java.math.BigInteger.valueOf(10).pow(38)
-          .subtract(java.math.BigInteger.ONE);
-  private static final BigDecimal MAX_DECIMAL = new java.math.BigDecimal(MAX_BIG_INT, 0);
-  private static final BigInteger MIN_BIG_INT = MAX_BIG_INT.multiply(BigInteger.valueOf(-1));
-  private static final BigDecimal MIN_DECIMAL = new java.math.BigDecimal(MIN_BIG_INT, 0);
+  private static final BigInteger[] MAX_BIG_INT = new BigInteger[]{BigInteger.valueOf(10).pow(38)
+          .subtract(java.math.BigInteger.ONE), java.math.BigInteger.valueOf(10).pow(76)};
+  private static final BigInteger[] MIN_BIG_INT = new BigInteger[]{MAX_BIG_INT[0].multiply(BigInteger.valueOf(-1)),
+     MAX_BIG_INT[1].multiply(BigInteger.valueOf(-1))};
 
   @Test
-  public void testSetByteArrayInDecimalArrowBuf() {
-    try (BufferAllocator allocator = new RootAllocator(128);
-         ArrowBuf buf = allocator.buffer(16);
-    ) {
-      int [] intValues = new int [] {Integer.MAX_VALUE, Integer.MIN_VALUE, 0};
-      for (int val : intValues) {
-        buf.clear();
-        DecimalUtility.writeByteArrayToArrowBuf(BigInteger.valueOf(val).toByteArray(), buf, 0);
-        BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0);
-        BigDecimal expected = BigDecimal.valueOf(val);
-        Assert.assertEquals(expected, actual);
+  public void testSetLongInDecimalArrowBuf() {
+    int[] byteLengths = new int[]{16, 32};
+    for (int x = 0; x < 2; x++) {
+      try (BufferAllocator allocator = new RootAllocator(128);
+           ArrowBuf buf = allocator.buffer(byteLengths[x]);
+      ) {
+        int [] intValues = new int [] {Integer.MAX_VALUE, Integer.MIN_VALUE, 0};
+        for (int val : intValues) {
+          buf.clear();
+          DecimalUtility.writeLongToArrowBuf((long) val, buf, 0, byteLengths[x]);
+          BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0, byteLengths[x]);
+          BigDecimal expected = BigDecimal.valueOf(val);
+          Assert.assertEquals(expected, actual);
+        }
       }
+    }
+  }
 
-      long [] longValues = new long[] {Long.MIN_VALUE, 0 , Long.MAX_VALUE};
-      for (long val : longValues) {
-        buf.clear();
-        DecimalUtility.writeByteArrayToArrowBuf(BigInteger.valueOf(val).toByteArray(), buf, 0);
-        BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0);
-        BigDecimal expected = BigDecimal.valueOf(val);
-        Assert.assertEquals(expected, actual);
-      }
+  @Test
+  public void testSetByteArrayInDecimalArrowBuf() {
+    int[] byteLengths = new int[]{16, 32};
+    for (int x = 0; x < 2; x++) {
+      try (BufferAllocator allocator = new RootAllocator(128);
+           ArrowBuf buf = allocator.buffer(byteLengths[x]);
+      ) {
+        int [] intValues = new int [] {Integer.MAX_VALUE, Integer.MIN_VALUE, 0};
+        for (int val : intValues) {
+          buf.clear();
+          DecimalUtility.writeByteArrayToArrowBuf(BigInteger.valueOf(val).toByteArray(), buf, 0, byteLengths[x]);
+          BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0, byteLengths[x]);
+          BigDecimal expected = BigDecimal.valueOf(val);
+          Assert.assertEquals(expected, actual);
+        }
+
+        long [] longValues = new long[] {Long.MIN_VALUE, 0 , Long.MAX_VALUE};
+        for (long val : longValues) {
+          buf.clear();
+          DecimalUtility.writeByteArrayToArrowBuf(BigInteger.valueOf(val).toByteArray(), buf, 0, byteLengths[x]);
+          BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0, byteLengths[x]);
+          BigDecimal expected = BigDecimal.valueOf(val);
+          Assert.assertEquals(expected, actual);
+        }
 
-      BigInteger [] decimals = new BigInteger[] {MAX_BIG_INT, new BigInteger("0"), MIN_BIG_INT};
-      for (BigInteger val : decimals) {
-        buf.clear();
-        DecimalUtility.writeByteArrayToArrowBuf(val.toByteArray(), buf, 0);
-        BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0);
-        BigDecimal expected = new BigDecimal(val);
-        Assert.assertEquals(expected, actual);
+        BigInteger [] decimals = new BigInteger[] {MAX_BIG_INT[x], new BigInteger("0"), MIN_BIG_INT[x]};
+        for (BigInteger val : decimals) {
+          buf.clear();
+          DecimalUtility.writeByteArrayToArrowBuf(val.toByteArray(), buf, 0, byteLengths[x]);
+          BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0, byteLengths[x]);
+          BigDecimal expected = new BigDecimal(val);
+          Assert.assertEquals(expected, actual);
+        }
       }
     }
   }
 
   @Test
   public void testSetBigDecimalInDecimalArrowBuf() {
-    try (BufferAllocator allocator = new RootAllocator(128);
-         ArrowBuf buf = allocator.buffer(16);
-    ) {
-      int [] intValues = new int [] {Integer.MAX_VALUE, Integer.MIN_VALUE, 0};
-      for (int val : intValues) {
-        buf.clear();
-        DecimalUtility.writeBigDecimalToArrowBuf(BigDecimal.valueOf(val), buf, 0);
-        BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0);
-        BigDecimal expected = BigDecimal.valueOf(val);
-        Assert.assertEquals(expected, actual);
-      }
+    int[] byteLengths = new int[]{16, 32};
+    for (int x = 0; x < 2; x++) {
+      try (BufferAllocator allocator = new RootAllocator(128);
+           ArrowBuf buf = allocator.buffer(byteLengths[x]);
+      ) {
+        int [] intValues = new int [] {Integer.MAX_VALUE, Integer.MIN_VALUE, 0};
+        for (int val : intValues) {
+          buf.clear();
+          DecimalUtility.writeBigDecimalToArrowBuf(BigDecimal.valueOf(val), buf, 0, byteLengths[x]);
+          BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0, byteLengths[x]);
+          BigDecimal expected = BigDecimal.valueOf(val);
+          Assert.assertEquals(expected, actual);
+        }
 
-      long [] longValues = new long[] {Long.MIN_VALUE, 0 , Long.MAX_VALUE};
-      for (long val : longValues) {
-        buf.clear();
-        DecimalUtility.writeBigDecimalToArrowBuf(BigDecimal.valueOf(val), buf, 0);
-        BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0);
-        BigDecimal expected = BigDecimal.valueOf(val);
-        Assert.assertEquals(expected, actual);
-      }
+        long [] longValues = new long[] {Long.MIN_VALUE, 0 , Long.MAX_VALUE};
+        for (long val : longValues) {
+          buf.clear();
+          DecimalUtility.writeBigDecimalToArrowBuf(BigDecimal.valueOf(val), buf, 0, byteLengths[x]);
+          BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0, byteLengths[x]);
+          BigDecimal expected = BigDecimal.valueOf(val);
+          Assert.assertEquals(expected, actual);
+        }
 
-      BigInteger [] decimals = new BigInteger[] {MAX_BIG_INT, new BigInteger("0"), MIN_BIG_INT};
-      for (BigInteger val : decimals) {
-        buf.clear();
-        DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(val), buf, 0);
-        BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0);
-        BigDecimal expected = new BigDecimal(val);
-        Assert.assertEquals(expected, actual);
+        BigInteger [] decimals = new BigInteger[] {MAX_BIG_INT[x], new BigInteger("0"), MIN_BIG_INT[x]};
+        for (BigInteger val : decimals) {
+          buf.clear();
+          DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(val), buf, 0, byteLengths[x]);
+          BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0, byteLengths[x]);
+          BigDecimal expected = new BigDecimal(val);
+          Assert.assertEquals(expected, actual);
+        }
       }
     }
   }
diff --git a/java/vector/src/test/java/org/apache/arrow/util/TestDataSizeRoundingUtil.java b/java/vector/src/test/java/org/apache/arrow/vector/util/TestDataSizeRoundingUtil.java
similarity index 98%
rename from java/vector/src/test/java/org/apache/arrow/util/TestDataSizeRoundingUtil.java
rename to java/vector/src/test/java/org/apache/arrow/vector/util/TestDataSizeRoundingUtil.java
index a04a9578440..4138ea9d7a1 100644
--- a/java/vector/src/test/java/org/apache/arrow/util/TestDataSizeRoundingUtil.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/util/TestDataSizeRoundingUtil.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.util;
+package org.apache.arrow.vector.util;
 
 import static org.junit.Assert.assertEquals;
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorVisitor.java b/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVector.java
similarity index 71%
rename from java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorVisitor.java
rename to java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVector.java
index b997c3141d8..2354b281ed4 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorVisitor.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVector.java
@@ -19,7 +19,6 @@
 
 import static org.apache.arrow.vector.testing.ValueVectorDataPopulator.setVector;
 import static org.apache.arrow.vector.util.ValueVectorUtility.validate;
-import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
@@ -30,9 +29,11 @@
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.BigIntVector;
 import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.LargeVarCharVector;
 import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.complex.DenseUnionVector;
 import org.apache.arrow.vector.complex.FixedSizeListVector;
+import org.apache.arrow.vector.complex.LargeListVector;
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.StructVector;
 import org.apache.arrow.vector.complex.UnionVector;
@@ -47,7 +48,7 @@
 import org.junit.Before;
 import org.junit.Test;
 
-public class TestValidateVectorVisitor {
+public class TestValidateVector {
 
   private BufferAllocator allocator;
 
@@ -74,9 +75,9 @@ public void testBaseFixedWidthVector() {
       validate(vector);
 
       vector.getDataBuffer().capacity(0);
-      IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+      ValidateUtil.ValidateException e = assertThrows(ValidateUtil.ValidateException.class,
           () -> validate(vector));
-      assertTrue(e.getMessage().contains("Buffer #1 too small in vector of type"));
+      assertTrue(e.getMessage().contains("Not enough capacity for fixed width data buffer"));
     }
   }
 
@@ -88,9 +89,23 @@ public void testBaseVariableWidthVector() {
       validate(vector);
 
       vector.getDataBuffer().capacity(0);
-      IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+      ValidateUtil.ValidateException e = assertThrows(ValidateUtil.ValidateException.class,
           () -> validate(vector));
-      assertTrue(e.getMessage().contains("Buffer #2 too small in vector of type"));
+      assertTrue(e.getMessage().contains("Not enough capacity for data buffer"));
+    }
+  }
+
+  @Test
+  public void testBaseLargeVariableWidthVector() {
+    try (final LargeVarCharVector vector = new LargeVarCharVector("v", allocator)) {
+      validate(vector);
+      setVector(vector, STR1, STR2, null, STR3);
+      validate(vector);
+
+      vector.getDataBuffer().capacity(0);
+      ValidateUtil.ValidateException e = assertThrows(ValidateUtil.ValidateException.class,
+          () -> validate(vector));
+      assertTrue(e.getMessage().contains("Not enough capacity for data buffer"));
     }
   }
 
@@ -102,10 +117,23 @@ public void testListVector() {
       validate(vector);
 
       vector.getDataVector().setValueCount(3);
-      IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+      ValidateUtil.ValidateException e = assertThrows(ValidateUtil.ValidateException.class,
           () -> validate(vector));
-      assertEquals("Length spanned by list offsets (5) larger than data vector valueCount (length 3)",
-          e.getMessage());
+      assertTrue(e.getMessage().contains("Inner vector does not contain enough elements."));
+    }
+  }
+
+  @Test
+  public void testLargeListVector() {
+    try (final LargeListVector vector = LargeListVector.empty("v", allocator)) {
+      validate(vector);
+      setVector(vector, Arrays.asList(1, 2, 3, 4), Arrays.asList(5, 6));
+      validate(vector);
+
+      vector.getDataVector().setValueCount(4);
+      ValidateUtil.ValidateException e = assertThrows(ValidateUtil.ValidateException.class,
+          () -> validate(vector));
+      assertTrue(e.getMessage().contains("Inner vector does not contain enough elements."));
     }
   }
 
@@ -117,10 +145,9 @@ public void testFixedSizeListVector() {
       validate(vector);
 
       vector.getDataVector().setValueCount(3);
-      IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+      ValidateUtil.ValidateException e = assertThrows(ValidateUtil.ValidateException.class,
           () -> validate(vector));
-      assertEquals("data vector valueCount invalid, expect 6, actual is: 3",
-          e.getMessage());
+      assertTrue(e.getMessage().contains("Inner vector does not contain enough elements."));
     }
   }
 
@@ -143,17 +170,17 @@ public void testStructVectorRangeEquals() {
       writer.setValueCount(5);
 
       vector.getChild("f0").setValueCount(2);
-      IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+      ValidateUtil.ValidateException e = assertThrows(ValidateUtil.ValidateException.class,
           () -> validate(vector));
-      assertTrue(e.getMessage().contains("valueCount is not equals with struct vector"));
+      assertTrue(e.getMessage().contains("Struct vector length not equal to child vector length"));
 
       vector.getChild("f0").setValueCount(5);
       validate(vector);
 
       vector.getChild("f0").getDataBuffer().capacity(0);
-      IllegalArgumentException e2 = assertThrows(IllegalArgumentException.class,
+      ValidateUtil.ValidateException e2 = assertThrows(ValidateUtil.ValidateException.class,
           () -> validate(vector));
-      assertTrue(e2.getMessage().contains("valueBuffer is null or capacity is 0"));
+      assertTrue(e2.getMessage().contains("Not enough capacity for fixed width data buffer"));
     }
   }
 
@@ -179,17 +206,17 @@ public void testUnionVector() {
       validate(vector);
 
       vector.getChildrenFromFields().get(0).setValueCount(1);
-      IllegalArgumentException e1 = assertThrows(IllegalArgumentException.class,
+      ValidateUtil.ValidateException e1 = assertThrows(ValidateUtil.ValidateException.class,
           () -> validate(vector));
-      assertTrue(e1.getMessage().contains("valueCount is not equals with union vector"));
+      assertTrue(e1.getMessage().contains("Union vector length not equal to child vector length"));
 
       vector.getChildrenFromFields().get(0).setValueCount(2);
       validate(vector);
 
       vector.getChildrenFromFields().get(0).getDataBuffer().capacity(0);
-      IllegalArgumentException e2 = assertThrows(IllegalArgumentException.class,
+      ValidateUtil.ValidateException e2 = assertThrows(ValidateUtil.ValidateException.class,
           () -> validate(vector));
-      assertTrue(e2.getMessage().contains("valueBuffer is null or capacity is 0"));
+      assertTrue(e2.getMessage().contains("Not enough capacity for fixed width data buffer"));
     }
   }
 
@@ -218,9 +245,9 @@ public void testDenseUnionVector() {
       validate(vector);
 
       vector.getChildrenFromFields().get(0).getDataBuffer().capacity(0);
-      IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+      ValidateUtil.ValidateException e = assertThrows(ValidateUtil.ValidateException.class,
           () -> validate(vector));
-      assertTrue(e.getMessage().contains("valueBuffer is null or capacity is 0"));
+      assertTrue(e.getMessage().contains("Not enough capacity for fixed width data buffer"));
     }
   }
 
@@ -230,6 +257,4 @@ private void writeStructVector(NullableStructWriter writer, int value1, long val
     writer.bigInt("f1").writeBigInt(value2);
     writer.end();
   }
-
-
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorFull.java b/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorFull.java
new file mode 100644
index 00000000000..4241a0d9cff
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorFull.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.validate;
+
+import static org.apache.arrow.vector.testing.ValueVectorDataPopulator.setVector;
+import static org.apache.arrow.vector.util.ValueVectorUtility.validateFull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.util.Arrays;
+
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.Float4Vector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.LargeVarCharVector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.complex.DenseUnionVector;
+import org.apache.arrow.vector.complex.LargeListVector;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.StructVector;
+import org.apache.arrow.vector.complex.UnionVector;
+import org.apache.arrow.vector.holders.NullableFloat4Holder;
+import org.apache.arrow.vector.holders.NullableFloat8Holder;
+import org.apache.arrow.vector.testing.ValueVectorDataPopulator;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestValidateVectorFull {
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new RootAllocator(Long.MAX_VALUE);
+  }
+
+  @After
+  public void terminate() throws Exception {
+    allocator.close();
+  }
+
+  @Test
+  public void testBaseVariableWidthVector() {
+    try (final VarCharVector vector = new VarCharVector("v", allocator)) {
+      validateFull(vector);
+      setVector(vector, "aaa", "bbb", "ccc");
+      validateFull(vector);
+
+      ArrowBuf offsetBuf = vector.getOffsetBuffer();
+      offsetBuf.setInt(0, 100);
+      offsetBuf.setInt(4, 50);
+
+      ValidateUtil.ValidateException e = assertThrows(ValidateUtil.ValidateException.class,
+          () -> validateFull(vector));
+      assertTrue(e.getMessage().contains("The values in positions 0 and 1 of the offset buffer are decreasing"));
+    }
+  }
+
+  @Test
+  public void testBaseLargeVariableWidthVector() {
+    try (final LargeVarCharVector vector = new LargeVarCharVector("v", allocator)) {
+      validateFull(vector);
+      setVector(vector, "aaa", "bbb", null, "ccc");
+      validateFull(vector);
+
+      ArrowBuf offsetBuf = vector.getOffsetBuffer();
+      offsetBuf.setLong(0, 100);
+      offsetBuf.setLong(8, 50);
+
+      ValidateUtil.ValidateException e = assertThrows(ValidateUtil.ValidateException.class,
+          () -> validateFull(vector));
+      assertTrue(e.getMessage().contains("The values in positions 0 and 1 of the large offset buffer are decreasing"));
+    }
+  }
+
+  @Test
+  public void testListVector() {
+    try (final ListVector vector = ListVector.empty("v", allocator)) {
+      validateFull(vector);
+      setVector(vector, Arrays.asList(1, 2, 3), Arrays.asList(4, 5), Arrays.asList(6, 7, 8, 9));
+      validateFull(vector);
+
+      ArrowBuf offsetBuf = vector.getOffsetBuffer();
+      offsetBuf.setInt(0, 100);
+      offsetBuf.setInt(8, 50);
+
+      ValidateUtil.ValidateException e = assertThrows(ValidateUtil.ValidateException.class,
+          () -> validateFull(vector));
+      assertTrue(e.getMessage().contains("The values in positions 0 and 1 of the offset buffer are decreasing"));
+    }
+  }
+
+  @Test
+  public void testLargeListVector() {
+    try (final LargeListVector vector = LargeListVector.empty("v", allocator)) {
+      validateFull(vector);
+      setVector(vector, Arrays.asList(1, 2, 3), Arrays.asList(4, 5), Arrays.asList(6, 7, 8, 9));
+      validateFull(vector);
+
+      ArrowBuf offsetBuf = vector.getOffsetBuffer();
+      offsetBuf.setLong(0, 100);
+      offsetBuf.setLong(16, 50);
+
+      ValidateUtil.ValidateException e = assertThrows(ValidateUtil.ValidateException.class,
+          () -> validateFull(vector));
+      assertTrue(e.getMessage().contains("The values in positions 0 and 1 of the large offset buffer are decreasing"));
+    }
+  }
+
+  @Test
+  public void testStructVectorRangeEquals() {
+    try (final StructVector vector = StructVector.empty("struct", allocator)) {
+      IntVector intVector =
+          vector.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class);
+      VarCharVector strVector =
+          vector.addOrGet("f1", FieldType.nullable(new ArrowType.Utf8()), VarCharVector.class);
+
+      validateFull(vector);
+      validateFull(intVector);
+      validateFull(strVector);
+
+      ValueVectorDataPopulator.setVector(intVector, 1, 2, 3, 4, 5);
+      ValueVectorDataPopulator.setVector(strVector, "a", "b", "c", "d", "e");
+      vector.setValueCount(5);
+
+      validateFull(vector);
+      validateFull(intVector);
+      validateFull(strVector);
+
+      ArrowBuf offsetBuf = strVector.getOffsetBuffer();
+      offsetBuf.setInt(0, 100);
+      offsetBuf.setInt(8, 50);
+
+      ValidateUtil.ValidateException e = assertThrows(ValidateUtil.ValidateException.class,
+          () -> validateFull(strVector));
+      assertTrue(e.getMessage().contains("The values in positions 0 and 1 of the offset buffer are decreasing"));
+
+      e = assertThrows(ValidateUtil.ValidateException.class,
+          () -> validateFull(vector));
+      assertTrue(e.getMessage().contains("The values in positions 0 and 1 of the offset buffer are decreasing"));
+    }
+  }
+
+  @Test
+  public void testUnionVector() {
+    try (final UnionVector vector = UnionVector.empty("union", allocator)) {
+      validateFull(vector);
+
+      final NullableFloat4Holder float4Holder = new NullableFloat4Holder();
+      float4Holder.value = 1.01f;
+      float4Holder.isSet = 1;
+
+      final NullableFloat8Holder float8Holder = new NullableFloat8Holder();
+      float8Holder.value = 2.02f;
+      float8Holder.isSet = 1;
+
+      vector.setType(0, Types.MinorType.FLOAT4);
+      vector.setSafe(0, float4Holder);
+      vector.setType(1, Types.MinorType.FLOAT8);
+      vector.setSafe(1, float8Holder);
+      vector.setValueCount(2);
+
+      validateFull(vector);
+
+      // negative type id
+      vector.getTypeBuffer().setByte(0, -1);
+
+      ValidateUtil.ValidateException e = assertThrows(ValidateUtil.ValidateException.class,
+          () -> validateFull(vector));
+      assertTrue(e.getMessage().contains("The type id at position 0 is negative"));
+    }
+  }
+
+  @Test
+  public void testDenseUnionVector() {
+    try (final DenseUnionVector vector = DenseUnionVector.empty("union", allocator)) {
+      validateFull(vector);
+
+      final NullableFloat4Holder float4Holder = new NullableFloat4Holder();
+      float4Holder.value = 1.01f;
+      float4Holder.isSet = 1;
+
+      final NullableFloat8Holder float8Holder = new NullableFloat8Holder();
+      float8Holder.value = 2.02f;
+      float8Holder.isSet = 1;
+
+      byte float4TypeId = vector.registerNewTypeId(Field.nullable("", Types.MinorType.FLOAT4.getType()));
+      byte float8TypeId = vector.registerNewTypeId(Field.nullable("", Types.MinorType.FLOAT8.getType()));
+
+      vector.setTypeId(0, float4TypeId);
+      vector.setSafe(0, float4Holder);
+      vector.setTypeId(1, float8TypeId);
+      vector.setSafe(1, float8Holder);
+      vector.setValueCount(2);
+
+      validateFull(vector);
+
+      ValueVector subVector = vector.getVectorByType(float4TypeId);
+      assertTrue(subVector instanceof Float4Vector);
+      assertEquals(1, subVector.getValueCount());
+
+      // shrink sub-vector
+      subVector.setValueCount(0);
+
+      ValidateUtil.ValidateException e = assertThrows(ValidateUtil.ValidateException.class,
+          () -> validateFull(vector));
+      assertTrue(e.getMessage().contains("Dense union vector offset exceeds sub-vector boundary"));
+    }
+  }
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorSchemaRoot.java b/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorSchemaRoot.java
new file mode 100644
index 00000000000..1885fb21f17
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorSchemaRoot.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.validate;
+
+import static org.apache.arrow.vector.util.ValueVectorUtility.validate;
+import static org.apache.arrow.vector.util.ValueVectorUtility.validateFull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.testing.ValueVectorDataPopulator;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestValidateVectorSchemaRoot {
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new RootAllocator(Long.MAX_VALUE);
+  }
+
+  @After
+  public void terminate() throws Exception {
+    allocator.close();
+  }
+
+  @Test
+  public void testValidatePositive() {
+    try (IntVector intVector = new IntVector("int vector", allocator);
+         VarCharVector strVector = new VarCharVector("var char vector", allocator)) {
+
+      VectorSchemaRoot root = VectorSchemaRoot.of(intVector, strVector);
+
+      validate(root);
+      validateFull(root);
+
+      ValueVectorDataPopulator.setVector(intVector, 1, 2, 3, 4, 5);
+      ValueVectorDataPopulator.setVector(strVector, "a", "b", "c", "d", "e");
+      root.setRowCount(5);
+
+      validate(root);
+      validateFull(root);
+    }
+  }
+
+  @Test
+  public void testValidateNegative() {
+    try (IntVector intVector = new IntVector("int vector", allocator);
+         VarCharVector strVector = new VarCharVector("var char vector", allocator)) {
+
+      VectorSchemaRoot root = VectorSchemaRoot.of(intVector, strVector);
+
+      ValueVectorDataPopulator.setVector(intVector, 1, 2, 3, 4, 5);
+      ValueVectorDataPopulator.setVector(strVector, "a", "b", "c", "d", "e");
+
+      // validate mismatching value counts
+      root.setRowCount(4);
+      intVector.setValueCount(5);
+      strVector.setValueCount(5);
+      ValidateUtil.ValidateException e = assertThrows(ValidateUtil.ValidateException.class,
+          () -> validate(root));
+      assertTrue(e.getMessage().contains("Child vector and vector schema root have different value counts"));
+      e = assertThrows(ValidateUtil.ValidateException.class,
+          () -> validateFull(root));
+      assertTrue(e.getMessage().contains("Child vector and vector schema root have different value counts"));
+
+      // valid problems with the child vector
+      root.setRowCount(5);
+      ArrowBuf offsetBuf = strVector.getOffsetBuffer();
+      offsetBuf.setInt(0, 100);
+      offsetBuf.setInt(8, 50);
+      validate(root);
+      e = assertThrows(ValidateUtil.ValidateException.class,
+          () -> validateFull(root));
+      assertTrue(e.getMessage().contains("The values in positions 0 and 1 of the offset buffer are decreasing"));
+    }
+  }
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorTypeVisitor.java b/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorTypeVisitor.java
new file mode 100644
index 00000000000..dcda6bfae3e
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorTypeVisitor.java
@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.validate;
+
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.util.function.Supplier;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.BitVector;
+import org.apache.arrow.vector.DateDayVector;
+import org.apache.arrow.vector.DateMilliVector;
+import org.apache.arrow.vector.DecimalVector;
+import org.apache.arrow.vector.DurationVector;
+import org.apache.arrow.vector.FixedSizeBinaryVector;
+import org.apache.arrow.vector.Float4Vector;
+import org.apache.arrow.vector.Float8Vector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.IntervalDayVector;
+import org.apache.arrow.vector.IntervalYearVector;
+import org.apache.arrow.vector.LargeVarBinaryVector;
+import org.apache.arrow.vector.LargeVarCharVector;
+import org.apache.arrow.vector.NullVector;
+import org.apache.arrow.vector.SmallIntVector;
+import org.apache.arrow.vector.TimeMicroVector;
+import org.apache.arrow.vector.TimeMilliVector;
+import org.apache.arrow.vector.TimeNanoVector;
+import org.apache.arrow.vector.TimeSecVector;
+import org.apache.arrow.vector.TimeStampMicroTZVector;
+import org.apache.arrow.vector.TimeStampMicroVector;
+import org.apache.arrow.vector.TimeStampMilliTZVector;
+import org.apache.arrow.vector.TimeStampMilliVector;
+import org.apache.arrow.vector.TimeStampNanoTZVector;
+import org.apache.arrow.vector.TimeStampNanoVector;
+import org.apache.arrow.vector.TimeStampSecTZVector;
+import org.apache.arrow.vector.TimeStampSecVector;
+import org.apache.arrow.vector.TinyIntVector;
+import org.apache.arrow.vector.UInt1Vector;
+import org.apache.arrow.vector.UInt2Vector;
+import org.apache.arrow.vector.UInt4Vector;
+import org.apache.arrow.vector.UInt8Vector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VarBinaryVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.complex.DenseUnionVector;
+import org.apache.arrow.vector.complex.FixedSizeListVector;
+import org.apache.arrow.vector.complex.LargeListVector;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.StructVector;
+import org.apache.arrow.vector.complex.UnionVector;
+import org.apache.arrow.vector.types.TimeUnit;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test cases for {@link ValidateVectorTypeVisitor}.
+ */
+public class TestValidateVectorTypeVisitor {
+
+  private BufferAllocator allocator;
+
+  private ValidateVectorTypeVisitor visitor = new ValidateVectorTypeVisitor();
+
+  @Before
+  public void init() {
+    allocator = new RootAllocator(Long.MAX_VALUE);
+  }
+
+  @After
+  public void terminate() throws Exception {
+    allocator.close();
+  }
+
+  private void testPositiveCase(Supplier<ValueVector> vectorGenerator) {
+    try (ValueVector vector = vectorGenerator.get();) {
+      vector.accept(visitor, null);
+    }
+  }
+
+  private void testNegativeCase(Supplier<ValueVector> vectorGenerator) {
+    try (ValueVector vector = vectorGenerator.get()) {
+      assertThrows(ValidateUtil.ValidateException.class, () -> {
+        vector.accept(visitor, null);
+      });
+    }
+  }
+
+  @Test
+  public void testFixedWidthVectorsPositive() {
+    // integer vectors
+    testPositiveCase(() -> new TinyIntVector("vector", allocator));
+    testPositiveCase(() -> new SmallIntVector("vector", allocator));
+    testPositiveCase(() -> new IntVector("vector", allocator));
+    testPositiveCase(() -> new BigIntVector("vector", allocator));
+    testPositiveCase(() -> new UInt1Vector("vector", allocator));
+    testPositiveCase(() -> new UInt2Vector("vector", allocator));
+    testPositiveCase(() -> new UInt4Vector("vector", allocator));
+    testPositiveCase(() -> new UInt8Vector("vector", allocator));
+
+    testPositiveCase(() -> new BitVector("vector", allocator));
+    testPositiveCase(() -> new DecimalVector("vector", allocator, 30, 16));
+
+    // date vectors
+    testPositiveCase(() -> new DateDayVector("vector", allocator));
+    testPositiveCase(() -> new DateMilliVector("vector", allocator));
+
+    testPositiveCase(() -> new DurationVector(
+        "vector", FieldType.nullable(new ArrowType.Duration(TimeUnit.SECOND)), allocator));
+
+    // float vectors
+    testPositiveCase(() -> new Float4Vector("vector", allocator));
+    testPositiveCase(() -> new Float8Vector("vector", allocator));
+
+    // interval vectors
+    testPositiveCase(() -> new IntervalDayVector("vector", allocator));
+    testPositiveCase(() -> new IntervalYearVector("vector", allocator));
+
+    // time vectors
+    testPositiveCase(() -> new TimeMicroVector("vector", allocator));
+    testPositiveCase(() -> new TimeMilliVector("vector", allocator));
+    testPositiveCase(() -> new TimeMicroVector("vector", allocator));
+    testPositiveCase(() -> new TimeSecVector("vector", allocator));
+
+    // time stamp vectors
+    testPositiveCase(() -> new TimeStampMicroTZVector("vector", allocator, "cn"));
+    testPositiveCase(() -> new TimeStampMicroVector("vector", allocator));
+    testPositiveCase(() -> new TimeStampMilliTZVector("vector", allocator, "cn"));
+    testPositiveCase(() -> new TimeStampMilliVector("vector", allocator));
+    testPositiveCase(() -> new TimeStampNanoTZVector("vector", allocator, "cn"));
+    testPositiveCase(() -> new TimeStampNanoVector("vector", allocator));
+    testPositiveCase(() -> new TimeStampSecTZVector("vector", allocator, "cn"));
+    testPositiveCase(() -> new TimeStampSecVector("vector", allocator));
+
+    testPositiveCase(() -> new FixedSizeBinaryVector("vector", allocator, 5));
+  }
+
+  @Test
+  public void testFixedWidthVectorsNegative() {
+    // integer vectors
+    testNegativeCase(
+        () -> new TinyIntVector("vector", FieldType.nullable(Types.MinorType.INT.getType()), allocator));
+    testNegativeCase(
+        () -> new SmallIntVector("vector", FieldType.nullable(Types.MinorType.INT.getType()), allocator));
+    testNegativeCase(
+        () -> new BigIntVector("vector", FieldType.nullable(Types.MinorType.SMALLINT.getType()), allocator));
+    testNegativeCase(
+        () -> new BigIntVector("vector", FieldType.nullable(Types.MinorType.SMALLINT.getType()), allocator));
+    testNegativeCase(
+        () -> new UInt1Vector("vector", FieldType.nullable(Types.MinorType.SMALLINT.getType()), allocator));
+    testNegativeCase(
+        () -> new UInt2Vector("vector", FieldType.nullable(Types.MinorType.SMALLINT.getType()), allocator));
+    testNegativeCase(
+        () -> new UInt4Vector("vector", FieldType.nullable(Types.MinorType.SMALLINT.getType()), allocator));
+    testNegativeCase(
+        () -> new UInt8Vector("vector", FieldType.nullable(Types.MinorType.SMALLINT.getType()), allocator));
+
+    testNegativeCase(
+        () -> new BitVector("vector", FieldType.nullable(Types.MinorType.BIGINT.getType()), allocator));
+    testNegativeCase(
+        () -> new DecimalVector("vector", allocator, 30, -16));
+
+    // date vectors
+    testNegativeCase(
+        () -> new DateDayVector("vector", FieldType.nullable(Types.MinorType.FLOAT4.getType()), allocator));
+    testNegativeCase(
+        () -> new DateMilliVector("vector", FieldType.nullable(Types.MinorType.BIGINT.getType()), allocator));
+
+    // float pont vectors
+    testNegativeCase(
+        () -> new Float4Vector("vector", FieldType.nullable(Types.MinorType.BIGINT.getType()), allocator));
+    testNegativeCase(
+        () -> new Float8Vector("vector", FieldType.nullable(Types.MinorType.BIGINT.getType()), allocator));
+
+    // interval vectors
+    testNegativeCase(
+        () -> new IntervalDayVector("vector", FieldType.nullable(Types.MinorType.INT.getType()), allocator));
+    testNegativeCase(
+        () -> new IntervalYearVector("vector", FieldType.nullable(Types.MinorType.BIGINT.getType()), allocator));
+
+    // time vectors
+    testNegativeCase(
+        () -> new TimeMilliVector("vector", FieldType.nullable(Types.MinorType.BIGINT.getType()), allocator));
+    testNegativeCase(
+        () -> new TimeMicroVector("vector", FieldType.nullable(Types.MinorType.BIGINT.getType()), allocator));
+    testNegativeCase(
+        () -> new TimeNanoVector("vector", FieldType.nullable(Types.MinorType.BIGINT.getType()), allocator));
+    testNegativeCase(
+        () -> new TimeSecVector("vector", FieldType.nullable(Types.MinorType.BIGINT.getType()), allocator));
+
+    // time stamp vectors
+    testNegativeCase(
+        () -> new TimeStampMicroTZVector("vector", allocator, null));
+    testNegativeCase(
+        () -> new TimeStampMicroVector("vector", FieldType.nullable(Types.MinorType.BIGINT.getType()), allocator));
+    testNegativeCase(
+        () -> new TimeStampMilliTZVector("vector", allocator, null));
+    testNegativeCase(
+        () -> new TimeStampMilliVector("vector", FieldType.nullable(Types.MinorType.BIGINT.getType()), allocator));
+    testNegativeCase(
+        () -> new TimeStampNanoTZVector("vector", allocator, null));
+    testNegativeCase(
+        () -> new TimeStampNanoVector("vector", FieldType.nullable(Types.MinorType.BIGINT.getType()), allocator));
+    testNegativeCase(
+        () -> new TimeStampSecTZVector("vector", allocator, null));
+    testNegativeCase(
+        () -> new TimeStampSecVector("vector", FieldType.nullable(Types.MinorType.BIGINT.getType()), allocator));
+  }
+
+  @Test
+  public void testVariableWidthVectorsPositive() {
+    testPositiveCase(() -> new VarCharVector("vector", allocator));
+    testPositiveCase(() -> new VarBinaryVector("vector", allocator));
+  }
+
+  @Test
+  public void testVariableWidthVectorsNegative() {
+    testNegativeCase(
+        () -> new VarCharVector("vector", FieldType.nullable(Types.MinorType.INT.getType()), allocator));
+    testNegativeCase(
+        () -> new VarBinaryVector("vector", FieldType.nullable(Types.MinorType.INT.getType()), allocator));
+  }
+
+  @Test
+  public void testLargeVariableWidthVectorsPositive() {
+    testPositiveCase(() -> new LargeVarCharVector("vector", allocator));
+    testPositiveCase(() -> new LargeVarBinaryVector("vector", allocator));
+  }
+
+  @Test
+  public void testLargeVariableWidthVectorsNegative() {
+    testNegativeCase(
+        () -> new LargeVarCharVector("vector", FieldType.nullable(Types.MinorType.INT.getType()), allocator));
+    testNegativeCase(
+        () -> new LargeVarBinaryVector("vector", FieldType.nullable(Types.MinorType.INT.getType()), allocator));
+  }
+
+  @Test
+  public void testListVector() {
+    testPositiveCase(() -> ListVector.empty("vector", allocator));
+
+    testNegativeCase(
+        () -> new ListVector("vector", allocator, FieldType.nullable(Types.MinorType.INT.getType()), null));
+  }
+
+  @Test
+  public void testLargeListVector() {
+    testPositiveCase(() -> LargeListVector.empty("vector", allocator));
+
+    testNegativeCase(
+        () -> new LargeListVector("vector", allocator, FieldType.nullable(Types.MinorType.INT.getType()), null));
+  }
+
+  @Test
+  public void testFixedSizeListVector() {
+    testPositiveCase(() -> FixedSizeListVector.empty("vector", 10, allocator));
+  }
+
+  @Test
+  public void testStructVector() {
+    testPositiveCase(() -> StructVector.empty("vector", allocator));
+
+    testNegativeCase(
+        () -> new StructVector("vector", allocator, FieldType.nullable(Types.MinorType.INT.getType()), null));
+  }
+
+  @Test
+  public void testUnionVector() {
+    testPositiveCase(() -> UnionVector.empty("vector", allocator));
+  }
+
+  @Test
+  public void testDenseUnionVector() {
+    testPositiveCase(() -> DenseUnionVector.empty("vector", allocator));
+  }
+
+  @Test
+  public void testNullVector() {
+    testPositiveCase(() -> new NullVector());
+  }
+}
diff --git a/js/README.md b/js/README.md
index 1ab8bde41b0..b592a1afa29 100644
--- a/js/README.md
+++ b/js/README.md
@@ -122,7 +122,7 @@ const rainfall = Table.new(
 ```js
 import { Table } from "apache-arrow";
 
-const table = await Table.from(fetch(("/simple.arrow")));
+const table = await Table.from(fetch("/simple.arrow"));
 console.log(table.toString());
 
 ```
diff --git a/js/gulp/arrow-task.js b/js/gulp/arrow-task.js
index 0b95440478d..e916cfc7f96 100644
--- a/js/gulp/arrow-task.js
+++ b/js/gulp/arrow-task.js
@@ -21,11 +21,11 @@ const {
 
 const del = require('del');
 const gulp = require('gulp');
-const { promisify } = require('util');
+const mkdirp = require('mkdirp');
 const gulpRename = require(`gulp-rename`);
 const { memoizeTask } = require('./memoize-task');
-const exec = promisify(require('child_process').exec);
 const { Observable, ReplaySubject } = require('rxjs');
+const pipeline = require('util').promisify(require('stream').pipeline);
 
 const arrowTask = ((cache) => memoizeTask(cache, function copyMain(target) {
     const out = targetDir(target);
@@ -53,8 +53,8 @@ const arrowTask = ((cache) => memoizeTask(cache, function copyMain(target) {
 
 const arrowTSTask = ((cache) => memoizeTask(cache, async function copyTS(target, format) {
     const out = targetDir(target, format);
-    await exec(`mkdirp ${out}`);
-    await exec(`shx cp -r src/* ${out}`);
+    await mkdirp(out);
+    await pipeline(gulp.src(`src/*`), gulp.dest(out));
     await del(`${out}/**/*.js`);
 }))({});
   
diff --git a/js/gulp/closure-task.js b/js/gulp/closure-task.js
index fff7f9c2e2d..bbb48cebcd3 100644
--- a/js/gulp/closure-task.js
+++ b/js/gulp/closure-task.js
@@ -29,10 +29,10 @@ const {
 const fs = require('fs');
 const gulp = require('gulp');
 const path = require('path');
+const mkdirp = require('mkdirp');
 const sourcemaps = require('gulp-sourcemaps');
 const { memoizeTask } = require('./memoize-task');
 const { compileBinFiles } = require('./typescript-task');
-const mkdirp = require('util').promisify(require('mkdirp'));
 const closureCompiler = require('google-closure-compiler').gulp();
 
 const closureTask = ((cache) => memoizeTask(cache, async function closure(target, format) {
@@ -93,7 +93,7 @@ const createClosureArgs = (entry_point, externs) => ({
     entry_point,
     third_party: true,
     warning_level: `QUIET`,
-    dependency_mode: `STRICT`,
+    dependency_mode: `PRUNE`,
     rewrite_polyfills: false,
     module_resolution: `NODE`,
     // formatting: `PRETTY_PRINT`,
diff --git a/js/gulp/test-task.js b/js/gulp/test-task.js
index 4993822e1fc..5680f91e4ce 100644
--- a/js/gulp/test-task.js
+++ b/js/gulp/test-task.js
@@ -17,10 +17,10 @@
 
 const del = require('del');
 const path = require('path');
+const mkdirp = require('mkdirp');
 const { argv } = require('./argv');
 const { promisify } = require('util');
 const glob = promisify(require('glob'));
-const mkdirp = promisify(require('mkdirp'));
 const rimraf = promisify(require('rimraf'));
 const child_process = require(`child_process`);
 const { memoizeTask } = require('./memoize-task');
diff --git a/js/package-lock.json b/js/package-lock.json
index 9472895f3d2..cee8185b19e 100644
--- a/js/package-lock.json
+++ b/js/package-lock.json
@@ -1,216 +1,483 @@
 {
   "name": "apache-arrow",
-  "version": "1.0.0-SNAPSHOT",
+  "version": "2.0.0-SNAPSHOT",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
     "@babel/code-frame": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0.tgz",
-      "integrity": "sha512-OfC2uemaknXr87bdLUkWog7nYuliM9Ij5HUcajsVcMCpQrcLmtxRbVFTIqmcSkSeYRBFBRxs2FiUqFJDLdiebA==",
+      "version": "7.10.4",
+      "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.10.4.tgz",
+      "integrity": "sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg==",
       "dev": true,
       "requires": {
-        "@babel/highlight": "^7.0.0"
+        "@babel/highlight": "^7.10.4"
       }
     },
     "@babel/core": {
-      "version": "7.4.5",
-      "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.4.5.tgz",
-      "integrity": "sha512-OvjIh6aqXtlsA8ujtGKfC7LYWksYSX8yQcM8Ay3LuvVeQ63lcOKgoZWVqcpFwkd29aYU9rVx7jxhfhiEDV9MZA==",
-      "dev": true,
-      "requires": {
-        "@babel/code-frame": "^7.0.0",
-        "@babel/generator": "^7.4.4",
-        "@babel/helpers": "^7.4.4",
-        "@babel/parser": "^7.4.5",
-        "@babel/template": "^7.4.4",
-        "@babel/traverse": "^7.4.5",
-        "@babel/types": "^7.4.4",
-        "convert-source-map": "^1.1.0",
+      "version": "7.11.6",
+      "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.11.6.tgz",
+      "integrity": "sha512-Wpcv03AGnmkgm6uS6k8iwhIwTrcP0m17TL1n1sy7qD0qelDu4XNeW0dN0mHfa+Gei211yDaLoEe/VlbXQzM4Bg==",
+      "dev": true,
+      "requires": {
+        "@babel/code-frame": "^7.10.4",
+        "@babel/generator": "^7.11.6",
+        "@babel/helper-module-transforms": "^7.11.0",
+        "@babel/helpers": "^7.10.4",
+        "@babel/parser": "^7.11.5",
+        "@babel/template": "^7.10.4",
+        "@babel/traverse": "^7.11.5",
+        "@babel/types": "^7.11.5",
+        "convert-source-map": "^1.7.0",
         "debug": "^4.1.0",
-        "json5": "^2.1.0",
-        "lodash": "^4.17.11",
+        "gensync": "^1.0.0-beta.1",
+        "json5": "^2.1.2",
+        "lodash": "^4.17.19",
         "resolve": "^1.3.2",
         "semver": "^5.4.1",
         "source-map": "^0.5.0"
       },
       "dependencies": {
         "debug": {
-          "version": "4.1.1",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
-          "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
+          "version": "4.2.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
+          "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
           "dev": true,
           "requires": {
-            "ms": "^2.1.1"
+            "ms": "2.1.2"
           }
         },
         "ms": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
-          "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==",
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
           "dev": true
         }
       }
     },
     "@babel/generator": {
-      "version": "7.4.4",
-      "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.4.4.tgz",
-      "integrity": "sha512-53UOLK6TVNqKxf7RUh8NE851EHRxOOeVXKbK2bivdb+iziMyk03Sr4eaE9OELCbyZAAafAKPDwF2TPUES5QbxQ==",
+      "version": "7.11.6",
+      "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.11.6.tgz",
+      "integrity": "sha512-DWtQ1PV3r+cLbySoHrwn9RWEgKMBLLma4OBQloPRyDYvc5msJM9kvTLo1YnlJd1P/ZuKbdli3ijr5q3FvAF3uA==",
       "dev": true,
       "requires": {
-        "@babel/types": "^7.4.4",
+        "@babel/types": "^7.11.5",
         "jsesc": "^2.5.1",
-        "lodash": "^4.17.11",
-        "source-map": "^0.5.0",
-        "trim-right": "^1.0.1"
+        "source-map": "^0.5.0"
       }
     },
     "@babel/helper-function-name": {
-      "version": "7.1.0",
-      "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.1.0.tgz",
-      "integrity": "sha512-A95XEoCpb3TO+KZzJ4S/5uW5fNe26DjBGqf1o9ucyLyCmi1dXq/B3c8iaWTfBk3VvetUxl16e8tIrd5teOCfGw==",
+      "version": "7.10.4",
+      "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.10.4.tgz",
+      "integrity": "sha512-YdaSyz1n8gY44EmN7x44zBn9zQ1Ry2Y+3GTA+3vH6Mizke1Vw0aWDM66FOYEPw8//qKkmqOckrGgTYa+6sceqQ==",
       "dev": true,
       "requires": {
-        "@babel/helper-get-function-arity": "^7.0.0",
-        "@babel/template": "^7.1.0",
-        "@babel/types": "^7.0.0"
+        "@babel/helper-get-function-arity": "^7.10.4",
+        "@babel/template": "^7.10.4",
+        "@babel/types": "^7.10.4"
       }
     },
     "@babel/helper-get-function-arity": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/@babel/helper-get-function-arity/-/helper-get-function-arity-7.0.0.tgz",
-      "integrity": "sha512-r2DbJeg4svYvt3HOS74U4eWKsUAMRH01Z1ds1zx8KNTPtpTL5JAsdFv8BNyOpVqdFhHkkRDIg5B4AsxmkjAlmQ==",
+      "version": "7.10.4",
+      "resolved": "https://registry.npmjs.org/@babel/helper-get-function-arity/-/helper-get-function-arity-7.10.4.tgz",
+      "integrity": "sha512-EkN3YDB+SRDgiIUnNgcmiD361ti+AVbL3f3Henf6dqqUyr5dMsorno0lJWJuLhDhkI5sYEpgj6y9kB8AOU1I2A==",
       "dev": true,
       "requires": {
-        "@babel/types": "^7.0.0"
+        "@babel/types": "^7.10.4"
+      }
+    },
+    "@babel/helper-member-expression-to-functions": {
+      "version": "7.11.0",
+      "resolved": "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.11.0.tgz",
+      "integrity": "sha512-JbFlKHFntRV5qKw3YC0CvQnDZ4XMwgzzBbld7Ly4Mj4cbFy3KywcR8NtNctRToMWJOVvLINJv525Gd6wwVEx/Q==",
+      "dev": true,
+      "requires": {
+        "@babel/types": "^7.11.0"
+      }
+    },
+    "@babel/helper-module-imports": {
+      "version": "7.10.4",
+      "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.10.4.tgz",
+      "integrity": "sha512-nEQJHqYavI217oD9+s5MUBzk6x1IlvoS9WTPfgG43CbMEeStE0v+r+TucWdx8KFGowPGvyOkDT9+7DHedIDnVw==",
+      "dev": true,
+      "requires": {
+        "@babel/types": "^7.10.4"
+      }
+    },
+    "@babel/helper-module-transforms": {
+      "version": "7.11.0",
+      "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.11.0.tgz",
+      "integrity": "sha512-02EVu8COMuTRO1TAzdMtpBPbe6aQ1w/8fePD2YgQmxZU4gpNWaL9gK3Jp7dxlkUlUCJOTaSeA+Hrm1BRQwqIhg==",
+      "dev": true,
+      "requires": {
+        "@babel/helper-module-imports": "^7.10.4",
+        "@babel/helper-replace-supers": "^7.10.4",
+        "@babel/helper-simple-access": "^7.10.4",
+        "@babel/helper-split-export-declaration": "^7.11.0",
+        "@babel/template": "^7.10.4",
+        "@babel/types": "^7.11.0",
+        "lodash": "^4.17.19"
+      }
+    },
+    "@babel/helper-optimise-call-expression": {
+      "version": "7.10.4",
+      "resolved": "https://registry.npmjs.org/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.10.4.tgz",
+      "integrity": "sha512-n3UGKY4VXwXThEiKrgRAoVPBMqeoPgHVqiHZOanAJCG9nQUL2pLRQirUzl0ioKclHGpGqRgIOkgcIJaIWLpygg==",
+      "dev": true,
+      "requires": {
+        "@babel/types": "^7.10.4"
       }
     },
     "@babel/helper-plugin-utils": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.0.0.tgz",
-      "integrity": "sha512-CYAOUCARwExnEixLdB6sDm2dIJ/YgEAKDM1MOeMeZu9Ld/bDgVo8aiWrXwcY7OBh+1Ea2uUcVRcxKk0GJvW7QA==",
+      "version": "7.10.4",
+      "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz",
+      "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==",
       "dev": true
     },
+    "@babel/helper-replace-supers": {
+      "version": "7.10.4",
+      "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.10.4.tgz",
+      "integrity": "sha512-sPxZfFXocEymYTdVK1UNmFPBN+Hv5mJkLPsYWwGBxZAxaWfFu+xqp7b6qWD0yjNuNL2VKc6L5M18tOXUP7NU0A==",
+      "dev": true,
+      "requires": {
+        "@babel/helper-member-expression-to-functions": "^7.10.4",
+        "@babel/helper-optimise-call-expression": "^7.10.4",
+        "@babel/traverse": "^7.10.4",
+        "@babel/types": "^7.10.4"
+      }
+    },
+    "@babel/helper-simple-access": {
+      "version": "7.10.4",
+      "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.10.4.tgz",
+      "integrity": "sha512-0fMy72ej/VEvF8ULmX6yb5MtHG4uH4Dbd6I/aHDb/JVg0bbivwt9Wg+h3uMvX+QSFtwr5MeItvazbrc4jtRAXw==",
+      "dev": true,
+      "requires": {
+        "@babel/template": "^7.10.4",
+        "@babel/types": "^7.10.4"
+      }
+    },
     "@babel/helper-split-export-declaration": {
-      "version": "7.4.4",
-      "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.4.4.tgz",
-      "integrity": "sha512-Ro/XkzLf3JFITkW6b+hNxzZ1n5OQ80NvIUdmHspih1XAhtN3vPTuUFT4eQnela+2MaZ5ulH+iyP513KJrxbN7Q==",
+      "version": "7.11.0",
+      "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.11.0.tgz",
+      "integrity": "sha512-74Vejvp6mHkGE+m+k5vHY93FX2cAtrw1zXrZXRlG4l410Nm9PxfEiVTn1PjDPV5SnmieiueY4AFg2xqhNFuuZg==",
       "dev": true,
       "requires": {
-        "@babel/types": "^7.4.4"
+        "@babel/types": "^7.11.0"
       }
     },
+    "@babel/helper-validator-identifier": {
+      "version": "7.10.4",
+      "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.4.tgz",
+      "integrity": "sha512-3U9y+43hz7ZM+rzG24Qe2mufW5KhvFg/NhnNph+i9mgCtdTCtMJuI1TMkrIUiK7Ix4PYlRF9I5dhqaLYA/ADXw==",
+      "dev": true
+    },
     "@babel/helpers": {
-      "version": "7.4.4",
-      "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.4.4.tgz",
-      "integrity": "sha512-igczbR/0SeuPR8RFfC7tGrbdTbFL3QTvH6D+Z6zNxnTe//GyqmtHmDkzrqDmyZ3eSwPqB/LhyKoU5DXsp+Vp2A==",
+      "version": "7.10.4",
+      "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.10.4.tgz",
+      "integrity": "sha512-L2gX/XeUONeEbI78dXSrJzGdz4GQ+ZTA/aazfUsFaWjSe95kiCuOZ5HsXvkiw3iwF+mFHSRUfJU8t6YavocdXA==",
       "dev": true,
       "requires": {
-        "@babel/template": "^7.4.4",
-        "@babel/traverse": "^7.4.4",
-        "@babel/types": "^7.4.4"
+        "@babel/template": "^7.10.4",
+        "@babel/traverse": "^7.10.4",
+        "@babel/types": "^7.10.4"
       }
     },
     "@babel/highlight": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.0.0.tgz",
-      "integrity": "sha512-UFMC4ZeFC48Tpvj7C8UgLvtkaUuovQX+5xNWrsIoMG8o2z+XFKjKaN9iVmS84dPwVN00W4wPmqvYoZF3EGAsfw==",
+      "version": "7.10.4",
+      "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.10.4.tgz",
+      "integrity": "sha512-i6rgnR/YgPEQzZZnbTHHuZdlE8qyoBNalD6F+q4vAFlcMEcqmkoG+mPqJYJCo63qPf74+Y1UZsl3l6f7/RIkmA==",
       "dev": true,
       "requires": {
+        "@babel/helper-validator-identifier": "^7.10.4",
         "chalk": "^2.0.0",
-        "esutils": "^2.0.2",
         "js-tokens": "^4.0.0"
-      },
-      "dependencies": {
-        "js-tokens": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
-          "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
-          "dev": true
-        }
       }
     },
     "@babel/parser": {
-      "version": "7.4.5",
-      "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.4.5.tgz",
-      "integrity": "sha512-9mUqkL1FF5T7f0WDFfAoDdiMVPWsdD1gZYzSnaXsxUCUqzuch/8of9G3VUSNiZmMBoRxT3neyVsqeiL/ZPcjew==",
+      "version": "7.11.5",
+      "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.11.5.tgz",
+      "integrity": "sha512-X9rD8qqm695vgmeaQ4fvz/o3+Wk4ZzQvSHkDBgpYKxpD4qTAUm88ZKtHkVqIOsYFFbIQ6wQYhC6q7pjqVK0E0Q==",
       "dev": true
     },
+    "@babel/plugin-syntax-async-generators": {
+      "version": "7.8.4",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz",
+      "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==",
+      "dev": true,
+      "requires": {
+        "@babel/helper-plugin-utils": "^7.8.0"
+      }
+    },
+    "@babel/plugin-syntax-bigint": {
+      "version": "7.8.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz",
+      "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==",
+      "dev": true,
+      "requires": {
+        "@babel/helper-plugin-utils": "^7.8.0"
+      }
+    },
+    "@babel/plugin-syntax-class-properties": {
+      "version": "7.10.4",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.10.4.tgz",
+      "integrity": "sha512-GCSBF7iUle6rNugfURwNmCGG3Z/2+opxAMLs1nND4bhEG5PuxTIggDBoeYYSujAlLtsupzOHYJQgPS3pivwXIA==",
+      "dev": true,
+      "requires": {
+        "@babel/helper-plugin-utils": "^7.10.4"
+      }
+    },
+    "@babel/plugin-syntax-import-meta": {
+      "version": "7.10.4",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz",
+      "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==",
+      "dev": true,
+      "requires": {
+        "@babel/helper-plugin-utils": "^7.10.4"
+      }
+    },
+    "@babel/plugin-syntax-json-strings": {
+      "version": "7.8.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz",
+      "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==",
+      "dev": true,
+      "requires": {
+        "@babel/helper-plugin-utils": "^7.8.0"
+      }
+    },
+    "@babel/plugin-syntax-logical-assignment-operators": {
+      "version": "7.10.4",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz",
+      "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==",
+      "dev": true,
+      "requires": {
+        "@babel/helper-plugin-utils": "^7.10.4"
+      }
+    },
+    "@babel/plugin-syntax-nullish-coalescing-operator": {
+      "version": "7.8.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz",
+      "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==",
+      "dev": true,
+      "requires": {
+        "@babel/helper-plugin-utils": "^7.8.0"
+      }
+    },
+    "@babel/plugin-syntax-numeric-separator": {
+      "version": "7.10.4",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz",
+      "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==",
+      "dev": true,
+      "requires": {
+        "@babel/helper-plugin-utils": "^7.10.4"
+      }
+    },
     "@babel/plugin-syntax-object-rest-spread": {
-      "version": "7.2.0",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.2.0.tgz",
-      "integrity": "sha512-t0JKGgqk2We+9may3t0xDdmneaXmyxq0xieYcKHxIsrJO64n1OiMWNUtc5gQK1PA0NpdCRrtZp4z+IUaKugrSA==",
+      "version": "7.8.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz",
+      "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==",
+      "dev": true,
+      "requires": {
+        "@babel/helper-plugin-utils": "^7.8.0"
+      }
+    },
+    "@babel/plugin-syntax-optional-catch-binding": {
+      "version": "7.8.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz",
+      "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==",
+      "dev": true,
+      "requires": {
+        "@babel/helper-plugin-utils": "^7.8.0"
+      }
+    },
+    "@babel/plugin-syntax-optional-chaining": {
+      "version": "7.8.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz",
+      "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==",
       "dev": true,
       "requires": {
-        "@babel/helper-plugin-utils": "^7.0.0"
+        "@babel/helper-plugin-utils": "^7.8.0"
       }
     },
     "@babel/template": {
-      "version": "7.4.4",
-      "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.4.4.tgz",
-      "integrity": "sha512-CiGzLN9KgAvgZsnivND7rkA+AeJ9JB0ciPOD4U59GKbQP2iQl+olF1l76kJOupqidozfZ32ghwBEJDhnk9MEcw==",
+      "version": "7.10.4",
+      "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.10.4.tgz",
+      "integrity": "sha512-ZCjD27cGJFUB6nmCB1Enki3r+L5kJveX9pq1SvAUKoICy6CZ9yD8xO086YXdYhvNjBdnekm4ZnaP5yC8Cs/1tA==",
       "dev": true,
       "requires": {
-        "@babel/code-frame": "^7.0.0",
-        "@babel/parser": "^7.4.4",
-        "@babel/types": "^7.4.4"
+        "@babel/code-frame": "^7.10.4",
+        "@babel/parser": "^7.10.4",
+        "@babel/types": "^7.10.4"
       }
     },
     "@babel/traverse": {
-      "version": "7.4.5",
-      "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.4.5.tgz",
-      "integrity": "sha512-Vc+qjynwkjRmIFGxy0KYoPj4FdVDxLej89kMHFsWScq999uX+pwcX4v9mWRjW0KcAYTPAuVQl2LKP1wEVLsp+A==",
+      "version": "7.11.5",
+      "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.11.5.tgz",
+      "integrity": "sha512-EjiPXt+r7LiCZXEfRpSJd+jUMnBd4/9OUv7Nx3+0u9+eimMwJmG0Q98lw4/289JCoxSE8OolDMNZaaF/JZ69WQ==",
       "dev": true,
       "requires": {
-        "@babel/code-frame": "^7.0.0",
-        "@babel/generator": "^7.4.4",
-        "@babel/helper-function-name": "^7.1.0",
-        "@babel/helper-split-export-declaration": "^7.4.4",
-        "@babel/parser": "^7.4.5",
-        "@babel/types": "^7.4.4",
+        "@babel/code-frame": "^7.10.4",
+        "@babel/generator": "^7.11.5",
+        "@babel/helper-function-name": "^7.10.4",
+        "@babel/helper-split-export-declaration": "^7.11.0",
+        "@babel/parser": "^7.11.5",
+        "@babel/types": "^7.11.5",
         "debug": "^4.1.0",
         "globals": "^11.1.0",
-        "lodash": "^4.17.11"
+        "lodash": "^4.17.19"
       },
       "dependencies": {
         "debug": {
-          "version": "4.1.1",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
-          "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
+          "version": "4.2.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
+          "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
           "dev": true,
           "requires": {
-            "ms": "^2.1.1"
+            "ms": "2.1.2"
           }
         },
         "ms": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
-          "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==",
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
           "dev": true
         }
       }
     },
     "@babel/types": {
-      "version": "7.4.4",
-      "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.4.4.tgz",
-      "integrity": "sha512-dOllgYdnEFOebhkKCjzSVFqw/PmmB8pH6RGOWkY4GsboQNd47b1fBThBSwlHAq9alF9vc1M3+6oqR47R50L0tQ==",
+      "version": "7.11.5",
+      "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.11.5.tgz",
+      "integrity": "sha512-bvM7Qz6eKnJVFIn+1LPtjlBFPVN5jNDc1XmN15vWe7Q3DPBufWWsLiIvUu7xW87uTG6QoggpIDnUgLQvPheU+Q==",
       "dev": true,
       "requires": {
-        "esutils": "^2.0.2",
-        "lodash": "^4.17.11",
+        "@babel/helper-validator-identifier": "^7.10.4",
+        "lodash": "^4.17.19",
         "to-fast-properties": "^2.0.0"
       }
     },
+    "@bcoe/v8-coverage": {
+      "version": "0.2.3",
+      "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz",
+      "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==",
+      "dev": true
+    },
     "@cnakazawa/watch": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@cnakazawa/watch/-/watch-1.0.3.tgz",
-      "integrity": "sha512-r5160ogAvGyHsal38Kux7YYtodEKOj89RGb28ht1jh3SJb08VwRwAKKJL0bGb04Zd/3r9FL3BFIc3bBidYffCA==",
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@cnakazawa/watch/-/watch-1.0.4.tgz",
+      "integrity": "sha512-v9kIhKwjeZThiWrLmj0y17CWoyddASLj9O2yvbZkbvw/N3rWOYy9zkV66ursAoVr0mV15bL8g0c4QZUE6cdDoQ==",
       "dev": true,
       "requires": {
         "exec-sh": "^0.3.2",
         "minimist": "^1.2.0"
       }
     },
+    "@evocateur/libnpmaccess": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/@evocateur/libnpmaccess/-/libnpmaccess-3.1.2.tgz",
+      "integrity": "sha512-KSCAHwNWro0CF2ukxufCitT9K5LjL/KuMmNzSu8wuwN2rjyKHD8+cmOsiybK+W5hdnwc5M1SmRlVCaMHQo+3rg==",
+      "dev": true,
+      "requires": {
+        "@evocateur/npm-registry-fetch": "^4.0.0",
+        "aproba": "^2.0.0",
+        "figgy-pudding": "^3.5.1",
+        "get-stream": "^4.0.0",
+        "npm-package-arg": "^6.1.0"
+      },
+      "dependencies": {
+        "aproba": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz",
+          "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==",
+          "dev": true
+        }
+      }
+    },
+    "@evocateur/libnpmpublish": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/@evocateur/libnpmpublish/-/libnpmpublish-1.2.2.tgz",
+      "integrity": "sha512-MJrrk9ct1FeY9zRlyeoyMieBjGDG9ihyyD9/Ft6MMrTxql9NyoEx2hw9casTIP4CdqEVu+3nQ2nXxoJ8RCXyFg==",
+      "dev": true,
+      "requires": {
+        "@evocateur/npm-registry-fetch": "^4.0.0",
+        "aproba": "^2.0.0",
+        "figgy-pudding": "^3.5.1",
+        "get-stream": "^4.0.0",
+        "lodash.clonedeep": "^4.5.0",
+        "normalize-package-data": "^2.4.0",
+        "npm-package-arg": "^6.1.0",
+        "semver": "^5.5.1",
+        "ssri": "^6.0.1"
+      },
+      "dependencies": {
+        "aproba": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz",
+          "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==",
+          "dev": true
+        }
+      }
+    },
+    "@evocateur/npm-registry-fetch": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/@evocateur/npm-registry-fetch/-/npm-registry-fetch-4.0.0.tgz",
+      "integrity": "sha512-k1WGfKRQyhJpIr+P17O5vLIo2ko1PFLKwoetatdduUSt/aQ4J2sJrJwwatdI5Z3SiYk/mRH9S3JpdmMFd/IK4g==",
+      "dev": true,
+      "requires": {
+        "JSONStream": "^1.3.4",
+        "bluebird": "^3.5.1",
+        "figgy-pudding": "^3.4.1",
+        "lru-cache": "^5.1.1",
+        "make-fetch-happen": "^5.0.0",
+        "npm-package-arg": "^6.1.0",
+        "safe-buffer": "^5.1.2"
+      }
+    },
+    "@evocateur/pacote": {
+      "version": "9.6.5",
+      "resolved": "https://registry.npmjs.org/@evocateur/pacote/-/pacote-9.6.5.tgz",
+      "integrity": "sha512-EI552lf0aG2nOV8NnZpTxNo2PcXKPmDbF9K8eCBFQdIZwHNGN/mi815fxtmUMa2wTa1yndotICIDt/V0vpEx2w==",
+      "dev": true,
+      "requires": {
+        "@evocateur/npm-registry-fetch": "^4.0.0",
+        "bluebird": "^3.5.3",
+        "cacache": "^12.0.3",
+        "chownr": "^1.1.2",
+        "figgy-pudding": "^3.5.1",
+        "get-stream": "^4.1.0",
+        "glob": "^7.1.4",
+        "infer-owner": "^1.0.4",
+        "lru-cache": "^5.1.1",
+        "make-fetch-happen": "^5.0.0",
+        "minimatch": "^3.0.4",
+        "minipass": "^2.3.5",
+        "mississippi": "^3.0.0",
+        "mkdirp": "^0.5.1",
+        "normalize-package-data": "^2.5.0",
+        "npm-package-arg": "^6.1.0",
+        "npm-packlist": "^1.4.4",
+        "npm-pick-manifest": "^3.0.0",
+        "osenv": "^0.1.5",
+        "promise-inflight": "^1.0.1",
+        "promise-retry": "^1.1.1",
+        "protoduck": "^5.0.1",
+        "rimraf": "^2.6.3",
+        "safe-buffer": "^5.2.0",
+        "semver": "^5.7.0",
+        "ssri": "^6.0.1",
+        "tar": "^4.4.10",
+        "unique-filename": "^1.1.1",
+        "which": "^1.3.1"
+      },
+      "dependencies": {
+        "mkdirp": {
+          "version": "0.5.5",
+          "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
+          "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
+          "dev": true,
+          "requires": {
+            "minimist": "^1.2.5"
+          }
+        }
+      }
+    },
     "@gulp-sourcemaps/identity-map": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/@gulp-sourcemaps/identity-map/-/identity-map-1.0.2.tgz",
@@ -224,6 +491,15 @@
         "through2": "^2.0.3"
       },
       "dependencies": {
+        "normalize-path": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.1.1.tgz",
+          "integrity": "sha1-GrKLVW4Zg2Oowab35vogE3/mrtk=",
+          "dev": true,
+          "requires": {
+            "remove-trailing-separator": "^1.0.1"
+          }
+        },
         "source-map": {
           "version": "0.6.1",
           "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
@@ -240,6397 +516,8284 @@
       "requires": {
         "normalize-path": "^2.0.1",
         "through2": "^2.0.3"
+      },
+      "dependencies": {
+        "normalize-path": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.1.1.tgz",
+          "integrity": "sha1-GrKLVW4Zg2Oowab35vogE3/mrtk=",
+          "dev": true,
+          "requires": {
+            "remove-trailing-separator": "^1.0.1"
+          }
+        }
       }
     },
-    "@jest/console": {
-      "version": "24.7.1",
-      "resolved": "https://registry.npmjs.org/@jest/console/-/console-24.7.1.tgz",
-      "integrity": "sha512-iNhtIy2M8bXlAOULWVTUxmnelTLFneTNEkHCgPmgd+zNwy9zVddJ6oS5rZ9iwoscNdT5mMwUd0C51v/fSlzItg==",
+    "@istanbuljs/load-nyc-config": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz",
+      "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==",
       "dev": true,
       "requires": {
-        "@jest/source-map": "^24.3.0",
-        "chalk": "^2.0.1",
-        "slash": "^2.0.0"
+        "camelcase": "^5.3.1",
+        "find-up": "^4.1.0",
+        "get-package-type": "^0.1.0",
+        "js-yaml": "^3.13.1",
+        "resolve-from": "^5.0.0"
       },
       "dependencies": {
-        "slash": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
-          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
+        "camelcase": {
+          "version": "5.3.1",
+          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
+          "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==",
+          "dev": true
+        },
+        "find-up": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz",
+          "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==",
+          "dev": true,
+          "requires": {
+            "locate-path": "^5.0.0",
+            "path-exists": "^4.0.0"
+          }
+        },
+        "path-exists": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
+          "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==",
           "dev": true
         }
       }
     },
-    "@jest/core": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/@jest/core/-/core-24.8.0.tgz",
-      "integrity": "sha512-R9rhAJwCBQzaRnrRgAdVfnglUuATXdwTRsYqs6NMdVcAl5euG8LtWDe+fVkN27YfKVBW61IojVsXKaOmSnqd/A==",
-      "dev": true,
-      "requires": {
-        "@jest/console": "^24.7.1",
-        "@jest/reporters": "^24.8.0",
-        "@jest/test-result": "^24.8.0",
-        "@jest/transform": "^24.8.0",
-        "@jest/types": "^24.8.0",
-        "ansi-escapes": "^3.0.0",
-        "chalk": "^2.0.1",
-        "exit": "^0.1.2",
-        "graceful-fs": "^4.1.15",
-        "jest-changed-files": "^24.8.0",
-        "jest-config": "^24.8.0",
-        "jest-haste-map": "^24.8.0",
-        "jest-message-util": "^24.8.0",
-        "jest-regex-util": "^24.3.0",
-        "jest-resolve-dependencies": "^24.8.0",
-        "jest-runner": "^24.8.0",
-        "jest-runtime": "^24.8.0",
-        "jest-snapshot": "^24.8.0",
-        "jest-util": "^24.8.0",
-        "jest-validate": "^24.8.0",
-        "jest-watcher": "^24.8.0",
-        "micromatch": "^3.1.10",
-        "p-each-series": "^1.0.0",
-        "pirates": "^4.0.1",
-        "realpath-native": "^1.1.0",
-        "rimraf": "^2.5.4",
-        "strip-ansi": "^5.0.0"
+    "@istanbuljs/schema": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.2.tgz",
+      "integrity": "sha512-tsAQNx32a8CoFhjhijUIhI4kccIAgmGhy8LZMZgGfmXcpMbPRUqn5LWmgRttILi6yeGmBJd2xsPkFMs0PzgPCw==",
+      "dev": true
+    },
+    "@jest/console": {
+      "version": "26.3.0",
+      "resolved": "https://registry.npmjs.org/@jest/console/-/console-26.3.0.tgz",
+      "integrity": "sha512-/5Pn6sJev0nPUcAdpJHMVIsA8sKizL2ZkcKPE5+dJrCccks7tcM7c9wbgHudBJbxXLoTbqsHkG1Dofoem4F09w==",
+      "dev": true,
+      "requires": {
+        "@jest/types": "^26.3.0",
+        "@types/node": "*",
+        "chalk": "^4.0.0",
+        "jest-message-util": "^26.3.0",
+        "jest-util": "^26.3.0",
+        "slash": "^3.0.0"
       },
       "dependencies": {
-        "ansi-regex": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz",
-          "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==",
-          "dev": true
-        },
-        "callsites": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
-          "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
-          "dev": true
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
+          }
         },
-        "ci-info": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
-          "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==",
-          "dev": true
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-report": "*"
+          }
         },
-        "is-ci": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz",
-          "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==",
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
           "dev": true,
           "requires": {
-            "ci-info": "^2.0.0"
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
           }
         },
-        "jest-message-util": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-24.8.0.tgz",
-          "integrity": "sha512-p2k71rf/b6ns8btdB0uVdljWo9h0ovpnEe05ZKWceQGfXYr4KkzgKo3PBi8wdnd9OtNh46VpNIJynUn/3MKm1g==",
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
           "dev": true,
           "requires": {
-            "@babel/code-frame": "^7.0.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
-            "@types/stack-utils": "^1.0.1",
-            "chalk": "^2.0.1",
-            "micromatch": "^3.1.10",
-            "slash": "^2.0.0",
-            "stack-utils": "^1.0.1"
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
           }
         },
-        "jest-util": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-24.8.0.tgz",
-          "integrity": "sha512-DYZeE+XyAnbNt0BG1OQqKy/4GVLPtzwGx5tsnDrFcax36rVE3lTA5fbvgmbVPUZf9w77AJ8otqR4VBbfFJkUZA==",
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
           "dev": true,
           "requires": {
-            "@jest/console": "^24.7.1",
-            "@jest/fake-timers": "^24.8.0",
-            "@jest/source-map": "^24.3.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
-            "callsites": "^3.0.0",
-            "chalk": "^2.0.1",
-            "graceful-fs": "^4.1.15",
-            "is-ci": "^2.0.0",
-            "mkdirp": "^0.5.1",
-            "slash": "^2.0.0",
-            "source-map": "^0.6.0"
+            "color-name": "~1.1.4"
           }
         },
-        "slash": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
-          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
           "dev": true
         },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
           "dev": true
         },
-        "strip-ansi": {
-          "version": "5.2.0",
-          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz",
-          "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==",
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
           "dev": true,
           "requires": {
-            "ansi-regex": "^4.1.0"
+            "has-flag": "^4.0.0"
           }
         }
       }
     },
-    "@jest/environment": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-24.8.0.tgz",
-      "integrity": "sha512-vlGt2HLg7qM+vtBrSkjDxk9K0YtRBi7HfRFaDxoRtyi+DyVChzhF20duvpdAnKVBV6W5tym8jm0U9EfXbDk1tw==",
-      "dev": true,
-      "requires": {
-        "@jest/fake-timers": "^24.8.0",
-        "@jest/transform": "^24.8.0",
-        "@jest/types": "^24.8.0",
-        "jest-mock": "^24.8.0"
-      }
-    },
-    "@jest/fake-timers": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-24.8.0.tgz",
-      "integrity": "sha512-2M4d5MufVXwi6VzZhJ9f5S/wU4ud2ck0kxPof1Iz3zWx6Y+V2eJrES9jEktB6O3o/oEyk+il/uNu9PvASjWXQw==",
-      "dev": true,
-      "requires": {
-        "@jest/types": "^24.8.0",
-        "jest-message-util": "^24.8.0",
-        "jest-mock": "^24.8.0"
+    "@jest/core": {
+      "version": "26.4.2",
+      "resolved": "https://registry.npmjs.org/@jest/core/-/core-26.4.2.tgz",
+      "integrity": "sha512-sDva7YkeNprxJfepOctzS8cAk9TOekldh+5FhVuXS40+94SHbiicRO1VV2tSoRtgIo+POs/Cdyf8p76vPTd6dg==",
+      "dev": true,
+      "requires": {
+        "@jest/console": "^26.3.0",
+        "@jest/reporters": "^26.4.1",
+        "@jest/test-result": "^26.3.0",
+        "@jest/transform": "^26.3.0",
+        "@jest/types": "^26.3.0",
+        "@types/node": "*",
+        "ansi-escapes": "^4.2.1",
+        "chalk": "^4.0.0",
+        "exit": "^0.1.2",
+        "graceful-fs": "^4.2.4",
+        "jest-changed-files": "^26.3.0",
+        "jest-config": "^26.4.2",
+        "jest-haste-map": "^26.3.0",
+        "jest-message-util": "^26.3.0",
+        "jest-regex-util": "^26.0.0",
+        "jest-resolve": "^26.4.0",
+        "jest-resolve-dependencies": "^26.4.2",
+        "jest-runner": "^26.4.2",
+        "jest-runtime": "^26.4.2",
+        "jest-snapshot": "^26.4.2",
+        "jest-util": "^26.3.0",
+        "jest-validate": "^26.4.2",
+        "jest-watcher": "^26.3.0",
+        "micromatch": "^4.0.2",
+        "p-each-series": "^2.1.0",
+        "rimraf": "^3.0.0",
+        "slash": "^3.0.0",
+        "strip-ansi": "^6.0.0"
       },
       "dependencies": {
-        "jest-message-util": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-24.8.0.tgz",
-          "integrity": "sha512-p2k71rf/b6ns8btdB0uVdljWo9h0ovpnEe05ZKWceQGfXYr4KkzgKo3PBi8wdnd9OtNh46VpNIJynUn/3MKm1g==",
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
           "dev": true,
           "requires": {
-            "@babel/code-frame": "^7.0.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
-            "@types/stack-utils": "^1.0.1",
-            "chalk": "^2.0.1",
-            "micromatch": "^3.1.10",
-            "slash": "^2.0.0",
-            "stack-utils": "^1.0.1"
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
           }
         },
-        "slash": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
-          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
-          "dev": true
-        }
-      }
-    },
-    "@jest/reporters": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-24.8.0.tgz",
-      "integrity": "sha512-eZ9TyUYpyIIXfYCrw0UHUWUvE35vx5I92HGMgS93Pv7du+GHIzl+/vh8Qj9MCWFK/4TqyttVBPakWMOfZRIfxw==",
-      "dev": true,
-      "requires": {
-        "@jest/environment": "^24.8.0",
-        "@jest/test-result": "^24.8.0",
-        "@jest/transform": "^24.8.0",
-        "@jest/types": "^24.8.0",
-        "chalk": "^2.0.1",
-        "exit": "^0.1.2",
-        "glob": "^7.1.2",
-        "istanbul-lib-coverage": "^2.0.2",
-        "istanbul-lib-instrument": "^3.0.1",
-        "istanbul-lib-report": "^2.0.4",
-        "istanbul-lib-source-maps": "^3.0.1",
-        "istanbul-reports": "^2.1.1",
-        "jest-haste-map": "^24.8.0",
-        "jest-resolve": "^24.8.0",
-        "jest-runtime": "^24.8.0",
-        "jest-util": "^24.8.0",
-        "jest-worker": "^24.6.0",
-        "node-notifier": "^5.2.1",
-        "slash": "^2.0.0",
-        "source-map": "^0.6.0",
-        "string-length": "^2.0.0"
-      },
-      "dependencies": {
-        "callsites": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
-          "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
-          "dev": true
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-report": "*"
+          }
         },
-        "ci-info": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
-          "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==",
-          "dev": true
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
         },
-        "is-ci": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz",
-          "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==",
+        "braces": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
+          "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
           "dev": true,
           "requires": {
-            "ci-info": "^2.0.0"
+            "fill-range": "^7.0.1"
           }
         },
-        "jest-util": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-24.8.0.tgz",
-          "integrity": "sha512-DYZeE+XyAnbNt0BG1OQqKy/4GVLPtzwGx5tsnDrFcax36rVE3lTA5fbvgmbVPUZf9w77AJ8otqR4VBbfFJkUZA==",
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
           "dev": true,
           "requires": {
-            "@jest/console": "^24.7.1",
-            "@jest/fake-timers": "^24.8.0",
-            "@jest/source-map": "^24.3.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
-            "callsites": "^3.0.0",
-            "chalk": "^2.0.1",
-            "graceful-fs": "^4.1.15",
-            "is-ci": "^2.0.0",
-            "mkdirp": "^0.5.1",
-            "slash": "^2.0.0",
-            "source-map": "^0.6.0"
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
           }
         },
-        "slash": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
-          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
-          "dev": true
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
         },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-          "dev": true
-        }
-      }
-    },
-    "@jest/source-map": {
-      "version": "24.3.0",
-      "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-24.3.0.tgz",
-      "integrity": "sha512-zALZt1t2ou8le/crCeeiRYzvdnTzaIlpOWaet45lNSqNJUnXbppUUFR4ZUAlzgDmKee4Q5P/tKXypI1RiHwgag==",
-      "dev": true,
-      "requires": {
-        "callsites": "^3.0.0",
-        "graceful-fs": "^4.1.15",
-        "source-map": "^0.6.0"
-      },
-      "dependencies": {
-        "callsites": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
-          "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
           "dev": true
         },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-          "dev": true
-        }
-      }
-    },
-    "@jest/test-result": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-24.8.0.tgz",
-      "integrity": "sha512-+YdLlxwizlfqkFDh7Mc7ONPQAhA4YylU1s529vVM1rsf67vGZH/2GGm5uO8QzPeVyaVMobCQ7FTxl38QrKRlng==",
-      "dev": true,
-      "requires": {
-        "@jest/console": "^24.7.1",
-        "@jest/types": "^24.8.0",
-        "@types/istanbul-lib-coverage": "^2.0.0"
-      }
-    },
-    "@jest/test-sequencer": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-24.8.0.tgz",
-      "integrity": "sha512-OzL/2yHyPdCHXEzhoBuq37CE99nkme15eHkAzXRVqthreWZamEMA0WoetwstsQBCXABhczpK03JNbc4L01vvLg==",
-      "dev": true,
-      "requires": {
-        "@jest/test-result": "^24.8.0",
-        "jest-haste-map": "^24.8.0",
-        "jest-runner": "^24.8.0",
-        "jest-runtime": "^24.8.0"
-      }
-    },
-    "@jest/transform": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-24.8.0.tgz",
-      "integrity": "sha512-xBMfFUP7TortCs0O+Xtez2W7Zu1PLH9bvJgtraN1CDST6LBM/eTOZ9SfwS/lvV8yOfcDpFmwf9bq5cYbXvqsvA==",
-      "dev": true,
-      "requires": {
-        "@babel/core": "^7.1.0",
-        "@jest/types": "^24.8.0",
-        "babel-plugin-istanbul": "^5.1.0",
-        "chalk": "^2.0.1",
-        "convert-source-map": "^1.4.0",
-        "fast-json-stable-stringify": "^2.0.0",
-        "graceful-fs": "^4.1.15",
-        "jest-haste-map": "^24.8.0",
-        "jest-regex-util": "^24.3.0",
-        "jest-util": "^24.8.0",
-        "micromatch": "^3.1.10",
-        "realpath-native": "^1.1.0",
-        "slash": "^2.0.0",
-        "source-map": "^0.6.1",
-        "write-file-atomic": "2.4.1"
-      },
-      "dependencies": {
-        "callsites": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
-          "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
+        "fill-range": {
+          "version": "7.0.1",
+          "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
+          "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+          "dev": true,
+          "requires": {
+            "to-regex-range": "^5.0.1"
+          }
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
           "dev": true
         },
-        "ci-info": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
-          "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==",
+        "is-number": {
+          "version": "7.0.0",
+          "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
+          "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
           "dev": true
         },
-        "is-ci": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz",
-          "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==",
+        "micromatch": {
+          "version": "4.0.2",
+          "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.2.tgz",
+          "integrity": "sha512-y7FpHSbMUMoyPbYUSzO6PaZ6FyRnQOpHuKwbo1G+Knck95XVU4QAiKdGEnj5wwoS7PlOgthX/09u5iFJ+aYf5Q==",
           "dev": true,
           "requires": {
-            "ci-info": "^2.0.0"
+            "braces": "^3.0.1",
+            "picomatch": "^2.0.5"
           }
         },
-        "jest-util": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-24.8.0.tgz",
-          "integrity": "sha512-DYZeE+XyAnbNt0BG1OQqKy/4GVLPtzwGx5tsnDrFcax36rVE3lTA5fbvgmbVPUZf9w77AJ8otqR4VBbfFJkUZA==",
+        "rimraf": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
+          "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
           "dev": true,
           "requires": {
-            "@jest/console": "^24.7.1",
-            "@jest/fake-timers": "^24.8.0",
-            "@jest/source-map": "^24.3.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
-            "callsites": "^3.0.0",
-            "chalk": "^2.0.1",
-            "graceful-fs": "^4.1.15",
-            "is-ci": "^2.0.0",
-            "mkdirp": "^0.5.1",
-            "slash": "^2.0.0",
-            "source-map": "^0.6.0"
+            "glob": "^7.1.3"
           }
         },
-        "slash": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
-          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
-          "dev": true
+        "strip-ansi": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
+          "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
+          "dev": true,
+          "requires": {
+            "ansi-regex": "^5.0.0"
+          }
         },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-          "dev": true
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
         },
-        "write-file-atomic": {
-          "version": "2.4.1",
-          "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.1.tgz",
-          "integrity": "sha512-TGHFeZEZMnv+gBFRfjAcxL5bPHrsGKtnb4qsFAws7/vlh+QfwAaySIw4AXP9ZskTTh5GWu3FLuJhsWVdiJPGvg==",
+        "to-regex-range": {
+          "version": "5.0.1",
+          "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
+          "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
           "dev": true,
           "requires": {
-            "graceful-fs": "^4.1.11",
-            "imurmurhash": "^0.1.4",
-            "signal-exit": "^3.0.2"
+            "is-number": "^7.0.0"
           }
         }
       }
     },
-    "@jest/types": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.8.0.tgz",
-      "integrity": "sha512-g17UxVr2YfBtaMUxn9u/4+siG1ptg9IGYAYwvpwn61nBg779RXnjE/m7CxYcIzEt0AbHZZAHSEZNhkE2WxURVg==",
-      "dev": true,
-      "requires": {
-        "@types/istanbul-lib-coverage": "^2.0.0",
-        "@types/istanbul-reports": "^1.1.1",
-        "@types/yargs": "^12.0.9"
-      }
-    },
-    "@lerna/add": {
-      "version": "3.10.6",
-      "resolved": "https://registry.npmjs.org/@lerna/add/-/add-3.10.6.tgz",
-      "integrity": "sha512-FxQ5Bmyb5fF+3BQiNffM6cTeGCrl4uaAuGvxFIWF6Pgz6U14tUc1e16xgKDvVb1CurzJgIV5sLOT5xmCOqv1kA==",
-      "dev": true,
-      "requires": {
-        "@lerna/bootstrap": "3.10.6",
-        "@lerna/command": "3.10.6",
-        "@lerna/filter-options": "3.10.6",
-        "@lerna/npm-conf": "3.7.0",
-        "@lerna/validation-error": "3.6.0",
-        "dedent": "^0.7.0",
-        "libnpm": "^2.0.1",
-        "p-map": "^1.2.0",
-        "semver": "^5.5.0"
-      }
-    },
-    "@lerna/batch-packages": {
-      "version": "3.10.6",
-      "resolved": "https://registry.npmjs.org/@lerna/batch-packages/-/batch-packages-3.10.6.tgz",
-      "integrity": "sha512-sInr3ZQJFMh9Zq+ZUoVjX8R67j9ViRkVy0uEMsOfG+jZlXj1lRPRMPRiRgU0jXSYEwCdwuAB5pTd9tTx0VCJUw==",
-      "dev": true,
-      "requires": {
-        "@lerna/package-graph": "3.10.6",
-        "@lerna/validation-error": "3.6.0",
-        "libnpm": "^2.0.1"
-      }
-    },
-    "@lerna/bootstrap": {
-      "version": "3.10.6",
-      "resolved": "https://registry.npmjs.org/@lerna/bootstrap/-/bootstrap-3.10.6.tgz",
-      "integrity": "sha512-qbGjAxRpV/eiI9CboUIpsPPGpSogs8mN2/iDaAUBTaWVFVz/YyU64nui84Gll0kbdaHOyPput+kk2S8NCSCCdg==",
-      "dev": true,
-      "requires": {
-        "@lerna/batch-packages": "3.10.6",
-        "@lerna/command": "3.10.6",
-        "@lerna/filter-options": "3.10.6",
-        "@lerna/has-npm-version": "3.10.0",
-        "@lerna/npm-install": "3.10.0",
-        "@lerna/package-graph": "3.10.6",
-        "@lerna/pulse-till-done": "3.7.1",
-        "@lerna/rimraf-dir": "3.10.0",
-        "@lerna/run-lifecycle": "3.10.5",
-        "@lerna/run-parallel-batches": "3.0.0",
-        "@lerna/symlink-binary": "3.10.0",
-        "@lerna/symlink-dependencies": "3.10.0",
-        "@lerna/validation-error": "3.6.0",
-        "dedent": "^0.7.0",
-        "get-port": "^3.2.0",
-        "libnpm": "^2.0.1",
-        "multimatch": "^2.1.0",
-        "p-finally": "^1.0.0",
-        "p-map": "^1.2.0",
-        "p-map-series": "^1.0.0",
-        "p-waterfall": "^1.0.0",
-        "read-package-tree": "^5.1.6",
-        "semver": "^5.5.0"
-      }
-    },
-    "@lerna/changed": {
-      "version": "3.10.6",
-      "resolved": "https://registry.npmjs.org/@lerna/changed/-/changed-3.10.6.tgz",
-      "integrity": "sha512-nZDVq/sKdhgoAg1BVnpqjqUUz5+zedG+AnU+6mjEN2f23YVtRCsW55N4I9eEdW2pxXUaCY85Hj/HPSA74BYaFg==",
-      "dev": true,
-      "requires": {
-        "@lerna/collect-updates": "3.10.1",
-        "@lerna/command": "3.10.6",
-        "@lerna/listable": "3.10.6",
-        "@lerna/output": "3.6.0",
-        "@lerna/version": "3.10.6"
-      }
-    },
-    "@lerna/check-working-tree": {
-      "version": "3.10.0",
-      "resolved": "https://registry.npmjs.org/@lerna/check-working-tree/-/check-working-tree-3.10.0.tgz",
-      "integrity": "sha512-NdIPhDgEtGHfeGjB9F0oAoPLywgMpjnJhLLwTNQkelDHo2xNAVpG8kV+A2UJ+cU5UXCZA4RZFxKNmw86rO+Drw==",
-      "dev": true,
-      "requires": {
-        "@lerna/describe-ref": "3.10.0",
-        "@lerna/validation-error": "3.6.0"
-      }
-    },
-    "@lerna/child-process": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/@lerna/child-process/-/child-process-3.3.0.tgz",
-      "integrity": "sha512-q2d/OPlNX/cBXB6Iz1932RFzOmOHq6ZzPjqebkINNaTojHWuuRpvJJY4Uz3NGpJ3kEtPDvBemkZqUBTSO5wb1g==",
+    "@jest/environment": {
+      "version": "26.3.0",
+      "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-26.3.0.tgz",
+      "integrity": "sha512-EW+MFEo0DGHahf83RAaiqQx688qpXgl99wdb8Fy67ybyzHwR1a58LHcO376xQJHfmoXTu89M09dH3J509cx2AA==",
       "dev": true,
       "requires": {
-        "chalk": "^2.3.1",
-        "execa": "^1.0.0",
-        "strong-log-transformer": "^2.0.0"
+        "@jest/fake-timers": "^26.3.0",
+        "@jest/types": "^26.3.0",
+        "@types/node": "*",
+        "jest-mock": "^26.3.0"
       },
       "dependencies": {
-        "cross-spawn": {
-          "version": "6.0.5",
-          "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz",
-          "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==",
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
           "dev": true,
           "requires": {
-            "nice-try": "^1.0.4",
-            "path-key": "^2.0.1",
-            "semver": "^5.5.0",
-            "shebang-command": "^1.2.0",
-            "which": "^1.2.9"
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
           }
         },
-        "execa": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz",
-          "integrity": "sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==",
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
           "dev": true,
           "requires": {
-            "cross-spawn": "^6.0.0",
-            "get-stream": "^4.0.0",
-            "is-stream": "^1.1.0",
-            "npm-run-path": "^2.0.0",
-            "p-finally": "^1.0.0",
-            "signal-exit": "^3.0.0",
-            "strip-eof": "^1.0.0"
+            "@types/istanbul-lib-report": "*"
           }
         },
-        "get-stream": {
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
+        },
+        "chalk": {
           "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz",
-          "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
           "dev": true,
           "requires": {
-            "pump": "^3.0.0"
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
           }
         },
-        "pump": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-          "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
           "dev": true,
           "requires": {
-            "end-of-stream": "^1.1.0",
-            "once": "^1.3.1"
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
           }
         }
       }
     },
-    "@lerna/clean": {
-      "version": "3.10.6",
-      "resolved": "https://registry.npmjs.org/@lerna/clean/-/clean-3.10.6.tgz",
-      "integrity": "sha512-MuL8HOwnyvVtr6GOiAN/Ofjbx+BJdCrtjrM1Uuh8FFnbnZTPVf+0MPxL2jVzPMo0PmoIrX3fvlwvzKNk/lH0Ug==",
+    "@jest/fake-timers": {
+      "version": "26.3.0",
+      "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-26.3.0.tgz",
+      "integrity": "sha512-ZL9ytUiRwVP8ujfRepffokBvD2KbxbqMhrXSBhSdAhISCw3gOkuntisiSFv+A6HN0n0fF4cxzICEKZENLmW+1A==",
       "dev": true,
       "requires": {
-        "@lerna/command": "3.10.6",
-        "@lerna/filter-options": "3.10.6",
-        "@lerna/prompt": "3.6.0",
-        "@lerna/pulse-till-done": "3.7.1",
-        "@lerna/rimraf-dir": "3.10.0",
-        "p-map": "^1.2.0",
-        "p-map-series": "^1.0.0",
-        "p-waterfall": "^1.0.0"
-      }
-    },
-    "@lerna/cli": {
-      "version": "3.10.7",
-      "resolved": "https://registry.npmjs.org/@lerna/cli/-/cli-3.10.7.tgz",
-      "integrity": "sha512-yuoz/24mIfYit3neKqoE5NVs42Rj9A6A6SlkNPDfsy3v/Vh7SgYkU3cwiGyvwBGzIdhqL4/SWYo8H7YJLs0C+g==",
-      "dev": true,
-      "requires": {
-        "@lerna/global-options": "3.10.6",
-        "dedent": "^0.7.0",
-        "libnpm": "^2.0.1",
-        "yargs": "^12.0.1"
+        "@jest/types": "^26.3.0",
+        "@sinonjs/fake-timers": "^6.0.1",
+        "@types/node": "*",
+        "jest-message-util": "^26.3.0",
+        "jest-mock": "^26.3.0",
+        "jest-util": "^26.3.0"
       },
       "dependencies": {
-        "ansi-regex": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
-          "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=",
-          "dev": true
-        },
-        "camelcase": {
-          "version": "5.0.0",
-          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.0.0.tgz",
-          "integrity": "sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA==",
-          "dev": true
-        },
-        "cliui": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/cliui/-/cliui-4.1.0.tgz",
-          "integrity": "sha512-4FG+RSG9DL7uEwRUZXZn3SS34DiDPfzP0VOiEwtUWlE+AR2EIg+hSyvrIgUUfhdgR/UkAeW2QHgeP+hWrXs7jQ==",
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
           "dev": true,
           "requires": {
-            "string-width": "^2.1.1",
-            "strip-ansi": "^4.0.0",
-            "wrap-ansi": "^2.0.0"
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
           }
         },
-        "cross-spawn": {
-          "version": "6.0.5",
-          "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz",
-          "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==",
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
           "dev": true,
           "requires": {
-            "nice-try": "^1.0.4",
-            "path-key": "^2.0.1",
-            "semver": "^5.5.0",
-            "shebang-command": "^1.2.0",
-            "which": "^1.2.9"
+            "@types/istanbul-lib-report": "*"
           }
         },
-        "execa": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz",
-          "integrity": "sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==",
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
           "dev": true,
           "requires": {
-            "cross-spawn": "^6.0.0",
-            "get-stream": "^4.0.0",
-            "is-stream": "^1.1.0",
-            "npm-run-path": "^2.0.0",
-            "p-finally": "^1.0.0",
-            "signal-exit": "^3.0.0",
-            "strip-eof": "^1.0.0"
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
           }
         },
-        "find-up": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz",
-          "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==",
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
           "dev": true,
           "requires": {
-            "locate-path": "^3.0.0"
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
           }
         },
-        "get-stream": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz",
-          "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==",
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
           "dev": true,
           "requires": {
-            "pump": "^3.0.0"
+            "color-name": "~1.1.4"
           }
         },
-        "invert-kv": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-2.0.0.tgz",
-          "integrity": "sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==",
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
           "dev": true
         },
-        "is-fullwidth-code-point": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
-          "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=",
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
           "dev": true
         },
-        "lcid": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/lcid/-/lcid-2.0.0.tgz",
-          "integrity": "sha512-avPEb8P8EGnwXKClwsNUgryVjllcRqtMYa49NTsbQagYuT1DcXnl1915oxWjoyGrXR6zH/Y0Zc96xWsPcoDKeA==",
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
           "dev": true,
           "requires": {
-            "invert-kv": "^2.0.0"
+            "has-flag": "^4.0.0"
           }
-        },
-        "locate-path": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz",
-          "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==",
+        }
+      }
+    },
+    "@jest/globals": {
+      "version": "26.4.2",
+      "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-26.4.2.tgz",
+      "integrity": "sha512-Ot5ouAlehhHLRhc+sDz2/9bmNv9p5ZWZ9LE1pXGGTCXBasmi5jnYjlgYcYt03FBwLmZXCZ7GrL29c33/XRQiow==",
+      "dev": true,
+      "requires": {
+        "@jest/environment": "^26.3.0",
+        "@jest/types": "^26.3.0",
+        "expect": "^26.4.2"
+      },
+      "dependencies": {
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
           "dev": true,
           "requires": {
-            "p-locate": "^3.0.0",
-            "path-exists": "^3.0.0"
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
           }
         },
-        "mem": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/mem/-/mem-4.0.0.tgz",
-          "integrity": "sha512-WQxG/5xYc3tMbYLXoXPm81ET2WDULiU5FxbuIoNbJqLOOI8zehXFdZuiUEgfdrU2mVB1pxBZUGlYORSrpuJreA==",
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
           "dev": true,
           "requires": {
-            "map-age-cleaner": "^0.1.1",
-            "mimic-fn": "^1.0.0",
-            "p-is-promise": "^1.1.0"
+            "@types/istanbul-lib-report": "*"
           }
         },
-        "os-locale": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-3.1.0.tgz",
-          "integrity": "sha512-Z8l3R4wYWM40/52Z+S265okfFj8Kt2cC2MKY+xNi3kFs+XGI7WXu/I309QQQYbRW4ijiZ+yxs9pqEhJh0DqW3Q==",
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
           "dev": true,
           "requires": {
-            "execa": "^1.0.0",
-            "lcid": "^2.0.0",
-            "mem": "^4.0.0"
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
           }
         },
-        "p-limit": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.1.0.tgz",
-          "integrity": "sha512-NhURkNcrVB+8hNfLuysU8enY5xn2KXphsHBaC2YmRNTZRc7RWusw6apSpdEj3jo4CMb6W9nrF6tTnsJsJeyu6g==",
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
           "dev": true,
           "requires": {
-            "p-try": "^2.0.0"
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
           }
         },
-        "p-locate": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz",
-          "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==",
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
           "dev": true,
           "requires": {
-            "p-limit": "^2.0.0"
+            "color-name": "~1.1.4"
           }
         },
-        "p-try": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.0.0.tgz",
-          "integrity": "sha512-hMp0onDKIajHfIkdRk3P4CdCmErkYAxxDtP3Wx/4nZ3aGlau2VKh3mZpcuFkH27WQkL/3WBCPOktzA9ZOAnMQQ==",
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
           "dev": true
         },
-        "path-exists": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
-          "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=",
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
           "dev": true
         },
-        "pump": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-          "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
           "dev": true,
           "requires": {
-            "end-of-stream": "^1.1.0",
-            "once": "^1.3.1"
+            "has-flag": "^4.0.0"
+          }
+        }
+      }
+    },
+    "@jest/reporters": {
+      "version": "26.4.1",
+      "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-26.4.1.tgz",
+      "integrity": "sha512-aROTkCLU8++yiRGVxLsuDmZsQEKO6LprlrxtAuzvtpbIFl3eIjgIf3EUxDKgomkS25R9ZzwGEdB5weCcBZlrpQ==",
+      "dev": true,
+      "requires": {
+        "@bcoe/v8-coverage": "^0.2.3",
+        "@jest/console": "^26.3.0",
+        "@jest/test-result": "^26.3.0",
+        "@jest/transform": "^26.3.0",
+        "@jest/types": "^26.3.0",
+        "chalk": "^4.0.0",
+        "collect-v8-coverage": "^1.0.0",
+        "exit": "^0.1.2",
+        "glob": "^7.1.2",
+        "graceful-fs": "^4.2.4",
+        "istanbul-lib-coverage": "^3.0.0",
+        "istanbul-lib-instrument": "^4.0.3",
+        "istanbul-lib-report": "^3.0.0",
+        "istanbul-lib-source-maps": "^4.0.0",
+        "istanbul-reports": "^3.0.2",
+        "jest-haste-map": "^26.3.0",
+        "jest-resolve": "^26.4.0",
+        "jest-util": "^26.3.0",
+        "jest-worker": "^26.3.0",
+        "node-notifier": "^8.0.0",
+        "slash": "^3.0.0",
+        "source-map": "^0.6.0",
+        "string-length": "^4.0.1",
+        "terminal-link": "^2.0.0",
+        "v8-to-istanbul": "^5.0.1"
+      },
+      "dependencies": {
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
           }
         },
-        "string-width": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",
-          "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==",
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
           "dev": true,
           "requires": {
-            "is-fullwidth-code-point": "^2.0.0",
-            "strip-ansi": "^4.0.0"
+            "@types/istanbul-lib-report": "*"
           }
         },
-        "strip-ansi": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
-          "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
           "dev": true,
           "requires": {
-            "ansi-regex": "^3.0.0"
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
           }
         },
-        "which-module": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz",
-          "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=",
-          "dev": true
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
         },
-        "yargs": {
-          "version": "12.0.5",
-          "resolved": "https://registry.npmjs.org/yargs/-/yargs-12.0.5.tgz",
-          "integrity": "sha512-Lhz8TLaYnxq/2ObqHDql8dX8CJi97oHxrjUcYtzKbbykPtVW9WB+poxI+NM2UIzsMgNCZTIf0AQwsjK5yMAqZw==",
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
           "dev": true,
           "requires": {
-            "cliui": "^4.0.0",
-            "decamelize": "^1.2.0",
-            "find-up": "^3.0.0",
-            "get-caller-file": "^1.0.1",
-            "os-locale": "^3.0.0",
-            "require-directory": "^2.1.1",
-            "require-main-filename": "^1.0.1",
-            "set-blocking": "^2.0.0",
-            "string-width": "^2.0.0",
-            "which-module": "^2.0.0",
-            "y18n": "^3.2.1 || ^4.0.0",
-            "yargs-parser": "^11.1.1"
+            "color-name": "~1.1.4"
           }
         },
-        "yargs-parser": {
-          "version": "11.1.1",
-          "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-11.1.1.tgz",
-          "integrity": "sha512-C6kB/WJDiaxONLJQnF8ccx9SEeoTTLek8RVbaOIsrAUS8VrBEXfmeSnCZxygc+XC2sNMBIwOOnfcxiynjHsVSQ==",
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "source-map": {
+          "version": "0.6.1",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
           "dev": true,
           "requires": {
-            "camelcase": "^5.0.0",
-            "decamelize": "^1.2.0"
+            "has-flag": "^4.0.0"
           }
         }
       }
     },
-    "@lerna/collect-updates": {
-      "version": "3.10.1",
-      "resolved": "https://registry.npmjs.org/@lerna/collect-updates/-/collect-updates-3.10.1.tgz",
-      "integrity": "sha512-vb0wEJ8k63G+2CR/ud1WeVHNJ21Fs6Ew6lbdGZXnF4ZvaFWxWJZpoHeWwzjhMdJ75QdTzUaIhTG1hnH9faQNMw==",
+    "@jest/source-map": {
+      "version": "26.3.0",
+      "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-26.3.0.tgz",
+      "integrity": "sha512-hWX5IHmMDWe1kyrKl7IhFwqOuAreIwHhbe44+XH2ZRHjrKIh0LO5eLQ/vxHFeAfRwJapmxuqlGAEYLadDq6ZGQ==",
       "dev": true,
       "requires": {
-        "@lerna/child-process": "3.3.0",
-        "@lerna/describe-ref": "3.10.0",
-        "libnpm": "^2.0.1",
-        "minimatch": "^3.0.4",
-        "slash": "^1.0.0"
+        "callsites": "^3.0.0",
+        "graceful-fs": "^4.2.4",
+        "source-map": "^0.6.0"
+      },
+      "dependencies": {
+        "source-map": {
+          "version": "0.6.1",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+          "dev": true
+        }
       }
     },
-    "@lerna/command": {
-      "version": "3.10.6",
-      "resolved": "https://registry.npmjs.org/@lerna/command/-/command-3.10.6.tgz",
-      "integrity": "sha512-jPZswMZXOpAaIuSF5hrz+eaWQzbDrvwbrkCoRJKfiAHx7URAkE6MQe9DeAnqrTKMqwfg0RciSrZLc8kWYfrzCQ==",
+    "@jest/test-result": {
+      "version": "26.3.0",
+      "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-26.3.0.tgz",
+      "integrity": "sha512-a8rbLqzW/q7HWheFVMtghXV79Xk+GWwOK1FrtimpI5n1la2SY0qHri3/b0/1F0Ve0/yJmV8pEhxDfVwiUBGtgg==",
       "dev": true,
       "requires": {
-        "@lerna/child-process": "3.3.0",
-        "@lerna/package-graph": "3.10.6",
-        "@lerna/project": "3.10.0",
-        "@lerna/validation-error": "3.6.0",
-        "@lerna/write-log-file": "3.6.0",
-        "dedent": "^0.7.0",
-        "execa": "^1.0.0",
-        "is-ci": "^1.0.10",
-        "libnpm": "^2.0.1",
-        "lodash": "^4.17.5"
+        "@jest/console": "^26.3.0",
+        "@jest/types": "^26.3.0",
+        "@types/istanbul-lib-coverage": "^2.0.0",
+        "collect-v8-coverage": "^1.0.0"
       },
       "dependencies": {
-        "cross-spawn": {
-          "version": "6.0.5",
-          "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz",
-          "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==",
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
           "dev": true,
           "requires": {
-            "nice-try": "^1.0.4",
-            "path-key": "^2.0.1",
-            "semver": "^5.5.0",
-            "shebang-command": "^1.2.0",
-            "which": "^1.2.9"
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
           }
         },
-        "execa": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz",
-          "integrity": "sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==",
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
           "dev": true,
           "requires": {
-            "cross-spawn": "^6.0.0",
-            "get-stream": "^4.0.0",
-            "is-stream": "^1.1.0",
-            "npm-run-path": "^2.0.0",
-            "p-finally": "^1.0.0",
-            "signal-exit": "^3.0.0",
-            "strip-eof": "^1.0.0"
+            "@types/istanbul-lib-report": "*"
           }
         },
-        "get-stream": {
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
+        },
+        "chalk": {
           "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz",
-          "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
           "dev": true,
           "requires": {
-            "pump": "^3.0.0"
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
           }
         },
-        "pump": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-          "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
           "dev": true,
           "requires": {
-            "end-of-stream": "^1.1.0",
-            "once": "^1.3.1"
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
           }
         }
       }
     },
-    "@lerna/conventional-commits": {
-      "version": "3.10.0",
-      "resolved": "https://registry.npmjs.org/@lerna/conventional-commits/-/conventional-commits-3.10.0.tgz",
-      "integrity": "sha512-8FvO0eR8g/tEgkb6eRVYaD39TsqMKsOXp17EV48jciciEqcrF/d1Ypu6ilK1GDp6R/1m2mbjt/b52a/qrO+xaw==",
+    "@jest/test-sequencer": {
+      "version": "26.4.2",
+      "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-26.4.2.tgz",
+      "integrity": "sha512-83DRD8N3M0tOhz9h0bn6Kl6dSp+US6DazuVF8J9m21WAp5x7CqSMaNycMP0aemC/SH/pDQQddbsfHRTBXVUgog==",
       "dev": true,
       "requires": {
-        "@lerna/validation-error": "3.6.0",
-        "conventional-changelog-angular": "^5.0.2",
-        "conventional-changelog-core": "^3.1.5",
-        "conventional-recommended-bump": "^4.0.4",
-        "fs-extra": "^7.0.0",
-        "get-stream": "^4.0.0",
-        "libnpm": "^2.0.1",
-        "semver": "^5.5.0"
+        "@jest/test-result": "^26.3.0",
+        "graceful-fs": "^4.2.4",
+        "jest-haste-map": "^26.3.0",
+        "jest-runner": "^26.4.2",
+        "jest-runtime": "^26.4.2"
+      }
+    },
+    "@jest/transform": {
+      "version": "26.3.0",
+      "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-26.3.0.tgz",
+      "integrity": "sha512-Isj6NB68QorGoFWvcOjlUhpkT56PqNIsXKR7XfvoDlCANn/IANlh8DrKAA2l2JKC3yWSMH5wS0GwuQM20w3b2A==",
+      "dev": true,
+      "requires": {
+        "@babel/core": "^7.1.0",
+        "@jest/types": "^26.3.0",
+        "babel-plugin-istanbul": "^6.0.0",
+        "chalk": "^4.0.0",
+        "convert-source-map": "^1.4.0",
+        "fast-json-stable-stringify": "^2.0.0",
+        "graceful-fs": "^4.2.4",
+        "jest-haste-map": "^26.3.0",
+        "jest-regex-util": "^26.0.0",
+        "jest-util": "^26.3.0",
+        "micromatch": "^4.0.2",
+        "pirates": "^4.0.1",
+        "slash": "^3.0.0",
+        "source-map": "^0.6.1",
+        "write-file-atomic": "^3.0.0"
       },
       "dependencies": {
-        "get-stream": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz",
-          "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==",
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
           "dev": true,
           "requires": {
-            "pump": "^3.0.0"
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
           }
         },
-        "pump": {
+        "@types/istanbul-reports": {
           "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-          "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
           "dev": true,
           "requires": {
-            "end-of-stream": "^1.1.0",
-            "once": "^1.3.1"
+            "@types/istanbul-lib-report": "*"
           }
-        }
-      }
-    },
-    "@lerna/create": {
-      "version": "3.10.6",
-      "resolved": "https://registry.npmjs.org/@lerna/create/-/create-3.10.6.tgz",
-      "integrity": "sha512-OddQtGBHM2/eJONggLWoTE6275XGbnJ6dIVF+fLsKS93o4GC6g+qcc6Y7lUWHm5bfpeOwNOVKwj0tvqBZ6MgoA==",
-      "dev": true,
-      "requires": {
-        "@lerna/child-process": "3.3.0",
-        "@lerna/command": "3.10.6",
-        "@lerna/npm-conf": "3.7.0",
-        "@lerna/validation-error": "3.6.0",
-        "camelcase": "^4.1.0",
-        "dedent": "^0.7.0",
-        "fs-extra": "^7.0.0",
-        "globby": "^8.0.1",
-        "init-package-json": "^1.10.3",
-        "libnpm": "^2.0.1",
-        "p-reduce": "^1.0.0",
-        "pify": "^3.0.0",
-        "semver": "^5.5.0",
-        "slash": "^1.0.0",
-        "validate-npm-package-license": "^3.0.3",
-        "validate-npm-package-name": "^3.0.0",
-        "whatwg-url": "^7.0.0"
-      },
-      "dependencies": {
-        "camelcase": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz",
-          "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=",
-          "dev": true
         },
-        "globby": {
-          "version": "8.0.2",
-          "resolved": "https://registry.npmjs.org/globby/-/globby-8.0.2.tgz",
-          "integrity": "sha512-yTzMmKygLp8RUpG1Ymu2VXPSJQZjNAZPD4ywgYEaG7e4tBJeUQBO8OpXrf1RCNcEs5alsoJYPAMiIHP0cmeC7w==",
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
           "dev": true,
           "requires": {
-            "array-union": "^1.0.1",
-            "dir-glob": "2.0.0",
-            "fast-glob": "^2.0.2",
-            "glob": "^7.1.2",
-            "ignore": "^3.3.5",
-            "pify": "^3.0.0",
-            "slash": "^1.0.0"
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
           }
         },
-        "whatwg-url": {
-          "version": "7.0.0",
-          "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.0.0.tgz",
-          "integrity": "sha512-37GeVSIJ3kn1JgKyjiYNmSLP1yzbpb29jdmwBSgkD9h40/hyrR/OifpVUndji3tmwGgD8qpw7iQu3RSbCrBpsQ==",
+        "braces": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
+          "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
           "dev": true,
           "requires": {
-            "lodash.sortby": "^4.7.0",
-            "tr46": "^1.0.1",
-            "webidl-conversions": "^4.0.2"
+            "fill-range": "^7.0.1"
           }
-        }
-      }
-    },
-    "@lerna/create-symlink": {
-      "version": "3.6.0",
-      "resolved": "https://registry.npmjs.org/@lerna/create-symlink/-/create-symlink-3.6.0.tgz",
-      "integrity": "sha512-YG3lTb6zylvmGqKU+QYA3ylSnoLn+FyLH5XZmUsD0i85R884+EyJJeHx/zUk+yrL2ZwHS4RBUgJfC24fqzgPoA==",
-      "dev": true,
-      "requires": {
-        "cmd-shim": "^2.0.2",
-        "fs-extra": "^7.0.0",
-        "libnpm": "^2.0.1"
-      }
-    },
-    "@lerna/describe-ref": {
-      "version": "3.10.0",
-      "resolved": "https://registry.npmjs.org/@lerna/describe-ref/-/describe-ref-3.10.0.tgz",
-      "integrity": "sha512-fouh3FQS07QxJJp/mW8LkGnH0xMRAzpBlejtZaiRwfDkW2kd6EuHaj8I/2/p21Wsprcvuu4dqmyia2YS1xFb/w==",
-      "dev": true,
-      "requires": {
-        "@lerna/child-process": "3.3.0",
-        "libnpm": "^2.0.1"
-      }
-    },
-    "@lerna/diff": {
-      "version": "3.10.6",
-      "resolved": "https://registry.npmjs.org/@lerna/diff/-/diff-3.10.6.tgz",
-      "integrity": "sha512-0MqFhosjrqsIdXiKIu7t3CiJELqiU9mkjFBhYPB7JruAzpPwjMXJnC6/Ur5/7LXJYYVpqGQwZI9ZaZlOYJhhrw==",
-      "dev": true,
-      "requires": {
-        "@lerna/child-process": "3.3.0",
-        "@lerna/command": "3.10.6",
-        "@lerna/validation-error": "3.6.0",
-        "libnpm": "^2.0.1"
-      }
-    },
-    "@lerna/exec": {
-      "version": "3.10.6",
-      "resolved": "https://registry.npmjs.org/@lerna/exec/-/exec-3.10.6.tgz",
-      "integrity": "sha512-cdHqaRBMYceJu8rZLO8b4ZeR27O+xKPHgzi13OOOfBJQjrTuacjMWyHgmpy8jWc/0f7QnTl4VsHks7VJ3UK+vw==",
-      "dev": true,
-      "requires": {
-        "@lerna/batch-packages": "3.10.6",
-        "@lerna/child-process": "3.3.0",
-        "@lerna/command": "3.10.6",
-        "@lerna/filter-options": "3.10.6",
-        "@lerna/run-parallel-batches": "3.0.0",
-        "@lerna/validation-error": "3.6.0"
-      }
-    },
-    "@lerna/filter-options": {
-      "version": "3.10.6",
-      "resolved": "https://registry.npmjs.org/@lerna/filter-options/-/filter-options-3.10.6.tgz",
-      "integrity": "sha512-r/dQbqN+RGFKZNn+DyWehswFmAkny/fkdMB2sRM2YVe7zRTtSl95YxD9DtdYnpJTG/jbOVICS/L5QJakrI6SSw==",
-      "dev": true,
-      "requires": {
-        "@lerna/collect-updates": "3.10.1",
-        "@lerna/filter-packages": "3.10.0",
-        "dedent": "^0.7.0"
-      }
-    },
-    "@lerna/filter-packages": {
-      "version": "3.10.0",
-      "resolved": "https://registry.npmjs.org/@lerna/filter-packages/-/filter-packages-3.10.0.tgz",
-      "integrity": "sha512-3Acdj+jbany6LnQSuImU4ttcK5ULHSVug8Gh/EvwTewKCDpHAuoI3eyuzZOnSBdMvDOjE03uIESQK0dNNsn6Ow==",
-      "dev": true,
-      "requires": {
-        "@lerna/validation-error": "3.6.0",
-        "libnpm": "^2.0.1",
-        "multimatch": "^2.1.0"
-      }
-    },
-    "@lerna/get-npm-exec-opts": {
-      "version": "3.6.0",
-      "resolved": "https://registry.npmjs.org/@lerna/get-npm-exec-opts/-/get-npm-exec-opts-3.6.0.tgz",
-      "integrity": "sha512-ruH6KuLlt75aCObXfUIdVJqmfVq7sgWGq5mXa05vc1MEqxTIiU23YiJdWzofQOOUOACaZkzZ4K4Nu7wXEg4Xgg==",
-      "dev": true,
-      "requires": {
-        "libnpm": "^2.0.1"
+        },
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "fill-range": {
+          "version": "7.0.1",
+          "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
+          "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+          "dev": true,
+          "requires": {
+            "to-regex-range": "^5.0.1"
+          }
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "is-number": {
+          "version": "7.0.0",
+          "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
+          "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
+          "dev": true
+        },
+        "micromatch": {
+          "version": "4.0.2",
+          "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.2.tgz",
+          "integrity": "sha512-y7FpHSbMUMoyPbYUSzO6PaZ6FyRnQOpHuKwbo1G+Knck95XVU4QAiKdGEnj5wwoS7PlOgthX/09u5iFJ+aYf5Q==",
+          "dev": true,
+          "requires": {
+            "braces": "^3.0.1",
+            "picomatch": "^2.0.5"
+          }
+        },
+        "source-map": {
+          "version": "0.6.1",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
+        },
+        "to-regex-range": {
+          "version": "5.0.1",
+          "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
+          "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
+          "dev": true,
+          "requires": {
+            "is-number": "^7.0.0"
+          }
+        }
       }
     },
-    "@lerna/get-packed": {
-      "version": "3.7.0",
-      "resolved": "https://registry.npmjs.org/@lerna/get-packed/-/get-packed-3.7.0.tgz",
-      "integrity": "sha512-yuFtjsUZIHjeIvIYQ/QuytC+FQcHwo3peB+yGBST2uWCLUCR5rx6knoQcPzbxdFDCuUb5IFccFGd3B1fHFg3RQ==",
+    "@jest/types": {
+      "version": "25.5.0",
+      "resolved": "https://registry.npmjs.org/@jest/types/-/types-25.5.0.tgz",
+      "integrity": "sha512-OXD0RgQ86Tu3MazKo8bnrkDRaDXXMGUqd+kTtLtK1Zb7CRzQcaSRPPPV37SvYTdevXEBVxe0HXylEjs8ibkmCw==",
       "dev": true,
       "requires": {
-        "fs-extra": "^7.0.0",
-        "ssri": "^6.0.1",
-        "tar": "^4.4.8"
+        "@types/istanbul-lib-coverage": "^2.0.0",
+        "@types/istanbul-reports": "^1.1.1",
+        "@types/yargs": "^15.0.0",
+        "chalk": "^3.0.0"
       },
       "dependencies": {
-        "tar": {
-          "version": "4.4.8",
-          "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.8.tgz",
-          "integrity": "sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ==",
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
           "dev": true,
           "requires": {
-            "chownr": "^1.1.1",
-            "fs-minipass": "^1.2.5",
-            "minipass": "^2.3.4",
-            "minizlib": "^1.1.1",
-            "mkdirp": "^0.5.0",
-            "safe-buffer": "^5.1.2",
-            "yallist": "^3.0.2"
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
           }
         },
-        "yallist": {
-          "version": "3.0.3",
-          "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz",
-          "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==",
+        "chalk": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz",
+          "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
           "dev": true
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
         }
       }
     },
-    "@lerna/global-options": {
-      "version": "3.10.6",
-      "resolved": "https://registry.npmjs.org/@lerna/global-options/-/global-options-3.10.6.tgz",
-      "integrity": "sha512-k5Xkq1M/uREFC2R9uwN5gcvIgjj4iOXo0YyeEXCMWBiW3j2GL9xN4d1MmAIcrYlAzVYh6kLlWaFWl/rNIneHIw==",
-      "dev": true
-    },
-    "@lerna/has-npm-version": {
-      "version": "3.10.0",
-      "resolved": "https://registry.npmjs.org/@lerna/has-npm-version/-/has-npm-version-3.10.0.tgz",
-      "integrity": "sha512-N4RRYxGeivuaKgPDzrhkQOQs1Sg4tOnxnEe3akfqu1wDA4Ng5V6Y2uW3DbkAjFL3aNJhWF5Vbf7sBsGtfgDQ8w==",
-      "dev": true,
-      "requires": {
-        "@lerna/child-process": "3.3.0",
-        "semver": "^5.5.0"
-      }
-    },
-    "@lerna/import": {
-      "version": "3.10.6",
-      "resolved": "https://registry.npmjs.org/@lerna/import/-/import-3.10.6.tgz",
-      "integrity": "sha512-LlGxhfDhovoNoBJLF3PYd3j/G2GFTnfLh0V38+hBQ6lomMNJbjkACfiLVomQxPWWpYLk0GTlpWYR8YGv6L7Ifw==",
+    "@lerna/add": {
+      "version": "3.21.0",
+      "resolved": "https://registry.npmjs.org/@lerna/add/-/add-3.21.0.tgz",
+      "integrity": "sha512-vhUXXF6SpufBE1EkNEXwz1VLW03f177G9uMOFMQkp6OJ30/PWg4Ekifuz9/3YfgB2/GH8Tu4Lk3O51P2Hskg/A==",
       "dev": true,
       "requires": {
-        "@lerna/child-process": "3.3.0",
-        "@lerna/command": "3.10.6",
-        "@lerna/prompt": "3.6.0",
-        "@lerna/pulse-till-done": "3.7.1",
-        "@lerna/validation-error": "3.6.0",
+        "@evocateur/pacote": "^9.6.3",
+        "@lerna/bootstrap": "3.21.0",
+        "@lerna/command": "3.21.0",
+        "@lerna/filter-options": "3.20.0",
+        "@lerna/npm-conf": "3.16.0",
+        "@lerna/validation-error": "3.13.0",
         "dedent": "^0.7.0",
-        "fs-extra": "^7.0.0",
-        "p-map-series": "^1.0.0"
+        "npm-package-arg": "^6.1.0",
+        "p-map": "^2.1.0",
+        "semver": "^6.2.0"
+      },
+      "dependencies": {
+        "p-map": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz",
+          "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==",
+          "dev": true
+        },
+        "semver": {
+          "version": "6.3.0",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
+          "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
+          "dev": true
+        }
       }
     },
-    "@lerna/init": {
-      "version": "3.10.6",
-      "resolved": "https://registry.npmjs.org/@lerna/init/-/init-3.10.6.tgz",
-      "integrity": "sha512-RIlEx+ofWLYRNjxCkkV3G0XQPM+/KA5RXRDb5wKQLYO1f+tZAaHoUh8fHDIvxGf/ohY/OIjYYGSsU+ysimfwiQ==",
-      "dev": true,
-      "requires": {
-        "@lerna/child-process": "3.3.0",
-        "@lerna/command": "3.10.6",
-        "fs-extra": "^7.0.0",
-        "p-map": "^1.2.0",
-        "write-json-file": "^2.3.0"
+    "@lerna/bootstrap": {
+      "version": "3.21.0",
+      "resolved": "https://registry.npmjs.org/@lerna/bootstrap/-/bootstrap-3.21.0.tgz",
+      "integrity": "sha512-mtNHlXpmvJn6JTu0KcuTTPl2jLsDNud0QacV/h++qsaKbhAaJr/FElNZ5s7MwZFUM3XaDmvWzHKaszeBMHIbBw==",
+      "dev": true,
+      "requires": {
+        "@lerna/command": "3.21.0",
+        "@lerna/filter-options": "3.20.0",
+        "@lerna/has-npm-version": "3.16.5",
+        "@lerna/npm-install": "3.16.5",
+        "@lerna/package-graph": "3.18.5",
+        "@lerna/pulse-till-done": "3.13.0",
+        "@lerna/rimraf-dir": "3.16.5",
+        "@lerna/run-lifecycle": "3.16.2",
+        "@lerna/run-topologically": "3.18.5",
+        "@lerna/symlink-binary": "3.17.0",
+        "@lerna/symlink-dependencies": "3.17.0",
+        "@lerna/validation-error": "3.13.0",
+        "dedent": "^0.7.0",
+        "get-port": "^4.2.0",
+        "multimatch": "^3.0.0",
+        "npm-package-arg": "^6.1.0",
+        "npmlog": "^4.1.2",
+        "p-finally": "^1.0.0",
+        "p-map": "^2.1.0",
+        "p-map-series": "^1.0.0",
+        "p-waterfall": "^1.0.0",
+        "read-package-tree": "^5.1.6",
+        "semver": "^6.2.0"
+      },
+      "dependencies": {
+        "p-map": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz",
+          "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==",
+          "dev": true
+        },
+        "semver": {
+          "version": "6.3.0",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
+          "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
+          "dev": true
+        }
       }
     },
-    "@lerna/link": {
-      "version": "3.10.6",
-      "resolved": "https://registry.npmjs.org/@lerna/link/-/link-3.10.6.tgz",
-      "integrity": "sha512-dwD6qftRWitgLDYbqtDrgO7c8uF5C0fHVew5M6gU5m9tBJidqd7cDwHv/bXboLEI63U7tt5y6LY+wEpYUFsBRw==",
+    "@lerna/changed": {
+      "version": "3.21.0",
+      "resolved": "https://registry.npmjs.org/@lerna/changed/-/changed-3.21.0.tgz",
+      "integrity": "sha512-hzqoyf8MSHVjZp0gfJ7G8jaz+++mgXYiNs9iViQGA8JlN/dnWLI5sWDptEH3/B30Izo+fdVz0S0s7ydVE3pWIw==",
       "dev": true,
       "requires": {
-        "@lerna/command": "3.10.6",
-        "@lerna/package-graph": "3.10.6",
-        "@lerna/symlink-dependencies": "3.10.0",
-        "p-map": "^1.2.0",
-        "slash": "^1.0.0"
+        "@lerna/collect-updates": "3.20.0",
+        "@lerna/command": "3.21.0",
+        "@lerna/listable": "3.18.5",
+        "@lerna/output": "3.13.0"
       }
     },
-    "@lerna/list": {
-      "version": "3.10.6",
-      "resolved": "https://registry.npmjs.org/@lerna/list/-/list-3.10.6.tgz",
-      "integrity": "sha512-3ElQBj2dOB4uUkpsjC1bxdeZwEzRBuV1pBBs5E1LncwsZf7D9D99Z32fuZsDaCHpEMgHAD4/j8juI3/7m5dkaQ==",
+    "@lerna/check-working-tree": {
+      "version": "3.16.5",
+      "resolved": "https://registry.npmjs.org/@lerna/check-working-tree/-/check-working-tree-3.16.5.tgz",
+      "integrity": "sha512-xWjVBcuhvB8+UmCSb5tKVLB5OuzSpw96WEhS2uz6hkWVa/Euh1A0/HJwn2cemyK47wUrCQXtczBUiqnq9yX5VQ==",
       "dev": true,
       "requires": {
-        "@lerna/command": "3.10.6",
-        "@lerna/filter-options": "3.10.6",
-        "@lerna/listable": "3.10.6",
-        "@lerna/output": "3.6.0"
+        "@lerna/collect-uncommitted": "3.16.5",
+        "@lerna/describe-ref": "3.16.5",
+        "@lerna/validation-error": "3.13.0"
       }
     },
-    "@lerna/listable": {
-      "version": "3.10.6",
-      "resolved": "https://registry.npmjs.org/@lerna/listable/-/listable-3.10.6.tgz",
-      "integrity": "sha512-F7ZuvesSgeuMiJf99eOum5p1MQGQStykcmHH1ek+LQRMiGGF1o3PkBxPvHTZBADGOFarek8bFA5TVmRAMX7NIw==",
+    "@lerna/child-process": {
+      "version": "3.16.5",
+      "resolved": "https://registry.npmjs.org/@lerna/child-process/-/child-process-3.16.5.tgz",
+      "integrity": "sha512-vdcI7mzei9ERRV4oO8Y1LHBZ3A5+ampRKg1wq5nutLsUA4mEBN6H7JqjWOMY9xZemv6+kATm2ofjJ3lW5TszQg==",
       "dev": true,
       "requires": {
-        "@lerna/batch-packages": "3.10.6",
         "chalk": "^2.3.1",
-        "columnify": "^1.5.4"
-      }
-    },
-    "@lerna/log-packed": {
-      "version": "3.6.0",
-      "resolved": "https://registry.npmjs.org/@lerna/log-packed/-/log-packed-3.6.0.tgz",
-      "integrity": "sha512-T/J41zMkzpWB5nbiTRS5PmYTFn74mJXe6RQA2qhkdLi0UqnTp97Pux1loz3jsJf2yJtiQUnyMM7KuKIAge0Vlw==",
-      "dev": true,
-      "requires": {
-        "byte-size": "^4.0.3",
-        "columnify": "^1.5.4",
-        "has-unicode": "^2.0.1",
-        "libnpm": "^2.0.1"
-      }
-    },
-    "@lerna/npm-conf": {
-      "version": "3.7.0",
-      "resolved": "https://registry.npmjs.org/@lerna/npm-conf/-/npm-conf-3.7.0.tgz",
-      "integrity": "sha512-+WSMDfPKcKzMfqq283ydz9RRpOU6p9wfx0wy4hVSUY/6YUpsyuk8SShjcRtY8zTM5AOrxvFBuuV90H4YpZ5+Ng==",
-      "dev": true,
-      "requires": {
-        "config-chain": "^1.1.11",
-        "pify": "^3.0.0"
-      }
-    },
-    "@lerna/npm-dist-tag": {
-      "version": "3.8.5",
-      "resolved": "https://registry.npmjs.org/@lerna/npm-dist-tag/-/npm-dist-tag-3.8.5.tgz",
-      "integrity": "sha512-VO57yKTB4NC2LZuTd4w0LmlRpoFm/gejQ1gqqLGzSJuSZaBXmieElFovzl21S07cqiy7FNVdz75x7/a6WCZ6XA==",
-      "dev": true,
-      "requires": {
-        "figgy-pudding": "^3.5.1",
-        "libnpm": "^2.0.1"
-      }
-    },
-    "@lerna/npm-install": {
-      "version": "3.10.0",
-      "resolved": "https://registry.npmjs.org/@lerna/npm-install/-/npm-install-3.10.0.tgz",
-      "integrity": "sha512-/6/XyLY9/4jaMPBOVYUr4wZxQURIfwoELY0qCQ8gZ5zv4cOiFiiCUxZ0i4fxqFtD7nJ084zq1DsZW0aH0CIWYw==",
-      "dev": true,
-      "requires": {
-        "@lerna/child-process": "3.3.0",
-        "@lerna/get-npm-exec-opts": "3.6.0",
-        "fs-extra": "^7.0.0",
-        "libnpm": "^2.0.1",
-        "signal-exit": "^3.0.2",
-        "write-pkg": "^3.1.0"
-      }
-    },
-    "@lerna/npm-publish": {
-      "version": "3.10.7",
-      "resolved": "https://registry.npmjs.org/@lerna/npm-publish/-/npm-publish-3.10.7.tgz",
-      "integrity": "sha512-oU3/Q+eHC1fRjh7bk6Nn4tRD1OLR6XZVs3v+UWMWMrF4hVSV61pxcP5tpeI1n4gDQjSgh7seI4EzKVJe/WfraA==",
-      "dev": true,
-      "requires": {
-        "@lerna/run-lifecycle": "3.10.5",
-        "figgy-pudding": "^3.5.1",
-        "fs-extra": "^7.0.0",
-        "libnpm": "^2.0.1"
-      }
-    },
-    "@lerna/npm-run-script": {
-      "version": "3.10.0",
-      "resolved": "https://registry.npmjs.org/@lerna/npm-run-script/-/npm-run-script-3.10.0.tgz",
-      "integrity": "sha512-c21tBXLF1Wje4tx/Td9jKIMrlZo/8QQiyyadjdKpwyyo7orSMsVNXGyJwvZ4JVVDcwC3GPU6HQvkt63v7rcyaw==",
-      "dev": true,
-      "requires": {
-        "@lerna/child-process": "3.3.0",
-        "@lerna/get-npm-exec-opts": "3.6.0",
-        "libnpm": "^2.0.1"
-      }
-    },
-    "@lerna/output": {
-      "version": "3.6.0",
-      "resolved": "https://registry.npmjs.org/@lerna/output/-/output-3.6.0.tgz",
-      "integrity": "sha512-9sjQouf6p7VQtVCRnzoTGlZyURd48i3ha3WBHC/UBJnHZFuXMqWVPKNuvnMf2kRXDyoQD+2mNywpmEJg5jOnRg==",
-      "dev": true,
-      "requires": {
-        "libnpm": "^2.0.1"
+        "execa": "^1.0.0",
+        "strong-log-transformer": "^2.0.0"
       }
     },
-    "@lerna/pack-directory": {
-      "version": "3.10.5",
-      "resolved": "https://registry.npmjs.org/@lerna/pack-directory/-/pack-directory-3.10.5.tgz",
-      "integrity": "sha512-Ulj24L9XdgjJIxBr6ZjRJEoBULVH3c10lqunUdW41bswXhzhirRtQIxv0+5shngNjDwgMmJfOBcuCVKPSez4tg==",
+    "@lerna/clean": {
+      "version": "3.21.0",
+      "resolved": "https://registry.npmjs.org/@lerna/clean/-/clean-3.21.0.tgz",
+      "integrity": "sha512-b/L9l+MDgE/7oGbrav6rG8RTQvRiZLO1zTcG17zgJAAuhlsPxJExMlh2DFwJEVi2les70vMhHfST3Ue1IMMjpg==",
       "dev": true,
       "requires": {
-        "@lerna/get-packed": "3.7.0",
-        "@lerna/package": "3.7.2",
-        "@lerna/run-lifecycle": "3.10.5",
-        "figgy-pudding": "^3.5.1",
-        "libnpm": "^2.0.1",
-        "npm-packlist": "^1.1.12",
-        "tar": "^4.4.8",
-        "temp-write": "^3.4.0"
+        "@lerna/command": "3.21.0",
+        "@lerna/filter-options": "3.20.0",
+        "@lerna/prompt": "3.18.5",
+        "@lerna/pulse-till-done": "3.13.0",
+        "@lerna/rimraf-dir": "3.16.5",
+        "p-map": "^2.1.0",
+        "p-map-series": "^1.0.0",
+        "p-waterfall": "^1.0.0"
       },
       "dependencies": {
-        "tar": {
-          "version": "4.4.8",
-          "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.8.tgz",
-          "integrity": "sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ==",
-          "dev": true,
-          "requires": {
-            "chownr": "^1.1.1",
-            "fs-minipass": "^1.2.5",
-            "minipass": "^2.3.4",
-            "minizlib": "^1.1.1",
-            "mkdirp": "^0.5.0",
-            "safe-buffer": "^5.1.2",
-            "yallist": "^3.0.2"
-          }
-        },
-        "yallist": {
-          "version": "3.0.3",
-          "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz",
-          "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==",
+        "p-map": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz",
+          "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==",
           "dev": true
         }
       }
     },
-    "@lerna/package": {
-      "version": "3.7.2",
-      "resolved": "https://registry.npmjs.org/@lerna/package/-/package-3.7.2.tgz",
-      "integrity": "sha512-8A5hN2CekM1a0Ix4VUO/g+REo+MsnXb8lnQ0bGjr1YGWzSL5NxYJ0Z9+0pwTfDpvRDYlFYO0rMVwBUW44b4dUw==",
+    "@lerna/cli": {
+      "version": "3.18.5",
+      "resolved": "https://registry.npmjs.org/@lerna/cli/-/cli-3.18.5.tgz",
+      "integrity": "sha512-erkbxkj9jfc89vVs/jBLY/fM0I80oLmJkFUV3Q3wk9J3miYhP14zgVEBsPZY68IZlEjT6T3Xlq2xO1AVaatHsA==",
       "dev": true,
       "requires": {
-        "libnpm": "^2.0.1",
-        "load-json-file": "^4.0.0",
-        "write-pkg": "^3.1.0"
+        "@lerna/global-options": "3.13.0",
+        "dedent": "^0.7.0",
+        "npmlog": "^4.1.2",
+        "yargs": "^14.2.2"
       },
       "dependencies": {
-        "load-json-file": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz",
-          "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=",
+        "ansi-regex": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz",
+          "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==",
+          "dev": true
+        },
+        "camelcase": {
+          "version": "5.3.1",
+          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
+          "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==",
+          "dev": true
+        },
+        "cliui": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz",
+          "integrity": "sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==",
           "dev": true,
           "requires": {
-            "graceful-fs": "^4.1.2",
-            "parse-json": "^4.0.0",
-            "pify": "^3.0.0",
-            "strip-bom": "^3.0.0"
+            "string-width": "^3.1.0",
+            "strip-ansi": "^5.2.0",
+            "wrap-ansi": "^5.1.0"
           }
         },
-        "parse-json": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
-          "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=",
+        "emoji-regex": {
+          "version": "7.0.3",
+          "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz",
+          "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==",
+          "dev": true
+        },
+        "find-up": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz",
+          "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==",
           "dev": true,
           "requires": {
-            "error-ex": "^1.3.1",
-            "json-parse-better-errors": "^1.0.1"
+            "locate-path": "^3.0.0"
           }
         },
-        "strip-bom": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
-          "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=",
+        "get-caller-file": {
+          "version": "2.0.5",
+          "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
+          "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
           "dev": true
+        },
+        "is-fullwidth-code-point": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
+          "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=",
+          "dev": true
+        },
+        "locate-path": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz",
+          "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==",
+          "dev": true,
+          "requires": {
+            "p-locate": "^3.0.0",
+            "path-exists": "^3.0.0"
+          }
+        },
+        "p-locate": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz",
+          "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==",
+          "dev": true,
+          "requires": {
+            "p-limit": "^2.0.0"
+          }
+        },
+        "path-exists": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
+          "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=",
+          "dev": true
+        },
+        "require-main-filename": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz",
+          "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==",
+          "dev": true
+        },
+        "string-width": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz",
+          "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==",
+          "dev": true,
+          "requires": {
+            "emoji-regex": "^7.0.1",
+            "is-fullwidth-code-point": "^2.0.0",
+            "strip-ansi": "^5.1.0"
+          }
+        },
+        "strip-ansi": {
+          "version": "5.2.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz",
+          "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==",
+          "dev": true,
+          "requires": {
+            "ansi-regex": "^4.1.0"
+          }
+        },
+        "which-module": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz",
+          "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=",
+          "dev": true
+        },
+        "wrap-ansi": {
+          "version": "5.1.0",
+          "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz",
+          "integrity": "sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^3.2.0",
+            "string-width": "^3.0.0",
+            "strip-ansi": "^5.0.0"
+          }
+        },
+        "y18n": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
+          "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
+          "dev": true
+        },
+        "yargs": {
+          "version": "14.2.3",
+          "resolved": "https://registry.npmjs.org/yargs/-/yargs-14.2.3.tgz",
+          "integrity": "sha512-ZbotRWhF+lkjijC/VhmOT9wSgyBQ7+zr13+YLkhfsSiTriYsMzkTUFP18pFhWwBeMa5gUc1MzbhrO6/VB7c9Xg==",
+          "dev": true,
+          "requires": {
+            "cliui": "^5.0.0",
+            "decamelize": "^1.2.0",
+            "find-up": "^3.0.0",
+            "get-caller-file": "^2.0.1",
+            "require-directory": "^2.1.1",
+            "require-main-filename": "^2.0.0",
+            "set-blocking": "^2.0.0",
+            "string-width": "^3.0.0",
+            "which-module": "^2.0.0",
+            "y18n": "^4.0.0",
+            "yargs-parser": "^15.0.1"
+          }
+        },
+        "yargs-parser": {
+          "version": "15.0.1",
+          "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-15.0.1.tgz",
+          "integrity": "sha512-0OAMV2mAZQrs3FkNpDQcBk1x5HXb8X4twADss4S0Iuk+2dGnLOE/fRHrsYm542GduMveyA77OF4wrNJuanRCWw==",
+          "dev": true,
+          "requires": {
+            "camelcase": "^5.0.0",
+            "decamelize": "^1.2.0"
+          }
         }
       }
     },
-    "@lerna/package-graph": {
-      "version": "3.10.6",
-      "resolved": "https://registry.npmjs.org/@lerna/package-graph/-/package-graph-3.10.6.tgz",
-      "integrity": "sha512-mpIOJbhi+xLqT9BcUrLVD4We8WUdousQf/QndbEWl8DWAW1ethtRHVsCm9ufdBB3F9nj4PH/hqnDWWwqE+rS4w==",
+    "@lerna/collect-uncommitted": {
+      "version": "3.16.5",
+      "resolved": "https://registry.npmjs.org/@lerna/collect-uncommitted/-/collect-uncommitted-3.16.5.tgz",
+      "integrity": "sha512-ZgqnGwpDZiWyzIQVZtQaj9tRizsL4dUOhuOStWgTAw1EMe47cvAY2kL709DzxFhjr6JpJSjXV5rZEAeU3VE0Hg==",
       "dev": true,
       "requires": {
-        "@lerna/validation-error": "3.6.0",
-        "libnpm": "^2.0.1",
-        "semver": "^5.5.0"
+        "@lerna/child-process": "3.16.5",
+        "chalk": "^2.3.1",
+        "figgy-pudding": "^3.5.1",
+        "npmlog": "^4.1.2"
       }
     },
-    "@lerna/project": {
-      "version": "3.10.0",
-      "resolved": "https://registry.npmjs.org/@lerna/project/-/project-3.10.0.tgz",
-      "integrity": "sha512-9QRl8aGHuyU4zVEELQmNPnJTlS7XHqX7w9I9isCXdnilKc2R0MyvUs21lj6Yyt6xTuQnqD158TR9tbS4QufYQQ==",
+    "@lerna/collect-updates": {
+      "version": "3.20.0",
+      "resolved": "https://registry.npmjs.org/@lerna/collect-updates/-/collect-updates-3.20.0.tgz",
+      "integrity": "sha512-qBTVT5g4fupVhBFuY4nI/3FSJtQVcDh7/gEPOpRxoXB/yCSnT38MFHXWl+y4einLciCjt/+0x6/4AG80fjay2Q==",
+      "dev": true,
+      "requires": {
+        "@lerna/child-process": "3.16.5",
+        "@lerna/describe-ref": "3.16.5",
+        "minimatch": "^3.0.4",
+        "npmlog": "^4.1.2",
+        "slash": "^2.0.0"
+      },
+      "dependencies": {
+        "slash": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
+          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
+          "dev": true
+        }
+      }
+    },
+    "@lerna/command": {
+      "version": "3.21.0",
+      "resolved": "https://registry.npmjs.org/@lerna/command/-/command-3.21.0.tgz",
+      "integrity": "sha512-T2bu6R8R3KkH5YoCKdutKv123iUgUbW8efVjdGCDnCMthAQzoentOJfDeodBwn0P2OqCl3ohsiNVtSn9h78fyQ==",
       "dev": true,
       "requires": {
-        "@lerna/package": "3.7.2",
-        "@lerna/validation-error": "3.6.0",
-        "cosmiconfig": "^5.0.2",
+        "@lerna/child-process": "3.16.5",
+        "@lerna/package-graph": "3.18.5",
+        "@lerna/project": "3.21.0",
+        "@lerna/validation-error": "3.13.0",
+        "@lerna/write-log-file": "3.13.0",
+        "clone-deep": "^4.0.1",
         "dedent": "^0.7.0",
-        "dot-prop": "^4.2.0",
-        "glob-parent": "^3.1.0",
-        "globby": "^8.0.1",
-        "libnpm": "^2.0.1",
-        "load-json-file": "^4.0.0",
-        "p-map": "^1.2.0",
-        "resolve-from": "^4.0.0",
-        "write-json-file": "^2.3.0"
+        "execa": "^1.0.0",
+        "is-ci": "^2.0.0",
+        "npmlog": "^4.1.2"
+      }
+    },
+    "@lerna/conventional-commits": {
+      "version": "3.22.0",
+      "resolved": "https://registry.npmjs.org/@lerna/conventional-commits/-/conventional-commits-3.22.0.tgz",
+      "integrity": "sha512-z4ZZk1e8Mhz7+IS8NxHr64wyklHctCJyWpJKEZZPJiLFJ8yKto/x38O80R10pIzC0rr8Sy/OsjSH4bl0TbbgqA==",
+      "dev": true,
+      "requires": {
+        "@lerna/validation-error": "3.13.0",
+        "conventional-changelog-angular": "^5.0.3",
+        "conventional-changelog-core": "^3.1.6",
+        "conventional-recommended-bump": "^5.0.0",
+        "fs-extra": "^8.1.0",
+        "get-stream": "^4.0.0",
+        "lodash.template": "^4.5.0",
+        "npm-package-arg": "^6.1.0",
+        "npmlog": "^4.1.2",
+        "pify": "^4.0.1",
+        "semver": "^6.2.0"
+      },
+      "dependencies": {
+        "pify": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz",
+          "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==",
+          "dev": true
+        },
+        "semver": {
+          "version": "6.3.0",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
+          "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
+          "dev": true
+        }
+      }
+    },
+    "@lerna/create": {
+      "version": "3.22.0",
+      "resolved": "https://registry.npmjs.org/@lerna/create/-/create-3.22.0.tgz",
+      "integrity": "sha512-MdiQQzCcB4E9fBF1TyMOaAEz9lUjIHp1Ju9H7f3lXze5JK6Fl5NYkouAvsLgY6YSIhXMY8AHW2zzXeBDY4yWkw==",
+      "dev": true,
+      "requires": {
+        "@evocateur/pacote": "^9.6.3",
+        "@lerna/child-process": "3.16.5",
+        "@lerna/command": "3.21.0",
+        "@lerna/npm-conf": "3.16.0",
+        "@lerna/validation-error": "3.13.0",
+        "camelcase": "^5.0.0",
+        "dedent": "^0.7.0",
+        "fs-extra": "^8.1.0",
+        "globby": "^9.2.0",
+        "init-package-json": "^1.10.3",
+        "npm-package-arg": "^6.1.0",
+        "p-reduce": "^1.0.0",
+        "pify": "^4.0.1",
+        "semver": "^6.2.0",
+        "slash": "^2.0.0",
+        "validate-npm-package-license": "^3.0.3",
+        "validate-npm-package-name": "^3.0.0",
+        "whatwg-url": "^7.0.0"
       },
       "dependencies": {
+        "camelcase": {
+          "version": "5.3.1",
+          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
+          "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==",
+          "dev": true
+        },
         "globby": {
-          "version": "8.0.2",
-          "resolved": "https://registry.npmjs.org/globby/-/globby-8.0.2.tgz",
-          "integrity": "sha512-yTzMmKygLp8RUpG1Ymu2VXPSJQZjNAZPD4ywgYEaG7e4tBJeUQBO8OpXrf1RCNcEs5alsoJYPAMiIHP0cmeC7w==",
+          "version": "9.2.0",
+          "resolved": "https://registry.npmjs.org/globby/-/globby-9.2.0.tgz",
+          "integrity": "sha512-ollPHROa5mcxDEkwg6bPt3QbEf4pDQSNtd6JPL1YvOvAo/7/0VAm9TccUeoTmarjPw4pfUthSCqcyfNB1I3ZSg==",
           "dev": true,
           "requires": {
-            "array-union": "^1.0.1",
-            "dir-glob": "2.0.0",
-            "fast-glob": "^2.0.2",
-            "glob": "^7.1.2",
-            "ignore": "^3.3.5",
-            "pify": "^3.0.0",
-            "slash": "^1.0.0"
+            "@types/glob": "^7.1.1",
+            "array-union": "^1.0.2",
+            "dir-glob": "^2.2.2",
+            "fast-glob": "^2.2.6",
+            "glob": "^7.1.3",
+            "ignore": "^4.0.3",
+            "pify": "^4.0.1",
+            "slash": "^2.0.0"
           }
         },
-        "load-json-file": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz",
-          "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=",
-          "dev": true,
-          "requires": {
-            "graceful-fs": "^4.1.2",
-            "parse-json": "^4.0.0",
-            "pify": "^3.0.0",
-            "strip-bom": "^3.0.0"
-          }
+        "pify": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz",
+          "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==",
+          "dev": true
         },
-        "parse-json": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
-          "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=",
+        "semver": {
+          "version": "6.3.0",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
+          "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
+          "dev": true
+        },
+        "slash": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
+          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
+          "dev": true
+        },
+        "tr46": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz",
+          "integrity": "sha1-qLE/1r/SSJUZZ0zN5VujaTtwbQk=",
           "dev": true,
           "requires": {
-            "error-ex": "^1.3.1",
-            "json-parse-better-errors": "^1.0.1"
+            "punycode": "^2.1.0"
           }
         },
-        "resolve-from": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
-          "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
+        "webidl-conversions": {
+          "version": "4.0.2",
+          "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz",
+          "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==",
           "dev": true
         },
-        "strip-bom": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
-          "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=",
-          "dev": true
+        "whatwg-url": {
+          "version": "7.1.0",
+          "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz",
+          "integrity": "sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==",
+          "dev": true,
+          "requires": {
+            "lodash.sortby": "^4.7.0",
+            "tr46": "^1.0.1",
+            "webidl-conversions": "^4.0.2"
+          }
         }
       }
     },
-    "@lerna/prompt": {
-      "version": "3.6.0",
-      "resolved": "https://registry.npmjs.org/@lerna/prompt/-/prompt-3.6.0.tgz",
-      "integrity": "sha512-nyAjPMolJ/ZRAAVcXrUH89C4n1SiWvLh4xWNvWYKLcf3PI5yges35sDFP/HYrM4+cEbkNFuJCRq6CxaET4PRsg==",
+    "@lerna/create-symlink": {
+      "version": "3.16.2",
+      "resolved": "https://registry.npmjs.org/@lerna/create-symlink/-/create-symlink-3.16.2.tgz",
+      "integrity": "sha512-pzXIJp6av15P325sgiIRpsPXLFmkisLhMBCy4764d+7yjf2bzrJ4gkWVMhsv4AdF0NN3OyZ5jjzzTtLNqfR+Jw==",
       "dev": true,
       "requires": {
-        "inquirer": "^6.2.0",
-        "libnpm": "^2.0.1"
-      }
-    },
-    "@lerna/publish": {
-      "version": "3.10.7",
-      "resolved": "https://registry.npmjs.org/@lerna/publish/-/publish-3.10.7.tgz",
-      "integrity": "sha512-Qd8pml2l9s6GIvNX1pTnia+Ddjsm9LF3pRRoOQeugAdv2IJNf45c/83AAEyE9M2ShG5VjgxEITNW4Lg49zipjQ==",
-      "dev": true,
-      "requires": {
-        "@lerna/batch-packages": "3.10.6",
-        "@lerna/check-working-tree": "3.10.0",
-        "@lerna/child-process": "3.3.0",
-        "@lerna/collect-updates": "3.10.1",
-        "@lerna/command": "3.10.6",
-        "@lerna/describe-ref": "3.10.0",
-        "@lerna/log-packed": "3.6.0",
-        "@lerna/npm-conf": "3.7.0",
-        "@lerna/npm-dist-tag": "3.8.5",
-        "@lerna/npm-publish": "3.10.7",
-        "@lerna/output": "3.6.0",
-        "@lerna/pack-directory": "3.10.5",
-        "@lerna/prompt": "3.6.0",
-        "@lerna/pulse-till-done": "3.7.1",
-        "@lerna/run-lifecycle": "3.10.5",
-        "@lerna/run-parallel-batches": "3.0.0",
-        "@lerna/validation-error": "3.6.0",
-        "@lerna/version": "3.10.6",
-        "figgy-pudding": "^3.5.1",
-        "fs-extra": "^7.0.0",
-        "libnpm": "^2.0.1",
-        "p-finally": "^1.0.0",
-        "p-map": "^1.2.0",
-        "p-pipe": "^1.2.0",
-        "p-reduce": "^1.0.0",
-        "semver": "^5.5.0"
+        "@zkochan/cmd-shim": "^3.1.0",
+        "fs-extra": "^8.1.0",
+        "npmlog": "^4.1.2"
       }
     },
-    "@lerna/pulse-till-done": {
-      "version": "3.7.1",
-      "resolved": "https://registry.npmjs.org/@lerna/pulse-till-done/-/pulse-till-done-3.7.1.tgz",
-      "integrity": "sha512-MzpesZeW3Mc+CiAq4zUt9qTXI9uEBBKrubYHE36voQTSkHvu/Rox6YOvfUr+U7P6k8frFPeCgGpfMDTLhiqe6w==",
+    "@lerna/describe-ref": {
+      "version": "3.16.5",
+      "resolved": "https://registry.npmjs.org/@lerna/describe-ref/-/describe-ref-3.16.5.tgz",
+      "integrity": "sha512-c01+4gUF0saOOtDBzbLMFOTJDHTKbDFNErEY6q6i9QaXuzy9LNN62z+Hw4acAAZuJQhrVWncVathcmkkjvSVGw==",
       "dev": true,
       "requires": {
-        "libnpm": "^2.0.1"
+        "@lerna/child-process": "3.16.5",
+        "npmlog": "^4.1.2"
       }
     },
-    "@lerna/resolve-symlink": {
-      "version": "3.6.0",
-      "resolved": "https://registry.npmjs.org/@lerna/resolve-symlink/-/resolve-symlink-3.6.0.tgz",
-      "integrity": "sha512-TVOAEqHJSQVhNDMFCwEUZPaOETqHDQV1TQWQfC8ZlOqyaUQ7veZUbg0yfG7RPNzlSpvF0ZaGFeR0YhYDAW03GA==",
+    "@lerna/diff": {
+      "version": "3.21.0",
+      "resolved": "https://registry.npmjs.org/@lerna/diff/-/diff-3.21.0.tgz",
+      "integrity": "sha512-5viTR33QV3S7O+bjruo1SaR40m7F2aUHJaDAC7fL9Ca6xji+aw1KFkpCtVlISS0G8vikUREGMJh+c/VMSc8Usw==",
       "dev": true,
       "requires": {
-        "fs-extra": "^7.0.0",
-        "libnpm": "^2.0.1",
-        "read-cmd-shim": "^1.0.1"
+        "@lerna/child-process": "3.16.5",
+        "@lerna/command": "3.21.0",
+        "@lerna/validation-error": "3.13.0",
+        "npmlog": "^4.1.2"
       }
     },
-    "@lerna/rimraf-dir": {
-      "version": "3.10.0",
-      "resolved": "https://registry.npmjs.org/@lerna/rimraf-dir/-/rimraf-dir-3.10.0.tgz",
-      "integrity": "sha512-RSKSfxPURc58ERCD/PuzorR86lWEvIWNclXYGvIYM76yNGrWiDF44pGHQvB4J+Lxa5M+52ZtZC/eOC7A7YCH4g==",
-      "dev": true,
-      "requires": {
-        "@lerna/child-process": "3.3.0",
-        "libnpm": "^2.0.1",
-        "path-exists": "^3.0.0",
-        "rimraf": "^2.6.2"
+    "@lerna/exec": {
+      "version": "3.21.0",
+      "resolved": "https://registry.npmjs.org/@lerna/exec/-/exec-3.21.0.tgz",
+      "integrity": "sha512-iLvDBrIE6rpdd4GIKTY9mkXyhwsJ2RvQdB9ZU+/NhR3okXfqKc6py/24tV111jqpXTtZUW6HNydT4dMao2hi1Q==",
+      "dev": true,
+      "requires": {
+        "@lerna/child-process": "3.16.5",
+        "@lerna/command": "3.21.0",
+        "@lerna/filter-options": "3.20.0",
+        "@lerna/profiler": "3.20.0",
+        "@lerna/run-topologically": "3.18.5",
+        "@lerna/validation-error": "3.13.0",
+        "p-map": "^2.1.0"
       },
       "dependencies": {
-        "path-exists": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
-          "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=",
+        "p-map": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz",
+          "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==",
           "dev": true
         }
       }
     },
-    "@lerna/run": {
-      "version": "3.10.6",
-      "resolved": "https://registry.npmjs.org/@lerna/run/-/run-3.10.6.tgz",
-      "integrity": "sha512-KS2lWbu/8WUUscQPi9U8sPO6yYpzf/0GmODjpruR1nRi1u/tuncdjTiG+hjGAeFC1BD7YktT9Za6imIpE8RXmA==",
+    "@lerna/filter-options": {
+      "version": "3.20.0",
+      "resolved": "https://registry.npmjs.org/@lerna/filter-options/-/filter-options-3.20.0.tgz",
+      "integrity": "sha512-bmcHtvxn7SIl/R9gpiNMVG7yjx7WyT0HSGw34YVZ9B+3xF/83N3r5Rgtjh4hheLZ+Q91Or0Jyu5O3Nr+AwZe2g==",
       "dev": true,
       "requires": {
-        "@lerna/batch-packages": "3.10.6",
-        "@lerna/command": "3.10.6",
-        "@lerna/filter-options": "3.10.6",
-        "@lerna/npm-run-script": "3.10.0",
-        "@lerna/output": "3.6.0",
-        "@lerna/run-parallel-batches": "3.0.0",
-        "@lerna/timer": "3.5.0",
-        "@lerna/validation-error": "3.6.0",
-        "p-map": "^1.2.0"
+        "@lerna/collect-updates": "3.20.0",
+        "@lerna/filter-packages": "3.18.0",
+        "dedent": "^0.7.0",
+        "figgy-pudding": "^3.5.1",
+        "npmlog": "^4.1.2"
       }
     },
-    "@lerna/run-lifecycle": {
-      "version": "3.10.5",
-      "resolved": "https://registry.npmjs.org/@lerna/run-lifecycle/-/run-lifecycle-3.10.5.tgz",
-      "integrity": "sha512-YPmXviaxVlhcKM6IkDTIpTq24mxOuMCilo+MTr1RLoafgB9ZTmP2AHRiFt/sy14wOsq2Zqr0wJyj8KFlDYLTkA==",
+    "@lerna/filter-packages": {
+      "version": "3.18.0",
+      "resolved": "https://registry.npmjs.org/@lerna/filter-packages/-/filter-packages-3.18.0.tgz",
+      "integrity": "sha512-6/0pMM04bCHNATIOkouuYmPg6KH3VkPCIgTfQmdkPJTullERyEQfNUKikrefjxo1vHOoCACDpy65JYyKiAbdwQ==",
       "dev": true,
       "requires": {
-        "@lerna/npm-conf": "3.7.0",
-        "figgy-pudding": "^3.5.1",
-        "libnpm": "^2.0.1"
+        "@lerna/validation-error": "3.13.0",
+        "multimatch": "^3.0.0",
+        "npmlog": "^4.1.2"
       }
     },
-    "@lerna/run-parallel-batches": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/@lerna/run-parallel-batches/-/run-parallel-batches-3.0.0.tgz",
-      "integrity": "sha512-Mj1ravlXF7AkkewKd9YFq9BtVrsStNrvVLedD/b2wIVbNqcxp8lS68vehXVOzoL/VWNEDotvqCQtyDBilCodGw==",
+    "@lerna/get-npm-exec-opts": {
+      "version": "3.13.0",
+      "resolved": "https://registry.npmjs.org/@lerna/get-npm-exec-opts/-/get-npm-exec-opts-3.13.0.tgz",
+      "integrity": "sha512-Y0xWL0rg3boVyJk6An/vurKzubyJKtrxYv2sj4bB8Mc5zZ3tqtv0ccbOkmkXKqbzvNNF7VeUt1OJ3DRgtC/QZw==",
       "dev": true,
       "requires": {
-        "p-map": "^1.2.0",
-        "p-map-series": "^1.0.0"
+        "npmlog": "^4.1.2"
       }
     },
-    "@lerna/symlink-binary": {
-      "version": "3.10.0",
-      "resolved": "https://registry.npmjs.org/@lerna/symlink-binary/-/symlink-binary-3.10.0.tgz",
-      "integrity": "sha512-6mQsG+iVjBo8cD8s24O+YgFrwDyUGfUQbK4ryalAXFHI817Zd4xlI3tjg3W99whCt6rt6D0s1fpf8eslMN6dSw==",
+    "@lerna/get-packed": {
+      "version": "3.16.0",
+      "resolved": "https://registry.npmjs.org/@lerna/get-packed/-/get-packed-3.16.0.tgz",
+      "integrity": "sha512-AjsFiaJzo1GCPnJUJZiTW6J1EihrPkc2y3nMu6m3uWFxoleklsSCyImumzVZJssxMi3CPpztj8LmADLedl9kXw==",
       "dev": true,
       "requires": {
-        "@lerna/create-symlink": "3.6.0",
-        "@lerna/package": "3.7.2",
-        "fs-extra": "^7.0.0",
-        "p-map": "^1.2.0"
+        "fs-extra": "^8.1.0",
+        "ssri": "^6.0.1",
+        "tar": "^4.4.8"
       }
     },
-    "@lerna/symlink-dependencies": {
-      "version": "3.10.0",
-      "resolved": "https://registry.npmjs.org/@lerna/symlink-dependencies/-/symlink-dependencies-3.10.0.tgz",
-      "integrity": "sha512-vGpg5ydwGgQCuWNX5y7CRL38mGpuLhf1GRq9wMm7IGwnctEsdSNqvvE+LDgqtwEZASu5+vffYUkL0VlFXl8uWA==",
+    "@lerna/github-client": {
+      "version": "3.22.0",
+      "resolved": "https://registry.npmjs.org/@lerna/github-client/-/github-client-3.22.0.tgz",
+      "integrity": "sha512-O/GwPW+Gzr3Eb5bk+nTzTJ3uv+jh5jGho9BOqKlajXaOkMYGBELEAqV5+uARNGWZFvYAiF4PgqHb6aCUu7XdXg==",
       "dev": true,
       "requires": {
-        "@lerna/create-symlink": "3.6.0",
-        "@lerna/resolve-symlink": "3.6.0",
-        "@lerna/symlink-binary": "3.10.0",
-        "fs-extra": "^7.0.0",
-        "p-finally": "^1.0.0",
-        "p-map": "^1.2.0",
-        "p-map-series": "^1.0.0"
+        "@lerna/child-process": "3.16.5",
+        "@octokit/plugin-enterprise-rest": "^6.0.1",
+        "@octokit/rest": "^16.28.4",
+        "git-url-parse": "^11.1.2",
+        "npmlog": "^4.1.2"
       }
     },
-    "@lerna/timer": {
-      "version": "3.5.0",
-      "resolved": "https://registry.npmjs.org/@lerna/timer/-/timer-3.5.0.tgz",
-      "integrity": "sha512-TAb99hqQN6E3JBGtG9iyZNPq1/DbmqgBOeNrKtdJsGvIeX/NGLgUDWMrj2h04V4O+jpBFmSf6HIld6triKmxCA==",
-      "dev": true
+    "@lerna/gitlab-client": {
+      "version": "3.15.0",
+      "resolved": "https://registry.npmjs.org/@lerna/gitlab-client/-/gitlab-client-3.15.0.tgz",
+      "integrity": "sha512-OsBvRSejHXUBMgwWQqNoioB8sgzL/Pf1pOUhHKtkiMl6aAWjklaaq5HPMvTIsZPfS6DJ9L5OK2GGZuooP/5c8Q==",
+      "dev": true,
+      "requires": {
+        "node-fetch": "^2.5.0",
+        "npmlog": "^4.1.2",
+        "whatwg-url": "^7.0.0"
+      },
+      "dependencies": {
+        "tr46": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz",
+          "integrity": "sha1-qLE/1r/SSJUZZ0zN5VujaTtwbQk=",
+          "dev": true,
+          "requires": {
+            "punycode": "^2.1.0"
+          }
+        },
+        "webidl-conversions": {
+          "version": "4.0.2",
+          "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz",
+          "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==",
+          "dev": true
+        },
+        "whatwg-url": {
+          "version": "7.1.0",
+          "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz",
+          "integrity": "sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==",
+          "dev": true,
+          "requires": {
+            "lodash.sortby": "^4.7.0",
+            "tr46": "^1.0.1",
+            "webidl-conversions": "^4.0.2"
+          }
+        }
+      }
     },
-    "@lerna/validation-error": {
-      "version": "3.6.0",
-      "resolved": "https://registry.npmjs.org/@lerna/validation-error/-/validation-error-3.6.0.tgz",
-      "integrity": "sha512-MWltncGO5VgMS0QedTlZCjFUMF/evRjDMMHrtVorkIB2Cp5xy0rkKa8iDBG43qpUWeG1giwi58yUlETBcWfILw==",
+    "@lerna/global-options": {
+      "version": "3.13.0",
+      "resolved": "https://registry.npmjs.org/@lerna/global-options/-/global-options-3.13.0.tgz",
+      "integrity": "sha512-SlZvh1gVRRzYLVluz9fryY1nJpZ0FHDGB66U9tFfvnnxmueckRQxLopn3tXj3NU1kc3QANT2I5BsQkOqZ4TEFQ==",
+      "dev": true
+    },
+    "@lerna/has-npm-version": {
+      "version": "3.16.5",
+      "resolved": "https://registry.npmjs.org/@lerna/has-npm-version/-/has-npm-version-3.16.5.tgz",
+      "integrity": "sha512-WL7LycR9bkftyqbYop5rEGJ9sRFIV55tSGmbN1HLrF9idwOCD7CLrT64t235t3t4O5gehDnwKI5h2U3oxTrF8Q==",
       "dev": true,
       "requires": {
-        "libnpm": "^2.0.1"
+        "@lerna/child-process": "3.16.5",
+        "semver": "^6.2.0"
+      },
+      "dependencies": {
+        "semver": {
+          "version": "6.3.0",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
+          "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
+          "dev": true
+        }
       }
     },
-    "@lerna/version": {
-      "version": "3.10.6",
-      "resolved": "https://registry.npmjs.org/@lerna/version/-/version-3.10.6.tgz",
-      "integrity": "sha512-77peW2ROlHHl1e/tHBUmhpb8tsO6CIdlx34XapZhUuIVykrkOuqVFFxqMecrGG8SJe0e3l1G+Fah7bJTQcG0kw==",
-      "dev": true,
-      "requires": {
-        "@lerna/batch-packages": "3.10.6",
-        "@lerna/check-working-tree": "3.10.0",
-        "@lerna/child-process": "3.3.0",
-        "@lerna/collect-updates": "3.10.1",
-        "@lerna/command": "3.10.6",
-        "@lerna/conventional-commits": "3.10.0",
-        "@lerna/output": "3.6.0",
-        "@lerna/prompt": "3.6.0",
-        "@lerna/run-lifecycle": "3.10.5",
-        "@lerna/validation-error": "3.6.0",
-        "chalk": "^2.3.1",
+    "@lerna/import": {
+      "version": "3.22.0",
+      "resolved": "https://registry.npmjs.org/@lerna/import/-/import-3.22.0.tgz",
+      "integrity": "sha512-uWOlexasM5XR6tXi4YehODtH9Y3OZrFht3mGUFFT3OIl2s+V85xIGFfqFGMTipMPAGb2oF1UBLL48kR43hRsOg==",
+      "dev": true,
+      "requires": {
+        "@lerna/child-process": "3.16.5",
+        "@lerna/command": "3.21.0",
+        "@lerna/prompt": "3.18.5",
+        "@lerna/pulse-till-done": "3.13.0",
+        "@lerna/validation-error": "3.13.0",
         "dedent": "^0.7.0",
-        "libnpm": "^2.0.1",
-        "minimatch": "^3.0.4",
-        "p-map": "^1.2.0",
-        "p-pipe": "^1.2.0",
-        "p-reduce": "^1.0.0",
-        "p-waterfall": "^1.0.0",
-        "semver": "^5.5.0",
-        "slash": "^1.0.0",
-        "temp-write": "^3.4.0"
+        "fs-extra": "^8.1.0",
+        "p-map-series": "^1.0.0"
       }
     },
-    "@lerna/write-log-file": {
-      "version": "3.6.0",
-      "resolved": "https://registry.npmjs.org/@lerna/write-log-file/-/write-log-file-3.6.0.tgz",
-      "integrity": "sha512-OkLK99V6sYXsJsYg+O9wtiFS3z6eUPaiz2e6cXJt80mfIIdI1t2dnmyua0Ib5cZWExQvx2z6Y32Wlf0MnsoNsA==",
+    "@lerna/info": {
+      "version": "3.21.0",
+      "resolved": "https://registry.npmjs.org/@lerna/info/-/info-3.21.0.tgz",
+      "integrity": "sha512-0XDqGYVBgWxUquFaIptW2bYSIu6jOs1BtkvRTWDDhw4zyEdp6q4eaMvqdSap1CG+7wM5jeLCi6z94wS0AuiuwA==",
       "dev": true,
       "requires": {
-        "libnpm": "^2.0.1",
-        "write-file-atomic": "^2.3.0"
+        "@lerna/command": "3.21.0",
+        "@lerna/output": "3.13.0",
+        "envinfo": "^7.3.1"
       }
     },
-    "@mrmlnc/readdir-enhanced": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz",
-      "integrity": "sha512-bPHp6Ji8b41szTOcaP63VlnbbO5Ny6dwAATtY6JTjh5N2OLrb5Qk/Th5cRkRQhkWCt+EJsYrNB0MiL+Gpn6e3g==",
+    "@lerna/init": {
+      "version": "3.21.0",
+      "resolved": "https://registry.npmjs.org/@lerna/init/-/init-3.21.0.tgz",
+      "integrity": "sha512-6CM0z+EFUkFfurwdJCR+LQQF6MqHbYDCBPyhu/d086LRf58GtYZYj49J8mKG9ktayp/TOIxL/pKKjgLD8QBPOg==",
       "dev": true,
       "requires": {
-        "call-me-maybe": "^1.0.1",
-        "glob-to-regexp": "^0.3.0"
+        "@lerna/child-process": "3.16.5",
+        "@lerna/command": "3.21.0",
+        "fs-extra": "^8.1.0",
+        "p-map": "^2.1.0",
+        "write-json-file": "^3.2.0"
+      },
+      "dependencies": {
+        "p-map": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz",
+          "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==",
+          "dev": true
+        }
       }
     },
-    "@nodelib/fs.stat": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz",
-      "integrity": "sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw==",
-      "dev": true
-    },
-    "@sindresorhus/df": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/@sindresorhus/df/-/df-2.1.0.tgz",
-      "integrity": "sha1-0gjPJ+BvC7R20U197M19cm6ao4k=",
+    "@lerna/link": {
+      "version": "3.21.0",
+      "resolved": "https://registry.npmjs.org/@lerna/link/-/link-3.21.0.tgz",
+      "integrity": "sha512-tGu9GxrX7Ivs+Wl3w1+jrLi1nQ36kNI32dcOssij6bg0oZ2M2MDEFI9UF2gmoypTaN9uO5TSsjCFS7aR79HbdQ==",
       "dev": true,
       "requires": {
-        "execa": "^0.2.2"
+        "@lerna/command": "3.21.0",
+        "@lerna/package-graph": "3.18.5",
+        "@lerna/symlink-dependencies": "3.17.0",
+        "p-map": "^2.1.0",
+        "slash": "^2.0.0"
       },
       "dependencies": {
-        "execa": {
-          "version": "0.2.2",
-          "resolved": "https://registry.npmjs.org/execa/-/execa-0.2.2.tgz",
-          "integrity": "sha1-4urUcsLDGq1vc/GslW7vReEjIMs=",
-          "dev": true,
-          "requires": {
-            "cross-spawn-async": "^2.1.1",
-            "npm-run-path": "^1.0.0",
-            "object-assign": "^4.0.1",
-            "path-key": "^1.0.0",
-            "strip-eof": "^1.0.0"
-          }
-        },
-        "npm-run-path": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-1.0.0.tgz",
-          "integrity": "sha1-9cMr9ZX+ga6Sfa7FLoL4sACsPI8=",
-          "dev": true,
-          "requires": {
-            "path-key": "^1.0.0"
-          }
+        "p-map": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz",
+          "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==",
+          "dev": true
         },
-        "path-key": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/path-key/-/path-key-1.0.0.tgz",
-          "integrity": "sha1-XVPVeAGWRsDWiADbThRua9wqx68=",
+        "slash": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
+          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
           "dev": true
         }
       }
     },
-    "@types/babel__core": {
-      "version": "7.1.2",
-      "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.2.tgz",
-      "integrity": "sha512-cfCCrFmiGY/yq0NuKNxIQvZFy9kY/1immpSpTngOnyIbD4+eJOG5mxphhHDv3CHL9GltO4GcKr54kGBg3RNdbg==",
+    "@lerna/list": {
+      "version": "3.21.0",
+      "resolved": "https://registry.npmjs.org/@lerna/list/-/list-3.21.0.tgz",
+      "integrity": "sha512-KehRjE83B1VaAbRRkRy6jLX1Cin8ltsrQ7FHf2bhwhRHK0S54YuA6LOoBnY/NtA8bHDX/Z+G5sMY78X30NS9tg==",
       "dev": true,
       "requires": {
-        "@babel/parser": "^7.1.0",
-        "@babel/types": "^7.0.0",
-        "@types/babel__generator": "*",
-        "@types/babel__template": "*",
-        "@types/babel__traverse": "*"
+        "@lerna/command": "3.21.0",
+        "@lerna/filter-options": "3.20.0",
+        "@lerna/listable": "3.18.5",
+        "@lerna/output": "3.13.0"
       }
     },
-    "@types/babel__generator": {
-      "version": "7.0.2",
-      "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.0.2.tgz",
-      "integrity": "sha512-NHcOfab3Zw4q5sEE2COkpfXjoE7o+PmqD9DQW4koUT3roNxwziUdXGnRndMat/LJNUtePwn1TlP4do3uoe3KZQ==",
+    "@lerna/listable": {
+      "version": "3.18.5",
+      "resolved": "https://registry.npmjs.org/@lerna/listable/-/listable-3.18.5.tgz",
+      "integrity": "sha512-Sdr3pVyaEv5A7ZkGGYR7zN+tTl2iDcinryBPvtuv20VJrXBE8wYcOks1edBTcOWsPjCE/rMP4bo1pseyk3UTsg==",
       "dev": true,
       "requires": {
-        "@babel/types": "^7.0.0"
+        "@lerna/query-graph": "3.18.5",
+        "chalk": "^2.3.1",
+        "columnify": "^1.5.4"
       }
     },
-    "@types/babel__template": {
-      "version": "7.0.2",
-      "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.0.2.tgz",
-      "integrity": "sha512-/K6zCpeW7Imzgab2bLkLEbz0+1JlFSrUMdw7KoIIu+IUdu51GWaBZpd3y1VXGVXzynvGa4DaIaxNZHiON3GXUg==",
+    "@lerna/log-packed": {
+      "version": "3.16.0",
+      "resolved": "https://registry.npmjs.org/@lerna/log-packed/-/log-packed-3.16.0.tgz",
+      "integrity": "sha512-Fp+McSNBV/P2mnLUYTaSlG8GSmpXM7krKWcllqElGxvAqv6chk2K3c2k80MeVB4WvJ9tRjUUf+i7HUTiQ9/ckQ==",
       "dev": true,
       "requires": {
-        "@babel/parser": "^7.1.0",
-        "@babel/types": "^7.0.0"
+        "byte-size": "^5.0.1",
+        "columnify": "^1.5.4",
+        "has-unicode": "^2.0.1",
+        "npmlog": "^4.1.2"
       }
     },
-    "@types/babel__traverse": {
-      "version": "7.0.6",
-      "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.0.6.tgz",
-      "integrity": "sha512-XYVgHF2sQ0YblLRMLNPB3CkFMewzFmlDsH/TneZFHUXDlABQgh88uOxuez7ZcXxayLFrqLwtDH1t+FmlFwNZxw==",
+    "@lerna/npm-conf": {
+      "version": "3.16.0",
+      "resolved": "https://registry.npmjs.org/@lerna/npm-conf/-/npm-conf-3.16.0.tgz",
+      "integrity": "sha512-HbO3DUrTkCAn2iQ9+FF/eisDpWY5POQAOF1m7q//CZjdC2HSW3UYbKEGsSisFxSfaF9Z4jtrV+F/wX6qWs3CuA==",
       "dev": true,
       "requires": {
-        "@babel/types": "^7.3.0"
+        "config-chain": "^1.1.11",
+        "pify": "^4.0.1"
+      },
+      "dependencies": {
+        "pify": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz",
+          "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==",
+          "dev": true
+        }
       }
     },
-    "@types/events": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/@types/events/-/events-3.0.0.tgz",
-      "integrity": "sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==",
-      "dev": true
-    },
-    "@types/flatbuffers": {
-      "version": "1.9.1",
-      "resolved": "https://registry.npmjs.org/@types/flatbuffers/-/flatbuffers-1.9.1.tgz",
-      "integrity": "sha512-TC3X0Nkj5wgvuY217VkodBtjbD3Yr0JNApDY1GW9IU5Mzm5ie1IJErqe4vRm+wy08IRz3bemaDATrdEw1CJlVQ=="
-    },
-    "@types/glob": {
-      "version": "7.1.1",
-      "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.1.tgz",
-      "integrity": "sha512-1Bh06cbWJUHMC97acuD6UMG29nMt0Aqz1vF3guLfG+kHHJhy3AyohZFFxYk2f7Q1SQIrNwvncxAE0N/9s70F2w==",
+    "@lerna/npm-dist-tag": {
+      "version": "3.18.5",
+      "resolved": "https://registry.npmjs.org/@lerna/npm-dist-tag/-/npm-dist-tag-3.18.5.tgz",
+      "integrity": "sha512-xw0HDoIG6HreVsJND9/dGls1c+lf6vhu7yJoo56Sz5bvncTloYGLUppIfDHQr4ZvmPCK8rsh0euCVh2giPxzKQ==",
       "dev": true,
       "requires": {
-        "@types/events": "*",
-        "@types/minimatch": "*",
-        "@types/node": "*"
+        "@evocateur/npm-registry-fetch": "^4.0.0",
+        "@lerna/otplease": "3.18.5",
+        "figgy-pudding": "^3.5.1",
+        "npm-package-arg": "^6.1.0",
+        "npmlog": "^4.1.2"
       }
     },
-    "@types/istanbul-lib-coverage": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.1.tgz",
-      "integrity": "sha512-hRJD2ahnnpLgsj6KWMYSrmXkM3rm2Dl1qkx6IOFD5FnuNPXJIG5L0dhgKXCYTRMGzU4n0wImQ/xfmRc4POUFlg==",
-      "dev": true
-    },
-    "@types/istanbul-lib-report": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-1.1.1.tgz",
-      "integrity": "sha512-3BUTyMzbZa2DtDI2BkERNC6jJw2Mr2Y0oGI7mRxYNBPxppbtEK1F66u3bKwU2g+wxwWI7PAoRpJnOY1grJqzHg==",
+    "@lerna/npm-install": {
+      "version": "3.16.5",
+      "resolved": "https://registry.npmjs.org/@lerna/npm-install/-/npm-install-3.16.5.tgz",
+      "integrity": "sha512-hfiKk8Eku6rB9uApqsalHHTHY+mOrrHeWEs+gtg7+meQZMTS3kzv4oVp5cBZigndQr3knTLjwthT/FX4KvseFg==",
       "dev": true,
       "requires": {
-        "@types/istanbul-lib-coverage": "*"
+        "@lerna/child-process": "3.16.5",
+        "@lerna/get-npm-exec-opts": "3.13.0",
+        "fs-extra": "^8.1.0",
+        "npm-package-arg": "^6.1.0",
+        "npmlog": "^4.1.2",
+        "signal-exit": "^3.0.2",
+        "write-pkg": "^3.1.0"
       }
     },
-    "@types/istanbul-reports": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-1.1.1.tgz",
-      "integrity": "sha512-UpYjBi8xefVChsCoBpKShdxTllC9pwISirfoZsUa2AAdQg/Jd2KQGtSbw+ya7GPo7x/wAPlH6JBhKhAsXUEZNA==",
+    "@lerna/npm-publish": {
+      "version": "3.18.5",
+      "resolved": "https://registry.npmjs.org/@lerna/npm-publish/-/npm-publish-3.18.5.tgz",
+      "integrity": "sha512-3etLT9+2L8JAx5F8uf7qp6iAtOLSMj+ZYWY6oUgozPi/uLqU0/gsMsEXh3F0+YVW33q0M61RpduBoAlOOZnaTg==",
       "dev": true,
       "requires": {
-        "@types/istanbul-lib-coverage": "*",
-        "@types/istanbul-lib-report": "*"
+        "@evocateur/libnpmpublish": "^1.2.2",
+        "@lerna/otplease": "3.18.5",
+        "@lerna/run-lifecycle": "3.16.2",
+        "figgy-pudding": "^3.5.1",
+        "fs-extra": "^8.1.0",
+        "npm-package-arg": "^6.1.0",
+        "npmlog": "^4.1.2",
+        "pify": "^4.0.1",
+        "read-package-json": "^2.0.13"
+      },
+      "dependencies": {
+        "pify": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz",
+          "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==",
+          "dev": true
+        }
       }
     },
-    "@types/jest": {
-      "version": "24.0.13",
-      "resolved": "https://registry.npmjs.org/@types/jest/-/jest-24.0.13.tgz",
-      "integrity": "sha512-3m6RPnO35r7Dg+uMLj1+xfZaOgIHHHut61djNjzwExXN4/Pm9has9C6I1KMYSfz7mahDhWUOVg4HW/nZdv5Pww==",
+    "@lerna/npm-run-script": {
+      "version": "3.16.5",
+      "resolved": "https://registry.npmjs.org/@lerna/npm-run-script/-/npm-run-script-3.16.5.tgz",
+      "integrity": "sha512-1asRi+LjmVn3pMjEdpqKJZFT/3ZNpb+VVeJMwrJaV/3DivdNg7XlPK9LTrORuKU4PSvhdEZvJmSlxCKyDpiXsQ==",
       "dev": true,
       "requires": {
-        "@types/jest-diff": "*"
+        "@lerna/child-process": "3.16.5",
+        "@lerna/get-npm-exec-opts": "3.13.0",
+        "npmlog": "^4.1.2"
       }
     },
-    "@types/jest-diff": {
-      "version": "20.0.1",
-      "resolved": "https://registry.npmjs.org/@types/jest-diff/-/jest-diff-20.0.1.tgz",
-      "integrity": "sha512-yALhelO3i0hqZwhjtcr6dYyaLoCHbAMshwtj6cGxTvHZAKXHsYGdff6E8EPw3xLKY0ELUTQ69Q1rQiJENnccMA==",
-      "dev": true
-    },
-    "@types/minimatch": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz",
-      "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==",
-      "dev": true
-    },
-    "@types/node": {
-      "version": "12.0.4",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.4.tgz",
-      "integrity": "sha512-j8YL2C0fXq7IONwl/Ud5Kt0PeXw22zGERt+HSSnwbKOJVsAGkEz3sFCYwaF9IOuoG1HOtE0vKCj6sXF7Q0+Vaw=="
-    },
-    "@types/stack-utils": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-1.0.1.tgz",
-      "integrity": "sha512-l42BggppR6zLmpfU6fq9HEa2oGPEI8yrSPL3GITjfRInppYFahObbIQOQK3UGxEnyQpltZLaPe75046NOZQikw==",
-      "dev": true
-    },
-    "@types/text-encoding-utf-8": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@types/text-encoding-utf-8/-/text-encoding-utf-8-1.0.1.tgz",
-      "integrity": "sha512-GpIEYaS+yNfYqpowLLziiY42pyaL+lThd/wMh6tTubaKuG4IRkXqqyxK7Nddn3BvpUg2+go3Gv/jbXvAFMRjiQ=="
-    },
-    "@types/yargs": {
-      "version": "12.0.12",
-      "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-12.0.12.tgz",
-      "integrity": "sha512-SOhuU4wNBxhhTHxYaiG5NY4HBhDIDnJF60GU+2LqHAdKKer86//e4yg69aENCtQ04n0ovz+tq2YPME5t5yp4pw==",
-      "dev": true
-    },
-    "@webassemblyjs/ast": {
-      "version": "1.7.11",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.7.11.tgz",
-      "integrity": "sha512-ZEzy4vjvTzScC+SH8RBssQUawpaInUdMTYwYYLh54/s8TuT0gBLuyUnppKsVyZEi876VmmStKsUs28UxPgdvrA==",
+    "@lerna/otplease": {
+      "version": "3.18.5",
+      "resolved": "https://registry.npmjs.org/@lerna/otplease/-/otplease-3.18.5.tgz",
+      "integrity": "sha512-S+SldXAbcXTEDhzdxYLU0ZBKuYyURP/ND2/dK6IpKgLxQYh/z4ScljPDMyKymmEvgiEJmBsPZAAPfmNPEzxjog==",
       "dev": true,
       "requires": {
-        "@webassemblyjs/helper-module-context": "1.7.11",
-        "@webassemblyjs/helper-wasm-bytecode": "1.7.11",
-        "@webassemblyjs/wast-parser": "1.7.11"
+        "@lerna/prompt": "3.18.5",
+        "figgy-pudding": "^3.5.1"
       }
     },
-    "@webassemblyjs/floating-point-hex-parser": {
-      "version": "1.7.11",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.7.11.tgz",
-      "integrity": "sha512-zY8dSNyYcgzNRNT666/zOoAyImshm3ycKdoLsyDw/Bwo6+/uktb7p4xyApuef1dwEBo/U/SYQzbGBvV+nru2Xg==",
-      "dev": true
-    },
-    "@webassemblyjs/helper-api-error": {
-      "version": "1.7.11",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.7.11.tgz",
-      "integrity": "sha512-7r1qXLmiglC+wPNkGuXCvkmalyEstKVwcueZRP2GNC2PAvxbLYwLLPr14rcdJaE4UtHxQKfFkuDFuv91ipqvXg==",
-      "dev": true
-    },
-    "@webassemblyjs/helper-buffer": {
-      "version": "1.7.11",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.7.11.tgz",
-      "integrity": "sha512-MynuervdylPPh3ix+mKZloTcL06P8tenNH3sx6s0qE8SLR6DdwnfgA7Hc9NSYeob2jrW5Vql6GVlsQzKQCa13w==",
-      "dev": true
-    },
-    "@webassemblyjs/helper-code-frame": {
-      "version": "1.7.11",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-code-frame/-/helper-code-frame-1.7.11.tgz",
-      "integrity": "sha512-T8ESC9KMXFTXA5urJcyor5cn6qWeZ4/zLPyWeEXZ03hj/x9weSokGNkVCdnhSabKGYWxElSdgJ+sFa9G/RdHNw==",
+    "@lerna/output": {
+      "version": "3.13.0",
+      "resolved": "https://registry.npmjs.org/@lerna/output/-/output-3.13.0.tgz",
+      "integrity": "sha512-7ZnQ9nvUDu/WD+bNsypmPG5MwZBwu86iRoiW6C1WBuXXDxM5cnIAC1m2WxHeFnjyMrYlRXM9PzOQ9VDD+C15Rg==",
       "dev": true,
       "requires": {
-        "@webassemblyjs/wast-printer": "1.7.11"
+        "npmlog": "^4.1.2"
       }
     },
-    "@webassemblyjs/helper-fsm": {
-      "version": "1.7.11",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-fsm/-/helper-fsm-1.7.11.tgz",
-      "integrity": "sha512-nsAQWNP1+8Z6tkzdYlXT0kxfa2Z1tRTARd8wYnc/e3Zv3VydVVnaeePgqUzFrpkGUyhUUxOl5ML7f1NuT+gC0A==",
-      "dev": true
+    "@lerna/pack-directory": {
+      "version": "3.16.4",
+      "resolved": "https://registry.npmjs.org/@lerna/pack-directory/-/pack-directory-3.16.4.tgz",
+      "integrity": "sha512-uxSF0HZeGyKaaVHz5FroDY9A5NDDiCibrbYR6+khmrhZtY0Bgn6hWq8Gswl9iIlymA+VzCbshWIMX4o2O8C8ng==",
+      "dev": true,
+      "requires": {
+        "@lerna/get-packed": "3.16.0",
+        "@lerna/package": "3.16.0",
+        "@lerna/run-lifecycle": "3.16.2",
+        "figgy-pudding": "^3.5.1",
+        "npm-packlist": "^1.4.4",
+        "npmlog": "^4.1.2",
+        "tar": "^4.4.10",
+        "temp-write": "^3.4.0"
+      }
     },
-    "@webassemblyjs/helper-module-context": {
-      "version": "1.7.11",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-module-context/-/helper-module-context-1.7.11.tgz",
-      "integrity": "sha512-JxfD5DX8Ygq4PvXDucq0M+sbUFA7BJAv/GGl9ITovqE+idGX+J3QSzJYz+LwQmL7fC3Rs+utvWoJxDb6pmC0qg==",
-      "dev": true
-    },
-    "@webassemblyjs/helper-wasm-bytecode": {
-      "version": "1.7.11",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.7.11.tgz",
-      "integrity": "sha512-cMXeVS9rhoXsI9LLL4tJxBgVD/KMOKXuFqYb5oCJ/opScWpkCMEz9EJtkonaNcnLv2R3K5jIeS4TRj/drde1JQ==",
-      "dev": true
-    },
-    "@webassemblyjs/helper-wasm-section": {
-      "version": "1.7.11",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.7.11.tgz",
-      "integrity": "sha512-8ZRY5iZbZdtNFE5UFunB8mmBEAbSI3guwbrsCl4fWdfRiAcvqQpeqd5KHhSWLL5wuxo53zcaGZDBU64qgn4I4Q==",
+    "@lerna/package": {
+      "version": "3.16.0",
+      "resolved": "https://registry.npmjs.org/@lerna/package/-/package-3.16.0.tgz",
+      "integrity": "sha512-2lHBWpaxcBoiNVbtyLtPUuTYEaB/Z+eEqRS9duxpZs6D+mTTZMNy6/5vpEVSCBmzvdYpyqhqaYjjSLvjjr5Riw==",
       "dev": true,
       "requires": {
-        "@webassemblyjs/ast": "1.7.11",
-        "@webassemblyjs/helper-buffer": "1.7.11",
-        "@webassemblyjs/helper-wasm-bytecode": "1.7.11",
-        "@webassemblyjs/wasm-gen": "1.7.11"
+        "load-json-file": "^5.3.0",
+        "npm-package-arg": "^6.1.0",
+        "write-pkg": "^3.1.0"
+      },
+      "dependencies": {
+        "load-json-file": {
+          "version": "5.3.0",
+          "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-5.3.0.tgz",
+          "integrity": "sha512-cJGP40Jc/VXUsp8/OrnyKyTZ1y6v/dphm3bioS+RrKXjK2BB6wHUd6JptZEFDGgGahMT+InnZO5i1Ei9mpC8Bw==",
+          "dev": true,
+          "requires": {
+            "graceful-fs": "^4.1.15",
+            "parse-json": "^4.0.0",
+            "pify": "^4.0.1",
+            "strip-bom": "^3.0.0",
+            "type-fest": "^0.3.0"
+          }
+        },
+        "parse-json": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
+          "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=",
+          "dev": true,
+          "requires": {
+            "error-ex": "^1.3.1",
+            "json-parse-better-errors": "^1.0.1"
+          }
+        },
+        "pify": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz",
+          "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==",
+          "dev": true
+        },
+        "strip-bom": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
+          "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=",
+          "dev": true
+        },
+        "type-fest": {
+          "version": "0.3.1",
+          "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.3.1.tgz",
+          "integrity": "sha512-cUGJnCdr4STbePCgqNFbpVNCepa+kAVohJs1sLhxzdH+gnEoOd8VhbYa7pD3zZYGiURWM2xzEII3fQcRizDkYQ==",
+          "dev": true
+        }
       }
     },
-    "@webassemblyjs/ieee754": {
-      "version": "1.7.11",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.7.11.tgz",
-      "integrity": "sha512-Mmqx/cS68K1tSrvRLtaV/Lp3NZWzXtOHUW2IvDvl2sihAwJh4ACE0eL6A8FvMyDG9abes3saB6dMimLOs+HMoQ==",
+    "@lerna/package-graph": {
+      "version": "3.18.5",
+      "resolved": "https://registry.npmjs.org/@lerna/package-graph/-/package-graph-3.18.5.tgz",
+      "integrity": "sha512-8QDrR9T+dBegjeLr+n9WZTVxUYUhIUjUgZ0gvNxUBN8S1WB9r6H5Yk56/MVaB64tA3oGAN9IIxX6w0WvTfFudA==",
       "dev": true,
       "requires": {
-        "@xtuc/ieee754": "^1.2.0"
+        "@lerna/prerelease-id-from-version": "3.16.0",
+        "@lerna/validation-error": "3.13.0",
+        "npm-package-arg": "^6.1.0",
+        "npmlog": "^4.1.2",
+        "semver": "^6.2.0"
+      },
+      "dependencies": {
+        "semver": {
+          "version": "6.3.0",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
+          "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
+          "dev": true
+        }
       }
     },
-    "@webassemblyjs/leb128": {
-      "version": "1.7.11",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.7.11.tgz",
-      "integrity": "sha512-vuGmgZjjp3zjcerQg+JA+tGOncOnJLWVkt8Aze5eWQLwTQGNgVLcyOTqgSCxWTR4J42ijHbBxnuRaL1Rv7XMdw==",
+    "@lerna/prerelease-id-from-version": {
+      "version": "3.16.0",
+      "resolved": "https://registry.npmjs.org/@lerna/prerelease-id-from-version/-/prerelease-id-from-version-3.16.0.tgz",
+      "integrity": "sha512-qZyeUyrE59uOK8rKdGn7jQz+9uOpAaF/3hbslJVFL1NqF9ELDTqjCPXivuejMX/lN4OgD6BugTO4cR7UTq/sZA==",
       "dev": true,
       "requires": {
-        "@xtuc/long": "4.2.1"
+        "semver": "^6.2.0"
+      },
+      "dependencies": {
+        "semver": {
+          "version": "6.3.0",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
+          "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
+          "dev": true
+        }
       }
     },
-    "@webassemblyjs/utf8": {
-      "version": "1.7.11",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.7.11.tgz",
-      "integrity": "sha512-C6GFkc7aErQIAH+BMrIdVSmW+6HSe20wg57HEC1uqJP8E/xpMjXqQUxkQw07MhNDSDcGpxI9G5JSNOQCqJk4sA==",
-      "dev": true
-    },
-    "@webassemblyjs/wasm-edit": {
-      "version": "1.7.11",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.7.11.tgz",
-      "integrity": "sha512-FUd97guNGsCZQgeTPKdgxJhBXkUbMTY6hFPf2Y4OedXd48H97J+sOY2Ltaq6WGVpIH8o/TGOVNiVz/SbpEMJGg==",
+    "@lerna/profiler": {
+      "version": "3.20.0",
+      "resolved": "https://registry.npmjs.org/@lerna/profiler/-/profiler-3.20.0.tgz",
+      "integrity": "sha512-bh8hKxAlm6yu8WEOvbLENm42i2v9SsR4WbrCWSbsmOElx3foRnMlYk7NkGECa+U5c3K4C6GeBbwgqs54PP7Ljg==",
       "dev": true,
       "requires": {
-        "@webassemblyjs/ast": "1.7.11",
-        "@webassemblyjs/helper-buffer": "1.7.11",
-        "@webassemblyjs/helper-wasm-bytecode": "1.7.11",
-        "@webassemblyjs/helper-wasm-section": "1.7.11",
-        "@webassemblyjs/wasm-gen": "1.7.11",
-        "@webassemblyjs/wasm-opt": "1.7.11",
-        "@webassemblyjs/wasm-parser": "1.7.11",
-        "@webassemblyjs/wast-printer": "1.7.11"
+        "figgy-pudding": "^3.5.1",
+        "fs-extra": "^8.1.0",
+        "npmlog": "^4.1.2",
+        "upath": "^1.2.0"
       }
     },
-    "@webassemblyjs/wasm-gen": {
-      "version": "1.7.11",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.7.11.tgz",
-      "integrity": "sha512-U/KDYp7fgAZX5KPfq4NOupK/BmhDc5Kjy2GIqstMhvvdJRcER/kUsMThpWeRP8BMn4LXaKhSTggIJPOeYHwISA==",
+    "@lerna/project": {
+      "version": "3.21.0",
+      "resolved": "https://registry.npmjs.org/@lerna/project/-/project-3.21.0.tgz",
+      "integrity": "sha512-xT1mrpET2BF11CY32uypV2GPtPVm6Hgtha7D81GQP9iAitk9EccrdNjYGt5UBYASl4CIDXBRxwmTTVGfrCx82A==",
       "dev": true,
       "requires": {
-        "@webassemblyjs/ast": "1.7.11",
-        "@webassemblyjs/helper-wasm-bytecode": "1.7.11",
-        "@webassemblyjs/ieee754": "1.7.11",
-        "@webassemblyjs/leb128": "1.7.11",
-        "@webassemblyjs/utf8": "1.7.11"
+        "@lerna/package": "3.16.0",
+        "@lerna/validation-error": "3.13.0",
+        "cosmiconfig": "^5.1.0",
+        "dedent": "^0.7.0",
+        "dot-prop": "^4.2.0",
+        "glob-parent": "^5.0.0",
+        "globby": "^9.2.0",
+        "load-json-file": "^5.3.0",
+        "npmlog": "^4.1.2",
+        "p-map": "^2.1.0",
+        "resolve-from": "^4.0.0",
+        "write-json-file": "^3.2.0"
+      },
+      "dependencies": {
+        "glob-parent": {
+          "version": "5.1.1",
+          "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
+          "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
+          "dev": true,
+          "requires": {
+            "is-glob": "^4.0.1"
+          }
+        },
+        "globby": {
+          "version": "9.2.0",
+          "resolved": "https://registry.npmjs.org/globby/-/globby-9.2.0.tgz",
+          "integrity": "sha512-ollPHROa5mcxDEkwg6bPt3QbEf4pDQSNtd6JPL1YvOvAo/7/0VAm9TccUeoTmarjPw4pfUthSCqcyfNB1I3ZSg==",
+          "dev": true,
+          "requires": {
+            "@types/glob": "^7.1.1",
+            "array-union": "^1.0.2",
+            "dir-glob": "^2.2.2",
+            "fast-glob": "^2.2.6",
+            "glob": "^7.1.3",
+            "ignore": "^4.0.3",
+            "pify": "^4.0.1",
+            "slash": "^2.0.0"
+          }
+        },
+        "load-json-file": {
+          "version": "5.3.0",
+          "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-5.3.0.tgz",
+          "integrity": "sha512-cJGP40Jc/VXUsp8/OrnyKyTZ1y6v/dphm3bioS+RrKXjK2BB6wHUd6JptZEFDGgGahMT+InnZO5i1Ei9mpC8Bw==",
+          "dev": true,
+          "requires": {
+            "graceful-fs": "^4.1.15",
+            "parse-json": "^4.0.0",
+            "pify": "^4.0.1",
+            "strip-bom": "^3.0.0",
+            "type-fest": "^0.3.0"
+          }
+        },
+        "p-map": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz",
+          "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==",
+          "dev": true
+        },
+        "parse-json": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
+          "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=",
+          "dev": true,
+          "requires": {
+            "error-ex": "^1.3.1",
+            "json-parse-better-errors": "^1.0.1"
+          }
+        },
+        "pify": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz",
+          "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==",
+          "dev": true
+        },
+        "resolve-from": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
+          "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
+          "dev": true
+        },
+        "slash": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
+          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
+          "dev": true
+        },
+        "strip-bom": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
+          "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=",
+          "dev": true
+        },
+        "type-fest": {
+          "version": "0.3.1",
+          "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.3.1.tgz",
+          "integrity": "sha512-cUGJnCdr4STbePCgqNFbpVNCepa+kAVohJs1sLhxzdH+gnEoOd8VhbYa7pD3zZYGiURWM2xzEII3fQcRizDkYQ==",
+          "dev": true
+        }
       }
     },
-    "@webassemblyjs/wasm-opt": {
-      "version": "1.7.11",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.7.11.tgz",
-      "integrity": "sha512-XynkOwQyiRidh0GLua7SkeHvAPXQV/RxsUeERILmAInZegApOUAIJfRuPYe2F7RcjOC9tW3Cb9juPvAC/sCqvg==",
+    "@lerna/prompt": {
+      "version": "3.18.5",
+      "resolved": "https://registry.npmjs.org/@lerna/prompt/-/prompt-3.18.5.tgz",
+      "integrity": "sha512-rkKj4nm1twSbBEb69+Em/2jAERK8htUuV8/xSjN0NPC+6UjzAwY52/x9n5cfmpa9lyKf/uItp7chCI7eDmNTKQ==",
       "dev": true,
       "requires": {
-        "@webassemblyjs/ast": "1.7.11",
-        "@webassemblyjs/helper-buffer": "1.7.11",
-        "@webassemblyjs/wasm-gen": "1.7.11",
-        "@webassemblyjs/wasm-parser": "1.7.11"
+        "inquirer": "^6.2.0",
+        "npmlog": "^4.1.2"
       }
     },
-    "@webassemblyjs/wasm-parser": {
-      "version": "1.7.11",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.7.11.tgz",
-      "integrity": "sha512-6lmXRTrrZjYD8Ng8xRyvyXQJYUQKYSXhJqXOBLw24rdiXsHAOlvw5PhesjdcaMadU/pyPQOJ5dHreMjBxwnQKg==",
-      "dev": true,
-      "requires": {
-        "@webassemblyjs/ast": "1.7.11",
-        "@webassemblyjs/helper-api-error": "1.7.11",
-        "@webassemblyjs/helper-wasm-bytecode": "1.7.11",
-        "@webassemblyjs/ieee754": "1.7.11",
-        "@webassemblyjs/leb128": "1.7.11",
-        "@webassemblyjs/utf8": "1.7.11"
+    "@lerna/publish": {
+      "version": "3.22.1",
+      "resolved": "https://registry.npmjs.org/@lerna/publish/-/publish-3.22.1.tgz",
+      "integrity": "sha512-PG9CM9HUYDreb1FbJwFg90TCBQooGjj+n/pb3gw/eH5mEDq0p8wKdLFe0qkiqUkm/Ub5C8DbVFertIo0Vd0zcw==",
+      "dev": true,
+      "requires": {
+        "@evocateur/libnpmaccess": "^3.1.2",
+        "@evocateur/npm-registry-fetch": "^4.0.0",
+        "@evocateur/pacote": "^9.6.3",
+        "@lerna/check-working-tree": "3.16.5",
+        "@lerna/child-process": "3.16.5",
+        "@lerna/collect-updates": "3.20.0",
+        "@lerna/command": "3.21.0",
+        "@lerna/describe-ref": "3.16.5",
+        "@lerna/log-packed": "3.16.0",
+        "@lerna/npm-conf": "3.16.0",
+        "@lerna/npm-dist-tag": "3.18.5",
+        "@lerna/npm-publish": "3.18.5",
+        "@lerna/otplease": "3.18.5",
+        "@lerna/output": "3.13.0",
+        "@lerna/pack-directory": "3.16.4",
+        "@lerna/prerelease-id-from-version": "3.16.0",
+        "@lerna/prompt": "3.18.5",
+        "@lerna/pulse-till-done": "3.13.0",
+        "@lerna/run-lifecycle": "3.16.2",
+        "@lerna/run-topologically": "3.18.5",
+        "@lerna/validation-error": "3.13.0",
+        "@lerna/version": "3.22.1",
+        "figgy-pudding": "^3.5.1",
+        "fs-extra": "^8.1.0",
+        "npm-package-arg": "^6.1.0",
+        "npmlog": "^4.1.2",
+        "p-finally": "^1.0.0",
+        "p-map": "^2.1.0",
+        "p-pipe": "^1.2.0",
+        "semver": "^6.2.0"
+      },
+      "dependencies": {
+        "p-map": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz",
+          "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==",
+          "dev": true
+        },
+        "semver": {
+          "version": "6.3.0",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
+          "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
+          "dev": true
+        }
       }
     },
-    "@webassemblyjs/wast-parser": {
-      "version": "1.7.11",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-parser/-/wast-parser-1.7.11.tgz",
-      "integrity": "sha512-lEyVCg2np15tS+dm7+JJTNhNWq9yTZvi3qEhAIIOaofcYlUp0UR5/tVqOwa/gXYr3gjwSZqw+/lS9dscyLelbQ==",
+    "@lerna/pulse-till-done": {
+      "version": "3.13.0",
+      "resolved": "https://registry.npmjs.org/@lerna/pulse-till-done/-/pulse-till-done-3.13.0.tgz",
+      "integrity": "sha512-1SOHpy7ZNTPulzIbargrgaJX387csN7cF1cLOGZiJQA6VqnS5eWs2CIrG8i8wmaUavj2QlQ5oEbRMVVXSsGrzA==",
       "dev": true,
       "requires": {
-        "@webassemblyjs/ast": "1.7.11",
-        "@webassemblyjs/floating-point-hex-parser": "1.7.11",
-        "@webassemblyjs/helper-api-error": "1.7.11",
-        "@webassemblyjs/helper-code-frame": "1.7.11",
-        "@webassemblyjs/helper-fsm": "1.7.11",
-        "@xtuc/long": "4.2.1"
+        "npmlog": "^4.1.2"
       }
     },
-    "@webassemblyjs/wast-printer": {
-      "version": "1.7.11",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.7.11.tgz",
-      "integrity": "sha512-m5vkAsuJ32QpkdkDOUPGSltrg8Cuk3KBx4YrmAGQwCZPRdUHXxG4phIOuuycLemHFr74sWL9Wthqss4fzdzSwg==",
+    "@lerna/query-graph": {
+      "version": "3.18.5",
+      "resolved": "https://registry.npmjs.org/@lerna/query-graph/-/query-graph-3.18.5.tgz",
+      "integrity": "sha512-50Lf4uuMpMWvJ306be3oQDHrWV42nai9gbIVByPBYJuVW8dT8O8pA3EzitNYBUdLL9/qEVbrR0ry1HD7EXwtRA==",
       "dev": true,
       "requires": {
-        "@webassemblyjs/ast": "1.7.11",
-        "@webassemblyjs/wast-parser": "1.7.11",
-        "@xtuc/long": "4.2.1"
+        "@lerna/package-graph": "3.18.5",
+        "figgy-pudding": "^3.5.1"
       }
     },
-    "@xtuc/ieee754": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz",
-      "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==",
-      "dev": true
-    },
-    "@xtuc/long": {
-      "version": "4.2.1",
-      "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.1.tgz",
-      "integrity": "sha512-FZdkNBDqBRHKQ2MEbSC17xnPFOhZxeJ2YGSfr2BKf3sujG49Qe3bB+rGCwQfIaA7WHnGeGkSijX4FuBCdrzW/g==",
-      "dev": true
-    },
-    "JSONStream": {
-      "version": "1.3.5",
-      "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz",
-      "integrity": "sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==",
+    "@lerna/resolve-symlink": {
+      "version": "3.16.0",
+      "resolved": "https://registry.npmjs.org/@lerna/resolve-symlink/-/resolve-symlink-3.16.0.tgz",
+      "integrity": "sha512-Ibj5e7njVHNJ/NOqT4HlEgPFPtPLWsO7iu59AM5bJDcAJcR96mLZ7KGVIsS2tvaO7akMEJvt2P+ErwCdloG3jQ==",
       "dev": true,
       "requires": {
-        "jsonparse": "^1.2.0",
-        "through": ">=2.2.7 <3"
+        "fs-extra": "^8.1.0",
+        "npmlog": "^4.1.2",
+        "read-cmd-shim": "^1.0.1"
       }
     },
-    "abab": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.0.tgz",
-      "integrity": "sha512-sY5AXXVZv4Y1VACTtR11UJCPHHudgY5i26Qj5TypE6DKlIApbwb5uqhXcJ5UUGbvZNRh7EeIoW+LrJumBsKp7w==",
-      "dev": true
-    },
-    "abbrev": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
-      "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
-      "dev": true
-    },
-    "acorn": {
-      "version": "5.7.3",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.3.tgz",
-      "integrity": "sha512-T/zvzYRfbVojPWahDsE5evJdHb3oJoQfFbsrKM7w5Zcs++Tr257tia3BmMP8XYVjp1S9RZXQMh7gao96BlqZOw==",
-      "dev": true
-    },
-    "acorn-dynamic-import": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/acorn-dynamic-import/-/acorn-dynamic-import-4.0.0.tgz",
-      "integrity": "sha512-d3OEjQV4ROpoflsnUA8HozoIR504TFxNivYEUi6uwz0IYhBkTDXGuWlNdMtybRt3nqVx/L6XqMt0FxkXuWKZhw==",
-      "dev": true
-    },
-    "acorn-globals": {
-      "version": "4.3.2",
-      "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-4.3.2.tgz",
-      "integrity": "sha512-BbzvZhVtZP+Bs1J1HcwrQe8ycfO0wStkSGxuul3He3GkHOIZ6eTqOkPuw9IP1X3+IkOo4wiJmwkobzXYz4wewQ==",
+    "@lerna/rimraf-dir": {
+      "version": "3.16.5",
+      "resolved": "https://registry.npmjs.org/@lerna/rimraf-dir/-/rimraf-dir-3.16.5.tgz",
+      "integrity": "sha512-bQlKmO0pXUsXoF8lOLknhyQjOZsCc0bosQDoX4lujBXSWxHVTg1VxURtWf2lUjz/ACsJVDfvHZbDm8kyBk5okA==",
       "dev": true,
       "requires": {
-        "acorn": "^6.0.1",
-        "acorn-walk": "^6.0.1"
+        "@lerna/child-process": "3.16.5",
+        "npmlog": "^4.1.2",
+        "path-exists": "^3.0.0",
+        "rimraf": "^2.6.2"
       },
       "dependencies": {
-        "acorn": {
-          "version": "6.1.1",
-          "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.1.1.tgz",
-          "integrity": "sha512-jPTiwtOxaHNaAPg/dmrJ/beuzLRnXtB0kQPQ8JpotKJgTB6rX6c8mlf315941pyjBSaPg8NHXS9fhP4u17DpGA==",
+        "path-exists": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
+          "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=",
           "dev": true
         }
       }
     },
-    "acorn-walk": {
-      "version": "6.1.1",
-      "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-6.1.1.tgz",
-      "integrity": "sha512-OtUw6JUTgxA2QoqqmrmQ7F2NYqiBPi/L2jqHyFtllhOUvXYQXf0Z1CYUinIfyT4bTCGmrA7gX9FvHA81uzCoVw==",
-      "dev": true
+    "@lerna/run": {
+      "version": "3.21.0",
+      "resolved": "https://registry.npmjs.org/@lerna/run/-/run-3.21.0.tgz",
+      "integrity": "sha512-fJF68rT3veh+hkToFsBmUJ9MHc9yGXA7LSDvhziAojzOb0AI/jBDp6cEcDQyJ7dbnplba2Lj02IH61QUf9oW0Q==",
+      "dev": true,
+      "requires": {
+        "@lerna/command": "3.21.0",
+        "@lerna/filter-options": "3.20.0",
+        "@lerna/npm-run-script": "3.16.5",
+        "@lerna/output": "3.13.0",
+        "@lerna/profiler": "3.20.0",
+        "@lerna/run-topologically": "3.18.5",
+        "@lerna/timer": "3.13.0",
+        "@lerna/validation-error": "3.13.0",
+        "p-map": "^2.1.0"
+      },
+      "dependencies": {
+        "p-map": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz",
+          "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==",
+          "dev": true
+        }
+      }
     },
-    "agent-base": {
-      "version": "4.2.1",
-      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz",
-      "integrity": "sha512-JVwXMr9nHYTUXsBFKUqhJwvlcYU/blreOEUkhNR2eXZIvwd+c+o5V4MgDPKWnMS/56awN3TRzIP+KoPn+roQtg==",
+    "@lerna/run-lifecycle": {
+      "version": "3.16.2",
+      "resolved": "https://registry.npmjs.org/@lerna/run-lifecycle/-/run-lifecycle-3.16.2.tgz",
+      "integrity": "sha512-RqFoznE8rDpyyF0rOJy3+KjZCeTkO8y/OB9orPauR7G2xQ7PTdCpgo7EO6ZNdz3Al+k1BydClZz/j78gNCmL2A==",
       "dev": true,
       "requires": {
-        "es6-promisify": "^5.0.0"
+        "@lerna/npm-conf": "3.16.0",
+        "figgy-pudding": "^3.5.1",
+        "npm-lifecycle": "^3.1.2",
+        "npmlog": "^4.1.2"
       }
     },
-    "agentkeepalive": {
-      "version": "3.5.2",
-      "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.5.2.tgz",
-      "integrity": "sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==",
+    "@lerna/run-topologically": {
+      "version": "3.18.5",
+      "resolved": "https://registry.npmjs.org/@lerna/run-topologically/-/run-topologically-3.18.5.tgz",
+      "integrity": "sha512-6N1I+6wf4hLOnPW+XDZqwufyIQ6gqoPfHZFkfWlvTQ+Ue7CuF8qIVQ1Eddw5HKQMkxqN10thKOFfq/9NQZ4NUg==",
       "dev": true,
       "requires": {
-        "humanize-ms": "^1.2.1"
+        "@lerna/query-graph": "3.18.5",
+        "figgy-pudding": "^3.5.1",
+        "p-queue": "^4.0.0"
       }
     },
-    "ajv": {
-      "version": "6.7.0",
-      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.7.0.tgz",
-      "integrity": "sha512-RZXPviBTtfmtka9n9sy1N5M5b82CbxWIR6HIis4s3WQTXDJamc/0gpCWNGz6EWdWp4DOfjzJfhz/AS9zVPjjWg==",
+    "@lerna/symlink-binary": {
+      "version": "3.17.0",
+      "resolved": "https://registry.npmjs.org/@lerna/symlink-binary/-/symlink-binary-3.17.0.tgz",
+      "integrity": "sha512-RLpy9UY6+3nT5J+5jkM5MZyMmjNHxZIZvXLV+Q3MXrf7Eaa1hNqyynyj4RO95fxbS+EZc4XVSk25DGFQbcRNSQ==",
       "dev": true,
       "requires": {
-        "fast-deep-equal": "^2.0.1",
-        "fast-json-stable-stringify": "^2.0.0",
-        "json-schema-traverse": "^0.4.1",
-        "uri-js": "^4.2.2"
+        "@lerna/create-symlink": "3.16.2",
+        "@lerna/package": "3.16.0",
+        "fs-extra": "^8.1.0",
+        "p-map": "^2.1.0"
+      },
+      "dependencies": {
+        "p-map": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz",
+          "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==",
+          "dev": true
+        }
       }
     },
-    "ajv-errors": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/ajv-errors/-/ajv-errors-1.0.1.tgz",
-      "integrity": "sha512-DCRfO/4nQ+89p/RK43i8Ezd41EqdGIU4ld7nGF8OQ14oc/we5rEntLCUa7+jrn3nn83BosfwZA0wb4pon2o8iQ==",
-      "dev": true
-    },
-    "ajv-keywords": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.3.0.tgz",
-      "integrity": "sha512-CMzN9S62ZOO4sA/mJZIO4S++ZM7KFWzH3PPWkveLhy4OZ9i1/VatgwWMD46w/XbGCBy7Ye0gCk+Za6mmyfKK7g==",
-      "dev": true
-    },
-    "ansi-colors": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-1.1.0.tgz",
-      "integrity": "sha512-SFKX67auSNoVR38N3L+nvsPjOE0bybKTYbkf5tRvushrAPQ9V75huw0ZxBkKVeRU9kqH3d6HA4xTckbwZ4ixmA==",
+    "@lerna/symlink-dependencies": {
+      "version": "3.17.0",
+      "resolved": "https://registry.npmjs.org/@lerna/symlink-dependencies/-/symlink-dependencies-3.17.0.tgz",
+      "integrity": "sha512-KmjU5YT1bpt6coOmdFueTJ7DFJL4H1w5eF8yAQ2zsGNTtZ+i5SGFBWpb9AQaw168dydc3s4eu0W0Sirda+F59Q==",
       "dev": true,
       "requires": {
-        "ansi-wrap": "^0.1.0"
+        "@lerna/create-symlink": "3.16.2",
+        "@lerna/resolve-symlink": "3.16.0",
+        "@lerna/symlink-binary": "3.17.0",
+        "fs-extra": "^8.1.0",
+        "p-finally": "^1.0.0",
+        "p-map": "^2.1.0",
+        "p-map-series": "^1.0.0"
+      },
+      "dependencies": {
+        "p-map": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz",
+          "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==",
+          "dev": true
+        }
       }
     },
-    "ansi-escapes": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz",
-      "integrity": "sha512-UgAb8H9D41AQnu/PbWlCofQVcnV4Gs2bBJi9eZPxfU/hgglFh3SMDMENRIqdr7H6XFnXdoknctFByVsCOotTVw==",
+    "@lerna/timer": {
+      "version": "3.13.0",
+      "resolved": "https://registry.npmjs.org/@lerna/timer/-/timer-3.13.0.tgz",
+      "integrity": "sha512-RHWrDl8U4XNPqY5MQHkToWS9jHPnkLZEt5VD+uunCKTfzlxGnRCr3/zVr8VGy/uENMYpVP3wJa4RKGY6M0vkRw==",
       "dev": true
     },
-    "ansi-gray": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/ansi-gray/-/ansi-gray-0.1.1.tgz",
-      "integrity": "sha1-KWLPVOyXksSFEKPetSRDaGHvclE=",
+    "@lerna/validation-error": {
+      "version": "3.13.0",
+      "resolved": "https://registry.npmjs.org/@lerna/validation-error/-/validation-error-3.13.0.tgz",
+      "integrity": "sha512-SiJP75nwB8GhgwLKQfdkSnDufAaCbkZWJqEDlKOUPUvVOplRGnfL+BPQZH5nvq2BYSRXsksXWZ4UHVnQZI/HYA==",
       "dev": true,
       "requires": {
-        "ansi-wrap": "0.1.0"
+        "npmlog": "^4.1.2"
       }
     },
-    "ansi-regex": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
-      "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=",
-      "dev": true
-    },
-    "ansi-styles": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-      "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-      "requires": {
-        "color-convert": "^1.9.0"
+    "@lerna/version": {
+      "version": "3.22.1",
+      "resolved": "https://registry.npmjs.org/@lerna/version/-/version-3.22.1.tgz",
+      "integrity": "sha512-PSGt/K1hVqreAFoi3zjD0VEDupQ2WZVlVIwesrE5GbrL2BjXowjCsTDPqblahDUPy0hp6h7E2kG855yLTp62+g==",
+      "dev": true,
+      "requires": {
+        "@lerna/check-working-tree": "3.16.5",
+        "@lerna/child-process": "3.16.5",
+        "@lerna/collect-updates": "3.20.0",
+        "@lerna/command": "3.21.0",
+        "@lerna/conventional-commits": "3.22.0",
+        "@lerna/github-client": "3.22.0",
+        "@lerna/gitlab-client": "3.15.0",
+        "@lerna/output": "3.13.0",
+        "@lerna/prerelease-id-from-version": "3.16.0",
+        "@lerna/prompt": "3.18.5",
+        "@lerna/run-lifecycle": "3.16.2",
+        "@lerna/run-topologically": "3.18.5",
+        "@lerna/validation-error": "3.13.0",
+        "chalk": "^2.3.1",
+        "dedent": "^0.7.0",
+        "load-json-file": "^5.3.0",
+        "minimatch": "^3.0.4",
+        "npmlog": "^4.1.2",
+        "p-map": "^2.1.0",
+        "p-pipe": "^1.2.0",
+        "p-reduce": "^1.0.0",
+        "p-waterfall": "^1.0.0",
+        "semver": "^6.2.0",
+        "slash": "^2.0.0",
+        "temp-write": "^3.4.0",
+        "write-json-file": "^3.2.0"
+      },
+      "dependencies": {
+        "load-json-file": {
+          "version": "5.3.0",
+          "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-5.3.0.tgz",
+          "integrity": "sha512-cJGP40Jc/VXUsp8/OrnyKyTZ1y6v/dphm3bioS+RrKXjK2BB6wHUd6JptZEFDGgGahMT+InnZO5i1Ei9mpC8Bw==",
+          "dev": true,
+          "requires": {
+            "graceful-fs": "^4.1.15",
+            "parse-json": "^4.0.0",
+            "pify": "^4.0.1",
+            "strip-bom": "^3.0.0",
+            "type-fest": "^0.3.0"
+          }
+        },
+        "p-map": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz",
+          "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==",
+          "dev": true
+        },
+        "parse-json": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
+          "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=",
+          "dev": true,
+          "requires": {
+            "error-ex": "^1.3.1",
+            "json-parse-better-errors": "^1.0.1"
+          }
+        },
+        "pify": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz",
+          "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==",
+          "dev": true
+        },
+        "semver": {
+          "version": "6.3.0",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
+          "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
+          "dev": true
+        },
+        "slash": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
+          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
+          "dev": true
+        },
+        "strip-bom": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
+          "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=",
+          "dev": true
+        },
+        "type-fest": {
+          "version": "0.3.1",
+          "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.3.1.tgz",
+          "integrity": "sha512-cUGJnCdr4STbePCgqNFbpVNCepa+kAVohJs1sLhxzdH+gnEoOd8VhbYa7pD3zZYGiURWM2xzEII3fQcRizDkYQ==",
+          "dev": true
+        }
       }
     },
-    "ansi-wrap": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/ansi-wrap/-/ansi-wrap-0.1.0.tgz",
-      "integrity": "sha1-qCJQ3bABXponyoLoLqYDu/pF768=",
-      "dev": true
-    },
-    "anymatch": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-2.0.0.tgz",
-      "integrity": "sha512-5teOsQWABXHHBFP9y3skS5P3d/WfWXpv3FUpy+LorMrNYaT9pI4oLMQX7jzQ2KklNpGpWHzdCXTDT2Y3XGlZBw==",
+    "@lerna/write-log-file": {
+      "version": "3.13.0",
+      "resolved": "https://registry.npmjs.org/@lerna/write-log-file/-/write-log-file-3.13.0.tgz",
+      "integrity": "sha512-RibeMnDPvlL8bFYW5C8cs4mbI3AHfQef73tnJCQ/SgrXZHehmHnsyWUiE7qDQCAo+B1RfTapvSyFF69iPj326A==",
       "dev": true,
       "requires": {
-        "micromatch": "^3.1.4",
-        "normalize-path": "^2.1.1"
+        "npmlog": "^4.1.2",
+        "write-file-atomic": "^2.3.0"
+      },
+      "dependencies": {
+        "write-file-atomic": {
+          "version": "2.4.3",
+          "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.3.tgz",
+          "integrity": "sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==",
+          "dev": true,
+          "requires": {
+            "graceful-fs": "^4.1.11",
+            "imurmurhash": "^0.1.4",
+            "signal-exit": "^3.0.2"
+          }
+        }
       }
     },
-    "append-buffer": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/append-buffer/-/append-buffer-1.0.2.tgz",
-      "integrity": "sha1-2CIM9GYIFSXv6lBhTz3mUU36WPE=",
+    "@mrmlnc/readdir-enhanced": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz",
+      "integrity": "sha512-bPHp6Ji8b41szTOcaP63VlnbbO5Ny6dwAATtY6JTjh5N2OLrb5Qk/Th5cRkRQhkWCt+EJsYrNB0MiL+Gpn6e3g==",
       "dev": true,
       "requires": {
-        "buffer-equal": "^1.0.0"
+        "call-me-maybe": "^1.0.1",
+        "glob-to-regexp": "^0.3.0"
       }
     },
-    "aproba": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
-      "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==",
-      "dev": true
-    },
-    "archy": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz",
-      "integrity": "sha1-+cjBN1fMHde8N5rHeyxipcKGjEA=",
+    "@nodelib/fs.stat": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz",
+      "integrity": "sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw==",
       "dev": true
     },
-    "are-we-there-yet": {
-      "version": "1.1.5",
-      "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz",
-      "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==",
+    "@npmcli/move-file": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-1.0.1.tgz",
+      "integrity": "sha512-Uv6h1sT+0DrblvIrolFtbvM1FgWm+/sy4B3pvLp67Zys+thcukzS5ekn7HsZFGpWP4Q3fYJCljbWQE/XivMRLw==",
       "dev": true,
       "requires": {
-        "delegates": "^1.0.0",
-        "readable-stream": "^2.0.6"
+        "mkdirp": "^1.0.4"
       }
     },
-    "arg": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.0.tgz",
-      "integrity": "sha512-ZWc51jO3qegGkVh8Hwpv636EkbesNV5ZNQPCtRa+0qytRYPEs9IYT9qITY9buezqUH5uqyzlWLcufrzU2rffdg==",
-      "dev": true
-    },
-    "argparse": {
-      "version": "1.0.10",
-      "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
-      "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
+    "@octokit/auth-token": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.4.2.tgz",
+      "integrity": "sha512-jE/lE/IKIz2v1+/P0u4fJqv0kYwXOTujKemJMFr6FeopsxlIK3+wKDCJGnysg81XID5TgZQbIfuJ5J0lnTiuyQ==",
       "dev": true,
       "requires": {
-        "sprintf-js": "~1.0.2"
+        "@octokit/types": "^5.0.0"
       }
     },
-    "argv-tools": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/argv-tools/-/argv-tools-0.1.1.tgz",
-      "integrity": "sha512-Cc0dBvx4dvrjjKpyDA6w8RlNAw8Su30NvZbWl/Tv9ZALEVlLVkWQiHMi84Q0xNfpVuSaiQbYkdmWK8g1PLGhKw==",
+    "@octokit/endpoint": {
+      "version": "6.0.6",
+      "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-6.0.6.tgz",
+      "integrity": "sha512-7Cc8olaCoL/mtquB7j/HTbPM+sY6Ebr4k2X2y4JoXpVKQ7r5xB4iGQE0IoO58wIPsUk4AzoT65AMEpymSbWTgQ==",
+      "dev": true,
       "requires": {
-        "array-back": "^2.0.0",
-        "find-replace": "^2.0.1"
+        "@octokit/types": "^5.0.0",
+        "is-plain-object": "^5.0.0",
+        "universal-user-agent": "^6.0.0"
+      },
+      "dependencies": {
+        "is-plain-object": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz",
+          "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==",
+          "dev": true
+        },
+        "universal-user-agent": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz",
+          "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==",
+          "dev": true
+        }
       }
     },
-    "arr-diff": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz",
-      "integrity": "sha1-1kYQdP6/7HHn4VI1dhoyml3HxSA=",
+    "@octokit/plugin-enterprise-rest": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/@octokit/plugin-enterprise-rest/-/plugin-enterprise-rest-6.0.1.tgz",
+      "integrity": "sha512-93uGjlhUD+iNg1iWhUENAtJata6w5nE+V4urXOAlIXdco6xNZtUSfYY8dzp3Udy74aqO/B5UZL80x/YMa5PKRw==",
       "dev": true
     },
-    "arr-filter": {
+    "@octokit/plugin-paginate-rest": {
       "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/arr-filter/-/arr-filter-1.1.2.tgz",
-      "integrity": "sha1-Q/3d0JHo7xGqTEXZzcGOLf8XEe4=",
+      "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-1.1.2.tgz",
+      "integrity": "sha512-jbsSoi5Q1pj63sC16XIUboklNw+8tL9VOnJsWycWYR78TKss5PVpIPb1TUUcMQ+bBh7cY579cVAWmf5qG+dw+Q==",
       "dev": true,
       "requires": {
-        "make-iterator": "^1.0.0"
+        "@octokit/types": "^2.0.1"
+      },
+      "dependencies": {
+        "@octokit/types": {
+          "version": "2.16.2",
+          "resolved": "https://registry.npmjs.org/@octokit/types/-/types-2.16.2.tgz",
+          "integrity": "sha512-O75k56TYvJ8WpAakWwYRN8Bgu60KrmX0z1KqFp1kNiFNkgW+JW+9EBKZ+S33PU6SLvbihqd+3drvPxKK68Ee8Q==",
+          "dev": true,
+          "requires": {
+            "@types/node": ">= 8"
+          }
+        }
       }
     },
-    "arr-flatten": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz",
-      "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==",
+    "@octokit/plugin-request-log": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/@octokit/plugin-request-log/-/plugin-request-log-1.0.0.tgz",
+      "integrity": "sha512-ywoxP68aOT3zHCLgWZgwUJatiENeHE7xJzYjfz8WI0goynp96wETBF+d95b8g/uL4QmS6owPVlaxiz3wyMAzcw==",
       "dev": true
     },
-    "arr-map": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/arr-map/-/arr-map-2.0.2.tgz",
-      "integrity": "sha1-Onc0X/wc814qkYJWAfnljy4kysQ=",
+    "@octokit/plugin-rest-endpoint-methods": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-2.4.0.tgz",
+      "integrity": "sha512-EZi/AWhtkdfAYi01obpX0DF7U6b1VRr30QNQ5xSFPITMdLSfhcBqjamE3F+sKcxPbD7eZuMHu3Qkk2V+JGxBDQ==",
       "dev": true,
       "requires": {
-        "make-iterator": "^1.0.0"
-      }
-    },
-    "arr-union": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz",
-      "integrity": "sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ=",
-      "dev": true
-    },
-    "array-back": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/array-back/-/array-back-2.0.0.tgz",
-      "integrity": "sha512-eJv4pLLufP3g5kcZry0j6WXpIbzYw9GUB4mVJZno9wfwiBxbizTnHCw3VJb07cBihbFX48Y7oSrW9y+gt4glyw==",
-      "requires": {
-        "typical": "^2.6.1"
+        "@octokit/types": "^2.0.1",
+        "deprecation": "^2.3.1"
+      },
+      "dependencies": {
+        "@octokit/types": {
+          "version": "2.16.2",
+          "resolved": "https://registry.npmjs.org/@octokit/types/-/types-2.16.2.tgz",
+          "integrity": "sha512-O75k56TYvJ8WpAakWwYRN8Bgu60KrmX0z1KqFp1kNiFNkgW+JW+9EBKZ+S33PU6SLvbihqd+3drvPxKK68Ee8Q==",
+          "dev": true,
+          "requires": {
+            "@types/node": ">= 8"
+          }
+        }
       }
     },
-    "array-differ": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/array-differ/-/array-differ-1.0.0.tgz",
-      "integrity": "sha1-7/UuN1gknTO+QCuLuOVkuytdQDE=",
-      "dev": true
-    },
-    "array-each": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/array-each/-/array-each-1.0.1.tgz",
-      "integrity": "sha1-p5SvDAWrF1KEbudTofIRoFugxE8=",
-      "dev": true
-    },
-    "array-equal": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz",
-      "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=",
-      "dev": true
-    },
-    "array-filter": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/array-filter/-/array-filter-0.0.1.tgz",
-      "integrity": "sha1-fajPLiZijtcygDWB/SH2fKzS7uw=",
-      "dev": true
-    },
-    "array-find-index": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/array-find-index/-/array-find-index-1.0.2.tgz",
-      "integrity": "sha1-3wEKoSh+Fku9pvlyOwqWoexBh6E=",
-      "dev": true
-    },
-    "array-ify": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/array-ify/-/array-ify-1.0.0.tgz",
-      "integrity": "sha1-nlKHYrSpBmrRY6aWKjZEGOlibs4=",
-      "dev": true
-    },
-    "array-initial": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/array-initial/-/array-initial-1.1.0.tgz",
-      "integrity": "sha1-L6dLJnOTccOUe9enrcc74zSz15U=",
+    "@octokit/request": {
+      "version": "5.4.9",
+      "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.4.9.tgz",
+      "integrity": "sha512-CzwVvRyimIM1h2n9pLVYfTDmX9m+KHSgCpqPsY8F1NdEK8IaWqXhSBXsdjOBFZSpEcxNEeg4p0UO9cQ8EnOCLA==",
       "dev": true,
       "requires": {
-        "array-slice": "^1.0.0",
-        "is-number": "^4.0.0"
+        "@octokit/endpoint": "^6.0.1",
+        "@octokit/request-error": "^2.0.0",
+        "@octokit/types": "^5.0.0",
+        "deprecation": "^2.0.0",
+        "is-plain-object": "^5.0.0",
+        "node-fetch": "^2.6.1",
+        "once": "^1.4.0",
+        "universal-user-agent": "^6.0.0"
       },
       "dependencies": {
-        "is-number": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/is-number/-/is-number-4.0.0.tgz",
-          "integrity": "sha512-rSklcAIlf1OmFdyAqbnWTLVelsQ58uvZ66S/ZyawjWqIviTWCjg2PzVGw8WUA+nNuPTqb4wgA+NszrJ+08LlgQ==",
+        "@octokit/request-error": {
+          "version": "2.0.2",
+          "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.0.2.tgz",
+          "integrity": "sha512-2BrmnvVSV1MXQvEkrb9zwzP0wXFNbPJij922kYBTLIlIafukrGOb+ABBT2+c6wZiuyWDH1K1zmjGQ0toN/wMWw==",
+          "dev": true,
+          "requires": {
+            "@octokit/types": "^5.0.1",
+            "deprecation": "^2.0.0",
+            "once": "^1.4.0"
+          }
+        },
+        "is-plain-object": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz",
+          "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==",
+          "dev": true
+        },
+        "universal-user-agent": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz",
+          "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==",
           "dev": true
         }
       }
     },
-    "array-last": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/array-last/-/array-last-1.3.0.tgz",
-      "integrity": "sha512-eOCut5rXlI6aCOS7Z7kCplKRKyiFQ6dHFBem4PwlwKeNFk2/XxTrhRh5T9PyaEWGy/NHTZWbY+nsZlNFJu9rYg==",
+    "@octokit/request-error": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-1.2.1.tgz",
+      "integrity": "sha512-+6yDyk1EES6WK+l3viRDElw96MvwfJxCt45GvmjDUKWjYIb3PJZQkq3i46TwGwoPD4h8NmTrENmtyA1FwbmhRA==",
       "dev": true,
       "requires": {
-        "is-number": "^4.0.0"
+        "@octokit/types": "^2.0.0",
+        "deprecation": "^2.0.0",
+        "once": "^1.4.0"
       },
       "dependencies": {
-        "is-number": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/is-number/-/is-number-4.0.0.tgz",
-          "integrity": "sha512-rSklcAIlf1OmFdyAqbnWTLVelsQ58uvZ66S/ZyawjWqIviTWCjg2PzVGw8WUA+nNuPTqb4wgA+NszrJ+08LlgQ==",
-          "dev": true
+        "@octokit/types": {
+          "version": "2.16.2",
+          "resolved": "https://registry.npmjs.org/@octokit/types/-/types-2.16.2.tgz",
+          "integrity": "sha512-O75k56TYvJ8WpAakWwYRN8Bgu60KrmX0z1KqFp1kNiFNkgW+JW+9EBKZ+S33PU6SLvbihqd+3drvPxKK68Ee8Q==",
+          "dev": true,
+          "requires": {
+            "@types/node": ">= 8"
+          }
         }
       }
     },
-    "array-map": {
-      "version": "0.0.0",
-      "resolved": "https://registry.npmjs.org/array-map/-/array-map-0.0.0.tgz",
-      "integrity": "sha1-iKK6tz0c97zVwbEYoAP2b2ZfpmI=",
-      "dev": true
-    },
-    "array-reduce": {
-      "version": "0.0.0",
-      "resolved": "https://registry.npmjs.org/array-reduce/-/array-reduce-0.0.0.tgz",
-      "integrity": "sha1-FziZ0//Rx9k4PkR5Ul2+J4yrXys=",
-      "dev": true
+    "@octokit/rest": {
+      "version": "16.43.2",
+      "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-16.43.2.tgz",
+      "integrity": "sha512-ngDBevLbBTFfrHZeiS7SAMAZ6ssuVmXuya+F/7RaVvlysgGa1JKJkKWY+jV6TCJYcW0OALfJ7nTIGXcBXzycfQ==",
+      "dev": true,
+      "requires": {
+        "@octokit/auth-token": "^2.4.0",
+        "@octokit/plugin-paginate-rest": "^1.1.1",
+        "@octokit/plugin-request-log": "^1.0.0",
+        "@octokit/plugin-rest-endpoint-methods": "2.4.0",
+        "@octokit/request": "^5.2.0",
+        "@octokit/request-error": "^1.0.2",
+        "atob-lite": "^2.0.0",
+        "before-after-hook": "^2.0.0",
+        "btoa-lite": "^1.0.0",
+        "deprecation": "^2.0.0",
+        "lodash.get": "^4.4.2",
+        "lodash.set": "^4.3.2",
+        "lodash.uniq": "^4.5.0",
+        "octokit-pagination-methods": "^1.1.0",
+        "once": "^1.4.0",
+        "universal-user-agent": "^4.0.0"
+      }
     },
-    "array-slice": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/array-slice/-/array-slice-1.1.0.tgz",
-      "integrity": "sha512-B1qMD3RBP7O8o0H2KbrXDyB0IccejMF15+87Lvlor12ONPRHP6gTjXMNkt/d3ZuOGbAe66hFmaCfECI24Ufp6w==",
-      "dev": true
+    "@octokit/types": {
+      "version": "5.5.0",
+      "resolved": "https://registry.npmjs.org/@octokit/types/-/types-5.5.0.tgz",
+      "integrity": "sha512-UZ1pErDue6bZNjYOotCNveTXArOMZQFG6hKJfOnGnulVCMcVVi7YIIuuR4WfBhjo7zgpmzn/BkPDnUXtNx+PcQ==",
+      "dev": true,
+      "requires": {
+        "@types/node": ">= 8"
+      }
     },
-    "array-sort": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/array-sort/-/array-sort-1.0.0.tgz",
-      "integrity": "sha512-ihLeJkonmdiAsD7vpgN3CRcx2J2S0TiYW+IS/5zHBI7mKUq3ySvBdzzBfD236ubDBQFiiyG3SWCPc+msQ9KoYg==",
+    "@sindresorhus/df": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/@sindresorhus/df/-/df-2.1.0.tgz",
+      "integrity": "sha1-0gjPJ+BvC7R20U197M19cm6ao4k=",
       "dev": true,
       "requires": {
-        "default-compare": "^1.0.0",
-        "get-value": "^2.0.6",
-        "kind-of": "^5.0.2"
+        "execa": "^0.2.2"
       },
       "dependencies": {
-        "kind-of": {
-          "version": "5.1.0",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
-          "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==",
+        "execa": {
+          "version": "0.2.2",
+          "resolved": "https://registry.npmjs.org/execa/-/execa-0.2.2.tgz",
+          "integrity": "sha1-4urUcsLDGq1vc/GslW7vReEjIMs=",
+          "dev": true,
+          "requires": {
+            "cross-spawn-async": "^2.1.1",
+            "npm-run-path": "^1.0.0",
+            "object-assign": "^4.0.1",
+            "path-key": "^1.0.0",
+            "strip-eof": "^1.0.0"
+          }
+        },
+        "npm-run-path": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-1.0.0.tgz",
+          "integrity": "sha1-9cMr9ZX+ga6Sfa7FLoL4sACsPI8=",
+          "dev": true,
+          "requires": {
+            "path-key": "^1.0.0"
+          }
+        },
+        "path-key": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/path-key/-/path-key-1.0.0.tgz",
+          "integrity": "sha1-XVPVeAGWRsDWiADbThRua9wqx68=",
           "dev": true
         }
       }
     },
-    "array-union": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz",
-      "integrity": "sha1-mjRBDk9OPaI96jdb5b5w8kd47Dk=",
+    "@sinonjs/commons": {
+      "version": "1.8.1",
+      "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.1.tgz",
+      "integrity": "sha512-892K+kWUUi3cl+LlqEWIDrhvLgdL79tECi8JZUyq6IviKy/DNhuzCRlbHUjxK89f4ypPMMaFnFuR9Ie6DoIMsw==",
       "dev": true,
       "requires": {
-        "array-uniq": "^1.0.1"
+        "type-detect": "4.0.8"
       }
     },
-    "array-uniq": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz",
-      "integrity": "sha1-r2rId6Jcx/dOBYiUdThY39sk/bY=",
-      "dev": true
-    },
-    "array-unique": {
-      "version": "0.3.2",
-      "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz",
-      "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=",
-      "dev": true
-    },
-    "arrify": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz",
-      "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=",
-      "dev": true
+    "@sinonjs/fake-timers": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-6.0.1.tgz",
+      "integrity": "sha512-MZPUxrmFubI36XS1DI3qmI0YdN1gks62JtFZvxR67ljjSNCeK6U08Zx4msEWOXuofgqUt6zPHSi1H9fbjR/NRA==",
+      "dev": true,
+      "requires": {
+        "@sinonjs/commons": "^1.7.0"
+      }
     },
-    "asap": {
-      "version": "2.0.6",
-      "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
-      "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=",
-      "dev": true
+    "@types/babel__core": {
+      "version": "7.1.9",
+      "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.9.tgz",
+      "integrity": "sha512-sY2RsIJ5rpER1u3/aQ8OFSI7qGIy8o1NEEbgb2UaJcvOtXOMpd39ko723NBpjQFg9SIX7TXtjejZVGeIMLhoOw==",
+      "dev": true,
+      "requires": {
+        "@babel/parser": "^7.1.0",
+        "@babel/types": "^7.0.0",
+        "@types/babel__generator": "*",
+        "@types/babel__template": "*",
+        "@types/babel__traverse": "*"
+      }
     },
-    "asn1": {
-      "version": "0.2.4",
-      "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
-      "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==",
+    "@types/babel__generator": {
+      "version": "7.6.1",
+      "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.1.tgz",
+      "integrity": "sha512-bBKm+2VPJcMRVwNhxKu8W+5/zT7pwNEqeokFOmbvVSqGzFneNxYcEBro9Ac7/N9tlsaPYnZLK8J1LWKkMsLAew==",
       "dev": true,
       "requires": {
-        "safer-buffer": "~2.1.0"
+        "@babel/types": "^7.0.0"
       }
     },
-    "asn1.js": {
-      "version": "4.10.1",
-      "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-4.10.1.tgz",
-      "integrity": "sha512-p32cOF5q0Zqs9uBiONKYLm6BClCoBCM5O9JfeUSlnQLBTxYdTK+pW+nXflm8UkKd2UYlEbYz5qEi0JuZR9ckSw==",
+    "@types/babel__template": {
+      "version": "7.0.2",
+      "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.0.2.tgz",
+      "integrity": "sha512-/K6zCpeW7Imzgab2bLkLEbz0+1JlFSrUMdw7KoIIu+IUdu51GWaBZpd3y1VXGVXzynvGa4DaIaxNZHiON3GXUg==",
       "dev": true,
       "requires": {
-        "bn.js": "^4.0.0",
-        "inherits": "^2.0.1",
-        "minimalistic-assert": "^1.0.0"
+        "@babel/parser": "^7.1.0",
+        "@babel/types": "^7.0.0"
       }
     },
-    "assert": {
-      "version": "1.4.1",
-      "resolved": "https://registry.npmjs.org/assert/-/assert-1.4.1.tgz",
-      "integrity": "sha1-mZEtWRg2tab1s0XA8H7vwI/GXZE=",
+    "@types/babel__traverse": {
+      "version": "7.0.14",
+      "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.0.14.tgz",
+      "integrity": "sha512-8w9szzKs14ZtBVuP6Wn7nMLRJ0D6dfB0VEBEyRgxrZ/Ln49aNMykrghM2FaNn4FJRzNppCSa0Rv9pBRM5Xc3wg==",
       "dev": true,
       "requires": {
-        "util": "0.10.3"
-      },
-      "dependencies": {
-        "inherits": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz",
-          "integrity": "sha1-sX0I0ya0Qj5Wjv9xn5GwscvfafE=",
-          "dev": true
-        },
-        "util": {
-          "version": "0.10.3",
-          "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz",
-          "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=",
-          "dev": true,
-          "requires": {
-            "inherits": "2.0.1"
-          }
-        }
+        "@babel/types": "^7.3.0"
       }
     },
-    "assert-plus": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
-      "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
+    "@types/color-name": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz",
+      "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==",
       "dev": true
     },
-    "assign-symbols": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz",
-      "integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c=",
+    "@types/events": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/@types/events/-/events-3.0.0.tgz",
+      "integrity": "sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==",
       "dev": true
     },
-    "astral-regex": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz",
-      "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==",
-      "dev": true
+    "@types/flatbuffers": {
+      "version": "1.10.0",
+      "resolved": "https://registry.npmjs.org/@types/flatbuffers/-/flatbuffers-1.10.0.tgz",
+      "integrity": "sha512-7btbphLrKvo5yl/5CC2OCxUSMx1wV1wvGT1qDXkSt7yi00/YW7E8k6qzXqJHsp+WU0eoG7r6MTQQXI9lIvd0qA=="
     },
-    "async": {
-      "version": "2.6.1",
-      "resolved": "https://registry.npmjs.org/async/-/async-2.6.1.tgz",
-      "integrity": "sha512-fNEiL2+AZt6AlAw/29Cr0UDe4sRAHCpEHh54WMz+Bb7QfNcFw4h3loofyJpLeQs4Yx7yuqu/2dLgM5hKOs6HlQ==",
+    "@types/glob": {
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.1.tgz",
+      "integrity": "sha512-1Bh06cbWJUHMC97acuD6UMG29nMt0Aqz1vF3guLfG+kHHJhy3AyohZFFxYk2f7Q1SQIrNwvncxAE0N/9s70F2w==",
       "dev": true,
       "requires": {
-        "lodash": "^4.17.10"
+        "@types/events": "*",
+        "@types/minimatch": "*",
+        "@types/node": "*"
       }
     },
-    "async-done": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/async-done/-/async-done-1.3.1.tgz",
-      "integrity": "sha512-R1BaUeJ4PMoLNJuk+0tLJgjmEqVsdN118+Z8O+alhnQDQgy0kmD5Mqi0DNEmMx2LM0Ed5yekKu+ZXYvIHceicg==",
+    "@types/graceful-fs": {
+      "version": "4.1.3",
+      "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.3.tgz",
+      "integrity": "sha512-AiHRaEB50LQg0pZmm659vNBb9f4SJ0qrAnteuzhSeAUcJKxoYgEnprg/83kppCnc2zvtCKbdZry1a5pVY3lOTQ==",
       "dev": true,
       "requires": {
-        "end-of-stream": "^1.1.0",
-        "once": "^1.3.2",
-        "process-nextick-args": "^1.0.7",
-        "stream-exhaust": "^1.0.1"
+        "@types/node": "*"
       }
     },
-    "async-each": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/async-each/-/async-each-1.0.1.tgz",
-      "integrity": "sha1-GdOGodntxufByF04iu28xW0zYC0=",
+    "@types/istanbul-lib-coverage": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz",
+      "integrity": "sha512-sz7iLqvVUg1gIedBOvlkxPlc8/uVzyS5OwGz1cKjXzkl3FpL3al0crU8YGU1WoHkxn0Wxbw5tyi6hvzJKNzFsw==",
       "dev": true
     },
-    "async-limiter": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz",
-      "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==",
-      "dev": true
+    "@types/istanbul-lib-report": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz",
+      "integrity": "sha512-plGgXAPfVKFoYfa9NpYDAkseG+g6Jr294RqeqcqDixSbU34MZVJRi/P+7Y8GDpzkEwLaGZZOpKIEmeVZNtKsrg==",
+      "dev": true,
+      "requires": {
+        "@types/istanbul-lib-coverage": "*"
+      }
     },
-    "async-settle": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/async-settle/-/async-settle-1.0.0.tgz",
-      "integrity": "sha1-HQqRS7Aldb7IqPOnTlCA9yssDGs=",
+    "@types/istanbul-reports": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-1.1.2.tgz",
+      "integrity": "sha512-P/W9yOX/3oPZSpaYOCQzGqgCQRXn0FFO/V8bWrCQs+wLmvVVxk6CRBXALEvNs9OHIatlnlFokfhuDo2ug01ciw==",
       "dev": true,
       "requires": {
-        "async-done": "^1.2.2"
+        "@types/istanbul-lib-coverage": "*",
+        "@types/istanbul-lib-report": "*"
       }
     },
-    "asynckit": {
-      "version": "0.4.0",
-      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=",
+    "@types/jest": {
+      "version": "25.2.2",
+      "resolved": "https://registry.npmjs.org/@types/jest/-/jest-25.2.2.tgz",
+      "integrity": "sha512-aRctFbG8Pb7DSLzUt/fEtL3q/GKb9mretFuYhRub2J0q6NhzBYbx9HTQzHrWgBNIxYOlxGNVe6Z54cpbUt+Few==",
+      "dev": true,
+      "requires": {
+        "jest-diff": "^25.2.1",
+        "pretty-format": "^25.2.1"
+      }
+    },
+    "@types/json-schema": {
+      "version": "7.0.6",
+      "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.6.tgz",
+      "integrity": "sha512-3c+yGKvVP5Y9TYBEibGNR+kLtijnj7mYrXRg+WpFb2X9xm04g/DXYkfg4hmzJQosc9snFNUPkbYIhu+KAm6jJw==",
       "dev": true
     },
-    "atob": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz",
-      "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==",
+    "@types/minimatch": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz",
+      "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==",
       "dev": true
     },
-    "aws-sign2": {
-      "version": "0.7.0",
-      "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
-      "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=",
+    "@types/minimist": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.0.tgz",
+      "integrity": "sha1-aaI6OtKcrwCX8G7aWbNh7i8GOfY=",
       "dev": true
     },
-    "aws4": {
-      "version": "1.8.0",
-      "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz",
-      "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==",
+    "@types/node": {
+      "version": "12.12.62",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.62.tgz",
+      "integrity": "sha512-qAfo81CsD7yQIM9mVyh6B/U47li5g7cfpVQEDMfQeF8pSZVwzbhwU3crc0qG4DmpsebpJPR49AKOExQyJ05Cpg=="
+    },
+    "@types/normalize-package-data": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz",
+      "integrity": "sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==",
+      "dev": true
+    },
+    "@types/prettier": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.1.1.tgz",
+      "integrity": "sha512-2zs+O+UkDsJ1Vcp667pd3f8xearMdopz/z54i99wtRDI5KLmngk7vlrYZD0ZjKHaROR03EznlBbVY9PfAEyJIQ==",
+      "dev": true
+    },
+    "@types/stack-utils": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-1.0.1.tgz",
+      "integrity": "sha512-l42BggppR6zLmpfU6fq9HEa2oGPEI8yrSPL3GITjfRInppYFahObbIQOQK3UGxEnyQpltZLaPe75046NOZQikw==",
       "dev": true
     },
-    "babel-code-frame": {
-      "version": "6.26.0",
-      "resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz",
-      "integrity": "sha1-Y/1D99weO7fONZR9uP42mj9Yx0s=",
+    "@types/text-encoding-utf-8": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@types/text-encoding-utf-8/-/text-encoding-utf-8-1.0.1.tgz",
+      "integrity": "sha512-GpIEYaS+yNfYqpowLLziiY42pyaL+lThd/wMh6tTubaKuG4IRkXqqyxK7Nddn3BvpUg2+go3Gv/jbXvAFMRjiQ=="
+    },
+    "@types/yargs": {
+      "version": "15.0.6",
+      "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-15.0.6.tgz",
+      "integrity": "sha512-sw6JBfeZeE7ep9Pmhcysxo4lgaMLlHuQ8ie0srx5VOEnN4RNJgLryCuvxCUotw6Jmd50zBwCJHs3iwjLyIJgbQ==",
       "dev": true,
       "requires": {
-        "chalk": "^1.1.3",
-        "esutils": "^2.0.2",
-        "js-tokens": "^3.0.2"
-      },
-      "dependencies": {
-        "ansi-styles": {
-          "version": "2.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
-          "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
-          "dev": true
-        },
-        "chalk": {
-          "version": "1.1.3",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
-          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
-          "dev": true,
-          "requires": {
-            "ansi-styles": "^2.2.1",
-            "escape-string-regexp": "^1.0.2",
-            "has-ansi": "^2.0.0",
-            "strip-ansi": "^3.0.0",
-            "supports-color": "^2.0.0"
-          }
-        },
-        "supports-color": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
-          "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=",
-          "dev": true
-        }
+        "@types/yargs-parser": "*"
       }
     },
-    "babel-jest": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-24.8.0.tgz",
-      "integrity": "sha512-+5/kaZt4I9efoXzPlZASyK/lN9qdRKmmUav9smVc0ruPQD7IsfucQ87gpOE8mn2jbDuS6M/YOW6n3v9ZoIfgnw==",
+    "@types/yargs-parser": {
+      "version": "15.0.0",
+      "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-15.0.0.tgz",
+      "integrity": "sha512-FA/BWv8t8ZWJ+gEOnLLd8ygxH/2UFbAvgEonyfN6yWGLKc7zVjbpl2Y4CTjid9h2RfgPP6SEt6uHwEOply00yw==",
+      "dev": true
+    },
+    "@webassemblyjs/ast": {
+      "version": "1.7.11",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.7.11.tgz",
+      "integrity": "sha512-ZEzy4vjvTzScC+SH8RBssQUawpaInUdMTYwYYLh54/s8TuT0gBLuyUnppKsVyZEi876VmmStKsUs28UxPgdvrA==",
       "dev": true,
       "requires": {
-        "@jest/transform": "^24.8.0",
-        "@jest/types": "^24.8.0",
-        "@types/babel__core": "^7.1.0",
-        "babel-plugin-istanbul": "^5.1.0",
-        "babel-preset-jest": "^24.6.0",
-        "chalk": "^2.4.2",
-        "slash": "^2.0.0"
-      },
-      "dependencies": {
-        "slash": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
-          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
-          "dev": true
-        }
+        "@webassemblyjs/helper-module-context": "1.7.11",
+        "@webassemblyjs/helper-wasm-bytecode": "1.7.11",
+        "@webassemblyjs/wast-parser": "1.7.11"
       }
     },
-    "babel-plugin-istanbul": {
-      "version": "5.1.4",
-      "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-5.1.4.tgz",
-      "integrity": "sha512-dySz4VJMH+dpndj0wjJ8JPs/7i1TdSPb1nRrn56/92pKOF9VKC1FMFJmMXjzlGGusnCAqujP6PBCiKq0sVA+YQ==",
+    "@webassemblyjs/floating-point-hex-parser": {
+      "version": "1.7.11",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.7.11.tgz",
+      "integrity": "sha512-zY8dSNyYcgzNRNT666/zOoAyImshm3ycKdoLsyDw/Bwo6+/uktb7p4xyApuef1dwEBo/U/SYQzbGBvV+nru2Xg==",
+      "dev": true
+    },
+    "@webassemblyjs/helper-api-error": {
+      "version": "1.7.11",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.7.11.tgz",
+      "integrity": "sha512-7r1qXLmiglC+wPNkGuXCvkmalyEstKVwcueZRP2GNC2PAvxbLYwLLPr14rcdJaE4UtHxQKfFkuDFuv91ipqvXg==",
+      "dev": true
+    },
+    "@webassemblyjs/helper-buffer": {
+      "version": "1.7.11",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.7.11.tgz",
+      "integrity": "sha512-MynuervdylPPh3ix+mKZloTcL06P8tenNH3sx6s0qE8SLR6DdwnfgA7Hc9NSYeob2jrW5Vql6GVlsQzKQCa13w==",
+      "dev": true
+    },
+    "@webassemblyjs/helper-code-frame": {
+      "version": "1.7.11",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-code-frame/-/helper-code-frame-1.7.11.tgz",
+      "integrity": "sha512-T8ESC9KMXFTXA5urJcyor5cn6qWeZ4/zLPyWeEXZ03hj/x9weSokGNkVCdnhSabKGYWxElSdgJ+sFa9G/RdHNw==",
       "dev": true,
       "requires": {
-        "find-up": "^3.0.0",
-        "istanbul-lib-instrument": "^3.3.0",
-        "test-exclude": "^5.2.3"
-      },
-      "dependencies": {
-        "find-up": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz",
-          "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==",
-          "dev": true,
-          "requires": {
-            "locate-path": "^3.0.0"
-          }
-        },
-        "locate-path": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz",
-          "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==",
-          "dev": true,
-          "requires": {
-            "p-locate": "^3.0.0",
-            "path-exists": "^3.0.0"
-          }
-        },
-        "p-limit": {
-          "version": "2.2.0",
-          "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.0.tgz",
-          "integrity": "sha512-pZbTJpoUsCzV48Mc9Nh51VbwO0X9cuPFE8gYwx9BTCt9SF8/b7Zljd2fVgOxhIF/HDTKgpVzs+GPhyKfjLLFRQ==",
-          "dev": true,
-          "requires": {
-            "p-try": "^2.0.0"
-          }
-        },
-        "p-locate": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz",
-          "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==",
-          "dev": true,
-          "requires": {
-            "p-limit": "^2.0.0"
-          }
-        },
-        "p-try": {
-          "version": "2.2.0",
-          "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
-          "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==",
-          "dev": true
-        },
-        "path-exists": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
-          "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=",
-          "dev": true
-        }
+        "@webassemblyjs/wast-printer": "1.7.11"
       }
     },
-    "babel-plugin-jest-hoist": {
-      "version": "24.6.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-24.6.0.tgz",
-      "integrity": "sha512-3pKNH6hMt9SbOv0F3WVmy5CWQ4uogS3k0GY5XLyQHJ9EGpAT9XWkFd2ZiXXtkwFHdAHa5j7w7kfxSP5lAIwu7w==",
+    "@webassemblyjs/helper-fsm": {
+      "version": "1.7.11",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-fsm/-/helper-fsm-1.7.11.tgz",
+      "integrity": "sha512-nsAQWNP1+8Z6tkzdYlXT0kxfa2Z1tRTARd8wYnc/e3Zv3VydVVnaeePgqUzFrpkGUyhUUxOl5ML7f1NuT+gC0A==",
+      "dev": true
+    },
+    "@webassemblyjs/helper-module-context": {
+      "version": "1.7.11",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-module-context/-/helper-module-context-1.7.11.tgz",
+      "integrity": "sha512-JxfD5DX8Ygq4PvXDucq0M+sbUFA7BJAv/GGl9ITovqE+idGX+J3QSzJYz+LwQmL7fC3Rs+utvWoJxDb6pmC0qg==",
+      "dev": true
+    },
+    "@webassemblyjs/helper-wasm-bytecode": {
+      "version": "1.7.11",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.7.11.tgz",
+      "integrity": "sha512-cMXeVS9rhoXsI9LLL4tJxBgVD/KMOKXuFqYb5oCJ/opScWpkCMEz9EJtkonaNcnLv2R3K5jIeS4TRj/drde1JQ==",
+      "dev": true
+    },
+    "@webassemblyjs/helper-wasm-section": {
+      "version": "1.7.11",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.7.11.tgz",
+      "integrity": "sha512-8ZRY5iZbZdtNFE5UFunB8mmBEAbSI3guwbrsCl4fWdfRiAcvqQpeqd5KHhSWLL5wuxo53zcaGZDBU64qgn4I4Q==",
       "dev": true,
       "requires": {
-        "@types/babel__traverse": "^7.0.6"
+        "@webassemblyjs/ast": "1.7.11",
+        "@webassemblyjs/helper-buffer": "1.7.11",
+        "@webassemblyjs/helper-wasm-bytecode": "1.7.11",
+        "@webassemblyjs/wasm-gen": "1.7.11"
       }
     },
-    "babel-preset-jest": {
-      "version": "24.6.0",
-      "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-24.6.0.tgz",
-      "integrity": "sha512-pdZqLEdmy1ZK5kyRUfvBb2IfTPb2BUvIJczlPspS8fWmBQslNNDBqVfh7BW5leOVJMDZKzjD8XEyABTk6gQ5yw==",
+    "@webassemblyjs/ieee754": {
+      "version": "1.7.11",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.7.11.tgz",
+      "integrity": "sha512-Mmqx/cS68K1tSrvRLtaV/Lp3NZWzXtOHUW2IvDvl2sihAwJh4ACE0eL6A8FvMyDG9abes3saB6dMimLOs+HMoQ==",
       "dev": true,
       "requires": {
-        "@babel/plugin-syntax-object-rest-spread": "^7.0.0",
-        "babel-plugin-jest-hoist": "^24.6.0"
+        "@xtuc/ieee754": "^1.2.0"
       }
     },
-    "bach": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/bach/-/bach-1.2.0.tgz",
-      "integrity": "sha1-Szzpa/JxNPeaG0FKUcFONMO9mIA=",
+    "@webassemblyjs/leb128": {
+      "version": "1.7.11",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.7.11.tgz",
+      "integrity": "sha512-vuGmgZjjp3zjcerQg+JA+tGOncOnJLWVkt8Aze5eWQLwTQGNgVLcyOTqgSCxWTR4J42ijHbBxnuRaL1Rv7XMdw==",
       "dev": true,
       "requires": {
-        "arr-filter": "^1.1.1",
-        "arr-flatten": "^1.0.1",
-        "arr-map": "^2.0.0",
-        "array-each": "^1.0.0",
-        "array-initial": "^1.0.0",
-        "array-last": "^1.1.1",
-        "async-done": "^1.2.2",
-        "async-settle": "^1.0.0",
-        "now-and-later": "^2.0.0"
+        "@xtuc/long": "4.2.1"
       }
     },
-    "backbone": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/backbone/-/backbone-1.4.0.tgz",
-      "integrity": "sha512-RLmDrRXkVdouTg38jcgHhyQ/2zjg7a8E6sz2zxfz21Hh17xDJYUHBZimVIt5fUyS8vbfpeSmTL3gUjTEvUV3qQ==",
+    "@webassemblyjs/utf8": {
+      "version": "1.7.11",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.7.11.tgz",
+      "integrity": "sha512-C6GFkc7aErQIAH+BMrIdVSmW+6HSe20wg57HEC1uqJP8E/xpMjXqQUxkQw07MhNDSDcGpxI9G5JSNOQCqJk4sA==",
+      "dev": true
+    },
+    "@webassemblyjs/wasm-edit": {
+      "version": "1.7.11",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.7.11.tgz",
+      "integrity": "sha512-FUd97guNGsCZQgeTPKdgxJhBXkUbMTY6hFPf2Y4OedXd48H97J+sOY2Ltaq6WGVpIH8o/TGOVNiVz/SbpEMJGg==",
       "dev": true,
       "requires": {
-        "underscore": ">=1.8.3"
+        "@webassemblyjs/ast": "1.7.11",
+        "@webassemblyjs/helper-buffer": "1.7.11",
+        "@webassemblyjs/helper-wasm-bytecode": "1.7.11",
+        "@webassemblyjs/helper-wasm-section": "1.7.11",
+        "@webassemblyjs/wasm-gen": "1.7.11",
+        "@webassemblyjs/wasm-opt": "1.7.11",
+        "@webassemblyjs/wasm-parser": "1.7.11",
+        "@webassemblyjs/wast-printer": "1.7.11"
       }
     },
-    "balanced-match": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
-      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
-      "dev": true
-    },
-    "base": {
-      "version": "0.11.2",
-      "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz",
-      "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==",
+    "@webassemblyjs/wasm-gen": {
+      "version": "1.7.11",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.7.11.tgz",
+      "integrity": "sha512-U/KDYp7fgAZX5KPfq4NOupK/BmhDc5Kjy2GIqstMhvvdJRcER/kUsMThpWeRP8BMn4LXaKhSTggIJPOeYHwISA==",
       "dev": true,
       "requires": {
-        "cache-base": "^1.0.1",
-        "class-utils": "^0.3.5",
-        "component-emitter": "^1.2.1",
-        "define-property": "^1.0.0",
-        "isobject": "^3.0.1",
-        "mixin-deep": "^1.2.0",
-        "pascalcase": "^0.1.1"
-      },
-      "dependencies": {
-        "define-property": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
-          "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=",
-          "dev": true,
-          "requires": {
-            "is-descriptor": "^1.0.0"
-          }
-        },
-        "is-accessor-descriptor": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
-          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
-          "dev": true,
-          "requires": {
-            "kind-of": "^6.0.0"
-          }
-        },
-        "is-data-descriptor": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
-          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
-          "dev": true,
-          "requires": {
-            "kind-of": "^6.0.0"
-          }
-        },
-        "is-descriptor": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
-          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
-          "dev": true,
-          "requires": {
-            "is-accessor-descriptor": "^1.0.0",
-            "is-data-descriptor": "^1.0.0",
-            "kind-of": "^6.0.2"
-          }
-        }
+        "@webassemblyjs/ast": "1.7.11",
+        "@webassemblyjs/helper-wasm-bytecode": "1.7.11",
+        "@webassemblyjs/ieee754": "1.7.11",
+        "@webassemblyjs/leb128": "1.7.11",
+        "@webassemblyjs/utf8": "1.7.11"
       }
     },
-    "base64-js": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz",
-      "integrity": "sha512-ccav/yGvoa80BQDljCxsmmQ3Xvx60/UpBIij5QN21W3wBi/hhIC9OoO+KLpu9IJTS9j4DRVJ3aDDF9cMSoa2lw==",
-      "dev": true
-    },
-    "bcrypt-pbkdf": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
-      "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=",
+    "@webassemblyjs/wasm-opt": {
+      "version": "1.7.11",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.7.11.tgz",
+      "integrity": "sha512-XynkOwQyiRidh0GLua7SkeHvAPXQV/RxsUeERILmAInZegApOUAIJfRuPYe2F7RcjOC9tW3Cb9juPvAC/sCqvg==",
       "dev": true,
       "requires": {
-        "tweetnacl": "^0.14.3"
+        "@webassemblyjs/ast": "1.7.11",
+        "@webassemblyjs/helper-buffer": "1.7.11",
+        "@webassemblyjs/wasm-gen": "1.7.11",
+        "@webassemblyjs/wasm-parser": "1.7.11"
       }
     },
-    "benchmark": {
-      "version": "2.1.4",
-      "resolved": "https://registry.npmjs.org/benchmark/-/benchmark-2.1.4.tgz",
-      "integrity": "sha1-CfPeMckWQl1JjMLuVloOvzwqVik=",
+    "@webassemblyjs/wasm-parser": {
+      "version": "1.7.11",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.7.11.tgz",
+      "integrity": "sha512-6lmXRTrrZjYD8Ng8xRyvyXQJYUQKYSXhJqXOBLw24rdiXsHAOlvw5PhesjdcaMadU/pyPQOJ5dHreMjBxwnQKg==",
       "dev": true,
       "requires": {
-        "lodash": "^4.17.4",
-        "platform": "^1.3.3"
+        "@webassemblyjs/ast": "1.7.11",
+        "@webassemblyjs/helper-api-error": "1.7.11",
+        "@webassemblyjs/helper-wasm-bytecode": "1.7.11",
+        "@webassemblyjs/ieee754": "1.7.11",
+        "@webassemblyjs/leb128": "1.7.11",
+        "@webassemblyjs/utf8": "1.7.11"
       }
     },
-    "big.js": {
-      "version": "5.2.2",
-      "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz",
-      "integrity": "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==",
-      "dev": true
-    },
-    "bin-links": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/bin-links/-/bin-links-1.1.2.tgz",
-      "integrity": "sha512-8eEHVgYP03nILphilltWjeIjMbKyJo3wvp9K816pHbhP301ismzw15mxAAEVQ/USUwcP++1uNrbERbp8lOA6Fg==",
+    "@webassemblyjs/wast-parser": {
+      "version": "1.7.11",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-parser/-/wast-parser-1.7.11.tgz",
+      "integrity": "sha512-lEyVCg2np15tS+dm7+JJTNhNWq9yTZvi3qEhAIIOaofcYlUp0UR5/tVqOwa/gXYr3gjwSZqw+/lS9dscyLelbQ==",
       "dev": true,
       "requires": {
-        "bluebird": "^3.5.0",
-        "cmd-shim": "^2.0.2",
-        "gentle-fs": "^2.0.0",
-        "graceful-fs": "^4.1.11",
-        "write-file-atomic": "^2.3.0"
+        "@webassemblyjs/ast": "1.7.11",
+        "@webassemblyjs/floating-point-hex-parser": "1.7.11",
+        "@webassemblyjs/helper-api-error": "1.7.11",
+        "@webassemblyjs/helper-code-frame": "1.7.11",
+        "@webassemblyjs/helper-fsm": "1.7.11",
+        "@xtuc/long": "4.2.1"
       }
     },
-    "binary-extensions": {
-      "version": "1.12.0",
-      "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.12.0.tgz",
-      "integrity": "sha512-DYWGk01lDcxeS/K9IHPGWfT8PsJmbXRtRd2Sx72Tnb8pcYZQFF1oSDb8hJtS1vhp212q1Rzi5dUf9+nq0o9UIg==",
-      "dev": true
-    },
-    "block-stream": {
-      "version": "0.0.9",
-      "resolved": "https://registry.npmjs.org/block-stream/-/block-stream-0.0.9.tgz",
-      "integrity": "sha1-E+v+d4oDIFz+A3UUgeu0szAMEmo=",
+    "@webassemblyjs/wast-printer": {
+      "version": "1.7.11",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.7.11.tgz",
+      "integrity": "sha512-m5vkAsuJ32QpkdkDOUPGSltrg8Cuk3KBx4YrmAGQwCZPRdUHXxG4phIOuuycLemHFr74sWL9Wthqss4fzdzSwg==",
       "dev": true,
       "requires": {
-        "inherits": "~2.0.0"
+        "@webassemblyjs/ast": "1.7.11",
+        "@webassemblyjs/wast-parser": "1.7.11",
+        "@xtuc/long": "4.2.1"
       }
     },
-    "bluebird": {
-      "version": "3.5.3",
-      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.3.tgz",
-      "integrity": "sha512-/qKPUQlaW1OyR51WeCPBvRnAlnZFUJkCSG5HzGnuIqhgyJtF+T94lFnn33eiazjRm2LAHVy2guNnaq48X9SJuw==",
+    "@xtuc/ieee754": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz",
+      "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==",
       "dev": true
     },
-    "bn.js": {
-      "version": "4.11.8",
-      "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
-      "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA==",
+    "@xtuc/long": {
+      "version": "4.2.1",
+      "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.1.tgz",
+      "integrity": "sha512-FZdkNBDqBRHKQ2MEbSC17xnPFOhZxeJ2YGSfr2BKf3sujG49Qe3bB+rGCwQfIaA7WHnGeGkSijX4FuBCdrzW/g==",
       "dev": true
     },
-    "brace-expansion": {
-      "version": "1.1.11",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
-      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+    "@zkochan/cmd-shim": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/@zkochan/cmd-shim/-/cmd-shim-3.1.0.tgz",
+      "integrity": "sha512-o8l0+x7C7sMZU3v9GuJIAU10qQLtwR1dtRQIOmlNMtyaqhmpXOzx1HWiYoWfmmf9HHZoAkXpc9TM9PQYF9d4Jg==",
       "dev": true,
       "requires": {
-        "balanced-match": "^1.0.0",
-        "concat-map": "0.0.1"
+        "is-windows": "^1.0.0",
+        "mkdirp-promise": "^5.0.1",
+        "mz": "^2.5.0"
       }
     },
-    "braces": {
-      "version": "2.3.2",
-      "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz",
-      "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==",
+    "JSONStream": {
+      "version": "1.3.5",
+      "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz",
+      "integrity": "sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==",
       "dev": true,
       "requires": {
-        "arr-flatten": "^1.1.0",
-        "array-unique": "^0.3.2",
-        "extend-shallow": "^2.0.1",
-        "fill-range": "^4.0.0",
-        "isobject": "^3.0.1",
-        "repeat-element": "^1.1.2",
-        "snapdragon": "^0.8.1",
-        "snapdragon-node": "^2.0.1",
-        "split-string": "^3.0.2",
-        "to-regex": "^3.0.1"
-      },
-      "dependencies": {
-        "extend-shallow": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
-          "dev": true,
-          "requires": {
-            "is-extendable": "^0.1.0"
-          }
-        }
+        "jsonparse": "^1.2.0",
+        "through": ">=2.2.7 <3"
       }
     },
-    "brorand": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz",
-      "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=",
+    "abab": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.5.tgz",
+      "integrity": "sha512-9IK9EadsbHo6jLWIpxpR6pL0sazTXV6+SQv25ZB+F7Bj9mJNaOc4nCRabwd5M/JwmUa8idz6Eci6eKfJryPs6Q==",
       "dev": true
     },
-    "browser-process-hrtime": {
-      "version": "0.1.3",
-      "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-0.1.3.tgz",
-      "integrity": "sha512-bRFnI4NnjO6cnyLmOV/7PVoDEMJChlcfN0z4s1YMBY989/SvlfMI1lgCnkFUs53e9gQF+w7qu7XdllSTiSl8Aw==",
+    "abbrev": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
+      "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
+      "dev": true
+    },
+    "acorn": {
+      "version": "5.7.4",
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.4.tgz",
+      "integrity": "sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg==",
+      "dev": true
+    },
+    "acorn-dynamic-import": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/acorn-dynamic-import/-/acorn-dynamic-import-4.0.0.tgz",
+      "integrity": "sha512-d3OEjQV4ROpoflsnUA8HozoIR504TFxNivYEUi6uwz0IYhBkTDXGuWlNdMtybRt3nqVx/L6XqMt0FxkXuWKZhw==",
       "dev": true
     },
-    "browser-resolve": {
-      "version": "1.11.3",
-      "resolved": "https://registry.npmjs.org/browser-resolve/-/browser-resolve-1.11.3.tgz",
-      "integrity": "sha512-exDi1BYWB/6raKHmDTCicQfTkqwN5fioMFV4j8BsfMU4R2DK/QfZfK7kOVkmWCNANf0snkBzqGqAJBao9gZMdQ==",
+    "acorn-globals": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-6.0.0.tgz",
+      "integrity": "sha512-ZQl7LOWaF5ePqqcX4hLuv/bLXYQNfNWw2c0/yX/TsPRKamzHcTGQnlCjHT3TsmkOUVEPS3crCxiPfdzE/Trlhg==",
       "dev": true,
       "requires": {
-        "resolve": "1.1.7"
+        "acorn": "^7.1.1",
+        "acorn-walk": "^7.1.1"
       },
       "dependencies": {
-        "resolve": {
-          "version": "1.1.7",
-          "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.1.7.tgz",
-          "integrity": "sha1-IDEU2CrSxe2ejgQRs5ModeiJ6Xs=",
+        "acorn": {
+          "version": "7.4.0",
+          "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz",
+          "integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==",
           "dev": true
         }
       }
     },
-    "browserify-aes": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz",
-      "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==",
+    "acorn-walk": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.2.0.tgz",
+      "integrity": "sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==",
+      "dev": true
+    },
+    "agent-base": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz",
+      "integrity": "sha512-salcGninV0nPrwpGNn4VTXBb1SOuXQBiqbrNXoeizJsHrsL6ERFM2Ne3JUSBWRE6aeNJI2ROP/WEEIDUiDe3cg==",
       "dev": true,
       "requires": {
-        "buffer-xor": "^1.0.3",
-        "cipher-base": "^1.0.0",
-        "create-hash": "^1.1.0",
-        "evp_bytestokey": "^1.0.3",
-        "inherits": "^2.0.1",
-        "safe-buffer": "^5.0.1"
+        "es6-promisify": "^5.0.0"
       }
     },
-    "browserify-cipher": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/browserify-cipher/-/browserify-cipher-1.0.1.tgz",
-      "integrity": "sha512-sPhkz0ARKbf4rRQt2hTpAHqn47X3llLkUGn+xEJzLjwY8LRs2p0v7ljvI5EyoRO/mexrNunNECisZs+gw2zz1w==",
+    "agentkeepalive": {
+      "version": "3.5.2",
+      "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.5.2.tgz",
+      "integrity": "sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==",
       "dev": true,
       "requires": {
-        "browserify-aes": "^1.0.4",
-        "browserify-des": "^1.0.0",
-        "evp_bytestokey": "^1.0.0"
+        "humanize-ms": "^1.2.1"
       }
     },
-    "browserify-des": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/browserify-des/-/browserify-des-1.0.2.tgz",
-      "integrity": "sha512-BioO1xf3hFwz4kc6iBhI3ieDFompMhrMlnDFC4/0/vd5MokpuAc3R+LYbwTA9A5Yc9pq9UYPqffKpW2ObuwX5A==",
+    "aggregate-error": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz",
+      "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==",
       "dev": true,
       "requires": {
-        "cipher-base": "^1.0.1",
-        "des.js": "^1.0.0",
-        "inherits": "^2.0.1",
-        "safe-buffer": "^5.1.2"
+        "clean-stack": "^2.0.0",
+        "indent-string": "^4.0.0"
       }
     },
-    "browserify-rsa": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz",
-      "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=",
+    "ajv": {
+      "version": "6.12.5",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.5.tgz",
+      "integrity": "sha512-lRF8RORchjpKG50/WFf8xmg7sgCLFiYNNnqdKflk63whMQcWR5ngGjiSXkL9bjxy6B2npOK2HSMN49jEBMSkag==",
       "dev": true,
       "requires": {
-        "bn.js": "^4.1.0",
-        "randombytes": "^2.0.1"
+        "fast-deep-equal": "^3.1.1",
+        "fast-json-stable-stringify": "^2.0.0",
+        "json-schema-traverse": "^0.4.1",
+        "uri-js": "^4.2.2"
       }
     },
-    "browserify-sign": {
-      "version": "4.0.4",
-      "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.0.4.tgz",
-      "integrity": "sha1-qk62jl17ZYuqa/alfmMMvXqT0pg=",
+    "ajv-errors": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/ajv-errors/-/ajv-errors-1.0.1.tgz",
+      "integrity": "sha512-DCRfO/4nQ+89p/RK43i8Ezd41EqdGIU4ld7nGF8OQ14oc/we5rEntLCUa7+jrn3nn83BosfwZA0wb4pon2o8iQ==",
+      "dev": true
+    },
+    "ajv-keywords": {
+      "version": "3.5.2",
+      "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz",
+      "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==",
+      "dev": true
+    },
+    "ansi-colors": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-1.1.0.tgz",
+      "integrity": "sha512-SFKX67auSNoVR38N3L+nvsPjOE0bybKTYbkf5tRvushrAPQ9V75huw0ZxBkKVeRU9kqH3d6HA4xTckbwZ4ixmA==",
       "dev": true,
       "requires": {
-        "bn.js": "^4.1.1",
-        "browserify-rsa": "^4.0.0",
-        "create-hash": "^1.1.0",
-        "create-hmac": "^1.1.2",
-        "elliptic": "^6.0.0",
-        "inherits": "^2.0.1",
-        "parse-asn1": "^5.0.0"
+        "ansi-wrap": "^0.1.0"
       }
     },
-    "browserify-zlib": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.2.0.tgz",
-      "integrity": "sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA==",
+    "ansi-escapes": {
+      "version": "4.3.1",
+      "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.1.tgz",
+      "integrity": "sha512-JWF7ocqNrp8u9oqpgV+wH5ftbt+cfvv+PTjOvKLT3AdYly/LmORARfEVT1iyjwN+4MqE5UmVKoAdIBqeoCHgLA==",
       "dev": true,
       "requires": {
-        "pako": "~1.0.5"
+        "type-fest": "^0.11.0"
+      },
+      "dependencies": {
+        "type-fest": {
+          "version": "0.11.0",
+          "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.11.0.tgz",
+          "integrity": "sha512-OdjXJxnCN1AvyLSzeKIgXTXxV+99ZuXl3Hpo9XpJAv9MBcHrrJOQ5kV7ypXOuQie+AmWG25hLbiKdwYTifzcfQ==",
+          "dev": true
+        }
       }
     },
-    "bs-logger": {
-      "version": "0.2.6",
-      "resolved": "https://registry.npmjs.org/bs-logger/-/bs-logger-0.2.6.tgz",
-      "integrity": "sha512-pd8DCoxmbgc7hyPKOvxtqNcjYoOsABPQdcCUjGp3d42VR2CX1ORhk2A87oqqu5R1kk+76nsxZupkmyd+MVtCog==",
+    "ansi-gray": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/ansi-gray/-/ansi-gray-0.1.1.tgz",
+      "integrity": "sha1-KWLPVOyXksSFEKPetSRDaGHvclE=",
       "dev": true,
       "requires": {
-        "fast-json-stable-stringify": "2.x"
+        "ansi-wrap": "0.1.0"
       }
     },
-    "bser": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/bser/-/bser-2.0.0.tgz",
-      "integrity": "sha1-mseNPtXZFYBP2HrLFYvHlxR6Fxk=",
-      "dev": true,
-      "requires": {
-        "node-int64": "^0.4.0"
-      }
+    "ansi-regex": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
+      "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
+      "dev": true
     },
-    "buffer": {
-      "version": "4.9.1",
-      "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
-      "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=",
-      "dev": true,
+    "ansi-styles": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+      "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
       "requires": {
-        "base64-js": "^1.0.2",
-        "ieee754": "^1.1.4",
-        "isarray": "^1.0.0"
+        "color-convert": "^1.9.0"
       }
     },
-    "buffer-equal": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-1.0.0.tgz",
-      "integrity": "sha1-WWFrSYME1Var1GaWayLu2j7KX74=",
-      "dev": true
-    },
-    "buffer-from": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
-      "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==",
-      "dev": true
-    },
-    "buffer-xor": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz",
-      "integrity": "sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=",
-      "dev": true
-    },
-    "builtin-modules": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz",
-      "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8=",
-      "dev": true
-    },
-    "builtin-status-codes": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz",
-      "integrity": "sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=",
-      "dev": true
-    },
-    "builtins": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/builtins/-/builtins-1.0.3.tgz",
-      "integrity": "sha1-y5T662HIaWRR2zZTThQi+U8K7og=",
-      "dev": true
-    },
-    "byline": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/byline/-/byline-5.0.0.tgz",
-      "integrity": "sha1-dBxSFkaOrcRXsDQQEYrXfejB3bE=",
+    "ansi-wrap": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/ansi-wrap/-/ansi-wrap-0.1.0.tgz",
+      "integrity": "sha1-qCJQ3bABXponyoLoLqYDu/pF768=",
       "dev": true
     },
-    "byte-size": {
-      "version": "4.0.4",
-      "resolved": "https://registry.npmjs.org/byte-size/-/byte-size-4.0.4.tgz",
-      "integrity": "sha512-82RPeneC6nqCdSwCX2hZUz3JPOvN5at/nTEw/CMf05Smu3Hrpo9Psb7LjN+k+XndNArG1EY8L4+BM3aTM4BCvw==",
+    "any-promise": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz",
+      "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8=",
       "dev": true
     },
-    "cacache": {
-      "version": "11.3.2",
-      "resolved": "https://registry.npmjs.org/cacache/-/cacache-11.3.2.tgz",
-      "integrity": "sha512-E0zP4EPGDOaT2chM08Als91eYnf8Z+eH1awwwVsngUmgppfM5jjJ8l3z5vO5p5w/I3LsiXawb1sW0VY65pQABg==",
+    "anymatch": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-2.0.0.tgz",
+      "integrity": "sha512-5teOsQWABXHHBFP9y3skS5P3d/WfWXpv3FUpy+LorMrNYaT9pI4oLMQX7jzQ2KklNpGpWHzdCXTDT2Y3XGlZBw==",
       "dev": true,
       "requires": {
-        "bluebird": "^3.5.3",
-        "chownr": "^1.1.1",
-        "figgy-pudding": "^3.5.1",
-        "glob": "^7.1.3",
-        "graceful-fs": "^4.1.15",
-        "lru-cache": "^5.1.1",
-        "mississippi": "^3.0.0",
-        "mkdirp": "^0.5.1",
-        "move-concurrently": "^1.0.1",
-        "promise-inflight": "^1.0.1",
-        "rimraf": "^2.6.2",
-        "ssri": "^6.0.1",
-        "unique-filename": "^1.1.1",
-        "y18n": "^4.0.0"
+        "micromatch": "^3.1.4",
+        "normalize-path": "^2.1.1"
       },
       "dependencies": {
-        "lru-cache": {
-          "version": "5.1.1",
-          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
-          "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
+        "normalize-path": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.1.1.tgz",
+          "integrity": "sha1-GrKLVW4Zg2Oowab35vogE3/mrtk=",
           "dev": true,
           "requires": {
-            "yallist": "^3.0.2"
+            "remove-trailing-separator": "^1.0.1"
           }
-        },
-        "y18n": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
-          "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
-          "dev": true
-        },
-        "yallist": {
-          "version": "3.0.3",
-          "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz",
-          "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==",
-          "dev": true
         }
       }
     },
-    "cache-base": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz",
-      "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==",
+    "append-buffer": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/append-buffer/-/append-buffer-1.0.2.tgz",
+      "integrity": "sha1-2CIM9GYIFSXv6lBhTz3mUU36WPE=",
       "dev": true,
       "requires": {
-        "collection-visit": "^1.0.0",
-        "component-emitter": "^1.2.1",
-        "get-value": "^2.0.6",
-        "has-value": "^1.0.0",
-        "isobject": "^3.0.1",
-        "set-value": "^2.0.0",
-        "to-object-path": "^0.3.0",
-        "union-value": "^1.0.0",
-        "unset-value": "^1.0.0"
+        "buffer-equal": "^1.0.0"
       }
     },
-    "call-me-maybe": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/call-me-maybe/-/call-me-maybe-1.0.1.tgz",
-      "integrity": "sha1-JtII6onje1y95gJQoV8DHBak1ms=",
+    "aproba": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
+      "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==",
       "dev": true
     },
-    "caller-callsite": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/caller-callsite/-/caller-callsite-2.0.0.tgz",
-      "integrity": "sha1-hH4PzgoiN1CpoCfFSzNzGtMVQTQ=",
-      "dev": true,
-      "requires": {
-        "callsites": "^2.0.0"
-      }
+    "archy": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz",
+      "integrity": "sha1-+cjBN1fMHde8N5rHeyxipcKGjEA=",
+      "dev": true
     },
-    "caller-path": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/caller-path/-/caller-path-2.0.0.tgz",
-      "integrity": "sha1-Ro+DBE42mrIBD6xfBs7uFbsssfQ=",
+    "are-we-there-yet": {
+      "version": "1.1.5",
+      "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz",
+      "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==",
       "dev": true,
       "requires": {
-        "caller-callsite": "^2.0.0"
+        "delegates": "^1.0.0",
+        "readable-stream": "^2.0.6"
       }
     },
-    "callsites": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/callsites/-/callsites-2.0.0.tgz",
-      "integrity": "sha1-BuuE8A7qQT2oav/vrL/7Ngk7PFA=",
-      "dev": true
-    },
-    "camelcase": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-3.0.0.tgz",
-      "integrity": "sha1-MvxLn82vhF/N9+c7uXysImHwqwo=",
+    "arg": {
+      "version": "4.1.3",
+      "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz",
+      "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==",
       "dev": true
     },
-    "camelcase-keys": {
-      "version": "4.2.0",
-      "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-4.2.0.tgz",
-      "integrity": "sha1-oqpfsa9oh1glnDLBQUJteJI7m3c=",
+    "argparse": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
+      "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
       "dev": true,
       "requires": {
-        "camelcase": "^4.1.0",
-        "map-obj": "^2.0.0",
-        "quick-lru": "^1.0.0"
-      },
-      "dependencies": {
-        "camelcase": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz",
-          "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=",
-          "dev": true
-        }
+        "sprintf-js": "~1.0.2"
       }
     },
-    "capture-exit": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/capture-exit/-/capture-exit-2.0.0.tgz",
-      "integrity": "sha512-PiT/hQmTonHhl/HFGN+Lx3JJUznrVYJ3+AQsnthneZbvW7x+f08Tk7yLJTLEOUvBTbduLeeBkxEaYXUOUrRq6g==",
-      "dev": true,
+    "argv-tools": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/argv-tools/-/argv-tools-0.1.2.tgz",
+      "integrity": "sha512-wxqoymY0BEu9NblZVQiOTOAiJUjPhaa/kbNMjC2h6bnrmUSgnxKgWJo3lzXvi3bHJRwXyqK/dHzMlZVRT89Cxg==",
       "requires": {
-        "rsvp": "^4.8.4"
+        "array-back": "^2.0.0",
+        "find-replace": "^2.0.1"
       }
     },
-    "caseless": {
-      "version": "0.12.0",
-      "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
-      "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=",
+    "arr-diff": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz",
+      "integrity": "sha1-1kYQdP6/7HHn4VI1dhoyml3HxSA=",
       "dev": true
     },
-    "chalk": {
-      "version": "2.4.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
-      "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
+    "arr-filter": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/arr-filter/-/arr-filter-1.1.2.tgz",
+      "integrity": "sha1-Q/3d0JHo7xGqTEXZzcGOLf8XEe4=",
+      "dev": true,
       "requires": {
-        "ansi-styles": "^3.2.1",
-        "escape-string-regexp": "^1.0.5",
-        "supports-color": "^5.3.0"
+        "make-iterator": "^1.0.0"
       }
     },
-    "chardet": {
-      "version": "0.7.0",
-      "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz",
-      "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==",
+    "arr-flatten": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz",
+      "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==",
       "dev": true
     },
-    "chokidar": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.0.4.tgz",
-      "integrity": "sha512-z9n7yt9rOvIJrMhvDtDictKrkFHeihkNl6uWMmZlmL6tJtX9Cs+87oK+teBx+JIgzvbX3yZHT3eF8vpbDxHJXQ==",
+    "arr-map": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/arr-map/-/arr-map-2.0.2.tgz",
+      "integrity": "sha1-Onc0X/wc814qkYJWAfnljy4kysQ=",
       "dev": true,
       "requires": {
-        "anymatch": "^2.0.0",
-        "async-each": "^1.0.0",
-        "braces": "^2.3.0",
-        "fsevents": "^1.2.2",
-        "glob-parent": "^3.1.0",
-        "inherits": "^2.0.1",
-        "is-binary-path": "^1.0.0",
-        "is-glob": "^4.0.0",
-        "lodash.debounce": "^4.0.8",
-        "normalize-path": "^2.1.1",
-        "path-is-absolute": "^1.0.0",
-        "readdirp": "^2.0.0",
-        "upath": "^1.0.5"
+        "make-iterator": "^1.0.0"
       }
     },
-    "chownr": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz",
-      "integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g==",
+    "arr-union": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz",
+      "integrity": "sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ=",
       "dev": true
     },
-    "chrome-trace-event": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.0.tgz",
-      "integrity": "sha512-xDbVgyfDTT2piup/h8dK/y4QZfJRSa73bw1WZ8b4XM1o7fsFubUVGYcE+1ANtOzJJELGpYoG2961z0Z6OAld9A==",
-      "dev": true,
+    "array-back": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/array-back/-/array-back-2.0.0.tgz",
+      "integrity": "sha512-eJv4pLLufP3g5kcZry0j6WXpIbzYw9GUB4mVJZno9wfwiBxbizTnHCw3VJb07cBihbFX48Y7oSrW9y+gt4glyw==",
       "requires": {
-        "tslib": "^1.9.0"
+        "typical": "^2.6.1"
       }
     },
-    "ci-info": {
-      "version": "1.6.0",
-      "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-1.6.0.tgz",
-      "integrity": "sha512-vsGdkwSCDpWmP80ncATX7iea5DWQemg1UgCW5J8tqjU3lYw4FBYuj89J0CTVomA7BEfvSZd84GmHko+MxFQU2A==",
+    "array-differ": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/array-differ/-/array-differ-2.1.0.tgz",
+      "integrity": "sha512-KbUpJgx909ZscOc/7CLATBFam7P1Z1QRQInvgT0UztM9Q72aGKCunKASAl7WNW0tnPmPyEMeMhdsfWhfmW037w==",
       "dev": true
     },
-    "cipher-base": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz",
-      "integrity": "sha512-Kkht5ye6ZGmwv40uUDZztayT2ThLQGfnj/T71N/XzeZeo3nf8foyW7zGTsPYkEya3m5f3cAypH+qe7YOrM1U2Q==",
+    "array-each": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/array-each/-/array-each-1.0.1.tgz",
+      "integrity": "sha1-p5SvDAWrF1KEbudTofIRoFugxE8=",
+      "dev": true
+    },
+    "array-find-index": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/array-find-index/-/array-find-index-1.0.2.tgz",
+      "integrity": "sha1-3wEKoSh+Fku9pvlyOwqWoexBh6E=",
+      "dev": true
+    },
+    "array-ify": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/array-ify/-/array-ify-1.0.0.tgz",
+      "integrity": "sha1-nlKHYrSpBmrRY6aWKjZEGOlibs4=",
+      "dev": true
+    },
+    "array-initial": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/array-initial/-/array-initial-1.1.0.tgz",
+      "integrity": "sha1-L6dLJnOTccOUe9enrcc74zSz15U=",
       "dev": true,
       "requires": {
-        "inherits": "^2.0.1",
-        "safe-buffer": "^5.0.1"
+        "array-slice": "^1.0.0",
+        "is-number": "^4.0.0"
+      },
+      "dependencies": {
+        "is-number": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/is-number/-/is-number-4.0.0.tgz",
+          "integrity": "sha512-rSklcAIlf1OmFdyAqbnWTLVelsQ58uvZ66S/ZyawjWqIviTWCjg2PzVGw8WUA+nNuPTqb4wgA+NszrJ+08LlgQ==",
+          "dev": true
+        }
       }
     },
-    "class-utils": {
-      "version": "0.3.6",
-      "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz",
-      "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==",
+    "array-last": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/array-last/-/array-last-1.3.0.tgz",
+      "integrity": "sha512-eOCut5rXlI6aCOS7Z7kCplKRKyiFQ6dHFBem4PwlwKeNFk2/XxTrhRh5T9PyaEWGy/NHTZWbY+nsZlNFJu9rYg==",
       "dev": true,
       "requires": {
-        "arr-union": "^3.1.0",
-        "define-property": "^0.2.5",
-        "isobject": "^3.0.0",
-        "static-extend": "^0.1.1"
+        "is-number": "^4.0.0"
       },
       "dependencies": {
-        "define-property": {
-          "version": "0.2.5",
-          "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
-          "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
-          "dev": true,
-          "requires": {
-            "is-descriptor": "^0.1.0"
-          }
+        "is-number": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/is-number/-/is-number-4.0.0.tgz",
+          "integrity": "sha512-rSklcAIlf1OmFdyAqbnWTLVelsQ58uvZ66S/ZyawjWqIviTWCjg2PzVGw8WUA+nNuPTqb4wgA+NszrJ+08LlgQ==",
+          "dev": true
         }
       }
     },
-    "cli-cursor": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz",
-      "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=",
+    "array-slice": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/array-slice/-/array-slice-1.1.0.tgz",
+      "integrity": "sha512-B1qMD3RBP7O8o0H2KbrXDyB0IccejMF15+87Lvlor12ONPRHP6gTjXMNkt/d3ZuOGbAe66hFmaCfECI24Ufp6w==",
+      "dev": true
+    },
+    "array-sort": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/array-sort/-/array-sort-1.0.0.tgz",
+      "integrity": "sha512-ihLeJkonmdiAsD7vpgN3CRcx2J2S0TiYW+IS/5zHBI7mKUq3ySvBdzzBfD236ubDBQFiiyG3SWCPc+msQ9KoYg==",
       "dev": true,
       "requires": {
-        "restore-cursor": "^2.0.0"
+        "default-compare": "^1.0.0",
+        "get-value": "^2.0.6",
+        "kind-of": "^5.0.2"
+      },
+      "dependencies": {
+        "kind-of": {
+          "version": "5.1.0",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
+          "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==",
+          "dev": true
+        }
       }
     },
-    "cli-width": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.0.tgz",
-      "integrity": "sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk=",
-      "dev": true
-    },
-    "cliui": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz",
-      "integrity": "sha1-EgYBU3qRbSmUD5NNo7SNWFo5IT0=",
+    "array-union": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz",
+      "integrity": "sha1-mjRBDk9OPaI96jdb5b5w8kd47Dk=",
       "dev": true,
       "requires": {
-        "string-width": "^1.0.1",
-        "strip-ansi": "^3.0.1",
-        "wrap-ansi": "^2.0.0"
+        "array-uniq": "^1.0.1"
       }
     },
-    "clone": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz",
-      "integrity": "sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18=",
+    "array-uniq": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz",
+      "integrity": "sha1-r2rId6Jcx/dOBYiUdThY39sk/bY=",
       "dev": true
     },
-    "clone-buffer": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/clone-buffer/-/clone-buffer-1.0.0.tgz",
-      "integrity": "sha1-4+JbIHrE5wGvch4staFnksrD3Fg=",
+    "array-unique": {
+      "version": "0.3.2",
+      "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz",
+      "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=",
       "dev": true
     },
-    "clone-stats": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/clone-stats/-/clone-stats-1.0.0.tgz",
-      "integrity": "sha1-s3gt/4u1R04Yuba/D9/ngvh3doA=",
+    "arrify": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz",
+      "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=",
       "dev": true
     },
-    "cloneable-readable": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/cloneable-readable/-/cloneable-readable-1.1.2.tgz",
-      "integrity": "sha512-Bq6+4t+lbM8vhTs/Bef5c5AdEMtapp/iFb6+s4/Hh9MVTt8OLKH7ZOOZSCT+Ys7hsHvqv0GuMPJ1lnQJVHvxpg==",
+    "asap": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
+      "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=",
+      "dev": true
+    },
+    "asn1": {
+      "version": "0.2.4",
+      "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
+      "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==",
+      "dev": true,
+      "requires": {
+        "safer-buffer": "~2.1.0"
+      }
+    },
+    "asn1.js": {
+      "version": "5.4.1",
+      "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz",
+      "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==",
       "dev": true,
       "requires": {
+        "bn.js": "^4.0.0",
         "inherits": "^2.0.1",
-        "process-nextick-args": "^2.0.0",
-        "readable-stream": "^2.3.5"
+        "minimalistic-assert": "^1.0.0",
+        "safer-buffer": "^2.1.0"
       },
       "dependencies": {
-        "process-nextick-args": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
-          "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==",
+        "bn.js": {
+          "version": "4.11.9",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==",
           "dev": true
         }
       }
     },
-    "cmd-shim": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/cmd-shim/-/cmd-shim-2.0.2.tgz",
-      "integrity": "sha1-b8vamUg6j9FdfTChlspp1oii79s=",
+    "assert": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/assert/-/assert-1.5.0.tgz",
+      "integrity": "sha512-EDsgawzwoun2CZkCgtxJbv392v4nbk9XDD06zI+kQYoBM/3RBWLlEyJARDOmhAAosBjWACEkKL6S+lIZtcAubA==",
       "dev": true,
       "requires": {
-        "graceful-fs": "^4.1.2",
-        "mkdirp": "~0.5.0"
+        "object-assign": "^4.1.1",
+        "util": "0.10.3"
+      },
+      "dependencies": {
+        "inherits": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz",
+          "integrity": "sha1-sX0I0ya0Qj5Wjv9xn5GwscvfafE=",
+          "dev": true
+        },
+        "util": {
+          "version": "0.10.3",
+          "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz",
+          "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=",
+          "dev": true,
+          "requires": {
+            "inherits": "2.0.1"
+          }
+        }
       }
     },
-    "co": {
-      "version": "4.6.0",
-      "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
-      "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=",
+    "assert-plus": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
+      "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
       "dev": true
     },
-    "code-point-at": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
-      "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=",
+    "assign-symbols": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz",
+      "integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c=",
       "dev": true
     },
-    "collection-map": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/collection-map/-/collection-map-1.0.0.tgz",
-      "integrity": "sha1-rqDwb40mx4DCt1SUOFVEsiVa8Yw=",
+    "async": {
+      "version": "2.6.3",
+      "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz",
+      "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==",
       "dev": true,
       "requires": {
-        "arr-map": "^2.0.2",
-        "for-own": "^1.0.0",
-        "make-iterator": "^1.0.0"
+        "lodash": "^4.17.14"
       }
     },
-    "collection-visit": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz",
-      "integrity": "sha1-S8A3PBZLwykbTTaMgpzxqApZ3KA=",
+    "async-done": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/async-done/-/async-done-1.3.1.tgz",
+      "integrity": "sha512-R1BaUeJ4PMoLNJuk+0tLJgjmEqVsdN118+Z8O+alhnQDQgy0kmD5Mqi0DNEmMx2LM0Ed5yekKu+ZXYvIHceicg==",
       "dev": true,
       "requires": {
-        "map-visit": "^1.0.0",
-        "object-visit": "^1.0.0"
-      }
-    },
-    "color-convert": {
-      "version": "1.9.3",
-      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
-      "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
-      "requires": {
-        "color-name": "1.1.3"
+        "end-of-stream": "^1.1.0",
+        "once": "^1.3.2",
+        "process-nextick-args": "^1.0.7",
+        "stream-exhaust": "^1.0.1"
       }
     },
-    "color-name": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
-      "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
-    },
-    "color-support": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz",
-      "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==",
+    "async-each": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/async-each/-/async-each-1.0.3.tgz",
+      "integrity": "sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ==",
       "dev": true
     },
-    "columnify": {
-      "version": "1.5.4",
-      "resolved": "https://registry.npmjs.org/columnify/-/columnify-1.5.4.tgz",
-      "integrity": "sha1-Rzfd8ce2mop8NAVweC6UfuyOeLs=",
+    "async-settle": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/async-settle/-/async-settle-1.0.0.tgz",
+      "integrity": "sha1-HQqRS7Aldb7IqPOnTlCA9yssDGs=",
       "dev": true,
       "requires": {
-        "strip-ansi": "^3.0.0",
-        "wcwidth": "^1.0.0"
+        "async-done": "^1.2.2"
       }
     },
-    "combined-stream": {
-      "version": "1.0.7",
-      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz",
-      "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==",
-      "dev": true,
-      "requires": {
-        "delayed-stream": "~1.0.0"
-      }
-    },
-    "command-line-args": {
-      "version": "5.0.2",
-      "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.0.2.tgz",
-      "integrity": "sha512-/qPcbL8zpqg53x4rAaqMFlRV4opN3pbla7I7k9x8kyOBMQoGT6WltjN6sXZuxOXw6DgdK7Ad+ijYS5gjcr7vlA==",
-      "requires": {
-        "argv-tools": "^0.1.1",
-        "array-back": "^2.0.0",
-        "find-replace": "^2.0.1",
-        "lodash.camelcase": "^4.3.0",
-        "typical": "^2.6.1"
-      }
-    },
-    "command-line-usage": {
-      "version": "5.0.5",
-      "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-5.0.5.tgz",
-      "integrity": "sha512-d8NrGylA5oCXSbGoKz05FkehDAzSmIm4K03S5VDh4d5lZAtTWfc3D1RuETtuQCn8129nYfJfDdF7P/lwcz1BlA==",
-      "requires": {
-        "array-back": "^2.0.0",
-        "chalk": "^2.4.1",
-        "table-layout": "^0.4.3",
-        "typical": "^2.6.1"
-      }
-    },
-    "commander": {
-      "version": "2.17.1",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-2.17.1.tgz",
-      "integrity": "sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg==",
+    "asynckit": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+      "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=",
       "dev": true
     },
-    "commondir": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz",
-      "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=",
+    "at-least-node": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz",
+      "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==",
       "dev": true
     },
-    "compare-func": {
-      "version": "1.3.2",
-      "resolved": "https://registry.npmjs.org/compare-func/-/compare-func-1.3.2.tgz",
-      "integrity": "sha1-md0LpFfh+bxyKxLAjsM+6rMfpkg=",
-      "dev": true,
-      "requires": {
-        "array-ify": "^1.0.0",
-        "dot-prop": "^3.0.0"
-      },
-      "dependencies": {
-        "dot-prop": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-3.0.0.tgz",
-          "integrity": "sha1-G3CK8JSknJoOfbyteQq6U52sEXc=",
-          "dev": true,
-          "requires": {
-            "is-obj": "^1.0.0"
-          }
-        }
-      }
-    },
-    "component-emitter": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz",
-      "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY=",
+    "atob": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz",
+      "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==",
       "dev": true
     },
-    "concat-map": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
-      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
+    "atob-lite": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/atob-lite/-/atob-lite-2.0.0.tgz",
+      "integrity": "sha1-D+9a1G8b16hQLGVyfwNn1e5D1pY=",
       "dev": true
     },
-    "concat-stream": {
-      "version": "1.6.2",
-      "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
-      "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
-      "dev": true,
-      "requires": {
-        "buffer-from": "^1.0.0",
-        "inherits": "^2.0.3",
-        "readable-stream": "^2.2.2",
-        "typedarray": "^0.0.6"
-      }
-    },
-    "config-chain": {
-      "version": "1.1.12",
-      "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.12.tgz",
-      "integrity": "sha512-a1eOIcu8+7lUInge4Rpf/n4Krkf3Dd9lqhljRzII1/Zno/kRtUWnznPO3jOKBmTEktkt3fkxisUcivoj0ebzoA==",
-      "dev": true,
-      "requires": {
-        "ini": "^1.3.4",
-        "proto-list": "~1.2.1"
-      }
-    },
-    "console-browserify": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/console-browserify/-/console-browserify-1.1.0.tgz",
-      "integrity": "sha1-8CQcRXMKn8YyOyBtvzjtx0HQuxA=",
-      "dev": true,
-      "requires": {
-        "date-now": "^0.1.4"
-      }
-    },
-    "console-control-strings": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
-      "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=",
+    "aws-sign2": {
+      "version": "0.7.0",
+      "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
+      "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=",
       "dev": true
     },
-    "constants-browserify": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/constants-browserify/-/constants-browserify-1.0.0.tgz",
-      "integrity": "sha1-wguW2MYXdIqvHBYCF2DNJ/y4y3U=",
+    "aws4": {
+      "version": "1.10.1",
+      "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.10.1.tgz",
+      "integrity": "sha512-zg7Hz2k5lI8kb7U32998pRRFin7zJlkfezGJjUc2heaD4Pw2wObakCDVzkKztTm/Ln7eiVvYsjqak0Ed4LkMDA==",
       "dev": true
     },
-    "conventional-changelog-angular": {
-      "version": "5.0.2",
-      "resolved": "https://registry.npmjs.org/conventional-changelog-angular/-/conventional-changelog-angular-5.0.2.tgz",
-      "integrity": "sha512-yx7m7lVrXmt4nKWQgWZqxSALEiAKZhOAcbxdUaU9575mB0CzXVbgrgpfSnSP7OqWDUTYGD0YVJ0MSRdyOPgAwA==",
-      "dev": true,
-      "requires": {
-        "compare-func": "^1.3.1",
-        "q": "^1.5.1"
-      }
-    },
-    "conventional-changelog-core": {
-      "version": "3.1.5",
-      "resolved": "https://registry.npmjs.org/conventional-changelog-core/-/conventional-changelog-core-3.1.5.tgz",
-      "integrity": "sha512-iwqAotS4zk0wA4S84YY1JCUG7X3LxaRjJxuUo6GI4dZuIy243j5nOg/Ora35ExT4DOiw5dQbMMQvw2SUjh6moQ==",
-      "dev": true,
-      "requires": {
-        "conventional-changelog-writer": "^4.0.2",
-        "conventional-commits-parser": "^3.0.1",
-        "dateformat": "^3.0.0",
-        "get-pkg-repo": "^1.0.0",
-        "git-raw-commits": "2.0.0",
-        "git-remote-origin-url": "^2.0.0",
-        "git-semver-tags": "^2.0.2",
-        "lodash": "^4.2.1",
-        "normalize-package-data": "^2.3.5",
-        "q": "^1.5.1",
-        "read-pkg": "^3.0.0",
-        "read-pkg-up": "^3.0.0",
-        "through2": "^2.0.0"
+    "babel-jest": {
+      "version": "26.3.0",
+      "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-26.3.0.tgz",
+      "integrity": "sha512-sxPnQGEyHAOPF8NcUsD0g7hDCnvLL2XyblRBcgrzTWBB/mAIpWow3n1bEL+VghnnZfreLhFSBsFluRoK2tRK4g==",
+      "dev": true,
+      "requires": {
+        "@jest/transform": "^26.3.0",
+        "@jest/types": "^26.3.0",
+        "@types/babel__core": "^7.1.7",
+        "babel-plugin-istanbul": "^6.0.0",
+        "babel-preset-jest": "^26.3.0",
+        "chalk": "^4.0.0",
+        "graceful-fs": "^4.2.4",
+        "slash": "^3.0.0"
       },
       "dependencies": {
-        "find-up": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz",
-          "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=",
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
           "dev": true,
           "requires": {
-            "locate-path": "^2.0.0"
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
           }
         },
-        "load-json-file": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz",
-          "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=",
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
           "dev": true,
           "requires": {
-            "graceful-fs": "^4.1.2",
-            "parse-json": "^4.0.0",
-            "pify": "^3.0.0",
-            "strip-bom": "^3.0.0"
+            "@types/istanbul-lib-report": "*"
           }
         },
-        "parse-json": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
-          "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=",
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
           "dev": true,
           "requires": {
-            "error-ex": "^1.3.1",
-            "json-parse-better-errors": "^1.0.1"
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
           }
         },
-        "path-type": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz",
-          "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==",
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
           "dev": true,
           "requires": {
-            "pify": "^3.0.0"
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
           }
         },
-        "read-pkg": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz",
-          "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=",
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
           "dev": true,
           "requires": {
-            "load-json-file": "^4.0.0",
-            "normalize-package-data": "^2.3.2",
-            "path-type": "^3.0.0"
+            "color-name": "~1.1.4"
           }
         },
-        "read-pkg-up": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-3.0.0.tgz",
-          "integrity": "sha1-PtSWaF26D4/hGNBpHcUfSh/5bwc=",
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
           "dev": true,
           "requires": {
-            "find-up": "^2.0.0",
-            "read-pkg": "^3.0.0"
+            "has-flag": "^4.0.0"
           }
-        },
-        "strip-bom": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
-          "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=",
-          "dev": true
         }
       }
     },
-    "conventional-changelog-preset-loader": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/conventional-changelog-preset-loader/-/conventional-changelog-preset-loader-2.0.2.tgz",
-      "integrity": "sha512-pBY+qnUoJPXAXXqVGwQaVmcye05xi6z231QM98wHWamGAmu/ghkBprQAwmF5bdmyobdVxiLhPY3PrCfSeUNzRQ==",
-      "dev": true
-    },
-    "conventional-changelog-writer": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/conventional-changelog-writer/-/conventional-changelog-writer-4.0.2.tgz",
-      "integrity": "sha512-d8/FQY/fix2xXEBUhOo8u3DCbyEw3UOQgYHxLsPDw+wHUDma/GQGAGsGtoH876WyNs32fViHmTOUrgRKVLvBug==",
-      "dev": true,
-      "requires": {
-        "compare-func": "^1.3.1",
-        "conventional-commits-filter": "^2.0.1",
-        "dateformat": "^3.0.0",
-        "handlebars": "^4.0.2",
-        "json-stringify-safe": "^5.0.1",
-        "lodash": "^4.2.1",
-        "meow": "^4.0.0",
-        "semver": "^5.5.0",
-        "split": "^1.0.0",
-        "through2": "^2.0.0"
-      }
-    },
-    "conventional-commits-filter": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/conventional-commits-filter/-/conventional-commits-filter-2.0.1.tgz",
-      "integrity": "sha512-92OU8pz/977udhBjgPEbg3sbYzIxMDFTlQT97w7KdhR9igNqdJvy8smmedAAgn4tPiqseFloKkrVfbXCVd+E7A==",
+    "babel-plugin-istanbul": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.0.0.tgz",
+      "integrity": "sha512-AF55rZXpe7trmEylbaE1Gv54wn6rwU03aptvRoVIGP8YykoSxqdVLV1TfwflBCE/QtHmqtP8SWlTENqbK8GCSQ==",
       "dev": true,
       "requires": {
-        "is-subset": "^0.1.1",
-        "modify-values": "^1.0.0"
+        "@babel/helper-plugin-utils": "^7.0.0",
+        "@istanbuljs/load-nyc-config": "^1.0.0",
+        "@istanbuljs/schema": "^0.1.2",
+        "istanbul-lib-instrument": "^4.0.0",
+        "test-exclude": "^6.0.0"
       }
     },
-    "conventional-commits-parser": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/conventional-commits-parser/-/conventional-commits-parser-3.0.1.tgz",
-      "integrity": "sha512-P6U5UOvDeidUJ8ebHVDIoXzI7gMlQ1OF/id6oUvp8cnZvOXMt1n8nYl74Ey9YMn0uVQtxmCtjPQawpsssBWtGg==",
+    "babel-plugin-jest-hoist": {
+      "version": "26.2.0",
+      "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-26.2.0.tgz",
+      "integrity": "sha512-B/hVMRv8Nh1sQ1a3EY8I0n4Y1Wty3NrR5ebOyVT302op+DOAau+xNEImGMsUWOC3++ZlMooCytKz+NgN8aKGbA==",
       "dev": true,
       "requires": {
-        "JSONStream": "^1.0.4",
-        "is-text-path": "^1.0.0",
-        "lodash": "^4.2.1",
-        "meow": "^4.0.0",
-        "split2": "^2.0.0",
-        "through2": "^2.0.0",
-        "trim-off-newlines": "^1.0.0"
+        "@babel/template": "^7.3.3",
+        "@babel/types": "^7.3.3",
+        "@types/babel__core": "^7.0.0",
+        "@types/babel__traverse": "^7.0.6"
       }
     },
-    "conventional-recommended-bump": {
-      "version": "4.0.4",
-      "resolved": "https://registry.npmjs.org/conventional-recommended-bump/-/conventional-recommended-bump-4.0.4.tgz",
-      "integrity": "sha512-9mY5Yoblq+ZMqJpBzgS+RpSq+SUfP2miOR3H/NR9drGf08WCrY9B6HAGJZEm6+ThsVP917VHAahSOjM6k1vhPg==",
+    "babel-preset-current-node-syntax": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-0.1.3.tgz",
+      "integrity": "sha512-uyexu1sVwcdFnyq9o8UQYsXwXflIh8LvrF5+cKrYam93ned1CStffB3+BEcsxGSgagoA3GEyjDqO4a/58hyPYQ==",
       "dev": true,
       "requires": {
-        "concat-stream": "^1.6.0",
-        "conventional-changelog-preset-loader": "^2.0.2",
-        "conventional-commits-filter": "^2.0.1",
-        "conventional-commits-parser": "^3.0.1",
-        "git-raw-commits": "2.0.0",
-        "git-semver-tags": "^2.0.2",
-        "meow": "^4.0.0",
-        "q": "^1.5.1"
+        "@babel/plugin-syntax-async-generators": "^7.8.4",
+        "@babel/plugin-syntax-bigint": "^7.8.3",
+        "@babel/plugin-syntax-class-properties": "^7.8.3",
+        "@babel/plugin-syntax-import-meta": "^7.8.3",
+        "@babel/plugin-syntax-json-strings": "^7.8.3",
+        "@babel/plugin-syntax-logical-assignment-operators": "^7.8.3",
+        "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3",
+        "@babel/plugin-syntax-numeric-separator": "^7.8.3",
+        "@babel/plugin-syntax-object-rest-spread": "^7.8.3",
+        "@babel/plugin-syntax-optional-catch-binding": "^7.8.3",
+        "@babel/plugin-syntax-optional-chaining": "^7.8.3"
       }
     },
-    "convert-source-map": {
-      "version": "1.6.0",
-      "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.6.0.tgz",
-      "integrity": "sha512-eFu7XigvxdZ1ETfbgPBohgyQ/Z++C0eEhTor0qRwBw9unw+L0/6V8wkSuGgzdThkiS5lSpdptOQPD8Ak40a+7A==",
+    "babel-preset-jest": {
+      "version": "26.3.0",
+      "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-26.3.0.tgz",
+      "integrity": "sha512-5WPdf7nyYi2/eRxCbVrE1kKCWxgWY4RsPEbdJWFm7QsesFGqjdkyLeu1zRkwM1cxK6EPIlNd6d2AxLk7J+t4pw==",
       "dev": true,
       "requires": {
-        "safe-buffer": "~5.1.1"
+        "babel-plugin-jest-hoist": "^26.2.0",
+        "babel-preset-current-node-syntax": "^0.1.3"
       }
     },
-    "copy-concurrently": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/copy-concurrently/-/copy-concurrently-1.0.5.tgz",
-      "integrity": "sha512-f2domd9fsVDFtaFcbaRZuYXwtdmnzqbADSwhSWYxYB/Q8zsdUUFMXVRwXGDMWmbEzAn1kdRrtI1T/KTFOL4X2A==",
+    "bach": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/bach/-/bach-1.2.0.tgz",
+      "integrity": "sha1-Szzpa/JxNPeaG0FKUcFONMO9mIA=",
       "dev": true,
       "requires": {
-        "aproba": "^1.1.1",
-        "fs-write-stream-atomic": "^1.0.8",
-        "iferr": "^0.1.5",
-        "mkdirp": "^0.5.1",
-        "rimraf": "^2.5.4",
-        "run-queue": "^1.0.0"
+        "arr-filter": "^1.1.1",
+        "arr-flatten": "^1.0.1",
+        "arr-map": "^2.0.0",
+        "array-each": "^1.0.0",
+        "array-initial": "^1.0.0",
+        "array-last": "^1.1.1",
+        "async-done": "^1.2.2",
+        "async-settle": "^1.0.0",
+        "now-and-later": "^2.0.0"
       }
     },
-    "copy-descriptor": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz",
-      "integrity": "sha1-Z29us8OZl8LuGsOpJP1hJHSPV40=",
+    "balanced-match": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
+      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
       "dev": true
     },
-    "copy-props": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/copy-props/-/copy-props-2.0.4.tgz",
-      "integrity": "sha512-7cjuUME+p+S3HZlbllgsn2CDwS+5eCCX16qBgNC4jgSTf49qR1VKy/Zhl400m0IQXl/bPGEVqncgUUMjrr4s8A==",
-      "dev": true,
-      "requires": {
-        "each-props": "^1.3.0",
-        "is-plain-object": "^2.0.1"
-      }
-    },
-    "core-util-is": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
-      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
-      "dev": true
-    },
-    "cosmiconfig": {
-      "version": "5.0.7",
-      "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-5.0.7.tgz",
-      "integrity": "sha512-PcLqxTKiDmNT6pSpy4N6KtuPwb53W+2tzNvwOZw0WH9N6O0vLIBq0x8aj8Oj75ere4YcGi48bDFCL+3fRJdlNA==",
+    "base": {
+      "version": "0.11.2",
+      "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz",
+      "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==",
       "dev": true,
       "requires": {
-        "import-fresh": "^2.0.0",
-        "is-directory": "^0.3.1",
-        "js-yaml": "^3.9.0",
-        "parse-json": "^4.0.0"
+        "cache-base": "^1.0.1",
+        "class-utils": "^0.3.5",
+        "component-emitter": "^1.2.1",
+        "define-property": "^1.0.0",
+        "isobject": "^3.0.1",
+        "mixin-deep": "^1.2.0",
+        "pascalcase": "^0.1.1"
       },
       "dependencies": {
-        "parse-json": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
-          "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=",
+        "define-property": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
+          "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=",
           "dev": true,
           "requires": {
-            "error-ex": "^1.3.1",
-            "json-parse-better-errors": "^1.0.1"
+            "is-descriptor": "^1.0.0"
+          }
+        },
+        "is-accessor-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
+          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
+          "dev": true,
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
+        },
+        "is-data-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
+          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
+          "dev": true,
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
+        },
+        "is-descriptor": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
+          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+          "dev": true,
+          "requires": {
+            "is-accessor-descriptor": "^1.0.0",
+            "is-data-descriptor": "^1.0.0",
+            "kind-of": "^6.0.2"
           }
         }
       }
     },
-    "coveralls": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/coveralls/-/coveralls-3.0.3.tgz",
-      "integrity": "sha512-viNfeGlda2zJr8Gj1zqXpDMRjw9uM54p7wzZdvLRyOgnAfCe974Dq4veZkjJdxQXbmdppu6flEajFYseHYaUhg==",
-      "dev": true,
-      "requires": {
-        "growl": "~> 1.10.0",
-        "js-yaml": "^3.11.0",
-        "lcov-parse": "^0.0.10",
-        "log-driver": "^1.2.7",
-        "minimist": "^1.2.0",
-        "request": "^2.86.0"
-      }
+    "base64-js": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz",
+      "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==",
+      "dev": true
     },
-    "create-ecdh": {
-      "version": "4.0.3",
-      "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.3.tgz",
-      "integrity": "sha512-GbEHQPMOswGpKXM9kCWVrremUcBmjteUaQ01T9rkKCPDXfUHX0IoP9LpHYo2NPFampa4e+/pFDc3jQdxrxQLaw==",
+    "bcrypt-pbkdf": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
+      "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=",
       "dev": true,
       "requires": {
-        "bn.js": "^4.1.0",
-        "elliptic": "^6.0.0"
+        "tweetnacl": "^0.14.3"
       }
     },
-    "create-hash": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz",
-      "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==",
-      "dev": true,
-      "requires": {
-        "cipher-base": "^1.0.1",
-        "inherits": "^2.0.1",
-        "md5.js": "^1.3.4",
-        "ripemd160": "^2.0.1",
-        "sha.js": "^2.4.0"
-      }
+    "before-after-hook": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.1.0.tgz",
+      "integrity": "sha512-IWIbu7pMqyw3EAJHzzHbWa85b6oud/yfKYg5rqB5hNE8CeMi3nX+2C2sj0HswfblST86hpVEOAb9x34NZd6P7A==",
+      "dev": true
     },
-    "create-hmac": {
-      "version": "1.1.7",
-      "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz",
-      "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==",
+    "benchmark": {
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/benchmark/-/benchmark-2.1.4.tgz",
+      "integrity": "sha1-CfPeMckWQl1JjMLuVloOvzwqVik=",
       "dev": true,
       "requires": {
-        "cipher-base": "^1.0.3",
-        "create-hash": "^1.1.0",
-        "inherits": "^2.0.1",
-        "ripemd160": "^2.0.0",
-        "safe-buffer": "^5.0.1",
-        "sha.js": "^2.4.8"
+        "lodash": "^4.17.4",
+        "platform": "^1.3.3"
       }
     },
-    "cross-spawn": {
-      "version": "6.0.5",
-      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz",
-      "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==",
-      "dev": true,
-      "requires": {
-        "nice-try": "^1.0.4",
-        "path-key": "^2.0.1",
-        "semver": "^5.5.0",
-        "shebang-command": "^1.2.0",
-        "which": "^1.2.9"
-      }
+    "big.js": {
+      "version": "5.2.2",
+      "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz",
+      "integrity": "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==",
+      "dev": true
     },
-    "cross-spawn-async": {
-      "version": "2.2.5",
-      "resolved": "https://registry.npmjs.org/cross-spawn-async/-/cross-spawn-async-2.2.5.tgz",
-      "integrity": "sha1-hF/wwINKPe2dFg2sptOQkGuyiMw=",
+    "binary-extensions": {
+      "version": "1.13.1",
+      "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.13.1.tgz",
+      "integrity": "sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw==",
+      "dev": true
+    },
+    "bindings": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
+      "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
       "dev": true,
+      "optional": true,
       "requires": {
-        "lru-cache": "^4.0.0",
-        "which": "^1.2.8"
+        "file-uri-to-path": "1.0.0"
       }
     },
-    "crypto-browserify": {
-      "version": "3.12.0",
-      "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz",
-      "integrity": "sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==",
+    "bluebird": {
+      "version": "3.7.2",
+      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
+      "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==",
+      "dev": true
+    },
+    "bn.js": {
+      "version": "5.1.3",
+      "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.1.3.tgz",
+      "integrity": "sha512-GkTiFpjFtUzU9CbMeJ5iazkCzGL3jrhzerzZIuqLABjbwRaFt33I9tUdSNryIptM+RxDet6OKm2WnLXzW51KsQ==",
+      "dev": true
+    },
+    "brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
       "dev": true,
       "requires": {
-        "browserify-cipher": "^1.0.0",
-        "browserify-sign": "^4.0.0",
-        "create-ecdh": "^4.0.0",
-        "create-hash": "^1.1.0",
-        "create-hmac": "^1.1.0",
-        "diffie-hellman": "^5.0.0",
-        "inherits": "^2.0.1",
-        "pbkdf2": "^3.0.3",
-        "public-encrypt": "^4.0.0",
-        "randombytes": "^2.0.0",
-        "randomfill": "^1.0.3"
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
       }
     },
-    "css": {
-      "version": "2.2.4",
-      "resolved": "https://registry.npmjs.org/css/-/css-2.2.4.tgz",
-      "integrity": "sha512-oUnjmWpy0niI3x/mPL8dVEI1l7MnG3+HHyRPHf+YFSbK+svOhXpmSOcDURUh2aOCgl2grzrOPt1nHLuCVFULLw==",
+    "braces": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz",
+      "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==",
       "dev": true,
       "requires": {
-        "inherits": "^2.0.3",
-        "source-map": "^0.6.1",
-        "source-map-resolve": "^0.5.2",
-        "urix": "^0.1.0"
+        "arr-flatten": "^1.1.0",
+        "array-unique": "^0.3.2",
+        "extend-shallow": "^2.0.1",
+        "fill-range": "^4.0.0",
+        "isobject": "^3.0.1",
+        "repeat-element": "^1.1.2",
+        "snapdragon": "^0.8.1",
+        "snapdragon-node": "^2.0.1",
+        "split-string": "^3.0.2",
+        "to-regex": "^3.0.1"
       },
       "dependencies": {
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-          "dev": true
+        "extend-shallow": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+          "dev": true,
+          "requires": {
+            "is-extendable": "^0.1.0"
+          }
         }
       }
     },
-    "cssom": {
-      "version": "0.3.6",
-      "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.6.tgz",
-      "integrity": "sha512-DtUeseGk9/GBW0hl0vVPpU22iHL6YB5BUX7ml1hB+GMpo0NX5G4voX3kdWiMSEguFtcW3Vh3djqNF4aIe6ne0A==",
+    "brorand": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz",
+      "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=",
       "dev": true
     },
-    "cssstyle": {
-      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-1.2.2.tgz",
-      "integrity": "sha512-43wY3kl1CVQSvL7wUY1qXkxVGkStjpkDmVjiIKX8R97uhajy8Bybay78uOtqvh7Q5GK75dNPfW0geWjE6qQQow==",
+    "browser-process-hrtime": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz",
+      "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==",
+      "dev": true
+    },
+    "browserify-aes": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz",
+      "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==",
       "dev": true,
       "requires": {
-        "cssom": "0.3.x"
+        "buffer-xor": "^1.0.3",
+        "cipher-base": "^1.0.0",
+        "create-hash": "^1.1.0",
+        "evp_bytestokey": "^1.0.3",
+        "inherits": "^2.0.1",
+        "safe-buffer": "^5.0.1"
       }
     },
-    "currently-unhandled": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/currently-unhandled/-/currently-unhandled-0.4.1.tgz",
-      "integrity": "sha1-mI3zP+qxke95mmE2nddsF635V+o=",
+    "browserify-cipher": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/browserify-cipher/-/browserify-cipher-1.0.1.tgz",
+      "integrity": "sha512-sPhkz0ARKbf4rRQt2hTpAHqn47X3llLkUGn+xEJzLjwY8LRs2p0v7ljvI5EyoRO/mexrNunNECisZs+gw2zz1w==",
       "dev": true,
       "requires": {
-        "array-find-index": "^1.0.1"
+        "browserify-aes": "^1.0.4",
+        "browserify-des": "^1.0.0",
+        "evp_bytestokey": "^1.0.0"
       }
     },
-    "cyclist": {
-      "version": "0.2.2",
-      "resolved": "https://registry.npmjs.org/cyclist/-/cyclist-0.2.2.tgz",
-      "integrity": "sha1-GzN5LhHpFKL9bW7WRHRkRE5fpkA=",
-      "dev": true
-    },
-    "d": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/d/-/d-1.0.0.tgz",
-      "integrity": "sha1-dUu1v+VUUdpppYuU1F9MWwRi1Y8=",
+    "browserify-des": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/browserify-des/-/browserify-des-1.0.2.tgz",
+      "integrity": "sha512-BioO1xf3hFwz4kc6iBhI3ieDFompMhrMlnDFC4/0/vd5MokpuAc3R+LYbwTA9A5Yc9pq9UYPqffKpW2ObuwX5A==",
       "dev": true,
       "requires": {
-        "es5-ext": "^0.10.9"
-      }
-    },
-    "dargs": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/dargs/-/dargs-4.1.0.tgz",
-      "integrity": "sha1-A6nbtLXC8Tm/FK5T8LiipqhvThc=",
-      "dev": true,
-      "requires": {
-        "number-is-nan": "^1.0.0"
+        "cipher-base": "^1.0.1",
+        "des.js": "^1.0.0",
+        "inherits": "^2.0.1",
+        "safe-buffer": "^5.1.2"
       }
     },
-    "dashdash": {
-      "version": "1.14.1",
-      "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
-      "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=",
+    "browserify-rsa": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz",
+      "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=",
       "dev": true,
       "requires": {
-        "assert-plus": "^1.0.0"
+        "bn.js": "^4.1.0",
+        "randombytes": "^2.0.1"
+      },
+      "dependencies": {
+        "bn.js": {
+          "version": "4.11.9",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==",
+          "dev": true
+        }
       }
     },
-    "data-urls": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-1.1.0.tgz",
-      "integrity": "sha512-YTWYI9se1P55u58gL5GkQHW4P6VJBJ5iBT+B5a7i2Tjadhv52paJG0qHX4A0OR6/t52odI64KP2YvFpkDOi3eQ==",
-      "dev": true,
-      "requires": {
-        "abab": "^2.0.0",
-        "whatwg-mimetype": "^2.2.0",
-        "whatwg-url": "^7.0.0"
+    "browserify-sign": {
+      "version": "4.2.1",
+      "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.1.tgz",
+      "integrity": "sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg==",
+      "dev": true,
+      "requires": {
+        "bn.js": "^5.1.1",
+        "browserify-rsa": "^4.0.1",
+        "create-hash": "^1.2.0",
+        "create-hmac": "^1.1.7",
+        "elliptic": "^6.5.3",
+        "inherits": "^2.0.4",
+        "parse-asn1": "^5.1.5",
+        "readable-stream": "^3.6.0",
+        "safe-buffer": "^5.2.0"
       },
       "dependencies": {
-        "whatwg-url": {
-          "version": "7.0.0",
-          "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.0.0.tgz",
-          "integrity": "sha512-37GeVSIJ3kn1JgKyjiYNmSLP1yzbpb29jdmwBSgkD9h40/hyrR/OifpVUndji3tmwGgD8qpw7iQu3RSbCrBpsQ==",
+        "readable-stream": {
+          "version": "3.6.0",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
+          "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
           "dev": true,
           "requires": {
-            "lodash.sortby": "^4.7.0",
-            "tr46": "^1.0.1",
-            "webidl-conversions": "^4.0.2"
+            "inherits": "^2.0.3",
+            "string_decoder": "^1.1.1",
+            "util-deprecate": "^1.0.1"
           }
         }
       }
     },
-    "date-now": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/date-now/-/date-now-0.1.4.tgz",
-      "integrity": "sha1-6vQ5/U1ISK105cx9vvIAZyueNFs=",
-      "dev": true
+    "browserify-zlib": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.2.0.tgz",
+      "integrity": "sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA==",
+      "dev": true,
+      "requires": {
+        "pako": "~1.0.5"
+      }
     },
-    "dateformat": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-3.0.3.tgz",
-      "integrity": "sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q==",
+    "bs-logger": {
+      "version": "0.2.6",
+      "resolved": "https://registry.npmjs.org/bs-logger/-/bs-logger-0.2.6.tgz",
+      "integrity": "sha512-pd8DCoxmbgc7hyPKOvxtqNcjYoOsABPQdcCUjGp3d42VR2CX1ORhk2A87oqqu5R1kk+76nsxZupkmyd+MVtCog==",
+      "dev": true,
+      "requires": {
+        "fast-json-stable-stringify": "2.x"
+      }
+    },
+    "bser": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz",
+      "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==",
+      "dev": true,
+      "requires": {
+        "node-int64": "^0.4.0"
+      }
+    },
+    "btoa-lite": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/btoa-lite/-/btoa-lite-1.0.0.tgz",
+      "integrity": "sha1-M3dm2hWAEhD92VbCLpxokaudAzc=",
       "dev": true
     },
-    "debug": {
-      "version": "2.6.9",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
-      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+    "buffer": {
+      "version": "4.9.2",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz",
+      "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==",
       "dev": true,
       "requires": {
-        "ms": "2.0.0"
+        "base64-js": "^1.0.2",
+        "ieee754": "^1.1.4",
+        "isarray": "^1.0.0"
       }
     },
-    "debug-fabulous": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/debug-fabulous/-/debug-fabulous-1.1.0.tgz",
-      "integrity": "sha512-GZqvGIgKNlUnHUPQhepnUZFIMoi3dgZKQBzKDeL2g7oJF9SNAji/AAu36dusFUas0O+pae74lNeoIPHqXWDkLg==",
+    "buffer-equal": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-1.0.0.tgz",
+      "integrity": "sha1-WWFrSYME1Var1GaWayLu2j7KX74=",
+      "dev": true
+    },
+    "buffer-from": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
+      "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==",
+      "dev": true
+    },
+    "buffer-xor": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz",
+      "integrity": "sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=",
+      "dev": true
+    },
+    "builtin-modules": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz",
+      "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8=",
+      "dev": true
+    },
+    "builtin-status-codes": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz",
+      "integrity": "sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=",
+      "dev": true
+    },
+    "builtins": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/builtins/-/builtins-1.0.3.tgz",
+      "integrity": "sha1-y5T662HIaWRR2zZTThQi+U8K7og=",
+      "dev": true
+    },
+    "byline": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/byline/-/byline-5.0.0.tgz",
+      "integrity": "sha1-dBxSFkaOrcRXsDQQEYrXfejB3bE=",
+      "dev": true
+    },
+    "byte-size": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/byte-size/-/byte-size-5.0.1.tgz",
+      "integrity": "sha512-/XuKeqWocKsYa/cBY1YbSJSWWqTi4cFgr9S6OyM7PBaPbr9zvNGwWP33vt0uqGhwDdN+y3yhbXVILEUpnwEWGw==",
+      "dev": true
+    },
+    "cacache": {
+      "version": "12.0.4",
+      "resolved": "https://registry.npmjs.org/cacache/-/cacache-12.0.4.tgz",
+      "integrity": "sha512-a0tMB40oefvuInr4Cwb3GerbL9xTj1D5yg0T5xrjGCGyfvbxseIXX7BAO/u/hIXdafzOI5JC3wDwHyf24buOAQ==",
       "dev": true,
       "requires": {
-        "debug": "3.X",
-        "memoizee": "0.4.X",
-        "object-assign": "4.X"
+        "bluebird": "^3.5.5",
+        "chownr": "^1.1.1",
+        "figgy-pudding": "^3.5.1",
+        "glob": "^7.1.4",
+        "graceful-fs": "^4.1.15",
+        "infer-owner": "^1.0.3",
+        "lru-cache": "^5.1.1",
+        "mississippi": "^3.0.0",
+        "mkdirp": "^0.5.1",
+        "move-concurrently": "^1.0.1",
+        "promise-inflight": "^1.0.1",
+        "rimraf": "^2.6.3",
+        "ssri": "^6.0.1",
+        "unique-filename": "^1.1.1",
+        "y18n": "^4.0.0"
       },
       "dependencies": {
-        "debug": {
-          "version": "3.2.6",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
-          "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
+        "mkdirp": {
+          "version": "0.5.5",
+          "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
+          "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
           "dev": true,
           "requires": {
-            "ms": "^2.1.1"
+            "minimist": "^1.2.5"
           }
         },
-        "ms": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
-          "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==",
+        "y18n": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
+          "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
           "dev": true
         }
       }
     },
-    "debuglog": {
+    "cache-base": {
       "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz",
-      "integrity": "sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI=",
-      "dev": true
+      "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz",
+      "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==",
+      "dev": true,
+      "requires": {
+        "collection-visit": "^1.0.0",
+        "component-emitter": "^1.2.1",
+        "get-value": "^2.0.6",
+        "has-value": "^1.0.0",
+        "isobject": "^3.0.1",
+        "set-value": "^2.0.0",
+        "to-object-path": "^0.3.0",
+        "union-value": "^1.0.0",
+        "unset-value": "^1.0.0"
+      }
     },
-    "decamelize": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
-      "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=",
+    "call-me-maybe": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/call-me-maybe/-/call-me-maybe-1.0.1.tgz",
+      "integrity": "sha1-JtII6onje1y95gJQoV8DHBak1ms=",
       "dev": true
     },
-    "decamelize-keys": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/decamelize-keys/-/decamelize-keys-1.1.0.tgz",
-      "integrity": "sha1-0XGoeTMlKAfrPLYdwcFEXQeN8tk=",
+    "caller-callsite": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/caller-callsite/-/caller-callsite-2.0.0.tgz",
+      "integrity": "sha1-hH4PzgoiN1CpoCfFSzNzGtMVQTQ=",
       "dev": true,
       "requires": {
-        "decamelize": "^1.1.0",
-        "map-obj": "^1.0.0"
+        "callsites": "^2.0.0"
       },
       "dependencies": {
-        "map-obj": {
-          "version": "1.0.1",
-          "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz",
-          "integrity": "sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=",
+        "callsites": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/callsites/-/callsites-2.0.0.tgz",
+          "integrity": "sha1-BuuE8A7qQT2oav/vrL/7Ngk7PFA=",
           "dev": true
         }
       }
     },
-    "decode-uri-component": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz",
-      "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=",
-      "dev": true
+    "caller-path": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/caller-path/-/caller-path-2.0.0.tgz",
+      "integrity": "sha1-Ro+DBE42mrIBD6xfBs7uFbsssfQ=",
+      "dev": true,
+      "requires": {
+        "caller-callsite": "^2.0.0"
+      }
     },
-    "dedent": {
-      "version": "0.7.0",
-      "resolved": "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz",
-      "integrity": "sha1-JJXduvbrh0q7Dhvp3yLS5aVEMmw=",
+    "callsites": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
+      "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
       "dev": true
     },
-    "deep-extend": {
-      "version": "0.6.0",
-      "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
-      "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="
-    },
-    "deep-is": {
-      "version": "0.1.3",
-      "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz",
-      "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=",
+    "camelcase": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-3.0.0.tgz",
+      "integrity": "sha1-MvxLn82vhF/N9+c7uXysImHwqwo=",
       "dev": true
     },
-    "default-compare": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/default-compare/-/default-compare-1.0.0.tgz",
-      "integrity": "sha512-QWfXlM0EkAbqOCbD/6HjdwT19j7WCkMyiRhWilc4H9/5h/RzTF9gv5LYh1+CmDV5d1rki6KAWLtQale0xt20eQ==",
+    "camelcase-keys": {
+      "version": "6.2.2",
+      "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-6.2.2.tgz",
+      "integrity": "sha512-YrwaA0vEKazPBkn0ipTiMpSajYDSe+KjQfrjhcBMxJt/znbvlHd8Pw/Vamaz5EB4Wfhs3SUR3Z9mwRu/P3s3Yg==",
       "dev": true,
       "requires": {
-        "kind-of": "^5.0.2"
+        "camelcase": "^5.3.1",
+        "map-obj": "^4.0.0",
+        "quick-lru": "^4.0.1"
       },
       "dependencies": {
-        "kind-of": {
-          "version": "5.1.0",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
-          "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==",
+        "camelcase": {
+          "version": "5.3.1",
+          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
+          "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==",
           "dev": true
         }
       }
     },
-    "default-resolution": {
+    "capture-exit": {
       "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/default-resolution/-/default-resolution-2.0.0.tgz",
-      "integrity": "sha1-vLgrqnKtebQmp2cy8aga1t8m1oQ=",
-      "dev": true
-    },
-    "defaults": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz",
-      "integrity": "sha1-xlYFHpgX2f8I7YgUd/P+QBnz730=",
-      "dev": true,
-      "requires": {
-        "clone": "^1.0.2"
-      },
-      "dependencies": {
-        "clone": {
-          "version": "1.0.4",
-          "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz",
-          "integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4=",
-          "dev": true
-        }
-      }
-    },
-    "define-properties": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz",
-      "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==",
+      "resolved": "https://registry.npmjs.org/capture-exit/-/capture-exit-2.0.0.tgz",
+      "integrity": "sha512-PiT/hQmTonHhl/HFGN+Lx3JJUznrVYJ3+AQsnthneZbvW7x+f08Tk7yLJTLEOUvBTbduLeeBkxEaYXUOUrRq6g==",
       "dev": true,
       "requires": {
-        "object-keys": "^1.0.12"
+        "rsvp": "^4.8.4"
       }
     },
-    "define-property": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz",
-      "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==",
-      "dev": true,
-      "requires": {
-        "is-descriptor": "^1.0.2",
-        "isobject": "^3.0.1"
-      },
-      "dependencies": {
-        "is-accessor-descriptor": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
-          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
-          "dev": true,
-          "requires": {
-            "kind-of": "^6.0.0"
-          }
-        },
-        "is-data-descriptor": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
-          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
-          "dev": true,
-          "requires": {
-            "kind-of": "^6.0.0"
-          }
-        },
-        "is-descriptor": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
-          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
-          "dev": true,
-          "requires": {
-            "is-accessor-descriptor": "^1.0.0",
-            "is-data-descriptor": "^1.0.0",
-            "kind-of": "^6.0.2"
-          }
-        }
-      }
+    "caseless": {
+      "version": "0.12.0",
+      "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
+      "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=",
+      "dev": true
     },
-    "del": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/del/-/del-3.0.0.tgz",
-      "integrity": "sha1-U+z2mf/LyzljdpGrE7rxYIGXZuU=",
-      "dev": true,
+    "chalk": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
+      "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
       "requires": {
-        "globby": "^6.1.0",
-        "is-path-cwd": "^1.0.0",
-        "is-path-in-cwd": "^1.0.0",
-        "p-map": "^1.1.1",
-        "pify": "^3.0.0",
-        "rimraf": "^2.2.8"
+        "ansi-styles": "^3.2.1",
+        "escape-string-regexp": "^1.0.5",
+        "supports-color": "^5.3.0"
       }
     },
-    "delayed-stream": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
-      "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=",
+    "char-regex": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz",
+      "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==",
       "dev": true
     },
-    "delegates": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
-      "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=",
+    "chardet": {
+      "version": "0.7.0",
+      "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz",
+      "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==",
       "dev": true
     },
-    "des.js": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.0.0.tgz",
-      "integrity": "sha1-wHTS4qpqipoH29YfmhXCzYPsjsw=",
+    "chokidar": {
+      "version": "2.1.8",
+      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.1.8.tgz",
+      "integrity": "sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg==",
       "dev": true,
       "requires": {
-        "inherits": "^2.0.1",
-        "minimalistic-assert": "^1.0.0"
+        "anymatch": "^2.0.0",
+        "async-each": "^1.0.1",
+        "braces": "^2.3.2",
+        "fsevents": "^1.2.7",
+        "glob-parent": "^3.1.0",
+        "inherits": "^2.0.3",
+        "is-binary-path": "^1.0.0",
+        "is-glob": "^4.0.0",
+        "normalize-path": "^3.0.0",
+        "path-is-absolute": "^1.0.0",
+        "readdirp": "^2.2.1",
+        "upath": "^1.1.1"
       }
     },
-    "detect-file": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/detect-file/-/detect-file-1.0.0.tgz",
-      "integrity": "sha1-8NZtA2cqglyxtzvbP+YjEMjlUrc=",
-      "dev": true
-    },
-    "detect-newline": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-2.1.0.tgz",
-      "integrity": "sha1-9B8cEL5LAOh7XxPaaAdZ8sW/0+I=",
+    "chownr": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
+      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
       "dev": true
     },
-    "dezalgo": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.3.tgz",
-      "integrity": "sha1-f3Qt4Gb8dIvI24IFad3c5Jvw1FY=",
+    "chrome-trace-event": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.2.tgz",
+      "integrity": "sha512-9e/zx1jw7B4CO+c/RXoCsfg/x1AfUBioy4owYH0bJprEYAx5hRFLRhWBqHAG57D0ZM4H7vxbP7bPe0VwhQRYDQ==",
       "dev": true,
       "requires": {
-        "asap": "^2.0.0",
-        "wrappy": "1"
+        "tslib": "^1.9.0"
       }
     },
-    "diff": {
-      "version": "3.5.0",
-      "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz",
-      "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==",
-      "dev": true
-    },
-    "diff-sequences": {
-      "version": "24.3.0",
-      "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-24.3.0.tgz",
-      "integrity": "sha512-xLqpez+Zj9GKSnPWS0WZw1igGocZ+uua8+y+5dDNTT934N3QuY1sp2LkHzwiaYQGz60hMq0pjAshdeXm5VUOEw==",
+    "ci-info": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
+      "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==",
       "dev": true
     },
-    "diffie-hellman": {
-      "version": "5.0.3",
-      "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz",
-      "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==",
+    "cipher-base": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz",
+      "integrity": "sha512-Kkht5ye6ZGmwv40uUDZztayT2ThLQGfnj/T71N/XzeZeo3nf8foyW7zGTsPYkEya3m5f3cAypH+qe7YOrM1U2Q==",
       "dev": true,
       "requires": {
-        "bn.js": "^4.1.0",
-        "miller-rabin": "^4.0.0",
-        "randombytes": "^2.0.0"
+        "inherits": "^2.0.1",
+        "safe-buffer": "^5.0.1"
       }
     },
-    "dir-glob": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-2.0.0.tgz",
-      "integrity": "sha512-37qirFDz8cA5fimp9feo43fSuRo2gHwaIn6dXL8Ber1dGwUosDrGZeCCXq57WnIqE4aQ+u3eQZzsk1yOzhdwag==",
+    "class-utils": {
+      "version": "0.3.6",
+      "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz",
+      "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==",
       "dev": true,
       "requires": {
-        "arrify": "^1.0.1",
-        "path-type": "^3.0.0"
+        "arr-union": "^3.1.0",
+        "define-property": "^0.2.5",
+        "isobject": "^3.0.0",
+        "static-extend": "^0.1.1"
       },
       "dependencies": {
-        "path-type": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz",
-          "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==",
+        "define-property": {
+          "version": "0.2.5",
+          "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
+          "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
           "dev": true,
           "requires": {
-            "pify": "^3.0.0"
+            "is-descriptor": "^0.1.0"
           }
         }
       }
     },
-    "domain-browser": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz",
-      "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==",
+    "clean-stack": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz",
+      "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==",
       "dev": true
     },
-    "domexception": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz",
-      "integrity": "sha512-raigMkn7CJNNo6Ihro1fzG7wr3fHuYVytzquZKX5n0yizGsTcYgzdIUwj1X9pK0VvjeihV+XiclP+DjwbsSKug==",
+    "cli-cursor": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz",
+      "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=",
       "dev": true,
       "requires": {
-        "webidl-conversions": "^4.0.2"
+        "restore-cursor": "^2.0.0"
       }
     },
-    "dot-prop": {
-      "version": "4.2.0",
-      "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.0.tgz",
-      "integrity": "sha512-tUMXrxlExSW6U2EXiiKGSBVdYgtV8qlHL+C10TsW4PURY/ic+eaysnSkwB4kA/mBlCyy/IKDJ+Lc3wbWeaXtuQ==",
+    "cli-width": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.1.tgz",
+      "integrity": "sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw==",
+      "dev": true
+    },
+    "cliui": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz",
+      "integrity": "sha1-EgYBU3qRbSmUD5NNo7SNWFo5IT0=",
       "dev": true,
       "requires": {
-        "is-obj": "^1.0.0"
+        "string-width": "^1.0.1",
+        "strip-ansi": "^3.0.1",
+        "wrap-ansi": "^2.0.0"
       }
     },
-    "duplexer": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz",
-      "integrity": "sha1-rOb/gIwc5mtX0ev5eXessCM0z8E=",
+    "clone": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz",
+      "integrity": "sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18=",
       "dev": true
     },
-    "duplexify": {
-      "version": "3.6.1",
-      "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.6.1.tgz",
-      "integrity": "sha512-vM58DwdnKmty+FSPzT14K9JXb90H+j5emaR4KYbr2KTIz00WHGbWOe5ghQTx233ZCLZtrGDALzKwcjEtSt35mA==",
-      "dev": true,
-      "requires": {
-        "end-of-stream": "^1.0.0",
-        "inherits": "^2.0.1",
-        "readable-stream": "^2.0.0",
-        "stream-shift": "^1.0.0"
-      }
+    "clone-buffer": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/clone-buffer/-/clone-buffer-1.0.0.tgz",
+      "integrity": "sha1-4+JbIHrE5wGvch4staFnksrD3Fg=",
+      "dev": true
     },
-    "each-props": {
-      "version": "1.3.2",
-      "resolved": "https://registry.npmjs.org/each-props/-/each-props-1.3.2.tgz",
-      "integrity": "sha512-vV0Hem3zAGkJAyU7JSjixeU66rwdynTAa1vofCrSA5fEln+m67Az9CcnkVD776/fsN/UjIWmBDoNRS6t6G9RfA==",
+    "clone-deep": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz",
+      "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==",
       "dev": true,
       "requires": {
-        "is-plain-object": "^2.0.1",
-        "object.defaults": "^1.1.0"
+        "is-plain-object": "^2.0.4",
+        "kind-of": "^6.0.2",
+        "shallow-clone": "^3.0.0"
       }
     },
-    "ecc-jsbn": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
-      "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=",
-      "dev": true,
-      "requires": {
-        "jsbn": "~0.1.0",
-        "safer-buffer": "^2.1.0"
-      }
+    "clone-stats": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/clone-stats/-/clone-stats-1.0.0.tgz",
+      "integrity": "sha1-s3gt/4u1R04Yuba/D9/ngvh3doA=",
+      "dev": true
     },
-    "elliptic": {
-      "version": "6.4.1",
-      "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.4.1.tgz",
-      "integrity": "sha512-BsXLz5sqX8OHcsh7CqBMztyXARmGQ3LWPtGjJi6DiJHq5C/qvi9P3OqgswKSDftbu8+IoI/QDTAm2fFnQ9SZSQ==",
+    "cloneable-readable": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/cloneable-readable/-/cloneable-readable-1.1.3.tgz",
+      "integrity": "sha512-2EF8zTQOxYq70Y4XKtorQupqF0m49MBz2/yf5Bj+MHjvpG3Hy7sImifnqD6UA+TKYxeSV+u6qqQPawN5UvnpKQ==",
       "dev": true,
       "requires": {
-        "bn.js": "^4.4.0",
-        "brorand": "^1.0.1",
-        "hash.js": "^1.0.0",
-        "hmac-drbg": "^1.0.0",
         "inherits": "^2.0.1",
-        "minimalistic-assert": "^1.0.0",
-        "minimalistic-crypto-utils": "^1.0.0"
-      }
-    },
-    "emojis-list": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-2.1.0.tgz",
-      "integrity": "sha1-TapNnbAPmBmIDHn6RXrlsJof04k=",
-      "dev": true
-    },
-    "encoding": {
-      "version": "0.1.12",
-      "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.12.tgz",
-      "integrity": "sha1-U4tm8+5izRq1HsMjgp0flIDHS+s=",
-      "dev": true,
-      "requires": {
-        "iconv-lite": "~0.4.13"
+        "process-nextick-args": "^2.0.0",
+        "readable-stream": "^2.3.5"
+      },
+      "dependencies": {
+        "process-nextick-args": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
+          "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
+          "dev": true
+        }
       }
     },
-    "end-of-stream": {
-      "version": "1.4.1",
-      "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz",
-      "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==",
-      "dev": true,
-      "requires": {
-        "once": "^1.4.0"
-      }
+    "co": {
+      "version": "4.6.0",
+      "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
+      "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=",
+      "dev": true
     },
-    "enhanced-resolve": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-4.1.0.tgz",
-      "integrity": "sha512-F/7vkyTtyc/llOIn8oWclcB25KdRaiPBpZYDgJHgh/UHtpgT2p2eldQgtQnLtUvfMKPKxbRaQM/hHkvLHt1Vng==",
-      "dev": true,
-      "requires": {
-        "graceful-fs": "^4.1.2",
-        "memory-fs": "^0.4.0",
-        "tapable": "^1.0.0"
-      }
+    "code-point-at": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
+      "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=",
+      "dev": true
     },
-    "err-code": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/err-code/-/err-code-1.1.2.tgz",
-      "integrity": "sha1-BuARbTAo9q70gGhJ6w6mp0iuaWA=",
+    "collect-v8-coverage": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.1.tgz",
+      "integrity": "sha512-iBPtljfCNcTKNAto0KEtDfZ3qzjJvqE3aTGZsbhjSBlorqpXJlaWWtPO35D+ZImoC3KWejX64o+yPGxhWSTzfg==",
       "dev": true
     },
-    "errno": {
-      "version": "0.1.7",
-      "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz",
-      "integrity": "sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==",
+    "collection-map": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/collection-map/-/collection-map-1.0.0.tgz",
+      "integrity": "sha1-rqDwb40mx4DCt1SUOFVEsiVa8Yw=",
       "dev": true,
       "requires": {
-        "prr": "~1.0.1"
+        "arr-map": "^2.0.2",
+        "for-own": "^1.0.0",
+        "make-iterator": "^1.0.0"
       }
     },
-    "error-ex": {
-      "version": "1.3.2",
-      "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz",
-      "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==",
+    "collection-visit": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz",
+      "integrity": "sha1-S8A3PBZLwykbTTaMgpzxqApZ3KA=",
       "dev": true,
       "requires": {
-        "is-arrayish": "^0.2.1"
+        "map-visit": "^1.0.0",
+        "object-visit": "^1.0.0"
       }
     },
-    "es-abstract": {
-      "version": "1.13.0",
-      "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.13.0.tgz",
-      "integrity": "sha512-vDZfg/ykNxQVwup/8E1BZhVzFfBxs9NqMzGcvIJrqg5k2/5Za2bWo40dK2J1pgLngZ7c+Shh8lwYtLGyrwPutg==",
-      "dev": true,
+    "color-convert": {
+      "version": "1.9.3",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
+      "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
       "requires": {
-        "es-to-primitive": "^1.2.0",
-        "function-bind": "^1.1.1",
-        "has": "^1.0.3",
-        "is-callable": "^1.1.4",
-        "is-regex": "^1.0.4",
-        "object-keys": "^1.0.12"
+        "color-name": "1.1.3"
       }
     },
-    "es-to-primitive": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.0.tgz",
-      "integrity": "sha512-qZryBOJjV//LaxLTV6UC//WewneB3LcXOL9NP++ozKVXsIIIpm/2c13UDiD9Jp2eThsecw9m3jPqDwTyobcdbg==",
-      "dev": true,
-      "requires": {
-        "is-callable": "^1.1.4",
-        "is-date-object": "^1.0.1",
-        "is-symbol": "^1.0.2"
-      }
+    "color-name": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
+      "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
     },
-    "es5-ext": {
-      "version": "0.10.47",
-      "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.47.tgz",
-      "integrity": "sha512-/1TItLfj+TTfWoeRcDn/0FbGV6SNo4R+On2GGVucPU/j3BWnXE2Co8h8CTo4Tu34gFJtnmwS9xiScKs4EjZhdw==",
-      "dev": true,
-      "requires": {
-        "es6-iterator": "~2.0.3",
-        "es6-symbol": "~3.1.1",
-        "next-tick": "1"
-      }
+    "color-support": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz",
+      "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==",
+      "dev": true
     },
-    "es6-iterator": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz",
-      "integrity": "sha1-p96IkUGgWpSwhUQDstCg+/qY87c=",
+    "columnify": {
+      "version": "1.5.4",
+      "resolved": "https://registry.npmjs.org/columnify/-/columnify-1.5.4.tgz",
+      "integrity": "sha1-Rzfd8ce2mop8NAVweC6UfuyOeLs=",
       "dev": true,
       "requires": {
-        "d": "1",
-        "es5-ext": "^0.10.35",
-        "es6-symbol": "^3.1.1"
+        "strip-ansi": "^3.0.0",
+        "wcwidth": "^1.0.0"
       }
     },
-    "es6-object-assign": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/es6-object-assign/-/es6-object-assign-1.1.0.tgz",
-      "integrity": "sha1-wsNYJlYkfDnqEHyx5mUrb58kUjw=",
-      "dev": true
-    },
-    "es6-promise": {
-      "version": "4.2.5",
-      "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.5.tgz",
-      "integrity": "sha512-n6wvpdE43VFtJq+lUDYDBFUwV8TZbuGXLV4D6wKafg13ldznKsyEvatubnmUe31zcvelSzOHF+XbaT+Bl9ObDg==",
-      "dev": true
-    },
-    "es6-promisify": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz",
-      "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=",
+    "combined-stream": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
       "dev": true,
       "requires": {
-        "es6-promise": "^4.0.3"
+        "delayed-stream": "~1.0.0"
       }
     },
-    "es6-symbol": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.1.tgz",
-      "integrity": "sha1-vwDvT9q2uhtG7Le2KbTH7VcVzHc=",
-      "dev": true,
+    "command-line-args": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.0.2.tgz",
+      "integrity": "sha512-/qPcbL8zpqg53x4rAaqMFlRV4opN3pbla7I7k9x8kyOBMQoGT6WltjN6sXZuxOXw6DgdK7Ad+ijYS5gjcr7vlA==",
       "requires": {
-        "d": "1",
-        "es5-ext": "~0.10.14"
+        "argv-tools": "^0.1.1",
+        "array-back": "^2.0.0",
+        "find-replace": "^2.0.1",
+        "lodash.camelcase": "^4.3.0",
+        "typical": "^2.6.1"
       }
     },
-    "es6-weak-map": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/es6-weak-map/-/es6-weak-map-2.0.2.tgz",
-      "integrity": "sha1-XjqzIlH/0VOKH45f+hNXdy+S2W8=",
-      "dev": true,
+    "command-line-usage": {
+      "version": "5.0.5",
+      "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-5.0.5.tgz",
+      "integrity": "sha512-d8NrGylA5oCXSbGoKz05FkehDAzSmIm4K03S5VDh4d5lZAtTWfc3D1RuETtuQCn8129nYfJfDdF7P/lwcz1BlA==",
       "requires": {
-        "d": "1",
-        "es5-ext": "^0.10.14",
-        "es6-iterator": "^2.0.1",
-        "es6-symbol": "^3.1.1"
+        "array-back": "^2.0.0",
+        "chalk": "^2.4.1",
+        "table-layout": "^0.4.3",
+        "typical": "^2.6.1"
       }
     },
-    "escape-string-applescript": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/escape-string-applescript/-/escape-string-applescript-2.0.0.tgz",
-      "integrity": "sha1-dgvKg4Zo5Aj+XuUs5CyvfLRsUnM=",
+    "commander": {
+      "version": "2.20.3",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
+      "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
       "dev": true
     },
-    "escape-string-regexp": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
-      "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ="
+    "commondir": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz",
+      "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=",
+      "dev": true
     },
-    "escodegen": {
-      "version": "1.11.1",
-      "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.11.1.tgz",
-      "integrity": "sha512-JwiqFD9KdGVVpeuRa68yU3zZnBEOcPs0nKW7wZzXky8Z7tffdYUHbe11bPCV5jYlK6DVdKLWLm0f5I/QlL0Kmw==",
+    "compare-func": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/compare-func/-/compare-func-2.0.0.tgz",
+      "integrity": "sha512-zHig5N+tPWARooBnb0Zx1MFcdfpyJrfTJ3Y5L+IFvUm8rM74hHz66z0gw0x4tijh5CorKkKUCnW82R2vmpeCRA==",
       "dev": true,
       "requires": {
-        "esprima": "^3.1.3",
-        "estraverse": "^4.2.0",
-        "esutils": "^2.0.2",
-        "optionator": "^0.8.1",
-        "source-map": "~0.6.1"
+        "array-ify": "^1.0.0",
+        "dot-prop": "^5.1.0"
       },
       "dependencies": {
-        "esprima": {
-          "version": "3.1.3",
-          "resolved": "https://registry.npmjs.org/esprima/-/esprima-3.1.3.tgz",
-          "integrity": "sha1-/cpRzuYTOJXjyI1TXOSdv/YqRjM=",
-          "dev": true
-        },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+        "dot-prop": {
+          "version": "5.3.0",
+          "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.3.0.tgz",
+          "integrity": "sha512-QM8q3zDe58hqUqjraQOmzZ1LIH9SWQJTlEKCH4kJ2oQvLZk7RbQXvtDM2XEq3fwkV9CCvvH4LA0AV+ogFsBM2Q==",
           "dev": true,
-          "optional": true
+          "requires": {
+            "is-obj": "^2.0.0"
+          }
+        },
+        "is-obj": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz",
+          "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==",
+          "dev": true
         }
       }
     },
-    "eslint-scope": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.0.tgz",
-      "integrity": "sha512-1G6UTDi7Jc1ELFwnR58HV4fK9OQK4S6N985f166xqXxpjU6plxFISJa2Ba9KCQuFa8RCnj/lSFJbHo7UFDBnUA==",
-      "dev": true,
-      "requires": {
-        "esrecurse": "^4.1.0",
-        "estraverse": "^4.1.1"
-      }
-    },
-    "esm": {
-      "version": "3.2.25",
-      "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz",
-      "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==",
+    "component-emitter": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz",
+      "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==",
       "dev": true
     },
-    "esprima": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
-      "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
+    "concat-map": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
       "dev": true
     },
-    "esrecurse": {
-      "version": "4.2.1",
-      "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz",
-      "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==",
+    "concat-stream": {
+      "version": "1.6.2",
+      "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
+      "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
       "dev": true,
       "requires": {
-        "estraverse": "^4.1.0"
+        "buffer-from": "^1.0.0",
+        "inherits": "^2.0.3",
+        "readable-stream": "^2.2.2",
+        "typedarray": "^0.0.6"
       }
     },
-    "estraverse": {
-      "version": "4.2.0",
-      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz",
-      "integrity": "sha1-De4/7TH81GlhjOc0IJn8GvoL2xM=",
-      "dev": true
-    },
-    "esutils": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz",
-      "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=",
-      "dev": true
-    },
-    "event-emitter": {
-      "version": "0.3.5",
-      "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz",
-      "integrity": "sha1-34xp7vFkeSPHFXuc6DhAYQsCzDk=",
+    "config-chain": {
+      "version": "1.1.12",
+      "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.12.tgz",
+      "integrity": "sha512-a1eOIcu8+7lUInge4Rpf/n4Krkf3Dd9lqhljRzII1/Zno/kRtUWnznPO3jOKBmTEktkt3fkxisUcivoj0ebzoA==",
       "dev": true,
       "requires": {
-        "d": "1",
-        "es5-ext": "~0.10.14"
+        "ini": "^1.3.4",
+        "proto-list": "~1.2.1"
       }
     },
-    "events": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/events/-/events-3.0.0.tgz",
-      "integrity": "sha512-Dc381HFWJzEOhQ+d8pkNon++bk9h6cdAoAj4iE6Q4y6xgTzySWXlKn05/TVNpjnfRqi/X0EpJEJohPjNI3zpVA==",
+    "console-browserify": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/console-browserify/-/console-browserify-1.2.0.tgz",
+      "integrity": "sha512-ZMkYO/LkF17QvCPqM0gxw8yUzigAOZOSWSHg91FH6orS7vcEj5dVZTidN2fQ14yBSdg97RqhSNwLUXInd52OTA==",
       "dev": true
     },
-    "evp_bytestokey": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz",
-      "integrity": "sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==",
-      "dev": true,
-      "requires": {
-        "md5.js": "^1.3.4",
-        "safe-buffer": "^5.1.1"
-      }
-    },
-    "exec-sh": {
-      "version": "0.3.2",
-      "resolved": "https://registry.npmjs.org/exec-sh/-/exec-sh-0.3.2.tgz",
-      "integrity": "sha512-9sLAvzhI5nc8TpuQUh4ahMdCrWT00wPWz7j47/emR5+2qEfoZP5zzUXvx+vdx+H6ohhnsYC31iX04QLYJK8zTg==",
+    "console-control-strings": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
+      "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=",
       "dev": true
     },
-    "execa": {
+    "constants-browserify": {
       "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz",
-      "integrity": "sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==",
+      "resolved": "https://registry.npmjs.org/constants-browserify/-/constants-browserify-1.0.0.tgz",
+      "integrity": "sha1-wguW2MYXdIqvHBYCF2DNJ/y4y3U=",
+      "dev": true
+    },
+    "conventional-changelog-angular": {
+      "version": "5.0.11",
+      "resolved": "https://registry.npmjs.org/conventional-changelog-angular/-/conventional-changelog-angular-5.0.11.tgz",
+      "integrity": "sha512-nSLypht/1yEflhuTogC03i7DX7sOrXGsRn14g131Potqi6cbGbGEE9PSDEHKldabB6N76HiSyw9Ph+kLmC04Qw==",
       "dev": true,
       "requires": {
-        "cross-spawn": "^6.0.0",
-        "get-stream": "^4.0.0",
-        "is-stream": "^1.1.0",
-        "npm-run-path": "^2.0.0",
-        "p-finally": "^1.0.0",
-        "signal-exit": "^3.0.0",
-        "strip-eof": "^1.0.0"
+        "compare-func": "^2.0.0",
+        "q": "^1.5.1"
+      }
+    },
+    "conventional-changelog-core": {
+      "version": "3.2.3",
+      "resolved": "https://registry.npmjs.org/conventional-changelog-core/-/conventional-changelog-core-3.2.3.tgz",
+      "integrity": "sha512-LMMX1JlxPIq/Ez5aYAYS5CpuwbOk6QFp8O4HLAcZxe3vxoCtABkhfjetk8IYdRB9CDQGwJFLR3Dr55Za6XKgUQ==",
+      "dev": true,
+      "requires": {
+        "conventional-changelog-writer": "^4.0.6",
+        "conventional-commits-parser": "^3.0.3",
+        "dateformat": "^3.0.0",
+        "get-pkg-repo": "^1.0.0",
+        "git-raw-commits": "2.0.0",
+        "git-remote-origin-url": "^2.0.0",
+        "git-semver-tags": "^2.0.3",
+        "lodash": "^4.2.1",
+        "normalize-package-data": "^2.3.5",
+        "q": "^1.5.1",
+        "read-pkg": "^3.0.0",
+        "read-pkg-up": "^3.0.0",
+        "through2": "^3.0.0"
       },
       "dependencies": {
-        "get-stream": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz",
-          "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==",
+        "find-up": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz",
+          "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=",
           "dev": true,
           "requires": {
-            "pump": "^3.0.0"
+            "locate-path": "^2.0.0"
           }
         },
-        "pump": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-          "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
+        "load-json-file": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz",
+          "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=",
           "dev": true,
           "requires": {
-            "end-of-stream": "^1.1.0",
-            "once": "^1.3.1"
+            "graceful-fs": "^4.1.2",
+            "parse-json": "^4.0.0",
+            "pify": "^3.0.0",
+            "strip-bom": "^3.0.0"
           }
-        }
-      }
-    },
-    "exit": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz",
-      "integrity": "sha1-BjJjj42HfMghB9MKD/8aF8uhzQw=",
-      "dev": true
-    },
-    "expand-brackets": {
-      "version": "2.1.4",
-      "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz",
-      "integrity": "sha1-t3c14xXOMPa27/D4OwQVGiJEliI=",
-      "dev": true,
-      "requires": {
-        "debug": "^2.3.3",
-        "define-property": "^0.2.5",
-        "extend-shallow": "^2.0.1",
-        "posix-character-classes": "^0.1.0",
-        "regex-not": "^1.0.0",
-        "snapdragon": "^0.8.1",
-        "to-regex": "^3.0.1"
-      },
-      "dependencies": {
-        "define-property": {
-          "version": "0.2.5",
-          "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
-          "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
+        },
+        "locate-path": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz",
+          "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=",
           "dev": true,
           "requires": {
-            "is-descriptor": "^0.1.0"
+            "p-locate": "^2.0.0",
+            "path-exists": "^3.0.0"
           }
         },
-        "extend-shallow": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+        "p-limit": {
+          "version": "1.3.0",
+          "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz",
+          "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==",
           "dev": true,
           "requires": {
-            "is-extendable": "^0.1.0"
+            "p-try": "^1.0.0"
           }
-        }
-      }
-    },
-    "expand-tilde": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/expand-tilde/-/expand-tilde-2.0.2.tgz",
-      "integrity": "sha1-l+gBqgUt8CRU3kawK/YhZCzchQI=",
-      "dev": true,
-      "requires": {
-        "homedir-polyfill": "^1.0.1"
-      }
-    },
-    "expect": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/expect/-/expect-24.8.0.tgz",
-      "integrity": "sha512-/zYvP8iMDrzaaxHVa724eJBCKqSHmO0FA7EDkBiRHxg6OipmMn1fN+C8T9L9K8yr7UONkOifu6+LLH+z76CnaA==",
-      "dev": true,
-      "requires": {
-        "@jest/types": "^24.8.0",
-        "ansi-styles": "^3.2.0",
-        "jest-get-type": "^24.8.0",
-        "jest-matcher-utils": "^24.8.0",
-        "jest-message-util": "^24.8.0",
-        "jest-regex-util": "^24.3.0"
-      },
-      "dependencies": {
-        "jest-message-util": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-24.8.0.tgz",
-          "integrity": "sha512-p2k71rf/b6ns8btdB0uVdljWo9h0ovpnEe05ZKWceQGfXYr4KkzgKo3PBi8wdnd9OtNh46VpNIJynUn/3MKm1g==",
+        },
+        "p-locate": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz",
+          "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=",
           "dev": true,
           "requires": {
-            "@babel/code-frame": "^7.0.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
-            "@types/stack-utils": "^1.0.1",
-            "chalk": "^2.0.1",
-            "micromatch": "^3.1.10",
-            "slash": "^2.0.0",
-            "stack-utils": "^1.0.1"
+            "p-limit": "^1.1.0"
           }
         },
-        "slash": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
-          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
+        "p-try": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz",
+          "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=",
+          "dev": true
+        },
+        "parse-json": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
+          "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=",
+          "dev": true,
+          "requires": {
+            "error-ex": "^1.3.1",
+            "json-parse-better-errors": "^1.0.1"
+          }
+        },
+        "path-exists": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
+          "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=",
+          "dev": true
+        },
+        "path-type": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz",
+          "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==",
+          "dev": true,
+          "requires": {
+            "pify": "^3.0.0"
+          }
+        },
+        "read-pkg": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz",
+          "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=",
+          "dev": true,
+          "requires": {
+            "load-json-file": "^4.0.0",
+            "normalize-package-data": "^2.3.2",
+            "path-type": "^3.0.0"
+          }
+        },
+        "read-pkg-up": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-3.0.0.tgz",
+          "integrity": "sha1-PtSWaF26D4/hGNBpHcUfSh/5bwc=",
+          "dev": true,
+          "requires": {
+            "find-up": "^2.0.0",
+            "read-pkg": "^3.0.0"
+          }
+        },
+        "strip-bom": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
+          "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=",
           "dev": true
+        },
+        "through2": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.2.tgz",
+          "integrity": "sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==",
+          "dev": true,
+          "requires": {
+            "inherits": "^2.0.4",
+            "readable-stream": "2 || 3"
+          }
         }
       }
     },
-    "extend": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
-      "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==",
+    "conventional-changelog-preset-loader": {
+      "version": "2.3.4",
+      "resolved": "https://registry.npmjs.org/conventional-changelog-preset-loader/-/conventional-changelog-preset-loader-2.3.4.tgz",
+      "integrity": "sha512-GEKRWkrSAZeTq5+YjUZOYxdHq+ci4dNwHvpaBC3+ENalzFWuCWa9EZXSuZBpkr72sMdKB+1fyDV4takK1Lf58g==",
       "dev": true
     },
-    "extend-shallow": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz",
-      "integrity": "sha1-Jqcarwc7OfshJxcnRhMcJwQCjbg=",
+    "conventional-changelog-writer": {
+      "version": "4.0.17",
+      "resolved": "https://registry.npmjs.org/conventional-changelog-writer/-/conventional-changelog-writer-4.0.17.tgz",
+      "integrity": "sha512-IKQuK3bib/n032KWaSb8YlBFds+aLmzENtnKtxJy3+HqDq5kohu3g/UdNbIHeJWygfnEbZjnCKFxAW0y7ArZAw==",
       "dev": true,
       "requires": {
-        "assign-symbols": "^1.0.0",
-        "is-extendable": "^1.0.1"
+        "compare-func": "^2.0.0",
+        "conventional-commits-filter": "^2.0.6",
+        "dateformat": "^3.0.0",
+        "handlebars": "^4.7.6",
+        "json-stringify-safe": "^5.0.1",
+        "lodash": "^4.17.15",
+        "meow": "^7.0.0",
+        "semver": "^6.0.0",
+        "split": "^1.0.0",
+        "through2": "^3.0.0"
       },
       "dependencies": {
-        "is-extendable": {
-          "version": "1.0.1",
-          "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz",
-          "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==",
+        "semver": {
+          "version": "6.3.0",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
+          "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
+          "dev": true
+        },
+        "through2": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.2.tgz",
+          "integrity": "sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==",
           "dev": true,
           "requires": {
-            "is-plain-object": "^2.0.4"
+            "inherits": "^2.0.4",
+            "readable-stream": "2 || 3"
           }
         }
       }
     },
-    "external-editor": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.0.3.tgz",
-      "integrity": "sha512-bn71H9+qWoOQKyZDo25mOMVpSmXROAsTJVVVYzrrtol3d4y+AsKjf4Iwl2Q+IuT0kFSQ1qo166UuIwqYq7mGnA==",
+    "conventional-commits-filter": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/conventional-commits-filter/-/conventional-commits-filter-2.0.6.tgz",
+      "integrity": "sha512-4g+sw8+KA50/Qwzfr0hL5k5NWxqtrOVw4DDk3/h6L85a9Gz0/Eqp3oP+CWCNfesBvZZZEFHF7OTEbRe+yYSyKw==",
       "dev": true,
       "requires": {
-        "chardet": "^0.7.0",
-        "iconv-lite": "^0.4.24",
-        "tmp": "^0.0.33"
+        "lodash.ismatch": "^4.4.0",
+        "modify-values": "^1.0.0"
       }
     },
-    "extglob": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz",
-      "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==",
+    "conventional-commits-parser": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/conventional-commits-parser/-/conventional-commits-parser-3.1.0.tgz",
+      "integrity": "sha512-RSo5S0WIwXZiRxUGTPuYFbqvrR4vpJ1BDdTlthFgvHt5kEdnd1+pdvwWphWn57/oIl4V72NMmOocFqqJ8mFFhA==",
       "dev": true,
       "requires": {
-        "array-unique": "^0.3.2",
-        "define-property": "^1.0.0",
-        "expand-brackets": "^2.1.4",
-        "extend-shallow": "^2.0.1",
-        "fragment-cache": "^0.2.1",
-        "regex-not": "^1.0.0",
-        "snapdragon": "^0.8.1",
-        "to-regex": "^3.0.1"
+        "JSONStream": "^1.0.4",
+        "is-text-path": "^1.0.1",
+        "lodash": "^4.17.15",
+        "meow": "^7.0.0",
+        "split2": "^2.0.0",
+        "through2": "^3.0.0",
+        "trim-off-newlines": "^1.0.0"
       },
       "dependencies": {
-        "define-property": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
-          "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=",
+        "through2": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.2.tgz",
+          "integrity": "sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==",
           "dev": true,
           "requires": {
-            "is-descriptor": "^1.0.0"
+            "inherits": "^2.0.4",
+            "readable-stream": "2 || 3"
           }
-        },
-        "extend-shallow": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+        }
+      }
+    },
+    "conventional-recommended-bump": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/conventional-recommended-bump/-/conventional-recommended-bump-5.0.1.tgz",
+      "integrity": "sha512-RVdt0elRcCxL90IrNP0fYCpq1uGt2MALko0eyeQ+zQuDVWtMGAy9ng6yYn3kax42lCj9+XBxQ8ZN6S9bdKxDhQ==",
+      "dev": true,
+      "requires": {
+        "concat-stream": "^2.0.0",
+        "conventional-changelog-preset-loader": "^2.1.1",
+        "conventional-commits-filter": "^2.0.2",
+        "conventional-commits-parser": "^3.0.3",
+        "git-raw-commits": "2.0.0",
+        "git-semver-tags": "^2.0.3",
+        "meow": "^4.0.0",
+        "q": "^1.5.1"
+      },
+      "dependencies": {
+        "camelcase": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz",
+          "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=",
+          "dev": true
+        },
+        "camelcase-keys": {
+          "version": "4.2.0",
+          "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-4.2.0.tgz",
+          "integrity": "sha1-oqpfsa9oh1glnDLBQUJteJI7m3c=",
           "dev": true,
           "requires": {
-            "is-extendable": "^0.1.0"
+            "camelcase": "^4.1.0",
+            "map-obj": "^2.0.0",
+            "quick-lru": "^1.0.0"
           }
         },
-        "is-accessor-descriptor": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
-          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
+        "concat-stream": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz",
+          "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==",
           "dev": true,
           "requires": {
-            "kind-of": "^6.0.0"
+            "buffer-from": "^1.0.0",
+            "inherits": "^2.0.3",
+            "readable-stream": "^3.0.2",
+            "typedarray": "^0.0.6"
           }
         },
-        "is-data-descriptor": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
-          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
+        "find-up": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz",
+          "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=",
           "dev": true,
           "requires": {
-            "kind-of": "^6.0.0"
+            "locate-path": "^2.0.0"
           }
         },
-        "is-descriptor": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
-          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+        "indent-string": {
+          "version": "3.2.0",
+          "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz",
+          "integrity": "sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok=",
+          "dev": true
+        },
+        "load-json-file": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz",
+          "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=",
           "dev": true,
           "requires": {
-            "is-accessor-descriptor": "^1.0.0",
-            "is-data-descriptor": "^1.0.0",
-            "kind-of": "^6.0.2"
+            "graceful-fs": "^4.1.2",
+            "parse-json": "^4.0.0",
+            "pify": "^3.0.0",
+            "strip-bom": "^3.0.0"
           }
-        }
-      }
-    },
-    "extsprintf": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
-      "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=",
-      "dev": true
-    },
-    "fancy-log": {
-      "version": "1.3.3",
-      "resolved": "https://registry.npmjs.org/fancy-log/-/fancy-log-1.3.3.tgz",
-      "integrity": "sha512-k9oEhlyc0FrVh25qYuSELjr8oxsCoc4/LEZfg2iJJrfEk/tZL9bCoJE47gqAvI2m/AUjluCS4+3I0eTx8n3AEw==",
-      "dev": true,
-      "requires": {
-        "ansi-gray": "^0.1.1",
-        "color-support": "^1.1.3",
-        "parse-node-version": "^1.0.0",
-        "time-stamp": "^1.0.0"
-      }
-    },
-    "fast-deep-equal": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz",
-      "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=",
-      "dev": true
-    },
-    "fast-extend": {
-      "version": "0.0.2",
-      "resolved": "https://registry.npmjs.org/fast-extend/-/fast-extend-0.0.2.tgz",
-      "integrity": "sha1-9exCz0C5Rg9SGmOH37Ut7u1nHb0=",
-      "dev": true
-    },
-    "fast-glob": {
-      "version": "2.2.6",
-      "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-2.2.6.tgz",
-      "integrity": "sha512-0BvMaZc1k9F+MeWWMe8pL6YltFzZYcJsYU7D4JyDA6PAczaXvxqQQ/z+mDF7/4Mw01DeUc+i3CTKajnkANkV4w==",
-      "dev": true,
-      "requires": {
-        "@mrmlnc/readdir-enhanced": "^2.2.1",
-        "@nodelib/fs.stat": "^1.1.2",
-        "glob-parent": "^3.1.0",
-        "is-glob": "^4.0.0",
-        "merge2": "^1.2.3",
-        "micromatch": "^3.1.10"
-      }
-    },
-    "fast-json-stable-stringify": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz",
-      "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=",
-      "dev": true
-    },
-    "fast-levenshtein": {
-      "version": "2.0.6",
-      "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
-      "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=",
-      "dev": true
-    },
-    "fb-watchman": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.0.tgz",
-      "integrity": "sha1-VOmr99+i8mzZsWNsWIwa/AXeXVg=",
-      "dev": true,
-      "requires": {
-        "bser": "^2.0.0"
-      }
-    },
-    "figgy-pudding": {
-      "version": "3.5.1",
-      "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.1.tgz",
-      "integrity": "sha512-vNKxJHTEKNThjfrdJwHc7brvM6eVevuO5nTj6ez8ZQ1qbXTvGthucRF7S4vf2cr71QVnT70V34v0S1DyQsti0w==",
-      "dev": true
-    },
-    "figures": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz",
-      "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=",
-      "dev": true,
-      "requires": {
-        "escape-string-regexp": "^1.0.5"
-      }
-    },
-    "fill-range": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz",
-      "integrity": "sha1-1USBHUKPmOsGpj3EAtJAPDKMOPc=",
-      "dev": true,
-      "requires": {
-        "extend-shallow": "^2.0.1",
-        "is-number": "^3.0.0",
-        "repeat-string": "^1.6.1",
-        "to-regex-range": "^2.1.0"
-      },
-      "dependencies": {
-        "extend-shallow": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+        },
+        "locate-path": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz",
+          "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=",
           "dev": true,
           "requires": {
-            "is-extendable": "^0.1.0"
+            "p-locate": "^2.0.0",
+            "path-exists": "^3.0.0"
           }
-        }
-      }
-    },
-    "find-cache-dir": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-2.0.0.tgz",
-      "integrity": "sha512-LDUY6V1Xs5eFskUVYtIwatojt6+9xC9Chnlk/jYOOvn3FAFfSaWddxahDGyNHh0b2dMXa6YW2m0tk8TdVaXHlA==",
-      "dev": true,
-      "requires": {
-        "commondir": "^1.0.1",
-        "make-dir": "^1.0.0",
-        "pkg-dir": "^3.0.0"
-      },
-      "dependencies": {
-        "find-up": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz",
-          "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==",
+        },
+        "map-obj": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-2.0.0.tgz",
+          "integrity": "sha1-plzSkIepJZi4eRJXpSPgISIqwfk=",
+          "dev": true
+        },
+        "meow": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/meow/-/meow-4.0.1.tgz",
+          "integrity": "sha512-xcSBHD5Z86zaOc+781KrupuHAzeGXSLtiAOmBsiLDiPSaYSB6hdew2ng9EBAnZ62jagG9MHAOdxpDi/lWBFJ/A==",
           "dev": true,
           "requires": {
-            "locate-path": "^3.0.0"
+            "camelcase-keys": "^4.0.0",
+            "decamelize-keys": "^1.0.0",
+            "loud-rejection": "^1.0.0",
+            "minimist": "^1.1.3",
+            "minimist-options": "^3.0.1",
+            "normalize-package-data": "^2.3.4",
+            "read-pkg-up": "^3.0.0",
+            "redent": "^2.0.0",
+            "trim-newlines": "^2.0.0"
           }
         },
-        "locate-path": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz",
-          "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==",
+        "minimist-options": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/minimist-options/-/minimist-options-3.0.2.tgz",
+          "integrity": "sha512-FyBrT/d0d4+uiZRbqznPXqw3IpZZG3gl3wKWiX784FycUKVwBt0uLBFkQrtE4tZOrgo78nZp2jnKz3L65T5LdQ==",
           "dev": true,
           "requires": {
-            "p-locate": "^3.0.0",
-            "path-exists": "^3.0.0"
+            "arrify": "^1.0.1",
+            "is-plain-obj": "^1.1.0"
           }
         },
         "p-limit": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.1.0.tgz",
-          "integrity": "sha512-NhURkNcrVB+8hNfLuysU8enY5xn2KXphsHBaC2YmRNTZRc7RWusw6apSpdEj3jo4CMb6W9nrF6tTnsJsJeyu6g==",
+          "version": "1.3.0",
+          "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz",
+          "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==",
           "dev": true,
           "requires": {
-            "p-try": "^2.0.0"
+            "p-try": "^1.0.0"
           }
         },
         "p-locate": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz",
-          "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==",
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz",
+          "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=",
           "dev": true,
           "requires": {
-            "p-limit": "^2.0.0"
+            "p-limit": "^1.1.0"
           }
         },
         "p-try": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.0.0.tgz",
-          "integrity": "sha512-hMp0onDKIajHfIkdRk3P4CdCmErkYAxxDtP3Wx/4nZ3aGlau2VKh3mZpcuFkH27WQkL/3WBCPOktzA9ZOAnMQQ==",
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz",
+          "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=",
           "dev": true
         },
+        "parse-json": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
+          "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=",
+          "dev": true,
+          "requires": {
+            "error-ex": "^1.3.1",
+            "json-parse-better-errors": "^1.0.1"
+          }
+        },
         "path-exists": {
           "version": "3.0.0",
           "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
           "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=",
           "dev": true
         },
-        "pkg-dir": {
+        "path-type": {
           "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz",
-          "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==",
+          "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz",
+          "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==",
           "dev": true,
           "requires": {
-            "find-up": "^3.0.0"
+            "pify": "^3.0.0"
+          }
+        },
+        "quick-lru": {
+          "version": "1.1.0",
+          "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-1.1.0.tgz",
+          "integrity": "sha1-Q2CxfGETatOAeDl/8RQW4Ybc+7g=",
+          "dev": true
+        },
+        "read-pkg": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz",
+          "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=",
+          "dev": true,
+          "requires": {
+            "load-json-file": "^4.0.0",
+            "normalize-package-data": "^2.3.2",
+            "path-type": "^3.0.0"
+          }
+        },
+        "read-pkg-up": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-3.0.0.tgz",
+          "integrity": "sha1-PtSWaF26D4/hGNBpHcUfSh/5bwc=",
+          "dev": true,
+          "requires": {
+            "find-up": "^2.0.0",
+            "read-pkg": "^3.0.0"
+          }
+        },
+        "readable-stream": {
+          "version": "3.6.0",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
+          "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
+          "dev": true,
+          "requires": {
+            "inherits": "^2.0.3",
+            "string_decoder": "^1.1.1",
+            "util-deprecate": "^1.0.1"
+          }
+        },
+        "redent": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/redent/-/redent-2.0.0.tgz",
+          "integrity": "sha1-wbIAe0LVfrE4kHmzyDM2OdXhzKo=",
+          "dev": true,
+          "requires": {
+            "indent-string": "^3.0.0",
+            "strip-indent": "^2.0.0"
           }
+        },
+        "strip-bom": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
+          "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=",
+          "dev": true
+        },
+        "strip-indent": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-2.0.0.tgz",
+          "integrity": "sha1-XvjbKV0B5u1sv3qrlpmNeCJSe2g=",
+          "dev": true
+        },
+        "trim-newlines": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-2.0.0.tgz",
+          "integrity": "sha1-tAPQuRvlDDMd/EuC7s6yLD3hbSA=",
+          "dev": true
         }
       }
     },
-    "find-npm-prefix": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/find-npm-prefix/-/find-npm-prefix-1.0.2.tgz",
-      "integrity": "sha512-KEftzJ+H90x6pcKtdXZEPsQse8/y/UnvzRKrOSQFprnrGaFuJ62fVkP34Iu2IYuMvyauCyoLTNkJZgrrGA2wkA==",
-      "dev": true
-    },
-    "find-replace": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/find-replace/-/find-replace-2.0.1.tgz",
-      "integrity": "sha512-LzDo3Fpa30FLIBsh6DCDnMN1KW2g4QKkqKmejlImgWY67dDFPX/x9Kh/op/GK522DchQXEvDi/wD48HKW49XOQ==",
-      "requires": {
-        "array-back": "^2.0.0",
-        "test-value": "^3.0.0"
-      }
-    },
-    "find-up": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz",
-      "integrity": "sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8=",
-      "dev": true,
-      "requires": {
-        "path-exists": "^2.0.0",
-        "pinkie-promise": "^2.0.0"
-      }
-    },
-    "findup-sync": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/findup-sync/-/findup-sync-3.0.0.tgz",
-      "integrity": "sha512-YbffarhcicEhOrm4CtrwdKBdCuz576RLdhJDsIfvNtxUuhdRet1qZcsMjqbePtAseKdAnDyM/IyXbu7PRPRLYg==",
+    "convert-source-map": {
+      "version": "1.7.0",
+      "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.7.0.tgz",
+      "integrity": "sha512-4FJkXzKXEDB1snCFZlLP4gpC3JILicCpGbzG9f9G7tGqGCzETQ2hWPrcinA9oU4wtf2biUaEH5065UnMeR33oA==",
       "dev": true,
       "requires": {
-        "detect-file": "^1.0.0",
-        "is-glob": "^4.0.0",
-        "micromatch": "^3.0.4",
-        "resolve-dir": "^1.0.1"
+        "safe-buffer": "~5.1.1"
+      },
+      "dependencies": {
+        "safe-buffer": {
+          "version": "5.1.2",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+          "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+          "dev": true
+        }
       }
     },
-    "fined": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/fined/-/fined-1.2.0.tgz",
-      "integrity": "sha512-ZYDqPLGxDkDhDZBjZBb+oD1+j0rA4E0pXY50eplAAOPg2N/gUBSSk5IM1/QhPfyVo19lJ+CvXpqfvk+b2p/8Ng==",
+    "copy-concurrently": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/copy-concurrently/-/copy-concurrently-1.0.5.tgz",
+      "integrity": "sha512-f2domd9fsVDFtaFcbaRZuYXwtdmnzqbADSwhSWYxYB/Q8zsdUUFMXVRwXGDMWmbEzAn1kdRrtI1T/KTFOL4X2A==",
       "dev": true,
       "requires": {
-        "expand-tilde": "^2.0.2",
-        "is-plain-object": "^2.0.3",
-        "object.defaults": "^1.1.0",
-        "object.pick": "^1.2.0",
-        "parse-filepath": "^1.0.1"
+        "aproba": "^1.1.1",
+        "fs-write-stream-atomic": "^1.0.8",
+        "iferr": "^0.1.5",
+        "mkdirp": "^0.5.1",
+        "rimraf": "^2.5.4",
+        "run-queue": "^1.0.0"
+      },
+      "dependencies": {
+        "mkdirp": {
+          "version": "0.5.5",
+          "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
+          "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
+          "dev": true,
+          "requires": {
+            "minimist": "^1.2.5"
+          }
+        }
       }
     },
-    "flagged-respawn": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/flagged-respawn/-/flagged-respawn-1.0.1.tgz",
-      "integrity": "sha512-lNaHNVymajmk0OJMBn8fVUAU1BtDeKIqKoVhk4xAALB57aALg6b4W0MfJ/cUE0g9YBXy5XhSlPIpYIJ7HaY/3Q==",
+    "copy-descriptor": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz",
+      "integrity": "sha1-Z29us8OZl8LuGsOpJP1hJHSPV40=",
       "dev": true
     },
-    "flatbuffers": {
-      "version": "1.11.0",
-      "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-1.11.0.tgz",
-      "integrity": "sha512-0PqFKtXI4MjxomI7jO4g5XfLPm/15g2R+5WGCHBGYGh0ihQiypnHlJ6bMmkkrAe0GzZ4d7PDAfCONKIPUxNF+A=="
-    },
-    "flush-write-stream": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-1.0.3.tgz",
-      "integrity": "sha512-calZMC10u0FMUqoiunI2AiGIIUtUIvifNwkHhNupZH4cbNnW1Itkoh/Nf5HFYmDrwWPjrUxpkZT0KhuCq0jmGw==",
+    "copy-props": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/copy-props/-/copy-props-2.0.4.tgz",
+      "integrity": "sha512-7cjuUME+p+S3HZlbllgsn2CDwS+5eCCX16qBgNC4jgSTf49qR1VKy/Zhl400m0IQXl/bPGEVqncgUUMjrr4s8A==",
       "dev": true,
       "requires": {
-        "inherits": "^2.0.1",
-        "readable-stream": "^2.0.4"
+        "each-props": "^1.3.0",
+        "is-plain-object": "^2.0.1"
       }
     },
-    "for-in": {
+    "core-util-is": {
       "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz",
-      "integrity": "sha1-gQaNKVqBQuwKxybG4iAMMPttXoA=",
+      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
+      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
       "dev": true
     },
-    "for-own": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/for-own/-/for-own-1.0.0.tgz",
-      "integrity": "sha1-xjMy9BXO3EsE2/5wz4NklMU8tEs=",
+    "cosmiconfig": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-5.2.1.tgz",
+      "integrity": "sha512-H65gsXo1SKjf8zmrJ67eJk8aIRKV5ff2D4uKZIBZShbhGSpEmsQOPW/SKMKYhSTrqR7ufy6RP69rPogdaPh/kA==",
       "dev": true,
       "requires": {
-        "for-in": "^1.0.1"
+        "import-fresh": "^2.0.0",
+        "is-directory": "^0.3.1",
+        "js-yaml": "^3.13.1",
+        "parse-json": "^4.0.0"
+      },
+      "dependencies": {
+        "parse-json": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
+          "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=",
+          "dev": true,
+          "requires": {
+            "error-ex": "^1.3.1",
+            "json-parse-better-errors": "^1.0.1"
+          }
+        }
       }
     },
-    "forever-agent": {
-      "version": "0.6.1",
-      "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
-      "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=",
-      "dev": true
-    },
-    "form-data": {
-      "version": "2.3.3",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
-      "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
+    "coveralls": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/coveralls/-/coveralls-3.0.3.tgz",
+      "integrity": "sha512-viNfeGlda2zJr8Gj1zqXpDMRjw9uM54p7wzZdvLRyOgnAfCe974Dq4veZkjJdxQXbmdppu6flEajFYseHYaUhg==",
       "dev": true,
       "requires": {
-        "asynckit": "^0.4.0",
-        "combined-stream": "^1.0.6",
-        "mime-types": "^2.1.12"
+        "growl": "~> 1.10.0",
+        "js-yaml": "^3.11.0",
+        "lcov-parse": "^0.0.10",
+        "log-driver": "^1.2.7",
+        "minimist": "^1.2.0",
+        "request": "^2.86.0"
       }
     },
-    "fragment-cache": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz",
-      "integrity": "sha1-QpD60n8T6Jvn8zeZxrxaCr//DRk=",
+    "create-ecdh": {
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz",
+      "integrity": "sha512-mf+TCx8wWc9VpuxfP2ht0iSISLZnt0JgWlrOKZiNqyUZWnjIaCIVNQArMHnCZKfEYRg6IM7A+NeJoN8gf/Ws0A==",
       "dev": true,
       "requires": {
-        "map-cache": "^0.2.2"
+        "bn.js": "^4.1.0",
+        "elliptic": "^6.5.3"
+      },
+      "dependencies": {
+        "bn.js": {
+          "version": "4.11.9",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==",
+          "dev": true
+        }
       }
     },
-    "from2": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz",
-      "integrity": "sha1-i/tVAr3kpNNs/e6gB/zKIdfjgq8=",
+    "create-hash": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz",
+      "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==",
       "dev": true,
       "requires": {
+        "cipher-base": "^1.0.1",
         "inherits": "^2.0.1",
-        "readable-stream": "^2.0.0"
+        "md5.js": "^1.3.4",
+        "ripemd160": "^2.0.1",
+        "sha.js": "^2.4.0"
       }
     },
-    "fs-extra": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz",
-      "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==",
+    "create-hmac": {
+      "version": "1.1.7",
+      "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz",
+      "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==",
       "dev": true,
       "requires": {
-        "graceful-fs": "^4.1.2",
-        "jsonfile": "^4.0.0",
-        "universalify": "^0.1.0"
+        "cipher-base": "^1.0.3",
+        "create-hash": "^1.1.0",
+        "inherits": "^2.0.1",
+        "ripemd160": "^2.0.0",
+        "safe-buffer": "^5.0.1",
+        "sha.js": "^2.4.8"
       }
     },
-    "fs-minipass": {
-      "version": "1.2.5",
-      "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.5.tgz",
-      "integrity": "sha512-JhBl0skXjUPCFH7x6x61gQxrKyXsxB5gcgePLZCwfyCGGsTISMoIeObbrvVeP6Xmyaudw4TT43qV2Gz+iyd2oQ==",
+    "cross-spawn": {
+      "version": "6.0.5",
+      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz",
+      "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==",
       "dev": true,
       "requires": {
-        "minipass": "^2.2.1"
+        "nice-try": "^1.0.4",
+        "path-key": "^2.0.1",
+        "semver": "^5.5.0",
+        "shebang-command": "^1.2.0",
+        "which": "^1.2.9"
       }
     },
-    "fs-mkdirp-stream": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/fs-mkdirp-stream/-/fs-mkdirp-stream-1.0.0.tgz",
-      "integrity": "sha1-C3gV/DIBxqaeFNuYzgmMFpNSWes=",
+    "cross-spawn-async": {
+      "version": "2.2.5",
+      "resolved": "https://registry.npmjs.org/cross-spawn-async/-/cross-spawn-async-2.2.5.tgz",
+      "integrity": "sha1-hF/wwINKPe2dFg2sptOQkGuyiMw=",
       "dev": true,
       "requires": {
-        "graceful-fs": "^4.1.11",
-        "through2": "^2.0.3"
+        "lru-cache": "^4.0.0",
+        "which": "^1.2.8"
+      },
+      "dependencies": {
+        "lru-cache": {
+          "version": "4.1.5",
+          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz",
+          "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==",
+          "dev": true,
+          "requires": {
+            "pseudomap": "^1.0.2",
+            "yallist": "^2.1.2"
+          }
+        },
+        "yallist": {
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
+          "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=",
+          "dev": true
+        }
       }
     },
-    "fs-monkey": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/fs-monkey/-/fs-monkey-0.3.3.tgz",
-      "integrity": "sha512-FNUvuTAJ3CqCQb5ELn+qCbGR/Zllhf2HtwsdAtBi59s1WeCjKMT81fHcSu7dwIskqGVK+MmOrb7VOBlq3/SItw==",
-      "dev": true
-    },
-    "fs-vacuum": {
-      "version": "1.2.10",
-      "resolved": "https://registry.npmjs.org/fs-vacuum/-/fs-vacuum-1.2.10.tgz",
-      "integrity": "sha1-t2Kb7AekAxolSP35n17PHMizHjY=",
+    "crypto-browserify": {
+      "version": "3.12.0",
+      "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz",
+      "integrity": "sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==",
       "dev": true,
       "requires": {
-        "graceful-fs": "^4.1.2",
-        "path-is-inside": "^1.0.1",
-        "rimraf": "^2.5.2"
+        "browserify-cipher": "^1.0.0",
+        "browserify-sign": "^4.0.0",
+        "create-ecdh": "^4.0.0",
+        "create-hash": "^1.1.0",
+        "create-hmac": "^1.1.0",
+        "diffie-hellman": "^5.0.0",
+        "inherits": "^2.0.1",
+        "pbkdf2": "^3.0.3",
+        "public-encrypt": "^4.0.0",
+        "randombytes": "^2.0.0",
+        "randomfill": "^1.0.3"
       }
     },
-    "fs-write-stream-atomic": {
-      "version": "1.0.10",
-      "resolved": "https://registry.npmjs.org/fs-write-stream-atomic/-/fs-write-stream-atomic-1.0.10.tgz",
-      "integrity": "sha1-tH31NJPvkR33VzHnCp3tAYnbQMk=",
+    "css": {
+      "version": "2.2.4",
+      "resolved": "https://registry.npmjs.org/css/-/css-2.2.4.tgz",
+      "integrity": "sha512-oUnjmWpy0niI3x/mPL8dVEI1l7MnG3+HHyRPHf+YFSbK+svOhXpmSOcDURUh2aOCgl2grzrOPt1nHLuCVFULLw==",
       "dev": true,
       "requires": {
-        "graceful-fs": "^4.1.2",
-        "iferr": "^0.1.5",
-        "imurmurhash": "^0.1.4",
-        "readable-stream": "1 || 2"
+        "inherits": "^2.0.3",
+        "source-map": "^0.6.1",
+        "source-map-resolve": "^0.5.2",
+        "urix": "^0.1.0"
+      },
+      "dependencies": {
+        "source-map": {
+          "version": "0.6.1",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+          "dev": true
+        }
       }
     },
-    "fs.realpath": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
-      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
+    "cssom": {
+      "version": "0.4.4",
+      "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz",
+      "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw==",
       "dev": true
     },
-    "fsevents": {
-      "version": "1.2.7",
-      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.7.tgz",
-      "integrity": "sha512-Pxm6sI2MeBD7RdD12RYsqaP0nMiwx8eZBXCa6z2L+mRHm2DYrOYwihmhjpkdjUHwQhslWQjRpEgNq4XvBmaAuw==",
+    "cssstyle": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz",
+      "integrity": "sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==",
       "dev": true,
-      "optional": true,
       "requires": {
-        "nan": "^2.9.2",
-        "node-pre-gyp": "^0.10.0"
+        "cssom": "~0.3.6"
       },
       "dependencies": {
-        "abbrev": {
-          "version": "1.1.1",
-          "bundled": true,
-          "dev": true,
-          "optional": true
-        },
-        "ansi-regex": {
-          "version": "2.1.1",
-          "bundled": true,
-          "dev": true,
-          "optional": true
-        },
-        "aproba": {
-          "version": "1.2.0",
-          "bundled": true,
-          "dev": true,
-          "optional": true
-        },
-        "are-we-there-yet": {
-          "version": "1.1.5",
-          "bundled": true,
+        "cssom": {
+          "version": "0.3.8",
+          "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz",
+          "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==",
+          "dev": true
+        }
+      }
+    },
+    "currently-unhandled": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/currently-unhandled/-/currently-unhandled-0.4.1.tgz",
+      "integrity": "sha1-mI3zP+qxke95mmE2nddsF635V+o=",
+      "dev": true,
+      "requires": {
+        "array-find-index": "^1.0.1"
+      }
+    },
+    "cyclist": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz",
+      "integrity": "sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk=",
+      "dev": true
+    },
+    "d": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz",
+      "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==",
+      "dev": true,
+      "requires": {
+        "es5-ext": "^0.10.50",
+        "type": "^1.0.1"
+      }
+    },
+    "dargs": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/dargs/-/dargs-4.1.0.tgz",
+      "integrity": "sha1-A6nbtLXC8Tm/FK5T8LiipqhvThc=",
+      "dev": true,
+      "requires": {
+        "number-is-nan": "^1.0.0"
+      }
+    },
+    "dashdash": {
+      "version": "1.14.1",
+      "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
+      "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=",
+      "dev": true,
+      "requires": {
+        "assert-plus": "^1.0.0"
+      }
+    },
+    "data-urls": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz",
+      "integrity": "sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ==",
+      "dev": true,
+      "requires": {
+        "abab": "^2.0.3",
+        "whatwg-mimetype": "^2.3.0",
+        "whatwg-url": "^8.0.0"
+      }
+    },
+    "dateformat": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-3.0.3.tgz",
+      "integrity": "sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q==",
+      "dev": true
+    },
+    "debug": {
+      "version": "2.6.9",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+      "dev": true,
+      "requires": {
+        "ms": "2.0.0"
+      }
+    },
+    "debug-fabulous": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/debug-fabulous/-/debug-fabulous-1.1.0.tgz",
+      "integrity": "sha512-GZqvGIgKNlUnHUPQhepnUZFIMoi3dgZKQBzKDeL2g7oJF9SNAji/AAu36dusFUas0O+pae74lNeoIPHqXWDkLg==",
+      "dev": true,
+      "requires": {
+        "debug": "3.X",
+        "memoizee": "0.4.X",
+        "object-assign": "4.X"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "3.2.6",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
+          "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
           "dev": true,
-          "optional": true,
           "requires": {
-            "delegates": "^1.0.0",
-            "readable-stream": "^2.0.6"
+            "ms": "^2.1.1"
           }
         },
-        "balanced-match": {
+        "ms": {
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+          "dev": true
+        }
+      }
+    },
+    "debuglog": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz",
+      "integrity": "sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI=",
+      "dev": true
+    },
+    "decamelize": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
+      "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=",
+      "dev": true
+    },
+    "decamelize-keys": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/decamelize-keys/-/decamelize-keys-1.1.0.tgz",
+      "integrity": "sha1-0XGoeTMlKAfrPLYdwcFEXQeN8tk=",
+      "dev": true,
+      "requires": {
+        "decamelize": "^1.1.0",
+        "map-obj": "^1.0.0"
+      },
+      "dependencies": {
+        "map-obj": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz",
+          "integrity": "sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=",
+          "dev": true
+        }
+      }
+    },
+    "decimal.js": {
+      "version": "10.2.0",
+      "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.2.0.tgz",
+      "integrity": "sha512-vDPw+rDgn3bZe1+F/pyEwb1oMG2XTlRVgAa6B4KccTEpYgF8w6eQllVbQcfIJnZyvzFtFpxnpGtx8dd7DJp/Rw==",
+      "dev": true
+    },
+    "decode-uri-component": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz",
+      "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=",
+      "dev": true
+    },
+    "dedent": {
+      "version": "0.7.0",
+      "resolved": "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz",
+      "integrity": "sha1-JJXduvbrh0q7Dhvp3yLS5aVEMmw=",
+      "dev": true
+    },
+    "deep-extend": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
+      "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="
+    },
+    "deep-is": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz",
+      "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=",
+      "dev": true
+    },
+    "deepmerge": {
+      "version": "4.2.2",
+      "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.2.2.tgz",
+      "integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==",
+      "dev": true
+    },
+    "default-compare": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/default-compare/-/default-compare-1.0.0.tgz",
+      "integrity": "sha512-QWfXlM0EkAbqOCbD/6HjdwT19j7WCkMyiRhWilc4H9/5h/RzTF9gv5LYh1+CmDV5d1rki6KAWLtQale0xt20eQ==",
+      "dev": true,
+      "requires": {
+        "kind-of": "^5.0.2"
+      },
+      "dependencies": {
+        "kind-of": {
+          "version": "5.1.0",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
+          "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==",
+          "dev": true
+        }
+      }
+    },
+    "default-resolution": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/default-resolution/-/default-resolution-2.0.0.tgz",
+      "integrity": "sha1-vLgrqnKtebQmp2cy8aga1t8m1oQ=",
+      "dev": true
+    },
+    "defaults": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz",
+      "integrity": "sha1-xlYFHpgX2f8I7YgUd/P+QBnz730=",
+      "dev": true,
+      "requires": {
+        "clone": "^1.0.2"
+      },
+      "dependencies": {
+        "clone": {
+          "version": "1.0.4",
+          "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz",
+          "integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4=",
+          "dev": true
+        }
+      }
+    },
+    "define-properties": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz",
+      "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==",
+      "dev": true,
+      "requires": {
+        "object-keys": "^1.0.12"
+      }
+    },
+    "define-property": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz",
+      "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==",
+      "dev": true,
+      "requires": {
+        "is-descriptor": "^1.0.2",
+        "isobject": "^3.0.1"
+      },
+      "dependencies": {
+        "is-accessor-descriptor": {
           "version": "1.0.0",
-          "bundled": true,
-          "dev": true,
-          "optional": true
-        },
-        "brace-expansion": {
-          "version": "1.1.11",
-          "bundled": true,
+          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
+          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
           "dev": true,
-          "optional": true,
           "requires": {
-            "balanced-match": "^1.0.0",
-            "concat-map": "0.0.1"
+            "kind-of": "^6.0.0"
           }
         },
-        "chownr": {
-          "version": "1.1.1",
-          "bundled": true,
-          "dev": true,
-          "optional": true
-        },
-        "code-point-at": {
-          "version": "1.1.0",
-          "bundled": true,
-          "dev": true,
-          "optional": true
-        },
-        "concat-map": {
-          "version": "0.0.1",
-          "bundled": true,
-          "dev": true,
-          "optional": true
-        },
-        "console-control-strings": {
-          "version": "1.1.0",
-          "bundled": true,
+        "is-data-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
+          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
           "dev": true,
-          "optional": true
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
         },
-        "core-util-is": {
+        "is-descriptor": {
           "version": "1.0.2",
-          "bundled": true,
+          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
+          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+          "dev": true,
+          "requires": {
+            "is-accessor-descriptor": "^1.0.0",
+            "is-data-descriptor": "^1.0.0",
+            "kind-of": "^6.0.2"
+          }
+        }
+      }
+    },
+    "del": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/del/-/del-3.0.0.tgz",
+      "integrity": "sha1-U+z2mf/LyzljdpGrE7rxYIGXZuU=",
+      "dev": true,
+      "requires": {
+        "globby": "^6.1.0",
+        "is-path-cwd": "^1.0.0",
+        "is-path-in-cwd": "^1.0.0",
+        "p-map": "^1.1.1",
+        "pify": "^3.0.0",
+        "rimraf": "^2.2.8"
+      }
+    },
+    "delayed-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+      "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=",
+      "dev": true
+    },
+    "delegates": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
+      "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=",
+      "dev": true
+    },
+    "deprecation": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz",
+      "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==",
+      "dev": true
+    },
+    "des.js": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.0.1.tgz",
+      "integrity": "sha512-Q0I4pfFrv2VPd34/vfLrFOoRmlYj3OV50i7fskps1jZWK1kApMWWT9G6RRUeYedLcBDIhnSDaUvJMb3AhUlaEA==",
+      "dev": true,
+      "requires": {
+        "inherits": "^2.0.1",
+        "minimalistic-assert": "^1.0.0"
+      }
+    },
+    "detect-file": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/detect-file/-/detect-file-1.0.0.tgz",
+      "integrity": "sha1-8NZtA2cqglyxtzvbP+YjEMjlUrc=",
+      "dev": true
+    },
+    "detect-indent": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/detect-indent/-/detect-indent-5.0.0.tgz",
+      "integrity": "sha1-OHHMCmoALow+Wzz38zYmRnXwa50=",
+      "dev": true
+    },
+    "detect-newline": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-2.1.0.tgz",
+      "integrity": "sha1-9B8cEL5LAOh7XxPaaAdZ8sW/0+I=",
+      "dev": true
+    },
+    "dezalgo": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.3.tgz",
+      "integrity": "sha1-f3Qt4Gb8dIvI24IFad3c5Jvw1FY=",
+      "dev": true,
+      "requires": {
+        "asap": "^2.0.0",
+        "wrappy": "1"
+      }
+    },
+    "diff": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
+      "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
+      "dev": true
+    },
+    "diff-sequences": {
+      "version": "25.2.6",
+      "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-25.2.6.tgz",
+      "integrity": "sha512-Hq8o7+6GaZeoFjtpgvRBUknSXNeJiCx7V9Fr94ZMljNiCr9n9L8H8aJqgWOQiDDGdyn29fRNcDdRVJ5fdyihfg==",
+      "dev": true
+    },
+    "diffie-hellman": {
+      "version": "5.0.3",
+      "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz",
+      "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==",
+      "dev": true,
+      "requires": {
+        "bn.js": "^4.1.0",
+        "miller-rabin": "^4.0.0",
+        "randombytes": "^2.0.0"
+      },
+      "dependencies": {
+        "bn.js": {
+          "version": "4.11.9",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==",
+          "dev": true
+        }
+      }
+    },
+    "dir-glob": {
+      "version": "2.2.2",
+      "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-2.2.2.tgz",
+      "integrity": "sha512-f9LBi5QWzIW3I6e//uxZoLBlUt9kcp66qo0sSCxL6YZKc75R1c4MFCoe/LaZiBGmgujvQdxc5Bn3QhfyvK5Hsw==",
+      "dev": true,
+      "requires": {
+        "path-type": "^3.0.0"
+      },
+      "dependencies": {
+        "path-type": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz",
+          "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==",
+          "dev": true,
+          "requires": {
+            "pify": "^3.0.0"
+          }
+        }
+      }
+    },
+    "domain-browser": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz",
+      "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==",
+      "dev": true
+    },
+    "domexception": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz",
+      "integrity": "sha512-yxJ2mFy/sibVQlu5qHjOkf9J3K6zgmCxgJ94u2EdvDOV09H+32LtRswEcUsmUWN72pVLOEnTSRaIVVzVQgS0dg==",
+      "dev": true,
+      "requires": {
+        "webidl-conversions": "^5.0.0"
+      },
+      "dependencies": {
+        "webidl-conversions": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz",
+          "integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA==",
+          "dev": true
+        }
+      }
+    },
+    "dot-prop": {
+      "version": "4.2.1",
+      "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.1.tgz",
+      "integrity": "sha512-l0p4+mIuJIua0mhxGoh4a+iNL9bmeK5DvnSVQa6T0OhrVmaEa1XScX5Etc673FePCJOArq/4Pa2cLGODUWTPOQ==",
+      "dev": true,
+      "requires": {
+        "is-obj": "^1.0.0"
+      }
+    },
+    "duplexer": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz",
+      "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==",
+      "dev": true
+    },
+    "duplexify": {
+      "version": "3.7.1",
+      "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz",
+      "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==",
+      "dev": true,
+      "requires": {
+        "end-of-stream": "^1.0.0",
+        "inherits": "^2.0.1",
+        "readable-stream": "^2.0.0",
+        "stream-shift": "^1.0.0"
+      }
+    },
+    "each-props": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/each-props/-/each-props-1.3.2.tgz",
+      "integrity": "sha512-vV0Hem3zAGkJAyU7JSjixeU66rwdynTAa1vofCrSA5fEln+m67Az9CcnkVD776/fsN/UjIWmBDoNRS6t6G9RfA==",
+      "dev": true,
+      "requires": {
+        "is-plain-object": "^2.0.1",
+        "object.defaults": "^1.1.0"
+      }
+    },
+    "ecc-jsbn": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
+      "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=",
+      "dev": true,
+      "requires": {
+        "jsbn": "~0.1.0",
+        "safer-buffer": "^2.1.0"
+      }
+    },
+    "elliptic": {
+      "version": "6.5.3",
+      "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.3.tgz",
+      "integrity": "sha512-IMqzv5wNQf+E6aHeIqATs0tOLeOTwj1QKbRcS3jBbYkl5oLAserA8yJTT7/VyHUYG91PRmPyeQDObKLPpeS4dw==",
+      "dev": true,
+      "requires": {
+        "bn.js": "^4.4.0",
+        "brorand": "^1.0.1",
+        "hash.js": "^1.0.0",
+        "hmac-drbg": "^1.0.0",
+        "inherits": "^2.0.1",
+        "minimalistic-assert": "^1.0.0",
+        "minimalistic-crypto-utils": "^1.0.0"
+      },
+      "dependencies": {
+        "bn.js": {
+          "version": "4.11.9",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==",
+          "dev": true
+        }
+      }
+    },
+    "emittery": {
+      "version": "0.7.1",
+      "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.7.1.tgz",
+      "integrity": "sha512-d34LN4L6h18Bzz9xpoku2nPwKxCPlPMr3EEKTkoEBi+1/+b0lcRkRJ1UVyyZaKNeqGR3swcGl6s390DNO4YVgQ==",
+      "dev": true
+    },
+    "emoji-regex": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
+      "dev": true
+    },
+    "emojis-list": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-3.0.0.tgz",
+      "integrity": "sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==",
+      "dev": true
+    },
+    "encoding": {
+      "version": "0.1.13",
+      "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz",
+      "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==",
+      "dev": true,
+      "requires": {
+        "iconv-lite": "^0.6.2"
+      },
+      "dependencies": {
+        "iconv-lite": {
+          "version": "0.6.2",
+          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.2.tgz",
+          "integrity": "sha512-2y91h5OpQlolefMPmUlivelittSWy0rP+oYVpn6A7GwVHNE8AWzoYOBNmlwks3LobaJxgHCYZAnyNo2GgpNRNQ==",
+          "dev": true,
+          "requires": {
+            "safer-buffer": ">= 2.1.2 < 3.0.0"
+          }
+        }
+      }
+    },
+    "end-of-stream": {
+      "version": "1.4.4",
+      "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
+      "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
+      "dev": true,
+      "requires": {
+        "once": "^1.4.0"
+      }
+    },
+    "enhanced-resolve": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-4.3.0.tgz",
+      "integrity": "sha512-3e87LvavsdxyoCfGusJnrZ5G8SLPOFeHSNpZI/ATL9a5leXo2k0w6MKnbqhdBad9qTobSfB20Ld7UmgoNbAZkQ==",
+      "dev": true,
+      "requires": {
+        "graceful-fs": "^4.1.2",
+        "memory-fs": "^0.5.0",
+        "tapable": "^1.0.0"
+      },
+      "dependencies": {
+        "memory-fs": {
+          "version": "0.5.0",
+          "resolved": "https://registry.npmjs.org/memory-fs/-/memory-fs-0.5.0.tgz",
+          "integrity": "sha512-jA0rdU5KoQMC0e6ppoNRtpp6vjFq6+NY7r8hywnC7V+1Xj/MtHwGIbB1QaK/dunyjWteJzmkpd7ooeWg10T7GA==",
+          "dev": true,
+          "requires": {
+            "errno": "^0.1.3",
+            "readable-stream": "^2.0.1"
+          }
+        }
+      }
+    },
+    "env-paths": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.0.tgz",
+      "integrity": "sha512-6u0VYSCo/OW6IoD5WCLLy9JUGARbamfSavcNXry/eu8aHVFei6CD3Sw+VGX5alea1i9pgPHW0mbu6Xj0uBh7gA==",
+      "dev": true
+    },
+    "envinfo": {
+      "version": "7.7.3",
+      "resolved": "https://registry.npmjs.org/envinfo/-/envinfo-7.7.3.tgz",
+      "integrity": "sha512-46+j5QxbPWza0PB1i15nZx0xQ4I/EfQxg9J8Had3b408SV63nEtor2e+oiY63amTo9KTuh2a3XLObNwduxYwwA==",
+      "dev": true
+    },
+    "err-code": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/err-code/-/err-code-1.1.2.tgz",
+      "integrity": "sha1-BuARbTAo9q70gGhJ6w6mp0iuaWA=",
+      "dev": true
+    },
+    "errno": {
+      "version": "0.1.7",
+      "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz",
+      "integrity": "sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==",
+      "dev": true,
+      "requires": {
+        "prr": "~1.0.1"
+      }
+    },
+    "error-ex": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz",
+      "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==",
+      "dev": true,
+      "requires": {
+        "is-arrayish": "^0.2.1"
+      }
+    },
+    "es-abstract": {
+      "version": "1.18.0-next.0",
+      "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.0-next.0.tgz",
+      "integrity": "sha512-elZXTZXKn51hUBdJjSZGYRujuzilgXo8vSPQzjGYXLvSlGiCo8VO8ZGV3kjo9a0WNJJ57hENagwbtlRuHuzkcQ==",
+      "dev": true,
+      "requires": {
+        "es-to-primitive": "^1.2.1",
+        "function-bind": "^1.1.1",
+        "has": "^1.0.3",
+        "has-symbols": "^1.0.1",
+        "is-callable": "^1.2.0",
+        "is-negative-zero": "^2.0.0",
+        "is-regex": "^1.1.1",
+        "object-inspect": "^1.8.0",
+        "object-keys": "^1.1.1",
+        "object.assign": "^4.1.0",
+        "string.prototype.trimend": "^1.0.1",
+        "string.prototype.trimstart": "^1.0.1"
+      }
+    },
+    "es-to-primitive": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz",
+      "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==",
+      "dev": true,
+      "requires": {
+        "is-callable": "^1.1.4",
+        "is-date-object": "^1.0.1",
+        "is-symbol": "^1.0.2"
+      }
+    },
+    "es5-ext": {
+      "version": "0.10.53",
+      "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.53.tgz",
+      "integrity": "sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q==",
+      "dev": true,
+      "requires": {
+        "es6-iterator": "~2.0.3",
+        "es6-symbol": "~3.1.3",
+        "next-tick": "~1.0.0"
+      }
+    },
+    "es6-iterator": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz",
+      "integrity": "sha1-p96IkUGgWpSwhUQDstCg+/qY87c=",
+      "dev": true,
+      "requires": {
+        "d": "1",
+        "es5-ext": "^0.10.35",
+        "es6-symbol": "^3.1.1"
+      }
+    },
+    "es6-object-assign": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/es6-object-assign/-/es6-object-assign-1.1.0.tgz",
+      "integrity": "sha1-wsNYJlYkfDnqEHyx5mUrb58kUjw=",
+      "dev": true
+    },
+    "es6-promise": {
+      "version": "4.2.8",
+      "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz",
+      "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==",
+      "dev": true
+    },
+    "es6-promisify": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz",
+      "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=",
+      "dev": true,
+      "requires": {
+        "es6-promise": "^4.0.3"
+      }
+    },
+    "es6-symbol": {
+      "version": "3.1.3",
+      "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz",
+      "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==",
+      "dev": true,
+      "requires": {
+        "d": "^1.0.1",
+        "ext": "^1.1.2"
+      }
+    },
+    "es6-weak-map": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/es6-weak-map/-/es6-weak-map-2.0.3.tgz",
+      "integrity": "sha512-p5um32HOTO1kP+w7PRnB+5lQ43Z6muuMuIMffvDN8ZB4GcnjLBV6zGStpbASIMk4DCAvEaamhe2zhyCb/QXXsA==",
+      "dev": true,
+      "requires": {
+        "d": "1",
+        "es5-ext": "^0.10.46",
+        "es6-iterator": "^2.0.3",
+        "es6-symbol": "^3.1.1"
+      }
+    },
+    "escape-string-applescript": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/escape-string-applescript/-/escape-string-applescript-2.0.0.tgz",
+      "integrity": "sha1-dgvKg4Zo5Aj+XuUs5CyvfLRsUnM=",
+      "dev": true
+    },
+    "escape-string-regexp": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
+      "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ="
+    },
+    "escodegen": {
+      "version": "1.14.3",
+      "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.14.3.tgz",
+      "integrity": "sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==",
+      "dev": true,
+      "requires": {
+        "esprima": "^4.0.1",
+        "estraverse": "^4.2.0",
+        "esutils": "^2.0.2",
+        "optionator": "^0.8.1",
+        "source-map": "~0.6.1"
+      },
+      "dependencies": {
+        "source-map": {
+          "version": "0.6.1",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
           "dev": true,
           "optional": true
+        }
+      }
+    },
+    "eslint-scope": {
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz",
+      "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==",
+      "dev": true,
+      "requires": {
+        "esrecurse": "^4.1.0",
+        "estraverse": "^4.1.1"
+      }
+    },
+    "esm": {
+      "version": "3.2.25",
+      "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz",
+      "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==",
+      "dev": true
+    },
+    "esprima": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
+      "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
+      "dev": true
+    },
+    "esrecurse": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
+      "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
+      "dev": true,
+      "requires": {
+        "estraverse": "^5.2.0"
+      },
+      "dependencies": {
+        "estraverse": {
+          "version": "5.2.0",
+          "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz",
+          "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==",
+          "dev": true
+        }
+      }
+    },
+    "estraverse": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz",
+      "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==",
+      "dev": true
+    },
+    "esutils": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
+      "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",
+      "dev": true
+    },
+    "event-emitter": {
+      "version": "0.3.5",
+      "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz",
+      "integrity": "sha1-34xp7vFkeSPHFXuc6DhAYQsCzDk=",
+      "dev": true,
+      "requires": {
+        "d": "1",
+        "es5-ext": "~0.10.14"
+      }
+    },
+    "eventemitter3": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz",
+      "integrity": "sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q==",
+      "dev": true
+    },
+    "events": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/events/-/events-3.2.0.tgz",
+      "integrity": "sha512-/46HWwbfCX2xTawVfkKLGxMifJYQBWMwY1mjywRtb4c9x8l5NP3KoJtnIOiL1hfdRkIuYhETxQlo62IF8tcnlg==",
+      "dev": true
+    },
+    "evp_bytestokey": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz",
+      "integrity": "sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==",
+      "dev": true,
+      "requires": {
+        "md5.js": "^1.3.4",
+        "safe-buffer": "^5.1.1"
+      }
+    },
+    "exec-sh": {
+      "version": "0.3.4",
+      "resolved": "https://registry.npmjs.org/exec-sh/-/exec-sh-0.3.4.tgz",
+      "integrity": "sha512-sEFIkc61v75sWeOe72qyrqg2Qg0OuLESziUDk/O/z2qgS15y2gWVFrI6f2Qn/qw/0/NCfCEsmNA4zOjkwEZT1A==",
+      "dev": true
+    },
+    "execa": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz",
+      "integrity": "sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==",
+      "dev": true,
+      "requires": {
+        "cross-spawn": "^6.0.0",
+        "get-stream": "^4.0.0",
+        "is-stream": "^1.1.0",
+        "npm-run-path": "^2.0.0",
+        "p-finally": "^1.0.0",
+        "signal-exit": "^3.0.0",
+        "strip-eof": "^1.0.0"
+      }
+    },
+    "exit": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz",
+      "integrity": "sha1-BjJjj42HfMghB9MKD/8aF8uhzQw=",
+      "dev": true
+    },
+    "expand-brackets": {
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz",
+      "integrity": "sha1-t3c14xXOMPa27/D4OwQVGiJEliI=",
+      "dev": true,
+      "requires": {
+        "debug": "^2.3.3",
+        "define-property": "^0.2.5",
+        "extend-shallow": "^2.0.1",
+        "posix-character-classes": "^0.1.0",
+        "regex-not": "^1.0.0",
+        "snapdragon": "^0.8.1",
+        "to-regex": "^3.0.1"
+      },
+      "dependencies": {
+        "define-property": {
+          "version": "0.2.5",
+          "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
+          "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
+          "dev": true,
+          "requires": {
+            "is-descriptor": "^0.1.0"
+          }
+        },
+        "extend-shallow": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+          "dev": true,
+          "requires": {
+            "is-extendable": "^0.1.0"
+          }
+        }
+      }
+    },
+    "expand-tilde": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/expand-tilde/-/expand-tilde-2.0.2.tgz",
+      "integrity": "sha1-l+gBqgUt8CRU3kawK/YhZCzchQI=",
+      "dev": true,
+      "requires": {
+        "homedir-polyfill": "^1.0.1"
+      }
+    },
+    "expect": {
+      "version": "26.4.2",
+      "resolved": "https://registry.npmjs.org/expect/-/expect-26.4.2.tgz",
+      "integrity": "sha512-IlJ3X52Z0lDHm7gjEp+m76uX46ldH5VpqmU0006vqDju/285twh7zaWMRhs67VpQhBwjjMchk+p5aA0VkERCAA==",
+      "dev": true,
+      "requires": {
+        "@jest/types": "^26.3.0",
+        "ansi-styles": "^4.0.0",
+        "jest-get-type": "^26.3.0",
+        "jest-matcher-utils": "^26.4.2",
+        "jest-message-util": "^26.3.0",
+        "jest-regex-util": "^26.0.0"
+      },
+      "dependencies": {
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
+          }
+        },
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-report": "*"
+          }
+        },
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
+        },
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "jest-get-type": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-26.3.0.tgz",
+          "integrity": "sha512-TpfaviN1R2pQWkIihlfEanwOXK0zcxrKEE4MlU6Tn7keoXdN6/3gK/xl0yEh8DOunn5pOVGKf8hB4R9gVh04ig==",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
+        }
+      }
+    },
+    "ext": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/ext/-/ext-1.4.0.tgz",
+      "integrity": "sha512-Key5NIsUxdqKg3vIsdw9dSuXpPCQ297y6wBjL30edxwPgt2E44WcWBZey/ZvUc6sERLTxKdyCu4gZFmUbk1Q7A==",
+      "dev": true,
+      "requires": {
+        "type": "^2.0.0"
+      },
+      "dependencies": {
+        "type": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/type/-/type-2.1.0.tgz",
+          "integrity": "sha512-G9absDWvhAWCV2gmF1zKud3OyC61nZDwWvBL2DApaVFogI07CprggiQAOOjvp2NRjYWFzPyu7vwtDrQFq8jeSA==",
+          "dev": true
+        }
+      }
+    },
+    "extend": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
+      "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==",
+      "dev": true
+    },
+    "extend-shallow": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz",
+      "integrity": "sha1-Jqcarwc7OfshJxcnRhMcJwQCjbg=",
+      "dev": true,
+      "requires": {
+        "assign-symbols": "^1.0.0",
+        "is-extendable": "^1.0.1"
+      },
+      "dependencies": {
+        "is-extendable": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz",
+          "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==",
+          "dev": true,
+          "requires": {
+            "is-plain-object": "^2.0.4"
+          }
+        }
+      }
+    },
+    "external-editor": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz",
+      "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==",
+      "dev": true,
+      "requires": {
+        "chardet": "^0.7.0",
+        "iconv-lite": "^0.4.24",
+        "tmp": "^0.0.33"
+      }
+    },
+    "extglob": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz",
+      "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==",
+      "dev": true,
+      "requires": {
+        "array-unique": "^0.3.2",
+        "define-property": "^1.0.0",
+        "expand-brackets": "^2.1.4",
+        "extend-shallow": "^2.0.1",
+        "fragment-cache": "^0.2.1",
+        "regex-not": "^1.0.0",
+        "snapdragon": "^0.8.1",
+        "to-regex": "^3.0.1"
+      },
+      "dependencies": {
+        "define-property": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
+          "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=",
+          "dev": true,
+          "requires": {
+            "is-descriptor": "^1.0.0"
+          }
+        },
+        "extend-shallow": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+          "dev": true,
+          "requires": {
+            "is-extendable": "^0.1.0"
+          }
+        },
+        "is-accessor-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
+          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
+          "dev": true,
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
+        },
+        "is-data-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
+          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
+          "dev": true,
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
+        },
+        "is-descriptor": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
+          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+          "dev": true,
+          "requires": {
+            "is-accessor-descriptor": "^1.0.0",
+            "is-data-descriptor": "^1.0.0",
+            "kind-of": "^6.0.2"
+          }
+        }
+      }
+    },
+    "extsprintf": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
+      "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=",
+      "dev": true
+    },
+    "fancy-log": {
+      "version": "1.3.3",
+      "resolved": "https://registry.npmjs.org/fancy-log/-/fancy-log-1.3.3.tgz",
+      "integrity": "sha512-k9oEhlyc0FrVh25qYuSELjr8oxsCoc4/LEZfg2iJJrfEk/tZL9bCoJE47gqAvI2m/AUjluCS4+3I0eTx8n3AEw==",
+      "dev": true,
+      "requires": {
+        "ansi-gray": "^0.1.1",
+        "color-support": "^1.1.3",
+        "parse-node-version": "^1.0.0",
+        "time-stamp": "^1.0.0"
+      }
+    },
+    "fast-deep-equal": {
+      "version": "3.1.3",
+      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
+      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
+      "dev": true
+    },
+    "fast-extend": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/fast-extend/-/fast-extend-0.0.2.tgz",
+      "integrity": "sha1-9exCz0C5Rg9SGmOH37Ut7u1nHb0=",
+      "dev": true
+    },
+    "fast-glob": {
+      "version": "2.2.7",
+      "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-2.2.7.tgz",
+      "integrity": "sha512-g1KuQwHOZAmOZMuBtHdxDtju+T2RT8jgCC9aANsbpdiDDTSnjgfuVsIBNKbUeJI3oKMRExcfNDtJl4OhbffMsw==",
+      "dev": true,
+      "requires": {
+        "@mrmlnc/readdir-enhanced": "^2.2.1",
+        "@nodelib/fs.stat": "^1.1.2",
+        "glob-parent": "^3.1.0",
+        "is-glob": "^4.0.0",
+        "merge2": "^1.2.3",
+        "micromatch": "^3.1.10"
+      }
+    },
+    "fast-json-stable-stringify": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
+      "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==",
+      "dev": true
+    },
+    "fast-levenshtein": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-1.1.4.tgz",
+      "integrity": "sha1-5qdUzI8V5YmHqpy9J69m/W9OWvk=",
+      "dev": true
+    },
+    "fb-watchman": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.1.tgz",
+      "integrity": "sha512-DkPJKQeY6kKwmuMretBhr7G6Vodr7bFwDYTXIkfG1gjvNpaxBTQV3PbXg6bR1c1UP4jPOX0jHUbbHANL9vRjVg==",
+      "dev": true,
+      "requires": {
+        "bser": "2.1.1"
+      }
+    },
+    "figgy-pudding": {
+      "version": "3.5.2",
+      "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.2.tgz",
+      "integrity": "sha512-0btnI/H8f2pavGMN8w40mlSKOfTK2SVJmBfBeVIj3kNw0swwgzyRq0d5TJVOwodFmtvpPeWPN/MCcfuWF0Ezbw==",
+      "dev": true
+    },
+    "figures": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz",
+      "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=",
+      "dev": true,
+      "requires": {
+        "escape-string-regexp": "^1.0.5"
+      }
+    },
+    "file-uri-to-path": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
+      "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==",
+      "dev": true,
+      "optional": true
+    },
+    "fill-range": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz",
+      "integrity": "sha1-1USBHUKPmOsGpj3EAtJAPDKMOPc=",
+      "dev": true,
+      "requires": {
+        "extend-shallow": "^2.0.1",
+        "is-number": "^3.0.0",
+        "repeat-string": "^1.6.1",
+        "to-regex-range": "^2.1.0"
+      },
+      "dependencies": {
+        "extend-shallow": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+          "dev": true,
+          "requires": {
+            "is-extendable": "^0.1.0"
+          }
+        }
+      }
+    },
+    "find-cache-dir": {
+      "version": "3.3.1",
+      "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.1.tgz",
+      "integrity": "sha512-t2GDMt3oGC/v+BMwzmllWDuJF/xcDtE5j/fCGbqDD7OLuJkj0cfh1YSA5VKPvwMeLFLNDBkwOKZ2X85jGLVftQ==",
+      "dev": true,
+      "requires": {
+        "commondir": "^1.0.1",
+        "make-dir": "^3.0.2",
+        "pkg-dir": "^4.1.0"
+      }
+    },
+    "find-replace": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/find-replace/-/find-replace-2.0.1.tgz",
+      "integrity": "sha512-LzDo3Fpa30FLIBsh6DCDnMN1KW2g4QKkqKmejlImgWY67dDFPX/x9Kh/op/GK522DchQXEvDi/wD48HKW49XOQ==",
+      "requires": {
+        "array-back": "^2.0.0",
+        "test-value": "^3.0.0"
+      }
+    },
+    "find-up": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz",
+      "integrity": "sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8=",
+      "dev": true,
+      "requires": {
+        "path-exists": "^2.0.0",
+        "pinkie-promise": "^2.0.0"
+      }
+    },
+    "findup-sync": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/findup-sync/-/findup-sync-3.0.0.tgz",
+      "integrity": "sha512-YbffarhcicEhOrm4CtrwdKBdCuz576RLdhJDsIfvNtxUuhdRet1qZcsMjqbePtAseKdAnDyM/IyXbu7PRPRLYg==",
+      "dev": true,
+      "requires": {
+        "detect-file": "^1.0.0",
+        "is-glob": "^4.0.0",
+        "micromatch": "^3.0.4",
+        "resolve-dir": "^1.0.1"
+      }
+    },
+    "fined": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/fined/-/fined-1.2.0.tgz",
+      "integrity": "sha512-ZYDqPLGxDkDhDZBjZBb+oD1+j0rA4E0pXY50eplAAOPg2N/gUBSSk5IM1/QhPfyVo19lJ+CvXpqfvk+b2p/8Ng==",
+      "dev": true,
+      "requires": {
+        "expand-tilde": "^2.0.2",
+        "is-plain-object": "^2.0.3",
+        "object.defaults": "^1.1.0",
+        "object.pick": "^1.2.0",
+        "parse-filepath": "^1.0.1"
+      }
+    },
+    "flagged-respawn": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/flagged-respawn/-/flagged-respawn-1.0.1.tgz",
+      "integrity": "sha512-lNaHNVymajmk0OJMBn8fVUAU1BtDeKIqKoVhk4xAALB57aALg6b4W0MfJ/cUE0g9YBXy5XhSlPIpYIJ7HaY/3Q==",
+      "dev": true
+    },
+    "flatbuffers": {
+      "version": "1.11.0",
+      "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-1.11.0.tgz",
+      "integrity": "sha512-0PqFKtXI4MjxomI7jO4g5XfLPm/15g2R+5WGCHBGYGh0ihQiypnHlJ6bMmkkrAe0GzZ4d7PDAfCONKIPUxNF+A=="
+    },
+    "flush-write-stream": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-1.1.1.tgz",
+      "integrity": "sha512-3Z4XhFZ3992uIq0XOqb9AreonueSYphE6oYbpt5+3u06JWklbsPkNv3ZKkP9Bz/r+1MWCaMoSQ28P85+1Yc77w==",
+      "dev": true,
+      "requires": {
+        "inherits": "^2.0.3",
+        "readable-stream": "^2.3.6"
+      }
+    },
+    "for-in": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz",
+      "integrity": "sha1-gQaNKVqBQuwKxybG4iAMMPttXoA=",
+      "dev": true
+    },
+    "for-own": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/for-own/-/for-own-1.0.0.tgz",
+      "integrity": "sha1-xjMy9BXO3EsE2/5wz4NklMU8tEs=",
+      "dev": true,
+      "requires": {
+        "for-in": "^1.0.1"
+      }
+    },
+    "forever-agent": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
+      "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=",
+      "dev": true
+    },
+    "form-data": {
+      "version": "2.3.3",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
+      "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
+      "dev": true,
+      "requires": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.6",
+        "mime-types": "^2.1.12"
+      }
+    },
+    "fragment-cache": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz",
+      "integrity": "sha1-QpD60n8T6Jvn8zeZxrxaCr//DRk=",
+      "dev": true,
+      "requires": {
+        "map-cache": "^0.2.2"
+      }
+    },
+    "from2": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz",
+      "integrity": "sha1-i/tVAr3kpNNs/e6gB/zKIdfjgq8=",
+      "dev": true,
+      "requires": {
+        "inherits": "^2.0.1",
+        "readable-stream": "^2.0.0"
+      }
+    },
+    "fs-extra": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz",
+      "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==",
+      "dev": true,
+      "requires": {
+        "graceful-fs": "^4.2.0",
+        "jsonfile": "^4.0.0",
+        "universalify": "^0.1.0"
+      }
+    },
+    "fs-minipass": {
+      "version": "1.2.7",
+      "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz",
+      "integrity": "sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==",
+      "dev": true,
+      "requires": {
+        "minipass": "^2.6.0"
+      }
+    },
+    "fs-mkdirp-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs-mkdirp-stream/-/fs-mkdirp-stream-1.0.0.tgz",
+      "integrity": "sha1-C3gV/DIBxqaeFNuYzgmMFpNSWes=",
+      "dev": true,
+      "requires": {
+        "graceful-fs": "^4.1.11",
+        "through2": "^2.0.3"
+      }
+    },
+    "fs-monkey": {
+      "version": "0.3.3",
+      "resolved": "https://registry.npmjs.org/fs-monkey/-/fs-monkey-0.3.3.tgz",
+      "integrity": "sha512-FNUvuTAJ3CqCQb5ELn+qCbGR/Zllhf2HtwsdAtBi59s1WeCjKMT81fHcSu7dwIskqGVK+MmOrb7VOBlq3/SItw==",
+      "dev": true
+    },
+    "fs-write-stream-atomic": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/fs-write-stream-atomic/-/fs-write-stream-atomic-1.0.10.tgz",
+      "integrity": "sha1-tH31NJPvkR33VzHnCp3tAYnbQMk=",
+      "dev": true,
+      "requires": {
+        "graceful-fs": "^4.1.2",
+        "iferr": "^0.1.5",
+        "imurmurhash": "^0.1.4",
+        "readable-stream": "1 || 2"
+      }
+    },
+    "fs.realpath": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
+      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
+      "dev": true
+    },
+    "fsevents": {
+      "version": "1.2.13",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz",
+      "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==",
+      "dev": true,
+      "optional": true,
+      "requires": {
+        "bindings": "^1.5.0",
+        "nan": "^2.12.1"
+      }
+    },
+    "function-bind": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
+      "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==",
+      "dev": true
+    },
+    "gauge": {
+      "version": "2.7.4",
+      "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
+      "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=",
+      "dev": true,
+      "requires": {
+        "aproba": "^1.0.3",
+        "console-control-strings": "^1.0.0",
+        "has-unicode": "^2.0.0",
+        "object-assign": "^4.1.0",
+        "signal-exit": "^3.0.0",
+        "string-width": "^1.0.1",
+        "strip-ansi": "^3.0.1",
+        "wide-align": "^1.1.0"
+      }
+    },
+    "genfun": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/genfun/-/genfun-5.0.0.tgz",
+      "integrity": "sha512-KGDOARWVga7+rnB3z9Sd2Letx515owfk0hSxHGuqjANb1M+x2bGZGqHLiozPsYMdM2OubeMni/Hpwmjq6qIUhA==",
+      "dev": true
+    },
+    "gensync": {
+      "version": "1.0.0-beta.1",
+      "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.1.tgz",
+      "integrity": "sha512-r8EC6NO1sngH/zdD9fiRDLdcgnbayXah+mLgManTaIZJqEC1MZstmnox8KpnI2/fxQwrp5OpCOYWLp4rBl4Jcg==",
+      "dev": true
+    },
+    "get-caller-file": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz",
+      "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==",
+      "dev": true
+    },
+    "get-package-type": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz",
+      "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==",
+      "dev": true
+    },
+    "get-pkg-repo": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/get-pkg-repo/-/get-pkg-repo-1.4.0.tgz",
+      "integrity": "sha1-xztInAbYDMVTbCyFP54FIyBWly0=",
+      "dev": true,
+      "requires": {
+        "hosted-git-info": "^2.1.4",
+        "meow": "^3.3.0",
+        "normalize-package-data": "^2.3.0",
+        "parse-github-repo-url": "^1.3.0",
+        "through2": "^2.0.0"
+      },
+      "dependencies": {
+        "camelcase": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-2.1.1.tgz",
+          "integrity": "sha1-fB0W1nmhu+WcoCys7PsBHiAfWh8=",
+          "dev": true
         },
-        "debug": {
-          "version": "2.6.9",
-          "bundled": true,
+        "camelcase-keys": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz",
+          "integrity": "sha1-MIvur/3ygRkFHvodkyITyRuPkuc=",
           "dev": true,
-          "optional": true,
           "requires": {
-            "ms": "2.0.0"
+            "camelcase": "^2.0.0",
+            "map-obj": "^1.0.0"
           }
         },
-        "deep-extend": {
-          "version": "0.6.0",
-          "bundled": true,
-          "dev": true,
-          "optional": true
-        },
-        "delegates": {
-          "version": "1.0.0",
-          "bundled": true,
+        "indent-string": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-2.1.0.tgz",
+          "integrity": "sha1-ji1INIdCEhtKghi3oTfppSBJ3IA=",
           "dev": true,
-          "optional": true
+          "requires": {
+            "repeating": "^2.0.0"
+          }
         },
-        "detect-libc": {
-          "version": "1.0.3",
-          "bundled": true,
-          "dev": true,
-          "optional": true
+        "map-obj": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz",
+          "integrity": "sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=",
+          "dev": true
         },
-        "fs-minipass": {
-          "version": "1.2.5",
-          "bundled": true,
+        "meow": {
+          "version": "3.7.0",
+          "resolved": "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz",
+          "integrity": "sha1-cstmi0JSKCkKu/qFaJJYcwioAfs=",
           "dev": true,
-          "optional": true,
           "requires": {
-            "minipass": "^2.2.1"
+            "camelcase-keys": "^2.0.0",
+            "decamelize": "^1.1.2",
+            "loud-rejection": "^1.0.0",
+            "map-obj": "^1.0.1",
+            "minimist": "^1.1.3",
+            "normalize-package-data": "^2.3.4",
+            "object-assign": "^4.0.1",
+            "read-pkg-up": "^1.0.1",
+            "redent": "^1.0.0",
+            "trim-newlines": "^1.0.0"
           }
         },
-        "fs.realpath": {
+        "redent": {
           "version": "1.0.0",
-          "bundled": true,
-          "dev": true,
-          "optional": true
-        },
-        "gauge": {
-          "version": "2.7.4",
-          "bundled": true,
+          "resolved": "https://registry.npmjs.org/redent/-/redent-1.0.0.tgz",
+          "integrity": "sha1-z5Fqsf1fHxbfsggi3W7H9zDCr94=",
           "dev": true,
-          "optional": true,
           "requires": {
-            "aproba": "^1.0.3",
-            "console-control-strings": "^1.0.0",
-            "has-unicode": "^2.0.0",
-            "object-assign": "^4.1.0",
-            "signal-exit": "^3.0.0",
-            "string-width": "^1.0.1",
-            "strip-ansi": "^3.0.1",
-            "wide-align": "^1.1.0"
+            "indent-string": "^2.1.0",
+            "strip-indent": "^1.0.1"
           }
         },
-        "glob": {
-          "version": "7.1.3",
-          "bundled": true,
+        "strip-indent": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-1.0.1.tgz",
+          "integrity": "sha1-DHlipq3vp7vUrDZkYKY4VSrhoKI=",
           "dev": true,
-          "optional": true,
           "requires": {
-            "fs.realpath": "^1.0.0",
-            "inflight": "^1.0.4",
-            "inherits": "2",
-            "minimatch": "^3.0.4",
-            "once": "^1.3.0",
-            "path-is-absolute": "^1.0.0"
+            "get-stdin": "^4.0.1"
           }
         },
-        "has-unicode": {
-          "version": "2.0.1",
-          "bundled": true,
-          "dev": true,
-          "optional": true
-        },
-        "iconv-lite": {
-          "version": "0.4.24",
-          "bundled": true,
+        "trim-newlines": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz",
+          "integrity": "sha1-WIeWa7WCpFA6QetST301ARgVphM=",
+          "dev": true
+        }
+      }
+    },
+    "get-port": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/get-port/-/get-port-4.2.0.tgz",
+      "integrity": "sha512-/b3jarXkH8KJoOMQc3uVGHASwGLPq3gSFJ7tgJm2diza+bydJPTGOibin2steecKeOylE8oY2JERlVWkAJO6yw==",
+      "dev": true
+    },
+    "get-stdin": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz",
+      "integrity": "sha1-uWjGsKBDhDJJAui/Gl3zJXmkUP4=",
+      "dev": true
+    },
+    "get-stream": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz",
+      "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==",
+      "dev": true,
+      "requires": {
+        "pump": "^3.0.0"
+      },
+      "dependencies": {
+        "pump": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
+          "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
           "dev": true,
-          "optional": true,
           "requires": {
-            "safer-buffer": ">= 2.1.2 < 3"
+            "end-of-stream": "^1.1.0",
+            "once": "^1.3.1"
           }
+        }
+      }
+    },
+    "get-value": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz",
+      "integrity": "sha1-3BXKHGcjh8p2vTesCjlbogQqLCg=",
+      "dev": true
+    },
+    "getpass": {
+      "version": "0.1.7",
+      "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
+      "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
+      "dev": true,
+      "requires": {
+        "assert-plus": "^1.0.0"
+      }
+    },
+    "git-raw-commits": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/git-raw-commits/-/git-raw-commits-2.0.0.tgz",
+      "integrity": "sha512-w4jFEJFgKXMQJ0H0ikBk2S+4KP2VEjhCvLCNqbNRQC8BgGWgLKNCO7a9K9LI+TVT7Gfoloje502sEnctibffgg==",
+      "dev": true,
+      "requires": {
+        "dargs": "^4.0.1",
+        "lodash.template": "^4.0.2",
+        "meow": "^4.0.0",
+        "split2": "^2.0.0",
+        "through2": "^2.0.0"
+      },
+      "dependencies": {
+        "camelcase": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz",
+          "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=",
+          "dev": true
         },
-        "ignore-walk": {
-          "version": "3.0.1",
-          "bundled": true,
+        "camelcase-keys": {
+          "version": "4.2.0",
+          "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-4.2.0.tgz",
+          "integrity": "sha1-oqpfsa9oh1glnDLBQUJteJI7m3c=",
           "dev": true,
-          "optional": true,
           "requires": {
-            "minimatch": "^3.0.4"
+            "camelcase": "^4.1.0",
+            "map-obj": "^2.0.0",
+            "quick-lru": "^1.0.0"
           }
         },
-        "inflight": {
-          "version": "1.0.6",
-          "bundled": true,
+        "find-up": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz",
+          "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=",
           "dev": true,
-          "optional": true,
           "requires": {
-            "once": "^1.3.0",
-            "wrappy": "1"
+            "locate-path": "^2.0.0"
           }
         },
-        "inherits": {
-          "version": "2.0.3",
-          "bundled": true,
-          "dev": true,
-          "optional": true
-        },
-        "ini": {
-          "version": "1.3.5",
-          "bundled": true,
-          "dev": true,
-          "optional": true
+        "indent-string": {
+          "version": "3.2.0",
+          "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz",
+          "integrity": "sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok=",
+          "dev": true
         },
-        "is-fullwidth-code-point": {
-          "version": "1.0.0",
-          "bundled": true,
+        "load-json-file": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz",
+          "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=",
           "dev": true,
-          "optional": true,
           "requires": {
-            "number-is-nan": "^1.0.0"
+            "graceful-fs": "^4.1.2",
+            "parse-json": "^4.0.0",
+            "pify": "^3.0.0",
+            "strip-bom": "^3.0.0"
           }
         },
-        "isarray": {
-          "version": "1.0.0",
-          "bundled": true,
-          "dev": true,
-          "optional": true
-        },
-        "minimatch": {
-          "version": "3.0.4",
-          "bundled": true,
+        "locate-path": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz",
+          "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=",
           "dev": true,
-          "optional": true,
           "requires": {
-            "brace-expansion": "^1.1.7"
+            "p-locate": "^2.0.0",
+            "path-exists": "^3.0.0"
           }
         },
-        "minimist": {
-          "version": "0.0.8",
-          "bundled": true,
-          "dev": true,
-          "optional": true
+        "map-obj": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-2.0.0.tgz",
+          "integrity": "sha1-plzSkIepJZi4eRJXpSPgISIqwfk=",
+          "dev": true
         },
-        "minipass": {
-          "version": "2.3.5",
-          "bundled": true,
+        "meow": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/meow/-/meow-4.0.1.tgz",
+          "integrity": "sha512-xcSBHD5Z86zaOc+781KrupuHAzeGXSLtiAOmBsiLDiPSaYSB6hdew2ng9EBAnZ62jagG9MHAOdxpDi/lWBFJ/A==",
           "dev": true,
-          "optional": true,
           "requires": {
-            "safe-buffer": "^5.1.2",
-            "yallist": "^3.0.0"
+            "camelcase-keys": "^4.0.0",
+            "decamelize-keys": "^1.0.0",
+            "loud-rejection": "^1.0.0",
+            "minimist": "^1.1.3",
+            "minimist-options": "^3.0.1",
+            "normalize-package-data": "^2.3.4",
+            "read-pkg-up": "^3.0.0",
+            "redent": "^2.0.0",
+            "trim-newlines": "^2.0.0"
           }
         },
-        "minizlib": {
-          "version": "1.2.1",
-          "bundled": true,
+        "minimist-options": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/minimist-options/-/minimist-options-3.0.2.tgz",
+          "integrity": "sha512-FyBrT/d0d4+uiZRbqznPXqw3IpZZG3gl3wKWiX784FycUKVwBt0uLBFkQrtE4tZOrgo78nZp2jnKz3L65T5LdQ==",
           "dev": true,
-          "optional": true,
           "requires": {
-            "minipass": "^2.2.1"
+            "arrify": "^1.0.1",
+            "is-plain-obj": "^1.1.0"
           }
         },
-        "mkdirp": {
-          "version": "0.5.1",
-          "bundled": true,
+        "p-limit": {
+          "version": "1.3.0",
+          "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz",
+          "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==",
           "dev": true,
-          "optional": true,
           "requires": {
-            "minimist": "0.0.8"
+            "p-try": "^1.0.0"
           }
         },
-        "ms": {
+        "p-locate": {
           "version": "2.0.0",
-          "bundled": true,
-          "dev": true,
-          "optional": true
-        },
-        "needle": {
-          "version": "2.2.4",
-          "bundled": true,
+          "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz",
+          "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=",
           "dev": true,
-          "optional": true,
           "requires": {
-            "debug": "^2.1.2",
-            "iconv-lite": "^0.4.4",
-            "sax": "^1.2.4"
+            "p-limit": "^1.1.0"
           }
         },
-        "node-pre-gyp": {
-          "version": "0.10.3",
-          "bundled": true,
+        "p-try": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz",
+          "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=",
+          "dev": true
+        },
+        "parse-json": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
+          "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=",
           "dev": true,
-          "optional": true,
           "requires": {
-            "detect-libc": "^1.0.2",
-            "mkdirp": "^0.5.1",
-            "needle": "^2.2.1",
-            "nopt": "^4.0.1",
-            "npm-packlist": "^1.1.6",
-            "npmlog": "^4.0.2",
-            "rc": "^1.2.7",
-            "rimraf": "^2.6.1",
-            "semver": "^5.3.0",
-            "tar": "^4"
+            "error-ex": "^1.3.1",
+            "json-parse-better-errors": "^1.0.1"
           }
         },
-        "nopt": {
-          "version": "4.0.1",
-          "bundled": true,
+        "path-exists": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
+          "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=",
+          "dev": true
+        },
+        "path-type": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz",
+          "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==",
           "dev": true,
-          "optional": true,
           "requires": {
-            "abbrev": "1",
-            "osenv": "^0.1.4"
+            "pify": "^3.0.0"
           }
         },
-        "npm-bundled": {
-          "version": "1.0.5",
-          "bundled": true,
-          "dev": true,
-          "optional": true
+        "quick-lru": {
+          "version": "1.1.0",
+          "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-1.1.0.tgz",
+          "integrity": "sha1-Q2CxfGETatOAeDl/8RQW4Ybc+7g=",
+          "dev": true
         },
-        "npm-packlist": {
-          "version": "1.2.0",
-          "bundled": true,
+        "read-pkg": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz",
+          "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=",
           "dev": true,
-          "optional": true,
           "requires": {
-            "ignore-walk": "^3.0.1",
-            "npm-bundled": "^1.0.1"
+            "load-json-file": "^4.0.0",
+            "normalize-package-data": "^2.3.2",
+            "path-type": "^3.0.0"
           }
         },
-        "npmlog": {
-          "version": "4.1.2",
-          "bundled": true,
+        "read-pkg-up": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-3.0.0.tgz",
+          "integrity": "sha1-PtSWaF26D4/hGNBpHcUfSh/5bwc=",
           "dev": true,
-          "optional": true,
           "requires": {
-            "are-we-there-yet": "~1.1.2",
-            "console-control-strings": "~1.1.0",
-            "gauge": "~2.7.3",
-            "set-blocking": "~2.0.0"
+            "find-up": "^2.0.0",
+            "read-pkg": "^3.0.0"
           }
         },
-        "number-is-nan": {
-          "version": "1.0.1",
-          "bundled": true,
+        "redent": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/redent/-/redent-2.0.0.tgz",
+          "integrity": "sha1-wbIAe0LVfrE4kHmzyDM2OdXhzKo=",
           "dev": true,
-          "optional": true
+          "requires": {
+            "indent-string": "^3.0.0",
+            "strip-indent": "^2.0.0"
+          }
         },
-        "object-assign": {
-          "version": "4.1.1",
-          "bundled": true,
-          "dev": true,
-          "optional": true
+        "strip-bom": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
+          "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=",
+          "dev": true
+        },
+        "strip-indent": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-2.0.0.tgz",
+          "integrity": "sha1-XvjbKV0B5u1sv3qrlpmNeCJSe2g=",
+          "dev": true
+        },
+        "trim-newlines": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-2.0.0.tgz",
+          "integrity": "sha1-tAPQuRvlDDMd/EuC7s6yLD3hbSA=",
+          "dev": true
+        }
+      }
+    },
+    "git-remote-origin-url": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/git-remote-origin-url/-/git-remote-origin-url-2.0.0.tgz",
+      "integrity": "sha1-UoJlna4hBxRaERJhEq0yFuxfpl8=",
+      "dev": true,
+      "requires": {
+        "gitconfiglocal": "^1.0.0",
+        "pify": "^2.3.0"
+      },
+      "dependencies": {
+        "pify": {
+          "version": "2.3.0",
+          "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
+          "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=",
+          "dev": true
+        }
+      }
+    },
+    "git-semver-tags": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/git-semver-tags/-/git-semver-tags-2.0.3.tgz",
+      "integrity": "sha512-tj4FD4ww2RX2ae//jSrXZzrocla9db5h0V7ikPl1P/WwoZar9epdUhwR7XHXSgc+ZkNq72BEEerqQuicoEQfzA==",
+      "dev": true,
+      "requires": {
+        "meow": "^4.0.0",
+        "semver": "^6.0.0"
+      },
+      "dependencies": {
+        "camelcase": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz",
+          "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=",
+          "dev": true
         },
-        "once": {
-          "version": "1.4.0",
-          "bundled": true,
+        "camelcase-keys": {
+          "version": "4.2.0",
+          "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-4.2.0.tgz",
+          "integrity": "sha1-oqpfsa9oh1glnDLBQUJteJI7m3c=",
           "dev": true,
-          "optional": true,
           "requires": {
-            "wrappy": "1"
+            "camelcase": "^4.1.0",
+            "map-obj": "^2.0.0",
+            "quick-lru": "^1.0.0"
           }
         },
-        "os-homedir": {
-          "version": "1.0.2",
-          "bundled": true,
+        "find-up": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz",
+          "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=",
           "dev": true,
-          "optional": true
+          "requires": {
+            "locate-path": "^2.0.0"
+          }
         },
-        "os-tmpdir": {
-          "version": "1.0.2",
-          "bundled": true,
-          "dev": true,
-          "optional": true
+        "indent-string": {
+          "version": "3.2.0",
+          "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz",
+          "integrity": "sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok=",
+          "dev": true
         },
-        "osenv": {
-          "version": "0.1.5",
-          "bundled": true,
+        "load-json-file": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz",
+          "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=",
           "dev": true,
-          "optional": true,
           "requires": {
-            "os-homedir": "^1.0.0",
-            "os-tmpdir": "^1.0.0"
+            "graceful-fs": "^4.1.2",
+            "parse-json": "^4.0.0",
+            "pify": "^3.0.0",
+            "strip-bom": "^3.0.0"
           }
         },
-        "path-is-absolute": {
-          "version": "1.0.1",
-          "bundled": true,
+        "locate-path": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz",
+          "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=",
           "dev": true,
-          "optional": true
+          "requires": {
+            "p-locate": "^2.0.0",
+            "path-exists": "^3.0.0"
+          }
         },
-        "process-nextick-args": {
+        "map-obj": {
           "version": "2.0.0",
-          "bundled": true,
-          "dev": true,
-          "optional": true
+          "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-2.0.0.tgz",
+          "integrity": "sha1-plzSkIepJZi4eRJXpSPgISIqwfk=",
+          "dev": true
         },
-        "rc": {
-          "version": "1.2.8",
-          "bundled": true,
+        "meow": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/meow/-/meow-4.0.1.tgz",
+          "integrity": "sha512-xcSBHD5Z86zaOc+781KrupuHAzeGXSLtiAOmBsiLDiPSaYSB6hdew2ng9EBAnZ62jagG9MHAOdxpDi/lWBFJ/A==",
           "dev": true,
-          "optional": true,
           "requires": {
-            "deep-extend": "^0.6.0",
-            "ini": "~1.3.0",
-            "minimist": "^1.2.0",
-            "strip-json-comments": "~2.0.1"
-          },
-          "dependencies": {
-            "minimist": {
-              "version": "1.2.0",
-              "bundled": true,
-              "dev": true,
-              "optional": true
-            }
+            "camelcase-keys": "^4.0.0",
+            "decamelize-keys": "^1.0.0",
+            "loud-rejection": "^1.0.0",
+            "minimist": "^1.1.3",
+            "minimist-options": "^3.0.1",
+            "normalize-package-data": "^2.3.4",
+            "read-pkg-up": "^3.0.0",
+            "redent": "^2.0.0",
+            "trim-newlines": "^2.0.0"
           }
         },
-        "readable-stream": {
-          "version": "2.3.6",
-          "bundled": true,
+        "minimist-options": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/minimist-options/-/minimist-options-3.0.2.tgz",
+          "integrity": "sha512-FyBrT/d0d4+uiZRbqznPXqw3IpZZG3gl3wKWiX784FycUKVwBt0uLBFkQrtE4tZOrgo78nZp2jnKz3L65T5LdQ==",
           "dev": true,
-          "optional": true,
           "requires": {
-            "core-util-is": "~1.0.0",
-            "inherits": "~2.0.3",
-            "isarray": "~1.0.0",
-            "process-nextick-args": "~2.0.0",
-            "safe-buffer": "~5.1.1",
-            "string_decoder": "~1.1.1",
-            "util-deprecate": "~1.0.1"
+            "arrify": "^1.0.1",
+            "is-plain-obj": "^1.1.0"
           }
         },
-        "rimraf": {
-          "version": "2.6.3",
-          "bundled": true,
+        "p-limit": {
+          "version": "1.3.0",
+          "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz",
+          "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==",
           "dev": true,
-          "optional": true,
           "requires": {
-            "glob": "^7.1.3"
+            "p-try": "^1.0.0"
           }
         },
-        "safe-buffer": {
-          "version": "5.1.2",
-          "bundled": true,
+        "p-locate": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz",
+          "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=",
           "dev": true,
-          "optional": true
+          "requires": {
+            "p-limit": "^1.1.0"
+          }
         },
-        "safer-buffer": {
-          "version": "2.1.2",
-          "bundled": true,
-          "dev": true,
-          "optional": true
+        "p-try": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz",
+          "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=",
+          "dev": true
         },
-        "sax": {
-          "version": "1.2.4",
-          "bundled": true,
+        "parse-json": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
+          "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=",
           "dev": true,
-          "optional": true
+          "requires": {
+            "error-ex": "^1.3.1",
+            "json-parse-better-errors": "^1.0.1"
+          }
         },
-        "semver": {
-          "version": "5.6.0",
-          "bundled": true,
-          "dev": true,
-          "optional": true
+        "path-exists": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
+          "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=",
+          "dev": true
         },
-        "set-blocking": {
-          "version": "2.0.0",
-          "bundled": true,
+        "path-type": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz",
+          "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==",
           "dev": true,
-          "optional": true
+          "requires": {
+            "pify": "^3.0.0"
+          }
         },
-        "signal-exit": {
-          "version": "3.0.2",
-          "bundled": true,
-          "dev": true,
-          "optional": true
+        "quick-lru": {
+          "version": "1.1.0",
+          "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-1.1.0.tgz",
+          "integrity": "sha1-Q2CxfGETatOAeDl/8RQW4Ybc+7g=",
+          "dev": true
         },
-        "string-width": {
-          "version": "1.0.2",
-          "bundled": true,
+        "read-pkg": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz",
+          "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=",
           "dev": true,
-          "optional": true,
           "requires": {
-            "code-point-at": "^1.0.0",
-            "is-fullwidth-code-point": "^1.0.0",
-            "strip-ansi": "^3.0.0"
+            "load-json-file": "^4.0.0",
+            "normalize-package-data": "^2.3.2",
+            "path-type": "^3.0.0"
           }
         },
-        "string_decoder": {
-          "version": "1.1.1",
-          "bundled": true,
+        "read-pkg-up": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-3.0.0.tgz",
+          "integrity": "sha1-PtSWaF26D4/hGNBpHcUfSh/5bwc=",
           "dev": true,
-          "optional": true,
           "requires": {
-            "safe-buffer": "~5.1.0"
+            "find-up": "^2.0.0",
+            "read-pkg": "^3.0.0"
           }
         },
-        "strip-ansi": {
-          "version": "3.0.1",
-          "bundled": true,
+        "redent": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/redent/-/redent-2.0.0.tgz",
+          "integrity": "sha1-wbIAe0LVfrE4kHmzyDM2OdXhzKo=",
           "dev": true,
-          "optional": true,
           "requires": {
-            "ansi-regex": "^2.0.0"
+            "indent-string": "^3.0.0",
+            "strip-indent": "^2.0.0"
           }
         },
-        "strip-json-comments": {
-          "version": "2.0.1",
-          "bundled": true,
-          "dev": true,
-          "optional": true
+        "semver": {
+          "version": "6.3.0",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
+          "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
+          "dev": true
         },
-        "tar": {
-          "version": "4.4.8",
-          "bundled": true,
-          "dev": true,
-          "optional": true,
-          "requires": {
-            "chownr": "^1.1.1",
-            "fs-minipass": "^1.2.5",
-            "minipass": "^2.3.4",
-            "minizlib": "^1.1.1",
-            "mkdirp": "^0.5.0",
-            "safe-buffer": "^5.1.2",
-            "yallist": "^3.0.2"
-          }
+        "strip-bom": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
+          "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=",
+          "dev": true
         },
-        "util-deprecate": {
-          "version": "1.0.2",
-          "bundled": true,
-          "dev": true,
-          "optional": true
+        "strip-indent": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-2.0.0.tgz",
+          "integrity": "sha1-XvjbKV0B5u1sv3qrlpmNeCJSe2g=",
+          "dev": true
         },
-        "wide-align": {
-          "version": "1.1.3",
-          "bundled": true,
+        "trim-newlines": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-2.0.0.tgz",
+          "integrity": "sha1-tAPQuRvlDDMd/EuC7s6yLD3hbSA=",
+          "dev": true
+        }
+      }
+    },
+    "git-up": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/git-up/-/git-up-4.0.2.tgz",
+      "integrity": "sha512-kbuvus1dWQB2sSW4cbfTeGpCMd8ge9jx9RKnhXhuJ7tnvT+NIrTVfYZxjtflZddQYcmdOTlkAcjmx7bor+15AQ==",
+      "dev": true,
+      "requires": {
+        "is-ssh": "^1.3.0",
+        "parse-url": "^5.0.0"
+      }
+    },
+    "git-url-parse": {
+      "version": "11.2.0",
+      "resolved": "https://registry.npmjs.org/git-url-parse/-/git-url-parse-11.2.0.tgz",
+      "integrity": "sha512-KPoHZg8v+plarZvto4ruIzzJLFQoRx+sUs5DQSr07By9IBKguVd+e6jwrFR6/TP6xrCJlNV1tPqLO1aREc7O2g==",
+      "dev": true,
+      "requires": {
+        "git-up": "^4.0.0"
+      }
+    },
+    "gitconfiglocal": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/gitconfiglocal/-/gitconfiglocal-1.0.0.tgz",
+      "integrity": "sha1-QdBF84UaXqiPA/JMocYXgRRGS5s=",
+      "dev": true,
+      "requires": {
+        "ini": "^1.3.2"
+      }
+    },
+    "glob": {
+      "version": "7.1.4",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",
+      "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==",
+      "dev": true,
+      "requires": {
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.0.4",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      }
+    },
+    "glob-parent": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-3.1.0.tgz",
+      "integrity": "sha1-nmr2KZ2NO9K9QEMIMr0RPfkGxa4=",
+      "dev": true,
+      "requires": {
+        "is-glob": "^3.1.0",
+        "path-dirname": "^1.0.0"
+      },
+      "dependencies": {
+        "is-glob": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-3.1.0.tgz",
+          "integrity": "sha1-e6WuJCF4BKxwcHuWkiVnSGzD6Eo=",
           "dev": true,
-          "optional": true,
           "requires": {
-            "string-width": "^1.0.2 || 2"
+            "is-extglob": "^2.1.0"
           }
-        },
-        "wrappy": {
-          "version": "1.0.2",
-          "bundled": true,
-          "dev": true,
-          "optional": true
-        },
-        "yallist": {
-          "version": "3.0.3",
-          "bundled": true,
-          "dev": true,
-          "optional": true
         }
       }
     },
-    "fstream": {
-      "version": "1.0.11",
-      "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.11.tgz",
-      "integrity": "sha1-XB+x8RdHcRTwYyoOtLcbPLD9MXE=",
+    "glob-stream": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/glob-stream/-/glob-stream-6.1.0.tgz",
+      "integrity": "sha1-cEXJlBOz65SIjYOrRtC0BMx73eQ=",
+      "dev": true,
+      "requires": {
+        "extend": "^3.0.0",
+        "glob": "^7.1.1",
+        "glob-parent": "^3.1.0",
+        "is-negated-glob": "^1.0.0",
+        "ordered-read-streams": "^1.0.0",
+        "pumpify": "^1.3.5",
+        "readable-stream": "^2.1.5",
+        "remove-trailing-separator": "^1.0.1",
+        "to-absolute-glob": "^2.0.0",
+        "unique-stream": "^2.0.2"
+      }
+    },
+    "glob-to-regexp": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz",
+      "integrity": "sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs=",
+      "dev": true
+    },
+    "glob-watcher": {
+      "version": "5.0.5",
+      "resolved": "https://registry.npmjs.org/glob-watcher/-/glob-watcher-5.0.5.tgz",
+      "integrity": "sha512-zOZgGGEHPklZNjZQaZ9f41i7F2YwE+tS5ZHrDhbBCk3stwahn5vQxnFmBJZHoYdusR6R1bLSXeGUy/BhctwKzw==",
+      "dev": true,
+      "requires": {
+        "anymatch": "^2.0.0",
+        "async-done": "^1.2.0",
+        "chokidar": "^2.0.0",
+        "is-negated-glob": "^1.0.0",
+        "just-debounce": "^1.0.0",
+        "normalize-path": "^3.0.0",
+        "object.defaults": "^1.1.0"
+      }
+    },
+    "global-modules": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/global-modules/-/global-modules-1.0.0.tgz",
+      "integrity": "sha512-sKzpEkf11GpOFuw0Zzjzmt4B4UZwjOcG757PPvrfhxcLFbq0wpsgpOqxpxtxFiCG4DtG93M6XRVbF2oGdev7bg==",
+      "dev": true,
+      "requires": {
+        "global-prefix": "^1.0.1",
+        "is-windows": "^1.0.1",
+        "resolve-dir": "^1.0.0"
+      }
+    },
+    "global-prefix": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/global-prefix/-/global-prefix-1.0.2.tgz",
+      "integrity": "sha1-2/dDxsFJklk8ZVVoy2btMsASLr4=",
+      "dev": true,
+      "requires": {
+        "expand-tilde": "^2.0.2",
+        "homedir-polyfill": "^1.0.1",
+        "ini": "^1.3.4",
+        "is-windows": "^1.0.1",
+        "which": "^1.2.14"
+      }
+    },
+    "globals": {
+      "version": "11.12.0",
+      "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz",
+      "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==",
+      "dev": true
+    },
+    "globby": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/globby/-/globby-6.1.0.tgz",
+      "integrity": "sha1-9abXDoOV4hyFj7BInWTfAkJNUGw=",
+      "dev": true,
+      "requires": {
+        "array-union": "^1.0.1",
+        "glob": "^7.0.3",
+        "object-assign": "^4.0.1",
+        "pify": "^2.0.0",
+        "pinkie-promise": "^2.0.0"
+      },
+      "dependencies": {
+        "pify": {
+          "version": "2.3.0",
+          "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
+          "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=",
+          "dev": true
+        }
+      }
+    },
+    "glogg": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/glogg/-/glogg-1.0.2.tgz",
+      "integrity": "sha512-5mwUoSuBk44Y4EshyiqcH95ZntbDdTQqA3QYSrxmzj28Ai0vXBGMH1ApSANH14j2sIRtqCEyg6PfsuP7ElOEDA==",
       "dev": true,
       "requires": {
-        "graceful-fs": "^4.1.2",
-        "inherits": "~2.0.0",
-        "mkdirp": ">=0.5 0",
-        "rimraf": "2"
+        "sparkles": "^1.0.0"
       }
     },
-    "function-bind": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
-      "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==",
+    "google-closure-compiler": {
+      "version": "20200830.0.0",
+      "resolved": "https://registry.npmjs.org/google-closure-compiler/-/google-closure-compiler-20200830.0.0.tgz",
+      "integrity": "sha512-Pri8kyHGmd2xqLM38QBarx+fdkm2HuLniGz7GimbdjQ1KUuPNIz7IJOYc8NGGwYPGAB45vg4IZRk/LepAqnoxg==",
+      "dev": true,
+      "requires": {
+        "chalk": "2.x",
+        "google-closure-compiler-java": "^20200830.0.0",
+        "google-closure-compiler-linux": "^20200830.0.0",
+        "google-closure-compiler-osx": "^20200830.0.0",
+        "google-closure-compiler-windows": "^20200830.0.0",
+        "minimist": "1.x",
+        "vinyl": "2.x",
+        "vinyl-sourcemaps-apply": "^0.2.0"
+      }
+    },
+    "google-closure-compiler-java": {
+      "version": "20200830.0.0",
+      "resolved": "https://registry.npmjs.org/google-closure-compiler-java/-/google-closure-compiler-java-20200830.0.0.tgz",
+      "integrity": "sha512-DLlcY875mQB7PA9wtfbPBVL9chJj+si/cmxyp3euw7x09MiFYynR4tmQJ9KjWUffPbhvCRDEO/jKcVyNWQVS1Q==",
       "dev": true
     },
-    "gauge": {
-      "version": "2.7.4",
-      "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
-      "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=",
+    "google-closure-compiler-linux": {
+      "version": "20200830.0.0",
+      "resolved": "https://registry.npmjs.org/google-closure-compiler-linux/-/google-closure-compiler-linux-20200830.0.0.tgz",
+      "integrity": "sha512-QfxFA3+fOrNe0RH2lcXmkdiaM97KvZQOtO3trobNvfkMNr2h9OUtpXkqWExwolo/jsJWNumsdaRnEAwEthMUOw==",
       "dev": true,
-      "requires": {
-        "aproba": "^1.0.3",
-        "console-control-strings": "^1.0.0",
-        "has-unicode": "^2.0.0",
-        "object-assign": "^4.1.0",
-        "signal-exit": "^3.0.0",
-        "string-width": "^1.0.1",
-        "strip-ansi": "^3.0.1",
-        "wide-align": "^1.1.0"
-      }
+      "optional": true
     },
-    "genfun": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/genfun/-/genfun-5.0.0.tgz",
-      "integrity": "sha512-KGDOARWVga7+rnB3z9Sd2Letx515owfk0hSxHGuqjANb1M+x2bGZGqHLiozPsYMdM2OubeMni/Hpwmjq6qIUhA==",
+    "google-closure-compiler-osx": {
+      "version": "20200830.0.0",
+      "resolved": "https://registry.npmjs.org/google-closure-compiler-osx/-/google-closure-compiler-osx-20200830.0.0.tgz",
+      "integrity": "sha512-qHKjRBJVq2+2mT25eoT6iOMVbUGT02sJUwkdLlsohWKV4sMEY8/nwnkZYsdm7KnPJnmQLlrfYJ1ZTh1VTlAJpQ==",
+      "dev": true,
+      "optional": true
+    },
+    "google-closure-compiler-windows": {
+      "version": "20200830.0.0",
+      "resolved": "https://registry.npmjs.org/google-closure-compiler-windows/-/google-closure-compiler-windows-20200830.0.0.tgz",
+      "integrity": "sha512-IpJAyxJo+GQ2DSVC4sslPydhIPyWRINkdNynIK/Bk+vbM/7i4LoEm/Y5rY/KJOLRCSds+s3Ov9LYdFkN8C//7g==",
+      "dev": true,
+      "optional": true
+    },
+    "graceful-fs": {
+      "version": "4.2.4",
+      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.4.tgz",
+      "integrity": "sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw==",
       "dev": true
     },
-    "gentle-fs": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/gentle-fs/-/gentle-fs-2.0.1.tgz",
-      "integrity": "sha512-cEng5+3fuARewXktTEGbwsktcldA+YsnUEaXZwcK/3pjSE1X9ObnTs+/8rYf8s+RnIcQm2D5x3rwpN7Zom8Bew==",
+    "growl": {
+      "version": "1.10.5",
+      "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz",
+      "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==",
+      "dev": true
+    },
+    "growly": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/growly/-/growly-1.3.0.tgz",
+      "integrity": "sha1-8QdIy+dq+WS3yWyTxrzCivEgwIE=",
+      "dev": true,
+      "optional": true
+    },
+    "gulp": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/gulp/-/gulp-4.0.2.tgz",
+      "integrity": "sha512-dvEs27SCZt2ibF29xYgmnwwCYZxdxhQ/+LFWlbAW8y7jt68L/65402Lz3+CKy0Ov4rOs+NERmDq7YlZaDqUIfA==",
       "dev": true,
       "requires": {
-        "aproba": "^1.1.2",
-        "fs-vacuum": "^1.2.10",
-        "graceful-fs": "^4.1.11",
-        "iferr": "^0.1.5",
-        "mkdirp": "^0.5.1",
-        "path-is-inside": "^1.0.2",
-        "read-cmd-shim": "^1.0.1",
-        "slide": "^1.1.6"
+        "glob-watcher": "^5.0.3",
+        "gulp-cli": "^2.2.0",
+        "undertaker": "^1.2.1",
+        "vinyl-fs": "^3.0.0"
+      },
+      "dependencies": {
+        "gulp-cli": {
+          "version": "2.3.0",
+          "resolved": "https://registry.npmjs.org/gulp-cli/-/gulp-cli-2.3.0.tgz",
+          "integrity": "sha512-zzGBl5fHo0EKSXsHzjspp3y5CONegCm8ErO5Qh0UzFzk2y4tMvzLWhoDokADbarfZRL2pGpRp7yt6gfJX4ph7A==",
+          "dev": true,
+          "requires": {
+            "ansi-colors": "^1.0.1",
+            "archy": "^1.0.0",
+            "array-sort": "^1.0.0",
+            "color-support": "^1.1.3",
+            "concat-stream": "^1.6.0",
+            "copy-props": "^2.0.1",
+            "fancy-log": "^1.3.2",
+            "gulplog": "^1.0.0",
+            "interpret": "^1.4.0",
+            "isobject": "^3.0.1",
+            "liftoff": "^3.1.0",
+            "matchdep": "^2.0.0",
+            "mute-stdout": "^1.0.0",
+            "pretty-hrtime": "^1.0.0",
+            "replace-homedir": "^1.0.0",
+            "semver-greatest-satisfied-range": "^1.1.0",
+            "v8flags": "^3.2.0",
+            "yargs": "^7.1.0"
+          }
+        }
       }
     },
-    "get-caller-file": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz",
-      "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==",
-      "dev": true
+    "gulp-json-transform": {
+      "version": "0.4.6",
+      "resolved": "https://registry.npmjs.org/gulp-json-transform/-/gulp-json-transform-0.4.6.tgz",
+      "integrity": "sha512-laPoNiJP/+lAeiyb0lgY3cynOOi7R/QbPvKBEXJY6bm836nYg90pwY4mgwR7w8nFDlXiCToUeaoQCBIc2NudjA==",
+      "dev": true,
+      "requires": {
+        "ansi-colors": "^1.0.1",
+        "fancy-log": "^1.3.2",
+        "plugin-error": "^1.0.1",
+        "promise": "^8.0.1",
+        "through2": "^2.0.3",
+        "vinyl": "^2.1.0"
+      }
     },
-    "get-pkg-repo": {
+    "gulp-rename": {
       "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/get-pkg-repo/-/get-pkg-repo-1.4.0.tgz",
-      "integrity": "sha1-xztInAbYDMVTbCyFP54FIyBWly0=",
+      "resolved": "https://registry.npmjs.org/gulp-rename/-/gulp-rename-1.4.0.tgz",
+      "integrity": "sha512-swzbIGb/arEoFK89tPY58vg3Ok1bw+d35PfUNwWqdo7KM4jkmuGA78JiDNqR+JeZFaeeHnRg9N7aihX3YPmsyg==",
+      "dev": true
+    },
+    "gulp-sourcemaps": {
+      "version": "2.6.5",
+      "resolved": "https://registry.npmjs.org/gulp-sourcemaps/-/gulp-sourcemaps-2.6.5.tgz",
+      "integrity": "sha512-SYLBRzPTew8T5Suh2U8jCSDKY+4NARua4aqjj8HOysBh2tSgT9u4jc1FYirAdPx1akUxxDeK++fqw6Jg0LkQRg==",
       "dev": true,
       "requires": {
-        "hosted-git-info": "^2.1.4",
-        "meow": "^3.3.0",
-        "normalize-package-data": "^2.3.0",
-        "parse-github-repo-url": "^1.3.0",
-        "through2": "^2.0.0"
+        "@gulp-sourcemaps/identity-map": "1.X",
+        "@gulp-sourcemaps/map-sources": "1.X",
+        "acorn": "5.X",
+        "convert-source-map": "1.X",
+        "css": "2.X",
+        "debug-fabulous": "1.X",
+        "detect-newline": "2.X",
+        "graceful-fs": "4.X",
+        "source-map": "~0.6.0",
+        "strip-bom-string": "1.X",
+        "through2": "2.X"
       },
       "dependencies": {
-        "camelcase": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-2.1.1.tgz",
-          "integrity": "sha1-fB0W1nmhu+WcoCys7PsBHiAfWh8=",
+        "source-map": {
+          "version": "0.6.1",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
           "dev": true
-        },
-        "camelcase-keys": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz",
-          "integrity": "sha1-MIvur/3ygRkFHvodkyITyRuPkuc=",
-          "dev": true,
-          "requires": {
-            "camelcase": "^2.0.0",
-            "map-obj": "^1.0.0"
-          }
-        },
-        "indent-string": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-2.1.0.tgz",
-          "integrity": "sha1-ji1INIdCEhtKghi3oTfppSBJ3IA=",
-          "dev": true,
-          "requires": {
-            "repeating": "^2.0.0"
-          }
-        },
-        "map-obj": {
-          "version": "1.0.1",
-          "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz",
-          "integrity": "sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=",
+        }
+      }
+    },
+    "gulp-typescript": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/gulp-typescript/-/gulp-typescript-5.0.1.tgz",
+      "integrity": "sha512-YuMMlylyJtUSHG1/wuSVTrZp60k1dMEFKYOvDf7OvbAJWrDtxxD4oZon4ancdWwzjj30ztiidhe4VXJniF0pIQ==",
+      "dev": true,
+      "requires": {
+        "ansi-colors": "^3.0.5",
+        "plugin-error": "^1.0.1",
+        "source-map": "^0.7.3",
+        "through2": "^3.0.0",
+        "vinyl": "^2.1.0",
+        "vinyl-fs": "^3.0.3"
+      },
+      "dependencies": {
+        "ansi-colors": {
+          "version": "3.2.4",
+          "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-3.2.4.tgz",
+          "integrity": "sha512-hHUXGagefjN2iRrID63xckIvotOXOojhQKWIPUZ4mNUZ9nLZW+7FMNoE1lOkEhNWYsx/7ysGIuJYCiMAA9FnrA==",
           "dev": true
         },
-        "meow": {
-          "version": "3.7.0",
-          "resolved": "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz",
-          "integrity": "sha1-cstmi0JSKCkKu/qFaJJYcwioAfs=",
-          "dev": true,
-          "requires": {
-            "camelcase-keys": "^2.0.0",
-            "decamelize": "^1.1.2",
-            "loud-rejection": "^1.0.0",
-            "map-obj": "^1.0.1",
-            "minimist": "^1.1.3",
-            "normalize-package-data": "^2.3.4",
-            "object-assign": "^4.0.1",
-            "read-pkg-up": "^1.0.1",
-            "redent": "^1.0.0",
-            "trim-newlines": "^1.0.0"
-          }
-        },
-        "redent": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/redent/-/redent-1.0.0.tgz",
-          "integrity": "sha1-z5Fqsf1fHxbfsggi3W7H9zDCr94=",
-          "dev": true,
-          "requires": {
-            "indent-string": "^2.1.0",
-            "strip-indent": "^1.0.1"
-          }
+        "source-map": {
+          "version": "0.7.3",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz",
+          "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==",
+          "dev": true
         },
-        "strip-indent": {
-          "version": "1.0.1",
-          "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-1.0.1.tgz",
-          "integrity": "sha1-DHlipq3vp7vUrDZkYKY4VSrhoKI=",
+        "through2": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.2.tgz",
+          "integrity": "sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==",
           "dev": true,
           "requires": {
-            "get-stdin": "^4.0.1"
+            "inherits": "^2.0.4",
+            "readable-stream": "2 || 3"
           }
-        },
-        "trim-newlines": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz",
-          "integrity": "sha1-WIeWa7WCpFA6QetST301ARgVphM=",
-          "dev": true
         }
       }
     },
-    "get-port": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/get-port/-/get-port-3.2.0.tgz",
-      "integrity": "sha1-3Xzn3hh8Bsi/NTeWrHHgmfCYDrw=",
-      "dev": true
-    },
-    "get-stdin": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz",
-      "integrity": "sha1-uWjGsKBDhDJJAui/Gl3zJXmkUP4=",
-      "dev": true
-    },
-    "get-stream": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
-      "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=",
-      "dev": true
-    },
-    "get-value": {
-      "version": "2.0.6",
-      "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz",
-      "integrity": "sha1-3BXKHGcjh8p2vTesCjlbogQqLCg=",
-      "dev": true
-    },
-    "getpass": {
-      "version": "0.1.7",
-      "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
-      "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
+    "gulplog": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/gulplog/-/gulplog-1.0.0.tgz",
+      "integrity": "sha1-4oxNRdBey77YGDY86PnFkmIp/+U=",
       "dev": true,
       "requires": {
-        "assert-plus": "^1.0.0"
+        "glogg": "^1.0.0"
       }
     },
-    "git-raw-commits": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/git-raw-commits/-/git-raw-commits-2.0.0.tgz",
-      "integrity": "sha512-w4jFEJFgKXMQJ0H0ikBk2S+4KP2VEjhCvLCNqbNRQC8BgGWgLKNCO7a9K9LI+TVT7Gfoloje502sEnctibffgg==",
+    "handlebars": {
+      "version": "4.7.6",
+      "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.6.tgz",
+      "integrity": "sha512-1f2BACcBfiwAfStCKZNrUCgqNZkGsAT7UM3kkYtXuLo0KnaVfjKOyf7PRzB6++aK9STyT1Pd2ZCPe3EGOXleXA==",
       "dev": true,
       "requires": {
-        "dargs": "^4.0.1",
-        "lodash.template": "^4.0.2",
-        "meow": "^4.0.0",
-        "split2": "^2.0.0",
-        "through2": "^2.0.0"
+        "minimist": "^1.2.5",
+        "neo-async": "^2.6.0",
+        "source-map": "^0.6.1",
+        "uglify-js": "^3.1.4",
+        "wordwrap": "^1.0.0"
+      },
+      "dependencies": {
+        "source-map": {
+          "version": "0.6.1",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+          "dev": true
+        }
       }
     },
-    "git-remote-origin-url": {
+    "har-schema": {
       "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/git-remote-origin-url/-/git-remote-origin-url-2.0.0.tgz",
-      "integrity": "sha1-UoJlna4hBxRaERJhEq0yFuxfpl8=",
+      "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
+      "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=",
+      "dev": true
+    },
+    "har-validator": {
+      "version": "5.1.5",
+      "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz",
+      "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==",
       "dev": true,
       "requires": {
-        "gitconfiglocal": "^1.0.0",
-        "pify": "^2.3.0"
-      },
-      "dependencies": {
-        "pify": {
-          "version": "2.3.0",
-          "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
-          "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=",
-          "dev": true
-        }
+        "ajv": "^6.12.3",
+        "har-schema": "^2.0.0"
       }
     },
-    "git-semver-tags": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/git-semver-tags/-/git-semver-tags-2.0.2.tgz",
-      "integrity": "sha512-34lMF7Yo1xEmsK2EkbArdoU79umpvm0MfzaDkSNYSJqtM5QLAVTPWgpiXSVI5o/O9EvZPSrP4Zvnec/CqhSd5w==",
+    "hard-rejection": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/hard-rejection/-/hard-rejection-2.1.0.tgz",
+      "integrity": "sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==",
+      "dev": true
+    },
+    "has": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
+      "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
       "dev": true,
       "requires": {
-        "meow": "^4.0.0",
-        "semver": "^5.5.0"
+        "function-bind": "^1.1.1"
       }
     },
-    "gitconfiglocal": {
+    "has-flag": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+      "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
+    },
+    "has-symbols": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz",
+      "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==",
+      "dev": true
+    },
+    "has-unicode": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
+      "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=",
+      "dev": true
+    },
+    "has-value": {
       "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/gitconfiglocal/-/gitconfiglocal-1.0.0.tgz",
-      "integrity": "sha1-QdBF84UaXqiPA/JMocYXgRRGS5s=",
+      "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz",
+      "integrity": "sha1-GLKB2lhbHFxR3vJMkw7SmgvmsXc=",
       "dev": true,
       "requires": {
-        "ini": "^1.3.2"
+        "get-value": "^2.0.6",
+        "has-values": "^1.0.0",
+        "isobject": "^3.0.0"
       }
     },
-    "glob": {
-      "version": "7.1.4",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",
-      "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==",
+    "has-values": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/has-values/-/has-values-1.0.0.tgz",
+      "integrity": "sha1-lbC2P+whRmGab+V/51Yo1aOe/k8=",
       "dev": true,
       "requires": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.0.4",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
+        "is-number": "^3.0.0",
+        "kind-of": "^4.0.0"
+      },
+      "dependencies": {
+        "kind-of": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-4.0.0.tgz",
+          "integrity": "sha1-IIE989cSkosgc3hpGkUGb65y3Vc=",
+          "dev": true,
+          "requires": {
+            "is-buffer": "^1.1.5"
+          }
+        }
       }
     },
-    "glob-parent": {
+    "hash-base": {
       "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-3.1.0.tgz",
-      "integrity": "sha1-nmr2KZ2NO9K9QEMIMr0RPfkGxa4=",
+      "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.1.0.tgz",
+      "integrity": "sha512-1nmYp/rhMDiE7AYkDw+lLwlAzz0AntGIe51F3RfFfEqyQ3feY2eI/NcwC6umIQVOASPMsWJLJScWKSSvzL9IVA==",
       "dev": true,
       "requires": {
-        "is-glob": "^3.1.0",
-        "path-dirname": "^1.0.0"
+        "inherits": "^2.0.4",
+        "readable-stream": "^3.6.0",
+        "safe-buffer": "^5.2.0"
       },
       "dependencies": {
-        "is-glob": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-3.1.0.tgz",
-          "integrity": "sha1-e6WuJCF4BKxwcHuWkiVnSGzD6Eo=",
+        "readable-stream": {
+          "version": "3.6.0",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
+          "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
           "dev": true,
           "requires": {
-            "is-extglob": "^2.1.0"
+            "inherits": "^2.0.3",
+            "string_decoder": "^1.1.1",
+            "util-deprecate": "^1.0.1"
           }
         }
       }
     },
-    "glob-stream": {
-      "version": "6.1.0",
-      "resolved": "https://registry.npmjs.org/glob-stream/-/glob-stream-6.1.0.tgz",
-      "integrity": "sha1-cEXJlBOz65SIjYOrRtC0BMx73eQ=",
+    "hash.js": {
+      "version": "1.1.7",
+      "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz",
+      "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==",
       "dev": true,
       "requires": {
-        "extend": "^3.0.0",
-        "glob": "^7.1.1",
-        "glob-parent": "^3.1.0",
-        "is-negated-glob": "^1.0.0",
-        "ordered-read-streams": "^1.0.0",
-        "pumpify": "^1.3.5",
-        "readable-stream": "^2.1.5",
-        "remove-trailing-separator": "^1.0.1",
-        "to-absolute-glob": "^2.0.0",
-        "unique-stream": "^2.0.2"
+        "inherits": "^2.0.3",
+        "minimalistic-assert": "^1.0.1"
       }
     },
-    "glob-to-regexp": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz",
-      "integrity": "sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs=",
+    "highlight.js": {
+      "version": "10.2.0",
+      "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.2.0.tgz",
+      "integrity": "sha512-OryzPiqqNCfO/wtFo619W+nPYALM6u7iCQkum4bqRmmlcTikOkmlL06i009QelynBPAlNByTQU6cBB2cOBQtCw==",
       "dev": true
     },
-    "glob-watcher": {
-      "version": "5.0.3",
-      "resolved": "https://registry.npmjs.org/glob-watcher/-/glob-watcher-5.0.3.tgz",
-      "integrity": "sha512-8tWsULNEPHKQ2MR4zXuzSmqbdyV5PtwwCaWSGQ1WwHsJ07ilNeN1JB8ntxhckbnpSHaf9dXFUHzIWvm1I13dsg==",
+    "hmac-drbg": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz",
+      "integrity": "sha1-0nRXAQJabHdabFRXk+1QL8DGSaE=",
       "dev": true,
       "requires": {
-        "anymatch": "^2.0.0",
-        "async-done": "^1.2.0",
-        "chokidar": "^2.0.0",
-        "is-negated-glob": "^1.0.0",
-        "just-debounce": "^1.0.0",
-        "object.defaults": "^1.1.0"
+        "hash.js": "^1.0.3",
+        "minimalistic-assert": "^1.0.0",
+        "minimalistic-crypto-utils": "^1.0.1"
       }
     },
-    "global-modules": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/global-modules/-/global-modules-1.0.0.tgz",
-      "integrity": "sha512-sKzpEkf11GpOFuw0Zzjzmt4B4UZwjOcG757PPvrfhxcLFbq0wpsgpOqxpxtxFiCG4DtG93M6XRVbF2oGdev7bg==",
+    "homedir-polyfill": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz",
+      "integrity": "sha512-eSmmWE5bZTK2Nou4g0AI3zZ9rswp7GRKoKXS1BLUkvPviOqs4YTN1djQIqrXy9k5gEtdLPy86JjRwsNM9tnDcA==",
       "dev": true,
       "requires": {
-        "global-prefix": "^1.0.1",
-        "is-windows": "^1.0.1",
-        "resolve-dir": "^1.0.0"
+        "parse-passwd": "^1.0.0"
       }
     },
-    "global-prefix": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/global-prefix/-/global-prefix-1.0.2.tgz",
-      "integrity": "sha1-2/dDxsFJklk8ZVVoy2btMsASLr4=",
+    "hosted-git-info": {
+      "version": "2.8.8",
+      "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.8.tgz",
+      "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==",
+      "dev": true
+    },
+    "html-encoding-sniffer": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz",
+      "integrity": "sha512-D5JbOMBIR/TVZkubHT+OyT2705QvogUW4IBn6nHd756OwieSF9aDYFj4dv6HHEVGYbHaLETa3WggZYWWMyy3ZQ==",
       "dev": true,
       "requires": {
-        "expand-tilde": "^2.0.2",
-        "homedir-polyfill": "^1.0.1",
-        "ini": "^1.3.4",
-        "is-windows": "^1.0.1",
-        "which": "^1.2.14"
+        "whatwg-encoding": "^1.0.5"
       }
     },
-    "globals": {
-      "version": "11.12.0",
-      "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz",
-      "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==",
+    "html-escaper": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz",
+      "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==",
       "dev": true
     },
-    "globby": {
-      "version": "6.1.0",
-      "resolved": "https://registry.npmjs.org/globby/-/globby-6.1.0.tgz",
-      "integrity": "sha1-9abXDoOV4hyFj7BInWTfAkJNUGw=",
+    "http-cache-semantics": {
+      "version": "3.8.1",
+      "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz",
+      "integrity": "sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w==",
+      "dev": true
+    },
+    "http-proxy-agent": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-2.1.0.tgz",
+      "integrity": "sha512-qwHbBLV7WviBl0rQsOzH6o5lwyOIvwp/BdFnvVxXORldu5TmjFfjzBcWUWS5kWAZhmv+JtiDhSuQCp4sBfbIgg==",
       "dev": true,
       "requires": {
-        "array-union": "^1.0.1",
-        "glob": "^7.0.3",
-        "object-assign": "^4.0.1",
-        "pify": "^2.0.0",
-        "pinkie-promise": "^2.0.0"
-      },
-      "dependencies": {
-        "pify": {
-          "version": "2.3.0",
-          "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
-          "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=",
-          "dev": true
+        "agent-base": "4",
+        "debug": "3.1.0"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+          "dev": true,
+          "requires": {
+            "ms": "2.0.0"
+          }
         }
       }
     },
-    "glogg": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/glogg/-/glogg-1.0.2.tgz",
-      "integrity": "sha512-5mwUoSuBk44Y4EshyiqcH95ZntbDdTQqA3QYSrxmzj28Ai0vXBGMH1ApSANH14j2sIRtqCEyg6PfsuP7ElOEDA==",
+    "http-signature": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
+      "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=",
       "dev": true,
       "requires": {
-        "sparkles": "^1.0.0"
+        "assert-plus": "^1.0.0",
+        "jsprim": "^1.2.2",
+        "sshpk": "^1.7.0"
       }
     },
-    "google-closure-compiler": {
-      "version": "20190513.0.0",
-      "resolved": "https://registry.npmjs.org/google-closure-compiler/-/google-closure-compiler-20190513.0.0.tgz",
-      "integrity": "sha512-86B1C1yR+Fvd60QFiI/AtMIJb55QCBHzEMEmxOhbzFfdAQ9TpTk3ZFI5NHOwjq2wKlj6tmvHSl5H/5UrdV24Vg==",
+    "https-browserify": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz",
+      "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=",
+      "dev": true
+    },
+    "https-proxy-agent": {
+      "version": "2.2.4",
+      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.4.tgz",
+      "integrity": "sha512-OmvfoQ53WLjtA9HeYP9RNrWMJzzAz1JGaSFr1nijg0PVR1JaD/xbJq1mdEIIlxGpXp9eSe/O2LgU9DJmTPd0Eg==",
       "dev": true,
       "requires": {
-        "chalk": "^1.0.0",
-        "google-closure-compiler-java": "^20190513.0.0",
-        "google-closure-compiler-js": "^20190513.0.0",
-        "google-closure-compiler-linux": "^20190513.0.0",
-        "google-closure-compiler-osx": "^20190513.0.0",
-        "minimist": "^1.2.0",
-        "vinyl": "^2.0.1",
-        "vinyl-sourcemaps-apply": "^0.2.0"
+        "agent-base": "^4.3.0",
+        "debug": "^3.1.0"
       },
       "dependencies": {
-        "ansi-styles": {
-          "version": "2.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
-          "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
-          "dev": true
-        },
-        "chalk": {
-          "version": "1.1.3",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
-          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
+        "debug": {
+          "version": "3.2.6",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
+          "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
           "dev": true,
           "requires": {
-            "ansi-styles": "^2.2.1",
-            "escape-string-regexp": "^1.0.2",
-            "has-ansi": "^2.0.0",
-            "strip-ansi": "^3.0.0",
-            "supports-color": "^2.0.0"
+            "ms": "^2.1.1"
           }
         },
-        "supports-color": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
-          "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=",
+        "ms": {
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
           "dev": true
         }
       }
     },
-    "google-closure-compiler-java": {
-      "version": "20190513.0.0",
-      "resolved": "https://registry.npmjs.org/google-closure-compiler-java/-/google-closure-compiler-java-20190513.0.0.tgz",
-      "integrity": "sha512-0dFZwAREck9W780tVW4AW1c4ztrigLXGJ+dLCXFCwVJZS0pmhmZx3YEByl/aFSU9etIjLwYkYo73xjPyZ/AbtA==",
-      "dev": true
-    },
-    "google-closure-compiler-js": {
-      "version": "20190513.0.0",
-      "resolved": "https://registry.npmjs.org/google-closure-compiler-js/-/google-closure-compiler-js-20190513.0.0.tgz",
-      "integrity": "sha512-BoKdVwowSKUUMGCWVBO9pngeep7B4ngLz5Qjm3VG7KVYZScybworgUjQKi4y8t1vPR8YbhdxQCvNaAqoyh+3ng==",
+    "human-signals": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-1.1.1.tgz",
+      "integrity": "sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw==",
       "dev": true
     },
-    "google-closure-compiler-linux": {
-      "version": "20190513.0.0",
-      "resolved": "https://registry.npmjs.org/google-closure-compiler-linux/-/google-closure-compiler-linux-20190513.0.0.tgz",
-      "integrity": "sha512-YI1EnD3v86g6IrkAQndrdlvHvH1/ACvbH9q33CKVMUpAnzOYh8dZF2J5i4AK2GUALnun8wL/MIk5SKTQBCSbjg==",
+    "humanize-ms": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
+      "integrity": "sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=",
       "dev": true,
-      "optional": true
+      "requires": {
+        "ms": "^2.0.0"
+      }
     },
-    "google-closure-compiler-osx": {
-      "version": "20190513.0.0",
-      "resolved": "https://registry.npmjs.org/google-closure-compiler-osx/-/google-closure-compiler-osx-20190513.0.0.tgz",
-      "integrity": "sha512-q4/e9eGNrl16oSTqgcDR+GK4GBDp27kn+cxICDYKgK/rLV8XmAW5HpI3kJty3mlWhO3SIbP5j4RMQuxqwjtZ2w==",
+    "iconv-lite": {
+      "version": "0.4.24",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
+      "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
       "dev": true,
-      "optional": true
+      "requires": {
+        "safer-buffer": ">= 2.1.2 < 3"
+      }
     },
-    "graceful-fs": {
-      "version": "4.1.15",
-      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz",
-      "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==",
+    "ieee754": {
+      "version": "1.1.13",
+      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
+      "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==",
       "dev": true
     },
-    "growl": {
-      "version": "1.10.5",
-      "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz",
-      "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==",
+    "iferr": {
+      "version": "0.1.5",
+      "resolved": "https://registry.npmjs.org/iferr/-/iferr-0.1.5.tgz",
+      "integrity": "sha1-xg7taebY/bazEEofy8ocGS3FtQE=",
       "dev": true
     },
-    "growly": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/growly/-/growly-1.3.0.tgz",
-      "integrity": "sha1-8QdIy+dq+WS3yWyTxrzCivEgwIE=",
+    "ignore": {
+      "version": "4.0.6",
+      "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz",
+      "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==",
       "dev": true
     },
-    "gulp": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/gulp/-/gulp-4.0.2.tgz",
-      "integrity": "sha512-dvEs27SCZt2ibF29xYgmnwwCYZxdxhQ/+LFWlbAW8y7jt68L/65402Lz3+CKy0Ov4rOs+NERmDq7YlZaDqUIfA==",
+    "ignore-walk": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz",
+      "integrity": "sha512-m7o6xuOaT1aqheYHKf8W6J5pYH85ZI9w077erOzLje3JsB1gkafkAhHHY19dqjulgIZHFm32Cp5uNZgcQqdJKw==",
       "dev": true,
       "requires": {
-        "glob-watcher": "^5.0.3",
-        "gulp-cli": "^2.2.0",
-        "undertaker": "^1.2.1",
-        "vinyl-fs": "^3.0.0"
+        "minimatch": "^3.0.4"
+      }
+    },
+    "import-fresh": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-2.0.0.tgz",
+      "integrity": "sha1-2BNVwVYS04bGH53dOSLUMEgipUY=",
+      "dev": true,
+      "requires": {
+        "caller-path": "^2.0.0",
+        "resolve-from": "^3.0.0"
       },
       "dependencies": {
-        "gulp-cli": {
-          "version": "2.2.0",
-          "resolved": "https://registry.npmjs.org/gulp-cli/-/gulp-cli-2.2.0.tgz",
-          "integrity": "sha512-rGs3bVYHdyJpLqR0TUBnlcZ1O5O++Zs4bA0ajm+zr3WFCfiSLjGwoCBqFs18wzN+ZxahT9DkOK5nDf26iDsWjA==",
-          "dev": true,
-          "requires": {
-            "ansi-colors": "^1.0.1",
-            "archy": "^1.0.0",
-            "array-sort": "^1.0.0",
-            "color-support": "^1.1.3",
-            "concat-stream": "^1.6.0",
-            "copy-props": "^2.0.1",
-            "fancy-log": "^1.3.2",
-            "gulplog": "^1.0.0",
-            "interpret": "^1.1.0",
-            "isobject": "^3.0.1",
-            "liftoff": "^3.1.0",
-            "matchdep": "^2.0.0",
-            "mute-stdout": "^1.0.0",
-            "pretty-hrtime": "^1.0.0",
-            "replace-homedir": "^1.0.0",
-            "semver-greatest-satisfied-range": "^1.1.0",
-            "v8flags": "^3.0.1",
-            "yargs": "^7.1.0"
-          }
+        "resolve-from": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-3.0.0.tgz",
+          "integrity": "sha1-six699nWiBvItuZTM17rywoYh0g=",
+          "dev": true
         }
       }
     },
-    "gulp-json-transform": {
-      "version": "0.4.6",
-      "resolved": "https://registry.npmjs.org/gulp-json-transform/-/gulp-json-transform-0.4.6.tgz",
-      "integrity": "sha512-laPoNiJP/+lAeiyb0lgY3cynOOi7R/QbPvKBEXJY6bm836nYg90pwY4mgwR7w8nFDlXiCToUeaoQCBIc2NudjA==",
+    "import-local": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.0.2.tgz",
+      "integrity": "sha512-vjL3+w0oulAVZ0hBHnxa/Nm5TAurf9YLQJDhqRZyqb+VKGOB6LU8t9H1Nr5CIo16vh9XfJTOoHwU0B71S557gA==",
       "dev": true,
       "requires": {
-        "ansi-colors": "^1.0.1",
-        "fancy-log": "^1.3.2",
-        "plugin-error": "^1.0.1",
-        "promise": "^8.0.1",
-        "through2": "^2.0.3",
-        "vinyl": "^2.1.0"
+        "pkg-dir": "^4.2.0",
+        "resolve-cwd": "^3.0.0"
       }
     },
-    "gulp-rename": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/gulp-rename/-/gulp-rename-1.4.0.tgz",
-      "integrity": "sha512-swzbIGb/arEoFK89tPY58vg3Ok1bw+d35PfUNwWqdo7KM4jkmuGA78JiDNqR+JeZFaeeHnRg9N7aihX3YPmsyg==",
+    "imurmurhash": {
+      "version": "0.1.4",
+      "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
+      "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=",
       "dev": true
     },
-    "gulp-sourcemaps": {
-      "version": "2.6.5",
-      "resolved": "https://registry.npmjs.org/gulp-sourcemaps/-/gulp-sourcemaps-2.6.5.tgz",
-      "integrity": "sha512-SYLBRzPTew8T5Suh2U8jCSDKY+4NARua4aqjj8HOysBh2tSgT9u4jc1FYirAdPx1akUxxDeK++fqw6Jg0LkQRg==",
+    "indent-string": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz",
+      "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==",
+      "dev": true
+    },
+    "infer-owner": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/infer-owner/-/infer-owner-1.0.4.tgz",
+      "integrity": "sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==",
+      "dev": true
+    },
+    "inflight": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+      "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
       "dev": true,
       "requires": {
-        "@gulp-sourcemaps/identity-map": "1.X",
-        "@gulp-sourcemaps/map-sources": "1.X",
-        "acorn": "5.X",
-        "convert-source-map": "1.X",
-        "css": "2.X",
-        "debug-fabulous": "1.X",
-        "detect-newline": "2.X",
-        "graceful-fs": "4.X",
-        "source-map": "~0.6.0",
-        "strip-bom-string": "1.X",
-        "through2": "2.X"
-      },
-      "dependencies": {
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-          "dev": true
-        }
+        "once": "^1.3.0",
+        "wrappy": "1"
       }
     },
-    "gulp-typescript": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/gulp-typescript/-/gulp-typescript-5.0.1.tgz",
-      "integrity": "sha512-YuMMlylyJtUSHG1/wuSVTrZp60k1dMEFKYOvDf7OvbAJWrDtxxD4oZon4ancdWwzjj30ztiidhe4VXJniF0pIQ==",
+    "inherits": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
+      "dev": true
+    },
+    "ini": {
+      "version": "1.3.5",
+      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
+      "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==",
+      "dev": true
+    },
+    "init-package-json": {
+      "version": "1.10.3",
+      "resolved": "https://registry.npmjs.org/init-package-json/-/init-package-json-1.10.3.tgz",
+      "integrity": "sha512-zKSiXKhQveNteyhcj1CoOP8tqp1QuxPIPBl8Bid99DGLFqA1p87M6lNgfjJHSBoWJJlidGOv5rWjyYKEB3g2Jw==",
       "dev": true,
       "requires": {
-        "ansi-colors": "^3.0.5",
-        "plugin-error": "^1.0.1",
-        "source-map": "^0.7.3",
-        "through2": "^3.0.0",
-        "vinyl": "^2.1.0",
-        "vinyl-fs": "^3.0.3"
+        "glob": "^7.1.1",
+        "npm-package-arg": "^4.0.0 || ^5.0.0 || ^6.0.0",
+        "promzard": "^0.3.0",
+        "read": "~1.0.1",
+        "read-package-json": "1 || 2",
+        "semver": "2.x || 3.x || 4 || 5",
+        "validate-npm-package-license": "^3.0.1",
+        "validate-npm-package-name": "^3.0.0"
+      }
+    },
+    "inquirer": {
+      "version": "6.5.2",
+      "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.5.2.tgz",
+      "integrity": "sha512-cntlB5ghuB0iuO65Ovoi8ogLHiWGs/5yNrtUcKjFhSSiVeAIVpD7koaSU9RM8mpXw5YDi9RdYXGQMaOURB7ycQ==",
+      "dev": true,
+      "requires": {
+        "ansi-escapes": "^3.2.0",
+        "chalk": "^2.4.2",
+        "cli-cursor": "^2.1.0",
+        "cli-width": "^2.0.0",
+        "external-editor": "^3.0.3",
+        "figures": "^2.0.0",
+        "lodash": "^4.17.12",
+        "mute-stream": "0.0.7",
+        "run-async": "^2.2.0",
+        "rxjs": "^6.4.0",
+        "string-width": "^2.1.0",
+        "strip-ansi": "^5.1.0",
+        "through": "^2.3.6"
       },
       "dependencies": {
-        "ansi-colors": {
-          "version": "3.2.4",
-          "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-3.2.4.tgz",
-          "integrity": "sha512-hHUXGagefjN2iRrID63xckIvotOXOojhQKWIPUZ4mNUZ9nLZW+7FMNoE1lOkEhNWYsx/7ysGIuJYCiMAA9FnrA==",
+        "ansi-escapes": {
+          "version": "3.2.0",
+          "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz",
+          "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==",
           "dev": true
         },
-        "source-map": {
-          "version": "0.7.3",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz",
-          "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==",
+        "ansi-regex": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
+          "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=",
+          "dev": true
+        },
+        "is-fullwidth-code-point": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
+          "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=",
           "dev": true
         },
-        "through2": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz",
-          "integrity": "sha512-M96dvTalPT3YbYLaKaCuwu+j06D/8Jfib0o/PxbVt6Amhv3dUAtW6rTV1jPgJSBG83I/e04Y6xkVdVhSRhi0ww==",
+        "rxjs": {
+          "version": "6.6.3",
+          "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.6.3.tgz",
+          "integrity": "sha512-trsQc+xYYXZ3urjOiJOuCOa5N3jAZ3eiSpQB5hIT8zGlL2QfnHLJ2r7GMkBGuIausdJN1OneaI6gQlsqNHHmZQ==",
+          "dev": true,
+          "requires": {
+            "tslib": "^1.9.0"
+          }
+        },
+        "string-width": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",
+          "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==",
           "dev": true,
           "requires": {
-            "readable-stream": "2 || 3"
+            "is-fullwidth-code-point": "^2.0.0",
+            "strip-ansi": "^4.0.0"
+          },
+          "dependencies": {
+            "strip-ansi": {
+              "version": "4.0.0",
+              "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
+              "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
+              "dev": true,
+              "requires": {
+                "ansi-regex": "^3.0.0"
+              }
+            }
+          }
+        },
+        "strip-ansi": {
+          "version": "5.2.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz",
+          "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==",
+          "dev": true,
+          "requires": {
+            "ansi-regex": "^4.1.0"
+          },
+          "dependencies": {
+            "ansi-regex": {
+              "version": "4.1.0",
+              "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz",
+              "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==",
+              "dev": true
+            }
           }
         }
       }
     },
-    "gulplog": {
+    "interpret": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz",
+      "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==",
+      "dev": true
+    },
+    "invert-kv": {
       "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/gulplog/-/gulplog-1.0.0.tgz",
-      "integrity": "sha1-4oxNRdBey77YGDY86PnFkmIp/+U=",
+      "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz",
+      "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY=",
+      "dev": true
+    },
+    "ip": {
+      "version": "1.1.5",
+      "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz",
+      "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=",
+      "dev": true
+    },
+    "ip-regex": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz",
+      "integrity": "sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk=",
+      "dev": true
+    },
+    "is-absolute": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-absolute/-/is-absolute-1.0.0.tgz",
+      "integrity": "sha512-dOWoqflvcydARa360Gvv18DZ/gRuHKi2NU/wU5X1ZFzdYfH29nkiNZsF3mp4OJ3H4yo9Mx8A/uAGNzpzPN3yBA==",
       "dev": true,
       "requires": {
-        "glogg": "^1.0.0"
+        "is-relative": "^1.0.0",
+        "is-windows": "^1.0.1"
       }
     },
-    "handlebars": {
-      "version": "4.0.12",
-      "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.0.12.tgz",
-      "integrity": "sha512-RhmTekP+FZL+XNhwS1Wf+bTTZpdLougwt5pcgA1tuz6Jcx0fpH/7z0qd71RKnZHBCxIRBHfBOnio4gViPemNzA==",
+    "is-accessor-descriptor": {
+      "version": "0.1.6",
+      "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
+      "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
       "dev": true,
       "requires": {
-        "async": "^2.5.0",
-        "optimist": "^0.6.1",
-        "source-map": "^0.6.1",
-        "uglify-js": "^3.1.4"
+        "kind-of": "^3.0.2"
       },
       "dependencies": {
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-          "dev": true
+        "kind-of": {
+          "version": "3.2.2",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
+          "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
+          "dev": true,
+          "requires": {
+            "is-buffer": "^1.1.5"
+          }
         }
       }
     },
-    "har-schema": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
-      "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=",
+    "is-arrayish": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz",
+      "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=",
       "dev": true
     },
-    "har-validator": {
-      "version": "5.1.3",
-      "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz",
-      "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==",
-      "dev": true,
-      "requires": {
-        "ajv": "^6.5.5",
-        "har-schema": "^2.0.0"
-      }
-    },
-    "has": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
-      "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
-      "dev": true,
-      "requires": {
-        "function-bind": "^1.1.1"
-      }
-    },
-    "has-ansi": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
-      "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
+    "is-binary-path": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-1.0.1.tgz",
+      "integrity": "sha1-dfFmQrSA8YenEcgUFh/TpKdlWJg=",
       "dev": true,
       "requires": {
-        "ansi-regex": "^2.0.0"
+        "binary-extensions": "^1.0.0"
       }
     },
-    "has-flag": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-      "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
-    },
-    "has-symbols": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz",
-      "integrity": "sha1-uhqPGvKg/DllD1yFA2dwQSIGO0Q=",
+    "is-buffer": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
+      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
       "dev": true
     },
-    "has-unicode": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
-      "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=",
+    "is-callable": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.2.tgz",
+      "integrity": "sha512-dnMqspv5nU3LoewK2N/y7KLtxtakvTuaCsU9FU50/QDmdbHNy/4/JuRtMHqRU22o3q+W89YQndQEeCVwK+3qrA==",
       "dev": true
     },
-    "has-value": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz",
-      "integrity": "sha1-GLKB2lhbHFxR3vJMkw7SmgvmsXc=",
+    "is-ci": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz",
+      "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==",
       "dev": true,
       "requires": {
-        "get-value": "^2.0.6",
-        "has-values": "^1.0.0",
-        "isobject": "^3.0.0"
+        "ci-info": "^2.0.0"
       }
     },
-    "has-values": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/has-values/-/has-values-1.0.0.tgz",
-      "integrity": "sha1-lbC2P+whRmGab+V/51Yo1aOe/k8=",
+    "is-data-descriptor": {
+      "version": "0.1.4",
+      "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
+      "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
       "dev": true,
       "requires": {
-        "is-number": "^3.0.0",
-        "kind-of": "^4.0.0"
+        "kind-of": "^3.0.2"
       },
       "dependencies": {
         "kind-of": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-4.0.0.tgz",
-          "integrity": "sha1-IIE989cSkosgc3hpGkUGb65y3Vc=",
+          "version": "3.2.2",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
+          "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
           "dev": true,
           "requires": {
             "is-buffer": "^1.1.5"
@@ -6638,1925 +8801,2738 @@
         }
       }
     },
-    "hash-base": {
-      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.0.4.tgz",
-      "integrity": "sha1-X8hoaEfs1zSZQDMZprCj8/auSRg=",
-      "dev": true,
-      "requires": {
-        "inherits": "^2.0.1",
-        "safe-buffer": "^5.0.1"
-      }
+    "is-date-object": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.2.tgz",
+      "integrity": "sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g==",
+      "dev": true
     },
-    "hash.js": {
-      "version": "1.1.7",
-      "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz",
-      "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==",
+    "is-descriptor": {
+      "version": "0.1.6",
+      "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
+      "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
       "dev": true,
       "requires": {
-        "inherits": "^2.0.3",
-        "minimalistic-assert": "^1.0.1"
+        "is-accessor-descriptor": "^0.1.6",
+        "is-data-descriptor": "^0.1.4",
+        "kind-of": "^5.0.0"
+      },
+      "dependencies": {
+        "kind-of": {
+          "version": "5.1.0",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
+          "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==",
+          "dev": true
+        }
       }
     },
-    "highlight.js": {
-      "version": "9.15.8",
-      "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-9.15.8.tgz",
-      "integrity": "sha512-RrapkKQWwE+wKdF73VsOa2RQdIoO3mxwJ4P8mhbI6KYJUraUHRKM5w5zQQKXNk0xNL4UVRdulV9SBJcmzJNzVA==",
+    "is-directory": {
+      "version": "0.3.1",
+      "resolved": "https://registry.npmjs.org/is-directory/-/is-directory-0.3.1.tgz",
+      "integrity": "sha1-YTObbyR1/Hcv2cnYP1yFddwVSuE=",
       "dev": true
     },
-    "hmac-drbg": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz",
-      "integrity": "sha1-0nRXAQJabHdabFRXk+1QL8DGSaE=",
-      "dev": true,
-      "requires": {
-        "hash.js": "^1.0.3",
-        "minimalistic-assert": "^1.0.0",
-        "minimalistic-crypto-utils": "^1.0.1"
-      }
-    },
-    "homedir-polyfill": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz",
-      "integrity": "sha512-eSmmWE5bZTK2Nou4g0AI3zZ9rswp7GRKoKXS1BLUkvPviOqs4YTN1djQIqrXy9k5gEtdLPy86JjRwsNM9tnDcA==",
+    "is-docker": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.1.1.tgz",
+      "integrity": "sha512-ZOoqiXfEwtGknTiuDEy8pN2CfE3TxMHprvNer1mXiqwkOT77Rw3YVrUQ52EqAOU3QAWDQ+bQdx7HJzrv7LS2Hw==",
       "dev": true,
-      "requires": {
-        "parse-passwd": "^1.0.0"
-      }
+      "optional": true
     },
-    "hosted-git-info": {
-      "version": "2.7.1",
-      "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.7.1.tgz",
-      "integrity": "sha512-7T/BxH19zbcCTa8XkMlbK5lTo1WtgkFi3GvdWEyNuc4Vex7/9Dqbnpsf4JMydcfj9HCg4zUWFTL3Za6lapg5/w==",
+    "is-extendable": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz",
+      "integrity": "sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik=",
       "dev": true
     },
-    "html-encoding-sniffer": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-1.0.2.tgz",
-      "integrity": "sha512-71lZziiDnsuabfdYiUeWdCVyKuqwWi23L8YeIgV9jSSZHCtb6wB1BKWooH7L3tn4/FuZJMVWyNaIDr4RGmaSYw==",
-      "dev": true,
-      "requires": {
-        "whatwg-encoding": "^1.0.1"
-      }
+    "is-extglob": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
+      "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=",
+      "dev": true
     },
-    "http-cache-semantics": {
-      "version": "3.8.1",
-      "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz",
-      "integrity": "sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w==",
+    "is-finite": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/is-finite/-/is-finite-1.1.0.tgz",
+      "integrity": "sha512-cdyMtqX/BOqqNBBiKlIVkytNHm49MtMlYyn1zxzvJKWmFMlGzm+ry5BBfYyeY9YmNKbRSo/o7OX9w9ale0wg3w==",
       "dev": true
     },
-    "http-proxy-agent": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-2.1.0.tgz",
-      "integrity": "sha512-qwHbBLV7WviBl0rQsOzH6o5lwyOIvwp/BdFnvVxXORldu5TmjFfjzBcWUWS5kWAZhmv+JtiDhSuQCp4sBfbIgg==",
-      "dev": true,
-      "requires": {
-        "agent-base": "4",
-        "debug": "3.1.0"
-      },
-      "dependencies": {
-        "debug": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
-          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
-          "dev": true,
-          "requires": {
-            "ms": "2.0.0"
-          }
-        }
+    "is-fullwidth-code-point": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
+      "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
+      "dev": true,
+      "requires": {
+        "number-is-nan": "^1.0.0"
       }
     },
-    "http-signature": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
-      "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=",
+    "is-generator-fn": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz",
+      "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==",
+      "dev": true
+    },
+    "is-glob": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz",
+      "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==",
       "dev": true,
       "requires": {
-        "assert-plus": "^1.0.0",
-        "jsprim": "^1.2.2",
-        "sshpk": "^1.7.0"
+        "is-extglob": "^2.1.1"
       }
     },
-    "https-browserify": {
+    "is-negated-glob": {
       "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz",
-      "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=",
+      "resolved": "https://registry.npmjs.org/is-negated-glob/-/is-negated-glob-1.0.0.tgz",
+      "integrity": "sha1-aRC8pdqMleeEtXUbl2z1oQ/uNtI=",
       "dev": true
     },
-    "https-proxy-agent": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz",
-      "integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==",
+    "is-negative-zero": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.0.tgz",
+      "integrity": "sha1-lVOxIbD6wohp2p7UWeIMdUN4hGE=",
+      "dev": true
+    },
+    "is-number": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz",
+      "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=",
       "dev": true,
       "requires": {
-        "agent-base": "^4.1.0",
-        "debug": "^3.1.0"
+        "kind-of": "^3.0.2"
       },
       "dependencies": {
-        "debug": {
-          "version": "3.2.6",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
-          "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
+        "kind-of": {
+          "version": "3.2.2",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
+          "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
           "dev": true,
           "requires": {
-            "ms": "^2.1.1"
+            "is-buffer": "^1.1.5"
           }
-        },
-        "ms": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
-          "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==",
-          "dev": true
         }
       }
     },
-    "humanize-ms": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
-      "integrity": "sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=",
+    "is-obj": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz",
+      "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=",
+      "dev": true
+    },
+    "is-path-cwd": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-1.0.0.tgz",
+      "integrity": "sha1-0iXsIxMuie3Tj9p2dHLmLmXxEG0=",
+      "dev": true
+    },
+    "is-path-in-cwd": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-path-in-cwd/-/is-path-in-cwd-1.0.1.tgz",
+      "integrity": "sha512-FjV1RTW48E7CWM7eE/J2NJvAEEVektecDBVBE5Hh3nM1Jd0kvhHtX68Pr3xsDf857xt3Y4AkwVULK1Vku62aaQ==",
       "dev": true,
       "requires": {
-        "ms": "^2.0.0"
+        "is-path-inside": "^1.0.0"
       }
     },
-    "iconv-lite": {
-      "version": "0.4.24",
-      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
-      "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
+    "is-path-inside": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-1.0.1.tgz",
+      "integrity": "sha1-jvW33lBDej/cprToZe96pVy0gDY=",
       "dev": true,
       "requires": {
-        "safer-buffer": ">= 2.1.2 < 3"
+        "path-is-inside": "^1.0.1"
       }
     },
-    "ieee754": {
-      "version": "1.1.12",
-      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.12.tgz",
-      "integrity": "sha512-GguP+DRY+pJ3soyIiGPTvdiVXjZ+DbXOxGpXn3eMvNW4x4irjqXm4wHKscC+TfxSJ0yw/S1F24tqdMNsMZTiLA==",
+    "is-plain-obj": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz",
+      "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=",
       "dev": true
     },
-    "iferr": {
-      "version": "0.1.5",
-      "resolved": "https://registry.npmjs.org/iferr/-/iferr-0.1.5.tgz",
-      "integrity": "sha1-xg7taebY/bazEEofy8ocGS3FtQE=",
+    "is-plain-object": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
+      "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
+      "dev": true,
+      "requires": {
+        "isobject": "^3.0.1"
+      }
+    },
+    "is-potential-custom-element-name": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.0.tgz",
+      "integrity": "sha1-DFLlS8yjkbssSUsh6GJtczbG45c=",
       "dev": true
     },
-    "ignore": {
-      "version": "3.3.10",
-      "resolved": "https://registry.npmjs.org/ignore/-/ignore-3.3.10.tgz",
-      "integrity": "sha512-Pgs951kaMm5GXP7MOvxERINe3gsaVjUWFm+UZPSq9xYriQAksyhg0csnS0KXSNRD5NmNdapXEpjxG49+AKh/ug==",
+    "is-promise": {
+      "version": "2.2.2",
+      "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.2.2.tgz",
+      "integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==",
       "dev": true
     },
-    "ignore-walk": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.1.tgz",
-      "integrity": "sha512-DTVlMx3IYPe0/JJcYP7Gxg7ttZZu3IInhuEhbchuqneY9wWe5Ojy2mXLBaQFUQmo0AW2r3qG7m1mg86js+gnlQ==",
+    "is-regex": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.1.tgz",
+      "integrity": "sha512-1+QkEcxiLlB7VEyFtyBg94e08OAsvq7FUBgApTq/w2ymCLyKJgDPsybBENVtA7XCQEgEXxKPonG+mvYRxh/LIg==",
       "dev": true,
       "requires": {
-        "minimatch": "^3.0.4"
+        "has-symbols": "^1.0.1"
       }
     },
-    "import-fresh": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-2.0.0.tgz",
-      "integrity": "sha1-2BNVwVYS04bGH53dOSLUMEgipUY=",
+    "is-relative": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-relative/-/is-relative-1.0.0.tgz",
+      "integrity": "sha512-Kw/ReK0iqwKeu0MITLFuj0jbPAmEiOsIwyIXvvbfa6QfmN9pkD1M+8pdk7Rl/dTKbH34/XBFMbgD4iMJhLQbGA==",
       "dev": true,
       "requires": {
-        "caller-path": "^2.0.0",
-        "resolve-from": "^3.0.0"
+        "is-unc-path": "^1.0.0"
       }
     },
-    "import-local": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/import-local/-/import-local-1.0.0.tgz",
-      "integrity": "sha512-vAaZHieK9qjGo58agRBg+bhHX3hoTZU/Oa3GESWLz7t1U62fk63aHuDJJEteXoDeTCcPmUT+z38gkHPZkkmpmQ==",
+    "is-ssh": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/is-ssh/-/is-ssh-1.3.2.tgz",
+      "integrity": "sha512-elEw0/0c2UscLrNG+OAorbP539E3rhliKPg+hDMWN9VwrDXfYK+4PBEykDPfxlYYtQvl84TascnQyobfQLHEhQ==",
       "dev": true,
       "requires": {
-        "pkg-dir": "^2.0.0",
-        "resolve-cwd": "^2.0.0"
+        "protocols": "^1.1.0"
       }
     },
-    "imurmurhash": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
-      "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=",
-      "dev": true
-    },
-    "indent-string": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz",
-      "integrity": "sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok=",
-      "dev": true
-    },
-    "indexof": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/indexof/-/indexof-0.0.1.tgz",
-      "integrity": "sha1-gtwzbSMrkGIXnQWrMpOmYFn9Q10=",
+    "is-stream": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
+      "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=",
       "dev": true
     },
-    "inflight": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
-      "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
+    "is-symbol": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz",
+      "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==",
       "dev": true,
       "requires": {
-        "once": "^1.3.0",
-        "wrappy": "1"
+        "has-symbols": "^1.0.1"
       }
     },
-    "inherits": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
-      "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=",
-      "dev": true
-    },
-    "ini": {
-      "version": "1.3.5",
-      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
-      "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==",
-      "dev": true
-    },
-    "init-package-json": {
-      "version": "1.10.3",
-      "resolved": "https://registry.npmjs.org/init-package-json/-/init-package-json-1.10.3.tgz",
-      "integrity": "sha512-zKSiXKhQveNteyhcj1CoOP8tqp1QuxPIPBl8Bid99DGLFqA1p87M6lNgfjJHSBoWJJlidGOv5rWjyYKEB3g2Jw==",
+    "is-text-path": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-text-path/-/is-text-path-1.0.1.tgz",
+      "integrity": "sha1-Thqg+1G/vLPpJogAE5cgLBd1tm4=",
       "dev": true,
       "requires": {
-        "glob": "^7.1.1",
-        "npm-package-arg": "^4.0.0 || ^5.0.0 || ^6.0.0",
-        "promzard": "^0.3.0",
-        "read": "~1.0.1",
-        "read-package-json": "1 || 2",
-        "semver": "2.x || 3.x || 4 || 5",
-        "validate-npm-package-license": "^3.0.1",
-        "validate-npm-package-name": "^3.0.0"
+        "text-extensions": "^1.0.0"
       }
     },
-    "inquirer": {
-      "version": "6.2.1",
-      "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.2.1.tgz",
-      "integrity": "sha512-088kl3DRT2dLU5riVMKKr1DlImd6X7smDhpXUCkJDCKvTEJeRiXh0G132HG9u5a+6Ylw9plFRY7RuTnwohYSpg==",
+    "is-typedarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
+      "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=",
+      "dev": true
+    },
+    "is-unc-path": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-unc-path/-/is-unc-path-1.0.0.tgz",
+      "integrity": "sha512-mrGpVd0fs7WWLfVsStvgF6iEJnbjDFZh9/emhRDcGWTduTfNHd9CHeUwH3gYIjdbwo4On6hunkztwOaAw0yllQ==",
       "dev": true,
       "requires": {
-        "ansi-escapes": "^3.0.0",
-        "chalk": "^2.0.0",
-        "cli-cursor": "^2.1.0",
-        "cli-width": "^2.0.0",
-        "external-editor": "^3.0.0",
-        "figures": "^2.0.0",
-        "lodash": "^4.17.10",
-        "mute-stream": "0.0.7",
-        "run-async": "^2.2.0",
-        "rxjs": "^6.1.0",
-        "string-width": "^2.1.0",
-        "strip-ansi": "^5.0.0",
-        "through": "^2.3.6"
-      },
-      "dependencies": {
-        "ansi-regex": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
-          "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=",
-          "dev": true
-        },
-        "is-fullwidth-code-point": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
-          "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=",
-          "dev": true
-        },
-        "rxjs": {
-          "version": "6.3.3",
-          "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.3.3.tgz",
-          "integrity": "sha512-JTWmoY9tWCs7zvIk/CvRjhjGaOd+OVBM987mxFo+OW66cGpdKjZcpmc74ES1sB//7Kl/PAe8+wEakuhG4pcgOw==",
-          "dev": true,
-          "requires": {
-            "tslib": "^1.9.0"
-          }
-        },
-        "string-width": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",
-          "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==",
-          "dev": true,
-          "requires": {
-            "is-fullwidth-code-point": "^2.0.0",
-            "strip-ansi": "^4.0.0"
-          },
-          "dependencies": {
-            "strip-ansi": {
-              "version": "4.0.0",
-              "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
-              "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
-              "dev": true,
-              "requires": {
-                "ansi-regex": "^3.0.0"
-              }
-            }
-          }
-        },
-        "strip-ansi": {
-          "version": "5.0.0",
-          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.0.0.tgz",
-          "integrity": "sha512-Uu7gQyZI7J7gn5qLn1Np3G9vcYGTVqB+lFTytnDJv83dd8T22aGH451P3jueT2/QemInJDfxHB5Tde5OzgG1Ow==",
-          "dev": true,
-          "requires": {
-            "ansi-regex": "^4.0.0"
-          },
-          "dependencies": {
-            "ansi-regex": {
-              "version": "4.0.0",
-              "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.0.0.tgz",
-              "integrity": "sha512-iB5Dda8t/UqpPI/IjsejXu5jOGDrzn41wJyljwPH65VCIbk6+1BzFIMJGFwTNrYXT1CrD+B4l19U7awiQ8rk7w==",
-              "dev": true
-            }
-          }
-        }
+        "unc-path-regex": "^0.1.2"
       }
     },
-    "interpret": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.2.0.tgz",
-      "integrity": "sha512-mT34yGKMNceBQUoVn7iCDKDntA7SC6gycMAWzGx1z/CMCTV7b2AAtXlo3nRyHZ1FelRkQbQjprHSYGwzLtkVbw==",
+    "is-utf8": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz",
+      "integrity": "sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI=",
+      "dev": true
+    },
+    "is-valid-glob": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-valid-glob/-/is-valid-glob-1.0.0.tgz",
+      "integrity": "sha1-Kb8+/3Ab4tTTFdusw5vDn+j2Aao=",
+      "dev": true
+    },
+    "is-windows": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz",
+      "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==",
       "dev": true
     },
-    "invariant": {
-      "version": "2.2.4",
-      "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz",
-      "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==",
+    "is-wsl": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz",
+      "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==",
       "dev": true,
+      "optional": true,
       "requires": {
-        "loose-envify": "^1.0.0"
+        "is-docker": "^2.0.0"
       }
     },
-    "invert-kv": {
+    "isarray": {
       "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz",
-      "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY=",
+      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
       "dev": true
     },
-    "ip": {
-      "version": "1.1.5",
-      "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz",
-      "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=",
+    "isexe": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
+      "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
       "dev": true
     },
-    "is-absolute": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-absolute/-/is-absolute-1.0.0.tgz",
-      "integrity": "sha512-dOWoqflvcydARa360Gvv18DZ/gRuHKi2NU/wU5X1ZFzdYfH29nkiNZsF3mp4OJ3H4yo9Mx8A/uAGNzpzPN3yBA==",
+    "isobject": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
+      "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=",
+      "dev": true
+    },
+    "isstream": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
+      "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=",
+      "dev": true
+    },
+    "istanbul-lib-coverage": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.0.0.tgz",
+      "integrity": "sha512-UiUIqxMgRDET6eR+o5HbfRYP1l0hqkWOs7vNxC/mggutCMUIhWMm8gAHb8tHlyfD3/l6rlgNA5cKdDzEAf6hEg==",
+      "dev": true
+    },
+    "istanbul-lib-instrument": {
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-4.0.3.tgz",
+      "integrity": "sha512-BXgQl9kf4WTCPCCpmFGoJkz/+uhvm7h7PFKUYxh7qarQd3ER33vHG//qaE8eN25l07YqZPpHXU9I09l/RD5aGQ==",
       "dev": true,
       "requires": {
-        "is-relative": "^1.0.0",
-        "is-windows": "^1.0.1"
+        "@babel/core": "^7.7.5",
+        "@istanbuljs/schema": "^0.1.2",
+        "istanbul-lib-coverage": "^3.0.0",
+        "semver": "^6.3.0"
+      },
+      "dependencies": {
+        "semver": {
+          "version": "6.3.0",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
+          "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
+          "dev": true
+        }
       }
     },
-    "is-accessor-descriptor": {
-      "version": "0.1.6",
-      "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
-      "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
+    "istanbul-lib-report": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz",
+      "integrity": "sha512-wcdi+uAKzfiGT2abPpKZ0hSU1rGQjUQnLvtY5MpQ7QCTahD3VODhcu4wcfY1YtkGaDD5yuydOLINXsfbus9ROw==",
       "dev": true,
       "requires": {
-        "kind-of": "^3.0.2"
+        "istanbul-lib-coverage": "^3.0.0",
+        "make-dir": "^3.0.0",
+        "supports-color": "^7.1.0"
       },
       "dependencies": {
-        "kind-of": {
-          "version": "3.2.2",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-          "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
           "dev": true,
           "requires": {
-            "is-buffer": "^1.1.5"
+            "has-flag": "^4.0.0"
           }
         }
       }
     },
-    "is-arrayish": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz",
-      "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=",
-      "dev": true
-    },
-    "is-binary-path": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-1.0.1.tgz",
-      "integrity": "sha1-dfFmQrSA8YenEcgUFh/TpKdlWJg=",
+    "istanbul-lib-source-maps": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.0.tgz",
+      "integrity": "sha512-c16LpFRkR8vQXyHZ5nLpY35JZtzj1PQY1iZmesUbf1FZHbIupcWfjgOXBY9YHkLEQ6puz1u4Dgj6qmU/DisrZg==",
       "dev": true,
       "requires": {
-        "binary-extensions": "^1.0.0"
+        "debug": "^4.1.1",
+        "istanbul-lib-coverage": "^3.0.0",
+        "source-map": "^0.6.1"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "4.2.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
+          "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
+          "dev": true,
+          "requires": {
+            "ms": "2.1.2"
+          }
+        },
+        "ms": {
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+          "dev": true
+        },
+        "source-map": {
+          "version": "0.6.1",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+          "dev": true
+        }
       }
     },
-    "is-buffer": {
-      "version": "1.1.6",
-      "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
-      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
-      "dev": true
-    },
-    "is-builtin-module": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz",
-      "integrity": "sha1-VAVy0096wxGfj3bDDLwbHgN6/74=",
+    "istanbul-reports": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.0.2.tgz",
+      "integrity": "sha512-9tZvz7AiR3PEDNGiV9vIouQ/EAcqMXFmkcA1CDFTwOB98OZVDL0PH9glHotf5Ugp6GCOTypfzGWI/OqjWNCRUw==",
       "dev": true,
       "requires": {
-        "builtin-modules": "^1.0.0"
+        "html-escaper": "^2.0.0",
+        "istanbul-lib-report": "^3.0.0"
       }
     },
-    "is-callable": {
-      "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.4.tgz",
-      "integrity": "sha512-r5p9sxJjYnArLjObpjA4xu5EKI3CuKHkJXMhT7kwbpUyIFD1n5PMAsoPvWnvtZiNz7LjkYDRZhd7FlI0eMijEA==",
-      "dev": true
-    },
-    "is-ci": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-1.2.1.tgz",
-      "integrity": "sha512-s6tfsaQaQi3JNciBH6shVqEDvhGut0SUXr31ag8Pd8BBbVVlcGfWhpPmEOoM6RJ5TFhbypvf5yyRw/VXW1IiWg==",
+    "ix": {
+      "version": "2.5.3",
+      "resolved": "https://registry.npmjs.org/ix/-/ix-2.5.3.tgz",
+      "integrity": "sha512-Syz3oYUx5+P0T9F3BZOE2DzvWYF31Mbha1VqDbJsr4e4A+twhxTAf2c1BO8TMlwon12oAvvxAtQ+NJm9P8ISYQ==",
       "dev": true,
       "requires": {
-        "ci-info": "^1.5.0"
+        "@types/node": "^11.11.6",
+        "tslib": "^1.9.3"
+      },
+      "dependencies": {
+        "@types/node": {
+          "version": "11.15.27",
+          "resolved": "https://registry.npmjs.org/@types/node/-/node-11.15.27.tgz",
+          "integrity": "sha512-LbLwyGC/ukDV0EbHFP1OCfs2V5h3vUS8ZXJJjS2L5YYg8rNkJe6Tl/yv+L+g94sbHllyXUCfUCn5+sZLBegvyw==",
+          "dev": true
+        }
       }
     },
-    "is-data-descriptor": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
-      "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
+    "jest": {
+      "version": "26.3.0",
+      "resolved": "https://registry.npmjs.org/jest/-/jest-26.3.0.tgz",
+      "integrity": "sha512-LFCry7NS6bTa4BUGUHC+NvZ3B9WG7Jv8F+Lb96dAJFM23LMwSsL5RiJcw9S+nejsh8lS1VxHq+RSH4Xa9tujpA==",
       "dev": true,
       "requires": {
-        "kind-of": "^3.0.2"
+        "@jest/core": "^26.3.0",
+        "import-local": "^3.0.2",
+        "jest-cli": "^26.3.0"
       },
       "dependencies": {
-        "kind-of": {
-          "version": "3.2.2",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-          "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
           "dev": true,
           "requires": {
-            "is-buffer": "^1.1.5"
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
+          }
+        },
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-report": "*"
+          }
+        },
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
+        },
+        "camelcase": {
+          "version": "5.3.1",
+          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
+          "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==",
+          "dev": true
+        },
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
+        },
+        "cliui": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz",
+          "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==",
+          "dev": true,
+          "requires": {
+            "string-width": "^4.2.0",
+            "strip-ansi": "^6.0.0",
+            "wrap-ansi": "^6.2.0"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "find-up": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz",
+          "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==",
+          "dev": true,
+          "requires": {
+            "locate-path": "^5.0.0",
+            "path-exists": "^4.0.0"
+          }
+        },
+        "get-caller-file": {
+          "version": "2.0.5",
+          "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
+          "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
+          "dev": true
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "is-fullwidth-code-point": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+          "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+          "dev": true
+        },
+        "jest-cli": {
+          "version": "26.4.2",
+          "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-26.4.2.tgz",
+          "integrity": "sha512-zb+lGd/SfrPvoRSC/0LWdaWCnscXc1mGYW//NP4/tmBvRPT3VntZ2jtKUONsRi59zc5JqmsSajA9ewJKFYp8Cw==",
+          "dev": true,
+          "requires": {
+            "@jest/core": "^26.4.2",
+            "@jest/test-result": "^26.3.0",
+            "@jest/types": "^26.3.0",
+            "chalk": "^4.0.0",
+            "exit": "^0.1.2",
+            "graceful-fs": "^4.2.4",
+            "import-local": "^3.0.2",
+            "is-ci": "^2.0.0",
+            "jest-config": "^26.4.2",
+            "jest-util": "^26.3.0",
+            "jest-validate": "^26.4.2",
+            "prompts": "^2.0.1",
+            "yargs": "^15.3.1"
+          }
+        },
+        "path-exists": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
+          "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==",
+          "dev": true
+        },
+        "require-main-filename": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz",
+          "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==",
+          "dev": true
+        },
+        "string-width": {
+          "version": "4.2.0",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz",
+          "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==",
+          "dev": true,
+          "requires": {
+            "emoji-regex": "^8.0.0",
+            "is-fullwidth-code-point": "^3.0.0",
+            "strip-ansi": "^6.0.0"
+          }
+        },
+        "strip-ansi": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
+          "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
+          "dev": true,
+          "requires": {
+            "ansi-regex": "^5.0.0"
+          }
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
+        },
+        "which-module": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz",
+          "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=",
+          "dev": true
+        },
+        "wrap-ansi": {
+          "version": "6.2.0",
+          "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz",
+          "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.0.0",
+            "string-width": "^4.1.0",
+            "strip-ansi": "^6.0.0"
+          }
+        },
+        "y18n": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
+          "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
+          "dev": true
+        },
+        "yargs": {
+          "version": "15.4.1",
+          "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz",
+          "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==",
+          "dev": true,
+          "requires": {
+            "cliui": "^6.0.0",
+            "decamelize": "^1.2.0",
+            "find-up": "^4.1.0",
+            "get-caller-file": "^2.0.1",
+            "require-directory": "^2.1.1",
+            "require-main-filename": "^2.0.0",
+            "set-blocking": "^2.0.0",
+            "string-width": "^4.2.0",
+            "which-module": "^2.0.0",
+            "y18n": "^4.0.0",
+            "yargs-parser": "^18.1.2"
+          }
+        },
+        "yargs-parser": {
+          "version": "18.1.3",
+          "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz",
+          "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==",
+          "dev": true,
+          "requires": {
+            "camelcase": "^5.0.0",
+            "decamelize": "^1.2.0"
           }
         }
       }
     },
-    "is-date-object": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz",
-      "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=",
-      "dev": true
-    },
-    "is-descriptor": {
-      "version": "0.1.6",
-      "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
-      "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
+    "jest-changed-files": {
+      "version": "26.3.0",
+      "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-26.3.0.tgz",
+      "integrity": "sha512-1C4R4nijgPltX6fugKxM4oQ18zimS7LqQ+zTTY8lMCMFPrxqBFb7KJH0Z2fRQJvw2Slbaipsqq7s1mgX5Iot+g==",
       "dev": true,
       "requires": {
-        "is-accessor-descriptor": "^0.1.6",
-        "is-data-descriptor": "^0.1.4",
-        "kind-of": "^5.0.0"
+        "@jest/types": "^26.3.0",
+        "execa": "^4.0.0",
+        "throat": "^5.0.0"
       },
       "dependencies": {
-        "kind-of": {
-          "version": "5.1.0",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
-          "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==",
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
+          }
+        },
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-report": "*"
+          }
+        },
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
+        },
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "cross-spawn": {
+          "version": "7.0.3",
+          "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+          "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+          "dev": true,
+          "requires": {
+            "path-key": "^3.1.0",
+            "shebang-command": "^2.0.0",
+            "which": "^2.0.1"
+          }
+        },
+        "execa": {
+          "version": "4.0.3",
+          "resolved": "https://registry.npmjs.org/execa/-/execa-4.0.3.tgz",
+          "integrity": "sha512-WFDXGHckXPWZX19t1kCsXzOpqX9LWYNqn4C+HqZlk/V0imTkzJZqf87ZBhvpHaftERYknpk0fjSylnXVlVgI0A==",
+          "dev": true,
+          "requires": {
+            "cross-spawn": "^7.0.0",
+            "get-stream": "^5.0.0",
+            "human-signals": "^1.1.1",
+            "is-stream": "^2.0.0",
+            "merge-stream": "^2.0.0",
+            "npm-run-path": "^4.0.0",
+            "onetime": "^5.1.0",
+            "signal-exit": "^3.0.2",
+            "strip-final-newline": "^2.0.0"
+          }
+        },
+        "get-stream": {
+          "version": "5.2.0",
+          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
+          "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==",
+          "dev": true,
+          "requires": {
+            "pump": "^3.0.0"
+          }
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "is-stream": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz",
+          "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==",
           "dev": true
+        },
+        "npm-run-path": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz",
+          "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==",
+          "dev": true,
+          "requires": {
+            "path-key": "^3.0.0"
+          }
+        },
+        "path-key": {
+          "version": "3.1.1",
+          "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
+          "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
+          "dev": true
+        },
+        "pump": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
+          "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
+          "dev": true,
+          "requires": {
+            "end-of-stream": "^1.1.0",
+            "once": "^1.3.1"
+          }
+        },
+        "shebang-command": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
+          "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
+          "dev": true,
+          "requires": {
+            "shebang-regex": "^3.0.0"
+          }
+        },
+        "shebang-regex": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
+          "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
+        },
+        "which": {
+          "version": "2.0.2",
+          "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+          "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+          "dev": true,
+          "requires": {
+            "isexe": "^2.0.0"
+          }
         }
       }
     },
-    "is-directory": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/is-directory/-/is-directory-0.3.1.tgz",
-      "integrity": "sha1-YTObbyR1/Hcv2cnYP1yFddwVSuE=",
-      "dev": true
-    },
-    "is-extendable": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz",
-      "integrity": "sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik=",
-      "dev": true
-    },
-    "is-extglob": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
-      "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=",
-      "dev": true
-    },
-    "is-finite": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/is-finite/-/is-finite-1.0.2.tgz",
-      "integrity": "sha1-zGZ3aVYCvlUO8R6LSqYwU0K20Ko=",
-      "dev": true,
-      "requires": {
-        "number-is-nan": "^1.0.0"
-      }
-    },
-    "is-fullwidth-code-point": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
-      "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
-      "dev": true,
-      "requires": {
-        "number-is-nan": "^1.0.0"
-      }
-    },
-    "is-generator-fn": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz",
-      "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==",
-      "dev": true
-    },
-    "is-glob": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.0.tgz",
-      "integrity": "sha1-lSHHaEXMJhCoUgPd8ICpWML/q8A=",
-      "dev": true,
-      "requires": {
-        "is-extglob": "^2.1.1"
-      }
-    },
-    "is-negated-glob": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-negated-glob/-/is-negated-glob-1.0.0.tgz",
-      "integrity": "sha1-aRC8pdqMleeEtXUbl2z1oQ/uNtI=",
-      "dev": true
-    },
-    "is-number": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz",
-      "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=",
+    "jest-config": {
+      "version": "26.4.2",
+      "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-26.4.2.tgz",
+      "integrity": "sha512-QBf7YGLuToiM8PmTnJEdRxyYy3mHWLh24LJZKVdXZ2PNdizSe1B/E8bVm+HYcjbEzGuVXDv/di+EzdO/6Gq80A==",
       "dev": true,
       "requires": {
-        "kind-of": "^3.0.2"
+        "@babel/core": "^7.1.0",
+        "@jest/test-sequencer": "^26.4.2",
+        "@jest/types": "^26.3.0",
+        "babel-jest": "^26.3.0",
+        "chalk": "^4.0.0",
+        "deepmerge": "^4.2.2",
+        "glob": "^7.1.1",
+        "graceful-fs": "^4.2.4",
+        "jest-environment-jsdom": "^26.3.0",
+        "jest-environment-node": "^26.3.0",
+        "jest-get-type": "^26.3.0",
+        "jest-jasmine2": "^26.4.2",
+        "jest-regex-util": "^26.0.0",
+        "jest-resolve": "^26.4.0",
+        "jest-util": "^26.3.0",
+        "jest-validate": "^26.4.2",
+        "micromatch": "^4.0.2",
+        "pretty-format": "^26.4.2"
       },
       "dependencies": {
-        "kind-of": {
-          "version": "3.2.2",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-          "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
           "dev": true,
           "requires": {
-            "is-buffer": "^1.1.5"
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
           }
-        }
-      }
-    },
-    "is-obj": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz",
-      "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=",
-      "dev": true
-    },
-    "is-path-cwd": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-1.0.0.tgz",
-      "integrity": "sha1-0iXsIxMuie3Tj9p2dHLmLmXxEG0=",
-      "dev": true
-    },
-    "is-path-in-cwd": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/is-path-in-cwd/-/is-path-in-cwd-1.0.1.tgz",
-      "integrity": "sha512-FjV1RTW48E7CWM7eE/J2NJvAEEVektecDBVBE5Hh3nM1Jd0kvhHtX68Pr3xsDf857xt3Y4AkwVULK1Vku62aaQ==",
-      "dev": true,
-      "requires": {
-        "is-path-inside": "^1.0.0"
-      }
-    },
-    "is-path-inside": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-1.0.1.tgz",
-      "integrity": "sha1-jvW33lBDej/cprToZe96pVy0gDY=",
-      "dev": true,
-      "requires": {
-        "path-is-inside": "^1.0.1"
-      }
-    },
-    "is-plain-obj": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz",
-      "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=",
-      "dev": true
-    },
-    "is-plain-object": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
-      "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
-      "dev": true,
-      "requires": {
-        "isobject": "^3.0.1"
+        },
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-report": "*"
+          }
+        },
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
+        },
+        "braces": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
+          "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+          "dev": true,
+          "requires": {
+            "fill-range": "^7.0.1"
+          }
+        },
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "fill-range": {
+          "version": "7.0.1",
+          "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
+          "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+          "dev": true,
+          "requires": {
+            "to-regex-range": "^5.0.1"
+          }
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "is-number": {
+          "version": "7.0.0",
+          "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
+          "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
+          "dev": true
+        },
+        "jest-get-type": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-26.3.0.tgz",
+          "integrity": "sha512-TpfaviN1R2pQWkIihlfEanwOXK0zcxrKEE4MlU6Tn7keoXdN6/3gK/xl0yEh8DOunn5pOVGKf8hB4R9gVh04ig==",
+          "dev": true
+        },
+        "micromatch": {
+          "version": "4.0.2",
+          "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.2.tgz",
+          "integrity": "sha512-y7FpHSbMUMoyPbYUSzO6PaZ6FyRnQOpHuKwbo1G+Knck95XVU4QAiKdGEnj5wwoS7PlOgthX/09u5iFJ+aYf5Q==",
+          "dev": true,
+          "requires": {
+            "braces": "^3.0.1",
+            "picomatch": "^2.0.5"
+          }
+        },
+        "pretty-format": {
+          "version": "26.4.2",
+          "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-26.4.2.tgz",
+          "integrity": "sha512-zK6Gd8zDsEiVydOCGLkoBoZuqv8VTiHyAbKznXe/gaph/DAeZOmit9yMfgIz5adIgAMMs5XfoYSwAX3jcCO1tA==",
+          "dev": true,
+          "requires": {
+            "@jest/types": "^26.3.0",
+            "ansi-regex": "^5.0.0",
+            "ansi-styles": "^4.0.0",
+            "react-is": "^16.12.0"
+          }
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
+        },
+        "to-regex-range": {
+          "version": "5.0.1",
+          "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
+          "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
+          "dev": true,
+          "requires": {
+            "is-number": "^7.0.0"
+          }
+        }
       }
     },
-    "is-promise": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz",
-      "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=",
-      "dev": true
-    },
-    "is-regex": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz",
-      "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=",
+    "jest-diff": {
+      "version": "25.5.0",
+      "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-25.5.0.tgz",
+      "integrity": "sha512-z1kygetuPiREYdNIumRpAHY6RXiGmp70YHptjdaxTWGmA085W3iCnXNx0DhflK3vwrKmrRWyY1wUpkPMVxMK7A==",
       "dev": true,
       "requires": {
-        "has": "^1.0.1"
+        "chalk": "^3.0.0",
+        "diff-sequences": "^25.2.6",
+        "jest-get-type": "^25.2.6",
+        "pretty-format": "^25.5.0"
+      },
+      "dependencies": {
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
+        },
+        "chalk": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz",
+          "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
+        }
       }
     },
-    "is-relative": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-relative/-/is-relative-1.0.0.tgz",
-      "integrity": "sha512-Kw/ReK0iqwKeu0MITLFuj0jbPAmEiOsIwyIXvvbfa6QfmN9pkD1M+8pdk7Rl/dTKbH34/XBFMbgD4iMJhLQbGA==",
+    "jest-docblock": {
+      "version": "26.0.0",
+      "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-26.0.0.tgz",
+      "integrity": "sha512-RDZ4Iz3QbtRWycd8bUEPxQsTlYazfYn/h5R65Fc6gOfwozFhoImx+affzky/FFBuqISPTqjXomoIGJVKBWoo0w==",
       "dev": true,
       "requires": {
-        "is-unc-path": "^1.0.0"
+        "detect-newline": "^3.0.0"
+      },
+      "dependencies": {
+        "detect-newline": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz",
+          "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==",
+          "dev": true
+        }
       }
     },
-    "is-stream": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
-      "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=",
-      "dev": true
-    },
-    "is-subset": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/is-subset/-/is-subset-0.1.1.tgz",
-      "integrity": "sha1-ilkRfZMt4d4A8kX83TnOQ/HpOaY=",
-      "dev": true
-    },
-    "is-symbol": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.2.tgz",
-      "integrity": "sha512-HS8bZ9ox60yCJLH9snBpIwv9pYUAkcuLhSA1oero1UB5y9aiQpRA8y2ex945AOtCZL1lJDeIk3G5LthswI46Lw==",
+    "jest-each": {
+      "version": "26.4.2",
+      "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-26.4.2.tgz",
+      "integrity": "sha512-p15rt8r8cUcRY0Mvo1fpkOGYm7iI8S6ySxgIdfh3oOIv+gHwrHTy5VWCGOecWUhDsit4Nz8avJWdT07WLpbwDA==",
       "dev": true,
       "requires": {
-        "has-symbols": "^1.0.0"
+        "@jest/types": "^26.3.0",
+        "chalk": "^4.0.0",
+        "jest-get-type": "^26.3.0",
+        "jest-util": "^26.3.0",
+        "pretty-format": "^26.4.2"
+      },
+      "dependencies": {
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
+          }
+        },
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-report": "*"
+          }
+        },
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
+        },
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "jest-get-type": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-26.3.0.tgz",
+          "integrity": "sha512-TpfaviN1R2pQWkIihlfEanwOXK0zcxrKEE4MlU6Tn7keoXdN6/3gK/xl0yEh8DOunn5pOVGKf8hB4R9gVh04ig==",
+          "dev": true
+        },
+        "pretty-format": {
+          "version": "26.4.2",
+          "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-26.4.2.tgz",
+          "integrity": "sha512-zK6Gd8zDsEiVydOCGLkoBoZuqv8VTiHyAbKznXe/gaph/DAeZOmit9yMfgIz5adIgAMMs5XfoYSwAX3jcCO1tA==",
+          "dev": true,
+          "requires": {
+            "@jest/types": "^26.3.0",
+            "ansi-regex": "^5.0.0",
+            "ansi-styles": "^4.0.0",
+            "react-is": "^16.12.0"
+          }
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
+        }
       }
     },
-    "is-text-path": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/is-text-path/-/is-text-path-1.0.1.tgz",
-      "integrity": "sha1-Thqg+1G/vLPpJogAE5cgLBd1tm4=",
-      "dev": true,
-      "requires": {
-        "text-extensions": "^1.0.0"
+    "jest-environment-jsdom": {
+      "version": "26.3.0",
+      "resolved": "https://registry.npmjs.org/jest-environment-jsdom/-/jest-environment-jsdom-26.3.0.tgz",
+      "integrity": "sha512-zra8He2btIMJkAzvLaiZ9QwEPGEetbxqmjEBQwhH3CA+Hhhu0jSiEJxnJMbX28TGUvPLxBt/zyaTLrOPF4yMJA==",
+      "dev": true,
+      "requires": {
+        "@jest/environment": "^26.3.0",
+        "@jest/fake-timers": "^26.3.0",
+        "@jest/types": "^26.3.0",
+        "@types/node": "*",
+        "jest-mock": "^26.3.0",
+        "jest-util": "^26.3.0",
+        "jsdom": "^16.2.2"
+      },
+      "dependencies": {
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
+          }
+        },
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-report": "*"
+          }
+        },
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
+        },
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
+        }
       }
     },
-    "is-typedarray": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
-      "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=",
-      "dev": true
-    },
-    "is-unc-path": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-unc-path/-/is-unc-path-1.0.0.tgz",
-      "integrity": "sha512-mrGpVd0fs7WWLfVsStvgF6iEJnbjDFZh9/emhRDcGWTduTfNHd9CHeUwH3gYIjdbwo4On6hunkztwOaAw0yllQ==",
+    "jest-environment-node": {
+      "version": "26.3.0",
+      "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-26.3.0.tgz",
+      "integrity": "sha512-c9BvYoo+FGcMj5FunbBgtBnbR5qk3uky8PKyRVpSfe2/8+LrNQMiXX53z6q2kY+j15SkjQCOSL/6LHnCPLVHNw==",
       "dev": true,
       "requires": {
-        "unc-path-regex": "^0.1.2"
-      }
-    },
-    "is-utf8": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz",
-      "integrity": "sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI=",
-      "dev": true
-    },
-    "is-valid-glob": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-valid-glob/-/is-valid-glob-1.0.0.tgz",
-      "integrity": "sha1-Kb8+/3Ab4tTTFdusw5vDn+j2Aao=",
-      "dev": true
-    },
-    "is-windows": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz",
-      "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==",
-      "dev": true
-    },
-    "is-wsl": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-1.1.0.tgz",
-      "integrity": "sha1-HxbkqiKwTRM2tmGIpmrzxgDDpm0=",
-      "dev": true
-    },
-    "isarray": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
-      "dev": true
-    },
-    "isexe": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
-      "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
-      "dev": true
-    },
-    "isobject": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
-      "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=",
-      "dev": true
-    },
-    "isstream": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
-      "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=",
+        "@jest/environment": "^26.3.0",
+        "@jest/fake-timers": "^26.3.0",
+        "@jest/types": "^26.3.0",
+        "@types/node": "*",
+        "jest-mock": "^26.3.0",
+        "jest-util": "^26.3.0"
+      },
+      "dependencies": {
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
+          }
+        },
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-report": "*"
+          }
+        },
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
+        },
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
+        }
+      }
+    },
+    "jest-environment-node-debug": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/jest-environment-node-debug/-/jest-environment-node-debug-2.0.0.tgz",
+      "integrity": "sha1-XvCYlC/sG2r17khB9Pii/0GVYvk=",
       "dev": true
     },
-    "istanbul-lib-coverage": {
-      "version": "2.0.5",
-      "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.5.tgz",
-      "integrity": "sha512-8aXznuEPCJvGnMSRft4udDRDtb1V3pkQkMMI5LI+6HuQz5oQ4J2UFn1H82raA3qJtyOLkkwVqICBQkjnGtn5mA==",
+    "jest-get-type": {
+      "version": "25.2.6",
+      "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-25.2.6.tgz",
+      "integrity": "sha512-DxjtyzOHjObRM+sM1knti6or+eOgcGU4xVSb2HNP1TqO4ahsT+rqZg+nyqHWJSvWgKC5cG3QjGFBqxLghiF/Ig==",
       "dev": true
     },
-    "istanbul-lib-instrument": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-3.3.0.tgz",
-      "integrity": "sha512-5nnIN4vo5xQZHdXno/YDXJ0G+I3dAm4XgzfSVTPLQpj/zAV2dV6Juy0yaf10/zrJOJeHoN3fraFe+XRq2bFVZA==",
+    "jest-haste-map": {
+      "version": "26.3.0",
+      "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-26.3.0.tgz",
+      "integrity": "sha512-DHWBpTJgJhLLGwE5Z1ZaqLTYqeODQIZpby0zMBsCU9iRFHYyhklYqP4EiG73j5dkbaAdSZhgB938mL51Q5LeZA==",
       "dev": true,
       "requires": {
-        "@babel/generator": "^7.4.0",
-        "@babel/parser": "^7.4.3",
-        "@babel/template": "^7.4.0",
-        "@babel/traverse": "^7.4.3",
-        "@babel/types": "^7.4.0",
-        "istanbul-lib-coverage": "^2.0.5",
-        "semver": "^6.0.0"
+        "@jest/types": "^26.3.0",
+        "@types/graceful-fs": "^4.1.2",
+        "@types/node": "*",
+        "anymatch": "^3.0.3",
+        "fb-watchman": "^2.0.0",
+        "fsevents": "^2.1.2",
+        "graceful-fs": "^4.2.4",
+        "jest-regex-util": "^26.0.0",
+        "jest-serializer": "^26.3.0",
+        "jest-util": "^26.3.0",
+        "jest-worker": "^26.3.0",
+        "micromatch": "^4.0.2",
+        "sane": "^4.0.3",
+        "walker": "^1.0.7"
       },
       "dependencies": {
-        "semver": {
-          "version": "6.1.0",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-6.1.0.tgz",
-          "integrity": "sha512-kCqEOOHoBcFs/2Ccuk4Xarm/KiWRSLEX9CAZF8xkJ6ZPlIoTZ8V5f7J16vYLJqDbR7KrxTJpR2lqjIEm2Qx9cQ==",
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
+          }
+        },
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-report": "*"
+          }
+        },
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
+        },
+        "anymatch": {
+          "version": "3.1.1",
+          "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz",
+          "integrity": "sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg==",
+          "dev": true,
+          "requires": {
+            "normalize-path": "^3.0.0",
+            "picomatch": "^2.0.4"
+          }
+        },
+        "braces": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
+          "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+          "dev": true,
+          "requires": {
+            "fill-range": "^7.0.1"
+          }
+        },
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "fill-range": {
+          "version": "7.0.1",
+          "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
+          "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+          "dev": true,
+          "requires": {
+            "to-regex-range": "^5.0.1"
+          }
+        },
+        "fsevents": {
+          "version": "2.1.3",
+          "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.3.tgz",
+          "integrity": "sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ==",
+          "dev": true,
+          "optional": true
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "is-number": {
+          "version": "7.0.0",
+          "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
+          "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
           "dev": true
+        },
+        "micromatch": {
+          "version": "4.0.2",
+          "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.2.tgz",
+          "integrity": "sha512-y7FpHSbMUMoyPbYUSzO6PaZ6FyRnQOpHuKwbo1G+Knck95XVU4QAiKdGEnj5wwoS7PlOgthX/09u5iFJ+aYf5Q==",
+          "dev": true,
+          "requires": {
+            "braces": "^3.0.1",
+            "picomatch": "^2.0.5"
+          }
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
+        },
+        "to-regex-range": {
+          "version": "5.0.1",
+          "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
+          "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
+          "dev": true,
+          "requires": {
+            "is-number": "^7.0.0"
+          }
         }
       }
     },
-    "istanbul-lib-report": {
-      "version": "2.0.8",
-      "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-2.0.8.tgz",
-      "integrity": "sha512-fHBeG573EIihhAblwgxrSenp0Dby6tJMFR/HvlerBsrCTD5bkUuoNtn3gVh29ZCS824cGGBPn7Sg7cNk+2xUsQ==",
+    "jest-jasmine2": {
+      "version": "26.4.2",
+      "resolved": "https://registry.npmjs.org/jest-jasmine2/-/jest-jasmine2-26.4.2.tgz",
+      "integrity": "sha512-z7H4EpCldHN1J8fNgsja58QftxBSL+JcwZmaXIvV9WKIM+x49F4GLHu/+BQh2kzRKHAgaN/E82od+8rTOBPyPA==",
       "dev": true,
       "requires": {
-        "istanbul-lib-coverage": "^2.0.5",
-        "make-dir": "^2.1.0",
-        "supports-color": "^6.1.0"
+        "@babel/traverse": "^7.1.0",
+        "@jest/environment": "^26.3.0",
+        "@jest/source-map": "^26.3.0",
+        "@jest/test-result": "^26.3.0",
+        "@jest/types": "^26.3.0",
+        "@types/node": "*",
+        "chalk": "^4.0.0",
+        "co": "^4.6.0",
+        "expect": "^26.4.2",
+        "is-generator-fn": "^2.0.0",
+        "jest-each": "^26.4.2",
+        "jest-matcher-utils": "^26.4.2",
+        "jest-message-util": "^26.3.0",
+        "jest-runtime": "^26.4.2",
+        "jest-snapshot": "^26.4.2",
+        "jest-util": "^26.3.0",
+        "pretty-format": "^26.4.2",
+        "throat": "^5.0.0"
       },
       "dependencies": {
-        "make-dir": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz",
-          "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==",
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
           "dev": true,
           "requires": {
-            "pify": "^4.0.1",
-            "semver": "^5.6.0"
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
           }
         },
-        "pify": {
-          "version": "4.0.1",
-          "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz",
-          "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==",
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-report": "*"
+          }
+        },
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
+        },
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
           "dev": true
         },
+        "pretty-format": {
+          "version": "26.4.2",
+          "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-26.4.2.tgz",
+          "integrity": "sha512-zK6Gd8zDsEiVydOCGLkoBoZuqv8VTiHyAbKznXe/gaph/DAeZOmit9yMfgIz5adIgAMMs5XfoYSwAX3jcCO1tA==",
+          "dev": true,
+          "requires": {
+            "@jest/types": "^26.3.0",
+            "ansi-regex": "^5.0.0",
+            "ansi-styles": "^4.0.0",
+            "react-is": "^16.12.0"
+          }
+        },
         "supports-color": {
-          "version": "6.1.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz",
-          "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==",
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
           "dev": true,
           "requires": {
-            "has-flag": "^3.0.0"
+            "has-flag": "^4.0.0"
           }
         }
       }
     },
-    "istanbul-lib-source-maps": {
-      "version": "3.0.6",
-      "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-3.0.6.tgz",
-      "integrity": "sha512-R47KzMtDJH6X4/YW9XTx+jrLnZnscW4VpNN+1PViSYTejLVPWv7oov+Duf8YQSPyVRUvueQqz1TcsC6mooZTXw==",
+    "jest-leak-detector": {
+      "version": "26.4.2",
+      "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-26.4.2.tgz",
+      "integrity": "sha512-akzGcxwxtE+9ZJZRW+M2o+nTNnmQZxrHJxX/HjgDaU5+PLmY1qnQPnMjgADPGCRPhB+Yawe1iij0REe+k/aHoA==",
       "dev": true,
       "requires": {
-        "debug": "^4.1.1",
-        "istanbul-lib-coverage": "^2.0.5",
-        "make-dir": "^2.1.0",
-        "rimraf": "^2.6.3",
-        "source-map": "^0.6.1"
+        "jest-get-type": "^26.3.0",
+        "pretty-format": "^26.4.2"
       },
       "dependencies": {
-        "debug": {
-          "version": "4.1.1",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
-          "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
           "dev": true,
           "requires": {
-            "ms": "^2.1.1"
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
           }
         },
-        "make-dir": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz",
-          "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==",
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
           "dev": true,
           "requires": {
-            "pify": "^4.0.1",
-            "semver": "^5.6.0"
+            "@types/istanbul-lib-report": "*"
           }
         },
-        "ms": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
-          "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==",
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
+        },
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
           "dev": true
         },
-        "pify": {
-          "version": "4.0.1",
-          "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz",
-          "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==",
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
           "dev": true
         },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+        "jest-get-type": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-26.3.0.tgz",
+          "integrity": "sha512-TpfaviN1R2pQWkIihlfEanwOXK0zcxrKEE4MlU6Tn7keoXdN6/3gK/xl0yEh8DOunn5pOVGKf8hB4R9gVh04ig==",
           "dev": true
+        },
+        "pretty-format": {
+          "version": "26.4.2",
+          "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-26.4.2.tgz",
+          "integrity": "sha512-zK6Gd8zDsEiVydOCGLkoBoZuqv8VTiHyAbKznXe/gaph/DAeZOmit9yMfgIz5adIgAMMs5XfoYSwAX3jcCO1tA==",
+          "dev": true,
+          "requires": {
+            "@jest/types": "^26.3.0",
+            "ansi-regex": "^5.0.0",
+            "ansi-styles": "^4.0.0",
+            "react-is": "^16.12.0"
+          }
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
         }
       }
     },
-    "istanbul-reports": {
-      "version": "2.2.6",
-      "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-2.2.6.tgz",
-      "integrity": "sha512-SKi4rnMyLBKe0Jy2uUdx28h8oG7ph2PPuQPvIAh31d+Ci+lSiEu4C+h3oBPuJ9+mPKhOyW0M8gY4U5NM1WLeXA==",
+    "jest-matcher-utils": {
+      "version": "26.4.2",
+      "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-26.4.2.tgz",
+      "integrity": "sha512-KcbNqWfWUG24R7tu9WcAOKKdiXiXCbMvQYT6iodZ9k1f7065k0keUOW6XpJMMvah+hTfqkhJhRXmA3r3zMAg0Q==",
       "dev": true,
       "requires": {
-        "handlebars": "^4.1.2"
+        "chalk": "^4.0.0",
+        "jest-diff": "^26.4.2",
+        "jest-get-type": "^26.3.0",
+        "pretty-format": "^26.4.2"
       },
       "dependencies": {
-        "handlebars": {
-          "version": "4.1.2",
-          "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.1.2.tgz",
-          "integrity": "sha512-nvfrjqvt9xQ8Z/w0ijewdD/vvWDTOweBUm96NTr66Wfvo1mJenBLwcYmPs3TIBP5ruzYGD7Hx/DaM9RmhroGPw==",
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
           "dev": true,
           "requires": {
-            "neo-async": "^2.6.0",
-            "optimist": "^0.6.1",
-            "source-map": "^0.6.1",
-            "uglify-js": "^3.1.4"
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
           }
         },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-report": "*"
+          }
+        },
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
+        },
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
           "dev": true
-        }
-      }
-    },
-    "ix": {
-      "version": "2.5.3",
-      "resolved": "https://registry.npmjs.org/ix/-/ix-2.5.3.tgz",
-      "integrity": "sha512-Syz3oYUx5+P0T9F3BZOE2DzvWYF31Mbha1VqDbJsr4e4A+twhxTAf2c1BO8TMlwon12oAvvxAtQ+NJm9P8ISYQ==",
-      "dev": true,
-      "requires": {
-        "@types/node": "^11.11.6",
-        "tslib": "^1.9.3"
-      },
-      "dependencies": {
-        "@types/node": {
-          "version": "11.13.11",
-          "resolved": "https://registry.npmjs.org/@types/node/-/node-11.13.11.tgz",
-          "integrity": "sha512-blLeR+KIy26km1OU8yTLUlSyVCOvT6+wPq/77tIA+uSHHa4yYQosn+bbaJqPtWId0wjVClUtD7aXzDbZeKWqig==",
+        },
+        "diff-sequences": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-26.3.0.tgz",
+          "integrity": "sha512-5j5vdRcw3CNctePNYN0Wy2e/JbWT6cAYnXv5OuqPhDpyCGc0uLu2TK0zOCJWNB9kOIfYMSpIulRaDgIi4HJ6Ig==",
+          "dev": true
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "jest-diff": {
+          "version": "26.4.2",
+          "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-26.4.2.tgz",
+          "integrity": "sha512-6T1XQY8U28WH0Z5rGpQ+VqZSZz8EN8rZcBtfvXaOkbwxIEeRre6qnuZQlbY1AJ4MKDxQF8EkrCvK+hL/VkyYLQ==",
+          "dev": true,
+          "requires": {
+            "chalk": "^4.0.0",
+            "diff-sequences": "^26.3.0",
+            "jest-get-type": "^26.3.0",
+            "pretty-format": "^26.4.2"
+          }
+        },
+        "jest-get-type": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-26.3.0.tgz",
+          "integrity": "sha512-TpfaviN1R2pQWkIihlfEanwOXK0zcxrKEE4MlU6Tn7keoXdN6/3gK/xl0yEh8DOunn5pOVGKf8hB4R9gVh04ig==",
           "dev": true
+        },
+        "pretty-format": {
+          "version": "26.4.2",
+          "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-26.4.2.tgz",
+          "integrity": "sha512-zK6Gd8zDsEiVydOCGLkoBoZuqv8VTiHyAbKznXe/gaph/DAeZOmit9yMfgIz5adIgAMMs5XfoYSwAX3jcCO1tA==",
+          "dev": true,
+          "requires": {
+            "@jest/types": "^26.3.0",
+            "ansi-regex": "^5.0.0",
+            "ansi-styles": "^4.0.0",
+            "react-is": "^16.12.0"
+          }
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
         }
       }
     },
-    "jest": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest/-/jest-24.8.0.tgz",
-      "integrity": "sha512-o0HM90RKFRNWmAWvlyV8i5jGZ97pFwkeVoGvPW1EtLTgJc2+jcuqcbbqcSZLE/3f2S5pt0y2ZBETuhpWNl1Reg==",
+    "jest-message-util": {
+      "version": "26.3.0",
+      "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-26.3.0.tgz",
+      "integrity": "sha512-xIavRYqr4/otGOiLxLZGj3ieMmjcNE73Ui+LdSW/Y790j5acqCsAdDiLIbzHCZMpN07JOENRWX5DcU+OQ+TjTA==",
       "dev": true,
       "requires": {
-        "import-local": "^2.0.0",
-        "jest-cli": "^24.8.0"
+        "@babel/code-frame": "^7.0.0",
+        "@jest/types": "^26.3.0",
+        "@types/stack-utils": "^1.0.1",
+        "chalk": "^4.0.0",
+        "graceful-fs": "^4.2.4",
+        "micromatch": "^4.0.2",
+        "slash": "^3.0.0",
+        "stack-utils": "^2.0.2"
       },
       "dependencies": {
-        "ansi-regex": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
-          "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=",
-          "dev": true
-        },
-        "callsites": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
-          "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
-          "dev": true
-        },
-        "camelcase": {
-          "version": "5.3.1",
-          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
-          "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==",
-          "dev": true
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
+          }
         },
-        "ci-info": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
-          "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==",
-          "dev": true
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-report": "*"
+          }
         },
-        "cliui": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/cliui/-/cliui-4.1.0.tgz",
-          "integrity": "sha512-4FG+RSG9DL7uEwRUZXZn3SS34DiDPfzP0VOiEwtUWlE+AR2EIg+hSyvrIgUUfhdgR/UkAeW2QHgeP+hWrXs7jQ==",
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
           "dev": true,
           "requires": {
-            "string-width": "^2.1.1",
-            "strip-ansi": "^4.0.0",
-            "wrap-ansi": "^2.0.0"
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
           }
         },
-        "find-up": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz",
-          "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==",
+        "braces": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
+          "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
           "dev": true,
           "requires": {
-            "locate-path": "^3.0.0"
+            "fill-range": "^7.0.1"
           }
         },
-        "import-local": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/import-local/-/import-local-2.0.0.tgz",
-          "integrity": "sha512-b6s04m3O+s3CGSbqDIyP4R6aAwAeYlVq9+WUWep6iHa8ETRf9yei1U48C5MmfJmV9AiLYYBKPMq/W+/WRpQmCQ==",
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
           "dev": true,
           "requires": {
-            "pkg-dir": "^3.0.0",
-            "resolve-cwd": "^2.0.0"
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
           }
         },
-        "invert-kv": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-2.0.0.tgz",
-          "integrity": "sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==",
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
           "dev": true
         },
-        "is-ci": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz",
-          "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==",
+        "fill-range": {
+          "version": "7.0.1",
+          "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
+          "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
           "dev": true,
           "requires": {
-            "ci-info": "^2.0.0"
+            "to-regex-range": "^5.0.1"
           }
         },
-        "is-fullwidth-code-point": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
-          "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=",
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
           "dev": true
         },
-        "jest-cli": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-24.8.0.tgz",
-          "integrity": "sha512-+p6J00jSMPQ116ZLlHJJvdf8wbjNbZdeSX9ptfHX06/MSNaXmKihQzx5vQcw0q2G6JsdVkUIdWbOWtSnaYs3yA==",
+        "is-number": {
+          "version": "7.0.0",
+          "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
+          "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
+          "dev": true
+        },
+        "micromatch": {
+          "version": "4.0.2",
+          "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.2.tgz",
+          "integrity": "sha512-y7FpHSbMUMoyPbYUSzO6PaZ6FyRnQOpHuKwbo1G+Knck95XVU4QAiKdGEnj5wwoS7PlOgthX/09u5iFJ+aYf5Q==",
           "dev": true,
           "requires": {
-            "@jest/core": "^24.8.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
-            "chalk": "^2.0.1",
-            "exit": "^0.1.2",
-            "import-local": "^2.0.0",
-            "is-ci": "^2.0.0",
-            "jest-config": "^24.8.0",
-            "jest-util": "^24.8.0",
-            "jest-validate": "^24.8.0",
-            "prompts": "^2.0.1",
-            "realpath-native": "^1.1.0",
-            "yargs": "^12.0.2"
+            "braces": "^3.0.1",
+            "picomatch": "^2.0.5"
           }
         },
-        "jest-util": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-24.8.0.tgz",
-          "integrity": "sha512-DYZeE+XyAnbNt0BG1OQqKy/4GVLPtzwGx5tsnDrFcax36rVE3lTA5fbvgmbVPUZf9w77AJ8otqR4VBbfFJkUZA==",
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
           "dev": true,
           "requires": {
-            "@jest/console": "^24.7.1",
-            "@jest/fake-timers": "^24.8.0",
-            "@jest/source-map": "^24.3.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
-            "callsites": "^3.0.0",
-            "chalk": "^2.0.1",
-            "graceful-fs": "^4.1.15",
-            "is-ci": "^2.0.0",
-            "mkdirp": "^0.5.1",
-            "slash": "^2.0.0",
-            "source-map": "^0.6.0"
+            "has-flag": "^4.0.0"
           }
         },
-        "lcid": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/lcid/-/lcid-2.0.0.tgz",
-          "integrity": "sha512-avPEb8P8EGnwXKClwsNUgryVjllcRqtMYa49NTsbQagYuT1DcXnl1915oxWjoyGrXR6zH/Y0Zc96xWsPcoDKeA==",
+        "to-regex-range": {
+          "version": "5.0.1",
+          "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
+          "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
+          "dev": true,
+          "requires": {
+            "is-number": "^7.0.0"
+          }
+        }
+      }
+    },
+    "jest-mock": {
+      "version": "26.3.0",
+      "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-26.3.0.tgz",
+      "integrity": "sha512-PeaRrg8Dc6mnS35gOo/CbZovoDPKAeB1FICZiuagAgGvbWdNNyjQjkOaGUa/3N3JtpQ/Mh9P4A2D4Fv51NnP8Q==",
+      "dev": true,
+      "requires": {
+        "@jest/types": "^26.3.0",
+        "@types/node": "*"
+      },
+      "dependencies": {
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
           "dev": true,
           "requires": {
-            "invert-kv": "^2.0.0"
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
           }
         },
-        "locate-path": {
+        "@types/istanbul-reports": {
           "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz",
-          "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
           "dev": true,
           "requires": {
-            "p-locate": "^3.0.0",
-            "path-exists": "^3.0.0"
+            "@types/istanbul-lib-report": "*"
           }
         },
-        "os-locale": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-3.1.0.tgz",
-          "integrity": "sha512-Z8l3R4wYWM40/52Z+S265okfFj8Kt2cC2MKY+xNi3kFs+XGI7WXu/I309QQQYbRW4ijiZ+yxs9pqEhJh0DqW3Q==",
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
           "dev": true,
           "requires": {
-            "execa": "^1.0.0",
-            "lcid": "^2.0.0",
-            "mem": "^4.0.0"
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
           }
         },
-        "p-limit": {
-          "version": "2.2.0",
-          "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.0.tgz",
-          "integrity": "sha512-pZbTJpoUsCzV48Mc9Nh51VbwO0X9cuPFE8gYwx9BTCt9SF8/b7Zljd2fVgOxhIF/HDTKgpVzs+GPhyKfjLLFRQ==",
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
           "dev": true,
           "requires": {
-            "p-try": "^2.0.0"
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
           }
         },
-        "p-locate": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz",
-          "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==",
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
           "dev": true,
           "requires": {
-            "p-limit": "^2.0.0"
+            "color-name": "~1.1.4"
           }
         },
-        "p-try": {
-          "version": "2.2.0",
-          "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
-          "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==",
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
           "dev": true
         },
-        "path-exists": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
-          "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=",
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
           "dev": true
         },
-        "pkg-dir": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz",
-          "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==",
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
           "dev": true,
           "requires": {
-            "find-up": "^3.0.0"
+            "has-flag": "^4.0.0"
+          }
+        }
+      }
+    },
+    "jest-pnp-resolver": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.2.tgz",
+      "integrity": "sha512-olV41bKSMm8BdnuMsewT4jqlZ8+3TCARAXjZGT9jcoSnrfUnRCqnMoF9XEeoWjbzObpqF9dRhHQj0Xb9QdF6/w==",
+      "dev": true
+    },
+    "jest-regex-util": {
+      "version": "26.0.0",
+      "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-26.0.0.tgz",
+      "integrity": "sha512-Gv3ZIs/nA48/Zvjrl34bf+oD76JHiGDUxNOVgUjh3j890sblXryjY4rss71fPtD/njchl6PSE2hIhvyWa1eT0A==",
+      "dev": true
+    },
+    "jest-resolve": {
+      "version": "26.4.0",
+      "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-26.4.0.tgz",
+      "integrity": "sha512-bn/JoZTEXRSlEx3+SfgZcJAVuTMOksYq9xe9O6s4Ekg84aKBObEaVXKOEilULRqviSLAYJldnoWV9c07kwtiCg==",
+      "dev": true,
+      "requires": {
+        "@jest/types": "^26.3.0",
+        "chalk": "^4.0.0",
+        "graceful-fs": "^4.2.4",
+        "jest-pnp-resolver": "^1.2.2",
+        "jest-util": "^26.3.0",
+        "read-pkg-up": "^7.0.1",
+        "resolve": "^1.17.0",
+        "slash": "^3.0.0"
+      },
+      "dependencies": {
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
           }
         },
-        "slash": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
-          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
-          "dev": true
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-report": "*"
+          }
         },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-          "dev": true
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
         },
-        "string-width": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",
-          "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==",
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
           "dev": true,
           "requires": {
-            "is-fullwidth-code-point": "^2.0.0",
-            "strip-ansi": "^4.0.0"
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
           }
         },
-        "strip-ansi": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
-          "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
           "dev": true,
           "requires": {
-            "ansi-regex": "^3.0.0"
+            "color-name": "~1.1.4"
           }
         },
-        "which-module": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz",
-          "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=",
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
           "dev": true
         },
-        "yargs": {
-          "version": "12.0.5",
-          "resolved": "https://registry.npmjs.org/yargs/-/yargs-12.0.5.tgz",
-          "integrity": "sha512-Lhz8TLaYnxq/2ObqHDql8dX8CJi97oHxrjUcYtzKbbykPtVW9WB+poxI+NM2UIzsMgNCZTIf0AQwsjK5yMAqZw==",
+        "find-up": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz",
+          "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==",
           "dev": true,
           "requires": {
-            "cliui": "^4.0.0",
-            "decamelize": "^1.2.0",
-            "find-up": "^3.0.0",
-            "get-caller-file": "^1.0.1",
-            "os-locale": "^3.0.0",
-            "require-directory": "^2.1.1",
-            "require-main-filename": "^1.0.1",
-            "set-blocking": "^2.0.0",
-            "string-width": "^2.0.0",
-            "which-module": "^2.0.0",
-            "y18n": "^3.2.1 || ^4.0.0",
-            "yargs-parser": "^11.1.1"
+            "locate-path": "^5.0.0",
+            "path-exists": "^4.0.0"
           }
         },
-        "yargs-parser": {
-          "version": "11.1.1",
-          "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-11.1.1.tgz",
-          "integrity": "sha512-C6kB/WJDiaxONLJQnF8ccx9SEeoTTLek8RVbaOIsrAUS8VrBEXfmeSnCZxygc+XC2sNMBIwOOnfcxiynjHsVSQ==",
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "parse-json": {
+          "version": "5.1.0",
+          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.1.0.tgz",
+          "integrity": "sha512-+mi/lmVVNKFNVyLXV31ERiy2CY5E1/F6QtJFEzoChPRwwngMNXRDQ9GJ5WdE2Z2P4AujsOi0/+2qHID68KwfIQ==",
           "dev": true,
           "requires": {
-            "camelcase": "^5.0.0",
-            "decamelize": "^1.2.0"
+            "@babel/code-frame": "^7.0.0",
+            "error-ex": "^1.3.1",
+            "json-parse-even-better-errors": "^2.3.0",
+            "lines-and-columns": "^1.1.6"
           }
-        }
-      }
-    },
-    "jest-changed-files": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-24.8.0.tgz",
-      "integrity": "sha512-qgANC1Yrivsq+UrLXsvJefBKVoCsKB0Hv+mBb6NMjjZ90wwxCDmU3hsCXBya30cH+LnPYjwgcU65i6yJ5Nfuug==",
-      "dev": true,
-      "requires": {
-        "@jest/types": "^24.8.0",
-        "execa": "^1.0.0",
-        "throat": "^4.0.0"
-      }
-    },
-    "jest-config": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-24.8.0.tgz",
-      "integrity": "sha512-Czl3Nn2uEzVGsOeaewGWoDPD8GStxCpAe0zOYs2x2l0fZAgPbCr3uwUkgNKV3LwE13VXythM946cd5rdGkkBZw==",
-      "dev": true,
-      "requires": {
-        "@babel/core": "^7.1.0",
-        "@jest/test-sequencer": "^24.8.0",
-        "@jest/types": "^24.8.0",
-        "babel-jest": "^24.8.0",
-        "chalk": "^2.0.1",
-        "glob": "^7.1.1",
-        "jest-environment-jsdom": "^24.8.0",
-        "jest-environment-node": "^24.8.0",
-        "jest-get-type": "^24.8.0",
-        "jest-jasmine2": "^24.8.0",
-        "jest-regex-util": "^24.3.0",
-        "jest-resolve": "^24.8.0",
-        "jest-util": "^24.8.0",
-        "jest-validate": "^24.8.0",
-        "micromatch": "^3.1.10",
-        "pretty-format": "^24.8.0",
-        "realpath-native": "^1.1.0"
-      },
-      "dependencies": {
-        "callsites": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
-          "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
-          "dev": true
         },
-        "ci-info": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
-          "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==",
+        "path-exists": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
+          "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==",
           "dev": true
         },
-        "is-ci": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz",
-          "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==",
+        "read-pkg": {
+          "version": "5.2.0",
+          "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-5.2.0.tgz",
+          "integrity": "sha512-Ug69mNOpfvKDAc2Q8DRpMjjzdtrnv9HcSMX+4VsZxD1aZ6ZzrIE7rlzXBtWTyhULSMKg076AW6WR5iZpD0JiOg==",
           "dev": true,
           "requires": {
-            "ci-info": "^2.0.0"
+            "@types/normalize-package-data": "^2.4.0",
+            "normalize-package-data": "^2.5.0",
+            "parse-json": "^5.0.0",
+            "type-fest": "^0.6.0"
+          },
+          "dependencies": {
+            "type-fest": {
+              "version": "0.6.0",
+              "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.6.0.tgz",
+              "integrity": "sha512-q+MB8nYR1KDLrgr4G5yemftpMC7/QLqVndBmEEdqzmNj5dcFOO4Oo8qlwZE3ULT3+Zim1F8Kq4cBnikNhlCMlg==",
+              "dev": true
+            }
           }
         },
-        "jest-util": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-24.8.0.tgz",
-          "integrity": "sha512-DYZeE+XyAnbNt0BG1OQqKy/4GVLPtzwGx5tsnDrFcax36rVE3lTA5fbvgmbVPUZf9w77AJ8otqR4VBbfFJkUZA==",
+        "read-pkg-up": {
+          "version": "7.0.1",
+          "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-7.0.1.tgz",
+          "integrity": "sha512-zK0TB7Xd6JpCLmlLmufqykGE+/TlOePD6qKClNW7hHDKFh/J7/7gCWGR7joEQEW1bKq3a3yUZSObOoWLFQ4ohg==",
           "dev": true,
           "requires": {
-            "@jest/console": "^24.7.1",
-            "@jest/fake-timers": "^24.8.0",
-            "@jest/source-map": "^24.3.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
-            "callsites": "^3.0.0",
-            "chalk": "^2.0.1",
-            "graceful-fs": "^4.1.15",
-            "is-ci": "^2.0.0",
-            "mkdirp": "^0.5.1",
-            "slash": "^2.0.0",
-            "source-map": "^0.6.0"
+            "find-up": "^4.1.0",
+            "read-pkg": "^5.2.0",
+            "type-fest": "^0.8.1"
           }
         },
-        "slash": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
-          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
-          "dev": true
-        },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-          "dev": true
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
         }
       }
     },
-    "jest-diff": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-24.8.0.tgz",
-      "integrity": "sha512-wxetCEl49zUpJ/bvUmIFjd/o52J+yWcoc5ZyPq4/W1LUKGEhRYDIbP1KcF6t+PvqNrGAFk4/JhtxDq/Nnzs66g==",
-      "dev": true,
-      "requires": {
-        "chalk": "^2.0.1",
-        "diff-sequences": "^24.3.0",
-        "jest-get-type": "^24.8.0",
-        "pretty-format": "^24.8.0"
-      }
-    },
-    "jest-docblock": {
-      "version": "24.3.0",
-      "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-24.3.0.tgz",
-      "integrity": "sha512-nlANmF9Yq1dufhFlKG9rasfQlrY7wINJbo3q01tu56Jv5eBU5jirylhF2O5ZBnLxzOVBGRDz/9NAwNyBtG4Nyg==",
-      "dev": true,
-      "requires": {
-        "detect-newline": "^2.1.0"
-      }
-    },
-    "jest-each": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-24.8.0.tgz",
-      "integrity": "sha512-NrwK9gaL5+XgrgoCsd9svsoWdVkK4gnvyhcpzd6m487tXHqIdYeykgq3MKI1u4I+5Zf0tofr70at9dWJDeb+BA==",
+    "jest-resolve-dependencies": {
+      "version": "26.4.2",
+      "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-26.4.2.tgz",
+      "integrity": "sha512-ADHaOwqEcVc71uTfySzSowA/RdxUpCxhxa2FNLiin9vWLB1uLPad3we+JSSROq5+SrL9iYPdZZF8bdKM7XABTQ==",
       "dev": true,
       "requires": {
-        "@jest/types": "^24.8.0",
-        "chalk": "^2.0.1",
-        "jest-get-type": "^24.8.0",
-        "jest-util": "^24.8.0",
-        "pretty-format": "^24.8.0"
+        "@jest/types": "^26.3.0",
+        "jest-regex-util": "^26.0.0",
+        "jest-snapshot": "^26.4.2"
       },
       "dependencies": {
-        "callsites": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
-          "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
-          "dev": true
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
+          }
         },
-        "ci-info": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
-          "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==",
-          "dev": true
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-report": "*"
+          }
         },
-        "is-ci": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz",
-          "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==",
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
           "dev": true,
           "requires": {
-            "ci-info": "^2.0.0"
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
           }
         },
-        "jest-util": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-24.8.0.tgz",
-          "integrity": "sha512-DYZeE+XyAnbNt0BG1OQqKy/4GVLPtzwGx5tsnDrFcax36rVE3lTA5fbvgmbVPUZf9w77AJ8otqR4VBbfFJkUZA==",
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
           "dev": true,
           "requires": {
-            "@jest/console": "^24.7.1",
-            "@jest/fake-timers": "^24.8.0",
-            "@jest/source-map": "^24.3.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
-            "callsites": "^3.0.0",
-            "chalk": "^2.0.1",
-            "graceful-fs": "^4.1.15",
-            "is-ci": "^2.0.0",
-            "mkdirp": "^0.5.1",
-            "slash": "^2.0.0",
-            "source-map": "^0.6.0"
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
           }
         },
-        "slash": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
-          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
           "dev": true
         },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
           "dev": true
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
         }
       }
     },
-    "jest-environment-jsdom": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest-environment-jsdom/-/jest-environment-jsdom-24.8.0.tgz",
-      "integrity": "sha512-qbvgLmR7PpwjoFjM/sbuqHJt/NCkviuq9vus9NBn/76hhSidO+Z6Bn9tU8friecegbJL8gzZQEMZBQlFWDCwAQ==",
-      "dev": true,
-      "requires": {
-        "@jest/environment": "^24.8.0",
-        "@jest/fake-timers": "^24.8.0",
-        "@jest/types": "^24.8.0",
-        "jest-mock": "^24.8.0",
-        "jest-util": "^24.8.0",
-        "jsdom": "^11.5.1"
+    "jest-runner": {
+      "version": "26.4.2",
+      "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-26.4.2.tgz",
+      "integrity": "sha512-FgjDHeVknDjw1gRAYaoUoShe1K3XUuFMkIaXbdhEys+1O4bEJS8Avmn4lBwoMfL8O5oFTdWYKcf3tEJyyYyk8g==",
+      "dev": true,
+      "requires": {
+        "@jest/console": "^26.3.0",
+        "@jest/environment": "^26.3.0",
+        "@jest/test-result": "^26.3.0",
+        "@jest/types": "^26.3.0",
+        "@types/node": "*",
+        "chalk": "^4.0.0",
+        "emittery": "^0.7.1",
+        "exit": "^0.1.2",
+        "graceful-fs": "^4.2.4",
+        "jest-config": "^26.4.2",
+        "jest-docblock": "^26.0.0",
+        "jest-haste-map": "^26.3.0",
+        "jest-leak-detector": "^26.4.2",
+        "jest-message-util": "^26.3.0",
+        "jest-resolve": "^26.4.0",
+        "jest-runtime": "^26.4.2",
+        "jest-util": "^26.3.0",
+        "jest-worker": "^26.3.0",
+        "source-map-support": "^0.5.6",
+        "throat": "^5.0.0"
       },
       "dependencies": {
-        "callsites": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
-          "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
-          "dev": true
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
+          }
         },
-        "ci-info": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
-          "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==",
-          "dev": true
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-report": "*"
+          }
         },
-        "is-ci": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz",
-          "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==",
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
           "dev": true,
           "requires": {
-            "ci-info": "^2.0.0"
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
           }
         },
-        "jest-util": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-24.8.0.tgz",
-          "integrity": "sha512-DYZeE+XyAnbNt0BG1OQqKy/4GVLPtzwGx5tsnDrFcax36rVE3lTA5fbvgmbVPUZf9w77AJ8otqR4VBbfFJkUZA==",
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
           "dev": true,
           "requires": {
-            "@jest/console": "^24.7.1",
-            "@jest/fake-timers": "^24.8.0",
-            "@jest/source-map": "^24.3.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
-            "callsites": "^3.0.0",
-            "chalk": "^2.0.1",
-            "graceful-fs": "^4.1.15",
-            "is-ci": "^2.0.0",
-            "mkdirp": "^0.5.1",
-            "slash": "^2.0.0",
-            "source-map": "^0.6.0"
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
           }
         },
-        "slash": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
-          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
           "dev": true
         },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
           "dev": true
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
         }
       }
     },
-    "jest-environment-node": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-24.8.0.tgz",
-      "integrity": "sha512-vIGUEScd1cdDgR6sqn2M08sJTRLQp6Dk/eIkCeO4PFHxZMOgy+uYLPMC4ix3PEfM5Au/x3uQ/5Tl0DpXXZsJ/Q==",
-      "dev": true,
-      "requires": {
-        "@jest/environment": "^24.8.0",
-        "@jest/fake-timers": "^24.8.0",
-        "@jest/types": "^24.8.0",
-        "jest-mock": "^24.8.0",
-        "jest-util": "^24.8.0"
+    "jest-runtime": {
+      "version": "26.4.2",
+      "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-26.4.2.tgz",
+      "integrity": "sha512-4Pe7Uk5a80FnbHwSOk7ojNCJvz3Ks2CNQWT5Z7MJo4tX0jb3V/LThKvD9tKPNVNyeMH98J/nzGlcwc00R2dSHQ==",
+      "dev": true,
+      "requires": {
+        "@jest/console": "^26.3.0",
+        "@jest/environment": "^26.3.0",
+        "@jest/fake-timers": "^26.3.0",
+        "@jest/globals": "^26.4.2",
+        "@jest/source-map": "^26.3.0",
+        "@jest/test-result": "^26.3.0",
+        "@jest/transform": "^26.3.0",
+        "@jest/types": "^26.3.0",
+        "@types/yargs": "^15.0.0",
+        "chalk": "^4.0.0",
+        "collect-v8-coverage": "^1.0.0",
+        "exit": "^0.1.2",
+        "glob": "^7.1.3",
+        "graceful-fs": "^4.2.4",
+        "jest-config": "^26.4.2",
+        "jest-haste-map": "^26.3.0",
+        "jest-message-util": "^26.3.0",
+        "jest-mock": "^26.3.0",
+        "jest-regex-util": "^26.0.0",
+        "jest-resolve": "^26.4.0",
+        "jest-snapshot": "^26.4.2",
+        "jest-util": "^26.3.0",
+        "jest-validate": "^26.4.2",
+        "slash": "^3.0.0",
+        "strip-bom": "^4.0.0",
+        "yargs": "^15.3.1"
       },
       "dependencies": {
-        "callsites": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
-          "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
-          "dev": true
-        },
-        "ci-info": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
-          "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==",
-          "dev": true
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
+          }
         },
-        "is-ci": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz",
-          "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==",
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
           "dev": true,
           "requires": {
-            "ci-info": "^2.0.0"
+            "@types/istanbul-lib-report": "*"
           }
         },
-        "jest-util": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-24.8.0.tgz",
-          "integrity": "sha512-DYZeE+XyAnbNt0BG1OQqKy/4GVLPtzwGx5tsnDrFcax36rVE3lTA5fbvgmbVPUZf9w77AJ8otqR4VBbfFJkUZA==",
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
           "dev": true,
           "requires": {
-            "@jest/console": "^24.7.1",
-            "@jest/fake-timers": "^24.8.0",
-            "@jest/source-map": "^24.3.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
-            "callsites": "^3.0.0",
-            "chalk": "^2.0.1",
-            "graceful-fs": "^4.1.15",
-            "is-ci": "^2.0.0",
-            "mkdirp": "^0.5.1",
-            "slash": "^2.0.0",
-            "source-map": "^0.6.0"
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
           }
         },
-        "slash": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
-          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
+        "camelcase": {
+          "version": "5.3.1",
+          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
+          "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==",
           "dev": true
         },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-          "dev": true
-        }
-      }
-    },
-    "jest-environment-node-debug": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/jest-environment-node-debug/-/jest-environment-node-debug-2.0.0.tgz",
-      "integrity": "sha1-XvCYlC/sG2r17khB9Pii/0GVYvk=",
-      "dev": true
-    },
-    "jest-get-type": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-24.8.0.tgz",
-      "integrity": "sha512-RR4fo8jEmMD9zSz2nLbs2j0zvPpk/KCEz3a62jJWbd2ayNo0cb+KFRxPHVhE4ZmgGJEQp0fosmNz84IfqM8cMQ==",
-      "dev": true
-    },
-    "jest-haste-map": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-24.8.0.tgz",
-      "integrity": "sha512-ZBPRGHdPt1rHajWelXdqygIDpJx8u3xOoLyUBWRW28r3tagrgoepPrzAozW7kW9HrQfhvmiv1tncsxqHJO1onQ==",
-      "dev": true,
-      "requires": {
-        "@jest/types": "^24.8.0",
-        "anymatch": "^2.0.0",
-        "fb-watchman": "^2.0.0",
-        "fsevents": "^1.2.7",
-        "graceful-fs": "^4.1.15",
-        "invariant": "^2.2.4",
-        "jest-serializer": "^24.4.0",
-        "jest-util": "^24.8.0",
-        "jest-worker": "^24.6.0",
-        "micromatch": "^3.1.10",
-        "sane": "^4.0.3",
-        "walker": "^1.0.7"
-      },
-      "dependencies": {
-        "callsites": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
-          "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
-          "dev": true
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
         },
-        "ci-info": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
-          "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==",
-          "dev": true
+        "cliui": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz",
+          "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==",
+          "dev": true,
+          "requires": {
+            "string-width": "^4.2.0",
+            "strip-ansi": "^6.0.0",
+            "wrap-ansi": "^6.2.0"
+          }
         },
-        "is-ci": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz",
-          "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==",
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
           "dev": true,
           "requires": {
-            "ci-info": "^2.0.0"
+            "color-name": "~1.1.4"
           }
         },
-        "jest-util": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-24.8.0.tgz",
-          "integrity": "sha512-DYZeE+XyAnbNt0BG1OQqKy/4GVLPtzwGx5tsnDrFcax36rVE3lTA5fbvgmbVPUZf9w77AJ8otqR4VBbfFJkUZA==",
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "find-up": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz",
+          "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==",
           "dev": true,
           "requires": {
-            "@jest/console": "^24.7.1",
-            "@jest/fake-timers": "^24.8.0",
-            "@jest/source-map": "^24.3.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
-            "callsites": "^3.0.0",
-            "chalk": "^2.0.1",
-            "graceful-fs": "^4.1.15",
-            "is-ci": "^2.0.0",
-            "mkdirp": "^0.5.1",
-            "slash": "^2.0.0",
-            "source-map": "^0.6.0"
+            "locate-path": "^5.0.0",
+            "path-exists": "^4.0.0"
           }
         },
-        "slash": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
-          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
+        "get-caller-file": {
+          "version": "2.0.5",
+          "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
+          "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
           "dev": true
         },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
           "dev": true
-        }
-      }
-    },
-    "jest-jasmine2": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest-jasmine2/-/jest-jasmine2-24.8.0.tgz",
-      "integrity": "sha512-cEky88npEE5LKd5jPpTdDCLvKkdyklnaRycBXL6GNmpxe41F0WN44+i7lpQKa/hcbXaQ+rc9RMaM4dsebrYong==",
-      "dev": true,
-      "requires": {
-        "@babel/traverse": "^7.1.0",
-        "@jest/environment": "^24.8.0",
-        "@jest/test-result": "^24.8.0",
-        "@jest/types": "^24.8.0",
-        "chalk": "^2.0.1",
-        "co": "^4.6.0",
-        "expect": "^24.8.0",
-        "is-generator-fn": "^2.0.0",
-        "jest-each": "^24.8.0",
-        "jest-matcher-utils": "^24.8.0",
-        "jest-message-util": "^24.8.0",
-        "jest-runtime": "^24.8.0",
-        "jest-snapshot": "^24.8.0",
-        "jest-util": "^24.8.0",
-        "pretty-format": "^24.8.0",
-        "throat": "^4.0.0"
-      },
-      "dependencies": {
-        "callsites": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
-          "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
+        },
+        "is-fullwidth-code-point": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+          "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
           "dev": true
         },
-        "ci-info": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
-          "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==",
+        "path-exists": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
+          "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==",
           "dev": true
         },
-        "is-ci": {
+        "require-main-filename": {
           "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz",
-          "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==",
+          "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz",
+          "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==",
+          "dev": true
+        },
+        "string-width": {
+          "version": "4.2.0",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz",
+          "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==",
           "dev": true,
           "requires": {
-            "ci-info": "^2.0.0"
+            "emoji-regex": "^8.0.0",
+            "is-fullwidth-code-point": "^3.0.0",
+            "strip-ansi": "^6.0.0"
           }
         },
-        "jest-message-util": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-24.8.0.tgz",
-          "integrity": "sha512-p2k71rf/b6ns8btdB0uVdljWo9h0ovpnEe05ZKWceQGfXYr4KkzgKo3PBi8wdnd9OtNh46VpNIJynUn/3MKm1g==",
+        "strip-ansi": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
+          "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
           "dev": true,
           "requires": {
-            "@babel/code-frame": "^7.0.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
-            "@types/stack-utils": "^1.0.1",
-            "chalk": "^2.0.1",
-            "micromatch": "^3.1.10",
-            "slash": "^2.0.0",
-            "stack-utils": "^1.0.1"
+            "ansi-regex": "^5.0.0"
           }
         },
-        "jest-util": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-24.8.0.tgz",
-          "integrity": "sha512-DYZeE+XyAnbNt0BG1OQqKy/4GVLPtzwGx5tsnDrFcax36rVE3lTA5fbvgmbVPUZf9w77AJ8otqR4VBbfFJkUZA==",
+        "strip-bom": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz",
+          "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
           "dev": true,
           "requires": {
-            "@jest/console": "^24.7.1",
-            "@jest/fake-timers": "^24.8.0",
-            "@jest/source-map": "^24.3.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
-            "callsites": "^3.0.0",
-            "chalk": "^2.0.1",
-            "graceful-fs": "^4.1.15",
-            "is-ci": "^2.0.0",
-            "mkdirp": "^0.5.1",
-            "slash": "^2.0.0",
-            "source-map": "^0.6.0"
+            "has-flag": "^4.0.0"
           }
         },
-        "slash": {
+        "which-module": {
           "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
-          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
+          "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz",
+          "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=",
           "dev": true
         },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-          "dev": true
-        }
-      }
-    },
-    "jest-leak-detector": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-24.8.0.tgz",
-      "integrity": "sha512-cG0yRSK8A831LN8lIHxI3AblB40uhv0z+SsQdW3GoMMVcK+sJwrIIyax5tu3eHHNJ8Fu6IMDpnLda2jhn2pD/g==",
-      "dev": true,
-      "requires": {
-        "pretty-format": "^24.8.0"
-      }
-    },
-    "jest-matcher-utils": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-24.8.0.tgz",
-      "integrity": "sha512-lex1yASY51FvUuHgm0GOVj7DCYEouWSlIYmCW7APSqB9v8mXmKSn5+sWVF0MhuASG0bnYY106/49JU1FZNl5hw==",
-      "dev": true,
-      "requires": {
-        "chalk": "^2.0.1",
-        "jest-diff": "^24.8.0",
-        "jest-get-type": "^24.8.0",
-        "pretty-format": "^24.8.0"
-      }
-    },
-    "jest-mock": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-24.8.0.tgz",
-      "integrity": "sha512-6kWugwjGjJw+ZkK4mDa0Df3sDlUTsV47MSrT0nGQ0RBWJbpODDQ8MHDVtGtUYBne3IwZUhtB7elxHspU79WH3A==",
-      "dev": true,
-      "requires": {
-        "@jest/types": "^24.8.0"
-      }
-    },
-    "jest-pnp-resolver": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.1.tgz",
-      "integrity": "sha512-pgFw2tm54fzgYvc/OHrnysABEObZCUNFnhjoRjaVOCN8NYc032/gVjPaHD4Aq6ApkSieWtfKAFQtmDKAmhupnQ==",
-      "dev": true
-    },
-    "jest-regex-util": {
-      "version": "24.3.0",
-      "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-24.3.0.tgz",
-      "integrity": "sha512-tXQR1NEOyGlfylyEjg1ImtScwMq8Oh3iJbGTjN7p0J23EuVX1MA8rwU69K4sLbCmwzgCUbVkm0FkSF9TdzOhtg==",
-      "dev": true
-    },
-    "jest-resolve": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-24.8.0.tgz",
-      "integrity": "sha512-+hjSzi1PoRvnuOICoYd5V/KpIQmkAsfjFO71458hQ2Whi/yf1GDeBOFj8Gxw4LrApHsVJvn5fmjcPdmoUHaVKw==",
-      "dev": true,
-      "requires": {
-        "@jest/types": "^24.8.0",
-        "browser-resolve": "^1.11.3",
-        "chalk": "^2.0.1",
-        "jest-pnp-resolver": "^1.2.1",
-        "realpath-native": "^1.1.0"
+        "wrap-ansi": {
+          "version": "6.2.0",
+          "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz",
+          "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.0.0",
+            "string-width": "^4.1.0",
+            "strip-ansi": "^6.0.0"
+          }
+        },
+        "y18n": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
+          "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
+          "dev": true
+        },
+        "yargs": {
+          "version": "15.4.1",
+          "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz",
+          "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==",
+          "dev": true,
+          "requires": {
+            "cliui": "^6.0.0",
+            "decamelize": "^1.2.0",
+            "find-up": "^4.1.0",
+            "get-caller-file": "^2.0.1",
+            "require-directory": "^2.1.1",
+            "require-main-filename": "^2.0.0",
+            "set-blocking": "^2.0.0",
+            "string-width": "^4.2.0",
+            "which-module": "^2.0.0",
+            "y18n": "^4.0.0",
+            "yargs-parser": "^18.1.2"
+          }
+        },
+        "yargs-parser": {
+          "version": "18.1.3",
+          "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz",
+          "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==",
+          "dev": true,
+          "requires": {
+            "camelcase": "^5.0.0",
+            "decamelize": "^1.2.0"
+          }
+        }
       }
     },
-    "jest-resolve-dependencies": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-24.8.0.tgz",
-      "integrity": "sha512-hyK1qfIf/krV+fSNyhyJeq3elVMhK9Eijlwy+j5jqmZ9QsxwKBiP6qukQxaHtK8k6zql/KYWwCTQ+fDGTIJauw==",
+    "jest-serializer": {
+      "version": "26.3.0",
+      "resolved": "https://registry.npmjs.org/jest-serializer/-/jest-serializer-26.3.0.tgz",
+      "integrity": "sha512-IDRBQBLPlKa4flg77fqg0n/pH87tcRKwe8zxOVTWISxGpPHYkRZ1dXKyh04JOja7gppc60+soKVZ791mruVdow==",
       "dev": true,
       "requires": {
-        "@jest/types": "^24.8.0",
-        "jest-regex-util": "^24.3.0",
-        "jest-snapshot": "^24.8.0"
+        "@types/node": "*",
+        "graceful-fs": "^4.2.4"
       }
     },
-    "jest-runner": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-24.8.0.tgz",
-      "integrity": "sha512-utFqC5BaA3JmznbissSs95X1ZF+d+4WuOWwpM9+Ak356YtMhHE/GXUondZdcyAAOTBEsRGAgH/0TwLzfI9h7ow==",
+    "jest-silent-reporter": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/jest-silent-reporter/-/jest-silent-reporter-0.1.2.tgz",
+      "integrity": "sha512-w/qc9NvWqdX0vZv6TUG4EE15d72+JxQJYh+3hqq8cTi3BnfBOtwNtL3T6TwkZSy/sfc3REW5niz0eSBPTIvWnA==",
       "dev": true,
       "requires": {
-        "@jest/console": "^24.7.1",
-        "@jest/environment": "^24.8.0",
-        "@jest/test-result": "^24.8.0",
-        "@jest/types": "^24.8.0",
-        "chalk": "^2.4.2",
-        "exit": "^0.1.2",
-        "graceful-fs": "^4.1.15",
-        "jest-config": "^24.8.0",
-        "jest-docblock": "^24.3.0",
-        "jest-haste-map": "^24.8.0",
-        "jest-jasmine2": "^24.8.0",
-        "jest-leak-detector": "^24.8.0",
-        "jest-message-util": "^24.8.0",
-        "jest-resolve": "^24.8.0",
-        "jest-runtime": "^24.8.0",
-        "jest-util": "^24.8.0",
-        "jest-worker": "^24.6.0",
-        "source-map-support": "^0.5.6",
-        "throat": "^4.0.0"
+        "chalk": "^2.3.1",
+        "jest-util": "^24.0.0"
       },
       "dependencies": {
-        "callsites": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
-          "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
-          "dev": true
-        },
-        "ci-info": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
-          "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==",
-          "dev": true
-        },
-        "is-ci": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz",
-          "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==",
+        "@jest/console": {
+          "version": "24.9.0",
+          "resolved": "https://registry.npmjs.org/@jest/console/-/console-24.9.0.tgz",
+          "integrity": "sha512-Zuj6b8TnKXi3q4ymac8EQfc3ea/uhLeCGThFqXeC8H9/raaH8ARPUTdId+XyGd03Z4In0/VjD2OYFcBF09fNLQ==",
           "dev": true,
           "requires": {
-            "ci-info": "^2.0.0"
+            "@jest/source-map": "^24.9.0",
+            "chalk": "^2.0.1",
+            "slash": "^2.0.0"
           }
         },
-        "jest-message-util": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-24.8.0.tgz",
-          "integrity": "sha512-p2k71rf/b6ns8btdB0uVdljWo9h0ovpnEe05ZKWceQGfXYr4KkzgKo3PBi8wdnd9OtNh46VpNIJynUn/3MKm1g==",
+        "@jest/fake-timers": {
+          "version": "24.9.0",
+          "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-24.9.0.tgz",
+          "integrity": "sha512-eWQcNa2YSwzXWIMC5KufBh3oWRIijrQFROsIqt6v/NS9Io/gknw1jsAC9c+ih/RQX4A3O7SeWAhQeN0goKhT9A==",
           "dev": true,
           "requires": {
-            "@babel/code-frame": "^7.0.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
-            "@types/stack-utils": "^1.0.1",
-            "chalk": "^2.0.1",
-            "micromatch": "^3.1.10",
-            "slash": "^2.0.0",
-            "stack-utils": "^1.0.1"
+            "@jest/types": "^24.9.0",
+            "jest-message-util": "^24.9.0",
+            "jest-mock": "^24.9.0"
           }
         },
-        "jest-util": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-24.8.0.tgz",
-          "integrity": "sha512-DYZeE+XyAnbNt0BG1OQqKy/4GVLPtzwGx5tsnDrFcax36rVE3lTA5fbvgmbVPUZf9w77AJ8otqR4VBbfFJkUZA==",
+        "@jest/source-map": {
+          "version": "24.9.0",
+          "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-24.9.0.tgz",
+          "integrity": "sha512-/Xw7xGlsZb4MJzNDgB7PW5crou5JqWiBQaz6xyPd3ArOg2nfn/PunV8+olXbbEZzNl591o5rWKE9BRDaFAuIBg==",
           "dev": true,
           "requires": {
-            "@jest/console": "^24.7.1",
-            "@jest/fake-timers": "^24.8.0",
-            "@jest/source-map": "^24.3.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
             "callsites": "^3.0.0",
-            "chalk": "^2.0.1",
             "graceful-fs": "^4.1.15",
-            "is-ci": "^2.0.0",
-            "mkdirp": "^0.5.1",
-            "slash": "^2.0.0",
             "source-map": "^0.6.0"
           }
         },
-        "slash": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
-          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
-          "dev": true
-        },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-          "dev": true
-        }
-      }
-    },
-    "jest-runtime": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-24.8.0.tgz",
-      "integrity": "sha512-Mq0aIXhvO/3bX44ccT+czU1/57IgOMyy80oM0XR/nyD5zgBcesF84BPabZi39pJVA6UXw+fY2Q1N+4BiVUBWOA==",
-      "dev": true,
-      "requires": {
-        "@jest/console": "^24.7.1",
-        "@jest/environment": "^24.8.0",
-        "@jest/source-map": "^24.3.0",
-        "@jest/transform": "^24.8.0",
-        "@jest/types": "^24.8.0",
-        "@types/yargs": "^12.0.2",
-        "chalk": "^2.0.1",
-        "exit": "^0.1.2",
-        "glob": "^7.1.3",
-        "graceful-fs": "^4.1.15",
-        "jest-config": "^24.8.0",
-        "jest-haste-map": "^24.8.0",
-        "jest-message-util": "^24.8.0",
-        "jest-mock": "^24.8.0",
-        "jest-regex-util": "^24.3.0",
-        "jest-resolve": "^24.8.0",
-        "jest-snapshot": "^24.8.0",
-        "jest-util": "^24.8.0",
-        "jest-validate": "^24.8.0",
-        "realpath-native": "^1.1.0",
-        "slash": "^2.0.0",
-        "strip-bom": "^3.0.0",
-        "yargs": "^12.0.2"
-      },
-      "dependencies": {
-        "ansi-regex": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
-          "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=",
-          "dev": true
-        },
-        "callsites": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
-          "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
-          "dev": true
-        },
-        "camelcase": {
-          "version": "5.3.1",
-          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
-          "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==",
-          "dev": true
-        },
-        "ci-info": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
-          "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==",
-          "dev": true
-        },
-        "cliui": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/cliui/-/cliui-4.1.0.tgz",
-          "integrity": "sha512-4FG+RSG9DL7uEwRUZXZn3SS34DiDPfzP0VOiEwtUWlE+AR2EIg+hSyvrIgUUfhdgR/UkAeW2QHgeP+hWrXs7jQ==",
+        "@jest/test-result": {
+          "version": "24.9.0",
+          "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-24.9.0.tgz",
+          "integrity": "sha512-XEFrHbBonBJ8dGp2JmF8kP/nQI/ImPpygKHwQ/SY+es59Z3L5PI4Qb9TQQMAEeYsThG1xF0k6tmG0tIKATNiiA==",
           "dev": true,
           "requires": {
-            "string-width": "^2.1.1",
-            "strip-ansi": "^4.0.0",
-            "wrap-ansi": "^2.0.0"
+            "@jest/console": "^24.9.0",
+            "@jest/types": "^24.9.0",
+            "@types/istanbul-lib-coverage": "^2.0.0"
           }
         },
-        "find-up": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz",
-          "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==",
+        "@jest/types": {
+          "version": "24.9.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz",
+          "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==",
           "dev": true,
           "requires": {
-            "locate-path": "^3.0.0"
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^1.1.1",
+            "@types/yargs": "^13.0.0"
           }
         },
-        "invert-kv": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-2.0.0.tgz",
-          "integrity": "sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==",
-          "dev": true
-        },
-        "is-ci": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz",
-          "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==",
+        "@types/yargs": {
+          "version": "13.0.10",
+          "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz",
+          "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==",
           "dev": true,
           "requires": {
-            "ci-info": "^2.0.0"
+            "@types/yargs-parser": "*"
           }
         },
-        "is-fullwidth-code-point": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
-          "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=",
-          "dev": true
-        },
         "jest-message-util": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-24.8.0.tgz",
-          "integrity": "sha512-p2k71rf/b6ns8btdB0uVdljWo9h0ovpnEe05ZKWceQGfXYr4KkzgKo3PBi8wdnd9OtNh46VpNIJynUn/3MKm1g==",
+          "version": "24.9.0",
+          "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-24.9.0.tgz",
+          "integrity": "sha512-oCj8FiZ3U0hTP4aSui87P4L4jC37BtQwUMqk+zk/b11FR19BJDeZsZAvIHutWnmtw7r85UmR3CEWZ0HWU2mAlw==",
           "dev": true,
           "requires": {
             "@babel/code-frame": "^7.0.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
+            "@jest/test-result": "^24.9.0",
+            "@jest/types": "^24.9.0",
             "@types/stack-utils": "^1.0.1",
             "chalk": "^2.0.1",
             "micromatch": "^3.1.10",
@@ -8564,17 +11540,26 @@
             "stack-utils": "^1.0.1"
           }
         },
+        "jest-mock": {
+          "version": "24.9.0",
+          "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-24.9.0.tgz",
+          "integrity": "sha512-3BEYN5WbSq9wd+SyLDES7AHnjH9A/ROBwmz7l2y+ol+NtSFO8DYiEBzoO1CeFc9a8DYy10EO4dDFVv/wN3zl1w==",
+          "dev": true,
+          "requires": {
+            "@jest/types": "^24.9.0"
+          }
+        },
         "jest-util": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-24.8.0.tgz",
-          "integrity": "sha512-DYZeE+XyAnbNt0BG1OQqKy/4GVLPtzwGx5tsnDrFcax36rVE3lTA5fbvgmbVPUZf9w77AJ8otqR4VBbfFJkUZA==",
+          "version": "24.9.0",
+          "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-24.9.0.tgz",
+          "integrity": "sha512-x+cZU8VRmOJxbA1K5oDBdxQmdq0OIdADarLxk0Mq+3XS4jgvhG/oKGWcIDCtPG0HgjxOYvF+ilPJQsAyXfbNOg==",
           "dev": true,
           "requires": {
-            "@jest/console": "^24.7.1",
-            "@jest/fake-timers": "^24.8.0",
-            "@jest/source-map": "^24.3.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
+            "@jest/console": "^24.9.0",
+            "@jest/fake-timers": "^24.9.0",
+            "@jest/source-map": "^24.9.0",
+            "@jest/test-result": "^24.9.0",
+            "@jest/types": "^24.9.0",
             "callsites": "^3.0.0",
             "chalk": "^2.0.1",
             "graceful-fs": "^4.1.15",
@@ -8584,385 +11569,544 @@
             "source-map": "^0.6.0"
           }
         },
-        "lcid": {
+        "mkdirp": {
+          "version": "0.5.5",
+          "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
+          "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
+          "dev": true,
+          "requires": {
+            "minimist": "^1.2.5"
+          }
+        },
+        "slash": {
           "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/lcid/-/lcid-2.0.0.tgz",
-          "integrity": "sha512-avPEb8P8EGnwXKClwsNUgryVjllcRqtMYa49NTsbQagYuT1DcXnl1915oxWjoyGrXR6zH/Y0Zc96xWsPcoDKeA==",
+          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
+          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
+          "dev": true
+        },
+        "source-map": {
+          "version": "0.6.1",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+          "dev": true
+        },
+        "stack-utils": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-1.0.2.tgz",
+          "integrity": "sha512-MTX+MeG5U994cazkjd/9KNAapsHnibjMLnfXodlkXw76JEea0UiNzrqidzo1emMwk7w5Qhc9jd4Bn9TBb1MFwA==",
+          "dev": true
+        }
+      }
+    },
+    "jest-snapshot": {
+      "version": "26.4.2",
+      "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-26.4.2.tgz",
+      "integrity": "sha512-N6Uub8FccKlf5SBFnL2Ri/xofbaA68Cc3MGjP/NuwgnsvWh+9hLIR/DhrxbSiKXMY9vUW5dI6EW1eHaDHqe9sg==",
+      "dev": true,
+      "requires": {
+        "@babel/types": "^7.0.0",
+        "@jest/types": "^26.3.0",
+        "@types/prettier": "^2.0.0",
+        "chalk": "^4.0.0",
+        "expect": "^26.4.2",
+        "graceful-fs": "^4.2.4",
+        "jest-diff": "^26.4.2",
+        "jest-get-type": "^26.3.0",
+        "jest-haste-map": "^26.3.0",
+        "jest-matcher-utils": "^26.4.2",
+        "jest-message-util": "^26.3.0",
+        "jest-resolve": "^26.4.0",
+        "natural-compare": "^1.4.0",
+        "pretty-format": "^26.4.2",
+        "semver": "^7.3.2"
+      },
+      "dependencies": {
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
           "dev": true,
           "requires": {
-            "invert-kv": "^2.0.0"
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
           }
         },
-        "locate-path": {
+        "@types/istanbul-reports": {
           "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz",
-          "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
           "dev": true,
           "requires": {
-            "p-locate": "^3.0.0",
-            "path-exists": "^3.0.0"
+            "@types/istanbul-lib-report": "*"
           }
         },
-        "os-locale": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-3.1.0.tgz",
-          "integrity": "sha512-Z8l3R4wYWM40/52Z+S265okfFj8Kt2cC2MKY+xNi3kFs+XGI7WXu/I309QQQYbRW4ijiZ+yxs9pqEhJh0DqW3Q==",
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
           "dev": true,
           "requires": {
-            "execa": "^1.0.0",
-            "lcid": "^2.0.0",
-            "mem": "^4.0.0"
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
           }
         },
-        "p-limit": {
-          "version": "2.2.0",
-          "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.0.tgz",
-          "integrity": "sha512-pZbTJpoUsCzV48Mc9Nh51VbwO0X9cuPFE8gYwx9BTCt9SF8/b7Zljd2fVgOxhIF/HDTKgpVzs+GPhyKfjLLFRQ==",
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
           "dev": true,
           "requires": {
-            "p-try": "^2.0.0"
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
           }
         },
-        "p-locate": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz",
-          "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==",
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
           "dev": true,
           "requires": {
-            "p-limit": "^2.0.0"
+            "color-name": "~1.1.4"
           }
         },
-        "p-try": {
-          "version": "2.2.0",
-          "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
-          "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==",
-          "dev": true
-        },
-        "path-exists": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
-          "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=",
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
           "dev": true
         },
-        "slash": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
-          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
+        "diff-sequences": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-26.3.0.tgz",
+          "integrity": "sha512-5j5vdRcw3CNctePNYN0Wy2e/JbWT6cAYnXv5OuqPhDpyCGc0uLu2TK0zOCJWNB9kOIfYMSpIulRaDgIi4HJ6Ig==",
           "dev": true
         },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
           "dev": true
         },
-        "string-width": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",
-          "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==",
-          "dev": true,
-          "requires": {
-            "is-fullwidth-code-point": "^2.0.0",
-            "strip-ansi": "^4.0.0"
-          }
-        },
-        "strip-ansi": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
-          "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
+        "jest-diff": {
+          "version": "26.4.2",
+          "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-26.4.2.tgz",
+          "integrity": "sha512-6T1XQY8U28WH0Z5rGpQ+VqZSZz8EN8rZcBtfvXaOkbwxIEeRre6qnuZQlbY1AJ4MKDxQF8EkrCvK+hL/VkyYLQ==",
           "dev": true,
           "requires": {
-            "ansi-regex": "^3.0.0"
+            "chalk": "^4.0.0",
+            "diff-sequences": "^26.3.0",
+            "jest-get-type": "^26.3.0",
+            "pretty-format": "^26.4.2"
           }
         },
-        "strip-bom": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
-          "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=",
-          "dev": true
-        },
-        "which-module": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz",
-          "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=",
+        "jest-get-type": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-26.3.0.tgz",
+          "integrity": "sha512-TpfaviN1R2pQWkIihlfEanwOXK0zcxrKEE4MlU6Tn7keoXdN6/3gK/xl0yEh8DOunn5pOVGKf8hB4R9gVh04ig==",
           "dev": true
         },
-        "yargs": {
-          "version": "12.0.5",
-          "resolved": "https://registry.npmjs.org/yargs/-/yargs-12.0.5.tgz",
-          "integrity": "sha512-Lhz8TLaYnxq/2ObqHDql8dX8CJi97oHxrjUcYtzKbbykPtVW9WB+poxI+NM2UIzsMgNCZTIf0AQwsjK5yMAqZw==",
+        "pretty-format": {
+          "version": "26.4.2",
+          "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-26.4.2.tgz",
+          "integrity": "sha512-zK6Gd8zDsEiVydOCGLkoBoZuqv8VTiHyAbKznXe/gaph/DAeZOmit9yMfgIz5adIgAMMs5XfoYSwAX3jcCO1tA==",
           "dev": true,
           "requires": {
-            "cliui": "^4.0.0",
-            "decamelize": "^1.2.0",
-            "find-up": "^3.0.0",
-            "get-caller-file": "^1.0.1",
-            "os-locale": "^3.0.0",
-            "require-directory": "^2.1.1",
-            "require-main-filename": "^1.0.1",
-            "set-blocking": "^2.0.0",
-            "string-width": "^2.0.0",
-            "which-module": "^2.0.0",
-            "y18n": "^3.2.1 || ^4.0.0",
-            "yargs-parser": "^11.1.1"
+            "@jest/types": "^26.3.0",
+            "ansi-regex": "^5.0.0",
+            "ansi-styles": "^4.0.0",
+            "react-is": "^16.12.0"
           }
         },
-        "yargs-parser": {
-          "version": "11.1.1",
-          "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-11.1.1.tgz",
-          "integrity": "sha512-C6kB/WJDiaxONLJQnF8ccx9SEeoTTLek8RVbaOIsrAUS8VrBEXfmeSnCZxygc+XC2sNMBIwOOnfcxiynjHsVSQ==",
+        "semver": {
+          "version": "7.3.2",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz",
+          "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
           "dev": true,
           "requires": {
-            "camelcase": "^5.0.0",
-            "decamelize": "^1.2.0"
+            "has-flag": "^4.0.0"
           }
         }
       }
     },
-    "jest-serializer": {
-      "version": "24.4.0",
-      "resolved": "https://registry.npmjs.org/jest-serializer/-/jest-serializer-24.4.0.tgz",
-      "integrity": "sha512-k//0DtglVstc1fv+GY/VHDIjrtNjdYvYjMlbLUed4kxrE92sIUewOi5Hj3vrpB8CXfkJntRPDRjCrCvUhBdL8Q==",
-      "dev": true
-    },
-    "jest-silent-reporter": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/jest-silent-reporter/-/jest-silent-reporter-0.1.2.tgz",
-      "integrity": "sha512-w/qc9NvWqdX0vZv6TUG4EE15d72+JxQJYh+3hqq8cTi3BnfBOtwNtL3T6TwkZSy/sfc3REW5niz0eSBPTIvWnA==",
-      "dev": true,
-      "requires": {
-        "chalk": "^2.3.1",
-        "jest-util": "^24.0.0"
-      }
-    },
-    "jest-snapshot": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-24.8.0.tgz",
-      "integrity": "sha512-5ehtWoc8oU9/cAPe6fez6QofVJLBKyqkY2+TlKTOf0VllBB/mqUNdARdcjlZrs9F1Cv+/HKoCS/BknT0+tmfPg==",
+    "jest-util": {
+      "version": "26.3.0",
+      "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-26.3.0.tgz",
+      "integrity": "sha512-4zpn6bwV0+AMFN0IYhH/wnzIQzRaYVrz1A8sYnRnj4UXDXbOVtWmlaZkO9mipFqZ13okIfN87aDoJWB7VH6hcw==",
       "dev": true,
       "requires": {
-        "@babel/types": "^7.0.0",
-        "@jest/types": "^24.8.0",
-        "chalk": "^2.0.1",
-        "expect": "^24.8.0",
-        "jest-diff": "^24.8.0",
-        "jest-matcher-utils": "^24.8.0",
-        "jest-message-util": "^24.8.0",
-        "jest-resolve": "^24.8.0",
-        "mkdirp": "^0.5.1",
-        "natural-compare": "^1.4.0",
-        "pretty-format": "^24.8.0",
-        "semver": "^5.5.0"
+        "@jest/types": "^26.3.0",
+        "@types/node": "*",
+        "chalk": "^4.0.0",
+        "graceful-fs": "^4.2.4",
+        "is-ci": "^2.0.0",
+        "micromatch": "^4.0.2"
       },
       "dependencies": {
-        "jest-message-util": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-24.8.0.tgz",
-          "integrity": "sha512-p2k71rf/b6ns8btdB0uVdljWo9h0ovpnEe05ZKWceQGfXYr4KkzgKo3PBi8wdnd9OtNh46VpNIJynUn/3MKm1g==",
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
+          }
+        },
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-report": "*"
+          }
+        },
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
+        },
+        "braces": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
+          "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+          "dev": true,
+          "requires": {
+            "fill-range": "^7.0.1"
+          }
+        },
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
           "dev": true,
           "requires": {
-            "@babel/code-frame": "^7.0.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
-            "@types/stack-utils": "^1.0.1",
-            "chalk": "^2.0.1",
-            "micromatch": "^3.1.10",
-            "slash": "^2.0.0",
-            "stack-utils": "^1.0.1"
+            "color-name": "~1.1.4"
           }
         },
-        "slash": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
-          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
           "dev": true
-        }
-      }
-    },
-    "jest-util": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-24.8.0.tgz",
-      "integrity": "sha512-DYZeE+XyAnbNt0BG1OQqKy/4GVLPtzwGx5tsnDrFcax36rVE3lTA5fbvgmbVPUZf9w77AJ8otqR4VBbfFJkUZA==",
-      "dev": true,
-      "requires": {
-        "@jest/console": "^24.7.1",
-        "@jest/fake-timers": "^24.8.0",
-        "@jest/source-map": "^24.3.0",
-        "@jest/test-result": "^24.8.0",
-        "@jest/types": "^24.8.0",
-        "callsites": "^3.0.0",
-        "chalk": "^2.0.1",
-        "graceful-fs": "^4.1.15",
-        "is-ci": "^2.0.0",
-        "mkdirp": "^0.5.1",
-        "slash": "^2.0.0",
-        "source-map": "^0.6.0"
-      },
-      "dependencies": {
-        "callsites": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
-          "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
+        },
+        "fill-range": {
+          "version": "7.0.1",
+          "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
+          "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+          "dev": true,
+          "requires": {
+            "to-regex-range": "^5.0.1"
+          }
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
           "dev": true
         },
-        "ci-info": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
-          "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==",
+        "is-number": {
+          "version": "7.0.0",
+          "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
+          "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
           "dev": true
         },
-        "is-ci": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz",
-          "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==",
+        "micromatch": {
+          "version": "4.0.2",
+          "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.2.tgz",
+          "integrity": "sha512-y7FpHSbMUMoyPbYUSzO6PaZ6FyRnQOpHuKwbo1G+Knck95XVU4QAiKdGEnj5wwoS7PlOgthX/09u5iFJ+aYf5Q==",
           "dev": true,
           "requires": {
-            "ci-info": "^2.0.0"
+            "braces": "^3.0.1",
+            "picomatch": "^2.0.5"
           }
         },
-        "slash": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
-          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
-          "dev": true
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
         },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-          "dev": true
+        "to-regex-range": {
+          "version": "5.0.1",
+          "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
+          "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
+          "dev": true,
+          "requires": {
+            "is-number": "^7.0.0"
+          }
         }
       }
     },
     "jest-validate": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-24.8.0.tgz",
-      "integrity": "sha512-+/N7VOEMW1Vzsrk3UWBDYTExTPwf68tavEPKDnJzrC6UlHtUDU/fuEdXqFoHzv9XnQ+zW6X3qMZhJ3YexfeLDA==",
+      "version": "26.4.2",
+      "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-26.4.2.tgz",
+      "integrity": "sha512-blft+xDX7XXghfhY0mrsBCYhX365n8K5wNDC4XAcNKqqjEzsRUSXP44m6PL0QJEW2crxQFLLztVnJ4j7oPlQrQ==",
       "dev": true,
       "requires": {
-        "@jest/types": "^24.8.0",
-        "camelcase": "^5.0.0",
-        "chalk": "^2.0.1",
-        "jest-get-type": "^24.8.0",
-        "leven": "^2.1.0",
-        "pretty-format": "^24.8.0"
+        "@jest/types": "^26.3.0",
+        "camelcase": "^6.0.0",
+        "chalk": "^4.0.0",
+        "jest-get-type": "^26.3.0",
+        "leven": "^3.1.0",
+        "pretty-format": "^26.4.2"
       },
       "dependencies": {
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
+          }
+        },
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-report": "*"
+          }
+        },
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
+        },
         "camelcase": {
-          "version": "5.3.1",
-          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
-          "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==",
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.0.0.tgz",
+          "integrity": "sha512-8KMDF1Vz2gzOq54ONPJS65IvTUaB1cHJ2DMM7MbPmLZljDH1qpzzLsWdiN9pHh6qvkRVDTi/07+eNGch/oLU4w==",
+          "dev": true
+        },
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "jest-get-type": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-26.3.0.tgz",
+          "integrity": "sha512-TpfaviN1R2pQWkIihlfEanwOXK0zcxrKEE4MlU6Tn7keoXdN6/3gK/xl0yEh8DOunn5pOVGKf8hB4R9gVh04ig==",
           "dev": true
+        },
+        "pretty-format": {
+          "version": "26.4.2",
+          "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-26.4.2.tgz",
+          "integrity": "sha512-zK6Gd8zDsEiVydOCGLkoBoZuqv8VTiHyAbKznXe/gaph/DAeZOmit9yMfgIz5adIgAMMs5XfoYSwAX3jcCO1tA==",
+          "dev": true,
+          "requires": {
+            "@jest/types": "^26.3.0",
+            "ansi-regex": "^5.0.0",
+            "ansi-styles": "^4.0.0",
+            "react-is": "^16.12.0"
+          }
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
         }
       }
     },
     "jest-watcher": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-24.8.0.tgz",
-      "integrity": "sha512-SBjwHt5NedQoVu54M5GEx7cl7IGEFFznvd/HNT8ier7cCAx/Qgu9ZMlaTQkvK22G1YOpcWBLQPFSImmxdn3DAw==",
-      "dev": true,
-      "requires": {
-        "@jest/test-result": "^24.8.0",
-        "@jest/types": "^24.8.0",
-        "@types/yargs": "^12.0.9",
-        "ansi-escapes": "^3.0.0",
-        "chalk": "^2.0.1",
-        "jest-util": "^24.8.0",
-        "string-length": "^2.0.0"
+      "version": "26.3.0",
+      "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-26.3.0.tgz",
+      "integrity": "sha512-XnLdKmyCGJ3VoF6G/p5ohbJ04q/vv5aH9ENI+i6BL0uu9WWB6Z7Z2lhQQk0d2AVZcRGp1yW+/TsoToMhBFPRdQ==",
+      "dev": true,
+      "requires": {
+        "@jest/test-result": "^26.3.0",
+        "@jest/types": "^26.3.0",
+        "@types/node": "*",
+        "ansi-escapes": "^4.2.1",
+        "chalk": "^4.0.0",
+        "jest-util": "^26.3.0",
+        "string-length": "^4.0.1"
       },
       "dependencies": {
-        "callsites": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
-          "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
-          "dev": true
+        "@jest/types": {
+          "version": "26.3.0",
+          "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",
+          "integrity": "sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-coverage": "^2.0.0",
+            "@types/istanbul-reports": "^3.0.0",
+            "@types/node": "*",
+            "@types/yargs": "^15.0.0",
+            "chalk": "^4.0.0"
+          }
         },
-        "ci-info": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
-          "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==",
-          "dev": true
+        "@types/istanbul-reports": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.0.tgz",
+          "integrity": "sha512-nwKNbvnwJ2/mndE9ItP/zc2TCzw6uuodnF4EHYWD+gCQDVBuRQL5UzbZD0/ezy1iKsFU2ZQiDqg4M9dN4+wZgA==",
+          "dev": true,
+          "requires": {
+            "@types/istanbul-lib-report": "*"
+          }
         },
-        "is-ci": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz",
-          "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==",
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
           "dev": true,
           "requires": {
-            "ci-info": "^2.0.0"
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
           }
         },
-        "jest-util": {
-          "version": "24.8.0",
-          "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-24.8.0.tgz",
-          "integrity": "sha512-DYZeE+XyAnbNt0BG1OQqKy/4GVLPtzwGx5tsnDrFcax36rVE3lTA5fbvgmbVPUZf9w77AJ8otqR4VBbfFJkUZA==",
+        "chalk": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
           "dev": true,
           "requires": {
-            "@jest/console": "^24.7.1",
-            "@jest/fake-timers": "^24.8.0",
-            "@jest/source-map": "^24.3.0",
-            "@jest/test-result": "^24.8.0",
-            "@jest/types": "^24.8.0",
-            "callsites": "^3.0.0",
-            "chalk": "^2.0.1",
-            "graceful-fs": "^4.1.15",
-            "is-ci": "^2.0.0",
-            "mkdirp": "^0.5.1",
-            "slash": "^2.0.0",
-            "source-map": "^0.6.0"
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
           }
         },
-        "slash": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
-          "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
           "dev": true
         },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
           "dev": true
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
         }
       }
     },
     "jest-worker": {
-      "version": "24.6.0",
-      "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-24.6.0.tgz",
-      "integrity": "sha512-jDwgW5W9qGNvpI1tNnvajh0a5IE/PuGLFmHk6aR/BZFz8tSgGw17GsDPXAJ6p91IvYDjOw8GpFbvvZGAK+DPQQ==",
+      "version": "26.3.0",
+      "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-26.3.0.tgz",
+      "integrity": "sha512-Vmpn2F6IASefL+DVBhPzI2J9/GJUsqzomdeN+P+dK8/jKxbh8R3BtFnx3FIta7wYlPU62cpJMJQo4kuOowcMnw==",
       "dev": true,
       "requires": {
-        "merge-stream": "^1.0.1",
-        "supports-color": "^6.1.0"
+        "@types/node": "*",
+        "merge-stream": "^2.0.0",
+        "supports-color": "^7.0.0"
       },
       "dependencies": {
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
         "supports-color": {
-          "version": "6.1.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz",
-          "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==",
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
           "dev": true,
           "requires": {
-            "has-flag": "^3.0.0"
+            "has-flag": "^4.0.0"
           }
         }
       }
     },
-    "jquery": {
-      "version": "2.2.4",
-      "resolved": "https://registry.npmjs.org/jquery/-/jquery-2.2.4.tgz",
-      "integrity": "sha1-LInWiJterFIqfuoywUUhVZxsvwI=",
-      "dev": true
-    },
     "js-tokens": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.2.tgz",
-      "integrity": "sha1-mGbfOVECEw449/mWvOtlRDIJwls=",
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
+      "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
       "dev": true
     },
     "js-yaml": {
-      "version": "3.12.1",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.12.1.tgz",
-      "integrity": "sha512-um46hB9wNOKlwkHgiuyEVAybXBjwFUV0Z/RaHJblRd9DXltue9FTYvzCr9ErQrK9Adz5MU4gHWVaNUfdmrC8qA==",
+      "version": "3.14.0",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.0.tgz",
+      "integrity": "sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A==",
       "dev": true,
       "requires": {
         "argparse": "^1.0.7",
@@ -8976,37 +12120,56 @@
       "dev": true
     },
     "jsdom": {
-      "version": "11.12.0",
-      "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-11.12.0.tgz",
-      "integrity": "sha512-y8Px43oyiBM13Zc1z780FrfNLJCXTL40EWlty/LXUtcjykRBNgLlCjWXpfSPBl2iv+N7koQN+dvqszHZgT/Fjw==",
-      "dev": true,
-      "requires": {
-        "abab": "^2.0.0",
-        "acorn": "^5.5.3",
-        "acorn-globals": "^4.1.0",
-        "array-equal": "^1.0.0",
-        "cssom": ">= 0.3.2 < 0.4.0",
-        "cssstyle": "^1.0.0",
-        "data-urls": "^1.0.0",
-        "domexception": "^1.0.1",
-        "escodegen": "^1.9.1",
-        "html-encoding-sniffer": "^1.0.2",
-        "left-pad": "^1.3.0",
-        "nwsapi": "^2.0.7",
-        "parse5": "4.0.0",
-        "pn": "^1.1.0",
-        "request": "^2.87.0",
-        "request-promise-native": "^1.0.5",
-        "sax": "^1.2.4",
-        "symbol-tree": "^3.2.2",
-        "tough-cookie": "^2.3.4",
-        "w3c-hr-time": "^1.0.1",
-        "webidl-conversions": "^4.0.2",
-        "whatwg-encoding": "^1.0.3",
-        "whatwg-mimetype": "^2.1.0",
-        "whatwg-url": "^6.4.1",
-        "ws": "^5.2.0",
+      "version": "16.4.0",
+      "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.4.0.tgz",
+      "integrity": "sha512-lYMm3wYdgPhrl7pDcRmvzPhhrGVBeVhPIqeHjzeiHN3DFmD1RBpbExbi8vU7BJdH8VAZYovR8DMt0PNNDM7k8w==",
+      "dev": true,
+      "requires": {
+        "abab": "^2.0.3",
+        "acorn": "^7.1.1",
+        "acorn-globals": "^6.0.0",
+        "cssom": "^0.4.4",
+        "cssstyle": "^2.2.0",
+        "data-urls": "^2.0.0",
+        "decimal.js": "^10.2.0",
+        "domexception": "^2.0.1",
+        "escodegen": "^1.14.1",
+        "html-encoding-sniffer": "^2.0.1",
+        "is-potential-custom-element-name": "^1.0.0",
+        "nwsapi": "^2.2.0",
+        "parse5": "5.1.1",
+        "request": "^2.88.2",
+        "request-promise-native": "^1.0.8",
+        "saxes": "^5.0.0",
+        "symbol-tree": "^3.2.4",
+        "tough-cookie": "^3.0.1",
+        "w3c-hr-time": "^1.0.2",
+        "w3c-xmlserializer": "^2.0.0",
+        "webidl-conversions": "^6.1.0",
+        "whatwg-encoding": "^1.0.5",
+        "whatwg-mimetype": "^2.3.0",
+        "whatwg-url": "^8.0.0",
+        "ws": "^7.2.3",
         "xml-name-validator": "^3.0.0"
+      },
+      "dependencies": {
+        "acorn": {
+          "version": "7.4.0",
+          "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz",
+          "integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==",
+          "dev": true
+        },
+        "tough-cookie": {
+          "version": "3.0.1",
+          "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-3.0.1.tgz",
+          "integrity": "sha512-yQyJ0u4pZsv9D4clxO69OEjLWYw+jbgspjTue4lTQZLfV0c5l1VmK2y1JK8E9ahdpltPOaAThPcp5nKPUgSnsg==",
+          "dev": true,
+          "requires": {
+            "ip-regex": "^2.1.0",
+            "psl": "^1.1.28",
+            "punycode": "^2.1.1"
+          }
+        }
       }
     },
     "jsesc": {
@@ -9032,6 +12195,12 @@
       "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==",
       "dev": true
     },
+    "json-parse-even-better-errors": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz",
+      "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==",
+      "dev": true
+    },
     "json-schema": {
       "version": "0.2.3",
       "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
@@ -9057,12 +12226,12 @@
       "dev": true
     },
     "json5": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/json5/-/json5-2.1.0.tgz",
-      "integrity": "sha512-8Mh9h6xViijj36g7Dxi+Y4S6hNGV96vcJZr/SrlHh1LR/pEn/8j/+qIBbs44YKl69Lrfctp4QD+AdWLTMqEZAQ==",
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/json5/-/json5-2.1.3.tgz",
+      "integrity": "sha512-KXPvOm8K9IJKFM0bmdn8QXh7udDh1g/giieX0NLCaMnb4hEiVFqnop2ImTXCc5e0/oHz3LTqmHGtExn5hfMkOA==",
       "dev": true,
       "requires": {
-        "minimist": "^1.2.0"
+        "minimist": "^1.2.5"
       }
     },
     "jsonfile": {
@@ -9074,12 +12243,6 @@
         "graceful-fs": "^4.1.6"
       }
     },
-    "jsonify": {
-      "version": "0.0.0",
-      "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz",
-      "integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM=",
-      "dev": true
-    },
     "jsonparse": {
       "version": "1.3.1",
       "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz",
@@ -9105,9 +12268,9 @@
       "dev": true
     },
     "kind-of": {
-      "version": "6.0.2",
-      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
-      "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==",
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
+      "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==",
       "dev": true
     },
     "klaw": {
@@ -9141,156 +12304,57 @@
       "integrity": "sha1-9plf4PggOS9hOWvolGJAe7dxaOQ=",
       "dev": true,
       "requires": {
-        "readable-stream": "^2.0.5"
-      }
-    },
-    "lcid": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz",
-      "integrity": "sha1-MIrMr6C8SDo4Z7S28rlQYlHRuDU=",
-      "dev": true,
-      "requires": {
-        "invert-kv": "^1.0.0"
-      }
-    },
-    "lcov-parse": {
-      "version": "0.0.10",
-      "resolved": "https://registry.npmjs.org/lcov-parse/-/lcov-parse-0.0.10.tgz",
-      "integrity": "sha1-GwuP+ayceIklBYK3C3ExXZ2m2aM=",
-      "dev": true
-    },
-    "lead": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/lead/-/lead-1.0.0.tgz",
-      "integrity": "sha1-bxT5mje+Op3XhPVJVpDlkDRm7kI=",
-      "dev": true,
-      "requires": {
-        "flush-write-stream": "^1.0.2"
-      }
-    },
-    "left-pad": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz",
-      "integrity": "sha512-XI5MPzVNApjAyhQzphX8BkmKsKUxD4LdyK24iZeQGinBN9yTQT3bFlCBy/aVx2HrNcqQGsdot8ghrjyrvMCoEA==",
-      "dev": true
-    },
-    "lerna": {
-      "version": "3.10.7",
-      "resolved": "https://registry.npmjs.org/lerna/-/lerna-3.10.7.tgz",
-      "integrity": "sha512-ha/dehl/L3Nw0pbdir5z6Hrv2oYBg5ym2fTcuk8HCLe7Zdb/ylIHdrgW8CU9eTVZkwr4et8RdVtxFA/+xa65/Q==",
-      "dev": true,
-      "requires": {
-        "@lerna/add": "3.10.6",
-        "@lerna/bootstrap": "3.10.6",
-        "@lerna/changed": "3.10.6",
-        "@lerna/clean": "3.10.6",
-        "@lerna/cli": "3.10.7",
-        "@lerna/create": "3.10.6",
-        "@lerna/diff": "3.10.6",
-        "@lerna/exec": "3.10.6",
-        "@lerna/import": "3.10.6",
-        "@lerna/init": "3.10.6",
-        "@lerna/link": "3.10.6",
-        "@lerna/list": "3.10.6",
-        "@lerna/publish": "3.10.7",
-        "@lerna/run": "3.10.6",
-        "@lerna/version": "3.10.6",
-        "import-local": "^1.0.0",
-        "libnpm": "^2.0.1"
-      }
-    },
-    "leven": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/leven/-/leven-2.1.0.tgz",
-      "integrity": "sha1-wuep93IJTe6dNCAq6KzORoeHVYA=",
-      "dev": true
-    },
-    "levn": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz",
-      "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=",
-      "dev": true,
-      "requires": {
-        "prelude-ls": "~1.1.2",
-        "type-check": "~0.3.2"
-      }
-    },
-    "libnpm": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/libnpm/-/libnpm-2.0.1.tgz",
-      "integrity": "sha512-qTKoxyJvpBxHZQB6k0AhSLajyXq9ZE/lUsZzuHAplr2Bpv9G+k4YuYlExYdUCeVRRGqcJt8hvkPh4tBwKoV98w==",
-      "dev": true,
-      "requires": {
-        "bin-links": "^1.1.2",
-        "bluebird": "^3.5.3",
-        "find-npm-prefix": "^1.0.2",
-        "libnpmaccess": "^3.0.1",
-        "libnpmconfig": "^1.2.1",
-        "libnpmhook": "^5.0.2",
-        "libnpmorg": "^1.0.0",
-        "libnpmpublish": "^1.1.0",
-        "libnpmsearch": "^2.0.0",
-        "libnpmteam": "^1.0.1",
-        "lock-verify": "^2.0.2",
-        "npm-lifecycle": "^2.1.0",
-        "npm-logical-tree": "^1.2.1",
-        "npm-package-arg": "^6.1.0",
-        "npm-profile": "^4.0.1",
-        "npm-registry-fetch": "^3.8.0",
-        "npmlog": "^4.1.2",
-        "pacote": "^9.2.3",
-        "read-package-json": "^2.0.13",
-        "stringify-package": "^1.0.0"
-      }
-    },
-    "libnpmaccess": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/libnpmaccess/-/libnpmaccess-3.0.1.tgz",
-      "integrity": "sha512-RlZ7PNarCBt+XbnP7R6PoVgOq9t+kou5rvhaInoNibhPO7eMlRfS0B8yjatgn2yaHIwWNyoJDolC/6Lc5L/IQA==",
-      "dev": true,
-      "requires": {
-        "aproba": "^2.0.0",
-        "get-stream": "^4.0.0",
-        "npm-package-arg": "^6.1.0",
-        "npm-registry-fetch": "^3.8.0"
-      },
-      "dependencies": {
-        "aproba": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz",
-          "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==",
-          "dev": true
-        },
-        "get-stream": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz",
-          "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==",
-          "dev": true,
-          "requires": {
-            "pump": "^3.0.0"
-          }
-        },
-        "pump": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-          "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
-          "dev": true,
-          "requires": {
-            "end-of-stream": "^1.1.0",
-            "once": "^1.3.1"
-          }
-        }
+        "readable-stream": "^2.0.5"
       }
     },
-    "libnpmconfig": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/libnpmconfig/-/libnpmconfig-1.2.1.tgz",
-      "integrity": "sha512-9esX8rTQAHqarx6qeZqmGQKBNZR5OIbl/Ayr0qQDy3oXja2iFVQQI81R6GZ2a02bSNZ9p3YOGX1O6HHCb1X7kA==",
+    "lcid": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz",
+      "integrity": "sha1-MIrMr6C8SDo4Z7S28rlQYlHRuDU=",
       "dev": true,
       "requires": {
-        "figgy-pudding": "^3.5.1",
-        "find-up": "^3.0.0",
-        "ini": "^1.3.5"
+        "invert-kv": "^1.0.0"
+      }
+    },
+    "lcov-parse": {
+      "version": "0.0.10",
+      "resolved": "https://registry.npmjs.org/lcov-parse/-/lcov-parse-0.0.10.tgz",
+      "integrity": "sha1-GwuP+ayceIklBYK3C3ExXZ2m2aM=",
+      "dev": true
+    },
+    "lead": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/lead/-/lead-1.0.0.tgz",
+      "integrity": "sha1-bxT5mje+Op3XhPVJVpDlkDRm7kI=",
+      "dev": true,
+      "requires": {
+        "flush-write-stream": "^1.0.2"
+      }
+    },
+    "lerna": {
+      "version": "3.22.1",
+      "resolved": "https://registry.npmjs.org/lerna/-/lerna-3.22.1.tgz",
+      "integrity": "sha512-vk1lfVRFm+UuEFA7wkLKeSF7Iz13W+N/vFd48aW2yuS7Kv0RbNm2/qcDPV863056LMfkRlsEe+QYOw3palj5Lg==",
+      "dev": true,
+      "requires": {
+        "@lerna/add": "3.21.0",
+        "@lerna/bootstrap": "3.21.0",
+        "@lerna/changed": "3.21.0",
+        "@lerna/clean": "3.21.0",
+        "@lerna/cli": "3.18.5",
+        "@lerna/create": "3.22.0",
+        "@lerna/diff": "3.21.0",
+        "@lerna/exec": "3.21.0",
+        "@lerna/import": "3.22.0",
+        "@lerna/info": "3.21.0",
+        "@lerna/init": "3.21.0",
+        "@lerna/link": "3.21.0",
+        "@lerna/list": "3.21.0",
+        "@lerna/publish": "3.22.1",
+        "@lerna/run": "3.21.0",
+        "@lerna/version": "3.22.1",
+        "import-local": "^2.0.0",
+        "npmlog": "^4.1.2"
       },
       "dependencies": {
         "find-up": {
@@ -9302,6 +12366,16 @@
             "locate-path": "^3.0.0"
           }
         },
+        "import-local": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/import-local/-/import-local-2.0.0.tgz",
+          "integrity": "sha512-b6s04m3O+s3CGSbqDIyP4R6aAwAeYlVq9+WUWep6iHa8ETRf9yei1U48C5MmfJmV9AiLYYBKPMq/W+/WRpQmCQ==",
+          "dev": true,
+          "requires": {
+            "pkg-dir": "^3.0.0",
+            "resolve-cwd": "^2.0.0"
+          }
+        },
         "locate-path": {
           "version": "3.0.0",
           "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz",
@@ -9312,15 +12386,6 @@
             "path-exists": "^3.0.0"
           }
         },
-        "p-limit": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.1.0.tgz",
-          "integrity": "sha512-NhURkNcrVB+8hNfLuysU8enY5xn2KXphsHBaC2YmRNTZRc7RWusw6apSpdEj3jo4CMb6W9nrF6tTnsJsJeyu6g==",
-          "dev": true,
-          "requires": {
-            "p-try": "^2.0.0"
-          }
-        },
         "p-locate": {
           "version": "3.0.0",
           "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz",
@@ -9330,211 +12395,52 @@
             "p-limit": "^2.0.0"
           }
         },
-        "p-try": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.0.0.tgz",
-          "integrity": "sha512-hMp0onDKIajHfIkdRk3P4CdCmErkYAxxDtP3Wx/4nZ3aGlau2VKh3mZpcuFkH27WQkL/3WBCPOktzA9ZOAnMQQ==",
-          "dev": true
-        },
         "path-exists": {
           "version": "3.0.0",
           "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
           "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=",
           "dev": true
-        }
-      }
-    },
-    "libnpmhook": {
-      "version": "5.0.2",
-      "resolved": "https://registry.npmjs.org/libnpmhook/-/libnpmhook-5.0.2.tgz",
-      "integrity": "sha512-vLenmdFWhRfnnZiNFPNMog6CK7Ujofy2TWiM2CrpZUjBRIhHkJeDaAbJdYCT6W4lcHtyrJR8yXW8KFyq6UAp1g==",
-      "dev": true,
-      "requires": {
-        "aproba": "^2.0.0",
-        "figgy-pudding": "^3.4.1",
-        "get-stream": "^4.0.0",
-        "npm-registry-fetch": "^3.8.0"
-      },
-      "dependencies": {
-        "aproba": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz",
-          "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==",
-          "dev": true
         },
-        "get-stream": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz",
-          "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==",
-          "dev": true,
-          "requires": {
-            "pump": "^3.0.0"
-          }
-        },
-        "pump": {
+        "pkg-dir": {
           "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-          "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
-          "dev": true,
-          "requires": {
-            "end-of-stream": "^1.1.0",
-            "once": "^1.3.1"
-          }
-        }
-      }
-    },
-    "libnpmorg": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/libnpmorg/-/libnpmorg-1.0.0.tgz",
-      "integrity": "sha512-o+4eVJBoDGMgRwh2lJY0a8pRV2c/tQM/SxlqXezjcAg26Qe9jigYVs+Xk0vvlYDWCDhP0g74J8UwWeAgsB7gGw==",
-      "dev": true,
-      "requires": {
-        "aproba": "^2.0.0",
-        "figgy-pudding": "^3.4.1",
-        "get-stream": "^4.0.0",
-        "npm-registry-fetch": "^3.8.0"
-      },
-      "dependencies": {
-        "aproba": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz",
-          "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==",
-          "dev": true
-        },
-        "get-stream": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz",
-          "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==",
+          "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz",
+          "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==",
           "dev": true,
           "requires": {
-            "pump": "^3.0.0"
+            "find-up": "^3.0.0"
           }
         },
-        "pump": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-          "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
-          "dev": true,
-          "requires": {
-            "end-of-stream": "^1.1.0",
-            "once": "^1.3.1"
-          }
-        }
-      }
-    },
-    "libnpmpublish": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/libnpmpublish/-/libnpmpublish-1.1.1.tgz",
-      "integrity": "sha512-nefbvJd/wY38zdt+b9SHL6171vqBrMtZ56Gsgfd0duEKb/pB8rDT4/ObUQLrHz1tOfht1flt2zM+UGaemzAG5g==",
-      "dev": true,
-      "requires": {
-        "aproba": "^2.0.0",
-        "figgy-pudding": "^3.5.1",
-        "get-stream": "^4.0.0",
-        "lodash.clonedeep": "^4.5.0",
-        "normalize-package-data": "^2.4.0",
-        "npm-package-arg": "^6.1.0",
-        "npm-registry-fetch": "^3.8.0",
-        "semver": "^5.5.1",
-        "ssri": "^6.0.1"
-      },
-      "dependencies": {
-        "aproba": {
+        "resolve-cwd": {
           "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz",
-          "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==",
-          "dev": true
-        },
-        "get-stream": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz",
-          "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==",
+          "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-2.0.0.tgz",
+          "integrity": "sha1-AKn3OHVW4nA46uIyyqNypqWbZlo=",
           "dev": true,
           "requires": {
-            "pump": "^3.0.0"
+            "resolve-from": "^3.0.0"
           }
         },
-        "pump": {
+        "resolve-from": {
           "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-          "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
-          "dev": true,
-          "requires": {
-            "end-of-stream": "^1.1.0",
-            "once": "^1.3.1"
-          }
+          "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-3.0.0.tgz",
+          "integrity": "sha1-six699nWiBvItuZTM17rywoYh0g=",
+          "dev": true
         }
       }
     },
-    "libnpmsearch": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/libnpmsearch/-/libnpmsearch-2.0.0.tgz",
-      "integrity": "sha512-vd+JWbTGzOSfiOc+72MU6y7WqmBXn49egCCrIXp27iE/88bX8EpG64ST1blWQI1bSMUr9l1AKPMVsqa2tS5KWA==",
-      "dev": true,
-      "requires": {
-        "figgy-pudding": "^3.5.1",
-        "get-stream": "^4.0.0",
-        "npm-registry-fetch": "^3.8.0"
-      },
-      "dependencies": {
-        "get-stream": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz",
-          "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==",
-          "dev": true,
-          "requires": {
-            "pump": "^3.0.0"
-          }
-        },
-        "pump": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-          "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
-          "dev": true,
-          "requires": {
-            "end-of-stream": "^1.1.0",
-            "once": "^1.3.1"
-          }
-        }
-      }
+    "leven": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz",
+      "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==",
+      "dev": true
     },
-    "libnpmteam": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/libnpmteam/-/libnpmteam-1.0.1.tgz",
-      "integrity": "sha512-gDdrflKFCX7TNwOMX1snWojCoDE5LoRWcfOC0C/fqF7mBq8Uz9zWAX4B2RllYETNO7pBupBaSyBDkTAC15cAMg==",
+    "levn": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz",
+      "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=",
       "dev": true,
       "requires": {
-        "aproba": "^2.0.0",
-        "figgy-pudding": "^3.4.1",
-        "get-stream": "^4.0.0",
-        "npm-registry-fetch": "^3.8.0"
-      },
-      "dependencies": {
-        "aproba": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz",
-          "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==",
-          "dev": true
-        },
-        "get-stream": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz",
-          "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==",
-          "dev": true,
-          "requires": {
-            "pump": "^3.0.0"
-          }
-        },
-        "pump": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-          "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
-          "dev": true,
-          "requires": {
-            "end-of-stream": "^1.1.0",
-            "once": "^1.3.1"
-          }
-        }
+        "prelude-ls": "~1.1.2",
+        "type-check": "~0.3.2"
       }
     },
     "liftoff": {
@@ -9553,6 +12459,12 @@
         "resolve": "^1.1.7"
       }
     },
+    "lines-and-columns": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.1.6.tgz",
+      "integrity": "sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA=",
+      "dev": true
+    },
     "load-json-file": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz",
@@ -9581,13 +12493,13 @@
       "dev": true
     },
     "loader-utils": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.2.3.tgz",
-      "integrity": "sha512-fkpz8ejdnEMG3s37wGL07iSBDg99O9D5yflE9RGNH3hRdx9SOwYfnGYdZOUIZitN8E+E2vkq3MUMYMvPYl5ZZA==",
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.4.0.tgz",
+      "integrity": "sha512-qH0WSMBtn/oHuwjy/NucEgbx5dbxxnxup9s4PVXJUDHZBQY+s0NWA9rJf53RBnQZxfch7euUui7hpoAPvALZdA==",
       "dev": true,
       "requires": {
         "big.js": "^5.2.2",
-        "emojis-list": "^2.0.0",
+        "emojis-list": "^3.0.0",
         "json5": "^1.0.1"
       },
       "dependencies": {
@@ -9603,37 +12515,18 @@
       }
     },
     "locate-path": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz",
-      "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=",
-      "dev": true,
-      "requires": {
-        "p-locate": "^2.0.0",
-        "path-exists": "^3.0.0"
-      },
-      "dependencies": {
-        "path-exists": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
-          "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=",
-          "dev": true
-        }
-      }
-    },
-    "lock-verify": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/lock-verify/-/lock-verify-2.0.2.tgz",
-      "integrity": "sha512-QNVwK0EGZBS4R3YQ7F1Ox8p41Po9VGl2QG/2GsuvTbkJZYSsPeWHKMbbH6iZMCHWSMww5nrJroZYnGzI4cePuw==",
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
+      "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==",
       "dev": true,
       "requires": {
-        "npm-package-arg": "^5.1.2 || 6",
-        "semver": "^5.4.1"
+        "p-locate": "^4.1.0"
       }
     },
     "lodash": {
-      "version": "4.17.11",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
-      "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==",
+      "version": "4.17.20",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
+      "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==",
       "dev": true
     },
     "lodash._reinterpolate": {
@@ -9653,10 +12546,22 @@
       "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=",
       "dev": true
     },
-    "lodash.debounce": {
-      "version": "4.0.8",
-      "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz",
-      "integrity": "sha1-gteb/zCmfEAF/9XiUVMArZyk168=",
+    "lodash.get": {
+      "version": "4.4.2",
+      "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
+      "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=",
+      "dev": true
+    },
+    "lodash.ismatch": {
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/lodash.ismatch/-/lodash.ismatch-4.4.0.tgz",
+      "integrity": "sha1-dWy1FQyjum8RCFp4hJZF8Yj4Xzc=",
+      "dev": true
+    },
+    "lodash.memoize": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz",
+      "integrity": "sha1-vMbEmkKihA7Zl/Mj6tpezRguC/4=",
       "dev": true
     },
     "lodash.padend": {
@@ -9664,6 +12569,12 @@
       "resolved": "https://registry.npmjs.org/lodash.padend/-/lodash.padend-4.6.1.tgz",
       "integrity": "sha1-U8y6BH0G4VjTEfRdpiX05J5vFm4="
     },
+    "lodash.set": {
+      "version": "4.3.2",
+      "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz",
+      "integrity": "sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM=",
+      "dev": true
+    },
     "lodash.sortby": {
       "version": "4.7.0",
       "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz",
@@ -9671,39 +12582,36 @@
       "dev": true
     },
     "lodash.template": {
-      "version": "4.4.0",
-      "resolved": "https://registry.npmjs.org/lodash.template/-/lodash.template-4.4.0.tgz",
-      "integrity": "sha1-5zoDhcg1VZF0bgILmWecaQ5o+6A=",
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/lodash.template/-/lodash.template-4.5.0.tgz",
+      "integrity": "sha512-84vYFxIkmidUiFxidA/KjjH9pAycqW+h980j7Fuz5qxRtO9pgB7MDFTdys1N7A5mcucRiDyEq4fusljItR1T/A==",
       "dev": true,
       "requires": {
-        "lodash._reinterpolate": "~3.0.0",
+        "lodash._reinterpolate": "^3.0.0",
         "lodash.templatesettings": "^4.0.0"
       }
     },
     "lodash.templatesettings": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/lodash.templatesettings/-/lodash.templatesettings-4.1.0.tgz",
-      "integrity": "sha1-K01OlbpEDZFf8IvImeRVNmZxMxY=",
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/lodash.templatesettings/-/lodash.templatesettings-4.2.0.tgz",
+      "integrity": "sha512-stgLz+i3Aa9mZgnjr/O+v9ruKZsPsndy7qPZOchbqk2cnTU1ZaldKK+v7m54WoKIyxiuMZTKT2H81F8BeAc3ZQ==",
       "dev": true,
       "requires": {
-        "lodash._reinterpolate": "~3.0.0"
+        "lodash._reinterpolate": "^3.0.0"
       }
     },
+    "lodash.uniq": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz",
+      "integrity": "sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=",
+      "dev": true
+    },
     "log-driver": {
       "version": "1.2.7",
       "resolved": "https://registry.npmjs.org/log-driver/-/log-driver-1.2.7.tgz",
       "integrity": "sha512-U7KCmLdqsGHBLeWqYlFA0V0Sl6P08EE1ZrmA9cxjUE0WVqT9qnyVDPz1kzpFEP0jdJuFnasWIfSd7fsaNXkpbg==",
       "dev": true
     },
-    "loose-envify": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
-      "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==",
-      "dev": true,
-      "requires": {
-        "js-tokens": "^3.0.0 || ^4.0.0"
-      }
-    },
     "loud-rejection": {
       "version": "1.6.0",
       "resolved": "https://registry.npmjs.org/loud-rejection/-/loud-rejection-1.6.0.tgz",
@@ -9715,13 +12623,12 @@
       }
     },
     "lru-cache": {
-      "version": "4.1.5",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz",
-      "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==",
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
+      "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
       "dev": true,
       "requires": {
-        "pseudomap": "^1.0.2",
-        "yallist": "^2.1.2"
+        "yallist": "^3.0.2"
       }
     },
     "lru-queue": {
@@ -9733,39 +12640,53 @@
         "es5-ext": "~0.10.2"
       }
     },
-    "lunr": {
-      "version": "2.3.6",
-      "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.6.tgz",
-      "integrity": "sha512-swStvEyDqQ85MGpABCMBclZcLI/pBIlu8FFDtmX197+oEgKloJ67QnB+Tidh0340HmLMs39c4GrkPY3cmkXp6Q==",
+    "lunr": {
+      "version": "2.3.9",
+      "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.9.tgz",
+      "integrity": "sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==",
+      "dev": true
+    },
+    "macos-release": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.4.1.tgz",
+      "integrity": "sha512-H/QHeBIN1fIGJX517pvK8IEK53yQOW7YcEI55oYtgjDdoCQQz7eJS94qt5kNrscReEyuD/JcdFCm2XBEcGOITg==",
       "dev": true
     },
     "make-dir": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz",
-      "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==",
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz",
+      "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==",
       "dev": true,
       "requires": {
-        "pify": "^3.0.0"
+        "semver": "^6.0.0"
+      },
+      "dependencies": {
+        "semver": {
+          "version": "6.3.0",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
+          "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
+          "dev": true
+        }
       }
     },
     "make-error": {
-      "version": "1.3.5",
-      "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.5.tgz",
-      "integrity": "sha512-c3sIjNUow0+8swNwVpqoH4YCShKNFkMaw6oH1mNS2haDZQqkeZFlHS3dhoeEbKKmJB4vXpJucU6oH75aDYeE9g==",
+      "version": "1.3.6",
+      "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz",
+      "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==",
       "dev": true
     },
     "make-fetch-happen": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-4.0.1.tgz",
-      "integrity": "sha512-7R5ivfy9ilRJ1EMKIOziwrns9fGeAD4bAha8EB7BIiBBLHm2KeTUGCrICFt2rbHfzheTLynv50GnNTK1zDTrcQ==",
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-5.0.2.tgz",
+      "integrity": "sha512-07JHC0r1ykIoruKO8ifMXu+xEU8qOXDFETylktdug6vJDACnP+HKevOu3PXyNPzFyTSlz8vrBYlBO1JZRe8Cag==",
       "dev": true,
       "requires": {
         "agentkeepalive": "^3.4.1",
-        "cacache": "^11.0.1",
+        "cacache": "^12.0.0",
         "http-cache-semantics": "^3.8.1",
         "http-proxy-agent": "^2.1.0",
-        "https-proxy-agent": "^2.2.1",
-        "lru-cache": "^4.1.2",
+        "https-proxy-agent": "^2.2.3",
+        "lru-cache": "^5.1.1",
         "mississippi": "^3.0.0",
         "node-fetch-npm": "^2.0.2",
         "promise-retry": "^1.1.1",
@@ -9791,15 +12712,6 @@
         "tmpl": "1.0.x"
       }
     },
-    "map-age-cleaner": {
-      "version": "0.1.3",
-      "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz",
-      "integrity": "sha512-bJzx6nMoP6PDLPBFmg7+xRKeFZvFboMrGlxmNj9ClvX53KrmvM5bXFXEWjbz4cz1AFn+jWJ9z/DJSz7hrs0w3w==",
-      "dev": true,
-      "requires": {
-        "p-defer": "^1.0.0"
-      }
-    },
     "map-cache": {
       "version": "0.2.2",
       "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz",
@@ -9807,9 +12719,9 @@
       "dev": true
     },
     "map-obj": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-2.0.0.tgz",
-      "integrity": "sha1-plzSkIepJZi4eRJXpSPgISIqwfk=",
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-4.1.0.tgz",
+      "integrity": "sha512-glc9y00wgtwcDmp7GaE/0b0OnxpNJsVf3ael/An6Fe2Q51LLwN1er6sdomLRzz5h0+yMpiYLhWYF5R7HeqVd4g==",
       "dev": true
     },
     "map-visit": {
@@ -9822,9 +12734,9 @@
       }
     },
     "marked": {
-      "version": "0.6.2",
-      "resolved": "https://registry.npmjs.org/marked/-/marked-0.6.2.tgz",
-      "integrity": "sha512-LqxwVH3P/rqKX4EKGz7+c2G9r98WeM/SW34ybhgNGhUQNKtf1GmmSkJ6cDGJ/t6tiyae49qRkpyTw2B9HOrgUA==",
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/marked/-/marked-1.1.1.tgz",
+      "integrity": "sha512-mJzT8D2yPxoPh7h0UXkB+dBj4FykPJ2OIfxAWeIHrvoHDkFxukV/29QxoFQoPM6RLEwhIFdJpmKBlqVM3s2ZIw==",
       "dev": true
     },
     "matchdep": {
@@ -9879,31 +12791,6 @@
         "safe-buffer": "^5.1.2"
       }
     },
-    "mem": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/mem/-/mem-4.3.0.tgz",
-      "integrity": "sha512-qX2bG48pTqYRVmDB37rn/6PT7LcR8T7oAX3bf99u1Tt1nzxYfxkgqDwUwolPlXweM0XzBOBFzSx4kfp7KP1s/w==",
-      "dev": true,
-      "requires": {
-        "map-age-cleaner": "^0.1.1",
-        "mimic-fn": "^2.0.0",
-        "p-is-promise": "^2.0.0"
-      },
-      "dependencies": {
-        "mimic-fn": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz",
-          "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==",
-          "dev": true
-        },
-        "p-is-promise": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-2.1.0.tgz",
-          "integrity": "sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg==",
-          "dev": true
-        }
-      }
-    },
     "memfs": {
       "version": "2.15.2",
       "resolved": "https://registry.npmjs.org/memfs/-/memfs-2.15.2.tgz",
@@ -9947,104 +12834,125 @@
       "dev": true
     },
     "meow": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/meow/-/meow-4.0.1.tgz",
-      "integrity": "sha512-xcSBHD5Z86zaOc+781KrupuHAzeGXSLtiAOmBsiLDiPSaYSB6hdew2ng9EBAnZ62jagG9MHAOdxpDi/lWBFJ/A==",
-      "dev": true,
-      "requires": {
-        "camelcase-keys": "^4.0.0",
-        "decamelize-keys": "^1.0.0",
-        "loud-rejection": "^1.0.0",
-        "minimist": "^1.1.3",
-        "minimist-options": "^3.0.1",
-        "normalize-package-data": "^2.3.4",
-        "read-pkg-up": "^3.0.0",
-        "redent": "^2.0.0",
-        "trim-newlines": "^2.0.0"
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/meow/-/meow-7.1.1.tgz",
+      "integrity": "sha512-GWHvA5QOcS412WCo8vwKDlTelGLsCGBVevQB5Kva961rmNfun0PCbv5+xta2kUMFJyR8/oWnn7ddeKdosbAPbA==",
+      "dev": true,
+      "requires": {
+        "@types/minimist": "^1.2.0",
+        "camelcase-keys": "^6.2.2",
+        "decamelize-keys": "^1.1.0",
+        "hard-rejection": "^2.1.0",
+        "minimist-options": "4.1.0",
+        "normalize-package-data": "^2.5.0",
+        "read-pkg-up": "^7.0.1",
+        "redent": "^3.0.0",
+        "trim-newlines": "^3.0.0",
+        "type-fest": "^0.13.1",
+        "yargs-parser": "^18.1.3"
       },
       "dependencies": {
-        "find-up": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz",
-          "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=",
-          "dev": true,
-          "requires": {
-            "locate-path": "^2.0.0"
-          }
+        "camelcase": {
+          "version": "5.3.1",
+          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
+          "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==",
+          "dev": true
         },
-        "load-json-file": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz",
-          "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=",
+        "find-up": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz",
+          "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==",
           "dev": true,
           "requires": {
-            "graceful-fs": "^4.1.2",
-            "parse-json": "^4.0.0",
-            "pify": "^3.0.0",
-            "strip-bom": "^3.0.0"
+            "locate-path": "^5.0.0",
+            "path-exists": "^4.0.0"
           }
         },
         "parse-json": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
-          "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=",
+          "version": "5.1.0",
+          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.1.0.tgz",
+          "integrity": "sha512-+mi/lmVVNKFNVyLXV31ERiy2CY5E1/F6QtJFEzoChPRwwngMNXRDQ9GJ5WdE2Z2P4AujsOi0/+2qHID68KwfIQ==",
           "dev": true,
           "requires": {
+            "@babel/code-frame": "^7.0.0",
             "error-ex": "^1.3.1",
-            "json-parse-better-errors": "^1.0.1"
+            "json-parse-even-better-errors": "^2.3.0",
+            "lines-and-columns": "^1.1.6"
           }
         },
-        "path-type": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz",
-          "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==",
-          "dev": true,
-          "requires": {
-            "pify": "^3.0.0"
-          }
+        "path-exists": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
+          "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==",
+          "dev": true
         },
         "read-pkg": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz",
-          "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=",
+          "version": "5.2.0",
+          "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-5.2.0.tgz",
+          "integrity": "sha512-Ug69mNOpfvKDAc2Q8DRpMjjzdtrnv9HcSMX+4VsZxD1aZ6ZzrIE7rlzXBtWTyhULSMKg076AW6WR5iZpD0JiOg==",
           "dev": true,
           "requires": {
-            "load-json-file": "^4.0.0",
-            "normalize-package-data": "^2.3.2",
-            "path-type": "^3.0.0"
+            "@types/normalize-package-data": "^2.4.0",
+            "normalize-package-data": "^2.5.0",
+            "parse-json": "^5.0.0",
+            "type-fest": "^0.6.0"
+          },
+          "dependencies": {
+            "type-fest": {
+              "version": "0.6.0",
+              "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.6.0.tgz",
+              "integrity": "sha512-q+MB8nYR1KDLrgr4G5yemftpMC7/QLqVndBmEEdqzmNj5dcFOO4Oo8qlwZE3ULT3+Zim1F8Kq4cBnikNhlCMlg==",
+              "dev": true
+            }
           }
         },
         "read-pkg-up": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-3.0.0.tgz",
-          "integrity": "sha1-PtSWaF26D4/hGNBpHcUfSh/5bwc=",
+          "version": "7.0.1",
+          "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-7.0.1.tgz",
+          "integrity": "sha512-zK0TB7Xd6JpCLmlLmufqykGE+/TlOePD6qKClNW7hHDKFh/J7/7gCWGR7joEQEW1bKq3a3yUZSObOoWLFQ4ohg==",
           "dev": true,
           "requires": {
-            "find-up": "^2.0.0",
-            "read-pkg": "^3.0.0"
+            "find-up": "^4.1.0",
+            "read-pkg": "^5.2.0",
+            "type-fest": "^0.8.1"
+          },
+          "dependencies": {
+            "type-fest": {
+              "version": "0.8.1",
+              "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz",
+              "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==",
+              "dev": true
+            }
           }
         },
-        "strip-bom": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
-          "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=",
+        "type-fest": {
+          "version": "0.13.1",
+          "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz",
+          "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==",
           "dev": true
+        },
+        "yargs-parser": {
+          "version": "18.1.3",
+          "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz",
+          "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==",
+          "dev": true,
+          "requires": {
+            "camelcase": "^5.0.0",
+            "decamelize": "^1.2.0"
+          }
         }
       }
     },
     "merge-stream": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-1.0.1.tgz",
-      "integrity": "sha1-QEEgLVCKNCugAXQAjfDCUbjBNeE=",
-      "dev": true,
-      "requires": {
-        "readable-stream": "^2.0.1"
-      }
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
+      "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==",
+      "dev": true
     },
     "merge2": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.2.3.tgz",
-      "integrity": "sha512-gdUU1Fwj5ep4kplwcmftruWofEFt6lfpkkr3h860CXbAB9c3hGb55EOL2ali0Td5oebvW0E1+3Sr+Ur7XfKpRA==",
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
+      "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==",
       "dev": true
     },
     "micromatch": {
@@ -10076,27 +12984,41 @@
       "requires": {
         "bn.js": "^4.0.0",
         "brorand": "^1.0.1"
+      },
+      "dependencies": {
+        "bn.js": {
+          "version": "4.11.9",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==",
+          "dev": true
+        }
       }
     },
     "mime-db": {
-      "version": "1.37.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.37.0.tgz",
-      "integrity": "sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg==",
+      "version": "1.44.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz",
+      "integrity": "sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg==",
       "dev": true
     },
     "mime-types": {
-      "version": "2.1.21",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.21.tgz",
-      "integrity": "sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg==",
+      "version": "2.1.27",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz",
+      "integrity": "sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==",
       "dev": true,
       "requires": {
-        "mime-db": "~1.37.0"
+        "mime-db": "1.44.0"
       }
     },
     "mimic-fn": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz",
-      "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==",
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz",
+      "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==",
+      "dev": true
+    },
+    "min-indent": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz",
+      "integrity": "sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==",
       "dev": true
     },
     "minimalistic-assert": {
@@ -10121,46 +13043,117 @@
       }
     },
     "minimist": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
-      "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=",
+      "version": "1.2.5",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
+      "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==",
       "dev": true
     },
     "minimist-options": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/minimist-options/-/minimist-options-3.0.2.tgz",
-      "integrity": "sha512-FyBrT/d0d4+uiZRbqznPXqw3IpZZG3gl3wKWiX784FycUKVwBt0uLBFkQrtE4tZOrgo78nZp2jnKz3L65T5LdQ==",
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/minimist-options/-/minimist-options-4.1.0.tgz",
+      "integrity": "sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==",
       "dev": true,
       "requires": {
         "arrify": "^1.0.1",
-        "is-plain-obj": "^1.1.0"
+        "is-plain-obj": "^1.1.0",
+        "kind-of": "^6.0.3"
       }
     },
     "minipass": {
-      "version": "2.3.5",
-      "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz",
-      "integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==",
+      "version": "2.9.0",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz",
+      "integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==",
       "dev": true,
       "requires": {
         "safe-buffer": "^5.1.2",
         "yallist": "^3.0.0"
+      }
+    },
+    "minipass-collect": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-1.0.2.tgz",
+      "integrity": "sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==",
+      "dev": true,
+      "requires": {
+        "minipass": "^3.0.0"
+      },
+      "dependencies": {
+        "minipass": {
+          "version": "3.1.3",
+          "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.1.3.tgz",
+          "integrity": "sha512-Mgd2GdMVzY+x3IJ+oHnVM+KG3lA5c8tnabyJKmHSaG2kAGpudxuOf8ToDkhumF7UzME7DecbQE9uOZhNm7PuJg==",
+          "dev": true,
+          "requires": {
+            "yallist": "^4.0.0"
+          }
+        },
+        "yallist": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+          "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+          "dev": true
+        }
+      }
+    },
+    "minipass-flush": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.5.tgz",
+      "integrity": "sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==",
+      "dev": true,
+      "requires": {
+        "minipass": "^3.0.0"
+      },
+      "dependencies": {
+        "minipass": {
+          "version": "3.1.3",
+          "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.1.3.tgz",
+          "integrity": "sha512-Mgd2GdMVzY+x3IJ+oHnVM+KG3lA5c8tnabyJKmHSaG2kAGpudxuOf8ToDkhumF7UzME7DecbQE9uOZhNm7PuJg==",
+          "dev": true,
+          "requires": {
+            "yallist": "^4.0.0"
+          }
+        },
+        "yallist": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+          "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+          "dev": true
+        }
+      }
+    },
+    "minipass-pipeline": {
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz",
+      "integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==",
+      "dev": true,
+      "requires": {
+        "minipass": "^3.0.0"
       },
       "dependencies": {
+        "minipass": {
+          "version": "3.1.3",
+          "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.1.3.tgz",
+          "integrity": "sha512-Mgd2GdMVzY+x3IJ+oHnVM+KG3lA5c8tnabyJKmHSaG2kAGpudxuOf8ToDkhumF7UzME7DecbQE9uOZhNm7PuJg==",
+          "dev": true,
+          "requires": {
+            "yallist": "^4.0.0"
+          }
+        },
         "yallist": {
-          "version": "3.0.3",
-          "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz",
-          "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==",
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+          "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
           "dev": true
         }
       }
     },
     "minizlib": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.2.1.tgz",
-      "integrity": "sha512-7+4oTUOWKg7AuL3vloEWekXY2/D20cevzsrNT2kGWm+39J9hGTCBv8VI5Pm5lXZ/o3/mdR4f8rflAPhnQb8mPA==",
+      "version": "1.3.3",
+      "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz",
+      "integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==",
       "dev": true,
       "requires": {
-        "minipass": "^2.2.1"
+        "minipass": "^2.9.0"
       }
     },
     "mississippi": {
@@ -10194,9 +13187,9 @@
       }
     },
     "mixin-deep": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.1.tgz",
-      "integrity": "sha512-8ZItLHeEgaqEvd5lYBXfm4EZSFCX29Jb9K+lAHhDKzReKBQKj3R+7NOF6tjqYi9t4oI8VUfaWITJQm86wnXGNQ==",
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz",
+      "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==",
       "dev": true,
       "requires": {
         "for-in": "^1.0.2",
@@ -10215,20 +13208,18 @@
       }
     },
     "mkdirp": {
-      "version": "0.5.1",
-      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
-      "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
+      "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==",
+      "dev": true
+    },
+    "mkdirp-promise": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/mkdirp-promise/-/mkdirp-promise-5.0.1.tgz",
+      "integrity": "sha1-6bj2jlUsaKnBcTuEiD96HdA5uKE=",
       "dev": true,
       "requires": {
-        "minimist": "0.0.8"
-      },
-      "dependencies": {
-        "minimist": {
-          "version": "0.0.8",
-          "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
-          "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=",
-          "dev": true
-        }
+        "mkdirp": "*"
       }
     },
     "modify-values": {
@@ -10274,6 +13265,17 @@
         "mkdirp": "^0.5.1",
         "rimraf": "^2.5.4",
         "run-queue": "^1.0.3"
+      },
+      "dependencies": {
+        "mkdirp": {
+          "version": "0.5.5",
+          "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
+          "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
+          "dev": true,
+          "requires": {
+            "minimist": "^1.2.5"
+          }
+        }
       }
     },
     "ms": {
@@ -10283,15 +13285,15 @@
       "dev": true
     },
     "multimatch": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/multimatch/-/multimatch-2.1.0.tgz",
-      "integrity": "sha1-nHkGoi+0wCkZ4vX3UWG0zb1LKis=",
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/multimatch/-/multimatch-3.0.0.tgz",
+      "integrity": "sha512-22foS/gqQfANZ3o+W7ST2x25ueHDVNWl/b9OlGcLpy/iKxjCpvcNCM51YCenUi7Mt/jAjjqv8JwZRs8YP5sRjA==",
       "dev": true,
       "requires": {
-        "array-differ": "^1.0.0",
-        "array-union": "^1.0.1",
-        "arrify": "^1.0.0",
-        "minimatch": "^3.0.0"
+        "array-differ": "^2.0.3",
+        "array-union": "^1.0.2",
+        "arrify": "^1.0.1",
+        "minimatch": "^3.0.4"
       }
     },
     "multistream": {
@@ -10316,10 +13318,21 @@
       "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=",
       "dev": true
     },
+    "mz": {
+      "version": "2.7.0",
+      "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
+      "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==",
+      "dev": true,
+      "requires": {
+        "any-promise": "^1.0.0",
+        "object-assign": "^4.0.1",
+        "thenify-all": "^1.0.0"
+      }
+    },
     "nan": {
-      "version": "2.12.1",
-      "resolved": "https://registry.npmjs.org/nan/-/nan-2.12.1.tgz",
-      "integrity": "sha512-JY7V6lRkStKcKTvHO5NVSQRv+RV+FIL5pvDoLiAtSL9pKlC5x9PKQcZDsq7m4FO4d57mkhC6Z+QhAh3Jdk5JFw==",
+      "version": "2.14.1",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz",
+      "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==",
       "dev": true,
       "optional": true
     },
@@ -10349,9 +13362,9 @@
       "dev": true
     },
     "neo-async": {
-      "version": "2.6.0",
-      "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.0.tgz",
-      "integrity": "sha512-MFh0d/Wa7vkKO3Y3LlacqAEeHK0mckVqzDieUKTT+KGxi+zIpeVsFxymkIiRpbpDziHc290Xr9A1O4Om7otoRA==",
+      "version": "2.6.2",
+      "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz",
+      "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==",
       "dev": true
     },
     "next-tick": {
@@ -10366,10 +13379,16 @@
       "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==",
       "dev": true
     },
+    "node-fetch": {
+      "version": "2.6.1",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
+      "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==",
+      "dev": true
+    },
     "node-fetch-npm": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/node-fetch-npm/-/node-fetch-npm-2.0.2.tgz",
-      "integrity": "sha512-nJIxm1QmAj4v3nfCvEeCrYSoVwXyxLnaPBK5W1W5DGEJwjlKuC2VEUycGw5oxk+4zZahRrB84PUJJgEmhFTDFw==",
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/node-fetch-npm/-/node-fetch-npm-2.0.4.tgz",
+      "integrity": "sha512-iOuIQDWDyjhv9qSDrj9aq/klt6F9z1p2otB3AV7v3zBDcL/x+OfGsvGQZZCcMZbUf4Ujw1xGNQkjvGnVT22cKg==",
       "dev": true,
       "requires": {
         "encoding": "^0.1.11",
@@ -10378,30 +13397,32 @@
       }
     },
     "node-gyp": {
-      "version": "3.8.0",
-      "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-3.8.0.tgz",
-      "integrity": "sha512-3g8lYefrRRzvGeSowdJKAKyks8oUpLEd/DyPV4eMhVlhJ0aNaZqIrNUIPuEWWTAoPqyFkfGrM67MC69baqn6vA==",
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-5.1.1.tgz",
+      "integrity": "sha512-WH0WKGi+a4i4DUt2mHnvocex/xPLp9pYt5R6M2JdFB7pJ7Z34hveZ4nDTGTiLXCkitA9T8HFZjhinBCiVHYcWw==",
       "dev": true,
       "requires": {
-        "fstream": "^1.0.0",
-        "glob": "^7.0.3",
-        "graceful-fs": "^4.1.2",
-        "mkdirp": "^0.5.0",
-        "nopt": "2 || 3",
-        "npmlog": "0 || 1 || 2 || 3 || 4",
-        "osenv": "0",
-        "request": "^2.87.0",
-        "rimraf": "2",
-        "semver": "~5.3.0",
-        "tar": "^2.0.0",
-        "which": "1"
+        "env-paths": "^2.2.0",
+        "glob": "^7.1.4",
+        "graceful-fs": "^4.2.2",
+        "mkdirp": "^0.5.1",
+        "nopt": "^4.0.1",
+        "npmlog": "^4.1.2",
+        "request": "^2.88.0",
+        "rimraf": "^2.6.3",
+        "semver": "^5.7.1",
+        "tar": "^4.4.12",
+        "which": "^1.3.1"
       },
       "dependencies": {
-        "semver": {
-          "version": "5.3.0",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz",
-          "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8=",
-          "dev": true
+        "mkdirp": {
+          "version": "0.5.5",
+          "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
+          "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
+          "dev": true,
+          "requires": {
+            "minimist": "^1.2.5"
+          }
         }
       }
     },
@@ -10412,9 +13433,9 @@
       "dev": true
     },
     "node-libs-browser": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.0.tgz",
-      "integrity": "sha512-5MQunG/oyOaBdttrL40dA7bUfPORLRWMUJLQtMg7nluxUvk5XwnLdL9twQHFAjRx/y7mIMkLKT9++qPbbk6BZA==",
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.1.tgz",
+      "integrity": "sha512-h/zcD8H9kaDZ9ALUWwlBUDo6TKF8a7qBSCSEGfjTVIYeqsioSKaAX+BN7NgiMGp6iSIXZ3PxgCu8KS3b71YK5Q==",
       "dev": true,
       "requires": {
         "assert": "^1.1.1",
@@ -10427,7 +13448,7 @@
         "events": "^3.0.0",
         "https-browserify": "^1.0.0",
         "os-browserify": "^0.3.0",
-        "path-browserify": "0.0.0",
+        "path-browserify": "0.0.1",
         "process": "^0.11.10",
         "punycode": "^1.2.4",
         "querystring-es3": "^0.2.0",
@@ -10439,7 +13460,7 @@
         "tty-browserify": "0.0.0",
         "url": "^0.11.0",
         "util": "^0.11.0",
-        "vm-browserify": "0.0.4"
+        "vm-browserify": "^1.0.1"
       },
       "dependencies": {
         "punycode": {
@@ -10457,72 +13478,107 @@
       "dev": true
     },
     "node-notifier": {
-      "version": "5.4.0",
-      "resolved": "https://registry.npmjs.org/node-notifier/-/node-notifier-5.4.0.tgz",
-      "integrity": "sha512-SUDEb+o71XR5lXSTyivXd9J7fCloE3SyP4lSgt3lU2oSANiox+SxlNRGPjDKrwU1YN3ix2KN/VGGCg0t01rttQ==",
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/node-notifier/-/node-notifier-8.0.0.tgz",
+      "integrity": "sha512-46z7DUmcjoYdaWyXouuFNNfUo6eFa94t23c53c+lG/9Cvauk4a98rAUp9672X5dxGdQmLpPzTxzu8f/OeEPaFA==",
       "dev": true,
+      "optional": true,
       "requires": {
         "growly": "^1.3.0",
-        "is-wsl": "^1.1.0",
-        "semver": "^5.5.0",
+        "is-wsl": "^2.2.0",
+        "semver": "^7.3.2",
         "shellwords": "^0.1.1",
-        "which": "^1.3.0"
+        "uuid": "^8.3.0",
+        "which": "^2.0.2"
+      },
+      "dependencies": {
+        "semver": {
+          "version": "7.3.2",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz",
+          "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==",
+          "dev": true,
+          "optional": true
+        },
+        "uuid": {
+          "version": "8.3.0",
+          "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.0.tgz",
+          "integrity": "sha512-fX6Z5o4m6XsXBdli9g7DtWgAx+osMsRRZFKma1mIUsLCz6vRvv+pz5VNbyu9UEDzpMWulZfvpgb/cmDXVulYFQ==",
+          "dev": true,
+          "optional": true
+        },
+        "which": {
+          "version": "2.0.2",
+          "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+          "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "isexe": "^2.0.0"
+          }
+        }
       }
     },
     "nopt": {
-      "version": "3.0.6",
-      "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz",
-      "integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=",
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz",
+      "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==",
       "dev": true,
       "requires": {
-        "abbrev": "1"
+        "abbrev": "1",
+        "osenv": "^0.1.4"
       }
     },
     "normalize-package-data": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.4.0.tgz",
-      "integrity": "sha512-9jjUFbTPfEy3R/ad/2oNbKtW9Hgovl5O1FvFWKkKblNXoN/Oou6+9+KKohPK13Yc3/TyunyWhJp6gvRNR/PPAw==",
+      "version": "2.5.0",
+      "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz",
+      "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==",
       "dev": true,
       "requires": {
         "hosted-git-info": "^2.1.4",
-        "is-builtin-module": "^1.0.0",
+        "resolve": "^1.10.0",
         "semver": "2 || 3 || 4 || 5",
-        "validate-npm-package-license": "^3.0.1"
-      }
-    },
-    "normalize-path": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.1.1.tgz",
-      "integrity": "sha1-GrKLVW4Zg2Oowab35vogE3/mrtk=",
-      "dev": true,
-      "requires": {
-        "remove-trailing-separator": "^1.0.1"
+        "validate-npm-package-license": "^3.0.1"
       }
     },
+    "normalize-path": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
+      "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==",
+      "dev": true
+    },
+    "normalize-url": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-3.3.0.tgz",
+      "integrity": "sha512-U+JJi7duF1o+u2pynbp2zXDW2/PADgC30f0GsHZtRh+HOcXHnw137TrNlyxxRvWW5fjKd3bcLHPxofWuCjaeZg==",
+      "dev": true
+    },
     "now-and-later": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/now-and-later/-/now-and-later-2.0.0.tgz",
-      "integrity": "sha1-vGHLtFbXnLMiB85HygUTb/Ln1u4=",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/now-and-later/-/now-and-later-2.0.1.tgz",
+      "integrity": "sha512-KGvQ0cB70AQfg107Xvs/Fbu+dGmZoTRJp2TaPwcwQm3/7PteUyN2BCgk8KBMPGBUXZdVwyWS8fDCGFygBm19UQ==",
       "dev": true,
       "requires": {
         "once": "^1.3.2"
       }
     },
     "npm-bundled": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.0.5.tgz",
-      "integrity": "sha512-m/e6jgWu8/v5niCUKQi9qQl8QdeEduFA96xHDDzFGqly0OOjI7c+60KM/2sppfnUU9JJagf+zs+yGhqSOFj71g==",
-      "dev": true
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.1.tgz",
+      "integrity": "sha512-gqkfgGePhTpAEgUsGEgcq1rqPXA+tv/aVBlgEzfXwA1yiUJF7xtEt3CtVwOjNYQOVknDk0F20w58Fnm3EtG0fA==",
+      "dev": true,
+      "requires": {
+        "npm-normalize-package-bin": "^1.0.1"
+      }
     },
     "npm-lifecycle": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/npm-lifecycle/-/npm-lifecycle-2.1.0.tgz",
-      "integrity": "sha512-QbBfLlGBKsktwBZLj6AviHC6Q9Y3R/AY4a2PYSIRhSKSS0/CxRyD/PfxEX6tPeOCXQgMSNdwGeECacstgptc+g==",
+      "version": "3.1.5",
+      "resolved": "https://registry.npmjs.org/npm-lifecycle/-/npm-lifecycle-3.1.5.tgz",
+      "integrity": "sha512-lDLVkjfZmvmfvpvBzA4vzee9cn+Me4orq0QF8glbswJVEbIcSNWib7qGOffolysc3teCqbbPZZkzbr3GQZTL1g==",
       "dev": true,
       "requires": {
         "byline": "^5.0.0",
-        "graceful-fs": "^4.1.11",
-        "node-gyp": "^3.8.0",
+        "graceful-fs": "^4.1.15",
+        "node-gyp": "^5.0.2",
         "resolve-from": "^4.0.0",
         "slide": "^1.1.6",
         "uid-number": "0.0.6",
@@ -10538,38 +13594,39 @@
         }
       }
     },
-    "npm-logical-tree": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/npm-logical-tree/-/npm-logical-tree-1.2.1.tgz",
-      "integrity": "sha512-AJI/qxDB2PWI4LG1CYN579AY1vCiNyWfkiquCsJWqntRu/WwimVrC8yXeILBFHDwxfOejxewlmnvW9XXjMlYIg==",
+    "npm-normalize-package-bin": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-1.0.1.tgz",
+      "integrity": "sha512-EPfafl6JL5/rU+ot6P3gRSCpPDW5VmIzX959Ob1+ySFUuuYHWHekXpwdUZcKP5C+DS4GEtdJluwBjnsNDl+fSA==",
       "dev": true
     },
     "npm-package-arg": {
-      "version": "6.1.0",
-      "resolved": "https://registry.npmjs.org/npm-package-arg/-/npm-package-arg-6.1.0.tgz",
-      "integrity": "sha512-zYbhP2k9DbJhA0Z3HKUePUgdB1x7MfIfKssC+WLPFMKTBZKpZh5m13PgexJjCq6KW7j17r0jHWcCpxEqnnncSA==",
+      "version": "6.1.1",
+      "resolved": "https://registry.npmjs.org/npm-package-arg/-/npm-package-arg-6.1.1.tgz",
+      "integrity": "sha512-qBpssaL3IOZWi5vEKUKW0cO7kzLeT+EQO9W8RsLOZf76KF9E/K9+wH0C7t06HXPpaH8WH5xF1MExLuCwbTqRUg==",
       "dev": true,
       "requires": {
-        "hosted-git-info": "^2.6.0",
+        "hosted-git-info": "^2.7.1",
         "osenv": "^0.1.5",
-        "semver": "^5.5.0",
+        "semver": "^5.6.0",
         "validate-npm-package-name": "^3.0.0"
       }
     },
     "npm-packlist": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.2.0.tgz",
-      "integrity": "sha512-7Mni4Z8Xkx0/oegoqlcao/JpPCPEMtUvsmB0q7mgvlMinykJLSRTYuFqoQLYgGY8biuxIeiHO+QNJKbCfljewQ==",
+      "version": "1.4.8",
+      "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.8.tgz",
+      "integrity": "sha512-5+AZgwru5IevF5ZdnFglB5wNlHG1AOOuw28WhUq8/8emhBmLv6jX5by4WJCh7lW0uSYZYS6DXqIsyZVIXRZU9A==",
       "dev": true,
       "requires": {
         "ignore-walk": "^3.0.1",
-        "npm-bundled": "^1.0.1"
+        "npm-bundled": "^1.0.1",
+        "npm-normalize-package-bin": "^1.0.1"
       }
     },
     "npm-pick-manifest": {
-      "version": "2.2.3",
-      "resolved": "https://registry.npmjs.org/npm-pick-manifest/-/npm-pick-manifest-2.2.3.tgz",
-      "integrity": "sha512-+IluBC5K201+gRU85vFlUwX3PFShZAbAgDNp2ewJdWMVSppdo/Zih0ul2Ecky/X7b51J7LrrUAP+XOmOCvYZqA==",
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/npm-pick-manifest/-/npm-pick-manifest-3.0.2.tgz",
+      "integrity": "sha512-wNprTNg+X5nf+tDi+hbjdHhM4bX+mKqv6XmPh7B5eG+QY9VARfQPfCEH013H5GqfNj6ee8Ij2fg8yk0mzps1Vw==",
       "dev": true,
       "requires": {
         "figgy-pudding": "^3.5.1",
@@ -10577,31 +13634,6 @@
         "semver": "^5.4.1"
       }
     },
-    "npm-profile": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/npm-profile/-/npm-profile-4.0.1.tgz",
-      "integrity": "sha512-NQ1I/1Q7YRtHZXkcuU1/IyHeLy6pd+ScKg4+DQHdfsm769TGq6HPrkbuNJVJS4zwE+0mvvmeULzQdWn2L2EsVA==",
-      "dev": true,
-      "requires": {
-        "aproba": "^1.1.2 || 2",
-        "figgy-pudding": "^3.4.1",
-        "npm-registry-fetch": "^3.8.0"
-      }
-    },
-    "npm-registry-fetch": {
-      "version": "3.9.0",
-      "resolved": "https://registry.npmjs.org/npm-registry-fetch/-/npm-registry-fetch-3.9.0.tgz",
-      "integrity": "sha512-srwmt8YhNajAoSAaDWndmZgx89lJwIZ1GWxOuckH4Coek4uHv5S+o/l9FLQe/awA+JwTnj4FJHldxhlXdZEBmw==",
-      "dev": true,
-      "requires": {
-        "JSONStream": "^1.3.4",
-        "bluebird": "^3.5.1",
-        "figgy-pudding": "^3.4.1",
-        "lru-cache": "^4.1.3",
-        "make-fetch-happen": "^4.0.1",
-        "npm-package-arg": "^6.1.0"
-      }
-    },
     "npm-run-all": {
       "version": "4.1.5",
       "resolved": "https://registry.npmjs.org/npm-run-all/-/npm-run-all-4.1.5.tgz",
@@ -10619,19 +13651,6 @@
         "string.prototype.padend": "^3.0.0"
       },
       "dependencies": {
-        "cross-spawn": {
-          "version": "6.0.5",
-          "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz",
-          "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==",
-          "dev": true,
-          "requires": {
-            "nice-try": "^1.0.4",
-            "path-key": "^2.0.1",
-            "semver": "^5.5.0",
-            "shebang-command": "^1.2.0",
-            "which": "^1.2.9"
-          }
-        },
         "load-json-file": {
           "version": "4.0.0",
           "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz",
@@ -10710,9 +13729,9 @@
       "dev": true
     },
     "nwsapi": {
-      "version": "2.1.4",
-      "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.1.4.tgz",
-      "integrity": "sha512-iGfd9Y6SFdTNldEy2L0GUhcarIutFmk+MPWIn9dmj8NMIup03G08uUF2KGbbmv/Ux4RT0VZJoP/sVbWA6d/VIw==",
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz",
+      "integrity": "sha512-h2AatdwYH+JHiZpv7pt/gSX1XoRGb7L/qSIeuqA6GwYoF9w1vP1cw42TO0aI2pNyshRK5893hNSl+1//vHK7hQ==",
       "dev": true
     },
     "oauth-sign": {
@@ -10758,10 +13777,16 @@
         }
       }
     },
+    "object-inspect": {
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.8.0.tgz",
+      "integrity": "sha512-jLdtEOB112fORuypAyl/50VRVIBIdVQOSUUGQHzJ4xBSbit81zRarz7GThkEFZy1RceYrWYcPcBFPQwHyAc1gA==",
+      "dev": true
+    },
     "object-keys": {
-      "version": "1.0.12",
-      "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.12.tgz",
-      "integrity": "sha512-FTMyFUm2wBcGHnH2eXmz7tC6IwlqQZ6mVZ+6dm6vZ4IQIHjs6FdNsQBuKGPuUUUY6NfJw2PshC08Tn6LzLDOag==",
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
+      "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==",
       "dev": true
     },
     "object-visit": {
@@ -10774,15 +13799,15 @@
       }
     },
     "object.assign": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz",
-      "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==",
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.1.tgz",
+      "integrity": "sha512-VT/cxmx5yaoHSOTSyrCygIDFco+RsibY2NM0a4RdEeY/4KgqezwFtK1yr3U67xYhqJSlASm2pKhLVzPj2lr4bA==",
       "dev": true,
       "requires": {
-        "define-properties": "^1.1.2",
-        "function-bind": "^1.1.1",
-        "has-symbols": "^1.0.0",
-        "object-keys": "^1.0.11"
+        "define-properties": "^1.1.3",
+        "es-abstract": "^1.18.0-next.0",
+        "has-symbols": "^1.0.1",
+        "object-keys": "^1.1.1"
       }
     },
     "object.defaults": {
@@ -10798,13 +13823,34 @@
       }
     },
     "object.getownpropertydescriptors": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.0.3.tgz",
-      "integrity": "sha1-h1jIRvW0B62rDyNuCYbxSwUcqhY=",
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.0.tgz",
+      "integrity": "sha512-Z53Oah9A3TdLoblT7VKJaTDdXdT+lQO+cNpKVnya5JDe9uLvzu1YyY1yFDFrcxrlRgWrEFH0jJtD/IbuwjcEVg==",
       "dev": true,
       "requires": {
-        "define-properties": "^1.1.2",
-        "es-abstract": "^1.5.1"
+        "define-properties": "^1.1.3",
+        "es-abstract": "^1.17.0-next.1"
+      },
+      "dependencies": {
+        "es-abstract": {
+          "version": "1.17.6",
+          "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz",
+          "integrity": "sha512-Fr89bON3WFyUi5EvAeI48QTWX0AyekGgLA8H+c+7fbfCkJwRWRMLd8CQedNEyJuoYYhmtEqY92pgte1FAhBlhw==",
+          "dev": true,
+          "requires": {
+            "es-to-primitive": "^1.2.1",
+            "function-bind": "^1.1.1",
+            "has": "^1.0.3",
+            "has-symbols": "^1.0.1",
+            "is-callable": "^1.2.0",
+            "is-regex": "^1.1.0",
+            "object-inspect": "^1.7.0",
+            "object-keys": "^1.1.1",
+            "object.assign": "^4.1.0",
+            "string.prototype.trimend": "^1.0.1",
+            "string.prototype.trimstart": "^1.0.1"
+          }
+        }
       }
     },
     "object.map": {
@@ -10836,6 +13882,12 @@
         "make-iterator": "^1.0.0"
       }
     },
+    "octokit-pagination-methods": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/octokit-pagination-methods/-/octokit-pagination-methods-1.1.0.tgz",
+      "integrity": "sha512-fZ4qZdQ2nxJvtcasX7Ghl+WlWS/d9IgnBIwFZXVNNZUmzpno91SX5bc5vuxiuKoCtK78XxGGNuSCrDC7xYB3OQ==",
+      "dev": true
+    },
     "once": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
@@ -10846,50 +13898,32 @@
       }
     },
     "onetime": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz",
-      "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=",
-      "dev": true,
-      "requires": {
-        "mimic-fn": "^1.0.0"
-      }
-    },
-    "optimist": {
-      "version": "0.6.1",
-      "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz",
-      "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=",
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz",
+      "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==",
       "dev": true,
       "requires": {
-        "minimist": "~0.0.1",
-        "wordwrap": "~0.0.2"
-      },
-      "dependencies": {
-        "minimist": {
-          "version": "0.0.10",
-          "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz",
-          "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=",
-          "dev": true
-        }
+        "mimic-fn": "^2.1.0"
       }
     },
     "optionator": {
-      "version": "0.8.2",
-      "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz",
-      "integrity": "sha1-NkxeQJ0/TWMB1sC0wFu6UBgK62Q=",
+      "version": "0.8.3",
+      "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz",
+      "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==",
       "dev": true,
       "requires": {
         "deep-is": "~0.1.3",
-        "fast-levenshtein": "~2.0.4",
+        "fast-levenshtein": "~2.0.6",
         "levn": "~0.3.0",
         "prelude-ls": "~1.1.2",
         "type-check": "~0.3.2",
-        "wordwrap": "~1.0.0"
+        "word-wrap": "~1.2.3"
       },
       "dependencies": {
-        "wordwrap": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz",
-          "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=",
+        "fast-levenshtein": {
+          "version": "2.0.6",
+          "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
+          "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=",
           "dev": true
         }
       }
@@ -10924,6 +13958,16 @@
         "lcid": "^1.0.0"
       }
     },
+    "os-name": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/os-name/-/os-name-3.1.0.tgz",
+      "integrity": "sha512-h8L+8aNjNcMpo/mAIBPn5PXCM16iyPGjHNWo6U1YO8sJTMHtEtyczI6QJnLoplswm6goopQkqc7OAnjhWcugVg==",
+      "dev": true,
+      "requires": {
+        "macos-release": "^2.2.0",
+        "windows-release": "^3.1.0"
+      }
+    },
     "os-tmpdir": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
@@ -10940,20 +13984,11 @@
         "os-tmpdir": "^1.0.0"
       }
     },
-    "p-defer": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz",
-      "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww=",
-      "dev": true
-    },
     "p-each-series": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/p-each-series/-/p-each-series-1.0.0.tgz",
-      "integrity": "sha1-kw89Et0fUOdDRFeiLNbwSsatf3E=",
-      "dev": true,
-      "requires": {
-        "p-reduce": "^1.0.0"
-      }
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/p-each-series/-/p-each-series-2.1.0.tgz",
+      "integrity": "sha512-ZuRs1miPT4HrjFa+9fRfOFXxGJfORgelKV9f9nNOWw2gl6gVsRaVDOQP0+MI0G0wGKns1Yacsu0GjOFbTK0JFQ==",
+      "dev": true
     },
     "p-finally": {
       "version": "1.0.0",
@@ -10961,28 +13996,22 @@
       "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=",
       "dev": true
     },
-    "p-is-promise": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-1.1.0.tgz",
-      "integrity": "sha1-nJRWmJ6fZYgBewQ01WCXZ1w9oF4=",
-      "dev": true
-    },
     "p-limit": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz",
-      "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==",
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
+      "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==",
       "dev": true,
       "requires": {
-        "p-try": "^1.0.0"
+        "p-try": "^2.0.0"
       }
     },
     "p-locate": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz",
-      "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=",
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz",
+      "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==",
       "dev": true,
       "requires": {
-        "p-limit": "^1.1.0"
+        "p-limit": "^2.2.0"
       }
     },
     "p-map": {
@@ -11006,6 +14035,15 @@
       "integrity": "sha1-SxoROZoRUgpneQ7loMHViB1r7+k=",
       "dev": true
     },
+    "p-queue": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-4.0.0.tgz",
+      "integrity": "sha512-3cRXXn3/O0o3+eVmUroJPSj/esxoEFIm0ZOno/T+NzG/VZgPOqQ8WKmlNqubSEpZmCIngEy34unkHGg83ZIBmg==",
+      "dev": true,
+      "requires": {
+        "eventemitter3": "^3.1.0"
+      }
+    },
     "p-reduce": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/p-reduce/-/p-reduce-1.0.0.tgz",
@@ -11013,9 +14051,9 @@
       "dev": true
     },
     "p-try": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz",
-      "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=",
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
+      "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==",
       "dev": true
     },
     "p-waterfall": {
@@ -11027,92 +14065,6 @@
         "p-reduce": "^1.0.0"
       }
     },
-    "pacote": {
-      "version": "9.4.1",
-      "resolved": "https://registry.npmjs.org/pacote/-/pacote-9.4.1.tgz",
-      "integrity": "sha512-YKSRsQqmeHxgra0KCdWA2FtVxDPUlBiCdmew+mSe44pzlx5t1ViRMWiQg18T+DREA+vSqYfKzynaToFR4hcKHw==",
-      "dev": true,
-      "requires": {
-        "bluebird": "^3.5.3",
-        "cacache": "^11.3.2",
-        "figgy-pudding": "^3.5.1",
-        "get-stream": "^4.1.0",
-        "glob": "^7.1.3",
-        "lru-cache": "^5.1.1",
-        "make-fetch-happen": "^4.0.1",
-        "minimatch": "^3.0.4",
-        "minipass": "^2.3.5",
-        "mississippi": "^3.0.0",
-        "mkdirp": "^0.5.1",
-        "normalize-package-data": "^2.4.0",
-        "npm-package-arg": "^6.1.0",
-        "npm-packlist": "^1.1.12",
-        "npm-pick-manifest": "^2.2.3",
-        "npm-registry-fetch": "^3.8.0",
-        "osenv": "^0.1.5",
-        "promise-inflight": "^1.0.1",
-        "promise-retry": "^1.1.1",
-        "protoduck": "^5.0.1",
-        "rimraf": "^2.6.2",
-        "safe-buffer": "^5.1.2",
-        "semver": "^5.6.0",
-        "ssri": "^6.0.1",
-        "tar": "^4.4.8",
-        "unique-filename": "^1.1.1",
-        "which": "^1.3.1"
-      },
-      "dependencies": {
-        "get-stream": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz",
-          "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==",
-          "dev": true,
-          "requires": {
-            "pump": "^3.0.0"
-          }
-        },
-        "lru-cache": {
-          "version": "5.1.1",
-          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
-          "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
-          "dev": true,
-          "requires": {
-            "yallist": "^3.0.2"
-          }
-        },
-        "pump": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-          "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
-          "dev": true,
-          "requires": {
-            "end-of-stream": "^1.1.0",
-            "once": "^1.3.1"
-          }
-        },
-        "tar": {
-          "version": "4.4.8",
-          "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.8.tgz",
-          "integrity": "sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ==",
-          "dev": true,
-          "requires": {
-            "chownr": "^1.1.1",
-            "fs-minipass": "^1.2.5",
-            "minipass": "^2.3.4",
-            "minizlib": "^1.1.1",
-            "mkdirp": "^0.5.0",
-            "safe-buffer": "^5.1.2",
-            "yallist": "^3.0.2"
-          }
-        },
-        "yallist": {
-          "version": "3.0.3",
-          "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz",
-          "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==",
-          "dev": true
-        }
-      }
-    },
     "pad-left": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/pad-left/-/pad-left-2.1.0.tgz",
@@ -11122,31 +14074,30 @@
       }
     },
     "pako": {
-      "version": "1.0.8",
-      "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.8.tgz",
-      "integrity": "sha512-6i0HVbUfcKaTv+EG8ZTr75az7GFXcLYk9UyLEg7Notv/Ma+z/UG3TCoz6GiNeOrn1E/e63I0X/Hpw18jHOTUnA==",
+      "version": "1.0.11",
+      "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz",
+      "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==",
       "dev": true
     },
     "parallel-transform": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/parallel-transform/-/parallel-transform-1.1.0.tgz",
-      "integrity": "sha1-1BDwZbBdojCB/NEPKIVMKb2jOwY=",
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/parallel-transform/-/parallel-transform-1.2.0.tgz",
+      "integrity": "sha512-P2vSmIu38uIlvdcU7fDkyrxj33gTUy/ABO5ZUbGowxNCopBq/OoD42bP4UmMrJoPyk4Uqf0mu3mtWBhHCZD8yg==",
       "dev": true,
       "requires": {
-        "cyclist": "~0.2.2",
+        "cyclist": "^1.0.1",
         "inherits": "^2.0.3",
         "readable-stream": "^2.1.5"
       }
     },
     "parse-asn1": {
-      "version": "5.1.3",
-      "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.3.tgz",
-      "integrity": "sha512-VrPoetlz7B/FqjBLD2f5wBVZvsZVLnRUrxVLfRYhGXCODa/NWE4p3Wp+6+aV3ZPL3KM7/OZmxDIwwijD7yuucg==",
+      "version": "5.1.6",
+      "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz",
+      "integrity": "sha512-RnZRo1EPU6JBnra2vGHj0yhp6ebyjBZpmUCLHWiFhxlzvBCCpAuZ7elsBp1PVAbQN0/04VD/19rfzlBSwLstMw==",
       "dev": true,
       "requires": {
-        "asn1.js": "^4.0.0",
+        "asn1.js": "^5.2.0",
         "browserify-aes": "^1.0.0",
-        "create-hash": "^1.1.0",
         "evp_bytestokey": "^1.0.0",
         "pbkdf2": "^3.0.3",
         "safe-buffer": "^5.1.1"
@@ -11179,9 +14130,9 @@
       }
     },
     "parse-node-version": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/parse-node-version/-/parse-node-version-1.0.0.tgz",
-      "integrity": "sha512-02GTVHD1u0nWc20n2G7WX/PgdhNFG04j5fi1OkaJzPWLTcf6vh6229Lta1wTmXG/7Dg42tCssgkccVt7qvd8Kg==",
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/parse-node-version/-/parse-node-version-1.0.1.tgz",
+      "integrity": "sha512-3YHlOa/JgH6Mnpr05jP9eDG254US9ek25LyIxZlDItp2iJtwyaXQb57lBYLdT3MowkUFYEV2XXNAYIPlESvJlA==",
       "dev": true
     },
     "parse-passwd": {
@@ -11190,10 +14141,32 @@
       "integrity": "sha1-bVuTSkVpk7I9N/QKOC1vFmao5cY=",
       "dev": true
     },
+    "parse-path": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/parse-path/-/parse-path-4.0.2.tgz",
+      "integrity": "sha512-HSqVz6iuXSiL8C1ku5Gl1Z5cwDd9Wo0q8CoffdAghP6bz8pJa1tcMC+m4N+z6VAS8QdksnIGq1TB6EgR4vPR6w==",
+      "dev": true,
+      "requires": {
+        "is-ssh": "^1.3.0",
+        "protocols": "^1.4.0"
+      }
+    },
+    "parse-url": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/parse-url/-/parse-url-5.0.2.tgz",
+      "integrity": "sha512-Czj+GIit4cdWtxo3ISZCvLiUjErSo0iI3wJ+q9Oi3QuMYTI6OZu+7cewMWZ+C1YAnKhYTk6/TLuhIgCypLthPA==",
+      "dev": true,
+      "requires": {
+        "is-ssh": "^1.3.0",
+        "normalize-url": "^3.3.0",
+        "parse-path": "^4.0.0",
+        "protocols": "^1.4.0"
+      }
+    },
     "parse5": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/parse5/-/parse5-4.0.0.tgz",
-      "integrity": "sha512-VrZ7eOd3T1Fk4XWNXMgiGBK/z0MG48BWG2uQNU4I72fkQuKUTZpl+u9k+CxEG0twMVzSmXEEz12z5Fnw1jIQFA==",
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/parse5/-/parse5-5.1.1.tgz",
+      "integrity": "sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug==",
       "dev": true
     },
     "pascalcase": {
@@ -11203,9 +14176,9 @@
       "dev": true
     },
     "path-browserify": {
-      "version": "0.0.0",
-      "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-0.0.0.tgz",
-      "integrity": "sha1-oLhwcpquIUAFt9UDLsLLuw+0RRo=",
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-0.0.1.tgz",
+      "integrity": "sha512-BapA40NHICOS+USX9SN4tyhq+A2RrN/Ws5F0Z5aMHDp98Fl86lX8Oti8B7uN93L4Ifv4fHOEA+pQw87gmMO/lQ==",
       "dev": true
     },
     "path-dirname": {
@@ -11282,9 +14255,9 @@
       }
     },
     "pbkdf2": {
-      "version": "3.0.17",
-      "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.0.17.tgz",
-      "integrity": "sha512-U/il5MsrZp7mGg3mSQfn742na2T+1/vHDCG5/iTI3X9MKUuYUZVLQhyRsg06mCgDBTd57TxzgZt7P+fYfjRLtA==",
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.1.tgz",
+      "integrity": "sha512-4Ejy1OPxi9f2tt1rRV7Go7zmfDQ+ZectEQz3VGUQhgq62HtIRPDyG/JtnwIxs6x3uNMwo2V7q1fMvKjb+Tnpqg==",
       "dev": true,
       "requires": {
         "create-hash": "^1.1.2",
@@ -11300,10 +14273,16 @@
       "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=",
       "dev": true
     },
+    "picomatch": {
+      "version": "2.2.2",
+      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz",
+      "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==",
+      "dev": true
+    },
     "pidtree": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/pidtree/-/pidtree-0.3.0.tgz",
-      "integrity": "sha512-9CT4NFlDcosssyg8KVFltgokyKZIFjoBxw8CTGy+5F38Y1eQWrt8tRayiUOXE+zVKQnYu5BR8JjCtvK3BcnBhg==",
+      "version": "0.3.1",
+      "resolved": "https://registry.npmjs.org/pidtree/-/pidtree-0.3.1.tgz",
+      "integrity": "sha512-qQbW94hLHEqCg7nhby4yRC7G2+jYHY4Rguc2bjw7Uug4GIJuu1tvf2uHaZv5Q8zdt+WKJ6qK1FOI6amaWUo5FA==",
       "dev": true
     },
     "pify": {
@@ -11337,29 +14316,36 @@
       }
     },
     "pkg-dir": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz",
-      "integrity": "sha1-9tXREJ4Z1j7fQo4L1X4Sd3YVM0s=",
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz",
+      "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==",
       "dev": true,
       "requires": {
-        "find-up": "^2.1.0"
+        "find-up": "^4.0.0"
       },
       "dependencies": {
         "find-up": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz",
-          "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=",
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz",
+          "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==",
           "dev": true,
           "requires": {
-            "locate-path": "^2.0.0"
+            "locate-path": "^5.0.0",
+            "path-exists": "^4.0.0"
           }
+        },
+        "path-exists": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
+          "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==",
+          "dev": true
         }
       }
     },
     "platform": {
-      "version": "1.3.5",
-      "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.5.tgz",
-      "integrity": "sha512-TuvHS8AOIZNAlE77WUDiR4rySV/VMptyMfcfeoMgs4P8apaZM3JrnbzBiixKUv+XR6i+BXrQh8WAnjaSPFO65Q==",
+      "version": "1.3.6",
+      "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz",
+      "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==",
       "dev": true
     },
     "plugin-error": {
@@ -11374,12 +14360,6 @@
         "extend-shallow": "^3.0.2"
       }
     },
-    "pn": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz",
-      "integrity": "sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==",
-      "dev": true
-    },
     "posix-character-classes": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz",
@@ -11393,21 +14373,40 @@
       "dev": true
     },
     "pretty-format": {
-      "version": "24.8.0",
-      "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-24.8.0.tgz",
-      "integrity": "sha512-P952T7dkrDEplsR+TuY7q3VXDae5Sr7zmQb12JU/NDQa/3CH7/QW0yvqLcGN6jL+zQFKaoJcPc+yJxMTGmosqw==",
+      "version": "25.5.0",
+      "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-25.5.0.tgz",
+      "integrity": "sha512-kbo/kq2LQ/A/is0PQwsEHM7Ca6//bGPPvU6UnsdDRSKTWxT/ru/xb88v4BJf6a69H+uTytOEsTusT9ksd/1iWQ==",
       "dev": true,
       "requires": {
-        "@jest/types": "^24.8.0",
-        "ansi-regex": "^4.0.0",
-        "ansi-styles": "^3.2.0",
-        "react-is": "^16.8.4"
+        "@jest/types": "^25.5.0",
+        "ansi-regex": "^5.0.0",
+        "ansi-styles": "^4.0.0",
+        "react-is": "^16.12.0"
       },
       "dependencies": {
-        "ansi-regex": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz",
-          "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==",
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
           "dev": true
         }
       }
@@ -11437,9 +14436,9 @@
       "dev": true
     },
     "promise": {
-      "version": "8.0.2",
-      "resolved": "https://registry.npmjs.org/promise/-/promise-8.0.2.tgz",
-      "integrity": "sha512-EIyzM39FpVOMbqgzEHhxdrEhtOSDOtjMZQ0M6iVfCE+kWNgCkAyOdnuCWqfmflylftfadU6FkiMgHZA2kUzwRw==",
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/promise/-/promise-8.1.0.tgz",
+      "integrity": "sha512-W04AqnILOL/sPRXziNicCjSNRruLAuIHEOVBazepu0545DDNGYHz7ar9ZgZ1fMU8/MA4mVxp5rkBWRi6OXIy3Q==",
       "dev": true,
       "requires": {
         "asap": "~2.0.6"
@@ -11462,13 +14461,13 @@
       }
     },
     "prompts": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.1.0.tgz",
-      "integrity": "sha512-+x5TozgqYdOwWsQFZizE/Tra3fKvAoy037kOyU6cgz84n8f6zxngLOV4O32kTwt9FcLCxAqw0P/c8rOr9y+Gfg==",
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.3.2.tgz",
+      "integrity": "sha512-Q06uKs2CkNYVID0VqwfAl9mipo99zkBv/n2JtWY89Yxa3ZabWSrs0e2KTudKVa3peLUvYXMefDqIleLPVUBZMA==",
       "dev": true,
       "requires": {
-        "kleur": "^3.0.2",
-        "sisteransi": "^1.0.0"
+        "kleur": "^3.0.3",
+        "sisteransi": "^1.0.4"
       }
     },
     "promzard": {
@@ -11486,6 +14485,12 @@
       "integrity": "sha1-IS1b/hMYMGpCD2QCuOJv85ZHqEk=",
       "dev": true
     },
+    "protocols": {
+      "version": "1.4.8",
+      "resolved": "https://registry.npmjs.org/protocols/-/protocols-1.4.8.tgz",
+      "integrity": "sha512-IgjKyaUSjsROSO8/D49Ab7hP8mJgTYcqApOqdPhLoPxAplXmkp+zRvsrSQjFn5by0rhm4VH0GAUELIPpx7B1yg==",
+      "dev": true
+    },
     "protoduck": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/protoduck/-/protoduck-5.0.1.tgz",
@@ -11508,9 +14513,9 @@
       "dev": true
     },
     "psl": {
-      "version": "1.1.31",
-      "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.31.tgz",
-      "integrity": "sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw==",
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz",
+      "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==",
       "dev": true
     },
     "public-encrypt": {
@@ -11525,6 +14530,14 @@
         "parse-asn1": "^5.0.0",
         "randombytes": "^2.0.1",
         "safe-buffer": "^5.1.2"
+      },
+      "dependencies": {
+        "bn.js": {
+          "version": "4.11.9",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==",
+          "dev": true
+        }
       }
     },
     "pump": {
@@ -11579,9 +14592,9 @@
       "dev": true
     },
     "quick-lru": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-1.1.0.tgz",
-      "integrity": "sha1-Q2CxfGETatOAeDl/8RQW4Ybc+7g=",
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-4.0.1.tgz",
+      "integrity": "sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==",
       "dev": true
     },
     "randomatic": {
@@ -11604,9 +14617,9 @@
       }
     },
     "randombytes": {
-      "version": "2.0.6",
-      "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.0.6.tgz",
-      "integrity": "sha512-CIQ5OFxf4Jou6uOKe9t1AOgqpeU5fd70A8NPdHSGeYXqXsPe6peOwI0cUl88RWZ6sP1vPMV3avd/R6cZ5/sP1A==",
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz",
+      "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==",
       "dev": true,
       "requires": {
         "safe-buffer": "^5.1.0"
@@ -11623,9 +14636,9 @@
       }
     },
     "react-is": {
-      "version": "16.8.6",
-      "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.8.6.tgz",
-      "integrity": "sha512-aUk3bHfZ2bRSVFFbbeVS4i+lNPZr3/WM5jT2J5omUVV1zzcs1nAaf3l51ctA5FFvCRbhrH0bdAsRRQddFJZPtA==",
+      "version": "16.13.1",
+      "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
+      "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==",
       "dev": true
     },
     "read": {
@@ -11638,38 +14651,35 @@
       }
     },
     "read-cmd-shim": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/read-cmd-shim/-/read-cmd-shim-1.0.1.tgz",
-      "integrity": "sha1-LV0Vd4ajfAVdIgd8MsU/gynpHHs=",
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/read-cmd-shim/-/read-cmd-shim-1.0.5.tgz",
+      "integrity": "sha512-v5yCqQ/7okKoZZkBQUAfTsQ3sVJtXdNfbPnI5cceppoxEVLYA3k+VtV2omkeo8MS94JCy4fSiUwlRBAwCVRPUA==",
       "dev": true,
       "requires": {
         "graceful-fs": "^4.1.2"
       }
     },
     "read-package-json": {
-      "version": "2.0.13",
-      "resolved": "https://registry.npmjs.org/read-package-json/-/read-package-json-2.0.13.tgz",
-      "integrity": "sha512-/1dZ7TRZvGrYqE0UAfN6qQb5GYBsNcqS1C0tNK601CFOJmtHI7NIGXwetEPU/OtoFHZL3hDxm4rolFFVE9Bnmg==",
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/read-package-json/-/read-package-json-2.1.2.tgz",
+      "integrity": "sha512-D1KmuLQr6ZSJS0tW8hf3WGpRlwszJOXZ3E8Yd/DNRaM5d+1wVRZdHlpGBLAuovjr28LbWvjpWkBHMxpRGGjzNA==",
       "dev": true,
       "requires": {
         "glob": "^7.1.1",
-        "graceful-fs": "^4.1.2",
-        "json-parse-better-errors": "^1.0.1",
+        "json-parse-even-better-errors": "^2.3.0",
         "normalize-package-data": "^2.0.0",
-        "slash": "^1.0.0"
+        "npm-normalize-package-bin": "^1.0.0"
       }
     },
     "read-package-tree": {
-      "version": "5.2.1",
-      "resolved": "https://registry.npmjs.org/read-package-tree/-/read-package-tree-5.2.1.tgz",
-      "integrity": "sha512-2CNoRoh95LxY47LvqrehIAfUVda2JbuFE/HaGYs42bNrGG+ojbw1h3zOcPcQ+1GQ3+rkzNndZn85u1XyZ3UsIA==",
+      "version": "5.3.1",
+      "resolved": "https://registry.npmjs.org/read-package-tree/-/read-package-tree-5.3.1.tgz",
+      "integrity": "sha512-mLUDsD5JVtlZxjSlPPx1RETkNjjvQYuweKwNVt1Sn8kP5Jh44pvYuUHCp6xSVDZWbNxVxG5lyZJ921aJH61sTw==",
       "dev": true,
       "requires": {
-        "debuglog": "^1.0.1",
-        "dezalgo": "^1.0.0",
-        "once": "^1.3.0",
         "read-package-json": "^2.0.0",
-        "readdir-scoped-modules": "^1.0.0"
+        "readdir-scoped-modules": "^1.0.0",
+        "util-promisify": "^2.1.0"
       }
     },
     "read-pkg": {
@@ -11694,9 +14704,9 @@
       }
     },
     "readable-stream": {
-      "version": "2.3.6",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
-      "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
+      "version": "2.3.7",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
+      "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
       "dev": true,
       "requires": {
         "core-util-is": "~1.0.0",
@@ -11709,17 +14719,23 @@
       },
       "dependencies": {
         "process-nextick-args": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
-          "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==",
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
+          "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
+          "dev": true
+        },
+        "safe-buffer": {
+          "version": "5.1.2",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+          "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
           "dev": true
         }
       }
     },
     "readdir-scoped-modules": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/readdir-scoped-modules/-/readdir-scoped-modules-1.0.2.tgz",
-      "integrity": "sha1-n6+jfShr5dksuuve4DDcm19AZ0c=",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/readdir-scoped-modules/-/readdir-scoped-modules-1.1.0.tgz",
+      "integrity": "sha512-asaikDeqAQg7JifRsZn1NJZXo9E+VwlyCfbkZhwyISinqk5zNS6266HS5kah6P0SaQKGF6SkNnZVHUzHFYxYDw==",
       "dev": true,
       "requires": {
         "debuglog": "^1.0.1",
@@ -11736,16 +14752,7 @@
       "requires": {
         "graceful-fs": "^4.1.11",
         "micromatch": "^3.1.10",
-        "readable-stream": "^2.0.2"
-      }
-    },
-    "realpath-native": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/realpath-native/-/realpath-native-1.1.0.tgz",
-      "integrity": "sha512-wlgPA6cCIIg9gKz0fgAPjnzh4yR/LnXovwuo9hvyGvx3h8nX4+/iLZplfUWasXpqD8BdnGnP5njOFjkUwPzvjA==",
-      "dev": true,
-      "requires": {
-        "util.promisify": "^1.0.0"
+        "readable-stream": "^2.0.2"
       }
     },
     "rechoir": {
@@ -11758,13 +14765,13 @@
       }
     },
     "redent": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/redent/-/redent-2.0.0.tgz",
-      "integrity": "sha1-wbIAe0LVfrE4kHmzyDM2OdXhzKo=",
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz",
+      "integrity": "sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==",
       "dev": true,
       "requires": {
-        "indent-string": "^3.0.0",
-        "strip-indent": "^2.0.0"
+        "indent-string": "^4.0.0",
+        "strip-indent": "^3.0.0"
       }
     },
     "reduce-flatten": {
@@ -11830,9 +14837,9 @@
       }
     },
     "replace-ext": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/replace-ext/-/replace-ext-1.0.0.tgz",
-      "integrity": "sha1-3mMSg3P8v3w8z6TeWkgMRaZ5WOs=",
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/replace-ext/-/replace-ext-1.0.1.tgz",
+      "integrity": "sha512-yD5BHCe7quCgBph4rMQ+0KkIRKwWCrHDOX1p1Gp6HwjPM5kVoCdKGNhN7ydqqsX6lJEnQDKZ/tFMiEdQ1dvPEw==",
       "dev": true
     },
     "replace-homedir": {
@@ -11847,9 +14854,9 @@
       }
     },
     "request": {
-      "version": "2.88.0",
-      "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz",
-      "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==",
+      "version": "2.88.2",
+      "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz",
+      "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==",
       "dev": true,
       "requires": {
         "aws-sign2": "~0.7.0",
@@ -11859,7 +14866,7 @@
         "extend": "~3.0.2",
         "forever-agent": "~0.6.1",
         "form-data": "~2.3.2",
-        "har-validator": "~5.1.0",
+        "har-validator": "~5.1.3",
         "http-signature": "~1.2.0",
         "is-typedarray": "~1.0.0",
         "isstream": "~0.1.2",
@@ -11869,27 +14876,27 @@
         "performance-now": "^2.1.0",
         "qs": "~6.5.2",
         "safe-buffer": "^5.1.2",
-        "tough-cookie": "~2.4.3",
+        "tough-cookie": "~2.5.0",
         "tunnel-agent": "^0.6.0",
         "uuid": "^3.3.2"
       }
     },
     "request-promise-core": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.2.tgz",
-      "integrity": "sha512-UHYyq1MO8GsefGEt7EprS8UrXsm1TxEvFUX1IMTuSLU2Rh7fTIdFtl8xD7JiEYiWU2dl+NYAjCTksTehQUxPag==",
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.4.tgz",
+      "integrity": "sha512-TTbAfBBRdWD7aNNOoVOBH4pN/KigV6LyapYNNlAPA8JwbovRti1E88m3sYAwsLi5ryhPKsE9APwnjFTgdUjTpw==",
       "dev": true,
       "requires": {
-        "lodash": "^4.17.11"
+        "lodash": "^4.17.19"
       }
     },
     "request-promise-native": {
-      "version": "1.0.7",
-      "resolved": "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.7.tgz",
-      "integrity": "sha512-rIMnbBdgNViL37nZ1b3L/VfPOpSi0TqVDQPAvO6U14lMzOLrt5nilxCQqtDKhZeDiW0/hkCXGoQjhgJd/tCh6w==",
+      "version": "1.0.9",
+      "resolved": "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.9.tgz",
+      "integrity": "sha512-wcW+sIUiWnKgNY0dqCpOZkUbF/I+YPi+f09JZIDa39Ec+q82CpSYniDp+ISgTTbKmnpJWASeJBPZmoxH84wt3g==",
       "dev": true,
       "requires": {
-        "request-promise-core": "1.1.2",
+        "request-promise-core": "1.1.4",
         "stealthy-require": "^1.1.1",
         "tough-cookie": "^2.3.3"
       }
@@ -11907,21 +14914,21 @@
       "dev": true
     },
     "resolve": {
-      "version": "1.10.0",
-      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.10.0.tgz",
-      "integrity": "sha512-3sUr9aq5OfSg2S9pNtPA9hL1FVEAjvfOC4leW0SNf/mpnaakz2a9femSd6LqAww2RaFctwyf1lCqnTHuF1rxDg==",
+      "version": "1.17.0",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.17.0.tgz",
+      "integrity": "sha512-ic+7JYiV8Vi2yzQGFWOkiZD5Z9z7O2Zhm9XMaTxdJExKasieFCr+yXZ/WmXsckHiKl12ar0y6XiXDx3m4RHn1w==",
       "dev": true,
       "requires": {
         "path-parse": "^1.0.6"
       }
     },
     "resolve-cwd": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-2.0.0.tgz",
-      "integrity": "sha1-AKn3OHVW4nA46uIyyqNypqWbZlo=",
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz",
+      "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==",
       "dev": true,
       "requires": {
-        "resolve-from": "^3.0.0"
+        "resolve-from": "^5.0.0"
       }
     },
     "resolve-dir": {
@@ -11935,9 +14942,9 @@
       }
     },
     "resolve-from": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-3.0.0.tgz",
-      "integrity": "sha1-six699nWiBvItuZTM17rywoYh0g=",
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz",
+      "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==",
       "dev": true
     },
     "resolve-options": {
@@ -11963,6 +14970,23 @@
       "requires": {
         "onetime": "^2.0.0",
         "signal-exit": "^3.0.2"
+      },
+      "dependencies": {
+        "mimic-fn": {
+          "version": "1.2.0",
+          "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz",
+          "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==",
+          "dev": true
+        },
+        "onetime": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz",
+          "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=",
+          "dev": true,
+          "requires": {
+            "mimic-fn": "^1.0.0"
+          }
+        }
       }
     },
     "ret": {
@@ -11997,9 +15021,9 @@
       }
     },
     "rsvp": {
-      "version": "4.8.4",
-      "resolved": "https://registry.npmjs.org/rsvp/-/rsvp-4.8.4.tgz",
-      "integrity": "sha512-6FomvYPfs+Jy9TfXmBpBuMWNH94SgCsZmJKcanySzgNNP6LjWxBvyLTa9KaMfDDM5oxRfrKDB0r/qeRsLwnBfA==",
+      "version": "4.8.5",
+      "resolved": "https://registry.npmjs.org/rsvp/-/rsvp-4.8.5.tgz",
+      "integrity": "sha512-nfMOlASu9OnRJo1mbEk2cz0D56a1MBNrJ7orjRZQG10XDyuvwksKbuXNp6qa+kbn839HwjwhBzhFmdsaEAfauA==",
       "dev": true
     },
     "run-applescript": {
@@ -12011,19 +15035,6 @@
         "execa": "^0.10.0"
       },
       "dependencies": {
-        "cross-spawn": {
-          "version": "6.0.5",
-          "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz",
-          "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==",
-          "dev": true,
-          "requires": {
-            "nice-try": "^1.0.4",
-            "path-key": "^2.0.1",
-            "semver": "^5.5.0",
-            "shebang-command": "^1.2.0",
-            "which": "^1.2.9"
-          }
-        },
         "execa": {
           "version": "0.10.0",
           "resolved": "https://registry.npmjs.org/execa/-/execa-0.10.0.tgz",
@@ -12038,17 +15049,20 @@
             "signal-exit": "^3.0.0",
             "strip-eof": "^1.0.0"
           }
+        },
+        "get-stream": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
+          "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=",
+          "dev": true
         }
       }
     },
     "run-async": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.3.0.tgz",
-      "integrity": "sha1-A3GrSuC91yDUFm19/aZP96RFpsA=",
-      "dev": true,
-      "requires": {
-        "is-promise": "^2.1.0"
-      }
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz",
+      "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==",
+      "dev": true
     },
     "run-queue": {
       "version": "1.0.3",
@@ -12069,9 +15083,9 @@
       }
     },
     "safe-buffer": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
-      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+      "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
       "dev": true
     },
     "safe-regex": {
@@ -12112,21 +15126,30 @@
       "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==",
       "dev": true
     },
+    "saxes": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz",
+      "integrity": "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==",
+      "dev": true,
+      "requires": {
+        "xmlchars": "^2.2.0"
+      }
+    },
     "schema-utils": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-1.0.0.tgz",
-      "integrity": "sha512-i27Mic4KovM/lnGsy8whRCHhc7VicJajAjTrYg11K9zfZXnYIt4k5F+kZkwjnrhKzLic/HLU4j11mjsz2G/75g==",
+      "version": "2.7.1",
+      "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.1.tgz",
+      "integrity": "sha512-SHiNtMOUGWBQJwzISiVYKu82GiV4QYGePp3odlY1tuKO7gPtphAT5R/py0fA6xtbgLL/RvtJZnU9b8s0F1q0Xg==",
       "dev": true,
       "requires": {
-        "ajv": "^6.1.0",
-        "ajv-errors": "^1.0.0",
-        "ajv-keywords": "^3.1.0"
+        "@types/json-schema": "^7.0.5",
+        "ajv": "^6.12.4",
+        "ajv-keywords": "^3.5.2"
       }
     },
     "semver": {
-      "version": "5.6.0",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz",
-      "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==",
+      "version": "5.7.1",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
+      "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
       "dev": true
     },
     "semver-greatest-satisfied-range": {
@@ -12139,10 +15162,13 @@
       }
     },
     "serialize-javascript": {
-      "version": "1.6.1",
-      "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-1.6.1.tgz",
-      "integrity": "sha512-A5MOagrPFga4YaKQSWHryl7AXvbQkEqpw4NNYMTNYUNV51bA8ABHgYFpqKx+YFFrw59xMV1qGH1R4AgoNIVgCw==",
-      "dev": true
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz",
+      "integrity": "sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==",
+      "dev": true,
+      "requires": {
+        "randombytes": "^2.1.0"
+      }
     },
     "set-blocking": {
       "version": "2.0.0",
@@ -12151,9 +15177,9 @@
       "dev": true
     },
     "set-value": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz",
-      "integrity": "sha512-hw0yxk9GT/Hr5yJEYnHNKYXkIA8mVJgd9ditYZCe16ZczcaELYYcfvaXesNACk2O8O0nTiPQcQhGUQj8JLzeeg==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz",
+      "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==",
       "dev": true,
       "requires": {
         "extend-shallow": "^2.0.1",
@@ -12189,6 +15215,15 @@
         "safe-buffer": "^5.0.1"
       }
     },
+    "shallow-clone": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz",
+      "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==",
+      "dev": true,
+      "requires": {
+        "kind-of": "^6.0.2"
+      }
+    },
     "shebang-command": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz",
@@ -12205,21 +15240,15 @@
       "dev": true
     },
     "shell-quote": {
-      "version": "1.6.1",
-      "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.6.1.tgz",
-      "integrity": "sha1-9HgZSczkAmlxJ0MOo7PFR29IF2c=",
-      "dev": true,
-      "requires": {
-        "array-filter": "~0.0.0",
-        "array-map": "~0.0.0",
-        "array-reduce": "~0.0.0",
-        "jsonify": "~0.0.0"
-      }
+      "version": "1.7.2",
+      "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.7.2.tgz",
+      "integrity": "sha512-mRz/m/JVscCrkMyPqHc/bczi3OQHkLTqXHEFu0zDhK/qfv3UcOA4SVmRCLmos4bhjr9ekVQubj/R7waKapmiQg==",
+      "dev": true
     },
     "shelljs": {
-      "version": "0.8.3",
-      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.3.tgz",
-      "integrity": "sha512-fc0BKlAWiLpwZljmOvAOTE/gXawtCoNrP5oaY7KIaQbbyHeQVg01pSEuEGvGh3HEdBU4baCD7wQBwADmM/7f7A==",
+      "version": "0.8.4",
+      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.4.tgz",
+      "integrity": "sha512-7gk3UZ9kOfPLIAbslLzyWeGiEqx9e3rxwZM0KE6EL8GlGwjym9Mrlx5/p33bWTu9YG6vcS4MBxYZDHYr5lr8BQ==",
       "dev": true,
       "requires": {
         "glob": "^7.0.0",
@@ -12231,7 +15260,8 @@
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/shellwords/-/shellwords-0.1.1.tgz",
       "integrity": "sha512-vFwSUfQvqybiICwZY5+DAWIPLKsWO31Q91JSKl3UYv+K5c2QRPzn0qzec6QPu1Qc9eHYItiP3NdJqNVqetYAww==",
-      "dev": true
+      "dev": true,
+      "optional": true
     },
     "shx": {
       "version": "0.3.2",
@@ -12245,21 +15275,21 @@
       }
     },
     "signal-exit": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
-      "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=",
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz",
+      "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==",
       "dev": true
     },
     "sisteransi": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.0.tgz",
-      "integrity": "sha512-N+z4pHB4AmUv0SjveWRd6q1Nj5w62m5jodv+GD8lvmbY/83T/rpbJGZOnK5T149OldDj4Db07BSv9xY4K6NTPQ==",
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz",
+      "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==",
       "dev": true
     },
     "slash": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/slash/-/slash-1.0.0.tgz",
-      "integrity": "sha1-xB8vbDn8FtHNF61LXYlhFK5HDVU=",
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
+      "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==",
       "dev": true
     },
     "slide": {
@@ -12269,9 +15299,9 @@
       "dev": true
     },
     "smart-buffer": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.0.2.tgz",
-      "integrity": "sha512-JDhEpTKzXusOqXZ0BUIdH+CjFdO/CR3tLlf5CN34IypI+xMmXW1uB16OOY8z3cICbJlDAVJzNbwBhNO0wt9OAw==",
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.1.0.tgz",
+      "integrity": "sha512-iVICrxOzCynf/SNaBQCw34eM9jROU/s5rzIhpOvzhzuYHfJR/DhZfDkXiZSgKXfgv26HT3Yni3AV/DGw0cGnnw==",
       "dev": true
     },
     "snapdragon": {
@@ -12382,23 +15412,34 @@
       }
     },
     "socks": {
-      "version": "2.2.3",
-      "resolved": "https://registry.npmjs.org/socks/-/socks-2.2.3.tgz",
-      "integrity": "sha512-+2r83WaRT3PXYoO/1z+RDEBE7Z2f9YcdQnJ0K/ncXXbV5gJ6wYfNAebYFYiiUjM6E4JyXnPY8cimwyvFYHVUUA==",
+      "version": "2.3.3",
+      "resolved": "https://registry.npmjs.org/socks/-/socks-2.3.3.tgz",
+      "integrity": "sha512-o5t52PCNtVdiOvzMry7wU4aOqYWL0PeCXRWBEiJow4/i/wr+wpsJQ9awEu1EonLIqsfGd5qSgDdxEOvCdmBEpA==",
       "dev": true,
       "requires": {
-        "ip": "^1.1.5",
-        "smart-buffer": "4.0.2"
+        "ip": "1.1.5",
+        "smart-buffer": "^4.1.0"
       }
     },
     "socks-proxy-agent": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-4.0.1.tgz",
-      "integrity": "sha512-Kezx6/VBguXOsEe5oU3lXYyKMi4+gva72TwJ7pQY5JfqUx2nMk7NXA6z/mpNqIlfQjWYVfeuNvQjexiTaTn6Nw==",
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-4.0.2.tgz",
+      "integrity": "sha512-NT6syHhI9LmuEMSK6Kd2V7gNv5KFZoLE7V5udWmn0de+3Mkj3UMA/AJPLyeNUVmElCurSHtUdM3ETpR3z770Wg==",
       "dev": true,
       "requires": {
-        "agent-base": "~4.2.0",
-        "socks": "~2.2.0"
+        "agent-base": "~4.2.1",
+        "socks": "~2.3.2"
+      },
+      "dependencies": {
+        "agent-base": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz",
+          "integrity": "sha512-JVwXMr9nHYTUXsBFKUqhJwvlcYU/blreOEUkhNR2eXZIvwd+c+o5V4MgDPKWnMS/56awN3TRzIP+KoPn+roQtg==",
+          "dev": true,
+          "requires": {
+            "es6-promisify": "^5.0.0"
+          }
+        }
       }
     },
     "sort-keys": {
@@ -12433,12 +15474,12 @@
       }
     },
     "source-map-resolve": {
-      "version": "0.5.2",
-      "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.2.tgz",
-      "integrity": "sha512-MjqsvNwyz1s0k81Goz/9vRBe9SZdB09Bdw+/zYyO+3CuPk6fouTaxscHkgtE8jKvf01kVfl8riHzERQ/kefaSA==",
+      "version": "0.5.3",
+      "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.3.tgz",
+      "integrity": "sha512-Htz+RnsXWk5+P2slx5Jh3Q66vhQj1Cllm0zvnaY98+NFx+Dv2CF/f5O/t8x+KaNdrdIAsruNzoh/KpialbqAnw==",
       "dev": true,
       "requires": {
-        "atob": "^2.1.1",
+        "atob": "^2.1.2",
         "decode-uri-component": "^0.2.0",
         "resolve-url": "^0.2.1",
         "source-map-url": "^0.4.0",
@@ -12446,9 +15487,9 @@
       }
     },
     "source-map-support": {
-      "version": "0.5.12",
-      "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.12.tgz",
-      "integrity": "sha512-4h2Pbvyy15EE02G+JOZpUCmqWJuqrs+sEkzewTm++BPi7Hvn/HwcqLAcNxYAyI0x13CpPPn+kMjl+hplXMHITQ==",
+      "version": "0.5.19",
+      "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz",
+      "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==",
       "dev": true,
       "requires": {
         "buffer-from": "^1.0.0",
@@ -12476,9 +15517,9 @@
       "dev": true
     },
     "spdx-correct": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.0.tgz",
-      "integrity": "sha512-lr2EZCctC2BNR7j7WzJ2FpDznxky1sjfxvvYEyzxNyb6lZXHODmEoJeFu4JupYlkfha1KZpJyoqiJ7pgA1qq8Q==",
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.1.tgz",
+      "integrity": "sha512-cOYcUWwhCuHCXi49RhFRCyJEK3iPj1Ziz9DpViV3tbZOwXD49QzIN3MpOLJNxh2qwq2lJJZaKMVw9qNi4jTC0w==",
       "dev": true,
       "requires": {
         "spdx-expression-parse": "^3.0.0",
@@ -12486,15 +15527,15 @@
       }
     },
     "spdx-exceptions": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.2.0.tgz",
-      "integrity": "sha512-2XQACfElKi9SlVb1CYadKDXvoajPgBVPn/gOQLrTvHdElaVhr7ZEbqJaRnJLVNeaI4cMEAgVCeBMKF6MWRDCRA==",
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.3.0.tgz",
+      "integrity": "sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A==",
       "dev": true
     },
     "spdx-expression-parse": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz",
-      "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==",
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz",
+      "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==",
       "dev": true,
       "requires": {
         "spdx-exceptions": "^2.1.0",
@@ -12502,9 +15543,9 @@
       }
     },
     "spdx-license-ids": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.3.tgz",
-      "integrity": "sha512-uBIcIl3Ih6Phe3XHK1NqboJLdGfwr1UN3k6wSD1dZpmPsIkb8AGNbZYJ1fOBk834+Gxy8rpfDxrS6XLEMZMY2g==",
+      "version": "3.0.6",
+      "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.6.tgz",
+      "integrity": "sha512-+orQK83kyMva3WyPf59k1+Y525csj5JejicWut55zeTWANuN17qSiSLUXWtzHeNWORSvT7GLDJ/E/XiIWoXBTw==",
       "dev": true
     },
     "split": {
@@ -12573,10 +15614,21 @@
       "dev": true
     },
     "stack-utils": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-1.0.2.tgz",
-      "integrity": "sha512-MTX+MeG5U994cazkjd/9KNAapsHnibjMLnfXodlkXw76JEea0UiNzrqidzo1emMwk7w5Qhc9jd4Bn9TBb1MFwA==",
-      "dev": true
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.2.tgz",
+      "integrity": "sha512-0H7QK2ECz3fyZMzQ8rH0j2ykpfbnd20BFtfg/SqVC2+sCTtcw0aDTGB7dk+de4U4uUeuz6nOtJcrkFFLG1B0Rg==",
+      "dev": true,
+      "requires": {
+        "escape-string-regexp": "^2.0.0"
+      },
+      "dependencies": {
+        "escape-string-regexp": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz",
+          "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==",
+          "dev": true
+        }
+      }
     },
     "static-extend": {
       "version": "0.1.2",
@@ -12645,34 +15697,28 @@
       }
     },
     "stream-shift": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz",
-      "integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=",
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz",
+      "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==",
       "dev": true
     },
     "string-length": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/string-length/-/string-length-2.0.0.tgz",
-      "integrity": "sha1-1A27aGo6zpYMHP/KVivyxF+DY+0=",
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.1.tgz",
+      "integrity": "sha512-PKyXUd0LK0ePjSOnWn34V2uD6acUWev9uy0Ft05k0E8xRW+SKcA0F7eMr7h5xlzfn+4O3N+55rduYyet3Jk+jw==",
       "dev": true,
       "requires": {
-        "astral-regex": "^1.0.0",
-        "strip-ansi": "^4.0.0"
+        "char-regex": "^1.0.2",
+        "strip-ansi": "^6.0.0"
       },
       "dependencies": {
-        "ansi-regex": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
-          "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=",
-          "dev": true
-        },
         "strip-ansi": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
-          "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
+          "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
           "dev": true,
           "requires": {
-            "ansi-regex": "^3.0.0"
+            "ansi-regex": "^5.0.0"
           }
         }
       }
@@ -12689,14 +15735,96 @@
       }
     },
     "string.prototype.padend": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/string.prototype.padend/-/string.prototype.padend-3.0.0.tgz",
-      "integrity": "sha1-86rvfBcZ8XDF6rHDK/eA2W4h8vA=",
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/string.prototype.padend/-/string.prototype.padend-3.1.0.tgz",
+      "integrity": "sha512-3aIv8Ffdp8EZj8iLwREGpQaUZiPyrWrpzMBHvkiSW/bK/EGve9np07Vwy7IJ5waydpGXzQZu/F8Oze2/IWkBaA==",
+      "dev": true,
+      "requires": {
+        "define-properties": "^1.1.3",
+        "es-abstract": "^1.17.0-next.1"
+      },
+      "dependencies": {
+        "es-abstract": {
+          "version": "1.17.6",
+          "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz",
+          "integrity": "sha512-Fr89bON3WFyUi5EvAeI48QTWX0AyekGgLA8H+c+7fbfCkJwRWRMLd8CQedNEyJuoYYhmtEqY92pgte1FAhBlhw==",
+          "dev": true,
+          "requires": {
+            "es-to-primitive": "^1.2.1",
+            "function-bind": "^1.1.1",
+            "has": "^1.0.3",
+            "has-symbols": "^1.0.1",
+            "is-callable": "^1.2.0",
+            "is-regex": "^1.1.0",
+            "object-inspect": "^1.7.0",
+            "object-keys": "^1.1.1",
+            "object.assign": "^4.1.0",
+            "string.prototype.trimend": "^1.0.1",
+            "string.prototype.trimstart": "^1.0.1"
+          }
+        }
+      }
+    },
+    "string.prototype.trimend": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.1.tgz",
+      "integrity": "sha512-LRPxFUaTtpqYsTeNKaFOw3R4bxIzWOnbQ837QfBylo8jIxtcbK/A/sMV7Q+OAV/vWo+7s25pOE10KYSjaSO06g==",
+      "dev": true,
+      "requires": {
+        "define-properties": "^1.1.3",
+        "es-abstract": "^1.17.5"
+      },
+      "dependencies": {
+        "es-abstract": {
+          "version": "1.17.6",
+          "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz",
+          "integrity": "sha512-Fr89bON3WFyUi5EvAeI48QTWX0AyekGgLA8H+c+7fbfCkJwRWRMLd8CQedNEyJuoYYhmtEqY92pgte1FAhBlhw==",
+          "dev": true,
+          "requires": {
+            "es-to-primitive": "^1.2.1",
+            "function-bind": "^1.1.1",
+            "has": "^1.0.3",
+            "has-symbols": "^1.0.1",
+            "is-callable": "^1.2.0",
+            "is-regex": "^1.1.0",
+            "object-inspect": "^1.7.0",
+            "object-keys": "^1.1.1",
+            "object.assign": "^4.1.0",
+            "string.prototype.trimend": "^1.0.1",
+            "string.prototype.trimstart": "^1.0.1"
+          }
+        }
+      }
+    },
+    "string.prototype.trimstart": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.1.tgz",
+      "integrity": "sha512-XxZn+QpvrBI1FOcg6dIpxUPgWCPuNXvMD72aaRaUQv1eD4e/Qy8i/hFTe0BUmD60p/QA6bh1avmuPTfNjqVWRw==",
       "dev": true,
       "requires": {
-        "define-properties": "^1.1.2",
-        "es-abstract": "^1.4.3",
-        "function-bind": "^1.0.2"
+        "define-properties": "^1.1.3",
+        "es-abstract": "^1.17.5"
+      },
+      "dependencies": {
+        "es-abstract": {
+          "version": "1.17.6",
+          "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz",
+          "integrity": "sha512-Fr89bON3WFyUi5EvAeI48QTWX0AyekGgLA8H+c+7fbfCkJwRWRMLd8CQedNEyJuoYYhmtEqY92pgte1FAhBlhw==",
+          "dev": true,
+          "requires": {
+            "es-to-primitive": "^1.2.1",
+            "function-bind": "^1.1.1",
+            "has": "^1.0.3",
+            "has-symbols": "^1.0.1",
+            "is-callable": "^1.2.0",
+            "is-regex": "^1.1.0",
+            "object-inspect": "^1.7.0",
+            "object-keys": "^1.1.1",
+            "object.assign": "^4.1.0",
+            "string.prototype.trimend": "^1.0.1",
+            "string.prototype.trimstart": "^1.0.1"
+          }
+        }
       }
     },
     "string_decoder": {
@@ -12706,14 +15834,16 @@
       "dev": true,
       "requires": {
         "safe-buffer": "~5.1.0"
+      },
+      "dependencies": {
+        "safe-buffer": {
+          "version": "5.1.2",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+          "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+          "dev": true
+        }
       }
     },
-    "stringify-package": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/stringify-package/-/stringify-package-1.0.0.tgz",
-      "integrity": "sha512-JIQqiWmLiEozOC0b0BtxZ/AOUtdUZHCBPgqIZ2kSJJqGwgb9neo44XdTHUC4HZSGqi03hOeB7W/E8rAlKnGe9g==",
-      "dev": true
-    },
     "strip-ansi": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
@@ -12721,6 +15851,14 @@
       "dev": true,
       "requires": {
         "ansi-regex": "^2.0.0"
+      },
+      "dependencies": {
+        "ansi-regex": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
+          "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=",
+          "dev": true
+        }
       }
     },
     "strip-bom": {
@@ -12744,12 +15882,21 @@
       "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=",
       "dev": true
     },
-    "strip-indent": {
+    "strip-final-newline": {
       "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-2.0.0.tgz",
-      "integrity": "sha1-XvjbKV0B5u1sv3qrlpmNeCJSe2g=",
+      "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz",
+      "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==",
       "dev": true
     },
+    "strip-indent": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-3.0.0.tgz",
+      "integrity": "sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==",
+      "dev": true,
+      "requires": {
+        "min-indent": "^1.0.0"
+      }
+    },
     "strong-log-transformer": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/strong-log-transformer/-/strong-log-transformer-2.1.0.tgz",
@@ -12769,6 +15916,33 @@
         "has-flag": "^3.0.0"
       }
     },
+    "supports-hyperlinks": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/supports-hyperlinks/-/supports-hyperlinks-2.1.0.tgz",
+      "integrity": "sha512-zoE5/e+dnEijk6ASB6/qrK+oYdm2do1hjoLWrqUC/8WEIW1gbxFcKuBof7sW8ArN6e+AYvsE8HBGiVRWL/F5CA==",
+      "dev": true,
+      "requires": {
+        "has-flag": "^4.0.0",
+        "supports-color": "^7.0.0"
+      },
+      "dependencies": {
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
+        }
+      }
+    },
     "sver-compat": {
       "version": "1.5.0",
       "resolved": "https://registry.npmjs.org/sver-compat/-/sver-compat-1.5.0.tgz",
@@ -12786,15 +15960,15 @@
       "dev": true
     },
     "symbol-tree": {
-      "version": "3.2.2",
-      "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.2.tgz",
-      "integrity": "sha1-rifbOPZgp64uHDt9G8KQgZuFGeY=",
+      "version": "3.2.4",
+      "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
+      "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==",
       "dev": true
     },
     "table-layout": {
-      "version": "0.4.4",
-      "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-0.4.4.tgz",
-      "integrity": "sha512-uNaR3SRMJwfdp9OUr36eyEi6LLsbcTqTO/hfTsNviKsNeyMBPICJCC7QXRF3+07bAP6FRwA8rczJPBqXDc0CkQ==",
+      "version": "0.4.5",
+      "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-0.4.5.tgz",
+      "integrity": "sha512-zTvf0mcggrGeTe/2jJ6ECkJHAQPIYEwDoqsiqBjI24mvRmQbInK5jq33fyypaCBxX08hMkfmdOqj6haT33EqWw==",
       "requires": {
         "array-back": "^2.0.0",
         "deep-extend": "~0.6.0",
@@ -12804,20 +15978,35 @@
       }
     },
     "tapable": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/tapable/-/tapable-1.1.1.tgz",
-      "integrity": "sha512-9I2ydhj8Z9veORCw5PRm4u9uebCn0mcCa6scWoNcbZ6dAtoo2618u9UUzxgmsCOreJpqDDuv61LvwofW7hLcBA==",
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/tapable/-/tapable-1.1.3.tgz",
+      "integrity": "sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==",
       "dev": true
     },
     "tar": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/tar/-/tar-2.2.1.tgz",
-      "integrity": "sha1-jk0qJWwOIYXGsYrWlK7JaLg8sdE=",
+      "version": "4.4.13",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.13.tgz",
+      "integrity": "sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA==",
       "dev": true,
       "requires": {
-        "block-stream": "*",
-        "fstream": "^1.0.2",
-        "inherits": "2"
+        "chownr": "^1.1.1",
+        "fs-minipass": "^1.2.5",
+        "minipass": "^2.8.6",
+        "minizlib": "^1.2.1",
+        "mkdirp": "^0.5.0",
+        "safe-buffer": "^5.1.2",
+        "yallist": "^3.0.3"
+      },
+      "dependencies": {
+        "mkdirp": {
+          "version": "0.5.5",
+          "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
+          "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
+          "dev": true,
+          "requires": {
+            "minimist": "^1.2.5"
+          }
+        }
       }
     },
     "temp-dir": {
@@ -12838,51 +16027,38 @@
         "pify": "^3.0.0",
         "temp-dir": "^1.0.0",
         "uuid": "^3.0.1"
-      }
-    },
-    "terser": {
-      "version": "3.14.1",
-      "resolved": "https://registry.npmjs.org/terser/-/terser-3.14.1.tgz",
-      "integrity": "sha512-NSo3E99QDbYSMeJaEk9YW2lTg3qS9V0aKGlb+PlOrei1X02r1wSBHCNX/O+yeTRFSWPKPIGj6MqvvdqV4rnVGw==",
-      "dev": true,
-      "requires": {
-        "commander": "~2.17.1",
-        "source-map": "~0.6.1",
-        "source-map-support": "~0.5.6"
       },
       "dependencies": {
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-          "dev": true
-        },
-        "source-map-support": {
-          "version": "0.5.10",
-          "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.10.tgz",
-          "integrity": "sha512-YfQ3tQFTK/yzlGJuX8pTwa4tifQj4QS2Mj7UegOu8jAz59MqIiMGPXxQhVQiIMNzayuUSF/jEuVnfFF5JqybmQ==",
+        "make-dir": {
+          "version": "1.3.0",
+          "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz",
+          "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==",
           "dev": true,
           "requires": {
-            "buffer-from": "^1.0.0",
-            "source-map": "^0.6.0"
+            "pify": "^3.0.0"
           }
         }
       }
     },
-    "terser-webpack-plugin": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-1.2.1.tgz",
-      "integrity": "sha512-GGSt+gbT0oKcMDmPx4SRSfJPE1XaN3kQRWG4ghxKQw9cn5G9x6aCKSsgYdvyM0na9NJ4Drv0RG6jbBByZ5CMjw==",
+    "terminal-link": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/terminal-link/-/terminal-link-2.1.1.tgz",
+      "integrity": "sha512-un0FmiRUQNr5PJqy9kP7c40F5BOfpGlYTrxonDChEZB7pzZxRNp/bt+ymiy9/npwXya9KH99nJ/GXFIiUkYGFQ==",
       "dev": true,
       "requires": {
-        "cacache": "^11.0.2",
-        "find-cache-dir": "^2.0.0",
-        "schema-utils": "^1.0.0",
-        "serialize-javascript": "^1.4.0",
-        "source-map": "^0.6.1",
-        "terser": "^3.8.1",
-        "webpack-sources": "^1.1.0",
-        "worker-farm": "^1.5.2"
+        "ansi-escapes": "^4.2.1",
+        "supports-hyperlinks": "^2.0.0"
+      }
+    },
+    "terser": {
+      "version": "5.3.2",
+      "resolved": "https://registry.npmjs.org/terser/-/terser-5.3.2.tgz",
+      "integrity": "sha512-H67sydwBz5jCUA32ZRL319ULu+Su1cAoZnnc+lXnenGRYWyLE3Scgkt8mNoAsMx0h5kdo758zdoS0LG9rYZXDQ==",
+      "dev": true,
+      "requires": {
+        "commander": "^2.20.0",
+        "source-map": "~0.6.1",
+        "source-map-support": "~0.5.12"
       },
       "dependencies": {
         "source-map": {
@@ -12893,133 +16069,166 @@
         }
       }
     },
-    "test-exclude": {
-      "version": "5.2.3",
-      "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-5.2.3.tgz",
-      "integrity": "sha512-M+oxtseCFO3EDtAaGH7iiej3CBkzXqFMbzqYAACdzKui4eZA+pq3tZEwChvOdNfa7xxy8BfbmgJSIr43cC/+2g==",
+    "terser-webpack-plugin": {
+      "version": "4.2.2",
+      "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-4.2.2.tgz",
+      "integrity": "sha512-3qAQpykRTD5DReLu5/cwpsg7EZFzP3Q0Hp2XUWJUw2mpq2jfgOKTZr8IZKKnNieRVVo1UauROTdhbQJZveGKtQ==",
       "dev": true,
       "requires": {
-        "glob": "^7.1.3",
-        "minimatch": "^3.0.4",
-        "read-pkg-up": "^4.0.0",
-        "require-main-filename": "^2.0.0"
+        "cacache": "^15.0.5",
+        "find-cache-dir": "^3.3.1",
+        "jest-worker": "^26.3.0",
+        "p-limit": "^3.0.2",
+        "schema-utils": "^2.7.1",
+        "serialize-javascript": "^5.0.1",
+        "source-map": "^0.6.1",
+        "terser": "^5.3.2",
+        "webpack-sources": "^1.4.3"
       },
       "dependencies": {
-        "find-up": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz",
-          "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==",
+        "cacache": {
+          "version": "15.0.5",
+          "resolved": "https://registry.npmjs.org/cacache/-/cacache-15.0.5.tgz",
+          "integrity": "sha512-lloiL22n7sOjEEXdL8NAjTgv9a1u43xICE9/203qonkZUCj5X1UEWIdf2/Y0d6QcCtMzbKQyhrcDbdvlZTs/+A==",
+          "dev": true,
+          "requires": {
+            "@npmcli/move-file": "^1.0.1",
+            "chownr": "^2.0.0",
+            "fs-minipass": "^2.0.0",
+            "glob": "^7.1.4",
+            "infer-owner": "^1.0.4",
+            "lru-cache": "^6.0.0",
+            "minipass": "^3.1.1",
+            "minipass-collect": "^1.0.2",
+            "minipass-flush": "^1.0.5",
+            "minipass-pipeline": "^1.2.2",
+            "mkdirp": "^1.0.3",
+            "p-map": "^4.0.0",
+            "promise-inflight": "^1.0.1",
+            "rimraf": "^3.0.2",
+            "ssri": "^8.0.0",
+            "tar": "^6.0.2",
+            "unique-filename": "^1.1.1"
+          }
+        },
+        "chownr": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz",
+          "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==",
+          "dev": true
+        },
+        "fs-minipass": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz",
+          "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==",
           "dev": true,
           "requires": {
-            "locate-path": "^3.0.0"
+            "minipass": "^3.0.0"
           }
         },
-        "load-json-file": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz",
-          "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=",
+        "lru-cache": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+          "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
           "dev": true,
           "requires": {
-            "graceful-fs": "^4.1.2",
-            "parse-json": "^4.0.0",
-            "pify": "^3.0.0",
-            "strip-bom": "^3.0.0"
+            "yallist": "^4.0.0"
           }
         },
-        "locate-path": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz",
-          "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==",
+        "minipass": {
+          "version": "3.1.3",
+          "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.1.3.tgz",
+          "integrity": "sha512-Mgd2GdMVzY+x3IJ+oHnVM+KG3lA5c8tnabyJKmHSaG2kAGpudxuOf8ToDkhumF7UzME7DecbQE9uOZhNm7PuJg==",
+          "dev": true,
+          "requires": {
+            "yallist": "^4.0.0"
+          }
+        },
+        "minizlib": {
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz",
+          "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==",
           "dev": true,
           "requires": {
-            "p-locate": "^3.0.0",
-            "path-exists": "^3.0.0"
+            "minipass": "^3.0.0",
+            "yallist": "^4.0.0"
           }
         },
         "p-limit": {
-          "version": "2.2.0",
-          "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.0.tgz",
-          "integrity": "sha512-pZbTJpoUsCzV48Mc9Nh51VbwO0X9cuPFE8gYwx9BTCt9SF8/b7Zljd2fVgOxhIF/HDTKgpVzs+GPhyKfjLLFRQ==",
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.0.2.tgz",
+          "integrity": "sha512-iwqZSOoWIW+Ew4kAGUlN16J4M7OB3ysMLSZtnhmqx7njIHFPlxWBX8xo3lVTyFVq6mI/lL9qt2IsN1sHwaxJkg==",
           "dev": true,
           "requires": {
             "p-try": "^2.0.0"
           }
         },
-        "p-locate": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz",
-          "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==",
+        "p-map": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz",
+          "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==",
           "dev": true,
           "requires": {
-            "p-limit": "^2.0.0"
+            "aggregate-error": "^3.0.0"
           }
         },
-        "p-try": {
-          "version": "2.2.0",
-          "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
-          "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==",
-          "dev": true
-        },
-        "parse-json": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
-          "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=",
+        "rimraf": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
+          "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
           "dev": true,
           "requires": {
-            "error-ex": "^1.3.1",
-            "json-parse-better-errors": "^1.0.1"
+            "glob": "^7.1.3"
           }
         },
-        "path-exists": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
-          "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=",
+        "source-map": {
+          "version": "0.6.1",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
           "dev": true
         },
-        "path-type": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz",
-          "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==",
+        "ssri": {
+          "version": "8.0.0",
+          "resolved": "https://registry.npmjs.org/ssri/-/ssri-8.0.0.tgz",
+          "integrity": "sha512-aq/pz989nxVYwn16Tsbj1TqFpD5LLrQxHf5zaHuieFV+R0Bbr4y8qUsOA45hXT/N4/9UNXTarBjnjVmjSOVaAA==",
           "dev": true,
           "requires": {
-            "pify": "^3.0.0"
+            "minipass": "^3.1.1"
           }
         },
-        "read-pkg": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz",
-          "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=",
+        "tar": {
+          "version": "6.0.5",
+          "resolved": "https://registry.npmjs.org/tar/-/tar-6.0.5.tgz",
+          "integrity": "sha512-0b4HOimQHj9nXNEAA7zWwMM91Zhhba3pspja6sQbgTpynOJf+bkjBnfybNYzbpLbnwXnbyB4LOREvlyXLkCHSg==",
           "dev": true,
           "requires": {
-            "load-json-file": "^4.0.0",
-            "normalize-package-data": "^2.3.2",
-            "path-type": "^3.0.0"
+            "chownr": "^2.0.0",
+            "fs-minipass": "^2.0.0",
+            "minipass": "^3.0.0",
+            "minizlib": "^2.1.1",
+            "mkdirp": "^1.0.3",
+            "yallist": "^4.0.0"
           }
         },
-        "read-pkg-up": {
+        "yallist": {
           "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-4.0.0.tgz",
-          "integrity": "sha512-6etQSH7nJGsK0RbG/2TeDzZFa8shjQ1um+SwQQ5cwKy0dhSXdOncEhb1CPpvQG4h7FyOV6EB6YlV0yJvZQNAkA==",
-          "dev": true,
-          "requires": {
-            "find-up": "^3.0.0",
-            "read-pkg": "^3.0.0"
-          }
-        },
-        "require-main-filename": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz",
-          "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==",
-          "dev": true
-        },
-        "strip-bom": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
-          "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=",
+          "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+          "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
           "dev": true
         }
       }
     },
+    "test-exclude": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz",
+      "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==",
+      "dev": true,
+      "requires": {
+        "@istanbuljs/schema": "^0.1.2",
+        "glob": "^7.1.4",
+        "minimatch": "^3.0.4"
+      }
+    },
     "test-value": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/test-value/-/test-value-3.0.0.tgz",
@@ -13040,10 +16249,28 @@
       "integrity": "sha512-wiBrwC1EhBelW12Zy26JeOUkQ5mRu+5o8rpsJk5+2t+Y5vE7e842qtZDQ2g1NpX/29HdyFeJ4nSIhI47ENSxlQ==",
       "dev": true
     },
+    "thenify": {
+      "version": "3.3.1",
+      "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz",
+      "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==",
+      "dev": true,
+      "requires": {
+        "any-promise": "^1.0.0"
+      }
+    },
+    "thenify-all": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz",
+      "integrity": "sha1-GhkY1ALY/D+Y+/I02wvMjMEOlyY=",
+      "dev": true,
+      "requires": {
+        "thenify": ">= 3.1.0 < 4"
+      }
+    },
     "throat": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/throat/-/throat-4.1.0.tgz",
-      "integrity": "sha1-iQN8vJLFarGJJua6TLsgDhVnKmo=",
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/throat/-/throat-5.0.0.tgz",
+      "integrity": "sha512-fcwX4mndzpLQKBS1DVYhGAcYaYt7vsHNIvQV+WXMvnow5cgjPphq5CaayLaGsjRdSCKZFNGt7/GYAuXaNOiYCA==",
       "dev": true
     },
     "through": {
@@ -13079,9 +16306,9 @@
       "dev": true
     },
     "timers-browserify": {
-      "version": "2.0.10",
-      "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.10.tgz",
-      "integrity": "sha512-YvC1SV1XdOUaL6gx5CoGroT3Gu49pK9+TZ38ErPldOWW4j49GI1HKs9DV+KGq/w6y+LZ72W1c8cKz2vzY+qpzg==",
+      "version": "2.0.11",
+      "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.11.tgz",
+      "integrity": "sha512-60aV6sgJ5YEbzUdn9c8kYGIqOubPoUdqQCul3SBAsRCZ40s6Y5cMcrW4dt3/k/EsbLVJNl9n6Vz3fTc+k2GeKQ==",
       "dev": true,
       "requires": {
         "setimmediate": "^1.0.4"
@@ -13186,30 +16413,22 @@
       }
     },
     "tough-cookie": {
-      "version": "2.4.3",
-      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz",
-      "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==",
+      "version": "2.5.0",
+      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz",
+      "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==",
       "dev": true,
       "requires": {
-        "psl": "^1.1.24",
-        "punycode": "^1.4.1"
-      },
-      "dependencies": {
-        "punycode": {
-          "version": "1.4.1",
-          "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
-          "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=",
-          "dev": true
-        }
+        "psl": "^1.1.28",
+        "punycode": "^2.1.1"
       }
     },
     "tr46": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz",
-      "integrity": "sha1-qLE/1r/SSJUZZ0zN5VujaTtwbQk=",
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.0.2.tgz",
+      "integrity": "sha512-3n1qG+/5kg+jrbTzwAykB5yRYtQCTqOGKq5U5PE3b0a1/mzo6snDhjGS0zJVJunO0NrT3Dg1MLy5TjWP/UJppg==",
       "dev": true,
       "requires": {
-        "punycode": "^2.1.0"
+        "punycode": "^2.1.1"
       }
     },
     "trash": {
@@ -13256,6 +16475,12 @@
             "slash": "^1.0.0"
           }
         },
+        "ignore": {
+          "version": "3.3.10",
+          "resolved": "https://registry.npmjs.org/ignore/-/ignore-3.3.10.tgz",
+          "integrity": "sha512-Pgs951kaMm5GXP7MOvxERINe3gsaVjUWFm+UZPSq9xYriQAksyhg0csnS0KXSNRD5NmNdapXEpjxG49+AKh/ug==",
+          "dev": true
+        },
         "jsonfile": {
           "version": "2.4.0",
           "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz",
@@ -13264,13 +16489,25 @@
           "requires": {
             "graceful-fs": "^4.1.6"
           }
+        },
+        "p-try": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz",
+          "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=",
+          "dev": true
+        },
+        "slash": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/slash/-/slash-1.0.0.tgz",
+          "integrity": "sha1-xB8vbDn8FtHNF61LXYlhFK5HDVU=",
+          "dev": true
         }
       }
     },
     "trim-newlines": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-2.0.0.tgz",
-      "integrity": "sha1-tAPQuRvlDDMd/EuC7s6yLD3hbSA=",
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-3.0.0.tgz",
+      "integrity": "sha512-C4+gOpvmxaSMKuEf9Qc134F1ZuOHVXKRbtEflf4NTtuuJDEIJ9p5PXsalL8SkeRw+qit1Mo+yuvMPAKwWg/1hA==",
       "dev": true
     },
     "trim-off-newlines": {
@@ -13279,90 +16516,107 @@
       "integrity": "sha1-n5up2e+odkw4dpi8v+sshI8RrbM=",
       "dev": true
     },
-    "trim-right": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/trim-right/-/trim-right-1.0.1.tgz",
-      "integrity": "sha1-yy4SAwZ+DI3h9hQJS5/kVwTqYAM=",
-      "dev": true
-    },
     "ts-jest": {
-      "version": "24.0.2",
-      "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-24.0.2.tgz",
-      "integrity": "sha512-h6ZCZiA1EQgjczxq+uGLXQlNgeg02WWJBbeT8j6nyIBRQdglqbvzDoHahTEIiS6Eor6x8mK6PfZ7brQ9Q6tzHw==",
+      "version": "26.3.0",
+      "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-26.3.0.tgz",
+      "integrity": "sha512-Jq2uKfx6bPd9+JDpZNMBJMdMQUC3sJ08acISj8NXlVgR2d5OqslEHOR2KHMgwymu8h50+lKIm0m0xj/ioYdW2Q==",
       "dev": true,
       "requires": {
+        "@types/jest": "26.x",
         "bs-logger": "0.x",
         "buffer-from": "1.x",
         "fast-json-stable-stringify": "2.x",
+        "jest-util": "26.x",
         "json5": "2.x",
+        "lodash.memoize": "4.x",
         "make-error": "1.x",
-        "mkdirp": "0.x",
-        "resolve": "1.x",
-        "semver": "^5.5",
-        "yargs-parser": "10.x"
+        "mkdirp": "1.x",
+        "semver": "7.x",
+        "yargs-parser": "18.x"
       },
       "dependencies": {
+        "@types/jest": {
+          "version": "26.0.14",
+          "resolved": "https://registry.npmjs.org/@types/jest/-/jest-26.0.14.tgz",
+          "integrity": "sha512-Hz5q8Vu0D288x3iWXePSn53W7hAjP0H7EQ6QvDO9c7t46mR0lNOLlfuwQ+JkVxuhygHzlzPX+0jKdA3ZgSh+Vg==",
+          "dev": true,
+          "requires": {
+            "jest-diff": "^25.2.1",
+            "pretty-format": "^25.2.1"
+          }
+        },
         "camelcase": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz",
-          "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=",
+          "version": "5.3.1",
+          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
+          "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==",
+          "dev": true
+        },
+        "semver": {
+          "version": "7.3.2",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz",
+          "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==",
           "dev": true
         },
         "yargs-parser": {
-          "version": "10.1.0",
-          "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-10.1.0.tgz",
-          "integrity": "sha512-VCIyR1wJoEBZUqk5PA+oOBF6ypbwh5aNB3I50guxAL/quggdfs4TtNHQrSazFA3fYZ+tEqfs0zIGlv0c/rgjbQ==",
+          "version": "18.1.3",
+          "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz",
+          "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==",
           "dev": true,
           "requires": {
-            "camelcase": "^4.1.0"
+            "camelcase": "^5.0.0",
+            "decamelize": "^1.2.0"
           }
         }
       }
     },
     "ts-node": {
-      "version": "8.2.0",
-      "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-8.2.0.tgz",
-      "integrity": "sha512-m8XQwUurkbYqXrKqr3WHCW310utRNvV5OnRVeISeea7LoCWVcdfeB/Ntl8JYWFh+WRoUAdBgESrzKochQt7sMw==",
+      "version": "9.0.0",
+      "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-9.0.0.tgz",
+      "integrity": "sha512-/TqB4SnererCDR/vb4S/QvSZvzQMJN8daAslg7MeaiHvD8rDZsSfXmNeNumyZZzMned72Xoq/isQljYSt8Ynfg==",
       "dev": true,
       "requires": {
         "arg": "^4.1.0",
         "diff": "^4.0.1",
         "make-error": "^1.1.1",
-        "source-map-support": "^0.5.6",
-        "yn": "^3.0.0"
-      },
-      "dependencies": {
-        "diff": {
-          "version": "4.0.1",
-          "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.1.tgz",
-          "integrity": "sha512-s2+XdvhPCOF01LRQBC8hf4vhbVmI2CGS5aZnxLJlT5FtdhPCDFq80q++zK2KlrVorVDdL5BOGZ/VfLrVtYNF+Q==",
-          "dev": true
-        }
+        "source-map-support": "^0.5.17",
+        "yn": "3.1.1"
       }
     },
     "tslib": {
-      "version": "1.9.3",
-      "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.9.3.tgz",
-      "integrity": "sha512-4krF8scpejhaOgqzBEcGM7yDIEfi0/8+8zDRZhNZZ2kjmHJ4hv3zCbQWxoJGz1iw5U0Jl0nma13xzHXcncMavQ=="
+      "version": "1.13.0",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.13.0.tgz",
+      "integrity": "sha512-i/6DQjL8Xf3be4K/E6Wgpekn5Qasl1usyw++dAA35Ue5orEn65VIxOA+YvNNl9HV3qv70T7CNwjODHZrLwvd1Q=="
     },
     "tslint": {
-      "version": "5.12.1",
-      "resolved": "https://registry.npmjs.org/tslint/-/tslint-5.12.1.tgz",
-      "integrity": "sha512-sfodBHOucFg6egff8d1BvuofoOQ/nOeYNfbp7LDlKBcLNrL3lmS5zoiDGyOMdT7YsEXAwWpTdAHwOGOc8eRZAw==",
+      "version": "6.1.3",
+      "resolved": "https://registry.npmjs.org/tslint/-/tslint-6.1.3.tgz",
+      "integrity": "sha512-IbR4nkT96EQOvKE2PW/djGz8iGNeJ4rF2mBfiYaR/nvUWYKJhLwimoJKgjIFEIDibBtOevj7BqCRL4oHeWWUCg==",
       "dev": true,
       "requires": {
-        "babel-code-frame": "^6.22.0",
+        "@babel/code-frame": "^7.0.0",
         "builtin-modules": "^1.1.1",
         "chalk": "^2.3.0",
         "commander": "^2.12.1",
-        "diff": "^3.2.0",
+        "diff": "^4.0.1",
         "glob": "^7.1.1",
-        "js-yaml": "^3.7.0",
+        "js-yaml": "^3.13.1",
         "minimatch": "^3.0.4",
+        "mkdirp": "^0.5.3",
         "resolve": "^1.3.2",
         "semver": "^5.3.0",
-        "tslib": "^1.8.0",
-        "tsutils": "^2.27.2"
+        "tslib": "^1.13.0",
+        "tsutils": "^2.29.0"
+      },
+      "dependencies": {
+        "mkdirp": {
+          "version": "0.5.5",
+          "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
+          "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
+          "dev": true,
+          "requires": {
+            "minimist": "^1.2.5"
+          }
+        }
       }
     },
     "tsutils": {
@@ -13395,6 +16649,12 @@
       "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=",
       "dev": true
     },
+    "type": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz",
+      "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==",
+      "dev": true
+    },
     "type-check": {
       "version": "0.3.2",
       "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz",
@@ -13404,72 +16664,98 @@
         "prelude-ls": "~1.1.2"
       }
     },
+    "type-detect": {
+      "version": "4.0.8",
+      "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz",
+      "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==",
+      "dev": true
+    },
+    "type-fest": {
+      "version": "0.8.1",
+      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz",
+      "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==",
+      "dev": true
+    },
     "typedarray": {
       "version": "0.0.6",
       "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
       "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=",
       "dev": true
     },
+    "typedarray-to-buffer": {
+      "version": "3.1.5",
+      "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz",
+      "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==",
+      "dev": true,
+      "requires": {
+        "is-typedarray": "^1.0.0"
+      }
+    },
     "typedoc": {
-      "version": "0.15.0-0",
-      "resolved": "github:TypeStrong/typedoc#f781b20dda3dfafd453cf540f58fcd8cf4473316",
+      "version": "0.19.1",
+      "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.19.1.tgz",
+      "integrity": "sha512-EqZpRJQUnkwHA1yBhaDExEXUZIiWKddkrDXhRcfUzpnu6pizxNmVTw5IZ3mu682Noa4zQCniE0YNjaAwHQodrA==",
       "dev": true,
       "requires": {
-        "@types/minimatch": "3.0.3",
-        "fs-extra": "^7.0.1",
-        "handlebars": "^4.1.2",
-        "highlight.js": "^9.13.1",
-        "lodash": "^4.17.11",
-        "marked": "^0.6.2",
+        "fs-extra": "^9.0.1",
+        "handlebars": "^4.7.6",
+        "highlight.js": "^10.0.0",
+        "lodash": "^4.17.20",
+        "lunr": "^2.3.9",
+        "marked": "^1.1.1",
         "minimatch": "^3.0.0",
         "progress": "^2.0.3",
-        "shelljs": "^0.8.3",
-        "typedoc-default-themes": "^0.6.0-0",
-        "typescript": "3.4.x"
+        "semver": "^7.3.2",
+        "shelljs": "^0.8.4",
+        "typedoc-default-themes": "^0.11.1"
       },
       "dependencies": {
-        "handlebars": {
-          "version": "4.1.2",
-          "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.1.2.tgz",
-          "integrity": "sha512-nvfrjqvt9xQ8Z/w0ijewdD/vvWDTOweBUm96NTr66Wfvo1mJenBLwcYmPs3TIBP5ruzYGD7Hx/DaM9RmhroGPw==",
+        "fs-extra": {
+          "version": "9.0.1",
+          "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.0.1.tgz",
+          "integrity": "sha512-h2iAoN838FqAFJY2/qVpzFXy+EBxfVE220PalAqQLDVsFOHLJrZvut5puAbCdNv6WJk+B8ihI+k0c7JK5erwqQ==",
           "dev": true,
           "requires": {
-            "neo-async": "^2.6.0",
-            "optimist": "^0.6.1",
-            "source-map": "^0.6.1",
-            "uglify-js": "^3.1.4"
+            "at-least-node": "^1.0.0",
+            "graceful-fs": "^4.2.0",
+            "jsonfile": "^6.0.1",
+            "universalify": "^1.0.0"
           }
         },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+        "jsonfile": {
+          "version": "6.0.1",
+          "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.0.1.tgz",
+          "integrity": "sha512-jR2b5v7d2vIOust+w3wtFKZIfpC2pnRmFAhAC/BuweZFQR8qZzxH1OyrQ10HmdVYiXWkYUqPVsz91cG7EL2FBg==",
+          "dev": true,
+          "requires": {
+            "graceful-fs": "^4.1.6",
+            "universalify": "^1.0.0"
+          }
+        },
+        "semver": {
+          "version": "7.3.2",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz",
+          "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==",
           "dev": true
         },
-        "typescript": {
-          "version": "3.4.5",
-          "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.4.5.tgz",
-          "integrity": "sha512-YycBxUb49UUhdNMU5aJ7z5Ej2XGmaIBL0x34vZ82fn3hGvD+bgrMrVDpatgz2f7YxUMJxMkbWxJZeAvDxVe7Vw==",
+        "universalify": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/universalify/-/universalify-1.0.0.tgz",
+          "integrity": "sha512-rb6X1W158d7pRQBg5gkR8uPaSfiids68LTJQYOtEUhoJUWBdaQHsuT/EUduxXYxcrt4r5PJ4fuHW1MHT6p0qug==",
           "dev": true
         }
       }
     },
     "typedoc-default-themes": {
-      "version": "0.6.0-0",
-      "resolved": "https://registry.npmjs.org/typedoc-default-themes/-/typedoc-default-themes-0.6.0-0.tgz",
-      "integrity": "sha512-O7hBMS1yBCozvVUntIIdlBk04WiqM+f6NOEc9p+LimJSFKJMF66cgzejeiybuTk6mgbMJW+olg42BNYC8E9x9Q==",
-      "dev": true,
-      "requires": {
-        "backbone": "^1.1.2",
-        "jquery": "^2.2.4",
-        "lunr": "^2.3.6",
-        "underscore": "^1.9.1"
-      }
+      "version": "0.11.4",
+      "resolved": "https://registry.npmjs.org/typedoc-default-themes/-/typedoc-default-themes-0.11.4.tgz",
+      "integrity": "sha512-Y4Lf+qIb9NTydrexlazAM46SSLrmrQRqWiD52593g53SsmUFioAsMWt8m834J6qsp+7wHRjxCXSZeiiW5cMUdw==",
+      "dev": true
     },
     "typescript": {
-      "version": "3.5.1",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.5.1.tgz",
-      "integrity": "sha512-64HkdiRv1yYZsSe4xC1WVgamNigVYjlssIoaH2HcZF0+ijsk5YK2g0G34w9wJkze8+5ow4STd22AynfO6ZYYLw==",
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.0.2.tgz",
+      "integrity": "sha512-e4ERvRV2wb+rRZ/IQeb3jm2VxBsirQLpQhdxplZ2MEzGvDkkMmPglecnNDfSUBivMjP93vRbngYYDQqQ/78bcQ==",
       "dev": true
     },
     "typical": {
@@ -13478,24 +16764,11 @@
       "integrity": "sha1-XAgOXWYcu+OCWdLnCjxyU+hziB0="
     },
     "uglify-js": {
-      "version": "3.4.9",
-      "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.4.9.tgz",
-      "integrity": "sha512-8CJsbKOtEbnJsTyv6LE6m6ZKniqMiFWmm9sRbopbkGs3gMPPfd3Fh8iIA4Ykv5MgaTbqHr4BaoGLJLZNhsrW1Q==",
+      "version": "3.10.4",
+      "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.10.4.tgz",
+      "integrity": "sha512-kBFT3U4Dcj4/pJ52vfjCSfyLyvG9VYYuGYPmrPvAxRw/i7xHiT4VvCev+uiEMcEEiu6UNB6KgWmGtSUYIWScbw==",
       "dev": true,
-      "optional": true,
-      "requires": {
-        "commander": "~2.17.1",
-        "source-map": "~0.6.1"
-      },
-      "dependencies": {
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-          "dev": true,
-          "optional": true
-        }
-      }
+      "optional": true
     },
     "uid-number": {
       "version": "0.0.6",
@@ -13515,16 +16788,10 @@
       "integrity": "sha1-5z3T17DXxe2G+6xrCufYxqadUPo=",
       "dev": true
     },
-    "underscore": {
-      "version": "1.9.1",
-      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.9.1.tgz",
-      "integrity": "sha512-5/4etnCkd9c8gwgowi5/om/mYO5ajCaOgdzj/oW+0eQV9WxKBDZw5+ycmKmeaTXjInS/W0BzpGLo2xR2aBwZdg==",
-      "dev": true
-    },
     "undertaker": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/undertaker/-/undertaker-1.2.1.tgz",
-      "integrity": "sha512-71WxIzDkgYk9ZS+spIB8iZXchFhAdEo2YU8xYqBYJ39DIUIqziK78ftm26eecoIY49X0J2MLhG4hr18Yp6/CMA==",
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/undertaker/-/undertaker-1.3.0.tgz",
+      "integrity": "sha512-/RXwi5m/Mu3H6IHQGww3GNt1PNXlbeCuclF2QYR14L/2CHPz3DFZkvB5hZ0N/QUkiXWCACML2jXViIQEQc2MLg==",
       "dev": true,
       "requires": {
         "arr-flatten": "^1.0.1",
@@ -13532,6 +16799,7 @@
         "bach": "^1.0.0",
         "collection-map": "^1.0.0",
         "es6-weak-map": "^2.0.1",
+        "fast-levenshtein": "^1.0.0",
         "last-run": "^1.1.0",
         "object.defaults": "^1.0.0",
         "object.reduce": "^1.0.0",
@@ -13545,38 +16813,15 @@
       "dev": true
     },
     "union-value": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz",
-      "integrity": "sha1-XHHDTLW61dzr4+oM0IIHulqhrqQ=",
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz",
+      "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==",
       "dev": true,
       "requires": {
         "arr-union": "^3.1.0",
         "get-value": "^2.0.6",
         "is-extendable": "^0.1.1",
-        "set-value": "^0.4.3"
-      },
-      "dependencies": {
-        "extend-shallow": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
-          "dev": true,
-          "requires": {
-            "is-extendable": "^0.1.0"
-          }
-        },
-        "set-value": {
-          "version": "0.4.3",
-          "resolved": "https://registry.npmjs.org/set-value/-/set-value-0.4.3.tgz",
-          "integrity": "sha1-fbCPnT0i3H945Trzw79GZuzfzPE=",
-          "dev": true,
-          "requires": {
-            "extend-shallow": "^2.0.1",
-            "is-extendable": "^0.1.1",
-            "is-plain-object": "^2.0.1",
-            "to-object-path": "^0.3.0"
-          }
-        }
+        "set-value": "^2.0.1"
       }
     },
     "unique-filename": {
@@ -13589,9 +16834,9 @@
       }
     },
     "unique-slug": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-2.0.1.tgz",
-      "integrity": "sha512-n9cU6+gITaVu7VGj1Z8feKMmfAjEAQGhwD9fE3zvpRRa0wEIx8ODYkVGfSc94M2OX00tUFV8wH3zYbm1I8mxFg==",
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-2.0.2.tgz",
+      "integrity": "sha512-zoWr9ObaxALD3DOPfjPSqxt4fnZiWblxHIgeWqW8x7UqDzEtHEQLzji2cuJYQFCU6KmoJikOYAZlrTHHebjx2w==",
       "dev": true,
       "requires": {
         "imurmurhash": "^0.1.4"
@@ -13607,6 +16852,15 @@
         "through2-filter": "^3.0.0"
       }
     },
+    "universal-user-agent": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-4.0.1.tgz",
+      "integrity": "sha512-LnST3ebHwVL2aNe4mejI9IQh2HfZ1RLo8Io2HugSif8ekzD1TlWpHpColOB/eh8JHMLkGH3Akqf040I+4ylNxg==",
+      "dev": true,
+      "requires": {
+        "os-name": "^3.1.0"
+      }
+    },
     "universalify": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
@@ -13654,15 +16908,15 @@
       }
     },
     "upath": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/upath/-/upath-1.1.0.tgz",
-      "integrity": "sha512-bzpH/oBhoS/QI/YtbkqCg6VEiPYjSZtrHQM6/QnJS6OL9pKUFLqb3aFh4Scvwm45+7iAgiMkLhSbaZxUqmrprw==",
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/upath/-/upath-1.2.0.tgz",
+      "integrity": "sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg==",
       "dev": true
     },
     "uri-js": {
-      "version": "4.2.2",
-      "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz",
-      "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==",
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.0.tgz",
+      "integrity": "sha512-B0yRTzYdUCCn9n+F4+Gh4yIDtMQcaJsmYBDsTSG8g/OejKBodLQ2IHfN3bM7jUsRXndopT7OIXWdYqc1fjmV6g==",
       "dev": true,
       "requires": {
         "punycode": "^2.1.0"
@@ -13714,6 +16968,14 @@
       "dev": true,
       "requires": {
         "inherits": "2.0.3"
+      },
+      "dependencies": {
+        "inherits": {
+          "version": "2.0.3",
+          "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
+          "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=",
+          "dev": true
+        }
       }
     },
     "util-deprecate": {
@@ -13722,26 +16984,44 @@
       "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=",
       "dev": true
     },
-    "util.promisify": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.0.0.tgz",
-      "integrity": "sha512-i+6qA2MPhvoKLuxnJNpXAGhg7HphQOSUq2LKMZD0m15EiskXUkMvKdF4Uui0WYeCUGea+o2cw/ZuwehtfsrNkA==",
+    "util-promisify": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/util-promisify/-/util-promisify-2.1.0.tgz",
+      "integrity": "sha1-PCI2R2xNMsX/PEcAKt18E7moKlM=",
       "dev": true,
       "requires": {
-        "define-properties": "^1.1.2",
         "object.getownpropertydescriptors": "^2.0.3"
       }
     },
     "uuid": {
-      "version": "3.3.2",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
-      "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==",
+      "version": "3.4.0",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
+      "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
       "dev": true
     },
+    "v8-to-istanbul": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-5.0.1.tgz",
+      "integrity": "sha512-mbDNjuDajqYe3TXFk5qxcQy8L1msXNE37WTlLoqqpBfRsimbNcrlhQlDPntmECEcUvdC+AQ8CyMMf6EUx1r74Q==",
+      "dev": true,
+      "requires": {
+        "@types/istanbul-lib-coverage": "^2.0.1",
+        "convert-source-map": "^1.6.0",
+        "source-map": "^0.7.3"
+      },
+      "dependencies": {
+        "source-map": {
+          "version": "0.7.3",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz",
+          "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==",
+          "dev": true
+        }
+      }
+    },
     "v8flags": {
-      "version": "3.1.3",
-      "resolved": "https://registry.npmjs.org/v8flags/-/v8flags-3.1.3.tgz",
-      "integrity": "sha512-amh9CCg3ZxkzQ48Mhcb8iX7xpAfYJgePHxWMQCBWECpOSqJUXgY26ncA61UTV0BkPqfhcy6mzwCIoP4ygxpW8w==",
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/v8flags/-/v8flags-3.2.0.tgz",
+      "integrity": "sha512-mH8etigqMfiGWdeXpaaqGfs6BndypxusHHcv2qSHyZkGEznCd/qAXCWWRzeowtL54147cktFOC4P5y+kl8d8Jg==",
       "dev": true,
       "requires": {
         "homedir-polyfill": "^1.0.1"
@@ -13784,9 +17064,9 @@
       }
     },
     "vinyl": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/vinyl/-/vinyl-2.2.0.tgz",
-      "integrity": "sha512-MBH+yP0kC/GQ5GwBqrTPTzEfiiLjta7hTtvQtbxBgTeSXsmKQRQecjibMbxIXzVT3Y9KJK+drOz1/k+vsu8Nkg==",
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/vinyl/-/vinyl-2.2.1.tgz",
+      "integrity": "sha512-LII3bXRFBZLlezoG5FfZVcXflZgWP/4dCwKtxd5ky9+LOtM4CS3bIRQsmR1KMnMW07jpE8fqR2lcxPZ+8sJIcw==",
       "dev": true,
       "requires": {
         "clone": "^2.1.1",
@@ -13835,6 +17115,17 @@
         "now-and-later": "^2.0.0",
         "remove-bom-buffer": "^3.0.0",
         "vinyl": "^2.0.0"
+      },
+      "dependencies": {
+        "normalize-path": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.1.1.tgz",
+          "integrity": "sha1-GrKLVW4Zg2Oowab35vogE3/mrtk=",
+          "dev": true,
+          "requires": {
+            "remove-trailing-separator": "^1.0.1"
+          }
+        }
       }
     },
     "vinyl-sourcemaps-apply": {
@@ -13847,21 +17138,27 @@
       }
     },
     "vm-browserify": {
-      "version": "0.0.4",
-      "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-0.0.4.tgz",
-      "integrity": "sha1-XX6kW7755Kb/ZflUOOCofDV9WnM=",
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz",
+      "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==",
+      "dev": true
+    },
+    "w3c-hr-time": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz",
+      "integrity": "sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ==",
       "dev": true,
       "requires": {
-        "indexof": "0.0.1"
+        "browser-process-hrtime": "^1.0.0"
       }
     },
-    "w3c-hr-time": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.1.tgz",
-      "integrity": "sha1-gqwr/2PZUOqeMYmlimViX+3xkEU=",
+    "w3c-xmlserializer": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-2.0.0.tgz",
+      "integrity": "sha512-4tzD0mF8iSiMiNs30BiLO3EpfGLZUT2MSX/G+o7ZywDzliWQ3OPtTZ0PTC3B3ca1UAf4cJMHB+2Bf56EriJuRA==",
       "dev": true,
       "requires": {
-        "browser-process-hrtime": "^0.1.2"
+        "xml-name-validator": "^3.0.0"
       }
     },
     "walker": {
@@ -13874,14 +17171,136 @@
       }
     },
     "watchpack": {
-      "version": "1.6.0",
-      "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.6.0.tgz",
-      "integrity": "sha512-i6dHe3EyLjMmDlU1/bGQpEw25XSjkJULPuAVKCbNRefQVq48yXKUpwg538F7AZTf9kyr57zj++pQFltUa5H7yA==",
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.7.4.tgz",
+      "integrity": "sha512-aWAgTW4MoSJzZPAicljkO1hsi1oKj/RRq/OJQh2PKI2UKL04c2Bs+MBOB+BBABHTXJpf9mCwHN7ANCvYsvY2sg==",
       "dev": true,
       "requires": {
-        "chokidar": "^2.0.2",
+        "chokidar": "^3.4.1",
         "graceful-fs": "^4.1.2",
-        "neo-async": "^2.5.0"
+        "neo-async": "^2.5.0",
+        "watchpack-chokidar2": "^2.0.0"
+      },
+      "dependencies": {
+        "anymatch": {
+          "version": "3.1.1",
+          "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz",
+          "integrity": "sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg==",
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "normalize-path": "^3.0.0",
+            "picomatch": "^2.0.4"
+          }
+        },
+        "binary-extensions": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.1.0.tgz",
+          "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==",
+          "dev": true,
+          "optional": true
+        },
+        "braces": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
+          "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "fill-range": "^7.0.1"
+          }
+        },
+        "chokidar": {
+          "version": "3.4.2",
+          "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.2.tgz",
+          "integrity": "sha512-IZHaDeBeI+sZJRX7lGcXsdzgvZqKv6sECqsbErJA4mHWfpRrD8B97kSFN4cQz6nGBGiuFia1MKR4d6c1o8Cv7A==",
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "anymatch": "~3.1.1",
+            "braces": "~3.0.2",
+            "fsevents": "~2.1.2",
+            "glob-parent": "~5.1.0",
+            "is-binary-path": "~2.1.0",
+            "is-glob": "~4.0.1",
+            "normalize-path": "~3.0.0",
+            "readdirp": "~3.4.0"
+          }
+        },
+        "fill-range": {
+          "version": "7.0.1",
+          "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
+          "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "to-regex-range": "^5.0.1"
+          }
+        },
+        "fsevents": {
+          "version": "2.1.3",
+          "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.3.tgz",
+          "integrity": "sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ==",
+          "dev": true,
+          "optional": true
+        },
+        "glob-parent": {
+          "version": "5.1.1",
+          "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
+          "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "is-glob": "^4.0.1"
+          }
+        },
+        "is-binary-path": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
+          "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==",
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "binary-extensions": "^2.0.0"
+          }
+        },
+        "is-number": {
+          "version": "7.0.0",
+          "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
+          "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
+          "dev": true,
+          "optional": true
+        },
+        "readdirp": {
+          "version": "3.4.0",
+          "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.4.0.tgz",
+          "integrity": "sha512-0xe001vZBnJEK+uKcj8qOhyAKPzIT+gStxWr3LCB0DwcXR5NZJ3IaC+yGnHCYzB/S7ov3m3EEbZI2zeNvX+hGQ==",
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "picomatch": "^2.2.1"
+          }
+        },
+        "to-regex-range": {
+          "version": "5.0.1",
+          "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
+          "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "is-number": "^7.0.0"
+          }
+        }
+      }
+    },
+    "watchpack-chokidar2": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/watchpack-chokidar2/-/watchpack-chokidar2-2.0.0.tgz",
+      "integrity": "sha512-9TyfOyN/zLUbA288wZ8IsMZ+6cbzvsNyEzSBp6e/zkifi6xxbl8SmQ/CxQq32k8NNqrdVEVUVSEf56L4rQ/ZxA==",
+      "dev": true,
+      "optional": true,
+      "requires": {
+        "chokidar": "^2.1.8"
       }
     },
     "wcwidth": {
@@ -13906,9 +17325,9 @@
       "dev": true
     },
     "webidl-conversions": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz",
-      "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==",
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz",
+      "integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==",
       "dev": true
     },
     "webpack": {
@@ -13944,11 +17363,96 @@
       },
       "dependencies": {
         "acorn": {
-          "version": "6.0.5",
-          "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.0.5.tgz",
-          "integrity": "sha512-i33Zgp3XWtmZBMNvCr4azvOFeWVw1Rk6p3hfi3LUDvIFraOMywb1kAtrbi+med14m4Xfpqm3zRZMT+c0FNE7kg==",
+          "version": "6.4.1",
+          "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz",
+          "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==",
+          "dev": true
+        },
+        "find-cache-dir": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-2.1.0.tgz",
+          "integrity": "sha512-Tq6PixE0w/VMFfCgbONnkiQIVol/JJL7nRMi20fqzA4NRs9AfeqMGeRdPi3wIhYkxjeBaWh2rxwapn5Tu3IqOQ==",
+          "dev": true,
+          "requires": {
+            "commondir": "^1.0.1",
+            "make-dir": "^2.0.0",
+            "pkg-dir": "^3.0.0"
+          }
+        },
+        "find-up": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz",
+          "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==",
+          "dev": true,
+          "requires": {
+            "locate-path": "^3.0.0"
+          }
+        },
+        "is-wsl": {
+          "version": "1.1.0",
+          "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-1.1.0.tgz",
+          "integrity": "sha1-HxbkqiKwTRM2tmGIpmrzxgDDpm0=",
+          "dev": true
+        },
+        "locate-path": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz",
+          "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==",
+          "dev": true,
+          "requires": {
+            "p-locate": "^3.0.0",
+            "path-exists": "^3.0.0"
+          }
+        },
+        "make-dir": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz",
+          "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==",
+          "dev": true,
+          "requires": {
+            "pify": "^4.0.1",
+            "semver": "^5.6.0"
+          }
+        },
+        "mkdirp": {
+          "version": "0.5.5",
+          "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
+          "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
+          "dev": true,
+          "requires": {
+            "minimist": "^1.2.5"
+          }
+        },
+        "p-locate": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz",
+          "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==",
+          "dev": true,
+          "requires": {
+            "p-limit": "^2.0.0"
+          }
+        },
+        "path-exists": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
+          "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=",
+          "dev": true
+        },
+        "pify": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz",
+          "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==",
           "dev": true
         },
+        "pkg-dir": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz",
+          "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==",
+          "dev": true,
+          "requires": {
+            "find-up": "^3.0.0"
+          }
+        },
         "schema-utils": {
           "version": "0.4.7",
           "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-0.4.7.tgz",
@@ -13958,13 +17462,69 @@
             "ajv": "^6.1.0",
             "ajv-keywords": "^3.1.0"
           }
+        },
+        "serialize-javascript": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-4.0.0.tgz",
+          "integrity": "sha512-GaNA54380uFefWghODBWEGisLZFj00nS5ACs6yHa9nLqlLpVLO8ChDGeKRjZnV4Nh4n0Qi7nhYZD/9fCPzEqkw==",
+          "dev": true,
+          "requires": {
+            "randombytes": "^2.1.0"
+          }
+        },
+        "source-map": {
+          "version": "0.6.1",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+          "dev": true
+        },
+        "terser": {
+          "version": "4.8.0",
+          "resolved": "https://registry.npmjs.org/terser/-/terser-4.8.0.tgz",
+          "integrity": "sha512-EAPipTNeWsb/3wLPeup1tVPaXfIaU68xMnVdPafIL1TV05OhASArYyIfFvnvJCNrR2NIOvDVNNTFRa+Re2MWyw==",
+          "dev": true,
+          "requires": {
+            "commander": "^2.20.0",
+            "source-map": "~0.6.1",
+            "source-map-support": "~0.5.12"
+          }
+        },
+        "terser-webpack-plugin": {
+          "version": "1.4.5",
+          "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-1.4.5.tgz",
+          "integrity": "sha512-04Rfe496lN8EYruwi6oPQkG0vo8C+HT49X687FZnpPF0qMAIHONI6HEXYPKDOE8e5HjXTyKfqRd/agHtH0kOtw==",
+          "dev": true,
+          "requires": {
+            "cacache": "^12.0.2",
+            "find-cache-dir": "^2.1.0",
+            "is-wsl": "^1.1.0",
+            "schema-utils": "^1.0.0",
+            "serialize-javascript": "^4.0.0",
+            "source-map": "^0.6.1",
+            "terser": "^4.1.2",
+            "webpack-sources": "^1.4.0",
+            "worker-farm": "^1.7.0"
+          },
+          "dependencies": {
+            "schema-utils": {
+              "version": "1.0.0",
+              "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-1.0.0.tgz",
+              "integrity": "sha512-i27Mic4KovM/lnGsy8whRCHhc7VicJajAjTrYg11K9zfZXnYIt4k5F+kZkwjnrhKzLic/HLU4j11mjsz2G/75g==",
+              "dev": true,
+              "requires": {
+                "ajv": "^6.1.0",
+                "ajv-errors": "^1.0.0",
+                "ajv-keywords": "^3.1.0"
+              }
+            }
+          }
         }
       }
     },
     "webpack-sources": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.3.0.tgz",
-      "integrity": "sha512-OiVgSrbGu7NEnEvQJJgdSFPl2qWKkWq5lHMhgiToIiN9w34EBnjYzSYs+VbL5KoYiLNtFFa7BZIKxRED3I32pA==",
+      "version": "1.4.3",
+      "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz",
+      "integrity": "sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==",
       "dev": true,
       "requires": {
         "source-list-map": "^2.0.0",
@@ -13995,14 +17555,14 @@
       "dev": true
     },
     "whatwg-url": {
-      "version": "6.5.0",
-      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-6.5.0.tgz",
-      "integrity": "sha512-rhRZRqx/TLJQWUpQ6bmrt2UV4f0HCQ463yQuONJqC6fO2VoEb1pTYddbe59SkYq87aoM5A3bdhMZiUiVws+fzQ==",
+      "version": "8.2.2",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.2.2.tgz",
+      "integrity": "sha512-PcVnO6NiewhkmzV0qn7A+UZ9Xx4maNTI+O+TShmfE4pqjoCMwUMjkvoNhNHPTvgR7QH9Xt3R13iHuWy2sToFxQ==",
       "dev": true,
       "requires": {
         "lodash.sortby": "^4.7.0",
-        "tr46": "^1.0.1",
-        "webidl-conversions": "^4.0.2"
+        "tr46": "^2.0.2",
+        "webidl-conversions": "^6.1.0"
       }
     },
     "which": {
@@ -14029,10 +17589,25 @@
         "string-width": "^1.0.2 || 2"
       }
     },
+    "windows-release": {
+      "version": "3.3.3",
+      "resolved": "https://registry.npmjs.org/windows-release/-/windows-release-3.3.3.tgz",
+      "integrity": "sha512-OSOGH1QYiW5yVor9TtmXKQvt2vjQqbYS+DqmsZw+r7xDwLXEeT3JGW0ZppFmHx4diyXmxt238KFR3N9jzevBRg==",
+      "dev": true,
+      "requires": {
+        "execa": "^1.0.0"
+      }
+    },
+    "word-wrap": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
+      "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
+      "dev": true
+    },
     "wordwrap": {
-      "version": "0.0.3",
-      "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz",
-      "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=",
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz",
+      "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=",
       "dev": true
     },
     "wordwrapjs": {
@@ -14045,9 +17620,9 @@
       }
     },
     "worker-farm": {
-      "version": "1.6.0",
-      "resolved": "https://registry.npmjs.org/worker-farm/-/worker-farm-1.6.0.tgz",
-      "integrity": "sha512-6w+3tHbM87WnSWnENBUvA2pxJPLhQUg5LKwUQHq3r+XPhIM+Gh2R5ycbwPCyuGbNg+lPgdcnQUhuC02kJCvffQ==",
+      "version": "1.7.0",
+      "resolved": "https://registry.npmjs.org/worker-farm/-/worker-farm-1.7.0.tgz",
+      "integrity": "sha512-rvw3QTZc8lAxyVrqcSGVm5yP/IJ2UcB3U0graE3LCFoZ0Yn2x4EoVSqJKdB/T5M+FLcRPjz4TDacRf3OCfNUzw==",
       "dev": true,
       "requires": {
         "errno": "~0.1.7"
@@ -14070,35 +17645,57 @@
       "dev": true
     },
     "write-file-atomic": {
-      "version": "2.4.2",
-      "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.2.tgz",
-      "integrity": "sha512-s0b6vB3xIVRLWywa6X9TOMA7k9zio0TMOsl9ZnDkliA/cfJlpHXAscj0gbHVJiTdIuAYpIyqS5GW91fqm6gG5g==",
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz",
+      "integrity": "sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==",
       "dev": true,
       "requires": {
-        "graceful-fs": "^4.1.11",
         "imurmurhash": "^0.1.4",
-        "signal-exit": "^3.0.2"
+        "is-typedarray": "^1.0.0",
+        "signal-exit": "^3.0.2",
+        "typedarray-to-buffer": "^3.1.5"
       }
     },
     "write-json-file": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/write-json-file/-/write-json-file-2.3.0.tgz",
-      "integrity": "sha1-K2TIozAE1UuGmMdtWFp3zrYdoy8=",
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/write-json-file/-/write-json-file-3.2.0.tgz",
+      "integrity": "sha512-3xZqT7Byc2uORAatYiP3DHUUAVEkNOswEWNs9H5KXiicRTvzYzYqKjYc4G7p+8pltvAw641lVByKVtMpf+4sYQ==",
       "dev": true,
       "requires": {
         "detect-indent": "^5.0.0",
-        "graceful-fs": "^4.1.2",
-        "make-dir": "^1.0.0",
-        "pify": "^3.0.0",
+        "graceful-fs": "^4.1.15",
+        "make-dir": "^2.1.0",
+        "pify": "^4.0.1",
         "sort-keys": "^2.0.0",
-        "write-file-atomic": "^2.0.0"
+        "write-file-atomic": "^2.4.2"
       },
       "dependencies": {
-        "detect-indent": {
-          "version": "5.0.0",
-          "resolved": "https://registry.npmjs.org/detect-indent/-/detect-indent-5.0.0.tgz",
-          "integrity": "sha1-OHHMCmoALow+Wzz38zYmRnXwa50=",
+        "make-dir": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz",
+          "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==",
+          "dev": true,
+          "requires": {
+            "pify": "^4.0.1",
+            "semver": "^5.6.0"
+          }
+        },
+        "pify": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz",
+          "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==",
           "dev": true
+        },
+        "write-file-atomic": {
+          "version": "2.4.3",
+          "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.3.tgz",
+          "integrity": "sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==",
+          "dev": true,
+          "requires": {
+            "graceful-fs": "^4.1.11",
+            "imurmurhash": "^0.1.4",
+            "signal-exit": "^3.0.2"
+          }
         }
       }
     },
@@ -14110,16 +17707,49 @@
       "requires": {
         "sort-keys": "^2.0.0",
         "write-json-file": "^2.2.0"
+      },
+      "dependencies": {
+        "make-dir": {
+          "version": "1.3.0",
+          "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz",
+          "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==",
+          "dev": true,
+          "requires": {
+            "pify": "^3.0.0"
+          }
+        },
+        "write-file-atomic": {
+          "version": "2.4.3",
+          "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.3.tgz",
+          "integrity": "sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==",
+          "dev": true,
+          "requires": {
+            "graceful-fs": "^4.1.11",
+            "imurmurhash": "^0.1.4",
+            "signal-exit": "^3.0.2"
+          }
+        },
+        "write-json-file": {
+          "version": "2.3.0",
+          "resolved": "https://registry.npmjs.org/write-json-file/-/write-json-file-2.3.0.tgz",
+          "integrity": "sha1-K2TIozAE1UuGmMdtWFp3zrYdoy8=",
+          "dev": true,
+          "requires": {
+            "detect-indent": "^5.0.0",
+            "graceful-fs": "^4.1.2",
+            "make-dir": "^1.0.0",
+            "pify": "^3.0.0",
+            "sort-keys": "^2.0.0",
+            "write-file-atomic": "^2.0.0"
+          }
+        }
       }
     },
     "ws": {
-      "version": "5.2.2",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-5.2.2.tgz",
-      "integrity": "sha512-jaHFD6PFv6UgoIVda6qZllptQsMlDEJkTQcybzzXDYM1XO9Y8em691FGMPmM46WGyLU4z9KMgQN+qrux/nhlHA==",
-      "dev": true,
-      "requires": {
-        "async-limiter": "~1.0.0"
-      }
+      "version": "7.3.1",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-7.3.1.tgz",
+      "integrity": "sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA==",
+      "dev": true
     },
     "xdg-basedir": {
       "version": "2.0.0",
@@ -14173,10 +17803,16 @@
       "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=",
       "dev": true
     },
+    "xmlchars": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",
+      "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==",
+      "dev": true
+    },
     "xtend": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
-      "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=",
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
+      "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
       "dev": true
     },
     "y18n": {
@@ -14186,15 +17822,15 @@
       "dev": true
     },
     "yallist": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
-      "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=",
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
+      "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==",
       "dev": true
     },
     "yargs": {
-      "version": "7.1.0",
-      "resolved": "https://registry.npmjs.org/yargs/-/yargs-7.1.0.tgz",
-      "integrity": "sha1-a6MY6xaWFyf10oT46gA+jWFU0Mg=",
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/yargs/-/yargs-7.1.1.tgz",
+      "integrity": "sha512-huO4Fr1f9PmiJJdll5kwoS2e4GqzGSsMT3PPMpOwoVkOK8ckqAewMTZyA6LXVQWflleb/Z8oPBEvNsMft0XE+g==",
       "dev": true,
       "requires": {
         "camelcase": "^3.0.0",
@@ -14209,22 +17845,23 @@
         "string-width": "^1.0.2",
         "which-module": "^1.0.0",
         "y18n": "^3.2.1",
-        "yargs-parser": "^5.0.0"
+        "yargs-parser": "5.0.0-security.0"
       }
     },
     "yargs-parser": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-5.0.0.tgz",
-      "integrity": "sha1-J17PDX/+Bcd+ZOfIbkzZS/DhIoo=",
+      "version": "5.0.0-security.0",
+      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-5.0.0-security.0.tgz",
+      "integrity": "sha512-T69y4Ps64LNesYxeYGYPvfoMTt/7y1XtfpIslUeK4um+9Hu7hlGoRtaDLvdXb7+/tfq4opVa2HRY5xGip022rQ==",
       "dev": true,
       "requires": {
-        "camelcase": "^3.0.0"
+        "camelcase": "^3.0.0",
+        "object.assign": "^4.1.0"
       }
     },
     "yn": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.0.tgz",
-      "integrity": "sha512-kKfnnYkbTfrAdd0xICNFw7Atm8nKpLcLv9AZGEt+kczL/WQVai4e2V6ZN8U/O+iI6WrNuJjNNOyu4zfhl9D3Hg==",
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz",
+      "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==",
       "dev": true
     }
   }
diff --git a/js/package.json b/js/package.json
index 2d5f94c0e08..6029c92fef9 100644
--- a/js/package.json
+++ b/js/package.json
@@ -66,31 +66,31 @@
     "json-bignum": "^0.0.3",
     "pad-left": "^2.1.0",
     "text-encoding-utf-8": "^1.0.2",
-    "tslib": "^1.9.3"
+    "tslib": "^1.12.0"
   },
   "devDependencies": {
     "@types/glob": "7.1.1",
-    "@types/jest": "24.0.13",
+    "@types/jest": "25.2.2",
     "async-done": "1.3.1",
     "benchmark": "2.1.4",
     "coveralls": "3.0.3",
     "del": "3.0.0",
     "esm": "3.2.25",
     "glob": "7.1.4",
-    "google-closure-compiler": "20190513.0.0",
+    "google-closure-compiler": "20200830.0.0",
     "gulp": "4.0.2",
     "gulp-json-transform": "0.4.6",
     "gulp-rename": "1.4.0",
     "gulp-sourcemaps": "2.6.5",
     "gulp-typescript": "5.0.1",
     "ix": "2.5.3",
-    "jest": "24.8.0",
+    "jest": "26.3.0",
     "jest-environment-node-debug": "2.0.0",
     "jest-silent-reporter": "0.1.2",
     "json": "9.0.6",
-    "lerna": "3.10.7",
+    "lerna": "3.22.1",
     "memfs": "2.15.2",
-    "mkdirp": "0.5.1",
+    "mkdirp": "1.0.4",
     "multistream": "2.1.1",
     "npm-run-all": "4.1.5",
     "randomatic": "3.1.1",
@@ -98,13 +98,13 @@
     "rxjs": "5.5.11",
     "shx": "0.3.2",
     "source-map-loader": "0.2.4",
-    "terser-webpack-plugin": "1.2.1",
+    "terser-webpack-plugin": "4.2.2",
     "trash": "4.3.0",
-    "ts-jest": "24.0.2",
-    "ts-node": "8.2.0",
-    "tslint": "5.12.1",
-    "typedoc": "0.15.0-0",
-    "typescript": "3.5.1",
+    "ts-jest": "26.3.0",
+    "ts-node": "9.0.0",
+    "tslint": "6.1.3",
+    "typedoc": "0.19.1",
+    "typescript": "4.0.2",
     "web-stream-tools": "0.0.1",
     "web-streams-polyfill": "2.0.3",
     "webpack": "4.29.0",
@@ -113,5 +113,5 @@
   "engines": {
     "node": ">=11.12"
   },
-  "version": "2.0.0-SNAPSHOT"
+  "version": "3.0.0-SNAPSHOT"
 }
diff --git a/js/src/builder.ts b/js/src/builder.ts
index d9f032b0c35..6065711dd79 100644
--- a/js/src/builder.ts
+++ b/js/src/builder.ts
@@ -493,7 +493,7 @@ export abstract class VariableWidthBuilder<T extends Binary | Utf8 | List | Map_
 type ThroughIterable<T extends DataType = any, TNull = any> = (source: Iterable<T['TValue'] | TNull>) => IterableIterator<V<T>>;
 
 /** @ignore */
-function throughIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>): ThroughIterable<T, TNull> {
+function throughIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) {
     const { ['queueingStrategy']: queueingStrategy = 'count' } = options;
     const { ['highWaterMark']: highWaterMark = queueingStrategy !== 'bytes' ? 1000 : 2 ** 14 } = options;
     const sizeProperty: 'length' | 'byteLength' = queueingStrategy !== 'bytes' ? 'length' : 'byteLength';
@@ -508,14 +508,14 @@ function throughIterable<T extends DataType = any, TNull = any>(options: Iterabl
         if (builder.finish().length > 0 || numChunks === 0) {
             yield builder.toVector();
         }
-    };
+    } as ThroughIterable<T, TNull>;
 }
 
 /** @ignore */
 type ThroughAsyncIterable<T extends DataType = any, TNull = any> = (source: Iterable<T['TValue'] | TNull> | AsyncIterable<T['TValue'] | TNull>) => AsyncIterableIterator<V<T>>;
 
 /** @ignore */
-function throughAsyncIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>): ThroughAsyncIterable<T, TNull> {
+function throughAsyncIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) {
     const { ['queueingStrategy']: queueingStrategy = 'count' } = options;
     const { ['highWaterMark']: highWaterMark = queueingStrategy !== 'bytes' ? 1000 : 2 ** 14 } = options;
     const sizeProperty: 'length' | 'byteLength' = queueingStrategy !== 'bytes' ? 'length' : 'byteLength';
@@ -530,5 +530,5 @@ function throughAsyncIterable<T extends DataType = any, TNull = any>(options: It
         if (builder.finish().length > 0 || numChunks === 0) {
             yield builder.toVector();
         }
-    };
+    } as ThroughAsyncIterable<T, TNull>;
 }
diff --git a/js/src/column.ts b/js/src/column.ts
index 3a2cb4a1d71..0336e884c7a 100644
--- a/js/src/column.ts
+++ b/js/src/column.ts
@@ -24,7 +24,6 @@ import { VectorCtorArgs, VectorType as V } from './interfaces';
 import { Chunked, SearchContinuation } from './vector/chunked';
 
 export interface Column<T extends DataType = any> {
-    typeId: T['TType'];
     concat(...others: Vector<T>[]): Column<T>;
     slice(begin?: number, end?: number): Column<T>;
     clone(chunks?: Vector<T>[], offsets?: Uint32Array): Column<T>;
@@ -36,10 +35,22 @@ export class Column<T extends DataType = any>
                Sliceable<Column<T>>,
                Applicative<T, Column<T>> {
 
+    public static new<T extends DataType>(data: Data<T>, ...args: VectorCtorArgs<V<T>>): Column<T>;
     public static new<T extends DataType>(field: string | Field<T>, ...chunks: (Vector<T> | Vector<T>[])[]): Column<T>;
     public static new<T extends DataType>(field: string | Field<T>, data: Data<T>, ...args: VectorCtorArgs<V<T>>): Column<T>;
     /** @nocollapse */
-    public static new<T extends DataType = any>(field: string | Field<T>, data: Data<T> | Vector<T> | (Data<T> | Vector<T>)[], ...rest: any[]) {
+    public static new<T extends DataType = any>(...args: any[]) {
+
+        let [field, data, ...rest] = args as [
+            string | Field<T>,
+            Data<T> | Vector<T> | (Data<T> | Vector<T>)[],
+            ...any[]
+        ];
+
+        if (typeof field !== 'string' && !(field instanceof Field)) {
+            data = <Data<T> | Vector<T> | (Data<T> | Vector<T>)[]> field;
+            field = '';
+        }
 
         const chunks = Chunked.flatten<T>(
             Array.isArray(data) ? [...data, ...rest] :
diff --git a/js/src/data.ts b/js/src/data.ts
index 59c785dd34c..47f644c0a4e 100644
--- a/js/src/data.ts
+++ b/js/src/data.ts
@@ -70,6 +70,7 @@ export class Data<T extends DataType = DataType> {
      */
     public dictionary?: Vector;
 
+    // @ts-ignore
     public readonly values: Buffers<T>[BufferType.DATA];
     // @ts-ignore
     public readonly typeIds: Buffers<T>[BufferType.TYPE];
@@ -253,20 +254,20 @@ export class Data<T extends DataType = DataType> {
         return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]);
     }
     /** @nocollapse */
-    public static Binary<T extends Binary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, valueOffsets: ValueOffsetsBuffer, data: Uint8Array) {
+    public static Binary<T extends Binary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, valueOffsets: ValueOffsetsBuffer, data: DataBuffer<T>) {
         return new Data(type, offset, length, nullCount, [toInt32Array(valueOffsets), toUint8Array(data), toUint8Array(nullBitmap)]);
     }
     /** @nocollapse */
-    public static Utf8<T extends Utf8>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, valueOffsets: ValueOffsetsBuffer, data: Uint8Array) {
+    public static Utf8<T extends Utf8>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, valueOffsets: ValueOffsetsBuffer, data: DataBuffer<T>) {
         return new Data(type, offset, length, nullCount, [toInt32Array(valueOffsets), toUint8Array(data), toUint8Array(nullBitmap)]);
     }
     /** @nocollapse */
     public static List<T extends List>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, valueOffsets: ValueOffsetsBuffer, child: Data<T['valueType']> | Vector<T['valueType']>) {
-        return new Data(type, offset, length, nullCount, [toInt32Array(valueOffsets), undefined, toUint8Array(nullBitmap)], [child]);
+        return new Data(type, offset, length, nullCount, [toInt32Array(valueOffsets), undefined, toUint8Array(nullBitmap)], child ? [child] : []);
     }
     /** @nocollapse */
     public static FixedSizeList<T extends FixedSizeList>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, child: Data<T['valueType']> | Vector<T['valueType']>) {
-        return new Data(type, offset, length, nullCount, [undefined, undefined, toUint8Array(nullBitmap)], [child]);
+        return new Data(type, offset, length, nullCount, [undefined, undefined, toUint8Array(nullBitmap)], child ? [child] : []);
     }
     /** @nocollapse */
     public static Struct<T extends Struct>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, children: (Data | Vector)[]) {
@@ -274,7 +275,7 @@ export class Data<T extends DataType = DataType> {
     }
     /** @nocollapse */
     public static Map<T extends Map_>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, valueOffsets: ValueOffsetsBuffer, child: (Data | Vector)) {
-        return new Data(type, offset, length, nullCount, [toInt32Array(valueOffsets), undefined, toUint8Array(nullBitmap)], [child]);
+        return new Data(type, offset, length, nullCount, [toInt32Array(valueOffsets), undefined, toUint8Array(nullBitmap)], child ? [child] : []);
     }
     public static Union<T extends SparseUnion>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, typeIds: TypeIdsBuffer, children: (Data | Vector)[], _?: any): Data<T>;
     public static Union<T extends DenseUnion>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, typeIds: TypeIdsBuffer, valueOffsets: ValueOffsetsBuffer, children: (Data | Vector)[]): Data<T>;
diff --git a/js/src/interfaces.ts b/js/src/interfaces.ts
index 54fb872190c..8e16c4150e3 100644
--- a/js/src/interfaces.ts
+++ b/js/src/interfaces.ts
@@ -113,9 +113,9 @@ export type ConstructorType<
 export type VectorCtorType<
     T extends VectorType<R>,
     R extends DataType = any,
-    TCtor extends new (data: Data<R>, ...args: VectorCtorArgs<T, R>) => T =
-                  new (data: Data<R>, ...args: VectorCtorArgs<T, R>) => T
-> = TCtor extends new (data: Data<R>, ...args: VectorCtorArgs<T, R>) => T ? TCtor : never;
+    TCtor extends new (type: R, data?: Data<R>[], offsets?: Uint32Array) => T =
+                  new (type: R, data?: Data<R>[], offsets?: Uint32Array) => T
+> = TCtor extends new (type: R, data?: Data<R>[], offsets?: Uint32Array) => T ? TCtor : never;
 
 /** @ignore */
 export type BuilderCtorType<
@@ -141,10 +141,10 @@ export type BuilderType<T extends Type | DataType = any, TNull = any> =
 
 /** @ignore */
 export type VectorCtor<T extends Type | DataType | VectorType> =
-    T extends VectorType    ? VectorCtorType<T>                      :
+    T extends VectorType    ? VectorCtorType<VectorType<T['TType']>> :
     T extends Type          ? VectorCtorType<VectorType<T>>          :
     T extends DataType      ? VectorCtorType<VectorType<T['TType']>> :
-                              VectorCtorType<vecs.BaseVector>
+                              VectorCtorType<vecs.BaseVector<any>>
     ;
 
 /** @ignore */
@@ -156,15 +156,15 @@ export type BuilderCtor<T extends Type | DataType = any> =
 
 /** @ignore */
 export type DataTypeCtor<T extends Type | DataType | VectorType = any> =
-    T extends DataType      ? ConstructorType<T>                 :
-    T extends VectorType    ? ConstructorType<T['type']>         :
-    T extends Type          ? ConstructorType<TypeToDataType<T>> :
-                              never
+    T extends DataType   ? ConstructorType<T>                 :
+    T extends VectorType ? ConstructorType<T['type']>         :
+    T extends Type       ? ConstructorType<TypeToDataType<T>> :
+                           never
     ;
 
 /** @ignore */
 type TypeToVector<T extends Type> = {
-    [key: number               ]: any                             ;
+    [key: number               ]: vecs.Vector<any>                ;
     [Type.Null                 ]: vecs.NullVector                 ;
     [Type.Bool                 ]: vecs.BoolVector                 ;
     [Type.Int8                 ]: vecs.Int8Vector                 ;
@@ -212,50 +212,50 @@ type TypeToVector<T extends Type> = {
 
 /** @ignore */
 type DataTypeToVector<T extends DataType = any> = {
-    [key: number               ]:                                                                                                            any ;
-    [Type.Null                 ]: T extends type.Null                 ? vecs.NullVector                                     : vecs.BaseVector<T> ;
-    [Type.Bool                 ]: T extends type.Bool                 ? vecs.BoolVector                                     : vecs.BaseVector<T> ;
-    [Type.Int8                 ]: T extends type.Int8                 ? vecs.Int8Vector                                     : vecs.BaseVector<T> ;
-    [Type.Int16                ]: T extends type.Int16                ? vecs.Int16Vector                                    : vecs.BaseVector<T> ;
-    [Type.Int32                ]: T extends type.Int32                ? vecs.Int32Vector                                    : vecs.BaseVector<T> ;
-    [Type.Int64                ]: T extends type.Int64                ? vecs.Int64Vector                                    : vecs.BaseVector<T> ;
-    [Type.Uint8                ]: T extends type.Uint8                ? vecs.Uint8Vector                                    : vecs.BaseVector<T> ;
-    [Type.Uint16               ]: T extends type.Uint16               ? vecs.Uint16Vector                                   : vecs.BaseVector<T> ;
-    [Type.Uint32               ]: T extends type.Uint32               ? vecs.Uint32Vector                                   : vecs.BaseVector<T> ;
-    [Type.Uint64               ]: T extends type.Uint64               ? vecs.Uint64Vector                                   : vecs.BaseVector<T> ;
-    [Type.Int                  ]: T extends type.Int                  ? vecs.IntVector                                      : vecs.BaseVector<T> ;
-    [Type.Float16              ]: T extends type.Float16              ? vecs.Float16Vector                                  : vecs.BaseVector<T> ;
-    [Type.Float32              ]: T extends type.Float32              ? vecs.Float32Vector                                  : vecs.BaseVector<T> ;
-    [Type.Float64              ]: T extends type.Float64              ? vecs.Float64Vector                                  : vecs.BaseVector<T> ;
-    [Type.Float                ]: T extends type.Float                ? vecs.FloatVector                                    : vecs.BaseVector<T> ;
-    [Type.Utf8                 ]: T extends type.Utf8                 ? vecs.Utf8Vector                                     : vecs.BaseVector<T> ;
-    [Type.Binary               ]: T extends type.Binary               ? vecs.BinaryVector                                   : vecs.BaseVector<T> ;
-    [Type.FixedSizeBinary      ]: T extends type.FixedSizeBinary      ? vecs.FixedSizeBinaryVector                          : vecs.BaseVector<T> ;
-    [Type.Date                 ]: T extends type.Date_                ? vecs.DateVector                                     : vecs.BaseVector<T> ;
-    [Type.DateDay              ]: T extends type.DateDay              ? vecs.DateDayVector                                  : vecs.BaseVector<T> ;
-    [Type.DateMillisecond      ]: T extends type.DateMillisecond      ? vecs.DateMillisecondVector                          : vecs.BaseVector<T> ;
-    [Type.Timestamp            ]: T extends type.Timestamp            ? vecs.TimestampVector                                : vecs.BaseVector<T> ;
-    [Type.TimestampSecond      ]: T extends type.TimestampSecond      ? vecs.TimestampSecondVector                          : vecs.BaseVector<T> ;
-    [Type.TimestampMillisecond ]: T extends type.TimestampMillisecond ? vecs.TimestampMillisecondVector                     : vecs.BaseVector<T> ;
-    [Type.TimestampMicrosecond ]: T extends type.TimestampMicrosecond ? vecs.TimestampMicrosecondVector                     : vecs.BaseVector<T> ;
-    [Type.TimestampNanosecond  ]: T extends type.TimestampNanosecond  ? vecs.TimestampNanosecondVector                      : vecs.BaseVector<T> ;
-    [Type.Time                 ]: T extends type.Time                 ? vecs.TimeVector                                     : vecs.BaseVector<T> ;
-    [Type.TimeSecond           ]: T extends type.TimeSecond           ? vecs.TimeSecondVector                               : vecs.BaseVector<T> ;
-    [Type.TimeMillisecond      ]: T extends type.TimeMillisecond      ? vecs.TimeMillisecondVector                          : vecs.BaseVector<T> ;
-    [Type.TimeMicrosecond      ]: T extends type.TimeMicrosecond      ? vecs.TimeMicrosecondVector                          : vecs.BaseVector<T> ;
-    [Type.TimeNanosecond       ]: T extends type.TimeNanosecond       ? vecs.TimeNanosecondVector                           : vecs.BaseVector<T> ;
-    [Type.Decimal              ]: T extends type.Decimal              ? vecs.DecimalVector                                  : vecs.BaseVector<T> ;
-    [Type.Union                ]: T extends type.Union                ? vecs.UnionVector                                    : vecs.BaseVector<T> ;
-    [Type.DenseUnion           ]: T extends type.DenseUnion           ? vecs.DenseUnionVector                               : vecs.BaseVector<T> ;
-    [Type.SparseUnion          ]: T extends type.SparseUnion          ? vecs.SparseUnionVector                              : vecs.BaseVector<T> ;
-    [Type.Interval             ]: T extends type.Interval             ? vecs.IntervalVector                                 : vecs.BaseVector<T> ;
-    [Type.IntervalDayTime      ]: T extends type.IntervalDayTime      ? vecs.IntervalDayTimeVector                          : vecs.BaseVector<T> ;
-    [Type.IntervalYearMonth    ]: T extends type.IntervalYearMonth    ? vecs.IntervalYearMonthVector                        : vecs.BaseVector<T> ;
-    [Type.Map                  ]: T extends type.Map_                 ? vecs.MapVector<T['keyType'], T['valueType']>        : vecs.BaseVector<T> ;
-    [Type.List                 ]: T extends type.List                 ? vecs.ListVector<T['valueType']>                     : vecs.BaseVector<T> ;
-    [Type.Struct               ]: T extends type.Struct               ? vecs.StructVector<T['dataTypes']>                   : vecs.BaseVector<T> ;
-    [Type.Dictionary           ]: T extends type.Dictionary           ? vecs.DictionaryVector<T['valueType'], T['indices']> : vecs.BaseVector<T> ;
-    [Type.FixedSizeList        ]: T extends type.FixedSizeList        ? vecs.FixedSizeListVector<T['valueType']>            : vecs.BaseVector<T> ;
+    [key: number               ]:                                       vecs.Vector<any>                                            ;
+    [Type.Null                 ]: T extends type.Null                 ? vecs.NullVector                                     : never ;
+    [Type.Bool                 ]: T extends type.Bool                 ? vecs.BoolVector                                     : never ;
+    [Type.Int8                 ]: T extends type.Int8                 ? vecs.Int8Vector                                     : never ;
+    [Type.Int16                ]: T extends type.Int16                ? vecs.Int16Vector                                    : never ;
+    [Type.Int32                ]: T extends type.Int32                ? vecs.Int32Vector                                    : never ;
+    [Type.Int64                ]: T extends type.Int64                ? vecs.Int64Vector                                    : never ;
+    [Type.Uint8                ]: T extends type.Uint8                ? vecs.Uint8Vector                                    : never ;
+    [Type.Uint16               ]: T extends type.Uint16               ? vecs.Uint16Vector                                   : never ;
+    [Type.Uint32               ]: T extends type.Uint32               ? vecs.Uint32Vector                                   : never ;
+    [Type.Uint64               ]: T extends type.Uint64               ? vecs.Uint64Vector                                   : never ;
+    [Type.Int                  ]: T extends type.Int                  ? vecs.IntVector                                      : never ;
+    [Type.Float16              ]: T extends type.Float16              ? vecs.Float16Vector                                  : never ;
+    [Type.Float32              ]: T extends type.Float32              ? vecs.Float32Vector                                  : never ;
+    [Type.Float64              ]: T extends type.Float64              ? vecs.Float64Vector                                  : never ;
+    [Type.Float                ]: T extends type.Float                ? vecs.FloatVector                                    : never ;
+    [Type.Utf8                 ]: T extends type.Utf8                 ? vecs.Utf8Vector                                     : never ;
+    [Type.Binary               ]: T extends type.Binary               ? vecs.BinaryVector                                   : never ;
+    [Type.FixedSizeBinary      ]: T extends type.FixedSizeBinary      ? vecs.FixedSizeBinaryVector                          : never ;
+    [Type.Date                 ]: T extends type.Date_                ? vecs.DateVector                                     : never ;
+    [Type.DateDay              ]: T extends type.DateDay              ? vecs.DateDayVector                                  : never ;
+    [Type.DateMillisecond      ]: T extends type.DateMillisecond      ? vecs.DateMillisecondVector                          : never ;
+    [Type.Timestamp            ]: T extends type.Timestamp            ? vecs.TimestampVector                                : never ;
+    [Type.TimestampSecond      ]: T extends type.TimestampSecond      ? vecs.TimestampSecondVector                          : never ;
+    [Type.TimestampMillisecond ]: T extends type.TimestampMillisecond ? vecs.TimestampMillisecondVector                     : never ;
+    [Type.TimestampMicrosecond ]: T extends type.TimestampMicrosecond ? vecs.TimestampMicrosecondVector                     : never ;
+    [Type.TimestampNanosecond  ]: T extends type.TimestampNanosecond  ? vecs.TimestampNanosecondVector                      : never ;
+    [Type.Time                 ]: T extends type.Time                 ? vecs.TimeVector                                     : never ;
+    [Type.TimeSecond           ]: T extends type.TimeSecond           ? vecs.TimeSecondVector                               : never ;
+    [Type.TimeMillisecond      ]: T extends type.TimeMillisecond      ? vecs.TimeMillisecondVector                          : never ;
+    [Type.TimeMicrosecond      ]: T extends type.TimeMicrosecond      ? vecs.TimeMicrosecondVector                          : never ;
+    [Type.TimeNanosecond       ]: T extends type.TimeNanosecond       ? vecs.TimeNanosecondVector                           : never ;
+    [Type.Decimal              ]: T extends type.Decimal              ? vecs.DecimalVector                                  : never ;
+    [Type.Union                ]: T extends type.Union                ? vecs.UnionVector                                    : never ;
+    [Type.DenseUnion           ]: T extends type.DenseUnion           ? vecs.DenseUnionVector                               : never ;
+    [Type.SparseUnion          ]: T extends type.SparseUnion          ? vecs.SparseUnionVector                              : never ;
+    [Type.Interval             ]: T extends type.Interval             ? vecs.IntervalVector                                 : never ;
+    [Type.IntervalDayTime      ]: T extends type.IntervalDayTime      ? vecs.IntervalDayTimeVector                          : never ;
+    [Type.IntervalYearMonth    ]: T extends type.IntervalYearMonth    ? vecs.IntervalYearMonthVector                        : never ;
+    [Type.Map                  ]: T extends type.Map_                 ? vecs.MapVector<T['keyType'], T['valueType']>        : never ;
+    [Type.List                 ]: T extends type.List                 ? vecs.ListVector<T['valueType']>                     : never ;
+    [Type.Struct               ]: T extends type.Struct               ? vecs.StructVector<T['dataTypes']>                   : never ;
+    [Type.Dictionary           ]: T extends type.Dictionary           ? vecs.DictionaryVector<T['valueType'], T['indices']> : never ;
+    [Type.FixedSizeList        ]: T extends type.FixedSizeList        ? vecs.FixedSizeListVector<T['valueType']>            : never ;
 }[T['TType']];
 
 /** @ignore */
@@ -356,48 +356,48 @@ type TypeToBuilder<T extends Type = any, TNull = any> = {
 
 /** @ignore */
 type DataTypeToBuilder<T extends DataType = any, TNull = any> = {
-    [key: number               ]:                                                                                                  builders.Builder<any, TNull> ;
-    [Type.Null                 ]: T extends type.Null                 ? builders.NullBuilder<TNull>                              : builders.Builder<any, TNull> ;
-    [Type.Bool                 ]: T extends type.Bool                 ? builders.BoolBuilder<TNull>                              : builders.Builder<any, TNull> ;
-    [Type.Int8                 ]: T extends type.Int8                 ? builders.Int8Builder<TNull>                              : builders.Builder<any, TNull> ;
-    [Type.Int16                ]: T extends type.Int16                ? builders.Int16Builder<TNull>                             : builders.Builder<any, TNull> ;
-    [Type.Int32                ]: T extends type.Int32                ? builders.Int32Builder<TNull>                             : builders.Builder<any, TNull> ;
-    [Type.Int64                ]: T extends type.Int64                ? builders.Int64Builder<TNull>                             : builders.Builder<any, TNull> ;
-    [Type.Uint8                ]: T extends type.Uint8                ? builders.Uint8Builder<TNull>                             : builders.Builder<any, TNull> ;
-    [Type.Uint16               ]: T extends type.Uint16               ? builders.Uint16Builder<TNull>                            : builders.Builder<any, TNull> ;
-    [Type.Uint32               ]: T extends type.Uint32               ? builders.Uint32Builder<TNull>                            : builders.Builder<any, TNull> ;
-    [Type.Uint64               ]: T extends type.Uint64               ? builders.Uint64Builder<TNull>                            : builders.Builder<any, TNull> ;
-    [Type.Int                  ]: T extends type.Int                  ? builders.IntBuilder<T, TNull>                            : builders.Builder<any, TNull> ;
-    [Type.Float16              ]: T extends type.Float16              ? builders.Float16Builder<TNull>                           : builders.Builder<any, TNull> ;
-    [Type.Float32              ]: T extends type.Float32              ? builders.Float32Builder<TNull>                           : builders.Builder<any, TNull> ;
-    [Type.Float64              ]: T extends type.Float64              ? builders.Float64Builder<TNull>                           : builders.Builder<any, TNull> ;
-    [Type.Float                ]: T extends type.Float                ? builders.FloatBuilder<T, TNull>                          : builders.Builder<any, TNull> ;
-    [Type.Utf8                 ]: T extends type.Utf8                 ? builders.Utf8Builder<TNull>                              : builders.Builder<any, TNull> ;
-    [Type.Binary               ]: T extends type.Binary               ? builders.BinaryBuilder<TNull>                            : builders.Builder<any, TNull> ;
-    [Type.FixedSizeBinary      ]: T extends type.FixedSizeBinary      ? builders.FixedSizeBinaryBuilder<TNull>                   : builders.Builder<any, TNull> ;
-    [Type.Date                 ]: T extends type.Date_                ? builders.DateBuilder<T, TNull>                           : builders.Builder<any, TNull> ;
-    [Type.DateDay              ]: T extends type.DateDay              ? builders.DateDayBuilder<TNull>                           : builders.Builder<any, TNull> ;
-    [Type.DateMillisecond      ]: T extends type.DateMillisecond      ? builders.DateMillisecondBuilder<TNull>                   : builders.Builder<any, TNull> ;
-    [Type.Timestamp            ]: T extends type.Timestamp            ? builders.TimestampBuilder<T, TNull>                      : builders.Builder<any, TNull> ;
-    [Type.TimestampSecond      ]: T extends type.TimestampSecond      ? builders.TimestampSecondBuilder<TNull>                   : builders.Builder<any, TNull> ;
-    [Type.TimestampMillisecond ]: T extends type.TimestampMillisecond ? builders.TimestampMillisecondBuilder<TNull>              : builders.Builder<any, TNull> ;
-    [Type.TimestampMicrosecond ]: T extends type.TimestampMicrosecond ? builders.TimestampMicrosecondBuilder<TNull>              : builders.Builder<any, TNull> ;
-    [Type.TimestampNanosecond  ]: T extends type.TimestampNanosecond  ? builders.TimestampNanosecondBuilder<TNull>               : builders.Builder<any, TNull> ;
-    [Type.Time                 ]: T extends type.Time                 ? builders.TimeBuilder<T, TNull>                           : builders.Builder<any, TNull> ;
-    [Type.TimeSecond           ]: T extends type.TimeSecond           ? builders.TimeSecondBuilder<TNull>                        : builders.Builder<any, TNull> ;
-    [Type.TimeMillisecond      ]: T extends type.TimeMillisecond      ? builders.TimeMillisecondBuilder<TNull>                   : builders.Builder<any, TNull> ;
-    [Type.TimeMicrosecond      ]: T extends type.TimeMicrosecond      ? builders.TimeMicrosecondBuilder<TNull>                   : builders.Builder<any, TNull> ;
-    [Type.TimeNanosecond       ]: T extends type.TimeNanosecond       ? builders.TimeNanosecondBuilder<TNull>                    : builders.Builder<any, TNull> ;
-    [Type.Decimal              ]: T extends type.Decimal              ? builders.DecimalBuilder<TNull>                           : builders.Builder<any, TNull> ;
-    [Type.Union                ]: T extends type.Union                ? builders.UnionBuilder<T, TNull>                          : builders.Builder<any, TNull> ;
-    [Type.DenseUnion           ]: T extends type.DenseUnion           ? builders.DenseUnionBuilder<T, TNull>                     : builders.Builder<any, TNull> ;
-    [Type.SparseUnion          ]: T extends type.SparseUnion          ? builders.SparseUnionBuilder<T, TNull>                    : builders.Builder<any, TNull> ;
-    [Type.Interval             ]: T extends type.Interval             ? builders.IntervalBuilder<T, TNull>                       : builders.Builder<any, TNull> ;
-    [Type.IntervalDayTime      ]: T extends type.IntervalDayTime      ? builders.IntervalDayTimeBuilder<TNull>                   : builders.Builder<any, TNull> ;
-    [Type.IntervalYearMonth    ]: T extends type.IntervalYearMonth    ? builders.IntervalYearMonthBuilder<TNull>                 : builders.Builder<any, TNull> ;
-    [Type.Map                  ]: T extends type.Map_                 ? builders.MapBuilder<T['keyType'], T['valueType'], TNull> : builders.Builder<any, TNull> ;
-    [Type.List                 ]: T extends type.List                 ? builders.ListBuilder<T['valueType'], TNull>              : builders.Builder<any, TNull> ;
-    [Type.Struct               ]: T extends type.Struct               ? builders.StructBuilder<T['dataTypes'], TNull>            : builders.Builder<any, TNull> ;
-    [Type.Dictionary           ]: T extends type.Dictionary           ? builders.DictionaryBuilder<T, TNull>                     : builders.Builder<any, TNull> ;
-    [Type.FixedSizeList        ]: T extends type.FixedSizeList        ? builders.FixedSizeListBuilder<T['valueType'], TNull>     : builders.Builder<any, TNull> ;
+    [key: number               ]:                                       builders.Builder<any, TNull>                                     ;
+    [Type.Null                 ]: T extends type.Null                 ? builders.NullBuilder<TNull>                              : never ;
+    [Type.Bool                 ]: T extends type.Bool                 ? builders.BoolBuilder<TNull>                              : never ;
+    [Type.Int8                 ]: T extends type.Int8                 ? builders.Int8Builder<TNull>                              : never ;
+    [Type.Int16                ]: T extends type.Int16                ? builders.Int16Builder<TNull>                             : never ;
+    [Type.Int32                ]: T extends type.Int32                ? builders.Int32Builder<TNull>                             : never ;
+    [Type.Int64                ]: T extends type.Int64                ? builders.Int64Builder<TNull>                             : never ;
+    [Type.Uint8                ]: T extends type.Uint8                ? builders.Uint8Builder<TNull>                             : never ;
+    [Type.Uint16               ]: T extends type.Uint16               ? builders.Uint16Builder<TNull>                            : never ;
+    [Type.Uint32               ]: T extends type.Uint32               ? builders.Uint32Builder<TNull>                            : never ;
+    [Type.Uint64               ]: T extends type.Uint64               ? builders.Uint64Builder<TNull>                            : never ;
+    [Type.Int                  ]: T extends type.Int                  ? builders.IntBuilder<T, TNull>                            : never ;
+    [Type.Float16              ]: T extends type.Float16              ? builders.Float16Builder<TNull>                           : never ;
+    [Type.Float32              ]: T extends type.Float32              ? builders.Float32Builder<TNull>                           : never ;
+    [Type.Float64              ]: T extends type.Float64              ? builders.Float64Builder<TNull>                           : never ;
+    [Type.Float                ]: T extends type.Float                ? builders.FloatBuilder<T, TNull>                          : never ;
+    [Type.Utf8                 ]: T extends type.Utf8                 ? builders.Utf8Builder<TNull>                              : never ;
+    [Type.Binary               ]: T extends type.Binary               ? builders.BinaryBuilder<TNull>                            : never ;
+    [Type.FixedSizeBinary      ]: T extends type.FixedSizeBinary      ? builders.FixedSizeBinaryBuilder<TNull>                   : never ;
+    [Type.Date                 ]: T extends type.Date_                ? builders.DateBuilder<T, TNull>                           : never ;
+    [Type.DateDay              ]: T extends type.DateDay              ? builders.DateDayBuilder<TNull>                           : never ;
+    [Type.DateMillisecond      ]: T extends type.DateMillisecond      ? builders.DateMillisecondBuilder<TNull>                   : never ;
+    [Type.Timestamp            ]: T extends type.Timestamp            ? builders.TimestampBuilder<T, TNull>                      : never ;
+    [Type.TimestampSecond      ]: T extends type.TimestampSecond      ? builders.TimestampSecondBuilder<TNull>                   : never ;
+    [Type.TimestampMillisecond ]: T extends type.TimestampMillisecond ? builders.TimestampMillisecondBuilder<TNull>              : never ;
+    [Type.TimestampMicrosecond ]: T extends type.TimestampMicrosecond ? builders.TimestampMicrosecondBuilder<TNull>              : never ;
+    [Type.TimestampNanosecond  ]: T extends type.TimestampNanosecond  ? builders.TimestampNanosecondBuilder<TNull>               : never ;
+    [Type.Time                 ]: T extends type.Time                 ? builders.TimeBuilder<T, TNull>                           : never ;
+    [Type.TimeSecond           ]: T extends type.TimeSecond           ? builders.TimeSecondBuilder<TNull>                        : never ;
+    [Type.TimeMillisecond      ]: T extends type.TimeMillisecond      ? builders.TimeMillisecondBuilder<TNull>                   : never ;
+    [Type.TimeMicrosecond      ]: T extends type.TimeMicrosecond      ? builders.TimeMicrosecondBuilder<TNull>                   : never ;
+    [Type.TimeNanosecond       ]: T extends type.TimeNanosecond       ? builders.TimeNanosecondBuilder<TNull>                    : never ;
+    [Type.Decimal              ]: T extends type.Decimal              ? builders.DecimalBuilder<TNull>                           : never ;
+    [Type.Union                ]: T extends type.Union                ? builders.UnionBuilder<T, TNull>                          : never ;
+    [Type.DenseUnion           ]: T extends type.DenseUnion           ? builders.DenseUnionBuilder<T, TNull>                     : never ;
+    [Type.SparseUnion          ]: T extends type.SparseUnion          ? builders.SparseUnionBuilder<T, TNull>                    : never ;
+    [Type.Interval             ]: T extends type.Interval             ? builders.IntervalBuilder<T, TNull>                       : never ;
+    [Type.IntervalDayTime      ]: T extends type.IntervalDayTime      ? builders.IntervalDayTimeBuilder<TNull>                   : never ;
+    [Type.IntervalYearMonth    ]: T extends type.IntervalYearMonth    ? builders.IntervalYearMonthBuilder<TNull>                 : never ;
+    [Type.Map                  ]: T extends type.Map_                 ? builders.MapBuilder<T['keyType'], T['valueType'], TNull> : never ;
+    [Type.List                 ]: T extends type.List                 ? builders.ListBuilder<T['valueType'], TNull>              : never ;
+    [Type.Struct               ]: T extends type.Struct               ? builders.StructBuilder<T['dataTypes'], TNull>            : never ;
+    [Type.Dictionary           ]: T extends type.Dictionary           ? builders.DictionaryBuilder<T, TNull>                     : never ;
+    [Type.FixedSizeList        ]: T extends type.FixedSizeList        ? builders.FixedSizeListBuilder<T['valueType'], TNull>     : never ;
 }[T['TType']];
diff --git a/js/src/io/adapters.ts b/js/src/io/adapters.ts
index 35b2016bc2d..d14dbe7982f 100644
--- a/js/src/io/adapters.ts
+++ b/js/src/io/adapters.ts
@@ -25,18 +25,22 @@ import {
 
 import { ReadableDOMStreamOptions } from './interfaces';
 
+interface ReadableStreamReadResult<T> { done: boolean; value: T; }
+type Uint8ArrayGenerator = Generator<Uint8Array, null, { cmd: 'peek' | 'read', size: number }>;
+type AsyncUint8ArrayGenerator = AsyncGenerator<Uint8Array, null, { cmd: 'peek' | 'read', size: number }>;
+
 /** @ignore */
 export default {
-    fromIterable<T extends ArrayBufferViewInput>(source: Iterable<T> | T): IterableIterator<Uint8Array> {
+    fromIterable<T extends ArrayBufferViewInput>(source: Iterable<T> | T): Uint8ArrayGenerator {
         return pump(fromIterable<T>(source));
     },
-    fromAsyncIterable<T extends ArrayBufferViewInput>(source: AsyncIterable<T> | PromiseLike<T>): AsyncIterableIterator<Uint8Array> {
+    fromAsyncIterable<T extends ArrayBufferViewInput>(source: AsyncIterable<T> | PromiseLike<T>): AsyncUint8ArrayGenerator {
         return pump(fromAsyncIterable<T>(source));
     },
-    fromDOMStream<T extends ArrayBufferViewInput>(source: ReadableStream<T>): AsyncIterableIterator<Uint8Array> {
+    fromDOMStream<T extends ArrayBufferViewInput>(source: ReadableStream<T>): AsyncUint8ArrayGenerator {
         return pump(fromDOMStream<T>(source));
     },
-    fromNodeStream(stream: NodeJS.ReadableStream): AsyncIterableIterator<Uint8Array> {
+    fromNodeStream(stream: NodeJS.ReadableStream): AsyncUint8ArrayGenerator {
         return pump(fromNodeStream(stream));
     },
     // @ts-ignore
@@ -50,12 +54,12 @@ export default {
 };
 
 /** @ignore */
-const pump = <T extends Iterator<any> | AsyncIterator<any>>(iterator: T) => { iterator.next(); return iterator; };
+const pump = <T extends Uint8ArrayGenerator | AsyncUint8ArrayGenerator>(iterator: T) => { iterator.next(); return iterator; };
 
 /** @ignore */
-function* fromIterable<T extends ArrayBufferViewInput>(source: Iterable<T> | T): IterableIterator<Uint8Array> {
+function* fromIterable<T extends ArrayBufferViewInput>(source: Iterable<T> | T): Uint8ArrayGenerator {
 
-    let done: boolean, threw = false;
+    let done: boolean | undefined, threw = false;
     let buffers: Uint8Array[] = [], buffer: Uint8Array;
     let cmd: 'peek' | 'read', size: number, bufferLength = 0;
 
@@ -93,14 +97,15 @@ function* fromIterable<T extends ArrayBufferViewInput>(source: Iterable<T> | T):
     } catch (e) {
         (threw = true) && (typeof it.throw === 'function') && (it.throw(e));
     } finally {
-        (threw === false) && (typeof it.return === 'function') && (it.return());
+        (threw === false) && (typeof it.return === 'function') && (it.return(null!));
     }
+    return null;
 }
 
 /** @ignore */
-async function* fromAsyncIterable<T extends ArrayBufferViewInput>(source: AsyncIterable<T> | PromiseLike<T>): AsyncIterableIterator<Uint8Array> {
+async function* fromAsyncIterable<T extends ArrayBufferViewInput>(source: AsyncIterable<T> | PromiseLike<T>): AsyncUint8ArrayGenerator {
 
-    let done: boolean, threw = false;
+    let done: boolean | undefined, threw = false;
     let buffers: Uint8Array[] = [], buffer: Uint8Array;
     let cmd: 'peek' | 'read', size: number, bufferLength = 0;
 
@@ -113,7 +118,7 @@ async function* fromAsyncIterable<T extends ArrayBufferViewInput>(source: AsyncI
     }
 
     // Yield so the caller can inject the read command before creating the source AsyncIterator
-    ({ cmd, size } = yield <any> null);
+    ({ cmd, size } = (yield <any> null)!);
 
     // initialize the iterator
     let it = toUint8ArrayAsyncIterator(source)[Symbol.asyncIterator]();
@@ -139,15 +144,16 @@ async function* fromAsyncIterable<T extends ArrayBufferViewInput>(source: AsyncI
     } catch (e) {
         (threw = true) && (typeof it.throw === 'function') && (await it.throw(e));
     } finally {
-        (threw === false) && (typeof it.return === 'function') && (await it.return());
+        (threw === false) && (typeof it.return === 'function') && (await it.return(new Uint8Array(0)));
     }
+    return null;
 }
 
 // All this manual Uint8Array chunk management can be avoided if/when engines
 // add support for ArrayBuffer.transfer() or ArrayBuffer.prototype.realloc():
 // https://github.com/domenic/proposal-arraybuffer-transfer
 /** @ignore */
-async function* fromDOMStream<T extends ArrayBufferViewInput>(source: ReadableStream<T>): AsyncIterableIterator<Uint8Array> {
+async function* fromDOMStream<T extends ArrayBufferViewInput>(source: ReadableStream<T>): AsyncUint8ArrayGenerator {
 
     let done = false, threw = false;
     let buffers: Uint8Array[] = [], buffer: Uint8Array;
@@ -191,6 +197,7 @@ async function* fromDOMStream<T extends ArrayBufferViewInput>(source: ReadableSt
         (threw === false) ? (await it['cancel']())
             : source['locked'] && it.releaseLock();
     }
+    return null;
 }
 
 /** @ignore */
@@ -278,10 +285,10 @@ async function readInto(reader: ReadableStreamBYOBReader, buffer: ArrayBufferLik
         return { done: false, value: new Uint8Array(buffer, 0, size) };
     }
     const { done, value } = await reader.read(new Uint8Array(buffer, offset, size - offset));
-    if (((offset += value.byteLength) < size) && !done) {
-        return await readInto(reader, value.buffer, offset, size);
+    if (((offset += value!.byteLength) < size) && !done) {
+        return await readInto(reader, value!.buffer, offset, size);
     }
-    return { done, value: new Uint8Array(value.buffer, 0, offset) };
+    return { done, value: new Uint8Array(value!.buffer, 0, offset) };
 }
 
 /** @ignore */
@@ -298,7 +305,7 @@ const onEvent = <T extends string>(stream: NodeJS.ReadableStream, event: T) => {
 };
 
 /** @ignore */
-async function* fromNodeStream(stream: NodeJS.ReadableStream): AsyncIterableIterator<Uint8Array> {
+async function* fromNodeStream(stream: NodeJS.ReadableStream): AsyncUint8ArrayGenerator {
 
     let events: Event[] = [];
     let event: EventName = 'error';
@@ -319,7 +326,10 @@ async function* fromNodeStream(stream: NodeJS.ReadableStream): AsyncIterableIter
     ({ cmd, size } = yield <any> null);
 
     // ignore stdin if it's a TTY
-    if ((stream as any)['isTTY']) { return yield new Uint8Array(0); }
+    if ((stream as any)['isTTY']) {
+        yield new Uint8Array(0);
+        return null;
+    }
 
     try {
         // initialize the stream event handlers
@@ -365,6 +375,8 @@ async function* fromNodeStream(stream: NodeJS.ReadableStream): AsyncIterableIter
         await cleanup(events, event === 'error' ? err : null);
     }
 
+    return null;
+
     function cleanup<T extends Error | null | void>(events: Event[], err?: T) {
         buffer = buffers = <any> null;
         return new Promise<T>(async (resolve, reject) => {
diff --git a/js/src/io/node/iterable.ts b/js/src/io/node/iterable.ts
index eb5542a1c54..b174d1eed44 100644
--- a/js/src/io/node/iterable.ts
+++ b/js/src/io/node/iterable.ts
@@ -20,11 +20,15 @@ import { isIterable, isAsyncIterable } from '../../util/compat';
 
 /** @ignore */
 type ReadableOptions = import('stream').ReadableOptions;
+/** @ignore */
+type SourceIterator<T> = Generator<T, void, number | null>;
+/** @ignore */
+type AsyncSourceIterator<T> = AsyncGenerator<T, void, number | null>;
 
 /** @ignore */
 export function toNodeStream<T>(source: Iterable<T> | AsyncIterable<T>, options?: ReadableOptions): Readable {
-    if (isAsyncIterable<T>(source)) { return new AsyncIterableReadable(source[Symbol.asyncIterator](), options); }
-    if (isIterable<T>(source)) { return new IterableReadable(source[Symbol.iterator](), options); }
+    if (isAsyncIterable<T>(source)) { return new AsyncIterableReadable(source[Symbol.asyncIterator]() as AsyncSourceIterator<T>, options); }
+    if (isIterable<T>(source)) { return new IterableReadable(source[Symbol.iterator]() as SourceIterator<T>, options); }
     /* istanbul ignore next */
     throw new Error(`toNodeStream() must be called with an Iterable or AsyncIterable`);
 }
@@ -33,8 +37,8 @@ export function toNodeStream<T>(source: Iterable<T> | AsyncIterable<T>, options?
 class IterableReadable<T extends Uint8Array | any> extends Readable {
     private _pulling: boolean;
     private _bytesMode: boolean;
-    private _iterator: Iterator<T>;
-    constructor(it: Iterator<T>, options?: ReadableOptions) {
+    private _iterator: SourceIterator<T>;
+    constructor(it: SourceIterator<T>, options?: ReadableOptions) {
         super(options);
         this._iterator = it;
         this._pulling = false;
@@ -52,7 +56,7 @@ class IterableReadable<T extends Uint8Array | any> extends Readable {
         fn && fn.call(it, e);
         cb && cb(null);
     }
-    private _pull(size: number, it: Iterator<T>) {
+    private _pull(size: number, it: SourceIterator<T>) {
         const bm = this._bytesMode;
         let r: IteratorResult<T> | null = null;
         while (this.readable && !(r = it.next(bm ? size : null)).done) {
@@ -72,8 +76,8 @@ class IterableReadable<T extends Uint8Array | any> extends Readable {
 class AsyncIterableReadable<T extends Uint8Array | any> extends Readable {
     private _pulling: boolean;
     private _bytesMode: boolean;
-    private _iterator: AsyncIterator<T>;
-    constructor(it: AsyncIterator<T>, options?: ReadableOptions) {
+    private _iterator: AsyncSourceIterator<T>;
+    constructor(it: AsyncSourceIterator<T>, options?: ReadableOptions) {
         super(options);
         this._iterator = it;
         this._pulling = false;
@@ -90,7 +94,7 @@ class AsyncIterableReadable<T extends Uint8Array | any> extends Readable {
         it && (fn = e != null && it.throw || it.return);
         fn && fn.call(it, e).then(() => cb && cb(null)) || (cb && cb(null));
     }
-    private async _pull(size: number, it: AsyncIterator<T>) {
+    private async _pull(size: number, it: AsyncSourceIterator<T>) {
         const bm = this._bytesMode;
         let r: IteratorResult<T> | null = null;
         while (this.readable && !(r = await it.next(bm ? size : null)).done) {
diff --git a/js/src/io/stream.ts b/js/src/io/stream.ts
index 8e0bb949794..c714925100c 100644
--- a/js/src/io/stream.ts
+++ b/js/src/io/stream.ts
@@ -110,14 +110,9 @@ export class AsyncByteStream implements Readable<Uint8Array>, AsyncIterableItera
 }
 
 /** @ignore */
-interface ByteStreamSourceIterator<T> extends IterableIterator<T> {
-    next(value?: { cmd: 'peek' | 'read', size?: number | null }): IteratorResult<T>;
-}
-
+type ByteStreamSourceIterator<T> = Generator<T, null, { cmd: 'peek' | 'read', size?: number | null }>;
 /** @ignore */
-interface AsyncByteStreamSourceIterator<T> extends AsyncIterableIterator<T> {
-    next(value?: { cmd: 'peek' | 'read', size?: number | null }): Promise<IteratorResult<T>>;
-}
+type AsyncByteStreamSourceIterator<T> = AsyncGenerator<T, null, { cmd: 'peek' | 'read', size?: number | null }>;
 
 /** @ignore */
 class ByteStreamSource<T> {
diff --git a/js/src/io/whatwg/iterable.ts b/js/src/io/whatwg/iterable.ts
index 31916f2a3bd..b8428f6852c 100644
--- a/js/src/io/whatwg/iterable.ts
+++ b/js/src/io/whatwg/iterable.ts
@@ -19,6 +19,11 @@ import { toUint8Array } from '../../util/buffer';
 import { ReadableDOMStreamOptions } from '../../io/interfaces';
 import { isIterable, isAsyncIterable } from '../../util/compat';
 
+/** @ignore */
+type SourceIterator<T> = Generator<T, void, number | null>;
+/** @ignore */
+type AsyncSourceIterator<T> = AsyncGenerator<T, void, number | null>;
+
 /** @ignore */
 export function toDOMStream<T>(source: Iterable<T> | AsyncIterable<T>, options?: ReadableDOMStreamOptions): ReadableStream<T> {
     if (isAsyncIterable<T>(source)) { return asyncIterableAsReadableDOMStream(source, options); }
@@ -30,18 +35,18 @@ export function toDOMStream<T>(source: Iterable<T> | AsyncIterable<T>, options?:
 /** @ignore */
 function iterableAsReadableDOMStream<T>(source: Iterable<T>, options?: ReadableDOMStreamOptions) {
 
-    let it: Iterator<T> | null = null;
+    let it: SourceIterator<T> | null = null;
     const bm = (options && options.type === 'bytes') || false;
     const hwm = options && options.highWaterMark || (2 ** 24);
 
     return new ReadableStream<T>({
         ...options as any,
-        start(controller) { next(controller, it || (it = source[Symbol.iterator]())); },
+        start(controller) { next(controller, it || (it = source[Symbol.iterator]() as SourceIterator<T>)); },
         pull(controller) { it ? (next(controller, it)) : controller.close(); },
         cancel() { (it && (it.return && it.return()) || true) && (it = null); }
     }, { highWaterMark: bm ? hwm : undefined, ...options });
 
-    function next(controller: ReadableStreamDefaultController<T>, it: Iterator<T>) {
+    function next(controller: ReadableStreamDefaultController<T>, it: SourceIterator<T>) {
         let buf: Uint8Array;
         let r: IteratorResult<T> | null = null;
         let size = controller.desiredSize || null;
@@ -60,18 +65,18 @@ function iterableAsReadableDOMStream<T>(source: Iterable<T>, options?: ReadableD
 /** @ignore */
 function asyncIterableAsReadableDOMStream<T>(source: AsyncIterable<T>, options?: ReadableDOMStreamOptions) {
 
-    let it: AsyncIterator<T> | null = null;
+    let it: AsyncSourceIterator<T> | null = null;
     const bm = (options && options.type === 'bytes') || false;
     const hwm = options && options.highWaterMark || (2 ** 24);
 
     return new ReadableStream<T>({
         ...options as any,
-        async start(controller) { await next(controller, it || (it = source[Symbol.asyncIterator]())); },
+        async start(controller) { await next(controller, it || (it = source[Symbol.asyncIterator]() as AsyncSourceIterator<T>)); },
         async pull(controller) { it ? (await next(controller, it)) : controller.close(); },
         async cancel() { (it && (it.return && await it.return()) || true) && (it = null); },
     }, { highWaterMark: bm ? hwm : undefined, ...options });
 
-    async function next(controller: ReadableStreamDefaultController<T>, it: AsyncIterator<T>) {
+    async function next(controller: ReadableStreamDefaultController<T>, it: AsyncSourceIterator<T>) {
         let buf: Uint8Array;
         let r: IteratorResult<T> | null = null;
         let size = controller.desiredSize || null;
diff --git a/js/src/ipc/reader.ts b/js/src/ipc/reader.ts
index d655d8e1dd6..e44361a7b9b 100644
--- a/js/src/ipc/reader.ts
+++ b/js/src/ipc/reader.ts
@@ -74,7 +74,7 @@ export class RecordBatchReader<T extends { [key: string]: DataType } = any> exte
     public get dictionaries() { return this._impl.dictionaries; }
     public get numDictionaries() { return this._impl.numDictionaries; }
     public get numRecordBatches() { return this._impl.numRecordBatches; }
-    public get footer() { return this._impl.isFile() ? this._impl.footer : null; }
+    public get footer(): Footer | null { return this._impl.isFile() ? this._impl.footer : null; }
 
     public isSync(): this is RecordBatchReaders<T> { return this._impl.isSync(); }
     public isAsync(): this is AsyncRecordBatchReaders<T> { return this._impl.isAsync(); }
@@ -247,13 +247,11 @@ export interface AsyncRecordBatchStreamReader<T extends { [key: string]: DataTyp
 
 /** @ignore */
 export interface RecordBatchFileReader<T extends { [key: string]: DataType } = any> extends RecordBatchStreamReader<T> {
-    footer: Footer;
     readRecordBatch(index: number): RecordBatch<T> | null;
 }
 
 /** @ignore */
 export interface AsyncRecordBatchFileReader<T extends { [key: string]: DataType } = any> extends AsyncRecordBatchStreamReader<T> {
-    footer: Footer;
     readRecordBatch(index: number): Promise<RecordBatch<T> | null>;
 }
 
diff --git a/js/src/type.ts b/js/src/type.ts
index 2f5e33ce1a1..69613b923c0 100644
--- a/js/src/type.ts
+++ b/js/src/type.ts
@@ -202,6 +202,7 @@ export class Float<T extends Floats = Floats> extends DataType<T> {
             case Precision.SINGLE: return Float32Array;
             case Precision.DOUBLE: return Float64Array;
         }
+        // @ts-ignore
         throw new Error(`Unrecognized ${this[Symbol.toStringTag]} type`);
     }
     public toString() { return `Float${(this.precision << 5) || 16}`; }
diff --git a/js/src/util/bn.ts b/js/src/util/bn.ts
index 46b394ec954..7e93dbbdca6 100644
--- a/js/src/util/bn.ts
+++ b/js/src/util/bn.ts
@@ -45,6 +45,7 @@ BigNum.prototype[Symbol.toPrimitive] = function<T extends BN<BigNumArray>>(this:
         case 'string': return bignumToString(this);
         case 'default': return bignumToBigInt(this);
     }
+    // @ts-ignore
     return bignumToString(this);
 };
 
diff --git a/js/src/util/buffer.ts b/js/src/util/buffer.ts
index 9c983705954..dfdfefc5f2e 100644
--- a/js/src/util/buffer.ts
+++ b/js/src/util/buffer.ts
@@ -141,12 +141,13 @@ export function* toArrayBufferViewIterator<T extends TypedArray>(ArrayCtor: Type
            : (source instanceof SharedArrayBuf) ? wrap(source)
     : !isIterable<ArrayBufferViewInput>(source) ? wrap(source) : source;
 
-    yield* pump((function* (it) {
+    yield* pump((function* (it: Iterator<ArrayBufferViewInput, any, number | undefined>): Generator<T, void, number | undefined> {
         let r: IteratorResult<any> = <any> null;
         do {
             r = it.next(yield toArrayBufferView(ArrayCtor, r));
         } while (!r.done);
     })(buffers[Symbol.iterator]()));
+    return new ArrayCtor();
 }
 
 /** @ignore */ export const toInt8ArrayIterator = (input: ArrayBufferViewIteratorInput) => toArrayBufferViewIterator(Int8Array, input);
@@ -163,7 +164,7 @@ export function* toArrayBufferViewIterator<T extends TypedArray>(ArrayCtor: Type
 type ArrayBufferViewAsyncIteratorInput = AsyncIterable<ArrayBufferViewInput> | Iterable<ArrayBufferViewInput> | PromiseLike<ArrayBufferViewInput> | ArrayBufferViewInput;
 
 /** @ignore */
-export async function* toArrayBufferViewAsyncIterator<T extends TypedArray>(ArrayCtor: TypedArrayConstructor<T>, source: ArrayBufferViewAsyncIteratorInput): AsyncIterableIterator<T> {
+export async function* toArrayBufferViewAsyncIterator<T extends TypedArray>(ArrayCtor: TypedArrayConstructor<T>, source: ArrayBufferViewAsyncIteratorInput): AsyncGenerator<T, T, number | undefined> {
 
     // if a Promise, unwrap the Promise and iterate the resolved value
     if (isPromise<ArrayBufferViewInput>(source)) {
@@ -189,12 +190,13 @@ export async function* toArrayBufferViewAsyncIterator<T extends TypedArray>(Arra
     : !isAsyncIterable<ArrayBufferViewInput>(source) ? wrap(source) // If not an AsyncIterable, treat as a sentinel and wrap in an AsyncIterableIterator
                                                      : source; // otherwise if AsyncIterable, use it
 
-    yield* pump((async function* (it) {
+    yield* pump((async function* (it: AsyncIterator<ArrayBufferViewInput, any, number | undefined>): AsyncGenerator<T, void, number | undefined> {
         let r: IteratorResult<any> = <any> null;
         do {
             r = await it.next(yield toArrayBufferView(ArrayCtor, r));
         } while (!r.done);
     })(buffers[Symbol.asyncIterator]()));
+    return new ArrayCtor();
 }
 
 /** @ignore */ export const toInt8ArrayAsyncIterator = (input: ArrayBufferViewAsyncIteratorInput) => toArrayBufferViewAsyncIterator(Int8Array, input);
diff --git a/js/src/vector.ts b/js/src/vector.ts
index 3bd71b984e1..bd7838cdf96 100644
--- a/js/src/vector.ts
+++ b/js/src/vector.ts
@@ -36,8 +36,8 @@ export interface Applicative<T extends DataType, R extends Chunked> {
 }
 
 export interface AbstractVector<T extends DataType = any>
-    extends Clonable<Vector<T>>,
-            Sliceable<Vector<T>>,
+    extends Clonable<AbstractVector<T>>,
+            Sliceable<AbstractVector<T>>,
             Applicative<T, Chunked<T>> {
 
     readonly TType: T['TType'];
@@ -68,4 +68,6 @@ export abstract class AbstractVector<T extends DataType = any> implements Iterab
     public abstract getChildAt<R extends DataType = any>(index: number): Vector<R> | null;
 }
 
+(AbstractVector.prototype as any).data = null;
+
 export { AbstractVector as Vector };
diff --git a/js/src/vector/bool.ts b/js/src/vector/bool.ts
index f3b6ba78079..b555f469271 100644
--- a/js/src/vector/bool.ts
+++ b/js/src/vector/bool.ts
@@ -26,10 +26,10 @@ import { VectorBuilderOptionsAsync } from './index';
 export class BoolVector extends BaseVector<Bool> {
     public static from<TNull = any>(input: Iterable<boolean | TNull>): BoolVector;
     public static from<TNull = any>(input: AsyncIterable<boolean | TNull>): Promise<BoolVector>;
-    public static from<TNull = any>(input: VectorBuilderOptions<Bool, TNull>): Chunked<Bool>;
-    public static from<TNull = any>(input: VectorBuilderOptionsAsync<Bool, TNull>): Promise<Chunked<Bool>>;
+    public static from<TNull = any>(input: VectorBuilderOptions<Bool, boolean | TNull>): Chunked<Bool>;
+    public static from<TNull = any>(input: VectorBuilderOptionsAsync<Bool, boolean | TNull>): Promise<Chunked<Bool>>;
     /** @nocollapse */
-    public static from<TNull = any>(input: Iterable<boolean | TNull> | AsyncIterable<boolean | TNull> | VectorBuilderOptions<Bool, TNull> | VectorBuilderOptionsAsync<Bool, TNull>) {
+    public static from<TNull = any>(input: Iterable<boolean | TNull> | AsyncIterable<boolean | TNull> | VectorBuilderOptions<Bool, boolean | TNull> | VectorBuilderOptionsAsync<Bool, boolean | TNull>) {
         return vectorFromValuesWithType(() => new Bool(), input);
     }
 }
diff --git a/js/src/vector/date.ts b/js/src/vector/date.ts
index 4b28dee0a80..8c2b7a56356 100644
--- a/js/src/vector/date.ts
+++ b/js/src/vector/date.ts
@@ -33,10 +33,10 @@ export class DateVector<T extends Date_ = Date_> extends BaseVector<T> {
     public static from<T extends DateUnit.MILLISECOND>(...args: FromArgs<DateMillisecond>): V<DateMillisecond>;
     public static from<T extends Date_, TNull = any>(input: Iterable<Date | TNull>): V<T>;
     public static from<T extends Date_, TNull = any>(input: AsyncIterable<Date | TNull>): Promise<V<T>>;
-    public static from<T extends Date_, TNull = any>(input: VectorBuilderOptions<T, TNull>): Chunked<T>;
-    public static from<T extends Date_, TNull = any>(input: VectorBuilderOptionsAsync<T, TNull>): Promise<Chunked<T>>;
+    public static from<T extends Date_, TNull = any>(input: VectorBuilderOptions<T, Date | TNull>): Chunked<T>;
+    public static from<T extends Date_, TNull = any>(input: VectorBuilderOptionsAsync<T, Date | TNull>): Promise<Chunked<T>>;
     /** @nocollapse */
-    public static from<T extends Date_, TNull = any>(...args: FromArgs<T> | [Iterable<Date | TNull> | AsyncIterable<Date | TNull> | VectorBuilderOptions<T, TNull> | VectorBuilderOptionsAsync<T, TNull>]) {
+    public static from<T extends Date_, TNull = any>(...args: FromArgs<T> | [Iterable<Date | TNull> | AsyncIterable<Date | TNull> | VectorBuilderOptions<T, Date | TNull> | VectorBuilderOptionsAsync<T, Date | TNull>]) {
         if (args.length === 2) {
             return vectorFromValuesWithType(() => args[1] === DateUnit.DAY ? new DateDay() : new DateMillisecond() as T, args[0]);
         }
diff --git a/js/src/vector/index.ts b/js/src/vector/index.ts
index c110dbaa7c8..4711a6be930 100644
--- a/js/src/vector/index.ts
+++ b/js/src/vector/index.ts
@@ -118,14 +118,14 @@ function vectorFrom<T extends DataType = any, TNull = any>(input: VectorBuilderO
     const { 'values': values = [], ...options } = { 'nullValues': [null, undefined], ...input } as VectorBuilderOptions<T, TNull> | VectorBuilderOptionsAsync<T, TNull>;
     if (isIterable<T['TValue'] | TNull>(values)) {
         const chunks = [...Builder.throughIterable(options)(values)];
-        return chunks.length === 1 ? chunks[0] : Chunked.concat<T>(chunks);
+        return (chunks.length === 1 ? chunks[0] : Chunked.concat<T>(chunks)) as Vector<T>;
     }
     return (async (chunks: V<T>[]) => {
         const transform = Builder.throughAsyncIterable(options);
         for await (const chunk of transform(values)) {
             chunks.push(chunk);
         }
-        return chunks.length === 1 ? chunks[0] : Chunked.concat<T>(chunks);
+        return (chunks.length === 1 ? chunks[0] : Chunked.concat<T>(chunks)) as Vector<T>;
     })([]);
 }
 
diff --git a/js/src/vector/utf8.ts b/js/src/vector/utf8.ts
index 16fe7c9c39d..a891c0dc5a6 100644
--- a/js/src/vector/utf8.ts
+++ b/js/src/vector/utf8.ts
@@ -27,10 +27,10 @@ import { VectorBuilderOptionsAsync } from './index';
 export class Utf8Vector extends BaseVector<Utf8> {
     public static from<TNull = any>(input: Iterable<string | TNull>): Utf8Vector;
     public static from<TNull = any>(input: AsyncIterable<string | TNull>): Promise<Utf8Vector>;
-    public static from<TNull = any>(input: VectorBuilderOptions<Utf8, TNull>): Chunked<Utf8>;
-    public static from<TNull = any>(input: VectorBuilderOptionsAsync<Utf8, TNull>): Promise<Chunked<Utf8>>;
+    public static from<TNull = any>(input: VectorBuilderOptions<Utf8, string | TNull>): Chunked<Utf8>;
+    public static from<TNull = any>(input: VectorBuilderOptionsAsync<Utf8, string | TNull>): Promise<Chunked<Utf8>>;
     /** @nocollapse */
-    public static from<TNull = any>(input: Iterable<string | TNull> | AsyncIterable<string | TNull> | VectorBuilderOptions<Utf8, TNull> | VectorBuilderOptionsAsync<Utf8, TNull>) {
+    public static from<TNull = any>(input: Iterable<string | TNull> | AsyncIterable<string | TNull> | VectorBuilderOptions<Utf8, string | TNull> | VectorBuilderOptionsAsync<Utf8, string | TNull>) {
         return vectorFromValuesWithType(() => new Utf8(), input);
     }
     public asBinary() {
diff --git a/js/src/visitor.ts b/js/src/visitor.ts
index f857a389525..9877a55b7b0 100644
--- a/js/src/visitor.ts
+++ b/js/src/visitor.ts
@@ -122,6 +122,7 @@ function inferDType<T extends DataType>(type: T): Type {
                 case 32: return isSigned ? Type.Int32 : Type.Uint32;
                 case 64: return isSigned ? Type.Int64 : Type.Uint64;
             }
+            // @ts-ignore
             return Type.Int;
         case Type.Float:
             switch((type as any as Float).precision) {
@@ -129,6 +130,7 @@ function inferDType<T extends DataType>(type: T): Type {
                 case Precision.SINGLE: return Type.Float32;
                 case Precision.DOUBLE: return Type.Float64;
             }
+            // @ts-ignore
             return Type.Float;
         case Type.Binary: return Type.Binary;
         case Type.Utf8: return Type.Utf8;
@@ -141,6 +143,7 @@ function inferDType<T extends DataType>(type: T): Type {
                 case TimeUnit.MICROSECOND: return Type.TimeMicrosecond;
                 case TimeUnit.NANOSECOND: return Type.TimeNanosecond;
             }
+            // @ts-ignore
             return Type.Time;
         case Type.Timestamp:
             switch ((type as any as Timestamp).unit) {
@@ -149,18 +152,21 @@ function inferDType<T extends DataType>(type: T): Type {
                 case TimeUnit.MICROSECOND: return Type.TimestampMicrosecond;
                 case TimeUnit.NANOSECOND: return Type.TimestampNanosecond;
             }
+            // @ts-ignore
             return Type.Timestamp;
         case Type.Date:
             switch ((type as any as Date_).unit) {
                 case DateUnit.DAY: return Type.DateDay;
                 case DateUnit.MILLISECOND: return Type.DateMillisecond;
             }
+            // @ts-ignore
             return Type.Date;
         case Type.Interval:
             switch ((type as any as Interval).unit) {
                 case IntervalUnit.DAY_TIME: return Type.IntervalDayTime;
                 case IntervalUnit.YEAR_MONTH: return Type.IntervalYearMonth;
             }
+            // @ts-ignore
             return Type.Interval;
         case Type.Map: return Type.Map;
         case Type.List: return Type.List;
@@ -170,6 +176,7 @@ function inferDType<T extends DataType>(type: T): Type {
                 case UnionMode.Dense: return Type.DenseUnion;
                 case UnionMode.Sparse: return Type.SparseUnion;
             }
+            // @ts-ignore
             return Type.Union;
         case Type.FixedSizeBinary: return Type.FixedSizeBinary;
         case Type.FixedSizeList: return Type.FixedSizeList;
diff --git a/js/src/visitor/indexof.ts b/js/src/visitor/indexof.ts
index 1b0ef0341f9..f853167e03b 100644
--- a/js/src/visitor/indexof.ts
+++ b/js/src/visitor/indexof.ts
@@ -95,7 +95,7 @@ function nullIndexOf(vector: VectorType<Null>, searchElement?: null) {
 
 /** @ignore */
 function indexOfNull<T extends DataType>(vector: VectorType<T>, fromIndex?: number): number {
-    const { nullBitmap } = vector;
+    const { nullBitmap } = vector.data;
     if (!nullBitmap || vector.nullCount <= 0) {
         return -1;
     }
diff --git a/js/src/visitor/iterator.ts b/js/src/visitor/iterator.ts
index 8e33e73d219..0449cbc2040 100644
--- a/js/src/visitor/iterator.ts
+++ b/js/src/visitor/iterator.ts
@@ -91,7 +91,7 @@ export class IteratorVisitor extends Visitor {}
 function nullableIterator<T extends DataType>(vector: VectorType<T>): IterableIterator<T['TValue'] | null> {
     const getFn = getVisitor.getVisitFn(vector);
     return iterateBits<T['TValue'] | null>(
-        vector.nullBitmap, vector.offset, vector.length, vector,
+        vector.data.nullBitmap, vector.data.offset, vector.length, vector,
         (vec: VectorType<T>, idx: number, nullByte: number, nullBit: number) =>
             ((nullByte & 1 << nullBit) !== 0) ? getFn(vec, idx) : null
     );
@@ -114,7 +114,7 @@ function vectorIterator<T extends DataType>(vector: VectorType<T>): IterableIter
         (typeId === Type.Time && (type as Time).bitWidth !== 64) ||
         (typeId === Type.Float && (type as Float).precision > 0 /* Precision.HALF */)
     )) {
-        return vector.values.subarray(0, length)[Symbol.iterator]();
+        return vector.data.values.subarray(0, length)[Symbol.iterator]();
     }
 
     // Otherwise, iterate manually
diff --git a/js/src/visitor/toarray.ts b/js/src/visitor/toarray.ts
index eeebaaa1314..395e9943c0b 100644
--- a/js/src/visitor/toarray.ts
+++ b/js/src/visitor/toarray.ts
@@ -96,7 +96,7 @@ function arrayOfVector<T extends DataType>(vector: VectorType<T>): T['TArray'] {
         case Type.Int:
         case Type.Float: case Type.Decimal:
         case Type.Time: case Type.Timestamp:
-            return vector.values.subarray(0, length * stride);
+            return vector.data.values.subarray(0, length * stride);
     }
 
     // Otherwise if not primitive, slow copy
diff --git a/js/test/Arrow.ts b/js/test/Arrow.ts
index 8ebf229a9c3..e81c07f5e6d 100644
--- a/js/test/Arrow.ts
+++ b/js/test/Arrow.ts
@@ -27,6 +27,13 @@ require('web-stream-tools');
 (<any> global).window = (<any> global).window || global;
 
 // Fix for Jest in node v10.x
+Object.defineProperty(Object, Symbol.hasInstance, {
+    writable: true,
+    configurable: true,
+    value(inst: any) {
+        return inst && inst.constructor && inst.constructor.name === 'Object';
+    }
+});
 Object.defineProperty(ArrayBuffer, Symbol.hasInstance, {
     writable: true,
     configurable: true,
diff --git a/js/test/inference/column.ts b/js/test/inference/column.ts
index 521cec52e33..442dd8c8749 100644
--- a/js/test/inference/column.ts
+++ b/js/test/inference/column.ts
@@ -19,7 +19,7 @@ import { Data } from '../../src/data';
 import { Field } from '../../src/schema';
 import { Column } from '../../src/column';
 import { Vector } from '../../src/vector';
-import { Bool, Int8, Utf8, List, Dictionary, Struct, Map_ } from '../../src/type';
+import { Bool, Int8, Utf8, List, Dictionary, Struct } from '../../src/type';
 
 const boolType = new Bool();
 const boolVector = Vector.new(Data.Bool(boolType, 0, 10, 0, null, new Uint8Array(2)));
@@ -33,33 +33,6 @@ const boolColumn = new Column(new Field('bool', boolType), [
 expect(typeof boolVector.get(0) === 'boolean').toBe(true);
 expect(typeof boolColumn.get(0) === 'boolean').toBe(true);
 
-type NamedSchema = {
-    a: Int8;
-    b: Utf8;
-    c: Dictionary<List<Bool>>;
-};
-
-const mapChildFields = [
-    { name: 'a', type: new Int8() },
-    { name: 'b', type: new Utf8() },
-    { name: 'c', type: new Dictionary<List<Bool>>(null!, null!) }
-].map(({ name, type }) => new Field(name, type));
-
-const mapType = new Map_<NamedSchema>(mapChildFields);
-
-const mapVector = Vector.new(Data.Map(mapType, 0, 0, 0, null, []));
-const mapColumn = new Column(new Field('map', mapType, false), [
-    Vector.new(Data.Map(mapType, 0, 0, 0, null, [])),
-    Vector.new(Data.Map(mapType, 0, 0, 0, null, [])),
-    Vector.new(Data.Map(mapType, 0, 0, 0, null, [])),
-]);
-
-const { a: a1, b: b1, c: c1 } = mapVector.get(0)!;
-const { a: a2, b: b2, c: c2 } = mapColumn.get(0)!;
-
-console.log(a1, b1, c1);
-console.log(a2, b2, c2);
-
 type IndexSchema = {
     0: Int8;
     1: Utf8;
diff --git a/js/test/inference/nested.ts b/js/test/inference/nested.ts
index ee4c76cd735..5a811b576b3 100644
--- a/js/test/inference/nested.ts
+++ b/js/test/inference/nested.ts
@@ -19,8 +19,7 @@ import { Data } from '../../src/data';
 import { Field } from '../../src/schema';
 import { DataType } from '../../src/type';
 import { Vector, BoolVector } from '../../src/vector/index';
-import { RowProxyGenerator as Row } from '../../src/vector/row';
-import { Bool, Int8, Utf8, List, Dictionary, Struct, Map_ } from '../../src/type';
+import { Bool, Int8, Utf8, List, Dictionary, Struct } from '../../src/type';
 
 type NamedSchema = { a: Int8, b: Utf8, c: Dictionary<List<Bool>>; [idx: string]: DataType; };
 type IndexSchema = { 0: Int8, 1: Utf8, 2: Dictionary<List<Bool>>; [idx: number]: DataType; };
@@ -34,30 +33,30 @@ function checkIndexTypes(schema: IndexSchema) {
         Object.keys(schema).map((x) => new Field(x, schema[(<any> x)]))
     ), 0, 0, 0, null, []);
 
-    const row = Row.new(Vector.new(data), schema).bind(0);
+    const row = Vector.new(data).bind(0);
 
     const check_0 = (x = row[0]) => expect(typeof x === 'number').toBe(true);
     const check_1 = (x = row[1]) => expect(typeof x === 'string').toBe(true);
     const check_2 = (x = row[2]) => expect(x instanceof BoolVector).toBe(true);
 
-    check_0() && check_0(row[0]) && check_0(row.get(0));
-    check_1() && check_1(row[1]) && check_1(row.get(1));
-    check_2() && check_2(row[2]) && check_2(row.get(2));
+    check_0(); check_0(row[0]); check_0(row.get(0));
+    check_1(); check_1(row[1]); check_1(row.get(1));
+    check_2(); check_2(row[2]); check_2(row.get(2));
 }
 
 function checkNamedTypes(schema: NamedSchema) {
 
-    const data = Data.Map(new Map_<NamedSchema>(
+    const data = Data.Struct(new Struct<NamedSchema>(
         Object.keys(schema).map((x) => new Field(x, schema[x]))
     ), 0, 0, 0, null, []);
 
-    const row = Row.new(Vector.new(data), schema).bind(0);
+    const row = Vector.new(data).bind(0);
 
     const check_a = (x = row.a) => expect(typeof x === 'number').toBe(true);
     const check_b = (x = row.b) => expect(typeof x === 'string').toBe(true);
     const check_c = (x = row.c) => expect(x instanceof BoolVector).toBe(true);
 
-    check_a() && check_a(row.a) && check_a(row.get('a'));
-    check_b() && check_b(row.b) && check_b(row.get('b'));
-    check_c() && check_c(row.c) && check_c(row.get('c'));
+    check_a(); check_a(row.a); check_a(row.get('a'));
+    check_b(); check_b(row.b); check_b(row.get('b'));
+    check_c(); check_c(row.c); check_c(row.get('c'));
 }
diff --git a/js/test/unit/builders/builder-tests.ts b/js/test/unit/builders/builder-tests.ts
index b6184f46863..6817999a034 100644
--- a/js/test/unit/builders/builder-tests.ts
+++ b/js/test/unit/builders/builder-tests.ts
@@ -155,7 +155,7 @@ function validateBuilderWithNullValues(suiteName: string, nullValues: any[], gen
     }
 
     if (DataType.isInt(type) && type.bitWidth === 64 && ArrayBuffer.isView(nullValues[0])) {
-        referenceNullValues[0] = util.BN.new(nullValues[0])[Symbol.toPrimitive]('default');
+        referenceNullValues[0] = util.BN.new<any>(nullValues[0])[Symbol.toPrimitive]('default');
     }
 
     describe(suiteName, () => {
diff --git a/js/test/unit/ipc/helpers.ts b/js/test/unit/ipc/helpers.ts
index 0b994792d6b..f8ae1609f3f 100644
--- a/js/test/unit/ipc/helpers.ts
+++ b/js/test/unit/ipc/helpers.ts
@@ -54,13 +54,13 @@ export abstract class ArrowIOTestHelper {
             await testFn(await this.writer(this.table).toUint8Array());
         };
     }
-    iterable(testFn: (iterable: Iterable<Uint8Array>) => void | Promise<void>) {
+    iterable(testFn: (iterable: Generator<Uint8Array>) => void | Promise<void>) {
         return async () => {
             expect.hasAssertions();
             await testFn(chunkedIterable(await this.writer(this.table).toUint8Array()));
         };
     }
-    asyncIterable(testFn: (asyncIterable: AsyncIterable<Uint8Array>) => void | Promise<void>) {
+    asyncIterable(testFn: (asyncIterable: AsyncGenerator<Uint8Array>) => void | Promise<void>) {
         return async () => {
             expect.hasAssertions();
             await testFn(asyncChunkedIterable(await this.writer(this.table).toUint8Array()));
diff --git a/js/test/unit/ipc/reader/streams-node-tests.ts b/js/test/unit/ipc/reader/streams-node-tests.ts
index b03b68c3185..fe0795e9ffa 100644
--- a/js/test/unit/ipc/reader/streams-node-tests.ts
+++ b/js/test/unit/ipc/reader/streams-node-tests.ts
@@ -209,7 +209,8 @@ import { validateRecordBatchAsyncIterator } from '../validate';
             }
         }
 
-        validateStreamState(reader, stream, true, true);
+        // stream.readable should be false here
+        validateStreamState(reader, stream, true);
         expect(tableIndex).toBe(tables.length / 2 | 0);
     });
 })();
diff --git a/js/test/unit/visitor-tests.ts b/js/test/unit/visitor-tests.ts
index 9a65b243bf6..9a6bf26ffd9 100644
--- a/js/test/unit/visitor-tests.ts
+++ b/js/test/unit/visitor-tests.ts
@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+import { Field } from '../Arrow';
 import { Visitor } from '../Arrow';
 import {
     DataType, Dictionary,
@@ -110,7 +111,7 @@ describe('Visitor', () => {
         test(`visits Dictionary types`, () => validateBasicVisitor(new Dictionary(null as any, null as any)));
         test(`visits Interval types`, () => validateBasicVisitor(new Interval(0)));
         test(`visits FixedSizeList types`, () => validateBasicVisitor(new FixedSizeList(2, null as any)));
-        test(`visits Map types`, () => validateBasicVisitor(new Map_([] as any[])));
+        test(`visits Map types`, () => validateBasicVisitor(new Map_(new Field('', new Struct<{ key: Int, value: Int }>([] as any[])))));
         function validateBasicVisitor<T extends DataType>(type: T) {
             const visitor = new BasicVisitor();
             const result = visitor.visit(type);
@@ -156,7 +157,7 @@ describe('Visitor', () => {
         test(`visits IntervalDayTime types`, () => validateFeatureVisitor(new IntervalDayTime()));
         test(`visits IntervalYearMonth types`, () => validateFeatureVisitor(new IntervalYearMonth()));
         test(`visits FixedSizeList types`, () => validateFeatureVisitor(new FixedSizeList(2, null as any)));
-        test(`visits Map types`, () => validateFeatureVisitor(new Map_([] as any[])));
+        test(`visits Map types`, () => validateFeatureVisitor(new Map_(new Field('', new Struct<{ key: Int, value: Int }>([] as any[])))));
 
         function validateFeatureVisitor<T extends DataType>(type: T) {
             const visitor = new FeatureVisitor();
diff --git a/js/tsconfig/tsconfig.base.json b/js/tsconfig/tsconfig.base.json
index e95ee1f8200..cb9785df1b4 100644
--- a/js/tsconfig/tsconfig.base.json
+++ b/js/tsconfig/tsconfig.base.json
@@ -22,16 +22,24 @@
 
     /* The most restrictive settings possible */
     "strict": true,
+    "noImplicitAny": true,
+    "noImplicitThis": true,
+    "alwaysStrict": true,
+    "strictBindCallApply": true,
+    "strictNullChecks": true,
+    "strictFunctionTypes": true,
+    "strictPropertyInitialization": true,
+    
     "skipLibCheck": false,
     "importHelpers": true,
     "noEmitHelpers": true,
-    "noImplicitAny": true,
     "noUnusedLocals": true,
     "noImplicitReturns": true,
-    "allowUnusedLabels": false,
     "noUnusedParameters": true,
+    "allowUnusedLabels": false,
     "allowUnreachableCode": false,
+    "noStrictGenericChecks": false,
     "noFallthroughCasesInSwitch": true,
-    "forceConsistentCasingInFileNames": true
+    "forceConsistentCasingInFileNames": true,
   }
 }
diff --git a/js/typedoc.js b/js/typedoc.js
index 641888b7344..b8c6e31bb47 100644
--- a/js/typedoc.js
+++ b/js/typedoc.js
@@ -16,7 +16,7 @@
 // under the License.
 
 module.exports = {
-    src: 'src',
+    inputFiles: 'src',
     out: 'doc',
     mode: 'file',
     name: 'Apache Arrow',
@@ -27,7 +27,6 @@ module.exports = {
     excludeProtected: true,
     excludeExternals: true,
     excludeNotExported: true,
-    includeDefinitions: true,
     ignoreCompilerErrors: true,
     exclude: [
         'src/fb/*.ts',
diff --git a/julia/Arrow/LICENSE.md b/julia/Arrow/LICENSE.md
new file mode 100644
index 00000000000..136e1c6a784
--- /dev/null
+++ b/julia/Arrow/LICENSE.md
@@ -0,0 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
\ No newline at end of file
diff --git a/julia/Arrow/Project.toml b/julia/Arrow/Project.toml
new file mode 100644
index 00000000000..3d5de48729a
--- /dev/null
+++ b/julia/Arrow/Project.toml
@@ -0,0 +1,34 @@
+name = "Arrow"
+uuid = "69666777-d1a9-59fb-9406-91d4454c9d45"
+authors = ["quinnj <quinn.jacobd@gmail.com>"]
+version = "0.3.0"
+
+[deps]
+CodecLz4 = "5ba52731-8f18-5e0d-9241-30f10d1ec561"
+CodecZstd = "6b39b394-51ab-5f42-8807-6242bab2b4c2"
+DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
+Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+Mmap = "a63ad114-7e13-5084-954f-fe012c677804"
+PooledArrays = "2dfb63ee-cc39-5dd5-95bd-886bf059d720"
+SentinelArrays = "91c51154-3ec4-41a3-a24f-3f23e20d615c"
+Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
+TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53"
+
+[compat]
+CodecLz4 = "0.4"
+CodecZstd = "0.7"
+julia = "1.3"
+DataAPI = "1"
+PooledArrays = "0.5"
+Tables = "1.1"
+SentinelArrays = "1"
+
+[extras]
+JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+
+[targets]
+test = ["Test", "Random", "JSON3", "StructTypes"]
diff --git a/julia/Arrow/README.md b/julia/Arrow/README.md
new file mode 100644
index 00000000000..188e858fa55
--- /dev/null
+++ b/julia/Arrow/README.md
@@ -0,0 +1,145 @@
+# Arrow
+
+[![Build Status](https://travis-ci.com/JuliaData/Arrow.jl.svg?branch=master)](https://travis-ci.com/JuliaData/Arrow.jl.svg?branch=master)
+[![codecov](https://codecov.io/gh/JuliaData/Arrow.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/JuliaData/Arrow.jl)
+
+This is a pure Julia implementation of the [Apache Arrow](https://arrow.apache.org) data standard.  This package provides Julia `AbstractVector` objects for
+referencing data that conforms to the Arrow standard.  This allows users to seamlessly interface Arrow formatted data with a great deal of existing Julia code.
+
+Please see this [document](https://arrow.apache.org/docs/format/Columnar.html#physical-memory-layout) for a description of the Arrow memory layout.
+
+## Format Support
+
+This implementation supports the 1.0 version of the specification, including support for:
+  * All primitive data types
+  * All nested data types
+  * Dictionary encodings and messages
+  * Extension types
+  * Streaming, file, record batch, and replacement and isdelta dictionary messages
+
+It currently doesn't include support for:
+  * Tensors or sparse tensors
+  * Flight RPC
+  * C data interface
+
+Third-party data formats:
+  * csv and parquet support via the existing CSV.jl and Parquet.jl packages
+  * Other Tables.jl-compatible packages automatically supported (DataFrames.jl, JSONTables.jl, JuliaDB.jl, SQLite.jl, MySQL.jl, JDBC.jl, ODBC.jl, XLSX.jl, etc.)
+  * No current Julia packages support ORC or Avro data formats
+
+## Basic usage:
+
+### Installation
+
+```julia
+] add Arrow
+```
+
+### Reading
+
+#### `Arrow.Table`
+
+    Arrow.Table(io::IO; convert::Bool=true)
+    Arrow.Table(file::String; convert::Bool=true)
+    Arrow.Table(bytes::Vector{UInt8}, pos=1, len=nothing; convert::Bool=true)
+
+Read an arrow formatted table, from:
+ * `io`, bytes will be read all at once via `read(io)`
+ * `file`, bytes will be read via `Mmap.mmap(file)`
+ * `bytes`, a byte vector directly, optionally allowing specifying the starting byte position `pos` and `len`
+
+Returns a `Arrow.Table` object that allows column access via `table.col1`, `table[:col1]`, or `table[1]`.
+
+The Apache Arrow standard is foremost a "columnar" format and saves a variety of metadata with each column (such as column name, type, length, etc.).
+A data set which has tens of thousands of columns is probably not well suited for the arrow format and may cause dramatic file size increases when one saves to a `arrow` file.
+If it is possible to reshape the data such that there are not as many columns, `Arrow.Table` should not have as many problems. 
+A simple method Julia provides is to simply execute `transpose(data)` to switch the rows and columns of your data if that does not interfere with one's analysis.
+
+NOTE: the columns in an `Arrow.Table` are views into the original arrow memory, and hence are not easily
+modifiable (with e.g. `push!`, `append!`, etc.). To mutate arrow columns, call `copy(x)` to materialize
+the arrow data as a normal Julia array.
+
+`Arrow.Table` also satisfies the Tables.jl interface, and so can easily be materialized via any supporting
+sink function: e.g. `DataFrame(Arrow.Table(file))`, `SQLite.load!(db, "table", Arrow.Table(file))`, etc.
+
+Supports the `convert` keyword argument which controls whether certain arrow primitive types will be
+lazily converted to more friendly Julia defaults; by default, `convert=true`.
+
+##### Examples
+
+```julia
+using Arrow
+
+# read arrow table from file format
+tbl = Arrow.Table(file)
+
+# read arrow table from IO
+tbl = Arrow.Table(io)
+
+# read arrow table directly from bytes, like from an HTTP request
+resp = HTTP.get(url)
+tbl = Arrow.Table(resp.body)
+```
+
+#### `Arrow.Stream`
+
+    Arrow.Stream(io::IO; convert::Bool=true)
+    Arrow.Stream(file::String; convert::Bool=true)
+    Arrow.Stream(bytes::Vector{UInt8}, pos=1, len=nothing; convert::Bool=true)
+
+Start reading an arrow formatted table, from:
+ * `io`, bytes will be read all at once via `read(io)`
+ * `file`, bytes will be read via `Mmap.mmap(file)`
+ * `bytes`, a byte vector directly, optionally allowing specifying the starting byte position `pos` and `len`
+
+Reads the initial schema message from the arrow stream/file, then returns an `Arrow.Stream` object
+which will iterate over record batch messages, producing an `Arrow.Table` on each iteration.
+
+By iterating `Arrow.Table`, `Arrow.Stream` satisfies the `Tables.partitions` interface, and as such can
+be passed to Tables.jl-compatible sink functions.
+
+This allows iterating over extremely large "arrow tables" in chunks represented as record batches.
+
+Supports the `convert` keyword argument which controls whether certain arrow primitive types will be
+lazily converted to more friendly Julia defaults; by default, `convert=true`.
+
+### Writing
+
+#### `Arrow.write`
+
+    Arrow.write(io::IO, tbl)
+    Arrow.write(file::String, tbl)
+
+Write any Tables.jl-compatible `tbl` out as arrow formatted data.
+Providing an `io::IO` argument will cause the data to be written to it
+in the "streaming" format, unless `file=true` keyword argument is passed.
+Providing a `file::String` argument will result in the "file" format being written.
+
+Multiple record batches will be written based on the number of
+`Tables.partitions(tbl)` that are provided; by default, this is just
+one for a given table, but some table sources support automatic
+partitioning. Note you can turn multiple table objects into partitions
+by doing `Tables.partitioner([tbl1, tbl2, ...])`, but note that
+each table must have the exact same `Tables.Schema`.
+
+By default, `Arrow.write` will use multiple threads to write multiple
+record batches simultaneously (e.g. if julia is started with `julia -t 8`).
+
+Supported keyword arguments to `Arrow.write` include:
+  * `compress`: possible values include `:lz4`, `:zstd`, or your own initialized `LZ4FrameCompressor` or `ZstdCompressor` objects; will cause all buffers in each record batch to use the respective compression encoding
+  * `alignment::Int=8`: specify the number of bytes to align buffers to when written in messages; strongly recommended to only use alignment values of 8 or 64 for modern memory cache line optimization
+  * `dictencode::Bool=false`: whether all columns should use dictionary encoding when being written
+  * `dictencodenested::Bool=false`: whether nested data type columns should also dict encode nested arrays/buffers; many other implementations don't support this
+  * `denseunions::Bool=true`: whether Julia `Vector{<:Union}` arrays should be written using the dense union layout; passing `false` will result in the sparse union layout
+  * `largelists::Bool=false`: causes list column types to be written with Int64 offset arrays; mainly for testing purposes; by default, Int64 offsets will be used only if needed
+  * `file::Bool=false`: if a an `io` argument is being written to, passing `file=true` will cause the arrow file format to be written instead of just IPC streaming
+
+##### Examples
+
+```julia
+# write directly to any IO in streaming format
+Arrow.write(io, tbl)
+
+# write to a file in file format
+Arrow.write("data.arrow", tbl)
+```
diff --git a/julia/Arrow/docs/.gitignore b/julia/Arrow/docs/.gitignore
new file mode 100644
index 00000000000..a303fff2035
--- /dev/null
+++ b/julia/Arrow/docs/.gitignore
@@ -0,0 +1,2 @@
+build/
+site/
diff --git a/julia/Arrow/docs/Manifest.toml b/julia/Arrow/docs/Manifest.toml
new file mode 100644
index 00000000000..69420b1937b
--- /dev/null
+++ b/julia/Arrow/docs/Manifest.toml
@@ -0,0 +1,204 @@
+# This file is machine-generated - editing it directly is not advised
+
+[[ArgTools]]
+uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
+
+[[Arrow]]
+deps = ["CodecLz4", "CodecZstd", "DataAPI", "Dates", "Mmap", "PooledArrays", "SentinelArrays", "Tables"]
+git-tree-sha1 = "76641f71ac332cd4d3cf54b98234a0f597bd7a2f"
+uuid = "69666777-d1a9-59fb-9406-91d4454c9d45"
+version = "0.3.0"
+
+[[Artifacts]]
+uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
+
+[[Base64]]
+uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
+
+[[CodecLz4]]
+deps = ["Lz4_jll", "TranscodingStreams"]
+git-tree-sha1 = "59fe0cb37784288d6b9f1baebddbf75457395d40"
+uuid = "5ba52731-8f18-5e0d-9241-30f10d1ec561"
+version = "0.4.0"
+
+[[CodecZstd]]
+deps = ["TranscodingStreams", "Zstd_jll"]
+git-tree-sha1 = "d19cd9ae79ef31774151637492291d75194fc5fa"
+uuid = "6b39b394-51ab-5f42-8807-6242bab2b4c2"
+version = "0.7.0"
+
+[[DataAPI]]
+git-tree-sha1 = "176e23402d80e7743fc26c19c681bfb11246af32"
+uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
+version = "1.3.0"
+
+[[DataValueInterfaces]]
+git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6"
+uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464"
+version = "1.0.0"
+
+[[Dates]]
+deps = ["Printf"]
+uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
+
+[[DocStringExtensions]]
+deps = ["LibGit2", "Markdown", "Pkg", "Test"]
+git-tree-sha1 = "50ddf44c53698f5e784bbebb3f4b21c5807401b1"
+uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
+version = "0.8.3"
+
+[[Documenter]]
+deps = ["Base64", "Dates", "DocStringExtensions", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"]
+git-tree-sha1 = "fb1ff838470573adc15c71ba79f8d31328f035da"
+uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+version = "0.25.2"
+
+[[DocumenterMarkdown]]
+deps = ["Documenter", "Test"]
+git-tree-sha1 = "c302ba512683c3db462ee4eff718ae6fedcbf380"
+uuid = "997ab1e6-3595-5248-9280-8efb232c3433"
+version = "0.2.0"
+
+[[Downloads]]
+deps = ["ArgTools", "LibCURL"]
+uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+
+[[InteractiveUtils]]
+deps = ["Markdown"]
+uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+
+[[IteratorInterfaceExtensions]]
+git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
+uuid = "82899510-4779-5014-852e-03e436cf321d"
+version = "1.0.0"
+
+[[JLLWrappers]]
+git-tree-sha1 = "c70593677bbf2c3ccab4f7500d0f4dacfff7b75c"
+uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
+version = "1.1.3"
+
+[[JSON]]
+deps = ["Dates", "Mmap", "Parsers", "Unicode"]
+git-tree-sha1 = "81690084b6198a2e1da36fcfda16eeca9f9f24e4"
+uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+version = "0.21.1"
+
+[[LibCURL]]
+deps = ["LibCURL_jll", "MozillaCACerts_jll"]
+uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
+
+[[LibCURL_jll]]
+deps = ["Libdl"]
+uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
+
+[[LibGit2]]
+deps = ["Printf"]
+uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
+
+[[Libdl]]
+uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
+
+[[LinearAlgebra]]
+deps = ["Libdl"]
+uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+
+[[Logging]]
+uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
+
+[[Lz4_jll]]
+deps = ["Libdl", "Pkg"]
+git-tree-sha1 = "51b1db0732bbdcfabb60e36095cc3ed9c0016932"
+uuid = "5ced341a-0733-55b8-9ab6-a4889d929147"
+version = "1.9.2+2"
+
+[[Markdown]]
+deps = ["Base64"]
+uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
+
+[[Mmap]]
+uuid = "a63ad114-7e13-5084-954f-fe012c677804"
+
+[[MozillaCACerts_jll]]
+uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
+
+[[Parsers]]
+deps = ["Dates"]
+git-tree-sha1 = "6fa4202675c05ba0f8268a6ddf07606350eda3ce"
+uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
+version = "1.0.11"
+
+[[Pkg]]
+deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "UUIDs"]
+uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+
+[[PooledArrays]]
+deps = ["DataAPI"]
+git-tree-sha1 = "b1333d4eced1826e15adbdf01a4ecaccca9d353c"
+uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720"
+version = "0.5.3"
+
+[[Printf]]
+deps = ["Unicode"]
+uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+
+[[REPL]]
+deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
+uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
+
+[[Random]]
+deps = ["Serialization"]
+uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+
+[[SHA]]
+uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+
+[[SentinelArrays]]
+deps = ["Dates", "Random"]
+git-tree-sha1 = "6ccde405cf0759eba835eb613130723cb8f10ff9"
+uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c"
+version = "1.2.16"
+
+[[Serialization]]
+uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+
+[[Sockets]]
+uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
+
+[[TOML]]
+deps = ["Dates"]
+uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+
+[[TableTraits]]
+deps = ["IteratorInterfaceExtensions"]
+git-tree-sha1 = "b1ad568ba658d8cbb3b892ed5380a6f3e781a81e"
+uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c"
+version = "1.0.0"
+
+[[Tables]]
+deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "TableTraits", "Test"]
+git-tree-sha1 = "24a584cf65e2cfabdadc21694fb69d2e74c82b44"
+uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
+version = "1.1.0"
+
+[[Test]]
+deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
+uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[[TranscodingStreams]]
+deps = ["Random", "Test"]
+git-tree-sha1 = "7c53c35547de1c5b9d46a4797cf6d8253807108c"
+uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
+version = "0.9.5"
+
+[[UUIDs]]
+deps = ["Random", "SHA"]
+uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+
+[[Unicode]]
+uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
+
+[[Zstd_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "6f1abcb0c44f184690912aa4b0ba861dd64f11b9"
+uuid = "3161d3a3-bdf6-5164-811a-617609db77b4"
+version = "1.4.5+2"
diff --git a/julia/Arrow/docs/Project.toml b/julia/Arrow/docs/Project.toml
new file mode 100644
index 00000000000..fa6129cfec8
--- /dev/null
+++ b/julia/Arrow/docs/Project.toml
@@ -0,0 +1,4 @@
+[deps]
+Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45"
+Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+DocumenterMarkdown = "997ab1e6-3595-5248-9280-8efb232c3433"
diff --git a/julia/Arrow/docs/make.jl b/julia/Arrow/docs/make.jl
new file mode 100644
index 00000000000..a8fbc9a4dbb
--- /dev/null
+++ b/julia/Arrow/docs/make.jl
@@ -0,0 +1,14 @@
+using Documenter, DocumenterMarkdown
+using Arrow
+
+makedocs(
+    format = Markdown(),
+    modules = [Arrow],
+    pages = [
+        "Home" => "index.md",
+        "User Manual" => "manual.md",
+        "API Reference" => "reference.md"
+    ]
+)
+
+deploydocs(repo = "github.com/JuliaData/Arrow.jl.git")
diff --git a/julia/Arrow/docs/mkdocs.yml b/julia/Arrow/docs/mkdocs.yml
new file mode 100644
index 00000000000..6d57305c4f6
--- /dev/null
+++ b/julia/Arrow/docs/mkdocs.yml
@@ -0,0 +1,27 @@
+# See the mkdocs user guide for more information on these settings.
+#   http://www.mkdocs.org/user-guide/configuration/
+
+site_name:        Arrow.jl
+#repo_url:         https://github.com/USER_NAME/PACKAGE_NAME.jl
+#site_description: Description...
+#site_author:      USER_NAME
+
+theme: readthedocs
+
+extra_css:
+  - assets/Documenter.css
+
+extra_javascript:
+  - https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS_HTML
+  - assets/mathjaxhelper.js
+
+markdown_extensions:
+  - extra
+  - tables
+  - fenced_code
+  - mdx_math
+
+docs_dir: 'build'
+
+pages:
+  - Home: index.md
diff --git a/julia/Arrow/docs/src/index.md b/julia/Arrow/docs/src/index.md
new file mode 100644
index 00000000000..ee4d141c670
--- /dev/null
+++ b/julia/Arrow/docs/src/index.md
@@ -0,0 +1,10 @@
+# Arrow.jl
+
+```@contents
+Pages = ["manual.md", "reference.md"]
+Depth = 3
+```
+
+```@docs
+Arrow
+```
\ No newline at end of file
diff --git a/julia/Arrow/docs/src/manual.md b/julia/Arrow/docs/src/manual.md
new file mode 100644
index 00000000000..a583b2adc9e
--- /dev/null
+++ b/julia/Arrow/docs/src/manual.md
@@ -0,0 +1,110 @@
+# User Manual
+
+The goal of this documentation is to provide a brief introduction to the arrow data format, then provide a walk-through of the functionality provided in the Arrow.jl Julia package, with an aim to expose a little of the machinery "under the hood" to help explain how things work and how that influences real-world use-cases for the arrow data format.
+
+The best place to learn about the Apache arrow project is [the website itself](https://arrow.apache.org/), specifically the data format [specification](https://arrow.apache.org/docs/format/Columnar.html). Put briefly, the arrow project provides a formal speficiation for how columnar, "table" data can be laid out efficiently in memory to standardize and maximize the ability to share data across languages/platforms. In the current [apache/arrow GitHub repository](https://github.com/apache/arrow), language implementations exist for C++, Java, Go, Javascript, Rust, to name a few. Other database vendors and data processing frameworks/applications have also built support for the arrow format, allowing for a wide breadth of possibility for applications to "speak the data language" of arrow.
+
+The [Arrow.jl](https://github.com/JuliaData/Arrow.jl) Julia package is another implementation, allowing the ability to both read and write data in the arrow format. As a data format, arrow specifies an exact memory layout to be used for columnar table data, and as such, "reading" involves custom Julia objects ([`Arrow.Table`](@ref) and [`Arrow.Stream`](@ref)), which read the *metadata* of an "arrow memory blob", then *wrap* the array data contained therein, having learned the type and size, amongst other properties, from the metadata. Let's take a closer look at what this "reading" of arrow memory really means/looks like.
+
+
+## Reading arrow data
+
+After installing the Arrow.jl Julia package (via `] add Arrow`), and if you have some arrow data, let's say a file named `data.arrow` generated from the [`pyarrow`](https://arrow.apache.org/docs/python/) library (a Python library for interfacing with arrow data), you can then read that arrow data into a Julia session by doing:
+
+```julia
+using Arrow
+
+table = Arrow.Table("data.arrow")
+```
+
+### `Arrow.Table`
+
+The type of `table` in this example will be an `Arrow.Table`. When "reading" the arrow data, `Arrow.Table` first ["mmapped"](https://en.wikipedia.org/wiki/Mmap) the `data.arrow` file, which is an important technique for dealing with data larger than available RAM on a system. By "mmapping" a file, the OS doesn't actually load the entire file contents into RAM at the same time, but file contents are "swapped" into RAM as different regions of a file are requested. Once "mmapped", `Arrow.Table` then inspected the metadata in the file to determine the number of columns, their names and types, at which byte offset each column begins in the file data, and even how many "batches" are included in this file (arrow tables may be partitioned into one ore more "record batches" each containing portions of the data). Armed with all the appropriate metadata, `Arrow.Table` then created custom array objects ([`ArrowVector`](@ref)), which act as "views" into the raw arrow memory bytes. This is a significant point in that no extra memory is allocated for "data" when reading arrow data. This is in contrast to if we wanted to read the data of a csv file as columns into Julia structures; we would need to allocate those array structures ourselves, then parse the file, "filling in" each element of the array with the data we parsed from the file. Arrow data, on the other hand, is *already laid out in memory or on disk* in a binary format, and as long as we have the metadata to interpret the raw bytes, we can figure out whether to treat those bytes as a `Vector{Float64}`, etc. A sample of the kinds of arrow array types you might see when deserializing arrow data, include:
+
+* [`Arrow.Primitive`](@ref): the most common array type for simple, fixed-size elements like integers, floats, time types, and decimals
+* [`Arrow.List`](@ref): an array type where its own elements are also arrays of some kind, like string columns, where each element can be thought of as an array of characters
+* [`Arrow.FixedSizeList`](@ref): similar to the `List` type, but where each array element has a fixed number of elements itself; you can think of this like a `Vector{NTuple{N, T}}`, where `N` is the fixed-size width
+* [`Arrow.Map`](@ref): an array type where each element is like a Julia `Dict`; a list of key value pairs like a `Vector{Dict}`
+* [`Arrow.Struct`](@ref): an array type where each element is an instance of a custom struct, i.e. an ordered collection of named & typed fields, kind of like a `Vector{NamedTuple}`
+* [`Arrow.DenseUnion`](@ref): an array type where elements may be of several different types, stored compactly; can be thought of like `Vector{Union{A, B}}`
+* [`Arrow.SparseUnion`](@ref): another array type where elements may be of several different types, but stored as if made up of identically lengthed child arrays for each possible type (less memory efficient than `DenseUnion`)
+* [`Arrow.DictEncoded`](@ref): a special array type where values are "dictionary encoded", meaning the list of unique, possible values for an array are stored internally in an "encoding pool", whereas each stored element of the array is just an integer "code" to index into the encoding pool for the actual value.
+
+And while these custom array types do subtype `AbstractArray`, there is only limited support for `setindex!`. Remember, these arrays are "views" into the raw arrow bytes, so for array types other than `Arrow.Primitive`, it gets pretty tricky to allow manipulating those raw arrow bytes. Nevetheless, it's as simple as calling `copy(x)` where `x` is any `ArrowVector` type, and a normal Julia `Vector` type will be fully materialized (which would then allow mutating/manipulating values).
+
+So, what can you do with an `Arrow.Table` full of data? Quite a bit actually!
+
+Because `Arrow.Table` implements the [Tables.jl](https://juliadata.github.io/Tables.jl/stable/) interface, it opens up a world of integrations for using arrow data. A few examples include:
+
+* `df = DataFrame(Arrow.Table(file))`: Build a [`DataFrame`](https://juliadata.github.io/DataFrames.jl/stable/), using the arrow vectors themselves; this allows utilizing a host of DataFrames.jl functionality directly on arrow data; grouping, joining, selecting, etc.
+* `Tables.datavaluerows(Arrow.Table(file)) |> @map(...) |> @filter(...) |> DataFrame`: use [`Query.jl`'s](https://www.queryverse.org/Query.jl/stable/standalonequerycommands/) row-processing utilities to map, group, filter, mutate, etc. directly over arrow data.
+* `Arrow.Table(file) |> SQLite.load!(db, "arrow_table")`: load arrow data directly into an sqlite database/table, where sql queries can be executed on the data
+* `Arrow.Table(file) |> CSV.write("arrow.csv")`: write arrow data out to a csv file
+
+A full list of Julia packages leveraging the Tables.jl inteface can be found [here](https://github.com/JuliaData/Tables.jl/blob/master/INTEGRATIONS.md).
+
+Apart from letting other packages have all the fun, an `Arrow.Table` itself can be plenty useful. For example, with `tbl = Arrow.Table(file)`:
+* `tbl[1]`: retrieve the first column via indexing; the number of columns can be queried via `length(tbl)`
+* `tbl[:col1]` or `tbl.col1`: retrieve the column named `col1`, either via indexing with the column name given as a `Symbol`, or via "dot-access"
+* `for col in tbl`: iterate through columns in the table
+* `AbstractDict` methods like `haskey(tbl, :col1)`, `get(tbl, :col1, nothing)`, `keys(tbl)`, or `values(tbl)`
+
+### Arrow types
+
+In the arrow data format, specific logical types are supported, a list of which can be found [here](https://arrow.apache.org/docs/status.html#data-types). These include booleans, integers of various bit widths, floats, decimals, time types, and binary/string. While most of these map naturally to types builtin to Julia itself, there are a few cases where the definitions are slightly different, and in these cases, by default, they are converted to more "friendly" Julia types (this auto conversion can be avoided by passing `convert=false` to `Arrow.Table`, like `Arrow.Table(file; convert=false)`). Examples of arrow to julia type mappings include:
+
+* `Date`, `Time`, `Timestamp`, and `Duration` all have natural Julia defintions in `Dates.Date`, `Dates.Time`, `TimeZones.ZonedDateTime`, and `Dates.Period` subtypes, respectively. 
+* `Char` and `Symbol` Julia types are mapped to arrow string types, with additional metadata of the original Julia type; this allows deserializing directly to `Char` and `Symbol` in Julia, while other language implementations will see these columns as just strings
+* `Decimal128` has no corresponding builtin Julia type, so it's deserialized using a compatible type definition in Arrow.jl itself: `Arrow.Decimal`
+
+Note that when `convert=false` is passed, data will be returned in Arrow.jl-defined types that exactly match the arrow definitions of those types; the authoritative source for how each type represents its data can be found in the arrow [`Schema.fbs`](https://github.com/apache/arrow/blob/master/format/Schema.fbs) file.
+
+### `Arrow.Stream`
+
+In addition to `Arrow.Table`, the Arrow.jl package also provides `Arrow.Stream` for processing arrow data. While `Arrow.Table` will iterate all record batches in an arrow file/stream, concatenating columns, `Arrow.Stream` provides a way to *iterate* through record batches, one at a time. Each iteration yields an `Arrow.Table` instance, with columns/data for a single record batch. This allows, if so desired, "batch processing" of arrow data, one record batch at a time, instead of creating a single long table via `Arrow.Table`.
+
+### Table and column metadata
+
+The arrow format allows attaching arbitrary metadata in the form of a `Dict{String, String}` to tables and individual columns. The Arrow.jl package supports retrieving serialized metadata by calling `Arrow.getmetadata(table)` or `Arrow.getmetadata(column)`.
+
+## Writing arrow data
+
+Ok, so that's a pretty good rundown of *reading* arrow data, but how do you *produce* arrow data? Enter `Arrow.write`.
+
+### `Arrow.write`
+
+With `Arrow.write`, you provide either an `io::IO` argument or `file::String` to write the arrow data to, as well as a Tables.jl-compatible source that contains the data to be written.
+
+What are some examples of Tables.jl-compatible sources? A few examples include:
+* `Arrow.write(io, df::DataFrame)`: A `DataFrame` is a collection of indexable columns
+* `Arrow.write(io, CSV.File(file))`: read data from a csv file and write out to arrow format
+* `Arrow.write(io, DBInterface.execute(db, sql_query))`: Execute an SQL query against a database via the [`DBInterface.jl`](https://github.com/JuliaDatabases/DBInterface.jl) interface, and write the query resultset out directly in the arrow format. Packages that implement DBInterface include [SQLite.jl](https://juliadatabases.github.io/SQLite.jl/stable/), [MySQL.jl](https://juliadatabases.github.io/MySQL.jl/dev/), and [ODBC.jl](http://juliadatabases.github.io/ODBC.jl/latest/). 
+* `df |> @map(...) |> Arrow.write(io)`: Write the results of a [Query.jl](https://www.queryverse.org/Query.jl/stable/) chain of operations directly out as arrow data
+* `jsontable(json) |> Arrow.write(io)`: Treat a json array of objects or object of arrays as a "table" and write it out as arrow data using the [JSONTables.jl](https://github.com/JuliaData/JSONTables.jl) package
+* `Arrow.write(io, (col1=data1, col2=data2, ...))`: a `NamedTuple` of `AbstractVector`s or an `AbstractVector` of `NamedTuple`s are both considered tables by default, so they can be quickly constructed for easy writing of arrow data if you already have columns of data
+
+And these are just a few examples of the numerous [integrations](https://github.com/JuliaData/Tables.jl/blob/master/INTEGRATIONS.md).
+
+In addition to just writing out a single "table" of data as a single arrow record batch, `Arrow.write` also supports writing out multiple record batches when the input supports the `Tables.partitions` functionality. One immediate, though perhaps not incredibly useful example, is `Arrow.Stream`. `Arrow.Stream` implements `Tables.partitions` in that it iterates "tables" (specifically `Arrow.Table`), and as such, `Arrow.write` will iterate an `Arrow.Stream`, and write out each `Arrow.Table` as a separate record batch. Another important point for why this example works is because an `Arrow.Stream` iterates `Arrow.Table`s that all have the same schema. This is important because when writing arrow data, a "schema" message is always written first, with all subsequent record batches written with data matching the initial schema.
+
+In addition to inputs that support `Tables.partitions`, note that the Tables.jl itself provides the `Tables.partitioner` function, which allows providing your own separate instances of similarly-schema-ed tables as "partitions", like:
+
+```julia
+# treat 2 separate NamedTuples of vectors with same schema as 1 table, 2 partitions
+tbl_parts = Tables.partitioner([(col1=data1, col2=data2), (col1=data3, col2=data4)])
+Arrow.write(io, tbl_parts)
+
+# treat an array of csv files with same schema where each file is a partition
+# in this form, a function `CSV.File` is applied to each element of 2nd argument
+csv_parts = Tables.partitioner(CSV.File, csv_files)
+Arrow.write(io, csv_parts)
+```
+
+### Multithreaded writing
+
+By default, `Arrow.write` will use multiple threads to write multiple
+record batches simultaneously (e.g. if julia is started with `julia -t 8` or the `JULIA_NUM_THREADS` environment variable is set).
+
+### Compression
+
+Compression is supported when writing via the `compress` keyword argument. Possible values include `:lz4`, `:zstd`, or your own initialized `LZ4FrameCompressor` or `ZstdCompressor` objects; will cause all buffers in each record batch to use the respective compression encoding or compressor.
diff --git a/julia/Arrow/docs/src/reference.md b/julia/Arrow/docs/src/reference.md
new file mode 100644
index 00000000000..9130082db9d
--- /dev/null
+++ b/julia/Arrow/docs/src/reference.md
@@ -0,0 +1,6 @@
+# API Reference
+
+```@autodocs
+Modules = [Arrow]
+Order   = [:type, :function]
+```
\ No newline at end of file
diff --git a/julia/Arrow/src/Arrow.jl b/julia/Arrow/src/Arrow.jl
new file mode 100644
index 00000000000..4eb5986e911
--- /dev/null
+++ b/julia/Arrow/src/Arrow.jl
@@ -0,0 +1,107 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+    Arrow.jl
+
+A pure Julia implementation of the [apache arrow](https://arrow.apache.org/) memory format specification.
+
+This implementation supports the 1.0 version of the specification, including support for:
+  * All primitive data types
+  * All nested data types
+  * Dictionary encodings, nested dictionary encodings, and messages
+  * Extension types
+  * Streaming, file, record batch, and replacement and isdelta dictionary messages
+  * Buffer compression/decompression via the standard LZ4 frame and Zstd formats
+
+It currently doesn't include support for:
+  * Tensors or sparse tensors
+  * Flight RPC
+  * C data interface
+
+Third-party data formats:
+  * csv and parquet support via the existing [CSV.jl](https://github.com/JuliaData/CSV.jl) and [Parquet.jl](https://github.com/JuliaIO/Parquet.jl) packages
+  * Other [Tables.jl](https://github.com/JuliaData/Tables.jl)-compatible packages automatically supported ([DataFrames.jl](https://github.com/JuliaData/DataFrames.jl), [JSONTables.jl](https://github.com/JuliaData/JSONTables.jl), [JuliaDB.jl](https://github.com/JuliaData/JuliaDB.jl), [SQLite.jl](https://github.com/JuliaDatabases/SQLite.jl), [MySQL.jl](https://github.com/JuliaDatabases/MySQL.jl), [JDBC.jl](https://github.com/JuliaDatabases/JDBC.jl), [ODBC.jl](https://github.com/JuliaDatabases/ODBC.jl), [XLSX.jl](https://github.com/felipenoris/XLSX.jl), etc.)
+  * No current Julia packages support ORC or Avro data formats
+
+See docs for official Arrow.jl API with the [User Manual](@ref) and reference docs for [`Arrow.Table`](@ref), [`Arrow.write`](@ref), and [`Arrow.Stream`](@ref).
+"""
+module Arrow
+
+using Mmap
+import Dates
+using DataAPI, Tables, SentinelArrays, PooledArrays, CodecLz4, CodecZstd, TimeZones
+
+using Base: @propagate_inbounds
+import Base: ==
+
+const DEBUG_LEVEL = Ref(0)
+
+function setdebug!(level::Int)
+    DEBUG_LEVEL[] = level
+    return
+end
+
+function withdebug(f, level)
+    lvl = DEBUG_LEVEL[]
+    try
+        setdebug!(level)
+        f()
+    finally
+        setdebug!(lvl)
+    end
+end
+
+macro debug(level, msg)
+    esc(quote
+        if DEBUG_LEVEL[] >= $level
+            println(string("DEBUG: ", $(QuoteNode(__source__.file)), ":", $(QuoteNode(__source__.line)), " ", $msg))
+        end
+    end)
+end
+
+const FILE_FORMAT_MAGIC_BYTES = b"ARROW1"
+const CONTINUATION_INDICATOR_BYTES = 0xffffffff
+
+# vendored flatbuffers code for now
+include("FlatBuffers/FlatBuffers.jl")
+using .FlatBuffers
+
+include("metadata/Flatbuf.jl")
+using .Flatbuf; const Meta = Flatbuf
+
+include("arrowtypes.jl")
+using .ArrowTypes
+include("utils.jl")
+include("arraytypes/arraytypes.jl")
+include("eltypes.jl")
+include("table.jl")
+include("write.jl")
+
+const LZ4_FRAME_COMPRESSOR = Ref{LZ4FrameCompressor}()
+const ZSTD_COMPRESSOR = Ref{ZstdCompressor}()
+
+function __init__()
+    zstd = ZstdCompressor(; level=3)
+    CodecZstd.TranscodingStreams.initialize(zstd)
+    ZSTD_COMPRESSOR[] = zstd
+    lz4 = LZ4FrameCompressor(; compressionlevel=4)
+    CodecLz4.TranscodingStreams.initialize(lz4)
+    LZ4_FRAME_COMPRESSOR[] = lz4
+    return
+end
+
+end  # module Arrow
diff --git a/julia/Arrow/src/FlatBuffers/FlatBuffers.jl b/julia/Arrow/src/FlatBuffers/FlatBuffers.jl
new file mode 100644
index 00000000000..f4c74777173
--- /dev/null
+++ b/julia/Arrow/src/FlatBuffers/FlatBuffers.jl
@@ -0,0 +1,153 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module FlatBuffers
+
+const UOffsetT = UInt32
+const SOffsetT = Int32
+const VOffsetT = UInt16
+const VtableMetadataFields = 2
+
+basetype(::Enum) = UInt8
+
+function readbuffer(t::AbstractVector{UInt8}, pos::Integer, ::Type{Bool})
+    @inbounds b = t[pos + 1]
+    return b === 0x01
+end
+
+function readbuffer(t::AbstractVector{UInt8}, pos::Integer, ::Type{T}) where {T}
+    GC.@preserve t begin
+        ptr = convert(Ptr{T}, pointer(t, pos + 1))
+        x = unsafe_load(ptr)
+    end
+end
+
+include("builder.jl")
+include("table.jl")
+
+function Base.show(io::IO, x::TableOrStruct)
+    print(io, "$(typeof(x))")
+    if isempty(propertynames(x))
+        print(io, "()")
+    else
+        show(io, NamedTuple{propertynames(x)}(Tuple(getproperty(x, y) for y in propertynames(x))))
+    end
+end
+
+abstract type ScopedEnum{T<:Integer} <: Enum{T} end
+
+macro scopedenum(T, syms...)
+    if isempty(syms)
+        throw(ArgumentError("no arguments given for ScopedEnum $T"))
+    end
+    basetype = Int32
+    typename = T
+    if isa(T, Expr) && T.head === :(::) && length(T.args) == 2 && isa(T.args[1], Symbol)
+        typename = T.args[1]
+        basetype = Core.eval(__module__, T.args[2])
+        if !isa(basetype, DataType) || !(basetype <: Integer) || !isbitstype(basetype)
+            throw(ArgumentError("invalid base type for ScopedEnum $typename, $T=::$basetype; base type must be an integer primitive type"))
+        end
+    elseif !isa(T, Symbol)
+        throw(ArgumentError("invalid type expression for ScopedEnum $T"))
+    end
+    values = basetype[]
+    seen = Set{Symbol}()
+    namemap = Dict{basetype,Symbol}()
+    lo = hi = 0
+    i = zero(basetype)
+    hasexpr = false
+
+    if length(syms) == 1 && syms[1] isa Expr && syms[1].head === :block
+        syms = syms[1].args
+    end
+    for s in syms
+        s isa LineNumberNode && continue
+        if isa(s, Symbol)
+            if i == typemin(basetype) && !isempty(values)
+                throw(ArgumentError("overflow in value \"$s\" of ScopedEnum $typename"))
+            end
+        elseif isa(s, Expr) &&
+               (s.head === :(=) || s.head === :kw) &&
+               length(s.args) == 2 && isa(s.args[1], Symbol)
+            i = Core.eval(__module__, s.args[2]) # allow exprs, e.g. uint128"1"
+            if !isa(i, Integer)
+                throw(ArgumentError("invalid value for ScopedEnum $typename, $s; values must be integers"))
+            end
+            i = convert(basetype, i)
+            s = s.args[1]
+            hasexpr = true
+        else
+            throw(ArgumentError(string("invalid argument for ScopedEnum ", typename, ": ", s)))
+        end
+        if !Base.isidentifier(s)
+            throw(ArgumentError("invalid name for ScopedEnum $typename; \"$s\" is not a valid identifier"))
+        end
+        if hasexpr && haskey(namemap, i)
+            throw(ArgumentError("both $s and $(namemap[i]) have value $i in ScopedEnum $typename; values must be unique"))
+        end
+        namemap[i] = s
+        push!(values, i)
+        if s in seen
+            throw(ArgumentError("name \"$s\" in ScopedEnum $typename is not unique"))
+        end
+        push!(seen, s)
+        if length(values) == 1
+            lo = hi = i
+        else
+            lo = min(lo, i)
+            hi = max(hi, i)
+        end
+        i += oneunit(i)
+    end
+    defs = Expr(:block)
+    if isa(typename, Symbol)
+        for (i, sym) in namemap
+            push!(defs.args, :(const $(esc(sym)) = $(esc(typename))($i)))
+        end
+    end
+    mod = Symbol(typename, "Module")
+    syms = Tuple(Base.values(namemap))
+    blk = quote
+        module $(esc(mod))
+            export $(esc(typename))
+            # enum definition
+            primitive type $(esc(typename)) <: ScopedEnum{$(basetype)} $(sizeof(basetype) * 8) end
+            function $(esc(typename))(x::Integer)
+                $(Base.Enums.membershiptest(:x, values)) || Base.Enums.enum_argument_error($(Expr(:quote, typename)), x)
+                return Core.bitcast($(esc(typename)), convert($(basetype), x))
+            end
+            if isdefined(Base.Enums, :namemap)
+                Base.Enums.namemap(::Type{$(esc(typename))}) = $(esc(namemap))
+            end
+            Base.getproperty(::Type{$(esc(typename))}, sym::Symbol) = sym in $syms ? getfield($(esc(mod)), sym) : getfield($(esc(typename)), sym)
+            Base.typemin(x::Type{$(esc(typename))}) = $(esc(typename))($lo)
+            Base.typemax(x::Type{$(esc(typename))}) = $(esc(typename))($hi)
+            let insts = (Any[ $(esc(typename))(v) for v in $values ]...,)
+                Base.instances(::Type{$(esc(typename))}) = insts
+            end
+            FlatBuffers.basetype(::$(esc(typename))) = $(basetype)
+            FlatBuffers.basetype(::Type{$(esc(typename))}) = $(basetype)
+            $defs
+        end
+    end
+    push!(blk.args, :nothing)
+    blk.head = :toplevel
+    push!(blk.args, :(using .$mod))
+    return blk
+end
+
+end # module
diff --git a/julia/Arrow/src/FlatBuffers/builder.jl b/julia/Arrow/src/FlatBuffers/builder.jl
new file mode 100644
index 00000000000..0c65c6f3a52
--- /dev/null
+++ b/julia/Arrow/src/FlatBuffers/builder.jl
@@ -0,0 +1,440 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+const fileIdentifierLength = 4
+
+"""
+Scalar
+A Union of the Julia types `T <: Number` that are allowed in FlatBuffers schema
+"""
+const Scalar = Union{Bool,
+Int8, Int16, Int32, Int64,
+UInt8, UInt16, UInt32, UInt64,
+Float32, Float64, Enum}
+
+"""
+Builder is a state machine for creating FlatBuffer objects.
+Use a Builder to construct object(s) starting from leaf nodes.
+
+A Builder constructs byte buffers in a last-first manner for simplicity and
+performance.
+"""
+mutable struct Builder
+    bytes::Vector{UInt8}
+    minalign::Int
+    vtable::Vector{UOffsetT}
+    objectend::UOffsetT
+    vtables::Vector{UOffsetT}
+    head::UOffsetT
+    nested::Bool
+    finished::Bool
+    sharedstrings::Dict{String, UOffsetT}
+end
+
+bytes(b::Builder) = getfield(b, :bytes)
+
+Builder(size=0) = Builder(zeros(UInt8, size), 1, UOffsetT[], UOffsetT(0), UOffsetT[], UOffsetT(size), false, false, Dict{String, UOffsetT}())
+
+function reset!(b::Builder)
+    empty!(b.bytes)
+    empty!(b.vtable)
+    emtpy!(b.vtables)
+    empty!(b.sharedstrings)
+    b.minalign = 1
+    b.nested = false
+    b.finished = false
+    b.head = 0
+    return
+end
+
+Base.write(sink::Builder, o, x::Union{Bool,UInt8}) = sink.bytes[o+1] = UInt8(x)
+function Base.write(sink::Builder, off, x::T) where {T}
+    off += 1
+    for (i, ind) = enumerate(off:(off + sizeof(T) - 1))
+        sink.bytes[ind] = (x >> ((i-1) * 8)) % UInt8
+    end
+end
+Base.write(b::Builder, o, x::Float32) = write(b, o, reinterpret(UInt32, x))
+Base.write(b::Builder, o, x::Float64) = write(b, o, reinterpret(UInt64, x))
+Base.write(b::Builder, o, x::Enum) = write(b, o, basetype(x)(x))
+
+"""
+`finishedbytes` returns a pointer to the written data in the byte buffer.
+Panics if the builder is not in a finished state (which is caused by calling
+`finish!()`).
+"""
+function finishedbytes(b::Builder)
+    assertfinished(b)
+    return view(b.bytes, (b.head + 1):length(b.bytes))
+end
+
+function startobject!(b::Builder, numfields)
+    assertnotnested(b)
+    b.nested = true
+    resize!(b.vtable, numfields)
+    fill!(b.vtable, 0)
+    b.objectend = offset(b)
+    return
+end
+
+"""
+WriteVtable serializes the vtable for the current object, if applicable.
+
+Before writing out the vtable, this checks pre-existing vtables for equality
+to this one. If an equal vtable is found, point the object to the existing
+vtable and return.
+
+Because vtable values are sensitive to alignment of object data, not all
+logically-equal vtables will be deduplicated.
+
+A vtable has the following format:
+<VOffsetT: size of the vtable in bytes, including this value>
+<VOffsetT: size of the object in bytes, including the vtable offset>
+<VOffsetT: offset for a field> * N, where N is the number of fields in
+the schema for this type. Includes deprecated fields.
+Thus, a vtable is made of 2 + N elements, each SizeVOffsetT bytes wide.
+
+An object has the following format:
+<SOffsetT: offset to this object's vtable (may be negative)>
+<byte: data>+
+"""
+function writevtable!(b::Builder)
+    # Prepend a zero scalar to the object. Later in this function we'll
+    # write an offset here that points to the object's vtable:
+    prepend!(b, SOffsetT(0))
+
+    objectOffset = offset(b)
+    existingVtable = UOffsetT(0)
+
+    # Trim vtable of trailing zeroes.
+    i = findlast(!iszero, b.vtable)
+    if i !== nothing
+        resize!(b.vtable, i)
+    end
+    
+    # Search backwards through existing vtables, because similar vtables
+    # are likely to have been recently appended. See
+    # BenchmarkVtableDeduplication for a case in which this heuristic
+    # saves about 30% of the time used in writing objects with duplicate
+    # tables.
+    for i = length(b.vtables):-1:1
+        # Find the other vtable, which is associated with `i`:
+        vt2Offset = b.vtables[i]
+        vt2Start = length(b.bytes) - vt2Offset
+        vt2Len = readbuffer(b.bytes, vt2Start, VOffsetT)
+
+        metadata = VtableMetadataFields * sizeof(VOffsetT)
+        vt2End = vt2Start + vt2Len
+        vt2 = view(b.bytes, (vt2Start + metadata + 1):vt2End) #TODO: might need a +1 on the start of range here
+
+        # Compare the other vtable to the one under consideration.
+        # If they are equal, store the offset and break:
+        if vtableEqual(b.vtable, objectOffset, vt2)
+            existingVtable = vt2Offset
+            break
+        end
+    end
+
+    if existingVtable == 0
+        # Did not find a vtable, so write this one to the buffer.
+
+        # Write out the current vtable in reverse , because
+        # serialization occurs in last-first order:
+        for i = length(b.vtable):-1:1
+            off::UOffsetT = 0
+            if b.vtable[i] != 0
+                # Forward reference to field;
+                # use 32bit number to assert no overflow:
+                off = objectOffset - b.vtable[i]
+            end
+            prepend!(b, VOffsetT(off))
+        end
+
+        # The two metadata fields are written last.
+
+        # First, store the object bytesize:
+        objectSize = objectOffset - b.objectend
+        prepend!(b, VOffsetT(objectSize))
+
+        # Second, store the vtable bytesize:
+        vbytes = (length(b.vtable) + VtableMetadataFields) * sizeof(VOffsetT)
+        prepend!(b, VOffsetT(vbytes))
+
+        # Next, write the offset to the new vtable in the
+        # already-allocated SOffsetT at the beginning of this object:
+        objectStart = SOffsetT(length(b.bytes) - objectOffset)
+        write(b, objectStart, SOffsetT(offset(b) - objectOffset))
+
+        # Finally, store this vtable in memory for future
+        # deduplication:
+        push!(b.vtables, offset(b))
+    else
+        # Found a duplicate vtable.
+
+        objectStart = SOffsetT(length(b.bytes) - objectOffset)
+        b.head = objectStart
+
+        # Write the offset to the found vtable in the
+        # already-allocated SOffsetT at the beginning of this object:
+        write(b, b.head, SOffsetT(existingVtable) - SOffsetT(objectOffset))
+    end
+
+    empty!(b.vtable)
+    return objectOffset
+end
+
+"""
+`endobject` writes data necessary to finish object construction.
+"""
+function endobject!(b::Builder)
+    assertnested(b)
+    n = writevtable!(b)
+    b.nested = false
+    return n
+end
+
+offset(b::Builder) = UOffsetT(length(b.bytes) - b.head)
+
+pad!(b::Builder, n) = foreach(x->place!(b, 0x00), 1:n)
+
+"""
+`prep!` prepares to write an element of `size` after `additionalbytes`
+have been written, e.g. if you write a string, you need to align such
+the int length field is aligned to sizeof(Int32), and the string data follows it
+directly.
+If all you need to do is align, `additionalbytes` will be 0.
+"""
+function prep!(b::Builder, size, additionalbytes)
+    # Track the biggest thing we've ever aligned to.
+    if size > b.minalign
+        b.minalign = size
+    end
+    # Find the amount of alignment needed such that `size` is properly
+    # aligned after `additionalBytes`:
+    alignsize = xor(Int(-1), (length(b.bytes) - b.head) + additionalbytes) + 1
+    alignsize &= (size - 1)
+
+    # Reallocate the buffer if needed:
+    totalsize = alignsize + size + additionalbytes
+    if b.head <= totalsize
+        len = length(b.bytes)
+        prepend!(b.bytes, zeros(UInt8, totalsize))
+        b.head += length(b.bytes) - len
+    end
+    pad!(b, alignsize)
+    return
+end
+
+function Base.prepend!(b::Builder, x::T) where {T}
+    prep!(b, sizeof(T), 0)
+    place!(b, x)
+    return
+end
+
+function prependoffset!(b::Builder, off)
+    prep!(b, sizeof(Int32), 0) # Ensure alignment is already done.
+    if !(off <= offset(b))
+        throw(ArgumentError("unreachable: $off <= $(offset(b))"))
+    end
+    place!(b, SOffsetT(offset(b) - off + sizeof(SOffsetT)))
+    return
+end
+
+function prependoffsetslot!(b::Builder, o::Int, x::T, d) where {T}
+    if x != T(d)
+        prependoffset!(b, x)
+        slot!(b, o)
+    end
+    return
+end
+
+"""
+`startvector` initializes bookkeeping for writing a new vector.
+
+A vector has the following format:
+<UOffsetT: number of elements in this vector>
+<T: data>+, where T is the type of elements of this vector.
+"""
+function startvector!(b::Builder, elemSize, numElems, alignment)
+    assertnotnested(b)
+    b.nested = true
+    prep!(b, sizeof(UInt32), elemSize * numElems)
+    prep!(b, alignment, elemSize * numElems)
+    return offset(b)
+end
+
+"""
+`endvector` writes data necessary to finish vector construction.
+"""
+function endvector!(b::Builder, vectorNumElems)
+    assertnested(b)
+    place!(b, UOffsetT(vectorNumElems))
+    b.nested = false
+    return offset(b)
+end
+
+function createsharedstring!(b::Builder, s::AbstractString)
+    get!(b.sharedstrings, s) do
+        createstring!(b, s)
+    end
+end
+
+"""
+`createstring!` writes a null-terminated string as a vector.
+"""
+function createstring!(b::Builder, s::Union{AbstractString, AbstractVector{UInt8}})
+    assertnotnested(b)
+    b.nested = true
+    s = codeunits(s)
+    prep!(b, sizeof(UInt32), sizeof(s) + 1)
+    place!(b, UInt8(0))
+
+    l = sizeof(s)
+
+    b.head -= l
+    copyto!(b.bytes, b.head+1, s, 1, l)
+    return endvector!(b, sizeof(s))
+end
+
+createbytevector(b::Builder, v) = createstring!(b, v)
+
+function assertnested(b::Builder)
+    # If you get this assert, you're in an object while trying to write
+    # data that belongs outside of an object.
+    # To fix this, write non-inline data (like vectors) before creating
+    # objects.
+    if !b.nested
+        throw(ArgumentError("Incorrect creation order: must be inside object."))
+    end
+    return
+end
+
+function assertnotnested(b::Builder)
+    # If you hit this, you're trying to construct a Table/Vector/String
+    # during the construction of its parent table (between the MyTableBuilder
+    # and builder.Finish()).
+    # Move the creation of these view-objects to above the MyTableBuilder to
+    # not get this assert.
+    # Ignoring this assert may appear to work in simple cases, but the reason
+    # it is here is that storing objects in-line may cause vtable offsets
+    # to not fit anymore. It also leads to vtable duplication.
+    if b.nested
+        throw(ArgumentError("Incorrect creation order: object must not be nested."))
+    end
+    return
+end
+
+function assertfinished(b::Builder)
+    # If you get this assert, you're attempting to get access a buffer
+    # which hasn't been finished yet. Be sure to call builder.Finish()
+    # with your root table.
+    # If you really need to access an unfinished buffer, use the bytes
+    # buffer directly.
+    if !b.finished
+        throw(ArgumentError("Incorrect use of FinishedBytes(): must call 'Finish' first."))
+    end
+end
+
+"""
+`prependslot!` prepends a `T` onto the object at vtable slot `o`.
+If value `x` equals default `d`, then the slot will be set to zero and no
+other data will be written.
+"""
+function prependslot!(b::Builder, o::Int, x::T, d, sh=false) where {T <: Scalar}
+    if x != T(d)
+        prepend!(b, x)
+        slot!(b, o)
+    end
+    return
+end
+
+"""
+`prependstructslot!` prepends a struct onto the object at vtable slot `o`.
+Structs are stored inline, so nothing additional is being added.
+In generated code, `d` is always 0.
+"""
+function prependstructslot!(b::Builder, voffset, x, d)
+    if x != d
+        assertnested(b)
+        if x != offset(b)
+            throw(ArgumentError("inline data write outside of object"))
+        end
+        slot!(b, voffset)
+    end
+    return
+end
+
+"""
+`slot!` sets the vtable key `voffset` to the current location in the buffer.
+"""
+function slot!(b::Builder, slotnum)
+    b.vtable[slotnum + 1] = offset(b)
+end
+
+# FinishWithFileIdentifier finalizes a buffer, pointing to the given `rootTable`.
+# as well as applys a file identifier
+function finishwithfileidentifier(b::Builder, rootTable, fid)
+    if length(fid) != fileIdentifierLength
+        error("incorrect file identifier length")
+    end
+    # In order to add a file identifier to the flatbuffer message, we need
+    # to prepare an alignment and file identifier length
+    prep!(b, b.minalign, sizeof(Int32) + fileIdentifierLength)
+    for i = fileIdentifierLength:-1:1
+        # place the file identifier
+        place!(b, fid[i])
+    end
+    # finish
+    finish!(b, rootTable)
+end
+
+"""
+`finish!` finalizes a buffer, pointing to the given `rootTable`.
+"""
+function finish!(b::Builder, rootTable)
+    assertnotnested(b)
+    prep!(b, b.minalign, sizeof(UOffsetT))
+    prependoffset!(b, UOffsetT(rootTable))
+    b.finished = true
+    return
+end
+
+"vtableEqual compares an unwritten vtable to a written vtable."
+function vtableEqual(a::Vector{UOffsetT}, objectStart, b::AbstractVector{UInt8})
+    if length(a) * sizeof(VOffsetT) != length(b)
+        return false
+    end
+
+    for i = 0:(length(a)-1)
+        x = read(IOBuffer(view(b, (i * sizeof(VOffsetT) + 1):length(b))), VOffsetT)
+
+        # Skip vtable entries that indicate a default value.
+        x == 0 && a[i+1] == 0 && continue
+
+        y = objectStart - a[i+1]
+        x != y && return false
+    end
+    return true
+end
+
+"""
+`place!` prepends a `T` to the Builder, without checking for space.
+"""
+function place!(b::Builder, x::T) where {T}
+    b.head -= sizeof(T)
+    write(b, b.head, x)
+    return
+end
diff --git a/julia/Arrow/src/FlatBuffers/table.jl b/julia/Arrow/src/FlatBuffers/table.jl
new file mode 100644
index 00000000000..cd87c4f1e02
--- /dev/null
+++ b/julia/Arrow/src/FlatBuffers/table.jl
@@ -0,0 +1,170 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Table
+
+The object containing the flatbuffer and positional information specific to the table.
+The `vtable` containing the offsets for specific members precedes `pos`.
+The actual values in the table follow `pos` offset and size of the vtable.
+
+- `bytes::Vector{UInt8}`: the flatbuffer itself
+- `pos::Integer`:  the base position in `bytes` of the table
+"""
+abstract type Table end
+abstract type Struct end
+
+const TableOrStruct = Union{Table, Struct}
+
+bytes(x::TableOrStruct) = getfield(x, :bytes)
+pos(x::TableOrStruct) = getfield(x, :pos)
+
+(::Type{T})(b::Builder) where {T <: TableOrStruct} = T(b.bytes[b.head+1:end], get(b, b.head, Int32))
+
+getrootas(::Type{T}, bytes::Vector{UInt8}, offset) where {T <: Table} = init(T, bytes, offset + readbuffer(bytes, offset, UOffsetT))
+init(::Type{T}, bytes::Vector{UInt8}, pos::Integer) where {T <: TableOrStruct} = T(bytes, pos)
+
+const TableOrBuilder = Union{Table, Struct, Builder}
+
+Base.get(t::TableOrBuilder, pos, ::Type{T}) where {T} = readbuffer(bytes(t), pos, T)
+Base.get(t::TableOrBuilder, pos, ::Type{T}) where {T <: Enum} = T(get(t, pos, basetype(T)))
+
+"""
+`offset` provides access into the Table's vtable.
+
+Deprecated fields are ignored by checking against the vtable's length.
+"""
+function offset(t::Table, vtableoffset)
+    vtable = pos(t) - get(t, pos(t), SOffsetT)
+    return vtableoffset < get(t, vtable, VOffsetT) ? get(t, vtable + vtableoffset, VOffsetT) : VOffsetT(0)
+end
+
+"`indirect` retrieves the relative offset stored at `offset`."
+indirect(t::Table, off) = off + get(t, off, UOffsetT)
+
+getvalue(t, o, ::Type{Nothing}) = nothing
+getvalue(t, o, ::Type{T}) where {T <: Scalar} = get(t, pos(t) + o, T)
+getvalue(t, o, ::Type{T}) where {T <: Enum} = T(get(t, pos(t) + o, enumtype(T)))
+
+function Base.String(t::Table, off)
+    off += get(t, off, UOffsetT)
+    start = off + sizeof(UOffsetT)
+    len = get(t, off, UOffsetT)
+    return unsafe_string(pointer(bytes(t), start + 1), len)
+end
+
+function bytevector(t::Table, off)
+    off += get(t, off, UOffsetT)
+    start = off + sizeof(UOffsetT)
+    len = get(t, off, UOffsetT)
+    return view(bytes(t), (start + 1):(start + len + 1))
+end
+
+"""
+`vectorlen` retrieves the length of the vector whose offset is stored at
+`off` in this object.
+"""
+function vectorlen(t::Table, off)
+    off += pos(t)
+    off += get(t, off, UOffsetT)
+    return Int(get(t, off, UOffsetT))
+end
+
+"""
+`vector` retrieves the start of data of the vector whose offset is stored
+at `off` in this object.
+"""
+function vector(t::Table, off)
+    off += pos(t)
+    x = off + get(t, off, UOffsetT)
+    # data starts after metadata containing the vector length
+    return x + sizeof(UOffsetT)
+end
+
+struct Array{T, S, TT} <: AbstractVector{T}
+    _tab::TT
+    pos::UOffsetT
+    data::Vector{S}
+end
+
+function Array{T}(t::Table, off) where {T}
+    a = vector(t, off)
+    S = T <: Table ? UOffsetT : T <: Struct ? NTuple{structsizeof(T), UInt8} : T
+    ptr = convert(Ptr{S}, pointer(bytes(t), pos(t) + a + 1))
+    data = unsafe_wrap(Base.Array, ptr, vectorlen(t, off))
+    return Array{T, S, typeof(t)}(t, a, data)
+end
+
+function structsizeof end
+
+Base.IndexStyle(::Type{<:Array}) = Base.IndexLinear()
+Base.size(x::Array) = size(x.data)
+Base.@propagate_inbounds function Base.getindex(A::Array{T, S}, i::Integer) where {T, S}
+    if T === S
+        return A.data[i]
+    elseif T <: Struct
+        return init(T, bytes(A._tab), A.pos + (i - 1) * structsizeof(T))
+    else # T isa Table
+        return init(T, bytes(A._tab), indirect(A._tab, A.pos + (i - 1) * 4))
+    end
+end
+
+Base.@propagate_inbounds function Base.setindex!(A::Array{T, S}, v, i::Integer) where {T, S}
+    if T === S
+        return setindex!(A.data, v, i)
+    else
+        error("setindex! not supported for reference/table types")
+    end
+end
+
+function union(t::Table, off)
+    off += pos(t)
+    return off + get(t, off, UOffsetT)
+end
+
+function union!(t::Table, t2::Table, off)
+    off += pos(t)
+    t2.pos = off + get(t, off, UOffsetT)
+    t2.bytes = bytes(t)
+    return
+end
+
+"""
+GetVOffsetTSlot retrieves the VOffsetT that the given vtable location
+points to. If the vtable value is zero, the default value `d`
+will be returned.
+"""
+function getoffsetslot(t::Table, slot, d)
+    off = offset(t, slot)
+    if off == 0
+        return d
+    end
+    return off
+end
+
+"""
+`getslot` retrieves the `T` that the given vtable location
+points to. If the vtable value is zero, the default value `d`
+will be returned.
+"""
+function getslot(t::Table, slot, d::T) where {T}
+    off = offset(t, slot)
+    if off == 0
+        return d
+    end
+
+    return get(t, pos(t) + off, T)
+end
diff --git a/julia/Arrow/src/arraytypes/arraytypes.jl b/julia/Arrow/src/arraytypes/arraytypes.jl
new file mode 100644
index 00000000000..92ba3e40af9
--- /dev/null
+++ b/julia/Arrow/src/arraytypes/arraytypes.jl
@@ -0,0 +1,180 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+    Arrow.ArrowVector
+
+An abstract type that subtypes `AbstractVector`. Each specific arrow array type
+subtypes `ArrowVector`. See [`BoolVector`](@ref), [`Primitive`](@ref), [`List`](@ref),
+[`Map`](@ref), [`FixedSizeList`](@ref), [`Struct`](@ref), [`DenseUnion`](@ref),
+[`SparseUnion`](@ref), and [`DictEncoded`](@ref) for more details.
+"""
+abstract type ArrowVector{T} <: AbstractVector{T} end
+
+Base.IndexStyle(::Type{A}) where {A <: ArrowVector} = Base.IndexLinear()
+Base.similar(::Type{A}, dims::Dims) where {T, A <: ArrowVector{T}} = Vector{T}(undef, dims)
+validitybitmap(x::ArrowVector) = x.validity
+nullcount(x::ArrowVector) = validitybitmap(x).nc
+getmetadata(x::ArrowVector) = x.metadata
+
+function toarrowvector(x, i=1, de=Dict{Int64, Any}(), ded=DictEncoding[], meta=getmetadata(x); compression::Union{Nothing, LZ4FrameCompressor, ZstdCompressor}=nothing, kw...)
+    @debug 2 "converting top-level column to arrow format: col = $(typeof(x)), compression = $compression, kw = $(kw.data)"
+    @debug 3 x
+    A = arrowvector(x, i, 0, 0, de, ded, meta; compression=compression, kw...)
+    if compression isa LZ4FrameCompressor
+        A = compress(Meta.CompressionType.LZ4_FRAME, compression, A)
+    elseif compression isa ZstdCompressor
+        A = compress(Meta.CompressionType.ZSTD, compression, A)
+    end
+    @debug 2 "converted top-level column to arrow format: $(typeof(A))"
+    @debug 3 A
+    return A
+end
+
+function arrowvector(x, i, nl, fi, de, ded, meta; dictencoding::Bool=false, dictencode::Bool=false, kw...)
+    if !(x isa DictEncode) && !dictencoding && (dictencode || (x isa AbstractArray && DataAPI.refarray(x) !== x))
+        x = DictEncode(x, dictencodeid(i, nl, fi))
+    end
+    S = maybemissing(eltype(x))
+    return arrowvector(S, x, i, nl, fi, de, ded, meta; dictencode=dictencode, kw...)
+end
+
+# conversions to arrow types
+arrowvector(::Type{Dates.Date}, x, i, nl, fi, de, ded, meta; kw...) =
+    arrowvector(converter(DATE, x), i, nl, fi, de, ded, meta; kw...)
+arrowvector(::Type{Dates.Time}, x, i, nl, fi, de, ded, meta; kw...) =
+    arrowvector(converter(TIME, x), i, nl, fi, de, ded, meta; kw...)
+arrowvector(::Type{Dates.DateTime}, x, i, nl, fi, de, ded, meta; kw...) =
+    arrowvector(converter(DATETIME, x), i, nl, fi, de, ded, meta; kw...)
+arrowvector(::Type{ZonedDateTime}, x, i, nl, fi, de, ded, meta; kw...) =
+    arrowvector(converter(Timestamp{Meta.TimeUnit.MILLISECOND, Symbol(x[1].timezone)}, x), i, nl, fi, de, ded, meta; kw...)
+arrowvector(::Type{P}, x, i, nl, fi, de, ded, meta; kw...) where {P <: Dates.Period} =
+    arrowvector(converter(Duration{arrowperiodtype(P)}, x), i, nl, fi, de, ded, meta; kw...)
+
+# fallback that calls ArrowType
+function arrowvector(::Type{S}, x, i, nl, fi, de, ded, meta; kw...) where {S}
+    if ArrowTypes.istyperegistered(S)
+        meta = meta === nothing ? Dict{String, String}() : meta
+        arrowtype = ArrowTypes.getarrowtype!(meta, S)
+        return arrowvector(converter(arrowtype, x), i, nl, fi, de, ded, meta; kw...)
+    end
+    return arrowvector(ArrowType(S), x, i, nl, fi, de, ded, meta; kw...)
+end
+
+arrowvector(::NullType, x, i, nl, fi, de, ded, meta; kw...) = MissingVector(length(x))
+compress(Z::Meta.CompressionType, comp, v::MissingVector) =
+    Compressed{Z, MissingVector}(v, CompressedBuffer[], length(v), length(v), Compressed[])
+
+function makenodesbuffers!(col::MissingVector, fieldnodes, fieldbuffers, bufferoffset, alignment)
+    push!(fieldnodes, FieldNode(length(col), length(col)))
+    @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
+    return bufferoffset
+end
+
+function writebuffer(io, col::MissingVector, alignment)
+    return
+end
+
+"""
+    Arrow.ValidityBitmap
+
+A bit-packed array type where each bit corresponds to an element in an
+[`ArrowVector`](@ref), indicating whether that element is "valid" (bit == 1),
+or not (bit == 0). Used to indicate element missingness (whether it's null).
+
+If the null count of an array is zero, the `ValidityBitmap` will be "emtpy"
+and all elements are treated as "valid"/non-null.
+"""
+struct ValidityBitmap <: ArrowVector{Bool}
+    bytes::Vector{UInt8} # arrow memory blob
+    pos::Int # starting byte of validity bitmap
+    ℓ::Int # # of _elements_ (not bytes!) in bitmap (because bitpacking)
+    nc::Int # null count
+end
+
+Base.size(p::ValidityBitmap) = (p.ℓ,)
+nullcount(x::ValidityBitmap) = x.nc
+
+function ValidityBitmap(x)
+    T = eltype(x)
+    if !(T >: Missing)
+        return ValidityBitmap(UInt8[], 1, length(x), 0)
+    end
+    len = length(x)
+    blen = cld(len, 8)
+    bytes = Vector{UInt8}(undef, blen)
+    st = iterate(x)
+    i = 0
+    nc = 0
+    for k = 1:blen
+        b = 0x00
+        for j = 1:8
+            if (i + j) <= len
+                y, state = st
+                if y === missing
+                    nc += 1
+                    b = setbit(b, false, j)
+                else
+                    b = setbit(b, true, j)
+                end
+                st = iterate(x, state)
+            end
+        end
+        i += 8
+        @inbounds bytes[k] = b
+    end
+    return ValidityBitmap(nc == 0 ? UInt8[] : bytes, 1, nc == 0 ? 0 : len, nc)
+end
+
+@propagate_inbounds function Base.getindex(p::ValidityBitmap, i::Integer)
+    # no boundscheck because parent array should do it
+    # if a validity bitmap is empty, it either means:
+    #   1) the parent array null_count is 0, so all elements are valid
+    #   2) parent array is also empty, so "all" elements are valid
+    p.nc == 0 && return true
+    # translate element index to bitpacked byte index
+    a, b = fldmod1(i, 8)
+    @inbounds byte = p.bytes[p.pos + a - 1]
+    # check individual bit of byte
+    return getbit(byte, b)
+end
+
+@propagate_inbounds function Base.setindex!(p::ValidityBitmap, v, i::Integer)
+    x = convert(Bool, v)
+    p.ℓ == 0 && !x && throw(BoundsError(p, i))
+    a, b = fldmod1(i, 8)
+    @inbounds byte = p.bytes[p.pos + a - 1]
+    @inbounds p.bytes[p.pos + a - 1] = setbit(byte, x, b)
+    return v
+end
+
+function writebitmap(io, col::ArrowVector, alignment)
+    v = col.validity
+    @debug 1 "writing validity bitmap: nc = $(v.nc), n = $(cld(v.ℓ, 8))"
+    v.nc == 0 && return 0
+    n = Base.write(io, view(v.bytes, v.pos:(v.pos + cld(v.ℓ, 8) - 1)))
+    return n + writezeros(io, paddinglength(n, alignment))
+end
+
+include("compressed.jl")
+include("primitive.jl")
+include("bool.jl")
+include("list.jl")
+include("fixedsizelist.jl")
+include("map.jl")
+include("struct.jl")
+include("unions.jl")
+include("dictencoding.jl")
diff --git a/julia/Arrow/src/arraytypes/bool.jl b/julia/Arrow/src/arraytypes/bool.jl
new file mode 100644
index 00000000000..07929030282
--- /dev/null
+++ b/julia/Arrow/src/arraytypes/bool.jl
@@ -0,0 +1,112 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+    Arrow.BoolVector
+
+A bit-packed array type, similar to [`ValidityBitmap`](@ref), but which
+holds boolean values, `true` or `false`.
+"""
+struct BoolVector{T} <: ArrowVector{T}
+    arrow::Vector{UInt8} # need to hold a reference to arrow memory blob
+    pos::Int
+    validity::ValidityBitmap
+    ℓ::Int64
+    metadata::Union{Nothing, Dict{String, String}}
+end
+
+Base.size(p::BoolVector) = (p.ℓ,)
+
+@propagate_inbounds function Base.getindex(p::BoolVector{T}, i::Integer) where {T}
+    @boundscheck checkbounds(p, i)
+    if T >: Missing
+        @inbounds !p.validity[i] && return missing
+    end
+    a, b = fldmod1(i, 8)
+    @inbounds byte = p.arrow[p.pos + a - 1]
+    # check individual bit of byte
+    return getbit(byte, b)
+end
+
+@propagate_inbounds function Base.setindex!(p::BoolVector, v, i::Integer)
+    @boundscheck checkbounds(p, i)
+    x = convert(Bool, v)
+    a, b = fldmod1(i, 8)
+    @inbounds byte = p.arrow[p.pos + a - 1]
+    @inbounds p.arrow[p.pos + a - 1] = setbit(byte, x, b)
+    return v
+end
+
+arrowvector(::BoolType, x::BoolVector, i, nl, fi, de, ded, meta; kw...) = x
+
+function arrowvector(::BoolType, x, i, nl, fi, de, ded, meta; kw...)
+    validity = ValidityBitmap(x)
+    len = length(x)
+    blen = cld(len, 8)
+    bytes = Vector{UInt8}(undef, blen)
+    st = iterate(x)
+    i = 0
+    for k = 1:blen
+        b = 0x00
+        for j = 1:8
+            if (i + j) <= len
+                y, state = st
+                if y === missing || !y
+                    b = setbit(b, false, j)
+                else
+                    b = setbit(b, true, j)
+                end
+                st = iterate(x, state)
+            end
+        end
+        i += 8
+        @inbounds bytes[k] = b
+    end
+    return BoolVector{eltype(x)}(bytes, 1, validity, len, meta)
+end
+
+function compress(Z::Meta.CompressionType, comp, p::P) where {P <: BoolVector}
+    len = length(p)
+    nc = nullcount(p)
+    validity = compress(Z, comp, p.validity)
+    data = compress(Z, comp, view(p.arrow, p.pos:(p.pos + cld(p.ℓ, 8) - 1)))
+    return Compressed{Z, P}(p, [validity, data], len, nc, Compressed[])
+end
+
+function makenodesbuffers!(col::BoolVector, fieldnodes, fieldbuffers, bufferoffset, alignment)
+    len = length(col)
+    nc = nullcount(col)
+    push!(fieldnodes, FieldNode(len, nc))
+    @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
+    # validity bitmap
+    blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
+    push!(fieldbuffers, Buffer(bufferoffset, blen))
+    @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
+    # adjust buffer offset, make primitive array buffer
+    bufferoffset += blen
+    blen = bitpackedbytes(len, alignment)
+    push!(fieldbuffers, Buffer(bufferoffset, blen))
+    @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
+    return bufferoffset + blen
+end
+
+function writebuffer(io, col::BoolVector, alignment)
+    @debug 1 "writebuffer: col = $(typeof(col))"
+    @debug 2 col
+    writebitmap(io, col, alignment)
+    n = Base.write(io, view(col.arrow, col.pos:(col.pos + cld(col.ℓ, 8) - 1)))
+    return n + writezeros(io, paddinglength(n, alignment))
+end
diff --git a/julia/Arrow/src/arraytypes/compressed.jl b/julia/Arrow/src/arraytypes/compressed.jl
new file mode 100644
index 00000000000..5f8e67e25d6
--- /dev/null
+++ b/julia/Arrow/src/arraytypes/compressed.jl
@@ -0,0 +1,90 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+struct CompressedBuffer
+    data::Vector{UInt8}
+    uncompressedlength::Int64
+end
+
+"""
+    Arrow.Compressed
+
+Represents the compressed version of an [`ArrowVector`](@ref).
+Holds a reference to the original column. May have `Compressed`
+children for nested array types.
+"""
+struct Compressed{Z, A}
+    data::A
+    buffers::Vector{CompressedBuffer}
+    len::Int64
+    nullcount::Int64
+    children::Vector{Compressed}
+end
+
+Base.length(c::Compressed) = c.len
+Base.eltype(c::Compressed{Z, A}) where {Z, A} = eltype(A)
+getmetadata(x::Compressed) = getmetadata(x.data)
+compressiontype(c::Compressed{Z}) where {Z} = Z
+
+function compress(Z::Meta.CompressionType, comp, x::Array)
+    GC.@preserve x begin
+        y = unsafe_wrap(Array, convert(Ptr{UInt8}, pointer(x)), sizeof(x))
+        return CompressedBuffer(transcode(comp, y), length(y))
+    end
+end
+
+compress(Z::Meta.CompressionType, comp, x) = compress(Z, comp, convert(Array, x))
+
+compress(Z::Meta.CompressionType, comp, v::ValidityBitmap) =
+    v.nc == 0 ? CompressedBuffer(UInt8[], 0) : compress(Z, comp, view(v.bytes, v.pos:(v.pos + cld(v.ℓ, 8) - 1)))
+
+function makenodesbuffers!(col::Compressed, fieldnodes, fieldbuffers, bufferoffset, alignment)
+    push!(fieldnodes, FieldNode(col.len, col.nullcount))
+    @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
+    for buffer in col.buffers
+        blen = length(buffer.data) == 0 ? 0 : 8 + length(buffer.data)
+        push!(fieldbuffers, Buffer(bufferoffset, blen))
+        @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
+        bufferoffset += padding(blen, alignment)
+    end
+    for child in col.children
+        bufferoffset = makenodesbuffers!(child, fieldnodes, fieldbuffers, bufferoffset, alignment)
+    end
+    return bufferoffset
+end
+
+function writearray(io, b::CompressedBuffer)
+    if length(b.data) > 0
+        n = Base.write(io, b.uncompressedlength)
+        @debug 1 "writing compressed buffer: uncompressedlength = $(b.uncompressedlength), n = $(length(b.data))"
+        @debug 2 b.data
+        return n + Base.write(io, b.data)
+    end
+    return 0
+end
+
+function writebuffer(io, col::Compressed, alignment)
+    @debug 1 "writebuffer: col = $(typeof(col))"
+    @debug 2 col
+    for buffer in col.buffers
+        n = writearray(io, buffer)
+        writezeros(io, paddinglength(n, alignment))
+    end
+    for child in col.children
+        writebuffer(io, child, alignment)
+    end
+    return
+end
diff --git a/julia/Arrow/src/arraytypes/dictencoding.jl b/julia/Arrow/src/arraytypes/dictencoding.jl
new file mode 100644
index 00000000000..af12447588c
--- /dev/null
+++ b/julia/Arrow/src/arraytypes/dictencoding.jl
@@ -0,0 +1,253 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+    Arrow.DictEncoding
+
+Represents the "pool" of possible values for a [`DictEncoded`](@ref)
+array type. Whether the order of values is significant can be checked
+by looking at the `isOrdered` boolean field.
+"""
+mutable struct DictEncoding{T, A} <: ArrowVector{T}
+    id::Int64
+    data::A
+    isOrdered::Bool
+end
+
+Base.size(d::DictEncoding) = size(d.data)
+
+@propagate_inbounds function Base.getindex(d::DictEncoding{T}, i::Integer) where {T}
+    @boundscheck checkbounds(d, i)
+    return @inbounds ArrowTypes.arrowconvert(T, d.data[i])
+end
+
+# convenience wrapper to signal that an input column should be
+# dict encoded when written to the arrow format
+struct DictEncodeType{T} end
+getT(::Type{DictEncodeType{T}}) where {T} = T
+
+"""
+    Arrow.DictEncode(::AbstractVector, id::Integer=nothing)
+
+Signals that a column/array should be dictionary encoded when serialized
+to the arrow streaming/file format. An optional `id` number may be provided
+to signal that multiple columns should use the same pool when being
+dictionary encoded.
+"""
+struct DictEncode{T, A} <: AbstractVector{DictEncodeType{T}}
+    id::Int64
+    data::A
+end
+
+DictEncode(x::A, id=-1) where {A} = DictEncode{eltype(A), A}(id, x)
+Base.IndexStyle(::Type{<:DictEncode}) = Base.IndexLinear()
+Base.size(x::DictEncode) = (length(x.data),)
+Base.iterate(x::DictEncode, st...) = iterate(x.data, st...)
+Base.getindex(x::DictEncode, i::Int) = getindex(x.data, i)
+ArrowTypes.ArrowType(::Type{<:DictEncodeType}) = DictEncodedType()
+
+"""
+    Arrow.DictEncoded
+
+A dictionary encoded array type (similar to a `PooledArray`). Behaves just
+like a normal array in most respects; internally, possible values are stored
+in the `encoding::DictEncoding` field, while the `indices::Vector{<:Integer}`
+field holds the "codes" of each element for indexing into the encoding pool.
+Any column/array can be dict encoding when serializing to the arrow format
+either by passing the `dictencode=true` keyword argument to [`Arrow.write`](@ref)
+(which causes _all_ columns to be dict encoded), or wrapping individual columns/
+arrays in [`Arrow.DictEncode(x)`](@ref).
+"""
+struct DictEncoded{T, S, A} <: ArrowVector{T}
+    arrow::Vector{UInt8} # need to hold a reference to arrow memory blob
+    validity::ValidityBitmap
+    indices::Vector{S}
+    encoding::DictEncoding{T, A}
+    metadata::Union{Nothing, Dict{String, String}}
+end
+
+DictEncoded(b::Vector{UInt8}, v::ValidityBitmap, inds::Vector{S}, encoding::DictEncoding{T, A}, meta) where {S, T, A} =
+    DictEncoded{T, S, A}(b, v, inds, encoding, meta)
+
+Base.size(d::DictEncoded) = size(d.indices)
+
+isdictencoded(d::DictEncoded) = true
+isdictencoded(x) = false
+isdictencoded(c::Compressed{Z, A}) where {Z, A <: DictEncoded} = true
+
+signedtype(::Type{UInt8}) = Int8
+signedtype(::Type{UInt16}) = Int16
+signedtype(::Type{UInt32}) = Int32
+signedtype(::Type{UInt64}) = Int64
+
+indtype(d::DictEncoded{T, S, A}) where {T, S, A} = S
+indtype(c::Compressed{Z, A}) where {Z, A <: DictEncoded} = indtype(c.data)
+
+dictencodeid(colidx, nestedlevel, fieldid) = (Int64(nestedlevel) << 48) | (Int64(fieldid) << 32) | Int64(colidx)
+
+getid(d::DictEncoded) = d.encoding.id
+getid(c::Compressed{Z, A}) where {Z, A <: DictEncoded} = c.data.encoding.id
+
+arrowvector(::DictEncodedType, x::DictEncoded, i, nl, fi, de, ded, meta; kw...) = x
+
+function arrowvector(::DictEncodedType, x, i, nl, fi, de, ded, meta; dictencode::Bool=false, dictencodenested::Bool=false, kw...)
+    @assert x isa DictEncode
+    id = x.id == -1 ? dictencodeid(i, nl, fi) : x.id
+    x = x.data
+    len = length(x)
+    validity = ValidityBitmap(x)
+    if !haskey(de, id)
+        # dict encoding doesn't exist yet, so create for 1st time
+        if DataAPI.refarray(x) === x
+            # need to encode ourselves
+            x = PooledArray(x, encodingtype(length(x)))
+            inds = DataAPI.refarray(x)
+        else
+            inds = copy(DataAPI.refarray(x))
+        end
+        # adjust to "offset" instead of index
+        for i = 1:length(inds)
+            @inbounds inds[i] -= 1
+        end
+        pool = DataAPI.refpool(x)
+        # horrible hack? yes. better than taking CategoricalArrays dependency? also yes.
+        if typeof(pool).name.name == :CategoricalRefPool
+            pool = [get(pool[i]) for i = 1:length(pool)]
+        end
+        data = arrowvector(pool, i, nl, fi, de, ded, nothing; dictencode=dictencodenested, dictencodenested=dictencodenested, dictencoding=true, kw...)
+        encoding = DictEncoding{eltype(data), typeof(data)}(id, data, false)
+        de[id] = Lockable(encoding)
+    else
+        # encoding already exists
+          # compute inds based on it
+          # if value doesn't exist in encoding, push! it
+          # also add to deltas updates
+        encodinglockable = de[id]
+        @lock encodinglockable begin
+            encoding = encodinglockable.x
+            pool = Dict(a => (b - 1) for (b, a) in enumerate(encoding))
+            deltas = eltype(x)[]
+            len = length(x)
+            inds = Vector{encodingtype(len)}(undef, len)
+            categorical = typeof(x).name.name == :CategoricalArray
+            for (j, val) in enumerate(x)
+                if categorical
+                    val = get(val)
+                end
+                @inbounds inds[j] = get!(pool, val) do
+                    push!(deltas, val)
+                    length(pool)
+                end
+            end
+            if !isempty(deltas)
+                data = arrowvector(deltas, i, nl, fi, de, ded, nothing; dictencode=dictencodenested, dictencodenested=dictencodenested, dictencoding=true, kw...)
+                push!(ded, DictEncoding{eltype(data), typeof(data)}(id, data, false))
+                if typeof(encoding.data) <: ChainedVector
+                    append!(encoding.data, data)
+                else
+                    data2 = ChainedVector([encoding.data, data])
+                    encoding = DictEncoding{eltype(data2), typeof(data2)}(id, data2, false)
+                    de[id] = Lockable(encoding)
+                end
+            end
+        end
+    end
+    if meta !== nothing && data.metadata !== nothing
+        merge!(meta, data.metadata)
+    elseif data.metadata !== nothing
+        meta = data.metadata
+    end
+    return DictEncoded(UInt8[], validity, inds, encoding, meta)
+end
+
+@propagate_inbounds function Base.getindex(d::DictEncoded, i::Integer)
+    @boundscheck checkbounds(d, i)
+    @inbounds valid = d.validity[i]
+    !valid && return missing
+    @inbounds idx = d.indices[i]
+    return @inbounds d.encoding[idx + 1]
+end
+
+@propagate_inbounds function Base.setindex!(d::DictEncoded{T}, v, i::Integer) where {T}
+    @boundscheck checkbounds(d, i)
+    if v === missing
+        @inbounds d.validity[i] = false
+    else
+        ix = findfirst(d.encoding.data, v)
+        if ix === nothing
+            push!(d.encoding.data, v)
+            @inbounds d.indices[i] = length(d.encoding.data) - 1
+        else
+            @inbounds d.indices[i] = ix - 1
+        end
+    end
+    return v
+end
+
+function Base.copy(x::DictEncoded{T, S}) where {T, S}
+    pool = copy(x.encoding.data)
+    valid = x.validity
+    inds = x.indices
+    if T >: Missing
+        refs = Vector{S}(undef, length(inds))
+        @inbounds for i = 1:length(inds)
+            refs[i] = ifelse(valid[i], inds[i] + one(S), missing)
+        end
+    else
+        refs = copy(inds)
+        @inbounds for i = 1:length(inds)
+            refs[i] = refs[i] + one(S)
+        end
+    end
+    return PooledArray(PooledArrays.RefArray(refs), Dict{T, S}(val => i for (i, val) in enumerate(pool)), pool)
+end
+
+function compress(Z::Meta.CompressionType, comp, x::A) where {A <: DictEncoded}
+    len = length(x)
+    nc = nullcount(x)
+    validity = compress(Z, comp, x.validity)
+    inds = compress(Z, comp, x.indices)
+    return Compressed{Z, A}(x, [validity, inds], len, nc, Compressed[])
+end
+
+function makenodesbuffers!(col::DictEncoded{T, S}, fieldnodes, fieldbuffers, bufferoffset, alignment) where {T, S}
+    len = length(col)
+    nc = nullcount(col)
+    push!(fieldnodes, FieldNode(len, nc))
+    @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
+    # validity bitmap
+    blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
+    push!(fieldbuffers, Buffer(bufferoffset, blen))
+    @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
+    bufferoffset += blen
+    # indices
+    blen = sizeof(S) * len
+    push!(fieldbuffers, Buffer(bufferoffset, blen))
+    @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
+    bufferoffset += padding(blen, alignment)
+    return bufferoffset
+end
+
+function writebuffer(io, col::DictEncoded, alignment)
+    @debug 1 "writebuffer: col = $(typeof(col))"
+    @debug 2 col
+    writebitmap(io, col, alignment)
+    # write indices
+    n = writearray(io, col.indices)
+    @debug 1 "writing array: col = $(typeof(col.indices)), n = $n, padded = $(padding(n, alignment))"
+    writezeros(io, paddinglength(n, alignment))
+    return
+end
diff --git a/julia/Arrow/src/arraytypes/fixedsizelist.jl b/julia/Arrow/src/arraytypes/fixedsizelist.jl
new file mode 100644
index 00000000000..9393049ab43
--- /dev/null
+++ b/julia/Arrow/src/arraytypes/fixedsizelist.jl
@@ -0,0 +1,153 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+    Arrow.FixedSizeList
+
+An `ArrowVector` where each element is a "fixed size" list of some kind, like a `NTuple{N, T}`.
+"""
+struct FixedSizeList{T, A <: AbstractVector} <: ArrowVector{T}
+    arrow::Vector{UInt8} # need to hold a reference to arrow memory blob
+    validity::ValidityBitmap
+    data::A
+    ℓ::Int
+    metadata::Union{Nothing, Dict{String, String}}
+end
+
+Base.size(l::FixedSizeList) = (l.ℓ,)
+
+@propagate_inbounds function Base.getindex(l::FixedSizeList{T}, i::Integer) where {T}
+    @boundscheck checkbounds(l, i)
+    N = ArrowTypes.getsize(Base.nonmissingtype(T))
+    off = (i - 1) * N
+    if Base.nonmissingtype(T) !== T
+        return l.validity[i] ? ArrowTypes.arrowconvert(T, ntuple(j->l.data[off + j], N)) : missing
+    else
+        return ArrowTypes.arrowconvert(T, ntuple(j->l.data[off + j], N))
+    end
+end
+
+@propagate_inbounds function Base.setindex!(l::FixedSizeList{T}, v::T, i::Integer) where {T}
+    @boundscheck checkbounds(l, i)
+    if v === missing
+        @inbounds l.validity[i] = false
+    else
+        N = ArrowTypes.getsize(Base.nonmissingtype(T))
+        off = (i - 1) * N
+        foreach(1:N) do j
+            @inbounds l.data[off + j] = v[j]
+        end
+    end
+    return v
+end
+
+# lazy equal-spaced flattener
+struct ToFixedSizeList{T, N, A} <: AbstractVector{T}
+    data::A # A is AbstractVector of AbstractVector or AbstractString
+end
+
+function ToFixedSizeList(input)
+    NT = Base.nonmissingtype(eltype(input)) # typically NTuple{N, T}
+    return ToFixedSizeList{ArrowTypes.gettype(NT), ArrowTypes.getsize(NT), typeof(input)}(input)
+end
+
+Base.IndexStyle(::Type{<:ToFixedSizeList}) = Base.IndexLinear()
+Base.size(x::ToFixedSizeList{T, N}) where {T, N} = (N * length(x.data),)
+
+Base.@propagate_inbounds function Base.getindex(A::ToFixedSizeList{T, N}, i::Integer) where {T, N}
+    @boundscheck checkbounds(A, i)
+    a, b = fldmod1(i, N)
+    @inbounds x = A.data[a]
+    return @inbounds x === missing ? ArrowTypes.default(T) : x[b]
+end
+
+# efficient iteration
+@inline function Base.iterate(A::ToFixedSizeList{T, N}, (i, chunk, chunk_i, len)=(1, 1, 1, length(A))) where {T, N}
+    i > len && return nothing
+    @inbounds y = A.data[chunk]
+    @inbounds x = y === missing ? ArrowTypes.default(T) : y[chunk_i]
+    if chunk_i == N
+        chunk += 1
+        chunk_i = 1
+    else
+        chunk_i += 1
+    end
+    return x, (i + 1, chunk, chunk_i, len)
+end
+
+arrowvector(::FixedSizeListType, x::FixedSizeList, i, nl, fi, de, ded, meta; kw...) = x
+
+function arrowvector(::FixedSizeListType, x, i, nl, fi, de, ded, meta; kw...)
+    len = length(x)
+    validity = ValidityBitmap(x)
+    flat = ToFixedSizeList(x)
+    if eltype(flat) == UInt8
+        data = flat
+    else
+        data = arrowvector(flat, i, nl + 1, fi, de, ded, nothing; kw...)
+    end
+    return FixedSizeList{eltype(x), typeof(data)}(UInt8[], validity, data, len, meta)
+end
+
+function compress(Z::Meta.CompressionType, comp, x::FixedSizeList{T, A}) where {T, A}
+    len = length(x)
+    nc = nullcount(x)
+    validity = compress(Z, comp, x.validity)
+    buffers = [validity]
+    children = Compressed[]
+    if eltype(A) == UInt8
+        push!(buffers, compress(Z, comp, x.data))
+    else
+        push!(children, compress(Z, comp, x.data))
+    end
+    return Compressed{Z, typeof(x)}(x, buffers, len, nc, children)
+end
+
+function makenodesbuffers!(col::FixedSizeList{T, A}, fieldnodes, fieldbuffers, bufferoffset, alignment) where {T, A}
+    len = length(col)
+    nc = nullcount(col)
+    push!(fieldnodes, FieldNode(len, nc))
+    @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
+    # validity bitmap
+    blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
+    push!(fieldbuffers, Buffer(bufferoffset, blen))
+    @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
+    bufferoffset += blen
+    if eltype(A) === UInt8
+        blen = ArrowTypes.getsize(Base.nonmissingtype(T)) * len
+        push!(fieldbuffers, Buffer(bufferoffset, blen))
+        @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
+        bufferoffset += padding(blen, alignment)
+    else
+        bufferoffset = makenodesbuffers!(col.data, fieldnodes, fieldbuffers, bufferoffset, alignment)
+    end
+    return bufferoffset
+end
+
+function writebuffer(io, col::FixedSizeList{T, A}, alignment) where {T, A}
+    @debug 1 "writebuffer: col = $(typeof(col))"
+    @debug 2 col
+    writebitmap(io, col, alignment)
+    # write values array
+    if eltype(A) === UInt8
+        n = writearray(io, UInt8, col.data)
+        @debug 1 "writing array: col = $(typeof(col.data)), n = $n, padded = $(padding(n, alignment))"
+        writezeros(io, paddinglength(n, alignment))
+    else
+        writebuffer(io, col.data, alignment)
+    end
+    return
+end
diff --git a/julia/Arrow/src/arraytypes/list.jl b/julia/Arrow/src/arraytypes/list.jl
new file mode 100644
index 00000000000..c975bb38f87
--- /dev/null
+++ b/julia/Arrow/src/arraytypes/list.jl
@@ -0,0 +1,209 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+struct Offsets{T <: Union{Int32, Int64}} <: ArrowVector{Tuple{T, T}}
+    arrow::Vector{UInt8} # need to hold a reference to arrow memory blob
+    offsets::Vector{T}
+end
+
+Base.size(o::Offsets) = (length(o.offsets) - 1,)
+
+@propagate_inbounds function Base.getindex(o::Offsets, i::Integer)
+    @boundscheck checkbounds(o, i)
+    @inbounds lo = o.offsets[i] + 1
+    @inbounds hi = o.offsets[i + 1]
+    return lo, hi
+end
+
+"""
+    Arrow.List
+
+An `ArrowVector` where each element is a variable sized list of some kind, like an `AbstractVector` or `AbstractString`.
+"""
+struct List{T, O, A} <: ArrowVector{T}
+    arrow::Vector{UInt8} # need to hold a reference to arrow memory blob
+    validity::ValidityBitmap
+    offsets::Offsets{O}
+    data::A
+    ℓ::Int
+    metadata::Union{Nothing, Dict{String, String}}
+end
+
+Base.size(l::List) = (l.ℓ,)
+
+@propagate_inbounds function Base.getindex(l::List{T}, i::Integer) where {T}
+    @boundscheck checkbounds(l, i)
+    @inbounds lo, hi = l.offsets[i]
+    if ArrowTypes.isstringtype(T)
+        if Base.nonmissingtype(T) !== T
+            return l.validity[i] ? ArrowTypes.arrowconvert(T, unsafe_string(pointer(l.data, lo), hi - lo + 1)) : missing
+        else
+            return ArrowTypes.arrowconvert(T, unsafe_string(pointer(l.data, lo), hi - lo + 1))
+        end
+    elseif Base.nonmissingtype(T) !== T
+        return l.validity[i] ? ArrowTypes.arrowconvert(T, view(l.data, lo:hi)) : missing
+    else
+        return ArrowTypes.arrowconvert(T, view(l.data, lo:hi))
+    end
+end
+
+# @propagate_inbounds function Base.setindex!(l::List{T}, v, i::Integer) where {T}
+
+# end
+
+# an AbstractVector version of Iterators.flatten
+# code based on SentinelArrays.ChainedVector
+struct ToList{T, stringtype, A, I} <: AbstractVector{T}
+    data::Vector{A} # A is AbstractVector or AbstractString
+    inds::Vector{I}
+end
+
+function ToList(input; largelists::Bool=false)
+    AT = eltype(input)
+    ST = Base.nonmissingtype(AT)
+    stringtype = ArrowTypes.isstringtype(ST)
+    T = stringtype ? UInt8 : eltype(ST)
+    len = stringtype ? ncodeunits : length
+    data = AT[]
+    I = largelists ? Int64 : Int32
+    inds = I[0]
+    sizehint!(data, length(input))
+    sizehint!(inds, length(input))
+    totalsize = I(0)
+    for x in input
+        if x === missing
+            push!(data, missing)
+        else
+            push!(data, x)
+            totalsize += len(x)
+            if I === Int32 && totalsize > 2147483647
+                I = Int64
+                inds = convert(Vector{Int64}, inds)
+            end
+        end
+        push!(inds, totalsize)
+    end
+    return ToList{T, stringtype, AT, I}(data, inds)
+end
+
+Base.IndexStyle(::Type{<:ToList}) = Base.IndexLinear()
+Base.size(x::ToList) = (length(x.inds) == 0 ? 0 : x.inds[end],)
+
+function Base.pointer(A::ToList{UInt8}, i::Integer)
+    chunk = searchsortedfirst(A.inds, i)
+    return pointer(A.data[chunk - 1])
+end
+
+@inline function index(A::ToList, i::Integer)
+    chunk = searchsortedfirst(A.inds, i)
+    return chunk - 1, i - (@inbounds A.inds[chunk - 1])
+end
+
+Base.@propagate_inbounds function Base.getindex(A::ToList{T, stringtype}, i::Integer) where {T, stringtype}
+    @boundscheck checkbounds(A, i)
+    chunk, ix = index(A, i)
+    @inbounds x = A.data[chunk]
+    return @inbounds stringtype ? codeunits(x)[ix] : x[ix]
+end
+
+Base.@propagate_inbounds function Base.setindex!(A::ToList{T, stringtype}, v, i::Integer) where {T, stringtype}
+    @boundscheck checkbounds(A, i)
+    chunk, ix = index(A, i)
+    @inbounds x = A.data[chunk]
+    if stringtype
+        codeunits(x)[ix] = v
+    else
+        x[ix] = v
+    end
+    return v
+end
+
+# efficient iteration
+@inline function Base.iterate(A::ToList{T, stringtype}) where {T, stringtype}
+    length(A) == 0 && return nothing
+    i = 1
+    chunk = 2
+    chunk_i = 1
+    chunk_len = A.inds[chunk]
+    while i > chunk_len
+        chunk += 1
+        chunk_len = A.inds[chunk]
+    end
+    val = A.data[chunk - 1]
+    x = stringtype ? codeunits(val)[1] : val[1]
+    # find next valid index
+    i += 1
+    if i > chunk_len
+        while true
+            chunk += 1
+            chunk > length(A.inds) && break
+            chunk_len = A.inds[chunk]
+            i <= chunk_len && break
+        end
+    else
+        chunk_i += 1
+    end
+    return x, (i, chunk, chunk_i, chunk_len, length(A))
+end
+
+@inline function Base.iterate(A::ToList{T, stringtype}, (i, chunk, chunk_i, chunk_len, len)) where {T, stringtype}
+    i > len && return nothing
+    @inbounds val = A.data[chunk - 1]
+    @inbounds x = stringtype ? codeunits(val)[chunk_i] : val[chunk_i]
+    i += 1
+    if i > chunk_len
+        chunk_i = 1
+        while true
+            chunk += 1
+            chunk > length(A.inds) && break
+            @inbounds chunk_len = A.inds[chunk]
+            i <= chunk_len && break
+        end
+    else
+        chunk_i += 1
+    end
+    return x, (i, chunk, chunk_i, chunk_len, len)
+end
+
+arrowvector(::ListType, x::List, i, nl, fi, de, ded, meta; kw...) = x
+
+function arrowvector(::ListType, x, i, nl, fi, de, ded, meta; largelists::Bool=false, kw...)
+    len = length(x)
+    validity = ValidityBitmap(x)
+    flat = ToList(x; largelists=largelists)
+    offsets = Offsets(UInt8[], flat.inds)
+    if eltype(flat) == UInt8 # binary or utf8string
+        data = flat
+    else
+        data = arrowvector(flat, i, nl + 1, fi, de, ded, nothing; lareglists=largelists, kw...)
+    end
+    return List{eltype(x), eltype(flat.inds), typeof(data)}(UInt8[], validity, offsets, data, len, meta)
+end
+
+function compress(Z::Meta.CompressionType, comp, x::List{T, O, A}) where {T, O, A}
+    len = length(x)
+    nc = nullcount(x)
+    validity = compress(Z, comp, x.validity)
+    offsets = compress(Z, comp, x.offsets.offsets)
+    buffers = [validity, offsets]
+    children = Compressed[]
+    if eltype(A) == UInt8
+        push!(buffers, compress(Z, comp, x.data))
+    else
+        push!(children, compress(Z, comp, x.data))
+    end
+    return Compressed{Z, typeof(x)}(x, buffers, len, nc, children)
+end
diff --git a/julia/Arrow/src/arraytypes/map.jl b/julia/Arrow/src/arraytypes/map.jl
new file mode 100644
index 00000000000..856517206c5
--- /dev/null
+++ b/julia/Arrow/src/arraytypes/map.jl
@@ -0,0 +1,115 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+    Arrow.Map
+
+An `ArrowVector` where each element is a "map" of some kind, like a `Dict`.
+"""
+struct Map{T, O, A} <: ArrowVector{T}
+    validity::ValidityBitmap
+    offsets::Offsets{O}
+    data::A
+    ℓ::Int
+    metadata::Union{Nothing, Dict{String, String}}
+end
+
+Base.size(l::Map) = (l.ℓ,)
+
+@propagate_inbounds function Base.getindex(l::Map{T}, i::Integer) where {T}
+    @boundscheck checkbounds(l, i)
+    @inbounds lo, hi = l.offsets[i]
+    if Base.nonmissingtype(T) !== T
+        return l.validity[i] ? ArrowTypes.arrowconvert(T, Dict(x.key => x.value for x in view(l.data, lo:hi))) : missing
+    else
+        return ArrowTypes.arrowconvert(T, Dict(x.key => x.value for x in view(l.data, lo:hi)))
+    end
+end
+
+keyvalues(KT, ::Missing) = missing
+keyvalues(KT, x::AbstractDict) = [KT(k, v) for (k, v) in pairs(x)]
+
+arrowvector(::MapType, x::Map, i, nl, fi, de, ded, meta; kw...) = x
+
+function arrowvector(::MapType, x, i, nl, fi, de, ded, meta; largelists::Bool=false, kw...)
+    len = length(x)
+    validity = ValidityBitmap(x)
+    ET = eltype(x)
+    DT = Base.nonmissingtype(ET)
+    KT = KeyValue{keytype(DT), valtype(DT)}
+    VT = Vector{KT}
+    T = DT !== ET ? Union{Missing, VT} : VT
+    flat = ToList(T[keyvalues(KT, y) for y in x]; largelists=largelists)
+    offsets = Offsets(UInt8[], flat.inds)
+    data = arrowvector(flat, i, nl + 1, fi, de, ded, nothing; lareglists=largelists, kw...)
+    return Map{ET, eltype(flat.inds), typeof(data)}(validity, offsets, data, len, meta)
+end
+
+function compress(Z::Meta.CompressionType, comp, x::A) where {A <: Map}
+    len = length(x)
+    nc = nullcount(x)
+    validity = compress(Z, comp, x.validity)
+    offsets = compress(Z, comp, x.offsets.offsets)
+    buffers = [validity, offsets]
+    children = Compressed[]
+    push!(children, compress(Z, comp, x.data))
+    return Compressed{Z, A}(x, buffers, len, nc, children)
+end
+
+function makenodesbuffers!(col::Union{Map{T, O, A}, List{T, O, A}}, fieldnodes, fieldbuffers, bufferoffset, alignment) where {T, O, A}
+    len = length(col)
+    nc = nullcount(col)
+    push!(fieldnodes, FieldNode(len, nc))
+    @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
+    # validity bitmap
+    blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
+    push!(fieldbuffers, Buffer(bufferoffset, blen))
+    @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
+    # adjust buffer offset, make array buffer
+    bufferoffset += blen
+    blen = sizeof(O) * (len + 1)
+    push!(fieldbuffers, Buffer(bufferoffset, blen))
+    @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
+    bufferoffset += padding(blen, alignment)
+    if eltype(A) == UInt8
+        blen = length(col.data)
+        push!(fieldbuffers, Buffer(bufferoffset, blen))
+        @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
+        bufferoffset += padding(blen, alignment)
+    else
+        bufferoffset = makenodesbuffers!(col.data, fieldnodes, fieldbuffers, bufferoffset, alignment)
+    end
+    return bufferoffset
+end
+
+function writebuffer(io, col::Union{Map{T, O, A}, List{T, O, A}}, alignment) where {T, O, A}
+    @debug 1 "writebuffer: col = $(typeof(col))"
+    @debug 2 col
+    writebitmap(io, col, alignment)
+    # write offsets
+    n = writearray(io, O, col.offsets.offsets)
+    @debug 1 "writing array: col = $(typeof(col.offsets.offsets)), n = $n, padded = $(padding(n, alignment))"
+    writezeros(io, paddinglength(n, alignment))
+    # write values array
+    if eltype(A) == UInt8
+        n = writearray(io, UInt8, col.data)
+        @debug 1 "writing array: col = $(typeof(col.data)), n = $n, padded = $(padding(n, alignment))"
+        writezeros(io, paddinglength(n, alignment))
+    else
+        writebuffer(io, col.data, alignment)
+    end
+    return
+end
diff --git a/julia/Arrow/src/arraytypes/primitive.jl b/julia/Arrow/src/arraytypes/primitive.jl
new file mode 100644
index 00000000000..fc7c79ab6d1
--- /dev/null
+++ b/julia/Arrow/src/arraytypes/primitive.jl
@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+    Arrow.Primitive
+
+An `ArrowVector` where each element is a "fixed size" scalar of some kind, like an integer, float, decimal, or time type.
+"""
+struct Primitive{T, A} <: ArrowVector{T}
+    arrow::Vector{UInt8} # need to hold a reference to arrow memory blob
+    validity::ValidityBitmap
+    data::A
+    ℓ::Int64
+    metadata::Union{Nothing, Dict{String, String}}
+end
+
+Primitive(::Type{T}, b::Vector{UInt8}, v::ValidityBitmap, data::A, l, meta) where {T, A} =
+    Primitive{T, A}(b, v, data, l, meta)
+
+Base.size(p::Primitive) = (p.ℓ,)
+
+function Base.copy(p::Primitive)
+    if nullcount(p) == 0
+        return copy(p.data)
+    else
+        return convert(Array, p)
+    end
+end
+
+@propagate_inbounds function Base.getindex(p::Primitive{T}, i::Integer) where {T}
+    @boundscheck checkbounds(p, i)
+    if T >: Missing
+        return @inbounds (p.validity[i] ? ArrowTypes.arrowconvert(T, p.data[i]) : missing)
+    else
+        return @inbounds ArrowTypes.arrowconvert(T, p.data[i])
+    end
+end
+
+@propagate_inbounds function Base.setindex!(p::Primitive{T}, v, i::Integer) where {T}
+    @boundscheck checkbounds(p, i)
+    if T >: Missing
+        if v === missing
+            @inbounds p.validity[i] = false
+        else
+            @inbounds p.data[i] = convert(Base.nonmissingtype(T), v)
+        end
+    else
+        @inbounds p.data[i] = convert(Base.nonmissingtype(T), v)
+    end
+    return v
+end
+
+arrowvector(::PrimitiveType, x::Primitive, i, nl, fi, de, ded, meta; kw...) = x
+
+function arrowvector(::PrimitiveType, x, i, nl, fi, de, ded, meta; kw...)
+    validity = ValidityBitmap(x)
+    return Primitive(eltype(x), UInt8[], validity, x, length(x), meta)
+end
+
+function compress(Z::Meta.CompressionType, comp, p::P) where {P <: Primitive}
+    len = length(p)
+    nc = nullcount(p)
+    validity = compress(Z, comp, p.validity)
+    data = compress(Z, comp, p.data)
+    return Compressed{Z, P}(p, [validity, data], len, nc, Compressed[])
+end
+
+function makenodesbuffers!(col::Primitive{T}, fieldnodes, fieldbuffers, bufferoffset, alignment) where {T}
+    len = length(col)
+    nc = nullcount(col)
+    push!(fieldnodes, FieldNode(len, nc))
+    @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
+    # validity bitmap
+    blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
+    push!(fieldbuffers, Buffer(bufferoffset, blen))
+    @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
+    # adjust buffer offset, make primitive array buffer
+    bufferoffset += blen
+    blen = len * sizeof(Base.nonmissingtype(T))
+    push!(fieldbuffers, Buffer(bufferoffset, blen))
+    @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
+    return bufferoffset + padding(blen, alignment)
+end
+
+function writebuffer(io, col::Primitive{T}, alignment) where {T}
+    @debug 1 "writebuffer: col = $(typeof(col))"
+    @debug 2 col
+    writebitmap(io, col, alignment)
+    n = writearray(io, Base.nonmissingtype(T), col.data)
+    @debug 1 "writing array: col = $(typeof(col.data)), n = $n, padded = $(padding(n, alignment))"
+    writezeros(io, paddinglength(n, alignment))
+    return
+end
diff --git a/julia/Arrow/src/arraytypes/struct.jl b/julia/Arrow/src/arraytypes/struct.jl
new file mode 100644
index 00000000000..96f67583639
--- /dev/null
+++ b/julia/Arrow/src/arraytypes/struct.jl
@@ -0,0 +1,125 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+    Arrow.Struct
+
+An `ArrowVector` where each element is a "struct" of some kind with ordered, named fields, like a `NamedTuple{names, types}` or regular julia `struct`.
+"""
+struct Struct{T, S} <: ArrowVector{T}
+    validity::ValidityBitmap
+    data::S # Tuple of ArrowVector
+    ℓ::Int
+    metadata::Union{Nothing, Dict{String, String}}
+end
+
+Base.size(s::Struct) = (s.ℓ,)
+
+@propagate_inbounds function Base.getindex(s::Struct{T}, i::Integer) where {T}
+    @boundscheck checkbounds(s, i)
+    NT = Base.nonmissingtype(T)
+    if ArrowTypes.structtype(NT) === ArrowTypes.NAMEDTUPLE
+        if NT !== T
+            return s.validity[i] ? NT(ntuple(j->s.data[j][i], fieldcount(NT))) : missing
+        else
+            return NT(ntuple(j->s.data[j][i], fieldcount(NT)))
+        end
+    elseif ArrowTypes.structtype(NT) === ArrowTypes.STRUCT
+        if NT !== T
+            return s.validity[i] ? NT(ntuple(j->s.data[j][i], fieldcount(NT))...) : missing
+        else
+            return NT(ntuple(j->s.data[j][i], fieldcount(NT))...)
+        end
+    end
+end
+
+@propagate_inbounds function Base.setindex!(s::Struct{T}, v::T, i::Integer) where {T}
+    @boundscheck checkbounds(s, i)
+    if v === missing
+        @inbounds s.validity[i] = false
+    else
+        NT = Base.nonmissingtype(T)
+        N = fieldcount(NT)
+        foreach(1:N) do j
+            @inbounds s.data[j][i] = getfield(v, j)
+        end
+    end
+    return v
+end
+
+struct ToStruct{T, i, A} <: AbstractVector{T}
+    data::A # eltype is NamedTuple or some struct
+end
+
+ToStruct(x::A, j::Integer) where {A} = ToStruct{fieldtype(Base.nonmissingtype(eltype(A)), j), j, A}(x)
+
+Base.IndexStyle(::Type{<:ToStruct}) = Base.IndexLinear()
+Base.size(x::ToStruct) = (length(x.data),)
+
+Base.@propagate_inbounds function Base.getindex(A::ToStruct{T, j}, i::Integer) where {T, j}
+    @boundscheck checkbounds(A, i)
+    @inbounds x = A.data[i]
+    return x === missing ? ArrowTypes.default(T) : getfield(x, j)
+end
+
+arrowvector(::StructType, x::Struct, i, nl, fi, de, ded, meta; kw...) = x
+
+function arrowvector(::StructType, x, i, nl, fi, de, ded, meta; kw...)
+    len = length(x)
+    validity = ValidityBitmap(x)
+    T = Base.nonmissingtype(eltype(x))
+    data = Tuple(arrowvector(ToStruct(x, j), i, nl + 1, j, de, ded, nothing; kw...) for j = 1:fieldcount(T))
+    return Struct{eltype(x), typeof(data)}(validity, data, len, meta)
+end
+
+function compress(Z::Meta.CompressionType, comp, x::A) where {A <: Struct}
+    len = length(x)
+    nc = nullcount(x)
+    validity = compress(Z, comp, x.validity)
+    buffers = [validity]
+    children = Compressed[]
+    for y in x.data
+        push!(children, compress(Z, comp, y))
+    end
+    return Compressed{Z, A}(x, buffers, len, nc, children)
+end
+
+function makenodesbuffers!(col::Struct{T}, fieldnodes, fieldbuffers, bufferoffset, alignment) where {T}
+    len = length(col)
+    nc = nullcount(col)
+    push!(fieldnodes, FieldNode(len, nc))
+    @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
+    # validity bitmap
+    blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
+    push!(fieldbuffers, Buffer(bufferoffset, blen))
+    @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
+    bufferoffset += blen
+    for child in col.data
+        bufferoffset = makenodesbuffers!(child, fieldnodes, fieldbuffers, bufferoffset, alignment)
+    end
+    return bufferoffset
+end
+
+function writebuffer(io, col::Struct, alignment)
+    @debug 1 "writebuffer: col = $(typeof(col))"
+    @debug 2 col
+    writebitmap(io, col, alignment)
+    # write values arrays
+    for child in col.data
+        writebuffer(io, child, alignment)
+    end
+    return
+end
diff --git a/julia/Arrow/src/arraytypes/unions.jl b/julia/Arrow/src/arraytypes/unions.jl
new file mode 100644
index 00000000000..64d86305f07
--- /dev/null
+++ b/julia/Arrow/src/arraytypes/unions.jl
@@ -0,0 +1,279 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Union arrays
+# need a custom representation of Union types since arrow unions
+# are ordered, and possibly indirected via separate typeIds array
+# here, T is Meta.UnionMode.Dense or Meta.UnionMode.Sparse,
+# typeIds is a NTuple{N, Int32}, and U is a Tuple{...} of the
+# unioned types
+struct UnionT{T, typeIds, U}
+end
+
+unionmode(::Type{UnionT{T, typeIds, U}}) where {T, typeIds, U} = T
+typeids(::Type{UnionT{T, typeIds, U}}) where {T, typeIds, U} = typeIds
+Base.eltype(::Type{UnionT{T, typeIds, U}}) where {T, typeIds, U} = U
+
+ArrowTypes.ArrowType(::Type{<:UnionT}) = ArrowTypes.UnionType()
+
+# iterate a Julia Union{...} type, producing an array of unioned types
+function eachunion(U::Union, elems=nothing)
+    if elems === nothing
+        return eachunion(U.b, Type[U.a])
+    else
+        push!(elems, U.a)
+        return eachunion(U.b, elems)
+    end
+end
+
+function eachunion(T, elems)
+    push!(elems, T)
+    return elems
+end
+
+# produce typeIds, offsets, data tuple for DenseUnion
+isatypeid(x::T, ::Type{types}) where {T, types} = isatypeid(x, fieldtype(types, 1), types, 1)
+isatypeid(x::T, ::Type{S}, ::Type{types}, i) where {T, S, types} = x isa S ? i : isatypeid(x, fieldtype(types, i + 1), types, i + 1)
+
+"""
+    Arrow.DenseUnion
+
+An `ArrowVector` where the type of each element is one of a fixed set of types, meaning its eltype is like a julia `Union{type1, type2, ...}`.
+An `Arrow.DenseUnion`, in comparison to `Arrow.SparseUnion`, stores elements in a set of arrays, one array per possible type, and an "offsets"
+array, where each offset element is the index into one of the typed arrays. This allows a sort of "compression", where no extra space is
+used/allocated to store all the elements.
+"""
+struct DenseUnion{T, S} <: ArrowVector{T}
+    arrow::Vector{UInt8} # need to hold a reference to arrow memory blob
+    arrow2::Vector{UInt8} # if arrow blob is compressed, need a 2nd reference for uncompressed offsets bytes
+    typeIds::Vector{UInt8}
+    offsets::Vector{Int32}
+    data::S # Tuple of ArrowVector
+    metadata::Union{Nothing, Dict{String, String}}
+end
+
+Base.size(s::DenseUnion) = size(s.typeIds)
+nullcount(x::DenseUnion) = 0 # DenseUnion has no validity bitmap; only children do
+
+@propagate_inbounds function Base.getindex(s::DenseUnion{T}, i::Integer) where {T}
+    @boundscheck checkbounds(s, i)
+    @inbounds typeId = s.typeIds[i]
+    @inbounds off = s.offsets[i]
+    @inbounds x = s.data[typeId + 1][off + 1]
+    return x
+end
+
+@propagate_inbounds function Base.setindex!(s::DenseUnion{UnionT{T, typeIds, U}}, v, i::Integer) where {T, typeIds, U}
+    @boundscheck checkbounds(s, i)
+    @inbounds typeId = s.typeIds[i]
+    typeids = typeIds === nothing ? (0:(fieldcount(U) - 1)) : typeIds
+    vtypeId = Int8(typeids[isatypeid(v, U)])
+    if typeId == vtypeId
+        @inbounds off = s.offsets[i]
+        @inbounds s.data[typeId +1][off + 1] = v
+    else
+        throw(ArgumentError("type of item to set $(typeof(v)) must match existing item $(fieldtype(U, typeid))"))
+    end
+    return v
+end
+
+# convenience wrappers for signaling that an array shoudld be written
+# as with dense/sparse union arrow buffers
+struct DenseUnionVector{T, U} <: AbstractVector{UnionT{Meta.UnionMode.Dense, nothing, U}}
+    itr::T
+end
+
+DenseUnionVector(x::T) where {T} = DenseUnionVector{T, Tuple{eachunion(eltype(x))...}}(x)
+Base.IndexStyle(::Type{<:DenseUnionVector}) = Base.IndexLinear()
+Base.size(x::DenseUnionVector) = (length(x.itr),)
+Base.iterate(x::DenseUnionVector, st...) = iterate(x.itr, st...)
+Base.getindex(x::DenseUnionVector, i::Int) = getindex(x.itr, i)
+
+function todense(::Type{UnionT{T, typeIds, U}}, x) where {T, typeIds, U}
+    typeids = typeIds === nothing ? (0:(fieldcount(U) - 1)) : typeIds
+    len = length(x)
+    types = Vector{UInt8}(undef, len)
+    offsets = Vector{Int32}(undef, len)
+    data = Tuple(Vector{i == 1 ? Union{Missing, fieldtype(U, i)} : fieldtype(U, i)}(undef, 0) for i = 1:fieldcount(U))
+    for (i, y) in enumerate(x)
+        typeid = y === missing ? 0x00 : UInt8(typeids[isatypeid(y, U)])
+        @inbounds types[i] = typeid
+        @inbounds offsets[i] = length(data[typeid + 1])
+        push!(data[typeid + 1], y)
+    end
+    return types, offsets, data
+end
+
+struct SparseUnionVector{T, U} <: AbstractVector{UnionT{Meta.UnionMode.Sparse, nothing, U}}
+    itr::T
+end
+
+SparseUnionVector(x::T) where {T} = SparseUnionVector{T, Tuple{eachunion(eltype(x))...}}(x)
+Base.IndexStyle(::Type{<:SparseUnionVector}) = Base.IndexLinear()
+Base.size(x::SparseUnionVector) = (length(x.itr),)
+Base.iterate(x::SparseUnionVector, st...) = iterate(x.itr, st...)
+Base.getindex(x::SparseUnionVector, i::Int) = getindex(x.itr, i)
+
+# sparse union child array producer
+# for sparse unions, we split the parent array into
+# N children arrays, each having the same length as the parent
+# but with one child array per unioned type; each child
+# should include the elements from parent of its type
+# and other elements can be missing/default
+function sparsetypeids(::Type{UnionT{T, typeIds, U}}, x) where {T, typeIds, U}
+    typeids = typeIds === nothing ? (0:(fieldcount(U) - 1)) : typeIds
+    len = length(x)
+    types = Vector{UInt8}(undef, len)
+    for (i, y) in enumerate(x)
+        typeid = y === missing ? 0x00 : UInt8(typeids[isatypeid(y, U)])
+        @inbounds types[i] = typeid
+    end
+    return types
+end
+
+struct ToSparseUnion{T, A} <: AbstractVector{T}
+    data::A
+end
+
+ToSparseUnion(::Type{T}, data::A) where {T, A} = ToSparseUnion{T, A}(data)
+
+Base.IndexStyle(::Type{<:ToSparseUnion}) = Base.IndexLinear()
+Base.size(x::ToSparseUnion) = (length(x.data),)
+
+Base.@propagate_inbounds function Base.getindex(A::ToSparseUnion{T}, i::Integer) where {T}
+    @boundscheck checkbounds(A, i)
+    @inbounds x = A.data[i]
+    return @inbounds x isa T ? x : ArrowTypes.default(T)
+end
+
+function compress(Z::Meta.CompressionType, comp, x::A) where {A <: DenseUnion}
+    len = length(x)
+    nc = nullcount(x)
+    typeIds = compress(Z, comp, x.typeIds)
+    offsets = compress(Z, comp, x.offsets)
+    buffers = [typeIds, offsets]
+    children = Compressed[]
+    for y in x.data
+        push!(children, compress(Z, comp, y))
+    end
+    return Compressed{Z, A}(x, buffers, len, nc, children)
+end
+
+"""
+    Arrow.SparseUnion
+
+An `ArrowVector` where the type of each element is one of a fixed set of types, meaning its eltype is like a julia `Union{type1, type2, ...}`.
+An `Arrow.SparseUnion`, in comparison to `Arrow.DenseUnion`, stores elements in a set of arrays, one array per possible type, and each typed
+array has the same length as the full array. This ends up with "wasted" space, since only one slot among the typed arrays is valid per full
+array element, but can allow for certain optimizations when each typed array has the same length.
+"""
+struct SparseUnion{T, S} <: ArrowVector{T}
+    arrow::Vector{UInt8} # need to hold a reference to arrow memory blob
+    typeIds::Vector{UInt8}
+    data::S # Tuple of ArrowVector
+    metadata::Union{Nothing, Dict{String, String}}
+end
+
+Base.size(s::SparseUnion) = size(s.typeIds)
+nullcount(x::SparseUnion) = 0
+
+@propagate_inbounds function Base.getindex(s::SparseUnion{T}, i::Integer) where {T}
+    @boundscheck checkbounds(s, i)
+    @inbounds typeId = s.typeIds[i]
+    @inbounds x = s.data[typeId + 1][i]
+    return x
+end
+
+@propagate_inbounds function Base.setindex!(s::SparseUnion{UnionT{T, typeIds, U}}, v, i::Integer) where {T, typeIds, U}
+    @boundscheck checkbounds(s, i)
+    typeids = typeIds === nothing ? (0:(fieldcount(U) - 1)) : typeIds
+    vtypeId = Int8(typeids[isatypeid(v, U)])
+    @inbounds s.typeIds[i] = vtypeId
+    @inbounds s.data[vtypeId + 1][i] = v
+    return v
+end
+
+arrowvector(U::Union, x, i, nl, fi, de, ded, meta; denseunions::Bool=true, kw...) =
+    arrowvector(denseunions ? DenseUnionVector(x) : SparseUnionVector(x), i, nl, fi, de, ded, meta; denseunions=denseunions, kw...)
+
+arrowvector(::UnionType, x::Union{DenseUnion, SparseUnion}, i, nl, fi, de, ded, meta; kw...) = x
+
+function arrowvector(::UnionType, x, i, nl, fi, de, ded, meta; kw...)
+    UT = eltype(x)
+    if unionmode(UT) == Meta.UnionMode.Dense
+        x = x isa DenseUnionVector ? x.itr : x
+        typeids, offsets, data = todense(UT, x)
+        data2 = map(y -> arrowvector(y[2], i, nl + 1, y[1], de, ded, nothing; kw...), enumerate(data))
+        return DenseUnion{UT, typeof(data2)}(UInt8[], UInt8[], typeids, offsets, data2, meta)
+    else
+        x = x isa SparseUnionVector ? x.itr : x
+        typeids = sparsetypeids(UT, x)
+        data3 = Tuple(arrowvector(ToSparseUnion(fieldtype(eltype(UT), j), x), i, nl + 1, j, de, ded, nothing; kw...) for j = 1:fieldcount(eltype(UT)))
+        return SparseUnion{UT, typeof(data3)}(UInt8[], typeids, data3, meta)
+    end
+end
+
+function compress(Z::Meta.CompressionType, comp, x::A) where {A <: SparseUnion}
+    len = length(x)
+    nc = nullcount(x)
+    typeIds = compress(Z, comp, x.typeIds)
+    buffers = [typeIds]
+    children = Compressed[]
+    for y in x.data
+        push!(children, compress(Z, comp, y))
+    end
+    return Compressed{Z, A}(x, buffers, len, nc, children)
+end
+
+function makenodesbuffers!(col::Union{DenseUnion, SparseUnion}, fieldnodes, fieldbuffers, bufferoffset, alignment)
+    len = length(col)
+    nc = nullcount(col)
+    push!(fieldnodes, FieldNode(len, nc))
+    @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
+    # typeIds buffer
+    push!(fieldbuffers, Buffer(bufferoffset, len))
+    @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
+    bufferoffset += padding(len, alignment)
+    if col isa DenseUnion
+        # offsets buffer
+        blen = sizeof(Int32) * len
+        push!(fieldbuffers, Buffer(bufferoffset, blen))
+        @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
+        bufferoffset += padding(blen, alignment)
+    end
+    for child in col.data
+        bufferoffset = makenodesbuffers!(child, fieldnodes, fieldbuffers, bufferoffset, alignment)
+    end
+    return bufferoffset
+end
+
+function writebuffer(io, col::Union{DenseUnion, SparseUnion}, alignment)
+    @debug 1 "writebuffer: col = $(typeof(col))"
+    @debug 2 col
+    # typeIds buffer
+    n = writearray(io, UInt8, col.typeIds)
+    @debug 1 "writing array: col = $(typeof(col.typeIds)), n = $n, padded = $(padding(n, alignment))"
+    writezeros(io, paddinglength(n, alignment))
+    if col isa DenseUnion
+        n = writearray(io, Int32, col.offsets)
+        @debug 1 "writing array: col = $(typeof(col.offsets)), n = $n, padded = $(padding(n, alignment))"
+        writezeros(io, paddinglength(n, alignment))
+    end
+    for child in col.data
+        writebuffer(io, child, alignment)
+    end
+    return
+end
diff --git a/julia/Arrow/src/arrowtypes.jl b/julia/Arrow/src/arrowtypes.jl
new file mode 100644
index 00000000000..2e9bf4c6830
--- /dev/null
+++ b/julia/Arrow/src/arrowtypes.jl
@@ -0,0 +1,153 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+The ArrowTypes module provides the [`ArrowTypes.Arrowtype`](@ref) interface trait that objects can define
+in order to signal how they should be serialized in the arrow format.
+"""
+module ArrowTypes
+
+export ArrowType, NullType, PrimitiveType, BoolType, ListType, FixedSizeListType, MapType, StructType, UnionType, DictEncodedType
+
+abstract type ArrowType end
+
+ArrowType(x::T) where {T} = ArrowType(T)
+ArrowType(::Type{T}) where {T} = isprimitivetype(T) ? PrimitiveType() : StructType()
+
+function arrowconvert end
+
+arrowconvert(T, x) = convert(T, x)
+arrowconvert(::Type{Union{T, Missing}}, x) where {T} = arrowconvert(T, x)
+arrowconvert(::Type{Union{T, Missing}}, ::Missing) where {T} = missing
+
+struct NullType <: ArrowType end
+
+ArrowType(::Type{Missing}) = NullType()
+
+struct PrimitiveType <: ArrowType end
+
+ArrowType(::Type{<:Integer}) = PrimitiveType()
+ArrowType(::Type{<:AbstractFloat}) = PrimitiveType()
+
+struct BoolType <: ArrowType end
+ArrowType(::Type{Bool}) = BoolType()
+
+struct ListType <: ArrowType end
+
+# isstringtype MUST BE UTF8 (other codeunit sizes not supported; arrow encoding for strings is specifically UTF8)
+isstringtype(T) = false
+isstringtype(::Type{Union{T, Missing}}) where {T} = isstringtype(T)
+
+ArrowType(::Type{<:AbstractString}) = ListType()
+isstringtype(::Type{<:AbstractString}) = true
+
+ArrowType(::Type{Symbol}) = ListType()
+isstringtype(::Type{Symbol}) = true
+arrowconvert(::Type{Symbol}, x::String) = Symbol(x)
+arrowconvert(::Type{String}, x::Symbol) = String(x)
+
+ArrowType(::Type{<:AbstractArray}) = ListType()
+
+struct FixedSizeListType <: ArrowType end
+
+ArrowType(::Type{NTuple{N, T}}) where {N, T} = FixedSizeListType()
+gettype(::Type{NTuple{N, T}}) where {N, T} = T
+getsize(::Type{NTuple{N, T}}) where {N, T} = N
+
+struct StructType <: ArrowType end
+
+ArrowType(::Type{<:NamedTuple}) = StructType()
+
+@enum STRUCT_TYPES NAMEDTUPLE STRUCT # KEYWORDARGS
+
+structtype(::Type{NamedTuple{N, T}}) where {N, T} = NAMEDTUPLE
+structtype(::Type{T}) where {T} = STRUCT
+
+# must implement keytype, valtype
+struct MapType <: ArrowType end
+
+ArrowType(::Type{<:AbstractDict}) = MapType()
+
+struct UnionType <: ArrowType end
+
+ArrowType(::Union) = UnionType()
+
+struct DictEncodedType <: ArrowType end
+
+"""
+There are a couple places when writing arrow buffers where
+we need to write a "dummy" value; it doesn't really matter
+what we write, but we need to write something of a specific
+type. So each supported writing type needs to define `default`.
+"""
+function default end
+
+default(T) = zero(T)
+default(::Type{Symbol}) = Symbol()
+default(::Type{Char}) = '\0'
+default(::Type{<:AbstractString}) = ""
+default(::Type{Union{T, Missing}}) where {T} = default(T)
+
+function default(::Type{A}) where {A <: AbstractVector{T}} where {T}
+    a = similar(A, 1)
+    a[1] = default(T)
+    return a
+end
+
+default(::Type{NTuple{N, T}}) where {N, T} = ntuple(i -> default(T), N)
+default(::Type{T}) where {T <: Tuple} = Tuple(default(fieldtype(T, i)) for i = 1:fieldcount(T))
+default(::Type{Dict{K, V}}) where {K, V} = Dict{K, V}()
+default(::Type{NamedTuple{names, types}}) where {names, types} = NamedTuple{names}(Tuple(default(fieldtype(types, i)) for i = 1:length(names)))
+
+const JULIA_TO_ARROW_TYPE_MAPPING = Dict{Type, Tuple{String, Type}}(
+    Char => ("JuliaLang.Char", UInt32),
+    Symbol => ("JuliaLang.Symbol", String),
+)
+
+istyperegistered(::Type{T}) where {T} = haskey(JULIA_TO_ARROW_TYPE_MAPPING, T)
+
+function getarrowtype!(meta, ::Type{T}) where {T}
+    arrowname, arrowtype = JULIA_TO_ARROW_TYPE_MAPPING[T]
+    meta["ARROW:extension:name"] = arrowname
+    meta["ARROW:extension:metadata"] = ""
+    return arrowtype
+end
+
+const ARROW_TO_JULIA_TYPE_MAPPING = Dict{String, Tuple{Type, Type}}(
+    "JuliaLang.Char" => (Char, UInt32),
+    "JuliaLang.Symbol" => (Symbol, String),
+)
+
+function extensiontype(meta)
+    if haskey(meta, "ARROW:extension:name")
+        typename = meta["ARROW:extension:name"]
+        if haskey(ARROW_TO_JULIA_TYPE_MAPPING, typename)
+            return ARROW_TO_JULIA_TYPE_MAPPING[typename][1]
+        else
+            @warn "unsupported ARROW:extension:name type: \"$typename\""
+        end
+    end
+    return nothing
+end
+
+function registertype!(juliatype::Type, arrowtype::Type, arrowname::String=string("JuliaLang.", string(juliatype)))
+    # TODO: validate that juliatype isn't already default arrow type
+    JULIA_TO_ARROW_TYPE_MAPPING[juliatype] = (arrowname, arrowtype)
+    ARROW_TO_JULIA_TYPE_MAPPING[arrowname] = (juliatype, arrowtype)
+    return
+end
+
+end # module ArrowTypes
\ No newline at end of file
diff --git a/julia/Arrow/src/eltypes.jl b/julia/Arrow/src/eltypes.jl
new file mode 100644
index 00000000000..fbf26ab23da
--- /dev/null
+++ b/julia/Arrow/src/eltypes.jl
@@ -0,0 +1,408 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Given a flatbuffers metadata type definition (a Field instance from Schema.fbs),
+translate to the appropriate Julia storage eltype
+"""
+function juliaeltype end
+
+finaljuliatype(T) = T
+finaljuliatype(::Type{Missing}) = Missing
+finaljuliatype(::Type{Union{T, Missing}}) where {T} = Union{Missing, finaljuliatype(T)}
+
+"""
+Given a FlatBuffers.Builder and a Julia column or column eltype,
+Write the field.type flatbuffer definition of the eltype
+"""
+function arrowtype end
+
+arrowtype(b, col::AbstractVector{T}) where {T} = arrowtype(b, maybemissing(T))
+arrowtype(b, col::DictEncoded) = arrowtype(b, col.encoding.data)
+arrowtype(b, col::Compressed) = arrowtype(b, col.data)
+
+function juliaeltype(f::Meta.Field, ::Nothing, convert::Bool)
+    T = juliaeltype(f, convert)
+    return convert ? finaljuliatype(T) : T
+end
+
+function juliaeltype(f::Meta.Field, meta::Dict{String, String}, convert::Bool)
+    TT = juliaeltype(f, convert)
+    !convert && return TT
+    T = finaljuliatype(TT)
+    TTT = ArrowTypes.extensiontype(meta)
+    return something(TTT, T)
+end
+
+function juliaeltype(f::Meta.Field, convert::Bool)
+    T = juliaeltype(f, f.type, convert)
+    return (f.nullable ? Union{T, Missing} : T)
+end
+
+juliaeltype(f::Meta.Field, ::Meta.Null, convert) = Missing
+
+function arrowtype(b, ::Type{Missing})
+    Meta.nullStart(b)
+    return Meta.Null, Meta.nullEnd(b), nothing
+end
+
+function juliaeltype(f::Meta.Field, int::Meta.Int, convert)
+    if int.is_signed
+        if int.bitWidth == 8
+            Int8
+        elseif int.bitWidth == 16
+            Int16
+        elseif int.bitWidth == 32
+            Int32
+        elseif int.bitWidth == 64
+            Int64
+        elseif int.bitWidth == 128
+            Int128
+        else
+            error("$int is not valid arrow type metadata")
+        end
+    else
+        if int.bitWidth == 8
+            UInt8
+        elseif int.bitWidth == 16
+            UInt16
+        elseif int.bitWidth == 32
+            UInt32
+        elseif int.bitWidth == 64
+            UInt64
+        elseif int.bitWidth == 128
+            UInt128
+        else
+            error("$int is not valid arrow type metadata")
+        end
+    end
+end
+
+function arrowtype(b, ::Type{T}) where {T <: Integer}
+    Meta.intStart(b)
+    Meta.intAddBitWidth(b, Int32(8 * sizeof(T)))
+    Meta.intAddIsSigned(b, T <: Signed)
+    return Meta.Int, Meta.intEnd(b), nothing
+end
+
+# primitive types
+function juliaeltype(f::Meta.Field, fp::Meta.FloatingPoint, convert)
+    if fp.precision == Meta.Precision.HALF
+        Float16
+    elseif fp.precision == Meta.Precision.SINGLE
+        Float32
+    elseif fp.precision == Meta.Precision.DOUBLE
+        Float64
+    end
+end
+
+function arrowtype(b, ::Type{T}) where {T <: AbstractFloat}
+    Meta.floatingPointStart(b)
+    Meta.floatingPointAddPrecision(b, T === Float16 ? Meta.Precision.HALF : T === Float32 ? Meta.Precision.SINGLE : Meta.Precision.DOUBLE)
+    return Meta.FloatingPoint, Meta.floatingPointEnd(b), nothing
+end
+
+juliaeltype(f::Meta.Field, b::Union{Meta.Utf8, Meta.LargeUtf8}, convert) = String
+
+datasizeof(x) = sizeof(x)
+datasizeof(x::AbstractVector) = sum(datasizeof, x)
+
+juliaeltype(f::Meta.Field, b::Union{Meta.Binary, Meta.LargeBinary}, convert) = Vector{UInt8}
+
+juliaeltype(f::Meta.Field, x::Meta.FixedSizeBinary, convert) = NTuple{Int(x.byteWidth), UInt8}
+
+# arggh!
+Base.write(io::IO, x::NTuple{N, T}) where {N, T} = sum(y -> Base.write(io, y), x)
+
+juliaeltype(f::Meta.Field, x::Meta.Bool, convert) = Bool
+
+function arrowtype(b, ::Type{Bool})
+    Meta.boolStart(b)
+    return Meta.Bool, Meta.boolEnd(b), nothing
+end
+
+struct Decimal{P, S}
+    value::Int128
+end
+
+Base.zero(::Type{Decimal{P, S}}) where {P, S} = Decimal{P, S}(Int128(0))
+==(a::Decimal{P, S}, b::Decimal{P, S}) where {P, S} = ==(a.value, b.value)
+Base.isequal(a::Decimal{P, S}, b::Decimal{P, S}) where {P, S} = isequal(a.value, b.value)
+
+function juliaeltype(f::Meta.Field, x::Meta.Decimal, convert)
+    return Decimal{x.precision, x.scale}
+end
+
+ArrowTypes.ArrowType(::Type{<:Decimal}) = PrimitiveType()
+
+function arrowtype(b, ::Type{Decimal{P, S}}) where {P, S}
+    Meta.decimalStart(b)
+    Meta.decimalAddPrecision(b, Int32(P))
+    Meta.decimalAddScale(b, Int32(S))
+    return Meta.Decimal, Meta.decimalEnd(b), nothing
+end
+
+Base.write(io::IO, x::Decimal) = Base.write(io, x.value)
+
+abstract type ArrowTimeType end
+Base.write(io::IO, x::ArrowTimeType) = Base.write(io, x.x)
+ArrowTypes.ArrowType(::Type{<:ArrowTimeType}) = PrimitiveType()
+
+struct Date{U, T} <: ArrowTimeType
+    x::T
+end
+
+Base.zero(::Type{Date{U, T}}) where {U, T} = Date{U, T}(T(0))
+storagetype(::Type{Date{U, T}}) where {U, T} = T
+bitwidth(x::Meta.DateUnit) = x == Meta.DateUnit.DAY ? Int32 : Int64
+Date{Meta.DateUnit.DAY}(days) = Date{Meta.DateUnit.DAY, Int32}(Int32(days))
+Date{Meta.DateUnit.MILLISECOND}(ms) = Date{Meta.DateUnit.MILLISECOND, Int64}(Int64(ms))
+const DATE = Date{Meta.DateUnit.DAY, Int32}
+const DATETIME = Date{Meta.DateUnit.MILLISECOND, Int64}
+
+juliaeltype(f::Meta.Field, x::Meta.Date, convert) = Date{x.unit, bitwidth(x.unit)}
+finaljuliatype(::Type{Date{Meta.DateUnit.DAY, Int32}}) = Dates.Date
+Base.convert(::Type{Dates.Date}, x::Date{Meta.DateUnit.DAY, Int32}) = Dates.Date(Dates.UTD(Int64(x.x + UNIX_EPOCH_DATE)))
+finaljuliatype(::Type{Date{Meta.DateUnit.MILLISECOND, Int64}}) = Dates.DateTime
+Base.convert(::Type{Dates.DateTime}, x::Date{Meta.DateUnit.MILLISECOND, Int64}) = Dates.DateTime(Dates.UTM(Int64(x.x + UNIX_EPOCH_DATETIME)))
+
+function arrowtype(b, ::Type{Date{U, T}}) where {U, T}
+    Meta.dateStart(b)
+    Meta.dateAddUnit(b, U)
+    return Meta.Date, Meta.dateEnd(b), nothing
+end
+
+const UNIX_EPOCH_DATE = Dates.value(Dates.Date(1970))
+Base.convert(::Type{Date{Meta.DateUnit.DAY, Int32}}, x::Dates.Date) = Date{Meta.DateUnit.DAY, Int32}(Int32(Dates.value(x) - UNIX_EPOCH_DATE))
+
+const UNIX_EPOCH_DATETIME = Dates.value(Dates.DateTime(1970))
+Base.convert(::Type{Date{Meta.DateUnit.MILLISECOND, Int64}}, x::Dates.DateTime) = Date{Meta.DateUnit.MILLISECOND, Int64}(Int64(Dates.value(x) - UNIX_EPOCH_DATETIME))
+
+struct Time{U, T} <: ArrowTimeType
+    x::T
+end
+
+Base.zero(::Type{Time{U, T}}) where {U, T} = Time{U, T}(T(0))
+const TIME = Time{Meta.TimeUnit.NANOSECOND, Int64}
+
+bitwidth(x::Meta.TimeUnit) = x == Meta.TimeUnit.SECOND || x == Meta.TimeUnit.MILLISECOND ? Int32 : Int64
+Time{U}(x) where {U <: Meta.TimeUnit} = Time{U, bitwidth(U)}(bitwidth(U)(x))
+storagetype(::Type{Time{U, T}}) where {U, T} = T
+juliaeltype(f::Meta.Field, x::Meta.Time, convert) = Time{x.unit, bitwidth(x.unit)}
+finaljuliatype(::Type{<:Time}) = Dates.Time
+periodtype(U::Meta.TimeUnit) = U === Meta.TimeUnit.SECOND ? Dates.Second :
+                               U === Meta.TimeUnit.MILLISECOND ? Dates.Millisecond :
+                               U === Meta.TimeUnit.MICROSECOND ? Dates.Microsecond : Dates.Nanosecond
+Base.convert(::Type{Dates.Time}, x::Time{U, T}) where {U, T} = Dates.Time(Dates.Nanosecond(Dates.tons(periodtype(U)(x.x))))
+
+function arrowtype(b, ::Type{Time{U, T}}) where {U, T}
+    Meta.timeStart(b)
+    Meta.timeAddUnit(b, U)
+    Meta.timeAddBitWidth(b, Int32(8 * sizeof(T)))
+    return Meta.Time, Meta.timeEnd(b), nothing
+end
+
+Base.convert(::Type{Time{Meta.TimeUnit.NANOSECOND, Int64}}, x::Dates.Time) = Time{Meta.TimeUnit.NANOSECOND, Int64}(Dates.value(x))
+
+struct Timestamp{U, TZ} <: ArrowTimeType
+    x::Int64
+end
+
+Base.zero(::Type{Timestamp{U, T}}) where {U, T} = Timestamp{U, T}(Int64(0))
+
+function juliaeltype(f::Meta.Field, x::Meta.Timestamp, convert)
+    return Timestamp{x.unit, x.timezone === nothing ? nothing : Symbol(x.timezone)}
+end
+
+finaljuliatype(::Type{<:Timestamp}) = ZonedDateTime
+Base.convert(::Type{ZonedDateTime}, x::Timestamp{U, TZ}) where {U, TZ} =
+    ZonedDateTime(Dates.DateTime(Dates.UTM(Int64(Dates.toms(periodtype(U)(x.x)) + UNIX_EPOCH_DATETIME))), TimeZone(String(TZ)))
+Base.convert(::Type{Timestamp{Meta.TimeUnit.MILLISECOND, TZ}}, x::ZonedDateTime) where {TZ} =
+    Timestamp{Meta.TimeUnit.MILLISECOND, TZ}(Int64(Dates.value(DateTime(x, Local)) - UNIX_EPOCH_DATETIME))
+
+function arrowtype(b, ::Type{Timestamp{U, TZ}}) where {U, TZ}
+    tz = TZ !== nothing ? FlatBuffers.createstring!(b, String(TZ)) : FlatBuffers.UOffsetT(0)
+    Meta.timestampStart(b)
+    Meta.timestampAddUnit(b, U)
+    Meta.timestampAddTimezone(b, tz)
+    return Meta.Timestamp, Meta.timestampEnd(b), nothing
+end
+
+struct Interval{U, T} <: ArrowTimeType
+    x::T
+end
+
+Base.zero(::Type{Interval{U, T}}) where {U, T} = Interval{U, T}(T(0))
+
+bitwidth(x::Meta.IntervalUnit) = x == Meta.IntervalUnit.YEAR_MONTH ? Int32 : Int64
+Interval{Meta.IntervalUnit.YEAR_MONTH}(x) = Interval{Meta.IntervalUnit.YEAR_MONTH, Int32}(Int32(x))
+Interval{Meta.IntervalUnit.DAY_TIME}(x) = Interval{Meta.IntervalUnit.DAY_TIME, Int64}(Int64(x))
+
+function juliaeltype(f::Meta.Field, x::Meta.Interval, convert)
+    return Interval{x.unit, bitwidth(x.unit)}
+end
+
+function arrowtype(b, ::Type{Interval{U, T}}) where {U, T}
+    Meta.intervalStart(b)
+    Meta.intervalAddUnit(b, U)
+    return Meta.Interval, Meta.intervalEnd(b), nothing
+end
+
+struct Duration{U} <: ArrowTimeType
+    x::Int64
+end
+
+Base.zero(::Type{Duration{U}}) where {U} = Duration{U}(Int64(0))
+
+function juliaeltype(f::Meta.Field, x::Meta.Duration, convert)
+    return Duration{x.unit}
+end
+
+finaljuliatype(::Type{Duration{U}}) where {U} = periodtype(U)
+Base.convert(::Type{P}, x::Duration{U}) where {P <: Dates.Period, U} = P(periodtype(U)(x.x))
+
+function arrowtype(b, ::Type{Duration{U}}) where {U}
+    Meta.durationStart(b)
+    Meta.durationAddUnit(b, U)
+    return Meta.Duration, Meta.durationEnd(b), nothing
+end
+
+arrowperiodtype(P) = Meta.TimeUnit.SECOND
+arrowperiodtype(::Type{Dates.Millisecond}) = Meta.TimeUnit.MILLISECOND
+arrowperiodtype(::Type{Dates.Microsecond}) = Meta.TimeUnit.MICROSECOND
+arrowperiodtype(::Type{Dates.Nanosecond}) = Meta.TimeUnit.NANOSECOND
+
+Base.convert(::Type{Duration{U}}, x::Dates.Period) where {U} = Duration{U}(Dates.value(periodtype(U)(x)))
+
+# nested types; call juliaeltype recursively on nested children
+function juliaeltype(f::Meta.Field, list::Union{Meta.List, Meta.LargeList}, convert)
+    return Vector{juliaeltype(f.children[1], buildmetadata(f.children[1]), convert)}
+end
+
+# arrowtype will call fieldoffset recursively for children
+function arrowtype(b, x::List{T, O, A}) where {T, O, A}
+    if eltype(A) == UInt8
+        if T <: AbstractString || T <: Union{AbstractString, Missing}
+            if O == Int32
+                Meta.utf8Start(b)
+                return Meta.Utf8, Meta.utf8End(b), nothing
+            else # if O == Int64
+                Meta.largUtf8Start(b)
+                return Meta.LargeUtf8, Meta.largUtf8End(b), nothing
+            end
+        else # if Vector{UInt8}
+            if O == Int32
+                Meta.binaryStart(b)
+                return Meta.Binary, Meta.binaryEnd(b), nothing
+            else # if O == Int64
+                Meta.largeBinaryStart(b)
+                return Meta.LargeBinary, Meta.largeBinaryEnd(b), nothing
+            end
+        end
+    else
+        children = [fieldoffset(b, "", x.data)]
+        if O == Int32
+            Meta.listStart(b)
+            return Meta.List, Meta.listEnd(b), children
+        else
+            Meta.largeListStart(b)
+            return Meta.LargeList, Meta.largeListEnd(b), children
+        end
+    end
+end
+
+function juliaeltype(f::Meta.Field, list::Meta.FixedSizeList, convert)
+    type = juliaeltype(f.children[1], buildmetadata(f.children[1]), convert)
+    return NTuple{Int(list.listSize), type}
+end
+
+function arrowtype(b, x::FixedSizeList{T, A}) where {T, A}
+    N = ArrowTypes.getsize(Base.nonmissingtype(T))
+    if eltype(A) == UInt8
+        Meta.fixedSizeBinaryStart(b)
+        Meta.fixedSizeBinaryAddByteWidth(b, Int32(N))
+        return Meta.FixedSizeBinary, Meta.fixedSizeBinaryEnd(b), nothing
+    else
+        children = [fieldoffset(b, "", x.data)]
+        Meta.fixedSizeListStart(b)
+        Meta.fixedSizeListAddListSize(b, Int32(N))
+        return Meta.FixedSizeList, Meta.fixedSizeListEnd(b), children
+    end
+end
+
+function juliaeltype(f::Meta.Field, map::Meta.Map, convert)
+    K = juliaeltype(f.children[1].children[1], buildmetadata(f.children[1].children[1]), convert)
+    V = juliaeltype(f.children[1].children[2], buildmetadata(f.children[1].children[2]), convert)
+    return Dict{K, V}
+end
+
+function arrowtype(b, x::Map)
+    children = [fieldoffset(b, "entries", x.data)]
+    Meta.mapStart(b)
+    return Meta.Map, Meta.mapEnd(b), children
+end
+
+struct KeyValue{K, V}
+    key::K
+    value::V
+end
+keyvalueK(::Type{KeyValue{K, V}}) where {K, V} = K
+keyvalueV(::Type{KeyValue{K, V}}) where {K, V} = V
+Base.length(kv::KeyValue) = 1
+Base.iterate(kv::KeyValue, st=1) = st === nothing ? nothing : (kv, nothing)
+ArrowTypes.default(::Type{KeyValue{K, V}}) where {K, V} = KeyValue(default(K), default(V))
+
+function arrowtype(b, ::Type{KeyValue{K, V}}) where {K, V}
+    children = [fieldoffset(b, "key", K), fieldoffset(b, "value", V)]
+    Meta.structStart(b)
+    return Meta.Struct, Meta.structEnd(b), children
+end
+
+function juliaeltype(f::Meta.Field, list::Meta.Struct, convert)
+    names = Tuple(Symbol(x.name) for x in f.children)
+    types = Tuple(juliaeltype(x, buildmetadata(x), convert) for x in f.children)
+    return NamedTuple{names, Tuple{types...}}
+end
+
+function arrowtype(b, x::Struct{T, S}) where {T, S}
+    names = fieldnames(Base.nonmissingtype(T))
+    children = [fieldoffset(b, names[i], x.data[i]) for i = 1:length(names)]
+    Meta.structStart(b)
+    return Meta.Struct, Meta.structEnd(b), children
+end
+
+# Unions
+function juliaeltype(f::Meta.Field, u::Meta.Union, convert)
+    return UnionT{u.mode, u.typeIds !== nothing ? Tuple(u.typeIds) : u.typeIds, Tuple{(juliaeltype(x, buildmetadata(x), convert) for x in f.children)...}}
+end
+
+arrowtype(b, x::Union{DenseUnion{TT, S}, SparseUnion{TT, S}}) where {TT, S} = arrowtype(b, TT, x)
+function arrowtype(b, ::Type{UnionT{T, typeIds, U}}, x::Union{DenseUnion{TT, S}, SparseUnion{TT, S}}) where {T, typeIds, U, TT, S}
+    if typeIds !== nothing
+        Meta.unionStartTypeIdsVector(b, length(typeIds))
+        for id in Iterators.reverse(typeIds)
+            FlatBuffers.prepend!(b, id)
+        end
+        TI = FlatBuffers.endvector!(b, length(typeIds))
+    end
+    children = [fieldoffset(b, "", x.data[i]) for i = 1:fieldcount(U)]
+    Meta.unionStart(b)
+    Meta.unionAddMode(b, T)
+    if typeIds !== nothing
+        Meta.unionAddTypeIds(b, TI)
+    end
+    return Meta.Union, Meta.unionEnd(b), children
+end
diff --git a/julia/Arrow/src/metadata/File.jl b/julia/Arrow/src/metadata/File.jl
new file mode 100644
index 00000000000..8c1622ce439
--- /dev/null
+++ b/julia/Arrow/src/metadata/File.jl
@@ -0,0 +1,90 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+struct Footer <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::Footer) = (:version, :schema, :dictionaries, :recordBatches, :custom_metadata)
+
+function Base.getproperty(x::Footer, field::Symbol)
+    if field === :version
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), MetadataVersion)
+        return MetadataVersion.V1
+    elseif field === :schema
+        o = FlatBuffers.offset(x, 6)
+        if o != 0
+            y = FlatBuffers.indirect(x, o + FlatBuffers.pos(x))
+            return FlatBuffers.init(Schema, FlatBuffers.bytes(x), y)
+        end
+    elseif field === :dictionaries
+        o = FlatBuffers.offset(x, 8)
+        if o != 0
+            return FlatBuffers.Array{Block}(x, o)
+        end
+    elseif field === :recordBatches
+        o = FlatBuffers.offset(x, 10)
+        if o != 0
+            return FlatBuffers.Array{Block}(x, o)
+        end
+    elseif field === :custom_metadata
+        o = FlatBuffers.offset(x, 12)
+        if o != 0
+            return FlatBuffers.Array{KeyValue}(x, o)
+        end
+    end
+    return nothing
+end
+
+footerStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 4)
+footerAddVersion(b::FlatBuffers.Builder, version::MetadataVersion) = FlatBuffers.prependslot!(b, 0, version, 0)
+footerAddSchema(b::FlatBuffers.Builder, schema::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 1, schema, 0)
+footerAddDictionaries(b::FlatBuffers.Builder, dictionaries::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 2, dictionaries, 0)
+footerStartDictionariesVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 24, numelems, 8)
+footerAddRecordBatches(b::FlatBuffers.Builder, recordbatches::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 3, recordbatches, 0)
+footerStartRecordBatchesVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 24, numelems, 8)
+footerEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+struct Block <: FlatBuffers.Struct
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+FlatBuffers.structsizeof(::Base.Type{Block}) = 24
+
+Base.propertynames(x::Block) = (:offset, :metaDataLength, :bodyLength)
+
+function Base.getproperty(x::Block, field::Symbol)
+    if field === :offset
+        return FlatBuffers.get(x, FlatBuffers.pos(x), Int64)
+    elseif field === :metaDataLength
+        return FlatBuffers.get(x, FlatBuffers.pos(x) + 8, Int32)
+    elseif field === :bodyLength
+        return FlatBuffers.get(x, FlatBuffers.pos(x) + 16, Int64)
+    end
+    return nothing
+end
+
+function createBlock(b::FlatBuffers.Builder, offset::Int64, metadatalength::Int32, bodylength::Int64)
+    FlatBuffers.prep!(b, 8, 24)
+    prepend!(b, bodylength)
+    FlatBuffers.pad!(b, 4)
+    prepend!(b, metadatalength)
+    prepend!(b, offset)
+    return FlatBuffers.offset(b)
+end
\ No newline at end of file
diff --git a/julia/Arrow/src/metadata/Flatbuf.jl b/julia/Arrow/src/metadata/Flatbuf.jl
new file mode 100644
index 00000000000..a69230d9247
--- /dev/null
+++ b/julia/Arrow/src/metadata/Flatbuf.jl
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Flatbuf
+
+using ..FlatBuffers
+
+include("Schema.jl")
+include("File.jl")
+include("Message.jl")
+
+end # module
\ No newline at end of file
diff --git a/julia/Arrow/src/metadata/Message.jl b/julia/Arrow/src/metadata/Message.jl
new file mode 100644
index 00000000000..4fe6253bbc0
--- /dev/null
+++ b/julia/Arrow/src/metadata/Message.jl
@@ -0,0 +1,202 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+struct FieldNode <: FlatBuffers.Struct
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+FlatBuffers.structsizeof(::Base.Type{FieldNode}) = 16
+
+Base.propertynames(x::FieldNode) = (:length, :null_count)
+
+function Base.getproperty(x::FieldNode, field::Symbol)
+    if field === :length
+        return FlatBuffers.get(x, FlatBuffers.pos(x), Int64)
+    elseif field === :null_count
+        return FlatBuffers.get(x, FlatBuffers.pos(x) + 8, Int64)
+    end
+    return nothing
+end
+
+function createFieldNode(b::FlatBuffers.Builder, length::Int64, nullCount::Int64)
+    FlatBuffers.prep!(b, 8, 16)
+    prepend!(b, nullCount)
+    prepend!(b, length)
+    return FlatBuffers.offset(b)
+end
+
+FlatBuffers.@scopedenum CompressionType::Int8 LZ4_FRAME ZSTD
+
+FlatBuffers.@scopedenum BodyCompressionMethod::Int8 BUFFER
+
+struct BodyCompression <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::BodyCompression) = (:codec, :method)
+
+function Base.getproperty(x::BodyCompression, field::Symbol)
+    if field === :codec
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), CompressionType)
+        return CompressionType.LZ4_FRAME
+    elseif field === :method
+        o = FlatBuffers.offset(x, 6)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), BodyCompressionMethod)
+        return BodyCompressionMethod.BUFFER
+    end
+    return nothing
+end
+
+bodyCompressionStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 2)
+bodyCompressionAddCodec(b::FlatBuffers.Builder, codec::CompressionType) = FlatBuffers.prependslot!(b, 0, codec, 0)
+bodyCompressionAddMethod(b::FlatBuffers.Builder, method::BodyCompressionMethod) = FlatBuffers.prependslot!(b, 1, method, 0)
+bodyCompressionEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+struct RecordBatch <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::RecordBatch) = (:length, :nodes, :buffers, :compression)
+
+function Base.getproperty(x::RecordBatch, field::Symbol)
+    if field === :length
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int64)
+    elseif field === :nodes
+        o = FlatBuffers.offset(x, 6)
+        if o != 0
+            return FlatBuffers.Array{FieldNode}(x, o)
+        end
+    elseif field === :buffers
+        o = FlatBuffers.offset(x, 8)
+        if o != 0
+            return FlatBuffers.Array{Buffer}(x, o)
+        end
+    elseif field === :compression
+        o = FlatBuffers.offset(x, 10)
+        if o != 0
+            y = FlatBuffers.indirect(x, o + FlatBuffers.pos(x))
+            return FlatBuffers.init(BodyCompression, FlatBuffers.bytes(x), y)
+        end
+    end
+    return nothing
+end
+
+recordBatchStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 4)
+recordBatchAddLength(b::FlatBuffers.Builder, length::Int64) = FlatBuffers.prependslot!(b, 0, length, 0)
+recordBatchAddNodes(b::FlatBuffers.Builder, nodes::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 1, nodes, 0)
+recordBatchStartNodesVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 16, numelems, 8)
+recordBatchAddBuffers(b::FlatBuffers.Builder, buffers::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 2, buffers, 0)
+recordBatchStartBuffersVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 16, numelems, 8)
+recordBatchAddCompression(b::FlatBuffers.Builder, c::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 3, c, 0)
+recordBatchEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+struct DictionaryBatch <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::DictionaryBatch) = (:id, :data, :isDelta)
+
+function Base.getproperty(x::DictionaryBatch, field::Symbol)
+    if field === :id
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int64)
+        return Int64(0)
+    elseif field === :data
+        o = FlatBuffers.offset(x, 6)
+        if o != 0
+            y = FlatBuffers.indirect(x, o + FlatBuffers.pos(x))
+            return FlatBuffers.init(RecordBatch, FlatBuffers.bytes(x), y)
+        end
+    elseif field === :isDelta
+        o = FlatBuffers.offset(x, 8)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Base.Bool)
+        return false
+    end
+    return nothing
+end
+
+dictionaryBatchStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 3)
+dictionaryBatchAddId(b::FlatBuffers.Builder, id::Int64) = FlatBuffers.prependslot!(b, 0, id, 0)
+dictionaryBatchAddData(b::FlatBuffers.Builder, data::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 1, data, 0)
+dictionaryBatchAddIsDelta(b::FlatBuffers.Builder, isdelta::Base.Bool) = FlatBuffers.prependslot!(b, 2, isdelta, false)
+dictionaryBatchEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+function MessageHeader(b::UInt8)
+    b == 1 && return Schema
+    b == 2 && return DictionaryBatch
+    b == 3 && return RecordBatch
+    # b == 4 && return Tensor
+    # b == 5 && return SparseTensor
+    return nothing
+end
+
+function MessageHeader(::Base.Type{T})::Int16 where {T}
+    T == Schema && return 1
+    T == DictionaryBatch && return 2
+    T == RecordBatch && return 3
+    # T == Tensor && return 4
+    # T == SparseTensor && return 5
+    return 0
+end
+
+struct Message <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::Message) = (:version, :header, :bodyLength, :custom_metadata)
+
+function Base.getproperty(x::Message, field::Symbol)
+    if field === :version
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), MetadataVersion)
+    elseif field === :header
+        o = FlatBuffers.offset(x, 6)
+        if o != 0
+            T = MessageHeader(FlatBuffers.get(x, o + FlatBuffers.pos(x), UInt8))
+            o = FlatBuffers.offset(x, 8)
+            pos = FlatBuffers.union(x, o)
+            if o != 0
+                return FlatBuffers.init(T, FlatBuffers.bytes(x), pos)
+            end
+        end
+    elseif field === :bodyLength
+        o = FlatBuffers.offset(x, 10)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int64)
+        return Int64(0)
+    elseif field === :custom_metadata
+        o = FlatBuffers.offset(x, 12)
+        if o != 0
+            return FlatBuffers.Array{KeyValue}(x, o)
+        end
+    end
+    return nothing
+end
+
+messageStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 5)
+messageAddVersion(b::FlatBuffers.Builder, version::MetadataVersion) = FlatBuffers.prependslot!(b, 0, version, 0)
+messageAddHeaderType(b::FlatBuffers.Builder, ::Core.Type{T}) where {T} = FlatBuffers.prependslot!(b, 1, MessageHeader(T), 0)
+messageAddHeader(b::FlatBuffers.Builder, header::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 2, header, 0)
+messageAddBodyLength(b::FlatBuffers.Builder, bodyLength::Int64) = FlatBuffers.prependslot!(b, 3, bodyLength, 0)
+messageAddCustomMetadata(b::FlatBuffers.Builder, meta::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 4, meta, 0)
+messageStartCustomMetadataVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 4, numelems, 4)
+messageEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
\ No newline at end of file
diff --git a/julia/Arrow/src/metadata/Schema.jl b/julia/Arrow/src/metadata/Schema.jl
new file mode 100644
index 00000000000..1ddf7485fc4
--- /dev/null
+++ b/julia/Arrow/src/metadata/Schema.jl
@@ -0,0 +1,605 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FlatBuffers.@scopedenum MetadataVersion::Int16 V1 V2 V3 V4 V5
+
+struct Null <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::Null) = ()
+
+nullStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0)
+nullEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+struct Struct <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::Struct) = ()
+
+structStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0)
+structEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+struct List <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::List) = ()
+
+listStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0)
+listEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+struct LargeList <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::LargeList) = ()
+
+largeListStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0)
+largeListEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+struct FixedSizeList <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::FixedSizeList) = (:listSize,)
+
+function Base.getproperty(x::FixedSizeList, field::Symbol)
+    if field === :listSize
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int32)
+        return Int32(0)
+    end
+    return nothing
+end
+
+fixedSizeListStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 1)
+fixedSizeListAddListSize(b::FlatBuffers.Builder, listSize::Int32) = FlatBuffers.prependslot!(b, 0, listSize, 0)
+fixedSizeListEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+struct Map <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::Map) = (:keysSorted,)
+
+function Base.getproperty(x::Map, field::Symbol)
+    if field === :keysSorted
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Base.Bool)
+    end
+    return nothing
+end
+
+mapStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 1)
+mapAddKeysSorted(b::FlatBuffers.Builder, keyssorted::Base.Bool) = FlatBuffers.prependslot!(b, 0, keyssorted, 0)
+mapEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+FlatBuffers.@scopedenum UnionMode::Int16 Sparse Dense
+
+struct Union <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::Union) = (:mode, :typeIds)
+
+function Base.getproperty(x::Union, field::Symbol)
+    if field === :mode
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), UnionMode)
+        return UnionMode.Sparse
+    elseif field === :typeIds
+        o = FlatBuffers.offset(x, 6)
+        o != 0 && return FlatBuffers.Array{Int32}(x, o)
+    end
+    return nothing
+end
+
+unionStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 2)
+unionAddMode(b::FlatBuffers.Builder, mode::UnionMode) = FlatBuffers.prependslot!(b, 0, mode, 0)
+unionAddTypeIds(b::FlatBuffers.Builder, typeIds::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 1, typeIds, 0)
+unionStartTypeIdsVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 4, numelems, 4)
+unionEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+struct Int <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::Int) = (:bitWidth, :is_signed)
+
+function Base.getproperty(x::Int, field::Symbol)
+    if field === :bitWidth
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int32)
+    elseif field === :is_signed
+        o = FlatBuffers.offset(x, 6)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Base.Bool)
+        return false
+    end
+    return nothing
+end
+
+intStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 2)
+intAddBitWidth(b::FlatBuffers.Builder, bitwidth::Int32) = FlatBuffers.prependslot!(b, 0, bitwidth, 0)
+intAddIsSigned(b::FlatBuffers.Builder, issigned::Base.Bool) = FlatBuffers.prependslot!(b, 1, issigned, 0)
+intEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+FlatBuffers.@scopedenum Precision::Int16 HALF SINGLE DOUBLE
+
+struct FloatingPoint <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::FloatingPoint) = (:precision,)
+
+function Base.getproperty(x::FloatingPoint, field::Symbol)
+    if field === :precision
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Precision)
+        return Precision.HALF
+    end
+    return nothing
+end
+
+floatingPointStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 1)
+floatingPointAddPrecision(b::FlatBuffers.Builder, precision::Precision) = FlatBuffers.prependslot!(b, 0, precision, 0)
+floatingPointEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+struct Utf8 <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::Utf8) = ()
+
+utf8Start(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0)
+utf8End(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+struct Binary <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::Binary) = ()
+
+binaryStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0)
+binaryEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+struct LargeUtf8 <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::LargeUtf8) = ()
+
+largUtf8Start(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0)
+largUtf8End(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+struct LargeBinary <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::LargeBinary) = ()
+
+largeBinaryStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0)
+largeBinaryEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+struct FixedSizeBinary <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::FixedSizeBinary) = (:byteWidth,)
+
+function Base.getproperty(x::FixedSizeBinary, field::Symbol)
+    if field === :byteWidth
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int32)
+    end
+    return nothing
+end
+
+fixedSizeBinaryStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 1)
+fixedSizeBinaryAddByteWidth(b::FlatBuffers.Builder, bytewidth::Int32) = FlatBuffers.prependslot!(b, 0, bytewidth, 0)
+fixedSizeBinaryEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+struct Bool <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::Bool) = ()
+
+boolStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0)
+boolEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+struct Decimal <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::Decimal) = (:precision, :scale)
+
+function Base.getproperty(x::Decimal, field::Symbol)
+    if field === :precision
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int32)
+        return Int32(0)
+    elseif field === :scale
+        o = FlatBuffers.offset(x, 6)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int32)
+        return Int32(0)
+    end
+    return nothing
+end
+
+decimalStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 2)
+decimalAddPrecision(b::FlatBuffers.Builder, precision::Int32) = FlatBuffers.prependslot!(b, 0, precision, 0)
+decimalAddScale(b::FlatBuffers.Builder, scale::Int32) = FlatBuffers.prependslot!(b, 1, scale, 0)
+decimalEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+FlatBuffers.@scopedenum DateUnit::Int16 DAY MILLISECOND
+
+struct Date <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::Date) = (:unit,)
+
+function Base.getproperty(x::Date, field::Symbol)
+    if field === :unit
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), DateUnit)
+        return DateUnit.MILLISECOND
+    end
+    return nothing
+end
+
+dateStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 1)
+dateAddUnit(b::FlatBuffers.Builder, unit::DateUnit) = FlatBuffers.prependslot!(b, 0, unit, 1)
+dateEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+FlatBuffers.@scopedenum TimeUnit::Int16 SECOND MILLISECOND MICROSECOND NANOSECOND
+
+struct Time <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::Time) = (:unit, :bitWidth)
+
+function Base.getproperty(x::Time, field::Symbol)
+    if field === :unit
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), TimeUnit)
+        return TimeUnit.MILLISECOND
+    elseif field === :bitWidth
+        o = FlatBuffers.offset(x, 6)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int32)
+        return 32
+    end
+    return nothing
+end
+
+timeStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 2)
+timeAddUnit(b::FlatBuffers.Builder, unit::TimeUnit) = FlatBuffers.prependslot!(b, 0, unit, 1)
+timeAddBitWidth(b::FlatBuffers.Builder, bitwidth::Int32) = FlatBuffers.prependslot!(b, 1, bitwidth, 32)
+timeEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+struct Timestamp <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::Timestamp) = (:unit, :timezone)
+
+function Base.getproperty(x::Timestamp, field::Symbol)
+    if field === :unit
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), TimeUnit)
+        return TimeUnit.SECOND
+    elseif field === :timezone
+        o = FlatBuffers.offset(x, 6)
+        o != 0 && return String(x, o + FlatBuffers.pos(x))
+    end
+    return nothing
+end
+
+timestampStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 2)
+timestampAddUnit(b::FlatBuffers.Builder, unit::TimeUnit) = FlatBuffers.prependslot!(b, 0, unit, 0)
+timestampAddTimezone(b::FlatBuffers.Builder, timezone::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 1, timezone, 0)
+timestampEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+FlatBuffers.@scopedenum IntervalUnit::Int16 YEAR_MONTH DAY_TIME
+
+struct Interval <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::Interval) = (:unit,)
+
+function Base.getproperty(x::Interval, field::Symbol)
+    if field === :unit
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), IntervalUnit)
+        return IntervalUnit.YEAR_MONTH
+    end
+    return nothing
+end
+
+intervalStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 1)
+intervalAddUnit(b::FlatBuffers.Builder, unit::IntervalUnit) = FlatBuffers.prependslot!(b, 0, unit, 0)
+intervalEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+struct Duration <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::Duration) = (:unit,)
+
+function Base.getproperty(x::Duration, field::Symbol)
+    if field === :unit
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), TimeUnit)
+        return TimeUnit.MILLISECOND
+    end
+    return nothing
+end
+
+durationStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 1)
+durationAddUnit(b::FlatBuffers.Builder, unit::TimeUnit) = FlatBuffers.prependslot!(b, 0, unit, 1)
+durationEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+function Type(b::UInt8)
+    b == 1 && return Null
+    b == 2 && return Int
+    b == 3 && return FloatingPoint
+    b == 4 && return Binary
+    b == 5 && return Utf8
+    b == 6 && return Bool
+    b == 7 && return Decimal
+    b == 8 && return Date
+    b == 9 && return Time
+    b == 10 && return Timestamp
+    b == 11 && return Interval
+    b == 12 && return List
+    b == 13 && return Struct
+    b == 14 && return Union
+    b == 15 && return FixedSizeBinary
+    b == 16 && return FixedSizeList
+    b == 17 && return Map
+    b == 18 && return Duration
+    b == 19 && return LargeBinary
+    b == 20 && return LargeUtf8
+    b == 21 && return LargeList
+    return nothing
+end
+
+function Type(::Base.Type{T})::Int16 where {T}
+    T == Null && return 1
+    T == Int && return 2
+    T == FloatingPoint && return 3
+    T == Binary && return 4
+    T == Utf8 && return 5
+    T == Bool && return 6
+    T == Decimal && return 7
+    T == Date && return 8
+    T == Time && return 9
+    T == Timestamp && return 10
+    T == Interval && return 11
+    T == List && return 12
+    T == Struct && return 13
+    T == Union && return 14
+    T == FixedSizeBinary && return 15
+    T == FixedSizeList && return 16
+    T == Map && return 17
+    T == Duration && return 18
+    T == LargeBinary && return 19
+    T == LargeUtf8 && return 20
+    T == LargeList && return 21
+    return 0
+end
+
+struct KeyValue <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::KeyValue) = (:key, :value)
+
+function Base.getproperty(x::KeyValue, field::Symbol)
+    if field === :key
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return String(x, o + FlatBuffers.pos(x))
+    elseif field === :value
+        o = FlatBuffers.offset(x, 6)
+        o != 0 && return String(x, o + FlatBuffers.pos(x))
+    end
+    return nothing
+end
+
+keyValueStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 2)
+keyValueAddKey(b::FlatBuffers.Builder, key::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 0, key, 0)
+keyValueAddValue(b::FlatBuffers.Builder, value::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 1, value, 0)
+keyValueEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+FlatBuffers.@scopedenum DictionaryKind::Int16 DenseArray
+
+struct DictionaryEncoding <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::DictionaryEncoding) = (:id, :indexType, :isOrdered, :dictionaryKind)
+
+function Base.getproperty(x::DictionaryEncoding, field::Symbol)
+    if field === :id
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int64)
+        return Int64(0)
+    elseif field === :indexType
+        o = FlatBuffers.offset(x, 6)
+        if o != 0
+            y = FlatBuffers.indirect(x, o + FlatBuffers.pos(x))
+            return FlatBuffers.init(Int, FlatBuffers.bytes(x), y)
+        end
+    elseif field === :isOrdered
+        o = FlatBuffers.offset(x, 8)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Base.Bool)
+        return false
+    elseif field === :dictionaryKind
+        o = FlatBuffers.offset(x, 10)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), DictionaryKind)
+    end
+    return nothing
+end
+
+dictionaryEncodingStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 3)
+dictionaryEncodingAddId(b::FlatBuffers.Builder, id::Int64) = FlatBuffers.prependslot!(b, 0, id, 0)
+dictionaryEncodingAddIndexType(b::FlatBuffers.Builder, indextype::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 1, indextype, 0)
+dictionaryEncodingAddIsOrdered(b::FlatBuffers.Builder, isordered::Base.Bool) = FlatBuffers.prependslot!(b, 1, isordered, 0)
+dictionaryEncodingEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+struct Field <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::Field) = (:name, :nullable, :type, :dictionary, :children, :custom_metadata)
+
+function Base.getproperty(x::Field, field::Symbol)
+    if field === :name
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return String(x, o + FlatBuffers.pos(x))
+    elseif field === :nullable
+        o = FlatBuffers.offset(x, 6)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Base.Bool)
+        return false
+    elseif field === :type
+        o = FlatBuffers.offset(x, 8)
+        if o != 0
+            T = Type(FlatBuffers.get(x, o + FlatBuffers.pos(x), UInt8))
+            o = FlatBuffers.offset(x, 10)
+            pos = FlatBuffers.union(x, o)
+            if o != 0
+                return FlatBuffers.init(T, FlatBuffers.bytes(x), pos)
+            end
+        end
+    elseif field === :dictionary
+        o = FlatBuffers.offset(x, 12)
+        if o != 0
+            y = FlatBuffers.indirect(x, o + FlatBuffers.pos(x))
+            return FlatBuffers.init(DictionaryEncoding, FlatBuffers.bytes(x), y)
+        end
+    elseif field === :children
+        o = FlatBuffers.offset(x, 14)
+        if o != 0
+            return FlatBuffers.Array{Field}(x, o)
+        end
+    elseif field === :custom_metadata
+        o = FlatBuffers.offset(x, 16)
+        if o != 0
+            return FlatBuffers.Array{KeyValue}(x, o)
+        end
+    end
+    return nothing
+end
+
+fieldStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 7)
+fieldAddName(b::FlatBuffers.Builder, name::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 0, name, 0)
+fieldAddNullable(b::FlatBuffers.Builder, nullable::Base.Bool) = FlatBuffers.prependslot!(b, 1, nullable, false)
+fieldAddTypeType(b::FlatBuffers.Builder, ::Core.Type{T}) where {T} = FlatBuffers.prependslot!(b, 2, Type(T), 0)
+fieldAddType(b::FlatBuffers.Builder, type::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 3, type, 0)
+fieldAddDictionary(b::FlatBuffers.Builder, dictionary::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 4, dictionary, 0)
+fieldAddChildren(b::FlatBuffers.Builder, children::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 5, children, 0)
+fieldStartChildrenVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 4, numelems, 4)
+fieldAddCustomMetadata(b::FlatBuffers.Builder, custommetadata::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 6, custommetadata, 0)
+fieldStartCustomMetadataVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 4, numelems, 4)
+fieldEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
+
+FlatBuffers.@scopedenum Endianness::Int16 Little Big
+
+struct Buffer <: FlatBuffers.Struct
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+FlatBuffers.structsizeof(::Base.Type{Buffer}) = 16
+
+Base.propertynames(x::Buffer) = (:offset, :length)
+
+function Base.getproperty(x::Buffer, field::Symbol)
+    if field === :offset
+        return FlatBuffers.get(x, FlatBuffers.pos(x), Int64)
+    elseif field === :length
+        return FlatBuffers.get(x, FlatBuffers.pos(x) + 8, Int64)
+    end
+    return nothing
+end
+
+function createBuffer(b::FlatBuffers.Builder, offset::Int64, length::Int64)
+    FlatBuffers.prep!(b, 8, 16)
+    prepend!(b, length)
+    prepend!(b, offset)
+    return FlatBuffers.offset(b)
+end
+
+struct Schema <: FlatBuffers.Table
+    bytes::Vector{UInt8}
+    pos::Base.Int
+end
+
+Base.propertynames(x::Schema) = (:endianness, :fields, :custom_metadata)
+
+function Base.getproperty(x::Schema, field::Symbol)
+    if field === :endianness
+        o = FlatBuffers.offset(x, 4)
+        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Endianness)
+    elseif field === :fields
+        o = FlatBuffers.offset(x, 6)
+        if o != 0
+            return FlatBuffers.Array{Field}(x, o)
+        end
+    elseif field === :custom_metadata
+        o = FlatBuffers.offset(x, 8)
+        if o != 0
+            return FlatBuffers.Array{KeyValue}(x, o)
+        end
+    end
+    return nothing
+end
+
+schemaStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 3)
+schemaAddEndianness(b::FlatBuffers.Builder, endianness::Endianness) = FlatBuffers.prependslot!(b, 0, endianness, 0)
+schemaAddFields(b::FlatBuffers.Builder, fields::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 1, fields, 0)
+schemaStartFieldsVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 4, numelems, 4)
+schemaAddCustomMetadata(b::FlatBuffers.Builder, custommetadata::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 2, custommetadata, 0)
+schemaStartCustomMetadataVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 4, numelems, 4)
+schemaEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
diff --git a/julia/Arrow/src/table.jl b/julia/Arrow/src/table.jl
new file mode 100644
index 00000000000..721d208e105
--- /dev/null
+++ b/julia/Arrow/src/table.jl
@@ -0,0 +1,556 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+struct BatchIterator
+    bytes::Vector{UInt8}
+    startpos::Int
+end
+
+"""
+    Arrow.Stream(io::IO; convert::Bool=true)
+    Arrow.Stream(file::String; convert::Bool=true)
+    Arrow.Stream(bytes::Vector{UInt8}, pos=1, len=nothing; convert::Bool=true)
+
+Start reading an arrow formatted table, from:
+ * `io`, bytes will be read all at once via `read(io)`
+ * `file`, bytes will be read via `Mmap.mmap(file)`
+ * `bytes`, a byte vector directly, optionally allowing specifying the starting byte position `pos` and `len`
+
+Reads the initial schema message from the arrow stream/file, then returns an `Arrow.Stream` object
+which will iterate over record batch messages, producing an [`Arrow.Table`](@ref) on each iteration.
+
+By iterating [`Arrow.Table`](@ref), `Arrow.Stream` satisfies the `Tables.partitions` interface, and as such can
+be passed to Tables.jl-compatible sink functions.
+
+This allows iterating over extremely large "arrow tables" in chunks represented as record batches.
+
+Supports the `convert` keyword argument which controls whether certain arrow primitive types will be
+lazily converted to more friendly Julia defaults; by default, `convert=true`.
+"""
+struct Stream
+    batchiterator::BatchIterator
+    pos::Int
+    names::Vector{Symbol}
+    schema::Meta.Schema
+    dictencodings::Dict{Int64, DictEncoding} # dictionary id => DictEncoding
+    dictencoded::Dict{Int64, Meta.Field} # dictionary id => field
+    convert::Bool
+end
+
+Tables.partitions(x::Stream) = x
+
+Stream(io::IO, pos::Integer=1, len=nothing; convert::Bool=true) = Stream(Base.read(io), pos, len; convert=convert)
+Stream(str::String, pos::Integer=1, len=nothing; convert::Bool=true) = isfile(str) ? Stream(Mmap.mmap(str), pos, len; convert=convert) :
+    throw(ArgumentError("$str is not a valid arrow file"))
+
+# will detect whether we're reading a Stream from a file or stream
+function Stream(bytes::Vector{UInt8}, off::Integer=1, tlen::Union{Integer, Nothing}=nothing; convert::Bool=true)
+    len = something(tlen, length(bytes))
+    if len > 24 &&
+        _startswith(bytes, off, FILE_FORMAT_MAGIC_BYTES) &&
+        _endswith(bytes, off + len - 1, FILE_FORMAT_MAGIC_BYTES)
+        off += 8 # skip past magic bytes + padding
+    end
+    dictencodings = Dict{Int64, DictEncoding}() # dictionary id => DictEncoding
+    dictencoded = Dict{Int64, Meta.Field}() # dictionary id => field
+    batchiterator = BatchIterator(bytes, off)
+    state = iterate(batchiterator)
+    state === nothing && throw(ArgumentError("no arrow ipc messages found in provided input"))
+    batch, (pos, id) = state
+    schema = batch.msg.header
+    schema isa Meta.Schema || throw(ArgumentError("first arrow ipc message MUST be a schema message"))
+    # assert endianness?
+    # store custom_metadata?
+    names = Symbol[]
+    for (i, field) in enumerate(schema.fields)
+        push!(names, Symbol(field.name))
+        # recursively find any dictionaries for any fields
+        getdictionaries!(dictencoded, field)
+        @debug 1 "parsed column from schema: field = $field"
+    end
+    return Stream(batchiterator, pos, names, schema, dictencodings, dictencoded, convert)
+end
+
+function Base.iterate(x::Stream, (pos, id)=(x.pos, 1))
+    columns = AbstractVector[]
+    while true
+        state = iterate(x.batchiterator, (pos, id))
+        state === nothing && return nothing
+        batch, (pos, id) = state
+        header = batch.msg.header
+        if header isa Meta.DictionaryBatch
+            id = header.id
+            recordbatch = header.data
+            @debug 1 "parsing dictionary batch message: id = $id, compression = $(recordbatch.compression)"
+            if haskey(x.dictencodings, id) && header.isDelta
+                # delta
+                field = x.dictencoded[id]
+                values, _, _ = build(field, field.type, batch, recordbatch, x.dictencodings, Int64(1), Int64(1), x.convert)
+                dictencoding = x.dictencodings[id]
+                append!(dictencoding.data, values)
+                continue
+            end
+            # new dictencoding or replace
+            field = x.dictencoded[id]
+            values, _, _ = build(field, field.type, batch, recordbatch, x.dictencodings, Int64(1), Int64(1), x.convert)
+            A = ChainedVector([values])
+            x.dictencodings[id] = DictEncoding{eltype(A), typeof(A)}(id, A, field.dictionary.isOrdered)
+            @debug 1 "parsed dictionary batch message: id=$id, data=$values\n"
+        elseif header isa Meta.RecordBatch
+            @debug 1 "parsing record batch message: compression = $(header.compression)"
+            for vec in VectorIterator(x.schema, batch, x.dictencodings, x.convert)
+                push!(columns, vec)
+            end
+            break
+        else
+            throw(ArgumentError("unsupported arrow message type: $(typeof(header))"))
+        end
+    end
+    lookup = Dict{Symbol, AbstractVector}()
+    types = Type[]
+    for (nm, col) in zip(x.names, columns)
+        lookup[nm] = col
+        push!(types, eltype(col))
+    end
+    return Table(x.names, types, columns, lookup, Ref(x.schema)), (pos, id)
+end
+
+"""
+    Arrow.Table(io::IO; convert::Bool=true)
+    Arrow.Table(file::String; convert::Bool=true)
+    Arrow.Table(bytes::Vector{UInt8}, pos=1, len=nothing; convert::Bool=true)
+
+Read an arrow formatted table, from:
+ * `io`, bytes will be read all at once via `read(io)`
+ * `file`, bytes will be read via `Mmap.mmap(file)`
+ * `bytes`, a byte vector directly, optionally allowing specifying the starting byte position `pos` and `len`
+
+Returns a `Arrow.Table` object that allows column access via `table.col1`, `table[:col1]`, or `table[1]`.
+
+NOTE: the columns in an `Arrow.Table` are views into the original arrow memory, and hence are not easily
+modifiable (with e.g. `push!`, `append!`, etc.). To mutate arrow columns, call `copy(x)` to materialize
+the arrow data as a normal Julia array.
+
+`Arrow.Table` also satisfies the [Tables.jl](https://github.com/JuliaData/Tables.jl) interface, and so can easily be materialied via any supporting
+sink function: e.g. `DataFrame(Arrow.Table(file))`, `SQLite.load!(db, "table", Arrow.Table(file))`, etc.
+
+Supports the `convert` keyword argument which controls whether certain arrow primitive types will be
+lazily converted to more friendly Julia defaults; by default, `convert=true`.
+"""
+struct Table <: Tables.AbstractColumns
+    names::Vector{Symbol}
+    types::Vector{Type}
+    columns::Vector{AbstractVector}
+    lookup::Dict{Symbol, AbstractVector}
+    schema::Ref{Meta.Schema}
+end
+
+Table() = Table(Symbol[], Type[], AbstractVector[], Dict{Symbol, AbstractVector}(), Ref{Meta.Schema}())
+
+names(t::Table) = getfield(t, :names)
+types(t::Table) = getfield(t, :types)
+columns(t::Table) = getfield(t, :columns)
+lookup(t::Table) = getfield(t, :lookup)
+schema(t::Table) = getfield(t, :schema)
+
+Tables.istable(::Table) = true
+Tables.columnaccess(::Table) = true
+Tables.columns(t::Table) = Tables.CopiedColumns(t)
+Tables.schema(t::Table) = Tables.Schema(names(t), types(t))
+Tables.columnnames(t::Table) = names(t)
+Tables.getcolumn(t::Table, i::Int) = columns(t)[i]
+Tables.getcolumn(t::Table, nm::Symbol) = lookup(t)[nm]
+
+# high-level user API functions
+Table(io::IO, pos::Integer=1, len=nothing; convert::Bool=true) = Table(Base.read(io), pos, len; convert=convert)
+Table(str::String, pos::Integer=1, len=nothing; convert::Bool=true) = isfile(str) ? Table(Mmap.mmap(str), pos, len; convert=convert) :
+    throw(ArgumentError("$str is not a valid arrow file"))
+
+# will detect whether we're reading a Table from a file or stream
+function Table(bytes::Vector{UInt8}, off::Integer=1, tlen::Union{Integer, Nothing}=nothing; convert::Bool=true)
+    len = something(tlen, length(bytes))
+    if len > 24 &&
+        _startswith(bytes, off, FILE_FORMAT_MAGIC_BYTES) &&
+        _endswith(bytes, off + len - 1, FILE_FORMAT_MAGIC_BYTES)
+        off += 8 # skip past magic bytes + padding
+    end
+    t = Table()
+    sch = nothing
+    dictencodings = Dict{Int64, DictEncoding}() # dictionary id => DictEncoding
+    dictencoded = Dict{Int64, Meta.Field}() # dictionary id => field
+    tsks = Channel{Task}(Inf)
+    tsk = Threads.@spawn begin
+        i = 1
+        for tsk in tsks
+            cols = fetch(tsk)
+            if i == 1
+                foreach(x -> push!(columns(t), x), cols)
+            elseif i == 2
+                foreach(1:length(cols)) do i
+                    columns(t)[i] = ChainedVector([columns(t)[i], cols[i]])
+                end
+            else
+                foreach(1:length(cols)) do i
+                    append!(columns(t)[i], cols[i])
+                end
+            end
+            i += 1
+        end
+    end
+    for batch in BatchIterator(bytes, off)
+        # store custom_metadata of batch.msg?
+        header = batch.msg.header
+        if header isa Meta.Schema
+            @debug 1 "parsing schema message"
+            # assert endianness?
+            # store custom_metadata?
+            for (i, field) in enumerate(header.fields)
+                push!(names(t), Symbol(field.name))
+                # recursively find any dictionaries for any fields
+                getdictionaries!(dictencoded, field)
+                @debug 1 "parsed column from schema: field = $field"
+            end
+            sch = header
+            schema(t)[] = sch
+        elseif header isa Meta.DictionaryBatch
+            id = header.id
+            recordbatch = header.data
+            @debug 1 "parsing dictionary batch message: id = $id, compression = $(recordbatch.compression)"
+            if haskey(dictencodings, id) && header.isDelta
+                # delta
+                field = dictencoded[id]
+                values, _, _ = build(field, field.type, batch, recordbatch, dictencodings, Int64(1), Int64(1), convert)
+                dictencoding = dictencodings[id]
+                append!(dictencoding.data, values)
+                continue
+            end
+            # new dictencoding or replace
+            field = dictencoded[id]
+            values, _, _ = build(field, field.type, batch, recordbatch, dictencodings, Int64(1), Int64(1), convert)
+            A = ChainedVector([values])
+            dictencodings[id] = DictEncoding{eltype(A), typeof(A)}(id, A, field.dictionary.isOrdered)
+            @debug 1 "parsed dictionary batch message: id=$id, data=$values\n"
+        elseif header isa Meta.RecordBatch
+            @debug 1 "parsing record batch message: compression = $(header.compression)"
+            put!(tsks, Threads.@spawn begin
+                collect(VectorIterator(sch, batch, dictencodings, convert))
+            end)
+        else
+            throw(ArgumentError("unsupported arrow message type: $(typeof(header))"))
+        end
+    end
+    close(tsks)
+    wait(tsk)
+    lu = lookup(t)
+    ty = types(t)
+    for (nm, col) in zip(names(t), columns(t))
+        lu[nm] = col
+        push!(ty, eltype(col))
+    end
+    meta = sch !== nothing ? sch.custom_metadata : nothing
+    if meta !== nothing
+        setmetadata!(t, Dict(String(kv.key) => String(kv.value) for kv in meta))
+    end
+    return t
+end
+
+function getdictionaries!(dictencoded, field)
+    d = field.dictionary
+    if d !== nothing
+        dictencoded[d.id] = field
+    end
+    for child in field.children
+        getdictionaries!(dictencoded, child)
+    end
+    return
+end
+
+struct Batch
+    msg::Meta.Message
+    bytes::Vector{UInt8}
+    pos::Int
+    id::Int
+end
+
+function Base.iterate(x::BatchIterator, (pos, id)=(x.startpos, 0))
+    @debug 1 "checking for next arrow message: pos = $pos"
+    if pos + 3 > length(x.bytes)
+        @debug 1 "not enough bytes left for another batch message"
+        return nothing
+    end
+    if readbuffer(x.bytes, pos, UInt32) != CONTINUATION_INDICATOR_BYTES
+        @debug 1 "didn't find continuation byte to keep parsing messages: $(readbuffer(x.bytes, pos, UInt32))"
+        return nothing
+    end
+    pos += 4
+    if pos + 3 > length(x.bytes)
+        @debug 1 "not enough bytes left to read length of another batch message"
+        return nothing
+    end
+    msglen = readbuffer(x.bytes, pos, Int32)
+    if msglen == 0
+        @debug 1 "message has 0 length; terminating message parsing"
+        return nothing
+    end
+    pos += 4
+    msg = FlatBuffers.getrootas(Meta.Message, x.bytes, pos-1)
+    pos += msglen
+    # pos now points to message body
+    @debug 1 "parsing message: pos = $pos, msglen = $msglen, bodyLength = $(msg.bodyLength)"
+    return Batch(msg, x.bytes, pos, id), (pos + msg.bodyLength, id + 1)
+end
+
+struct VectorIterator
+    schema::Meta.Schema
+    batch::Batch # batch.msg.header MUST BE RecordBatch
+    dictencodings::Dict{Int64, DictEncoding}
+    convert::Bool
+end
+
+buildmetadata(f::Meta.Field) = buildmetadata(f.custom_metadata)
+buildmetadata(meta) = Dict(String(kv.key) => String(kv.value) for kv in meta)
+buildmetadata(::Nothing) = nothing
+
+function Base.iterate(x::VectorIterator, (columnidx, nodeidx, bufferidx)=(Int64(1), Int64(1), Int64(1)))
+    columnidx > length(x.schema.fields) && return nothing
+    field = x.schema.fields[columnidx]
+    @debug 2 "building top-level column: field = $(field), columnidx = $columnidx, nodeidx = $nodeidx, bufferidx = $bufferidx"
+    A, nodeidx, bufferidx = build(field, x.batch, x.batch.msg.header, x.dictencodings, nodeidx, bufferidx, x.convert)
+    @debug 2 "built top-level column: A = $(typeof(A)), columnidx = $columnidx, nodeidx = $nodeidx, bufferidx = $bufferidx"
+    @debug 3 A
+    return A, (columnidx + 1, nodeidx, bufferidx)
+end
+
+Base.length(x::VectorIterator) = length(x.schema.fields)
+
+const ListTypes = Union{Meta.Utf8, Meta.LargeUtf8, Meta.Binary, Meta.LargeBinary, Meta.List, Meta.LargeList}
+const LargeLists = Union{Meta.LargeUtf8, Meta.LargeBinary, Meta.LargeList}
+
+function build(field::Meta.Field, batch, rb, de, nodeidx, bufferidx, convert)
+    d = field.dictionary
+    if d !== nothing
+        validity = buildbitmap(batch, rb, nodeidx, bufferidx)
+        bufferidx += 1
+        buffer = rb.buffers[bufferidx]
+        S = d.indexType === nothing ? Int32 : juliaeltype(field, d.indexType, false)
+        bytes, indices = reinterp(S, batch, buffer, rb.compression)
+        encoding = de[d.id]
+        A = DictEncoded(bytes, validity, indices, encoding, buildmetadata(field.custom_metadata))
+        nodeidx += 1
+        bufferidx += 1
+    else
+        A, nodeidx, bufferidx = build(field, field.type, batch, rb, de, nodeidx, bufferidx, convert)
+    end
+    return A, nodeidx, bufferidx
+end
+
+function buildbitmap(batch, rb, nodeidx, bufferidx)
+    buffer = rb.buffers[bufferidx]
+    voff = batch.pos + buffer.offset
+    node = rb.nodes[nodeidx]
+    if rb.compression === nothing
+        return ValidityBitmap(batch.bytes, voff, node.length, node.null_count)
+    else
+        # compressed
+        ptr = pointer(batch.bytes, voff)
+        _, decodedbytes = uncompress(ptr, buffer, rb.compression)
+        return ValidityBitmap(decodedbytes, 1, node.length, node.null_count)
+    end
+end
+
+function uncompress(ptr::Ptr{UInt8}, buffer, compression)
+    if buffer.length == 0
+        return 0, UInt8[]
+    end
+    len = unsafe_load(convert(Ptr{Int64}, ptr))
+    ptr += 8 # skip past uncompressed length as Int64
+    encodedbytes = unsafe_wrap(Array, ptr, buffer.length - 8)
+    if compression.codec === Meta.CompressionType.LZ4_FRAME
+        decodedbytes = transcode(LZ4FrameDecompressor, encodedbytes)
+    elseif compression.codec === Meta.CompressionType.ZSTD
+        decodedbytes = transcode(ZstdDecompressor, encodedbytes)
+    else
+        error("unsupported compression type when reading arrow buffers: $(typeof(compression.codec))")
+    end
+    return len, decodedbytes
+end
+
+function reinterp(::Type{T}, batch, buf, compression) where {T}
+    ptr = pointer(batch.bytes, batch.pos + buf.offset)
+    if compression === nothing
+        return batch.bytes, unsafe_wrap(Array, convert(Ptr{T}, ptr), div(buf.length, sizeof(T)))
+    else
+        # compressed
+        len, decodedbytes = uncompress(ptr, buf, compression)
+        return decodedbytes, unsafe_wrap(Array, convert(Ptr{T}, pointer(decodedbytes)), div(len, sizeof(T)))
+    end
+end
+
+function build(f::Meta.Field, L::ListTypes, batch, rb, de, nodeidx, bufferidx, convert)
+    @debug 2 "building array: L = $L"
+    validity = buildbitmap(batch, rb, nodeidx, bufferidx)
+    bufferidx += 1
+    buffer = rb.buffers[bufferidx]
+    ooff = batch.pos + buffer.offset
+    OT = L isa LargeLists ? Int64 : Int32
+    bytes, offs = reinterp(OT, batch, buffer, rb.compression)
+    offsets = Offsets(bytes, offs)
+    bufferidx += 1
+    len = rb.nodes[nodeidx].length
+    nodeidx += 1
+    if L isa Meta.Utf8 || L isa Meta.LargeUtf8 || L isa Meta.Binary || L isa Meta.LargeBinary
+        buffer = rb.buffers[bufferidx]
+        bytes, A = reinterp(UInt8, batch, buffer, rb.compression)
+        bufferidx += 1
+    else
+        bytes = UInt8[]
+        A, nodeidx, bufferidx = build(f.children[1], batch, rb, de, nodeidx, bufferidx, convert)
+    end
+    meta = buildmetadata(f.custom_metadata)
+    T = juliaeltype(f, meta, convert)
+    return List{T, OT, typeof(A)}(bytes, validity, offsets, A, len, meta), nodeidx, bufferidx
+end
+
+function build(f::Meta.Field, L::Union{Meta.FixedSizeBinary, Meta.FixedSizeList}, batch, rb, de, nodeidx, bufferidx, convert)
+    @debug 2 "building array: L = $L"
+    validity = buildbitmap(batch, rb, nodeidx, bufferidx)
+    bufferidx += 1
+    len = rb.nodes[nodeidx].length
+    nodeidx += 1
+    if L isa Meta.FixedSizeBinary
+        buffer = rb.buffers[bufferidx]
+        bytes, A = reinterp(UInt8, batch, buffer, rb.compression)
+        bufferidx += 1
+    else
+        bytes = UInt8[]
+        A, nodeidx, bufferidx = build(f.children[1], batch, rb, de, nodeidx, bufferidx, convert)
+    end
+    meta = buildmetadata(f.custom_metadata)
+    T = juliaeltype(f, meta, convert)
+    return FixedSizeList{T, typeof(A)}(bytes, validity, A, len, meta), nodeidx, bufferidx
+end
+
+function build(f::Meta.Field, L::Meta.Map, batch, rb, de, nodeidx, bufferidx, convert)
+    @debug 2 "building array: L = $L"
+    validity = buildbitmap(batch, rb, nodeidx, bufferidx)
+    bufferidx += 1
+    buffer = rb.buffers[bufferidx]
+    ooff = batch.pos + buffer.offset
+    OT = Int32
+    bytes, offs = reinterp(OT, batch, buffer, rb.compression)
+    offsets = Offsets(bytes, offs)
+    bufferidx += 1
+    len = rb.nodes[nodeidx].length
+    nodeidx += 1
+    A, nodeidx, bufferidx = build(f.children[1], batch, rb, de, nodeidx, bufferidx, convert)
+    meta = buildmetadata(f.custom_metadata)
+    T = juliaeltype(f, meta, convert)
+    return Map{T, OT, typeof(A)}(validity, offsets, A, len, meta), nodeidx, bufferidx
+end
+
+function build(f::Meta.Field, L::Meta.Struct, batch, rb, de, nodeidx, bufferidx, convert)
+    @debug 2 "building array: L = $L"
+    validity = buildbitmap(batch, rb, nodeidx, bufferidx)
+    bufferidx += 1
+    len = rb.nodes[nodeidx].length
+    vecs = []
+    nodeidx += 1
+    for child in f.children
+        A, nodeidx, bufferidx = build(child, batch, rb, de, nodeidx, bufferidx, convert)
+        push!(vecs, A)
+    end
+    data = Tuple(vecs)
+    meta = buildmetadata(f.custom_metadata)
+    T = juliaeltype(f, meta, convert)
+    return Struct{T, typeof(data)}(validity, data, len, meta), nodeidx, bufferidx
+end
+
+function build(f::Meta.Field, L::Meta.Union, batch, rb, de, nodeidx, bufferidx, convert)
+    @debug 2 "building array: L = $L"
+    buffer = rb.buffers[bufferidx]
+    bytes, typeIds = reinterp(UInt8, batch, buffer, rb.compression)
+    bufferidx += 1
+    if L.mode == Meta.UnionMode.Dense
+        buffer = rb.buffers[bufferidx]
+        bytes2, offsets = reinterp(Int32, batch, buffer, rb.compression)
+        bufferidx += 1
+    end
+    vecs = []
+    nodeidx += 1
+    for child in f.children
+        A, nodeidx, bufferidx = build(child, batch, rb, de, nodeidx, bufferidx, convert)
+        push!(vecs, A)
+    end
+    data = Tuple(vecs)
+    meta = buildmetadata(f.custom_metadata)
+    T = juliaeltype(f, meta, convert)
+    if L.mode == Meta.UnionMode.Dense
+        B = DenseUnion{T, typeof(data)}(bytes, bytes2, typeIds, offsets, data, meta)
+    else
+        B = SparseUnion{T, typeof(data)}(bytes, typeIds, data, meta)
+    end
+    return B, nodeidx, bufferidx
+end
+
+function build(f::Meta.Field, L::Meta.Null, batch, rb, de, nodeidx, bufferidx, convert)
+    @debug 2 "building array: L = $L"
+    return MissingVector(rb.nodes[nodeidx].length), nodeidx + 1, bufferidx
+end
+
+# primitives
+function build(f::Meta.Field, ::L, batch, rb, de, nodeidx, bufferidx, convert) where {L}
+    @debug 2 "building array: L = $L"
+    validity = buildbitmap(batch, rb, nodeidx, bufferidx)
+    bufferidx += 1
+    buffer = rb.buffers[bufferidx]
+    meta = buildmetadata(f.custom_metadata)
+    # get storage type (non-converted)
+    T = juliaeltype(f, nothing, false)
+    @debug 2 "storage type for primitive: T = $T"
+    bytes, A = reinterp(Base.nonmissingtype(T), batch, buffer, rb.compression)
+    len = rb.nodes[nodeidx].length
+    T = juliaeltype(f, meta, convert)
+    @debug 2 "final julia type for primitive: T = $T"
+    return Primitive(T, bytes, validity, A, len, meta), nodeidx + 1, bufferidx + 1
+end
+
+function build(f::Meta.Field, L::Meta.Bool, batch, rb, de, nodeidx, bufferidx, convert)
+    @debug 2 "building array: L = $L"
+    validity = buildbitmap(batch, rb, nodeidx, bufferidx)
+    bufferidx += 1
+    buffer = rb.buffers[bufferidx]
+    meta = buildmetadata(f.custom_metadata)
+    # get storage type (non-converted)
+    T = juliaeltype(f, nothing, false)
+    @debug 2 "storage type for primitive: T = $T"
+    buffer = rb.buffers[bufferidx]
+    voff = batch.pos + buffer.offset
+    node = rb.nodes[nodeidx]
+    if rb.compression === nothing
+        decodedbytes = batch.bytes
+        pos = voff
+        # return ValidityBitmap(batch.bytes, voff, node.length, node.null_count)
+    else
+        # compressed
+        ptr = pointer(batch.bytes, voff)
+        _, decodedbytes = uncompress(ptr, buffer, rb.compression)
+        pos = 1
+        # return ValidityBitmap(decodedbytes, 1, node.length, node.null_count)
+    end
+    len = rb.nodes[nodeidx].length
+    T = juliaeltype(f, meta, convert)
+    return BoolVector{T}(decodedbytes, pos, validity, len, meta), nodeidx + 1, bufferidx + 1
+end
diff --git a/julia/Arrow/src/utils.jl b/julia/Arrow/src/utils.jl
new file mode 100644
index 00000000000..d1b851bcccd
--- /dev/null
+++ b/julia/Arrow/src/utils.jl
@@ -0,0 +1,197 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Determines the total number of bytes needed to store `n` bytes with padding.
+# Note that the Arrow standard requires buffers to be aligned to 8-byte boundaries.
+padding(n::Integer, alignment) = ((n + alignment - 1) ÷ alignment) * alignment
+
+paddinglength(n::Integer, alignment) = padding(n, alignment) - n
+
+function writezeros(io::IO, n::Integer)
+    s = 0
+    for i ∈ 1:n
+        s += Base.write(io, 0x00)
+    end
+    s
+end
+
+# efficient writing of arrays
+writearray(io, col) = writearray(io, maybemissing(eltype(col)), col)
+
+function writearray(io::IO, ::Type{T}, col) where {T}
+    if col isa Vector{T}
+        n = Base.write(io, col)
+    elseif isbitstype(T) && (col isa Vector{Union{T, Missing}} || col isa SentinelVector{T, T, Missing, Vector{T}})
+        # need to write the non-selector bytes of isbits Union Arrays
+        n = Base.unsafe_write(io, pointer(col), sizeof(T) * length(col))
+    elseif col isa ChainedVector
+        n = 0
+        for A in col.arrays
+            n += writearray(io, T, A)
+        end
+    else
+        n = 0
+        for x in col
+            n += Base.write(io, coalesce(x, ArrowTypes.default(T)))
+        end
+    end
+    return n
+end
+
+getbit(v::UInt8, n::Integer) = Bool((v & 0x02^(n - 1)) >> (n - 1))
+
+function setbit(v::UInt8, b::Bool, n::Integer)
+    if b
+        v | 0x02^(n - 1)
+    else
+        v & (0xff ⊻ 0x02^(n - 1))
+    end
+end
+
+# Determines the number of bytes used by `n` bits, optionally with padding.
+function bitpackedbytes(n::Integer, alignment)
+    ℓ = cld(n, 8)
+    return ℓ + paddinglength(ℓ, alignment)
+end
+
+# count # of missing elements in an iterable
+nullcount(col) = count(ismissing, col)
+
+# like startswith/endswith for strings, but on byte buffers
+function _startswith(a::AbstractVector{UInt8}, pos::Integer, b::AbstractVector{UInt8})
+    for i = 1:length(b)
+        @inbounds check = a[pos + i - 1] == b[i]
+        check || return false
+    end
+    return true
+end
+
+function _endswith(a::AbstractVector{UInt8}, endpos::Integer, b::AbstractVector{UInt8})
+    aoff = endpos - length(b) + 1
+    for i = 1:length(b)
+        @inbounds check = a[aoff] == b[i]
+        check || return false
+        aoff += 1
+    end
+    return true
+end
+
+# read a single element from a byte vector
+# copied from read(::IOBuffer, T) in Base
+function readbuffer(t::AbstractVector{UInt8}, pos::Integer, ::Type{T}) where {T}
+    GC.@preserve t begin
+        ptr::Ptr{T} = pointer(t, pos)
+        x = unsafe_load(ptr)
+    end
+end
+
+# given a number of unique values; what dict encoding _index_ type is most appropriate
+encodingtype(n) = n < div(typemax(Int8), 2) ? Int8 : n < div(typemax(Int16), 2) ? Int16 : n < div(typemax(Int32), 2) ? Int32 : Int64
+
+# lazily call convert(T, x) on getindex for each x in data
+struct Converter{T, A} <: AbstractVector{T}
+    data::A
+end
+
+converter(::Type{T}, x::A) where {T, A} = Converter{eltype(A) >: Missing ? Union{T, Missing} : T, A}(x)
+converter(::Type{T}, x::ChainedVector{A}) where {T, A} = ChainedVector([converter(T, x) for x in x.arrays])
+
+Base.IndexStyle(::Type{<:Converter}) = Base.IndexLinear()
+Base.size(x::Converter) = (length(x.data),)
+Base.eltype(x::Converter{T, A}) where {T, A} = T
+Base.getindex(x::Converter{T}, i::Int) where {T} = ArrowTypes.arrowconvert(T, getindex(x.data, i))
+
+maybemissing(::Type{T}) where {T} = T === Missing ? Missing : Base.nonmissingtype(T)
+
+function getfooter(filebytes)
+    len = readbuffer(filebytes, length(filebytes) - 9, Int32)
+    FlatBuffers.getrootas(Meta.Footer, filebytes[end-(9 + len):end-10], 0)
+end
+
+function getrb(filebytes)
+    f = getfooter(filebytes)
+    rb = f.recordBatches[1]
+    return filebytes[rb.offset+1:(rb.offset+1+rb.metaDataLength)]
+    # FlatBuffers.getrootas(Meta.Message, filebytes, rb.offset)
+end
+
+function readmessage(filebytes, off=9)
+    @assert readbuffer(filebytes, off, UInt32) === 0xFFFFFFFF
+    len = readbuffer(filebytes, off + 4, Int32)
+
+    FlatBuffers.getrootas(Meta.Message, filebytes, off + 8)
+end
+
+# a custom Channel type that only allows put!-ing objects in a specific, monotonically increasing order
+struct OrderedChannel{T}
+    chan::Channel{T}
+    cond::Threads.Condition
+    i::Ref{Int}
+end
+
+OrderedChannel{T}(sz) where {T} = OrderedChannel{T}(Channel{T}(sz), Threads.Condition(), Ref(1))
+Base.iterate(ch::OrderedChannel, st...) = iterate(ch.chan, st...)
+
+macro lock(obj, expr)
+    esc(quote
+        lock($obj)
+        try
+            $expr
+        finally
+            unlock($obj)
+        end
+    end)
+end
+
+# when put!-ing an object, operation may have to wait until other tasks have put their
+# objects to ensure the channel is ordered correctly
+function Base.put!(ch::OrderedChannel{T}, x::T, i::Integer, incr::Bool=false) where {T}
+    @lock ch.cond begin
+        while ch.i[] < i
+            # channel index too early, need to wait for other tasks to put their objects first
+            wait(ch.cond)
+        end
+        # now it's our turn
+        put!(ch.chan, x)
+        if incr
+            ch.i[] += 1
+        end
+        # wake up tasks that may be waiting to put their objects
+        notify(ch.cond)
+    end
+    return
+end
+
+function Base.close(ch::OrderedChannel)
+    @lock ch.cond begin
+        # just need to ensure any tasks waiting to put their tasks have had a chance to put
+        while Base.n_waiters(ch.cond) > 0
+            wait(ch.cond)
+        end
+        close(ch.chan)
+    end
+    return
+end
+
+struct Lockable{T}
+    x::T
+    lock::ReentrantLock
+end
+
+Lockable(x::T) where {T} = Lockable{T}(x, ReentrantLock())
+
+Base.lock(x::Lockable) = lock(x.lock)
+Base.unlock(x::Lockable) = unlock(x.lock)
diff --git a/julia/Arrow/src/write.jl b/julia/Arrow/src/write.jl
new file mode 100644
index 00000000000..77d954f14ac
--- /dev/null
+++ b/julia/Arrow/src/write.jl
@@ -0,0 +1,454 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+const OBJ_METADATA = IdDict{Any, Dict{String, String}}()
+
+"""
+    Arrow.setmetadata!(x, metadata::Dict{String, String})
+
+Set the metadata for any object, provided as a `Dict{String, String}`.
+Metadata attached to a table or column will be serialized when written
+as a stream or file.
+"""
+function setmetadata!(x, meta::Dict{String, String})
+    OBJ_METADATA[x] = meta
+    return
+end
+
+"""
+    Arrow.getmetadata(x) => Dict{String, String}
+
+Retrieve any metadata (as a `Dict{String, String}`) attached to an object.
+
+Metadata may be attached to any object via [`Arrow.setmetadata!`](@ref),
+or deserialized via the arrow format directly (the format allows attaching metadata
+to table, column, and other objects).
+"""
+getmetadata(x, default=nothing) = get(OBJ_METADATA, x, default)
+
+"""
+    Arrow.write(io::IO, tbl)
+    Arrow.write(file::String, tbl)
+    tbl |> Arrow.write(io_or_file)
+
+Write any [Tables.jl](https://github.com/JuliaData/Tables.jl)-compatible `tbl` out as arrow formatted data.
+Providing an `io::IO` argument will cause the data to be written to it
+in the ["streaming" format](https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format), unless `file=true` keyword argument is passed.
+Providing a `file::String` argument will result in the ["file" format](https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format) being written.
+
+Multiple record batches will be written based on the number of
+`Tables.partitions(tbl)` that are provided; by default, this is just
+one for a given table, but some table sources support automatic
+partitioning. Note you can turn multiple table objects into partitions
+by doing `Tables.partitioner([tbl1, tbl2, ...])`, but note that
+each table must have the exact same `Tables.Schema`.
+
+By default, `Arrow.write` will use multiple threads to write multiple
+record batches simultaneously (e.g. if julia is started with `julia -t 8` or the `JULIA_NUM_THREADS` environment variable is set).
+
+Supported keyword arguments to `Arrow.write` include:
+  * `compress`: possible values include `:lz4`, `:zstd`, or your own initialized `LZ4FrameCompressor` or `ZstdCompressor` objects; will cause all buffers in each record batch to use the respective compression encoding
+  * `alignment::Int=8`: specify the number of bytes to align buffers to when written in messages; strongly recommended to only use alignment values of 8 or 64 for modern memory cache line optimization
+  * `dictencode::Bool=false`: whether all columns should use dictionary encoding when being written; to dict encode specific columns, wrap the column/array in `Arrow.DictEncode(col)`
+  * `dictencodenested::Bool=false`: whether nested data type columns should also dict encode nested arrays/buffers; other language implementations [may not support this](https://arrow.apache.org/docs/status.html)
+  * `denseunions::Bool=true`: whether Julia `Vector{<:Union}` arrays should be written using the dense union layout; passing `false` will result in the sparse union layout
+  * `largelists::Bool=false`: causes list column types to be written with Int64 offset arrays; mainly for testing purposes; by default, Int64 offsets will be used only if needed
+  * `file::Bool=false`: if a an `io` argument is being written to, passing `file=true` will cause the arrow file format to be written instead of just IPC streaming
+"""
+function write end
+
+write(io_or_file; kw...) = x -> write(io_or_file, x; kw...)
+
+function write(file::String, tbl; largelists::Bool=false, compress::Union{Nothing, Symbol, LZ4FrameCompressor, ZstdCompressor}=nothing, denseunions::Bool=true, dictencode::Bool=false, dictencodenested::Bool=false, alignment::Int=8)
+    open(file, "w") do io
+        write(io, tbl, true, largelists, compress, denseunions, dictencode, dictencodenested, alignment)
+    end
+    return file
+end
+
+function write(io::IO, tbl; largelists::Bool=false, compress::Union{Nothing, Symbol, LZ4FrameCompressor, ZstdCompressor}=nothing, denseunions::Bool=true, dictencode::Bool=false, dictencodenested::Bool=false, alignment::Int=8, file::Bool=false)
+    return write(io, tbl, file, largelists, compress, denseunions, dictencode, dictencodenested, alignment)
+end
+
+function write(io, source, writetofile, largelists, compress, denseunions, dictencode, dictencodenested, alignment)
+    if compress === :lz4
+        compress = LZ4_FRAME_COMPRESSOR[]
+    elseif compress === :zstd
+        compress = ZSTD_COMPRESSOR[]
+    elseif compress isa Symbol
+        throw(ArgumentError("unsupported compress keyword argument value: $compress. Valid values include `:lz4` or `:zstd`"))
+    end
+    if writetofile
+        @debug 1 "starting write of arrow formatted file"
+        Base.write(io, "ARROW1\0\0")
+    end
+    msgs = OrderedChannel{Message}(Inf)
+    # build messages
+    sch = Ref{Tables.Schema}()
+    firstcols = Ref{Any}()
+    dictencodings = Dict{Int64, Any}() # Lockable{DictEncoding}
+    blocks = (Block[], Block[])
+    # start message writing from channel
+    tsk = Threads.@spawn for msg in msgs
+        Base.write(io, msg, blocks, sch, alignment)
+    end
+    @sync for (i, tbl) in enumerate(Tables.partitions(source))
+        @debug 1 "processing table partition i = $i"
+        if i == 1
+            cols = toarrowtable(tbl, dictencodings, largelists, compress, denseunions, dictencode, dictencodenested)
+            sch[] = Tables.schema(cols)
+            firstcols[] = cols
+            put!(msgs, makeschemamsg(sch[], cols), i)
+            if !isempty(dictencodings)
+                des = sort!(collect(dictencodings); by=x->x.first, rev=true)
+                for (id, delock) in des
+                    # assign dict encoding ids
+                    de = delock.x
+                    dictsch = Tables.Schema((:col,), (eltype(de.data),))
+                    put!(msgs, makedictionarybatchmsg(dictsch, (col=de.data,), id, false, alignment), i)
+                end
+            end
+            put!(msgs, makerecordbatchmsg(sch[], cols, alignment), i, true)
+        else
+            Threads.@spawn begin
+                cols = toarrowtable(tbl, dictencodings, largelists, compress, denseunions, dictencode, dictencodenested)
+                if !isempty(cols.dictencodingdeltas)
+                    for de in cols.dictencodingdeltas
+                        dictsch = Tables.Schema((:col,), (eltype(de.data),))
+                        put!(msgs, makedictionarybatchmsg(dictsch, (col=de.data,), de.id, true, alignment), i)
+                    end
+                end
+                put!(msgs, makerecordbatchmsg(sch[], cols, alignment), i, true)
+            end
+        end
+    end
+    # close our message-writing channel, no further put!-ing is allowed
+    close(msgs)
+    # now wait for our message-writing task to finish writing
+    wait(tsk)
+    # write empty message
+    if !writetofile
+        Base.write(io, Message(UInt8[], nothing, 0, true, false), blocks, sch, alignment)
+    end
+    if writetofile
+        b = FlatBuffers.Builder(1024)
+        schfoot = makeschema(b, sch[], firstcols[])
+        if !isempty(blocks[1])
+            N = length(blocks[1])
+            Meta.footerStartRecordBatchesVector(b, N)
+            for blk in Iterators.reverse(blocks[1])
+                Meta.createBlock(b, blk.offset, blk.metaDataLength, blk.bodyLength)
+            end
+            recordbatches = FlatBuffers.endvector!(b, N)
+        else
+            recordbatches = FlatBuffers.UOffsetT(0)
+        end
+        if !isempty(blocks[2])
+            N = length(blocks[2])
+            Meta.footerStartDictionariesVector(b, N)
+            for blk in Iterators.reverse(blocks[2])
+                Meta.createBlock(b, blk.offset, blk.metaDataLength, blk.bodyLength)
+            end
+            dicts = FlatBuffers.endvector!(b, N)
+        else
+            dicts = FlatBuffers.UOffsetT(0)
+        end
+        Meta.footerStart(b)
+        Meta.footerAddVersion(b, Meta.MetadataVersion.V4)
+        Meta.footerAddSchema(b, schfoot)
+        Meta.footerAddDictionaries(b, dicts)
+        Meta.footerAddRecordBatches(b, recordbatches)
+        foot = Meta.footerEnd(b)
+        FlatBuffers.finish!(b, foot)
+        footer = FlatBuffers.finishedbytes(b)
+        Base.write(io, footer)
+        Base.write(io, Int32(length(footer)))
+        Base.write(io, "ARROW1")
+    end
+    return io
+end
+
+struct ToArrowTable
+    sch::Tables.Schema
+    cols::Vector{Any}
+    metadata::Union{Nothing, Dict{String, String}}
+    dictencodingdeltas::Vector{DictEncoding}
+end
+
+function toarrowtable(x, dictencodings, largelists, compress, denseunions, dictencode, dictencodenested)
+    @debug 1 "converting input table to arrow formatted columns"
+    cols = Tables.columns(x)
+    meta = getmetadata(cols)
+    sch = Tables.schema(cols)
+    types = collect(sch.types)
+    N = length(types)
+    newcols = Vector{Any}(undef, N)
+    newtypes = Vector{Type}(undef, N)
+    dictencodingdeltas = DictEncoding[]
+    Tables.eachcolumn(sch, cols) do col, i, nm
+        newcol = toarrowvector(col, i, dictencodings, dictencodingdeltas; compression=compress, largelists=largelists, denseunions=denseunions, dictencode=dictencode, dictencodenested=dictencodenested)
+        newtypes[i] = eltype(newcol)
+        newcols[i] = newcol
+    end
+    return ToArrowTable(Tables.Schema(sch.names, newtypes), newcols, meta, dictencodingdeltas)
+end
+
+Tables.columns(x::ToArrowTable) = x
+Tables.rowcount(x::ToArrowTable) = length(x.cols) == 0 ? 0 : length(x.cols[1])
+Tables.schema(x::ToArrowTable) = x.sch
+Tables.columnnames(x::ToArrowTable) = x.sch.names
+Tables.getcolumn(x::ToArrowTable, i::Int) = x.cols[i]
+
+struct Message
+    msgflatbuf
+    columns
+    bodylen
+    isrecordbatch::Bool
+    blockmsg::Bool
+end
+
+struct Block
+    offset::Int64
+    metaDataLength::Int32
+    bodyLength::Int64
+end
+
+function Base.write(io::IO, msg::Message, blocks, sch, alignment)
+    metalen = padding(length(msg.msgflatbuf), alignment)
+    @debug 1 "writing message: metalen = $metalen, bodylen = $(msg.bodylen), isrecordbatch = $(msg.isrecordbatch)"
+    if msg.blockmsg
+        push!(blocks[msg.isrecordbatch ? 1 : 2], Block(position(io), metalen + 8, msg.bodylen))
+    end
+    # now write the final message spec out
+    # continuation byte
+    n = Base.write(io, 0xFFFFFFFF)
+    # metadata length
+    n += Base.write(io, Int32(metalen))
+    # message flatbuffer
+    n += Base.write(io, msg.msgflatbuf)
+    n += writezeros(io, paddinglength(length(msg.msgflatbuf), alignment))
+    # message body
+    if msg.columns !== nothing
+        # write out buffers
+        for col in Tables.Columns(msg.columns)
+            writebuffer(io, col, alignment)
+        end
+    end
+    return n
+end
+
+function makemessage(b, headerType, header, columns=nothing, bodylen=0)
+    # write the message flatbuffer object
+    Meta.messageStart(b)
+    Meta.messageAddVersion(b, Meta.MetadataVersion.V5)
+    Meta.messageAddHeaderType(b, headerType)
+    Meta.messageAddHeader(b, header)
+    Meta.messageAddBodyLength(b, Int64(bodylen))
+    # Meta.messageAddCustomMetadata(b, meta)
+    # Meta.messageStartCustomMetadataVector(b, num_meta_elems)
+    msg = Meta.messageEnd(b)
+    FlatBuffers.finish!(b, msg)
+    return Message(FlatBuffers.finishedbytes(b), columns, bodylen, headerType == Meta.RecordBatch, headerType == Meta.RecordBatch || headerType == Meta.DictionaryBatch)
+end
+
+function makeschema(b, sch::Tables.Schema{names}, columns) where {names}
+    # build Field objects
+    N = length(names)
+    fieldoffsets = [fieldoffset(b, names[i], columns.cols[i]) for i = 1:N]
+    Meta.schemaStartFieldsVector(b, N)
+    for off in Iterators.reverse(fieldoffsets)
+        FlatBuffers.prependoffset!(b, off)
+    end
+    fields = FlatBuffers.endvector!(b, N)
+    if columns.metadata !== nothing
+        kvs = columns.metadata
+        kvoffs = Vector{FlatBuffers.UOffsetT}(undef, length(kvs))
+        for (i, (k, v)) in enumerate(kvs)
+            koff = FlatBuffers.createstring!(b, String(k))
+            voff = FlatBuffers.createstring!(b, String(v))
+            Meta.keyValueStart(b)
+            Meta.keyValueAddKey(b, koff)
+            Meta.keyValueAddValue(b, voff)
+            kvoffs[i] = Meta.keyValueEnd(b)
+        end
+        Meta.schemaStartCustomMetadataVector(b, length(kvs))
+        for off in Iterators.reverse(kvoffs)
+            FlatBuffers.prependoffset!(b, off)
+        end
+        meta = FlatBuffers.endvector!(b, length(kvs))
+    else
+        meta = FlatBuffers.UOffsetT(0)
+    end
+    # write schema object
+    Meta.schemaStart(b)
+    Meta.schemaAddEndianness(b, Meta.Endianness.Little)
+    Meta.schemaAddFields(b, fields)
+    Meta.schemaAddCustomMetadata(b, meta)
+    return Meta.schemaEnd(b)
+end
+
+function makeschemamsg(sch::Tables.Schema, columns)
+    @debug 1 "building schema message: sch = $sch"
+    b = FlatBuffers.Builder(1024)
+    schema = makeschema(b, sch, columns)
+    return makemessage(b, Meta.Schema, schema)
+end
+
+function fieldoffset(b, name, col)
+    nameoff = FlatBuffers.createstring!(b, String(name))
+    T = eltype(col)
+    nullable = T >: Missing
+    # check for custom metadata
+    if getmetadata(col) !== nothing
+        kvs = getmetadata(col)
+        kvoffs = Vector{FlatBuffers.UOffsetT}(undef, length(kvs))
+        for (i, (k, v)) in enumerate(kvs)
+            koff = FlatBuffers.createstring!(b, String(k))
+            voff = FlatBuffers.createstring!(b, String(v))
+            Meta.keyValueStart(b)
+            Meta.keyValueAddKey(b, koff)
+            Meta.keyValueAddValue(b, voff)
+            kvoffs[i] = Meta.keyValueEnd(b)
+        end
+        Meta.fieldStartCustomMetadataVector(b, length(kvs))
+        for off in Iterators.reverse(kvoffs)
+            FlatBuffers.prependoffset!(b, off)
+        end
+        meta = FlatBuffers.endvector!(b, length(kvs))
+    else
+        meta = FlatBuffers.UOffsetT(0)
+    end
+    # build dictionary
+    if isdictencoded(col)
+        encodingtype = indtype(col)
+        IT, inttype, _ = arrowtype(b, encodingtype)
+        Meta.dictionaryEncodingStart(b)
+        Meta.dictionaryEncodingAddId(b, Int64(getid(col)))
+        Meta.dictionaryEncodingAddIndexType(b, inttype)
+        # TODO: support isOrdered?
+        Meta.dictionaryEncodingAddIsOrdered(b, false)
+        dict = Meta.dictionaryEncodingEnd(b)
+    else
+        dict = FlatBuffers.UOffsetT(0)
+    end
+    type, typeoff, children = arrowtype(b, col)
+    if children !== nothing
+        Meta.fieldStartChildrenVector(b, length(children))
+        for off in Iterators.reverse(children)
+            FlatBuffers.prependoffset!(b, off)
+        end
+        children = FlatBuffers.endvector!(b, length(children))
+    else
+        Meta.fieldStartChildrenVector(b, 0)
+        children = FlatBuffers.endvector!(b, 0)
+    end
+    # build field object
+    if isdictencoded(col)
+        @debug 1 "building field: name = $name, nullable = $nullable, T = $T, type = $type, inttype = $IT, dictionary id = $(getid(col))"
+    else
+        @debug 1 "building field: name = $name, nullable = $nullable, T = $T, type = $type"
+    end
+    Meta.fieldStart(b)
+    Meta.fieldAddName(b, nameoff)
+    Meta.fieldAddNullable(b, nullable)
+    Meta.fieldAddTypeType(b, type)
+    Meta.fieldAddType(b, typeoff)
+    Meta.fieldAddDictionary(b, dict)
+    Meta.fieldAddChildren(b, children)
+    Meta.fieldAddCustomMetadata(b, meta)
+    return Meta.fieldEnd(b)
+end
+
+struct FieldNode
+    length::Int64
+    null_count::Int64
+end
+
+struct Buffer
+    offset::Int64
+    length::Int64
+end
+
+function makerecordbatchmsg(sch::Tables.Schema{names, types}, columns, alignment) where {names, types}
+    b = FlatBuffers.Builder(1024)
+    recordbatch, bodylen = makerecordbatch(b, sch, columns, alignment)
+    return makemessage(b, Meta.RecordBatch, recordbatch, columns, bodylen)
+end
+
+function makerecordbatch(b, sch::Tables.Schema{names, types}, columns, alignment) where {names, types}
+    nrows = Tables.rowcount(columns)
+    
+    compress = nothing
+    fieldnodes = FieldNode[]
+    fieldbuffers = Buffer[]
+    bufferoffset = 0
+    for col in Tables.Columns(columns)
+        if col isa Compressed
+            compress = compressiontype(col)
+        end
+        bufferoffset = makenodesbuffers!(col, fieldnodes, fieldbuffers, bufferoffset, alignment)
+    end
+    @debug 1 "building record batch message: nrows = $nrows, sch = $sch, compress = $compress"
+
+    # write field nodes objects
+    FN = length(fieldnodes)
+    Meta.recordBatchStartNodesVector(b, FN)
+    for fn in Iterators.reverse(fieldnodes)
+        Meta.createFieldNode(b, fn.length, fn.null_count)
+    end
+    nodes = FlatBuffers.endvector!(b, FN)
+
+    # write buffer objects
+    bodylen = 0
+    BN = length(fieldbuffers)
+    Meta.recordBatchStartBuffersVector(b, BN)
+    for buf in Iterators.reverse(fieldbuffers)
+        Meta.createBuffer(b, buf.offset, buf.length)
+        bodylen += padding(buf.length, alignment)
+    end
+    buffers = FlatBuffers.endvector!(b, BN)
+
+    # compression
+    if compress !== nothing
+        Meta.bodyCompressionStart(b)
+        Meta.bodyCompressionAddCodec(b, compress)
+        Meta.bodyCompressionAddMethod(b, Meta.BodyCompressionMethod.BUFFER)
+        compression = Meta.bodyCompressionEnd(b)
+    else
+        compression = FlatBuffers.UOffsetT(0)
+    end
+
+    # write record batch object
+    @debug 1 "built record batch message: nrows = $nrows, nodes = $fieldnodes, buffers = $fieldbuffers, compress = $compress, bodylen = $bodylen"
+    Meta.recordBatchStart(b)
+    Meta.recordBatchAddLength(b, Int64(nrows))
+    Meta.recordBatchAddNodes(b, nodes)
+    Meta.recordBatchAddBuffers(b, buffers)
+    Meta.recordBatchAddCompression(b, compression)
+    return Meta.recordBatchEnd(b), bodylen
+end
+
+function makedictionarybatchmsg(sch, columns, id, isdelta, alignment)
+    @debug 1 "building dictionary message: id = $id, sch = $sch, isdelta = $isdelta"
+    b = FlatBuffers.Builder(1024)
+    recordbatch, bodylen = makerecordbatch(b, sch, columns, alignment)
+    Meta.dictionaryBatchStart(b)
+    Meta.dictionaryBatchAddId(b, Int64(id))
+    Meta.dictionaryBatchAddData(b, recordbatch)
+    Meta.dictionaryBatchAddIsDelta(b, isdelta)
+    dictionarybatch = Meta.dictionaryBatchEnd(b)
+    return makemessage(b, Meta.DictionaryBatch, dictionarybatch, columns, bodylen)
+end
diff --git a/julia/Arrow/test/arrowjson.jl b/julia/Arrow/test/arrowjson.jl
new file mode 100644
index 00000000000..7921e1f11b7
--- /dev/null
+++ b/julia/Arrow/test/arrowjson.jl
@@ -0,0 +1,611 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module ArrowJSON
+
+using Mmap
+using StructTypes, JSON3, Tables, SentinelArrays, Arrow
+
+# read json files as "table"
+# write to arrow stream/file
+# read arrow stream/file back
+
+abstract type Type end
+Type() = Null("null")
+StructTypes.StructType(::Base.Type{Type}) = StructTypes.AbstractType()
+
+children(::Base.Type{T}) where {T} = Field[]
+
+mutable struct Int <: Type
+    name::String
+    bitWidth::Int64
+    isSigned::Base.Bool
+end
+
+Int() = Int("", 0, true)
+Type(::Base.Type{T}) where {T <: Integer} = Int("int", 8 * sizeof(T), T <: Signed)
+StructTypes.StructType(::Base.Type{Int}) = StructTypes.Mutable()
+function juliatype(f, x::Int)
+    T = x.bitWidth == 8 ? Int8 : x.bitWidth == 16 ? Int16 :
+        x.bitWidth == 32 ? Int32 : x.bitWidth == 64 ? Int64 : Int128
+    return x.isSigned ? T : unsigned(T)
+end
+
+struct FloatingPoint <: Type
+    name::String
+    precision::String
+end
+
+Type(::Base.Type{T}) where {T <: AbstractFloat} = FloatingPoint("floatingpoint", T == Float16 ? "HALF" : T == Float32 ? "SINGLE" : "DOUBLE")
+StructTypes.StructType(::Base.Type{FloatingPoint}) = StructTypes.Struct()
+juliatype(f, x::FloatingPoint) = x.precision == "HALF" ? Float16 : x.precision == "SINGLE" ? Float32 : Float64
+
+struct FixedSizeBinary <: Type
+    name::String
+    byteWidth::Int64
+end
+
+Type(::Base.Type{NTuple{N, UInt8}}) where {N} = FixedSizeBinary("fixedsizebinary", N)
+children(::Base.Type{NTuple{N, UInt8}}) where {N} = Field[]
+StructTypes.StructType(::Base.Type{FixedSizeBinary}) = StructTypes.Struct()
+juliatype(f, x::FixedSizeBinary) = NTuple{x.byteWidth, UInt8}
+
+struct Decimal <: Type
+    name::String
+    precision::Int32
+    scale::Int32
+end
+
+Type(::Base.Type{Arrow.Decimal{P, S}}) where {P, S} = Decimal("decimal", P, S)
+StructTypes.StructType(::Base.Type{Decimal}) = StructTypes.Struct()
+juliatype(f, x::Decimal) = Arrow.Decimal{x.precision, x.scale}
+
+mutable struct Timestamp <: Type
+    name::String
+    unit::String
+    timezone::Union{Nothing ,String}
+end
+
+Timestamp() = Timestamp("", "", nothing)
+unit(U) = U == Arrow.Meta.TimeUnit.SECOND ? "SECOND" :
+          U == Arrow.Meta.TimeUnit.MILLISECOND ? "MILLISECOND" :
+          U == Arrow.Meta.TimeUnit.MICROSECOND ? "MICROSECOND" : "NANOSECOND"
+Type(::Base.Type{Arrow.Timestamp{U, TZ}}) where {U, TZ} = Timestamp("timestamp", unit(U), TZ === nothing ? nothing : String(TZ))
+StructTypes.StructType(::Base.Type{Timestamp}) = StructTypes.Mutable()
+unitT(u) = u == "SECOND" ? Arrow.Meta.TimeUnit.SECOND :
+           u == "MILLISECOND" ? Arrow.Meta.TimeUnit.MILLISECOND :
+           u == "MICROSECOND" ? Arrow.Meta.TimeUnit.MICROSECOND : Arrow.Meta.TimeUnit.NANOSECOND
+juliatype(f, x::Timestamp) = Arrow.Timestamp{unitT(x.unit), x.timezone === nothing ? nothing : Symbol(x.timezone)}
+
+struct Duration <: Type
+    name::String
+    unit::String
+end
+
+Type(::Base.Type{Arrow.Duration{U}}) where {U} = Duration("duration", unit(U))
+StructTypes.StructType(::Base.Type{Duration}) = StructTypes.Struct()
+juliatype(f, x::Duration) = Arrow.Duration{unit%(x.unit)}
+
+struct Date <: Type
+    name::String
+    unit::String
+end
+
+Type(::Base.Type{Arrow.Date{U, T}}) where {U, T} = Date("date", U == Arrow.Meta.DateUnit.DAY ? "DAY" : "MILLISECOND")
+StructTypes.StructType(::Base.Type{Date}) = StructTypes.Struct()
+juliatype(f, x::Date) = Arrow.Date{x.unit == "DAY" ? Arrow.Meta.DateUnit.DAY : Arrow.Meta.DateUnit.MILLISECOND, x.unit == "DAY" ? Int32 : Int64}
+
+struct Time <: Type
+    name::String
+    unit::String
+    bitWidth::Int64
+end
+
+Type(::Base.Type{Arrow.Time{U, T}}) where {U, T} = Time("time", unit(U), 8 * sizeof(T))
+StructTypes.StructType(::Base.Type{Time}) = StructTypes.Struct()
+juliatype(f, x::Time) = Arrow.Time{unitT(x.unit), x.unit == "SECOND" || x.unit == "MILLISECOND" ? Int32 : Int64}
+
+struct Interval <: Type
+    name::String
+    unit::String
+end
+
+Type(::Base.Type{Arrow.Interval{U, T}}) where {U, T} = Interval("interval", U == Arrow.Meta.IntervalUnit.YEAR_MONTH ? "YEAR_MONTH" : "DAY_TIME")
+StructTypes.StructType(::Base.Type{Interval}) = StructTypes.Struct()
+juliatype(f, x::Interval) = Arrow.Interval{x.unit == "YEAR_MONTH" ? Arrow.Meta.IntervalUnit.YEAR_MONTH : Arrow.Meta.IntervalUnit.DAY_TIME, x.unit == "YEAR_MONTH" ? Int32 : Int64}
+
+struct UnionT <: Type
+    name::String
+    mode::String
+    typIds::Vector{Int64}
+end
+
+Type(::Base.Type{Arrow.UnionT{T, typeIds, U}}) where {T, typeIds, U} = UnionT("union", T == Arrow.Meta.UnionMode.Dense ? "DENSE" : "SPARSE", collect(typeIds))
+children(::Base.Type{Arrow.UnionT{T, typeIds, U}}) where {T, typeIds, U} = Field[Field("", fieldtype(U, i), nothing) for i = 1:fieldcount(U)]
+StructTypes.StructType(::Base.Type{UnionT}) = StructTypes.Struct()
+juliatype(f, x::UnionT) = Arrow.UnionT{x.mode == "DENSE" ? Arrow.Meta.UnionMode.DENSE : Arrow.Meta.UnionMode.SPARSE, Tuple(x.typeIds), Tuple{(juliatype(y) for y in f.children)...}}
+
+struct List <: Type
+    name::String
+end
+
+Type(::Base.Type{Vector{T}}) where {T} = List("list")
+children(::Base.Type{Vector{T}}) where {T} = [Field("item", T, nothing)]
+StructTypes.StructType(::Base.Type{List}) = StructTypes.Struct()
+juliatype(f, x::List) = Vector{juliatype(f.children[1])}
+
+struct LargeList <: Type
+    name::String
+end
+
+StructTypes.StructType(::Base.Type{LargeList}) = StructTypes.Struct()
+juliatype(f, x::LargeList) = Vector{juliatype(f.children[1])}
+
+struct FixedSizeList <: Type
+    name::String
+    listSize::Int64
+end
+
+Type(::Base.Type{NTuple{N, T}}) where {N, T} = FixedSizeList("fixedsizelist", N)
+children(::Base.Type{NTuple{N, T}}) where {N, T} = [Field("item", T, nothing)]
+StructTypes.StructType(::Base.Type{FixedSizeList}) = StructTypes.Struct()
+juliatype(f, x::FixedSizeList) = NTuple{x.listSize, juliatype(f.children[1])}
+
+struct Struct <: Type
+    name::String
+end
+
+Type(::Base.Type{NamedTuple{names, types}}) where {names, types} = Struct("struct")
+children(::Base.Type{NamedTuple{names, types}}) where {names, types} = [Field(names[i], fieldtype(types, i), nothing) for i = 1:length(names)]
+StructTypes.StructType(::Base.Type{Struct}) = StructTypes.Struct()
+juliatype(f, x::Struct) = NamedTuple{Tuple(Symbol(x.name) for x in f.children), Tuple{(juliatype(y) for y in f.children)...}}
+
+struct Map <: Type
+    name::String
+    keysSorted::Base.Bool
+end
+
+Type(::Base.Type{Dict{K, V}}) where {K, V} = Map("map", false)
+children(::Base.Type{Dict{K, V}}) where {K, V} = [Field("entries", Arrow.KeyValue{K, V}, nothing)]
+StructTypes.StructType(::Base.Type{Map}) = StructTypes.Struct()
+juliatype(f, x::Map) = Dict{juliatype(f.children[1].children[1]), juliatype(f.children[1].children[2])}
+
+Type(::Base.Type{Arrow.KeyValue{K, V}}) where {K, V} = Struct("struct")
+children(::Base.Type{Arrow.KeyValue{K, V}}) where {K, V} = [Field("key", K, nothing), Field("value", V, nothing)]
+
+struct Null <: Type
+    name::String
+end
+
+Type(::Base.Type{Missing}) = Null("null")
+StructTypes.StructType(::Base.Type{Null}) = StructTypes.Struct()
+juliatype(f, x::Null) = Missing
+
+struct Utf8 <: Type
+    name::String
+end
+
+Type(::Base.Type{<:String}) = Utf8("utf8")
+StructTypes.StructType(::Base.Type{Utf8}) = StructTypes.Struct()
+juliatype(f, x::Utf8) = String
+
+struct LargeUtf8 <: Type
+    name::String
+end
+
+StructTypes.StructType(::Base.Type{LargeUtf8}) = StructTypes.Struct()
+juliatype(f, x::LargeUtf8) = String
+
+struct Binary <: Type
+    name::String
+end
+
+Type(::Base.Type{Vector{UInt8}}) = Binary("binary")
+children(::Base.Type{Vector{UInt8}}) = Field[]
+StructTypes.StructType(::Base.Type{Binary}) = StructTypes.Struct()
+juliatype(f, x::Binary) = Vector{UInt8}
+
+struct LargeBinary <: Type
+    name::String
+end
+
+StructTypes.StructType(::Base.Type{LargeBinary}) = StructTypes.Struct()
+juliatype(f, x::LargeBinary) = Vector{UInt8}
+
+struct Bool <: Type
+    name::String
+end
+
+Type(::Base.Type{Base.Bool}) = Bool("bool")
+StructTypes.StructType(::Base.Type{Bool}) = StructTypes.Struct()
+juliatype(f, x::Bool) = Base.Bool
+
+StructTypes.subtypekey(::Base.Type{Type}) = :name
+
+const SUBTYPES = @eval (
+    int=Int,
+    floatingpoint=FloatingPoint,
+    fixedsizebinary=FixedSizeBinary,
+    decimal=Decimal,
+    timestamp=Timestamp,
+    duration=Duration,
+    date=Date,
+    time=Time,
+    interval=Interval,
+    union=UnionT,
+    list=List,
+    largelist=LargeList,
+    fixedsizelist=FixedSizeList,
+    $(Symbol("struct"))=Struct,
+    map=Map,
+    null=Null,
+    utf8=Utf8,
+    largeutf8=LargeUtf8,
+    binary=Binary,
+    largebinary=LargeBinary,
+    bool=Bool
+)
+
+StructTypes.subtypes(::Base.Type{Type}) = SUBTYPES
+
+const Metadata = Union{Nothing, Vector{NamedTuple{(:key, :value), Tuple{String, String}}}}
+Metadata() = nothing
+
+mutable struct DictEncoding
+    id::Int64
+    indexType::Type
+    isOrdered::Base.Bool
+end
+
+DictEncoding() = DictEncoding(0, Type(), false)
+StructTypes.StructType(::Base.Type{DictEncoding}) = StructTypes.Mutable()
+
+mutable struct Field
+    name::String
+    nullable::Base.Bool
+    type::Type
+    children::Vector{Field}
+    dictionary::Union{DictEncoding, Nothing}
+    metadata::Metadata
+end
+
+Field() = Field("", true, Type(), Field[], nothing, Metadata())
+StructTypes.StructType(::Base.Type{Field}) = StructTypes.Mutable()
+Base.copy(f::Field) = Field(f.name, f.nullable, f.type, f.children, f.dictionary, f.metadata)
+
+function juliatype(f::Field)
+    T = juliatype(f, f.type)
+    return f.nullable ? Union{T, Missing} : T
+end
+
+function Field(nm, ::Base.Type{T}, dictencodings) where {T}
+    S = Arrow.maybemissing(T)
+    type = Type(S)
+    ch = children(S)
+    if dictencodings !== nothing && haskey(dictencodings, nm)
+        dict = dictencodings[nm]
+    else
+        dict = nothing
+    end
+    return Field(nm, T !== S, type, ch, dict, nothing)
+end
+
+mutable struct Schema
+    fields::Vector{Field}
+    metadata::Metadata
+end
+
+Schema() = Schema(Field[], Metadata())
+StructTypes.StructType(::Base.Type{Schema}) = StructTypes.Mutable()
+
+struct Offsets{T} <: AbstractVector{T}
+    data::Vector{T}
+end
+
+Base.size(x::Offsets) = size(x.data)
+Base.getindex(x::Offsets, i::Base.Int) = getindex(x.data, i)
+
+mutable struct FieldData
+    name::String
+    count::Int64
+    VALIDITY::Union{Nothing, Vector{Int8}}
+    OFFSET::Union{Nothing, Offsets}
+    TYPE_ID::Union{Nothing, Vector{Int8}}
+    DATA::Union{Nothing, Vector{Any}}
+    children::Vector{FieldData}
+end
+
+FieldData() = FieldData("", 0, nothing, nothing, nothing, nothing, FieldData[])
+StructTypes.StructType(::Base.Type{FieldData}) = StructTypes.Mutable()
+
+function FieldData(nm, ::Base.Type{T}, col, dictencodings) where {T}
+    if dictencodings !== nothing && haskey(dictencodings, nm)
+        refvals = DataAPI.refarray(col.data)
+        if refvals !== col.data
+            IT = eltype(refvals)
+            col = (x - one(T) for x in refvals)
+        else
+            _, de = dictencodings[nm]
+            IT = de.indexType
+            vals = unique(col)
+            col = Arrow.DictEncoder(col, vals, Arrow.encodingtype(length(vals)))
+        end
+        return FieldData(nm, IT, col, nothing)
+    end
+    S = Arrow.maybemissing(T)
+    len = Arrow._length(col)
+    VALIDITY = OFFSET = TYPE_ID = DATA = nothing
+    children = FieldData[]
+    if S <: Pair
+        return FieldData(nm, Vector{Arrow.KeyValue{Arrow._keytype(S), Arrow._valtype(S)}}, (Arrow.KeyValue(k, v) for (k, v) in pairs(col)))
+    elseif S !== Missing
+        # VALIDITY
+        VALIDITY = Int8[!ismissing(x) for x in col]
+        # OFFSET
+        if S <: Vector || S == String
+            lenfun = S == String ? x->ismissing(x) ? 0 : sizeof(x) : x->ismissing(x) ? 0 : length(x)
+            tot = sum(lenfun, col)
+            if tot > 2147483647
+                OFFSET = String[String(lenfun(x)) for x in col]
+                pushfirst!(OFFSET, "0")
+            else
+                OFFSET = Int32[ismissing(x) ? 0 : lenfun(x) for x in col]
+                pushfirst!(OFFSET, 0)
+            end
+            OFFSET = Offsets(OFFSET)
+            push!(children, FieldData("item", eltype(S), Arrow.flatten(skipmissing(col)), dictencodings))
+        elseif S <: NTuple
+            if Arrow.ArrowTypes.gettype(S) == UInt8
+                DATA = [ismissing(x) ? Arrow.ArrowTypes.default(S) : String(collect(x)) for x in col]
+            else
+                push!(children, FieldData("item", Arrow.ArrowTypes.gettype(S), Arrow.flatten(coalesce(x, Arrow.ArrowTypes.default(S)) for x in col), dictencodings))
+            end
+        elseif S <: NamedTuple
+            for (nm, typ) in zip(fieldnames(S), fieldtypes(S))
+                push!(children, FieldData(String(nm), typ, (getfield(x, nm) for x in col), dictencodings))
+            end
+        elseif S <: Arrow.UnionT
+            U = eltype(S)
+            tids = Arrow.typeids(S) === nothing ? (0:fieldcount(U)) : Arrow.typeids(S)
+            TYPE_ID = [x === missing ? 0 : tids[Arrow.isatypeid(x, U)] for x in col]
+            if Arrow.unionmode(S) == Arrow.Meta.UnionMode.Dense
+                offs = zeros(Int32, fieldcount(U))
+                OFFSET = Int32[]
+                for x in col
+                    idx = x === missing ? 1 : Arrow.isatypeid(x, U)
+                    push!(OFFSET, offs[idx])
+                    offs[idx] += 1
+                end
+                for i = 1:fieldcount(U)
+                    SS = fieldtype(U, i)
+                    push!(children, FieldData("$i", SS, Arrow.filtered(i == 1 ? Union{SS, Missing} : Arrow.maybemissing(SS), col), dictencodings))
+                end
+            else
+                for i = 1:fieldcount(U)
+                    SS = fieldtype(U, i)
+                    push!(children, FieldData("$i", SS, Arrow.replaced(SS, col), dictencodings))
+                end
+            end
+        elseif S <: KeyValue
+            push!(children, FieldData("key", Arrow.keyvalueK(S), (x.key for x in col), dictencodings))
+            push!(children, FieldData("value", Arrow.keyvalueV(S), (x.value for x in col), dictencodings))
+        end
+    end
+    return FieldData(nm, len, VALIDITY, OFFSET, TYPE_ID, DATA, children)
+end
+
+mutable struct RecordBatch
+    count::Int64
+    columns::Vector{FieldData}
+end
+
+RecordBatch() = RecordBatch(0, FieldData[])
+StructTypes.StructType(::Base.Type{RecordBatch}) = StructTypes.Mutable()
+
+mutable struct DictionaryBatch
+    id::Int64
+    data::RecordBatch
+end
+
+DictionaryBatch() = DictionaryBatch(0, RecordBatch())
+StructTypes.StructType(::Base.Type{DictionaryBatch}) = StructTypes.Mutable()
+
+mutable struct DataFile <: Tables.AbstractColumns
+    schema::Schema
+    batches::Vector{RecordBatch}
+    dictionaries::Vector{DictionaryBatch}
+end
+
+Base.propertynames(x::DataFile) = (:schema, :batches, :dictionaries)
+
+function Base.getproperty(df::DataFile, nm::Symbol)
+    if nm === :schema
+        return getfield(df, :schema)
+    elseif nm === :batches
+        return getfield(df, :batches)
+    elseif nm === :dictionaries
+        return getfield(df, :dictionaries)
+    end
+    return Tables.getcolumn(df, nm)
+end
+
+DataFile() = DataFile(Schema(), RecordBatch[], DictionaryBatch[])
+StructTypes.StructType(::Base.Type{DataFile}) = StructTypes.Mutable()
+
+parsefile(file) = JSON3.read(Mmap.mmap(file), DataFile)
+
+# make DataFile satisfy Tables.jl interface
+function Tables.partitions(x::DataFile)
+    if isempty(x.batches)
+        # special case empty batches by producing a single DataFile w/ schema
+        return (DataFile(x.schema, RecordBatch[], x.dictionaries),)
+    else
+        return (DataFile(x.schema, [x.batches[i]], x.dictionaries) for i = 1:length(x.batches))
+    end
+end
+
+Tables.columns(x::DataFile) = x
+
+function Tables.schema(x::DataFile)
+    names = map(x -> x.name, x.schema.fields)
+    types = map(x -> juliatype(x), x.schema.fields)
+    return Tables.Schema(names, types)
+end
+
+Tables.columnnames(x::DataFile) =  map(x -> Symbol(x.name), x.schema.fields)
+
+function Tables.getcolumn(x::DataFile, i::Base.Int)
+    field = x.schema.fields[i]
+    type = juliatype(field)
+    return ChainedVector(ArrowArray{type}[ArrowArray{type}(field, length(x.batches) > 0 ? x.batches[j].columns[i] : FieldData(), x.dictionaries) for j = 1:length(x.batches)])
+end
+
+function Tables.getcolumn(x::DataFile, nm::Symbol)
+    i = findfirst(x -> x.name == String(nm), x.schema.fields)
+    return Tables.getcolumn(x, i)
+end
+
+struct ArrowArray{T} <: AbstractVector{T}
+    field::Field
+    fielddata::FieldData
+    dictionaries::Vector{DictionaryBatch}
+end
+ArrowArray(f::Field, fd::FieldData, d) = ArrowArray{juliatype(f)}(f, fd, d)
+Base.size(x::ArrowArray) = (x.fielddata.count,)
+
+function Base.getindex(x::ArrowArray{T}, i::Base.Int) where {T}
+    @boundscheck checkbounds(x, i)
+    S = Base.nonmissingtype(T)
+    if x.field.dictionary !== nothing
+        fielddata = x.dictionaries[findfirst(y -> y.id == x.field.dictionary.id, x.dictionaries)].data.columns[1]
+        field = copy(x.field)
+        field.dictionary = nothing
+        idx = x.fielddata.DATA[i] + 1
+        return ArrowArray(field, fielddata, x.dictionaries)[idx]
+    end
+    if T === Missing
+        return missing
+    elseif S <: UnionT
+        U = eltype(S)
+        tids = Arrow.typeids(S) === nothing ? (0:fieldcount(U)) : Arrow.typeids(S)
+        typeid = tids[x.fielddata.TYPE_ID[i]]
+        if Arrow.unionmode(S) == Arrow.Meta.UnionMode.DENSE
+            off = x.fielddata.OFFSET[i]
+            return ArrowArray(x.field.children[typeid+1], x.fielddata.children[typeid+1], x.dictionaries)[off]
+        else
+            return ArrowArray(x.field.children[typeid+1], x.fielddata.children[typeid+1], x.dictionaries)[i]
+        end
+    end
+    x.fielddata.VALIDITY[i] == 0 && return missing
+    if S <: Vector{UInt8}
+        return copy(codeunits(x.fielddata.DATA[i]))
+    elseif S <: String
+        return x.fielddata.DATA[i]
+    elseif S <: Vector
+        offs = x.fielddata.OFFSET
+        A = ArrowArray{eltype(S)}(x.field.children[1], x.fielddata.children[1], x.dictionaries)
+        return A[(offs[i] + 1):offs[i + 1]]
+    elseif S <: Dict
+        offs = x.fielddata.OFFSET
+        A = ArrowArray(x.field.children[1], x.fielddata.children[1], x.dictionaries)
+        return Dict(y.key => y.value for y in A[(offs[i] + 1):offs[i + 1]])
+    elseif S <: Tuple
+        if Arrow.ArrowTypes.gettype(S) == UInt8
+            A = x.fielddata.DATA
+            return Tuple(map(UInt8, collect(A[i][1:x.field.type.byteWidth])))
+        else
+            sz = x.field.type.listSize
+            A = ArrowArray{Arrow.ArrowTypes.gettype(S)}(x.field.children[1], x.fielddata.children[1], x.dictionaries)
+            off = (i - 1) * sz + 1
+            return Tuple(A[off:(off + sz - 1)])
+        end
+    elseif S <: NamedTuple
+        data = (ArrowArray(x.field.children[j], x.fielddata.children[j], x.dictionaries)[i] for j = 1:length(x.field.children))
+        return NamedTuple{fieldnames(S)}(Tuple(data))
+    elseif S == Int64 || S == UInt64
+        return parse(S, x.fielddata.DATA[i])
+    elseif S <: Arrow.Decimal
+        str = x.fielddata.DATA[i]
+        return S(parse(Int128, str))
+    elseif S <: Arrow.Date || S <: Arrow.Time
+        val = x.fielddata.DATA[i]
+        return Arrow.storagetype(S) == Int32 ? S(val) : S(parse(Int64, val))
+    elseif S <: Arrow.Timestamp
+        return S(parse(Int64, x.fielddata.DATA[i]))
+    else
+        return S(x.fielddata.DATA[i])
+    end
+end
+
+# take any Tables.jl source and write out arrow json datafile
+function DataFile(source)
+    fields = Field[]
+    metadata = nothing # TODO?
+    batches = RecordBatch[]
+    dictionaries = DictionaryBatch[]
+    dictencodings = Dict{String, Tuple{Base.Type, DictEncoding}}()
+    dictid = Ref(0)
+    for (i, tbl1) in Tables.partitions(source)
+        tbl = Arrow.toarrowtable(tbl1)
+        if i == 1
+            sch = Tables.schema(tbl)
+            for (nm, T, col) in zip(sch.names, sch.types, Tables.Columns(tbl))
+                if col isa Arrow.DictEncode
+                    id = dictid[]
+                    dictid[] += 1
+                    codes = DataAPI.refarray(col.data)
+                    if codes !== col.data
+                        IT = Type(eltype(codes))
+                    else
+                        IT = Type(Arrow.encodingtype(length(unique(col))))
+                    end
+                    dictencodings[String(nm)] = (T, DictEncoding(id, IT, false))
+                end
+                push!(fields, Field(String(nm), T, dictencodings))
+            end
+        end
+        # build record batch
+        len = Tables.rowcount(tbl)
+        columns = FieldData[]
+        for (nm, T, col) in zip(sch.names, sch.types, Tables.Columns(tbl))
+            push!(columns, FieldData(String(nm), T, col, dictencodings))
+        end
+        push!(batches, RecordBatch(len, columns))
+        # build dictionaries
+        for (nm, (T, dictencoding)) in dictencodings
+            column = FieldData(nm, T, Tables.getcolumn(tbl, nm), nothing)
+            recordbatch = RecordBatch(len, [column])
+            push!(dictionaries, DictionaryBatch(dictencoding.id, recordbatch))
+        end
+    end
+    schema = Schema(fields, metadata)
+    return DataFile(schema, batches, dictionaries)
+end
+
+function Base.isequal(df::DataFile, tbl::Arrow.Table)
+    Tables.schema(df) == Tables.schema(tbl) || return false
+    i = 1
+    for (col1, col2) in zip(Tables.Columns(df), Tables.Columns(tbl))
+        if !isequal(col1, col2)
+            @show i
+            return false
+        end
+        i += 1
+    end
+    return true
+end
+
+end
\ No newline at end of file
diff --git a/julia/Arrow/test/arrowjson/datetime.json b/julia/Arrow/test/arrowjson/datetime.json
new file mode 100644
index 00000000000..35eace4e8c5
--- /dev/null
+++ b/julia/Arrow/test/arrowjson/datetime.json
@@ -0,0 +1,911 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "f0",
+        "type": {
+          "name": "date",
+          "unit": "DAY"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f1",
+        "type": {
+          "name": "date",
+          "unit": "MILLISECOND"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f2",
+        "type": {
+          "name": "time",
+          "unit": "SECOND",
+          "bitWidth": 32
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f3",
+        "type": {
+          "name": "time",
+          "unit": "MILLISECOND",
+          "bitWidth": 32
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f4",
+        "type": {
+          "name": "time",
+          "unit": "MICROSECOND",
+          "bitWidth": 64
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f5",
+        "type": {
+          "name": "time",
+          "unit": "NANOSECOND",
+          "bitWidth": 64
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f6",
+        "type": {
+          "name": "timestamp",
+          "unit": "SECOND"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f7",
+        "type": {
+          "name": "timestamp",
+          "unit": "MILLISECOND"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f8",
+        "type": {
+          "name": "timestamp",
+          "unit": "MICROSECOND"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f9",
+        "type": {
+          "name": "timestamp",
+          "unit": "NANOSECOND"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f10",
+        "type": {
+          "name": "timestamp",
+          "unit": "MILLISECOND"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f11",
+        "type": {
+          "name": "timestamp",
+          "unit": "SECOND",
+          "timezone": "UTC"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f12",
+        "type": {
+          "name": "timestamp",
+          "unit": "MILLISECOND",
+          "timezone": "US/Eastern"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f13",
+        "type": {
+          "name": "timestamp",
+          "unit": "MICROSECOND",
+          "timezone": "Europe/Paris"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f14",
+        "type": {
+          "name": "timestamp",
+          "unit": "NANOSECOND",
+          "timezone": "US/Pacific"
+        },
+        "nullable": true,
+        "children": []
+      }
+    ]
+  },
+  "batches": [
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            -719162,
+            2932896,
+            1251583,
+            -354578,
+            1947695,
+            -669151,
+            26653
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-62135596800000",
+            "253402214400000",
+            "-2820221740189",
+            "71549882314362",
+            "125067746235030",
+            "-27038791348254",
+            "42137473450326"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            0,
+            86400,
+            76127,
+            6700,
+            27103,
+            56151,
+            56654
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            0,
+            86400000,
+            17984286,
+            76370591,
+            60937117,
+            2240575,
+            8788989
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "0",
+            "86400000000",
+            "74085029005",
+            "23058796418",
+            "5827157574",
+            "30753386088",
+            "41165364667"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "0",
+            "86400000000000",
+            "23907445718784",
+            "74345421086418",
+            "75233481254444",
+            "82172159793710",
+            "58497242525071"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-62135596800",
+            "253402214400",
+            "-61131551586",
+            "109841560561",
+            "-3916465142",
+            "146694684650",
+            "138850275868"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-62135596800000",
+            "253402214400000",
+            "73881152631437",
+            "69908713976427",
+            "252339755754438",
+            "-24746530024729",
+            "169302540975380"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-62135596800000000",
+            "253402214400000000",
+            "201464679490582249",
+            "168126161155504013",
+            "-23403778250906066",
+            "205706554937392102",
+            "45776665091115087"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-9223372036854775808",
+            "9223372036854775807",
+            "-6357255048670867880",
+            "-8661447973993819541",
+            "-8212788386909103318",
+            "-8530954041419345600",
+            "-4218486829304453721"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-62135596800000",
+            "253402214400000",
+            "206983911256609",
+            "94973546379549",
+            "-18473207641060",
+            "36529119814530",
+            "143273969098011"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-62135596800",
+            "253402214400",
+            "225567855249",
+            "18090198256",
+            "-18645631593",
+            "1683299996",
+            "240974238031"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-62135596800000",
+            "253402214400000",
+            "-41888047432132",
+            "142167692985706",
+            "96450253340232",
+            "-28800292871111",
+            "31551906541089"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-62135596800000000",
+            "253402214400000000",
+            "-55035741541368439",
+            "110555619232926841",
+            "13584197914180451",
+            "-40099398122556776",
+            "154575532939365500"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-9223372036854775808",
+            "9223372036854775807",
+            "5175948389020765869",
+            "5557679156666679724",
+            "4250919303876106324",
+            "9160676477011889469",
+            "8585006913301874724"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            -719162,
+            2932896,
+            1879965,
+            -566367,
+            37728,
+            1761040,
+            279144,
+            1056794,
+            756303,
+            525725
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-62135596800000",
+            "253402214400000",
+            "-38092110755085",
+            "-28445105640862",
+            "82150583163219",
+            "54419670636015",
+            "157522888788052",
+            "-41135178703404",
+            "-55692081078291",
+            "23161948344048"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            0,
+            86400,
+            68158,
+            43645,
+            82390,
+            63272,
+            8687,
+            73663,
+            41080,
+            16606
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            0,
+            86400000,
+            4300160,
+            71414742,
+            77204018,
+            20632726,
+            31365614,
+            66601445,
+            59573489,
+            62138475
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "0",
+            "86400000000",
+            "28651881349",
+            "76962235570",
+            "72557141940",
+            "81551970477",
+            "20740172257",
+            "69927019990",
+            "76743031592",
+            "82821335874"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "0",
+            "86400000000000",
+            "69757112637445",
+            "57552711513942",
+            "29426946816946",
+            "34878855922929",
+            "33344575898987",
+            "80887006335433",
+            "34037765279999",
+            "51577535310194"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-62135596800",
+            "253402214400",
+            "17362133914",
+            "-45891238920",
+            "184875963653",
+            "189805054827",
+            "-58368591641",
+            "11979945774",
+            "-42159999942",
+            "-40114167869"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-62135596800000",
+            "253402214400000",
+            "228991365143433",
+            "169961535994833",
+            "200469360821110",
+            "69234108321383",
+            "198319780924526",
+            "191497586634193",
+            "175427870270356",
+            "57342673854963"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-62135596800000000",
+            "253402214400000000",
+            "56980200145982394",
+            "181929648203745781",
+            "139102923751831867",
+            "208997257530770666",
+            "16136961511557279",
+            "41555612864958844",
+            "13419848118557598",
+            "87383692083185618"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-9223372036854775808",
+            "9223372036854775807",
+            "-7031715621840828119",
+            "6059066797068981521",
+            "-1646261793033501417",
+            "-4220806875454311426",
+            "-5197114782094970725",
+            "-8786449967831538943",
+            "2974021310284646715",
+            "-8065049992539820014"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-62135596800000",
+            "253402214400000",
+            "178310981489839",
+            "-57632494932898",
+            "-49036210908585",
+            "52895117552494",
+            "171282515196488",
+            "132014017559614",
+            "37589110284897",
+            "-58565057255450"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-62135596800",
+            "253402214400",
+            "-25861647763",
+            "225566577159",
+            "239974631847",
+            "9594019763",
+            "195861877033",
+            "-19961060193",
+            "75621579368",
+            "-37516489502"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-62135596800000",
+            "253402214400000",
+            "197770622812426",
+            "195993483135189",
+            "187243903796151",
+            "193213107363200",
+            "83570298570259",
+            "252571502045214",
+            "129428288356579",
+            "-6553516468568"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-62135596800000000",
+            "253402214400000000",
+            "227405247332731417",
+            "-23876629555725661",
+            "172967389442803645",
+            "214366955371313241",
+            "86933375268516953",
+            "162567381239071692",
+            "40270626452354761",
+            "-9766478375147980"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-9223372036854775808",
+            "9223372036854775807",
+            "-3367778002617009344",
+            "-921482854487607004",
+            "1280046021732622411",
+            "-2084512789553264359",
+            "6774927372092824293",
+            "2301253808511314195",
+            "-447823511949250637",
+            "-9137589927882857097"
+          ]
+        }
+      ]
+    }
+  ]
+}
\ No newline at end of file
diff --git a/julia/Arrow/test/arrowjson/decimal.json b/julia/Arrow/test/arrowjson/decimal.json
new file mode 100644
index 00000000000..6c27800acbf
--- /dev/null
+++ b/julia/Arrow/test/arrowjson/decimal.json
@@ -0,0 +1,32948 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "f0",
+        "type": {
+          "name": "decimal",
+          "precision": 3,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f1",
+        "type": {
+          "name": "decimal",
+          "precision": 4,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f2",
+        "type": {
+          "name": "decimal",
+          "precision": 5,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f3",
+        "type": {
+          "name": "decimal",
+          "precision": 6,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f4",
+        "type": {
+          "name": "decimal",
+          "precision": 7,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f5",
+        "type": {
+          "name": "decimal",
+          "precision": 8,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f6",
+        "type": {
+          "name": "decimal",
+          "precision": 9,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f7",
+        "type": {
+          "name": "decimal",
+          "precision": 10,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f8",
+        "type": {
+          "name": "decimal",
+          "precision": 11,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f9",
+        "type": {
+          "name": "decimal",
+          "precision": 12,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f10",
+        "type": {
+          "name": "decimal",
+          "precision": 13,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f11",
+        "type": {
+          "name": "decimal",
+          "precision": 14,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f12",
+        "type": {
+          "name": "decimal",
+          "precision": 15,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f13",
+        "type": {
+          "name": "decimal",
+          "precision": 16,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f14",
+        "type": {
+          "name": "decimal",
+          "precision": 17,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f15",
+        "type": {
+          "name": "decimal",
+          "precision": 18,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f16",
+        "type": {
+          "name": "decimal",
+          "precision": 19,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f17",
+        "type": {
+          "name": "decimal",
+          "precision": 20,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f18",
+        "type": {
+          "name": "decimal",
+          "precision": 21,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f19",
+        "type": {
+          "name": "decimal",
+          "precision": 22,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f20",
+        "type": {
+          "name": "decimal",
+          "precision": 23,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f21",
+        "type": {
+          "name": "decimal",
+          "precision": 24,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f22",
+        "type": {
+          "name": "decimal",
+          "precision": 25,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f23",
+        "type": {
+          "name": "decimal",
+          "precision": 26,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f24",
+        "type": {
+          "name": "decimal",
+          "precision": 27,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f25",
+        "type": {
+          "name": "decimal",
+          "precision": 28,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f26",
+        "type": {
+          "name": "decimal",
+          "precision": 29,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f27",
+        "type": {
+          "name": "decimal",
+          "precision": 30,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f28",
+        "type": {
+          "name": "decimal",
+          "precision": 31,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f29",
+        "type": {
+          "name": "decimal",
+          "precision": 32,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f30",
+        "type": {
+          "name": "decimal",
+          "precision": 33,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f31",
+        "type": {
+          "name": "decimal",
+          "precision": 34,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f32",
+        "type": {
+          "name": "decimal",
+          "precision": 35,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f33",
+        "type": {
+          "name": "decimal",
+          "precision": 36,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f34",
+        "type": {
+          "name": "decimal",
+          "precision": 37,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "f35",
+        "type": {
+          "name": "decimal",
+          "precision": 38,
+          "scale": 2
+        },
+        "nullable": true,
+        "children": []
+      }
+    ]
+  },
+  "batches": [
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-14265",
+            "-3301",
+            "-20359",
+            "5259",
+            "-10833",
+            "-19794",
+            "22904"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-7020",
+            "7396",
+            "9702",
+            "-29862",
+            "25915",
+            "17934",
+            "23441"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-6799031",
+            "-6350361",
+            "-7919057",
+            "-2688856",
+            "-2460218",
+            "-5498780",
+            "5580928"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-3533649",
+            "643928",
+            "2739361",
+            "-2006582",
+            "7501109",
+            "-2029241",
+            "-6554749"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-1383097445",
+            "-317364446",
+            "39137303",
+            "-203304373",
+            "284864794",
+            "-1902924832",
+            "694509255"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "1445584597",
+            "-1479256303",
+            "1986678586",
+            "653640925",
+            "-1377647126",
+            "163472005",
+            "-985666433"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-1032313347",
+            "1968833128",
+            "2108909581",
+            "1727353475",
+            "912414766",
+            "-635823470",
+            "1712241290"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "154945363887",
+            "416184610386",
+            "339640718067",
+            "517615365091",
+            "-225329280656",
+            "170183803130",
+            "-151819857736"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "415862710871",
+            "-497367616337",
+            "-513589024815",
+            "365532163027",
+            "162771025458",
+            "-217545761908",
+            "-177764161272"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "139749710369981",
+            "-83794245890178",
+            "-91856283653391",
+            "136480936919831",
+            "139751556918685",
+            "-128711073562986",
+            "57528145123438"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "52917228652967",
+            "-94943960868145",
+            "-95787469176401",
+            "138513251818631",
+            "72391840904205",
+            "15456369093039",
+            "43627762817987"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-55939733680307",
+            "109774177882810",
+            "-18751094013174",
+            "66217182114674",
+            "54552088012955",
+            "10184533351464",
+            "-42623419179005"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "5323426016158598",
+            "-14889118959500841",
+            "17725275930019497",
+            "8976278451851381",
+            "-13047873474817232",
+            "-35698038632707430",
+            "35553131382011652"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "22248347423524568",
+            "21382628128477388",
+            "10729166497278728",
+            "33552782977485637",
+            "-27049764851522362",
+            "-22662588671534830",
+            "34450594992450092"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "1793092340943680288",
+            "6687492261570159758",
+            "-8215335212004105263",
+            "4478178378408252470",
+            "163214921176454093",
+            "8663994206540321487",
+            "-7336047061160684053"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-8553613303342920727",
+            "2303004565228604365",
+            "4387581718276767427",
+            "-9190240538897215152",
+            "-2359611618219705826",
+            "-4394705260138635628",
+            "-5098494694180487526"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-2066857943452927830158",
+            "903014216615599800157",
+            "-550458216531247342288",
+            "2017634601926315282348",
+            "848752616371714143556",
+            "1142061465959813542452",
+            "1385502695475659971908"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-2114118262682166206082",
+            "1902359162690886678168",
+            "416686278682682920537",
+            "1670406372017974831657",
+            "1672178198289799650411",
+            "1290828393749566925129",
+            "477509832520793484657"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-761286776745227142069",
+            "-519173571795485925518",
+            "-1152371818412313184820",
+            "-1693854308571834153795",
+            "-2338147549154180008842",
+            "-722980793690531284755",
+            "-308453791763555488903"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-363139155978036972417607",
+            "-67592154020748584606597",
+            "229641197562527780640996",
+            "-346708211516644122649482",
+            "-1250859224618770910347",
+            "-324094365896255288686912",
+            "587754147918610672847722"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-260044582391618089260319",
+            "-571174996847059618302630",
+            "-109712997473546598340520",
+            "-504887878361994310822232",
+            "-79933142040493800702044",
+            "391593215117382882517477",
+            "253411954510878438009626"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "78355683904145007775288840",
+            "-119384708322041630321535990",
+            "-100920737167981116982294279",
+            "83826802691186830123887155",
+            "-142861875795075155563507787",
+            "-145798388286082279657253594",
+            "-25329913740178438718347749"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "2728894406873273319876599",
+            "-26824077854062930008319059",
+            "59282342970725803527905051",
+            "-102858143095488614827267122",
+            "-86746033992194633744162126",
+            "151758737255322612002715481",
+            "-7882766239372883938103148"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-79786812518583385435068134",
+            "-60631765880118763666388680",
+            "-6864681441231111204611249",
+            "13050926774682302829016376",
+            "-70829524080848069150697712",
+            "-82269446396753351313982437",
+            "-64035656687419899366647308"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "3955725931924121460907631531",
+            "-21274213085809772733586643935",
+            "-12093030231048856037056919233",
+            "1591415318816864321772119800",
+            "15686477033795236409156083537",
+            "35545421326820517900037881163",
+            "22425766368694890650567525141"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "29018707613584242707294878326",
+            "17791141760763489912825555133",
+            "16531138885547875931113958621",
+            "-30513053118126324165999828125",
+            "15960846333545963474586001659",
+            "36917332355702074649716398096",
+            "-23564003365919469704120973292"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "9213099815853647106837374141135",
+            "-2927321977169069503340222047887",
+            "6334956127042213437236348937051",
+            "421008034487997672364705982278",
+            "-8216535847017377542720322569325",
+            "9010595708854142678424220967811",
+            "4188490944857456828411574213081"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-8400462145843923761912990483225",
+            "-9532520685799568447195818877630",
+            "6941627626857671447061360399506",
+            "-1279966284289260405732779325351",
+            "1498483597016609631513149683949",
+            "-1529118713555007787773854671666",
+            "8107117611898163619507416633696"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-2915756808824546582752189119657",
+            "689849096940399949037495607644",
+            "6949792968358403147068782203692",
+            "-9209851065708847333103701669232",
+            "-9846735769797668003358129310662",
+            "-828325389508874726576445634733",
+            "-548287410953591263203641276502"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "2329344668638312257280241058269271",
+            "-752344765844397811032529707212356",
+            "147451832276294483763966453842451",
+            "2043297198679260475716881353145488",
+            "-1700852196536476328772217191241342",
+            "395231723125832272118398102832808",
+            "1021064751191444192162606435133102"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "1059349538643979619795009909676873",
+            "1077517333972932491341963859241254",
+            "-1269329697335528941024149247301687",
+            "1492334714292199711015081732157237",
+            "1164940978846205931486582828955479",
+            "1226701026828005364221866490976254",
+            "-2246642220670050837117291416085961"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-453427466718461089284644212936857043",
+            "401968647744354893846373970134628607",
+            "296564973980089833086319484394180932",
+            "300255396607176659100995955356045655",
+            "-128210286701929565620215086982795175",
+            "-76575845460847329054975157675278272",
+            "150610860039362064995272346289079343"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "421046835537097379208584523685561537",
+            "-12336249922904854000276177315936166",
+            "-643245709818789885569562430725515554",
+            "28125849177083902136768190312091645",
+            "-144070901322770869842559076535428168",
+            "404632889001498427711768281303970316",
+            "484413224979251839912542328701460679"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-74076465584946023556734876157740408561",
+            "48775465105440153520372738047296715437",
+            "-27256511570535798216063178902711504522",
+            "17556306662405983159878112274642973563",
+            "-100278707484244689483962841662232621044",
+            "61342616787415403733699116416597761322",
+            "65469616326882669620827062523697818819"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "160988666415816646705442929165000844171",
+            "-150079614246235114587531344502672720143",
+            "130458754430497218349443163826152965786",
+            "162593370774960128510764087469483971628",
+            "100335479568560073327353331226427021396",
+            "154469868317889131666703500879972187378",
+            "109851602321776291877341624608658829262"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "90507703396950205824782315866890653146",
+            "33101701517846982239639071710846694736",
+            "-162364169137146690363873495754205930932",
+            "6552361136224904568043204720462599330",
+            "55934073603932057089039430724186991179",
+            "-40518421289171987097923903446970623529",
+            "21964873705187790865866750301783750442"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-4326",
+            "-4742",
+            "24127",
+            "-28455",
+            "-25093",
+            "18893",
+            "6010",
+            "18847",
+            "-29502",
+            "32119"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-4350",
+            "6406",
+            "-22249",
+            "-3026",
+            "-27892",
+            "12582",
+            "-26038",
+            "-26810",
+            "-10412",
+            "12059"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-5437050",
+            "-2975935",
+            "-2708551",
+            "5905122",
+            "7598831",
+            "1837938",
+            "3161296",
+            "4144482",
+            "7971237",
+            "3812932"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "8307058",
+            "5437059",
+            "7848415",
+            "210888",
+            "-7438002",
+            "1251661",
+            "1558298",
+            "-8295786",
+            "-1337540",
+            "4329127"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "2143281149",
+            "1036205716",
+            "-483960365",
+            "464783350",
+            "44702295",
+            "1500496151",
+            "2007499481",
+            "-1980167635",
+            "-491561248",
+            "137927958"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "385115557",
+            "-468314229",
+            "-1806182189",
+            "826968059",
+            "1973820915",
+            "-1678321968",
+            "282860534",
+            "-429116070",
+            "-485370474",
+            "-1175964348"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1660401943",
+            "858637970",
+            "1954585251",
+            "896247082",
+            "-1324508546",
+            "1856733620",
+            "-508486488",
+            "-2144890404",
+            "-595464217",
+            "-541883970"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-475292084638",
+            "-151527930749",
+            "-136244496864",
+            "475724641544",
+            "490808185901",
+            "-380524076056",
+            "347742169602",
+            "-208359763265",
+            "500407573029",
+            "-390649746248"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-463878565883",
+            "-440350084389",
+            "-489309228695",
+            "403236358497",
+            "22420180364",
+            "461550541832",
+            "166802945594",
+            "-75395045157",
+            "349097107753",
+            "-31923850582"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "13351341741346",
+            "132672835377977",
+            "-104121852651737",
+            "-116578873884859",
+            "-93368786381550",
+            "-130866514927019",
+            "-130245922532357",
+            "119422720388976",
+            "39514069962031",
+            "129449354245745"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "117562734639864",
+            "-69335041870511",
+            "88065868490266",
+            "-88825080775852",
+            "-33000609471354",
+            "45253821354104",
+            "-19572766862990",
+            "-20998291433421",
+            "-138710717300131",
+            "105346771642837"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "23613726765605",
+            "-122540956874454",
+            "43298244246934",
+            "38914627030131",
+            "-96822287655033",
+            "-117561922155895",
+            "-116901550232631",
+            "-33238646153535",
+            "87255230245952",
+            "-103811103056354"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "14326292611678389",
+            "11679216137082255",
+            "30602968082225040",
+            "-35083463698552038",
+            "13793460528237121",
+            "29475754221579432",
+            "11956818381525770",
+            "25264515687485849",
+            "-9442305734363215",
+            "32709604673194504"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "10752127983223695",
+            "-17671172332493417",
+            "23204581064269137",
+            "25912317958066911",
+            "-6684281122027701",
+            "-5393708227406178",
+            "-705876736520327",
+            "-8271615012961841",
+            "-2359308340581953",
+            "-10178212042804721"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-785858494820999025",
+            "-1596606269767712061",
+            "-1418974761760001783",
+            "5406463592868106826",
+            "8383949914321587435",
+            "-8953833508549969767",
+            "7221522693757617783",
+            "7517732845880583602",
+            "-919064621805709634",
+            "-5495596733594283265"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-8483275882311463690",
+            "8973190785377545539",
+            "5542339023127975216",
+            "-1516710247958149161",
+            "-6505879801430717005",
+            "2140391223240971073",
+            "-582269641777058598",
+            "-5184242226957355683",
+            "5206992463328246120",
+            "-6079741749013867959"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "1593226799266187966947",
+            "-1812161934828717889262",
+            "-2122590131716118763227",
+            "-1027437168205961683538",
+            "1846540008444089681004",
+            "-1156292451907349126710",
+            "-1957435108076953499486",
+            "-950458759560036353191",
+            "-1125440584483224915212",
+            "2247219347276423832973"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-635230024201890240957",
+            "180963226287776474583",
+            "-1878152791054730240606",
+            "-1904422293952570992101",
+            "1052835439226626094544",
+            "-1858821189756108906198",
+            "1853668161015001150059",
+            "511899624049166705494",
+            "610084498432376101375",
+            "-1029828641365425116105"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "1885658882244978382784",
+            "565125911580367694969",
+            "-1652560178353597573029",
+            "2133173545067165238416",
+            "-486940365592293262895",
+            "-2060709217874894710408",
+            "1586493850244052619259",
+            "2178470762303491581080",
+            "-1293832362817784170345",
+            "-1858110114261315174517"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-369225411953896208185462",
+            "-432125813663454581808312",
+            "-144256737060926287909147",
+            "437531850902150565537267",
+            "-135152047695882388519227",
+            "200989985671128391254839",
+            "412064182108667107703143",
+            "155783836458581176854752",
+            "-227172778832455276260133",
+            "212767460062823457968604"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "403503153546865033134324",
+            "286440584824595490396339",
+            "191634056603308191156299",
+            "530416421236246257260161",
+            "-317587706869080743041040",
+            "-171530648886789759204250",
+            "222798032246148363279215",
+            "72219173848107712300756",
+            "-281960273386152702267354",
+            "-495494776734887426338667"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "122137352759155073399221518",
+            "-78109201384474477075454853",
+            "-119068241308521301444272353",
+            "-108756965225823937786847241",
+            "-15428358494009873779245484",
+            "19614899796809003767318009",
+            "-115295286224016867112586811",
+            "-8633742807795124483538483",
+            "-150410722825301769338372723",
+            "124250719445278286695627796"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-99183180993590467578076798",
+            "-36808132925962854413539892",
+            "-125679607816807665333773367",
+            "-136622111609623584591267546",
+            "-17266188400280841341905013",
+            "-71769330665427864647331885",
+            "-145814031084605056753603019",
+            "-125056942233093610827766296",
+            "9249166279701860198655516",
+            "126878824313892664602568663"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "69744825329510933906746340",
+            "-30122586028854363823971329",
+            "22212149164944035060004867",
+            "-25973928380545009708020589",
+            "50567555142199657207098957",
+            "-40021098669624532176421808",
+            "-33330934345289112585983041",
+            "-124012214741805393399462125",
+            "-138226312121129798637686830",
+            "2273005251425309465489619"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "26842282533862498382939405997",
+            "17787994211657168910392131484",
+            "7523952681100831594207033636",
+            "18257283108725825658512455316",
+            "-15469531382712880431453128966",
+            "-5009369258227088832031787745",
+            "-35598690213497115663830724084",
+            "22246953732785347649151349772",
+            "9330915514049730010034508310",
+            "-26800441656550330938009306707"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-36331212919615640889272411104",
+            "36192450254329815033783240839",
+            "37749265523377026412760210674",
+            "1776086174556031952147671519",
+            "15362218353534633991283119679",
+            "34413146026389235392142970662",
+            "14292777901465553588604435837",
+            "-17090082469158479303098433453",
+            "18688801441734020761551793736",
+            "-23859805527670127110643731069"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "2220103303731789573334199701663",
+            "6237111177014715580774407162796",
+            "-6338471790385983381447055194941",
+            "2250388387229112362557012222382",
+            "-3919498269201660475165992143963",
+            "-1799934177913445118487985955729",
+            "-2323304519983949754039480799426",
+            "-1301131662575316944670816531905",
+            "-4265160933868529886993472786495",
+            "-4292953046034527295812111533975"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-3484925219213002239805523064883",
+            "6528552859477682991951516050785",
+            "-765962107198382441283095455373",
+            "-1397137479831402026027607048248",
+            "3961682476407229854971103624981",
+            "5975195883181022585096451556775",
+            "-9676800477604513004081790052790",
+            "-5703219619447316623534739055181",
+            "6129137758274201914352345247446",
+            "771936840958552475578155475366"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "7726836802487400261012835642739",
+            "-2899737712900797452349686903367",
+            "-4699841764135292946953397202917",
+            "7418179251651709234455608105088",
+            "8120397131521471010221697932900",
+            "6526213244777384369820937871939",
+            "-3774567743925527043971960901589",
+            "-8873736034246294395374537274419",
+            "488289844645088756680442972516",
+            "-2388144175550709792515001316277"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-47477340538926222211104914237698",
+            "320639937921939986635581989855928",
+            "861736289842228881011999521004060",
+            "-1205256013177842982651274099236361",
+            "-39643995957385740378372042150003",
+            "-1210790336528949301250817830946444",
+            "1182423445445031739603473987778350",
+            "-640337212803879852231150942968813",
+            "-567587726713873060654848198630762",
+            "280641360776654488558911759135115"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-866285575875610221959113146637304",
+            "2344770413948196340883160931087052",
+            "1889857580890564350097047877514790",
+            "-2568726710340227178480991779371723",
+            "-2485213596075291138559974328614959",
+            "-62378076407090361966505761644273",
+            "-2113433394374145039336066697772466",
+            "-2389429054181728925012886258903305",
+            "-2525262726376576108925615177598533",
+            "-604526323558062739370715910785200"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-411175611485478784812936027850047826",
+            "457804241550769132910430204506150717",
+            "1549607974636127184352732575182742",
+            "390025168459008888512891394303962448",
+            "-490664276836807094735857019444661312",
+            "-450640457243898505844766529165221166",
+            "-553339316764622659577593048830066531",
+            "19986969431028149966548563329309114",
+            "512190967497020823088167980283378342",
+            "-587244163212803971173675798133411979"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "246512063483920590600659938977719645",
+            "483324212708049237205831199756205159",
+            "195186088040813738593870463346937474",
+            "-331572691964567402179138728488891845",
+            "466186594735574058231047101478621597",
+            "80512186957271594252263140708656889",
+            "-621018503026533776126752549530840325",
+            "-484705410290628923401085082564299400",
+            "-322852670767952007714303037031011136",
+            "306969564371703225579907932616560177"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "52513592802194657300546985265901321701",
+            "32396354459564619210286503831104805894",
+            "-152800597997887046528471654246818061800",
+            "16886738551251131147537573820997523099",
+            "-108529890244550965992486427743581407764",
+            "-107805100101953838884357149548341742683",
+            "-152428185005223769908895650096091179800",
+            "38697638049920608411728929261890502602",
+            "-53183835204559125450983905289890638199",
+            "-25198915522830164616311861007733226015"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-29473659368749332611063061562898178665",
+            "-64181392425515474831866849063339846755",
+            "90557155313004461723928910484647118568",
+            "69982410088115917703700629187142714952",
+            "137281427883875761429267863353473899975",
+            "66221749426872522877433672036305282198",
+            "139898216716793863940328165403822713329",
+            "-17999019985160863257629297875199464798",
+            "74582888771441381839018581558057879959",
+            "153431389525940565745457493557728519648"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "132130437221517226508875210221998086214",
+            "-53955540660976668373224815268780394481",
+            "-94671974318719360571824313264981742616",
+            "75940549933612625873217731049581943788",
+            "-58970299111050400318748873232294460623",
+            "106861808580585226388022589188291458328",
+            "151810935532684619700572858784852122078",
+            "147244409300271119764509412340177481716",
+            "139138676242120433411406371456479693081",
+            "58927563101546159870009467050083778558"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "25114",
+            "-12925",
+            "-1074",
+            "28646",
+            "-16569",
+            "-16770",
+            "-27151"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "26533",
+            "12085",
+            "25312",
+            "23445",
+            "13356",
+            "-2369",
+            "-29785"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-4380859",
+            "-157657",
+            "-4200925",
+            "6061632",
+            "1972423",
+            "-3016747",
+            "2864606"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-7045373",
+            "-7371800",
+            "1427921",
+            "-7601068",
+            "2525157",
+            "-6433403",
+            "-2477014"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-141345816",
+            "1543059503",
+            "1211501603",
+            "369888761",
+            "894246844",
+            "736166799",
+            "409408642"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-1553193945",
+            "1491657644",
+            "376067040",
+            "-1100453955",
+            "1763626627",
+            "1237652218",
+            "-1680624462"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "102290167",
+            "1044278987",
+            "501196467",
+            "32275276",
+            "-1384353672",
+            "594927697",
+            "-1482240101"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "431716525885",
+            "-396649752792",
+            "-252538061544",
+            "148825582538",
+            "536082535524",
+            "-135755487371",
+            "425356768774"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "137916147840",
+            "21393720222",
+            "-325824439981",
+            "278743651518",
+            "-49501503483",
+            "-419736800011",
+            "503854890502"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-137642247974694",
+            "-126053382015273",
+            "32896127803947",
+            "17450989064436",
+            "-120690899531858",
+            "109217312768874",
+            "-79259001409277"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "114207744387703",
+            "-79306413416767",
+            "139536513022560",
+            "-98423452444221",
+            "78791182271051",
+            "-35285009371089",
+            "41940974948329"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-27264096490547",
+            "136834819442083",
+            "-27088394932437",
+            "-98987047379901",
+            "125550952272750",
+            "-104804800493942",
+            "8768888141448"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "23677368293754508",
+            "-30279797614274598",
+            "-30317510840672041",
+            "-3538291640098903",
+            "-2087381848617356",
+            "-23713764981307104",
+            "-11713886893415435"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-26406417185878787",
+            "-14282719185754107",
+            "-34042541212866444",
+            "24219940349910515",
+            "-23950715983191837",
+            "9125287296093962",
+            "1340350995572397"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "1175386256159753067",
+            "9132036865814002849",
+            "2073302245565101007",
+            "5198424551239400626",
+            "1162180176286851142",
+            "-5469754044264044482",
+            "6861255327733562384"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "6025723723135521005",
+            "-7217621567135194958",
+            "-875939541579707257",
+            "8236552400788004405",
+            "-4353971909177851416",
+            "4189966288831554693",
+            "-2170856197554954993"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "1984509802579054321521",
+            "-1041998696764751045250",
+            "501861238173417097382",
+            "2266100162043156929049",
+            "1092677900985431385639",
+            "2290545820208575246231",
+            "-1929677784513148966088"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-2343674427840265383765",
+            "2230129800929309514822",
+            "1298773822649292467354",
+            "-1658379353955811152293",
+            "460872411226138227967",
+            "-366025556585308681996",
+            "2303266251171038691928"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-1553211803508188723315",
+            "-307288386314104261659",
+            "-147859719179575239406",
+            "316737260848156543682",
+            "1901202079912459192862",
+            "-2066795118493461325326",
+            "1097271871910371248765"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "60159431599803576006229",
+            "-349523214568312595512352",
+            "-479410413534686721947634",
+            "-438205000235664595511857",
+            "71434087716492159731908",
+            "313701681861148862347175",
+            "592529110781630294785385"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-41073349320046663529611",
+            "-460196612402794801624059",
+            "-85219737286050217930644",
+            "-430653003445193867062723",
+            "212772177419770007462828",
+            "533773505614581994442892",
+            "588510537528110040152211"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "31643252440554433191940460",
+            "-93699577774061168478202291",
+            "24154537866604719834628304",
+            "30696729192090200230120983",
+            "-99615616997150346574018879",
+            "-570384960873190266772386",
+            "60124053065001805618112484"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "43634928021314649415598611",
+            "-71376931904073326185512276",
+            "-24045621000779198743707062",
+            "6686912739698933453916262",
+            "-154458317914070878667147081",
+            "-48373280695426803475799028",
+            "76169691954204618596160986"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-114582957647416816237217281",
+            "-12748630690377976467956388",
+            "111731504296816655416680693",
+            "4019447671331190931268009",
+            "82801431291231910751983771",
+            "108497088164927504518108753",
+            "19137086084026561540969316"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "11411634354069865542863757714",
+            "4367992064855387626281818334",
+            "21423224512954024770097152486",
+            "-32916059460547773682793752250",
+            "-37598962143905700881222660514",
+            "-26791606874412750044572561764",
+            "-1640980913390047815572783266"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "34745452020371732029340755584",
+            "38633937182521363276851366299",
+            "13907386367292796497248233421",
+            "3516712513415277325145153684",
+            "3056849635253718194064084992",
+            "4551068805171087267034288791",
+            "-17898292752703455443559461457"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-782556900320991870801777710984",
+            "1691629456944775593578700272292",
+            "-3817502570462777233621604475255",
+            "-4582592814198566350977961347647",
+            "-8757778015646360699558792162096",
+            "-3372484932226680791305443517921",
+            "-6773601661940427910934854617600"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-3034111110316845464241842333639",
+            "-1781275911958673372639492265462",
+            "-1931637931243318072822423804991",
+            "-2889899890039585265079105752268",
+            "8441716411760182969194644963692",
+            "7571628885131553956380784504990",
+            "6417570389157484165557130988430"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-8637275815739683704408877390855",
+            "-5205784754631510403458931812287",
+            "5903473813623392609711242445408",
+            "2196392642166569937732303950298",
+            "4322897470867933044080337659838",
+            "-295215953845026604294742492576",
+            "-9707682435519997757567927666374"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1371746941161753233161616198879974",
+            "1982136493212029919715565342281105",
+            "-1917002754487970574686778478783266",
+            "-57865905347137694987990288364929",
+            "2249945384795180946501174810954490",
+            "-428951216116914987102881673111066",
+            "-2274002923925825152016895839891593"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "1092814442698298966273488282610980",
+            "1144413745884554568972739530468097",
+            "-75300812750583864582975138768425",
+            "789050129071516591209015420836704",
+            "-1164029127634305816704854164256453",
+            "2432154715761013584642826292686689",
+            "-1892357160522293273795945838702878"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-122183630549108645927458587184832899",
+            "-243792853164469971347092559188097429",
+            "586041069230257431808895634457019729",
+            "643508290616316791666365172242311322",
+            "140267869100082091472571196986321703",
+            "367901398863058047260102259430609991",
+            "663427402521611050810057766609452801"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "442484503124860279043087511624488704",
+            "-281538291396564587147893059598980836",
+            "-346402069367877555608660697944397703",
+            "121136520298204023131218573046733183",
+            "-297635365268234967618402017598530084",
+            "-374122180189017501660740031013019545",
+            "-581737813428791636660311570131585053"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "133154724605091554639809801656916837785",
+            "75830729335101076443136841481038070489",
+            "120313524491479335277638759335459890905",
+            "61287945625316584877200472317882946434",
+            "101401636551502086495943764753580195892",
+            "-67491973819108763200100227319414962675",
+            "1469727002050376143004735317308620121"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "163689960486968830624238755496353069266",
+            "120222413117901199440645458344006405188",
+            "48580226284038590005071569547583521787",
+            "8088270627535987928187148843962238997",
+            "81865070062931483309512859489282733063",
+            "-10237210340155912860809619233603401462",
+            "83075135112485276897941678118419444602"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-165871274022360766986351083915813500328",
+            "-8686579665026772053190367047771284024",
+            "-159204341085506490972656979604179139282",
+            "109873262356509061148465005250720029433",
+            "-106075570921623241101438726440228153847",
+            "94260033293871207317716609052204748035",
+            "-20194157317096074536517680199183372407"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-30954",
+            "20758",
+            "27427",
+            "20450",
+            "3843",
+            "-28014",
+            "-9284",
+            "-6624",
+            "17505",
+            "-15123"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "685",
+            "-5310",
+            "-781",
+            "-19286",
+            "-2510",
+            "-7200",
+            "3827",
+            "-1198",
+            "-27211",
+            "20037"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "805128",
+            "-8334205",
+            "-8322011",
+            "7348287",
+            "-2733195",
+            "426429",
+            "3846540",
+            "-6903563",
+            "6889801",
+            "-7512693"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "2488446",
+            "3102298",
+            "-476499",
+            "-2639540",
+            "3282590",
+            "6092430",
+            "-1675823",
+            "5601386",
+            "1196495",
+            "5499393"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-30691203",
+            "661960132",
+            "-1935743351",
+            "1575466801",
+            "256013883",
+            "1160657488",
+            "-777972530",
+            "-1493606470",
+            "486831696",
+            "-523869499"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "1802142466",
+            "1751466925",
+            "1524761681",
+            "1127842279",
+            "1711088497",
+            "993428343",
+            "1169779239",
+            "1513015569",
+            "-1133724977",
+            "2014476536"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "1960388178",
+            "1348475329",
+            "862499131",
+            "1640384591",
+            "-1548686121",
+            "-109978873",
+            "574685145",
+            "-1936942220",
+            "-828064162",
+            "-2111967247"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-10546471442",
+            "410574147711",
+            "-159310325390",
+            "-342068240706",
+            "268451689904",
+            "-188950021888",
+            "174863595081",
+            "75886283067",
+            "204533910683",
+            "-236375793932"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "395399290333",
+            "398808017348",
+            "470458425747",
+            "544514539160",
+            "106030474107",
+            "-220167092355",
+            "403504352884",
+            "-215936236845",
+            "157621638402",
+            "-84528308243"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-11872483439739",
+            "27198663837500",
+            "24092599125599",
+            "94647109181593",
+            "-25234549831778",
+            "18418685805939",
+            "-48015201340553",
+            "94192389246447",
+            "114313155048141",
+            "-8129249986173"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "77060681503342",
+            "-17103335031331",
+            "62085210064375",
+            "-4113774722062",
+            "-39621802698178",
+            "82862411524800",
+            "-80722206760975",
+            "-71853038428164",
+            "74451847897808",
+            "138561565698569"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "11425892925558",
+            "-1067102949941",
+            "-67195834692311",
+            "132254465032616",
+            "81094671882564",
+            "-16512810337582",
+            "31369523023438",
+            "-25612899826767",
+            "-92714617670958",
+            "11132766799365"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "31425051479846585",
+            "23658872883751119",
+            "-22841020643211561",
+            "-5901149141492493",
+            "-9081601009126169",
+            "2009456704881039",
+            "-5853653315964572",
+            "-17548466661612427",
+            "31243548103064502",
+            "-2990459100082881"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-12114077651218054",
+            "34893217784362181",
+            "20057977432890552",
+            "33560592225527770",
+            "-6087365862911550",
+            "35528436105980777",
+            "-18908989690611601",
+            "-23450050973056772",
+            "-24497468547358805",
+            "35295281517022055"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "7512947852738829754",
+            "1506648420706893967",
+            "5351706185125823777",
+            "-8979604389662071296",
+            "2474510455479584357",
+            "-979503341239819500",
+            "2879236724021058276",
+            "901640783116791589",
+            "-1968701555733090102",
+            "9048825040146306689"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "3527698479739214995",
+            "-9168040692056637587",
+            "4303558786305470137",
+            "-6434168737271011073",
+            "-5987724165431090257",
+            "-6342874047925994892",
+            "-5023544862753808662",
+            "8349405051483082410",
+            "-5427848340450674266",
+            "4416839962565899579"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "122719954361389244782",
+            "-1715029242671728269265",
+            "1046814468132539634758",
+            "1829993007839921437821",
+            "1961163586998902612001",
+            "-1718264963912709868519",
+            "-142775659012283376739",
+            "-227979474167608252692",
+            "-1284136730697963933939",
+            "-1188947702777815289337"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1540168519262718909498",
+            "1444272924270897021948",
+            "305426275834034841423",
+            "-2023121412074327344395",
+            "356123154623624533101",
+            "1851993497777674677711",
+            "-1548911725567051530406",
+            "-1688091710111258813903",
+            "547593933625451476915",
+            "-1136296230783566148328"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "93977861990199039012",
+            "-725203385003516503766",
+            "-2136208269275109184730",
+            "-1443758672222546024674",
+            "1770464701250178930017",
+            "-1503951609214983767704",
+            "580922121807638712625",
+            "-1715487409849159976347",
+            "-1581681782950208904546",
+            "-345107986036541695760"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-112509459909310806801713",
+            "-318071601726453617114288",
+            "-377093818504933877006627",
+            "-412251199154230320946975",
+            "14121963032451866972322",
+            "127959674785194294868765",
+            "274741358639653860200873",
+            "-10201412639728798865544",
+            "-484633295332684921844249",
+            "325728689994021020089522"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "102768059460000020082850",
+            "-582172705216901696514361",
+            "385654623152916059129826",
+            "-375444952943658610617412",
+            "-559762648769206534338228",
+            "-455225699143700230588676",
+            "414928288856671138989242",
+            "-239573759040735488345533",
+            "-151621506107277003038408",
+            "238527569805762630418752"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "105793826786409638098848324",
+            "-105930031172236153458218556",
+            "54215836825825339383685314",
+            "153999900608311376451254881",
+            "-60781620142763268768239184",
+            "129045242635307916195795557",
+            "142609636641396599505824777",
+            "25997822165486291786720766",
+            "-24843271142157773732898696",
+            "27667985603420108003309363"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "151727563990526262987631020",
+            "-153515663086836104138582144",
+            "34269843807095303273290014",
+            "116745487266501279093435997",
+            "19697558732934548814393639",
+            "116467090539752813149322652",
+            "-49597590215015448554625243",
+            "904912798075267997581911",
+            "-39665579024769081440226797",
+            "-148107902864927713797008511"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-83573886433003161782682974",
+            "-61690975717719632051567161",
+            "87221161283509159769937332",
+            "1424308408986182936520347",
+            "-97659156978716488999314621",
+            "-17069082270469318857712868",
+            "-81196418361561342109282996",
+            "89508773441843844208807904",
+            "144076371744089019894053325",
+            "46698503394106850398118272"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-4870804024939332050762868706",
+            "-7019153028274263100384076399",
+            "-38347116871041435121142912404",
+            "6752651380046831311117204924",
+            "32477977767491441764479499613",
+            "-17206384783392201002575380629",
+            "-27155762594673709466833763677",
+            "-7707471006910257215683844696",
+            "-12836204732773392557927387161",
+            "760807996077187416324370561"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-17307596853554170302945978542",
+            "-29291868274857325327495085418",
+            "39520334649804167187543012083",
+            "-29850376686502020869058128768",
+            "-7769023746550159417970381944",
+            "13728883947958157003762954783",
+            "-32237971097073321264822351426",
+            "5786340858732715593319648571",
+            "16603037987628417795944460761",
+            "23450906837191853879925326964"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-9980201576657945144842882236365",
+            "-7407461373350157855602680465891",
+            "-8849151930540839739287729740462",
+            "-2082450369562757904475450406520",
+            "5688213649545258906102340952545",
+            "8532260187438278666751588130781",
+            "-102084121821929763174388250038",
+            "7229993189261659865189536694418",
+            "6276298854863176977991964120714",
+            "1342743900941457696012670661953"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-2978344615310830117461734259477",
+            "-2947273335992494396678516995826",
+            "-5244820858348685929338162101277",
+            "-2640927544212422127465200234510",
+            "1768434382958061290101667511624",
+            "-3963482937808073490523756242168",
+            "3374307591928127235727040204209",
+            "-3382678138467009217749439169317",
+            "8801701051040866573946377274029",
+            "-9104355797002943521132479841900"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "8221768927807895270484160091896",
+            "-8900000026189067281738645101281",
+            "-8281746127211712061601232308528",
+            "3023240804949676140399638485768",
+            "-4102245491799843728882823944331",
+            "7251501339276578447235263420202",
+            "8751503650699867906907117450087",
+            "-3729506782195610335248190443752",
+            "-10033160781867769412699287534647",
+            "-1250783004758791761527983017975"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-2562167817474569120675956515425419",
+            "-2458571516901048848163704156892937",
+            "-36875395321508089748922725678113",
+            "1960304726616291730330575204501959",
+            "2055550886455178489271580894649114",
+            "-2358720546112845253144262770181075",
+            "1595191993485611135276028859826726",
+            "1566529993850113106475509726619617",
+            "-22596759305941524381058617032547",
+            "-498743035152497361574808016716787"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-47643018102478418616183822213670",
+            "-278640293644733515826236620350821",
+            "-1488199042811674371836519886289180",
+            "-818674521344108691664748392501099",
+            "2556172025333762565075594479138752",
+            "-1334377085385061017822352624180696",
+            "-2000158271833880683895816854711292",
+            "-368140045799924875460317772104607",
+            "-1585782388250425609895544646403971",
+            "1824564753054254497659039848918675"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "479525603135366804733516578850615870",
+            "-495446691382547674716309382436075953",
+            "502812585466542988435075354889645892",
+            "476416535889813426699310670064638917",
+            "-379315281173603247308987567117630647",
+            "607404279943635423469568736054452550",
+            "-629067677540132407295995323971248814",
+            "-62158935415489079301126585948586031",
+            "461986658609980837096262131134918226",
+            "-589275703863823715586136613904443930"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "435625147644630830418626494902722848",
+            "-478332198115914941416962365458575509",
+            "558636662901555390631163102871642769",
+            "355599683788663730974008140088569567",
+            "415606584067363911029530290237838841",
+            "398064603694355571202936926109341982",
+            "-633945976114528848016756385029022841",
+            "457577785385893587465074783208649049",
+            "-616523624972085533886453431515327099",
+            "-248511133054735912122810606062669023"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-57306327392984906276930890383185368223",
+            "72793461807811385650658673069344386369",
+            "-109687531415068367399736525485032846349",
+            "60356410425087146413082196521754844149",
+            "26662652717210939669243284822836719981",
+            "112603258741679461947852680773641769673",
+            "4457422288602224460620455185955042948",
+            "103794515803470385910175949232361042030",
+            "-119185317915451074176129503358012831945",
+            "65467100518198811141064173626789800823"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-68618397724637272526572961589871509577",
+            "7721187761487257785999877840186156513",
+            "5605947584439501391665551914895560442",
+            "68369687165773392157960709162408033710",
+            "-96656920099210957452259257889912964672",
+            "-94072454264796947158929675025216943665",
+            "144069747100296579445022554416200691423",
+            "-87560734517757835226646920956502241626",
+            "-136164322966740604273148909571047647101",
+            "19429304069228502711385669535598831748"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "68390208110339728548757304121957739423",
+            "51149440950819465758183122582606174618",
+            "12265984607323358363395800066794656038",
+            "-150000242236153252396886212934934551728",
+            "122779026475879408492238567997677881221",
+            "131874778752322502170775011108192661329",
+            "165901435173219254996114701417463409790",
+            "-152003734738506573690620489933044183326",
+            "156016528459691385523457640405842540870",
+            "-74618395994120066096952180078019949350"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "23815",
+            "-30659",
+            "-17127",
+            "-19868",
+            "13628",
+            "31416",
+            "18133"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-22400",
+            "-14679",
+            "619",
+            "32351",
+            "-31747",
+            "-7719",
+            "-29774"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "3520430",
+            "-8276056",
+            "-827609",
+            "2153003",
+            "178182",
+            "-4215376",
+            "3779312"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "3689483",
+            "-735347",
+            "-5360898",
+            "-5764362",
+            "4660427",
+            "-7905626",
+            "8154092"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-706829888",
+            "1167156309",
+            "1518125729",
+            "975860423",
+            "995194446",
+            "-1865080408",
+            "-665865487"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-1701907448",
+            "1352934836",
+            "286059016",
+            "-1400717695",
+            "-452649966",
+            "-1922657966",
+            "2070417333"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "1935351572",
+            "765282566",
+            "1987622027",
+            "1741583735",
+            "932400011",
+            "167151325",
+            "-355472668"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-396859212345",
+            "-358864231013",
+            "411692595169",
+            "-32032800170",
+            "79797592838",
+            "494642730187",
+            "482700694650"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-94529105733",
+            "-481091693177",
+            "317977953065",
+            "427273590013",
+            "-227681357693",
+            "99285197947",
+            "-49749811753"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-117118949406128",
+            "-16338400756989",
+            "121098098144191",
+            "17786074861365",
+            "139116588792903",
+            "-104900603769656",
+            "77460829002909"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-64152968257972",
+            "81807906354014",
+            "-9962726454967",
+            "86559569733926",
+            "-10083720575770",
+            "-14825156695938",
+            "-122499888153949"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-133378720489228",
+            "-59578984792077",
+            "48240930882493",
+            "-81399868956224",
+            "132623478058564",
+            "-17946185246720",
+            "108220579003501"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "27603984537822762",
+            "24285467590617142",
+            "-24610833591141255",
+            "-13707671138044065",
+            "-24079809168036321",
+            "-3426285774015157",
+            "-16661207174485960"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "25341970114930753",
+            "-12578463204570261",
+            "-27319367490414208",
+            "-27862145691252063",
+            "-9374098211284058",
+            "-224593245961164",
+            "34774375621000453"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1508260868549245163",
+            "6273375209195432213",
+            "-9078384653770555315",
+            "5236211367446183278",
+            "4843815885489957283",
+            "-2504671565165730378",
+            "-8947215402970277018"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "7409921464496531061",
+            "1356029422123982187",
+            "-1364654770344853581",
+            "8837499222236300067",
+            "5132751121490100122",
+            "-502781462678344483",
+            "4467369067622920214"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "2154925127820502346293",
+            "-1306492928287846607375",
+            "-278826605081884366316",
+            "882916332012524932211",
+            "1867667626776166611792",
+            "-1257524031015343087488",
+            "994948667658237479955"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "539210603887413500119",
+            "-1351832695016986241156",
+            "-1062904294743508083575",
+            "1450483825798880009652",
+            "-881626741922698995815",
+            "304292780803779267384",
+            "-371319374125766711584"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "243226565466619142907",
+            "-121955990464408141742",
+            "-636816055580076932255",
+            "-513308924855956320428",
+            "-1586486017550068684983",
+            "-197644649227952900350",
+            "1748345750409685925848"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "202061337421568637115802",
+            "185405094041080393854566",
+            "-147903622544387135396853",
+            "207229453925763208622831",
+            "-343183304767942752569844",
+            "-263692037517810930229757",
+            "111266436442025136177952"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "36355222093341524385642",
+            "218768782508355752146729",
+            "-200541398482024855947446",
+            "28474517856269610750246",
+            "-475859723844988539708245",
+            "-300510275404419103333742",
+            "546674288910101331332936"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "3601936927556237989024608",
+            "-3692430919094941422100028",
+            "97748690453108599708927434",
+            "-116661353115321540714375347",
+            "-71879985746676902756048669",
+            "-121080329577202994807755612",
+            "124157596086231452131087011"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-121811982953161783629612240",
+            "1607554249264877581531325",
+            "123072061986545439633145",
+            "-73550837036916705784352874",
+            "-117419921478539398829685153",
+            "-74399538740549801431316551",
+            "-53072519800681711726312500"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "128436901337483555656980766",
+            "113598952940587548025391029",
+            "88411366081005561182297775",
+            "14815457804520490927167142",
+            "75638196408599796922678025",
+            "-108087849560548352593752169",
+            "-11833635007482033712808763"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-6781678961533213012517697687",
+            "33224612367898641792526203105",
+            "19741448595364719290571423993",
+            "34895232778404370472470020141",
+            "12658156696934366509236475832",
+            "-1339988824059695372854332699",
+            "-15240202918093452204022855813"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-23304905447440712184522177464",
+            "38854699585363157300282622119",
+            "25197020277731879349242146247",
+            "37654843997064076575431679106",
+            "-37694332242432992461842973886",
+            "38735096335824029183056699705",
+            "1113712061187242088526216287"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "4872842181239913302717185661993",
+            "8110981189886162420036583272468",
+            "1846154365393107878136252015134",
+            "-7355182167045667821268714569976",
+            "-456342272099675035018815637968",
+            "-4741310371570496786057876108775",
+            "8692654330279187212246590575934"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "789321336877871879752144980182",
+            "-3592030043211469330492425640956",
+            "5543560868004228342173534568596",
+            "-8310481621300119615028360910615",
+            "7240613238493658400891584464926",
+            "6698325613041346941747416888641",
+            "-8117880941116130680718189365519"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-3557820883496744079071315236925",
+            "7037071399112810484115914176160",
+            "-1767092291265591600640701935680",
+            "-6045295616444022343488779010934",
+            "9259710329844166183361141015059",
+            "-1492876309663740055714983855690",
+            "-8427791476065938140828199154822"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "1158759477908482981738237800163588",
+            "-1173316691740305096863218708950191",
+            "723215252639660437402175676963743",
+            "-1879559334466643497978379903607782",
+            "1983927892969038400024936381013185",
+            "1149553546691385176645909195969356",
+            "1166342561375174956477479521466108"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "626529133445278098702551621416112",
+            "-1408837087511730448581596972023260",
+            "1906905437026049050822161959468707",
+            "-1929693317190147781992957517985541",
+            "-378172107382327021537808059758513",
+            "-2104995076904948347049313544321337",
+            "1294253937974735117873956527399622"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-523256123496869429061522979056899046",
+            "-18365757383114987172474795440552034",
+            "-204316577987191697375229787073710633",
+            "330181965802203666233116582657210837",
+            "-329328597335009708600913451090602124",
+            "203539490982126102130410391161030455",
+            "417738506604065828522825064729683710"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-321000364277288528283109856107655940",
+            "580137014281216496641049610076428712",
+            "-624714144804715993809991357206356781",
+            "-407262969746696369705355949281205801",
+            "481317944191441079472434015349123595",
+            "217536632380375131817558978089536953",
+            "481697272008029766433028053959999813"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-144116647597826623855199496157914248857",
+            "52208802527320772539382939270524160714",
+            "-150911491870745668133846108351239739078",
+            "85442639480033500108059636873587453266",
+            "73892340483729688596932981784301693030",
+            "-21983833769824359378093314118115605654",
+            "155791999129521894360881202697471787485"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-161222642718655133616350033662785590309",
+            "-144401176588064295883792554624730874458",
+            "-152972600928373921945993474844975784915",
+            "-138536917768299990575075582087577539387",
+            "-162776200467637379804062098437779005139",
+            "49042684400063856845441060397203950029",
+            "95767276168967643036870259756918109375"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-39863797120385387096682208969556052403",
+            "82559022270581885556475272207184980799",
+            "117510934395578824951787214497627148540",
+            "-83476646270141960487173091399240016205",
+            "-1758233846624033403425881485773452965",
+            "-135007718267876949864445576181493416395",
+            "-141767357710928769494049512437739952119"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "31322",
+            "-12998",
+            "-6668",
+            "-18050",
+            "20870",
+            "7407",
+            "31517",
+            "30964",
+            "-19025",
+            "-26502"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "1882",
+            "-19708",
+            "26796",
+            "32398",
+            "-3699",
+            "4487",
+            "11540",
+            "13472",
+            "32388",
+            "-24551"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-4766986",
+            "6654828",
+            "-7036686",
+            "-5243986",
+            "6660194",
+            "4688952",
+            "-6565312",
+            "7846449",
+            "-5876806",
+            "-491641"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-3816976",
+            "-5268710",
+            "-1742516",
+            "3856652",
+            "4463747",
+            "4089790",
+            "3013827",
+            "-2806363",
+            "-5993564",
+            "-4334975"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1939111885",
+            "1089839138",
+            "-423110561",
+            "-238112023",
+            "-1060805567",
+            "1288567695",
+            "820382341",
+            "-1689688269",
+            "966765872",
+            "-1935608267"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-806572859",
+            "2123921205",
+            "-1951507311",
+            "-1152859862",
+            "1529442938",
+            "1573780525",
+            "374401763",
+            "835164738",
+            "41728321",
+            "761957516"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "1756123501",
+            "1520929506",
+            "-284683124",
+            "1144653924",
+            "-552942804",
+            "-2120298060",
+            "30126814",
+            "643208321",
+            "1693512950",
+            "1968260177"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-543650305893",
+            "-278464771584",
+            "-373909651582",
+            "549153445885",
+            "-242295101951",
+            "491322040017",
+            "-328958769652",
+            "-24343088729",
+            "379399746035",
+            "-103071495012"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "445831171065",
+            "223984208224",
+            "183764160166",
+            "489898253722",
+            "-352256812260",
+            "132712896758",
+            "-213452847196",
+            "-458421076936",
+            "-337947454298",
+            "-466450812833"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-124879858110290",
+            "135519222186375",
+            "85699747562265",
+            "-85042659127554",
+            "75573515688713",
+            "-58488578730867",
+            "-41028103971674",
+            "50130623698029",
+            "-47534808179200",
+            "-74283087483722"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-108796237752434",
+            "-77267431737175",
+            "51570892952731",
+            "95335199636762",
+            "-120149530213851",
+            "65347251353625",
+            "-20627278920101",
+            "-88198554155288",
+            "-7858756951448",
+            "36719638530326"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "37989750263669",
+            "-7110474133358",
+            "-8142079359472",
+            "-109838866716358",
+            "-109570651345817",
+            "-54071131597599",
+            "-5091765250455",
+            "-88967218959552",
+            "-26610598798817",
+            "135506189432092"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "4927887103256557",
+            "-18754894942080393",
+            "-4912007726659186",
+            "21247575662236694",
+            "2797426944150268",
+            "-6616351412144510",
+            "-11858257489510497",
+            "-23188104958701432",
+            "-8877819571973802",
+            "1148489375738940"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-7825477566817051",
+            "22013917265275897",
+            "7697670787874498",
+            "-9332621858164545",
+            "-2912282090647944",
+            "22112963575670946",
+            "16681702598755920",
+            "-27753683646566454",
+            "6945904955659446",
+            "2442936651087792"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "7675767189136958218",
+            "4376177190988076013",
+            "135914493625679976",
+            "135689732946467188",
+            "-4157597916871123429",
+            "-3202587243297732574",
+            "8570927145651867846",
+            "966150880672931514",
+            "3340433868757095215",
+            "5044440160311120876"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "5133284943736156946",
+            "4845520967720236486",
+            "3808412010600114265",
+            "2390070001401807490",
+            "-2075341331489173819",
+            "7269598711719193440",
+            "6639116030361989708",
+            "3990965590531735058",
+            "6049354888289615042",
+            "4801937515511789680"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1064355765009099408700",
+            "-321943498165958622892",
+            "1201671142156144011576",
+            "1520237466996047474259",
+            "-58924034610474786199",
+            "1188600272040920538189",
+            "1973243532799526539431",
+            "-1589754043423107159161",
+            "1069084636593819465866",
+            "-2225914539586890120093"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "612790270140756043932",
+            "632096876907248352897",
+            "-614445606437892415535",
+            "-1538574921109377688884",
+            "686539122267683104890",
+            "-176896054618733272731",
+            "2112326625855675905778",
+            "1807700217188087573935",
+            "2250554304073443683530",
+            "2138601425060065078780"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-2178158937595239087007",
+            "2148658261965542925597",
+            "-1786860273154704658204",
+            "-2284942685968830599534",
+            "1135179943332399887722",
+            "-1850089368721794824531",
+            "-2107516150972307207136",
+            "-898833291339509074760",
+            "-1560291136165671096302",
+            "635830273331647493827"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "545257608208761234784401",
+            "-478027394265681567220331",
+            "23160102878411060468572",
+            "169851178877058380224552",
+            "-125063444295476267854414",
+            "-511661104901652851551160",
+            "-211091539176940271013432",
+            "-260892604441446245832984",
+            "334071806187190766343296",
+            "225652640236774742401805"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-125435440033823628710650",
+            "134873701855643950966740",
+            "-327244905598551933045573",
+            "392112435242139096124146",
+            "-392770190944746034104800",
+            "-444456098660755532238056",
+            "1948570436020667687581",
+            "334511617766662717840293",
+            "459630175785578568834194",
+            "295768689726958240598895"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "110764581191900446903790037",
+            "-142251040052070922679906758",
+            "4521074766144521546724589",
+            "-77954834062755231010987947",
+            "132206695009844374949071160",
+            "-115611166334940287023117326",
+            "65706530528500814014886945",
+            "132926781394523064114813219",
+            "56946888735676586779415707",
+            "44978068193037238045912896"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "79398695417455848981455196",
+            "41296350950824030267042588",
+            "-44890473232271813368854431",
+            "-93845863653461695811544157",
+            "-87673990780796119057141077",
+            "-79728712608821225741329213",
+            "133650017782788394397592147",
+            "-113944397482899885204627134",
+            "8147591003946322262360118",
+            "-88347016701263595509923031"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-97315706287462853410250215",
+            "-19297884994476314092921079",
+            "-138156245563462225217055556",
+            "140082648536915847982187381",
+            "-116717372906786739105728127",
+            "74725068650981797124372774",
+            "115115819529181418953999340",
+            "-104692994218324284120901952",
+            "-9339214268717197582310194",
+            "34586415164615511096229871"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "30452540938400888864313250312",
+            "-2802879378440933700326909109",
+            "-26896580846052300048510748778",
+            "13956100052440828740572023117",
+            "35092158772914033116097802591",
+            "33847590995690391053397721124",
+            "-6872021573016953238017942938",
+            "9337504336202853974096101152",
+            "-8139564364961519011852381841",
+            "33614459434990180894524889482"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "37905470020515740655644953427",
+            "15095043730643435565566661915",
+            "-16054339514326668180752045709",
+            "20206326378427568201157613061",
+            "36409538996611392332800791135",
+            "3986926868645572538910763024",
+            "3619435534252705282359941231",
+            "-28939638180493501947435665546",
+            "27752306518932422634994807389",
+            "5299661533164103665947907747"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "6105089286518535143859193272997",
+            "-1558013028566094088457684756095",
+            "267447018904579079104631493606",
+            "8716274064389440981111441588450",
+            "-8523502283934172783824252808972",
+            "7211587162269446695523784988664",
+            "7349202932851786545780603930347",
+            "6835046636946129018720692573076",
+            "-8333744634109651799888167347108",
+            "-2544747380910102910856810717389"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-6352412655736206068742730200924",
+            "-2123175620394690676305322433458",
+            "-8836382719876284105833993896413",
+            "5250128511200984854060353093256",
+            "10117785086990984372555197700371",
+            "-7553019007793539065221400188849",
+            "487332886938332402116864111164",
+            "-5531527231691294946689230570557",
+            "2033195130458534180764526511590",
+            "8418380572809691654984623098184"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-50075375212232172188420411196",
+            "4809924043195256813690822020596",
+            "1275374378747394211129459178045",
+            "-5194775733165378826646329523658",
+            "-8899088366764662523040904888258",
+            "-7065161673811581445863416064724",
+            "3448352081115495266715396247033",
+            "-772423708139560818203910008799",
+            "-3716422423592581053204772584415",
+            "-7073723704623872354556609478292"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "1662201874412358678818367544757313",
+            "767632843116354510741172561523193",
+            "-2574598778497120728925932302019195",
+            "822408943008991309024189274780663",
+            "1442207677524639837617120203090781",
+            "-2329938247319978136011213385811089",
+            "-844653930290095052875421889147387",
+            "597209484634644162562668851350801",
+            "-1653776464297020640113550379942905",
+            "1712355361051968314047230183856759"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-934154790921317246804917189844883",
+            "2011996172621693547547251717415199",
+            "-524198925581653076023930890792205",
+            "1460963164679956289335063949063860",
+            "916547324738412638652836279760881",
+            "-2181480489373236251201848006843613",
+            "2227194053359186942441098677400067",
+            "-2070283524814069666316978457528190",
+            "-2557886824011973362420531382315055",
+            "2359841497596077952056722100997103"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-244737191158099062140625546830357355",
+            "-558098781397099000894773898031616911",
+            "289872866815248801825838878888401974",
+            "92794478567836555966241203902901710",
+            "244269753519144613414831030049656050",
+            "488344310874378240044622887144568360",
+            "251880555776188717959293063210533656",
+            "650424241144052628672668970511093431",
+            "-490149649831311456535113419187895360",
+            "587815026061852779759502140522870365"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "372180445289567816296974210709998262",
+            "649322101444650143089117190099040098",
+            "-261315503489678515225280175375928660",
+            "-522083930998546622308238829319416419",
+            "142235256333754647127831983081551205",
+            "-166535174840652571060843789537974209",
+            "-466105320259826015444363036165976483",
+            "385149522321114928821540160206410292",
+            "-41443525693038526295999228480843805",
+            "-313051347741978666120077422667886094"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-167439439468877308797697749268322963993",
+            "-28242537581789671178678277260561674281",
+            "130587384617519710400745151774880402494",
+            "31589263496961519660807111701435015251",
+            "13326683789176912862324807380879018213",
+            "69851378199071338355039736806394653047",
+            "59447861241933516870558227322235615556",
+            "125427854769440091466924747763902774485",
+            "113841414377172020356030816155031973980",
+            "162459891215489371457804173662865844603"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-9113228729168889224553302821519848527",
+            "147906368719665671855384750115565399639",
+            "142408267705249397493244319994103808727",
+            "-158173827712728250139318830913214547586",
+            "90457292148533516388688156622880399300",
+            "-28762317200208469682474790209657754600",
+            "-147267994027501072294873134903092981782",
+            "73349578944467970931133887330177437829",
+            "-109559717261425791283100898451546302168",
+            "118124046303323736396291668012601665112"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-92141949280218575485253751932496543626",
+            "138068266350197970395619500727113936796",
+            "-57057880351028370035446451706682228912",
+            "140046511005251583337314294112597416965",
+            "5313615659947632571000091701353480420",
+            "163618210678848808873963839458536967363",
+            "-83020522347645062930841470958792915560",
+            "103388812228278752501831367543501892798",
+            "105919923091739023980005477488419598090",
+            "-15642676827816127683186187091061039771"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "1173",
+            "-25049",
+            "26985",
+            "-3279",
+            "-32375",
+            "11902",
+            "31640"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "23711",
+            "10618",
+            "1182",
+            "26210",
+            "-29042",
+            "-26592",
+            "-32390"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-3383836",
+            "-330331",
+            "7170648",
+            "5645424",
+            "5519488",
+            "5932048",
+            "-2721797"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-5680425",
+            "5530032",
+            "8360972",
+            "-1579435",
+            "1281443",
+            "4742079",
+            "-5190818"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "791809523",
+            "-1726831311",
+            "-306766525",
+            "1510344688",
+            "1557526531",
+            "-711666607",
+            "1285681573"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1588122426",
+            "-1825651554",
+            "-854908281",
+            "-1682899390",
+            "-386907059",
+            "1124699281",
+            "76032732"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "21696611",
+            "1114662058",
+            "-398379994",
+            "-37751028",
+            "-1858929126",
+            "-1571912144",
+            "1357766028"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-207926907151",
+            "-108845084422",
+            "483890046680",
+            "32829509085",
+            "276444998455",
+            "254643057519",
+            "162285997419"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "51727593810",
+            "-334061936348",
+            "30150675407",
+            "-331201335155",
+            "-501714900752",
+            "122994273993",
+            "-313427617689"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-38665729792403",
+            "113150504075646",
+            "-13283184519777",
+            "-122268737353293",
+            "7114949849358",
+            "-76982898428076",
+            "63703755155967"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-631968410916",
+            "-129868532669274",
+            "-138633149135123",
+            "90053223046351",
+            "130107621630474",
+            "-126528386107052",
+            "39830786351675"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-36266273353597",
+            "107910261195518",
+            "57616988744255",
+            "44429564088387",
+            "-52991708070673",
+            "18856755200112",
+            "-88910410722229"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-10733019451407946",
+            "-4637031550251259",
+            "29038597614687455",
+            "-2255766756551662",
+            "-6357871065260787",
+            "31141018628250396",
+            "20316885381831172"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "32276626527716918",
+            "1143035765853136",
+            "-35727015302634840",
+            "-26484971152317146",
+            "-32859967854440443",
+            "-32563301944872556",
+            "1238987785486865"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-8755802922599672235",
+            "-7396181851925542211",
+            "6711186749545114591",
+            "5590049579316174533",
+            "-3165569916335790183",
+            "-316326350454384555",
+            "4929314904948241377"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-397606177624923608",
+            "-8426436663324202280",
+            "4326137699230271144",
+            "6586316848218801011",
+            "-8091380893667926501",
+            "-893221278247799034",
+            "3462259882527377933"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-123690421269484703760",
+            "-1267016901460774922921",
+            "299161792651827136146",
+            "-1604897520961911844593",
+            "2037882344461012186736",
+            "1195943811863661819614",
+            "1207132107388937164804"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "2101648800869166208330",
+            "2274724666892686998217",
+            "-391260607449403060721",
+            "-539342914187070082276",
+            "-1440115909019570656078",
+            "1971609407978951853616",
+            "-1123962484080572650793"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "1226504478266363310149",
+            "-1336799843493576778914",
+            "-1930283161332942670068",
+            "-1331434065508788145751",
+            "-2246411720300345670491",
+            "407117263123141212856",
+            "1931974044378992306595"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-361522891823295867238478",
+            "523641781996221269629669",
+            "493463729381734013522117",
+            "132573578585472506680774",
+            "94595327070388358792040",
+            "-475960519364994802696270",
+            "-400496792855539234277056"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-83327604823851708214213",
+            "-21750678079158391741287",
+            "303079990127928613453200",
+            "221033538070216052397251",
+            "349922191060743894146041",
+            "373614674984738281954995",
+            "35960928414177633291510"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-35263044120328302148125905",
+            "119902294752181475961380716",
+            "-143462823401917067325450150",
+            "76224465900409165961186326",
+            "129425531295697896902175651",
+            "71961437570078598659959749",
+            "-50542941964441222281681861"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "21273851514521946688339521",
+            "-114682664467937571727316318",
+            "-87665586498201782141883102",
+            "-37148627339581480801973740",
+            "148715674812065621394995997",
+            "-11949810986329503823448971",
+            "19902828440769121844807521"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-82305650498237902006296677",
+            "13405144161056810438291528",
+            "-55538748559515558770298655",
+            "47675008502605479352214681",
+            "144416507011350857414309005",
+            "29779427576479727347932773",
+            "76124247118585246765896254"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "5201245639226497436237918218",
+            "32394407131966840913830741879",
+            "17275077614546392015968733648",
+            "19680639349359359322458447337",
+            "27317116065055333169824937849",
+            "-12746107751769296292286176888",
+            "24689167759685843386070714338"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "10674526914216998427441154557",
+            "-34360582954885045536029895204",
+            "-30414741650041537218707792172",
+            "-36100155712815379900420808706",
+            "-9135627534401552379066184552",
+            "24739179676383158523222359862",
+            "36247531023134689615999840678"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "7728506401956815228532038088763",
+            "-2837122511678749937658222392985",
+            "-5052751030412737719634091683608",
+            "6267848105390862826653603709957",
+            "-7256424895588253047138206019502",
+            "-1465562444156137793663621199168",
+            "4747031647779859427140201067878"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-4615761627649385445936725928202",
+            "-10039625391960422316333169130120",
+            "-5864157091799713513469653318184",
+            "2127816811402154227868729483435",
+            "3887900797372126787242346200516",
+            "-1886685381966172685286274273546",
+            "2174897160271324780574975683648"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-8147543028570098325830702033212",
+            "-8166347530503241567728663732295",
+            "-2400152589461801277732655340968",
+            "5816370023067145183233116808816",
+            "4507165196511235167888642848780",
+            "2286027903888132217006101817455",
+            "-2479439547905569438663575023840"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "1828555317791876329482620856869918",
+            "-1789881062592639492432854851944912",
+            "1997518813798357124855744528406909",
+            "1983001417735981157064295426331782",
+            "-469859785106586780643284066769335",
+            "-299069863301987956957383980507514",
+            "1236379623573944194394212606448257"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "888629009780912538325980386511617",
+            "27717296778921441430770399511331",
+            "-1037544761462996724025231718395229",
+            "-2569726393180105871878430433407165",
+            "1050434002808238746161893184408066",
+            "1631428512670349250299644326318484",
+            "-2300221672111018999696902542660406"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "117708577531638404447859755168996138",
+            "499980650229427236421511725860459336",
+            "-11095778332866038210737561230496935",
+            "-545351732363151371037809309111972618",
+            "-247593431450103821518249361274651770",
+            "-237793299013077356563098927682674357",
+            "390680910028520326666704921002649497"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "451974522107762371225049594944291931",
+            "-105576308356387969354658158113358895",
+            "131444584597404940972776636357533837",
+            "511081773875049253533205440188898072",
+            "-152320758908721137229679446546078860",
+            "440168332116421922278190781376074082",
+            "422466258163678281216225964929359699"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-157339119467114553152239726866392495",
+            "116897764313121017716807765578012098753",
+            "74820773589003012395258853673597787962",
+            "38832900549006200655588104913719049413",
+            "31809184412064166455927543375666243747",
+            "-26330135739473810705643960803773520617",
+            "-118932439894932248602393041154357721181"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-25109297380056962038867001208740287232",
+            "-41675596142284332174303529214392893663",
+            "5630763719989167974840000823279633369",
+            "160052832392163305375298352935547628724",
+            "4354500338288930103695848603658025851",
+            "-79240341020657408457197673660870174461",
+            "-105142963009963436353607279364134642269"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "98982080008354791864222553245931141612",
+            "-14255476744570986458215294770192518639",
+            "75075249011469661237617297182263259411",
+            "9960334250615541536807663596098652831",
+            "77610568733923384998121313535231559129",
+            "-124628932104686355631960480356638386284",
+            "-54909581662559425139035632209125077908"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "3832",
+            "1722",
+            "16659",
+            "30179",
+            "-24098",
+            "-17484",
+            "-18903",
+            "26194",
+            "-29015",
+            "-22705"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-28693",
+            "-22191",
+            "-31428",
+            "3880",
+            "2222",
+            "29307",
+            "-1198",
+            "-17255",
+            "-8873",
+            "-16187"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-3944372",
+            "-7965251",
+            "776136",
+            "7478406",
+            "479063",
+            "-5101226",
+            "5809084",
+            "581754",
+            "3714208",
+            "-78676"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-2197720",
+            "-5978443",
+            "6040915",
+            "1115058",
+            "4078042",
+            "256907",
+            "93983",
+            "5378385",
+            "-2933287",
+            "7437051"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "461104817",
+            "-336460791",
+            "-1866209508",
+            "981614060",
+            "-496512504",
+            "-1842967640",
+            "2028811575",
+            "-108083496",
+            "648817093",
+            "-911531546"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1855062455",
+            "-1848423958",
+            "-417919719",
+            "-1550324221",
+            "-226454251",
+            "1897143013",
+            "-1816207352",
+            "-1890930858",
+            "-1050656588",
+            "1380332427"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-125228080",
+            "147425902",
+            "-1803257123",
+            "1133430594",
+            "1104179937",
+            "-1155651362",
+            "930863958",
+            "-1372345558",
+            "204045790",
+            "325493759"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-43969925332",
+            "-423782533903",
+            "410599621485",
+            "-384348480285",
+            "410563996751",
+            "420331896854",
+            "-451260071343",
+            "-239682713334",
+            "258658798561",
+            "-490359578846"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-117878031314",
+            "160469154039",
+            "265139111256",
+            "479577383616",
+            "-359749140482",
+            "223868593231",
+            "-322904355205",
+            "350722353309",
+            "408233732665",
+            "297350052059"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-11402130119583",
+            "115585152210399",
+            "109875950346614",
+            "-26937162370986",
+            "81368062928951",
+            "-101279737882071",
+            "-79021567150608",
+            "33691525873326",
+            "94018510142332",
+            "110175572468354"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-10950828627484",
+            "-45009616517875",
+            "-33740390224925",
+            "14272990246117",
+            "-77929003338940",
+            "-102300441781475",
+            "-88070246222892",
+            "46505217555818",
+            "33852556004566",
+            "88581112696689"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "120432203274419",
+            "-15955787808560",
+            "34948631108647",
+            "-15994595995724",
+            "-27803388562197",
+            "-63066930361908",
+            "-3386645943750",
+            "41101017052909",
+            "46443568439036",
+            "-130299379020281"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-31001354878037938",
+            "-760641451157916",
+            "-4527935698981333",
+            "20422875389727418",
+            "-25561171622510273",
+            "6859613823629514",
+            "-31967296838595675",
+            "-26772749609822819",
+            "35113008647366866",
+            "-21247928407771622"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "16605450096694934",
+            "2713804935498894",
+            "3748040022825592",
+            "-35884925413042953",
+            "-28842895601915851",
+            "5002209115350832",
+            "3529334705060475",
+            "16112399859517651",
+            "26710552206102001",
+            "33271555054113480"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-6948678853416573641",
+            "8088096296940258606",
+            "1788919402499424939",
+            "-9045098789632663964",
+            "923217667787488454",
+            "2113233926511753642",
+            "7674437743867941425",
+            "-5483523786006335142",
+            "-4901566326841744867",
+            "-2159924707245153950"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "7483952831904713529",
+            "3981681156262502150",
+            "1574702197696520191",
+            "30404321841739229",
+            "-16526697791658885",
+            "-5887792884766481411",
+            "6480092953553123685",
+            "7930432301453660255",
+            "-950334699167714593",
+            "8999305224741512342"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "2275373047011451096072",
+            "1663741816301024567004",
+            "-1866492858541257742972",
+            "-2109145933811908015055",
+            "205702837245156733125",
+            "1635077412214287742642",
+            "685205307788793936320",
+            "1889860958451267333814",
+            "-1166602441424869654187",
+            "745504509663417227326"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "1798836380972328231764",
+            "-1406223119331403767737",
+            "1660047033970478613098",
+            "1778052602148636827065",
+            "2250316159389610073455",
+            "-1298324302681600099899",
+            "1334078532892003231310",
+            "729325161456818198272",
+            "191058750304629973982",
+            "51010503554643728898"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "1847340334856600128313",
+            "638235693099620437223",
+            "-478038004560773338092",
+            "-1391998540736358145230",
+            "-626383328490987256762",
+            "-2352574400149631230455",
+            "-1938941449061847831383",
+            "-526607055338931858789",
+            "1981655780781527447679",
+            "1545396686837274666506"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-312442107296555228736666",
+            "356012445393395243219201",
+            "505759552363745699184467",
+            "-468676292296122067383179",
+            "144650032031703545186687",
+            "-26329362228718986332534",
+            "23980362387843468011798",
+            "-365244050437485466278021",
+            "-243995662429130162331158",
+            "-204320605551784536257214"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-45575166340192063213221",
+            "-573081770149787292165894",
+            "-265165687156774270417407",
+            "142789058928229337895172",
+            "191301704388489595602183",
+            "-526986468133892967767446",
+            "477245941154730186128251",
+            "85837727744584298073480",
+            "124665454939314352568168",
+            "-538236817734442648171792"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-31883334627754186218955050",
+            "-132647558483145987490002060",
+            "152621615451762950923746193",
+            "-123974549416047643124802713",
+            "-16114612038666942583062670",
+            "-64534320970354404240221340",
+            "-52121728535745783604990414",
+            "-94488926190663283007722749",
+            "-142818466323905379126560097",
+            "-107911476720388673547687265"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "85894584079322001591239437",
+            "15214409976212802853836897",
+            "-38019531841082459805788664",
+            "37616670829337571764768074",
+            "97369332259197095859847091",
+            "73772357245924419843255015",
+            "-30815777011884938279345071",
+            "-48639604753962660078735675",
+            "115468028917195898397228106",
+            "127926946301889108741445954"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "133666088035672066583524552",
+            "-130261035231129416075204296",
+            "126288325871568137717986524",
+            "-86845343621023472085945052",
+            "107973761265249358331477587",
+            "-129772336942492423381739079",
+            "107590361807184920300114694",
+            "-15925934067100476069944577",
+            "40349773922798370876587602",
+            "10237290409613547079342522"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "23802936985404384833087799850",
+            "1891175066573372917579879025",
+            "-3414847211423679295209241879",
+            "26381566039446157654614814849",
+            "-27987571844346932586533728809",
+            "19734355232998847428072102124",
+            "18813566938676176789254455011",
+            "-36503995951399515178819792357",
+            "-14530867618962366745963282995",
+            "-8849346604262124212079590667"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "5181898859266891446958312760",
+            "35107669831244838496494366387",
+            "-14410695362339474219725640659",
+            "-11591179713532133832522279888",
+            "1322035740173050242109098628",
+            "-31330745687123561728243165717",
+            "26792385853476620307255105569",
+            "-239671734221072238217774753",
+            "19102158857542933762328806730",
+            "22327891859950141710268230741"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "7218784186398202012713467930238",
+            "-3350235717963492612747364671098",
+            "6231440398343118743080139900050",
+            "3480339439152451974013676380123",
+            "1802222928179100234342084269172",
+            "-6158375179378131392944011341388",
+            "-3504666018658185926339154570280",
+            "-8475441398809634118910111467555",
+            "-3093801338563751188575901637318",
+            "-5478326702705048089627298988765"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-7070255220529922623499452044373",
+            "-6392207880846944173023026301966",
+            "8940482032109394222736828154202",
+            "-5046383856173347902770746632199",
+            "-8627626268162429998930354857590",
+            "2158989160600634779705496438004",
+            "-9013792196143998093187489447006",
+            "3476375415358910964701042924829",
+            "10085089756175313037187138824355",
+            "9740053811517652352505683816287"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-3787663618609735534843807278261",
+            "-10131105569536599796884449801691",
+            "503460830171569790542890271138",
+            "3434572231596005739225905817721",
+            "-8956835466725390809094014178560",
+            "9973500372789618871294599932426",
+            "-3436060305237781453095163214550",
+            "8450001797448541892298668439883",
+            "3361188979149779406972984767155",
+            "-4211051396445129112263473603788"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1400604743906584917351131252499625",
+            "-1478062509467749242110369849940624",
+            "-1939775116070969545684922458204716",
+            "-1226594030808115703890602534185886",
+            "41411606629976216575859515000588",
+            "-715726075155862393171728350610649",
+            "-1104837525141739703305043777788269",
+            "-33170364036380943653659197533160",
+            "1222412368140077471220168238614012",
+            "-1553672963167499514929514167750075"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "414812444544593930314879937191796",
+            "1065092968521305037826601157806551",
+            "-1284545736853571361465316035084177",
+            "1913928682493656194884471773672006",
+            "2437413883363914925409924254101191",
+            "-63504718310075110440759765897849",
+            "1142014044009551494450713989884540",
+            "-467473985179578855121650728580161",
+            "-1958496203046466422013468466142816",
+            "-228498365668817309414956102721486"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-165107146600836184242508744416846410",
+            "-173159599341289250114641790452112880",
+            "519587843813546128711723035455451318",
+            "556002998269748818044894693715246655",
+            "340836458025000843041754711402673966",
+            "107273561582467343086931264151931276",
+            "198750867607657619623292872099343193",
+            "177053814334723551673700131599196681",
+            "12660542230757668534285774032322329",
+            "-571464786368411573231284232367569493"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "162179090271636955997303945925436088",
+            "-313756832601974313228748371796894029",
+            "-479290633805655842621397357725946736",
+            "-511500927368428520551421482817201088",
+            "275242443136997885046391431699367254",
+            "-577640700565811102564722509570839516",
+            "-603313958861080101850976268791504285",
+            "-4642553372325527067527268751318523",
+            "585149029480379128228533930021017452",
+            "-447624843136876027041393586846771215"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "41501085361451418692451075430330699392",
+            "14066756191036297756834058689490291695",
+            "-164566469205370280337196747799862639279",
+            "97007518905806845925433851317020841140",
+            "157835227951254852214602652284268481160",
+            "25273736966363553104566178634088585028",
+            "-150537488010280684140991686519150494549",
+            "-3078910762200766609996313414922029941",
+            "120706844608861526819394623973916082740",
+            "77402899003013119828768311588427778798"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "17155049489457881084030524101061900599",
+            "55722466817055128162883412356150704414",
+            "160092374234431079498351920387290796423",
+            "-81606519127017628665631973004297421491",
+            "89375409744877313802670851685928047535",
+            "-65994081055910568627784130329323864802",
+            "-120630990114675994405665160336455407373",
+            "-137414651139106149921771512358288667134",
+            "121174892531857852483583596440818324131",
+            "-165550685491777099186772462955579166433"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-121678364556055223485335845024798563730",
+            "67906221171052313441526344645309169424",
+            "135318293234875105755625723591118174647",
+            "112856765176163913169894958961682862611",
+            "-119771416342600648005955621630454841004",
+            "-163903534238729373402472686918104270530",
+            "161195965019384544589699058515114347869",
+            "-116587313658056174021511449216936299530",
+            "64724954802600766569366551938003781249",
+            "-93125379513310589407008602216272655289"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-24289",
+            "10573",
+            "18399",
+            "8295",
+            "-20661",
+            "1262",
+            "5316"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-18813",
+            "1196",
+            "14506",
+            "27699",
+            "17260",
+            "12672",
+            "-26712"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1816906",
+            "1950630",
+            "869435",
+            "-4057560",
+            "4225264",
+            "1863554",
+            "514068"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "5766833",
+            "-4566718",
+            "6185712",
+            "-78858",
+            "-4404053",
+            "-1932082",
+            "677179"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "383051226",
+            "-485372892",
+            "-903491488",
+            "1167906715",
+            "882058907",
+            "-1145532790",
+            "1869671269"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1968019064",
+            "-1799227215",
+            "-485104210",
+            "1368798640",
+            "-1217008177",
+            "-1227552350",
+            "1050282791"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "1331776197",
+            "-1523716584",
+            "-280980420",
+            "-199394097",
+            "220377273",
+            "993543643",
+            "152558613"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "383358476044",
+            "-549418593050",
+            "-95127854833",
+            "233604693010",
+            "241360165929",
+            "-244114921916",
+            "500668910648"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-53958362657",
+            "294585258987",
+            "-297555344274",
+            "208136066113",
+            "306802849873",
+            "-56933397445",
+            "356355330533"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-5931341008015",
+            "-85194361749904",
+            "113496344354023",
+            "110910986479677",
+            "99037670275407",
+            "-29955950246308",
+            "85080062899793"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-135973903302549",
+            "106511990735629",
+            "-68491062797510",
+            "-67355500519171",
+            "-90256403179030",
+            "-67513824484015",
+            "-117243873668118"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-100006712547242",
+            "-834254212295",
+            "123327381870828",
+            "-47227724753173",
+            "133999742949746",
+            "-18734787485868",
+            "114072994453650"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "26713924791564839",
+            "8130537069775545",
+            "-12577784584283418",
+            "-14275066705387698",
+            "-13753875085512229",
+            "25629684304719161",
+            "14775029745555656"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-11254714743780398",
+            "10473563621744326",
+            "33848039724296273",
+            "18968181007120441",
+            "23801217524644040",
+            "-26532014979132953",
+            "-27043991311421046"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-4338512873691242591",
+            "-2471396968750798701",
+            "3382716270394008960",
+            "-5536237505914333541",
+            "-5670361551544238026",
+            "3775163668383883090",
+            "-6190677986420548067"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-6811298520974526893",
+            "5349469892177723131",
+            "-5310390011268146908",
+            "-6394607027886038057",
+            "-8840279685184811878",
+            "-866886887884920562",
+            "9106364263997684727"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-2273577051086748127343",
+            "2421110092972784882",
+            "1009841336246568968530",
+            "-945047452043730572283",
+            "1410134830331293109638",
+            "-875429235306554264942",
+            "2114146046206025795606"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "2175914150359547376158",
+            "-1748223628477889384538",
+            "2082301040631284414221",
+            "1223990770664639894611",
+            "-1827289874583926966422",
+            "2188372252873287105874",
+            "949660314176845084876"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "756120269623394822481",
+            "1401758245452765618339",
+            "893699234552132134074",
+            "-1016562455395417533129",
+            "982560837349843281349",
+            "1535949657790955542784",
+            "339935115964234018700"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "160187131709749816068707",
+            "-445406864840389796795963",
+            "-364502369523172897909179",
+            "-483741481440179619808869",
+            "-363648688131558141030019",
+            "-317255639295947426340261",
+            "76880251422851734014297"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "83513645025310150339154",
+            "-17033888719894118308423",
+            "139753445403319042050931",
+            "-311922677215511648704490",
+            "6614071525705894990176",
+            "-347208526123811355958702",
+            "-331153931143899968239074"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "66937915462785606409890218",
+            "18402929024834529807256440",
+            "-93539116539481832132130597",
+            "129533946990250575778822929",
+            "-35405911050564377669039868",
+            "-55753625842174688440739221",
+            "73274477544376298222515222"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "102320970151175129799043835",
+            "101371657730044294428907301",
+            "30763947066697758866083944",
+            "125632427074532822811696240",
+            "67850157625702978153575284",
+            "-122847530486522138663035099",
+            "-51630753075526948960226330"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-5570796149032603732772443",
+            "112962510499227217269980469",
+            "153243861290764227222710433",
+            "53437571104225666803388890",
+            "61392106787538463166584259",
+            "117240447134205597906098644",
+            "60147122185044384664950672"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "25440681132059718117365230086",
+            "36918164020123084473254266745",
+            "21550823988603061803770774218",
+            "23314393410588498837026340595",
+            "6838667569178219678527118012",
+            "30164610006780796397257146070",
+            "3856966332228692365975619319"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "5081531030911632494676413206",
+            "-28718432056933259489318591062",
+            "-2050881926509555330776684358",
+            "-20284056359541046744053496159",
+            "27742784349335929086470876577",
+            "-23732359260540767132988499358",
+            "5634504669489702253559754357"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-3834438486720541018898060342396",
+            "757405556339035170443861964454",
+            "-8580423491565526971567650107331",
+            "-8945834070161348883399473301073",
+            "-5974403419273083538716640073054",
+            "-5722317395097101795951011911463",
+            "-5741054963623745695692184888504"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-8177592807298787544136200201401",
+            "-8737768036785410829204705458137",
+            "553812896188979268090905076767",
+            "-4021415115048931628518326506347",
+            "3924889803517574116437664136082",
+            "-8427629394384391196070489759500",
+            "4826578044967295536262713612905"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "7431086482852698405039291580358",
+            "-9383454246382299183181940802026",
+            "110780578216389910057065519226",
+            "2997597313220317309864866164017",
+            "-8403711664152768877667363976799",
+            "9484934137736292785001782174450",
+            "-1992550614924900659257438994754"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "2373418612554887808061895526341322",
+            "-875917552657361505492351804147083",
+            "-1898458663708254096173410834725291",
+            "290657266522040208254177101419000",
+            "1934693067868945395803972374030326",
+            "2035880975736537411226150165204569",
+            "-295118642138847608101651879233223"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "1104518889416566749000188799690936",
+            "2455684185543033692597997273850286",
+            "82712851079703819661084329753293",
+            "-410022871087898657272365213961056",
+            "820614625160173093789799139004878",
+            "-1777050003863809764253453475090598",
+            "-679519739224009400590313815819330"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-304159418192963037178769623546586814",
+            "-128686367459246117841134563726117419",
+            "-168852189389901620284754066083296623",
+            "-368361507059344027859608313034640451",
+            "406153616128697783106760476034969080",
+            "108280339193448166575810552660499327",
+            "-272917413831425220737435317129240149"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-521853357960558893783030665586434328",
+            "-343385983135307684129023060457977073",
+            "-506847810949852766012501602241167446",
+            "-320653206306289485921441154060647458",
+            "-321419500062747985044408215510498559",
+            "360059077100522851702855678971772481",
+            "-18193071889338165594572934630198306"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "135198259031897173424041180828687717318",
+            "166925950940460213304747134486553784378",
+            "91770708378464964972025279535217192484",
+            "-74034082172265236168468281957827129715",
+            "-162271236574132326749987187978014522711",
+            "-149970538812246904199283660851953012127",
+            "-33876209446467800177023509622790899587"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-61149961594095579505494563784365845076",
+            "55447541107548109725109916957958410011",
+            "-52782699847581819420368482823281766808",
+            "-61428023230847673671328947277950650863",
+            "-77167370229187325704618575761108559915",
+            "120671975447784265485882220443592967003",
+            "-31941156027929177364936983495588921100"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-46517984021030779553174610760370691361",
+            "113342560476092078423544510209910511761",
+            "-48453285416719025136328005628829738351",
+            "-156382820183969337136212148728801068593",
+            "92013865013427171673489829654013623118",
+            "-149825413114872556503600279014768710017",
+            "-41842153899195246897218693785873371052"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "22560",
+            "893",
+            "13712",
+            "26425",
+            "-31443",
+            "13197",
+            "-2033",
+            "-5158",
+            "4272",
+            "-10000"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "7568",
+            "5297",
+            "30894",
+            "17857",
+            "-2276",
+            "-23490",
+            "-21021",
+            "-25476",
+            "22182",
+            "29801"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-6043434",
+            "-7233989",
+            "-4760386",
+            "1954881",
+            "6262402",
+            "-5432480",
+            "8247099",
+            "8104542",
+            "7559904",
+            "-1468008"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-6009851",
+            "-3463144",
+            "1902131",
+            "-6065871",
+            "5322522",
+            "-5281546",
+            "-3949430",
+            "3706705",
+            "-1404042",
+            "3601699"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "865502134",
+            "1417082065",
+            "-629664388",
+            "-807386741",
+            "-932260324",
+            "-2100594554",
+            "1317544706",
+            "-1659287662",
+            "1747762053",
+            "-396035137"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "2009773453",
+            "1452039055",
+            "275687455",
+            "1229090980",
+            "1647739999",
+            "-2068020074",
+            "1116868483",
+            "-925674893",
+            "-276647664",
+            "-2124051141"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-652548523",
+            "-734515103",
+            "288855150",
+            "2003714209",
+            "-2050720739",
+            "-439439213",
+            "-597448192",
+            "1032305015",
+            "1396503618",
+            "1112795070"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "279530206260",
+            "-37787584606",
+            "-469700465159",
+            "505547261276",
+            "302063628730",
+            "232557570106",
+            "-400244326424",
+            "-500191912404",
+            "25916715111",
+            "434497276265"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-350338719945",
+            "82553936444",
+            "-303574370324",
+            "243609147006",
+            "-406918086752",
+            "-88984556473",
+            "414777054250",
+            "-65665371841",
+            "-214549984551",
+            "-160387504734"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "111115937773490",
+            "93917065955228",
+            "-96942832219834",
+            "-33245985521466",
+            "-14808380437022",
+            "-4337750370781",
+            "135538464766646",
+            "-89018159354352",
+            "107684460266199",
+            "54644567775645"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-4281375143734",
+            "-90555175252799",
+            "111437459291494",
+            "131211416029101",
+            "121367162942964",
+            "41802077584397",
+            "-15870395962588",
+            "88293896383396",
+            "5103371203595",
+            "872421043658"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-27344466164330",
+            "-64277006280629",
+            "-115200530257737",
+            "-96185525872446",
+            "-362102981191",
+            "41076176727009",
+            "97608038546518",
+            "-21909959043825",
+            "25684146129386",
+            "31887816743174"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "15805681068407129",
+            "-31632108725366096",
+            "-28430420565549892",
+            "-5685098119469251",
+            "-19359673914220516",
+            "34927075452235781",
+            "3646450258861011",
+            "23186600653867991",
+            "23870113232131846",
+            "24001450908961341"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-22903183193209387",
+            "-4042683237615665",
+            "-2730673747391472",
+            "26722058029955156",
+            "6157099985124238",
+            "20691656929186432",
+            "23541903929043982",
+            "30481714773589336",
+            "33354780396992065",
+            "5937390849575047"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "7111976505000124157",
+            "2513743790789558989",
+            "-5854964827243221874",
+            "1781418857572271171",
+            "117740201774951719",
+            "5464574628727265159",
+            "-5417016541752216420",
+            "9177082172732916663",
+            "160231964650370102",
+            "5198235504762424898"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-6959774822262382900",
+            "-5918004911967555855",
+            "-8785219904076781827",
+            "761518306349693777",
+            "-3727781912512154347",
+            "-7347378201324932678",
+            "-7407703676847128120",
+            "4192766707565496640",
+            "1019303794324002221",
+            "7284694249235670661"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-518706270599601321284",
+            "48296813336873849945",
+            "-757973019222940359901",
+            "475279089186345082149",
+            "-1857051161967159641567",
+            "-1773449378603505991898",
+            "-214240478131630835049",
+            "-286969346226347726119",
+            "-1377522753231516069488",
+            "1534327277443683237083"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "1261639334685978762634",
+            "-863251376487397515329",
+            "744768807612401835863",
+            "431090683079445762349",
+            "-1430008898903453262513",
+            "114432723295067239367",
+            "-1659415137482284952255",
+            "1956229533554192977542",
+            "-1285276599317909495561",
+            "-681033070942915028623"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "701567718623448740657",
+            "-770039146384883281781",
+            "569393332403074078663",
+            "-1154152301490888490101",
+            "1384935762862036151746",
+            "1292652059491738064552",
+            "-1901055548660828668152",
+            "-511050274918474237607",
+            "1514545794119297935503",
+            "-1582702036644800787512"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "525113524901831930800014",
+            "595544469553401116745247",
+            "531694395790167129406901",
+            "239795230404718942193473",
+            "47476260565690957269128",
+            "546393256156863431003393",
+            "-61262268239682896208557",
+            "401221833939464710528829",
+            "-300134644671240853134013",
+            "161147446687616707600843"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "219138432946234236649127",
+            "-306826762775104215875477",
+            "-122309794531015699303879",
+            "-484390629051561859282314",
+            "-484657707904256453204167",
+            "-64172125540895226776153",
+            "-142182935337162713560713",
+            "-425794894066631241856692",
+            "519332704943479362574611",
+            "-39607393270604766751662"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "117923250549289739547762878",
+            "-12796961197805647943116125",
+            "116897654782120455942318786",
+            "13956190831994724793021311",
+            "130567329542252884379704341",
+            "-151472228305034689251068623",
+            "117154604904044151278566211",
+            "-124756046758763955771939571",
+            "27553563965416818601126828",
+            "-6385245309521757072601303"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "19686173839910969340904667",
+            "67019339050451913372960344",
+            "-911760385169067625107436",
+            "-122602505415495630018637800",
+            "31593555949045884898723169",
+            "77102155795733768673367210",
+            "-88704824409118698981385569",
+            "-42307172980137248443961447",
+            "-4446290230048200090988726",
+            "74356647474998795935207344"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-133795720031537585830253848",
+            "-78089793241413791675804237",
+            "-146996109993430013479432790",
+            "23768136773568272325587897",
+            "-106683272894876981717341824",
+            "-80841936515652353087463629",
+            "42569237872224343987105320",
+            "49071873620207544414478264",
+            "-4630034012596569492380443",
+            "-10118444529078594226753741"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-24951441861790452223123296595",
+            "11386091573469723230249468441",
+            "357584100357389438039201862",
+            "-33986416720044219140319583496",
+            "23627532621110890373222649794",
+            "10017504927354356464772980155",
+            "19362269204948111616152932551",
+            "22750490848378999604678136199",
+            "-9083598083282026421514694140",
+            "-38727975416599385546239458974"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "12488280666875253249236254683",
+            "33422133051718018037208075117",
+            "-33776414240503546307124956974",
+            "-21484107419134338520121082452",
+            "23100427643229869917783680858",
+            "-1650226751931233622609376911",
+            "16906427077642537130376239920",
+            "-10289660454830607858626682520",
+            "-19484893295053214460865724970",
+            "-35952156995798958657792954473"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-4735298964100410577985543000395",
+            "-530020970016182803114777861279",
+            "-1131349626390530283723194772589",
+            "4317710074952661856482753164067",
+            "4095402110127114773697774757004",
+            "-5294319980949349221369104353569",
+            "-3048927073292851696777488488702",
+            "7634763577542382376327040341451",
+            "3115557643024666160471563838459",
+            "2926203117946390190222302068793"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1864945236373483656597257239959",
+            "405034584047843548124421910439",
+            "-3218977334120915262993640819384",
+            "-7142961612298989061130444521419",
+            "9307783629655608942945035515333",
+            "9723234906857782536374491021270",
+            "7650370082443858700585254978477",
+            "-9310868784454660367988144252963",
+            "7465773199661339364694190806591",
+            "8097845381146606888985497903233"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "2675669436671053895099186040413",
+            "6895644571078189440948660962592",
+            "4090458704020642720758701259110",
+            "3437150126960216898595459196390",
+            "-9705511727116548664311748297324",
+            "-9305522884159558955104155253458",
+            "-1180186469868495251804854919531",
+            "-9796644010646070627213151591605",
+            "7106015483167238861336648112959",
+            "3331018800384614291320519803020"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "1148597479167394014443766490953345",
+            "2591895812533789822937308818442541",
+            "-326160310667933246866417794387851",
+            "-790306080625502821695639512319184",
+            "1545324828980338934424171461753128",
+            "793479953457908571858622935305894",
+            "2257543789427477727739601853041034",
+            "2444208137442914637553318787476074",
+            "1042890315365379193825287662234332",
+            "-1210287719815272207093258458478606"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-582733012088132875430838862529115",
+            "-660250931409288922636705489970214",
+            "2138527182118079262740549727787747",
+            "-2553900648599237329560900255287994",
+            "-818610871820914906670478944208743",
+            "-236741329036188976369182885941428",
+            "2127636973795770254193330141423807",
+            "-431302383365507717941243837426762",
+            "1669665166902924237503748195025488",
+            "2520650475436221136120486524455387"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "437759108051476501968471376399124534",
+            "-243331297098327608976864926703591085",
+            "-312583708013495348602181422788137336",
+            "22789177262884761633952302550572526",
+            "-120348891169104228873609073292813076",
+            "-659626114206394610798043021957841914",
+            "70595700062132590775654277374702719",
+            "3451720916899085830004332720776845",
+            "-34955038635481392379717309447778322",
+            "611134933947728200993481817988506418"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "20428164720806545350579904191767053",
+            "-99366399181425663956407652863667479",
+            "-35862534912239206067332175346609413",
+            "163496968286287738613339640023311242",
+            "-591381118335160981639939232133613024",
+            "254858944488772779912509622495446951",
+            "-460577318473676675239602831217222871",
+            "-616355768017271613012286794164991144",
+            "569247714773488133953930390229542704",
+            "202465807813752300281405751766207660"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "50315610630704356585322877730374837077",
+            "-83098067869442334415610073682313947471",
+            "161365526209509778448649648555742998425",
+            "-34910129007785397951451736271574830290",
+            "-144028959288053184684355803368266298023",
+            "-83817363552302335547213001246996745217",
+            "66067533735734112278087882548638909770",
+            "120913070546179844966164010066001238079",
+            "11384422330934700411529312190354569876",
+            "24772640548850140487291953136913828116"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-33028393081806170731403400067321655079",
+            "-11463174497738743284114180047596875486",
+            "32254998885085389327900118303255482169",
+            "-143483787323458716777165205934466305713",
+            "-112857257202882526765530368259175091713",
+            "162300394541802787040467843446805562970",
+            "129149449533069183017216694533759537096",
+            "-64890619893840122581192195171585201603",
+            "92544985698450754032974399211134668880",
+            "-54842712948210953012984037318158437034"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "132478357453720312304052372488893354300",
+            "-32685327131103251164646706295319990901",
+            "137916669492333509181609802124805628257",
+            "87478174466828645335209662043146117532",
+            "-61373497372439972480430645755553315203",
+            "5894673095536466472878429843436835629",
+            "-11619138682611129587257806902056620796",
+            "154151469754446633748882542018886132033",
+            "133733328652939407559328194516170213391",
+            "-126168609308185025841853092223452728320"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "29705",
+            "-26987",
+            "583",
+            "14792",
+            "9066",
+            "1503",
+            "25039"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-11868",
+            "5868",
+            "1573",
+            "22308",
+            "27942",
+            "13220",
+            "25806"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "6498297",
+            "316060",
+            "-6879700",
+            "6560750",
+            "-578892",
+            "-4785342",
+            "7293532"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-355167",
+            "6077145",
+            "456351",
+            "-6459738",
+            "2475530",
+            "-2801348",
+            "2836470"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-763391790",
+            "-1421479470",
+            "1842219939",
+            "-1396384047",
+            "1279169338",
+            "-974221132",
+            "786409383"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-701822471",
+            "-624495191",
+            "1950360306",
+            "-937735710",
+            "891136351",
+            "1299679601",
+            "-1631128813"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "375062411",
+            "101064405",
+            "1860934989",
+            "852100817",
+            "1489729732",
+            "-1386032321",
+            "1372454919"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "211103860560",
+            "165685254110",
+            "475179443594",
+            "83982532429",
+            "-110181124820",
+            "260080395391",
+            "396030998948"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-345799561802",
+            "385403135660",
+            "394672914473",
+            "-41576529948",
+            "-89134685382",
+            "-3611010438",
+            "-230138775672"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-44213782783569",
+            "-20824315185336",
+            "119392219688726",
+            "-48280571708474",
+            "93590107082594",
+            "-73857195319550",
+            "-11092149545600"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-3391364501302",
+            "-76804068374074",
+            "-5330058597257",
+            "-19337292494715",
+            "138572773907931",
+            "21930262754355",
+            "19745392677247"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "85902098584199",
+            "111873133532349",
+            "-68545677680071",
+            "34394580702899",
+            "-129707208239806",
+            "124130433864871",
+            "34089590991835"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "11547754029851805",
+            "13929638569741680",
+            "-35234779220736599",
+            "1556424996390195",
+            "-9941069445592524",
+            "-10351643092326233",
+            "-25407298114074662"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-25715001828139556",
+            "219757435776379",
+            "-15386649204339974",
+            "14219377191186674",
+            "7764182966113153",
+            "-17037164141869666",
+            "-2475550776888956"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-9013210997202018091",
+            "-1797035859465177385",
+            "-5039026628271053798",
+            "-2279961464871578084",
+            "-3332468041413232867",
+            "5196784857810113987",
+            "-816043157177424446"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "6230937120482135875",
+            "4406903786753583285",
+            "-5727776665901788833",
+            "6110238893930148669",
+            "-145374459534268305",
+            "-5408134830337680351",
+            "4578682454615398942"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "2280996778076363880015",
+            "-1105914554647913546919",
+            "-2340456352988668320804",
+            "-2286379749373826695506",
+            "886840160760479118799",
+            "-41094253890685508209",
+            "548649222178985137197"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "2101991590910127128698",
+            "-831048801347409813644",
+            "283006935997353787913",
+            "1014149330696637995585",
+            "806441932790678928581",
+            "1292353558196559947530",
+            "1691441804823003410715"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "514147725545916528450",
+            "-193880206083679947659",
+            "937498535568154555134",
+            "-112013115494854394797",
+            "1838267628545853841229",
+            "1697251710819151266567",
+            "1334759843181386854916"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "140211044419225416059523",
+            "22798773775711118695439",
+            "185167441622179764743229",
+            "-341069468014651486949028",
+            "-311767107447079981815378",
+            "-365574085382023414765745",
+            "16721445196171820250327"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "486153544869280373342335",
+            "-242241326842640368113620",
+            "-562468626770589338549903",
+            "353095362900612835555266",
+            "497938272554417006820815",
+            "90302429955937293267553",
+            "526449366929722769596410"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "46664866598514825254083995",
+            "14396224856981525909820752",
+            "-107662311974370582851318011",
+            "-116315009943902083038882452",
+            "56805605268567131709840635",
+            "-285698053798149060391610",
+            "-33274177545126848030882515"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "33427646005521871613027485",
+            "2068649850038034587145992",
+            "100657143587737579826250182",
+            "-101842456842860935986029777",
+            "-24678241147837041282536443",
+            "-137081933839393292380849579",
+            "80701010466319784595704954"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-125204491775364926553434405",
+            "-68666196727647600858615388",
+            "112354660219741185829905397",
+            "-7303433862000761726317213",
+            "-11446693224295095219494490",
+            "-2156245347683069299464550",
+            "154672444469077398685327566"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "23329815819835763885234375029",
+            "17840398455337288079751030137",
+            "-25733526444924201365089723197",
+            "11519925701526868606303914066",
+            "-8122188835588210950596509581",
+            "20364966273892581445915895222",
+            "33135945479485532338472222967"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "34850681321476402094647761148",
+            "-8083975349781433603028632490",
+            "-26785483364705248388507635494",
+            "9143502111139064417571274857",
+            "14109565008946975910759730825",
+            "23146095454100851136277162012",
+            "1964738793442941286156606155"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-830794414147442758629680858917",
+            "3974824330694591996054219287782",
+            "5212798423257109428476944724672",
+            "4136148464931397492769112444980",
+            "8716912509663310300081342235480",
+            "6130432387446754799113484089575",
+            "6936235167270048720910192819573"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-3535004387728242781406510654821",
+            "-2981058632797045086695893841072",
+            "-7090056840341574165394710895073",
+            "2937436279617800594018353925694",
+            "-6164350919245055090956637473538",
+            "-6513208139095775762754457986006",
+            "9324466091637799474852790812941"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "2028288650797899768325565085648",
+            "2900055817977855811722193499494",
+            "6905622275677150166335492710080",
+            "-3160034744597195167509247764978",
+            "-1770329430972443791351959666851",
+            "-6751176363700567085044496183977",
+            "391291090038019520158630200495"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1144504652616585466559731200361813",
+            "-1411714359592296247517000426578522",
+            "1466595389854363706316815721880003",
+            "742237093197555043514198686208810",
+            "632837196426513796812458196446553",
+            "1024964501312689171613773158523485",
+            "-847167208284549920541560580297353"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-1379322307328006592168092307072132",
+            "387617426942146023411122884259110",
+            "343500850419230433556982280364993",
+            "2429014391360178084347952076151804",
+            "-1271102713054563598822742297111938",
+            "1935501224421101911088894464720498",
+            "891196511387998706976306482945567"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "327017567565957860896642013563192207",
+            "-197957402373048855115227470112922554",
+            "-274157035197732503264012623953912376",
+            "-403813151150633640598242397849702978",
+            "-607653760517680240576727061740741951",
+            "-484513780480873333770895576601263141",
+            "-364508171932424706077901220225164024"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-655778660285369872582403556455076537",
+            "-305967296614644034918759561910261868",
+            "408652192029803274954960477706952597",
+            "491158410408462445362863360612553421",
+            "190090738690396233311384708778934496",
+            "-650782116647010450130445692263218670",
+            "466999684855637663153637496446512539"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-120922442637350268547173038619758155686",
+            "-14798309877377243156606971473100481720",
+            "-158319052033044572444106251098798964128",
+            "-87790259877385234658887405184771282386",
+            "93922935281981262808730125220213226026",
+            "152362390413913740334911190667453959896",
+            "-144716399672639547866469402888393372244"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-44740461288862326679317055888851447788",
+            "-19261460967897542219529042766754216418",
+            "32572085694274520626483716255202096142",
+            "61807114741875177639257174914455169635",
+            "142136086887672692821719902247408741632",
+            "-53665021237760509021660371077876689014",
+            "135482047221090055561713975617737312735"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "151443922013736797164062148265526547401",
+            "35894129120522936623313378082863551111",
+            "165517845554666361071462006124502354697",
+            "-161960863802263743123033369079756629461",
+            "-13661079351804691514638526655907412444",
+            "-88694839241604621649741179075868872513",
+            "58239606879205464966310459406863836884"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "21865",
+            "-9513",
+            "-7941",
+            "-4293",
+            "-1219",
+            "-10549",
+            "-2196",
+            "21139",
+            "21478",
+            "-6001"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "8460",
+            "-5982",
+            "-23734",
+            "-11998",
+            "17806",
+            "-21183",
+            "13348",
+            "-23465",
+            "-7244",
+            "-15957"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1268458",
+            "-118370",
+            "-1893479",
+            "6108358",
+            "1102635",
+            "701748",
+            "-4079505",
+            "3897123",
+            "2836441",
+            "-5694828"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "3354881",
+            "3381813",
+            "6090423",
+            "7476959",
+            "-6152439",
+            "-3812109",
+            "-4999299",
+            "708401",
+            "1857502",
+            "1616326"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "1829093763",
+            "-4407926",
+            "2068340761",
+            "234009195",
+            "98079137",
+            "759088995",
+            "1573936081",
+            "-1886084340",
+            "1660264264",
+            "-703900633"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "1724311878",
+            "-1770100972",
+            "2005560287",
+            "1002317505",
+            "564934293",
+            "-1056620575",
+            "-1507809740",
+            "1202563854",
+            "-1605647773",
+            "965485898"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "1993753383",
+            "-1867185966",
+            "-1797347612",
+            "1892035404",
+            "-1434558470",
+            "200775493",
+            "-1667955972",
+            "-1185419659",
+            "-1021155420",
+            "-1017852104"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-103795691472",
+            "50179936568",
+            "-305052039589",
+            "-310642809394",
+            "298713016856",
+            "430578254618",
+            "544157970272",
+            "161717434423",
+            "-82568595668",
+            "-69760173661"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-375428789801",
+            "229743413651",
+            "-385129101606",
+            "-341027447447",
+            "-292292749496",
+            "262095081246",
+            "399108175194",
+            "-436877011553",
+            "-184609205075",
+            "-525138717918"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "109490881395508",
+            "-2794346701792",
+            "-136768254893717",
+            "98238674344928",
+            "-71024502877527",
+            "36337204077446",
+            "25777777072393",
+            "-76891055436936",
+            "-20658339677220",
+            "107548266692467"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "65741531641303",
+            "-120604147256920",
+            "-71319706911283",
+            "12568908870250",
+            "20874672241429",
+            "-103855183735668",
+            "-53956000337623",
+            "67466545861242",
+            "-40824456528955",
+            "-50502795434656"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-30593444826663",
+            "-105265259920813",
+            "90066025218056",
+            "72114368716596",
+            "88265148520874",
+            "-26982520274013",
+            "-56277959562553",
+            "-46821369439330",
+            "-84272522150036",
+            "-67142476625898"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "22942929546037422",
+            "20172954267975860",
+            "9006547764775266",
+            "5211508666269417",
+            "30624269047382012",
+            "-12672445409868765",
+            "-32303326488011005",
+            "21742501359293496",
+            "18870493623228669",
+            "-15114725655588776"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "7991033936095510",
+            "-25436300164394417",
+            "-4116325708088739",
+            "-8818216017431285",
+            "-18277587083906160",
+            "14144228659002687",
+            "-4827602588444601",
+            "-6917969417697944",
+            "34390202143918890",
+            "26364020049473109"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-5565127045385194966",
+            "-7684916584653270186",
+            "-6526101704890152726",
+            "5758616101220042812",
+            "4277401219812095069",
+            "-1156200668055380488",
+            "-3665026910524676836",
+            "7227753205619980282",
+            "-9117970762813305675",
+            "3814443356076271226"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-4056124162849183791",
+            "-3091054243487850690",
+            "-1005186283000031535",
+            "4264379098205926012",
+            "-5816101399981242529",
+            "657803495947876179",
+            "-7620099954905920196",
+            "1165400408553884079",
+            "-4242137285679032126",
+            "4883600219746027602"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-2075689333753747735685",
+            "2356020644862916379319",
+            "2167344894833235655242",
+            "-1233676921087779374936",
+            "-472138285531110689106",
+            "-1007733287828797731354",
+            "-1150820475775440441805",
+            "1849909446162084231324",
+            "40485887422211501292",
+            "-1531296877121202039185"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "2248262662434579442174",
+            "-24067525590648914392",
+            "-46650276984820791770",
+            "-2184543362428055235823",
+            "1974794946687090550573",
+            "2197581555630725738513",
+            "-1036621057391423397578",
+            "1122920389059697318393",
+            "-947702755738865994434",
+            "1634754840733349282174"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-859921720417617681444",
+            "-2190420634375059510979",
+            "-1434691179630443364753",
+            "-518741106894841329345",
+            "1093338259353051655764",
+            "2211352717790797000825",
+            "907793405354409516774",
+            "220145636554287824727",
+            "509691887514400022545",
+            "-1294594023424323107387"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-526990487436843194807840",
+            "596575213433002438735038",
+            "-71236910029025414107990",
+            "99178925759657972243233",
+            "-440284224066490812643561",
+            "-428326164367815704740490",
+            "-429553654939143998439803",
+            "-301485703325840334321221",
+            "-205335077080231282419987",
+            "-586491652024392408421712"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-543688845315372053238708",
+            "-163909849043312495770038",
+            "-389689726424929488513817",
+            "-114895253663121558845496",
+            "348640253272437667385550",
+            "16575128907035554845887",
+            "597210944202677679801832",
+            "220285812487408184784084",
+            "-42608008827610055995030",
+            "-96942244128145461390288"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "143527171537167288568334963",
+            "63922968735872942354971347",
+            "16148467038136997458380707",
+            "-153693517794242110028457288",
+            "19318376645712741297662190",
+            "-12794255459929906566054319",
+            "-37650752907879433922942673",
+            "-150166615652066337683957386",
+            "57329802738501405002838890",
+            "84897963838366989876933017"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "96124972909010047517011683",
+            "140953422210565403195650941",
+            "5311198421189536461110037",
+            "121439717040191911388251790",
+            "41648789220821764736893059",
+            "-34464297128753801791226983",
+            "-130145493103864535084592015",
+            "85478399528142555833134891",
+            "14109865387017479696802124",
+            "-35883514830521563515868006"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "7929873152858210171340528",
+            "146946594934440045255005503",
+            "-63984879531222004851746641",
+            "-98404545568865647913143114",
+            "147368438153309689359575404",
+            "-125299622361117017666281396",
+            "50888897645607457371420920",
+            "35436333051448932231106760",
+            "127474433365288821803863819",
+            "75480346343825103775154749"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-8746748457647904867496756699",
+            "13886890262009261987702677945",
+            "30610269704200657332639932690",
+            "1988011012396012136828019533",
+            "27809766702732680733200157750",
+            "-13397700579544534835060124865",
+            "25003563641560620051585929625",
+            "-36124205442930485039742412898",
+            "-26618151134121364917297218959",
+            "37250266093672499484998022951"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-14142713495768494830041644219",
+            "-28826123233012884759325086616",
+            "2139184291361898827778357325",
+            "-20691297473114691371314913993",
+            "34039896653596475818834199231",
+            "-1467759043071863209472101520",
+            "18433718648399999914637264302",
+            "-24512577345718204022326385307",
+            "-17460565004252223611174273742",
+            "30220497605213358105564998501"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "10121526385221565224067878565679",
+            "2761189092075112373016075500931",
+            "8854254270554901165807339966360",
+            "-4844293746135774082587580385559",
+            "3690381197256143789522141159908",
+            "1243411595693866754538784391256",
+            "9189681722403043058688029968097",
+            "-9423820843284925151006522300117",
+            "6829156521436853695351870360315",
+            "8184630808925606776348432717524"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-1198084970702764472492358102739",
+            "3152710836296673619366574627547",
+            "-3203266598150150519017435963345",
+            "-3401665769768522116199310644875",
+            "6607270485165065820523048271021",
+            "-5180376940463540157931412688597",
+            "3550769633626957920535415329535",
+            "-3923968524034696093714727315085",
+            "-4025531993361587655896041896848",
+            "-188450492479152136768542630378"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-9632952156052895819744271379361",
+            "-4884426114142676161683078815655",
+            "1839921941370111674080951268455",
+            "-8242503028303467805329769377593",
+            "-8839943769048065292932997859326",
+            "-7067251768733441732414994112967",
+            "4523867687328920768404651884895",
+            "4871179227553544008972518619850",
+            "-6057694139116711199472594493097",
+            "4059555045350317377455403338609"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-473059709957241261159046271732783",
+            "1980262112011223122009659766419299",
+            "-12462833368369094491640116529143",
+            "-1189936951170501491703143185809120",
+            "1637235101820702334443602525486638",
+            "-2031029039226482674092805839096008",
+            "1673643328081327255616083846013952",
+            "1515455878435179342948918055102912",
+            "-249177468182679514366885827523443",
+            "-1462399863102676315035000336009503"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "404343261308094133306771010587258",
+            "-2509027130906219806348103780125817",
+            "-2235370619058054613214526829812775",
+            "-2180139572932375906797575417923006",
+            "1786990964606634697851218573276596",
+            "-222676010348749137040908415328994",
+            "2148962315756674060907600200655578",
+            "1131231890777254296826085678598694",
+            "-209000893739050027703488315683250",
+            "-2491331756189203304481882066855639"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-185542581219629399352774704587796685",
+            "-394902121829278205437359525805807857",
+            "590713072897259853019691937013503396",
+            "502758727281988975427078318543416657",
+            "-443747842243734666697335489578837039",
+            "5042221370262609425633645769887675",
+            "-591977482927777323690326997409238587",
+            "292422542569899682160719976080427761",
+            "605833396059257331402310165203375182",
+            "-9563525518418491930653260896379776"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "334166123380339702684858383425230913",
+            "-637729563446579910264265173398457924",
+            "472769213866685799294818860568678399",
+            "447146791822178801527521581709580511",
+            "96610773493845656033107109605172170",
+            "-142602384805506729332865726614124340",
+            "-310601730706741597007959967153005294",
+            "354280041691278605239105907920727029",
+            "320355952808759932183649996729527966",
+            "331701763416087509468808713667332601"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "49073461277677362785760660857736919947",
+            "-169416908345824675858705165355626189812",
+            "84167001824144890089789137366638401928",
+            "-23409106148842837235476212707694687403",
+            "-30271474341226497841818553413331361753",
+            "-106504152229661458377059178821397168599",
+            "47171294945627565796345213643071625567",
+            "22970191271966763243715811330493877198",
+            "14863926779301434690221505350909619916",
+            "110474698934476487574067856333615082472"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "125849800007107590516369492180395552578",
+            "143148433387379660832193328066338030792",
+            "-72853653543700073732079719513168773515",
+            "123778760286761947686806074556748610584",
+            "-105565654733279392109260625797964696434",
+            "83264246581235221617996610103577182490",
+            "-48415572079837050298521191611988793535",
+            "55401352808652663867123392021289274501",
+            "18767844067668706651480236415919760235",
+            "149963850691371825124263493422131099303"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "125053867195111854928850289325494913185",
+            "-141558821655686526589894984641859254355",
+            "-70235167406408960299018082636911869127",
+            "-93931785069493972666863586219173004358",
+            "-33652465762715830748074030542843712877",
+            "43131529119131579740991924094501938303",
+            "-56826490505262593726163351863722744454",
+            "26080328815831670206668453834325133999",
+            "-118301668341141860875674987759645097183",
+            "-137560743311784101467304719508764176954"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-13389",
+            "8685",
+            "-12385",
+            "8428",
+            "-23847",
+            "2877",
+            "27301"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-20601",
+            "22726",
+            "-19831",
+            "11763",
+            "-21024",
+            "-26976",
+            "-3495"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-7613329",
+            "-6191296",
+            "-6562342",
+            "-2775043",
+            "-4626420",
+            "2253566",
+            "-1356456"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1881110",
+            "-3472963",
+            "-1986841",
+            "8110500",
+            "6613282",
+            "5706723",
+            "4659122"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-845165868",
+            "-123936040",
+            "-118407476",
+            "-872134111",
+            "250077717",
+            "48927889",
+            "-1242740962"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1257436866",
+            "1664895293",
+            "-1592574645",
+            "1054532878",
+            "1586386182",
+            "-1423067232",
+            "-433225350"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1448117074",
+            "991933747",
+            "-2007741814",
+            "1096958894",
+            "909454330",
+            "1945230524",
+            "-1135940304"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "454731687426",
+            "-410445542500",
+            "-275815453925",
+            "232188471199",
+            "74176210838",
+            "172694874896",
+            "-432229161881"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-237742907009",
+            "291725362921",
+            "-375456133851",
+            "-172555537749",
+            "-77222985727",
+            "75467983132",
+            "355603655065"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-103229358570432",
+            "-52384313969783",
+            "125123784330760",
+            "49147811785670",
+            "-76188420587913",
+            "62077347858917",
+            "113236111966942"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "16839675737350",
+            "117528188223946",
+            "-83215066360284",
+            "-106240126895133",
+            "-140610775675863",
+            "69161046340219",
+            "-5367620164209"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-28041939750005",
+            "-9849374173718",
+            "-104280463802980",
+            "68340765792259",
+            "50866601864200",
+            "-124901901785729",
+            "94320105961599"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "12964636865091758",
+            "-30900961707076249",
+            "-34235023432781587",
+            "31452849974220059",
+            "-32116433493349645",
+            "15056454641844697",
+            "3220380516817924"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "11565990132652309",
+            "-1979115219549902",
+            "-9506105015296441",
+            "-29947064953359026",
+            "-35723741290819777",
+            "1692216018015076",
+            "16174544022195084"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-7310963412698136424",
+            "-165597943071933948",
+            "-7957288198886101326",
+            "-1416681132847790179",
+            "2279955654231160293",
+            "-9208861139945806119",
+            "-2871834669649394603"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-4691141329322093701",
+            "8948849172004914148",
+            "4989049806853973423",
+            "-5955920418702428305",
+            "-5005462481578120888",
+            "-4824298625293485550",
+            "-992020992384580885"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-27391602612782822280",
+            "2319008273932661576621",
+            "1379487111365773121977",
+            "694284593718405260502",
+            "1761355855168617262841",
+            "-1515770924964548454668",
+            "-989979187498153392043"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "198638501199435366825",
+            "1125019815452792395133",
+            "-1049896359176778070473",
+            "-1601152615237961695468",
+            "1050439216436645606582",
+            "1663568252582913300370",
+            "-2061203660241370183042"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "2113036292339854435848",
+            "-1939642742757517734568",
+            "-28192641056500022357",
+            "-739364128466490143441",
+            "-814580364884709918305",
+            "-1154955975185230844627",
+            "-1921376171526945915117"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "155234916517253223214669",
+            "54569816973154769361790",
+            "-396264903716832274146123",
+            "-233058506077293724737866",
+            "-259977111446647844862201",
+            "168457257727401353456526",
+            "-440383324891821228732709"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-317523940996880385737765",
+            "-93963745003535018073676",
+            "424089378343927508686049",
+            "-547783720395248736053345",
+            "-573590139412320114240415",
+            "-279824181416708109089356",
+            "151946399597755342049492"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "28344600382890942689933714",
+            "61364243140907583311935231",
+            "-154265345511422448987361885",
+            "119378359869607162352295161",
+            "94263708499987060050544722",
+            "-133072175861722803758097907",
+            "-11704454779169713276909351"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-98751232093166611752967870",
+            "-139312664474034269271128429",
+            "35063444117900395179797093",
+            "102516806810490076849188468",
+            "74192717816812033285854691",
+            "-146743149862829000033130770",
+            "-141832950190420181845392446"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "78201965083863028268161981",
+            "-9069855773037171038336750",
+            "132188050847166348693715478",
+            "45312541275344755216581315",
+            "-57072509286905908805327196",
+            "-17639505153037819959882217",
+            "78921165012102074114569802"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "1613854197834032136730063430",
+            "-8470408631766558968143580401",
+            "-26796998771160976867597896819",
+            "-13968703974822115617423495877",
+            "16391297684868529484428246628",
+            "31996253723693099758880092348",
+            "-28515449027235056800026707822"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "29003901243034261051999340651",
+            "-9217872751721264488488548966",
+            "-36037333564320399226575507178",
+            "13184620706272928711369564930",
+            "-3484312485760312610634417682",
+            "26298119669598487256561916979",
+            "-10337073506268829385902024212"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-8395953463471428775558712544937",
+            "3280829292043794132363911327400",
+            "-713339165879923703523124951611",
+            "6966592500081948880026911531684",
+            "9458860222968810665112995514695",
+            "-478657998044149303771109777456",
+            "-7822981040467488960380110589645"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-7687380421442439073984466158397",
+            "1255125729759094668996545862678",
+            "4087370506300493221899403471926",
+            "2239990096185047591766535855780",
+            "324855577134097258958507078312",
+            "-3747605668465760865974870493838",
+            "-7517467382536093238063723586773"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "2264136733622961048042169712287",
+            "-3680763311587562301144245715244",
+            "-6085544815496196698212373259054",
+            "-4725158409155890778538182765011",
+            "4471247291507677347209093541425",
+            "-7596184803597155735867437286639",
+            "-5336764833430717836720119834307"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "2366929765190575517610613077354934",
+            "-933660721467627547628108668944304",
+            "275402050086771070341080860167297",
+            "-364049334222814377243385829475539",
+            "1802721796043278974720830665216534",
+            "1876897254307941830406818341784439",
+            "-407308419988080444605787064480067"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "1653560379001616793860692123874411",
+            "150784992794639309979528542957154",
+            "1110971968475734226805305846564008",
+            "2480908389064352508977069518804243",
+            "-271941929813911201105163823221822",
+            "2522624759865770340700105525537365",
+            "-1998298580902951627476767136760592"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-303303768433761525871131397626734735",
+            "580204425169654418247447034004051762",
+            "501559035276733743240608210997496006",
+            "434014792885862484071273298300463825",
+            "103329211286006486949233240613271101",
+            "-505666350544432180911151793361598853",
+            "64165233750039998288773750264121396"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "424864295799616426484212056964871888",
+            "-152810042904774611567602887301612969",
+            "477355529458883774205927095018255750",
+            "-613775486154323397471667167311948258",
+            "-30862514129841545507008489633366896",
+            "262491469819294261426016375408380504",
+            "128848914605896127620809659768891190"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1176728107084355805100360783355792854",
+            "-7236188428562180415879913333530957262",
+            "47292783010039120381519447577145042690",
+            "-81565099348678451031823780150331852600",
+            "78688059613057943024694270282505199422",
+            "147423850372516204461538795771199943752",
+            "-82721829556506668110250246860473354784"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "151022194687816907306712480755439984350",
+            "147565573622060292887781241703078453904",
+            "-166782516227114338666555252025171372201",
+            "6746685360027376523977385526199820516",
+            "58540530404008066096527055300466378045",
+            "25319977592399722647228094404431005983",
+            "75030966136806031996061647057535591049"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "121830200727005461151499890045616240332",
+            "131014485689583600627833063453380752758",
+            "-30720940443795603452473820184705126436",
+            "-15668097962838507353292677213849146239",
+            "104461885360038312673700885512336746198",
+            "-119938884501349847589471389439448814622",
+            "55172679667339024457950426429614765191"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "23120",
+            "-32572",
+            "-22215",
+            "16976",
+            "6717",
+            "17948",
+            "-11665",
+            "-24766",
+            "3353",
+            "23320"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1104",
+            "-30742",
+            "17502",
+            "24977",
+            "31980",
+            "-21331",
+            "11269",
+            "19058",
+            "11761",
+            "-29057"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "8257019",
+            "-1751304",
+            "-7837619",
+            "3010154",
+            "-2333966",
+            "-204198",
+            "-7074295",
+            "-6464090",
+            "7365046",
+            "4895822"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "1458664",
+            "5699996",
+            "3620616",
+            "-6624045",
+            "2550814",
+            "-3036255",
+            "1587012",
+            "-3365811",
+            "790235",
+            "3027254"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "442486938",
+            "441457258",
+            "-74436116",
+            "-923568700",
+            "1944924352",
+            "-1941482279",
+            "-1125035682",
+            "-102135544",
+            "-1196111163",
+            "1765877471"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "1765031209",
+            "-1507660239",
+            "-1844477793",
+            "-672110186",
+            "1975877216",
+            "-1203240404",
+            "274438170",
+            "-1671550411",
+            "-554361568",
+            "-163722651"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-1263789103",
+            "489447307",
+            "-566090243",
+            "284044302",
+            "-1836581764",
+            "1548735264",
+            "-191701487",
+            "-173609011",
+            "-166293908",
+            "-1503453842"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-7765917028",
+            "-67711994811",
+            "457867171231",
+            "335464190692",
+            "-319749552548",
+            "-252965983097",
+            "537149590956",
+            "-106161385680",
+            "-489362544078",
+            "-324766745567"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-229499927632",
+            "65253348718",
+            "104144370442",
+            "-291236114406",
+            "397153316552",
+            "-2446986870",
+            "-123088300656",
+            "-203970057575",
+            "-542590736363",
+            "-190268824826"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "53278510251890",
+            "27395352245810",
+            "43165128890490",
+            "-125549371097904",
+            "45913496601859",
+            "-88777298033268",
+            "-44578278857847",
+            "-63182483121634",
+            "-615189926077",
+            "-76357287782650"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "125574339986876",
+            "14897658308787",
+            "72773174883515",
+            "42123525241235",
+            "-92023130876220",
+            "-137779513626577",
+            "128721948417075",
+            "-131284549575073",
+            "-72682195461575",
+            "-92062818915183"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-56667925886322",
+            "48625292283487",
+            "118021699876201",
+            "-129645086891796",
+            "105886580754602",
+            "-81412745473268",
+            "22079958372753",
+            "130176891204108",
+            "96533194170193",
+            "11884025784665"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-506532673811626",
+            "-35650957401620111",
+            "3797359512863061",
+            "12274676970376749",
+            "-7643350403625116",
+            "-22952047899787933",
+            "-31801747497539952",
+            "34825544608002068",
+            "-2163219849477806",
+            "-26717933056837916"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-18117789352537263",
+            "-9588508057343771",
+            "14147544466407231",
+            "5358112868530619",
+            "15194998186633362",
+            "-26368440181218293",
+            "-12926433949412576",
+            "-23715017719955956",
+            "2124625137004533",
+            "-28761382221979366"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-611587277931329314",
+            "4788391799910085501",
+            "2015606813848422210",
+            "-8162441603711661172",
+            "-3434406396031032576",
+            "4201771398235074913",
+            "-7185844575809467367",
+            "-6124286884318139459",
+            "9104770916622956359",
+            "776894361998019584"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "2442968139182218751",
+            "-6191133470861120460",
+            "39377695956604263",
+            "-1911371627459726404",
+            "-2032902584307265533",
+            "-3918625665352066431",
+            "-3011351064796167752",
+            "-4761219044304366605",
+            "-5873710765610430386",
+            "-4865925097841000964"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "254629702353421818300",
+            "-2201401895035914287579",
+            "162176927797961571359",
+            "-887977538520723081192",
+            "1559680077891008200419",
+            "-1522754607817991562870",
+            "-1961188309718634913132",
+            "1306926884755953118990",
+            "-2195000881232701833676",
+            "269742032865994089514"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "506826741901505999759",
+            "-2184273395087508003256",
+            "924141038875577992170",
+            "-206907928631225444602",
+            "1962143049953611087716",
+            "743436553398815951437",
+            "931022352993641629959",
+            "-1854695194903132671547",
+            "-1981929463828562067483",
+            "-276972122424764490843"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1493690505338340259685",
+            "1586809240658822057888",
+            "494350090577058363533",
+            "-441261947278991503330",
+            "227960094120329115559",
+            "1848792322278602742922",
+            "1557617272354574911225",
+            "1731256272922066434300",
+            "2146230871407347076666",
+            "-918839131781624132904"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "22548387681410857803984",
+            "-486740942160285075283728",
+            "-18757726112678919772843",
+            "290325417808337495708055",
+            "-493613286382529175399469",
+            "-543640826835836745923553",
+            "305872812141626487961328",
+            "182680085006567454977369",
+            "-381084632278529532946167",
+            "-438390880171661736974263"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "456312474649470033816393",
+            "-115833852286009708732615",
+            "68058901001059387892663",
+            "449536434823641408121541",
+            "550535224734224663866259",
+            "600242006088744661574795",
+            "-474373221742191222036625",
+            "-240661321200669878886195",
+            "510976975891324563556258",
+            "56581084876567011909938"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-55030941453838611172063173",
+            "154562453921435135381589722",
+            "43654862091121183246335073",
+            "-37043561098272538846899535",
+            "-84012525778134679892504063",
+            "90521181578289106753547859",
+            "-121101532794400375638986820",
+            "-48962126032349923649363889",
+            "-49416297762321316497159289",
+            "91097613555149997187554026"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "740754556792408749652437",
+            "-48364577771068363588552848",
+            "22689046979811309216159063",
+            "-49805825990806434744728742",
+            "-129575325280951011614339770",
+            "33686330327503040241226442",
+            "132098550234954179034221650",
+            "-19568317425318962814852019",
+            "-85488236163679977350384849",
+            "-32900134218496126753349349"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "129277452246091321049329712",
+            "70220503267517710030799158",
+            "27372362666191002140346309",
+            "150017068938796641247123814",
+            "-83181229560758393426889380",
+            "78268145282577350329747809",
+            "-92583446733322174938991138",
+            "83334736171733181604682678",
+            "5792916388018467792831905",
+            "-79110296716121790367484224"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-6762857221159250494826450291",
+            "-10332119938961796101160722490",
+            "-6827819665107707846347452358",
+            "6329384513346831358274334163",
+            "-35228914666708609766335933234",
+            "-18048951629266414989346849863",
+            "-27226935845799367007517234093",
+            "-33993966153233641238453751768",
+            "-5449254845364088654662599852",
+            "7541505715737170338078113293"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-27624667007791025693852828942",
+            "29791087942990315789537070065",
+            "-31210197914933328681784492817",
+            "-20492515931057444475484015790",
+            "-34897600909921210666914133162",
+            "21001866866717994908958026000",
+            "34533628665541062590101591605",
+            "9988772685214552755442833227",
+            "22086548704119049555142576363",
+            "-16156072487413105847081955974"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "2364616944071636050694451730445",
+            "-4605392974112860339491707249622",
+            "7762016214564113231716204977147",
+            "-5804616094053219373113235353220",
+            "-7149030242509106300895187420705",
+            "-8733720516856953446393989573010",
+            "3766997683445225481997098610374",
+            "-4681729003393749052094837537164",
+            "-3858591495312700256431295473053",
+            "-4277910894337725265278367498545"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-8345846769042618988327927260668",
+            "3034158729600895891904384787679",
+            "-121178995572767479680968825369",
+            "2753730356624797067210787714567",
+            "8954963579041823082665982027392",
+            "1270742333952628369439000703669",
+            "-3722978202569846960374550742749",
+            "8821702309083726610115002889572",
+            "-4646910584068123191099120325769",
+            "-8235420195270182384004027445084"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-4599292704126802281290827035912",
+            "-314623101152193205334021409871",
+            "-629541047462259285195633937282",
+            "3573126210904165261265049508014",
+            "-1688995622757664479276042953337",
+            "-954432643427450256096079589970",
+            "4799594864439399541828732234249",
+            "7068551937371368971824412562795",
+            "2938601954614403464560201414377",
+            "2583981951692907177619688538873"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "1801390123686604711395138083864347",
+            "2408175799973869350806431040308722",
+            "-1778542198055167111435408713744051",
+            "2503317898880974086644685985961064",
+            "-1868573144190196894242583180400118",
+            "2413853351573616737379149363424791",
+            "1791133303754307763813804844006693",
+            "1880584363502969839626017340127067",
+            "2319612620308107999768959466515839",
+            "-1312849382673356132056970324156720"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-2310170941545151210165819902499827",
+            "-737480739959391674903038969492449",
+            "-2309694820066177770799513479215373",
+            "1423472051904245134846442992358971",
+            "260457318140005904743337186826997",
+            "-1594690225193818094407079633825817",
+            "-2186698024851903659624127278863647",
+            "1502160181086743651783854079119431",
+            "-249207087322062171035117292968436",
+            "131669942196137162081550281921265"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-221840655914556742984533980869327276",
+            "541288680814766363243902199787661875",
+            "259521991726914299693855180749498836",
+            "-642098028427571595255754897557274431",
+            "102852117125804159452269997724256795",
+            "-619484029498534054293488403236201260",
+            "-345175355591714711803667121166327599",
+            "-301314861888172625598139699349559259",
+            "-609730883416842409062992565799074655",
+            "230365679128699815563056808949145224"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "148237310969220211917376683153947791",
+            "-210879956694236410559514093773799189",
+            "68825095267034031802766092342803122",
+            "21069100059826162651376609239596285",
+            "595526602428177494983219729529252005",
+            "95671763294916433447636192505612543",
+            "-121882858245568611085988770451861605",
+            "206325916742129711002135164503598699",
+            "57651974744686292971156031485809986",
+            "-320734725725255645082933245576075729"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-138584416420268103904526275311550821586",
+            "5083386037235063597175109391737322324",
+            "26128220661314654796494761140594032479",
+            "12192498025672996200511955739264829627",
+            "85947824515275949989859778737001518967",
+            "132920098023348124785802079171904221898",
+            "111246451182495893420608179633240106537",
+            "-4547602311363102763260302847649859534",
+            "-43987674707274853293509031074465543646",
+            "-7024578484047164009719166261798276072"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-98438730789672041852593692005666781163",
+            "-50079767906520589446732692469907844466",
+            "-161434501044028603998001795580533038376",
+            "137619979466523372085843169543259266571",
+            "30806070627846697101821976362585151873",
+            "-20974889368336688743196860981435677443",
+            "-118421326318547135923082579581184125588",
+            "-36658255136000073539449905192106631248",
+            "41010547609721177330707111502953515617",
+            "116618718482647675602042707539708842430"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "56783177633153952864226073173557138209",
+            "-106939684114816004877527397111208670733",
+            "-53010195457974371105397833315577691496",
+            "-71944649577959370180890395897943740754",
+            "97436272237031860506529444641492486685",
+            "75452790723745404169344305366343804695",
+            "112051243374889808475698084777418001497",
+            "28127293814479165375232961041795735138",
+            "-79340368891394516117475712925796793216",
+            "161366149054331422136847320127220486305"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "30263",
+            "-23222",
+            "-24262",
+            "30483",
+            "-5838",
+            "-11101",
+            "-7596"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-28613",
+            "17283",
+            "12286",
+            "-15277",
+            "-25348",
+            "5314",
+            "14687"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "1641157",
+            "7771466",
+            "-5863310",
+            "-7519258",
+            "892965",
+            "-936146",
+            "-8184473"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "1892203",
+            "-6603897",
+            "8118473",
+            "4954349",
+            "5950071",
+            "-4001964",
+            "-563295"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "2105362262",
+            "-569111656",
+            "1919017986",
+            "1876055546",
+            "-1146939193",
+            "620406159",
+            "716539407"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "2083506192",
+            "-1628538025",
+            "-479438046",
+            "-1422963534",
+            "736679219",
+            "-1232279592",
+            "-724157284"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "1286129607",
+            "745445705",
+            "1008263313",
+            "50113482",
+            "-1025981626",
+            "-96567663",
+            "-1070688111"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "418265202101",
+            "-131132103981",
+            "476092459444",
+            "-88185972192",
+            "208316876344",
+            "287012809896",
+            "-90799129473"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-364027495907",
+            "-518301781733",
+            "-87293329789",
+            "131975972315",
+            "532952706734",
+            "65088425600",
+            "-389552284897"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "31618280405118",
+            "-136363944920194",
+            "-24219400967405",
+            "41351280642367",
+            "-133525694845951",
+            "33222176685107",
+            "-48907606129826"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-74180834940463",
+            "-120438687871068",
+            "95491516668458",
+            "111229832917865",
+            "33731924194747",
+            "-126511275125177",
+            "-139437230946699"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-81052755845134",
+            "-75947916229442",
+            "105789356551279",
+            "-77360764385979",
+            "125756275622911",
+            "92621084693706",
+            "-8993873340097"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-218063399041728",
+            "15987427800048505",
+            "5452953879269108",
+            "-2907032193248868",
+            "762605768430446",
+            "-1095257289404828",
+            "12083366342031533"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "10657562176134064",
+            "13537179043764834",
+            "4147191201684311",
+            "-15913012985064112",
+            "18305247661337724",
+            "25826108878037669",
+            "27185142855300649"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "7185681950550884166",
+            "1151981239380928488",
+            "8244169834678820702",
+            "-304666117465982921",
+            "981971288330668381",
+            "5795574764637974961",
+            "-5602631342767621026"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "8858282230609918710",
+            "-5845597164953171622",
+            "3853395376773770978",
+            "-5420964350807318652",
+            "-556878098560662410",
+            "5157835722577117011",
+            "-3095549053985906766"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-980234476785800984103",
+            "1785410146071585960280",
+            "1741125755543744357745",
+            "-2307966711214941804481",
+            "-373078769867949398924",
+            "-1086641478410581865639",
+            "690547933962974045594"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-885097049671576555174",
+            "-1893945141563298628450",
+            "-1652678503714833255734",
+            "-1522111312512895545246",
+            "-1668888046878977777801",
+            "1230021831666666579759",
+            "1588026688627154194388"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-917162439303146482401",
+            "1294154478309086604917",
+            "-1353063508538165826259",
+            "1022085074719266892796",
+            "-1562172028448696446044",
+            "1827858998664290349067",
+            "-1884237502299830467832"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-484334248622047005148813",
+            "148191614369576787178350",
+            "-41916037311516330068416",
+            "564042302843825437811083",
+            "-223411708998089161582419",
+            "-493169217694889240887888",
+            "168822761929986834961244"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-76559387748666164545163",
+            "-9254395158022788547985",
+            "-533412485676057805450960",
+            "149982043973998367780013",
+            "331053556044991173393445",
+            "514169254104754203169529",
+            "-24148845097144387020494"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-110794762039637748701547811",
+            "-112781226990835664252750191",
+            "-126691789751592065497849314",
+            "144891347683530500307750752",
+            "-95002620477467921055244746",
+            "40308052656613620648154342",
+            "134142133925977560550824382"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "46547963198244968911697774",
+            "97558079483281440143910654",
+            "42302799957358224994647507",
+            "-23653725784983135812047124",
+            "-73629112624555133955638884",
+            "147336720269797653946599523",
+            "49289602849638790286981280"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-70628982960116517292670785",
+            "89897841125429743744022708",
+            "40246726976710463297014640",
+            "-18507266936458323746034023",
+            "-127317537089789264088083641",
+            "147526034684993650522315979",
+            "104950772823463920929967725"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-7959063299212980364945888769",
+            "-12214855654426558559365015380",
+            "25951542396314432482289253877",
+            "-1020221313976375715276389621",
+            "-23858719187545844530085394517",
+            "26756899139375533317181309141",
+            "4149356920572734945386012056"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "30516050835182806794120674336",
+            "-24853046938355580136642360705",
+            "17673256930214912186178705872",
+            "3794080846648515873847989399",
+            "-27519999940588367143793580557",
+            "-24055845379132928502612582185",
+            "1667643406878176303891605843"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "7481053395824836436290423884846",
+            "-593931320626780956465988943779",
+            "-6000659640663712751730791207087",
+            "-90100969836256549622480172858",
+            "8857339616569823715364996577725",
+            "-4550234737408799149611645636613",
+            "-5298498692956294201665781009355"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "2183123309024430871493750993081",
+            "-7601532421099550450707534366673",
+            "-7300980343536532697821828736934",
+            "1354900935970415000060545678696",
+            "8198340038238087506252124691672",
+            "-865479512261563494978410780512",
+            "-5046161384947842589369454777111"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "7048037667368946471394838100018",
+            "-6071852502042552382040623004150",
+            "-29904067906303750173605027886",
+            "671351089519547849465678121449",
+            "3143301561492705185485961790751",
+            "-6145271153767518350310952751439",
+            "2739965492773135139400075441125"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-2567371035309291572933015418896592",
+            "-1613418673846353226340454921565057",
+            "-2365064503966708848862797241011316",
+            "-1542776028214362902586142929151475",
+            "-2077125689533596049348241259664237",
+            "-2395709824654377963034101663677500",
+            "2440524930359057180265130201332264"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "126271990461855730642053909769616",
+            "-2141603070621807963066899994069276",
+            "-2176132802609488796741574791492255",
+            "2332352886535253698579202178516320",
+            "-1988660127485179422598316866522748",
+            "617466151275332301720527118576755",
+            "2496619525601425389231233102949834"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "231635227467662539191613532700092322",
+            "357886376162746208586158220391320643",
+            "372166497885582630320382738692891137",
+            "-387074150560768584146102878863840104",
+            "-397494095284070251215509382375703502",
+            "540335898477711720687165346731799594",
+            "154690828219053775405144687447110697"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "656778981191586847022137324312215960",
+            "378543348336505847394761525402250912",
+            "-170070094972872823236801280112198457",
+            "-10242741317397583042355113808738618",
+            "89873395803487425288992885513379780",
+            "304757291680949813380778307009872780",
+            "-139647342743838007971248857009724115"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-131203220135616441025612961636698464749",
+            "-136533237651213770170219781290216699170",
+            "-115015389154631152533856730671459648641",
+            "-94487920510079112453883494033826826524",
+            "56128697101429071079534166874945886309",
+            "62210939351804546691896889001498845763",
+            "116499323952286447489475520935718675981"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "111388442425565536328400022570545560941",
+            "-128274800782884675622442061938835593669",
+            "-21259026684892744639073607446054060342",
+            "-112426553286465994874443695017393042924",
+            "-6478832771211729175837331925924806054",
+            "88007792352876262937416934703934442548",
+            "-124891073986219943979478446893457381063"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-53247472166305185804034744604113912186",
+            "63873846134112392155257205697721451801",
+            "-44185307140014507458077369663628065550",
+            "82459261534617980418389187926742556129",
+            "-91555320299720623336768030419645817723",
+            "-11478808234125256915529445366937625010",
+            "-284101804019488359263241192605820668"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-21109",
+            "-3732",
+            "-7946",
+            "-2257",
+            "-3925",
+            "-30469",
+            "19503",
+            "8362",
+            "12991",
+            "7380"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "7887",
+            "17931",
+            "-27968",
+            "-3196",
+            "-31776",
+            "31067",
+            "32277",
+            "-8764",
+            "-31456",
+            "-29467"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-8341014",
+            "-1946977",
+            "6040551",
+            "-2571346",
+            "-6392818",
+            "2115232",
+            "-2447337",
+            "4046082",
+            "-2062834",
+            "2887198"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-184184",
+            "1975807",
+            "-2304689",
+            "-5917706",
+            "-7687940",
+            "1428040",
+            "1997257",
+            "-4160391",
+            "942644",
+            "-5375074"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "461428707",
+            "-1396927537",
+            "-778825702",
+            "-1909514860",
+            "-890594087",
+            "-360960780",
+            "1809000906",
+            "635847266",
+            "310435728",
+            "-904068800"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "1160190062",
+            "-145728656",
+            "1783609630",
+            "-146939811",
+            "-842989885",
+            "-1768700091",
+            "988218247",
+            "148328980",
+            "-2071975098",
+            "670872405"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-824363066",
+            "-368967389",
+            "1685375538",
+            "1129837916",
+            "22659316",
+            "-1334068320",
+            "442107323",
+            "-1555351812",
+            "1240907608",
+            "32207466"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-467348909083",
+            "161337736582",
+            "534040697288",
+            "147508961254",
+            "-55643966274",
+            "338027786439",
+            "-116692579226",
+            "-269893444320",
+            "196082834886",
+            "251179928243"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-470003994676",
+            "322677755949",
+            "240704268230",
+            "503601093424",
+            "-298384610687",
+            "400606205208",
+            "-418170073845",
+            "-133930730273",
+            "134075361471",
+            "481801156891"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "3339778666379",
+            "-52347274197695",
+            "69720482084171",
+            "98924128655576",
+            "12928263807684",
+            "-6851555304470",
+            "51780495955443",
+            "83679464532035",
+            "-127505675951073",
+            "138274778683006"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "125929064682987",
+            "113730203099244",
+            "-100066179950776",
+            "-85878295227479",
+            "19299300138707",
+            "-108539192136351",
+            "99138900176365",
+            "69954325267229",
+            "-74544791461254",
+            "18922888907199"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "45485569650205",
+            "-128562887157102",
+            "-97873632139797",
+            "30278398120164",
+            "50653763282967",
+            "51088353337222",
+            "70065775683504",
+            "-138552521631772",
+            "-77979604107570",
+            "48798953674588"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "9248286127591027",
+            "6920015266380308",
+            "-31298699416650301",
+            "-32620123808864302",
+            "-8198521317945035",
+            "-31784959087849283",
+            "-13290835987057648",
+            "31560002024245360",
+            "15568532723510774",
+            "-1887653675336702"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "6474236439552209",
+            "-15895902060435063",
+            "-28567392174760254",
+            "28377048461337220",
+            "-4788924744301655",
+            "-35902541227577686",
+            "28213667178040981",
+            "27374686655345350",
+            "-9347347706094784",
+            "-22558945363830613"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "4315782565441521668",
+            "8173082059740912464",
+            "-4674157300377953730",
+            "439018152229220757",
+            "7259585234958472340",
+            "3209551603044381642",
+            "-2593722905832184497",
+            "2027467253721370766",
+            "4711777015859247337",
+            "-1809157338227183996"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-694544658659246762",
+            "-4156045606313107281",
+            "2281142887622773123",
+            "-624104445088573071",
+            "-3987668248832946150",
+            "-1357628067592529656",
+            "-7104181881423591417",
+            "2383273081562892502",
+            "4853234144609282656",
+            "6571580420114899404"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "1101678549357629021477",
+            "2009977653178126846280",
+            "2037345716315195820691",
+            "-2096610127883926523376",
+            "2194680562745348805943",
+            "-2143848724610431766410",
+            "700093503999313554640",
+            "6460670107803288774",
+            "1291134572995199955560",
+            "-717095483870334127208"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1449785013926244904068",
+            "1586111973600917779568",
+            "1975689855672951561335",
+            "1946100167589360666679",
+            "265628521821771152170",
+            "-948749623912750799141",
+            "-1084337593623251031397",
+            "776286080436824555132",
+            "-494554638041660841031",
+            "-401685002986893137940"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-178940137819013596278",
+            "-370142275253865396698",
+            "-156303150104810810130",
+            "-822078323048785455176",
+            "535756974328325498532",
+            "-890371668586461515474",
+            "279474091115067270095",
+            "2075769840894911144578",
+            "2271112074461256742768",
+            "1263298342536869880178"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "107390994480706624333240",
+            "-272629086063015754481125",
+            "-125381849980531312111076",
+            "323365287095819927005598",
+            "166918591791270045548611",
+            "541466210731753290375920",
+            "412426989884687859377573",
+            "-373858895152711407562897",
+            "349597921666518744534201",
+            "-477064957800482399571210"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-350456949692319622870317",
+            "105130553074956289619769",
+            "-454855427531984650401861",
+            "389828363885037529338500",
+            "-45149503960310017108207",
+            "-364809035488363237149074",
+            "-212095481090572510124715",
+            "208001365502923021156304",
+            "-132773639805291311517839",
+            "524948545150003066797425"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "35259018232949564354576762",
+            "36343724900171873095814458",
+            "-47507355485359376375717880",
+            "-57533411745939815466380662",
+            "-60092847738452003317331023",
+            "3800735336914918445567351",
+            "137812953107886858371793906",
+            "34225766901520865002566632",
+            "-132405725520797481081665087",
+            "-153398838732410672130385737"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "128319155253991829531389637",
+            "43165460069726070921482804",
+            "85467445526390304512322840",
+            "-146762181884606435611421185",
+            "-126844884930818972344264748",
+            "34361835173349993323133217",
+            "-68547202891841302450583744",
+            "-40167006696488837265069472",
+            "-74901074229595686899584654",
+            "-68749948940724462847250795"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-38290033550964318063135667",
+            "-20967313541687634038075040",
+            "-107264911137060066098734846",
+            "89584407711339849997950109",
+            "109335186095373309975629513",
+            "-14494027827192960963596349",
+            "-67509403216005821639193263",
+            "109578027592798760144924982",
+            "116834891439714631179635997",
+            "109674768496669329805900472"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-13706828639541826056708910505",
+            "-23779415475695806226285335701",
+            "-20403054366535638844892685460",
+            "17926682575342545857820109930",
+            "-29969666635246903811911577159",
+            "4322913582730437845509857109",
+            "-5090037359725837740805001421",
+            "18248885943719255743411625340",
+            "-12657516075527000586763065411",
+            "33671850711466671188705676643"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "21441226402320291786796151469",
+            "-489973429642209279383578568",
+            "-36568669593686005393185481762",
+            "13796971801458472172520434088",
+            "-7994017766016214184641016553",
+            "1561490110301188737024730447",
+            "38074453848227878221879512526",
+            "-6611262569813139733823741726",
+            "23022812151767616798249226140",
+            "26695316798647697932393341179"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-6126422073163258556989145139770",
+            "7257045667636323814171654348911",
+            "-5274091940900155174972411849180",
+            "-6812696260414552303993540333266",
+            "-8500309156232696643300971495939",
+            "5321769864659588392049817499314",
+            "-5039830093949469211483714156812",
+            "-6633027512060316645790696983007",
+            "4340429228631480058324378799636",
+            "-1630878746182595626723753451608"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-7314945488320426565820671586637",
+            "-7406514689693791623767541892888",
+            "-4830710931884289793431794199249",
+            "-3758167604048365457370249362898",
+            "5317862646659352563988454094829",
+            "-748850916109857148485910907057",
+            "-2993387002194928323024015231901",
+            "4211954164724882453586630364110",
+            "-7408921561114481661493286530975",
+            "10033288108411665452743017188443"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "8053572570098281818356964596376",
+            "1908605622493118035491961007084",
+            "-5937699460286270647647349131356",
+            "-1407164418808891695839274066972",
+            "1026031202093754747593428440664",
+            "9128075080231853627740826681361",
+            "1150734293920781967788066491325",
+            "3575593240656154263696537723347",
+            "-9578952592909740118052990539663",
+            "9788570458322925906731674034571"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "2546167711754510531653417471247239",
+            "1864915514804198253374517101253374",
+            "1572251731593167469136110804788032",
+            "-1816600003045057303909283771795164",
+            "1520652080462583011477916567108957",
+            "-1734147169849373491334306258765538",
+            "-2447614261820877423091669867951531",
+            "-1826396544811346698392237371607395",
+            "1502568816413254345265156473409654",
+            "-2014610313148128366910629261789779"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-1855163614050249786123016316161336",
+            "893899277550555639055767796543205",
+            "686227439600851989589634986801690",
+            "-1484196902391738874780681638450259",
+            "1181153298481188649464625411550110",
+            "-257877919422309608044093539127999",
+            "2468479739575998772676829454099709",
+            "-1539963498806165921795637993389302",
+            "-2243382068201233838876165984414063",
+            "491038991263905025700206542020861"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-597285041655777508836370117424546286",
+            "-212915674112372713563243155334464735",
+            "-169733400631355230665246923231564106",
+            "-257695180843856835976546761814974502",
+            "-159199708769938756688005712105595720",
+            "155463228870223105562821637784150991",
+            "-421604544463741431747680657332448481",
+            "192822447775661213463129358744172317",
+            "-286626526760237236497088410330567742",
+            "-601517958441081087199398706025786382"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-82547444875501132761785352487359116",
+            "-246593569791118450786584777558750625",
+            "-91844724030484160072926648255143672",
+            "-69252035825753540780599407037419460",
+            "644781206633544756310104714559447256",
+            "-262845596528177736487020213988699925",
+            "-375729034479299520248775176155207770",
+            "-479941467647966077592734215031877163",
+            "286117330727391846690235616502481186",
+            "-7294278388158531393325689448264285"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-169130959752592548102532341751116281504",
+            "82587899769437528472252758495774486404",
+            "-77723031252037678358942277533026624307",
+            "-39787175488615223575962629928173399959",
+            "-103079418527774040874191095440451368115",
+            "143404785392510227499220384378768451023",
+            "98751717881147885089187606548239643742",
+            "-80471675701993721480829704293444582397",
+            "-13432971458738065305485925445592722750",
+            "-68143198119846648770859620166719802"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "82277432281188444270047990740605736453",
+            "-140504957823814927502089640850816061182",
+            "48534660051854785030554289649407895158",
+            "6422771152783983559395644304289119920",
+            "162012152615078454305645544989583091731",
+            "37269704291699978324341128764605257057",
+            "-58248048235361087656770889990621980447",
+            "134745944265183112641665968501631304489",
+            "-9136434632101510662083878921980605716",
+            "-11817844733230924939043201144789838478"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-17709917998817602362457338930665430374",
+            "100705413630779255802749096858430372336",
+            "-65652752408617833644830081929182653005",
+            "78721980566008540939950320927524467430",
+            "-157085052161045001481271504295196155720",
+            "21700226768392088200691166712143650173",
+            "89253038833049482190072700821475812886",
+            "-42386752776532873392991410023518397456",
+            "34328803512588938135564393642000374714",
+            "126797458053317029746311576365125139325"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-3880",
+            "28077",
+            "4580",
+            "16362",
+            "-18975",
+            "-13634",
+            "29579"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "22466",
+            "-19394",
+            "4211",
+            "-30616",
+            "-20114",
+            "24229",
+            "-5149"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-896877",
+            "-3561706",
+            "459204",
+            "-7986999",
+            "-3891501",
+            "591660",
+            "-123828"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "7455787",
+            "-2954938",
+            "8061044",
+            "-7704103",
+            "-4283355",
+            "-2688704",
+            "-3286813"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-1733165912",
+            "1441221142",
+            "-711447900",
+            "-1370755477",
+            "-566470981",
+            "-1689066002",
+            "-1518943706"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1167592421",
+            "-791885241",
+            "-1679872731",
+            "-41746999",
+            "1369029801",
+            "-2099305061",
+            "-380693461"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "1876525998",
+            "99057337",
+            "-1107715390",
+            "-1940677144",
+            "-240324753",
+            "55473962",
+            "1797328299"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "295503234342",
+            "231420286475",
+            "-7245442623",
+            "-506030630708",
+            "112634015563",
+            "18061760994",
+            "-112548836018"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-365159103775",
+            "12635282946",
+            "-514650575463",
+            "-389272826188",
+            "-475262038569",
+            "8128202421",
+            "349784367667"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "105099742726339",
+            "89319467966898",
+            "-45683396076389",
+            "-21990516959323",
+            "58719877204527",
+            "24673773662587",
+            "109803579959540"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "117460253140304",
+            "53310537831787",
+            "-50260601769156",
+            "-86329701674689",
+            "4117449846060",
+            "-101524170218096",
+            "-39178939801212"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "116693494366735",
+            "-98720295094103",
+            "-3387455956043",
+            "33422455841875",
+            "72646014377284",
+            "-54216173525379",
+            "-107085335515558"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "33982846762129193",
+            "18070113177751228",
+            "-15344533360160850",
+            "22967009312426432",
+            "-27693105018949244",
+            "-19447372654909311",
+            "5526992817880686"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "22051072322444431",
+            "25739994011608686",
+            "17590147555692963",
+            "-22084188037496810",
+            "-10818514363912056",
+            "3629814651214680",
+            "-13864921149293605"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "1477505808877528725",
+            "-7970299900821284128",
+            "797529455893208185",
+            "-4380462832684600",
+            "-7512565283888184630",
+            "5963752963276377801",
+            "5383033152533304312"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-4687001791650362047",
+            "6189713819111473364",
+            "6157195100346646067",
+            "-7354274856118962216",
+            "-2096503882079843374",
+            "-2989958959527548003",
+            "-6990431314703212402"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "1543053224502515824700",
+            "572944953559834661337",
+            "-463017450276577476402",
+            "-814539245715762117156",
+            "2125610462348570304645",
+            "-1998974164978877476896",
+            "218777334538713492026"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1737306797700142536355",
+            "1531804891962626834702",
+            "-1195046210647179987202",
+            "2315927533477943693088",
+            "1955617169829065459239",
+            "-622025738336427455680",
+            "1268127968239165761772"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1605308004306756881486",
+            "2011474158613556880412",
+            "-399796168662135031326",
+            "202881347434850907252",
+            "2019703887423617334184",
+            "716418631018448992152",
+            "1206220294128077626983"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-439233385375980423579188",
+            "-282388747616905814633943",
+            "89357877650443882966242",
+            "-141495542322748990965271",
+            "394310253761853869007932",
+            "408555292651658924207417",
+            "31352288232225179050347"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "336925051011166695076987",
+            "-104340070149410931907855",
+            "-577270465951816735176538",
+            "-257830029049203338174931",
+            "-208291608961639617152387",
+            "581220927173979115429157",
+            "-486086206006317637743805"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-29078699886191649638085668",
+            "-58403200017753206157268711",
+            "-30728609198387149996274670",
+            "-64955510531916364593021196",
+            "34821390565272973390205800",
+            "-93422825258783366802978133",
+            "-32247179645343680253023813"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-77365678100681876623290641",
+            "-86628826177603420327288998",
+            "9220094682835113941854095",
+            "93882093552015539929945204",
+            "-150721224176965634213377599",
+            "43753026435426938659269571",
+            "66570450267194288255165042"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "26300179792340119797864079",
+            "142433122880117613936146124",
+            "-15440885853316070864672346",
+            "-77502061475155985937091480",
+            "-112852326686297479782561254",
+            "72434505811059791598110246",
+            "-7668392160648564895359533"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "29239766717040730171831641774",
+            "-21707593058421917306282917041",
+            "34516607235563142943063617602",
+            "1193640916706101190985687069",
+            "35195812678373567674988713830",
+            "-4604316293532843686268059747",
+            "11180352632046834701829653097"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1512268160667408885922336144",
+            "10844547914295271624025530713",
+            "-36215819512055547591428396750",
+            "-2810331321951135608956464976",
+            "-4386752948121646037249312193",
+            "-17767553783444186492160681129",
+            "-3024939524547086439531257635"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-20864326575772010142787316454",
+            "7488713151399696482410740417311",
+            "6381276422733482520358034285380",
+            "8884761350831459122090428512810",
+            "9969195609900524295444588216806",
+            "5543397900882544648824130766348",
+            "2837938767703679300265023672696"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1882854523229835368724322985294",
+            "2966116522928193897186452665796",
+            "6823275008349281974665560465394",
+            "-8062354025481937215315488548524",
+            "6765883098885847750323695485797",
+            "2224258611402637372812075498606",
+            "1096213966837685819229458326999"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1594645032934746910233970766445",
+            "-7888607865300585433234547082596",
+            "4568621404199252787882732641526",
+            "-8445702981648004651693271114722",
+            "-7061451521435267187519509650848",
+            "6696570366244357679289327894924",
+            "8441199880199722664043107998393"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-968352259610893197064576275508718",
+            "718654181183934094051419872500011",
+            "2514405299754440481837754186402421",
+            "1800516623297456299756852785790978",
+            "1735762811642360734642777500928026",
+            "-818427028473555572485098076829014",
+            "-1159041509361012508054712578014787"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-53546281846072725871658001435887",
+            "-784630734436542795241759169811923",
+            "-718526749795888797678293952454287",
+            "1156515964581299989003720410728362",
+            "-838405070125273270332512952593048",
+            "1940449904167314426051872957919891",
+            "-68651512522935345206569837233327"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "572850491437435230154655178596404256",
+            "255907180028216856698708583682335802",
+            "467380937533420611084983609639927678",
+            "-578776848219875222429940192462905503",
+            "-305437010432528222408867961277657933",
+            "444241914984458649589370317558151484",
+            "-437642956538688234838513076311369227"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "358312056397600081928352831952683806",
+            "230966521966579976309413468582413234",
+            "-454572317314346054050950281223398637",
+            "-514645527798405383965741984798628527",
+            "178604228774099539826674396572478175",
+            "-617509561026367858818205129274682551",
+            "516749383280600313391592174692222877"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-123120621344099237933440205349374488066",
+            "-32225408253558378935144372215319566228",
+            "11301001788743914587399424830593135310",
+            "-52673174720412201741335076711264329300",
+            "-118712794710184498260756820285222988657",
+            "28574446760716238823537127611396566250",
+            "46215799970023783748448065488919483477"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "58353390412480195555770446835562760276",
+            "4833546377833498062150899905495355127",
+            "58373007706619991431855338114454320484",
+            "-70703504770529836816766481390613100369",
+            "143910284033581946098683712226489458302",
+            "117235434241994039601754355029907027922",
+            "100290083016626428361996754831114825823"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "77825035066067814510953781223725256637",
+            "100986125463015864365041226255651190014",
+            "119992813429982672016306816732813811809",
+            "138695082079580523445519172685734788839",
+            "-93300896559078568544280378088130535661",
+            "-21512916011354134398283958037251369915",
+            "-59254623236285041945205874576696705580"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "18754",
+            "-17855",
+            "-30143",
+            "11781",
+            "1051",
+            "-18103",
+            "3181",
+            "30569",
+            "-22565",
+            "-27059"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-10966",
+            "-15974",
+            "20424",
+            "21589",
+            "-20157",
+            "30323",
+            "18376",
+            "8005",
+            "30511",
+            "-8141"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1843361",
+            "-3517196",
+            "4625468",
+            "2321631",
+            "-7687911",
+            "2778344",
+            "125068",
+            "-4052771",
+            "7945865",
+            "6630921"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-2367184",
+            "-4826645",
+            "384943",
+            "-2111816",
+            "-1600561",
+            "-570653",
+            "-4103404",
+            "743335",
+            "-1107912",
+            "-6354480"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "1229589203",
+            "-140746934",
+            "-1838286677",
+            "-1878305406",
+            "1961215412",
+            "1250128901",
+            "-1166900225",
+            "275507073",
+            "1193731503",
+            "117609926"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-2112929173",
+            "2144863319",
+            "1992254245",
+            "-1937627007",
+            "1514668427",
+            "-1839905388",
+            "-1528455662",
+            "684780309",
+            "285489556",
+            "1981384197"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1886553183",
+            "-442912398",
+            "32808274",
+            "-183696654",
+            "-36971109",
+            "1971012659",
+            "-1278373247",
+            "-1688683464",
+            "1543369024",
+            "-1978222221"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "516696037327",
+            "-133356328060",
+            "366887410082",
+            "229447810393",
+            "-446554774147",
+            "-403087474672",
+            "-332879324979",
+            "-201414959734",
+            "69326493096",
+            "425678448609"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-439293553812",
+            "81933805742",
+            "-183895960348",
+            "-496786511195",
+            "313438779807",
+            "490045940301",
+            "475167170007",
+            "-102296474200",
+            "-69324984155",
+            "-419967280054"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-96551143452313",
+            "122054609553289",
+            "77940246274132",
+            "58988438553627",
+            "-37681781403480",
+            "-19891162801465",
+            "135285406649690",
+            "-79327539900948",
+            "132470331657036",
+            "-6480637466609"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "139797814993535",
+            "30274745342542",
+            "112817172422674",
+            "-52131523918633",
+            "-137459606352282",
+            "84222662933576",
+            "-98419098319107",
+            "6437228832991",
+            "81243750066175",
+            "-6421108664511"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "77769134597368",
+            "72127816622848",
+            "16103132204416",
+            "-7118137065494",
+            "49038964068498",
+            "86622864654107",
+            "32928568991190",
+            "-93632446337391",
+            "72127053286297",
+            "-103694778424640"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "22116565043461009",
+            "-31711615551037511",
+            "2821246731201391",
+            "-13351247986191283",
+            "-15623501709378161",
+            "25262423813103532",
+            "-22343569817384923",
+            "25928473716871331",
+            "32038496333487076",
+            "-6890071520503891"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-16647068290977681",
+            "-732120677335230",
+            "-33532489576560866",
+            "17792605822942940",
+            "24198243709120098",
+            "-23282344775422150",
+            "-28675151281288448",
+            "-14495501421676227",
+            "-1863883724884560",
+            "10312379020348421"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "5502360856538125985",
+            "-32644019016852151",
+            "1564692446235537384",
+            "-117754471439160800",
+            "-9013223642801616724",
+            "-5023267401335837159",
+            "699232645927291479",
+            "-3584329191084872093",
+            "-1311864165271950824",
+            "-1215291485581331201"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-2526466500631342193",
+            "-5409822186679340641",
+            "971670265469264458",
+            "-7179072119717361795",
+            "7670806788162789754",
+            "-8318882969144120324",
+            "-9117830171517155584",
+            "6877332662750891661",
+            "1068609140965782417",
+            "-4925546114463183284"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "57583945723378415772",
+            "-342232111928136964405",
+            "-1195237978858213658562",
+            "-2068806789801380063600",
+            "-345026535447781896266",
+            "133610950746109877229",
+            "-450656040906061504998",
+            "-1893962609238091988400",
+            "2350599054313986251061",
+            "1880733516481080050212"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-2249590711027756564860",
+            "452844566121110339696",
+            "-1922925439694884164421",
+            "-1550876878150832998572",
+            "-1185260961134674742569",
+            "-2039283323590053548044",
+            "-1687563918950261572963",
+            "-1944597896900829991591",
+            "-2359460784165763037898",
+            "-849961236635718175763"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "39964951314827779563",
+            "-372589960613063031098",
+            "-438035602871932031986",
+            "-664149217874104492300",
+            "-775029718832463979623",
+            "-1435477218213206627730",
+            "-777662353097751064825",
+            "-262402700201170887022",
+            "-1133936324748952947710",
+            "1327754548066739872375"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-190787733056613599495219",
+            "-515734754496362833319535",
+            "-574734864992365444843708",
+            "156606116703304701360073",
+            "-548629563358608342499088",
+            "314352279479464825255103",
+            "244688836299970664478495",
+            "223517627384801717925625",
+            "317460265141706669756597",
+            "-143088760391778698353724"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-121682002183738855835622",
+            "33829240294738771467442",
+            "561539377338103121964304",
+            "272589000480746001955982",
+            "476056122948827385095616",
+            "-483571374749297317898197",
+            "281143130465809637777850",
+            "-252495987618098543389302",
+            "394527250215970829835810",
+            "-371881433858127168330471"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-57554220355698316288465256",
+            "-76017540520824653711945808",
+            "-119283376705753351997748235",
+            "135285600664968889113349822",
+            "-56831347642867226553961594",
+            "-16457633406117235070315269",
+            "94381218510174633599528949",
+            "139553957700687764923892613",
+            "-53986781389720021194280790",
+            "-19994198089108196976112296"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-82576387180835452318210228",
+            "-49951144242867126451613703",
+            "62645934474937602509471162",
+            "-22809010733349138696349721",
+            "92648271949016552757106864",
+            "88129191312931452389255255",
+            "48011438537493230295219061",
+            "-78095235259711088591460034",
+            "132652997750328176815451796",
+            "-127305214162620317777045127"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-130726889859542614597966076",
+            "-56150188916918748446918856",
+            "131676166120048204530142114",
+            "-94402515536541962124681830",
+            "-141920082631622521770167745",
+            "138796510001418272242292175",
+            "-25121938667485895566695315",
+            "-121172442589103978423419243",
+            "-20614593640357879353182241",
+            "-82772743389453364089604477"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-29032726342801534955983348833",
+            "26158471499703236988192974887",
+            "-20114116207465444344361601128",
+            "-8681577131743649610696414703",
+            "32981177147751632167653843307",
+            "10715468167954459453141416666",
+            "2302451135115289498612037956",
+            "23070910964316738454734382371",
+            "-33075125932954753049741121323",
+            "10179913539917748846617161898"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-18085869738221726864797815861",
+            "-22337076672294140460410142760",
+            "-36126257950244891489093166216",
+            "32551071786984003673681244792",
+            "-2850364559142517973517854327",
+            "30988135871042625915127743204",
+            "6816014999562698392234131960",
+            "11494536675490631889014354130",
+            "19518893252904531723609256491",
+            "30427336543604275087551984403"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "2206367769081757187272650503139",
+            "9091361631393367270835435652597",
+            "-8424167241080231285345475239019",
+            "5834959797812027161305666236164",
+            "5192168644968070279507303594280",
+            "8341506527326618523722473942428",
+            "8594436209974939893446936907176",
+            "8681682547098842826265765019748",
+            "-5209020223570640459802520771750",
+            "6129943703376515947722773721600"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "9047109249979470996987065428892",
+            "-9799662424789913378125551587573",
+            "-10747887838497246955184907970",
+            "-7651592187822100343817178833722",
+            "6010095747552236822560336347090",
+            "-9204712857936864198407587203238",
+            "9268329208382510346394692142454",
+            "8372710454424791445966082256342",
+            "-9589445085108256491425243830947",
+            "-2939036102518584405497956526954"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-5392971967770555579833623091396",
+            "7010238060604152385637944463012",
+            "-952157695963924409916700180566",
+            "9774556009383087545392285658966",
+            "5770727225973437448322228561807",
+            "8118034587530249294761621086593",
+            "4629045275285811417199668511474",
+            "-3061439446531807421688403151892",
+            "-1370462548021359419092251720599",
+            "3949264862782238848089672597134"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-2122708269622350729686602450849918",
+            "133816358693598033876477261324945",
+            "2416602162368187687173545193833207",
+            "1033421043633734739603132153227481",
+            "1940044255763482239251273519489903",
+            "122404165677034509854998880971649",
+            "2115709380653150711430911081473587",
+            "-954697183455914334514087563443341",
+            "-2123319112143807840315352640840117",
+            "2069575612420264676616069495687633"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "1692523376605227556299724175506026",
+            "-560133778627251677678012123636755",
+            "1544300548641566325099709876685852",
+            "745074432149870035759595939596681",
+            "-1954945128503766900016994683433198",
+            "-2250046900317020802445312714834438",
+            "-2385830584607782067723440542099762",
+            "132290783232525067349828502622213",
+            "422538990938605269068919642797432",
+            "-1355056714352703736683115064904273"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "569637448050074280085390862207666873",
+            "-493782672094432637986628201979332547",
+            "559997888965253767870152932047889365",
+            "-192294596493342415461227479481558644",
+            "-467823747943914593693180397447717393",
+            "-180358592750289795012663508384651702",
+            "-277633416434123648393250047500800405",
+            "-161316286985680702736242426514896331",
+            "-107565346718017079645076760979976254",
+            "-343028720252730035978891475737130091"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "234202228460068979568708307032847949",
+            "-40201170775316769341182286764965389",
+            "-548980577113599607444799254971433151",
+            "541309219470209179356416121398240002",
+            "-434133022569940359270892323517092074",
+            "-273560039796167418597819102027246382",
+            "-476192949086818994497703488377408870",
+            "538618952671800675393447490544116473",
+            "93123165923308661421936492785506839",
+            "-637742333921319974000307483554747580"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-57037547117361632743720819923766599836",
+            "-98043608090325780586069692996244560497",
+            "-58505160656452186950211173738911382547",
+            "-101832799685426843314478830415915453714",
+            "88565658613827585716212234902487873212",
+            "37993662471162893207714631860591274024",
+            "142774451425889543215285746860042086262",
+            "16769512069264042461946104154625579535",
+            "-47552828674887709372081650891476109271",
+            "-42213620497501762639143979558467451605"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-147371756269307190381561519577768333146",
+            "50001054768402174217442186921551216212",
+            "-37909857998845694340361772993373171274",
+            "26488762795056480696073949053739280517",
+            "148945896988690949823595206483101743044",
+            "72542903931524796290465216241047509873",
+            "-167126191952176091620300581762459791416",
+            "7988430434861579711046106333746313228",
+            "-93130434033478424433799959885140345146",
+            "-3441472474979605847594343058062446207"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "46136426115595487614083206992753880226",
+            "-152038433918307384545322896116023034965",
+            "78938265199256609159965233504637641814",
+            "-92829030003981682199112271693126981849",
+            "10428026972710553277643628999246199628",
+            "-733720183320183133584540075760187261",
+            "-21944082237248471904000697107198113161",
+            "77626504391158473624750083478122906448",
+            "-159953023283206971997917750080649331297",
+            "67741969038939169538736690551212738571"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-14965",
+            "4396",
+            "-28370",
+            "13432",
+            "-19302",
+            "-25134",
+            "-2325"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "17774",
+            "12772",
+            "4697",
+            "6699",
+            "10195",
+            "7948",
+            "-1073"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "2170666",
+            "7768056",
+            "1730086",
+            "-575771",
+            "6874141",
+            "-2918802",
+            "-4009750"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-1145774",
+            "7588162",
+            "2538941",
+            "2814807",
+            "6420688",
+            "5552162",
+            "61716"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "475579788",
+            "-2110736469",
+            "-1038349020",
+            "860173314",
+            "685087056",
+            "2099281387",
+            "274916519"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1655873120",
+            "-1731031947",
+            "561894379",
+            "867850098",
+            "-1594780953",
+            "467730702",
+            "296381326"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-270809066",
+            "-1724753578",
+            "1076930264",
+            "1046992208",
+            "-900063126",
+            "453135840",
+            "-1402444227"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "543756676973",
+            "13894519068",
+            "-471273054319",
+            "-361667009872",
+            "347117046185",
+            "352117261872",
+            "428679947623"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-414574595169",
+            "536827742264",
+            "73835628644",
+            "274376152363",
+            "294589390066",
+            "-11991628898",
+            "-253347644755"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-19215636889418",
+            "119919740703605",
+            "-53215288815658",
+            "-33327361054276",
+            "45198301326977",
+            "90776449725838",
+            "138054869379894"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-119367237365312",
+            "-140091389309758",
+            "15475687696688",
+            "-38031515520518",
+            "42841758149017",
+            "69022243350084",
+            "9179513583233"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "41132552781971",
+            "41253830277347",
+            "87359934843401",
+            "-95181726796502",
+            "27768710470810",
+            "65809922501001",
+            "-24863795602983"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "292236195126093",
+            "22719593938564618",
+            "-19218388071157930",
+            "33981265825341884",
+            "-18509717253371672",
+            "30580146808867540",
+            "-25999233818244727"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "28135088215434228",
+            "24773690318810918",
+            "-26724803220639117",
+            "-29814061818363701",
+            "-10766489482800560",
+            "30758073781326745",
+            "8776059520448382"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "409254481003117766",
+            "-378161134041351348",
+            "-226782297600641326",
+            "-2884040847263852027",
+            "-8247477639592596195",
+            "4107975741440930492",
+            "6909135551839426970"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "2652868014887595347",
+            "-938854012354185866",
+            "-1598192800799770751",
+            "-8491269256089485178",
+            "-872167601828525845",
+            "8058502300329141671",
+            "3061608648711594103"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "146769264186631567632",
+            "731409928392690360238",
+            "201264820200577960524",
+            "-1212063678184800014575",
+            "1402459273537500628832",
+            "-1321984359334412491423",
+            "2211457652738431111859"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1847588624832707279911",
+            "-1813942494389916835789",
+            "-1958404120436196946126",
+            "1723793223754906327044",
+            "91104121605339071868",
+            "-37127427643123176929",
+            "-1759281127620720025265"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "621171459171060457840",
+            "207620657443600214064",
+            "-404297398702992699514",
+            "1641661352402425721053",
+            "-1968173283051339556925",
+            "-1975771117070391869479",
+            "-2326206316062007904468"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "157960797511886924834578",
+            "-341648681181914186464072",
+            "-496733650079984967364838",
+            "-444089180371365298276212",
+            "545885788394993954738089",
+            "335428473992315516320770",
+            "-8497165329921324825223"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-38994153758728840237480",
+            "300952907927512214570952",
+            "-312533402564800726680664",
+            "-384239115396249647073708",
+            "-93282765318781444842024",
+            "551248528363972986694971",
+            "287731359865140964525224"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "101025247865168925900922830",
+            "54857286231629651663501039",
+            "53544859669524419312467353",
+            "-124478605232939875253548620",
+            "17285305918706892564565360",
+            "-109106338103303620219905680",
+            "-39866206645357083787763508"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-85724769944287781078609302",
+            "-11704976431025150432522098",
+            "-140709446005420876354105607",
+            "118949605388441528679717522",
+            "147415669254850277731131009",
+            "-2518242221011392198813403",
+            "-110299103632345032256326636"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "132459645720204436026597326",
+            "-75684750458331130703402700",
+            "122558708866408634795154753",
+            "-110485976053300938316559281",
+            "49084715923367850211269998",
+            "96960123064705845023119238",
+            "123180430855167057989302957"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-18936205707758319951200347669",
+            "-37258565469174214424589588093",
+            "-24453048320489113118458406056",
+            "-10587746588216050364022206474",
+            "9748825791372076261497902071",
+            "36545682723514548638183652663",
+            "-25861598120327052219989832099"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "34252212325207935596745602246",
+            "-23818237350808788371236620972",
+            "36226019636259967938400145050",
+            "37192973754494387619137336106",
+            "-5057174993081801990711137100",
+            "19315470624682365399645492525",
+            "-32772028697629595902245624670"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-6792023322456998237800922079479",
+            "6549013706052367027552378139504",
+            "-9355871158471909412376307831412",
+            "-2030077667778674905770995357453",
+            "4297878705058522858422532653176",
+            "-2236484269757156229891142724479",
+            "-6040135630828160347217599800148"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-3599439677962085785168307719800",
+            "-4245900385327794216010405726579",
+            "9450764155817872025990436327715",
+            "8656233260540192979968625227977",
+            "1160854863482601034702346632871",
+            "3956026374789035457631650901065",
+            "1453599630928365440553293992121"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-8915771384062939324293344072095",
+            "777633297413063771199232835099",
+            "130337830279869604841710324103",
+            "-8373921460052275293875653314023",
+            "-5246842589420902120766262884279",
+            "3729544269328940809924381217876",
+            "6272763819900035423590366605426"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1617716673371462402162303323419703",
+            "26336765354063275118084261528831",
+            "1492712908830805398072707084234922",
+            "-219159852068967847470786902528212",
+            "126542964660097922190866932619843",
+            "889878290610547866308346440500254",
+            "2086140459688639708299971129281292"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "111351977899132479809610701899734",
+            "883054552995427224030821200597311",
+            "-2065448421761013752757262983625041",
+            "1742123659066570365356027485580166",
+            "-1214139633564246959114082011858323",
+            "-1329417508775576485092806838462803",
+            "1116415564152726849396179280195975"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "25792382598206920383056857113445849",
+            "-225206309711067108325919531991796201",
+            "404950774727900383585705746884848243",
+            "-470637440392131155991087245263637994",
+            "230147369983625933489964301980421634",
+            "-371436034781333002283883627046974041",
+            "560240201817281749776706447503520796"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-184751107296655537872267341147777926",
+            "329426487040671280863523825102062539",
+            "-418105543114253529014238351683881534",
+            "-617932739175887659472533031137397189",
+            "498336206481632051733278557649532891",
+            "580985420014217396923072878340987450",
+            "-205539611656251655979522315213206497"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-140752175559703681946897226667461340778",
+            "6561236723375434393948178304575297047",
+            "-51434996264522492401719681071804511089",
+            "-125946647710302371061500790543557458955",
+            "-133398205151504361614506341930758954997",
+            "-25733469644094471567565487452545498665",
+            "44048098477441865782455673692289495153"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-145448294825692173492641127474455264094",
+            "-121372206006141296987868818880832869230",
+            "-141786136125809700329851146916010465431",
+            "-64699237500646340071065763833604265646",
+            "-57587914054490682697769053906782102087",
+            "32192058203820777323675957815817034772",
+            "111541312639824620042046577015667034684"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-120964441901874362290104187440391939364",
+            "-96084745151601772727177422680207591805",
+            "-166536914660320577793788842820747499277",
+            "61156293370756011979833006538029339427",
+            "94985231961279433991237396849461866817",
+            "43959734247759347414496739857098340817",
+            "-72248466678766711563463040091419174562"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-6120",
+            "7057",
+            "-6295",
+            "14831",
+            "20732",
+            "-7444",
+            "29138",
+            "89",
+            "-31431",
+            "-8017"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "32489",
+            "16222",
+            "-21842",
+            "-24054",
+            "-29957",
+            "6345",
+            "-3733",
+            "-22295",
+            "-1301",
+            "10500"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-72740",
+            "-2017731",
+            "8013529",
+            "8012822",
+            "8352456",
+            "6421813",
+            "-2425529",
+            "-6607883",
+            "1881808",
+            "-2234254"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "5181661",
+            "-3577958",
+            "-4493738",
+            "-3295135",
+            "2544246",
+            "447427",
+            "-4432217",
+            "5249513",
+            "-4448902",
+            "-7992598"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1569315361",
+            "418335588",
+            "341076905",
+            "-730203548",
+            "1269521259",
+            "1784051884",
+            "510427112",
+            "-1268247751",
+            "2018024071",
+            "997087873"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-162075630",
+            "1319926606",
+            "169298867",
+            "-921839919",
+            "421705930",
+            "1771873235",
+            "437874475",
+            "-312980929",
+            "-392069627",
+            "-1446413247"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-699761346",
+            "-1098413742",
+            "800470076",
+            "-841985366",
+            "1782178243",
+            "-89008181",
+            "-1616551429",
+            "1946816503",
+            "484187966",
+            "-128499562"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-15033977299",
+            "-446057719500",
+            "481043288478",
+            "205263251758",
+            "-56863332253",
+            "159382478645",
+            "15497510247",
+            "-11166860082",
+            "459845389844",
+            "-315779255031"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "13619924768",
+            "-108585103579",
+            "-286333703121",
+            "-22225593305",
+            "-4214443693",
+            "404081003905",
+            "-266115340439",
+            "-522963986885",
+            "-81541686728",
+            "311449667067"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "3440291968557",
+            "39790668036317",
+            "121143735330951",
+            "60300700594106",
+            "-138565677223306",
+            "26990651262138",
+            "-69507089026735",
+            "28346271500562",
+            "-68434454958",
+            "45308688118409"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-30985211143486",
+            "14497903153172",
+            "108091472626361",
+            "-132443509279150",
+            "-124710978475843",
+            "-22621813043867",
+            "43348381090099",
+            "54224054718466",
+            "43935291459066",
+            "106409780408478"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-37426243585738",
+            "-136731105244763",
+            "132386170102737",
+            "18729575753580",
+            "-61381755765885",
+            "-67683935577429",
+            "-130715456782347",
+            "135506741441431",
+            "64115570795686",
+            "23904178460900"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-3109726999519689",
+            "1933126768037472",
+            "-324967063575891",
+            "-30747070522132113",
+            "-27829185300215897",
+            "-24226320094880357",
+            "8149118499338522",
+            "-14537608810060536",
+            "-22181098580998350",
+            "-1543766855453763"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-19154567959160505",
+            "-33714007900076962",
+            "29435912661346051",
+            "35448248561967655",
+            "27478015216841137",
+            "-9642727074664549",
+            "20762876950068092",
+            "12674801929871187",
+            "-29272042854662323",
+            "23087590778346538"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-6178990937608128005",
+            "-7489771860953428316",
+            "7138996857544191473",
+            "6891721369949758163",
+            "-2873210693718606357",
+            "-985023314566108394",
+            "-8482975523737005038",
+            "1933121339588994379",
+            "-6237609370968336133",
+            "-6006493915891200615"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "4784228838256112666",
+            "-5639521520326372798",
+            "-6372119935029533050",
+            "-13173761350189796",
+            "5256559774234064648",
+            "-7639941297675590428",
+            "8339054501857458814",
+            "-5851965443287017683",
+            "8471016463241511679",
+            "241927931394313236"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-483218027063613629703",
+            "2158782735740085418214",
+            "2086327309425994289085",
+            "-1775936067034501989571",
+            "87437755811834396167",
+            "-28401153096633450165",
+            "-1133004021229840529093",
+            "1871364394906942992468",
+            "-1367249310926305189706",
+            "2201573961610082633279"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-534665456095867317406",
+            "-2091145637438156527744",
+            "1002549198487665026269",
+            "-1829053960401122456656",
+            "-507210040486661453955",
+            "1045481287999809175028",
+            "1364780996043256384861",
+            "136073480977825489240",
+            "-1037248082018171019376",
+            "186526137004977787352"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-1046613079302385912677",
+            "1021357072861918022642",
+            "-1212895629888241074297",
+            "54602995841860468967",
+            "-2328862578607448318042",
+            "480770704064136552652",
+            "-2205462302944763257608",
+            "154486372431065433621",
+            "-1627545550801203651319",
+            "1661236305135671491366"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-25780050424235659153148",
+            "529910266796805109246568",
+            "-524035727043667850771484",
+            "158037899710407903896118",
+            "17775205387059778764118",
+            "183414271106923495875624",
+            "564101025301896167399950",
+            "174082107953749604159417",
+            "-510247241020792959694652",
+            "-31790257572439119641150"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "97909310721294570420676",
+            "-370685916895270859148094",
+            "-207704549226119447779963",
+            "426685027916926492394771",
+            "-443922064163037127693132",
+            "-359165394706192831593487",
+            "232450412648501709276260",
+            "236476482318617346453058",
+            "-444855507997557301532066",
+            "-449511488318489004842616"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "38297941459228367137032480",
+            "37693160058585205753349341",
+            "-135230435965114188640670954",
+            "63954600949868348155966793",
+            "-18059488984707719081771228",
+            "12523259782961827464033972",
+            "-154035891838969004456038899",
+            "-143740984881393038015437656",
+            "-297107082262287737774124",
+            "-130063144978385075413421572"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-121901231315087756982129772",
+            "21641765111158040359351177",
+            "79510642319690549494318636",
+            "99992072265161738668933440",
+            "-127544045958283514926879895",
+            "-114814479146472538283905870",
+            "-6956674791158590081188973",
+            "-152063567955740133520378519",
+            "35283887938627339173013183",
+            "-77488852854891430841283266"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-35297860946578727078961377",
+            "60914664428732973645334926",
+            "-45881788007565335539581505",
+            "-16228932271534174997165412",
+            "-71589554478597723028529083",
+            "125241806056796930297219831",
+            "-5330340415825203344097529",
+            "51775257591241051145892674",
+            "-15094772395361486242521560",
+            "25938645275060872598806563"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-37399043524295253068535150516",
+            "30416211250451302199476070853",
+            "-33596129413262219560422312517",
+            "5313931682258453540118628812",
+            "21048003994659574916664477845",
+            "-37382894423738432073629190847",
+            "-15318095353333670892374528768",
+            "9624056340871881017715059148",
+            "-16108266238759019605768116174",
+            "13607318011608209290042469481"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-29339095371120961510655959746",
+            "34499077642097337163200996883",
+            "12438610605314642729365206169",
+            "30026217552330912860684967035",
+            "5868323165870187705662338709",
+            "-22530513597085742496802571404",
+            "20187377513734548340151159804",
+            "19309445557615337894124357368",
+            "-9834900335171895983684326501",
+            "10977735104041542369404969198"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-1375924286262127484011633937832",
+            "-2147016509826786120906015229213",
+            "8312572357614785735606012528886",
+            "-4051638098169024644782191731949",
+            "-3093628440576237730249167513209",
+            "5666878520633789566902037223594",
+            "8891965700337502827869504398273",
+            "-3425704790604702995311162396482",
+            "-6523722055221807095862112250242",
+            "6355813979375858041213525598182"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-5058425412398335303061912485394",
+            "-7219572559975239271635947264369",
+            "726522491715344256514490058497",
+            "-7042445872695173559471595541793",
+            "2824402214773339173380186665274",
+            "-5015784004233448712622655297019",
+            "5486760745287225222007329633188",
+            "10065222761695430251439328556933",
+            "6166669897599573455003032087428",
+            "-2172662097799258720481847032564"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "8036993285327716214363526967047",
+            "4430074085127306659301038354391",
+            "3362256777260202908602746872178",
+            "-8231357790598764797150571723919",
+            "-10113083855460126423773664315153",
+            "4304195785667284780679333446327",
+            "7529022456114985130352846237161",
+            "-1617155253974148221187702205421",
+            "8306989535252982450393679766069",
+            "-9917879652040843830362880242798"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "2025150411878375809601981318007687",
+            "733154270680517806621380045304125",
+            "237851062444860227206988707561392",
+            "-2559173073631870285694845698600192",
+            "-1612871136126666613840893942772465",
+            "-907529648416027223413166136860388",
+            "-1839189396233732473729596583035690",
+            "88713624503691341082574108422443",
+            "338402803145217006589478543497297",
+            "-880122644115442177811028160833635"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "1839742964889166195100300692473716",
+            "885434974408265966958762326684680",
+            "-1621828050026346614658514674603315",
+            "1905665376759464813614892131583581",
+            "-2388201929527636648329898098952877",
+            "2481877047862950402685641670256262",
+            "2023252640349615986505312617764433",
+            "-1573486913273646294139059870810833",
+            "1805414520297065811367995244619334",
+            "1843061646556820707723637177021074"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-191847997784246847874745009499399054",
+            "372786599802160285128273261461255881",
+            "376288273710337416992985070127844806",
+            "188093861376600388183605420152756308",
+            "571207793825131270518817095820136067",
+            "-29919619103217554294662107906751719",
+            "-452678708024430178885781299576494992",
+            "535406994206915746718046269323872661",
+            "640280552739110271190723844875415792",
+            "-600187294174532632129083155832394691"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "160335860551254932275470660660164261",
+            "-471326848390488335045668822156269822",
+            "-495691107470112053256769582261403309",
+            "-393200617684151831876978692678316885",
+            "297414113438652864360156678843959711",
+            "-268776834626793693334448405382040859",
+            "-117780897867754527460881147518546805",
+            "298322417140430849049049944396274055",
+            "234866336354615854416417135899865645",
+            "48595083212054770623391355746687188"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "46767536875344963516160138900400417992",
+            "140769783767864571592597846120527955449",
+            "-87767956607006201530442382053058191437",
+            "-147467086345147382218132338434959967344",
+            "-63040799227207808871886672405579600857",
+            "-163413759426610481573012906592956738336",
+            "-51038824199236863589719906709018954970",
+            "-144531515418551447921937889834182877513",
+            "-167321975433722954448458398645759050579",
+            "-123967516821302237595171386295592191011"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "79453031253319883075571062614534868736",
+            "123921984663798339646770377207635911524",
+            "-157303675968253574763215319217464671001",
+            "-156891299361397755454566840636980983938",
+            "-145862034846283521653632276945573288715",
+            "-28726972658144915880841740075557050787",
+            "-59175893681203716281306184534788761148",
+            "-116003172142553199810126172180490708929",
+            "76631143543919820921892254735365792596",
+            "97582911312902206555664841122180441248"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "7093132471014474767844887751733331003",
+            "-108686417660834376788570147811161987220",
+            "-163594850104237544086018972477724405791",
+            "-2302501739484662175406128065151266870",
+            "-97343550981288966396082969127785946405",
+            "-96030927988302472038807926974869041673",
+            "-154167090334610438942150064192357641402",
+            "-18793807036998611053269951477295987184",
+            "-70904740499999719165305256191226042599",
+            "70653097581681099304462157018443053691"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "26050",
+            "10037",
+            "-10374",
+            "-29490",
+            "14331",
+            "-16851",
+            "-12187"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "13120",
+            "30961",
+            "18595",
+            "11399",
+            "9159",
+            "-1887",
+            "-30813"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1464633",
+            "220199",
+            "2712771",
+            "-7495838",
+            "-7981890",
+            "-4964125",
+            "1595391"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-5853521",
+            "4344274",
+            "-7018099",
+            "2293236",
+            "-7612443",
+            "-2167093",
+            "-2845086"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "95875595",
+            "-609963850",
+            "896088118",
+            "-1715535486",
+            "-102098264",
+            "1291776262",
+            "-576308153"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1236728557",
+            "-1637854635",
+            "-984551882",
+            "795795740",
+            "8028990",
+            "901192316",
+            "-1111344946"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "378575634",
+            "-419525563",
+            "1883074043",
+            "-1173948468",
+            "-1824740989",
+            "-522033444",
+            "-144793007"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "370947827710",
+            "-353447519869",
+            "284585997441",
+            "274833182439",
+            "46832561565",
+            "371244974672",
+            "202518766395"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "218531362099",
+            "125513046194",
+            "203189347634",
+            "-304680830682",
+            "-463095454301",
+            "-6527992404",
+            "-65734664745"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "124491338796971",
+            "-71234357373577",
+            "-70640224281818",
+            "-115120848140363",
+            "46780581253253",
+            "-5239172476241",
+            "-84243714348014"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-43659843831402",
+            "97675487713738",
+            "-120205430205763",
+            "55637124317579",
+            "-2982496726764",
+            "-52205341209619",
+            "61907219984235"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "40565389677572",
+            "-115932875026862",
+            "-59257993113702",
+            "45346953112251",
+            "48999715544038",
+            "-93859032164586",
+            "-82647138297887"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-31698638660098966",
+            "-5631854766304352",
+            "17449252429242489",
+            "28180900120216510",
+            "5008123877732808",
+            "-34000957537070508",
+            "-9375521335589905"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "9961273701300998",
+            "-2079937397053597",
+            "26948861976924314",
+            "-33274483408865913",
+            "30025685782393962",
+            "23738162907963992",
+            "493130406875726"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-1979493224765362120",
+            "2703732660426547803",
+            "-1393112844070577191",
+            "-4589313647595954574",
+            "-6650779199917767681",
+            "4898222384404144581",
+            "136147557489562317"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-3646410955003742974",
+            "8191219785186527321",
+            "1272327194703892541",
+            "7200053761957267992",
+            "-4756848297671371938",
+            "465404651157646586",
+            "5030791210523335471"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "471782014697430155157",
+            "-737522866566807476021",
+            "-1150886932732683487297",
+            "2216993989609573889751",
+            "1791930017506828908157",
+            "-844823775260624667509",
+            "544085813983975919632"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "1674397379408819628225",
+            "-916247777654751098512",
+            "770173233696709315682",
+            "740475606804091311338",
+            "1832098966387094053802",
+            "-846509827577616730614",
+            "1310459528272118181618"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "74011478604212791529",
+            "-331788012397048455114",
+            "-1505535048061915354043",
+            "2196671714385448598750",
+            "-350496961296855157879",
+            "-1047520580803370592747",
+            "1407733904504194944452"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "334057475536999391388584",
+            "167091731454929333593163",
+            "44371453303518972747966",
+            "569151268328963633065480",
+            "502857981392722544973772",
+            "-295294362379075981585508",
+            "231490385988114470344787"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "67874870060011234362111",
+            "-134871563655837319141577",
+            "-165820628960372861946363",
+            "-391238480407275084709510",
+            "-514577690826104710765865",
+            "-67979890712897414695858",
+            "-408586208764886574226242"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-92558993866389986507056248",
+            "141516883852752944766615012",
+            "-91570642406924423409457354",
+            "142445780080422570580978556",
+            "6629554290181300402543099",
+            "-13376427830750013064535819",
+            "-128889999820989497872482505"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "109570534710933797396607895",
+            "44325882156245683484525132",
+            "-78370164483658463265616463",
+            "21269267048775656453724437",
+            "-30434466531056204756754318",
+            "-146721097875248361900368890",
+            "16341220289441428332578520"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "149460511934390160021660960",
+            "85813150145346742579054179",
+            "10395595941021601123326187",
+            "60742333060570890451324525",
+            "-69744724753060176636143795",
+            "85805585820369366977540699",
+            "-118751782594197400635380486"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "25165154960101426243162674534",
+            "-10828793959658115401846998769",
+            "-15232188130968548112534724088",
+            "37859448133639721078872362278",
+            "-7447685266469730873213326336",
+            "-6630414361379515759165805767",
+            "-7233008330319849805678054088"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "31352291163482491119505968125",
+            "30926463871793539317337614369",
+            "-26565368808580343750512847154",
+            "-24547829992985946652408320126",
+            "-18355556958734667364187798032",
+            "1438511591107866365310462118",
+            "-28634765763897394391222662353"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "5950176580747626568664247515800",
+            "-8881211795988992975636965132300",
+            "-2313319268134714731343698733583",
+            "-9280914263091091040890831012745",
+            "4639580868256294580236676918210",
+            "-9335839789223791032443503646374",
+            "-972698824035541842758234702650"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-9116871202867186426233180444277",
+            "1252769228624504890201406664824",
+            "-4441678277221598037171990272524",
+            "-3497493513724645633286134222906",
+            "1362086037022057987770970014654",
+            "2526310726561592465837221454378",
+            "197133844356911321653053339758"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-6708417134328302521535165500528",
+            "1259667415597434897205018980503",
+            "-8667405068669984535223184571634",
+            "-5739153439279994263124834718486",
+            "-4659094842072120505966726511778",
+            "8300992321238715729975176338668",
+            "8305762879837071470447907044080"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "1855360572508690069751614139995240",
+            "913127495556918482781605103352223",
+            "-1634237774328458406341648362825579",
+            "2012213887653411326935188001113650",
+            "1854844808900330191204660335054080",
+            "1943836864945319401575398852606071",
+            "-2512140816283887752192691293590826"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "2347346757752941921138702653567730",
+            "2471720490360303419055064438101411",
+            "-637525221921571876383204743520988",
+            "-313171852811679112466522221783714",
+            "1342661303952287975125536927727615",
+            "2299702375882500241106744128006573",
+            "-1943268622964000029081987879045476"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "654404760071118946096668775855737838",
+            "-134614183491363562333301742647543553",
+            "193373068724594777442105443158226074",
+            "82128803304304898958437601302688987",
+            "231616933503852269212186042174117433",
+            "314512202435083245896423396120681411",
+            "-560893487306878937612307892677922361"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "61682174231144284695206586082327571",
+            "-129722213375102804822534176471924759",
+            "-470843635185932669298551792606832749",
+            "217981712620797098953060036718615794",
+            "524439585175580833405205537825140874",
+            "-195520289865208736968677273971035370",
+            "74511399356182107059807662782310402"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "26120113210988095907759127757085647952",
+            "-62165437783621924594919739022258953866",
+            "104747421037847971642972590567045904195",
+            "53605601765747459417883834171905176717",
+            "-16460101511938648810934533084724073149",
+            "87326150749852540273294368718992775275",
+            "16683834934928424633510098174197548524"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "26497149875196872156784837245492823091",
+            "92947688966177192205895235501995800904",
+            "-128897447396708280338335738279922653247",
+            "94341806002152925510991334018527791058",
+            "-112437238136605911451884994064234664417",
+            "-149010327216271969260130899539508200431",
+            "-32345160422388704273102616927971726501"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-166189853516680988544038633926086071136",
+            "-51412708930825095905795826687352289630",
+            "-8345130886957441238365575658116421520",
+            "1835330936447412638754115271624712637",
+            "149041866461530306889089342295575456556",
+            "16212245285186773325597991037224844276",
+            "-44514833992738879886424708523206514266"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "19322",
+            "19888",
+            "-20267",
+            "-32519",
+            "-22546",
+            "19491",
+            "-12006",
+            "24475",
+            "7982",
+            "-4810"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-32156",
+            "-17491",
+            "30836",
+            "18967",
+            "26344",
+            "4302",
+            "21070",
+            "-3741",
+            "21984",
+            "20912"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "7093261",
+            "-5853548",
+            "7803379",
+            "-2652515",
+            "4959963",
+            "8087528",
+            "1598575",
+            "7487372",
+            "-3584523",
+            "476277"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "5614990",
+            "1605944",
+            "2158854",
+            "-5461816",
+            "1168967",
+            "61828",
+            "5746557",
+            "1413562",
+            "-7871958",
+            "6056696"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "1038435346",
+            "-2012132702",
+            "-903699578",
+            "-1322624637",
+            "-895719624",
+            "1176645993",
+            "-1538792708",
+            "-499541589",
+            "-1303395798",
+            "1511778212"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "592833317",
+            "-1551950984",
+            "1787888535",
+            "-810984372",
+            "-1974703475",
+            "493667730",
+            "-1121250454",
+            "376054319",
+            "567834251",
+            "1569097131"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-57685736",
+            "123800228",
+            "519450088",
+            "1079630019",
+            "-1254308991",
+            "590805934",
+            "1765339826",
+            "-1234550696",
+            "-824160185",
+            "-2094981215"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "426101820801",
+            "16436911678",
+            "-359476250071",
+            "-443809896688",
+            "320952321382",
+            "23645867134",
+            "341575522947",
+            "-454866134852",
+            "140063572947",
+            "-548803465692"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-206275629357",
+            "-54167785313",
+            "468398637611",
+            "-162195244968",
+            "238463244086",
+            "-168579433699",
+            "32497103886",
+            "379346160025",
+            "541309413663",
+            "187332535600"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "49075412231132",
+            "27312750405736",
+            "-22512416476092",
+            "112383900203683",
+            "-102429519587353",
+            "49012285726128",
+            "49380517981528",
+            "100096915120043",
+            "-135774832908052",
+            "-26743412282490"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "92838931686727",
+            "1117744174578",
+            "95749441954752",
+            "83774056709063",
+            "-80485560556004",
+            "31410756920117",
+            "-36487657625110",
+            "-17337070785087",
+            "-79624495118450",
+            "125937356449490"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-110960987245002",
+            "-130196863138687",
+            "112521051717937",
+            "22370815190676",
+            "31270809959205",
+            "121855313393173",
+            "8699496453047",
+            "-91463041032135",
+            "120519868366012",
+            "-133257403196016"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-32069587070878257",
+            "-17919111411953962",
+            "-18001660931442915",
+            "-1767754592277008",
+            "20250526488187012",
+            "-15695508630258145",
+            "-29837740769794226",
+            "-4501074155399095",
+            "12225232288612758",
+            "22313124950526297"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-34480102097926967",
+            "6344112291297418",
+            "29002029974785001",
+            "-31888655934628218",
+            "9139086331212235",
+            "2897750950926006",
+            "-27160809101795530",
+            "21028873622945161",
+            "15244977183310528",
+            "-24528972829842178"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-4068321997152171044",
+            "-4123931468795332282",
+            "-1917336520397580474",
+            "7863490469535036896",
+            "-6249110314124783127",
+            "-8225844925803348664",
+            "915179017540453539",
+            "3123520133722242387",
+            "7605858146072323334",
+            "3943872340866871094"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "8702547793259894978",
+            "6483962839585495096",
+            "-6872654786944293818",
+            "6033338727343656838",
+            "2377337619842982990",
+            "-6489810207155348116",
+            "1074629160217093102",
+            "4824865193186455416",
+            "-8000377732360383694",
+            "-7092995275501204023"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "392214659868220760435",
+            "2001707758785583846477",
+            "1973056242078770230918",
+            "-1958109206556224024652",
+            "1085003398984190910582",
+            "122474924730748113507",
+            "851293406060406798668",
+            "1832853642649160529635",
+            "-2111330458370422801057",
+            "-2122277960278850970588"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-2252221810790127710310",
+            "1477205316920429874421",
+            "-961506947433404737273",
+            "-245308458478085909173",
+            "357740715501747274650",
+            "-319267889025708102500",
+            "1094364280480774943760",
+            "-127746993279336115173",
+            "1782157345845552907164",
+            "238471321966246221649"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1691113452288155441823",
+            "-837928306525549514253",
+            "-1479432108576007046222",
+            "1105760639244233719583",
+            "-1237878533885748567381",
+            "-2185454825581323446807",
+            "163923040453210887447",
+            "-1194656668349891851539",
+            "-1697451474233505816259",
+            "-1346443165535100297337"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "234606311866196287199252",
+            "-280339717607805536038094",
+            "174162930698908947490765",
+            "161585530398926918641746",
+            "149110074266065657434558",
+            "39179047469267699561279",
+            "-562131507501967025468563",
+            "353803686657854340422174",
+            "491584079078747706104003",
+            "-361063318154572608483058"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "271411052426415377865185",
+            "-444592158660720424250257",
+            "-255709913155980939726276",
+            "365550349570545751906209",
+            "-391044572823762759576713",
+            "183367216650532855576691",
+            "-25390552937951110044900",
+            "184949987773880185014120",
+            "-267534620027159821434853",
+            "132847257761444427988262"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-50975717540435075836318695",
+            "1378903033683610665195435",
+            "-140258399598530936546358654",
+            "-87594983716718223901386738",
+            "-62237980291352487006786500",
+            "-91499845570297449212552189",
+            "-66208718275179977437401191",
+            "71189572681527467501510610",
+            "-1306164209002345386773606",
+            "-49463313801741626591705131"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-128773881103768533353175446",
+            "-92989705198857589828906933",
+            "135169579794423165966163763",
+            "-74770130369838071149350912",
+            "66417301185305992075241814",
+            "-141076205408915017047534288",
+            "139087458245468156248483059",
+            "-113662293865384133240833450",
+            "-33825577590872840313241969",
+            "-14077765969840480002036401"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "83003901123187485851103193",
+            "-2681382205142334385409300",
+            "-63331882652492224187298286",
+            "-110680165287453809004856775",
+            "147522995254483971982329487",
+            "75382769928196858065260409",
+            "68276261988199153282208869",
+            "-87269314773681476775194807",
+            "51406486149759545599895079",
+            "40564670281024197061922198"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-26644049411043351785728824141",
+            "-24105812251123083991288681115",
+            "-13313117369621013301443559549",
+            "-15343445014815373573159127244",
+            "5755441206841059236617659163",
+            "-19137609460361732913808417597",
+            "9820328360260799657364770926",
+            "33319966440642342193193092160",
+            "-2450510718121406857997296905",
+            "19633470714253525250579882639"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-36841392181436281670139286742",
+            "35355431815928332051589081557",
+            "12542032737254605451781545802",
+            "-10487948400679804251659176662",
+            "33809232792580693385020804592",
+            "-1286410851551344895362223329",
+            "7299687240575999746351638266",
+            "-17358024343798554471827987834",
+            "-29156466412538498088209136035",
+            "-17589710144892474177196649606"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1569098507666310750400243811856",
+            "-758234951593444817309034952464",
+            "-9334359694680896788581614109081",
+            "1374116640047423155668561508442",
+            "-173162738574107411965825187244",
+            "-7788646219993207807828069988222",
+            "4582300712366442004380627595004",
+            "4100619504434533691239280199560",
+            "-468263237010220777440412988226",
+            "-6978567372677615209814697240488"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-2976231597733439456905633512789",
+            "5127694050127805803249796769468",
+            "2812957412066585231215442690760",
+            "-8402845131237017711805113621227",
+            "-3787750299715284207846037832827",
+            "-2942880408698349918786267176602",
+            "-3869882757561265792893437687111",
+            "-5705588172653590518606599812141",
+            "-7302962533971552771835642777347",
+            "-3268746639752527561077535734497"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "249971857312161074689173876267",
+            "-242404619148151547781226427874",
+            "9581515616266968745257306192718",
+            "-9310338506009408582538577250155",
+            "-6804608278893940697882201407664",
+            "3099137104427329585255879001438",
+            "-7567579129514718223375916274942",
+            "-1792499874699902261296010365098",
+            "1639591688226140840628852768785",
+            "-7608440910939533514456193631459"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1478066879273161886647445158949772",
+            "1837220622062178767494109366244231",
+            "-92877272254401719687763113724897",
+            "-294736309677963931197700276380980",
+            "-1131638498094855104492084200292876",
+            "-1707583373250833555270111202726089",
+            "2530465654746538337722203391829038",
+            "-2249523431616310451983315854427522",
+            "626377764251003872328656072958391",
+            "-2410247195955462616836551577048770"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-146608320296582083230297319921561",
+            "-2472542853509176412554981102318147",
+            "-771312358883127333171922117508275",
+            "-1484063485715773612247024703348500",
+            "679091726809387681196416537626957",
+            "-2001457836812405622324166631537924",
+            "-631900342171956815934243043067448",
+            "711088708081508591363639203425914",
+            "1216331827955829693179876936054302",
+            "-1299482886169500029305135186407016"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "200012218527021640529683894748995748",
+            "66764908937162351505130243349347892",
+            "524610465017564034853035195648393051",
+            "-558690033622104772858687622389660622",
+            "-544523595431390359103420871965400054",
+            "245517776515380942196684249862925365",
+            "212945120221746468714378114960847869",
+            "-170782019116161816032185050362150848",
+            "-496641648487716226900749343652261406",
+            "648453852192358064522416946780906606"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "255949560736548808004852239641213264",
+            "-324087200268712128351855136105640947",
+            "-6820351909324143280484118195131375",
+            "-319504958870141568923172587852075104",
+            "523488920375771830477498953239602567",
+            "194332300193562216166276513041179721",
+            "-544008549012482630748073246202260640",
+            "159071387785195543622408425447358845",
+            "76192843992111999275453592662491096",
+            "-473054289942710741781585231767065104"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-139359679896732931766919659337245494846",
+            "-35545345721518693494350745566292372621",
+            "-39327020288096966885458128121852385736",
+            "7493284389294749331633573014767224917",
+            "77381066407576226608915901862097770212",
+            "157711333385356156585898944278021995524",
+            "49883214513209304705821748055646582719",
+            "146274344215989941722162867924721361448",
+            "29434461275514518797058993245422008579",
+            "129948671740191770106010881205508710601"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "78106395943978345433881498304008994451",
+            "-160930953760985927349252065503891837878",
+            "-44960294156570777230598296792605054671",
+            "140984650860694548582383830944572104234",
+            "-139997530771990989911328151859950982105",
+            "-56309689550359535719606358796078640634",
+            "56779686305520783893926725471749235781",
+            "64518970089533247978457584756917625030",
+            "-119002062601850172159773604273618695770",
+            "99267635189640639519242595654851971448"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-72992812885182321071012171435152271739",
+            "-55130623505249212822546937141683067554",
+            "-72150173497072821802434510525219140741",
+            "-135826146169714584749451385037745056378",
+            "130698346424652975123966744269457918140",
+            "-9260092987630855868815559991510953109",
+            "687286835945665138009166453051890054",
+            "-152468467896817629814259003076281203489",
+            "-162737426598589027675694265384434548482",
+            "130527505687529038013531520307450723070"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-20673",
+            "4891",
+            "21555",
+            "-23205",
+            "24226",
+            "-28935",
+            "-9747"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-7505",
+            "4860",
+            "-25545",
+            "20070",
+            "25803",
+            "-29488",
+            "25570"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "1487976",
+            "1762770",
+            "-4729573",
+            "1860064",
+            "4929956",
+            "5522284",
+            "6549685"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "4049960",
+            "-970587",
+            "-1321157",
+            "-3312445",
+            "2703511",
+            "6707801",
+            "-5572805"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1030375303",
+            "850853352",
+            "1617347935",
+            "-420547740",
+            "653990365",
+            "353737175",
+            "887141994"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "974530540",
+            "1286390864",
+            "-1834371078",
+            "-873048386",
+            "-1662709861",
+            "-1479216424",
+            "841702675"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "2111707600",
+            "-254659598",
+            "486623667",
+            "714212951",
+            "1069946182",
+            "1654147123",
+            "818253616"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "158613753089",
+            "458774753508",
+            "183875050170",
+            "-405459377062",
+            "-76041901937",
+            "422167883413",
+            "-534535030458"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-189858741056",
+            "-350281980076",
+            "75626581117",
+            "-124609040589",
+            "16597068948",
+            "210894043846",
+            "147277886747"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-138751689194121",
+            "-103042931779161",
+            "-112178692829012",
+            "29511789263726",
+            "46232439685971",
+            "48353150098534",
+            "117568446080604"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-61910798890980",
+            "-140506887267184",
+            "112641714944989",
+            "3355252674878",
+            "-15180008486174",
+            "-77620968921810",
+            "88635255157246"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "54206304886847",
+            "-101630474357009",
+            "96590747853874",
+            "-118603139384744",
+            "5675249069187",
+            "-45370506866973",
+            "-36853313570962"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "12547998279054852",
+            "32079636436024400",
+            "-8837895237364730",
+            "-35089444786916540",
+            "11941345258052284",
+            "11998082458470934",
+            "9244690049555110"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "10936921703728198",
+            "-21359025539153981",
+            "-17919755145627767",
+            "21949453057332109",
+            "31577422472892467",
+            "-2175882775298387",
+            "22366796586949547"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1991785497370767870",
+            "6209395018704963569",
+            "5802436034677451710",
+            "-7064032955625756220",
+            "-425594384187662050",
+            "-5892871677969375177",
+            "6618055914134267928"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-9127990150600728413",
+            "-5735750909960740797",
+            "3801973574837623883",
+            "-5415111795522083403",
+            "1905105625346053106",
+            "6231685532968733711",
+            "-467857863906217121"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "85863289349977744466",
+            "1143903301173917583596",
+            "343013108718986345006",
+            "143977393722524587619",
+            "-331683688552855746072",
+            "-1255008151495427109419",
+            "-367830894824428134772"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-1380262755120611917815",
+            "-2015184270487688482207",
+            "421650876988531672681",
+            "1400851929839245970743",
+            "-247535080568189823163",
+            "-2113957207159701869965",
+            "661753800403331824022"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1409117742382861999337",
+            "1964560064468429764667",
+            "1142331396214943706208",
+            "2120139614999229044568",
+            "1947051906742118331879",
+            "-2255925732391731193539",
+            "-706486902189772690761"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "476431701466362805331100",
+            "-102676717243207894859018",
+            "594838701556832564313786",
+            "-362093575874816415536617",
+            "-298506140329613600409281",
+            "113749430332489269875681",
+            "197169454728417077892949"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-427853124038909455080219",
+            "66701743587659683001174",
+            "-2597511976969857899394",
+            "60778757968804042956061",
+            "-544547149575175129004139",
+            "-301565858845410225401166",
+            "-265399533535019605844097"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-68618783730059765480823635",
+            "-148575561591978834067415296",
+            "34414152208973548229001993",
+            "-13643309285265500492215602",
+            "102647265825215089997130781",
+            "137227738860777958998527556",
+            "89495140415736266671994902"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-136993255643844109804321368",
+            "95521822621879235917169875",
+            "15444326698047673705527386",
+            "66587355931325005334848104",
+            "-145548966688252050709592565",
+            "-114049305396707161765587204",
+            "-108330685803294296438944164"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-35055644549667511602728701",
+            "-151408885763406254266448065",
+            "148986972316531200205118208",
+            "-41264915535541021357739976",
+            "136259859352059633203136439",
+            "50887898641106552333186126",
+            "-125285378380785532529599731"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "12941490015157247768713837736",
+            "6835075928133812117006091940",
+            "-22465188887333363863609842193",
+            "-7126648403394390900967730280",
+            "-28393957274203526260439553432",
+            "-38741812277298816415321041919",
+            "13338414913605328772627511898"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-20110672505237420917225518191",
+            "-27596411499885490649297106626",
+            "-12603225725813579881802031040",
+            "7543351889165927098806258732",
+            "-36704747017636431900889798792",
+            "21396851540234206754769905295",
+            "-26912455145404876756203862275"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-3387707576912848734654159629359",
+            "7428755755160883095771560631147",
+            "4569162969402491573365786468348",
+            "-2490525292701388792253454545621",
+            "2994837362157224884748366006872",
+            "8916264544517216999577433705932",
+            "5151024374664908204375853841885"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-5663811480390229830172653604467",
+            "7710539150230289621751006180128",
+            "7797728575196483778606701813486",
+            "9630462872777850496307059381475",
+            "-7728740627143181530379709906473",
+            "-7781087371673710496980289208707",
+            "-343446002946672747378655962906"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-7557416815553078625043545761338",
+            "-2302505358387623502188356176192",
+            "8409779146217467126417839702826",
+            "-9562089631495606093360342875300",
+            "-8213115909769479205332628941563",
+            "-3559170898222778482528552837035",
+            "515283880696723945344198400802"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "1367641768492784058807182267663971",
+            "465298046667559350852539820375203",
+            "-574478792401028873492458572224839",
+            "671454055256727078424602503807202",
+            "-1606532935067250255205206946799560",
+            "2502175580278378863196680582577687",
+            "-158821802300435835463112943740860"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "215412964590856924411049997929445",
+            "2425019052313712110758162804934291",
+            "-2146281446953297190248965140031957",
+            "2313116073547801117262414478722805",
+            "2445004820402073310655244369464935",
+            "2131568767414300517904191007403641",
+            "-1647531519212921581334819756802357"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "85439167121228840311602468684207718",
+            "81462378211070190176464268885238633",
+            "-520261127199435865904944398789158800",
+            "-373378915703658263067219343884833015",
+            "303273353020035808553785984579294122",
+            "-14150871600103744151748032687048184",
+            "187409089415128244172342086722110778"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "426903599253969182311652501209225081",
+            "386418676761755887855863585696427033",
+            "-12458025011532389171806041982715946",
+            "209536513458228602712253512458307460",
+            "-606973411546439030075456065531198766",
+            "292324957531014750417545379298738712",
+            "-283787856374051062869985486340352964"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "63865640843944075030209963280445799369",
+            "-18593804610606078796694478405978654849",
+            "-714714379007990044568901737313354522",
+            "-143842961516453470477418544067319181336",
+            "153172084381164114541423265053189318206",
+            "12685456738195001579207153151789306138",
+            "-29575308771159264213086650783552893870"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "46883541681382683193881459486746081613",
+            "-125705053509498712301219791891009785852",
+            "62669131077706452018756304286321569553",
+            "22514282939897237627731060578196173718",
+            "-75575161713087697602988263826550011240",
+            "46505563659899129005157792634893169283",
+            "-119258175090032575441578768164878731653"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-55040330949772143040530849272221593209",
+            "135813211061156674505901218739644432898",
+            "169896929526665904477525636205199802874",
+            "-144923747172463589788289511293391386010",
+            "52887701532943518748533315738175210189",
+            "129347148437659340779376391685263512814",
+            "-119470842876568878095060418047486776106"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "20439",
+            "22663",
+            "-27261",
+            "23271",
+            "-18990",
+            "15944",
+            "-28565",
+            "24963",
+            "30418",
+            "-30608"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-22057",
+            "3217",
+            "10476",
+            "15694",
+            "-6027",
+            "-15715",
+            "-23007",
+            "-31432",
+            "-4516",
+            "3674"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-7442230",
+            "2338346",
+            "-2738445",
+            "-5985510",
+            "2142214",
+            "52299",
+            "-8166525",
+            "4690970",
+            "-5712690",
+            "-5048341"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "4260441",
+            "4125202",
+            "2115968",
+            "-5649222",
+            "-3760260",
+            "-4042562",
+            "-2350779",
+            "96900",
+            "346378",
+            "4216085"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "1097374735",
+            "-1284838828",
+            "1061029847",
+            "1269187264",
+            "2093057513",
+            "-552929762",
+            "-1669807006",
+            "48971987",
+            "-806857042",
+            "-1203491271"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "486603994",
+            "-1677973283",
+            "-925436252",
+            "1756729478",
+            "2007578960",
+            "841852782",
+            "291979431",
+            "1252234044",
+            "-1077400305",
+            "-2004190290"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-2012058775",
+            "567356812",
+            "1905525934",
+            "-1504705027",
+            "-66129469",
+            "-562262896",
+            "732024217",
+            "-1858174448",
+            "-1128956516",
+            "1788342603"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "435657141702",
+            "-478490014750",
+            "360566647940",
+            "-451660074249",
+            "-473020751145",
+            "392377661939",
+            "-482632860063",
+            "-338086490878",
+            "309007405872",
+            "-353813708003"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-459773526894",
+            "-430668239377",
+            "98090610437",
+            "75247578430",
+            "-324388649864",
+            "416582345348",
+            "-62640371660",
+            "-284829666000",
+            "381139880955",
+            "-4984998478"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-20005911336447",
+            "72447015480327",
+            "45875595264956",
+            "-76521915606081",
+            "-34727867953542",
+            "-137384660844194",
+            "52697015634263",
+            "103452307321328",
+            "-122001791486213",
+            "-41151199613229"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "70939096990350",
+            "-88819773553707",
+            "-56139660451437",
+            "65674495025011",
+            "26628603487458",
+            "-135902172047580",
+            "-103682776666204",
+            "132777414234749",
+            "-95404715253975",
+            "132858731621408"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-132374262406977",
+            "-23628151618575",
+            "-119413346104562",
+            "-55036741341427",
+            "-35811108191614",
+            "-1926885375842",
+            "-45365158169918",
+            "-114661135140377",
+            "10473703483140",
+            "-74303850742113"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "25306738711824109",
+            "32509922510474623",
+            "707085602821514",
+            "9763707224974397",
+            "-11143100390263700",
+            "11596259349396834",
+            "24075139416501450",
+            "8655375690994432",
+            "-18063068128417348",
+            "-19863328717071201"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "5637539114033423",
+            "-5001319200678838",
+            "14212933040227288",
+            "-33982239913110551",
+            "9485454238704206",
+            "22215509542635843",
+            "-32953189204876032",
+            "-24998747694223021",
+            "-2633537128076828",
+            "20245361874996758"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "4985469615170125407",
+            "3351154357521105423",
+            "-1334855791139741442",
+            "-8797809572977779612",
+            "-8598556274215160807",
+            "-3639608199332585325",
+            "4875792833346596793",
+            "-180163167139705625",
+            "476427630399101153",
+            "-2608044133530342885"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-5157228090470659511",
+            "7212202656918885732",
+            "1392041261546235241",
+            "6666890125347683",
+            "-3802346357954570034",
+            "7105488401526240167",
+            "-4672914031095531393",
+            "-6827233405808698926",
+            "3167057671289501060",
+            "-3717235008913756239"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "1189803804746023930692",
+            "-625825002864892544122",
+            "-1629596920467579686358",
+            "1736386230972209967360",
+            "700381856353328623786",
+            "-654232777049969789030",
+            "-74724090947393824084",
+            "1167695718621359342916",
+            "982240876673049294819",
+            "-2253964136358443531770"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "1734021379805203643295",
+            "2025823067259793494492",
+            "-1597926848173583043168",
+            "2016223048238280625792",
+            "542138962133905683759",
+            "879305494437540350029",
+            "24106321453602867849",
+            "-23115421710948790743",
+            "-1394212641438626790639",
+            "900304908524327153009"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-815922170546921752952",
+            "2345528514985286552061",
+            "-730780927736831533616",
+            "1069468118213157740893",
+            "-1997524152172607320568",
+            "-1013146875657159489822",
+            "-1973113169762410936751",
+            "1200927779840472350073",
+            "-1766824844262595417616",
+            "525685124341004293754"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-170913940873623401775282",
+            "179981267419149838084057",
+            "-370082002746130953305180",
+            "-131843472762558045740043",
+            "-552464426412919975259597",
+            "24312389763321025014440",
+            "-117006219956015698142037",
+            "-294656127901854426623108",
+            "211409385267588640410423",
+            "-401701449753764845530816"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "261336340980586457175758",
+            "-200294377526284545507585",
+            "-545096901807334905127689",
+            "374297611040715654546064",
+            "242023592078112962668392",
+            "519439377841947434686615",
+            "-474543347988347187804194",
+            "-394392300302764104998174",
+            "-373120785523247440518771",
+            "588720790387528574187381"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-139459348306311875888174515",
+            "31473852940256986171518891",
+            "-9887972468444270565467209",
+            "93418597022785737089002364",
+            "65646938038849350878907951",
+            "-13454108861792806928065988",
+            "-49131526937539933217982757",
+            "133308858070906575237327213",
+            "152947960978969273942320012",
+            "-99442640752857756581864595"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "2115383285040758640199104",
+            "73985143764269122713862311",
+            "61649530092631838157886499",
+            "8213049092689795695398840",
+            "-80077933562147893493089163",
+            "78551679595459404249914332",
+            "19850268736295064479696418",
+            "124684888502488128271726633",
+            "37465268414151592422174391",
+            "-81920045880589413421302411"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "146919906028535181219026866",
+            "-89543083067520077646503602",
+            "-57021118916205567449809506",
+            "123915826284653894642187831",
+            "45309385836027080396265975",
+            "108158825422372930192837799",
+            "66329707408865417911755413",
+            "-118958587438073608674047710",
+            "-91357950068726260281104280",
+            "-29334802080406579764325519"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "29971765432798246511193470890",
+            "-14358861096333026316517453356",
+            "-20738805649133754102834682386",
+            "7189350037292179268626580107",
+            "28348942101195751039833924545",
+            "23861168485253092769676627300",
+            "16588967227968379293949586116",
+            "-6873327380426872277945057200",
+            "10424009656429165482189206264",
+            "-25621000662753343435110968263"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-39356814765782174445810187495",
+            "-5109629353343354164045003706",
+            "-16869835388192806041216992477",
+            "-37493550780440739680416813425",
+            "14541598222000814429300992064",
+            "-38988737188675992487467355111",
+            "-5094561545550859820454915644",
+            "-12905499536852970749799435474",
+            "-21653278306908362352193117144",
+            "-993708667489234042402387802"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "92415685774358925610111866673",
+            "9229208403038084890901114521746",
+            "-1766812394389966971613945075397",
+            "-414903185248431735357057732437",
+            "128208901508290284695665875545",
+            "-2137916455945877871317699306510",
+            "533853066501738976626621316273",
+            "-8107875934613418676045246063445",
+            "-2240122730023917069540851608907",
+            "-581801766541597034700720337725"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1815876156537294905389859319649",
+            "-4726996253616925415079686314941",
+            "568839304816351244324261819932",
+            "-5163478890877212498625611240707",
+            "-662774202918173654109667847074",
+            "-475889796722743921788914957564",
+            "-2257429926736080379736282434990",
+            "1726653147329710337174488734580",
+            "-9363121838594629328777547224810",
+            "-7533717992682225954740846518262"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-943679878919633167252094145878",
+            "8714088656079526494903956041252",
+            "8411215846063249276308781788948",
+            "3664164290851654853119485847374",
+            "-6596298511313995187171485387486",
+            "-1888023965534996939643588527874",
+            "9495642065507897069829420814146",
+            "-944088385505063048706519524817",
+            "-1682255994436138829648213443423",
+            "-10107759010048745798990652968169"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-1264118898522834382835880961293103",
+            "175335396147539366747199378984306",
+            "-1480903532901286343978092781681499",
+            "-1192391017717251896100729091650720",
+            "1838460835508247778407301651152350",
+            "-218273162958334959288494722385785",
+            "-266094035120591056222534225926678",
+            "2529452524986175024854639128506489",
+            "809296180497156248692989775063626",
+            "-2062841016560545591440823200686963"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "2170563804277303232741531700370893",
+            "-748753118748469760570998958561667",
+            "916018945705575050717700426827152",
+            "-2184472322228724823602690942023783",
+            "931638193487785096171421593175885",
+            "2048640881231787711057765873886397",
+            "-1717558044075129776250633892902155",
+            "-2313071463687411409263290533428648",
+            "-2151217590832164263816790880109487",
+            "233680104831469375662508975501380"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "386287962215958396998754574574732663",
+            "-631240026268303131318958821819142960",
+            "-525325746870165944407752164089651120",
+            "226083291950635135851013581626817238",
+            "-248862800389803494487190616128802768",
+            "-504010349662551275225205203827997278",
+            "105517119082759430552188298694313003",
+            "485997620144390375276743453768158813",
+            "-31031956533367407083197254732570524",
+            "105470167017873351672604929878914316"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-483310365266775246988333151327302729",
+            "632542156756981952900968647179691109",
+            "-199686287544647062245243545386295768",
+            "-346768888815915145525346260667844925",
+            "-208917751619615348174629828212087836",
+            "301058117385654246394042436715253681",
+            "72505593750960021178904279470923063",
+            "-533222490246855780568633480920745903",
+            "218758514092273334444751656476699625",
+            "508751967807589006206145598647069503"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-42741017306389443150668859837924739566",
+            "5211098457938490686418682329098486838",
+            "-129969844595428022866869746117998789324",
+            "-94012368654033828127179346577805729097",
+            "35086650888287131920843468969962483767",
+            "-64547523416114956219242396283441262524",
+            "-78626332557361945510915241951325611430",
+            "130701192029686880255650902394824091256",
+            "12989363125669154968811991286890110074",
+            "-19085080263288132794957946754147030146"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "126277014750196607230171475233679635500",
+            "-134087443068240928447419624897745395408",
+            "85434520441165433374732053355016779518",
+            "61561576435107610545214363658645070358",
+            "-35737267355118855527515802602250082633",
+            "11868126818381274069679858879103809834",
+            "87005374759848073229912677602906854442",
+            "90719075388093079581459445478934240458",
+            "-21323038801308634493038758759215276208",
+            "150665237002876200779610299709080110866"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-100088170739481113599978032714704061913",
+            "87230434305877189656761352602340853350",
+            "150644562584291617352733319990979317142",
+            "2355696344715547537955189102551329553",
+            "158450070722937152732551704806734037987",
+            "148116003453147564228147700395660475174",
+            "-115716266063960239779150802300625543144",
+            "-158188859732442401491344398356371993578",
+            "-99767424729140507618824095329194502037",
+            "62003921486566133988081889333737019283"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-535",
+            "-25514",
+            "12848",
+            "27508",
+            "-25029",
+            "-8130",
+            "-25697"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "21948",
+            "-6803",
+            "12985",
+            "-11290",
+            "21862",
+            "-7890",
+            "7067"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-7148303",
+            "-4971329",
+            "-7052595",
+            "4735130",
+            "5847325",
+            "-617961",
+            "4747637"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-5671537",
+            "4122273",
+            "5239819",
+            "5517428",
+            "7254910",
+            "75359",
+            "2582957"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "1425327362",
+            "-959771469",
+            "934816401",
+            "1185248203",
+            "-959453286",
+            "392425494",
+            "16338144"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1025370776",
+            "-572097770",
+            "1425619841",
+            "-380090147",
+            "-1914141867",
+            "-910187147",
+            "-1412977686"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-434340667",
+            "-1848805050",
+            "743213334",
+            "-324842742",
+            "292670937",
+            "-1900640639",
+            "-933474826"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "239743079568",
+            "-159475972033",
+            "-482663945868",
+            "482237623461",
+            "-423374537953",
+            "-467862671850",
+            "-529681543215"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-9094038950",
+            "163457626005",
+            "-104843767567",
+            "-374539130257",
+            "-470191859293",
+            "80376556424",
+            "384398483083"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1769074858487",
+            "83312732731055",
+            "-41445089577043",
+            "-93201479369899",
+            "-119343260700090",
+            "-68212756586620",
+            "-42877401031196"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "133242907747130",
+            "50022504118929",
+            "2658782627429",
+            "10941894796686",
+            "-80085216865476",
+            "71240008008818",
+            "42520497691986"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "82862936532274",
+            "-110751109078729",
+            "-88636454965649",
+            "89575893784635",
+            "-113792597216820",
+            "16670045303954",
+            "72512396633875"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-18708696209667035",
+            "-29729465986356868",
+            "7302038187080545",
+            "24647819387887401",
+            "-24274097067646472",
+            "8981835671516785",
+            "23227166627732592"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "15050466481240323",
+            "-24020755744281234",
+            "30234678633583819",
+            "-32972592121664681",
+            "-31358688408363897",
+            "14381538054568157",
+            "-21711802576304571"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-2980656018277684855",
+            "2130261546356574299",
+            "-408075321227727159",
+            "8286713698598987178",
+            "2810785916900593847",
+            "4658344625420734801",
+            "-85955245053273680"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-5045733282821378120",
+            "-5160561427270835799",
+            "7274527635644960075",
+            "7450048726897605750",
+            "1192508141799813218",
+            "-6935690825612507890",
+            "4890840298456771251"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-2095147600329911271060",
+            "1933472672626912165692",
+            "-1347315152343358826298",
+            "277387945491924944621",
+            "-769378651588853688675",
+            "1228702994603312892693",
+            "-573226298560349349125"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "949955811513626444069",
+            "2348831874784760844285",
+            "-1005596983583704859249",
+            "-160987643119919574079",
+            "-226311796781453099711",
+            "-777866278163013786942",
+            "-119808887292662530484"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1078706430442981789252",
+            "-830548988474988813900",
+            "252173722280322777802",
+            "686764913963945110534",
+            "-300321502707885602741",
+            "-2341175459570789936839",
+            "1104794651616574646405"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "81847995433813126101147",
+            "-122374233040857342961394",
+            "-86035995805787828514539",
+            "126804988483725019773355",
+            "173993212271803349346646",
+            "-479430025014835876660217",
+            "-107472624962357365868332"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-394077530826954690702589",
+            "-558490301095757507242556",
+            "-521490538576210425997803",
+            "457192467034944298375289",
+            "-514616069321096954519829",
+            "478488130853503189910654",
+            "-577821496311805716252063"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-25044801706278642778701324",
+            "12810834987050168365242160",
+            "31708912178391271824492022",
+            "29246422252469203846996284",
+            "29386096276019991509643500",
+            "49156718482999311956010351",
+            "-146178818417047615416629788"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-113609160992875298395963861",
+            "-24138001744338957650286105",
+            "-67004809197973811498505516",
+            "-102688488214775613024684556",
+            "139561555742232655615687767",
+            "-148612499920539242012129054",
+            "-60197558339677876239964540"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "20195825920279795940729324",
+            "-119334106986421277897505308",
+            "151659610074839765261403354",
+            "-118170878491501590658833395",
+            "50153305614953569413375206",
+            "101283064071515743153346905",
+            "-8052314554905129705402755"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "23932763385345968456601647803",
+            "15652781673781689996402901658",
+            "-25714911140136998783717424112",
+            "-24063108817371419630967954407",
+            "15277383588988737590192970511",
+            "-19221555964336177191153095738",
+            "8525778421472359673700475873"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-29702031637985515415546806692",
+            "-13859102307407213719104673225",
+            "-25067618688733109656139385633",
+            "-24682956579651842668991571498",
+            "13711565425088715334400148186",
+            "-19366222915962629805772891305",
+            "-33545282070076066792477610200"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "1577579865090943191123249075079",
+            "-3525655228029257341271764365663",
+            "-1050223731403232946069011798659",
+            "9184101680926576354398417891029",
+            "1090277651050981596352138640337",
+            "3043069214224348874921088096672",
+            "9123125546671244166682999591141"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "5230742018439185774351510283447",
+            "-3712455999261483372027089691766",
+            "-2990152400744505134513538814664",
+            "7316572061546595364944616807071",
+            "5602827451767519820566081421704",
+            "-9545247122155861772969509882401",
+            "7032145480601368225417113061961"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "2664567996265385909226813270251",
+            "5361353871754409746653525259200",
+            "2256932189357666584876220153595",
+            "-5654300048316572634355784474153",
+            "7631572574661069365537533985027",
+            "-2645945495156645486998250219170",
+            "2020870690070534491429622566455"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "2007985045286249696289152653639706",
+            "255131400056657258321022115794563",
+            "297050787060326049481336858216178",
+            "884106234438982336563352611661346",
+            "-2587734425716673025569123990231730",
+            "1957450916757365456976201890886857",
+            "-368089571172755563470896249505623"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1196387814490529999805386914015829",
+            "1038685272748955982421814739654364",
+            "-1193187060865049033560022310447241",
+            "-1172542451530951320883605317070612",
+            "2109806848803753145281118730755717",
+            "-1123127061355414245949691913443684",
+            "1576461862603874408384234984377650"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-622450033088710447603286926602835722",
+            "647404094951668977726470009898385198",
+            "-226340752111389342737654608992675255",
+            "288035057938916398743906657372237781",
+            "-398948521522675079718064199905049101",
+            "210834703229901379528655619684415338",
+            "-151997602879056519490824338774903819"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "599661339271767435241765491583593057",
+            "-405698515950459980602780956812973563",
+            "-651291563428939025100760124053290332",
+            "-657745631936196547863976296056517471",
+            "-600407163951273202005007415374308699",
+            "-238929313727252416233275164296771369",
+            "558497244260050726702362474833539378"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "47697045732499570659740940309604340801",
+            "49094216418383588916902774794940184651",
+            "107458416864580553906480322750546108468",
+            "-126667437332361921585431044198585948515",
+            "-53936578834290623326458038655308336078",
+            "104673595473921192102042280196362912884",
+            "87274371731568548795590851613278065574"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-57994691774977410461009466091670483898",
+            "-139290829877786761805444219267269803497",
+            "125557441407240749013229104923768296949",
+            "-149517211669454181272347743110271361442",
+            "-61144120978857663327902638374458613629",
+            "98284527118323795858458311216632999342",
+            "-121317326075664247739969129996213981006"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "31724754377471299103514825907465147200",
+            "-68683175193407119606110759218116809794",
+            "51929217413585477513343109403149838340",
+            "-145027261082478191639375237467632529703",
+            "-49770426552521492900748892821647544270",
+            "144672559328394602382003101451778075572",
+            "28345129948566882228213841519097444442"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "19013",
+            "16476",
+            "-20629",
+            "-13807",
+            "-4458",
+            "-14679",
+            "13895",
+            "-6591",
+            "28846",
+            "-910"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "7624",
+            "20461",
+            "18204",
+            "566",
+            "-19039",
+            "17312",
+            "-16104",
+            "-30039",
+            "28542",
+            "-31955"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "630872",
+            "-3149478",
+            "7212148",
+            "-5625858",
+            "4187377",
+            "-7592605",
+            "6361212",
+            "1360513",
+            "5585631",
+            "-1637645"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "1903211",
+            "81573",
+            "7930229",
+            "673455",
+            "-5195870",
+            "5653300",
+            "756828",
+            "2999469",
+            "-5710166",
+            "5301041"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-1853757120",
+            "1047124920",
+            "560213792",
+            "-566455513",
+            "1050311723",
+            "1598909497",
+            "-1861461622",
+            "1007453606",
+            "186779504",
+            "-563460356"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1383656932",
+            "883468748",
+            "-1292278154",
+            "-903131312",
+            "-1071814745",
+            "-1300060646",
+            "-837242798",
+            "-200746325",
+            "714599424",
+            "-6503541"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-460640621",
+            "1452598777",
+            "-1043355004",
+            "601836614",
+            "-387360778",
+            "-1842549950",
+            "-1286045229",
+            "1011279045",
+            "897922597",
+            "-2075789221"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "268979329564",
+            "516400770279",
+            "-330547198646",
+            "-138555837050",
+            "326543329356",
+            "-135436080610",
+            "-179952442222",
+            "300111077580",
+            "-276023308365",
+            "-379534546872"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-11746421841",
+            "-355409685643",
+            "-176229485899",
+            "-335104431722",
+            "162586810440",
+            "-417505572341",
+            "-327503218560",
+            "-237592706942",
+            "284154570293",
+            "-247114028858"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "81497388294647",
+            "-105089606867026",
+            "-85485884825237",
+            "-84732918845306",
+            "8357901780398",
+            "-59052160981538",
+            "-118765758366224",
+            "76774046345709",
+            "-122272430514436",
+            "89632957325007"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-49023243323234",
+            "-19691482514190",
+            "81020005388878",
+            "-13049232621282",
+            "-64400521716964",
+            "-44556778372970",
+            "-126760338329887",
+            "-95545268831523",
+            "41700405063963",
+            "108419780726149"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-107526557921638",
+            "107593946350906",
+            "15717527533250",
+            "3802176381376",
+            "-95371837173682",
+            "31982300862443",
+            "11262311488441",
+            "-71783765256670",
+            "7543990555883",
+            "6433455104518"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "15370190057594307",
+            "25338659396844360",
+            "-12395415092069244",
+            "-14339801329528901",
+            "-4669889778034838",
+            "-25293753369826946",
+            "-3864029849168179",
+            "-2361320120283656",
+            "-1383816312214096",
+            "34795785958072458"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "3783311029197913",
+            "6715667064352355",
+            "20491322855651616",
+            "-18892806810730476",
+            "-29298088411781325",
+            "-18004762134322096",
+            "-6287459475318319",
+            "-28607277114462589",
+            "-21423194975668894",
+            "-34665915056050672"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-3801841962478573811",
+            "-7405589676368624854",
+            "8312654202555889027",
+            "-8469153922901405899",
+            "9033342105804955460",
+            "353262572711500661",
+            "-3106215324351420666",
+            "831047326133816380",
+            "8150006237115616315",
+            "-4283399264798386963"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-8901584431726569509",
+            "3464311601859524575",
+            "-5869426028134580414",
+            "-2688169706060456011",
+            "2295784898660947844",
+            "8661201927268530958",
+            "-1291556588632435723",
+            "7508890651106821342",
+            "-552620568374393213",
+            "-2414552209269209516"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "1268774911788677821466",
+            "-1468173730086716488054",
+            "2333556357660662925914",
+            "1125086588336622451903",
+            "-1307938906371302432016",
+            "-1335315025974812297181",
+            "-1770591667023996442654",
+            "-1618151743678349641488",
+            "-1953870570892738899528",
+            "-1420698506208780653516"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "97145774678299519394",
+            "1227557973951035235823",
+            "98483114001898133318",
+            "-241607937190185623252",
+            "-1824624845562727979837",
+            "898642316179641750374",
+            "-917227455439067325819",
+            "1391756574048152055440",
+            "-2338483366275639941073",
+            "-236784562298273813783"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "1385658483208773656383",
+            "988352401637819464904",
+            "-834024798417789410134",
+            "253470261013770477298",
+            "1027824244823037033941",
+            "-295271284578270208942",
+            "2092749933699556157881",
+            "-212909696600914004341",
+            "430816063369429652642",
+            "-2118940369361998675417"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-72495282431672509310211",
+            "-260808002784871326009195",
+            "367911180219297507096976",
+            "475567389581679354296412",
+            "283347954325215180669562",
+            "-247672665222729098007990",
+            "212556445759539758862792",
+            "95137611300085883049338",
+            "-253373421791320529250788",
+            "222529485178868323798812"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-198192142423639484697068",
+            "177859535515358311773279",
+            "-558532822597349100035295",
+            "-340243398464679108564091",
+            "265469123993517923432529",
+            "345718000847558318478738",
+            "378018566902132428188504",
+            "105885046930552921000134",
+            "-160893570768315261756622",
+            "-387610702742418384935858"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-144968858380916871672160248",
+            "52324015489202530187648533",
+            "-60539725259061399057418114",
+            "5951149345478800610815784",
+            "57415730864791909115606042",
+            "-52801875272942350510376204",
+            "134592231723492901663336639",
+            "-140309028873563970501229424",
+            "-108507437605812788817593167",
+            "34131217432985750294482026"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "101925561886985522713167496",
+            "64068475454853961876718594",
+            "52868179913899871904937921",
+            "47228119380503897884367815",
+            "-46788993674462462832287570",
+            "-104610259825858116764781983",
+            "32354527810168930868473679",
+            "45056639449462594887058208",
+            "-135796831562998221931983419",
+            "-118863104811282298311979720"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-144935023148179749058126736",
+            "48227314826112309666538622",
+            "82605524173281841359546240",
+            "138426913971094444509629819",
+            "133303171716402294169615293",
+            "-142698067685030493395426562",
+            "-16118795484757783951829634",
+            "-127345535900813079037928402",
+            "31052925027573527366876496",
+            "6085416657630733491512226"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-35364635610587093818111370252",
+            "4086803468490861639250374276",
+            "-1753919885518013415897000208",
+            "26862553012392198245742349664",
+            "10669372287960510828219956384",
+            "-1007097557406650376921479141",
+            "-1580693773196331906210657222",
+            "25812917857711965549650468024",
+            "18724993038796357154863599325",
+            "-24508123889671650819840888793"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-18902279341581100470708119631",
+            "-36522283854701472155628358904",
+            "39532617789123607497080494567",
+            "27117584983573687565416505488",
+            "21636499255472286057566042509",
+            "18733621370093464247720680288",
+            "-39480534968471587456182921828",
+            "-37721391776569371748588571168",
+            "35468843832781032242301509843",
+            "-16165717202124486435967229402"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-4522816763549834982990574959364",
+            "1373820747630420236782319267397",
+            "-5973300398871900188157345915779",
+            "-3481250147289716375798626146235",
+            "-1694496501940066046436069391978",
+            "-7462383927751567171069774651591",
+            "8912249954123145763780574884843",
+            "9381633652197120079484670939176",
+            "-1098878014354252629151676845724",
+            "-2173222031454195847214087975766"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "8975610890580938533873895882166",
+            "6181193231273971804885197385850",
+            "441052995054523394237285556116",
+            "-8766763319719487961533692298280",
+            "-842714990971736191972834107071",
+            "6840968783732306639312843567376",
+            "3841095572745602224231284364820",
+            "9467751369945105582701807843797",
+            "-5351035191765711075838610630694",
+            "-5308772832124983511335493031630"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "9347392750318791423094898319330",
+            "4183549304829613232393877274325",
+            "-2300378611286319596549279057011",
+            "-6937551197751554358853541320617",
+            "4564202808321596383974663617261",
+            "-5190481232725371819825118580382",
+            "1822212646901438322631749853066",
+            "1485920178528983047791595729329",
+            "-1229068069521960212083202938332",
+            "237689578861440280652794100924"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "115106178081535021784505745982211",
+            "2419329305084042487339107025104645",
+            "-912291369331954853157175763841502",
+            "-900563205408184843681209632304310",
+            "-2409941993002043200510899606638580",
+            "207954557233582697210944508650947",
+            "2523362759243854886179529293350534",
+            "-1209388681794143621587165255741373",
+            "259026751367118542127035660878113",
+            "1003660757951177805123793137984087"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "1364136601439660016967396025012011",
+            "-2319775878890049929548330062341070",
+            "-1449285195280197573540980195382578",
+            "107888626176858249017005749097741",
+            "-2047490557041291301727159113491968",
+            "-879558452744622993459196325149602",
+            "-1386527790084045558776796472278147",
+            "133271536923889936499750446388741",
+            "-2505010338752155970620613001087615",
+            "666690558706649829294657481518697"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "647827261339309139672072271604218401",
+            "-258811588602379592317988185007366025",
+            "19118764232643136124664526632131448",
+            "-342358790376296733951169651683991194",
+            "323251023333708738421455258993841741",
+            "379547465547461884608218077364934561",
+            "-377241696764273343171368191818828042",
+            "-474120316689176518053512200120507400",
+            "572902079758457973787032998836436243",
+            "-170695774138811733345401262184630875"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "305708404293242487627320811047392371",
+            "-531504659332015189310098462346024467",
+            "38731992465228164632556252163535297",
+            "663006990236245838431214438209595319",
+            "-118885733274822068648133582054828225",
+            "-144829449643189775388924778145388908",
+            "472872218845792265593284840437270542",
+            "-310671243825518439944530367781619370",
+            "100999780786464624037618413063970122",
+            "-140342096347493988777682079106141362"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "32980532793889149400753838532332705439",
+            "-164344442015956226607797018737685311450",
+            "-141620964137040177729886345057833620325",
+            "129406137990155072039956432976677981821",
+            "110860822470333733093650034824108278208",
+            "-55257524399499752297571098737083027353",
+            "21667380566056781477986443622263599784",
+            "-139527484497317282946286760556136131579",
+            "2993193228803848771670718010890478039",
+            "-17314058829458659076458760524200313761"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "136785414456456523483802852002988078554",
+            "158300488297788966764944523807878924712",
+            "-139182135932402606222753976139330995413",
+            "-65945426315720446378481044342262546327",
+            "150507222652197288662681358385272850559",
+            "103871248288732570387585721016195272008",
+            "71995969230355239685697947093141718491",
+            "123362031595790252645978327397128737346",
+            "-9750682546488018096795492364860912234",
+            "-154627508798373930419429872814370749557"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "119842907203642501605603704672016604700",
+            "-17268268161596497482469861499561728202",
+            "10844152782523585961703939469633384076",
+            "-62997709460500166787346095846904613371",
+            "88308111493725127276829193825175861897",
+            "51565439967137952485926321683961024024",
+            "47601722115696675135575316380394827453",
+            "-103072164908596700312680385416990765141",
+            "99066862329124132138915843909816085618",
+            "29881936603443354551348559988907820317"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "11695",
+            "-23366",
+            "12098",
+            "-5922",
+            "-24360",
+            "-26490",
+            "-8451"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "8356",
+            "15273",
+            "13350",
+            "13154",
+            "-24436",
+            "17249",
+            "18193"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-7959866",
+            "-4584859",
+            "-1849516",
+            "-5743547",
+            "-6549993",
+            "-2265282",
+            "-7704024"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "7862991",
+            "1982106",
+            "3725606",
+            "791227",
+            "4292112",
+            "-7820825",
+            "-463905"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "393711805",
+            "722994374",
+            "-1096064610",
+            "529484077",
+            "1342578673",
+            "866470525",
+            "205716816"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-791379162",
+            "-1640580986",
+            "762691364",
+            "1995469295",
+            "-421319712",
+            "1271534201",
+            "1429378995"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "302750528",
+            "-120846658",
+            "796589794",
+            "959422940",
+            "1637146863",
+            "-2105168751",
+            "959083607"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "369772287616",
+            "151347559746",
+            "310899836472",
+            "-72176434390",
+            "-96924947011",
+            "-228897695327",
+            "-200002009593"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-457074057554",
+            "-375995209273",
+            "316810959776",
+            "232806094204",
+            "494582201448",
+            "527142467595",
+            "429029914811"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-2749358243231",
+            "-16732337474393",
+            "-41981779092256",
+            "-31047429629139",
+            "85922781452400",
+            "-110248875388850",
+            "14775617632206"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-89773154375777",
+            "108323854266884",
+            "-25662671385030",
+            "43022064801421",
+            "136623279644254",
+            "59073010679225",
+            "-68467564739828"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-72813466497130",
+            "71140769259291",
+            "-80317050256498",
+            "-54904445398065",
+            "58067517976047",
+            "140251327559406",
+            "-19836322904046"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "32549494969775706",
+            "-440355605481950",
+            "11791444108889307",
+            "12475855272248987",
+            "-2784519143284935",
+            "32466210616699342",
+            "27675300306271448"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-2902857186001800",
+            "23303428815434999",
+            "22978749219565527",
+            "18265376949989171",
+            "31457574635371153",
+            "-7781184835189876",
+            "-14565659292923040"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-430513575868348426",
+            "-8790392468356940146",
+            "-4699312496566636040",
+            "-3993022130065929131",
+            "-4879440303771900917",
+            "-6552655591468880275",
+            "1746806308890936437"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-721862386033128083",
+            "-4475260144423557773",
+            "-2570981490582934517",
+            "-7711961915298512607",
+            "-4822749646811070045",
+            "5836898402712974203",
+            "1646816636258502903"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-196690994284317985909",
+            "-1104147372526288843337",
+            "-1412688911560211238060",
+            "2156517522260848540623",
+            "1969864666336817403192",
+            "-424864910108053276903",
+            "344362455866908271873"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "1977829057280933277101",
+            "-1853366049465512956083",
+            "2335230945236060536049",
+            "-1075731044607594025734",
+            "1859892436977688408623",
+            "-315861879615022790690",
+            "-406105163821827431877"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "417852109818843056255",
+            "1752679773930708218188",
+            "-1943388216652054599366",
+            "-714786275398685414265",
+            "585659101400025508462",
+            "-2096928276954243783388",
+            "-5439936016422902751"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-371691511788058976981831",
+            "468331324108156702082525",
+            "-306216617268143895601700",
+            "585404166947186691975366",
+            "288996272789179095805972",
+            "404343176683141658586544",
+            "191866472956788824823964"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "315327112788086641525519",
+            "196752770980596165008391",
+            "-572486369362609381413082",
+            "157070379247532783973389",
+            "526043469975661293749107",
+            "-332418593565470406233968",
+            "-456053081488540795834833"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-32587082314199631587299920",
+            "133861603238939464220682787",
+            "-83661874551945050110053855",
+            "88800710199138738791487458",
+            "-148303404783980031631890281",
+            "-92723105772899258187193672",
+            "8503332868372431603113821"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-19332793743633931994349116",
+            "-96326365761369957850035834",
+            "-131011081713605299896434887",
+            "4272403363996546916496923",
+            "-136751423492618686805500248",
+            "151616186823003014418871731",
+            "-1671600429980516271659907"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-140178641895610194038703824",
+            "-101032891978279919640123306",
+            "81660784678037787563074285",
+            "39964525581604073164209878",
+            "-27374188725798306962666133",
+            "30132624370004367295733964",
+            "-24808854061327935626508114"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-18346621296739073329815384276",
+            "8129555162832277260205547644",
+            "36653480194016965780646040547",
+            "30413178297188904152302651584",
+            "-3894577824082035416332316895",
+            "-36007834468089853892594019849",
+            "-15800624696484429406311297074"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "22023236505727198042054799637",
+            "25162398511869781136612460948",
+            "-27627237437030757915583163269",
+            "2399929206249180782631036076",
+            "-28662435722237160171923737772",
+            "35814219127521949420744820118",
+            "-19892289081278817747121556433"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-5833740800009897266898626041467",
+            "-33244429845422337192185559070",
+            "8607576650630464089887525333430",
+            "-2116847884082967129690884147276",
+            "-1688186241717588214733687423048",
+            "8336863793337204216701546448411",
+            "-7776559106309167653701679757019"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "7677666198324057071013895406917",
+            "8312123351615753410594523358329",
+            "-9172950392603423861689575769225",
+            "3983044414175939736030503134666",
+            "8631672985863441241981992212823",
+            "-5092486822105432034536308372124",
+            "-9513508243773819164622502221862"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "5586571833059467170325581422044",
+            "580606001351331867421453366092",
+            "5459008382235569678361699614142",
+            "5167556446031629490226831649463",
+            "5538049048887584844681848848766",
+            "6285520963756397613259958855517",
+            "-6303661112625789803890168279755"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "2026431296682674250857553865416840",
+            "2156728720055476065414283902556465",
+            "-2113626044266072567032924427618725",
+            "1577105749037891124349723340806308",
+            "2341510691513285685324249715992360",
+            "1152985290466370765853992549550372",
+            "2257606784668378735530248971238219"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1909890587318622197067905430014808",
+            "287541947253583018938971686094227",
+            "2133113520020164331925609988632397",
+            "877412966607662750851739644288192",
+            "-1354720000623937488986892371455979",
+            "26739837041119641398066396760126",
+            "-188838172981149861365490159579856"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-474782395129122353298074619574123333",
+            "475246432403218151819336842343219369",
+            "-173617778400150860563781368814715566",
+            "-185846112444193317541092054250459789",
+            "530657363533857853032769680847569540",
+            "-624548224136809783035837658411029096",
+            "16767939677460215918277184102392562"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-8794439856560424693485039347212630",
+            "-74573922642298735959395658944476425",
+            "50121637564100226023942304740008235",
+            "100186315919309941191487123083722961",
+            "-656685831019371144979565967330203175",
+            "598127327210536287666688394187681062",
+            "469011031931868490962046889570742506"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "132571096983923187471923107191482189670",
+            "159416816456247505951844178439417096312",
+            "-46858620574250494092983344457915710534",
+            "-32188937023483923298551233297633219014",
+            "23399433488123673778344704553087116420",
+            "49411831195906345612909906764377215656",
+            "-120433578968253846639971258003920226422"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-79060725002651426090157731360159425117",
+            "23590752860694088349229816117245792147",
+            "122068332660626412558932914786262751393",
+            "58567440788739372959622438841791498887",
+            "-151757958624575453818610406998254514825",
+            "-46670330230498566773973502369131285704",
+            "-67623110355947762123153960505167699960"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "19323325648279215563974499442806584258",
+            "160743730096300729908285974982614804782",
+            "98979412153227101016992388437605176538",
+            "169051965254068352712380666347973917472",
+            "33410045474491953996554951161727411456",
+            "87639385406550804599722815989154690961",
+            "-127263625338574842387718381281492249303"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "30246",
+            "4750",
+            "-19198",
+            "21754",
+            "25255",
+            "-32398",
+            "-217",
+            "32432",
+            "7160",
+            "2502"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-22430",
+            "3984",
+            "2068",
+            "-17474",
+            "-3604",
+            "15015",
+            "-28407",
+            "13805",
+            "3348",
+            "31779"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "2080492",
+            "-5892388",
+            "-4427617",
+            "-1926570",
+            "7753036",
+            "8027624",
+            "-2138635",
+            "-7355969",
+            "-5207656",
+            "565924"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-7581394",
+            "-1145315",
+            "-5530746",
+            "-1343136",
+            "-1965882",
+            "3388545",
+            "5063475",
+            "-4724789",
+            "-5369747",
+            "-4555562"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "12542417",
+            "1259749219",
+            "-475800794",
+            "2133922027",
+            "584606746",
+            "1701844663",
+            "1570293589",
+            "93985663",
+            "-1590446304",
+            "38383855"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "1335818368",
+            "1814707665",
+            "-1447700920",
+            "-13690005",
+            "863389254",
+            "-1088930392",
+            "2068779612",
+            "-431001990",
+            "-133595395",
+            "-687033383"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "1433448420",
+            "-2037717833",
+            "-681228088",
+            "566270991",
+            "1714830138",
+            "1383333290",
+            "-870073230",
+            "1018390804",
+            "-1658508550",
+            "-1107870810"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-183566186766",
+            "22430286822",
+            "-216962582571",
+            "-466722944551",
+            "-27556463961",
+            "546569939868",
+            "-276694721505",
+            "-48879453774",
+            "260500893747",
+            "-442983780322"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-537420559602",
+            "-20381317872",
+            "-238654812999",
+            "162311255965",
+            "-380528677930",
+            "460248798746",
+            "-470403136829",
+            "41925493634",
+            "118729323873",
+            "152145641309"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "19115429868946",
+            "13475764429133",
+            "-134296438665814",
+            "106743314332396",
+            "-53710936617611",
+            "55826528957975",
+            "-6112400391073",
+            "29771890922370",
+            "12948136568295",
+            "70280112187588"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "75635401406293",
+            "29801992058998",
+            "-104075542445723",
+            "96149637189488",
+            "-92247279012539",
+            "-15060636948068",
+            "-56990909320532",
+            "-42148465517830",
+            "4636483585350",
+            "36352680284144"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "109325942422503",
+            "89437875709036",
+            "-69763342392522",
+            "-109570939746784",
+            "-51557696910336",
+            "106618186702949",
+            "-52418370534901",
+            "-41131112323835",
+            "97009168063041",
+            "15052093903552"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-12596475918000306",
+            "-10440190089661389",
+            "15930071291826631",
+            "-14154914571137080",
+            "-20679980275492503",
+            "-5555224487780486",
+            "19901627911606993",
+            "2179801741655552",
+            "32519637793270999",
+            "-12190525011728478"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-21675467468906896",
+            "14317453815507613",
+            "-19766776814356123",
+            "-15076731200801441",
+            "-22431380700345243",
+            "-6473570686950262",
+            "-8373384674763888",
+            "-20901722770742947",
+            "5175325370040911",
+            "24214606141441984"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-6586995582558976488",
+            "7822902623338681773",
+            "6785324312631495786",
+            "533690903523561420",
+            "-6270244316286526849",
+            "171361209170531205",
+            "-8970550092894953161",
+            "5694205347266685846",
+            "5128833225945547275",
+            "-6526220735860764396"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "1525290815331568444",
+            "8786542072042599843",
+            "3211936611572870417",
+            "-1018796928748300854",
+            "-5419478240797376370",
+            "2820144392948109276",
+            "-8097542825869754965",
+            "-1815427842766140431",
+            "7519915858501526150",
+            "6718055695379996173"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "1633852390483743207506",
+            "-1111545040340809206837",
+            "244724811785233487882",
+            "-693891632794029583107",
+            "2048439746612005438484",
+            "-2139374266382006896259",
+            "824431366346499761550",
+            "-2178751263030852933012",
+            "-1927196521844590849943",
+            "988141071239138916737"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "2257309169850384513902",
+            "-574793864866852081089",
+            "-1948188754190708684653",
+            "-1683360294804356257358",
+            "1047989496843790848714",
+            "-1920909202759912391700",
+            "-524370318736298792785",
+            "-2270280452792278665082",
+            "-1977443531683898225523",
+            "996281040520873028206"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1794161457257457767507",
+            "-579049890249855445335",
+            "-1877229817480332151412",
+            "1516286188346047750655",
+            "1679089559862183244805",
+            "1899758075679851870084",
+            "187132781949105016179",
+            "1867935632353572731320",
+            "-1296841752077650621812",
+            "1300114283317135818081"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "158042936756450218355580",
+            "490847485432966130350847",
+            "529866300289645094459968",
+            "-197353870044211794005526",
+            "87425841556036243951489",
+            "373006885981853652482697",
+            "-287764822688722336029390",
+            "-190352821874203518937063",
+            "279464221696482690177922",
+            "456648278931495677210040"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "62449586174442498659851",
+            "91984117770625787970323",
+            "-386017533006981165958695",
+            "-252962587854796012161774",
+            "-41590821064900391076386",
+            "-482091979386779184677156",
+            "91503772892342811308019",
+            "-283504385220883938631335",
+            "68755932698885547223664",
+            "-379651999761873616235097"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-126087576931161225003970774",
+            "127239018834369984513800503",
+            "70141064110265604854079590",
+            "154422224583900250448650270",
+            "77231480791207775951166590",
+            "99492320089316975380000682",
+            "-153576307293383539264890034",
+            "-116341636204244696735578596",
+            "-15786876758860723008109946",
+            "-34149255348815753839077980"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "143153832119590577378917526",
+            "-23514921785325046830312515",
+            "-77172712985345097430419628",
+            "-14243153727807957632994400",
+            "-58696496713845538380317529",
+            "69538635574941878208722809",
+            "-26166185622705445056596537",
+            "-62690164472796028840316596",
+            "21101480017403262376196510",
+            "-26276281990810217165918990"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-40333807166719138901605362",
+            "-37987667593309107331951459",
+            "113271936526201411790724188",
+            "-45961181592872567354204492",
+            "-55914627445218102108646757",
+            "-104465866552974080878275370",
+            "123873735196193599773252021",
+            "-69984072371083501932162355",
+            "60408753566988145951843027",
+            "84960611400280128174752202"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "36246979996368740974728708372",
+            "17919686541827178802317575872",
+            "26685274428819984663215559869",
+            "-26830140388908281565789341267",
+            "-2931873170900073034034488176",
+            "33796846013743474981760651067",
+            "-28384722395594446851545817635",
+            "-22654753522493569083372946875",
+            "-32663458189555845828697337353",
+            "-35127901885613742916339593277"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "20660444307893572140115730346",
+            "-3785075277916670195560917943",
+            "505119675982960330877221727",
+            "6140046975369405521925948823",
+            "4736562894917206280220185620",
+            "34734639626599117839339792477",
+            "-37238839274684550289419840966",
+            "-21192307666520898522344443325",
+            "-25731253359857977263230009203",
+            "9784873768788010897306473642"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-5228112207960439532634245527389",
+            "5167964205023544904856267703266",
+            "-3153830415354733573037966725748",
+            "7450883686345014593203587360908",
+            "-6190984559160100747667669453446",
+            "9158379943043982798000435871289",
+            "-5143633836669582578660334205312",
+            "3857866493487182226354931788449",
+            "-1995529191506094552567237788773",
+            "-1939757494685073234017884338772"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "2983226908002458413591863312573",
+            "-532752777081931294958349500501",
+            "1691976820460575299172383596135",
+            "9026960176016243081320400536118",
+            "-6149275498377603931086308362289",
+            "1877762879546909895671695219561",
+            "-5543502816818045577508158461912",
+            "-9721729867149888495285508938337",
+            "-8920180100768743706171922537830",
+            "-5169979352443901899206068056740"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "8702135094804887254191507204010",
+            "7975177441141361458695114972857",
+            "4215501985734737238170439787178",
+            "6544777372478132898809386710105",
+            "2765576461694941619080416997749",
+            "-1508947428389258215537026450679",
+            "5110812407105601388124457686907",
+            "75095474631054730513072856363",
+            "-4279937210566724408068745471134",
+            "-4330630809933126585531848495211"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1504559618154883384814296581669162",
+            "-773390196196490039153882388890703",
+            "-1897752906170923065951917060777094",
+            "142958130493415644926376088950273",
+            "-779836256990434274653725306832085",
+            "-1753325836605413516914447603525700",
+            "-1542378627150307800859873587627455",
+            "-1939970582128933270354383457813584",
+            "-1411234208839859424724556632640107",
+            "-2126244899381031545661748447926471"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1307160371865080630115651820675170",
+            "456003844170818774621646351482357",
+            "631068765196513436260956185922434",
+            "-1965243043509431543877873260642837",
+            "1924094736312307345032078234417418",
+            "-1875239514046642535329927412930852",
+            "928252274243472509987779134671194",
+            "978995848734671073561475467379020",
+            "-2564475835488959647742888457890405",
+            "-570152793522093518749573568631120"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "480389537893603636983558781276270114",
+            "-322627233363211525549283706077204323",
+            "-175773029789054851482330028411371014",
+            "244330382338385690861905900080229456",
+            "-392607940616669928826392153052775593",
+            "-107267728131376637957275222403768541",
+            "-545143790608966683829710014599747249",
+            "364869469591341322932716899752758323",
+            "394858084329837561588098411142592437",
+            "-610840159277877166690985420617619153"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-192669740281245461265542023292550989",
+            "-12603685317720603777562699557006198",
+            "187283379151228677500952715809692379",
+            "513546955692369120286734180331431234",
+            "-482110749762971834845638869557855315",
+            "91471688959029993916111990585487772",
+            "220809860807910859636036119386009639",
+            "652850877928760022088469019476234455",
+            "111668879690942453024321503787659219",
+            "73373210253636230671660785291826427"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "44996590607819977856286386900323601897",
+            "-68502769785321582579070708156248378078",
+            "114506708080602131052528148832194410156",
+            "-83232569015900232819092415317550976561",
+            "-97716408046524726095696560404660126718",
+            "32040614956823782468855275605877008019",
+            "54753377444348733160915719409913741068",
+            "146560799167787084071378433728834118133",
+            "-46659169011613354044672675978630700660",
+            "-18373617990019061846229769982316099682"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "67505677281734524965060455064837775711",
+            "-28713223195193703313300117213126061466",
+            "114670528108596039585803474305454053759",
+            "-109328515397473152172020766238706523521",
+            "-57747808164476452164582125724287659511",
+            "-115039911119596976728830311034338189054",
+            "90118785065736856720547824894744092178",
+            "-89659383715739235835540012679951558145",
+            "35536568601909522172926452268384931891",
+            "-133420441752119085196081353415527833003"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-8627303427894096841021728602542914202",
+            "-157430702291942847264537187308979825421",
+            "117495445521490567278694563084720233793",
+            "60657311361342276211434917495326305014",
+            "-46555659115647145900825174050414002012",
+            "38569430641717011417297628092264146455",
+            "104117949912584083295919883245212540224",
+            "126517129168323484921506157524766805721",
+            "-89395247495161073292219697890210210916",
+            "93186204422346566052866526301928198107"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-28716",
+            "-7461",
+            "5893",
+            "1161",
+            "-17035",
+            "-15574",
+            "4237"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-5295",
+            "-19936",
+            "26923",
+            "2510",
+            "31124",
+            "9353",
+            "-28774"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-3553533",
+            "7148395",
+            "3061750",
+            "-1009165",
+            "4381645",
+            "-1818667",
+            "-5730247"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1477251",
+            "-5800734",
+            "4881425",
+            "-7296550",
+            "-1526219",
+            "659450",
+            "-8356198"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-263873982",
+            "-1472352429",
+            "787206308",
+            "761788112",
+            "-2134545176",
+            "1917444990",
+            "-145750272"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-433966181",
+            "-1545963916",
+            "814812077",
+            "1593862724",
+            "-701833517",
+            "706907239",
+            "-544182188"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "805594080",
+            "-1195679709",
+            "882347557",
+            "-1853839955",
+            "646020133",
+            "1484971204",
+            "1300988856"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "4438499204",
+            "135820810048",
+            "-298194796045",
+            "-167913299917",
+            "-257288837544",
+            "289738017191",
+            "-418170170454"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "17906290840",
+            "125414876927",
+            "-275575916904",
+            "86104924740",
+            "-164076860424",
+            "390339975362",
+            "-408791420440"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-43710596027703",
+            "112836996919727",
+            "-105977145834636",
+            "-112904124134240",
+            "94001656040741",
+            "65183044734455",
+            "-131866661735203"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-90977051496274",
+            "-10596476734437",
+            "61560055683609",
+            "133871189520814",
+            "574814014019",
+            "-38018623146961",
+            "68859957958301"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "26944281635514",
+            "79179252721889",
+            "120097494144511",
+            "-76597855106777",
+            "72037045699022",
+            "127107312225912",
+            "-51976566719455"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "386005180963195",
+            "17673684201876788",
+            "-32819779628129716",
+            "-26142734584760798",
+            "-21415081378865369",
+            "27386071060762895",
+            "-25035767854560405"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "28727678942578358",
+            "3716854509922265",
+            "-30765891503268996",
+            "-35428939049280802",
+            "31032813396558579",
+            "21132833493728424",
+            "18323423600720926"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-7388758649832651560",
+            "4890796540741969531",
+            "-6494554253322577454",
+            "5501693823164720471",
+            "-8097047242278575855",
+            "-6038810107886633396",
+            "-2806525281648999824"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "5813079697128581888",
+            "4678184506304863002",
+            "-456855996733358303",
+            "-2121277677973694726",
+            "-2248413943539318365",
+            "6272381020409312259",
+            "1274893351361543565"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "1566501292452282639110",
+            "847789298808224186659",
+            "-1381494557864830190709",
+            "-389660191209952286951",
+            "-191009516626814231082",
+            "-96116899012721630801",
+            "1868221459505859782879"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1469408605037106208705",
+            "120169104670163997156",
+            "885213643526004078499",
+            "1656898855207585194216",
+            "1868990072585461344802",
+            "-933261783269672604472",
+            "274307016875459849352"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1897801203902311359499",
+            "-1638391687561633219399",
+            "809233679262405093278",
+            "-1426552107698825838210",
+            "-2156155154558519949123",
+            "375648567641874627428",
+            "-83641445825921590041"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-54554428886951413193957",
+            "-422146139334599760408373",
+            "-181537137117950003153154",
+            "565479726514324786737359",
+            "424173391709408884362222",
+            "-426382471585898970959321",
+            "378989549983955912678977"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "332482827287198457983040",
+            "26269806073413044497682",
+            "221494228699356468610270",
+            "-214742700742291383196050",
+            "-485063025095965310275166",
+            "-206922662880263609184941",
+            "-12269499779054433613668"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-2079153784240544880908553",
+            "-108538000488167946369982799",
+            "1337702113416699090369723",
+            "51201975526129906812967855",
+            "95133477031983503992368010",
+            "99560753378096547573231892",
+            "94680749872380168429064405"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-117274302077491880342412021",
+            "133235309557643707264847345",
+            "136425992851983814996249850",
+            "141388968148866833783730856",
+            "136798627735587050485647952",
+            "-19126693188647933554261649",
+            "48757965489875879851975965"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-8286635741174299975868485",
+            "51609959172902905910311173",
+            "-109111488591575501922430237",
+            "-78557116791558802282576538",
+            "-80656136078314199047657721",
+            "-113053382890647628720214794",
+            "-118160471943246572510407260"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-25813553783572142627468328674",
+            "-25977427310356228285126178554",
+            "-23185854957388926526847402942",
+            "-21707862041932744516393466772",
+            "-6421519562360989032505224517",
+            "569338598241172908008799687",
+            "-31836549906764569188743203758"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "37793044332277423478767661867",
+            "20748364196925817852337750787",
+            "392549789846734208161909185",
+            "26297790988587948005258680240",
+            "21586249354499693315147989389",
+            "22288577155677496694419036361",
+            "7345604052580273885509583438"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "9904495274881627900139080199187",
+            "7482827727017947275973272651641",
+            "5495560179907337187222597837509",
+            "-4055007536622081915505189176936",
+            "-9465371767163898005811811099232",
+            "-7331632251430461716607678930512",
+            "7141099397756868279221372659248"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "4169713351778166668753234914191",
+            "5859067319849858664254420101042",
+            "-3754282354044818620313472353105",
+            "2568048327396196032282025142854",
+            "9311607892968389058421897582763",
+            "5977797766778831148484981898236",
+            "-6645965265653096929117017458058"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-5801906969242261868600254035787",
+            "-3336257298871507164494778520275",
+            "-2636835112016446965166101852431",
+            "-6885572586792737606252971841764",
+            "1603907679825345662447639552058",
+            "-6879056187486539384923466888462",
+            "-1790707021629779711475082448245"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1034675525852537762364236474862575",
+            "1711882430155290136112735898040514",
+            "-95920345818202387596164300835452",
+            "-2249141525032950659874767503830521",
+            "-1652263065729457379526511839069047",
+            "69137670296199427018181414584483",
+            "-1688424122016258582447918142136846"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1068287357075711311089614491385749",
+            "-950554951242207664534573691846304",
+            "-1956036184083258810589554917733280",
+            "-896537215061273632781289017902857",
+            "-2536258376526122583868387765640523",
+            "-1792994625229025457039629491080247",
+            "1098569487026708584137620128184550"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "175091933681780709419449962505368809",
+            "452587234945438576574474457110999168",
+            "551706621587607711559343522731729738",
+            "205869761865384575133175619802827356",
+            "421828722673799022708318616794744473",
+            "284099079750264743351804395834429375",
+            "-227206504976250724681610321278346086"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "653258876517803228567267877074483421",
+            "88498832885550519275280233629225185",
+            "-76011122884235139182377528836347729",
+            "-306036684553026363758974895394410014",
+            "507096898707325051267095986297417498",
+            "7244969811766625291666288991989734",
+            "-573031189194397866985879798798669571"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-78707895658241450417047565739309187422",
+            "-109360485739432018575519398965631343423",
+            "-2428635583984270308797486033895962820",
+            "42753163486976599252181370759774389112",
+            "50819284555128794452586051816572760071",
+            "-144242532387316071424543222681940814787",
+            "-36156399915684139601256093989051959166"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "162984880645317865260475927948552198958",
+            "85240365845562965434048060592369612866",
+            "-158859671947676245091304105755240830347",
+            "-106305457729982387569448077637482766080",
+            "112063647808441935089339730874648100522",
+            "-89567526888097522782942387372735070337",
+            "-106346834442166901546233152794820385747"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "49579890402336664433922726332197359945",
+            "73089275354843686046803386174212568686",
+            "-114653905042244944982579349957812151500",
+            "127934754145313083729997784370486712512",
+            "-153960964783146902490613407319798440760",
+            "61282287594543529304868203724834609850",
+            "-95735253186395895118322435677049180510"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "27248",
+            "21926",
+            "22498",
+            "-21109",
+            "-13865",
+            "12084",
+            "-10974",
+            "-7421",
+            "11109",
+            "9914"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "24181",
+            "-10020",
+            "-21281",
+            "21049",
+            "19869",
+            "30238",
+            "16713",
+            "3878",
+            "-3800",
+            "12113"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-2112580",
+            "-5802445",
+            "-78712",
+            "-4496866",
+            "-5053677",
+            "2079868",
+            "6150553",
+            "-4483834",
+            "-8023488",
+            "6603846"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "1291524",
+            "3929930",
+            "-7350947",
+            "1175586",
+            "-3405008",
+            "-5060125",
+            "-8206503",
+            "-1303953",
+            "-6095530",
+            "-2760640"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-2147181345",
+            "-267715231",
+            "1287957097",
+            "604610482",
+            "-1548079692",
+            "770882347",
+            "-1892574560",
+            "-253216281",
+            "-399486830",
+            "1325921155"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "1675268343",
+            "554051112",
+            "-1764481009",
+            "-1789514237",
+            "-175051236",
+            "805874123",
+            "1858433827",
+            "-799955854",
+            "-253242360",
+            "31754547"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "861980915",
+            "-1734894499",
+            "5375232",
+            "-1346978107",
+            "-705850170",
+            "2133236007",
+            "759674577",
+            "1654618415",
+            "-1064241799",
+            "205825786"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-439739629851",
+            "546687129596",
+            "-183193050720",
+            "-19380489102",
+            "-295956872287",
+            "185301861617",
+            "-539483275533",
+            "174936140900",
+            "-216157665368",
+            "-323903818433"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-206902602028",
+            "-25523711337",
+            "329376290572",
+            "-547086561793",
+            "-420540489807",
+            "533906771992",
+            "-69356183432",
+            "-15636527669",
+            "-53282559214",
+            "-290867614525"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-71228950033910",
+            "-25738844726735",
+            "7537765437683",
+            "112839875804433",
+            "97538539157933",
+            "23979706973815",
+            "99830033351181",
+            "-98591255279452",
+            "103618021816339",
+            "18626785293021"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-63486282809713",
+            "101701065580704",
+            "70027248433374",
+            "-55296910174840",
+            "-79310707567811",
+            "71500005915799",
+            "-70746573691925",
+            "88747230972798",
+            "37301558421391",
+            "51854510134738"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-87693600270321",
+            "-12454309380745",
+            "-78257696973308",
+            "80285965677822",
+            "125254246716243",
+            "-117675991977555",
+            "10663316581359",
+            "24255903525594",
+            "81254209111391",
+            "-72113987036299"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "29148599662921759",
+            "19685775734681446",
+            "3333245217877997",
+            "2995823413324396",
+            "5176366661056475",
+            "12337102603147756",
+            "-20628365543902019",
+            "-22418243161590754",
+            "-29067802239769240",
+            "-23018730875097178"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-6176426923194882",
+            "-8149067582880393",
+            "-11749166095143035",
+            "18401921003901093",
+            "-1996265112064578",
+            "10197501871192290",
+            "-32253621163897299",
+            "-20847987719333936",
+            "-10659701083886535",
+            "-8278323833998893"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "5136857587879831048",
+            "-4226582843485454109",
+            "-4390294981117989663",
+            "4994818351801512739",
+            "-8267282930090047916",
+            "3039283087909496060",
+            "-4294189184656602394",
+            "-1041549980371122641",
+            "-3656126503669238317",
+            "9105664583047399153"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "8890565452093789405",
+            "516172262246414024",
+            "1562263263000381143",
+            "5536662842752237787",
+            "-8351947536553705606",
+            "6464103264200892007",
+            "-8790661469847258111",
+            "-6209442610558714656",
+            "435574662691177275",
+            "-4406978950610351547"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-288778447471678108496",
+            "217217798831157629251",
+            "-608988687910814932673",
+            "-2240048037329221421107",
+            "141299084981472691413",
+            "-2195387109122670914754",
+            "192149022897618665404",
+            "667039189961523482717",
+            "-2155889747007763781525",
+            "882853814250996329398"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1751402159298369348560",
+            "1451882575998488934096",
+            "1558108778681655864225",
+            "820503763199449595318",
+            "1623615880148157710724",
+            "-1033406684162170713762",
+            "1930159737768844808076",
+            "-940015457688702701592",
+            "25356638204045989132",
+            "758908114106815880742"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "1716471103883506022556",
+            "1848997534114484794729",
+            "-1094717836300829658295",
+            "1157808654390779724644",
+            "1974960526678787421302",
+            "-791852644564930461068",
+            "1518936215943205538052",
+            "993164305481300592037",
+            "-2190917609278580708946",
+            "1868797478910203743832"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "107250259596579898525274",
+            "36231469749575876646505",
+            "329436325938797206808627",
+            "-280578826478526573569133",
+            "522936041458815155368518",
+            "192285773258090139021578",
+            "-54285825226099356185035",
+            "-147961354881812346695825",
+            "600347692235230272507966",
+            "-353688820541444702551558"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "365491534656040934517771",
+            "-547465713497612713037502",
+            "116041079260709253587380",
+            "119630447821474156181639",
+            "522081240494430350219263",
+            "-474030018965150641292577",
+            "50270333922267433091553",
+            "-24643818854330102652483",
+            "245112928498664779365511",
+            "-121521490474436234210533"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "26593659860935441804716440",
+            "75646833540201692356918621",
+            "142130608477738687479162886",
+            "-83412542972966251252815465",
+            "-151155951774532976658715599",
+            "76154233043964388249198213",
+            "-46853272352917278017975341",
+            "32486164613383508372076408",
+            "-66829799824523238584057062",
+            "95792417909842220506235101"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "63730088675558873019939745",
+            "143548109718428079482639663",
+            "-27693260291056796500091225",
+            "-72348766321107397898380234",
+            "138677152483427690881537874",
+            "58516156596033347413669914",
+            "-107433759528371577596908748",
+            "-78141621760355595625354145",
+            "57091054796595629043991192",
+            "-123271655549899762394142143"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-38477417050477847045725516",
+            "-95888838803508348417829785",
+            "-93625160413861971562012467",
+            "117087970522726692597218305",
+            "5900192167374488789348559",
+            "-24674580893344344193379627",
+            "140998648377023443587850974",
+            "-66274038957839610112342957",
+            "114522458983086926572397822",
+            "101447863265640837765853036"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-55906306943028913908051930",
+            "-26457941841314857799301954576",
+            "21759711912953355183243610182",
+            "19401890234998887013156330687",
+            "-15933074126275848181591729540",
+            "27293805787497685180871118517",
+            "13704320194409856542280625225",
+            "25216830024686698097083609599",
+            "-33566241896826513638448226962",
+            "-6406030957083785743431792377"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "13296281254042409316527886192",
+            "5338707057590130523225415567",
+            "8819455897090902612324273291",
+            "10450242868251146464691055988",
+            "3984408170044123068649964640",
+            "31173755255592998054061179789",
+            "-19050412661501894628584863098",
+            "24644713794131254858462909618",
+            "25395276076315730035919041533",
+            "-10189071623516303081731732147"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-2273158346503188393179401750902",
+            "568616855647559589863643266508",
+            "-4280161068916792824269044568609",
+            "-5682425054604041528765672948927",
+            "2331397238480003292876981020193",
+            "3147988283813769672815110099822",
+            "-7632390386845877850077854509431",
+            "9840981189438011830733564669593",
+            "5308479141310485223873660191378",
+            "118932930720936450551429018688"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "741957915704735308305168907322",
+            "-9762021206507044939694536775050",
+            "-8250703757145415326282325826063",
+            "-1178202689406322490432919428434",
+            "589668456154125344611661965499",
+            "3829126770246418858041384052999",
+            "-4152282954150010503796558904446",
+            "2039899673016755337154781922078",
+            "4170256258494884707646640540653",
+            "8171545852305751294990084682300"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-6962616269350967896600526637270",
+            "-5635024743757428123953554602073",
+            "-3202590711760239385953136717255",
+            "9910046190612877780929595691738",
+            "-3777668629625810788137576085843",
+            "4614409143748488045192848474816",
+            "3938785692531490617884534958248",
+            "5437946965836538604426507932284",
+            "-2545690782086924734038765495913",
+            "616702678702978670407261429789"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-1801934502106174445176675096416695",
+            "-1684879803716391669315795499837023",
+            "1886852307739025680614537841032728",
+            "-2087562101318915052443253201738269",
+            "-289564041897678470358656657180353",
+            "1866276858800525721011618985335357",
+            "1482056324539718482806113984364738",
+            "1379406656844501099108553390445618",
+            "447265769955522315901489749442865",
+            "-1608978736986184364139419097387143"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "959483969247063574072923758413354",
+            "451668303782354619245629961105613",
+            "377534722039293380340942675240743",
+            "122765139177423843232789966201962",
+            "-1265814388489880705763330194364716",
+            "-2555618459717759708270107188671150",
+            "2134470347388221582022629725907413",
+            "104244807033941042242397388741541",
+            "-1292911051966102235019381832015662",
+            "2097241241602887003970802334050941"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "547484906495898917724309897034434982",
+            "187296642318494648639964887497107585",
+            "195010449729172050428280546508810633",
+            "594640376620694404142214250754771517",
+            "616088304853668191153025331194418912",
+            "-379044114359687531180765592396048067",
+            "-486181591192683117321273734043119484",
+            "-334064376837986313664550095407721034",
+            "290486366889561681226507320645315096",
+            "-347474821649864554819987979701785016"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "592690007814067700303615752186669589",
+            "-33347738259818305399177703203880591",
+            "-382688857036730057091326853248715857",
+            "340921527686390758273070134422955102",
+            "-246344114285772780156570736620316760",
+            "-87281520666834090344267836809132652",
+            "537245232536815311537081321427376564",
+            "-493728711296235677585761940344917200",
+            "-377081442166516277552025620543264222",
+            "2719688994411558148859579112050541"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-164648696059184796053055559504365982538",
+            "161879775934873933398660278767200250776",
+            "-29323675327274284512731239968352351966",
+            "22303285730412507017282414138333454321",
+            "37099625903665917412076794028736570815",
+            "-19515107152254927806948029059117184611",
+            "156049973457253022366642762662774258557",
+            "-77241044079010949387624313535905707837",
+            "162149982017470420229015929290220014122",
+            "-165344491933828504403531276296434192122"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "65054581186254715937680104906134532841",
+            "-60909757301953794888323625879233446757",
+            "114089158661229718075421585158899052989",
+            "107965178723789597153491785237079504326",
+            "52247619354554812540456199083118815935",
+            "146474344956199824278086927555782062440",
+            "-39425662132630652063257104658393984944",
+            "93755642240344437526921615504564431807",
+            "-151156356500508700657655200155812034666",
+            "-148636865825500491359194083438394642725"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-130344022211343875529807825195567803183",
+            "40014360360807493456218651568578676997",
+            "135874153827020334699855960081428792617",
+            "120470160757792083712774060829445555003",
+            "-146085386552734357896040388977994934530",
+            "64861335611000408584376324518321919018",
+            "6601563302721623021480813949509601323",
+            "-114901836202314275670611766582662166373",
+            "-106800567037776957620875222943893747296",
+            "-163938636407835983157260355753705412692"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "5622",
+            "-4826",
+            "-23503",
+            "-23236",
+            "-24902",
+            "-27443",
+            "-2505"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-18970",
+            "2792",
+            "9559",
+            "29810",
+            "20493",
+            "-9442",
+            "-2618"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-7747776",
+            "7378127",
+            "5914824",
+            "-7141645",
+            "-462534",
+            "7839055",
+            "-7683709"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "89932",
+            "427016",
+            "6342615",
+            "2937627",
+            "1091159",
+            "216727",
+            "-6413737"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1508343646",
+            "-714398970",
+            "686052863",
+            "2122626443",
+            "-2055953734",
+            "-1487184058",
+            "-327233497"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "46693013",
+            "-185591736",
+            "-1319806396",
+            "1714735002",
+            "-2133394566",
+            "-125069560",
+            "1637341168"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-24918835",
+            "413164011",
+            "-2057922849",
+            "2056580479",
+            "-457426615",
+            "536434479",
+            "-326452423"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "324353482493",
+            "-224184500030",
+            "-290036567736",
+            "-161876501106",
+            "275173073296",
+            "-357018570713",
+            "-297896568716"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-427281617130",
+            "-497367589935",
+            "73820145481",
+            "-254972813415",
+            "-60399473453",
+            "232313028413",
+            "-327759701859"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-54535480365073",
+            "61199946515142",
+            "-9892000149714",
+            "97913524605648",
+            "-85221226793073",
+            "-12545722671788",
+            "-68773831012552"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-126861055347150",
+            "57631355795057",
+            "9186109685969",
+            "-63002972283329",
+            "-58346149025908",
+            "115782687691468",
+            "139849718208364"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-93813523241112",
+            "119065079857991",
+            "-51189229753892",
+            "15646183077138",
+            "59056284602860",
+            "-92669522393207",
+            "2260801424852"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "1036615300047300",
+            "-28482170773083910",
+            "2678469281283015",
+            "25704309658360513",
+            "-12028318394798682",
+            "-17245374013553022",
+            "-16000168049485216"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-33753168399749839",
+            "-27740542059245417",
+            "-8681968124590170",
+            "32278807399459945",
+            "4851945585153961",
+            "14217157851976499",
+            "-8425777401033048"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-2766941444469888895",
+            "1729136918975836660",
+            "-1477513497854998037",
+            "1433050305337001645",
+            "-8233598361619818600",
+            "996969961284840283",
+            "6329284992445382418"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-718575473243337232",
+            "-7375018409999376865",
+            "8283859855328355879",
+            "-4138525682818595012",
+            "-4431847243389561533",
+            "-8084764830291418974",
+            "-3798062670523971527"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-565050117259497763461",
+            "-281556879669412834613",
+            "1705137738829453364168",
+            "-501989730825366398918",
+            "-1424226998969964624602",
+            "-2198902125834852658804",
+            "382157856805932111771"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-303105514873146686437",
+            "-1865673314410343902064",
+            "-1739893893314072462832",
+            "199753736014467107913",
+            "277323073744563230355",
+            "-1117856351014668186043",
+            "2336687337270391579606"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-285211332299958353660",
+            "-1511512024403761316173",
+            "-1568309493555178992532",
+            "1640762644342585812496",
+            "-393568821945143862673",
+            "1038579276245861014094",
+            "-862919179403309677858"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "592348192569902724704840",
+            "-26223245863895268294610",
+            "-338508635422316199664947",
+            "-486334758177154752056666",
+            "-326020328807272087893596",
+            "306693173070483770415643",
+            "-231885616351380144700249"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "147342012649640932496745",
+            "-365590949715364851039873",
+            "-238949724961989355524360",
+            "-20804180906842992000207",
+            "98422693216257524451712",
+            "99343894264393231770703",
+            "5240277192766874192955"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "97249345115448597432142646",
+            "8875195937074039828120226",
+            "1542565882778023075834635",
+            "11772316605485844570699635",
+            "70103217733879642573345968",
+            "-102512631348831506793442303",
+            "-30574606355593089220215659"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "119081714072400880086737325",
+            "-117489206105111877901926548",
+            "128872288419083705914038986",
+            "130736458515802378822355935",
+            "-121556241169153639422095686",
+            "72946657503235618393823866",
+            "-125663836339334145024409857"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-140434705281044213047311138",
+            "-28116089490630336228630975",
+            "103267266309928582800251241",
+            "-109508303463489441066925072",
+            "-37447408761258714290948079",
+            "-48348712769317916926342261",
+            "-140369639756363883929708964"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-9728758914128859071321834279",
+            "33309572658775989148537936017",
+            "-24668259330599725242298254275",
+            "32381561123351360763354670731",
+            "27825802768031595606954105738",
+            "12938216164626281948188731232",
+            "38931244077102153608387825162"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "10382809776177658355331432551",
+            "5168110327267343534258764218",
+            "-3267242975961960293350853731",
+            "18434029108189696411322869404",
+            "35091091873636227218260118505",
+            "21108165552752916181263958022",
+            "3661245166176387607640692752"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "7740613132371376696717495398418",
+            "-7707941191956096108812155008476",
+            "-577241601325072443290122836094",
+            "600167824497967521284893595791",
+            "3272067843081989787146120493930",
+            "-1030145054800689456877805292349",
+            "-3650234270988646330203349279059"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "7680286774308848437433168536793",
+            "6647565199451142367949930365916",
+            "-2999133161143853022620246013872",
+            "8804348153856092578496854714407",
+            "-4952285004482144846215389827781",
+            "-5318507214755856895477853590188",
+            "3057348036468872181146029422349"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "3851649300561843980682812308976",
+            "-5882755312906144634290218148759",
+            "8705902980707176681205204470899",
+            "-7226642038551930671682828321801",
+            "-437315629757074034698382119925",
+            "-7001534311562577654291968216882",
+            "-5286321524483593992727653959615"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-2529962123499222848219244545038324",
+            "395962666703149974739931139610831",
+            "57550468475234656216408472967844",
+            "16338486361970318802019682813255",
+            "-1828779660992541246630598587074810",
+            "-1623395548149665302699111049897215",
+            "-1192580008134980600742905854055406"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "1236661467304426090883166445264402",
+            "-371744076376489711195615032257743",
+            "2054398363329009304225110885914248",
+            "2282254142354611780924986446846003",
+            "1335098616305012872647529459955419",
+            "-2333515867164879976023231383269539",
+            "114078543021693938325750457951500"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "464104358696147522307181233407201836",
+            "528236181487245695215532819816780192",
+            "-49582519618663647096439258967309615",
+            "179810448282591210237518073467287227",
+            "641083621863607981646658838917064705",
+            "-588820141403146033804863670906323881",
+            "-209172547311926173294677521407025768"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "27559909638983289023355911880960481",
+            "-225884217842248648518867386017704678",
+            "179899274801825611492587037729547162",
+            "-645804265572868735329412546507385081",
+            "-182001003702797629603891592835717884",
+            "-196383748000158684449875627005718638",
+            "168348539043763218469877957630344248"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "117107436058598227102369350940703879876",
+            "109883501439450473736462013508285281794",
+            "-101387654941698733435373306911208832174",
+            "120539446302766831060730844387212578122",
+            "36010116834354981290778572290676615335",
+            "70903063265292543021168853120103056998",
+            "110744444106697685492892984617486125322"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "139776289567340167051908795098181848110",
+            "24016711019305619625037775723051412673",
+            "151016471675979627009310128091857241769",
+            "136878330327275804538678500765431659719",
+            "-8175489881016674742304549992648258102",
+            "25386838486740756955389088300546155350",
+            "-58709249744971886150026166430504011518"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-156704989705626079171987231966068215654",
+            "144772934450911580494762210707331833351",
+            "-62369821560188986732383850394596682312",
+            "80247096547059282305305172602938473045",
+            "10035303749721112586071998129175567777",
+            "92716485473901404146553068374146350189",
+            "161388554243068610590428401872902762267"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-17578",
+            "-532",
+            "-29058",
+            "3924",
+            "-4368",
+            "19542",
+            "-9728",
+            "-30043",
+            "8547",
+            "-22611"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-8228",
+            "28186",
+            "27587",
+            "1067",
+            "-14722",
+            "-4268",
+            "19776",
+            "32332",
+            "24421",
+            "-18365"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-1997107",
+            "1099651",
+            "-6097698",
+            "-5444647",
+            "2942038",
+            "6725671",
+            "-1283058",
+            "-2877113",
+            "7395450",
+            "-3898388"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "3550186",
+            "1817302",
+            "2814668",
+            "-2514642",
+            "-135513",
+            "5746437",
+            "5761493",
+            "-2082501",
+            "-2006387",
+            "-32607"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-1478559549",
+            "-1775015351",
+            "1327820002",
+            "2075513820",
+            "535530878",
+            "1674041100",
+            "38780695",
+            "-618881922",
+            "-1971347341",
+            "271829544"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-140335353",
+            "1487240277",
+            "232889439",
+            "-1753621058",
+            "-1650070658",
+            "-1744522644",
+            "445853726",
+            "1638982616",
+            "-2109491538",
+            "-2031080162"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-316389553",
+            "625297203",
+            "-876965397",
+            "-450718571",
+            "943406065",
+            "1809117943",
+            "-1839008717",
+            "-502467791",
+            "-1208669208",
+            "-640379010"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-183748795627",
+            "277910510532",
+            "142611883754",
+            "444329076374",
+            "347496352230",
+            "-310879611982",
+            "-445269624502",
+            "-110940203350",
+            "414382789989",
+            "-299514156855"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "219186267200",
+            "-45309151531",
+            "361805447006",
+            "429814377541",
+            "185451738517",
+            "-10717952791",
+            "-455894293535",
+            "-331945266020",
+            "434683169128",
+            "-427092008228"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-35787970178921",
+            "126503320146241",
+            "-47160409383224",
+            "59076874352542",
+            "-54218256211143",
+            "131768259093096",
+            "-140694998895085",
+            "139329340093293",
+            "2946344253399",
+            "-11352879480875"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-47865164294131",
+            "18996772330627",
+            "82776417283585",
+            "-48069371769128",
+            "121080667078025",
+            "-69990715553720",
+            "-100167359977899",
+            "-117897159499581",
+            "-37425616598022",
+            "-61336459221473"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "24962306301532",
+            "1342549247965",
+            "69591424177484",
+            "20605092392111",
+            "96006776802363",
+            "103132673511125",
+            "-74474597325207",
+            "123891091031891",
+            "64647602566462",
+            "-33817908700461"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-19023776449141216",
+            "-16616980776723277",
+            "3679087078886167",
+            "667623252211990",
+            "28848050655097494",
+            "35358382636133989",
+            "-5528487795104389",
+            "-35321874926701485",
+            "33150704456809249",
+            "-35673170700881073"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "9170335166359676",
+            "-29152376325887866",
+            "27394714588707273",
+            "-18771083725437163",
+            "-22856196754531896",
+            "30245430696795348",
+            "-11339487304468022",
+            "11795737209688712",
+            "17769114732159794",
+            "-31196892450377756"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-7484701409133991703",
+            "446279878089310952",
+            "-9094698745269615531",
+            "-635203874310126070",
+            "-3266768293641445962",
+            "8803106083192049543",
+            "-3447545194464331596",
+            "-74015213448162434",
+            "-3494919863524679499",
+            "4736749662952258658"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-7862885775777111821",
+            "-2258339050710865419",
+            "8640626623613280261",
+            "1725474079450247834",
+            "718161694297051028",
+            "8263883702397073425",
+            "7878929139943328067",
+            "4794616785419809038",
+            "1846785796472594179",
+            "1051708514867826862"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-290275855847029699526",
+            "2324764544272157302715",
+            "-172538644237854396534",
+            "-1015287161642688964086",
+            "1083861575526398116043",
+            "390994735554610775082",
+            "-1196474592645325657445",
+            "1865761624558280233031",
+            "-1180403655805921813793",
+            "530628104087968783213"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "954362787050824981929",
+            "-667973333297772026679",
+            "-1003872443597233202904",
+            "421871596895408794289",
+            "-1020239034548744444861",
+            "718583437320525106540",
+            "-379642667164083852664",
+            "-202130841394416991102",
+            "-1679999734562681423770",
+            "-1700441593695289764413"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-2355756035699002368979",
+            "1788582608362609979068",
+            "976621959978375116033",
+            "551918345212846459001",
+            "2038738001872254112254",
+            "-104241764012041061597",
+            "1815004322628837717025",
+            "-366030095473004039888",
+            "-212188636908643977409",
+            "-2212197067753662412081"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-157029864053237435585940",
+            "580349464450639935626054",
+            "274606111559531706158496",
+            "-271531107176514984144572",
+            "-283271260947610926629996",
+            "-102992963880630057053386",
+            "-151438874243484297766280",
+            "267231765639661935826287",
+            "6412575179562556131083",
+            "-447284995394712155520012"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "82122697814709630556960",
+            "-173674944109113014109917",
+            "-545487196009632239998847",
+            "497347624990809943036106",
+            "-403342273894709674210901",
+            "-408545942001809410098345",
+            "-596472526090909588749503",
+            "-516275782103386775147265",
+            "571758314045638582710655",
+            "-260772039821490889738006"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "151997540068811973892699427",
+            "75525965773514350955041445",
+            "86334400095685499194704768",
+            "-99446476458777770659737285",
+            "-36773561228702583990719229",
+            "114833233963970911445935291",
+            "-53580285648134851613933202",
+            "-107255936095104666984539508",
+            "11411674730798920037840357",
+            "144310491490477735136741915"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "111539513842329174253547474",
+            "134891345910083773681115377",
+            "4883280022715795599398831",
+            "-93475601770663945655266004",
+            "69463434873811940745671423",
+            "123064229528351805775861869",
+            "-150461278409905335393794684",
+            "5582686657272910816484271",
+            "-41856573658852576713916684",
+            "138680097641830104297300861"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-151772029737772929726728403",
+            "35642043181692385372706993",
+            "22083653620753836019287236",
+            "-132784292547512147409668783",
+            "-68828393278683857783700197",
+            "-76581151082179815290212905",
+            "39692809847260519264925858",
+            "100268541173636388889856795",
+            "-84443541640950891574906466",
+            "-56403642052264302619910023"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-28164201707581895339796457396",
+            "18512437657026840590853158320",
+            "-725631091722477987250581723",
+            "12378738266464188323153378043",
+            "1884164162561779726605459613",
+            "27835210450137278935849561805",
+            "-33957093897194026793294927588",
+            "24705712105631658413881985230",
+            "-20752699574766143719562108952",
+            "-25390963174699799711871467610"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-30515239169198036972672053279",
+            "-16714522371588133629312659687",
+            "20089473716367036753654096200",
+            "-26078971629524511775014771330",
+            "17501247017496478629908047849",
+            "-2928637003025782311993801087",
+            "21291695736216764696592795686",
+            "31536521455601627910426594990",
+            "23365283652803300319589541711",
+            "-21352428429043964655987017640"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-1871820427329445630900753275833",
+            "3231762858097800568377706484053",
+            "-7593147127830990171207926819911",
+            "-1328646573449438418929234745911",
+            "7929494242245932071338824047776",
+            "-3966690046585197915967741713421",
+            "-2665638134601027112326964326194",
+            "9493723499691953230194922419753",
+            "-192784910368186052857726341060",
+            "968457352886861403532133623589"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1799593923874731933560217881204",
+            "-4108892863880965470839888494269",
+            "-9288585435136926537350897814111",
+            "5657445890165360156029413482367",
+            "9566538541825565997512161866226",
+            "-8660816252467569077590355298227",
+            "-8872018686223204958393278690220",
+            "2747605566704311318475704614163",
+            "-5898641587743100416084001786515",
+            "-3297823611414655609450786579230"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "3031062632211056878773391959697",
+            "4539157404236993497036298560754",
+            "-89011930344651313281485505381",
+            "6329124475577017732572836547528",
+            "10013839443292860323774055369057",
+            "-1017595829075308067813401189014",
+            "2867701530636012507846076321736",
+            "5049199938873542782410309132373",
+            "-8360248192335873845319826952230",
+            "-8573205393401942663978510341713"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1119504478489389158114715132398217",
+            "-1196861483850018707503891203312216",
+            "1577676525610019362542143335287170",
+            "-828524918774712338254714242796201",
+            "2195372295538823359262046660137804",
+            "-2090592767226916350765817876441046",
+            "1267778113117072331278680263282405",
+            "1899343874503183337053685132483541",
+            "-1547991409572962043085814686518526",
+            "1902230290372078686946188304673897"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "693293489583149990962930554361914",
+            "440506011505476370060528227243171",
+            "-1325383072032732660105399745630800",
+            "917607968990684241788700315356108",
+            "1743560964081196656600523814562949",
+            "-1632234260262962178728375599295415",
+            "-1245497842582367367033445464435845",
+            "-1845276210960982951119705348291628",
+            "-852049233534078951923277630775327",
+            "-1061875438802298101158411197338174"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "662874337110619540246196737572533840",
+            "-646712275200560229865863897728322608",
+            "378197263526199456522160076139313754",
+            "-655367409488714240118815274020981441",
+            "-117363023724417392064609064984193546",
+            "379968998004796503715316062979365386",
+            "581971463075521396879850081437709346",
+            "486590909317284352288769404708849420",
+            "339153058170383739672141264076027632",
+            "368011261202186797510454029551864010"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "611471791995589932000290011287342073",
+            "-238320595795246885451815237904493030",
+            "129885954954066668429052045867305389",
+            "-593881411984223406801058087690854990",
+            "208716878349646304461338632058153168",
+            "294341067601902100416283102151104072",
+            "48828028516396476570316169415440949",
+            "621606094524397617471379378110976418",
+            "-24429954817222979379361646729750851",
+            "174309609239946237584153302119176933"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "38764350872143269830409050492580474774",
+            "-1783556143128429347444249923707622466",
+            "-73796525965417826742774297120572978723",
+            "130398981933491578275105091583750862109",
+            "166691015795481448891415156743596861442",
+            "137059374902539614554733085225910695585",
+            "-52077108529588489247990938655971916018",
+            "91346637728026911154761148563604037923",
+            "33500089399138437200753844820194648392",
+            "-129284621583923871181594899462237973804"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "155321932412457790454079866994618338188",
+            "27813009907673742133398066472232685666",
+            "39272698207745895563541959640721556926",
+            "-122722120683994266643963991712613165291",
+            "-130174397926461656752602397741063391588",
+            "82090308774368119635497681294792018586",
+            "-59839129775569455414641041372080794175",
+            "-24104505156877231797915313008748415591",
+            "-108881215793562491957481808068270876072",
+            "76694630138799355396392615062949976885"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "165547688447137835687250676649889094237",
+            "-163339935116527850906013137286281712121",
+            "-71022445758504577564847869973739799734",
+            "145194176986878243647893329429477538354",
+            "64521796946193208156404560040536806244",
+            "39300237731247748947718794529964696310",
+            "-57247755378136552231880640476421831435",
+            "-46273099289129264614986219324959153834",
+            "-55042141398285192896194894621330572941",
+            "-152341554718452744063215754531967241888"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "18698",
+            "5226",
+            "22280",
+            "556",
+            "8799",
+            "29440",
+            "451"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-23904",
+            "31878",
+            "21043",
+            "7840",
+            "-4061",
+            "973",
+            "-1228"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "2724325",
+            "-4798020",
+            "1908664",
+            "-1806352",
+            "-6499685",
+            "-7280271",
+            "-2257879"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-4011001",
+            "6863924",
+            "-6697732",
+            "-2217484",
+            "1921902",
+            "4699368",
+            "5306140"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-121080047",
+            "1970737025",
+            "177569772",
+            "-675421071",
+            "-412793147",
+            "1025987892",
+            "-1489708604"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-2045004499",
+            "539874307",
+            "-1997427023",
+            "1708485386",
+            "541003597",
+            "-2142889980",
+            "-1571929387"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-1277841834",
+            "369437735",
+            "115131390",
+            "1767292952",
+            "-737609351",
+            "-1060380702",
+            "1495387052"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "197387161100",
+            "-254436366157",
+            "286370837399",
+            "19374886452",
+            "-98965142936",
+            "-44474737185",
+            "443205205790"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-11626723519",
+            "446888208290",
+            "385477838312",
+            "269092016708",
+            "-115706867693",
+            "186662180742",
+            "443402603648"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-101903913010014",
+            "-17846116772043",
+            "-127568468992217",
+            "138034513755219",
+            "-22440181460716",
+            "63908344819874",
+            "71092752187712"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-10514368675102",
+            "18576548276665",
+            "11989898233479",
+            "-94964748076982",
+            "-25130417074236",
+            "-12150591292188",
+            "29627314247145"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-128065487316393",
+            "-51607804296775",
+            "9339364520761",
+            "136634838298533",
+            "-51187175560258",
+            "120582565367596",
+            "-113392529877605"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-6014210637334865",
+            "8607066096461461",
+            "24796296381889212",
+            "-21982938527186312",
+            "22830978621256619",
+            "-26845542068057233",
+            "-1207835268472696"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "3833038092952487",
+            "22879010815516439",
+            "35249449760784991",
+            "-17068864041186991",
+            "-26423231151454707",
+            "-3215687274393412",
+            "-23458194802999605"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "3125947421774122944",
+            "-8281228069349294860",
+            "-8475285212132358531",
+            "-5284781313458999470",
+            "8164712809196122859",
+            "1812915631649193077",
+            "4405900031418971351"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "4032410419461492356",
+            "-3461877490707766960",
+            "-1034170339182657409",
+            "-3390159984142309139",
+            "-8014068544953344330",
+            "-2322898039699171514",
+            "-4866093356373866447"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "746866223721616048807",
+            "-1092102356202329139788",
+            "-1546383537243345586308",
+            "-147480023935690463533",
+            "-1613816966398143873591",
+            "-1366145607207253623527",
+            "-1277834349517264956600"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-779628592648894155798",
+            "833551847015048377342",
+            "2350685094213395401616",
+            "-646450799987582052640",
+            "512671457318054258029",
+            "889600361052664942457",
+            "-1281226453859217419451"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-189794651087146365523",
+            "1730990859510508481978",
+            "84730118226610466482",
+            "-2271149966448749710655",
+            "-457693344089241611732",
+            "220532349183084708104",
+            "667161504208011399088"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-84007006749190537913356",
+            "-246149963758082151700698",
+            "58599320110198036377337",
+            "419084729775542661330649",
+            "-414234870389293282071427",
+            "-525262946081080156584518",
+            "136640685388491267545797"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-119294228731180826417958",
+            "8761229936388021822082",
+            "487012390130773891175460",
+            "-311845427865456314818185",
+            "559648900054229910709345",
+            "139393885514427519830624",
+            "107325890167816018081235"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "17089147827907561627837757",
+            "-94429323577754654200100992",
+            "-4473969571146714207261485",
+            "141782659389643845693155915",
+            "-93892358423502207196670410",
+            "-145554766670598055216282741",
+            "37597111607123600576198257"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-137314033837313163998637",
+            "72449797169876132616353909",
+            "-70161038289080296659386196",
+            "-43462961961878972425525973",
+            "42101201201192529980096415",
+            "-81066870591720969078480725",
+            "90267013220354821004402329"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "117713797316457619471160158",
+            "-70167250212359158255102362",
+            "-11552912460279859107138583",
+            "-29885413874861556795770843",
+            "4421888524449146349137592",
+            "104643958570092735443054627",
+            "50425644614078606322832703"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-30334928923978531796147629088",
+            "27288518731802156666631771366",
+            "13824378221071923474948637231",
+            "4644693338626225213907016302",
+            "11634130769035763812486389217",
+            "7684952376163662025235478239",
+            "-33239678106944925691053930083"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-11528961365515593522622576679",
+            "17133737086982694882922251878",
+            "13646045761710072112428841241",
+            "-11389171932161374967609239435",
+            "4257823513423339355354889471",
+            "-28413425236629841784233274651",
+            "39588941214204919522713866880"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1793447129255862870401371387102",
+            "-4521183177824322286238857131690",
+            "-7015296797702626396918062112070",
+            "-8405199754772912051218827051767",
+            "-5847103288844519371863856372805",
+            "-4986004223819988818141294208919",
+            "1152074979591001029103861569975"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-6349141188765631755738960028456",
+            "-2660433342897400293313691984408",
+            "8110414928660755884872240488403",
+            "-3255774704770639692268716192582",
+            "-5624565439076989481811168712985",
+            "8242854258628200994627074750006",
+            "-632483613318590601476620619686"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1906797773505486015084475661167",
+            "1831532006604140257182189698277",
+            "8658935091181123523496177509351",
+            "4083753014008812543656172710815",
+            "-703732616202838478277623562522",
+            "-6578720101664337167243397144316",
+            "-4231412865947716065230260553709"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "1481793740573226627257507227084301",
+            "1987274798160197648897728495716203",
+            "-2253699799337436551660654810527205",
+            "-1664891149611323142615003737883555",
+            "-812147342830058889078154049835125",
+            "1133591471991032412967586858920266",
+            "373066807517996728955491798981682"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1308500817719514731390159580539356",
+            "-2472263601229082178453567330649492",
+            "806886383595845079530117586003857",
+            "1483914875922945517574389647107123",
+            "-2447637035834905490061551060827517",
+            "-2584877975554282277356767257919022",
+            "1137264392772462522125091324886529"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-202694815737861835748583337438619141",
+            "-567475487177382119320244018738653105",
+            "240870396569462382448904278424042886",
+            "565877554788860310777372508885337792",
+            "-16105501605830402808140696954471983",
+            "-608678904857721513901782286805392766",
+            "89500838347412802178721155164713004"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-92477909222259270232496020800286668",
+            "-253868659242284214237281896032894612",
+            "-571911363839430504720309156797127441",
+            "-630228046136182620305988015405040527",
+            "-145796823877333917115220895453473596",
+            "-544416500996245714271727684022145559",
+            "62108147716790867959393946884530570"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "30196559748662473034683824360446700718",
+            "-145993220223129636351442638734923612124",
+            "-31744581905865529412669045728747964315",
+            "-42689243253906548960505899054623799466",
+            "-3278727297709430585641819106144408715",
+            "-139194553088228069126405551246144950996",
+            "-18860658848674927408656595634636715361"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "14098620813169356460637219212014489435",
+            "-25133450166458778557679753986808320092",
+            "-10717831626644290242659474112607023736",
+            "61898419746062756560226489821314179510",
+            "19338062857571632481395798424446979882",
+            "-80563484496287165417026379634301213058",
+            "38933787489872054716116934865976579056"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-18244737545312093171930563627683906325",
+            "-69817450526652822027490267764416914485",
+            "-161214969184468418385603069469725822547",
+            "132041756256234747183020020523975575773",
+            "-57875703366203729680850430132447338202",
+            "133741584114580239596914015216689701177",
+            "-104202051211600429556397759132978847898"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-25325",
+            "10828",
+            "-13355",
+            "8467",
+            "-14673",
+            "13128",
+            "-11338",
+            "-7976",
+            "-2935",
+            "28799"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-25899",
+            "15808",
+            "5036",
+            "5817",
+            "21295",
+            "6991",
+            "30262",
+            "-29206",
+            "22545",
+            "14281"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "5139488",
+            "-4603800",
+            "-4322358",
+            "5568494",
+            "2927252",
+            "-2770864",
+            "3950127",
+            "-402389",
+            "-1551328",
+            "4307703"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "1273196",
+            "-6043029",
+            "-2124152",
+            "6239939",
+            "486272",
+            "1367187",
+            "-4466034",
+            "-7300079",
+            "1984393",
+            "3056986"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "389208900",
+            "185157703",
+            "-2134187956",
+            "217215497",
+            "-2063962325",
+            "284328714",
+            "-1798439100",
+            "-1345866522",
+            "-44759387",
+            "1568455956"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-1988645525",
+            "793006818",
+            "-251491455",
+            "247123050",
+            "2046505050",
+            "-1728516425",
+            "-435424560",
+            "-308364872",
+            "-1892211705",
+            "-126815551"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-773671966",
+            "1358683392",
+            "-1755115842",
+            "1643669576",
+            "-878872476",
+            "-1809618284",
+            "-1045892795",
+            "-1775853987",
+            "-1403818658",
+            "482449875"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "241099970526",
+            "514993919499",
+            "-496567360082",
+            "502604322768",
+            "-28679500586",
+            "519917943945",
+            "413447742457",
+            "221064262119",
+            "-391763012706",
+            "-181222719579"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "509145597895",
+            "47721898418",
+            "201713081857",
+            "183700101234",
+            "314513786407",
+            "-33405756748",
+            "-532243541621",
+            "-107548030858",
+            "126151420688",
+            "285956004348"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "116275668335152",
+            "122141214739207",
+            "-63430344564549",
+            "-111832080367574",
+            "112155501448142",
+            "19656387304429",
+            "96601821131959",
+            "57545046918456",
+            "107104563639101",
+            "-59870112678790"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "45356471648224",
+            "14779609611199",
+            "-33962279108195",
+            "-98944840181309",
+            "91625511308220",
+            "-110850158882954",
+            "87916556152463",
+            "80467617162481",
+            "93925586648150",
+            "-113372051519249"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "19606854898583",
+            "-55504890190933",
+            "-15147315545027",
+            "40456424289866",
+            "-6247604760181",
+            "-93639696496745",
+            "-84665878592107",
+            "-100053850686284",
+            "16367469522358",
+            "99753371265059"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "24282640953032762",
+            "10091709048435857",
+            "-21974172647175281",
+            "14073179053430222",
+            "34109384946231056",
+            "5226653906582893",
+            "-33819520810414763",
+            "-22931875185563773",
+            "-24002071163260726",
+            "-1715146803898567"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-30659716847381795",
+            "-15836145091878170",
+            "-30830890473364625",
+            "-25423275503125707",
+            "-15385992609067487",
+            "-27328827163588642",
+            "-26765399453321294",
+            "21083654833354492",
+            "-19809350790025647",
+            "-2472980520466774"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "946065176298096897",
+            "520206069634010822",
+            "-2372009740528541735",
+            "-3605114825675420401",
+            "-4132067089809310737",
+            "705141097139262042",
+            "-7256635106430568358",
+            "-8105301464259245359",
+            "599492648809263584",
+            "4365466939375502140"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "8277129333648717097",
+            "-8448354335880876032",
+            "2913319982453504823",
+            "8736245116433424334",
+            "6387017288276853607",
+            "-4081208528945745050",
+            "46515463897966715",
+            "-1495751169585677679",
+            "-6665852657838015529",
+            "1497421528845365759"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "867602266070578599219",
+            "2247746875415261031076",
+            "1934169958376882247885",
+            "-1983965311174074790761",
+            "994286831208120914465",
+            "-2053360980630830501597",
+            "-352322923068724990708",
+            "-275529843474582648430",
+            "117569307044473834744",
+            "-2057119187357998311386"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-983989144481816608421",
+            "-227041553316359265394",
+            "-2338398780830635151885",
+            "-1425288624511611388041",
+            "-1200460011151461084288",
+            "-37860255133068080397",
+            "168815163262920902402",
+            "-1589043880708207878151",
+            "1999861056072554689170",
+            "-1063012832871660661701"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-934629893504979755792",
+            "1330023560256145633360",
+            "471809371618383733343",
+            "2321922756694883212096",
+            "-1145621103195820184004",
+            "432986056770166636184",
+            "527912618672815906694",
+            "-1383597906408951487454",
+            "-1840681975964590350984",
+            "1003431919766458755951"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-295917082043583082770621",
+            "-231343988449013068326631",
+            "5312383381819468949750",
+            "-139157207556611762942758",
+            "-241655483652404463403757",
+            "-512476477719184343596604",
+            "599410803946810481042938",
+            "67078249271685046714164",
+            "409870524862445838681667",
+            "-198986259949659924965247"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-278030681373064235595749",
+            "393693012596137044946808",
+            "113099188405109327204683",
+            "415775568903274402438459",
+            "-42076558072135973151428",
+            "-219378272136268135495431",
+            "-582613624074409026792710",
+            "191194427072366781075751",
+            "-9835457748214809790370",
+            "442340177819471571537141"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-125407723627032808185149189",
+            "-127693057085441986254509099",
+            "143049524315591523657757604",
+            "66172074121241277336416638",
+            "105043654403595832965542333",
+            "88321001961584231003626866",
+            "43148804784313926260363839",
+            "33183564004779395803780452",
+            "-145197017290377600759079842",
+            "119167929014336096222821000"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "119039071826650750074654630",
+            "44751775197664106490961349",
+            "27864406843200354265918393",
+            "15658656552850314500730985",
+            "-8220185761405769037321341",
+            "-83569483397839034074232516",
+            "-15999238535942008358971638",
+            "51943353221378823545548335",
+            "124460320105770697840371762",
+            "96188240730380539592133999"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-23740546198484978874875407",
+            "76636632473705803978381047",
+            "-56311328440996031370265536",
+            "-17243365633279424695862822",
+            "-86114647089246738996946962",
+            "77779733094924174077783969",
+            "-154377399990419141152689913",
+            "-58108558577026614888610010",
+            "32011657995884029676678558",
+            "-85849178273273349349099772"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "38268503482208460783689168776",
+            "415498958203045688529744846",
+            "-5469243189225899466540717631",
+            "-27069951860473838447007129302",
+            "2300819963695030297957821979",
+            "27013864454418030545712879052",
+            "33795947339603501913179687623",
+            "-36867304093401804903210992872",
+            "6652565972383263112675657430",
+            "21252184018008067722844465790"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "20488449679781423383625400864",
+            "28945181199727711698700316165",
+            "9535784301505998719738045771",
+            "-20854663557065706429017034430",
+            "-8496482734431467304571929654",
+            "39529776626237782521053868212",
+            "20282963743253661057874523422",
+            "39168299995389261713687626653",
+            "-2895774161826797358583901454",
+            "27415786790395091445855072594"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "7274372610033635352026015117499",
+            "-9390827953122518631957223639932",
+            "-2203836129140192889937266167905",
+            "4596600214642310892345897847127",
+            "983389576577548413649282888500",
+            "1964366100910371172669585857215",
+            "-4168439792076922929733693975481",
+            "3161618514384549987863457101037",
+            "-7790441321891779327083714442512",
+            "-7686467965572840837470847326318"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "9529887427430029315427723731097",
+            "4336235122315854607986015683663",
+            "-9706927952229870397920454025649",
+            "4499727370692865250957373478909",
+            "-5923209179547863758996820315917",
+            "-2626628904315332468993978106025",
+            "685916912461697873229704570250",
+            "-2838726730006480465962589525945",
+            "2039415276453860859795422056351",
+            "7628942068105897131623029254615"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "126535036553237060539012806767",
+            "-6939129554517585745715778319177",
+            "-7532363115381746277033981571367",
+            "7225306379513685172919544982624",
+            "2081180225728056445004841883009",
+            "6331729734153410979163746567493",
+            "5450234392691226980537948053828",
+            "-3339385174687947070096507831241",
+            "-6555844461598841325077996060011",
+            "9279765521180487317778800396224"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "2228023644252832655241598402411269",
+            "431676701798775425411616196133933",
+            "478260014584588677437872352249827",
+            "-1507262382979849604564706747863757",
+            "-2288462503836870608309925185139281",
+            "49393515288688670462360817470707",
+            "-2579210515571147629042989663475866",
+            "589507628712335934139197077166366",
+            "1989953354399049606431153717140929",
+            "-965660095005622252483438158567899"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1816053260927088757790173812582767",
+            "-2405480415192625953224616207989887",
+            "-2328044646297041482107281923018610",
+            "872987268389289710890938214776251",
+            "643667272856490093246696929059356",
+            "2135937810867387131247124235309141",
+            "-348035586025203024239919226158367",
+            "1200248021086499011518529585918610",
+            "2026575827504135717840864217744664",
+            "-1601765059510950727183880983778417"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "65074355550728584791094991091725510",
+            "655697096560546628903998665713057646",
+            "-526498334932676728875521670389483820",
+            "-660406393116184719913940975963998661",
+            "198613257448920495951253934679652556",
+            "-43713779768026581579783258854735948",
+            "24173852238556377483338028432771484",
+            "-323536680150552559099100992362365559",
+            "216972493779019710120712154999597849",
+            "391545022377807490738436750954965632"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-128389779235154808800484297421230508",
+            "491001613177916842185576260455747135",
+            "-455096123184868191344797938977008190",
+            "-309905590996917800177765369517052165",
+            "349603248086460094879044594595289453",
+            "173998676594990878495537914769309496",
+            "580673547960002977806631871571859052",
+            "304595872857331938936691935503967494",
+            "-309341788503071907892541600073298273",
+            "-445388828339388242207829894494395187"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-116759118799696740987833268006312595507",
+            "-240339187668656185170580579099921345",
+            "-165524819667082744814917029626210589845",
+            "112546391769223555027822608825303955489",
+            "-48744565225770459292490161985480679384",
+            "-61852298378918957927732680541183991273",
+            "-109815699785003222394734941309206527496",
+            "-161541054014304007303713671841308424373",
+            "38232801342956757143716333070831819830",
+            "-82876982197717752657894986064058509459"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "166626734583125926562909642736739648863",
+            "-537561525487190105228039554746904245",
+            "56724835995658384592515263322742572769",
+            "154738535117659769768747849261742373987",
+            "133841382783265265669107205599295242333",
+            "-82481830171752743723415208240895428708",
+            "47401574357301211579457100619126454320",
+            "101450166334449946734731337888765885526",
+            "-9655734409491446777009001964705514650",
+            "-20474375954705981387014226963404713903"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "97166129096528442889485736329241954705",
+            "-20654655171391656743269829102430663484",
+            "-19774868324035406536117826166021377349",
+            "-14595360169302341242602534087132194253",
+            "-77171889502077963021661942682977180668",
+            "-67805573373674104708253116675406619851",
+            "107903125463174461399865488130215282736",
+            "13666179504242759743615908229099766154",
+            "-14923042555307260112864102940917241528",
+            "-93735255641317514829426639104109826896"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-15637",
+            "-21826",
+            "23207",
+            "24299",
+            "9996",
+            "18466",
+            "7067"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-26781",
+            "22205",
+            "-30862",
+            "4191",
+            "22380",
+            "-31397",
+            "-16113"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-5606512",
+            "-841159",
+            "-1722865",
+            "-7412802",
+            "3421093",
+            "1404395",
+            "1763678"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-4174091",
+            "-6797376",
+            "-6735178",
+            "3654374",
+            "433672",
+            "1994930",
+            "1820130"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-27695193",
+            "235417726",
+            "1925960030",
+            "-1068170333",
+            "-1618156350",
+            "-317495919",
+            "1423560212"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-613673600",
+            "875021835",
+            "1453808997",
+            "1717180769",
+            "1580762908",
+            "1840132086",
+            "-1345743252"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "582856351",
+            "1284574124",
+            "340320670",
+            "-1092859214",
+            "1110297876",
+            "1357226248",
+            "1402761903"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-3179119623",
+            "450280234735",
+            "-246304555798",
+            "-119827255352",
+            "-525274364087",
+            "-230257922796",
+            "256828899002"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "172087714641",
+            "446734015461",
+            "215626082740",
+            "310099426632",
+            "262400844124",
+            "-403709578552",
+            "-543955348408"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "79497412466729",
+            "-17627606697345",
+            "-56306219857785",
+            "-44419610515947",
+            "-10505875477448",
+            "-137201986324719",
+            "-41490578530344"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "87785849879213",
+            "72153641766797",
+            "81344597122488",
+            "116608220424101",
+            "120127927299953",
+            "91392864468246",
+            "789277742054"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "16742530979330",
+            "-27276173304601",
+            "112787046376927",
+            "80422014882468",
+            "-53996799818821",
+            "-51482063936845",
+            "-109285996873057"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-27011971722513841",
+            "-25715520475567957",
+            "-31692200074647251",
+            "19797016206092429",
+            "13033529853072966",
+            "-34190512535950112",
+            "21391090647858827"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-2689551968658208",
+            "-11978754119332225",
+            "27943059826706703",
+            "30925593314872225",
+            "-25299415665884227",
+            "-6588871840929638",
+            "-13529612932491138"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-5877424150753455644",
+            "-750157543599527648",
+            "7844548247743823168",
+            "1983390184771773250",
+            "8109668973322170403",
+            "4662100181419928346",
+            "-7990733746417221683"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-4001316145812913855",
+            "8636276014947384575",
+            "255439123904346461",
+            "-5910657878241983749",
+            "-8574194995013797995",
+            "6656912349600694116",
+            "-153848629100633274"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "251402997439201416147",
+            "-773260098260956584464",
+            "-219271115207772204546",
+            "-1036039623941759511644",
+            "359482472533162316496",
+            "1965913949245283018894",
+            "599718122444765908168"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1438319238194982234339",
+            "1577759903359206474839",
+            "188186678199824158300",
+            "1276273466028897015455",
+            "-1836059768603829828246",
+            "-571809172339201000757",
+            "-1423862618055329967277"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "1054493543273151908590",
+            "-996798431306463901700",
+            "1030912998214022157210",
+            "1165909344715451535690",
+            "1678758991439476296198",
+            "-1261232816117225764157",
+            "-275535923846867412400"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-265478781031709465954466",
+            "-45885532800194657389599",
+            "47123250342923949029333",
+            "-28918098710716253013235",
+            "-216245678184360771226258",
+            "-67592927029058001825983",
+            "-216606971904693341187546"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-454834495768691198335625",
+            "-121587423919595203596630",
+            "-180702613619856867200053",
+            "-586233612775783081876233",
+            "-146313510918473805475897",
+            "-19249352342706744822274",
+            "602537232274909092052368"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-138817970110646913589715549",
+            "71212874681781732508199238",
+            "-36634575975569063090531384",
+            "-118838660864234798642972931",
+            "-63677435476346500972940088",
+            "-143253302639820088137695781",
+            "-10005431182625326908184063"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-24132845688739832600198664",
+            "70223772618132296366621411",
+            "39697053813238829872031989",
+            "-4463405036654040634416246",
+            "-114850114696918748689370315",
+            "24197714526405477903473970",
+            "-85613480517918645906993903"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-78615821810256843179885264",
+            "-9435616174371737446306696",
+            "14724235985325181617831683",
+            "-77554225473063349798518023",
+            "-151687049843944286803049399",
+            "152830772381840371720809534",
+            "41342447912435620711044487"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-7452626331051061693783872103",
+            "-37477886858762005988951878072",
+            "29024564721311845466178480817",
+            "27253762906471645828276940493",
+            "-19788635697675024252549748007",
+            "33227312025585737614263740454",
+            "-37713444931406134334378281082"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-25886791622624942457173736636",
+            "34370751906575221175682318842",
+            "-9932886803302690926326494218",
+            "3324237332180497322466960837",
+            "21125203591871926273778796597",
+            "-11276967068824629033564949936",
+            "28441370093039134290858059521"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "8912358001421025895832248632442",
+            "-8916798266948379553115396290654",
+            "-161154141964359329876263021833",
+            "3858086459715378537257910522289",
+            "8040021746182360241076098633516",
+            "4716833692114616471008198034491",
+            "3496285339125640710658712687020"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-9184049173255469554998225870729",
+            "-1938356198264242814346866467899",
+            "-4206826697895630793180053762195",
+            "-9497517245200463036658297067684",
+            "-5584086501913874255219443240156",
+            "8412140854426525037463120473138",
+            "-6042832331708829719042645761969"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-6643586848533296755722567177838",
+            "-1193404893030123421103724263326",
+            "-7351808613718633597834537706852",
+            "-4546872325918455995201693474980",
+            "5044513193984398500316947717350",
+            "-3564202300291159147721754279587",
+            "9899326223400708541507662547701"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "2538490507550203557654476537964592",
+            "-171852815793316039580834325499002",
+            "579673614387874228055248480157807",
+            "1547316751704920851604992499434244",
+            "82956532861960233155706332616871",
+            "2270532599099227110701079190556481",
+            "-1551256727215408667915785904404011"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-1697323961096836989990113960348683",
+            "2269492088271616903476594268476027",
+            "563384931675005850944800674271399",
+            "1829983204109953557442975861814188",
+            "-2099108441853345032440560303311823",
+            "-1159901622227230738337097223322335",
+            "1320399948477362419558701661162682"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-227649207549335409555288103485379868",
+            "541380090188013381867176032991003854",
+            "259040738101837697543772263422611758",
+            "468607824887048977270254889483333053",
+            "-37414275284716090078813150850969270",
+            "414632729703080747958410749710577970",
+            "-628951504936585588032719800558474149"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-585201720045860251224192209434417167",
+            "468582923778969233979378862262684223",
+            "134642897337460953327600517844593153",
+            "36157897287467373309609536222700125",
+            "42661449938741236454803304780412295",
+            "212096786713676119368154634300273677",
+            "585017229375111099843942749410082953"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-138591685930594567129919412699083355323",
+            "77060413350994913049260494214760663038",
+            "39700904836694418018645309917065762876",
+            "-31664403115710800671259751306315806980",
+            "63928998864039165846312590158916247833",
+            "-3586180789611960233999050211148582684",
+            "-42878742931457980860298222400389073217"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-102158662182543310745650009612358481115",
+            "-110662400233487451872440612242183030235",
+            "-4734079005823009383561681338545373884",
+            "25428205135817899705171309423993396942",
+            "119935163858170485692044331984703633688",
+            "27795979506302295159346827031425803891",
+            "-17630802775799110077215936154218710903"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-167820728718740728757244841820454776505",
+            "-95721966128938572947248972085749061555",
+            "140529351854457382631585268999194010048",
+            "-169071957026901707361219133333712549941",
+            "-70939221559844798247914114191797953094",
+            "-127614892614854677120213423457262257204",
+            "-102794811166159877868688547277286760364"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "21474",
+            "4094",
+            "16234",
+            "-10475",
+            "-25906",
+            "-12657",
+            "13211",
+            "20482",
+            "29407",
+            "-3604"
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "9720",
+            "-23643",
+            "9540",
+            "-23720",
+            "31318",
+            "-25292",
+            "18536",
+            "-9162",
+            "58",
+            "-7786"
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-7627572",
+            "-38899",
+            "2443108",
+            "-438758",
+            "-2210056",
+            "2273095",
+            "3114432",
+            "-124491",
+            "-1605021",
+            "8009315"
+          ]
+        },
+        {
+          "name": "f3",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "574992",
+            "-1257531",
+            "-2806474",
+            "-3352205",
+            "2915457",
+            "-7014958",
+            "-7579817",
+            "-2215757",
+            "-576189",
+            "-8360149"
+          ]
+        },
+        {
+          "name": "f4",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "-2041265396",
+            "732584234",
+            "-1999454822",
+            "1704993828",
+            "-210935878",
+            "929286432",
+            "1109525853",
+            "582363395",
+            "-1825179353",
+            "1409680841"
+          ]
+        },
+        {
+          "name": "f5",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "1991977558",
+            "1762056301",
+            "1455216951",
+            "1449185192",
+            "254303971",
+            "140252798",
+            "935225387",
+            "1899985780",
+            "-927824213",
+            "-208960670"
+          ]
+        },
+        {
+          "name": "f6",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "1480625771",
+            "1040969656",
+            "-1003085364",
+            "-553744456",
+            "1343292649",
+            "-325165443",
+            "-1346706809",
+            "-965803077",
+            "723994410",
+            "-20304136"
+          ]
+        },
+        {
+          "name": "f7",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-101818947634",
+            "-229200189261",
+            "-424288334493",
+            "227262504132",
+            "432555641000",
+            "328751277087",
+            "50785160524",
+            "387280003539",
+            "323702572325",
+            "-130440892777"
+          ]
+        },
+        {
+          "name": "f8",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "285258640838",
+            "-31233562378",
+            "413535063358",
+            "366447174903",
+            "51794529119",
+            "81104642011",
+            "-229146846972",
+            "-500253706209",
+            "26427555726",
+            "238540328738"
+          ]
+        },
+        {
+          "name": "f9",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "72984542244109",
+            "-85349566014644",
+            "-61750715247480",
+            "-107963169778383",
+            "-72064927212019",
+            "74634683457234",
+            "-5692709274261",
+            "-137801733360366",
+            "66863319822840",
+            "53790391904569"
+          ]
+        },
+        {
+          "name": "f10",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-30506162400564",
+            "44134792179061",
+            "43208822949445",
+            "116111507835088",
+            "-52652763068808",
+            "30227545743701",
+            "54132435978158",
+            "-84979416074353",
+            "-77767629770125",
+            "-49700250599015"
+          ]
+        },
+        {
+          "name": "f11",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-33646284874231",
+            "-39843156046137",
+            "135855350625826",
+            "67173686029441",
+            "47373973603503",
+            "-81315792116685",
+            "126362930268105",
+            "62326866041404",
+            "-79917419899133",
+            "116346318505097"
+          ]
+        },
+        {
+          "name": "f12",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-17623564370335172",
+            "13940102868728695",
+            "31159892605228020",
+            "19288139749976327",
+            "-32162377831099808",
+            "-25334882778439881",
+            "-9293849274157177",
+            "25753846620057242",
+            "-2715500844487831",
+            "17297835213572781"
+          ]
+        },
+        {
+          "name": "f13",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-24802265556783267",
+            "-1071851274851707",
+            "-20708121978765252",
+            "21106785059579409",
+            "895452156504814",
+            "-17439295017291261",
+            "-29860734942289352",
+            "-24366392879871907",
+            "-24844781251317766",
+            "5691042499081415"
+          ]
+        },
+        {
+          "name": "f14",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            "-950656153293958588",
+            "-7026895800290111295",
+            "-2906486337048393634",
+            "-2487298974163906537",
+            "-7240801780393004497",
+            "-5928987398455158527",
+            "91252511169998609",
+            "-4434565223927059941",
+            "6114237547594359253",
+            "-5649241382570014194"
+          ]
+        },
+        {
+          "name": "f15",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-9125944744613843645",
+            "2767912703851717499",
+            "-5594414080371870722",
+            "1326878363883364298",
+            "6250061466047884457",
+            "4167357360257738660",
+            "3185444479792598034",
+            "2850893211432323540",
+            "4099400621617587983",
+            "1911788567774553475"
+          ]
+        },
+        {
+          "name": "f16",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "2243798170232923011194",
+            "1260790852369524872646",
+            "1926129514370913291084",
+            "1923482934016917609948",
+            "-1337597225338052576342",
+            "-1396976913710878087379",
+            "1760383914304461201041",
+            "-2084424105474946896583",
+            "546847183858239970168",
+            "-950422829529740640046"
+          ]
+        },
+        {
+          "name": "f17",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "2278561335636612407116",
+            "2226551432527285815529",
+            "1102614999141365247364",
+            "282444931594177029536",
+            "1707175857213323708724",
+            "-1813457685182877716617",
+            "1969563893365610364950",
+            "-1294229647104229980192",
+            "1058028980431521243786",
+            "1425804620843155435443"
+          ]
+        },
+        {
+          "name": "f18",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-666605901201678736091",
+            "2294247840458377820087",
+            "-120100005268248092109",
+            "-2117030120296482991359",
+            "-936538133650490467428",
+            "1909556929153544496029",
+            "668148430777958474013",
+            "1300546260152428176254",
+            "-1307847030795641403330",
+            "962024918692270891085"
+          ]
+        },
+        {
+          "name": "f19",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "429007353965964812441246",
+            "-539416061647283176800407",
+            "-246119986866263363195168",
+            "519288706159858079097852",
+            "145290868494128799776491",
+            "435391066127563014503225",
+            "-212517380359649480302803",
+            "229639965616877780412338",
+            "-228549196983767607874750",
+            "254322220652598819404701"
+          ]
+        },
+        {
+          "name": "f20",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "302744770146464834277007",
+            "349890188007940826732505",
+            "-395767415302716050551365",
+            "-55506738483590549708985",
+            "-42546607637771046763757",
+            "599890205870578843548554",
+            "-566080596849854240867084",
+            "240819660280603920408014",
+            "490355076012318604086830",
+            "-444452608521493383189339"
+          ]
+        },
+        {
+          "name": "f21",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "140098423234498101984955781",
+            "114972555620228418385387046",
+            "-103855128687529946029391332",
+            "-56453596465478079539961341",
+            "112899417755966489798807822",
+            "92479416756872049033014234",
+            "-20220375538499957175667596",
+            "49619147776130236286406382",
+            "75868169169602577596893114",
+            "-33991855971785669167684812"
+          ]
+        },
+        {
+          "name": "f22",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-27955923849772806651093661",
+            "-15134268062501921536631099",
+            "-76180691187952624554132166",
+            "8417609600056713913269682",
+            "122210737333544046425988875",
+            "-8255971110072598553761320",
+            "72059712120402848706275333",
+            "136669805857501664860962453",
+            "117226067843670223572552823",
+            "9538509353520989293188292"
+          ]
+        },
+        {
+          "name": "f23",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "140326260020124618484882670",
+            "-7604232162274599406456263",
+            "-37578322955437058565218623",
+            "82251242821235569071412696",
+            "48934521009186759426554144",
+            "-111502425314556830584865049",
+            "143399285732546087389279250",
+            "-145031030627834093577440455",
+            "-56753812486185622944448939",
+            "-131020484598181539722605940"
+          ]
+        },
+        {
+          "name": "f24",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "7163957691918374413440392122",
+            "29895408949428293949825637565",
+            "24953244787599080815469023689",
+            "12933037616695046483291587011",
+            "12911061711671395434197711654",
+            "-662465023864870915737125277",
+            "32902737578296234199724114196",
+            "-13332149788556638691010135470",
+            "28732116950622282825950924835",
+            "-606082615276279539679739878"
+          ]
+        },
+        {
+          "name": "f25",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-2225441768919818543256115590",
+            "36300525663713292309356899587",
+            "22146605949936394498917396829",
+            "-13799763611538096768550340984",
+            "13697106267730772282848050012",
+            "37987370502999491375497665834",
+            "-15959212476158770181464939730",
+            "37740896499376085097100094638",
+            "-3277570747229261911544243044",
+            "34646859779346958641044018113"
+          ]
+        },
+        {
+          "name": "f26",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "8464727847598438214667266343191",
+            "3659092136516017614153405697408",
+            "8898836896743628102604107262318",
+            "-5124678781048461345216331096076",
+            "7048310126754458455427182472252",
+            "-7485149236907192851539148176189",
+            "4765944987612723450221212104859",
+            "3507852604123913333443796199231",
+            "-3638691289771499903484086224708",
+            "-1548294157076139554924261118715"
+          ]
+        },
+        {
+          "name": "f27",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "1960248254240505083559553622556",
+            "6334649991475602141352059045232",
+            "5429973568679996304865347312737",
+            "-5574768356761641457841827808332",
+            "2921634229104833962007853636995",
+            "6645606916555794304028632924702",
+            "7402711760974614198662134839264",
+            "-2430215409038042282074825546118",
+            "9189456250491239952337287624422",
+            "-6746525342617036236303085693943"
+          ]
+        },
+        {
+          "name": "f28",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "-2303252604266546830450692395647",
+            "24931561165839460953255704242",
+            "3539774568243302781625620724843",
+            "-4119755165665461319885896453094",
+            "9980798001735069049033995442687",
+            "6400453710768026204203981933498",
+            "-1785561568444900500617881894975",
+            "-1238657151175445676210732724686",
+            "-4115303461735084227982174922353",
+            "-4830151905286143893643036559970"
+          ]
+        },
+        {
+          "name": "f29",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-744732514813865310675102410677837",
+            "586923153993520038833541626305861",
+            "631405070127015389249055779739635",
+            "2375722591461493864257181946361349",
+            "536327416123747764992062327728379",
+            "4403250929897292075500020549198",
+            "-920102429093431209270145990206427",
+            "-1943957685901230324468842397328959",
+            "-1793159792601568795672835117263773",
+            "2539832186764726126109095059589925"
+          ]
+        },
+        {
+          "name": "f30",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "1736943855683500361344260536843977",
+            "-1041915987085848668127887646443139",
+            "-2073351044084729515377756324458994",
+            "-2499860712727710930829417979309376",
+            "-1334132038144948687896176896558215",
+            "2497357047334342837216894037461981",
+            "-2593640175489681036725320634980356",
+            "83143781366050188466245498529346",
+            "-1907502149335575039303599318337661",
+            "769063293620504337906197233553371"
+          ]
+        },
+        {
+          "name": "f31",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-250745229891973039581681177880223505",
+            "327446904412556324074846927736374445",
+            "-549111820347488138618064952270999012",
+            "-107667493114547862984588921499141796",
+            "-213442200532407781713670495439019398",
+            "-357538486078878970354893736102694429",
+            "556639003636036852879832060230162340",
+            "-263609850789709040712893190708950606",
+            "83507483892195138402414967192218438",
+            "122845115500690774528208372788074092"
+          ]
+        },
+        {
+          "name": "f32",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            "155941392458135251949747004728456096",
+            "-351841600639304120253871824228008989",
+            "13489132228213437891031310407260140",
+            "98346439010414420839370528660894148",
+            "110095662267504106346877402686062417",
+            "272056686592458102197893958630607521",
+            "-568455603353918918719268251254453391",
+            "-641181026054843530064046004084619896",
+            "-153598039697896424251301260354494095",
+            "-238169729901866634377216907979943085"
+          ]
+        },
+        {
+          "name": "f33",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "-76486506229583148953575426031378420083",
+            "-25268246003138515244204194206040713884",
+            "151680852047855885899827416129131343552",
+            "-65868597846063019193194432937954865449",
+            "-37353304147507335221409678836410716004",
+            "-83375093114704443137943855826794265870",
+            "58765750477038687599983055893883073980",
+            "-124247461021880357230108144628851373656",
+            "-97168006373071667635001937044389371121",
+            "147332252855265140513988431868146207163"
+          ]
+        },
+        {
+          "name": "f34",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "-93004622235567469002573559852323599733",
+            "-136486146006693484453265708648174772555",
+            "50333349154635707168287760155624850500",
+            "56065748751155225550881020072304819143",
+            "-89047403980608497862076968023015862559",
+            "-138188932216419239057464674728657893212",
+            "73805817848460887264703472962238324754",
+            "-146887621381330685164768807944413153241",
+            "-58194523976849290302949585080581301830",
+            "-147081260537816177520155637200701099095"
+          ]
+        },
+        {
+          "name": "f35",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "-40110048976612650804598426597711067016",
+            "124144731433352682852630636540845469156",
+            "145245442856314145141703280651554450780",
+            "-144190927886671777348422824785547904228",
+            "51752695865288273345113477117624659857",
+            "147665067149574600679327527424233686202",
+            "51752916559949464310444304180016487848",
+            "65187705658195518441807850839061980569",
+            "-132615731354653260112463459795520015874",
+            "-16458882186479214359093418885286436364"
+          ]
+        }
+      ]
+    }
+  ]
+}
\ No newline at end of file
diff --git a/julia/Arrow/test/arrowjson/dictionary.json b/julia/Arrow/test/arrowjson/dictionary.json
new file mode 100644
index 00000000000..2307f3e3a47
--- /dev/null
+++ b/julia/Arrow/test/arrowjson/dictionary.json
@@ -0,0 +1,422 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "dict0",
+        "type": {
+          "name": "utf8"
+        },
+        "nullable": true,
+        "children": [],
+        "dictionary": {
+          "id": 0,
+          "indexType": {
+            "name": "int",
+            "isSigned": true,
+            "bitWidth": 8
+          },
+          "isOrdered": false
+        }
+      },
+      {
+        "name": "dict1",
+        "type": {
+          "name": "utf8"
+        },
+        "nullable": true,
+        "children": [],
+        "dictionary": {
+          "id": 1,
+          "indexType": {
+            "name": "int",
+            "isSigned": true,
+            "bitWidth": 32
+          },
+          "isOrdered": false
+        }
+      },
+      {
+        "name": "dict2",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 64
+        },
+        "nullable": true,
+        "children": [],
+        "dictionary": {
+          "id": 2,
+          "indexType": {
+            "name": "int",
+            "isSigned": true,
+            "bitWidth": 16
+          },
+          "isOrdered": false
+        }
+      }
+    ]
+  },
+  "dictionaries": [
+    {
+      "id": 0,
+      "data": {
+        "count": 10,
+        "columns": [
+          {
+            "name": "DICT0",
+            "count": 10,
+            "VALIDITY": [
+              0,
+              0,
+              1,
+              0,
+              1,
+              1,
+              1,
+              1,
+              0,
+              0
+            ],
+            "OFFSET": [
+              0,
+              0,
+              0,
+              7,
+              7,
+              20,
+              27,
+              36,
+              45,
+              45,
+              45
+            ],
+            "DATA": [
+              "",
+              "",
+              "kfii3eo",
+              "",
+              "\u00f4eh5\u77e2\u20ac\u00a3",
+              "wa3fdmj",
+              "3\u00b5i\u00a3146",
+              "eo\u00f4rp3\u00b5",
+              "",
+              ""
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "id": 1,
+      "data": {
+        "count": 5,
+        "columns": [
+          {
+            "name": "DICT1",
+            "count": 5,
+            "VALIDITY": [
+              1,
+              0,
+              1,
+              0,
+              0
+            ],
+            "OFFSET": [
+              0,
+              11,
+              11,
+              20,
+              20,
+              20
+            ],
+            "DATA": [
+              "n6\u20ac\u00b54g\u00b0",
+              "",
+              "o3\u77e2kbr2",
+              "",
+              ""
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "id": 2,
+      "data": {
+        "count": 50,
+        "columns": [
+          {
+            "name": "DICT2",
+            "count": 50,
+            "VALIDITY": [
+              1,
+              1,
+              0,
+              0,
+              0,
+              1,
+              1,
+              1,
+              0,
+              1,
+              0,
+              1,
+              1,
+              1,
+              1,
+              1,
+              1,
+              0,
+              1,
+              1,
+              0,
+              1,
+              1,
+              0,
+              1,
+              1,
+              0,
+              0,
+              1,
+              1,
+              0,
+              1,
+              1,
+              1,
+              1,
+              1,
+              0,
+              0,
+              1,
+              0,
+              0,
+              1,
+              1,
+              1,
+              0,
+              0,
+              1,
+              0,
+              1,
+              1
+            ],
+            "DATA": [
+              "-2147483648",
+              "2147483647",
+              "2063303031",
+              "575556936",
+              "994232893",
+              "-733018692",
+              "-1869839205",
+              "818048194",
+              "-780203826",
+              "1956041779",
+              "1352257839",
+              "-411979330",
+              "-1676682633",
+              "-284358770",
+              "-1409435091",
+              "722395458",
+              "2117191004",
+              "-1448425086",
+              "-1128563576",
+              "-1190200669",
+              "1780667813",
+              "-987872595",
+              "-160215270",
+              "-2042614145",
+              "-801319359",
+              "940981753",
+              "-1432601950",
+              "-546534432",
+              "970337383",
+              "319587202",
+              "494356019",
+              "1832436202",
+              "-932834552",
+              "1463915583",
+              "1779474803",
+              "253308913",
+              "1534215558",
+              "1946313834",
+              "87626535",
+              "-2043979004",
+              "1625504442",
+              "-1819229860",
+              "-176171291",
+              "135039336",
+              "-836907545",
+              "792617942",
+              "602157568",
+              "81724808",
+              "103096742",
+              "282026629"
+            ]
+          }
+        ]
+      }
+    }
+  ],
+  "batches": [
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "dict0",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            5,
+            9,
+            1,
+            8,
+            0,
+            8,
+            5
+          ]
+        },
+        {
+          "name": "dict1",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            4,
+            2,
+            2,
+            3,
+            0,
+            0,
+            3
+          ]
+        },
+        {
+          "name": "dict2",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            8,
+            44,
+            46,
+            42,
+            11,
+            26,
+            15
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "dict0",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            6,
+            8,
+            2,
+            3,
+            8,
+            4,
+            9,
+            2,
+            4,
+            8
+          ]
+        },
+        {
+          "name": "dict1",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            4,
+            2,
+            3,
+            3,
+            4,
+            1,
+            3,
+            0,
+            2,
+            4
+          ]
+        },
+        {
+          "name": "dict2",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            1,
+            38,
+            40,
+            32,
+            19,
+            46,
+            30,
+            25,
+            32,
+            28
+          ]
+        }
+      ]
+    }
+  ]
+}
\ No newline at end of file
diff --git a/julia/Arrow/test/arrowjson/dictionary_unsigned.json b/julia/Arrow/test/arrowjson/dictionary_unsigned.json
new file mode 100644
index 00000000000..f6a61dfb02c
--- /dev/null
+++ b/julia/Arrow/test/arrowjson/dictionary_unsigned.json
@@ -0,0 +1,323 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "f0",
+        "type": {
+          "name": "utf8"
+        },
+        "nullable": true,
+        "children": [],
+        "dictionary": {
+          "id": 0,
+          "indexType": {
+            "name": "int",
+            "isSigned": false,
+            "bitWidth": 8
+          },
+          "isOrdered": false
+        }
+      },
+      {
+        "name": "f1",
+        "type": {
+          "name": "utf8"
+        },
+        "nullable": true,
+        "children": [],
+        "dictionary": {
+          "id": 1,
+          "indexType": {
+            "name": "int",
+            "isSigned": false,
+            "bitWidth": 16
+          },
+          "isOrdered": false
+        }
+      },
+      {
+        "name": "f2",
+        "type": {
+          "name": "utf8"
+        },
+        "nullable": true,
+        "children": [],
+        "dictionary": {
+          "id": 2,
+          "indexType": {
+            "name": "int",
+            "isSigned": false,
+            "bitWidth": 32
+          },
+          "isOrdered": false
+        }
+      }
+    ]
+  },
+  "dictionaries": [
+    {
+      "id": 0,
+      "data": {
+        "count": 5,
+        "columns": [
+          {
+            "name": "DICT0",
+            "count": 5,
+            "VALIDITY": [
+              1,
+              1,
+              1,
+              1,
+              1
+            ],
+            "OFFSET": [
+              0,
+              7,
+              17,
+              26,
+              37,
+              45
+            ],
+            "DATA": [
+              "nai4kkd",
+              "1\u00a3n\u00a3\u00a336",
+              "2f\u77e2wei4",
+              "f\u00c2g\u00b5\u00b5m\u00f4",
+              "\u00b5r2bkd1"
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "id": 1,
+      "data": {
+        "count": 5,
+        "columns": [
+          {
+            "name": "DICT1",
+            "count": 5,
+            "VALIDITY": [
+              1,
+              0,
+              0,
+              1,
+              1
+            ],
+            "OFFSET": [
+              0,
+              7,
+              7,
+              7,
+              15,
+              23
+            ],
+            "DATA": [
+              "bor21pg",
+              "",
+              "",
+              "erohj\u00c2d",
+              "5\u00b0jogf2"
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "id": 2,
+      "data": {
+        "count": 5,
+        "columns": [
+          {
+            "name": "DICT2",
+            "count": 5,
+            "VALIDITY": [
+              0,
+              1,
+              1,
+              1,
+              1
+            ],
+            "OFFSET": [
+              0,
+              0,
+              10,
+              18,
+              29,
+              39
+            ],
+            "DATA": [
+              "",
+              "\u77e2\u00a3efhcg",
+              "lii\u00b5wwm",
+              "1\u20ace2\u77e24c",
+              "bio\u20ac\u00b04l"
+            ]
+          }
+        ]
+      }
+    }
+  ],
+  "batches": [
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            3,
+            1,
+            1,
+            0,
+            4,
+            1,
+            0
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            1,
+            2,
+            0,
+            2,
+            1,
+            0,
+            0
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            0,
+            3,
+            4,
+            3,
+            2,
+            0,
+            1
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "f0",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            4,
+            3,
+            2,
+            2,
+            0,
+            2,
+            4,
+            2,
+            2,
+            0
+          ]
+        },
+        {
+          "name": "f1",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            1,
+            0,
+            4,
+            4,
+            4,
+            2,
+            4,
+            2,
+            2,
+            4
+          ]
+        },
+        {
+          "name": "f2",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            1,
+            1,
+            4,
+            0,
+            1,
+            1,
+            1,
+            4,
+            3,
+            2
+          ]
+        }
+      ]
+    }
+  ]
+}
\ No newline at end of file
diff --git a/julia/Arrow/test/arrowjson/map.json b/julia/Arrow/test/arrowjson/map.json
new file mode 100644
index 00000000000..78a215969cc
--- /dev/null
+++ b/julia/Arrow/test/arrowjson/map.json
@@ -0,0 +1,291 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "map_nullable",
+        "type": {
+          "name": "map",
+          "keysSorted": false
+        },
+        "nullable": true,
+        "children": [
+          {
+            "name": "entries",
+            "type": {
+              "name": "struct"
+            },
+            "nullable": false,
+            "children": [
+              {
+                "name": "key",
+                "type": {
+                  "name": "utf8"
+                },
+                "nullable": false,
+                "children": []
+              },
+              {
+                "name": "value",
+                "type": {
+                  "name": "int",
+                  "isSigned": true,
+                  "bitWidth": 32
+                },
+                "nullable": true,
+                "children": []
+              }
+            ]
+          }
+        ]
+      }
+    ]
+  },
+  "batches": [
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "map_nullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "OFFSET": [
+            0,
+            3,
+            3,
+            3,
+            3,
+            5,
+            6,
+            9
+          ],
+          "children": [
+            {
+              "name": "entries",
+              "count": 9,
+              "VALIDITY": [
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                1
+              ],
+              "children": [
+                {
+                  "name": "key",
+                  "count": 9,
+                  "VALIDITY": [
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1
+                  ],
+                  "OFFSET": [
+                    0,
+                    7,
+                    18,
+                    26,
+                    35,
+                    42,
+                    50,
+                    59,
+                    67,
+                    77
+                  ],
+                  "DATA": [
+                    "nk23ghl",
+                    "hr\u00b0rr\u77e2\u00b0",
+                    "b\u00b5reapd",
+                    "\u00a3\u00a3fprlo",
+                    "42lfc64",
+                    "ifcmf\u00b54",
+                    "mo\u00b51\u00b5gr",
+                    "alfkm\u00c2m",
+                    "r\u20acj333\u00f4"
+                  ]
+                },
+                {
+                  "name": "value",
+                  "count": 9,
+                  "VALIDITY": [
+                    1,
+                    0,
+                    1,
+                    0,
+                    1,
+                    1,
+                    1,
+                    0,
+                    1
+                  ],
+                  "DATA": [
+                    -2147483648,
+                    2147483647,
+                    -364117737,
+                    -1036293739,
+                    595547911,
+                    -136984751,
+                    1113642047,
+                    -174490757,
+                    247058944
+                  ]
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "map_nullable",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "OFFSET": [
+            0,
+            0,
+            3,
+            5,
+            5,
+            5,
+            9,
+            9,
+            9,
+            12,
+            12
+          ],
+          "children": [
+            {
+              "name": "entries",
+              "count": 12,
+              "VALIDITY": [
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                1
+              ],
+              "children": [
+                {
+                  "name": "key",
+                  "count": 12,
+                  "VALIDITY": [
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1
+                  ],
+                  "OFFSET": [
+                    0,
+                    7,
+                    15,
+                    23,
+                    31,
+                    41,
+                    48,
+                    57,
+                    67,
+                    76,
+                    88,
+                    95,
+                    106
+                  ],
+                  "DATA": [
+                    "nii3ppw",
+                    "am3a\u00c2f4",
+                    "pc\u00a3heh3",
+                    "d\u00c2a6n5b",
+                    "\u00f4mf2\u20acc4",
+                    "r1jdf4r",
+                    "njo4\u00b0\u00b0k",
+                    "or\u20acm\u00f4lr",
+                    "em\u00f4i\u00f4ab",
+                    "\u00a3\u20acg\u00c2\u00b53a",
+                    "nfrim36",
+                    "\u00b0\u00c2b\u00f4a\u00b5w"
+                  ]
+                },
+                {
+                  "name": "value",
+                  "count": 12,
+                  "VALIDITY": [
+                    1,
+                    0,
+                    1,
+                    1,
+                    1,
+                    0,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    0
+                  ],
+                  "DATA": [
+                    -2147483648,
+                    2147483647,
+                    -296180340,
+                    -44923686,
+                    2123774743,
+                    1831761340,
+                    1290652534,
+                    68203103,
+                    1896803149,
+                    -1837559303,
+                    -517631115,
+                    289945925
+                  ]
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}
\ No newline at end of file
diff --git a/julia/Arrow/test/arrowjson/nested.json b/julia/Arrow/test/arrowjson/nested.json
new file mode 100644
index 00000000000..50a73a18cc5
--- /dev/null
+++ b/julia/Arrow/test/arrowjson/nested.json
@@ -0,0 +1,537 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "list_nullable",
+        "type": {
+          "name": "list"
+        },
+        "nullable": true,
+        "children": [
+          {
+            "name": "item",
+            "type": {
+              "name": "int",
+              "isSigned": true,
+              "bitWidth": 32
+            },
+            "nullable": true,
+            "children": []
+          }
+        ]
+      },
+      {
+        "name": "fixedsizelist_nullable",
+        "type": {
+          "name": "fixedsizelist",
+          "listSize": 4
+        },
+        "nullable": true,
+        "children": [
+          {
+            "name": "item",
+            "type": {
+              "name": "int",
+              "isSigned": true,
+              "bitWidth": 32
+            },
+            "nullable": true,
+            "children": []
+          }
+        ]
+      },
+      {
+        "name": "struct_nullable",
+        "type": {
+          "name": "struct"
+        },
+        "nullable": true,
+        "children": [
+          {
+            "name": "f1",
+            "type": {
+              "name": "int",
+              "isSigned": true,
+              "bitWidth": 32
+            },
+            "nullable": true,
+            "children": []
+          },
+          {
+            "name": "f2",
+            "type": {
+              "name": "utf8"
+            },
+            "nullable": true,
+            "children": []
+          }
+        ]
+      }
+    ]
+  },
+  "batches": [
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "list_nullable",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1
+          ],
+          "OFFSET": [
+            0,
+            0,
+            4,
+            4,
+            4,
+            4,
+            4,
+            7
+          ],
+          "children": [
+            {
+              "name": "item",
+              "count": 7,
+              "VALIDITY": [
+                1,
+                0,
+                0,
+                0,
+                1,
+                1,
+                1
+              ],
+              "DATA": [
+                -2147483648,
+                2147483647,
+                1151956889,
+                -381487027,
+                873948027,
+                1643402405,
+                1658614443
+              ]
+            }
+          ]
+        },
+        {
+          "name": "fixedsizelist_nullable",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "children": [
+            {
+              "name": "item",
+              "count": 28,
+              "VALIDITY": [
+                0,
+                0,
+                0,
+                1,
+                0,
+                0,
+                1,
+                0,
+                1,
+                1,
+                0,
+                1,
+                1,
+                0,
+                0,
+                1,
+                0,
+                0,
+                1,
+                0,
+                1,
+                1,
+                0,
+                1,
+                1,
+                0,
+                0,
+                1
+              ],
+              "DATA": [
+                -2147483648,
+                2147483647,
+                1455417103,
+                -386721267,
+                1200555928,
+                416671823,
+                -426485775,
+                1024100580,
+                1264993422,
+                -1001974859,
+                1688456056,
+                1130147884,
+                -1020099019,
+                1848995454,
+                2101085138,
+                1609275375,
+                1517442455,
+                -1648565699,
+                994186796,
+                2006991970,
+                -620340903,
+                -846162493,
+                1646215825,
+                468219400,
+                1580372269,
+                -1426972627,
+                1957501966,
+                1539427720
+              ]
+            }
+          ]
+        },
+        {
+          "name": "struct_nullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "children": [
+            {
+              "name": "f1",
+              "count": 7,
+              "VALIDITY": [
+                1,
+                1,
+                1,
+                1,
+                0,
+                0,
+                0
+              ],
+              "DATA": [
+                -2147483648,
+                2147483647,
+                531016632,
+                1805412611,
+                -1194513600,
+                -1267423429,
+                1000139479
+              ]
+            },
+            {
+              "name": "f2",
+              "count": 7,
+              "VALIDITY": [
+                1,
+                0,
+                0,
+                0,
+                1,
+                0,
+                0
+              ],
+              "OFFSET": [
+                0,
+                9,
+                9,
+                9,
+                9,
+                21,
+                21,
+                21
+              ],
+              "DATA": [
+                "dej\u00c2\u00a3pr",
+                "",
+                "",
+                "",
+                "i\u77e2j\u00a3\u00b0\u00b5m",
+                "",
+                ""
+              ]
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "list_nullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "OFFSET": [
+            0,
+            2,
+            5,
+            9,
+            9,
+            9,
+            9,
+            11,
+            13,
+            13,
+            13
+          ],
+          "children": [
+            {
+              "name": "item",
+              "count": 13,
+              "VALIDITY": [
+                0,
+                1,
+                1,
+                1,
+                0,
+                1,
+                1,
+                1,
+                0,
+                0,
+                1,
+                0,
+                1
+              ],
+              "DATA": [
+                -2147483648,
+                2147483647,
+                248935155,
+                -322392706,
+                -115480117,
+                -1828031254,
+                -1766278277,
+                444059471,
+                -805219796,
+                1061714770,
+                423821865,
+                1278080160,
+                -1936049755
+              ]
+            }
+          ]
+        },
+        {
+          "name": "fixedsizelist_nullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "children": [
+            {
+              "name": "item",
+              "count": 40,
+              "VALIDITY": [
+                0,
+                1,
+                1,
+                0,
+                0,
+                1,
+                1,
+                0,
+                0,
+                1,
+                1,
+                0,
+                0,
+                1,
+                0,
+                0,
+                1,
+                0,
+                1,
+                0,
+                1,
+                1,
+                0,
+                0,
+                1,
+                0,
+                1,
+                0,
+                0,
+                1,
+                1,
+                1,
+                0,
+                1,
+                0,
+                0,
+                0,
+                0,
+                0,
+                0
+              ],
+              "DATA": [
+                -2147483648,
+                2147483647,
+                -376399341,
+                -1632957441,
+                -2143340689,
+                -159866261,
+                419032293,
+                1196087612,
+                784095846,
+                -435963426,
+                -574319754,
+                534646696,
+                -444534026,
+                1281811213,
+                -105831675,
+                -23408446,
+                348235550,
+                -1986936151,
+                -1558484167,
+                941484009,
+                2144967535,
+                830399805,
+                31470129,
+                -792586642,
+                -29778291,
+                1445584989,
+                -1344862682,
+                -1973973713,
+                -875795314,
+                556638175,
+                1282112437,
+                -2054079892,
+                -881420523,
+                1038384760,
+                846630800,
+                -442527632,
+                -975132694,
+                84102477,
+                1771796204,
+                -153752454
+              ]
+            }
+          ]
+        },
+        {
+          "name": "struct_nullable",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1
+          ],
+          "children": [
+            {
+              "name": "f1",
+              "count": 10,
+              "VALIDITY": [
+                0,
+                0,
+                1,
+                1,
+                1,
+                1,
+                0,
+                1,
+                1,
+                1
+              ],
+              "DATA": [
+                -2147483648,
+                2147483647,
+                -1568585736,
+                1732529716,
+                1236302061,
+                -232126550,
+                -1554873925,
+                191321500,
+                570901234,
+                -2063634007
+              ]
+            },
+            {
+              "name": "f2",
+              "count": 10,
+              "VALIDITY": [
+                0,
+                0,
+                0,
+                1,
+                1,
+                1,
+                1,
+                0,
+                0,
+                1
+              ],
+              "OFFSET": [
+                0,
+                0,
+                0,
+                0,
+                9,
+                20,
+                28,
+                36,
+                36,
+                36,
+                46
+              ],
+              "DATA": [
+                "",
+                "",
+                "",
+                "h3\u00f46c\u00b0i",
+                "opa\u20acdf\u20ac",
+                "ncdcpe\u00f4",
+                "4fwflo\u00b0",
+                "",
+                "",
+                "pf41\u00c2\u77e24"
+              ]
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}
\ No newline at end of file
diff --git a/julia/Arrow/test/arrowjson/primitive-empty.json b/julia/Arrow/test/arrowjson/primitive-empty.json
new file mode 100644
index 00000000000..1e162592015
--- /dev/null
+++ b/julia/Arrow/test/arrowjson/primitive-empty.json
@@ -0,0 +1,879 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "bool_nullable",
+        "type": {
+          "name": "bool"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "bool_nonnullable",
+        "type": {
+          "name": "bool"
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "int8_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 8
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "int8_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 8
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "int16_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 16
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "int16_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 16
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "int32_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 32
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "int32_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 32
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "int64_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 64
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "int64_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 64
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "uint8_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 8
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "uint8_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 8
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "uint16_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 16
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "uint16_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 16
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "uint32_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 32
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "uint32_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 32
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "uint64_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 64
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "uint64_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 64
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "float32_nullable",
+        "type": {
+          "name": "floatingpoint",
+          "precision": "SINGLE"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "float32_nonnullable",
+        "type": {
+          "name": "floatingpoint",
+          "precision": "SINGLE"
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "float64_nullable",
+        "type": {
+          "name": "floatingpoint",
+          "precision": "DOUBLE"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "float64_nonnullable",
+        "type": {
+          "name": "floatingpoint",
+          "precision": "DOUBLE"
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "binary_nullable",
+        "type": {
+          "name": "binary"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "binary_nonnullable",
+        "type": {
+          "name": "binary"
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "utf8_nullable",
+        "type": {
+          "name": "utf8"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "utf8_nonnullable",
+        "type": {
+          "name": "utf8"
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "fixedsizebinary_19_nullable",
+        "type": {
+          "name": "fixedsizebinary",
+          "byteWidth": 19
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "fixedsizebinary_19_nonnullable",
+        "type": {
+          "name": "fixedsizebinary",
+          "byteWidth": 19
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "fixedsizebinary_120_nullable",
+        "type": {
+          "name": "fixedsizebinary",
+          "byteWidth": 120
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "fixedsizebinary_120_nonnullable",
+        "type": {
+          "name": "fixedsizebinary",
+          "byteWidth": 120
+        },
+        "nullable": false,
+        "children": []
+      }
+    ]
+  },
+  "batches": [
+    {
+      "count": 0,
+      "columns": [
+        {
+          "name": "bool_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "bool_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int8_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int8_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int16_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int16_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int32_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int32_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int64_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int64_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint8_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint8_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint16_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint16_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint32_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint32_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint64_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint64_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "float32_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "float32_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "float64_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "float64_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "binary_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "OFFSET": [
+            0
+          ],
+          "DATA": []
+        },
+        {
+          "name": "binary_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "OFFSET": [
+            0
+          ],
+          "DATA": []
+        },
+        {
+          "name": "utf8_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "OFFSET": [
+            0
+          ],
+          "DATA": []
+        },
+        {
+          "name": "utf8_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "OFFSET": [
+            0
+          ],
+          "DATA": []
+        },
+        {
+          "name": "fixedsizebinary_19_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "fixedsizebinary_19_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "fixedsizebinary_120_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "fixedsizebinary_120_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        }
+      ]
+    },
+    {
+      "count": 0,
+      "columns": [
+        {
+          "name": "bool_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "bool_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int8_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int8_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int16_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int16_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int32_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int32_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int64_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int64_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint8_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint8_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint16_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint16_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint32_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint32_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint64_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint64_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "float32_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "float32_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "float64_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "float64_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "binary_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "OFFSET": [
+            0
+          ],
+          "DATA": []
+        },
+        {
+          "name": "binary_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "OFFSET": [
+            0
+          ],
+          "DATA": []
+        },
+        {
+          "name": "utf8_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "OFFSET": [
+            0
+          ],
+          "DATA": []
+        },
+        {
+          "name": "utf8_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "OFFSET": [
+            0
+          ],
+          "DATA": []
+        },
+        {
+          "name": "fixedsizebinary_19_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "fixedsizebinary_19_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "fixedsizebinary_120_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "fixedsizebinary_120_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        }
+      ]
+    },
+    {
+      "count": 0,
+      "columns": [
+        {
+          "name": "bool_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "bool_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int8_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int8_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int16_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int16_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int32_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int32_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int64_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "int64_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint8_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint8_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint16_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint16_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint32_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint32_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint64_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "uint64_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "float32_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "float32_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "float64_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "float64_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "binary_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "OFFSET": [
+            0
+          ],
+          "DATA": []
+        },
+        {
+          "name": "binary_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "OFFSET": [
+            0
+          ],
+          "DATA": []
+        },
+        {
+          "name": "utf8_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "OFFSET": [
+            0
+          ],
+          "DATA": []
+        },
+        {
+          "name": "utf8_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "OFFSET": [
+            0
+          ],
+          "DATA": []
+        },
+        {
+          "name": "fixedsizebinary_19_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "fixedsizebinary_19_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "fixedsizebinary_120_nullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        },
+        {
+          "name": "fixedsizebinary_120_nonnullable",
+          "count": 0,
+          "VALIDITY": [],
+          "DATA": []
+        }
+      ]
+    }
+  ]
+}
\ No newline at end of file
diff --git a/julia/Arrow/test/arrowjson/primitive.json b/julia/Arrow/test/arrowjson/primitive.json
new file mode 100644
index 00000000000..1b0ff10be2a
--- /dev/null
+++ b/julia/Arrow/test/arrowjson/primitive.json
@@ -0,0 +1,1890 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "bool_nullable",
+        "type": {
+          "name": "bool"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "bool_nonnullable",
+        "type": {
+          "name": "bool"
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "int8_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 8
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "int8_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 8
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "int16_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 16
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "int16_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 16
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "int32_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 32
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "int32_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 32
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "int64_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 64
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "int64_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 64
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "uint8_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 8
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "uint8_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 8
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "uint16_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 16
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "uint16_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 16
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "uint32_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 32
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "uint32_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 32
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "uint64_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 64
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "uint64_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 64
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "float32_nullable",
+        "type": {
+          "name": "floatingpoint",
+          "precision": "SINGLE"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "float32_nonnullable",
+        "type": {
+          "name": "floatingpoint",
+          "precision": "SINGLE"
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "float64_nullable",
+        "type": {
+          "name": "floatingpoint",
+          "precision": "DOUBLE"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "float64_nonnullable",
+        "type": {
+          "name": "floatingpoint",
+          "precision": "DOUBLE"
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "binary_nullable",
+        "type": {
+          "name": "binary"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "binary_nonnullable",
+        "type": {
+          "name": "binary"
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "utf8_nullable",
+        "type": {
+          "name": "utf8"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "utf8_nonnullable",
+        "type": {
+          "name": "utf8"
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "fixedsizebinary_19_nullable",
+        "type": {
+          "name": "fixedsizebinary",
+          "byteWidth": 19
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "fixedsizebinary_19_nonnullable",
+        "type": {
+          "name": "fixedsizebinary",
+          "byteWidth": 19
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "fixedsizebinary_120_nullable",
+        "type": {
+          "name": "fixedsizebinary",
+          "byteWidth": 120
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "fixedsizebinary_120_nonnullable",
+        "type": {
+          "name": "fixedsizebinary",
+          "byteWidth": 120
+        },
+        "nullable": false,
+        "children": []
+      }
+    ]
+  },
+  "batches": [
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "bool_nullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            true,
+            false,
+            true,
+            true,
+            true,
+            true,
+            true
+          ]
+        },
+        {
+          "name": "bool_nonnullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            true,
+            true,
+            false,
+            false,
+            true,
+            true,
+            true
+          ]
+        },
+        {
+          "name": "int8_nullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            -128,
+            127,
+            -122,
+            28,
+            -83,
+            84,
+            6
+          ]
+        },
+        {
+          "name": "int8_nonnullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            -128,
+            127,
+            70,
+            -103,
+            82,
+            -1,
+            32
+          ]
+        },
+        {
+          "name": "int16_nullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            -32768,
+            32767,
+            -28794,
+            -4798,
+            -9144,
+            -6471,
+            -28809
+          ]
+        },
+        {
+          "name": "int16_nonnullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            -32768,
+            32767,
+            -29297,
+            -8851,
+            -7684,
+            22714,
+            16282
+          ]
+        },
+        {
+          "name": "int32_nullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            -2147483648,
+            2147483647,
+            -279511779,
+            789939398,
+            1566952883,
+            459144944,
+            112645645
+          ]
+        },
+        {
+          "name": "int32_nonnullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            -2147483648,
+            2147483647,
+            1460305573,
+            -781342537,
+            -894485209,
+            -611261236,
+            -1733393626
+          ]
+        },
+        {
+          "name": "int64_nullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-2147483648",
+            "2147483647",
+            "-1338619198",
+            "1195898730",
+            "-196833435",
+            "65722453",
+            "227331787"
+          ]
+        },
+        {
+          "name": "int64_nonnullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-2147483648",
+            "2147483647",
+            "134673412",
+            "-1352599527",
+            "799755210",
+            "-1393335016",
+            "-1401944839"
+          ]
+        },
+        {
+          "name": "uint8_nullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            0,
+            255,
+            120,
+            36,
+            174,
+            74,
+            109
+          ]
+        },
+        {
+          "name": "uint8_nonnullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            0,
+            255,
+            0,
+            157,
+            120,
+            198,
+            153
+          ]
+        },
+        {
+          "name": "uint16_nullable",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            0,
+            65535,
+            58782,
+            40076,
+            39160,
+            7864,
+            28844
+          ]
+        },
+        {
+          "name": "uint16_nonnullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            0,
+            65535,
+            19124,
+            52442,
+            22182,
+            56048,
+            17204
+          ]
+        },
+        {
+          "name": "uint32_nullable",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1
+          ],
+          "DATA": [
+            0,
+            2147483647,
+            919579300,
+            484217231,
+            723422877,
+            219335907,
+            1153201262
+          ]
+        },
+        {
+          "name": "uint32_nonnullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            0,
+            2147483647,
+            1433308719,
+            68323683,
+            1195265875,
+            2106315800,
+            348904267
+          ]
+        },
+        {
+          "name": "uint64_nullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "0",
+            "2147483647",
+            "731435646",
+            "90213947",
+            "1630338070",
+            "1558676470",
+            "1207455205"
+          ]
+        },
+        {
+          "name": "uint64_nonnullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "0",
+            "2147483647",
+            "1521875652",
+            "1531293596",
+            "277775570",
+            "618952130",
+            "1948103139"
+          ]
+        },
+        {
+          "name": "float32_nullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            138.786,
+            335.978,
+            868.094,
+            -252.928,
+            433.858,
+            -1044.852,
+            -540.461
+          ]
+        },
+        {
+          "name": "float32_nonnullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            -1645.756,
+            -1634.941,
+            -469.511,
+            489.177,
+            89.502,
+            -1068.882,
+            -1212.052
+          ]
+        },
+        {
+          "name": "float64_nullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            915.264,
+            45.397,
+            1247.676,
+            724.292,
+            -78.253,
+            -751.91,
+            1169.159
+          ]
+        },
+        {
+          "name": "float64_nonnullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            -404.256,
+            -348.975,
+            273.1,
+            691.512,
+            -448.703,
+            632.765,
+            -1195.87
+          ]
+        },
+        {
+          "name": "binary_nullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0
+          ],
+          "OFFSET": [
+            0,
+            0,
+            5,
+            7,
+            8,
+            14,
+            14,
+            14
+          ],
+          "DATA": [
+            "",
+            "51D99B869B",
+            "F442",
+            "83",
+            "7C2BEBE65E6C",
+            "",
+            ""
+          ]
+        },
+        {
+          "name": "binary_nonnullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "OFFSET": [
+            0,
+            10,
+            14,
+            15,
+            16,
+            17,
+            26,
+            26
+          ],
+          "DATA": [
+            "D6DDC948702117DC0840",
+            "533708DF",
+            "FB",
+            "EA",
+            "44",
+            "BCCD4C61F06C883F73",
+            ""
+          ]
+        },
+        {
+          "name": "utf8_nullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "OFFSET": [
+            0,
+            12,
+            12,
+            12,
+            12,
+            20,
+            27,
+            35
+          ],
+          "DATA": [
+            "\u00b0\u00c2mgw\u20ac\u00b0",
+            "",
+            "",
+            "",
+            "iah6c\u00a36",
+            "rokidwr",
+            "6ja\u00b5fa1"
+          ]
+        },
+        {
+          "name": "utf8_nonnullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "OFFSET": [
+            0,
+            10,
+            19,
+            28,
+            36,
+            44,
+            52,
+            59
+          ],
+          "DATA": [
+            "g\u77e2\u00a3k4c2",
+            "orfbb\u77e2a",
+            "m2k23\u00a3\u00b5",
+            "5rl6\u00f45d",
+            "egijr\u00b0b",
+            "jple\u00b0ch",
+            "rkiw42p"
+          ]
+        },
+        {
+          "name": "fixedsizebinary_19_nullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "50243F2281B31338EF651FCF9256388A319101",
+            "9BECDDC65F62CFAB152F965E8C5311662E972F",
+            "C7599811DDB43DFD35EAD3D3FB7FAF0AAFB9F2",
+            "619D0F38B15A69CEA060657758CD9F6820C3DE",
+            "C977E16BB93CFE804EB187E0DB4FB5C7E5E613",
+            "2339A0EE888B27EFC693DF65DD717AB624DF0F",
+            "F8F3629704844DAEF6E952A498504192D33913"
+          ]
+        },
+        {
+          "name": "fixedsizebinary_19_nonnullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "9AD533F0DD44FC7229F465337FDAC9BEF5FACA",
+            "E0EC91C117CC0C3E82BF9B288D7C3A7CB48A2E",
+            "CC930B39BCAA159A659588751DC6EEA7F3F933",
+            "2B236ECB372BF9606EC626C1CFA6928D7D0CE2",
+            "DDF3F3C8D9DAA76406FCF6D6BEA375FF474C0E",
+            "828A465C7EF01C9341782A7421C877718F2282",
+            "FC34EC8BFEEF50D51F04B14DA64D1CD180757A"
+          ]
+        },
+        {
+          "name": "fixedsizebinary_120_nullable",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            "8B96BE80C69B5CEA1C7535D822E8E8B93D22836318AD95BD33F2502084DF803C94735559F3191BB6D44DCE09BB77F9AF48E1851C0483B47A8EAE37ED92ABC071E46975F17685E9DD44640414E24F9C9F150BD29B0E2CABD4FB235A6D2CD5655D647724520C47AB1FE4DC7D418CDF3F55459E6DDF5B46F726",
+            "ED0007F8435C9848C5F886245DC9C0AF5CA5796136146A967771E50DE82CA664B3C94C5AF0B10C8A6AB36B51D88E761181CEC6B8729FD6EB781255084F3A19F2CE1B0B2469B1E5B41468C125A391AED7313F8D088CB50C1B4090BA34BE96532377215DB8D02740C3577AEAB769CDC11F5C45EEFA54607F95",
+            "E8253141328AE443B2DBC4217FFC251D7C8501924A54071DDEDF06D249F88119161DE1B4CFA929868E965E2EA102BBECA7908180C867F24E5CE1519DD734BC1E8EC77F12D9BF2876077B2A497E03B46B0203FB7BB9173B5A401B4A9D8DA152A13CC5AC218B0BED3131DE47F99909103754CB890104DCB4CD",
+            "EE2CA14099A48D8E69A3EF6FBC20389F83F39692BD5E2490D25542EACB623FB79578CDD66C40A26B7732D1EFD5B5930E851F9E8C1E18C18455DA76D8BDB1562F7C6704F071B7707863AB91C9B3D10B5DB17BB451F3EF1FF1F1F2DD4AC2FB10FE6CB43E8F00EDDC5AF4785E7C4EED05BC5FEA73B9D08FDFB8",
+            "277EB8E2CDCE71E471B8AF106BC682F76FA50F580EA09031E797D68464736EA29C50326845A935E1E8EFE77387B4DC4424DC1AA84834BB07779869BF50059953020034FF201E10CC710C76F87366435FD20AE6292712BB4A746ED0B8D263D3BE2A8E8DE8AF54E0667D04B1F8B3147206C74BECFD010320FC",
+            "784E3DF66A677FD1D5D30EC37B3A9D8E77D876D1FE23EB4EDA9B721CF62D0C1AACBAEE2A4AF953547E472AF6D5F78BAE10D6FA7A2ED19F0C3D716F9C955157245EA7BAED58F6473DD03B6891CAA7AD551B1847A5C196A98716782C4C8479F0ED5BF5E505F01C0A772FA1805E2E7A81BC1409AC4D33CA25C7",
+            "51BD3890033C88951DE28101F9A5B90D77083C4342065B86B5D7057F0E07827E70F398DAD0CF60AEAF30CADF988F6CD096F1B58AC48DBB4928BB0831A10D6180F300CBE31AF3D8388C8AF52EEA5DD317A213DFE69F53516DD90A2EDC015B29E73C07ED04C2DABC2EB63C42C474D26DCC4CBAD785D39533D3"
+          ]
+        },
+        {
+          "name": "fixedsizebinary_120_nonnullable",
+          "count": 7,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "E6439C34B42D20E636A6263B095E98EE7BFE0B33DFFF009EA4B9A14E15E5C7898D0408BB4FB41E73E50A25DE7210FDAF36B1DC57ACD9516236DE6C2B1D6B22618EE24EE657C4DE5001CF05088BCBDFD3B9B1048329DF8A9DF6AAF7A4747461A11E1A425D9007217C9542B7F93DAA9766338C4276A5D00B79",
+            "D721489C85BF19A80DB99F84C529B900677A8647FFB6704A33914E8A80448E50143E2F7805F54D52B4B5C6920D57023D1B7FC4B8C4720F8768A9455D35F28097709CB8BE7EED669F20FFF7D8A11903AD66C61F1FC4D3761A487010F8164A20C02F7381F08C335DF4EF28E7E07E8CD1956B73C3805728CCF7",
+            "D29C90A96829D519E24EA6E91C59CA3484BCD7E3E7ABD9FEA6358355C67A6F1E903466D1DCCF99ED5205E19C0B3E744827B1F1F4BF1F1438F818482591EC4CED55A21B0F8FE49E18E6CF8BC17920F62710F193A9D7CE786D4FFEA5AB2F86FC375B25239203F9CEF85144252D8D51D953A0AD549D82FF24E1",
+            "665D643C48F33A6DC1C6D2A182F693BE49729C1976886AA5709E10B47992D4AC81682FD815447EC072852A112C5E056B21ACA45C4259DC6101893EF693CC83AB8023D431FD226C30DD9D543004D4655F18F1176B0A216FA425482244AB58E7B35C98E87A67D24723E17FB6009CD7520C1DDD6965F028CECF",
+            "B6F739AA817BF09B1D324D2ED41CAC7BF2BB48DFA36539661EB015619365D109D56DD3B0FC0D6D1F88A3C14D20E130CE3F2AC7544983145B33D0F431556093958077967AC3420545A786C18F3C3C8A3358D5EB4EFE6829F4A7E989810578C9A1B2CEDC6A4154DE1138497E7EF69DCBCC96FC28DD50CAB9B2",
+            "FF506B98BF6FB212E31A785A4B31BF897BBA4043C90FA9BE028183FB7A00B27055D4524EA9996A5CBC8C1A1EFC85D24A88385064245D9F694B504FCAB55A6DA2462FA3A0238990C3EBED78291272520D8C4DF6A8AB930562A0372C15CE02C89CF157819C8C8C1D39767BC413B64587EE41B95145B4C458E3",
+            "EF0D024C1652A39F750EEEDC0C116634A844F238724C6825D30D228DA51CC7CFFA0F249B46A72B3A50C80593D070C91424625A738F810238EBE5DB26B3ED0D3358E3E762EDB538FB7A6CEBEDAA0EC2606061DC7AFD2F2707AC1EE64D25053BCE5351F15C297CC17382C81C1F6AEBBACBB54C6452E0B28F94"
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "bool_nullable",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            false,
+            false,
+            false,
+            true,
+            true,
+            false,
+            true,
+            true,
+            true,
+            false
+          ]
+        },
+        {
+          "name": "bool_nonnullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            true,
+            true,
+            false,
+            false,
+            true,
+            true,
+            false,
+            true,
+            false,
+            true
+          ]
+        },
+        {
+          "name": "int8_nullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            -128,
+            127,
+            -65,
+            89,
+            -62,
+            -22,
+            -34,
+            65,
+            120,
+            15
+          ]
+        },
+        {
+          "name": "int8_nonnullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            -128,
+            127,
+            123,
+            70,
+            100,
+            -36,
+            113,
+            -108,
+            -6,
+            86
+          ]
+        },
+        {
+          "name": "int16_nullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            -32768,
+            32767,
+            9740,
+            -7733,
+            2637,
+            -2516,
+            -17569,
+            11730,
+            -2908,
+            -8593
+          ]
+        },
+        {
+          "name": "int16_nonnullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            -32768,
+            32767,
+            11718,
+            15880,
+            -6679,
+            -22888,
+            23270,
+            18088,
+            20637,
+            -15533
+          ]
+        },
+        {
+          "name": "int32_nullable",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            -2147483648,
+            2147483647,
+            477524758,
+            -606134564,
+            393807314,
+            -128112077,
+            717176459,
+            -1600977792,
+            978954235,
+            1860414687
+          ]
+        },
+        {
+          "name": "int32_nonnullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            -2147483648,
+            2147483647,
+            -829757574,
+            -578351845,
+            -1348186787,
+            1890352675,
+            1086502698,
+            855304232,
+            181164543,
+            1057305604
+          ]
+        },
+        {
+          "name": "int64_nullable",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "DATA": [
+            "-2147483648",
+            "2147483647",
+            "794060722",
+            "198709512",
+            "-8995664",
+            "-620701969",
+            "1130202008",
+            "415910253",
+            "-84549711",
+            "1106943323"
+          ]
+        },
+        {
+          "name": "int64_nonnullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "-2147483648",
+            "2147483647",
+            "674480972",
+            "1486723112",
+            "633112390",
+            "245208998",
+            "-1174152026",
+            "-321990725",
+            "-902510995",
+            "2067119255"
+          ]
+        },
+        {
+          "name": "uint8_nullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            0,
+            255,
+            239,
+            62,
+            105,
+            206,
+            159,
+            158,
+            131,
+            187
+          ]
+        },
+        {
+          "name": "uint8_nonnullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            0,
+            255,
+            228,
+            195,
+            119,
+            246,
+            233,
+            73,
+            227,
+            240
+          ]
+        },
+        {
+          "name": "uint16_nullable",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            0,
+            65535,
+            45066,
+            6178,
+            34711,
+            47964,
+            27512,
+            44887,
+            24908,
+            55701
+          ]
+        },
+        {
+          "name": "uint16_nonnullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            0,
+            65535,
+            34339,
+            47690,
+            52236,
+            13,
+            16102,
+            11524,
+            57102,
+            29731
+          ]
+        },
+        {
+          "name": "uint32_nullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            0,
+            2147483647,
+            1647048552,
+            2065866351,
+            1252022470,
+            1848990303,
+            16726832,
+            597785424,
+            1240213577,
+            867494515
+          ]
+        },
+        {
+          "name": "uint32_nonnullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            0,
+            2147483647,
+            1852519148,
+            672549034,
+            1636451193,
+            2103372287,
+            680709579,
+            969728024,
+            959488127,
+            712543217
+          ]
+        },
+        {
+          "name": "uint64_nullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0
+          ],
+          "DATA": [
+            "0",
+            "2147483647",
+            "706989612",
+            "89417634",
+            "753617711",
+            "863868960",
+            "844763261",
+            "1277577256",
+            "520788201",
+            "1438973022"
+          ]
+        },
+        {
+          "name": "uint64_nonnullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "0",
+            "2147483647",
+            "343654149",
+            "1874151572",
+            "1405305125",
+            "1695099154",
+            "1823719899",
+            "1099199355",
+            "649954155",
+            "1371446104"
+          ]
+        },
+        {
+          "name": "float32_nullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            0,
+            0
+          ],
+          "DATA": [
+            -1963.405,
+            -521.792,
+            -1054.599,
+            -908.552,
+            478.058,
+            689.984,
+            -728.344,
+            -328.966,
+            -992.419,
+            357.03
+          ]
+        },
+        {
+          "name": "float32_nonnullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            -3537.833,
+            887.934,
+            -844.73,
+            -564.775,
+            -1264.466,
+            45.573,
+            -1352.096,
+            -1115.417,
+            62.628,
+            19.499
+          ]
+        },
+        {
+          "name": "float64_nullable",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            1,
+            1,
+            0
+          ],
+          "DATA": [
+            2290.58,
+            1382.481,
+            304.522,
+            919.074,
+            -224.893,
+            941.403,
+            746.143,
+            -945.379,
+            -335.496,
+            -92.736
+          ]
+        },
+        {
+          "name": "float64_nonnullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            1537.239,
+            1200.259,
+            900.506,
+            -183.952,
+            1331.255,
+            2447.965,
+            -394.255,
+            -1930.895,
+            -85.89,
+            -67.091
+          ]
+        },
+        {
+          "name": "binary_nullable",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            0,
+            1,
+            0,
+            0,
+            1,
+            0
+          ],
+          "OFFSET": [
+            0,
+            0,
+            9,
+            11,
+            11,
+            11,
+            13,
+            13,
+            13,
+            13,
+            13
+          ],
+          "DATA": [
+            "",
+            "EA766EF95797A0C9B8",
+            "84D5",
+            "",
+            "",
+            "AD57",
+            "",
+            "",
+            "",
+            ""
+          ]
+        },
+        {
+          "name": "binary_nonnullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "OFFSET": [
+            0,
+            0,
+            1,
+            3,
+            4,
+            6,
+            7,
+            10,
+            12,
+            13,
+            14
+          ],
+          "DATA": [
+            "",
+            "3D",
+            "9E86",
+            "EA",
+            "C091",
+            "AD",
+            "2D3828",
+            "3EE2",
+            "84",
+            "F8"
+          ]
+        },
+        {
+          "name": "utf8_nullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            1,
+            1,
+            1
+          ],
+          "OFFSET": [
+            0,
+            10,
+            10,
+            19,
+            28,
+            39,
+            39,
+            49,
+            59,
+            73,
+            83
+          ],
+          "DATA": [
+            "f5\u77e2jnm\u00a3",
+            "",
+            "lnc\u77e2bbe",
+            "a\u77e2r2pd1",
+            "2d\u00f4\u00c2e\u20acg",
+            "",
+            "\u00b5\u00c2rh\u00c2rn",
+            "g\u00b5e\u00f4h\u00f4r",
+            "\u00a32\u77e2a\u77e2\u00b5\u00c2",
+            "m\u00f4\u00f4\u00b56in"
+          ]
+        },
+        {
+          "name": "utf8_nonnullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "OFFSET": [
+            0,
+            10,
+            19,
+            26,
+            37,
+            46,
+            57,
+            65,
+            77,
+            86,
+            95
+          ],
+          "DATA": [
+            "5j\u00a3n\u77e265",
+            "pdwf\u00f4\u00f45",
+            "ifwnble",
+            "1g\u00c2\u00b0h\u00b0\u00c2",
+            "4e\u00c2fkb\u00c2",
+            "\u00a3\u20ac\u00b55omr",
+            "4f36\u00a3rp",
+            "jhw\u20ac\u20ac\u00f4c",
+            "5r\u77e2wl2h",
+            "6\u77e2bnin1"
+          ]
+        },
+        {
+          "name": "fixedsizebinary_19_nullable",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1
+          ],
+          "DATA": [
+            "C85E25A3CCE5865762DCFC6E07E23481189581",
+            "833BACCDA190F764DEDB416EE08B8BC302AB63",
+            "D7573E24B94D3B70E23743918DBE9BA3272FA5",
+            "E3AE2D60B3E7513863A71E78874BE0B714524A",
+            "833E27429BC71A6CBCC0A1349F58112AA5307E",
+            "CB3F3571BCA366F6DA6AD3B182452529BDEA1C",
+            "5B4F8DD3A1694411902AAF6C0087143581B2CB",
+            "D7C308DBC4A59CCD176076941788705CFDA2ED",
+            "D5C307EEE12A6E4D0B2AD848DC52EB252ECDEF",
+            "D778B9EB37E4D5FFC1BF753ABAEB069A5E45E2"
+          ]
+        },
+        {
+          "name": "fixedsizebinary_19_nonnullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "C75F1645509DC011AB32AC2F576CAE7ECA7B16",
+            "CB8E1A1F0A4B75B4E8E199EE6959E2C0CC64B6",
+            "9D53CFDBD99B04F4FB034256CDDAAC86CD134A",
+            "0B78B4D2F163FC63BF2F36FEC8E9466B79A3C8",
+            "D89F13F6448B6E7FD84B2E19A460AD8E105E31",
+            "CE7121C392A8560257CB0F36A3D4CEFEAA120D",
+            "540FE8D61D90A7B5984B45A082BF2842126253",
+            "97EC73B3236B90B522AAC15817626E0566780D",
+            "3A8B546C8FE7624CD4E1EE8B3E0D77D89D5EC1",
+            "3E601ED2EA586F69F56387556D17512A482A08"
+          ]
+        },
+        {
+          "name": "fixedsizebinary_120_nullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            0,
+            1,
+            1,
+            1,
+            1,
+            0,
+            0,
+            1,
+            1
+          ],
+          "DATA": [
+            "D49D31D22EB091811343B3B583AE276D5DF5BC8EA636000DE72FCBB3031FBA4D05F433EBC75124D60C56656F5411B3A0E6EECDA4346CF3105F87E7B1DEBF3C6FCF938F07B3C470DDE33436D19C07D9BA128B47554E5276D47C21C715BEF67E3802ECA55BB6759E50F130590ADEDDCD86A958F56C9D077C70",
+            "6AC6110369AFA04648213B52CD109586E85CD77D5B3F262576BC3FF98A9EC90761912FC88940F798CDA6163F3E3D7994285A903470365243F551111621F6350B078CA055001EB406B5A7C07706F2E04B2730A2335D87C151368A160EDBC0C26201EA6441990354ED302894B8ACC1A257D6DE9E17B2564898",
+            "DCE126B0DCB6195E93DC878CAF5F064BAE1184A6DBC06C2C8A1628D019BCEC17411521E3BC58A8ACBC73FCCE887B9D6E45645B7C2100AD624FF6734241676C747A3C6DC61C52D605A379F906C662215DCE3115A7BD1213B6E7F5C9AFF2D6221BFE4C0C4FF4AF9B18BA2E7D425BD8D98C0D7BDA249EA37BA8",
+            "4B18DD4F1927AD7AF60F7C11C982FCCCFD42A17849EA8654A30DDC58C1CBD641C1B70A924EA1F3D2DCF750DBF87B16CBC4E00D9617E88737A22B9F9B7D20F779381A71879FC46D591A7C0426A2BC5FF563C6F33CA0566A6E7D471033961D45F685D403F4E9D3164FE8CF5ECBF131D3EFBA371DAE1C6C2E6B",
+            "33C9DE972AD49D58C52F895B0F81EEE1E819BA427E9197F2DC3F37FD9D6B31E847016B4590BC72C60BA40C89D0647A0064C5192FD917CCBC7351650885DAA03217E8FFA1B568582AC2354499852CDCC17967F3301DE60EAB044A66AD0A0175DCE71229D5ECD83325B0D2A4D75169A38DC11F6504A8CF9F92",
+            "943EA9C4927C15AB07594F7FABFAE47061670FF75663C9B3AF5F08C8EE1523AACA9E631996D3A17644B23DC1CC6FB3157A177340CA9F93FC7D0E2C3BF907EB74AB4C5DE8B49319B86BB9457649360DA39B66BC145721A6BAF056961172EC6C93CB7CBFC9A2C682DBE8A74F8CDC20324CDAB7388480908F18",
+            "00FA841BCDD7C5D584C9C27E3577C7CC3FD1AC6C760B1914D1CBF765418EF21005AB7E065AF3C08424F7CEF98EECAE1694939728B71F750A096B3FF8F78843AC768C37FC7712A075786C7F1FC90AD316D20838693002DEA9B32B5864948709B200FF3E278B578E0B47A1E63583DAB18AABED963207642992",
+            "4DCB937A9E3144A2383FFF5C3EF93EFFD128FC3C131DD9469A2CB0E59DA7EBDBA77AD01A0979B99FBA10055567EDA3CFCA77A515BD1D4D632C6BA9B698813F1C0AFFBEFF227D140407BF5728EC943F9D3833DA1B8D2B5B60398CB219500022B1AB9312273C5275D6282EC4A4E648B6491212B954470CA6BE",
+            "A0BBC95D4894952D2E4945538AF4CE60828A1B94949143EEA065C0917BD5CDDBD68AA838AA1F75EFED37DC7256618C0CA762EE7CF75A287D529ADBFC81FD11A5F75CE23BF27940150E737E73992ACF085F52AE14E02BCEE2D229F20AEB862A8CC3D5881AE4E6267CC8B1EEF1D2AC97D4769CAD14DD4BFB96",
+            "F54B84661C426B3D0E37A455738000E50EF024258EBCE2AA3FEC5AA4E2EC3C48150B56BD44E96658C7CCB6267DB8C5484DA153FBFE0478073720BCBB65927681B36C17AE976568129E3C2FD1CBBDE89DBEE1D461FD7C6DBA455D80DC02A107855C5D04808AA164D85CF1F72EDB153685B2046E12968DB052"
+          ]
+        },
+        {
+          "name": "fixedsizebinary_120_nonnullable",
+          "count": 10,
+          "VALIDITY": [
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1
+          ],
+          "DATA": [
+            "383BB0920976B030D92B51AA60DC74922A0EB112955D1D934E19EC0CDE12CAB87C6E069439EB4F9FC105F7250BFE7DD983CC10728B8445735A7E52720FBDCD4AB7113BF52B797A05B187F572A330DABEA5E8DB5698168538775825001E7EB6CA0468F06F623A77AD99E5BB63D90A267AF243D7840B099F2C",
+            "074AC397B235ADA4A90B56B3B37D5148BEA22D6A6746B308DEA7E81A7294AEF6F3C6097A0941CA8DF01147B27040A672D5DCCF073DEB5DC0AC360E083BFB8378D7364EB2E4A09D629264BC9ED28D967184A6710424DC36D80E7D4FF499FA3CF1C0D77117CFB3E5783735F8B00D52A1243521A598F44A139F",
+            "0612AE357E65463291258B15782DA04D8B8F004F4360EB1DAB17FB57B5A5772ECCF419EE132FC94A1C0D00A2C731B887C25CDAF192BE78E6C94145421884243B234DC7A80592E70D6ECCCAEA8C08AEAE2A1AD70D786B039DA894B6FE7996F5BF6EC6670FCBBE7042430D0174D5CA9043DB7B79899A360C94",
+            "FE0A6A2B3A475AF6AA63CE24997BCAFA096B9269DA9368CEBD03444223F2497907FBD66CA071A7BEAA68D76CFD9A507280BF6108303F16558225883351E2FDF88E425DBE6ACEBE29E69CF57E41B74952391DF1B04F80363E3879A1326B9DD2D4AD94092D1FDD8EE574AC69D1F22A1BC1EC7F04BBAEC32AB7",
+            "588F15533772FD3C26415F6DC5DDB98264BEB79868908B6CB216D59A3A227E75F443A448B0D7A405AA248F6A6F049EB247B1826A02DA6F2624E16B0A899056B1D0565429434006CB11D5E3D224CA4AE7A4C19817685CB46DC0BF789AC94BF755A47B4668EBA8898CF85CD0C4F2309E97F0FE25EFB508B635",
+            "5351D21576971DFFDF48693B65895450FC098F21722B44E4CE583E969ACDF47A686AA849B265CECEAAD67CD122C10FBA7069E52326B64FD146054EE8C3F621AF28FFB2127E75FFC99B0E72B72BC615C4E9B7310C088CE5883CF41AC717DABCDF6F140B9933C831CB828EF9C02E02DC35501B527E46B1933C",
+            "6B7D83AB9AC067A38C0F2A95A4B4A3CE63C4731C729C9ED568FDA35D750262B5E7B44809D131F9F7996B2630C6075C083F31736EBA9CBB1CC62A257EBE294C55B750154E004F768EB670F648E17EB5DA40E28350B19070E924001F16846F6BC197897F189E31F30C93D43098DBE39F0EBF2E4A7FD7A25A55",
+            "26C342B09B0D6D690753743D15F78B9F2CDBAF14B95DF0D5BC9D7E1AC0C4416F3852AD0CD3D51610A6B7B6BEC766672E274C971E5692FA9AB982C3C828EA9A270BFB524EDCA50D0E8444540B82752201BDC9BA56CE6A01A04E25D5514C045515FDE12AE0B0A0533AAFD5E5FF18ABD22312DFEC854C496391",
+            "7C6D76970A57EFEC88857971E84675A3EDC47F0B2E1AA20B916601842394BC674F3CF10729DF32D10BC7BE05B74B8F68E24884637F0553A7EC84353C43B4640AC5582EBAF0827D247CE49BC95B74D842E2EE25D80741F80C3019D7EDF58CCB47112C70835F32E538F3664B5625C5FC3C761FE96E329AF03C",
+            "3A97DD53C812EBB096458C9B8E382F9404BB5F1F29FFF7973E15FB547A06F32DAFD300719AEE30E11DF794603B5D196AAAB5BE44A127590388E420367C3F542E6288A1505D83283B63417F9DBD5E2609D436773106940756DDB71910ACFD5FEF8A1F92AD3366E778FD1849C1CB2C0439B6D5719299E59D96"
+          ]
+        }
+      ]
+    }
+  ]
+}
\ No newline at end of file
diff --git a/julia/Arrow/test/arrowjson/primitive_no_batches.json b/julia/Arrow/test/arrowjson/primitive_no_batches.json
new file mode 100644
index 00000000000..e9eac55ae47
--- /dev/null
+++ b/julia/Arrow/test/arrowjson/primitive_no_batches.json
@@ -0,0 +1,287 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "bool_nullable",
+        "type": {
+          "name": "bool"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "bool_nonnullable",
+        "type": {
+          "name": "bool"
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "int8_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 8
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "int8_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 8
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "int16_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 16
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "int16_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 16
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "int32_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 32
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "int32_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 32
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "int64_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 64
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "int64_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": true,
+          "bitWidth": 64
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "uint8_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 8
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "uint8_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 8
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "uint16_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 16
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "uint16_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 16
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "uint32_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 32
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "uint32_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 32
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "uint64_nullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 64
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "uint64_nonnullable",
+        "type": {
+          "name": "int",
+          "isSigned": false,
+          "bitWidth": 64
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "float32_nullable",
+        "type": {
+          "name": "floatingpoint",
+          "precision": "SINGLE"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "float32_nonnullable",
+        "type": {
+          "name": "floatingpoint",
+          "precision": "SINGLE"
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "float64_nullable",
+        "type": {
+          "name": "floatingpoint",
+          "precision": "DOUBLE"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "float64_nonnullable",
+        "type": {
+          "name": "floatingpoint",
+          "precision": "DOUBLE"
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "binary_nullable",
+        "type": {
+          "name": "binary"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "binary_nonnullable",
+        "type": {
+          "name": "binary"
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "utf8_nullable",
+        "type": {
+          "name": "utf8"
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "utf8_nonnullable",
+        "type": {
+          "name": "utf8"
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "fixedsizebinary_19_nullable",
+        "type": {
+          "name": "fixedsizebinary",
+          "byteWidth": 19
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "fixedsizebinary_19_nonnullable",
+        "type": {
+          "name": "fixedsizebinary",
+          "byteWidth": 19
+        },
+        "nullable": false,
+        "children": []
+      },
+      {
+        "name": "fixedsizebinary_120_nullable",
+        "type": {
+          "name": "fixedsizebinary",
+          "byteWidth": 120
+        },
+        "nullable": true,
+        "children": []
+      },
+      {
+        "name": "fixedsizebinary_120_nonnullable",
+        "type": {
+          "name": "fixedsizebinary",
+          "byteWidth": 120
+        },
+        "nullable": false,
+        "children": []
+      }
+    ]
+  },
+  "batches": []
+}
\ No newline at end of file
diff --git a/julia/Arrow/test/integrationtest.jl b/julia/Arrow/test/integrationtest.jl
new file mode 100644
index 00000000000..7bca4f64228
--- /dev/null
+++ b/julia/Arrow/test/integrationtest.jl
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+using Arrow, Tables, Test
+
+include(joinpath(dirname(pathof(Arrow)), "../test/arrowjson.jl"))
+# using .ArrowJSON
+
+function runcommand(jsonname, arrowname, mode, verbose)
+    if jsonname == ""
+        error("must provide json file name")
+    end
+    if arrowname == ""
+        error("must provide arrow file name")
+    end
+
+    if mode == "ARROW_TO_JSON"
+        tbl = Arrow.Table(arrowname)
+        df = ArrowJSON.DataFile(tbl)
+        open(jsonname, "w") do io
+            JSON3.write(io, df)
+        end
+    elseif mode == "JSON_TO_ARROW"
+        df = ArrowJSON.parsefile(jsonname)
+        open(arrowname, "w") do io
+            Arrow.write(io, df)
+        end
+    elseif mode == "VALIDATE"
+        df = ArrowJSON.parsefile(jsonname)
+        tbl = Arrow.Table(arrowname)
+        @test isequal(df, tbl)
+    else
+        error("unknown integration test mode: $mode")
+    end
+    return
+end
diff --git a/julia/Arrow/test/pyarrow_roundtrip.jl b/julia/Arrow/test/pyarrow_roundtrip.jl
new file mode 100644
index 00000000000..537ff9200b1
--- /dev/null
+++ b/julia/Arrow/test/pyarrow_roundtrip.jl
@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ENV["PYTHON"] = "python3"
+import PyCall
+pa = PyCall.pyimport("pyarrow")
+include(joinpath(dirname(pathof(Arrow)), "../test/testtables.jl"))
+
+for (nm, t, writekw, readkw, extratests) in testtables
+    nm == "unions" && continue
+    println("pyarrow roundtrip: $nm")
+    io = IOBuffer()
+    Arrow.write(io, t; writekw...)
+    seekstart(io)
+    buf = PyCall.pybytes(take!(io))
+    reader = pa.ipc.open_stream(buf)
+    sink = pa.BufferOutputStream()
+    writer = pa.ipc.new_stream(sink, reader.schema)
+    for batch in reader
+        writer.write_batch(batch)
+    end
+    writer.close()
+    buf = sink.getvalue()
+    jbytes = copy(reinterpret(UInt8, buf))
+    tt = Arrow.Table(jbytes)
+    println("pyarrow roundtrip w/ compression: $nm")
+    io = IOBuffer()
+    Arrow.write(io, t; compress=((:lz4, :zstd)[rand(1:2)]), writekw...)
+    seekstart(io)
+    buf = PyCall.pybytes(take!(io))
+    reader = pa.ipc.open_stream(buf)
+    sink = pa.BufferOutputStream()
+    writer = pa.ipc.new_stream(sink, reader.schema)
+    for batch in reader
+        writer.write_batch(batch)
+    end
+    writer.close()
+    buf = sink.getvalue()
+    jbytes = copy(reinterpret(UInt8, buf))
+    tt = Arrow.Table(jbytes)
+end
+
+f1 = pa.field("f1", pa.float64(), true)
+f2 = pa.field("f2", pa.int64(), false)
+fu = pa.field("col1", pa.union([f1, f2], "dense"))
+sch = pa.schema([fu])
+
+xs = pa.array([2.0, 4.0, PyCall.pynothing[]], type=pa.float64())
+ys = pa.array([1, 3], type=pa.int64())
+types = pa.array([0, 1, 0, 1, 1], type=pa.int8())
+offsets = pa.array([0, 0, 1, 1, 2], type=pa.int32())
+union_arr = pa.UnionArray.from_dense(types, offsets, [xs, ys])
+data = [union_arr]
+batch = pa.record_batch(data, names=["col1"])
+sink = pa.BufferOutputStream()
+writer = pa.ipc.new_stream(sink, batch.schema)
+writer.write_batch(batch)
+writer.close()
+buf = sink.getvalue()
+jbytes = copy(reinterpret(UInt8, buf))
+tt = Arrow.Table(jbytes)
\ No newline at end of file
diff --git a/julia/Arrow/test/runtests.jl b/julia/Arrow/test/runtests.jl
new file mode 100644
index 00000000000..9efc10067c2
--- /dev/null
+++ b/julia/Arrow/test/runtests.jl
@@ -0,0 +1,162 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+using Test, Arrow, Tables, Dates, PooledArrays, TimeZones
+
+include(joinpath(dirname(pathof(Arrow)), "../test/testtables.jl"))
+include(joinpath(dirname(pathof(Arrow)), "../test/integrationtest.jl"))
+
+@testset "Arrow" begin
+
+@testset "table roundtrips" begin
+
+for case in testtables
+    testtable(case...)
+end
+
+end # @testset "table roundtrips"
+
+@testset "arrow json integration tests" begin
+
+for file in readdir(joinpath(dirname(pathof(Arrow)), "../test/arrowjson"))
+    jsonfile = joinpath(joinpath(dirname(pathof(Arrow)), "../test/arrowjson"), file)
+    println("integration test for $jsonfile")
+    df = ArrowJSON.parsefile(jsonfile);
+    io = IOBuffer()
+    Arrow.write(io, df)
+    seekstart(io)
+    tbl = Arrow.Table(io; convert=false);
+    @test isequal(df, tbl)
+end
+
+end # @testset "arrow json integration tests"
+
+@testset "misc" begin
+
+# multiple record batches
+t = Tables.partitioner(((col1=Union{Int64, Missing}[1,2,3,4,5,6,7,8,9,missing],), (col1=Union{Int64, Missing}[1,2,3,4,5,6,7,8,9,missing],)))
+io = IOBuffer()
+Arrow.write(io, t)
+seekstart(io)
+tt = Arrow.Table(io)
+@test length(tt) == 1
+@test isequal(tt.col1, vcat([1,2,3,4,5,6,7,8,9,missing], [1,2,3,4,5,6,7,8,9,missing]))
+@test eltype(tt.col1) === Union{Int64, Missing}
+
+# Arrow.Stream
+seekstart(io)
+str = Arrow.Stream(io)
+state = iterate(str)
+@test state !== nothing
+tt, st = state
+@test length(tt) == 1
+@test isequal(tt.col1, [1,2,3,4,5,6,7,8,9,missing])
+
+state = iterate(str, st)
+@test state !== nothing
+tt, st = state
+@test length(tt) == 1
+@test isequal(tt.col1, [1,2,3,4,5,6,7,8,9,missing])
+
+@test iterate(str, st) === nothing
+
+# dictionary batch isDelta
+t = (
+    col1=Int64[1,2,3,4],
+    col2=Union{String, Missing}["hey", "there", "sailor", missing],
+    col3=NamedTuple{(:a, :b), Tuple{Int64, Union{Missing, NamedTuple{(:c,), Tuple{String}}}}}[(a=Int64(1), b=missing), (a=Int64(1), b=missing), (a=Int64(3), b=(c="sailor",)), (a=Int64(4), b=(c="jo-bob",))]
+)
+t2 = (
+    col1=Int64[1,2,5,6],
+    col2=Union{String, Missing}["hey", "there", "sailor2", missing],
+    col3=NamedTuple{(:a, :b), Tuple{Int64, Union{Missing, NamedTuple{(:c,), Tuple{String}}}}}[(a=Int64(1), b=missing), (a=Int64(1), b=missing), (a=Int64(5), b=(c="sailor2",)), (a=Int64(4), b=(c="jo-bob",))]
+)
+tt = Tables.partitioner((t, t2))
+io = IOBuffer()
+Arrow.write(io, tt; dictencode=true, dictencodenested=true)
+seekstart(io)
+tt = Arrow.Table(io)
+@test tt.col1 == [1,2,3,4,1,2,5,6]
+@test isequal(tt.col2, ["hey", "there", "sailor", missing, "hey", "there", "sailor2", missing])
+@test isequal(tt.col3, vcat(NamedTuple{(:a, :b), Tuple{Int64, Union{Missing, NamedTuple{(:c,), Tuple{String}}}}}[(a=Int64(1), b=missing), (a=Int64(1), b=missing), (a=Int64(3), b=(c="sailor",)), (a=Int64(4), b=(c="jo-bob",))], NamedTuple{(:a, :b), Tuple{Int64, Union{Missing, NamedTuple{(:c,), Tuple{String}}}}}[(a=Int64(1), b=missing), (a=Int64(1), b=missing), (a=Int64(5), b=(c="sailor2",)), (a=Int64(4), b=(c="jo-bob",))]))
+
+t = (col1=Int64[1,2,3,4,5,6,7,8,9,10],)
+meta = Dict("key1" => "value1", "key2" => "value2")
+Arrow.setmetadata!(t, meta)
+meta2 = Dict("colkey1" => "colvalue1", "colkey2" => "colvalue2")
+Arrow.setmetadata!(t.col1, meta2)
+io = IOBuffer()
+Arrow.write(io, t)
+seekstart(io)
+tt = Arrow.Table(io)
+@test length(tt) == length(t)
+@test tt.col1 == t.col1
+@test eltype(tt.col1) === Int64
+@test Arrow.getmetadata(tt) == meta
+@test Arrow.getmetadata(tt.col1) == meta2
+
+# custom compressors
+lz4 = Arrow.CodecLz4.LZ4FrameCompressor(; compressionlevel=8)
+Arrow.CodecLz4.TranscodingStreams.initialize(lz4)
+t = (col1=Int64[1,2,3,4,5,6,7,8,9,10],)
+io = IOBuffer()
+Arrow.write(io, t; compress=lz4)
+seekstart(io)
+tt = Arrow.Table(io)
+@test length(tt) == length(t)
+@test all(isequal.(values(t), values(tt)))
+
+zstd = Arrow.CodecZstd.ZstdCompressor(; level=8)
+Arrow.CodecZstd.TranscodingStreams.initialize(zstd)
+t = (col1=Int64[1,2,3,4,5,6,7,8,9,10],)
+io = IOBuffer()
+Arrow.write(io, t; compress=zstd)
+seekstart(io)
+tt = Arrow.Table(io)
+@test length(tt) == length(t)
+@test all(isequal.(values(t), values(tt)))
+
+# custom alignment
+t = (col1=Int64[1,2,3,4,5,6,7,8,9,10],)
+io = IOBuffer()
+Arrow.write(io, t; alignment=64)
+seekstart(io)
+tt = Arrow.Table(io)
+@test length(tt) == length(t)
+@test all(isequal.(values(t), values(tt)))
+
+# 53
+s = "a" ^ 100
+t = (a=[SubString(s, 1:10), SubString(s, 11:20)],)
+io = IOBuffer()
+Arrow.write(io, t)
+seekstart(io)
+tt = Arrow.Table(io)
+@test tt.a == ["aaaaaaaaaa", "aaaaaaaaaa"]
+
+# 49
+@test_throws ArgumentError Arrow.Table("file_that_doesnt_exist")
+
+# 52
+t = (a=Arrow.DictEncode(string.(1:129)),)
+io = IOBuffer()
+Arrow.write(io, t)
+seekstart(io)
+tt = Arrow.Table(io)
+
+end # @testset "misc"
+
+end
diff --git a/julia/Arrow/test/testtables.jl b/julia/Arrow/test/testtables.jl
new file mode 100644
index 00000000000..d33fc9369af
--- /dev/null
+++ b/julia/Arrow/test/testtables.jl
@@ -0,0 +1,242 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+testtables = [
+  (
+    "basic",
+    (col1=Int64[1,2,3,4,5,6,7,8,9,10],),
+    NamedTuple(),
+    NamedTuple(),
+    nothing
+  ),
+  (
+    "missing values",
+    (col1=Union{Int64, Missing}[1,2,3,4,5,6,7,8,9,missing],),
+    NamedTuple(),
+    NamedTuple(),
+    nothing
+  ),
+  (
+    "primitive types",
+    (
+      col1=[missing, missing, missing, missing],
+      col2=Union{UInt8, Missing}[0, 1, 2, missing],
+      col3=Union{UInt16, Missing}[0, 1, 2, missing],
+      col4=Union{UInt32, Missing}[0, 1, 2, missing],
+      col5=Union{UInt64, Missing}[0, 1, 2, missing],
+      col6=Union{Int8, Missing}[0, 1, 2, missing],
+      col7=Union{Int16, Missing}[0, 1, 2, missing],
+      col8=Union{Int32, Missing}[0, 1, 2, missing],
+      col9=Union{Int64, Missing}[0, 1, 2, missing],
+      col10=Union{Float16, Missing}[0, 1, 2, missing],
+      col11=Union{Float32, Missing}[0, 1, 2, missing],
+      col12=Union{Float64, Missing}[0, 1, 2, missing],
+      col13=[true, false, true, missing],
+    ),
+    NamedTuple(),
+    NamedTuple(),
+    nothing
+  ),
+  (
+    "arrow date/time types",
+    (
+      col14=[zero(Arrow.Decimal{Int32(2), Int32(2)}), zero(Arrow.Decimal{Int32(2), Int32(2)}), zero(Arrow.Decimal{Int32(2), Int32(2)}), missing],
+      col15=[zero(Arrow.Date{Arrow.Meta.DateUnit.DAY, Int32}), zero(Arrow.Date{Arrow.Meta.DateUnit.DAY, Int32}), zero(Arrow.Date{Arrow.Meta.DateUnit.DAY, Int32}), missing],
+      col16=[zero(Arrow.Time{Arrow.Meta.TimeUnit.SECOND, Int32}), zero(Arrow.Time{Arrow.Meta.TimeUnit.SECOND, Int32}), zero(Arrow.Time{Arrow.Meta.TimeUnit.SECOND, Int32}), missing],
+      col17=[zero(Arrow.Timestamp{Arrow.Meta.TimeUnit.SECOND, nothing}), zero(Arrow.Timestamp{Arrow.Meta.TimeUnit.SECOND, nothing}), zero(Arrow.Timestamp{Arrow.Meta.TimeUnit.SECOND, nothing}), missing],
+      col18=[zero(Arrow.Interval{Arrow.Meta.IntervalUnit.YEAR_MONTH, Int32}), zero(Arrow.Interval{Arrow.Meta.IntervalUnit.YEAR_MONTH, Int32}), zero(Arrow.Interval{Arrow.Meta.IntervalUnit.YEAR_MONTH, Int32}), missing],
+      col19=[zero(Arrow.Duration{Arrow.Meta.TimeUnit.SECOND}), zero(Arrow.Duration{Arrow.Meta.TimeUnit.SECOND}), zero(Arrow.Duration{Arrow.Meta.TimeUnit.SECOND}), missing],
+    ),
+    NamedTuple(),
+    (convert=false,),
+    nothing
+  ),
+  (
+    "list types",
+    (
+      col1=Union{String, Missing}["hey", "there", "sailor", missing],
+      col2=Union{Vector{UInt8}, Missing}[b"hey", b"there", b"sailor", missing],
+      col3=Union{Vector{Int64}, Missing}[Int64[1], Int64[2], Int64[3], missing],
+      col4=Union{NTuple{2, Vector{Int64}},Missing}[(Int64[1], Int64[2]), missing, missing, (Int64[3], Int64[4])],
+      col5=Union{NTuple{2, UInt8}, Missing}[(0x01, 0x02), (0x03, 0x04), missing, (0x05, 0x06)],
+      col6=NamedTuple{(:a, :b), Tuple{Int64, String}}[(a=Int64(1), b="hey"), (a=Int64(2), b="there"), (a=Int64(3), b="sailor"), (a=Int64(4), b="jo-bob")],
+    ),
+    NamedTuple(),
+    NamedTuple(),
+    nothing
+  ),
+  (
+    "unions",
+    (
+      col1=Arrow.DenseUnionVector( Union{Int64, Float64, Missing}[1, 2.0, 3, 4.0, missing]),
+      col2=Arrow.SparseUnionVector(Union{Int64, Float64, Missing}[1, 2.0, 3, 4.0, missing]),
+    ),
+    NamedTuple(),
+    NamedTuple(),
+    nothing
+  ),
+  (
+    "dict encodings",
+    (
+      col1=Arrow.DictEncode(Int64[4, 5, 6]),
+    ),
+    NamedTuple(),
+    NamedTuple(),
+    function (tt)
+      col1 = copy(tt.col1)
+      @test typeof(col1) == PooledVector{Int64, Int8, Vector{Int8}}
+    end
+  ),
+  (
+    "more dict encodings",
+    (
+      col1=Arrow.DictEncode(NamedTuple{(:a, :b), Tuple{Int64, Union{String, Missing}}}[(a=Int64(1), b=missing), (a=Int64(1), b=missing), (a=Int64(3), b="sailor"), (a=Int64(4), b="jo-bob")]),
+    ),
+    NamedTuple(),
+    NamedTuple(),
+    nothing
+  ),
+  (
+    "PooledArray",
+    (
+      col1=PooledArray([4,5,6,6]),
+    ),
+    NamedTuple(),
+    NamedTuple(),
+    nothing
+  ),
+  (
+    "auto-converting types",
+    (
+      col1=[Date(2001, 1, 2), Date(2010, 10, 10), Date(2020, 12, 1)],
+      col2=[Time(1, 1, 2), Time(13, 10, 10), Time(22, 12, 1)],
+      col3=[DateTime(2001, 1, 2), DateTime(2010, 10, 10), DateTime(2020, 12, 1)],
+      col4=[ZonedDateTime(2001, 1, 2, TimeZone("America/Denver")), ZonedDateTime(2010, 10, 10, TimeZone("America/Denver")), ZonedDateTime(2020, 12, 1, TimeZone("America/Denver"))]
+    ),
+    NamedTuple(),
+    NamedTuple(),
+    nothing
+  ),
+  (
+    "Map",
+    (
+      col1=[Dict(Int32(1) => Float32(3.14)), missing],
+    ),
+    NamedTuple(),
+    NamedTuple(),
+    nothing
+  ),
+  (
+    "non-standard types",
+    (
+      col1=[:hey, :there, :sailor],
+      col2=['a', 'b', 'c'],
+      col3=Arrow.DictEncode(['a', 'a', 'b'])
+    ),
+    NamedTuple(),
+    NamedTuple(),
+    nothing
+  ),
+  (
+    "large lists",
+    (
+      col1=Union{String, Missing}["hey", "there", "sailor", missing],
+      col2=Union{Vector{UInt8}, Missing}[b"hey", b"there", b"sailor", missing],
+      col3=Union{Vector{Int64}, Missing}[Int64[1], Int64[2], Int64[3], missing],
+      col4=Union{NTuple{2, Vector{Int64}},Missing}[(Int64[1], Int64[2]), missing, missing, (Int64[3], Int64[4])],
+      col5=Union{NTuple{2, UInt8}, Missing}[(0x01, 0x02), (0x03, 0x04), missing, (0x05, 0x06)],
+      col6=NamedTuple{(:a, :b), Tuple{Int64, String}}[(a=Int64(1), b="hey"), (a=Int64(2), b="there"), (a=Int64(3), b="sailor"), (a=Int64(4), b="jo-bob")],
+    ),
+    (largelists=true,),
+    NamedTuple(),
+    nothing
+  ),
+  (
+    "dictencode keyword",
+    (
+      col1=Int64[1,2,3,4],
+      col2=Union{String, Missing}["hey", "there", "sailor", missing],
+      col3=Arrow.DictEncode(NamedTuple{(:a, :b), Tuple{Int64, Union{String, Missing}}}[(a=Int64(1), b=missing), (a=Int64(1), b=missing), (a=Int64(3), b="sailor"), (a=Int64(4), b="jo-bob")]),
+      col4=[:a, :b, :c, missing],
+      col5=[Date(2020, 1, 1) for x = 1:4]
+    ),
+    (dictencode=true,),
+    NamedTuple(),
+    nothing
+  ),
+  (
+    "nesteddictencode keyword",
+    (
+      col1=NamedTuple{(:a, :b), Tuple{Int64, Union{Missing, NamedTuple{(:c,), Tuple{String}}}}}[(a=Int64(1), b=missing), (a=Int64(1), b=missing), (a=Int64(3), b=(c="sailor",)), (a=Int64(4), b=(c="jo-bob",))],
+    ),
+    (dictencode=true, dictencodenested=true,),
+    NamedTuple(),
+    nothing
+  ),
+  (
+    "Julia unions",
+    (
+      col1=Union{Int, String}[1, "hey", 2, "ho"],
+      col2=Union{Char, NamedTuple{(:a,), Tuple{Symbol}}}['a', (a=:hey,), 'b', (a=:ho,)],
+    ),
+    (denseunions=false,),
+    NamedTuple(),
+    nothing
+  ),
+];
+
+function testtable(nm, t, writekw, readkw, extratests)
+  println("testing: $nm")
+  io = IOBuffer()
+  Arrow.write(io, t; writekw...)
+  seekstart(io)
+  tt = Arrow.Table(io; readkw...)
+  @test length(tt) == length(t)
+  @test all(isequal.(values(t), values(tt)))
+  extratests !== nothing && extratests(tt)
+  seekstart(io)
+  str = Arrow.Stream(io; readkw...)
+  tt = first(str)
+  @test length(tt) == length(t)
+  @test all(isequal.(values(t), values(tt)))
+  # compressed
+  io = IOBuffer()
+  Arrow.write(io, t; compress=((:lz4, :zstd)[rand(1:2)]), writekw...)
+  seekstart(io)
+  tt = Arrow.Table(io; readkw...)
+  @test length(tt) == length(t)
+  @test all(isequal.(values(t), values(tt)))
+  extratests !== nothing && extratests(tt)
+  seekstart(io)
+  str = Arrow.Stream(io; readkw...)
+  tt = first(str)
+  @test length(tt) == length(t)
+  @test all(isequal.(values(t), values(tt)))
+  # file
+  io = IOBuffer()
+  Arrow.write(io, t; file=true, writekw...)
+  seekstart(io)
+  tt = Arrow.Table(io; readkw...)
+  @test length(tt) == length(t)
+  @test all(isequal.(values(t), values(tt)))
+  extratests !== nothing && extratests(tt)
+  seekstart(io)
+  str = Arrow.Stream(io; readkw...)
+  tt = first(str)
+  @test length(tt) == length(t)
+  @test all(isequal.(values(t), values(tt)))
+  return
+end
\ No newline at end of file
diff --git a/matlab/CMakeLists.txt b/matlab/CMakeLists.txt
index 81bec8aadab..993497006e1 100755
--- a/matlab/CMakeLists.txt
+++ b/matlab/CMakeLists.txt
@@ -18,7 +18,7 @@
 cmake_minimum_required(VERSION 3.2)
 set(CMAKE_CXX_STANDARD 11)
 
-set(MLARROW_VERSION "2.0.0-SNAPSHOT")
+set(MLARROW_VERSION "3.0.0-SNAPSHOT")
 string(REGEX MATCH
   "^[0-9]+\\.[0-9]+\\.[0-9]+" MLARROW_BASE_VERSION "${MLARROW_VERSION}")
 
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 2a43b165d7e..5bf991f89b0 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -97,6 +97,11 @@ include(BuildUtils)
 # Cython generated code emits way to many warnings at CHECKIN and EVERYTHING
 set(BUILD_WARNING_LEVEL "PRODUCTION")
 
+# This must be synchronized with the definition in
+# cpp/cmake_modules/DefineOptions.cmake.
+set(ARROW_ARMV8_ARCH
+    "armv8-a"
+    CACHE STRING "Arm64 arch and extensions: armv8-a, armv8-a or armv8-a+crc+crypto")
 include(SetupCxxFlags)
 
 # Add common flags
@@ -174,6 +179,7 @@ else()
   set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${BUILD_SUBDIR_NAME}")
 endif()
 
+message(STATUS "Generator: ${CMAKE_GENERATOR}")
 message(STATUS "Build output directory: ${BUILD_OUTPUT_ROOT_DIRECTORY}")
 
 # where to put generated archives (.a files)
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 5ffd25c5009..ab67fced573 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -71,11 +71,6 @@ PIP="${CPYTHON_PATH}/bin/pip"
 # Put our Python first to avoid picking up an antiquated Python from CMake
 PATH="${CPYTHON_PATH}/bin:${PATH}"
 
-# XXX The Docker image doesn't include Python libs, this confuses CMake
-# (https://github.com/pypa/manylinux/issues/484)
-py_libname=$(${PYTHON_INTERPRETER} -c "import sysconfig; print(sysconfig.get_config_var('LDLIBRARY'))")
-touch ${CPYTHON_PATH}/lib/${py_libname}
-
 echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ==="
 $PIP install -r requirements-wheel-build.txt
 
@@ -87,9 +82,6 @@ export BUILD_ARROW_DATASET=ON
 export BUILD_ARROW_FLIGHT=ON
 export BUILD_ARROW_GANDIVA=OFF
 
-# ARROW-3052(wesm): ORC is being bundled until it can be added to the
-# manylinux1 image
-
 echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 ARROW_BUILD_DIR=/tmp/build-PY${PYTHON_VERSION}
 mkdir -p "${ARROW_BUILD_DIR}"
diff --git a/python/manylinux1/scripts/build_aws_sdk.sh b/python/manylinux1/scripts/build_aws_sdk.sh
index fd19c35ed8e..f33f3e4aacb 100755
--- a/python/manylinux1/scripts/build_aws_sdk.sh
+++ b/python/manylinux1/scripts/build_aws_sdk.sh
@@ -34,7 +34,7 @@ cmake .. -GNinja \
     -DCMAKE_C_FLAGS=${CFLAGS} \
     -DCMAKE_CXX_FLAGS=${CFLAGS} \
     -DCMAKE_INSTALL_PREFIX=${PREFIX} \
-    -DBUILD_ONLY='s3;core;transfer;config' \
+    -DBUILD_ONLY='s3;core;transfer;config;identity-management;sts' \
     -DBUILD_SHARED_LIBS=OFF \
     -DENABLE_CURL_LOGGING=ON \
     -DENABLE_UNITY_BUILD=ON \
diff --git a/python/manylinux1/scripts/install_cmake.sh b/python/manylinux1/scripts/install_cmake.sh
index c105aae9d74..5a51b3561c7 100755
--- a/python/manylinux1/scripts/install_cmake.sh
+++ b/python/manylinux1/scripts/install_cmake.sh
@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-export CMAKE_VERSION=3.17.3
+export CMAKE_VERSION=3.18.2.post1
 /opt/python/cp37-cp37m/bin/pip install cmake==${CMAKE_VERSION} ninja
 ln -s /opt/python/cp37-cp37m/bin/cmake /usr/bin/cmake
 ln -s /opt/python/cp37-cp37m/bin/ninja /usr/bin/ninja
diff --git a/python/manylinux201x/build_arrow.sh b/python/manylinux201x/build_arrow.sh
index 2f5134ba2f1..3913b46b680 100755
--- a/python/manylinux201x/build_arrow.sh
+++ b/python/manylinux201x/build_arrow.sh
@@ -73,11 +73,6 @@ PATH="${PATH}:${CPYTHON_PATH}"
 # Will be "manylinux2010" or "manylinux2014"
 manylinux_kind=$(${PYTHON_INTERPRETER} -c "import os; print(os.environ['AUDITWHEEL_PLAT'].split('_')[0], end='')")
 
-# XXX The Docker image doesn't include Python libs, this confuses CMake
-# (https://github.com/pypa/manylinux/issues/484)
-py_libname=$(${PYTHON_INTERPRETER} -c "import sysconfig; print(sysconfig.get_config_var('LDLIBRARY'))")
-touch ${CPYTHON_PATH}/lib/${py_libname}
-
 echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ==="
 $PIP install -r requirements-wheel-build.txt
 
@@ -89,9 +84,6 @@ export BUILD_ARROW_DATASET=ON
 export BUILD_ARROW_FLIGHT=ON
 export BUILD_ARROW_GANDIVA=OFF
 
-# ARROW-3052(wesm): ORC is being bundled until it can be added to the
-# manylinux1 image
-
 echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 ARROW_BUILD_DIR=/tmp/build-PY${PYTHON_VERSION}
 mkdir -p "${ARROW_BUILD_DIR}"
diff --git a/python/manylinux201x/scripts/build_aws_sdk.sh b/python/manylinux201x/scripts/build_aws_sdk.sh
index 0471b4d1458..8271b7467a9 100755
--- a/python/manylinux201x/scripts/build_aws_sdk.sh
+++ b/python/manylinux201x/scripts/build_aws_sdk.sh
@@ -30,7 +30,7 @@ pushd build
 cmake .. -GNinja \
     -DCMAKE_BUILD_TYPE=Release \
     -DCMAKE_INSTALL_PREFIX=${PREFIX} \
-    -DBUILD_ONLY='s3;core;transfer;config' \
+    -DBUILD_ONLY='s3;core;transfer;config;identity-management;sts' \
     -DBUILD_SHARED_LIBS=OFF \
     -DENABLE_CURL_LOGGING=ON \
     -DENABLE_UNITY_BUILD=ON \
diff --git a/python/manylinux201x/scripts/install_cmake.sh b/python/manylinux201x/scripts/install_cmake.sh
index 637ea05340e..2afef8e70c9 100755
--- a/python/manylinux201x/scripts/install_cmake.sh
+++ b/python/manylinux201x/scripts/install_cmake.sh
@@ -16,7 +16,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-/opt/python/cp37-cp37m/bin/pip install cmake ninja
+export CMAKE_VERSION=3.18.2.post1
+/opt/python/cp37-cp37m/bin/pip install cmake==${CMAKE_VERSION} ninja
 ln -s /opt/python/cp37-cp37m/bin/cmake /usr/local/bin/cmake
 ln -s /opt/python/cp37-cp37m/bin/ninja /usr/local/bin/ninja
 strip /opt/_internal/cpython-3.*/lib/python3.7/site-packages/cmake/data/bin/*
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 9e22cc013f8..bc669325c2e 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -32,6 +32,7 @@
 import gc as _gc
 import os as _os
 import sys as _sys
+import warnings as _warnings
 
 try:
     from ._generated_version import version as __version__
@@ -94,7 +95,7 @@ def show_versions():
                          float16, float32, float64,
                          binary, string, utf8,
                          large_binary, large_string, large_utf8,
-                         decimal128,
+                         decimal128, decimal256,
                          list_, large_list, map_, struct, union, dictionary,
                          field,
                          type_for_alias,
@@ -102,7 +103,7 @@ def show_versions():
                          ListType, LargeListType, MapType, FixedSizeListType,
                          UnionType,
                          TimestampType, Time32Type, Time64Type, DurationType,
-                         FixedSizeBinaryType, Decimal128Type,
+                         FixedSizeBinaryType, Decimal128Type, Decimal256Type,
                          BaseExtensionType, ExtensionType,
                          PyExtensionType, UnknownExtensionType,
                          register_extension_type, unregister_extension_type,
@@ -132,13 +133,13 @@ def show_versions():
                          DictionaryArray,
                          Date32Array, Date64Array, TimestampArray,
                          Time32Array, Time64Array, DurationArray,
-                         Decimal128Array, StructArray, ExtensionArray,
+                         Decimal128Array, Decimal256Array, StructArray, ExtensionArray,
                          scalar, NA, _NULL as NULL, Scalar,
                          NullScalar, BooleanScalar,
                          Int8Scalar, Int16Scalar, Int32Scalar, Int64Scalar,
                          UInt8Scalar, UInt16Scalar, UInt32Scalar, UInt64Scalar,
                          HalfFloatScalar, FloatScalar, DoubleScalar,
-                         Decimal128Scalar,
+                         Decimal128Scalar, Decimal256Scalar,
                          ListScalar, LargeListScalar, FixedSizeListScalar,
                          Date32Scalar, Date64Scalar,
                          Time32Scalar, Time64Scalar,
@@ -186,27 +187,80 @@ def show_versions():
 from pyarrow.lib import (deserialize_from, deserialize,
                          deserialize_components,
                          serialize, serialize_to, read_serialized,
-                         SerializedPyObject, SerializationContext,
                          SerializationCallbackError,
                          DeserializationCallbackError)
 
-from pyarrow.filesystem import FileSystem, LocalFileSystem
-
-from pyarrow.hdfs import HadoopFileSystem
 import pyarrow.hdfs as hdfs
 
 from pyarrow.ipc import serialize_pandas, deserialize_pandas
 import pyarrow.ipc as ipc
 
-
-localfs = LocalFileSystem.get_instance()
-
 from pyarrow.serialization import (default_serialization_context,
                                    register_default_serialization_handlers,
                                    register_torch_serialization_handlers)
 
 import pyarrow.types as types
 
+
+# deprecated top-level access
+
+
+from pyarrow.filesystem import FileSystem as _FileSystem
+from pyarrow.filesystem import LocalFileSystem as _LocalFileSystem
+from pyarrow.hdfs import HadoopFileSystem as _HadoopFileSystem
+
+from pyarrow.lib import SerializationContext as _SerializationContext
+from pyarrow.lib import SerializedPyObject as _SerializedPyObject
+
+
+_localfs = _LocalFileSystem._get_instance()
+
+
+_msg = (
+    "pyarrow.{0} is deprecated as of 2.0.0, please use pyarrow.fs.{1} instead."
+)
+
+_serialization_msg = (
+    "'pyarrow.{0}' is deprecated and will be removed in a future version. "
+    "Use pickle or the pyarrow IPC functionality instead."
+)
+
+_deprecated = {
+    "localfs": (_localfs, "LocalFileSystem"),
+    "FileSystem": (_FileSystem, "FileSystem"),
+    "LocalFileSystem": (_LocalFileSystem, "LocalFileSystem"),
+    "HadoopFileSystem": (_HadoopFileSystem, "HadoopFileSystem"),
+}
+
+_serialization_deprecatd = {
+    "SerializationContext": _SerializationContext,
+    "SerializedPyObject": _SerializedPyObject,
+}
+
+if _sys.version_info >= (3, 7):
+    def __getattr__(name):
+        if name in _deprecated:
+            obj, new_name = _deprecated[name]
+            _warnings.warn(_msg.format(name, new_name),
+                           DeprecationWarning, stacklevel=2)
+            return obj
+        elif name in _serialization_deprecatd:
+            _warnings.warn(_serialization_msg.format(name),
+                           DeprecationWarning, stacklevel=2)
+            return _serialization_deprecatd[name]
+
+        raise AttributeError(
+            "module 'pyarrow' has no attribute '{0}'".format(name)
+        )
+else:
+    localfs = _localfs
+    FileSystem = _FileSystem
+    LocalFileSystem = _LocalFileSystem
+    HadoopFileSystem = _HadoopFileSystem
+    SerializationContext = _SerializationContext
+    SerializedPyObject = _SerializedPyObject
+
+
 # Entry point for starting the plasma store
 
 
@@ -290,6 +344,7 @@ def _deprecate_scalar(ty, symbol):
 FixedSizeBinaryValue = _deprecate_scalar("FixedSizeBinary",
                                          FixedSizeBinaryScalar)
 Decimal128Value = _deprecate_scalar("Decimal128", Decimal128Scalar)
+Decimal256Value = _deprecate_scalar("Decimal256", Decimal256Scalar)
 UnionValue = _deprecate_scalar("Union", UnionScalar)
 StructValue = _deprecate_scalar("Struct", StructScalar)
 DictionaryValue = _deprecate_scalar("Dictionary", DictionaryScalar)
@@ -391,7 +446,7 @@ def get_symlink_path(hard_path):
         except PermissionError:
             print("Tried creating symlink {}. If you need to link to "
                   "bundled shared libraries, run "
-                  "pyarrow._setup_bundled_symlinks() as root")
+                  "pyarrow.create_library_symlinks() as root")
 
 
 def get_library_dirs():
diff --git a/python/pyarrow/_compute.pxd b/python/pyarrow/_compute.pxd
index a3b020352f7..e187ed75b69 100644
--- a/python/pyarrow/_compute.pxd
+++ b/python/pyarrow/_compute.pxd
@@ -25,13 +25,3 @@ from pyarrow.includes.libarrow cimport *
 cdef class FunctionOptions(_Weakrefable):
 
     cdef const CFunctionOptions* get_options(self) except NULL
-
-
-cdef class CastOptions(FunctionOptions):
-    cdef:
-        CCastOptions options
-
-    @staticmethod
-    cdef wrap(CCastOptions options)
-
-    cdef inline CCastOptions unwrap(self) nogil
diff --git a/python/pyarrow/_compute.pyx b/python/pyarrow/_compute.pyx
index 1b5535032f0..af9eb473cf5 100644
--- a/python/pyarrow/_compute.pyx
+++ b/python/pyarrow/_compute.pyx
@@ -17,6 +17,10 @@
 
 # cython: language_level = 3
 
+from cython.operator cimport dereference as deref
+
+from collections import namedtuple
+
 from pyarrow.lib import frombytes, tobytes, ordered_dict
 from pyarrow.lib cimport *
 from pyarrow.includes.libarrow cimport *
@@ -26,18 +30,27 @@ import numpy as np
 
 
 cdef wrap_scalar_function(const shared_ptr[CFunction]& sp_func):
+    """
+    Wrap a C++ scalar Function in a ScalarFunction object.
+    """
     cdef ScalarFunction func = ScalarFunction.__new__(ScalarFunction)
     func.init(sp_func)
     return func
 
 
 cdef wrap_vector_function(const shared_ptr[CFunction]& sp_func):
+    """
+    Wrap a C++ vector Function in a VectorFunction object.
+    """
     cdef VectorFunction func = VectorFunction.__new__(VectorFunction)
     func.init(sp_func)
     return func
 
 
 cdef wrap_scalar_aggregate_function(const shared_ptr[CFunction]& sp_func):
+    """
+    Wrap a C++ aggregate Function in a ScalarAggregateFunction object.
+    """
     cdef ScalarAggregateFunction func = (
         ScalarAggregateFunction.__new__(ScalarAggregateFunction)
     )
@@ -46,6 +59,9 @@ cdef wrap_scalar_aggregate_function(const shared_ptr[CFunction]& sp_func):
 
 
 cdef wrap_meta_function(const shared_ptr[CFunction]& sp_func):
+    """
+    Wrap a C++ meta Function in a MetaFunction object.
+    """
     cdef MetaFunction func = (
         MetaFunction.__new__(MetaFunction)
     )
@@ -54,6 +70,11 @@ cdef wrap_meta_function(const shared_ptr[CFunction]& sp_func):
 
 
 cdef wrap_function(const shared_ptr[CFunction]& sp_func):
+    """
+    Wrap a C++ Function in a Function object.
+
+    This dispatches to specialized wrappers depending on the function kind.
+    """
     if sp_func.get() == NULL:
         raise ValueError('Function was NULL')
 
@@ -97,6 +118,11 @@ cdef wrap_scalar_aggregate_kernel(const CScalarAggregateKernel* c_kernel):
 
 
 cdef class Kernel(_Weakrefable):
+    """
+    A kernel object.
+
+    Kernels handle the execution of a Function for a certain signature.
+    """
 
     def __init__(self):
         raise TypeError("Do not call {}'s constructor directly"
@@ -139,7 +165,36 @@ cdef class ScalarAggregateKernel(Kernel):
                 .format(frombytes(self.kernel.signature.get().ToString())))
 
 
+FunctionDoc = namedtuple(
+    "FunctionDoc",
+    ("summary", "description", "arg_names", "options_class"))
+
+
 cdef class Function(_Weakrefable):
+    """
+    A compute function.
+
+    A function implements a certain logical computation over a range of
+    possible input signatures.  Each signature accepts a range of input
+    types and is implemented by a given Kernel.
+
+    Functions can be of different kinds:
+
+    * "scalar" functions apply an item-wise computation over all items
+      of their inputs.  Each item in the output only depends on the values
+      of the inputs at the same position.  Examples: addition, comparisons,
+      string predicates...
+
+    * "vector" functions apply a collection-wise computation, such that
+      each item in the output may depend on the values of several items
+      in each input.  Examples: dictionary encoding, sorting, extracting
+      unique values...
+
+    * "aggregate" functions reduce the dimensionality of the inputs by
+      applying a reduction function.  Examples: sum, minmax, mode...
+
+    * "meta" functions dispatch to other functions.
+    """
     cdef:
         shared_ptr[CFunction] sp_func
         CFunction* base_func
@@ -153,13 +208,40 @@ cdef class Function(_Weakrefable):
         self.base_func = sp_func.get()
 
     def __repr__(self):
-        return """arrow.compute.Function
-kind: {}
-num_kernels: {}
-""".format(self.kind, self.num_kernels)
+        return ("arrow.compute.Function<name={}, kind={}, "
+                "arity={}, num_kernels={}>"
+                ).format(self.name, self.kind, self.arity, self.num_kernels)
+
+    def __reduce__(self):
+        # Reduction uses the global registry
+        return get_function, (self.name,)
+
+    @property
+    def name(self):
+        """
+        The function name.
+        """
+        return frombytes(self.base_func.name())
+
+    @property
+    def arity(self):
+        """
+        The function arity.
+
+        If Ellipsis (i.e. `...`) is returned, the function takes a variable
+        number of arguments.
+        """
+        cdef CArity arity = self.base_func.arity()
+        if arity.is_varargs:
+            return ...
+        else:
+            return arity.num_args
 
     @property
     def kind(self):
+        """
+        The function kind.
+        """
         cdef FunctionKind c_kind = self.base_func.kind()
         if c_kind == FunctionKind_SCALAR:
             return 'scalar'
@@ -167,15 +249,35 @@ num_kernels: {}
             return 'vector'
         elif c_kind == FunctionKind_SCALAR_AGGREGATE:
             return 'scalar_aggregate'
+        elif c_kind == FunctionKind_META:
+            return 'meta'
         else:
             raise NotImplementedError("Unknown Function::Kind")
 
+    @property
+    def _doc(self):
+        """
+        The C++-like function documentation (for internal use).
+        """
+        cdef CFunctionDoc c_doc = self.base_func.doc()
+
+        return FunctionDoc(frombytes(c_doc.summary),
+                           frombytes(c_doc.description),
+                           [frombytes(s) for s in c_doc.arg_names],
+                           frombytes(c_doc.options_class))
+
     @property
     def num_kernels(self):
+        """
+        The number of kernels implementing this function.
+        """
         return self.base_func.num_kernels()
 
     def call(self, args, FunctionOptions options=None,
              MemoryPool memory_pool=None):
+        """
+        Call the function on the given arguments.
+        """
         cdef:
             const CFunctionOptions* c_options = NULL
             CMemoryPool* pool = maybe_unbox_memory_pool(memory_pool)
@@ -204,7 +306,11 @@ cdef class ScalarFunction(Function):
         Function.init(self, sp_func)
         self.func = <const CScalarFunction*> sp_func.get()
 
-    def list_kernels(self):
+    @property
+    def kernels(self):
+        """
+        The kernels implementing this function.
+        """
         cdef vector[const CScalarKernel*] kernels = self.func.kernels()
         return [wrap_scalar_kernel(k) for k in kernels]
 
@@ -217,7 +323,11 @@ cdef class VectorFunction(Function):
         Function.init(self, sp_func)
         self.func = <const CVectorFunction*> sp_func.get()
 
-    def list_kernels(self):
+    @property
+    def kernels(self):
+        """
+        The kernels implementing this function.
+        """
         cdef vector[const CVectorKernel*] kernels = self.func.kernels()
         return [wrap_vector_kernel(k) for k in kernels]
 
@@ -230,7 +340,11 @@ cdef class ScalarAggregateFunction(Function):
         Function.init(self, sp_func)
         self.func = <const CScalarAggregateFunction*> sp_func.get()
 
-    def list_kernels(self):
+    @property
+    def kernels(self):
+        """
+        The kernels implementing this function.
+        """
         cdef vector[const CScalarAggregateKernel*] kernels = (
             self.func.kernels()
         )
@@ -245,6 +359,15 @@ cdef class MetaFunction(Function):
         Function.init(self, sp_func)
         self.func = <const CMetaFunction*> sp_func.get()
 
+    # Since num_kernels is exposed, also expose a kernels property
+
+    @property
+    def kernels(self):
+        """
+        The kernels implementing this function.
+        """
+        return []
+
 
 cdef _pack_compute_args(object values, vector[CDatum]* out):
     for val in values:
@@ -253,17 +376,32 @@ cdef _pack_compute_args(object values, vector[CDatum]* out):
 
         if isinstance(val, Array):
             out.push_back(CDatum((<Array> val).sp_array))
+            continue
         elif isinstance(val, ChunkedArray):
             out.push_back(CDatum((<ChunkedArray> val).sp_chunked_array))
+            continue
         elif isinstance(val, Scalar):
             out.push_back(CDatum((<Scalar> val).unwrap()))
+            continue
         elif isinstance(val, RecordBatch):
             out.push_back(CDatum((<RecordBatch> val).sp_batch))
+            continue
         elif isinstance(val, Table):
             out.push_back(CDatum((<Table> val).sp_table))
+            continue
         else:
-            raise TypeError("Got unexpected argument type {} "
-                            "for compute function".format(type(val)))
+            # Is it a Python scalar?
+            try:
+                scal = lib.scalar(val)
+            except Exception:
+                # Raise dedicated error below
+                pass
+            else:
+                out.push_back(CDatum((<Scalar> scal).unwrap()))
+                continue
+
+        raise TypeError("Got unexpected argument type {} "
+                        "for compute function".format(type(val)))
 
 
 cdef class FunctionRegistry(_Weakrefable):
@@ -274,10 +412,16 @@ cdef class FunctionRegistry(_Weakrefable):
         self.registry = GetFunctionRegistry()
 
     def list_functions(self):
+        """
+        Return all function names in the registry.
+        """
         cdef vector[c_string] names = self.registry.GetFunctionNames()
         return [frombytes(name) for name in names]
 
     def get_function(self, name):
+        """
+        Look up a function by name in the registry.
+        """
         cdef:
             c_string c_name = tobytes(name)
             shared_ptr[CFunction] func
@@ -293,7 +437,30 @@ def function_registry():
     return _global_func_registry
 
 
+def get_function(name):
+    """
+    Get a function by name.
+
+    The function is looked up in the global registry
+    (as returned by `function_registry()`).
+    """
+    return _global_func_registry.get_function(name)
+
+
+def list_functions():
+    """
+    Return all function names in the global registry.
+    """
+    return _global_func_registry.list_functions()
+
+
 def call_function(name, args, options=None, memory_pool=None):
+    """
+    Call a named function.
+
+    The function is looked up in the global registry
+    (as returned by `function_registry()`).
+    """
     func = _global_func_registry.get_function(name)
     return func.call(args, options=options, memory_pool=memory_pool)
 
@@ -304,13 +471,27 @@ cdef class FunctionOptions(_Weakrefable):
         raise NotImplementedError("Unimplemented base options")
 
 
-cdef class CastOptions(FunctionOptions):
+# NOTE:
+# To properly expose the constructor signature of FunctionOptions
+# subclasses, we use a two-level inheritance:
+# 1. a C extension class that implements option validation and setting
+#    (won't expose function signatures because of
+#     https://github.com/cython/cython/issues/3873)
+# 2. a Python derived class that implements the constructor
+
+cdef class _CastOptions(FunctionOptions):
+    cdef:
+        CCastOptions options
 
     __slots__ = ()  # avoid mistakingly creating attributes
 
-    def __init__(self, DataType target_type=None, allow_int_overflow=None,
-                 allow_time_truncate=None, allow_time_overflow=None,
-                 allow_float_truncate=None, allow_invalid_utf8=None):
+    cdef const CFunctionOptions* get_options(self) except NULL:
+        return &self.options
+
+    def _set_options(self, DataType target_type, allow_int_overflow,
+                     allow_time_truncate, allow_time_overflow,
+                     allow_float_truncate, allow_invalid_utf8):
+        self._set_type(target_type)
         if allow_int_overflow is not None:
             self.allow_int_overflow = allow_int_overflow
         if allow_time_truncate is not None:
@@ -322,36 +503,18 @@ cdef class CastOptions(FunctionOptions):
         if allow_invalid_utf8 is not None:
             self.allow_invalid_utf8 = allow_invalid_utf8
 
-    cdef const CFunctionOptions* get_options(self) except NULL:
-        return &self.options
-
-    @staticmethod
-    cdef wrap(CCastOptions options):
-        cdef CastOptions self = CastOptions.__new__(CastOptions)
-        self.options = options
-        return self
-
-    cdef inline CCastOptions unwrap(self) nogil:
-        return self.options
-
-    @staticmethod
-    def safe(target_type=None):
-        options = CastOptions.wrap(CCastOptions.Safe())
-        options._set_type(target_type)
-        return options
-
-    @staticmethod
-    def unsafe(target_type=None):
-        options = CastOptions.wrap(CCastOptions.Unsafe())
-        options._set_type(target_type)
-        return options
-
     def _set_type(self, target_type=None):
         if target_type is not None:
             self.options.to_type = (
                 (<DataType> ensure_type(target_type)).sp_type
             )
 
+    def _set_safe(self):
+        self.options = CCastOptions.Safe()
+
+    def _set_unsafe(self):
+        self.options = CCastOptions.Unsafe()
+
     def is_safe(self):
         return not (
             self.options.allow_int_overflow or
@@ -402,23 +565,55 @@ cdef class CastOptions(FunctionOptions):
         self.options.allow_invalid_utf8 = flag
 
 
-cdef class MatchSubstringOptions(FunctionOptions):
+class CastOptions(_CastOptions):
+
+    def __init__(self, target_type=None, *, allow_int_overflow=None,
+                 allow_time_truncate=None, allow_time_overflow=None,
+                 allow_float_truncate=None, allow_invalid_utf8=None):
+        self._set_options(target_type, allow_int_overflow,
+                          allow_time_truncate, allow_time_overflow,
+                          allow_float_truncate, allow_invalid_utf8)
+
+    @staticmethod
+    def safe(target_type=None):
+        self = CastOptions()
+        self._set_safe()
+        self._set_type(target_type)
+        return self
+
+    @staticmethod
+    def unsafe(target_type=None):
+        self = CastOptions()
+        self._set_unsafe()
+        self._set_type(target_type)
+        return self
+
+
+cdef class _MatchSubstringOptions(FunctionOptions):
     cdef:
         unique_ptr[CMatchSubstringOptions] match_substring_options
 
-    def __init__(self, pattern):
+    cdef const CFunctionOptions* get_options(self) except NULL:
+        return self.match_substring_options.get()
+
+    def _set_options(self, pattern):
         self.match_substring_options.reset(
             new CMatchSubstringOptions(tobytes(pattern)))
 
-    cdef const CFunctionOptions* get_options(self) except NULL:
-        return self.match_substring_options.get()
+
+class MatchSubstringOptions(_MatchSubstringOptions):
+    def __init__(self, pattern):
+        self._set_options(pattern)
 
 
-cdef class FilterOptions(FunctionOptions):
+cdef class _FilterOptions(FunctionOptions):
     cdef:
         CFilterOptions filter_options
 
-    def __init__(self, null_selection_behavior='drop'):
+    cdef const CFunctionOptions* get_options(self) except NULL:
+        return &self.filter_options
+
+    def _set_options(self, null_selection_behavior):
         if null_selection_behavior == 'drop':
             self.filter_options.null_selection_behavior = (
                 CFilterNullSelectionBehavior_DROP
@@ -429,20 +624,198 @@ cdef class FilterOptions(FunctionOptions):
             )
         else:
             raise ValueError(
-                '"{}" is not a valid null_selection_behavior'.format(
-                    null_selection_behavior)
-            )
+                '"{}" is not a valid null_selection_behavior'
+                .format(null_selection_behavior))
 
-    cdef const CFunctionOptions* get_options(self) except NULL:
-        return &self.filter_options
+
+class FilterOptions(_FilterOptions):
+    def __init__(self, null_selection_behavior='drop'):
+        self._set_options(null_selection_behavior)
 
 
-cdef class TakeOptions(FunctionOptions):
+cdef class _TakeOptions(FunctionOptions):
     cdef:
         CTakeOptions take_options
 
-    def __init__(self, boundscheck=True):
+    cdef const CFunctionOptions* get_options(self) except NULL:
+        return &self.take_options
+
+    def _set_options(self, boundscheck):
         self.take_options.boundscheck = boundscheck
 
+
+class TakeOptions(_TakeOptions):
+    def __init__(self, *, boundscheck=True):
+        self._set_options(boundscheck)
+
+
+cdef class _PartitionNthOptions(FunctionOptions):
+    cdef:
+        unique_ptr[CPartitionNthOptions] partition_nth_options
+
     cdef const CFunctionOptions* get_options(self) except NULL:
-        return &self.take_options
+        return self.partition_nth_options.get()
+
+    def _set_options(self, int64_t pivot):
+        self.partition_nth_options.reset(new CPartitionNthOptions(pivot))
+
+
+class PartitionNthOptions(_PartitionNthOptions):
+    def __init__(self, int64_t pivot):
+        self._set_options(pivot)
+
+
+cdef class _MinMaxOptions(FunctionOptions):
+    cdef:
+        CMinMaxOptions min_max_options
+
+    cdef const CFunctionOptions* get_options(self) except NULL:
+        return &self.min_max_options
+
+    def _set_options(self, null_handling):
+        if null_handling == 'skip':
+            self.min_max_options.null_handling = CMinMaxMode_SKIP
+        elif null_handling == 'emit_null':
+            self.min_max_options.null_handling = CMinMaxMode_EMIT_NULL
+        else:
+            raise ValueError(
+                '{!r} is not a valid null_handling'
+                .format(null_handling))
+
+
+class MinMaxOptions(_MinMaxOptions):
+    def __init__(self, null_handling='skip'):
+        self._set_options(null_handling)
+
+
+cdef class _CountOptions(FunctionOptions):
+    cdef:
+        CCountOptions count_options
+
+    cdef const CFunctionOptions* get_options(self) except NULL:
+        return &self.count_options
+
+    def _set_options(self, count_mode):
+        if count_mode == 'count_null':
+            self.count_options.count_mode = CCountMode_COUNT_NULL
+        elif count_mode == 'count_non_null':
+            self.count_options.count_mode = CCountMode_COUNT_NON_NULL
+        else:
+            raise ValueError(
+                '{!r} is not a valid count_mode'
+                .format(count_mode))
+
+
+class CountOptions(_CountOptions):
+    def __init__(self, count_mode='count_non_null'):
+        self._set_options(count_mode)
+
+
+cdef class _SetLookupOptions(FunctionOptions):
+    cdef:
+        unique_ptr[CSetLookupOptions] set_lookup_options
+        unique_ptr[CDatum] valset
+
+    cdef const CFunctionOptions* get_options(self) except NULL:
+        return self.set_lookup_options.get()
+
+    def _set_options(self, value_set, c_bool skip_null):
+        if isinstance(value_set, Array):
+            self.valset.reset(new CDatum((<Array> value_set).sp_array))
+        elif isinstance(value_set, ChunkedArray):
+            self.valset.reset(
+                new CDatum((<ChunkedArray> value_set).sp_chunked_array)
+            )
+        elif isinstance(value_set, Scalar):
+            self.valset.reset(new CDatum((<Scalar> value_set).unwrap()))
+        else:
+            raise ValueError('"{}" is not a valid value_set'.format(value_set))
+
+        self.set_lookup_options.reset(
+            new CSetLookupOptions(deref(self.valset), skip_null)
+        )
+
+
+class SetLookupOptions(_SetLookupOptions):
+    def __init__(self, *, value_set, skip_null=False):
+        self._set_options(value_set, skip_null)
+
+
+cdef class _StrptimeOptions(FunctionOptions):
+    cdef:
+        unique_ptr[CStrptimeOptions] strptime_options
+        TimeUnit time_unit
+
+    cdef const CFunctionOptions* get_options(self) except NULL:
+        return self.strptime_options.get()
+
+    def _set_options(self, format, unit):
+        if unit == 's':
+            self.time_unit = TimeUnit_SECOND
+        elif unit == 'ms':
+            self.time_unit = TimeUnit_MILLI
+        elif unit == 'us':
+            self.time_unit = TimeUnit_MICRO
+        elif unit == 'ns':
+            self.time_unit = TimeUnit_NANO
+        else:
+            raise ValueError('"{}" is not a valid time unit'.format(unit))
+
+        self.strptime_options.reset(
+            new CStrptimeOptions(tobytes(format), self.time_unit)
+        )
+
+
+class StrptimeOptions(_StrptimeOptions):
+    def __init__(self, format, unit):
+        self._set_options(format, unit)
+
+
+cdef class _VarianceOptions(FunctionOptions):
+    cdef:
+        CVarianceOptions variance_options
+
+    cdef const CFunctionOptions* get_options(self) except NULL:
+        return &self.variance_options
+
+    def _set_options(self, ddof):
+        self.variance_options.ddof = ddof
+
+
+class VarianceOptions(_VarianceOptions):
+    def __init__(self, *, ddof=0):
+        self._set_options(ddof)
+
+
+cdef class _SplitOptions(FunctionOptions):
+    cdef:
+        unique_ptr[CSplitOptions] split_options
+
+    cdef const CFunctionOptions* get_options(self) except NULL:
+        return self.split_options.get()
+
+    def _set_options(self, max_splits, reverse):
+        self.split_options.reset(
+            new CSplitOptions(max_splits, reverse))
+
+
+class SplitOptions(_SplitOptions):
+    def __init__(self, *, max_splits=-1, reverse=False):
+        self._set_options(max_splits, reverse)
+
+
+cdef class _SplitPatternOptions(FunctionOptions):
+    cdef:
+        unique_ptr[CSplitPatternOptions] split_pattern_options
+
+    cdef const CFunctionOptions* get_options(self) except NULL:
+        return self.split_pattern_options.get()
+
+    def _set_options(self, pattern, max_splits, reverse):
+        self.split_pattern_options.reset(
+            new CSplitPatternOptions(tobytes(pattern), max_splits, reverse))
+
+
+class SplitPatternOptions(_SplitPatternOptions):
+    def __init__(self, *, pattern, max_splits=-1, reverse=False):
+        self._set_options(pattern, max_splits, reverse)
diff --git a/python/pyarrow/_csv.pyx b/python/pyarrow/_csv.pyx
index 028ddc6f43c..34c6693c51e 100644
--- a/python/pyarrow/_csv.pyx
+++ b/python/pyarrow/_csv.pyx
@@ -28,7 +28,7 @@ from collections.abc import Mapping
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *
 from pyarrow.lib cimport (check_status, Field, MemoryPool, Schema,
-                          _CRecordBatchReader, ensure_type,
+                          RecordBatchReader, ensure_type,
                           maybe_unbox_memory_pool, get_input_stream,
                           native_transcoding_input_stream,
                           pyarrow_wrap_schema, pyarrow_wrap_table,
@@ -633,7 +633,7 @@ cdef _get_convert_options(ConvertOptions convert_options,
         out[0] = convert_options.options
 
 
-cdef class CSVStreamingReader(_CRecordBatchReader):
+cdef class CSVStreamingReader(RecordBatchReader):
     """An object that reads record batches incrementally from a CSV file.
 
     Should not be instantiated directly by user code.
diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx
index c71ea8d3220..393071acb80 100644
--- a/python/pyarrow/_dataset.pyx
+++ b/python/pyarrow/_dataset.pyx
@@ -22,15 +22,21 @@
 from cpython.object cimport Py_LT, Py_EQ, Py_GT, Py_LE, Py_NE, Py_GE
 from cython.operator cimport dereference as deref
 
+import os
+
 import pyarrow as pa
 from pyarrow.lib cimport *
 from pyarrow.lib import frombytes, tobytes
 from pyarrow.includes.libarrow_dataset cimport *
 from pyarrow._fs cimport FileSystem, FileInfo, FileSelector
 from pyarrow._csv cimport ParseOptions
-from pyarrow._compute cimport CastOptions
 from pyarrow.util import _is_path_like, _stringify_path
 
+from pyarrow._parquet cimport (
+    _create_writer_properties, _create_arrow_writer_properties,
+    FileMetaData, RowGroupMetaData, ColumnChunkMetaData
+)
+
 
 def _forbid_instantiation(klass, subclasses_instead=True):
     msg = '{} is an abstract class thus cannot be initialized.'.format(
@@ -196,11 +202,13 @@ cdef class Expression(_Weakrefable):
 
     def cast(self, type, bint safe=True):
         """Explicitly change the expression's data type"""
-        cdef CastOptions options
-        options = CastOptions.safe() if safe else CastOptions.unsafe()
+        cdef CCastOptions options
+        if safe:
+            options = CCastOptions.Safe()
+        else:
+            options = CCastOptions.Unsafe()
         c_type = pyarrow_unwrap_data_type(ensure_type(type))
-        return Expression.wrap(self.expr.CastTo(c_type,
-                                                options.unwrap()).Copy())
+        return Expression.wrap(self.expr.CastTo(c_type, options).Copy())
 
     def isin(self, values):
         """Checks whether the expression is contained in values"""
@@ -353,7 +361,7 @@ cdef class Dataset(_Weakrefable):
             partition information or internal metadata found in the data
             source, e.g. Parquet statistics. Otherwise filters the loaded
             RecordBatches before yielding them.
-        batch_size : int, default 32K
+        batch_size : int, default 1M
             The maximum row count for scanned record batches. If scanned
             record batches are overflowing memory then this method can be
             called to reduce their size.
@@ -460,6 +468,8 @@ cdef class FileSystemDataset(Dataset):
     format : FileFormat
         File format of the fragments, currently only ParquetFileFormat,
         IpcFileFormat, and CsvFileFormat are supported.
+    filesystem : FileSystem
+        FileSystem of the fragments.
     root_partition : Expression, optional
         The top-level partition of the DataDataset.
     """
@@ -468,11 +478,12 @@ cdef class FileSystemDataset(Dataset):
         CFileSystemDataset* filesystem_dataset
 
     def __init__(self, fragments, Schema schema, FileFormat format,
-                 root_partition=None):
+                 FileSystem filesystem=None, root_partition=None):
         cdef:
-            FileFragment fragment
+            FileFragment fragment=None
             vector[shared_ptr[CFileFragment]] c_fragments
             CResult[shared_ptr[CDataset]] result
+            shared_ptr[CFileSystem] c_filesystem
 
         root_partition = root_partition or _true
         if not isinstance(root_partition, Expression):
@@ -486,14 +497,25 @@ cdef class FileSystemDataset(Dataset):
                 static_pointer_cast[CFileFragment, CFragment](
                     fragment.unwrap()))
 
+            if filesystem is None:
+                filesystem = fragment.filesystem
+
+        if filesystem is not None:
+            c_filesystem = filesystem.unwrap()
+
         result = CFileSystemDataset.Make(
             pyarrow_unwrap_schema(schema),
             (<Expression> root_partition).unwrap(),
-            (<FileFormat> format).unwrap(),
+            format.unwrap(),
+            c_filesystem,
             c_fragments
         )
         self.init(GetResultValue(result))
 
+    @property
+    def filesystem(self):
+        return FileSystem.wrap(self.filesystem_dataset.filesystem())
+
     cdef void init(self, const shared_ptr[CDataset]& sp):
         Dataset.init(self, sp)
         self.filesystem_dataset = <CFileSystemDataset*> sp.get()
@@ -503,6 +525,7 @@ cdef class FileSystemDataset(Dataset):
             list(self.get_fragments()),
             self.schema,
             self.format,
+            self.filesystem,
             self.partition_expression
         )
 
@@ -555,7 +578,8 @@ cdef class FileSystemDataset(Dataset):
             format.make_fragment(path, filesystem, partitions[i])
             for i, path in enumerate(paths)
         ]
-        return FileSystemDataset(fragments, schema, format, root_partition)
+        return FileSystemDataset(fragments, schema, format,
+                                 filesystem, root_partition)
 
     @property
     def files(self):
@@ -584,6 +608,44 @@ cdef shared_ptr[CExpression] _insert_implicit_casts(Expression filter,
     )
 
 
+cdef class FileWriteOptions(_Weakrefable):
+
+    cdef:
+        shared_ptr[CFileWriteOptions] wrapped
+        CFileWriteOptions* options
+
+    def __init__(self):
+        _forbid_instantiation(self.__class__)
+
+    cdef void init(self, const shared_ptr[CFileWriteOptions]& sp):
+        self.wrapped = sp
+        self.options = sp.get()
+
+    @staticmethod
+    cdef wrap(const shared_ptr[CFileWriteOptions]& sp):
+        type_name = frombytes(sp.get().type_name())
+
+        classes = {
+            'ipc': IpcFileWriteOptions,
+            'parquet': ParquetFileWriteOptions,
+        }
+
+        class_ = classes.get(type_name, None)
+        if class_ is None:
+            raise TypeError(type_name)
+
+        cdef FileWriteOptions self = class_.__new__(class_)
+        self.init(sp)
+        return self
+
+    @property
+    def format(self):
+        return FileFormat.wrap(self.options.format())
+
+    cdef inline shared_ptr[CFileWriteOptions] unwrap(self):
+        return self.wrapped
+
+
 cdef class FileFormat(_Weakrefable):
 
     cdef:
@@ -640,6 +702,13 @@ cdef class FileFormat(_Weakrefable):
                                      <shared_ptr[CSchema]>nullptr))
         return Fragment.wrap(move(c_fragment))
 
+    def make_write_options(self):
+        return FileWriteOptions.wrap(self.format.DefaultWriteOptions())
+
+    @property
+    def default_extname(self):
+        return frombytes(self.format.type_name())
+
     def __eq__(self, other):
         try:
             return self.equals(other)
@@ -732,7 +801,7 @@ cdef class Fragment(_Weakrefable):
             partition information or internal metadata found in the data
             source, e.g. Parquet statistics. Otherwise filters the loaded
             RecordBatches before yielding them.
-        batch_size : int, default 32K
+        batch_size : int, default 1M
             The maximum row count for scanned record batches. If scanned
             record batches are overflowing memory then this method can be
             called to reduce their size.
@@ -839,66 +908,56 @@ cdef class FileFragment(Fragment):
         return FileFormat.wrap(self.file_fragment.format())
 
 
-cdef class RowGroupInfo(_Weakrefable):
+class RowGroupInfo:
     """A wrapper class for RowGroup information"""
 
-    cdef:
-        CRowGroupInfo info
-
-    def __init__(self, int id):
-        cdef CRowGroupInfo info = CRowGroupInfo(id)
-        self.init(info)
-
-    cdef void init(self, CRowGroupInfo info):
-        self.info = info
-
-    @staticmethod
-    cdef wrap(CRowGroupInfo info):
-        cdef RowGroupInfo self = RowGroupInfo.__new__(RowGroupInfo)
-        self.init(info)
-        return self
-
-    @property
-    def id(self):
-        return self.info.id()
+    def __init__(self, id, metadata, schema):
+        self.id = id
+        self.metadata = metadata
+        self.schema = schema
 
     @property
     def num_rows(self):
-        return self.info.num_rows()
+        return self.metadata.num_rows
 
     @property
     def total_byte_size(self):
-        return self.info.total_byte_size()
+        return self.metadata.total_byte_size
 
     @property
     def statistics(self):
-        if not self.info.HasStatistics():
-            return None
+        def name_stats(i):
+            col = self.metadata.column(i)
 
-        cdef:
-            CStructScalar* c_statistics
-            CStructScalar* c_minmax
-
-        statistics = dict()
-        c_statistics = self.info.statistics().get()
-        for i in range(c_statistics.value.size()):
-            name = frombytes(c_statistics.type.get().field(i).get().name())
-            c_minmax = <CStructScalar*> c_statistics.value[i].get()
-
-            statistics[name] = {
-                'min': pyarrow_wrap_scalar(c_minmax.value[0]).as_py(),
-                'max': pyarrow_wrap_scalar(c_minmax.value[1]).as_py(),
+            if not col.statistics.has_min_max:
+                return None, None
+
+            name = col.path_in_schema
+            field_index = self.schema.get_field_index(name)
+            if field_index < 0:
+                return None, None
+
+            typ = self.schema.field(field_index).type
+            return col.path_in_schema, {
+                'min': pa.scalar(col.statistics.min, type=typ).as_py(),
+                'max': pa.scalar(col.statistics.max, type=typ).as_py()
             }
 
-        return statistics
+        return {
+            name: stats for name, stats
+            in map(name_stats, range(self.metadata.num_columns))
+            if stats is not None
+        }
+
+    def __repr__(self):
+        return "RowGroupInfo({})".format(self.id)
 
     def __eq__(self, other):
+        if isinstance(other, int):
+            return self.id == other
         if not isinstance(other, RowGroupInfo):
             return False
-        cdef:
-            RowGroupInfo row_group = other
-            CRowGroupInfo c_info = row_group.info
-        return self.info.Equals(c_info)
+        return self.id == other.id
 
 
 cdef class ParquetFileFragment(FileFragment):
@@ -913,10 +972,7 @@ cdef class ParquetFileFragment(FileFragment):
 
     def __reduce__(self):
         buffer = self.buffer
-        if self.row_groups is not None:
-            row_groups = [row_group.id for row_group in self.row_groups]
-        else:
-            row_groups = None
+        row_groups = [row_group.id for row_group in self.row_groups]
         return self.format.make_fragment, (
             self.path if buffer is None else buffer,
             self.filesystem,
@@ -933,12 +989,26 @@ cdef class ParquetFileFragment(FileFragment):
 
     @property
     def row_groups(self):
-        cdef:
-            vector[CRowGroupInfo] c_row_groups
-        c_row_groups = self.parquet_file_fragment.row_groups()
-        if c_row_groups.empty():
-            return None
-        return [RowGroupInfo.wrap(row_group) for row_group in c_row_groups]
+        metadata = self.metadata
+        cdef vector[int] row_groups = self.parquet_file_fragment.row_groups()
+        return [RowGroupInfo(i, metadata.row_group(i), self.physical_schema)
+                for i in row_groups]
+
+    @property
+    def metadata(self):
+        self.ensure_complete_metadata()
+        cdef FileMetaData metadata = FileMetaData()
+        metadata.init(self.parquet_file_fragment.metadata())
+        return metadata
+
+    @property
+    def num_row_groups(self):
+        """
+        Return the number of row groups viewed by this fragment (not the
+        number of row groups in the origin file).
+        """
+        self.ensure_complete_metadata()
+        return self.parquet_file_fragment.row_groups().size()
 
     def split_by_row_group(self, Expression filter=None,
                            Schema schema=None):
@@ -960,7 +1030,7 @@ cdef class ParquetFileFragment(FileFragment):
 
         Returns
         -------
-        A list of Fragment.
+        A list of Fragments
         """
         cdef:
             vector[shared_ptr[CFragment]] c_fragments
@@ -975,6 +1045,63 @@ cdef class ParquetFileFragment(FileFragment):
 
         return [Fragment.wrap(c_fragment) for c_fragment in c_fragments]
 
+    def subset(self, Expression filter=None, Schema schema=None,
+               object row_group_ids=None):
+        """
+        Create a subset of the fragment (viewing a subset of the row groups).
+
+        Subset can be specified by either a filter predicate (with optional
+        schema) or by a list of row group IDs. Note that when using a filter,
+        the resulting fragment can be empty (viewing no row groups).
+
+        Parameters
+        ----------
+        filter : Expression, default None
+            Only include the row groups which satisfy this predicate (using
+            the Parquet RowGroup statistics).
+        schema : Schema, default None
+            Schema to use when filtering row groups. Defaults to the
+            Fragment's phsyical schema
+        row_group_ids : list of ints
+            The row group IDs to include in the subset. Can only be specified
+            if `filter` is None.
+
+        Returns
+        -------
+        ParquetFileFragment
+        """
+        cdef:
+            shared_ptr[CExpression] c_filter
+            vector[int] c_row_group_ids
+            shared_ptr[CFragment] c_fragment
+
+        if filter is not None and row_group_ids is not None:
+            raise ValueError(
+                "Cannot specify both 'filter' and 'row_group_ids'."
+            )
+
+        if filter is not None:
+            schema = schema or self.physical_schema
+            c_filter = _insert_implicit_casts(filter, schema)
+            with nogil:
+                c_fragment = move(GetResultValue(
+                    self.parquet_file_fragment.SubsetWithFilter(
+                        move(c_filter))))
+        elif row_group_ids is not None:
+            c_row_group_ids = [
+                <int> row_group for row_group in sorted(set(row_group_ids))
+            ]
+            with nogil:
+                c_fragment = move(GetResultValue(
+                    self.parquet_file_fragment.SubsetWithIds(
+                        move(c_row_group_ids))))
+        else:
+            raise ValueError(
+                "Need to specify one of 'filter' or 'row_group_ids'"
+            )
+
+        return Fragment.wrap(c_fragment)
+
 
 cdef class ParquetReadOptions(_Weakrefable):
     """
@@ -991,27 +1118,38 @@ cdef class ParquetReadOptions(_Weakrefable):
     dictionary_columns : list of string, default None
         Names of columns which should be dictionary encoded as
         they are read.
+    enable_parallel_column_conversion : bool, default False
+        EXPERIMENTAL: Parallelize conversion across columns. This option is
+        ignored if a scan is already parallelized across input files to avoid
+        thread contention. This option will be removed after support is added
+        for simultaneous parallelization across files and columns.
     """
 
     cdef public:
         bint use_buffered_stream
         uint32_t buffer_size
         set dictionary_columns
+        bint enable_parallel_column_conversion
 
     def __init__(self, bint use_buffered_stream=False,
                  buffer_size=8192,
-                 dictionary_columns=None):
+                 dictionary_columns=None,
+                 bint enable_parallel_column_conversion=False):
         self.use_buffered_stream = use_buffered_stream
         if buffer_size <= 0:
             raise ValueError("Buffer size must be larger than zero")
         self.buffer_size = buffer_size
         self.dictionary_columns = set(dictionary_columns or set())
+        self.enable_parallel_column_conversion = \
+            enable_parallel_column_conversion
 
     def equals(self, ParquetReadOptions other):
         return (
             self.use_buffered_stream == other.use_buffered_stream and
             self.buffer_size == other.buffer_size and
-            self.dictionary_columns == other.dictionary_columns
+            self.dictionary_columns == other.dictionary_columns and
+            self.enable_parallel_column_conversion ==
+            other.enable_parallel_column_conversion
         )
 
     def __eq__(self, other):
@@ -1021,6 +1159,82 @@ cdef class ParquetReadOptions(_Weakrefable):
             return False
 
 
+cdef class ParquetFileWriteOptions(FileWriteOptions):
+
+    cdef:
+        CParquetFileWriteOptions* parquet_options
+        object _properties
+
+    def update(self, **kwargs):
+        arrow_fields = {
+            "use_deprecated_int96_timestamps",
+            "coerce_timestamps",
+            "allow_truncated_timestamps",
+        }
+
+        setters = set()
+        for name, value in kwargs.items():
+            if name not in self._properties:
+                raise TypeError("unexpected parquet write option: " + name)
+            self._properties[name] = value
+            if name in arrow_fields:
+                setters.add(self._set_arrow_properties)
+            else:
+                setters.add(self._set_properties)
+
+        for setter in setters:
+            setter()
+
+    def _set_properties(self):
+        cdef CParquetFileWriteOptions* opts = self.parquet_options
+
+        opts.writer_properties = _create_writer_properties(
+            use_dictionary=self._properties["use_dictionary"],
+            compression=self._properties["compression"],
+            version=self._properties["version"],
+            write_statistics=self._properties["write_statistics"],
+            data_page_size=self._properties["data_page_size"],
+            compression_level=self._properties["compression_level"],
+            use_byte_stream_split=(
+                self._properties["use_byte_stream_split"]
+            ),
+            data_page_version=self._properties["data_page_version"],
+        )
+
+    def _set_arrow_properties(self):
+        cdef CParquetFileWriteOptions* opts = self.parquet_options
+
+        opts.arrow_writer_properties = _create_arrow_writer_properties(
+            use_deprecated_int96_timestamps=(
+                self._properties["use_deprecated_int96_timestamps"]
+            ),
+            coerce_timestamps=self._properties["coerce_timestamps"],
+            allow_truncated_timestamps=(
+                self._properties["allow_truncated_timestamps"]
+            ),
+            writer_engine_version="V2",
+        )
+
+    cdef void init(self, const shared_ptr[CFileWriteOptions]& sp):
+        FileWriteOptions.init(self, sp)
+        self.parquet_options = <CParquetFileWriteOptions*> sp.get()
+        self._properties = dict(
+            use_dictionary=True,
+            compression="snappy",
+            version="1.0",
+            write_statistics=None,
+            data_page_size=None,
+            compression_level=None,
+            use_byte_stream_split=False,
+            data_page_version="1.0",
+            use_deprecated_int96_timestamps=False,
+            coerce_timestamps=None,
+            allow_truncated_timestamps=False,
+        )
+        self._set_properties()
+        self._set_arrow_properties()
+
+
 cdef class ParquetFileFormat(FileFormat):
 
     cdef:
@@ -1031,6 +1245,8 @@ cdef class ParquetFileFormat(FileFormat):
             shared_ptr[CParquetFileFormat] wrapped
             CParquetFileFormatReaderOptions* options
 
+        # Read options
+
         if read_options is None:
             read_options = ParquetReadOptions()
         elif isinstance(read_options, dict):
@@ -1043,6 +1259,8 @@ cdef class ParquetFileFormat(FileFormat):
         options = &(wrapped.get().reader_options)
         options.use_buffered_stream = read_options.use_buffered_stream
         options.buffer_size = read_options.buffer_size
+        options.enable_parallel_column_conversion = \
+            read_options.enable_parallel_column_conversion
         if read_options.dictionary_columns is not None:
             for column in read_options.dictionary_columns:
                 options.dict_columns.insert(tobytes(column))
@@ -1060,20 +1278,30 @@ cdef class ParquetFileFormat(FileFormat):
         return ParquetReadOptions(
             use_buffered_stream=options.use_buffered_stream,
             buffer_size=options.buffer_size,
-            dictionary_columns={frombytes(col) for col in options.dict_columns}
+            dictionary_columns={frombytes(col)
+                                for col in options.dict_columns},
+            enable_parallel_column_conversion=(
+                options.enable_parallel_column_conversion
+            )
         )
 
+    def make_write_options(self, **kwargs):
+        opts = FileFormat.make_write_options(self)
+        (<ParquetFileWriteOptions> opts).update(**kwargs)
+        return opts
+
     def equals(self, ParquetFileFormat other):
-        return self.read_options.equals(other.read_options)
+        return (
+            self.read_options.equals(other.read_options)
+        )
 
     def __reduce__(self):
-        return ParquetFileFormat, (self.read_options,)
+        return ParquetFileFormat, (self.read_options, )
 
     def make_fragment(self, file, filesystem=None,
                       Expression partition_expression=None, row_groups=None):
         cdef:
-            vector[int] c_row_group_ids
-            vector[CRowGroupInfo] c_row_groups
+            vector[int] c_row_groups
 
         partition_expression = partition_expression or _true
 
@@ -1082,17 +1310,22 @@ cdef class ParquetFileFormat(FileFormat):
                                          partition_expression)
 
         c_source = _make_file_source(file, filesystem)
-        c_row_group_ids = [<int> row_group for row_group in set(row_groups)]
-        c_row_groups = CRowGroupInfo.FromIdentifiers(move(c_row_group_ids))
+        c_row_groups = [<int> row_group for row_group in set(row_groups)]
 
         c_fragment = <shared_ptr[CFragment]> GetResultValue(
             self.parquet_format.MakeFragment(move(c_source),
                                              partition_expression.unwrap(),
-                                             move(c_row_groups),
-                                             <shared_ptr[CSchema]>nullptr))
+                                             <shared_ptr[CSchema]>nullptr,
+                                             move(c_row_groups)))
         return Fragment.wrap(move(c_fragment))
 
 
+cdef class IpcFileWriteOptions(FileWriteOptions):
+
+    def __init__(self):
+        _forbid_instantiation(self.__class__)
+
+
 cdef class IpcFileFormat(FileFormat):
 
     def __init__(self):
@@ -1101,6 +1334,10 @@ cdef class IpcFileFormat(FileFormat):
     def equals(self, IpcFileFormat other):
         return True
 
+    @property
+    def default_extname(self):
+        return "feather"
+
     def __reduce__(self):
         return IpcFileFormat, tuple()
 
@@ -1118,6 +1355,9 @@ cdef class CsvFileFormat(FileFormat):
         FileFormat.init(self, sp)
         self.csv_format = <CCsvFileFormat*> sp.get()
 
+    def make_write_options(self):
+        raise NotImplemented("writing CSV datasets")
+
     @property
     def parse_options(self):
         return ParseOptions.wrap(self.csv_format.parse_options)
@@ -1244,7 +1484,8 @@ cdef class DirectoryPartitioning(Partitioning):
         self.directory_partitioning = <CDirectoryPartitioning*> sp.get()
 
     @staticmethod
-    def discover(field_names, object max_partition_dictionary_size=0):
+    def discover(field_names, infer_dictionary=False,
+                 max_partition_dictionary_size=0):
         """
         Discover a DirectoryPartitioning.
 
@@ -1252,11 +1493,16 @@ cdef class DirectoryPartitioning(Partitioning):
         ----------
         field_names : list of str
             The names to associate with the values from the subdirectory names.
-        max_partition_dictionary_size : int or None, default 0
-            The maximum number of unique values to consider for dictionary
-            encoding. By default no field will be inferred as dictionary
-            encoded. If None is provided dictionary encoding will be used for
-            every string field.
+        infer_dictionary : bool, default False
+            When inferring a schema for partition fields, yield dictionary
+            encoded types instead of plain types. This can be more efficient
+            when materializing virtual columns, and Expressions parsed by the
+            finished Partitioning will include dictionaries of all unique
+            inspected values for each field.
+        max_partition_dictionary_size : int, default 0
+            Synonymous with infer_dictionary for backwards compatibility with
+            1.0: setting this to -1 or None is equivalent to passing
+            infer_dictionary=True.
 
         Returns
         -------
@@ -1264,18 +1510,21 @@ cdef class DirectoryPartitioning(Partitioning):
             To be used in the FileSystemFactoryOptions.
         """
         cdef:
-            CPartitioningFactoryOptions options
+            CPartitioningFactoryOptions c_options
             vector[c_string] c_field_names
 
-        if max_partition_dictionary_size is None:
-            max_partition_dictionary_size = -1
+        if max_partition_dictionary_size in {-1, None}:
+            infer_dictionary = True
+        elif max_partition_dictionary_size != 0:
+            raise NotImplemented("max_partition_dictionary_size must be "
+                                 "0, -1, or None")
 
-        options.max_partition_dictionary_size = \
-            int(max_partition_dictionary_size)
+        if infer_dictionary:
+            c_options.infer_dictionary = True
 
         c_field_names = [tobytes(s) for s in field_names]
         return PartitioningFactory.wrap(
-            CDirectoryPartitioning.MakeFactory(c_field_names, options))
+            CDirectoryPartitioning.MakeFactory(c_field_names, c_options))
 
 
 cdef class HivePartitioning(Partitioning):
@@ -1326,17 +1575,22 @@ cdef class HivePartitioning(Partitioning):
         self.hive_partitioning = <CHivePartitioning*> sp.get()
 
     @staticmethod
-    def discover(object max_partition_dictionary_size=0):
+    def discover(infer_dictionary=False, max_partition_dictionary_size=0):
         """
         Discover a HivePartitioning.
 
-        Params
-        ------
-        max_partition_dictionary_size : int or None, default 0
-            The maximum number of unique values to consider for dictionary
-            encoding. By default no field will be inferred as dictionary
-            encoded. If -1 is provided dictionary encoding will be used for
-            every string field.
+        Parameters
+        ----------
+        infer_dictionary : bool, default False
+            When inferring a schema for partition fields, yield dictionary
+            encoded types instead of plain. This can be more efficient when
+            materializing virtual columns, and Expressions parsed by the
+            finished Partitioning will include dictionaries of all unique
+            inspected values for each field.
+        max_partition_dictionary_size : int, default 0
+            Synonymous with infer_dictionary for backwards compatibility with
+            1.0: setting this to -1 or None is equivalent to passing
+            infer_dictionary=True.
 
         Returns
         -------
@@ -1344,16 +1598,19 @@ cdef class HivePartitioning(Partitioning):
             To be used in the FileSystemFactoryOptions.
         """
         cdef:
-            CPartitioningFactoryOptions options
+            CPartitioningFactoryOptions c_options
 
-        if max_partition_dictionary_size is None:
-            max_partition_dictionary_size = -1
+        if max_partition_dictionary_size in {-1, None}:
+            infer_dictionary = True
+        elif max_partition_dictionary_size != 0:
+            raise NotImplemented("max_partition_dictionary_size must be "
+                                 "0, -1, or None")
 
-        options.max_partition_dictionary_size = \
-            int(max_partition_dictionary_size)
+        if infer_dictionary:
+            c_options.infer_dictionary = True
 
         return PartitioningFactory.wrap(
-            CHivePartitioning.MakeFactory(options))
+            CHivePartitioning.MakeFactory(c_options))
 
 
 cdef class DatasetFactory(_Weakrefable):
@@ -1669,6 +1926,12 @@ cdef class ParquetFactoryOptions(_Weakrefable):
         have partition information.
     partitioning : Partitioning, PartitioningFactory, optional
         The partitioning scheme applied to fragments, see ``Partitioning``.
+    validate_column_chunk_paths : bool, default False
+        Assert that all ColumnChunk paths are consistent. The parquet spec
+        allows for ColumnChunk data to be stored in multiple files, but
+        ParquetDatasetFactory supports only a single file with all ColumnChunk
+        data. If this flag is set construction of a ParquetDatasetFactory will
+        raise an error if ColumnChunk data is not resident in a single file.
     """
 
     cdef:
@@ -1676,7 +1939,8 @@ cdef class ParquetFactoryOptions(_Weakrefable):
 
     __slots__ = ()  # avoid mistakingly creating attributes
 
-    def __init__(self, partition_base_dir=None, partitioning=None):
+    def __init__(self, partition_base_dir=None, partitioning=None,
+                 validate_column_chunk_paths=False):
         if isinstance(partitioning, PartitioningFactory):
             self.partitioning_factory = partitioning
         elif isinstance(partitioning, Partitioning):
@@ -1685,6 +1949,8 @@ cdef class ParquetFactoryOptions(_Weakrefable):
         if partition_base_dir is not None:
             self.partition_base_dir = partition_base_dir
 
+        self.options.validate_column_chunk_paths = validate_column_chunk_paths
+
     cdef inline CParquetFactoryOptions unwrap(self):
         return self.options
 
@@ -1730,6 +1996,17 @@ cdef class ParquetFactoryOptions(_Weakrefable):
     def partition_base_dir(self, value):
         self.options.partition_base_dir = tobytes(value)
 
+    @property
+    def validate_column_chunk_paths(self):
+        """
+        Base directory to strip paths before applying the partitioning.
+        """
+        return self.options.validate_column_chunk_paths
+
+    @validate_column_chunk_paths.setter
+    def validate_column_chunk_paths(self, value):
+        self.options.validate_column_chunk_paths = value
+
 
 cdef class ParquetDatasetFactory(DatasetFactory):
     """
@@ -1835,9 +2112,12 @@ cdef shared_ptr[CScanContext] _build_scan_context(bint use_threads=True,
     return context
 
 
+_DEFAULT_BATCH_SIZE = 2**20
+
+
 cdef void _populate_builder(const shared_ptr[CScannerBuilder]& ptr,
                             list columns=None, Expression filter=None,
-                            int batch_size=32*2**10) except *:
+                            int batch_size=_DEFAULT_BATCH_SIZE) except *:
     cdef:
         CScannerBuilder *builder
 
@@ -1875,7 +2155,7 @@ cdef class Scanner(_Weakrefable):
         partition information or internal metadata found in the data
         source, e.g. Parquet statistics. Otherwise filters the loaded
         RecordBatches before yielding them.
-    batch_size : int, default 32K
+    batch_size : int, default 1M
         The maximum row count for scanned record batches. If scanned
         record batches are overflowing memory then this method can be
         called to reduce their size.
@@ -1911,7 +2191,7 @@ cdef class Scanner(_Weakrefable):
     def from_dataset(Dataset dataset not None,
                      bint use_threads=True, MemoryPool memory_pool=None,
                      list columns=None, Expression filter=None,
-                     int batch_size=32*2**10):
+                     int batch_size=_DEFAULT_BATCH_SIZE):
         cdef:
             shared_ptr[CScanContext] context
             shared_ptr[CScannerBuilder] builder
@@ -1930,7 +2210,7 @@ cdef class Scanner(_Weakrefable):
     def from_fragment(Fragment fragment not None, Schema schema=None,
                       bint use_threads=True, MemoryPool memory_pool=None,
                       list columns=None, Expression filter=None,
-                      int batch_size=32*2**10):
+                      int batch_size=_DEFAULT_BATCH_SIZE):
         cdef:
             shared_ptr[CScanContext] context
             shared_ptr[CScannerBuilder] builder
@@ -2022,3 +2302,44 @@ def _get_partition_keys(Expression partition_expression):
         frombytes(name_val.first): pyarrow_wrap_scalar(name_val.second).as_py()
         for name_val in GetResultValue(CGetPartitionKeys(deref(expr.get())))
     }
+
+
+def _filesystemdataset_write(
+    data not None, object base_dir not None, str basename_template not None,
+    Schema schema not None, FileSystem filesystem not None,
+    Partitioning partitioning not None,
+    FileWriteOptions file_options not None, bint use_threads,
+):
+    """
+    CFileSystemDataset.Write wrapper
+    """
+    cdef:
+        CFileSystemDatasetWriteOptions c_options
+        shared_ptr[CScanner] c_scanner
+        vector[shared_ptr[CRecordBatch]] c_batches
+
+    c_options.file_write_options = file_options.unwrap()
+    c_options.filesystem = filesystem.unwrap()
+    c_options.base_dir = tobytes(_stringify_path(base_dir))
+    c_options.partitioning = partitioning.unwrap()
+    c_options.basename_template = tobytes(basename_template)
+
+    if isinstance(data, Dataset):
+        scanner = data._scanner(use_threads=use_threads)
+    else:
+        # data is list of batches/tables
+        for table in data:
+            if isinstance(table, Table):
+                for batch in table.to_batches():
+                    c_batches.push_back((<RecordBatch> batch).sp_batch)
+            else:
+                c_batches.push_back((<RecordBatch> table).sp_batch)
+
+        data = Fragment.wrap(shared_ptr[CFragment](
+            new CInMemoryFragment(move(c_batches), _true.unwrap())))
+
+        scanner = Scanner.from_fragment(data, schema, use_threads=use_threads)
+
+    c_scanner = (<Scanner> scanner).unwrap()
+    with nogil:
+        check_status(CFileSystemDataset.Write(c_options, c_scanner))
diff --git a/python/pyarrow/_flight.pyx b/python/pyarrow/_flight.pyx
index b7459d21da1..64b8548f64d 100644
--- a/python/pyarrow/_flight.pyx
+++ b/python/pyarrow/_flight.pyx
@@ -35,7 +35,7 @@ from pyarrow.lib cimport *
 from pyarrow.lib import ArrowException, ArrowInvalid
 from pyarrow.lib import as_buffer, frombytes, tobytes
 from pyarrow.includes.libarrow_flight cimport *
-from pyarrow.ipc import _ReadPandasOption, _get_legacy_format_default
+from pyarrow.ipc import _get_legacy_format_default, _ReadPandasMixin
 import pyarrow.lib as lib
 
 
@@ -51,7 +51,7 @@ cdef int check_flight_status(const CStatus& status) nogil except -1:
     detail = FlightStatusDetail.UnwrapStatus(status)
     if detail:
         with gil:
-            message = frombytes(status.message())
+            message = frombytes(status.message(), safe=True)
             detail_msg = detail.get().extra_info()
             if detail.get().code() == CFlightStatusInternal:
                 raise FlightInternalError(message, detail_msg)
@@ -72,7 +72,7 @@ cdef int check_flight_status(const CStatus& status) nogil except -1:
     size_detail = FlightWriteSizeStatusDetail.UnwrapStatus(status)
     if size_detail:
         with gil:
-            message = frombytes(status.message())
+            message = frombytes(status.message(), safe=True)
             raise FlightWriteSizeExceededError(
                 message,
                 size_detail.get().limit(), size_detail.get().actual())
@@ -812,7 +812,7 @@ cdef class FlightStreamChunk(_Weakrefable):
             self.chunk.data != NULL, self.chunk.app_metadata != NULL)
 
 
-cdef class _MetadataRecordBatchReader(_Weakrefable):
+cdef class _MetadataRecordBatchReader(_Weakrefable, _ReadPandasMixin):
     """A reader for Flight streams."""
 
     # Needs to be separate class so the "real" class can subclass the
@@ -869,8 +869,7 @@ cdef class _MetadataRecordBatchReader(_Weakrefable):
         return chunk
 
 
-cdef class MetadataRecordBatchReader(_MetadataRecordBatchReader,
-                                     _ReadPandasOption):
+cdef class MetadataRecordBatchReader(_MetadataRecordBatchReader):
     """The virtual base class for readers for Flight streams."""
 
 
@@ -1012,6 +1011,10 @@ cdef class FlightClient(_Weakrefable):
         batch that (when serialized) exceeds this limit will raise an
         exception; the client can retry the write with a smaller
         batch.
+    disable_server_verification : boolean optional, default False
+        A flag that indicates that, if the client is connecting
+        with TLS, that it skips server verification. If this is
+        enabled, all other TLS settings are overridden.
     generic_options : list optional, default None
         A list of generic (string, int or string) option tuples passed
         to the underlying transport. Effect is implementation
@@ -1022,12 +1025,13 @@ cdef class FlightClient(_Weakrefable):
 
     def __init__(self, location, *, tls_root_certs=None, cert_chain=None,
                  private_key=None, override_hostname=None, middleware=None,
-                 write_size_limit_bytes=None, generic_options=None):
+                 write_size_limit_bytes=None,
+                 disable_server_verification=None, generic_options=None):
         if isinstance(location, (bytes, str)):
             location = Location(location)
         elif isinstance(location, tuple):
             host, port = location
-            if tls_root_certs:
+            if tls_root_certs or disable_server_verification is not None:
                 location = Location.for_grpc_tls(host, port)
             else:
                 location = Location.for_grpc_tcp(host, port)
@@ -1036,11 +1040,12 @@ cdef class FlightClient(_Weakrefable):
                             'Location instance')
         self.init(location, tls_root_certs, cert_chain, private_key,
                   override_hostname, middleware, write_size_limit_bytes,
-                  generic_options)
+                  disable_server_verification, generic_options)
 
     cdef init(self, Location location, tls_root_certs, cert_chain,
               private_key, override_hostname, middleware,
-              write_size_limit_bytes, generic_options):
+              write_size_limit_bytes, disable_server_verification,
+              generic_options):
         cdef:
             int c_port = 0
             CLocation c_location = Location.unwrap(location)
@@ -1057,6 +1062,8 @@ cdef class FlightClient(_Weakrefable):
             c_options.private_key = tobytes(private_key)
         if override_hostname:
             c_options.override_hostname = tobytes(override_hostname)
+        if disable_server_verification is not None:
+            c_options.disable_server_verification = disable_server_verification
         if middleware:
             for factory in middleware:
                 c_options.middleware.push_back(
@@ -1107,13 +1114,17 @@ cdef class FlightClient(_Weakrefable):
 
     @classmethod
     def connect(cls, location, tls_root_certs=None, cert_chain=None,
-                private_key=None, override_hostname=None):
+                private_key=None, override_hostname=None,
+                disable_server_verification=None):
         warnings.warn("The 'FlightClient.connect' method is deprecated, use "
                       "FlightClient constructor or pyarrow.flight.connect "
                       "function instead")
-        return FlightClient(location, tls_root_certs=tls_root_certs,
-                            cert_chain=cert_chain, private_key=private_key,
-                            override_hostname=override_hostname)
+        return FlightClient(
+            location, tls_root_certs=tls_root_certs,
+            cert_chain=cert_chain, private_key=private_key,
+            override_hostname=override_hostname,
+            disable_server_verification=disable_server_verification
+        )
 
     def authenticate(self, auth_handler, options: FlightCallOptions = None):
         """Authenticate to the server.
@@ -1365,7 +1376,7 @@ cdef class RecordBatchStream(FlightDataStream):
         data_source : RecordBatchReader or Table
         options : pyarrow.ipc.IpcWriteOptions, optional
         """
-        if (not isinstance(data_source, _CRecordBatchReader) and
+        if (not isinstance(data_source, RecordBatchReader) and
                 not isinstance(data_source, lib.Table)):
             raise TypeError("Expected RecordBatchReader or Table, "
                             "but got: {}".format(type(data_source)))
@@ -1375,8 +1386,8 @@ cdef class RecordBatchStream(FlightDataStream):
     cdef CFlightDataStream* to_stream(self) except *:
         cdef:
             shared_ptr[CRecordBatchReader] reader
-        if isinstance(self.data_source, _CRecordBatchReader):
-            reader = (<_CRecordBatchReader> self.data_source).reader
+        if isinstance(self.data_source, RecordBatchReader):
+            reader = (<RecordBatchReader> self.data_source).reader
         elif isinstance(self.data_source, lib.Table):
             table = (<Table> self.data_source).table
             reader.reset(new TableBatchReader(deref(table)))
@@ -1435,7 +1446,8 @@ cdef class ServerCallContext(_Weakrefable):
 
     def peer(self):
         """Get the address of the peer."""
-        return frombytes(self.context.peer())
+        # Set safe=True as gRPC on Windows sometimes gives garbage bytes
+        return frombytes(self.context.peer(), safe=True)
 
     def get_middleware(self, key):
         """
@@ -1616,7 +1628,7 @@ cdef CStatus _data_stream_next(void* self, CFlightPayload* payload) except *:
     else:
         result, metadata = result, None
 
-    if isinstance(result, (Table, _CRecordBatchReader)):
+    if isinstance(result, (Table, RecordBatchReader)):
         if metadata:
             raise ValueError("Can only return metadata alongside a "
                              "RecordBatch.")
@@ -2516,6 +2528,9 @@ def connect(location, **kwargs):
         batch that (when serialized) exceeds this limit will raise an
         exception; the client can retry the write with a smaller
         batch.
+    disable_server_verification : boolean or None
+        Disable verifying the server when using TLS.
+        Insecure, use with caution.
     generic_options : list or None
         A list of generic (string, int or string) options to pass to
         the underlying transport.
diff --git a/python/pyarrow/_fs.pyx b/python/pyarrow/_fs.pyx
index d435de14d58..d881c749c71 100644
--- a/python/pyarrow/_fs.pyx
+++ b/python/pyarrow/_fs.pyx
@@ -405,16 +405,21 @@ cdef class FileSystem(_Weakrefable):
 
         Parameters
         ----------
-        paths_or_selector: FileSelector or list of path-likes
-            Either a selector object or a list of path-like objects.
-            The selector's base directory will not be part of the results, even
-            if it exists. If it doesn't exist, use `allow_not_found`.
+        paths_or_selector: FileSelector, path-like or list of path-likes
+            Either a selector object, a path-like object or a list of
+            path-like objects. The selector's base directory will not be
+            part of the results, even if it exists. If it doesn't exist,
+            use `allow_not_found`.
 
         Returns
         -------
-        file_infos : list of FileInfo
+        FileInfo or list of FileInfo
+            Single FileInfo object is returned for a single path, otherwise
+            a list of FileInfo objects is returned.
         """
         cdef:
+            CFileInfo info
+            c_string path
             vector[CFileInfo] infos
             vector[c_string] paths
             CFileSelector selector
@@ -427,8 +432,13 @@ cdef class FileSystem(_Weakrefable):
             paths = [_path_as_bytes(s) for s in paths_or_selector]
             with nogil:
                 infos = GetResultValue(self.fs.GetFileInfo(paths))
+        elif isinstance(paths_or_selector, (bytes, str)):
+            path =_path_as_bytes(paths_or_selector)
+            with nogil:
+                info = GetResultValue(self.fs.GetFileInfo(path))
+            return FileInfo.wrap(info)
         else:
-            raise TypeError('Must pass either paths or a FileSelector')
+            raise TypeError('Must pass either path(s) or a FileSelector')
 
         return [FileInfo.wrap(info) for info in infos]
 
diff --git a/python/pyarrow/_json.pyx b/python/pyarrow/_json.pyx
index 533ece9a906..ef2567a009a 100644
--- a/python/pyarrow/_json.pyx
+++ b/python/pyarrow/_json.pyx
@@ -213,7 +213,8 @@ def read_json(input_file, read_options=None, parse_options=None,
     Parameters
     ----------
     input_file: string, path or file-like object
-        The location of JSON data.
+        The location of JSON data. Currently only the line-delimited JSON
+        format is supported.
     read_options: pyarrow.json.ReadOptions, optional
         Options for the JSON reader (see ReadOptions constructor for defaults)
     parse_options: pyarrow.json.ParseOptions, optional
diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index 5b1317eb0b4..f1c0abfbdd8 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -24,6 +24,7 @@ from pyarrow.includes.libarrow cimport (CChunkedArray, CSchema, CStatus,
                                         CKeyValueMetadata,
                                         CRandomAccessFile, COutputStream,
                                         TimeUnit)
+from pyarrow.lib cimport _Weakrefable
 
 
 cdef extern from "parquet/api/schema.h" namespace "parquet::schema" nogil:
@@ -240,6 +241,7 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
         int64_t distinct_count() const
         int64_t num_values() const
         bint HasMinMax()
+        c_bool Equals(const CStatistics&) const
         void Reset()
         c_string EncodeMin()
         c_string EncodeMax()
@@ -288,6 +290,7 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
         shared_ptr[CStatistics] statistics() const
         ParquetCompression compression() const
         const vector[ParquetEncoding]& encodings() const
+        c_bool Equals(const CColumnChunkMetaData&) const
 
         int64_t has_dictionary_page() const
         int64_t dictionary_page_offset() const
@@ -297,12 +300,14 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
         int64_t total_uncompressed_size() const
 
     cdef cppclass CRowGroupMetaData" parquet::RowGroupMetaData":
+        c_bool Equals(const CRowGroupMetaData&) const
         int num_columns()
         int64_t num_rows()
         int64_t total_byte_size()
         unique_ptr[CColumnChunkMetaData] ColumnChunk(int i) const
 
     cdef cppclass CFileMetaData" parquet::FileMetaData":
+        c_bool Equals(const CFileMetaData&) const
         uint32_t size()
         int num_columns()
         int64_t num_rows()
@@ -471,3 +476,64 @@ cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil:
     CStatus WriteMetaDataFile(
         const CFileMetaData& file_metadata,
         const COutputStream* sink)
+
+
+cdef shared_ptr[WriterProperties] _create_writer_properties(
+    use_dictionary=*,
+    compression=*,
+    version=*,
+    write_statistics=*,
+    data_page_size=*,
+    compression_level=*,
+    use_byte_stream_split=*,
+    data_page_version=*) except *
+
+
+cdef shared_ptr[ArrowWriterProperties] _create_arrow_writer_properties(
+    use_deprecated_int96_timestamps=*,
+    coerce_timestamps=*,
+    allow_truncated_timestamps=*,
+    writer_engine_version=*) except *
+
+cdef class ParquetSchema(_Weakrefable):
+    cdef:
+        FileMetaData parent  # the FileMetaData owning the SchemaDescriptor
+        const SchemaDescriptor* schema
+
+cdef class FileMetaData(_Weakrefable):
+    cdef:
+        shared_ptr[CFileMetaData] sp_metadata
+        CFileMetaData* _metadata
+        ParquetSchema _schema
+
+    cdef inline init(self, const shared_ptr[CFileMetaData]& metadata):
+        self.sp_metadata = metadata
+        self._metadata = metadata.get()
+
+cdef class RowGroupMetaData(_Weakrefable):
+    cdef:
+        int index  # for pickling support
+        unique_ptr[CRowGroupMetaData] up_metadata
+        CRowGroupMetaData* metadata
+        FileMetaData parent
+
+cdef class ColumnChunkMetaData(_Weakrefable):
+    cdef:
+        unique_ptr[CColumnChunkMetaData] up_metadata
+        CColumnChunkMetaData* metadata
+        RowGroupMetaData parent
+
+    cdef inline init(self, RowGroupMetaData parent, int i):
+        self.up_metadata = parent.metadata.ColumnChunk(i)
+        self.metadata = self.up_metadata.get()
+        self.parent = parent
+
+cdef class Statistics(_Weakrefable):
+    cdef:
+        shared_ptr[CStatistics] statistics
+        ColumnChunkMetaData parent
+
+    cdef inline init(self, const shared_ptr[CStatistics]& statistics,
+                     ColumnChunkMetaData parent):
+        self.statistics = statistics
+        self.parent = parent
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index 7982d3d1895..be8e3634852 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -46,18 +46,9 @@ cimport cpython as cp
 
 
 cdef class Statistics(_Weakrefable):
-    cdef:
-        shared_ptr[CStatistics] statistics
-        ColumnChunkMetaData parent
-
     def __cinit__(self):
         pass
 
-    cdef init(self, const shared_ptr[CStatistics]& statistics,
-              ColumnChunkMetaData parent):
-        self.statistics = statistics
-        self.parent = parent
-
     def __repr__(self):
         return """{}
   has_min_max: {}
@@ -98,14 +89,7 @@ cdef class Statistics(_Weakrefable):
             return NotImplemented
 
     def equals(self, Statistics other):
-        # TODO(kszucs): implement native Equals method for Statistics
-        return (self.has_min_max == other.has_min_max and
-                self.min == other.min and
-                self.max == other.max and
-                self.null_count == other.null_count and
-                self.distinct_count == other.distinct_count and
-                self.num_values == other.num_values and
-                self.physical_type == other.physical_type)
+        return self.statistics.get().Equals(deref(other.statistics.get()))
 
     @property
     def has_min_max(self):
@@ -177,7 +161,7 @@ cdef class ParquetLogicalType(_Weakrefable):
         self.type = type
 
     def __str__(self):
-        return frombytes(self.type.get().ToString())
+        return frombytes(self.type.get().ToString(), safe=True)
 
     def to_json(self):
         return frombytes(self.type.get().ToJSON())
@@ -297,19 +281,9 @@ cdef _box_flba(ParquetFLBA val, uint32_t len):
 
 
 cdef class ColumnChunkMetaData(_Weakrefable):
-    cdef:
-        unique_ptr[CColumnChunkMetaData] up_metadata
-        CColumnChunkMetaData* metadata
-        RowGroupMetaData parent
-
     def __cinit__(self):
         pass
 
-    cdef init(self, RowGroupMetaData parent, int i):
-        self.up_metadata = parent.metadata.ColumnChunk(i)
-        self.metadata = self.up_metadata.get()
-        self.parent = parent
-
     def __repr__(self):
         statistics = indent(repr(self.statistics), 4 * ' ')
         return """{0}
@@ -369,21 +343,7 @@ cdef class ColumnChunkMetaData(_Weakrefable):
             return NotImplemented
 
     def equals(self, ColumnChunkMetaData other):
-        # TODO(kszucs): implement native Equals method for CColumnChunkMetaData
-        return (self.file_offset == other.file_offset and
-                self.file_path == other.file_path and
-                self.physical_type == other.physical_type and
-                self.num_values == other.num_values and
-                self.path_in_schema == other.path_in_schema and
-                self.is_stats_set == other.is_stats_set and
-                self.statistics == other.statistics and
-                self.compression == other.compression and
-                self.encodings == other.encodings and
-                self.has_dictionary_page == other.has_dictionary_page and
-                self.dictionary_page_offset == other.dictionary_page_offset and
-                self.data_page_offset == other.data_page_offset and
-                self.total_compressed_size == other.total_compressed_size and
-                self.total_uncompressed_size == other.total_uncompressed_size)
+        return self.metadata.Equals(deref(other.metadata))
 
     @property
     def file_offset(self):
@@ -459,12 +419,6 @@ cdef class ColumnChunkMetaData(_Weakrefable):
 
 
 cdef class RowGroupMetaData(_Weakrefable):
-    cdef:
-        int index  # for pickling support
-        unique_ptr[CRowGroupMetaData] up_metadata
-        CRowGroupMetaData* metadata
-        FileMetaData parent
-
     def __cinit__(self, FileMetaData parent, int index):
         if index < 0 or index >= parent.num_row_groups:
             raise IndexError('{0} out of bounds'.format(index))
@@ -483,16 +437,7 @@ cdef class RowGroupMetaData(_Weakrefable):
             return NotImplemented
 
     def equals(self, RowGroupMetaData other):
-        if not (self.num_columns == other.num_columns and
-                self.num_rows == other.num_rows and
-                self.total_byte_size == other.total_byte_size):
-            return False
-
-        for i in range(self.num_columns):
-            if self.column(i) != other.column(i):
-                return False
-
-        return True
+        return self.metadata.Equals(deref(other.metadata))
 
     def column(self, int i):
         if i < 0 or i >= self.num_columns:
@@ -547,18 +492,9 @@ def _reconstruct_filemetadata(Buffer serialized):
 
 
 cdef class FileMetaData(_Weakrefable):
-    cdef:
-        shared_ptr[CFileMetaData] sp_metadata
-        CFileMetaData* _metadata
-        ParquetSchema _schema
-
     def __cinit__(self):
         pass
 
-    cdef init(self, const shared_ptr[CFileMetaData]& metadata):
-        self.sp_metadata = metadata
-        self._metadata = metadata.get()
-
     def __reduce__(self):
         cdef:
             NativeFile sink = BufferOutputStream()
@@ -604,13 +540,7 @@ cdef class FileMetaData(_Weakrefable):
             return NotImplemented
 
     def equals(self, FileMetaData other):
-        # TODO(kszucs): use native method after ARROW-4970 is implemented
-        for prop in ('schema', 'serialized_size', 'num_columns', 'num_rows',
-                     'num_row_groups', 'format_version', 'created_by',
-                     'metadata'):
-            if getattr(self, prop) != getattr(other, prop):
-                return False
-        return True
+        return self._metadata.Equals(deref(other._metadata))
 
     @property
     def schema(self):
@@ -706,17 +636,14 @@ cdef class FileMetaData(_Weakrefable):
 
 
 cdef class ParquetSchema(_Weakrefable):
-    cdef:
-        FileMetaData parent  # the FileMetaData owning the SchemaDescriptor
-        const SchemaDescriptor* schema
-
     def __cinit__(self, FileMetaData container):
         self.parent = container
         self.schema = container._metadata.schema()
 
     def __repr__(self):
-        return """{0}
-{1}""".format(object.__repr__(self), frombytes(self.schema.ToString()))
+        return "{0}\n{1}".format(
+            object.__repr__(self),
+            frombytes(self.schema.ToString(), safe=True))
 
     def __reduce__(self):
         return ParquetSchema, (self.parent,)
@@ -1188,6 +1115,154 @@ cdef class ParquetReader(_Weakrefable):
         return pyarrow_wrap_chunked_array(out)
 
 
+cdef shared_ptr[WriterProperties] _create_writer_properties(
+        use_dictionary=None,
+        compression=None,
+        version=None,
+        write_statistics=None,
+        data_page_size=None,
+        compression_level=None,
+        use_byte_stream_split=False,
+        data_page_version=None) except *:
+    """General writer properties"""
+    cdef:
+        shared_ptr[WriterProperties] properties
+        WriterProperties.Builder props
+
+    # data_page_version
+
+    if data_page_version is not None:
+        if data_page_version == "1.0":
+            props.data_page_version(ParquetDataPageVersion_V1)
+        elif data_page_version == "2.0":
+            props.data_page_version(ParquetDataPageVersion_V2)
+        else:
+            raise ValueError("Unsupported Parquet data page version: {0}"
+                             .format(data_page_version))
+
+    # version
+
+    if version is not None:
+        if version == "1.0":
+            props.version(ParquetVersion_V1)
+        elif version == "2.0":
+            props.version(ParquetVersion_V2)
+        else:
+            raise ValueError("Unsupported Parquet format version: {0}"
+                             .format(version))
+
+    # compression
+
+    if isinstance(compression, basestring):
+        check_compression_name(compression)
+        props.compression(compression_from_name(compression))
+    elif compression is not None:
+        for column, codec in compression.iteritems():
+            check_compression_name(codec)
+            props.compression(tobytes(column), compression_from_name(codec))
+
+    if isinstance(compression_level, int):
+        props.compression_level(compression_level)
+    elif compression_level is not None:
+        for column, level in compression_level.iteritems():
+            props.compression_level(tobytes(column), level)
+
+    # use_dictionary
+
+    if isinstance(use_dictionary, bool):
+        if use_dictionary:
+            props.enable_dictionary()
+        else:
+            props.disable_dictionary()
+    elif use_dictionary is not None:
+        # Deactivate dictionary encoding by default
+        props.disable_dictionary()
+        for column in use_dictionary:
+            props.enable_dictionary(tobytes(column))
+
+    # write_statistics
+
+    if isinstance(write_statistics, bool):
+        if write_statistics:
+            props.enable_statistics()
+        else:
+            props.disable_statistics()
+    elif write_statistics is not None:
+        # Deactivate statistics by default and enable for specified columns
+        props.disable_statistics()
+        for column in write_statistics:
+            props.enable_statistics(tobytes(column))
+
+    # use_byte_stream_split
+
+    if isinstance(use_byte_stream_split, bool):
+        if use_byte_stream_split:
+            props.encoding(ParquetEncoding_BYTE_STREAM_SPLIT)
+    elif use_byte_stream_split is not None:
+        for column in use_byte_stream_split:
+            props.encoding(tobytes(column),
+                           ParquetEncoding_BYTE_STREAM_SPLIT)
+
+    if data_page_size is not None:
+        props.data_pagesize(data_page_size)
+
+    properties = props.build()
+
+    return properties
+
+
+cdef shared_ptr[ArrowWriterProperties] _create_arrow_writer_properties(
+        use_deprecated_int96_timestamps=False,
+        coerce_timestamps=None,
+        allow_truncated_timestamps=False,
+        writer_engine_version=None) except *:
+    """Arrow writer properties"""
+    cdef:
+        shared_ptr[ArrowWriterProperties] arrow_properties
+        ArrowWriterProperties.Builder arrow_props
+
+    # Store the original Arrow schema so things like dictionary types can
+    # be automatically reconstructed
+    arrow_props.store_schema()
+
+    # int96 support
+
+    if use_deprecated_int96_timestamps:
+        arrow_props.enable_deprecated_int96_timestamps()
+    else:
+        arrow_props.disable_deprecated_int96_timestamps()
+
+    # coerce_timestamps
+
+    if coerce_timestamps == 'ms':
+        arrow_props.coerce_timestamps(TimeUnit_MILLI)
+    elif coerce_timestamps == 'us':
+        arrow_props.coerce_timestamps(TimeUnit_MICRO)
+    elif coerce_timestamps is not None:
+        raise ValueError('Invalid value for coerce_timestamps: {0}'
+                         .format(coerce_timestamps))
+
+    # allow_truncated_timestamps
+
+    if allow_truncated_timestamps:
+        arrow_props.allow_truncated_timestamps()
+    else:
+        arrow_props.disallow_truncated_timestamps()
+
+    # writer_engine_version
+
+    if writer_engine_version == "V1":
+        warnings.warn("V1 parquet writer engine is a no-op.  Use V2.")
+        arrow_props.set_engine_version(ArrowWriterEngineVersion.V1)
+    elif writer_engine_version != "V2":
+        raise ValueError("Unsupported Writer Engine Version: {0}"
+                         .format(writer_engine_version))
+
+    arrow_properties = arrow_props.build()
+
+    return arrow_properties
+
+
 cdef class ParquetWriter(_Weakrefable):
     cdef:
         unique_ptr[FileWriter] writer
@@ -1223,6 +1298,7 @@ cdef class ParquetWriter(_Weakrefable):
                   data_page_version=None):
         cdef:
             shared_ptr[WriterProperties] properties
+            shared_ptr[ArrowWriterProperties] arrow_properties
             c_string c_where
             CMemoryPool* pool
 
@@ -1237,43 +1313,22 @@ cdef class ParquetWriter(_Weakrefable):
                 self.sink = GetResultValue(FileOutputStream.Open(c_where))
             self.own_sink = True
 
-        self.use_dictionary = use_dictionary
-        self.compression = compression
-        self.compression_level = compression_level
-        self.version = version
-        self.write_statistics = write_statistics
-        self.use_deprecated_int96_timestamps = use_deprecated_int96_timestamps
-        self.coerce_timestamps = coerce_timestamps
-        self.allow_truncated_timestamps = allow_truncated_timestamps
-        self.use_byte_stream_split = use_byte_stream_split
-        self.writer_engine_version = writer_engine_version
-        self.data_page_version = data_page_version
-
-        cdef WriterProperties.Builder properties_builder
-        self._set_data_page_version(&properties_builder)
-        self._set_version(&properties_builder)
-        self._set_compression_props(&properties_builder)
-        self._set_dictionary_props(&properties_builder)
-        self._set_statistics_props(&properties_builder)
-        self._set_byte_stream_split_props(&properties_builder)
-
-        if data_page_size is not None:
-            properties_builder.data_pagesize(data_page_size)
-
-        properties = properties_builder.build()
-
-        cdef ArrowWriterProperties.Builder arrow_properties_builder
-
-        # Store the original Arrow schema so things like dictionary types can
-        # be automatically reconstructed
-        arrow_properties_builder.store_schema()
-
-        self._set_int96_support(&arrow_properties_builder)
-        self._set_coerce_timestamps(&arrow_properties_builder)
-        self._set_allow_truncated_timestamps(&arrow_properties_builder)
-        self._set_writer_engine_version(&arrow_properties_builder)
-
-        arrow_properties = arrow_properties_builder.build()
+        properties = _create_writer_properties(
+            use_dictionary=use_dictionary,
+            compression=compression,
+            version=version,
+            write_statistics=write_statistics,
+            data_page_size=data_page_size,
+            compression_level=compression_level,
+            use_byte_stream_split=use_byte_stream_split,
+            data_page_version=data_page_version
+        )
+        arrow_properties = _create_arrow_writer_properties(
+            use_deprecated_int96_timestamps=use_deprecated_int96_timestamps,
+            coerce_timestamps=coerce_timestamps,
+            allow_truncated_timestamps=allow_truncated_timestamps,
+            writer_engine_version=writer_engine_version
+        )
 
         pool = maybe_unbox_memory_pool(memory_pool)
         with nogil:
@@ -1282,110 +1337,6 @@ cdef class ParquetWriter(_Weakrefable):
                                 self.sink, properties, arrow_properties,
                                 &self.writer))
 
-    cdef void _set_int96_support(self, ArrowWriterProperties.Builder* props):
-        if self.use_deprecated_int96_timestamps:
-            props.enable_deprecated_int96_timestamps()
-        else:
-            props.disable_deprecated_int96_timestamps()
-
-    cdef int _set_coerce_timestamps(
-            self, ArrowWriterProperties.Builder* props) except -1:
-        if self.coerce_timestamps == 'ms':
-            props.coerce_timestamps(TimeUnit_MILLI)
-        elif self.coerce_timestamps == 'us':
-            props.coerce_timestamps(TimeUnit_MICRO)
-        elif self.coerce_timestamps is not None:
-            raise ValueError('Invalid value for coerce_timestamps: {0}'
-                             .format(self.coerce_timestamps))
-
-    cdef void _set_allow_truncated_timestamps(
-            self, ArrowWriterProperties.Builder* props):
-        if self.allow_truncated_timestamps:
-            props.allow_truncated_timestamps()
-        else:
-            props.disallow_truncated_timestamps()
-
-    cdef int _set_writer_engine_version(
-            self, ArrowWriterProperties.Builder* props) except -1:
-        if self.writer_engine_version == "V1":
-            props.set_engine_version(ArrowWriterEngineVersion.V1)
-        elif self.writer_engine_version != "V2":
-            raise ValueError("Unsupported Writer Engine Version: {0}"
-                             .format(self.writer_engine_version))
-
-    cdef int _set_version(self, WriterProperties.Builder* props) except -1:
-        if self.version is not None:
-            if self.version == "1.0":
-                props.version(ParquetVersion_V1)
-            elif self.version == "2.0":
-                props.version(ParquetVersion_V2)
-            else:
-                raise ValueError("Unsupported Parquet format version: {0}"
-                                 .format(self.version))
-
-    cdef int _set_data_page_version(self, WriterProperties.Builder* props) \
-            except -1:
-        if self.data_page_version is not None:
-            if self.data_page_version == "1.0":
-                props.data_page_version(ParquetDataPageVersion_V1)
-            elif self.data_page_version == "2.0":
-                props.data_page_version(ParquetDataPageVersion_V2)
-            else:
-                raise ValueError("Unsupported Parquet data page version: {0}"
-                                 .format(self.data_page_version))
-
-    cdef void _set_compression_props(self, WriterProperties.Builder* props) \
-            except *:
-        if isinstance(self.compression, basestring):
-            check_compression_name(self.compression)
-            props.compression(compression_from_name(self.compression))
-        elif self.compression is not None:
-            for column, codec in self.compression.iteritems():
-                check_compression_name(codec)
-                props.compression(column, compression_from_name(codec))
-
-        if isinstance(self.compression_level, int):
-            props.compression_level(self.compression_level)
-        elif self.compression_level is not None:
-            for column, level in self.compression_level.iteritems():
-                props.compression_level(tobytes(column), level)
-
-    cdef void _set_dictionary_props(self, WriterProperties.Builder* props) \
-            except *:
-        if isinstance(self.use_dictionary, bool):
-            if self.use_dictionary:
-                props.enable_dictionary()
-            else:
-                props.disable_dictionary()
-        elif self.use_dictionary is not None:
-            # Deactivate dictionary encoding by default
-            props.disable_dictionary()
-            for column in self.use_dictionary:
-                props.enable_dictionary(tobytes(column))
-
-    cdef void _set_byte_stream_split_props(
-            self, WriterProperties.Builder* props) except *:
-        if isinstance(self.use_byte_stream_split, bool):
-            if self.use_byte_stream_split:
-                props.encoding(ParquetEncoding_BYTE_STREAM_SPLIT)
-        elif self.use_byte_stream_split is not None:
-            for column in self.use_byte_stream_split:
-                props.encoding(tobytes(column),
-                               ParquetEncoding_BYTE_STREAM_SPLIT)
-
-    cdef void _set_statistics_props(self, WriterProperties.Builder* props) \
-            except *:
-        if isinstance(self.write_statistics, bool):
-            if self.write_statistics:
-                props.enable_statistics()
-            else:
-                props.disable_statistics()
-        elif self.write_statistics is not None:
-            # Deactivate statistics by default and enable for specified columns
-            props.disable_statistics()
-            for column in self.write_statistics:
-                props.enable_statistics(tobytes(column))
-
     def close(self):
         with nogil:
             check_status(self.writer.get().Close())
diff --git a/python/pyarrow/_plasma.pyx b/python/pyarrow/_plasma.pyx
index 157fa7a4d79..5cab1030169 100644
--- a/python/pyarrow/_plasma.pyx
+++ b/python/pyarrow/_plasma.pyx
@@ -535,7 +535,13 @@ cdef class PlasmaClient(_Weakrefable):
         """
         cdef ObjectID target_id = (object_id if object_id
                                    else ObjectID.from_random())
-        serialized = pyarrow.serialize(value, serialization_context)
+        if serialization_context is not None:
+            warnings.warn(
+                "'serialization_context' is deprecated and will be removed "
+                "in a future version.",
+                DeprecationWarning, stacklevel=2
+            )
+        serialized = pyarrow.lib._serialize(value, serialization_context)
         buffer = self.create(target_id, serialized.total_bytes)
         stream = pyarrow.FixedSizeBufferWriter(buffer)
         stream.set_memcopy_threads(memcopy_threads)
@@ -566,6 +572,12 @@ cdef class PlasmaClient(_Weakrefable):
             the object_ids and ObjectNotAvailable if the object was not
             available.
         """
+        if serialization_context is not None:
+            warnings.warn(
+                "'serialization_context' is deprecated and will be removed "
+                "in a future version.",
+                DeprecationWarning, stacklevel=2
+            )
         if isinstance(object_ids, Sequence):
             results = []
             buffers = self.get_buffers(object_ids, timeout_ms)
@@ -573,8 +585,8 @@ cdef class PlasmaClient(_Weakrefable):
                 # buffers[i] is None if this object was not available within
                 # the timeout
                 if buffers[i]:
-                    val = pyarrow.deserialize(buffers[i],
-                                              serialization_context)
+                    val = pyarrow.lib._deserialize(buffers[i],
+                                                   serialization_context)
                     results.append(val)
                 else:
                     results.append(ObjectNotAvailable)
diff --git a/python/pyarrow/_s3fs.pyx b/python/pyarrow/_s3fs.pyx
index 3e6a03e82e7..ccec4600d41 100644
--- a/python/pyarrow/_s3fs.pyx
+++ b/python/pyarrow/_s3fs.pyx
@@ -48,9 +48,13 @@ def finalize_s3():
 cdef class S3FileSystem(FileSystem):
     """S3-backed FileSystem implementation
 
-    If neither access_key nor secret_key are provided then attempts to
-    initialize from AWS environment variables, otherwise both access_key and
-    secret_key must be provided.
+    If neither access_key nor secret_key are provided, and role_arn is also not
+    provided, then attempts to initialize from AWS environment variables,
+    otherwise both access_key and secret_key must be provided.
+
+    If role_arn is provided instead of access_key and secret_key, temporary
+    credentials will be fetched by issuing a request to STS to assume the
+    specified role.
 
     Note: S3 buckets are special and the operations available on them may be
     limited or more expensive than desired.
@@ -63,10 +67,24 @@ cdef class S3FileSystem(FileSystem):
     secret_key: str, default None
         AWS Secret Access key. Pass None to use the standard AWS environment
         variables and/or configuration file.
+    session_token: str, default None
+        AWS Session Token.  An optional session token, required if access_key
+        and secret_key are temporary credentials from STS.
     anonymous: boolean, default False
         Whether to connect anonymously if access_key and secret_key are None.
         If true, will not attempt to look up credentials using standard AWS
         configuration methods.
+    role_arn: str, default None
+        AWS Role ARN.  If provided instead of access_key and secret_key,
+        temporary credentials will be fetched by assuming this role.
+    session_name: str, default None
+        An optional identifier for the assumed role session.
+    external_id: str, default None
+        An optional unique identifier that might be required when you assume
+        a role in another account.
+    load_frequency: int, default 900
+        The frequency (in seconds) with which temporary credentials from an
+        assumed role session will be refreshed.
     region: str, default 'us-east-1'
         AWS region to connect to.
     scheme: str, default 'https'
@@ -81,9 +99,11 @@ cdef class S3FileSystem(FileSystem):
     cdef:
         CS3FileSystem* s3fs
 
-    def __init__(self, *, access_key=None, secret_key=None, anonymous=False,
-                 region=None, scheme=None, endpoint_override=None,
-                 bint background_writes=True):
+    def __init__(self, *, access_key=None, secret_key=None, session_token=None,
+                 anonymous=False, region=None, scheme=None,
+                 endpoint_override=None, bint background_writes=True,
+                 role_arn=None, session_name=None, external_id=None,
+                 load_frequency=900):
         cdef:
             CS3Options options
             shared_ptr[CS3FileSystem] wrapped
@@ -100,17 +120,42 @@ cdef class S3FileSystem(FileSystem):
                 'access_key and secret_key must be provided, '
                 '`access_key` is not set.'
             )
-        elif access_key is not None or secret_key is not None:
+
+        elif session_token is not None and (access_key is None or
+                                            secret_key is None):
+            raise ValueError(
+                'In order to initialize a session with temporary credentials, '
+                'both secret_key and access_key must be provided in addition '
+                'to session_token.'
+            )
+
+        elif (access_key is not None or secret_key is not None):
             if anonymous:
                 raise ValueError(
                     'Cannot pass anonymous=True together with access_key '
                     'and secret_key.')
+
+            if role_arn:
+                raise ValueError(
+                    'Cannot provide role_arn with access_key and secret_key')
+
+            if session_token is None:
+                session_token = ""
+
             options = CS3Options.FromAccessKey(
                 tobytes(access_key),
-                tobytes(secret_key)
+                tobytes(secret_key),
+                tobytes(session_token)
             )
         elif anonymous:
             options = CS3Options.Anonymous()
+        elif role_arn is not None:
+            options = CS3Options.FromAssumeRole(
+                tobytes(role_arn),
+                tobytes(session_name),
+                tobytes(external_id),
+                load_frequency
+            )
         else:
             options = CS3Options.Defaults()
 
@@ -138,13 +183,39 @@ cdef class S3FileSystem(FileSystem):
 
     def __reduce__(self):
         cdef CS3Options opts = self.s3fs.options()
+
+        role_arn = frombytes(opts.role_arn)
+
+        # if role_arn is set, we should not re-use temporary credentials
+        # but instead recreate a new assume role session
+        if role_arn:
+            access_key = None
+            secret_key = None
+            session_token = None
+        else:
+            access_key = frombytes(opts.GetAccessKey())
+            secret_key = frombytes(opts.GetSecretKey())
+            session_token = frombytes(opts.GetSessionToken())
+
         return (
             S3FileSystem._reconstruct, (dict(
-                access_key=frombytes(opts.GetAccessKey()),
-                secret_key=frombytes(opts.GetSecretKey()),
+                access_key=access_key,
+                secret_key=secret_key,
+                session_token=session_token,
                 region=frombytes(opts.region),
                 scheme=frombytes(opts.scheme),
                 endpoint_override=frombytes(opts.endpoint_override),
+                role_arn=role_arn,
+                session_name=frombytes(opts.session_name),
+                external_id=frombytes(opts.external_id),
+                load_frequency=opts.load_frequency,
                 background_writes=opts.background_writes
             ),)
         )
+
+    @property
+    def region(self):
+        """
+        The AWS region this filesystem connects to.
+        """
+        return frombytes(self.s3fs.region())
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index ac26ecca601..006e35150bf 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -15,13 +15,16 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import os
 import warnings
 
 
 cdef _sequence_to_array(object sequence, object mask, object size,
                         DataType type, CMemoryPool* pool, c_bool from_pandas):
-    cdef int64_t c_size
-    cdef PyConversionOptions options
+    cdef:
+        int64_t c_size
+        PyConversionOptions options
+        shared_ptr[CChunkedArray] chunked
 
     if type is not None:
         options.type = type.sp_type
@@ -29,18 +32,18 @@ cdef _sequence_to_array(object sequence, object mask, object size,
     if size is not None:
         options.size = size
 
-    options.pool = pool
     options.from_pandas = from_pandas
-
-    cdef shared_ptr[CChunkedArray] out
+    options.ignore_timezone = os.environ.get('PYARROW_IGNORE_TIMEZONE', False)
 
     with nogil:
-        check_status(ConvertPySequence(sequence, mask, options, &out))
+        chunked = GetResultValue(
+            ConvertPySequence(sequence, mask, options, pool)
+        )
 
-    if out.get().num_chunks() == 1:
-        return pyarrow_wrap_array(out.get().chunk(0))
+    if chunked.get().num_chunks() == 1:
+        return pyarrow_wrap_array(chunked.get().chunk(0))
     else:
-        return pyarrow_wrap_chunked_array(out)
+        return pyarrow_wrap_chunked_array(chunked)
 
 
 cdef inline _is_array_like(obj):
@@ -156,28 +159,52 @@ def array(object obj, type=None, mask=None, size=None, from_pandas=None,
     Notes
     -----
     Localized timestamps will currently be returned as UTC (pandas's native
-    representation).  Timezone-naive data will be implicitly interpreted as
+    representation). Timezone-naive data will be implicitly interpreted as
     UTC.
 
+    Converting to dictionary array will promote to a wider integer type for
+    indices if the number of distinct values cannot be represented, even if
+    the index type was explicitly set. This means that if there are more than
+    127 values the returned dictionary array's index type will be at least
+    pa.int16() even if pa.int8() was passed to the function. Note that an
+    explicit index type will not be demoted even if it is wider than required.
+
     Examples
     --------
     >>> import pandas as pd
     >>> import pyarrow as pa
     >>> pa.array(pd.Series([1, 2]))
-    <pyarrow.array.Int64Array object at 0x7f674e4c0e10>
+    <pyarrow.lib.Int64Array object at 0x7f674e4c0e10>
     [
       1,
       2
     ]
 
+    >>> pa.array(["a", "b", "a"], type=pa.dictionary(pa.int8(), pa.string()))
+    <pyarrow.lib.DictionaryArray object at 0x7feb288d9040>
+    -- dictionary:
+    [
+      "a",
+      "b"
+    ]
+    -- indices:
+    [
+      0,
+      1,
+      0
+    ]
+
     >>> import numpy as np
-    >>> pa.array(pd.Series([1, 2]), np.array([0, 1],
-    ... dtype=bool))
-    <pyarrow.array.Int64Array object at 0x7f9019e11208>
+    >>> pa.array(pd.Series([1, 2]), mask=np.array([0, 1], dtype=bool))
+    <pyarrow.lib.Int64Array object at 0x7f9019e11208>
     [
       1,
       null
     ]
+
+    >>> arr = pa.array(range(1024), type=pa.dictionary(pa.int8(), pa.int64()))
+    >>> arr.type.index_type
+    DataType(int16)
     """
     cdef:
         CMemoryPool* pool = maybe_unbox_memory_pool(memory_pool)
@@ -458,7 +485,7 @@ def infer_type(values, mask=None, from_pandas=False):
     if mask is not None and not isinstance(mask, np.ndarray):
         mask = np.array(mask, dtype=bool)
 
-    check_status(InferArrowType(values, mask, use_pandas_sentinels, &out))
+    out = GetResultValue(InferArrowType(values, mask, use_pandas_sentinels))
     return pyarrow_wrap_data_type(out)
 
 
@@ -730,6 +757,7 @@ cdef PandasOptions _convert_pandas_options(dict options):
     result.safe_cast = options['safe']
     result.split_blocks = options['split_blocks']
     result.self_destruct = options['self_destruct']
+    result.ignore_timezone = os.environ.get('PYARROW_IGNORE_TIMEZONE', False)
     return result
 
 
@@ -762,7 +790,7 @@ cdef class Array(_PandasConvertible):
         cdef c_string result
         with nogil:
             result = self.ap.Diff(deref(other.ap))
-        return frombytes(result)
+        return frombytes(result, safe=True)
 
     def cast(self, object target_type, safe=True):
         """
@@ -964,7 +992,7 @@ cdef class Array(_PandasConvertible):
                 )
             )
 
-        return frombytes(result)
+        return frombytes(result, safe=True)
 
     def format(self, **kwargs):
         import warnings
@@ -1054,6 +1082,7 @@ cdef class Array(_PandasConvertible):
         if offset < 0:
             raise IndexError('Offset must be non-negative')
 
+        offset = min(len(self), offset)
         if length is None:
             result = self.ap.Slice(offset)
         else:
@@ -1287,7 +1316,9 @@ cdef _array_like_to_pandas(obj, options):
     result = pandas_api.series(arr, dtype=dtype, name=name)
 
     if (isinstance(original_type, TimestampType) and
-            original_type.tz is not None):
+            original_type.tz is not None and
+            # can be object dtype for non-ns and timestamp_as_object=True
+            result.dtype.kind == "M"):
         from pyarrow.pandas_compat import make_tz_aware
         result = make_tz_aware(result, original_type.tz)
 
@@ -1455,6 +1486,12 @@ cdef class Decimal128Array(FixedSizeBinaryArray):
     Concrete class for Arrow arrays of decimal128 data type.
     """
 
+
+cdef class Decimal256Array(FixedSizeBinaryArray):
+    """
+    Concrete class for Arrow arrays of decimal256 data type.
+    """
+
 cdef class BaseListArray(Array):
 
     def flatten(self):
@@ -2245,7 +2282,8 @@ cdef dict _array_classes = {
     _Type_LARGE_STRING: LargeStringArray,
     _Type_DICTIONARY: DictionaryArray,
     _Type_FIXED_SIZE_BINARY: FixedSizeBinaryArray,
-    _Type_DECIMAL: Decimal128Array,
+    _Type_DECIMAL128: Decimal128Array,
+    _Type_DECIMAL256: Decimal256Array,
     _Type_STRUCT: StructArray,
     _Type_EXTENSION: ExtensionArray,
 }
@@ -2301,6 +2339,9 @@ def concat_arrays(arrays, MemoryPool memory_pool=None):
         CMemoryPool* pool = maybe_unbox_memory_pool(memory_pool)
 
     for array in arrays:
+        if not isinstance(array, Array):
+            raise TypeError("Iterable should contain Array objects, "
+                            "got {0} instead".format(type(array)))
         c_arrays.push_back(pyarrow_unwrap_array(array))
 
     with nogil:
diff --git a/python/pyarrow/cffi.py b/python/pyarrow/cffi.py
index 8880c25a035..961b61dee59 100644
--- a/python/pyarrow/cffi.py
+++ b/python/pyarrow/cffi.py
@@ -52,6 +52,18 @@
       // Opaque producer-specific data
       void* private_data;
     };
+
+    struct ArrowArrayStream {
+      int (*get_schema)(struct ArrowArrayStream*, struct ArrowSchema* out);
+      int (*get_next)(struct ArrowArrayStream*, struct ArrowArray* out);
+
+      const char* (*get_last_error)(struct ArrowArrayStream*);
+
+      // Release callback
+      void (*release)(struct ArrowArrayStream*);
+      // Opaque producer-specific data
+      void* private_data;
+    };
     """
 
 # TODO use out-of-line mode for faster import and avoid C parsing
diff --git a/python/pyarrow/compute.py b/python/pyarrow/compute.py
index cc9847e2dce..48bf9b25022 100644
--- a/python/pyarrow/compute.py
+++ b/python/pyarrow/compute.py
@@ -15,18 +15,206 @@
 # specific language governing permissions and limitations
 # under the License.
 
-
 from pyarrow._compute import (  # noqa
-    FilterOptions,
     Function,
+    FunctionOptions,
     FunctionRegistry,
+    Kernel,
+    ScalarAggregateFunction,
+    ScalarAggregateKernel,
+    ScalarFunction,
+    ScalarKernel,
+    VectorFunction,
+    VectorKernel,
+    # Option classes
+    CastOptions,
+    CountOptions,
+    FilterOptions,
+    MatchSubstringOptions,
+    SplitOptions,
+    SplitPatternOptions,
+    MinMaxOptions,
+    PartitionNthOptions,
+    SetLookupOptions,
+    StrptimeOptions,
+    TakeOptions,
+    VarianceOptions,
+    # Functions
     function_registry,
     call_function,
-    TakeOptions
+    get_function,
+    list_functions,
 )
 
+from textwrap import dedent
+import warnings
+
 import pyarrow as pa
-import pyarrow._compute as _pc
+
+
+def _get_arg_names(func):
+    arg_names = func._doc.arg_names
+    if not arg_names:
+        if func.arity == 1:
+            arg_names = ["arg"]
+        elif func.arity == 2:
+            arg_names = ["left", "right"]
+        else:
+            raise NotImplementedError(
+                "unsupported arity: {}".format(func.arity))
+
+    return arg_names
+
+
+def _decorate_compute_function(wrapper, exposed_name, func, option_class):
+    wrapper.__arrow_compute_function__ = dict(name=func.name,
+                                              arity=func.arity)
+    wrapper.__name__ = exposed_name
+    wrapper.__qualname__ = exposed_name
+
+    doc_pieces = []
+
+    cpp_doc = func._doc
+    summary = cpp_doc.summary
+    if not summary:
+        arg_str = "arguments" if func.arity > 1 else "argument"
+        summary = ("Call compute function {!r} with the given {}"
+                   .format(func.name, arg_str))
+
+    description = cpp_doc.description
+    arg_names = _get_arg_names(func)
+
+    doc_pieces.append("""\
+        {}.
+
+        """.format(summary))
+
+    if description:
+        doc_pieces.append("{}\n\n".format(description))
+
+    doc_pieces.append("""\
+        Parameters
+        ----------
+        """)
+
+    for arg_name in arg_names:
+        if func.kind in ('vector', 'scalar_aggregate'):
+            arg_type = 'Array-like'
+        else:
+            arg_type = 'Array-like or scalar-like'
+        doc_pieces.append("""\
+            {} : {}
+                Argument to compute function
+            """.format(arg_name, arg_type))
+
+    doc_pieces.append("""\
+        memory_pool : pyarrow.MemoryPool, optional
+            If not passed, will allocate memory from the default memory pool.
+        """)
+    if option_class is not None:
+        doc_pieces.append("""\
+            options : pyarrow.compute.{0}, optional
+                Parameters altering compute function semantics
+            **kwargs: optional
+                Parameters for {0} constructor.  Either `options`
+                or `**kwargs` can be passed, but not both at the same time.
+            """.format(option_class.__name__))
+
+    wrapper.__doc__ = "".join(dedent(s) for s in doc_pieces)
+    return wrapper
+
+
+def _get_options_class(func):
+    class_name = func._doc.options_class
+    if not class_name:
+        return None
+    try:
+        return globals()[class_name]
+    except KeyError:
+        warnings.warn("Python binding for {} not exposed"
+                      .format(class_name), RuntimeWarning)
+        return None
+
+
+def _handle_options(name, option_class, options, kwargs):
+    if kwargs:
+        if options is None:
+            return option_class(**kwargs)
+        raise TypeError(
+            "Function {!r} called with both an 'options' argument "
+            "and additional named arguments"
+            .format(name))
+
+    if options is not None:
+        if isinstance(options, dict):
+            return option_class(**options)
+        elif isinstance(options, option_class):
+            return options
+        raise TypeError(
+            "Function {!r} expected a {} parameter, got {}"
+            .format(name, option_class, type(options)))
+
+    return options
+
+
+_wrapper_template = dedent("""\
+    def make_wrapper(func, option_class):
+        def {func_name}({args_sig}, *, memory_pool=None):
+            return func.call([{args_sig}], None, memory_pool)
+        return {func_name}
+    """)
+
+_wrapper_options_template = dedent("""\
+    def make_wrapper(func, option_class):
+        def {func_name}({args_sig}, *, options=None, memory_pool=None,
+                        **kwargs):
+            options = _handle_options({func_name!r}, option_class, options,
+                                      kwargs)
+            return func.call([{args_sig}], options, memory_pool)
+        return {func_name}
+    """)
+
+
+def _wrap_function(name, func):
+    option_class = _get_options_class(func)
+    arg_names = _get_arg_names(func)
+    args_sig = ', '.join(arg_names)
+
+    # Generate templated wrapper, so that the signature matches
+    # the documented argument names.
+    ns = {}
+    if option_class is not None:
+        template = _wrapper_options_template
+    else:
+        template = _wrapper_template
+    exec(template.format(func_name=name, args_sig=args_sig), globals(), ns)
+    wrapper = ns['make_wrapper'](func, option_class)
+
+    return _decorate_compute_function(wrapper, name, func, option_class)
+
+
+def _make_global_functions():
+    """
+    Make global functions wrapping each compute function.
+
+    Note that some of the automatically-generated wrappers may be overriden
+    by custom versions below.
+    """
+    g = globals()
+    reg = function_registry()
+
+    # Avoid clashes with Python keywords
+    rewrites = {'and': 'and_',
+                'or': 'or_'}
+
+    for cpp_name in reg.list_functions():
+        name = rewrites.get(cpp_name, cpp_name)
+        func = reg.get_function(cpp_name)
+        assert name not in g, name
+        g[cpp_name] = g[name] = _wrap_function(name, func)
+
+
+_make_global_functions()
 
 
 def cast(arr, target_type, safe=True):
@@ -81,75 +269,12 @@ def cast(arr, target_type, safe=True):
     if target_type is None:
         raise ValueError("Cast target type must not be None")
     if safe:
-        options = _pc.CastOptions.safe(target_type)
+        options = CastOptions.safe(target_type)
     else:
-        options = _pc.CastOptions.unsafe(target_type)
+        options = CastOptions.unsafe(target_type)
     return call_function("cast", [arr], options)
 
 
-def _decorate_compute_function(func, name, *, arity):
-    func.__arrow_compute_function__ = dict(name=name, arity=arity)
-    return func
-
-
-def _simple_unary_function(name):
-    def func(arg):
-        return call_function(name, [arg])
-    return _decorate_compute_function(func, name, arity=1)
-
-
-def _simple_binary_function(name):
-    def func(left, right):
-        return call_function(name, [left, right])
-    return _decorate_compute_function(func, name, arity=2)
-
-
-binary_length = _simple_unary_function('binary_length')
-ascii_upper = _simple_unary_function('ascii_upper')
-ascii_lower = _simple_unary_function('ascii_lower')
-utf8_upper = _simple_unary_function('utf8_upper')
-utf8_lower = _simple_unary_function('utf8_lower')
-
-string_is_ascii = _simple_unary_function('string_is_ascii')
-
-ascii_is_alnum = _simple_unary_function('ascii_is_alnum')
-utf8_is_alnum = _simple_unary_function('utf8_is_alnum')
-ascii_is_alpha = _simple_unary_function('ascii_is_alpha')
-utf8_is_alpha = _simple_unary_function('utf8_is_alpha')
-ascii_is_decimal = _simple_unary_function('ascii_is_decimal')
-utf8_is_decimal = _simple_unary_function('utf8_is_decimal')
-ascii_is_digit = ascii_is_decimal  # alias
-utf8_is_digit = _simple_unary_function('utf8_is_digit')
-ascii_is_lower = _simple_unary_function('ascii_is_lower')
-utf8_is_lower = _simple_unary_function('utf8_is_lower')
-ascii_is_numeric = ascii_is_decimal  # alias
-utf8_is_numeric = _simple_unary_function('utf8_is_numeric')
-ascii_is_printable = _simple_unary_function('ascii_is_printable')
-utf8_is_printable = _simple_unary_function('utf8_is_printable')
-ascii_is_title = _simple_unary_function('ascii_is_title')
-utf8_is_title = _simple_unary_function('utf8_is_title')
-ascii_is_upper = _simple_unary_function('ascii_is_upper')
-utf8_is_upper = _simple_unary_function('utf8_is_upper')
-
-is_valid = _simple_unary_function('is_valid')
-is_null = _simple_unary_function('is_null')
-
-list_flatten = _simple_unary_function('list_flatten')
-list_parent_indices = _simple_unary_function('list_parent_indices')
-list_value_length = _simple_unary_function('list_value_length')
-
-add = _simple_binary_function('add')
-subtract = _simple_binary_function('subtract')
-multiply = _simple_binary_function('multiply')
-
-equal = _simple_binary_function('equal')
-not_equal = _simple_binary_function('not_equal')
-greater = _simple_binary_function('greater')
-greater_equal = _simple_binary_function('greater_equal')
-less = _simple_binary_function('less')
-less_equal = _simple_binary_function('less_equal')
-
-
 def match_substring(array, pattern):
     """
     Test if substring *pattern* is contained within a value of a string array.
@@ -165,7 +290,7 @@ def match_substring(array, pattern):
     result : pyarrow.Array or pyarrow.ChunkedArray
     """
     return call_function("match_substring", [array],
-                         _pc.MatchSubstringOptions(pattern))
+                         MatchSubstringOptions(pattern))
 
 
 def sum(array):
@@ -183,6 +308,32 @@ def sum(array):
     return call_function('sum', [array])
 
 
+def mode(array):
+    """
+    Return the mode (most common value) of a passed numerical
+    (chunked) array. If there is more than one such value, only
+    the smallest is returned.
+
+    Parameters
+    ----------
+    array : pyarrow.Array or pyarrow.ChunkedArray
+
+    Returns
+    -------
+    mode : pyarrow.StructScalar
+
+    Examples
+    --------
+    >>> import pyarrow as pa
+    >>> import pyarrow.compute as pc
+    >>> arr = pa.array([1, 1, 2, 2, 3, 2, 2, 2])
+    >>> pc.mode(arr)
+    <pyarrow.StructScalar: {'mode': 2, 'count': 5}>
+
+    """
+    return call_function("mode", [array])
+
+
 def filter(data, mask, null_selection_behavior='drop'):
     """
     Select values (or records) from array- or table-like data given boolean
@@ -227,7 +378,7 @@ def filter(data, mask, null_selection_behavior='drop'):
     return call_function('filter', [data, mask], options)
 
 
-def take(data, indices, boundscheck=True):
+def take(data, indices, *, boundscheck=True, memory_pool=None):
     """
     Select values (or records) from array- or table-like data given integer
     selection indices.
@@ -264,8 +415,8 @@ def take(data, indices, boundscheck=True):
       null
     ]
     """
-    options = TakeOptions(boundscheck)
-    return call_function('take', [data, indices], options)
+    options = TakeOptions(boundscheck=boundscheck)
+    return call_function('take', [data, indices], options, memory_pool)
 
 
 def fill_null(values, fill_value):
diff --git a/python/pyarrow/dataset.py b/python/pyarrow/dataset.py
index 19d39f12701..f278e05c007 100644
--- a/python/pyarrow/dataset.py
+++ b/python/pyarrow/dataset.py
@@ -32,13 +32,16 @@
     FileSystemDataset,
     FileSystemDatasetFactory,
     FileSystemFactoryOptions,
+    FileWriteOptions,
     Fragment,
     HivePartitioning,
     IpcFileFormat,
+    IpcFileWriteOptions,
     ParquetDatasetFactory,
     ParquetFactoryOptions,
     ParquetFileFormat,
     ParquetFileFragment,
+    ParquetFileWriteOptions,
     ParquetReadOptions,
     Partitioning,
     PartitioningFactory,
@@ -47,7 +50,8 @@
     ScanTask,
     UnionDataset,
     UnionDatasetFactory,
-    _get_partition_keys
+    _get_partition_keys,
+    _filesystemdataset_write,
 )
 
 
@@ -187,7 +191,7 @@ def _ensure_partitioning(scheme):
     """
     Validate input and return a Partitioning(Factory).
 
-    It passes None through if no partitioning scheme is defiend.
+    It passes None through if no partitioning scheme is defined.
     """
     if scheme is None:
         pass
@@ -682,3 +686,91 @@ def dataset(source, schema=None, format=None, filesystem=None,
             'Expected a path-like, list of path-likes or a list of Datasets '
             'instead of the given type: {}'.format(type(source).__name__)
         )
+
+
+def _ensure_write_partitioning(scheme):
+    if scheme is None:
+        scheme = partitioning(pa.schema([]))
+    if not isinstance(scheme, Partitioning):
+        # TODO support passing field names, and get types from schema
+        raise ValueError("partitioning needs to be actual Partitioning object")
+    return scheme
+
+
+def write_dataset(data, base_dir, basename_template=None, format=None,
+                  partitioning=None, schema=None,
+                  filesystem=None, file_options=None, use_threads=True):
+    """
+    Write a dataset to a given format and partitioning.
+
+    Parameters
+    ----------
+    data : Dataset, Table/RecordBatch, or list of Table/RecordBatch
+        The data to write. This can be a Dataset instance or
+        in-memory Arrow data.
+    base_dir : str
+        The root directory where to write the dataset.
+    basename_template : str, optional
+        A template string used to generate basenames of written data files.
+        The token '{i}' will be replaced with an automatically incremented
+        integer. If not specified, it defaults to
+        "part-{i}." + format.default_extname
+    format : FileFormat or str
+        The format in which to write the dataset. Currently supported:
+        "parquet", "ipc"/"feather". If a FileSystemDataset is being written
+        and `format` is not specified, it defaults to the same format as the
+        specified FileSystemDataset. When writing a Table or RecordBatch, this
+        keyword is required.
+    partitioning : Partitioning, optional
+        The partitioning scheme specified with the ``partitioning()``
+        function.
+    schema : Schema, optional
+    filesystem : FileSystem, optional
+    file_options : FileWriteOptions, optional
+        FileFormat specific write options, created using the
+        ``FileFormat.make_write_options()`` function.
+    use_threads : bool, default True
+        Write files in parallel. If enabled, then maximum parallelism will be
+        used determined by the number of available CPU cores.
+    """
+    if isinstance(data, Dataset):
+        schema = schema or data.schema
+    elif isinstance(data, (pa.Table, pa.RecordBatch)):
+        schema = schema or data.schema
+        data = [data]
+    elif isinstance(data, list):
+        schema = schema or data[0].schema
+    else:
+        raise ValueError(
+            "Only Dataset, Table/RecordBatch or a list of Table/RecordBatch "
+            "objects are supported."
+        )
+
+    if format is None and isinstance(data, FileSystemDataset):
+        format = data.format
+    else:
+        format = _ensure_format(format)
+
+    if file_options is None:
+        file_options = format.make_write_options()
+
+    if format != file_options.format:
+        raise TypeError("Supplied FileWriteOptions have format {}, "
+                        "which doesn't match supplied FileFormat {}".format(
+                            format, file_options))
+
+    if basename_template is None:
+        basename_template = "part-{i}." + format.default_extname
+
+    partitioning = _ensure_write_partitioning(partitioning)
+
+    if filesystem is None:
+        # fall back to local file system as the default
+        from pyarrow.fs import LocalFileSystem
+        filesystem = LocalFileSystem()
+    filesystem, _ = _ensure_fs(filesystem)
+
+    _filesystemdataset_write(
+        data, base_dir, basename_template, schema,
+        filesystem, partitioning, file_options, use_threads,
+    )
diff --git a/python/pyarrow/feather.py b/python/pyarrow/feather.py
index 8830838be37..d42662bbf1a 100644
--- a/python/pyarrow/feather.py
+++ b/python/pyarrow/feather.py
@@ -42,7 +42,6 @@ class FeatherDataset:
     """
 
     def __init__(self, path_or_paths, validate_schema=True):
-        _check_pandas_version()
         self.paths = path_or_paths
         self.validate_schema = validate_schema
 
@@ -94,6 +93,7 @@ def read_pandas(self, columns=None, use_threads=True):
         pandas.DataFrame
             Content of the file as a pandas DataFrame (of columns)
         """
+        _check_pandas_version()
         return self.read_table(columns=columns).to_pandas(
             use_threads=use_threads)
 
diff --git a/python/pyarrow/filesystem.py b/python/pyarrow/filesystem.py
index 7c7f09a5095..d6c11f5e8dd 100644
--- a/python/pyarrow/filesystem.py
+++ b/python/pyarrow/filesystem.py
@@ -19,12 +19,19 @@
 import os
 import inspect
 import posixpath
+import sys
 import urllib.parse
+import warnings
 
 from os.path import join as pjoin
 
 import pyarrow as pa
-from pyarrow.util import implements, _stringify_path, _is_path_like
+from pyarrow.util import implements, _stringify_path, _is_path_like, _DEPR_MSG
+
+
+_FS_DEPR_MSG = _DEPR_MSG.format(
+    "filesystem.LocalFileSystem", "2.0.0", "fs.LocalFileSystem"
+)
 
 
 class FileSystem:
@@ -237,12 +244,23 @@ class LocalFileSystem(FileSystem):
 
     _instance = None
 
+    def __init__(self):
+        warnings.warn(_FS_DEPR_MSG, DeprecationWarning, stacklevel=2)
+        super().__init__()
+
     @classmethod
-    def get_instance(cls):
+    def _get_instance(cls):
         if cls._instance is None:
-            cls._instance = LocalFileSystem()
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+                cls._instance = LocalFileSystem()
         return cls._instance
 
+    @classmethod
+    def get_instance(cls):
+        warnings.warn(_FS_DEPR_MSG, DeprecationWarning, stacklevel=2)
+        return cls._get_instance()
+
     @implements(FileSystem.ls)
     def ls(self, path):
         path = _stringify_path(path)
@@ -428,10 +446,14 @@ def _ensure_filesystem(fs):
         for mro in inspect.getmro(fs_type):
             if mro.__name__ == 'S3FileSystem':
                 return S3FSWrapper(fs)
-            # In case its a simple LocalFileSystem (e.g. dask) use native arrow
-            # FS
-            elif mro.__name__ == 'LocalFileSystem':
-                return LocalFileSystem.get_instance()
+
+        if "fsspec" in sys.modules:
+            fsspec = sys.modules["fsspec"]
+            if isinstance(fs, fsspec.AbstractFileSystem):
+                # for recent fsspec versions that stop inheriting from
+                # pyarrow.filesystem.FileSystem, still allow fsspec
+                # filesystems (which should be compatible with our legacy fs)
+                return fs
 
         raise OSError('Unrecognized filesystem: {}'.format(fs_type))
     else:
@@ -449,10 +471,20 @@ def resolve_filesystem_and_path(where, filesystem=None):
                              " there is nothing to open with filesystem.")
         return filesystem, where
 
-    path = _stringify_path(where)
-
     if filesystem is not None:
-        return _ensure_filesystem(filesystem), path
+        filesystem = _ensure_filesystem(filesystem)
+        if isinstance(filesystem, LocalFileSystem):
+            path = _stringify_path(where)
+        elif not isinstance(where, str):
+            raise TypeError(
+                "Expected string path; path-like objects are only allowed "
+                "with a local filesystem"
+            )
+        else:
+            path = where
+        return filesystem, path
+
+    path = _stringify_path(where)
 
     parsed_uri = urllib.parse.urlparse(path)
     if parsed_uri.scheme == 'hdfs' or parsed_uri.scheme == 'viewfs':
@@ -466,15 +498,15 @@ def resolve_filesystem_and_path(where, filesystem=None):
         port = 0
         if len(netloc_split) == 2 and netloc_split[1].isnumeric():
             port = int(netloc_split[1])
-        fs = pa.hdfs.connect(host=host, port=port)
+        fs = pa.hdfs._connect(host=host, port=port)
         fs_path = parsed_uri.path
     elif parsed_uri.scheme == 'file':
         # Input is local URI such as file:///home/user/myfile.parquet
-        fs = LocalFileSystem.get_instance()
+        fs = LocalFileSystem._get_instance()
         fs_path = parsed_uri.path
     else:
         # Input is local path such as /home/user/myfile.parquet
-        fs = LocalFileSystem.get_instance()
-        fs_path = where
+        fs = LocalFileSystem._get_instance()
+        fs_path = path
 
     return fs, fs_path
diff --git a/python/pyarrow/fs.py b/python/pyarrow/fs.py
index 69404b2a48e..eced037d08c 100644
--- a/python/pyarrow/fs.py
+++ b/python/pyarrow/fs.py
@@ -19,6 +19,8 @@
 FileSystem abstraction to interact with various local and remote filesystems.
 """
 
+from pyarrow.util import _is_path_like, _stringify_path
+
 from pyarrow._fs import (  # noqa
     FileSelector,
     FileType,
@@ -62,7 +64,9 @@ def __getattr__(name):
     )
 
 
-def _ensure_filesystem(filesystem, use_mmap=False):
+def _ensure_filesystem(
+    filesystem, use_mmap=False, allow_legacy_filesystem=False
+):
     if isinstance(filesystem, FileSystem):
         return filesystem
 
@@ -79,15 +83,67 @@ def _ensure_filesystem(filesystem, use_mmap=False):
             return PyFileSystem(FSSpecHandler(filesystem))
 
     # map old filesystems to new ones
-    from pyarrow.filesystem import LocalFileSystem as LegacyLocalFileSystem
+    import pyarrow.filesystem as legacyfs
 
-    if isinstance(filesystem, LegacyLocalFileSystem):
+    if isinstance(filesystem, legacyfs.LocalFileSystem):
         return LocalFileSystem(use_mmap=use_mmap)
     # TODO handle HDFS?
+    if allow_legacy_filesystem and isinstance(filesystem, legacyfs.FileSystem):
+        return filesystem
 
     raise TypeError("Unrecognized filesystem: {}".format(type(filesystem)))
 
 
+def _resolve_filesystem_and_path(
+    path, filesystem=None, allow_legacy_filesystem=False
+):
+    """
+    Return filesystem/path from path which could be an URI or a plain
+    filesystem path.
+    """
+    if not _is_path_like(path):
+        if filesystem is not None:
+            raise ValueError(
+                "'filesystem' passed but the specified path is file-like, so"
+                " there is nothing to open with 'filesystem'."
+            )
+        return filesystem, path
+
+    path = _stringify_path(path)
+
+    if filesystem is not None:
+        filesystem = _ensure_filesystem(
+            filesystem, allow_legacy_filesystem=allow_legacy_filesystem
+        )
+        return filesystem, path
+
+    # if filesystem is not given, try to automatically determine one
+    # first check if the file exists as a local (relative) file path
+    # if not then try to parse the path as an URI
+    filesystem = LocalFileSystem()
+    try:
+        file_info = filesystem.get_file_info(path)
+    except OSError:
+        file_info = None
+        exists_locally = False
+    else:
+        exists_locally = (file_info.type != FileType.NotFound)
+
+    # if the file or directory doesn't exists locally, then assume that
+    # the path is an URI describing the file system as well
+    if not exists_locally:
+        try:
+            filesystem, path = FileSystem.from_uri(path)
+        except ValueError as e:
+            # neither an URI nor a locally existing path, so assume that
+            # local path was given and propagate a nicer file not found error
+            # instead of a more confusing scheme parsing error
+            if "empty scheme" not in str(e):
+                raise
+
+    return filesystem, path
+
+
 class FSSpecHandler(FileSystemHandler):
     """
     Handler for fsspec-based Python filesystems.
diff --git a/python/pyarrow/hdfs.py b/python/pyarrow/hdfs.py
index f4beec37387..eb1b019bf91 100644
--- a/python/pyarrow/hdfs.py
+++ b/python/pyarrow/hdfs.py
@@ -19,8 +19,9 @@
 import os
 import posixpath
 import sys
+import warnings
 
-from pyarrow.util import implements
+from pyarrow.util import implements, _DEPR_MSG
 from pyarrow.filesystem import FileSystem
 import pyarrow.lib as lib
 
@@ -34,6 +35,10 @@ class HadoopFileSystem(lib.HadoopFileSystem, FileSystem):
 
     def __init__(self, host="default", port=0, user=None, kerb_ticket=None,
                  driver='libhdfs', extra_conf=None):
+        warnings.warn(
+            _DEPR_MSG.format(
+                "hdfs.HadoopFileSystem", "2.0.0", "fs.HadoopFileSystem"),
+            DeprecationWarning, stacklevel=2)
         if driver == 'libhdfs':
             _maybe_set_hadoop_classpath()
 
@@ -205,7 +210,21 @@ def connect(host="default", port=0, user=None, kerb_ticket=None,
     -------
     filesystem : HadoopFileSystem
     """
-    fs = HadoopFileSystem(host=host, port=port, user=user,
-                          kerb_ticket=kerb_ticket,
-                          extra_conf=extra_conf)
+    warnings.warn(
+        _DEPR_MSG.format("hdfs.connect", "2.0.0", "fs.HadoopFileSystem"),
+        DeprecationWarning, stacklevel=2
+    )
+    return _connect(
+        host=host, port=port, user=user, kerb_ticket=kerb_ticket,
+        extra_conf=extra_conf
+    )
+
+
+def _connect(host="default", port=0, user=None, kerb_ticket=None,
+             extra_conf=None):
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        fs = HadoopFileSystem(host=host, port=port, user=user,
+                              kerb_ticket=kerb_ticket,
+                              extra_conf=extra_conf)
     return fs
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 555688a8385..ddec351ba6b 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -49,6 +49,11 @@ cdef extern from "arrow/util/decimal.h" namespace "arrow" nogil:
         c_string ToString(int32_t scale) const
 
 
+cdef extern from "arrow/util/decimal.h" namespace "arrow" nogil:
+    cdef cppclass CDecimal256" arrow::Decimal256":
+        c_string ToString(int32_t scale) const
+
+
 cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
     cdef cppclass CBuildInfo "arrow::BuildInfo":
@@ -86,7 +91,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         _Type_FLOAT" arrow::Type::FLOAT"
         _Type_DOUBLE" arrow::Type::DOUBLE"
 
-        _Type_DECIMAL" arrow::Type::DECIMAL"
+        _Type_DECIMAL128" arrow::Type::DECIMAL128"
+        _Type_DECIMAL256" arrow::Type::DECIMAL256"
 
         _Type_DATE32" arrow::Type::DATE32"
         _Type_DATE64" arrow::Type::DATE64"
@@ -356,6 +362,12 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         int precision()
         int scale()
 
+    cdef cppclass CDecimal256Type \
+            " arrow::Decimal256Type"(CFixedSizeBinaryType):
+        CDecimal256Type(int precision, int scale)
+        int precision()
+        int scale()
+
     cdef cppclass CField" arrow::Field":
         cppclass CMergeOptions "arrow::Field::MergeOptions":
             c_bool promote_nullability
@@ -541,6 +553,11 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     ):
         c_string FormatValue(int i)
 
+    cdef cppclass CDecimal256Array" arrow::Decimal256Array"(
+        CFixedSizeBinaryArray
+    ):
+        c_string FormatValue(int i)
+
     cdef cppclass CListArray" arrow::ListArray"(CArray):
         @staticmethod
         CResult[shared_ptr[CArray]] FromArrays(
@@ -935,6 +952,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CDecimal128Scalar" arrow::Decimal128Scalar"(CScalar):
         CDecimal128 value
 
+    cdef cppclass CDecimal256Scalar" arrow::Decimal256Scalar"(CScalar):
+        CDecimal256 value
+
     cdef cppclass CDate32Scalar" arrow::Date32Scalar"(CScalar):
         int32_t value
 
@@ -969,12 +989,15 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         vector[shared_ptr[CScalar]] value
         CResult[shared_ptr[CScalar]] field(CFieldRef ref) const
 
-    cdef cppclass CDictionaryScalar" arrow::DictionaryScalar"(CScalar):
-        cppclass CDictionaryValue "arrow::DictionaryScalar::ValueType":
-            shared_ptr[CScalar] index
-            shared_ptr[CArray] dictionary
+    cdef cppclass CDictionaryScalarIndexAndDictionary \
+            "arrow::DictionaryScalar::ValueType":
+        shared_ptr[CScalar] index
+        shared_ptr[CArray] dictionary
 
-        CDictionaryValue value
+    cdef cppclass CDictionaryScalar" arrow::DictionaryScalar"(CScalar):
+        CDictionaryScalar(CDictionaryScalarIndexAndDictionary value,
+                          shared_ptr[CDataType], c_bool is_valid)
+        CDictionaryScalarIndexAndDictionary value
         CResult[shared_ptr[CScalar]] GetEncodedValue()
 
     cdef cppclass CUnionScalar" arrow::UnionScalar"(CScalar):
@@ -1334,7 +1357,7 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
         c_bool write_legacy_ipc_format
         CMemoryPool* memory_pool
         CMetadataVersion metadata_version
-        CCompressionType compression
+        shared_ptr[CCodec] codec
         c_bool use_threads
 
         @staticmethod
@@ -1390,19 +1413,19 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
             " arrow::ipc::RecordBatchStreamReader"(CRecordBatchReader):
         @staticmethod
         CResult[shared_ptr[CRecordBatchReader]] Open(
-            const CInputStream* stream, const CIpcReadOptions& options)
+            const shared_ptr[CInputStream], const CIpcReadOptions&)
 
         @staticmethod
         CResult[shared_ptr[CRecordBatchReader]] Open2" Open"(
             unique_ptr[CMessageReader] message_reader,
             const CIpcReadOptions& options)
 
-    CResult[shared_ptr[CRecordBatchWriter]] NewStreamWriter(
-        COutputStream* sink, const shared_ptr[CSchema]& schema,
+    CResult[shared_ptr[CRecordBatchWriter]] MakeStreamWriter(
+        shared_ptr[COutputStream] sink, const shared_ptr[CSchema]& schema,
         CIpcWriteOptions& options)
 
-    CResult[shared_ptr[CRecordBatchWriter]] NewFileWriter(
-        COutputStream* sink, const shared_ptr[CSchema]& schema,
+    CResult[shared_ptr[CRecordBatchWriter]] MakeFileWriter(
+        shared_ptr[COutputStream] sink, const shared_ptr[CSchema]& schema,
         CIpcWriteOptions& options)
 
     cdef cppclass CRecordBatchFileReader \
@@ -1441,8 +1464,7 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
         const CIpcReadOptions& options)
 
     CResult[shared_ptr[CBuffer]] SerializeSchema(
-        const CSchema& schema, CDictionaryMemo* dictionary_memo,
-        CMemoryPool* pool)
+        const CSchema& schema, CMemoryPool* pool)
 
     CResult[shared_ptr[CBuffer]] SerializeRecordBatch(
         const CRecordBatch& schema, const CIpcWriteOptions& options)
@@ -1636,6 +1658,12 @@ cdef extern from "arrow/compute/api.h" namespace "arrow::compute" nogil:
         FunctionKind_META \
             " arrow::compute::Function::META"
 
+    cdef cppclass CFunctionDoc" arrow::compute::FunctionDoc":
+        c_string summary
+        c_string description
+        vector[c_string] arg_names
+        c_string options_class
+
     cdef cppclass CFunctionOptions" arrow::compute::FunctionOptions":
         pass
 
@@ -1643,6 +1671,7 @@ cdef extern from "arrow/compute/api.h" namespace "arrow::compute" nogil:
         const c_string& name() const
         FunctionKind kind() const
         const CArity& arity() const
+        const CFunctionDoc& doc() const
         int num_kernels() const
         CResult[CDatum] Execute(const vector[CDatum]& args,
                                 const CFunctionOptions* options,
@@ -1675,6 +1704,18 @@ cdef extern from "arrow/compute/api.h" namespace "arrow::compute" nogil:
         CMatchSubstringOptions(c_string pattern)
         c_string pattern
 
+    cdef cppclass CSplitOptions \
+            "arrow::compute::SplitOptions"(CFunctionOptions):
+        CSplitOptions(int64_t max_splits, c_bool reverse)
+        int64_t max_splits
+        c_bool reverse
+
+    cdef cppclass CSplitPatternOptions \
+            "arrow::compute::SplitPatternOptions"(CSplitOptions):
+        CSplitPatternOptions(c_string pattern, int64_t max_splits,
+                             c_bool reverse)
+        c_string pattern
+
     cdef cppclass CCastOptions" arrow::compute::CastOptions"(CFunctionOptions):
         CCastOptions()
         CCastOptions(c_bool safe)
@@ -1708,6 +1749,41 @@ cdef extern from "arrow/compute/api.h" namespace "arrow::compute" nogil:
             " arrow::compute::TakeOptions"(CFunctionOptions):
         c_bool boundscheck
 
+    cdef cppclass CStrptimeOptions \
+            "arrow::compute::StrptimeOptions"(CFunctionOptions):
+        CStrptimeOptions(c_string format, TimeUnit unit)
+
+    cdef cppclass CVarianceOptions \
+            "arrow::compute::VarianceOptions"(CFunctionOptions):
+        int ddof
+
+    enum CMinMaxMode \
+            "arrow::compute::MinMaxOptions::Mode":
+        CMinMaxMode_SKIP \
+            "arrow::compute::MinMaxOptions::SKIP"
+        CMinMaxMode_EMIT_NULL \
+            "arrow::compute::MinMaxOptions::EMIT_NULL"
+
+    cdef cppclass CMinMaxOptions \
+            "arrow::compute::MinMaxOptions"(CFunctionOptions):
+        CMinMaxMode null_handling
+
+    enum CCountMode \
+            "arrow::compute::CountOptions::Mode":
+        CCountMode_COUNT_NON_NULL \
+            "arrow::compute::CountOptions::COUNT_NON_NULL"
+        CCountMode_COUNT_NULL \
+            "arrow::compute::CountOptions::COUNT_NULL"
+
+    cdef cppclass CCountOptions \
+            "arrow::compute::CountOptions"(CFunctionOptions):
+        CCountMode count_mode
+
+    cdef cppclass CPartitionNthOptions \
+            "arrow::compute::PartitionNthOptions"(CFunctionOptions):
+        CPartitionNthOptions(int64_t pivot)
+        int64_t pivot
+
     enum DatumType" arrow::Datum::type":
         DatumType_NONE" arrow::Datum::NONE"
         DatumType_SCALAR" arrow::Datum::SCALAR"
@@ -1733,12 +1809,17 @@ cdef extern from "arrow/compute/api.h" namespace "arrow::compute" nogil:
         shared_ptr[CTable] table()
         shared_ptr[CScalar] scalar()
 
+    cdef cppclass CSetLookupOptions \
+            "arrow::compute::SetLookupOptions"(CFunctionOptions):
+        CSetLookupOptions(CDatum value_set, c_bool skip_nulls)
+        CDatum value_set
+        c_bool skip_nulls
+
 
 cdef extern from "arrow/python/api.h" namespace "arrow::py":
     # Requires GIL
-    CStatus InferArrowType(object obj, object mask,
-                           c_bool pandas_null_sentinels,
-                           shared_ptr[CDataType]* out_type)
+    CResult[shared_ptr[CDataType]] InferArrowType(
+        object obj, object mask, c_bool pandas_null_sentinels)
 
 
 cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
@@ -1753,12 +1834,14 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
         int64_t size
         CMemoryPool* pool
         c_bool from_pandas
+        c_bool ignore_timezone
+        c_bool strict
 
     # TODO Some functions below are not actually "nogil"
 
-    CStatus ConvertPySequence(object obj, object mask,
-                              const PyConversionOptions& options,
-                              shared_ptr[CChunkedArray]* out)
+    CResult[shared_ptr[CChunkedArray]] ConvertPySequence(
+        object obj, object mask, const PyConversionOptions& options,
+        CMemoryPool* pool)
 
     CStatus NumPyDtypeToArrow(object dtype, shared_ptr[CDataType]* type)
 
@@ -1875,6 +1958,7 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
         c_bool timestamp_as_object
         c_bool use_threads
         c_bool coerce_temporal_nanoseconds
+        c_bool ignore_timezone
         c_bool deduplicate_objects
         c_bool safe_cast
         c_bool split_blocks
@@ -1927,6 +2011,9 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py::internal" nogil:
     CTimePoint TimePoint_from_s(double val)
     CTimePoint TimePoint_from_ns(int64_t val)
 
+    CResult[c_string] TzinfoToString(PyObject* pytzinfo)
+    CResult[PyObject*] StringToTzinfo(c_string)
+
 
 cdef extern from 'arrow/python/init.h':
     int arrow_init_numpy() except -1
@@ -1947,6 +2034,14 @@ cdef extern from 'arrow/python/inference.h' namespace 'arrow::py':
     c_bool IsPyFloat(object o)
 
 
+cdef extern from 'arrow/python/ipc.h' namespace 'arrow::py':
+    cdef cppclass CPyRecordBatchReader" arrow::py::PyRecordBatchReader" \
+            (CRecordBatchReader):
+        @staticmethod
+        CResult[shared_ptr[CRecordBatchReader]] Make(shared_ptr[CSchema],
+                                                     object)
+
+
 cdef extern from 'arrow/extension_type.h' namespace 'arrow':
     cdef cppclass CExtensionTypeRegistry" arrow::ExtensionTypeRegistry":
         @staticmethod
@@ -2010,7 +2105,7 @@ cdef extern from 'arrow/util/compression.h' namespace 'arrow' nogil:
         CResult[int64_t] Compress(int64_t input_len, const uint8_t* input,
                                   int64_t output_buffer_len,
                                   uint8_t* output_buffer)
-        const char* name() const
+        c_string name() const
         int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input)
 
 
@@ -2028,6 +2123,7 @@ cdef extern from 'arrow/util/iterator.h' namespace 'arrow' nogil:
             bint operator!=(RangeIterator) const
         RangeIterator begin()
         RangeIterator end()
+    CIterator[T] MakeVectorIterator[T](vector[T] v)
 
 cdef extern from 'arrow/util/thread_pool.h' namespace 'arrow' nogil:
     int GetCpuThreadPoolCapacity()
@@ -2045,6 +2141,9 @@ cdef extern from 'arrow/c/abi.h':
     cdef struct ArrowArray:
         pass
 
+    cdef struct ArrowArrayStream:
+        pass
+
 cdef extern from 'arrow/c/bridge.h' namespace 'arrow' nogil:
     CStatus ExportType(CDataType&, ArrowSchema* out)
     CResult[shared_ptr[CDataType]] ImportType(ArrowSchema*)
@@ -2065,3 +2164,8 @@ cdef extern from 'arrow/c/bridge.h' namespace 'arrow' nogil:
                                                         shared_ptr[CSchema])
     CResult[shared_ptr[CRecordBatch]] ImportRecordBatch(ArrowArray*,
                                                         ArrowSchema*)
+
+    CStatus ExportRecordBatchReader(shared_ptr[CRecordBatchReader],
+                                    ArrowArrayStream*)
+    CResult[shared_ptr[CRecordBatchReader]] ImportRecordBatchReader(
+        ArrowArrayStream*)
diff --git a/python/pyarrow/includes/libarrow_dataset.pxd b/python/pyarrow/includes/libarrow_dataset.pxd
index ab18300e65f..a81042920d5 100644
--- a/python/pyarrow/includes/libarrow_dataset.pxd
+++ b/python/pyarrow/includes/libarrow_dataset.pxd
@@ -22,6 +22,7 @@ from libcpp.unordered_map cimport unordered_map
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *
 from pyarrow.includes.libarrow_fs cimport *
+from pyarrow._parquet cimport *
 
 
 cdef extern from "arrow/api.h" namespace "arrow" nogil:
@@ -126,6 +127,11 @@ cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil:
     ctypedef CIterator[shared_ptr[CFragment]] CFragmentIterator \
         "arrow::dataset::FragmentIterator"
 
+    cdef cppclass CInMemoryFragment "arrow::dataset::InMemoryFragment"(
+            CFragment):
+        CInMemoryFragment(vector[shared_ptr[CRecordBatch]] record_batches,
+                          shared_ptr[CExpression] partition_expression)
+
     cdef cppclass CScanner "arrow::dataset::Scanner":
         CScanner(shared_ptr[CDataset], shared_ptr[CScanOptions],
                  shared_ptr[CScanContext])
@@ -205,6 +211,11 @@ cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil:
         # the generated C++ is compiled).
         CFileSource(...)
 
+    cdef cppclass CFileWriteOptions \
+            "arrow::dataset::FileWriteOptions":
+        const shared_ptr[CFileFormat]& format() const
+        c_string type_name() const
+
     cdef cppclass CFileFormat "arrow::dataset::FileFormat":
         c_string type_name() const
         CResult[shared_ptr[CSchema]] Inspect(const CFileSource&) const
@@ -212,32 +223,38 @@ cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil:
             CFileSource source,
             shared_ptr[CExpression] partition_expression,
             shared_ptr[CSchema] physical_schema)
+        shared_ptr[CFileWriteOptions] DefaultWriteOptions()
 
     cdef cppclass CFileFragment "arrow::dataset::FileFragment"(
             CFragment):
         const CFileSource& source() const
         const shared_ptr[CFileFormat]& format() const
 
-    cdef cppclass CRowGroupInfo "arrow::dataset::RowGroupInfo":
-        CRowGroupInfo()
-        CRowGroupInfo(int id)
-        int id() const
-        int64_t num_rows() const
-        int64_t total_byte_size() const
-        bint Equals(const CRowGroupInfo& other)
-        c_bool HasStatistics() const
-        shared_ptr[CStructScalar] statistics() const
-
-        @staticmethod
-        vector[CRowGroupInfo] FromIdentifiers(vector[int])
+    cdef cppclass CParquetFileWriteOptions \
+            "arrow::dataset::ParquetFileWriteOptions"(CFileWriteOptions):
+        shared_ptr[WriterProperties] writer_properties
+        shared_ptr[ArrowWriterProperties] arrow_writer_properties
 
     cdef cppclass CParquetFileFragment "arrow::dataset::ParquetFileFragment"(
             CFileFragment):
-        const vector[CRowGroupInfo]& row_groups() const
+        const vector[int]& row_groups() const
+        shared_ptr[CFileMetaData] metadata() const
         CResult[vector[shared_ptr[CFragment]]] SplitByRowGroup(
             shared_ptr[CExpression] predicate)
+        CResult[shared_ptr[CFragment]] SubsetWithFilter "Subset"(
+            shared_ptr[CExpression] predicate)
+        CResult[shared_ptr[CFragment]] SubsetWithIds "Subset"(
+            vector[int] row_group_ids)
         CStatus EnsureCompleteMetadata()
 
+    cdef cppclass CFileSystemDatasetWriteOptions \
+            "arrow::dataset::FileSystemDatasetWriteOptions":
+        shared_ptr[CFileWriteOptions] file_write_options
+        shared_ptr[CFileSystem] filesystem
+        c_string base_dir
+        shared_ptr[CPartitioning] partitioning
+        c_string basename_template
+
     cdef cppclass CFileSystemDataset \
             "arrow::dataset::FileSystemDataset"(CDataset):
         @staticmethod
@@ -245,16 +262,25 @@ cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil:
             shared_ptr[CSchema] schema,
             shared_ptr[CExpression] source_partition,
             shared_ptr[CFileFormat] format,
+            shared_ptr[CFileSystem] filesystem,
             vector[shared_ptr[CFileFragment]] fragments)
+
+        @staticmethod
+        CStatus Write(
+            const CFileSystemDatasetWriteOptions& write_options,
+            shared_ptr[CScanner] scanner)
+
         c_string type()
         vector[c_string] files()
         const shared_ptr[CFileFormat]& format() const
+        const shared_ptr[CFileSystem]& filesystem() const
 
     cdef cppclass CParquetFileFormatReaderOptions \
             "arrow::dataset::ParquetFileFormat::ReaderOptions":
         c_bool use_buffered_stream
         int64_t buffer_size
         unordered_set[c_string] dict_columns
+        c_bool enable_parallel_column_conversion
 
     cdef cppclass CParquetFileFormat "arrow::dataset::ParquetFileFormat"(
             CFileFormat):
@@ -262,8 +288,12 @@ cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil:
         CResult[shared_ptr[CFileFragment]] MakeFragment(
             CFileSource source,
             shared_ptr[CExpression] partition_expression,
-            vector[CRowGroupInfo] row_groups,
-            shared_ptr[CSchema] physical_schema)
+            shared_ptr[CSchema] physical_schema,
+            vector[int] row_groups)
+
+    cdef cppclass CIpcFileWriteOptions \
+            "arrow::dataset::IpcFileWriteOptions"(CFileWriteOptions):
+        pass
 
     cdef cppclass CIpcFileFormat "arrow::dataset::IpcFileFormat"(
             CFileFormat):
@@ -280,7 +310,7 @@ cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil:
 
     cdef cppclass CPartitioningFactoryOptions \
             "arrow::dataset::PartitioningFactoryOptions":
-        int max_partition_dictionary_size
+        c_bool infer_dictionary
 
     cdef cppclass CPartitioningFactory "arrow::dataset::PartitioningFactory":
         pass
@@ -350,6 +380,7 @@ cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil:
             "arrow::dataset::ParquetFactoryOptions":
         CPartitioningOrFactory partitioning
         c_string partition_base_dir
+        c_bool validate_column_chunk_paths
 
     cdef cppclass CParquetDatasetFactory \
             "arrow::dataset::ParquetDatasetFactory"(CDatasetFactory):
diff --git a/python/pyarrow/includes/libarrow_flight.pxd b/python/pyarrow/includes/libarrow_flight.pxd
index e9179dd333c..8d79e21e1ee 100644
--- a/python/pyarrow/includes/libarrow_flight.pxd
+++ b/python/pyarrow/includes/libarrow_flight.pxd
@@ -296,6 +296,7 @@ cdef extern from "arrow/flight/api.h" namespace "arrow" nogil:
         vector[shared_ptr[CClientMiddlewareFactory]] middleware
         int64_t write_size_limit_bytes
         vector[pair[c_string, CIntStringVariant]] generic_options
+        c_bool disable_server_verification
 
     cdef cppclass CFlightClient" arrow::flight::FlightClient":
         @staticmethod
diff --git a/python/pyarrow/includes/libarrow_fs.pxd b/python/pyarrow/includes/libarrow_fs.pxd
index 95fe6a35318..35d630d85da 100644
--- a/python/pyarrow/includes/libarrow_fs.pxd
+++ b/python/pyarrow/includes/libarrow_fs.pxd
@@ -130,11 +130,17 @@ cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil:
         c_string endpoint_override
         c_string scheme
         c_bool background_writes
+        c_string role_arn
+        c_string session_name
+        c_string external_id
+        int load_frequency
         void ConfigureDefaultCredentials()
         void ConfigureAccessKey(const c_string& access_key,
-                                const c_string& secret_key)
+                                const c_string& secret_key,
+                                const c_string& session_token)
         c_string GetAccessKey()
         c_string GetSecretKey()
+        c_string GetSessionToken()
         c_bool Equals(const CS3Options& other)
 
         @staticmethod
@@ -145,12 +151,20 @@ cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil:
 
         @staticmethod
         CS3Options FromAccessKey(const c_string& access_key,
-                                 const c_string& secret_key)
+                                 const c_string& secret_key,
+                                 const c_string& session_token)
+
+        @staticmethod
+        CS3Options FromAssumeRole(const c_string& role_arn,
+                                  const c_string& session_name,
+                                  const c_string& external_id,
+                                  const int load_frequency)
 
     cdef cppclass CS3FileSystem "arrow::fs::S3FileSystem"(CFileSystem):
         @staticmethod
         CResult[shared_ptr[CS3FileSystem]] Make(const CS3Options& options)
         CS3Options options()
+        c_string region()
 
     cdef CStatus CInitializeS3 "arrow::fs::InitializeS3"(
         const CS3GlobalOptions& options)
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index a12915ec616..3fc098478d6 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -1557,25 +1557,6 @@ cdef CCompressionType _ensure_compression(str name) except *:
         raise ValueError('Invalid value for compression: {!r}'.format(name))
 
 
-cdef str _compression_name(CCompressionType ctype):
-    if ctype == CCompressionType_GZIP:
-        return 'gzip'
-    elif ctype == CCompressionType_BROTLI:
-        return 'brotli'
-    elif ctype == CCompressionType_BZ2:
-        return 'bz2'
-    elif ctype == CCompressionType_LZ4_FRAME:
-        return 'lz4'
-    elif ctype == CCompressionType_LZ4:
-        return 'lz4_raw'
-    elif ctype == CCompressionType_SNAPPY:
-        return 'snappy'
-    elif ctype == CCompressionType_ZSTD:
-        return 'zstd'
-    else:
-        raise RuntimeError('Unexpected CCompressionType value')
-
-
 cdef class Codec(_Weakrefable):
     """
     Compression codec.
diff --git a/python/pyarrow/ipc.pxi b/python/pyarrow/ipc.pxi
index 4572d05adb8..5c8194d10c7 100644
--- a/python/pyarrow/ipc.pxi
+++ b/python/pyarrow/ipc.pxi
@@ -94,17 +94,18 @@ cdef class IpcWriteOptions(_Weakrefable):
 
     @property
     def compression(self):
-        if self.c_options.compression == CCompressionType_UNCOMPRESSED:
+        if self.c_options.codec == nullptr:
             return None
         else:
-            return _compression_name(self.c_options.compression)
+            return frombytes(self.c_options.codec.get().name())
 
     @compression.setter
     def compression(self, value):
         if value is None:
-            self.c_options.compression = CCompressionType_UNCOMPRESSED
+            self.c_options.codec.reset()
         else:
-            self.c_options.compression = _ensure_compression(value)
+            self.c_options.codec = shared_ptr[CCodec](GetResultValue(
+                CCodec.Create(_ensure_compression(value))).release())
 
     @property
     def use_threads(self):
@@ -358,7 +359,6 @@ cdef class _CRecordBatchWriter(_Weakrefable):
 
 cdef class _RecordBatchStreamWriter(_CRecordBatchWriter):
     cdef:
-        shared_ptr[COutputStream] sink
         CIpcWriteOptions options
         bint closed
 
@@ -378,14 +378,17 @@ cdef class _RecordBatchStreamWriter(_CRecordBatchWriter):
         # For testing (see test_ipc.py)
         return _wrap_metadata_version(self.options.metadata_version)
 
-    def _open(self, sink, Schema schema,
+    def _open(self, sink, Schema schema not None,
               IpcWriteOptions options=IpcWriteOptions()):
+        cdef:
+            shared_ptr[COutputStream] c_sink
+
         self.options = options.c_options
-        get_writer(sink, &self.sink)
+        get_writer(sink, &c_sink)
         with nogil:
             self.writer = GetResultValue(
-                NewStreamWriter(self.sink.get(), schema.sp_schema,
-                                self.options))
+                MakeStreamWriter(c_sink, schema.sp_schema,
+                                 self.options))
 
 
 cdef _get_input_stream(object source, shared_ptr[CInputStream]* out):
@@ -398,8 +401,29 @@ cdef _get_input_stream(object source, shared_ptr[CInputStream]* out):
     get_input_stream(source, True, out)
 
 
-cdef class _CRecordBatchReader(_Weakrefable):
-    """The base RecordBatchReader wrapper.
+class _ReadPandasMixin:
+
+    def read_pandas(self, **options):
+        """
+        Read contents of stream to a pandas.DataFrame.
+
+        Read all record batches as a pyarrow.Table then convert it to a
+        pandas.DataFrame using Table.to_pandas.
+
+        Parameters
+        ----------
+        **options : arguments to forward to Table.to_pandas
+
+        Returns
+        -------
+        df : pandas.DataFrame
+        """
+        table = self.read_all()
+        return table.to_pandas(**options)
+
+
+cdef class RecordBatchReader(_Weakrefable):
+    """Base class for reading stream of record batches.
 
     Provides common implementations of convenience methods. Should not
     be instantiated directly by user code.
@@ -411,6 +435,18 @@ cdef class _CRecordBatchReader(_Weakrefable):
         while True:
             yield self.read_next_batch()
 
+    @property
+    def schema(self):
+        """
+        Shared schema of the record batches in the stream.
+        """
+        cdef shared_ptr[CSchema] c_schema
+
+        with nogil:
+            c_schema = self.reader.get().schema()
+
+        return pyarrow_wrap_schema(c_schema)
+
     def get_next_batch(self):
         import warnings
         warnings.warn('Please use read_next_batch instead of '
@@ -445,21 +481,91 @@ cdef class _CRecordBatchReader(_Weakrefable):
             check_status(self.reader.get().ReadAll(&table))
         return pyarrow_wrap_table(table)
 
+    read_pandas = _ReadPandasMixin.read_pandas
+
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         pass
 
+    def _export_to_c(self, uintptr_t out_ptr):
+        """
+        Export to a C ArrowArrayStream struct, given its pointer.
+
+        Parameters
+        ----------
+        out_ptr: int
+            The raw pointer to a C ArrowArrayStream struct.
+
+        Be careful: if you don't pass the ArrowArrayStream struct to a
+        consumer, array memory will leak.  This is a low-level function
+        intended for expert users.
+        """
+        with nogil:
+            check_status(ExportRecordBatchReader(
+                self.reader, <ArrowArrayStream*> out_ptr))
+
+    @staticmethod
+    def _import_from_c(uintptr_t in_ptr):
+        """
+        Import RecordBatchReader from a C ArrowArrayStream struct,
+        given its pointer.
 
-cdef class _RecordBatchStreamReader(_CRecordBatchReader):
+        Parameters
+        ----------
+        in_ptr: int
+            The raw pointer to a C ArrowArrayStream struct.
+
+        This is a low-level function intended for expert users.
+        """
+        cdef:
+            shared_ptr[CRecordBatchReader] c_reader
+            RecordBatchReader self
+
+        with nogil:
+            c_reader = GetResultValue(ImportRecordBatchReader(
+                <ArrowArrayStream*> in_ptr))
+
+        self = RecordBatchReader.__new__(RecordBatchReader)
+        self.reader = c_reader
+        return self
+
+    @staticmethod
+    def from_batches(schema, batches):
+        """
+        Create RecordBatchReader from an iterable of batches.
+
+        Parameters
+        ----------
+        schema : Schema
+            The shared schema of the record batches
+        batches : Iterable[RecordBatch]
+            The batches that this reader will return.
+
+        Returns
+        -------
+        reader : RecordBatchReader
+        """
+        cdef:
+            shared_ptr[CSchema] c_schema
+            shared_ptr[CRecordBatchReader] c_reader
+            RecordBatchReader self
+
+        c_schema = pyarrow_unwrap_schema(schema)
+        c_reader = GetResultValue(CPyRecordBatchReader.Make(
+            c_schema, batches))
+
+        self = RecordBatchReader.__new__(RecordBatchReader)
+        self.reader = c_reader
+        return self
+
+
+cdef class _RecordBatchStreamReader(RecordBatchReader):
     cdef:
         shared_ptr[CInputStream] in_stream
         CIpcReadOptions options
 
-    cdef readonly:
-        Schema schema
-
     def __cinit__(self):
         pass
 
@@ -467,20 +573,21 @@ cdef class _RecordBatchStreamReader(_CRecordBatchReader):
         _get_input_stream(source, &self.in_stream)
         with nogil:
             self.reader = GetResultValue(CRecordBatchStreamReader.Open(
-                self.in_stream.get(), self.options))
-
-        self.schema = pyarrow_wrap_schema(self.reader.get().schema())
+                self.in_stream, self.options))
 
 
 cdef class _RecordBatchFileWriter(_RecordBatchStreamWriter):
 
-    def _open(self, sink, Schema schema,
+    def _open(self, sink, Schema schema not None,
               IpcWriteOptions options=IpcWriteOptions()):
+        cdef:
+            shared_ptr[COutputStream] c_sink
+
         self.options = options.c_options
-        get_writer(sink, &self.sink)
+        get_writer(sink, &c_sink)
         with nogil:
             self.writer = GetResultValue(
-                NewFileWriter(self.sink.get(), schema.sp_schema, self.options))
+                MakeFileWriter(c_sink, schema.sp_schema, self.options))
 
 
 cdef class _RecordBatchFileReader(_Weakrefable):
@@ -560,6 +667,8 @@ cdef class _RecordBatchFileReader(_Weakrefable):
 
         return pyarrow_wrap_table(table)
 
+    read_pandas = _ReadPandasMixin.read_pandas
+
     def __enter__(self):
         return self
 
diff --git a/python/pyarrow/ipc.py b/python/pyarrow/ipc.py
index 19e80baa8df..65325c483c4 100644
--- a/python/pyarrow/ipc.py
+++ b/python/pyarrow/ipc.py
@@ -22,6 +22,7 @@
 import pyarrow as pa
 
 from pyarrow.lib import (IpcWriteOptions, Message, MessageReader,  # noqa
+                         RecordBatchReader, _ReadPandasMixin,
                          MetadataVersion,
                          read_message, read_record_batch, read_schema,
                          read_tensor, write_tensor,
@@ -29,28 +30,7 @@
 import pyarrow.lib as lib
 
 
-class _ReadPandasOption:
-
-    def read_pandas(self, **options):
-        """
-        Read contents of stream to a pandas.DataFrame.
-
-        Read all record batches as a pyarrow.Table then convert it to a
-        pandas.DataFrame using Table.to_pandas.
-
-        Parameters
-        ----------
-        **options : arguments to forward to Table.to_pandas
-
-        Returns
-        -------
-        df : pandas.DataFrame
-        """
-        table = self.read_all()
-        return table.to_pandas(**options)
-
-
-class RecordBatchStreamReader(lib._RecordBatchStreamReader, _ReadPandasOption):
+class RecordBatchStreamReader(lib._RecordBatchStreamReader):
     """
     Reader for the Arrow streaming binary format.
 
@@ -97,7 +77,7 @@ def __init__(self, sink, schema, *, use_legacy_format=None, options=None):
         self._open(sink, schema, options=options)
 
 
-class RecordBatchFileReader(lib._RecordBatchFileReader, _ReadPandasOption):
+class RecordBatchFileReader(lib._RecordBatchFileReader):
     """
     Class for reading Arrow record batch data from the Arrow binary file format
 
diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd
index 8e06dcd1d9b..fb390e1af42 100644
--- a/python/pyarrow/lib.pxd
+++ b/python/pyarrow/lib.pxd
@@ -135,6 +135,11 @@ cdef class Decimal128Type(FixedSizeBinaryType):
         const CDecimal128Type* decimal128_type
 
 
+cdef class Decimal256Type(FixedSizeBinaryType):
+    cdef:
+        const CDecimal256Type* decimal256_type
+
+
 cdef class BaseExtensionType(DataType):
     cdef:
         const CExtensionType* ext_type
@@ -345,6 +350,10 @@ cdef class Decimal128Array(FixedSizeBinaryArray):
     pass
 
 
+cdef class Decimal256Array(FixedSizeBinaryArray):
+    pass
+
+
 cdef class StructArray(Array):
     pass
 
@@ -483,7 +492,7 @@ cdef class _CRecordBatchWriter(_Weakrefable):
         shared_ptr[CRecordBatchWriter] writer
 
 
-cdef class _CRecordBatchReader(_Weakrefable):
+cdef class RecordBatchReader(_Weakrefable):
     cdef:
         shared_ptr[CRecordBatchReader] reader
 
diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx
index 0f57ced3745..eba0f5a47af 100644
--- a/python/pyarrow/lib.pyx
+++ b/python/pyarrow/lib.pyx
@@ -73,7 +73,8 @@ Type_INT64 = _Type_INT64
 Type_HALF_FLOAT = _Type_HALF_FLOAT
 Type_FLOAT = _Type_FLOAT
 Type_DOUBLE = _Type_DOUBLE
-Type_DECIMAL = _Type_DECIMAL
+Type_DECIMAL128 = _Type_DECIMAL128
+Type_DECIMAL256 = _Type_DECIMAL256
 Type_DATE32 = _Type_DATE32
 Type_DATE64 = _Type_DATE64
 Type_TIMESTAMP = _Type_TIMESTAMP
diff --git a/python/pyarrow/orc.py b/python/pyarrow/orc.py
index d1f0925bd96..226d7e36cfb 100644
--- a/python/pyarrow/orc.py
+++ b/python/pyarrow/orc.py
@@ -27,7 +27,7 @@
 def _is_map(typ):
     return (types.is_list(typ) and
             types.is_struct(typ.value_type) and
-            typ.value_type.num_children == 2 and
+            typ.value_type.num_fields == 2 and
             typ.value_type[0].name == 'key' and
             typ.value_type[1].name == 'value')
 
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index b94abe4e2e6..7ff8e3d7c90 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -466,11 +466,20 @@ def _get_index_level(df, name):
     return df.index.get_level_values(key)
 
 
+def _level_name(name):
+    # preserve type when default serializable, otherwise str it
+    try:
+        json.dumps(name)
+        return name
+    except TypeError:
+        return str(name)
+
+
 def _get_range_index_descriptor(level):
     # public start/stop/step attributes added in pandas 0.25.0
     return {
         'kind': 'range',
-        'name': level.name,
+        'name': _level_name(level.name),
         'start': _pandas_api.get_rangeindex_attribute(level, 'start'),
         'stop': _pandas_api.get_rangeindex_attribute(level, 'stop'),
         'step': _pandas_api.get_rangeindex_attribute(level, 'step')
@@ -969,9 +978,8 @@ def _extract_index_level(table, result_table, field_name,
         # non-writeable arrays when calling MultiIndex.from_arrays
         values = values.copy()
 
-    if isinstance(col.type, pa.lib.TimestampType):
-        index_level = (pd.Series(values).dt.tz_localize('utc')
-                       .dt.tz_convert(col.type.tz))
+    if isinstance(col.type, pa.lib.TimestampType) and col.type.tz is not None:
+        index_level = make_tz_aware(pd.Series(values), col.type.tz)
     else:
         index_level = pd.Series(values, dtype=values.dtype)
     result_table = result_table.remove_column(
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index b5be07f778b..0d1bfa8e1ca 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -35,8 +35,9 @@
                               FileMetaData, RowGroupMetaData,
                               ColumnChunkMetaData,
                               ParquetSchema, ColumnSchema)
-from pyarrow.filesystem import (LocalFileSystem, _ensure_filesystem,
-                                resolve_filesystem_and_path)
+from pyarrow.fs import (LocalFileSystem, FileSystem,
+                        _resolve_filesystem_and_path, _ensure_filesystem)
+from pyarrow import filesystem as legacyfs
 from pyarrow.util import guid, _is_path_like, _stringify_path
 
 _URI_STRIP_SCHEMES = ('hdfs',)
@@ -55,9 +56,9 @@ def _parse_uri(path):
 
 def _get_filesystem_and_path(passed_filesystem, path):
     if passed_filesystem is None:
-        return resolve_filesystem_and_path(path, passed_filesystem)
+        return legacyfs.resolve_filesystem_and_path(path, passed_filesystem)
     else:
-        passed_filesystem = _ensure_filesystem(passed_filesystem)
+        passed_filesystem = legacyfs._ensure_filesystem(passed_filesystem)
         parsed_path = _parse_uri(path)
         return passed_filesystem, parsed_path
 
@@ -480,11 +481,6 @@ def _sanitize_table(table, new_schema, flavor):
     enabled, then dictionary is preferred.
     The byte_stream_split encoding is valid only for floating-point data types
     and should be combined with a compression codec.
-writer_engine_version: str, default "V2"
-    The engine version to use when writing out Arrow data.  V2 supports
-    all nested types. V1 is legacy and will be removed in a future release.
-    Setting the environment variable ARROW_PARQUET_WRITER_ENGINE will
-    override the default.
 data_page_version : {"1.0", "2.0"}, default "1.0"
     The serialized Parquet data page format version to write, defaults to
     1.0. This does not impact the file schema logical types and Arrow to
@@ -541,13 +537,20 @@ def __init__(self, where, schema, filesystem=None,
         # sure to close it when `self.close` is called.
         self.file_handle = None
 
-        filesystem, path = resolve_filesystem_and_path(where, filesystem)
+        filesystem, path = _resolve_filesystem_and_path(
+            where, filesystem, allow_legacy_filesystem=True
+        )
         if filesystem is not None:
-            sink = self.file_handle = filesystem.open(path, 'wb')
+            if isinstance(filesystem, legacyfs.FileSystem):
+                # legacy filesystem (eg custom subclass)
+                # TODO deprecate
+                sink = self.file_handle = filesystem.open(path, 'wb')
+            else:
+                sink = self.file_handle = filesystem.open_output_stream(path)
         else:
             sink = where
         self._metadata_collector = options.pop('metadata_collector', None)
-        engine_version = os.environ.get('ARROW_PARQUET_WRITER_ENGINE', 'V2')
+        engine_version = 'V2'
         self.writer = _parquet.ParquetWriter(
             sink, schema,
             version=version,
@@ -1040,7 +1043,8 @@ class _ParquetDatasetMetadata:
 
 
 def _open_dataset_file(dataset, path, meta=None):
-    if dataset.fs is not None and not isinstance(dataset.fs, LocalFileSystem):
+    if (dataset.fs is not None and
+            not isinstance(dataset.fs, legacyfs.LocalFileSystem)):
         path = dataset.fs.open(path, mode='rb')
     return ParquetFile(
         path,
@@ -1122,7 +1126,15 @@ def __new__(cls, path_or_paths=None, filesystem=None, schema=None,
                 metadata=None, split_row_groups=False, validate_schema=True,
                 filters=None, metadata_nthreads=1, read_dictionary=None,
                 memory_map=False, buffer_size=0, partitioning="hive",
-                use_legacy_dataset=True):
+                use_legacy_dataset=None):
+        if use_legacy_dataset is None:
+            # if a new filesystem is passed -> default to new implementation
+            if isinstance(filesystem, FileSystem):
+                use_legacy_dataset = False
+            # otherwise the default is still True
+            else:
+                use_legacy_dataset = True
+
         if not use_legacy_dataset:
             return _ParquetDatasetV2(path_or_paths, filesystem=filesystem,
                                      filters=filters,
@@ -1344,7 +1356,7 @@ def _make_manifest(path_or_paths, fs, pathsep='/', metadata_nthreads=1,
     if _is_path_like(path_or_paths) and fs.isdir(path_or_paths):
         manifest = ParquetManifest(path_or_paths, filesystem=fs,
                                    open_file_func=open_file_func,
-                                   pathsep=fs.pathsep,
+                                   pathsep=getattr(fs, "pathsep", "/"),
                                    metadata_nthreads=metadata_nthreads)
         common_metadata_path = manifest.common_metadata_path
         metadata_path = manifest.metadata_path
@@ -1378,7 +1390,6 @@ def __init__(self, path_or_paths, filesystem=None, filters=None,
                  partitioning="hive", read_dictionary=None, buffer_size=None,
                  memory_map=False, ignore_prefixes=None, **kwargs):
         import pyarrow.dataset as ds
-        import pyarrow.fs
 
         # Raise error for not supported keywords
         for keyword, default in [
@@ -1397,36 +1408,61 @@ def __init__(self, path_or_paths, filesystem=None, filters=None,
                                 buffer_size=buffer_size)
         if read_dictionary is not None:
             read_options.update(dictionary_columns=read_dictionary)
-        parquet_format = ds.ParquetFileFormat(read_options=read_options)
 
         # map filters to Expressions
         self._filters = filters
         self._filter_expression = filters and _filters_to_expression(filters)
 
-        # check for single NativeFile dataset
-        if not isinstance(path_or_paths, list):
-            if not _is_path_like(path_or_paths):
-                fragment = parquet_format.make_fragment(path_or_paths)
-                self._dataset = ds.FileSystemDataset(
-                    [fragment], schema=fragment.physical_schema,
-                    format=parquet_format
-                )
-                return
-
-        # check partitioning to enable dictionary encoding
-        if partitioning == "hive":
-            partitioning = ds.HivePartitioning.discover(
-                max_partition_dictionary_size=-1
-            )
-
         # map old filesystems to new one
         if filesystem is not None:
-            filesystem = pyarrow.fs._ensure_filesystem(
+            filesystem = _ensure_filesystem(
                 filesystem, use_mmap=memory_map)
         elif filesystem is None and memory_map:
             # if memory_map is specified, assume local file system (string
             # path can in principle be URI for any filesystem)
-            filesystem = pyarrow.fs.LocalFileSystem(use_mmap=True)
+            filesystem = LocalFileSystem(use_mmap=memory_map)
+
+        # check for single fragment dataset
+        single_file = None
+        if isinstance(path_or_paths, list):
+            if len(path_or_paths) == 1:
+                single_file = path_or_paths[0]
+        else:
+            if _is_path_like(path_or_paths):
+                path = str(path_or_paths)
+                if filesystem is None:
+                    # path might be a URI describing the FileSystem as well
+                    try:
+                        filesystem, path = FileSystem.from_uri(path)
+                    except ValueError:
+                        filesystem = LocalFileSystem(use_mmap=memory_map)
+                if filesystem.get_file_info(path).is_file:
+                    single_file = path
+            else:
+                single_file = path_or_paths
+
+        if single_file is not None:
+            self._enable_parallel_column_conversion = True
+            read_options.update(enable_parallel_column_conversion=True)
+
+            parquet_format = ds.ParquetFileFormat(read_options=read_options)
+            fragment = parquet_format.make_fragment(single_file, filesystem)
+
+            self._dataset = ds.FileSystemDataset(
+                [fragment], schema=fragment.physical_schema,
+                format=parquet_format,
+                filesystem=fragment.filesystem
+            )
+            return
+        else:
+            self._enable_parallel_column_conversion = False
+
+        parquet_format = ds.ParquetFileFormat(read_options=read_options)
+
+        # check partitioning to enable dictionary encoding
+        if partitioning == "hive":
+            partitioning = ds.HivePartitioning.discover(
+                infer_dictionary=True)
 
         self._dataset = ds.dataset(path_or_paths, filesystem=filesystem,
                                    format=parquet_format,
@@ -1470,6 +1506,12 @@ def read(self, columns=None, use_threads=True, use_pandas_metadata=False):
                 ]
                 columns = columns + list(set(index_columns) - set(columns))
 
+        if self._enable_parallel_column_conversion:
+            if use_threads:
+                # Allow per-column parallelism; would otherwise cause
+                # contention in the presence of per-file parallelism.
+                use_threads = False
+
         table = self._dataset.to_table(
             columns=columns, filter=self._filter_expression,
             use_threads=use_threads
@@ -1521,7 +1563,7 @@ def pieces(self):
     pyarrow 1.0.0. Among other things, this allows to pass `filters`
     for all columns and not only the partition keys, enables
     different partitioning schemes, etc.
-    Set to False to use the legacy behaviour.
+    Set to True to use the legacy behaviour.
 ignore_prefixes : list, optional
     Files matching any of these prefixes will be ignored by the
     discovery process if use_legacy_dataset=False.
@@ -1585,8 +1627,10 @@ def read_table(source, columns=None, use_threads=True, metadata=None,
                     "the 'partitioning' keyword is not supported when the "
                     "pyarrow.dataset module is not available"
                 )
+            filesystem, path = _resolve_filesystem_and_path(source, filesystem)
+            if filesystem is not None:
+                source = filesystem.open_input_file(path)
             # TODO test that source is not a directory or a list
-            # TODO check filesystem?
             dataset = ParquetFile(
                 source, metadata=metadata, read_dictionary=read_dictionary,
                 memory_map=memory_map, buffer_size=buffer_size)
@@ -1718,7 +1762,8 @@ def _mkdir_if_not_exists(fs, path):
 
 
 def write_to_dataset(table, root_path, partition_cols=None,
-                     partition_filename_cb=None, filesystem=None, **kwargs):
+                     partition_filename_cb=None, filesystem=None,
+                     use_legacy_dataset=True, **kwargs):
     """Wrapper around parquet.write_table for writing a Table to
     Parquet format by partitions.
     For each combination of partition columns and values,
@@ -1740,7 +1785,7 @@ def write_to_dataset(table, root_path, partition_cols=None,
     Parameters
     ----------
     table : pyarrow.Table
-    root_path : str,
+    root_path : str, pathlib.Path
         The root directory of the dataset
     filesystem : FileSystem, default None
         If nothing passed, paths assumed to be found in the local on-disk
@@ -1752,6 +1797,11 @@ def write_to_dataset(table, root_path, partition_cols=None,
         A callback function that takes the partition key(s) as an argument
         and allow you to override the partition filename. If nothing is
         passed, the filename will consist of a uuid.
+    use_legacy_dataset : bool, default True
+        Set to False to enable the new code path (experimental, using the
+        new Arrow Dataset API). This is more efficient when using partition
+        columns, but does not (yet) support `partition_filename_cb` and
+        `metadata_collector` keywords.
     **kwargs : dict,
         Additional kwargs for write_table function. See docstring for
         `write_table` or `ParquetWriter` for more information.
@@ -1759,7 +1809,44 @@ def write_to_dataset(table, root_path, partition_cols=None,
         file metadata instances of dataset pieces. The file paths in the
         ColumnChunkMetaData will be set relative to `root_path`.
     """
-    fs, root_path = _get_filesystem_and_path(filesystem, root_path)
+    if not use_legacy_dataset:
+        import pyarrow.dataset as ds
+
+        # extract non-file format options
+        schema = kwargs.pop("schema", None)
+        use_threads = kwargs.pop("use_threads", True)
+
+        # raise for unsupported keywords
+        msg = (
+            "The '{}' argument is not supported with the new dataset "
+            "implementation."
+        )
+        metadata_collector = kwargs.pop('metadata_collector', None)
+        if metadata_collector is not None:
+            raise ValueError(msg.format("metadata_collector"))
+        if partition_filename_cb is not None:
+            raise ValueError(msg.format("partition_filename_cb"))
+
+        # map format arguments
+        parquet_format = ds.ParquetFileFormat()
+        write_options = parquet_format.make_write_options(**kwargs)
+
+        # map old filesystems to new one
+        if filesystem is not None:
+            filesystem = _ensure_filesystem(filesystem)
+
+        partitioning = None
+        if partition_cols:
+            part_schema = table.select(partition_cols).schema
+            partitioning = ds.partitioning(part_schema, flavor="hive")
+
+        ds.write_dataset(
+            table, root_path, filesystem=filesystem,
+            format=parquet_format, file_options=write_options, schema=schema,
+            partitioning=partitioning, use_threads=use_threads)
+        return
+
+    fs, root_path = legacyfs.resolve_filesystem_and_path(root_path, filesystem)
 
     _mkdir_if_not_exists(fs, root_path)
 
diff --git a/python/pyarrow/plasma.py b/python/pyarrow/plasma.py
index a4bf79bb42b..239d29094e6 100644
--- a/python/pyarrow/plasma.py
+++ b/python/pyarrow/plasma.py
@@ -106,6 +106,10 @@ def start_plasma_store(plasma_store_memory,
         plasma_store_name = os.path.join(tmpdir, 'plasma.sock')
         plasma_store_executable = os.path.join(
             pa.__path__[0], "plasma-store-server")
+        if not os.path.exists(plasma_store_executable):
+            # Fallback to sys.prefix/bin/ (conda)
+            plasma_store_executable = os.path.join(
+                sys.prefix, "bin", "plasma-store-server")
         command = [plasma_store_executable,
                    "-s", plasma_store_name,
                    "-m", str(plasma_store_memory)]
diff --git a/python/pyarrow/public-api.pxi b/python/pyarrow/public-api.pxi
index 81c57b11926..aa738f9aaea 100644
--- a/python/pyarrow/public-api.pxi
+++ b/python/pyarrow/public-api.pxi
@@ -103,8 +103,10 @@ cdef api object pyarrow_wrap_data_type(
         out = DurationType.__new__(DurationType)
     elif type.get().id() == _Type_FIXED_SIZE_BINARY:
         out = FixedSizeBinaryType.__new__(FixedSizeBinaryType)
-    elif type.get().id() == _Type_DECIMAL:
+    elif type.get().id() == _Type_DECIMAL128:
         out = Decimal128Type.__new__(Decimal128Type)
+    elif type.get().id() == _Type_DECIMAL256:
+        out = Decimal256Type.__new__(Decimal256Type)
     elif type.get().id() == _Type_EXTENSION:
         ext_type = <const CExtensionType*> type.get()
         cpy_ext_type = dynamic_cast[_CPyExtensionTypePtr](ext_type)
diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi
index 7f35419345c..effe60c73b2 100644
--- a/python/pyarrow/scalar.pxi
+++ b/python/pyarrow/scalar.pxi
@@ -15,7 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-
 import collections
 
 
@@ -307,6 +306,26 @@ cdef class Decimal128Scalar(Scalar):
             return None
 
 
+cdef class Decimal256Scalar(Scalar):
+    """
+    Concrete class for decimal256 scalars.
+    """
+
+    def as_py(self):
+        """
+        Return this value as a Python Decimal.
+        """
+        cdef:
+            CDecimal256Scalar* sp = <CDecimal256Scalar*> self.wrapped.get()
+            CDecimal256Type* dtype = <CDecimal256Type*> sp.type.get()
+        if sp.is_valid:
+            return _pydecimal.Decimal(
+                frombytes(sp.value.ToString(dtype.scale()))
+            )
+        else:
+            return None
+
+
 cdef class Date32Scalar(Scalar):
     """
     Concrete class for date32 scalars.
@@ -687,6 +706,50 @@ cdef class DictionaryScalar(Scalar):
     Concrete class for dictionary-encoded scalars.
     """
 
+    @classmethod
+    def _reconstruct(cls, type, is_valid, index, dictionary):
+        cdef:
+            CDictionaryScalarIndexAndDictionary value
+            shared_ptr[CDictionaryScalar] wrapped
+            DataType type_
+            Scalar index_
+            Array dictionary_
+
+        type_ = ensure_type(type, allow_none=False)
+        if not isinstance(type_, DictionaryType):
+            raise TypeError('Must pass a DictionaryType instance')
+
+        if isinstance(index, Scalar):
+            if not index.type.equals(type.index_type):
+                raise TypeError("The Scalar value passed as index must have "
+                                "identical type to the dictionary type's "
+                                "index_type")
+            index_ = index
+        else:
+            index_ = scalar(index, type=type_.index_type)
+
+        if isinstance(dictionary, Array):
+            if not dictionary.type.equals(type.value_type):
+                raise TypeError("The Array passed as dictionary must have "
+                                "identical type to the dictionary type's "
+                                "value_type")
+            dictionary_ = dictionary
+        else:
+            dictionary_ = array(dictionary, type=type_.value_type)
+
+        value.index = pyarrow_unwrap_scalar(index_)
+        value.dictionary = pyarrow_unwrap_array(dictionary_)
+
+        wrapped = make_shared[CDictionaryScalar](
+            value, pyarrow_unwrap_data_type(type_), <c_bool>(is_valid)
+        )
+        return Scalar.wrap(<shared_ptr[CScalar]> wrapped)
+
+    def __reduce__(self):
+        return DictionaryScalar._reconstruct, (
+            self.type, self.is_valid, self.index, self.dictionary
+        )
+
     @property
     def index(self):
         """
@@ -739,7 +802,7 @@ cdef class UnionScalar(Scalar):
         Return underlying value as a scalar.
         """
         cdef CUnionScalar* sp = <CUnionScalar*> self.wrapped.get()
-        return Scalar.wrap(sp.value)
+        return Scalar.wrap(sp.value) if sp.is_valid else None
 
     def as_py(self):
         """
@@ -762,7 +825,8 @@ cdef dict _scalar_classes = {
     _Type_HALF_FLOAT: HalfFloatScalar,
     _Type_FLOAT: FloatScalar,
     _Type_DOUBLE: DoubleScalar,
-    _Type_DECIMAL: Decimal128Scalar,
+    _Type_DECIMAL128: Decimal128Scalar,
+    _Type_DECIMAL256: Decimal256Scalar,
     _Type_DATE32: Date32Scalar,
     _Type_DATE64: Date64Scalar,
     _Type_TIME32: Time32Scalar,
@@ -831,14 +895,15 @@ def scalar(value, type=None, *, from_pandas=None, MemoryPool memory_pool=None):
         shared_ptr[CArray] array
         shared_ptr[CChunkedArray] chunked
         bint is_pandas_object = False
+        CMemoryPool* pool
 
     type = ensure_type(type, allow_none=True)
+    pool = maybe_unbox_memory_pool(memory_pool)
 
     if _is_array_like(value):
         value = get_values(value, &is_pandas_object)
 
     options.size = 1
-    options.pool = maybe_unbox_memory_pool(memory_pool)
 
     if type is not None:
         ty = ensure_type(type)
@@ -851,9 +916,12 @@ def scalar(value, type=None, *, from_pandas=None, MemoryPool memory_pool=None):
 
     value = [value]
     with nogil:
-        check_status(ConvertPySequence(value, None, options, &chunked))
+        chunked = GetResultValue(ConvertPySequence(value, None, options, pool))
 
+    # get the first chunk
     assert chunked.get().num_chunks() == 1
     array = chunked.get().chunk(0)
+
+    # retrieve the scalar from the first position
     scalar = GetResultValue(array.get().GetScalar(0))
     return Scalar.wrap(scalar)
diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi
index ee999b5aa8b..b8d7df54b15 100644
--- a/python/pyarrow/serialization.pxi
+++ b/python/pyarrow/serialization.pxi
@@ -17,6 +17,16 @@
 
 from cpython.ref cimport PyObject
 
+import warnings
+
+
+def _deprecate_serialization(name):
+    msg = (
+        "'pyarrow.{}' is deprecated as of 2.0.0 and will be removed in a "
+        "future version. Use pickle or the pyarrow IPC functionality instead."
+    ).format(name)
+    warnings.warn(msg, DeprecationWarning, stacklevel=3)
+
 
 def is_named_tuple(cls):
     """
@@ -224,9 +234,10 @@ _default_context_initialized = False
 
 def _get_default_context():
     global _default_context_initialized
-    from pyarrow.serialization import register_default_serialization_handlers
+    from pyarrow.serialization import _register_default_serialization_handlers
     if not _default_context_initialized:
-        register_default_serialization_handlers(_default_serialization_context)
+        _register_default_serialization_handlers(
+            _default_serialization_context)
         _default_context_initialized = True
     return _default_serialization_context
 
@@ -369,6 +380,11 @@ def serialize(object value, SerializationContext context=None):
     serialized : SerializedPyObject
 
     """
+    _deprecate_serialization("serialize")
+    return _serialize(value, context)
+
+
+def _serialize(object value, SerializationContext context=None):
     cdef SerializedPyObject serialized = SerializedPyObject()
     wrapped_value = [value]
 
@@ -394,7 +410,8 @@ def serialize_to(object value, sink, SerializationContext context=None):
         Custom serialization and deserialization context, uses a default
         context with some standard type handlers if not specified.
     """
-    serialized = serialize(value, context)
+    _deprecate_serialization("serialize_to")
+    serialized = _serialize(value, context)
     serialized.write_to(sink)
 
 
@@ -414,6 +431,11 @@ def read_serialized(source, base=None):
     -------
     serialized : the serialized data
     """
+    _deprecate_serialization("read_serialized")
+    return _read_serialized(source, base=base)
+
+
+def _read_serialized(source, base=None):
     cdef shared_ptr[CRandomAccessFile] stream
     get_reader(source, True, &stream)
 
@@ -447,7 +469,8 @@ def deserialize_from(source, object base, SerializationContext context=None):
     object
         Python object for the deserialized sequence.
     """
-    serialized = read_serialized(source, base=base)
+    _deprecate_serialization("deserialize_from")
+    serialized = _read_serialized(source, base=base)
     return serialized.deserialize(context)
 
 
@@ -465,6 +488,7 @@ def deserialize_components(components, SerializationContext context=None):
     -------
     object : the Python object that was originally serialized
     """
+    _deprecate_serialization("deserialize_components")
     serialized = SerializedPyObject.from_components(components)
     return serialized.deserialize(context)
 
@@ -487,5 +511,11 @@ def deserialize(obj, SerializationContext context=None):
     -------
     deserialized : object
     """
+    _deprecate_serialization("deserialize")
+    return _deserialize(obj, context=context)
+
+
+def _deserialize(obj, SerializationContext context=None):
     source = BufferReader(obj)
-    return deserialize_from(source, obj, context)
+    serialized = _read_serialized(source, base=obj)
+    return serialized.deserialize(context)
diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py
index 9c1d53ea646..5e8ea697ccb 100644
--- a/python/pyarrow/serialization.py
+++ b/python/pyarrow/serialization.py
@@ -16,6 +16,7 @@
 # under the License.
 
 import collections
+import warnings
 
 import numpy as np
 
@@ -75,6 +76,14 @@ def descr_to_dtype(descr):
                          'offsets': offsets, 'itemsize': offset})
 
 
+def _deprecate_serialization(name):
+    msg = (
+        "'pyarrow.{}' is deprecated as of 2.0.0 and will be removed in a "
+        "future version. Use pickle or the pyarrow IPC functionality instead."
+    ).format(name)
+    warnings.warn(msg, DeprecationWarning, stacklevel=3)
+
+
 # ----------------------------------------------------------------------
 # Set up serialization for numpy with dtype object (primitive types are
 # handled efficiently with Arrow's Tensor facilities, see
@@ -261,6 +270,7 @@ def _deserialize_pandas_series(data):
 def register_torch_serialization_handlers(serialization_context):
     # ----------------------------------------------------------------------
     # Set up serialization for pytorch tensors
+    _deprecate_serialization("register_torch_serialization_handlers")
 
     try:
         import torch
@@ -432,7 +442,7 @@ def _deserialize_pydata_sparse(data):
         pass
 
 
-def register_default_serialization_handlers(serialization_context):
+def _register_default_serialization_handlers(serialization_context):
 
     # ----------------------------------------------------------------------
     # Set up serialization for primitive datatypes
@@ -482,7 +492,13 @@ def register_default_serialization_handlers(serialization_context):
     _register_pydata_sparse_handlers(serialization_context)
 
 
+def register_default_serialization_handlers(serialization_context):
+    _deprecate_serialization("register_default_serialization_handlers")
+    _register_default_serialization_handlers(serialization_context)
+
+
 def default_serialization_context():
+    _deprecate_serialization("default_serialization_context")
     context = SerializationContext()
-    register_default_serialization_handlers(context)
+    _register_default_serialization_handlers(context)
     return context
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index b8205a31c02..56ac805978a 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -79,7 +79,7 @@ cdef class ChunkedArray(_PandasConvertible):
                 )
             )
 
-        return frombytes(result)
+        return frombytes(result, safe=True)
 
     def format(self, **kwargs):
         import warnings
@@ -158,15 +158,12 @@ cdef class ChunkedArray(_PandasConvertible):
         """
         if isinstance(key, slice):
             return _normalize_slice(self, key)
-        elif isinstance(key, int):
-            return self.getitem(key)
-        else:
-            raise TypeError("key must either be a slice or integer")
 
-    cdef getitem(self, int64_t i):
+        return self.getitem(_normalize_index(key, self.chunked_array.length()))
+
+    cdef getitem(self, int64_t index):
         cdef int j
 
-        index = _normalize_index(i, self.chunked_array.length())
         for j in range(self.num_chunks):
             if index < self.chunked_array.chunk(j).get().length():
                 return self.chunk(j)[index]
@@ -354,6 +351,7 @@ cdef class ChunkedArray(_PandasConvertible):
         if offset < 0:
             raise IndexError('Offset must be non-negative')
 
+        offset = min(len(self), offset)
         if length is None:
             result = self.chunked_array.Slice(offset)
         else:
@@ -773,6 +771,7 @@ cdef class RecordBatch(_PandasConvertible):
         if offset < 0:
             raise IndexError('Offset must be non-negative')
 
+        offset = min(len(self), offset)
         if length is None:
             result = self.batch.Slice(offset)
         else:
@@ -1142,6 +1141,7 @@ cdef class Table(_PandasConvertible):
         if offset < 0:
             raise IndexError('Offset must be non-negative')
 
+        offset = min(len(self), offset)
         if length is None:
             result = self.table.Slice(offset)
         else:
diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py
index 43a64964519..45cf2a36858 100644
--- a/python/pyarrow/tests/conftest.py
+++ b/python/pyarrow/tests/conftest.py
@@ -28,7 +28,7 @@
 
 # setup hypothesis profiles
 h.settings.register_profile('ci', max_examples=1000)
-h.settings.register_profile('dev', max_examples=10)
+h.settings.register_profile('dev', max_examples=50)
 h.settings.register_profile('debug', max_examples=10,
                             verbosity=h.Verbosity.verbose)
 
@@ -38,6 +38,11 @@
 # pytest pyarrow -sv --enable-hypothesis --hypothesis-profile=debug
 h.settings.load_profile(os.environ.get('HYPOTHESIS_PROFILE', 'dev'))
 
+# Set this at the beginning before the AWS SDK was loaded to avoid reading in
+# user configuration values.
+os.environ['AWS_CONFIG_FILE'] = "/dev/null"
+
+
 groups = [
     'cython',
     'dataset',
diff --git a/python/pyarrow/tests/strategies.py b/python/pyarrow/tests/strategies.py
index 088f29185bd..92b0d3617c0 100644
--- a/python/pyarrow/tests/strategies.py
+++ b/python/pyarrow/tests/strategies.py
@@ -15,6 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import datetime
+
 import pytz
 import hypothesis as h
 import hypothesis.strategies as st
@@ -40,6 +42,17 @@
 string_type = st.just(pa.string())
 large_binary_type = st.just(pa.large_binary())
 large_string_type = st.just(pa.large_string())
+fixed_size_binary_type = st.builds(
+    pa.binary,
+    st.integers(min_value=0, max_value=16)
+)
+binary_like_types = st.one_of(
+    binary_type,
+    string_type,
+    large_binary_type,
+    large_string_type,
+    fixed_size_binary_type
+)
 
 signed_integer_types = st.sampled_from([
     pa.int8(),
@@ -60,12 +73,18 @@
     pa.float32(),
     pa.float64()
 ])
-decimal_type = st.builds(
+decimal128_type = st.builds(
     pa.decimal128,
     precision=st.integers(min_value=1, max_value=38),
     scale=st.integers(min_value=1, max_value=38)
 )
-numeric_types = st.one_of(integer_types, floating_types, decimal_type)
+decimal256_type = st.builds(
+    pa.decimal256,
+    precision=st.integers(min_value=1, max_value=76),
+    scale=st.integers(min_value=1, max_value=76)
+)
+numeric_types = st.one_of(integer_types, floating_types,
+                          decimal128_type, decimal256_type)
 
 date_types = st.sampled_from([
     pa.date32(),
@@ -82,55 +101,86 @@
     unit=st.sampled_from(['s', 'ms', 'us', 'ns']),
     tz=tzst.timezones()
 )
-duration_types = st.sampled_from([
-    pa.duration(unit) for unit in ['s', 'ms', 'us', 'ns']])
+duration_types = st.builds(
+    pa.duration,
+    st.sampled_from(['s', 'ms', 'us', 'ns'])
+)
 temporal_types = st.one_of(
-    date_types, time_types, timestamp_types, duration_types)
+    date_types,
+    time_types,
+    timestamp_types,
+    duration_types
+)
 
 primitive_types = st.one_of(
     null_type,
     bool_type,
-    binary_type,
-    string_type,
-    large_binary_type,
-    large_string_type,
     numeric_types,
-    temporal_types
+    temporal_types,
+    binary_like_types
 )
 
 metadata = st.dictionaries(st.text(), st.text())
 
 
-def fields(type_strategy=primitive_types):
-    return st.builds(pa.field, name=custom_text, type=type_strategy,
-                     nullable=st.booleans(), metadata=metadata)
+@st.composite
+def fields(draw, type_strategy=primitive_types):
+    name = draw(custom_text)
+    typ = draw(type_strategy)
+    if pa.types.is_null(typ):
+        nullable = True
+    else:
+        nullable = draw(st.booleans())
+    meta = draw(metadata)
+    return pa.field(name, type=typ, nullable=nullable, metadata=meta)
 
 
 def list_types(item_strategy=primitive_types):
     return (
         st.builds(pa.list_, item_strategy) |
-        st.builds(pa.large_list, item_strategy)
+        st.builds(pa.large_list, item_strategy) |
+        st.builds(
+            pa.list_,
+            item_strategy,
+            st.integers(min_value=0, max_value=16)
+        )
     )
 
 
-def struct_types(item_strategy=primitive_types):
-    return st.builds(pa.struct, st.lists(fields(item_strategy)))
-
-
-def complex_types(inner_strategy=primitive_types):
-    return list_types(inner_strategy) | struct_types(inner_strategy)
-
-
-def nested_list_types(item_strategy=primitive_types, max_leaves=3):
-    return st.recursive(item_strategy, list_types, max_leaves=max_leaves)
+@st.composite
+def struct_types(draw, item_strategy=primitive_types):
+    fields_strategy = st.lists(fields(item_strategy))
+    fields_rendered = draw(fields_strategy)
+    field_names = [field.name for field in fields_rendered]
+    # check that field names are unique, see ARROW-9997
+    h.assume(len(set(field_names)) == len(field_names))
+    return pa.struct(fields_rendered)
+
+
+def dictionary_types(key_strategy=None, value_strategy=None):
+    key_strategy = key_strategy or signed_integer_types
+    value_strategy = value_strategy or st.one_of(
+        bool_type,
+        integer_types,
+        st.sampled_from([pa.float32(), pa.float64()]),
+        binary_type,
+        string_type,
+        fixed_size_binary_type,
+    )
+    return st.builds(pa.dictionary, key_strategy, value_strategy)
 
 
-def nested_struct_types(item_strategy=primitive_types, max_leaves=3):
-    return st.recursive(item_strategy, struct_types, max_leaves=max_leaves)
+@st.composite
+def map_types(draw, key_strategy=primitive_types,
+              item_strategy=primitive_types):
+    key_type = draw(key_strategy)
+    h.assume(not pa.types.is_null(key_type))
+    value_type = draw(item_strategy)
+    return pa.map_(key_type, value_type)
 
 
-def nested_complex_types(inner_strategy=primitive_types, max_leaves=3):
-    return st.recursive(inner_strategy, complex_types, max_leaves=max_leaves)
+# union type
+# extension type
 
 
 def schemas(type_strategy=primitive_types, max_fields=None):
@@ -138,10 +188,17 @@ def schemas(type_strategy=primitive_types, max_fields=None):
     return st.builds(pa.schema, children)
 
 
-complex_schemas = schemas(complex_types())
-
-
-all_types = st.one_of(primitive_types, complex_types(), nested_complex_types())
+all_types = st.deferred(
+    lambda: (
+        primitive_types |
+        list_types() |
+        struct_types() |
+        dictionary_types() |
+        map_types() |
+        list_types(all_types) |
+        struct_types(all_types)
+    )
+)
 all_fields = fields(all_types)
 all_schemas = schemas(all_types)
 
@@ -150,10 +207,26 @@ def schemas(type_strategy=primitive_types, max_fields=None):
 
 
 @st.composite
-def arrays(draw, type, size=None):
+def _pylist(draw, value_type, size, nullable=True):
+    arr = draw(arrays(value_type, size=size, nullable=False))
+    return arr.to_pylist()
+
+
+@st.composite
+def _pymap(draw, key_type, value_type, size, nullable=True):
+    length = draw(size)
+    keys = draw(_pylist(key_type, size=length, nullable=False))
+    values = draw(_pylist(value_type, size=length, nullable=nullable))
+    return list(zip(keys, values))
+
+
+@st.composite
+def arrays(draw, type, size=None, nullable=True):
     if isinstance(type, st.SearchStrategy):
-        type = draw(type)
-    elif not isinstance(type, pa.DataType):
+        ty = draw(type)
+    elif isinstance(type, pa.DataType):
+        ty = type
+    else:
         raise TypeError('Type must be a pyarrow DataType')
 
     if isinstance(size, st.SearchStrategy):
@@ -163,59 +236,75 @@ def arrays(draw, type, size=None):
     elif not isinstance(size, int):
         raise TypeError('Size must be an integer')
 
-    shape = (size,)
-
-    if pa.types.is_list(type) or pa.types.is_large_list(type):
-        offsets = draw(npst.arrays(np.uint8(), shape=shape)).cumsum() // 20
-        offsets = np.insert(offsets, 0, 0, axis=0)  # prepend with zero
-        values = draw(arrays(type.value_type, size=int(offsets.sum())))
-        array_type = (
-            pa.LargeListArray if pa.types.is_large_list(type)
-            else pa.ListArray)
-        return array_type.from_arrays(offsets, values)
-
-    if pa.types.is_struct(type):
-        h.assume(len(type) > 0)
-        fields, child_arrays = [], []
-        for field in type:
-            fields.append(field)
-            child_arrays.append(draw(arrays(field.type, size=size)))
-        return pa.StructArray.from_arrays(child_arrays, fields=fields)
-
-    if (pa.types.is_boolean(type) or pa.types.is_integer(type) or
-            pa.types.is_floating(type)):
-        values = npst.arrays(type.to_pandas_dtype(), shape=(size,))
-        np_arr = draw(values)
-        if pa.types.is_floating(type):
-            # Workaround ARROW-4952: no easy way to assert array equality
-            # in a NaN-tolerant way.
-            np_arr[np.isnan(np_arr)] = -42.0
-        return pa.array(np_arr, type=type)
-
-    if pa.types.is_null(type):
+    if pa.types.is_null(ty):
+        h.assume(nullable)
         value = st.none()
-    elif pa.types.is_time(type):
+    elif pa.types.is_boolean(ty):
+        value = st.booleans()
+    elif pa.types.is_integer(ty):
+        values = draw(npst.arrays(ty.to_pandas_dtype(), shape=(size,)))
+        return pa.array(values, type=ty)
+    elif pa.types.is_floating(ty):
+        values = draw(npst.arrays(ty.to_pandas_dtype(), shape=(size,)))
+        # Workaround ARROW-4952: no easy way to assert array equality
+        # in a NaN-tolerant way.
+        values[np.isnan(values)] = -42.0
+        return pa.array(values, type=ty)
+    elif pa.types.is_decimal(ty):
+        # TODO(kszucs): properly limit the precision
+        # value = st.decimals(places=type.scale, allow_infinity=False)
+        h.reject()
+    elif pa.types.is_time(ty):
         value = st.times()
-    elif pa.types.is_date(type):
+    elif pa.types.is_date(ty):
         value = st.dates()
-    elif pa.types.is_timestamp(type):
-        tz = pytz.timezone(type.tz) if type.tz is not None else None
-        value = st.datetimes(timezones=st.just(tz))
-    elif pa.types.is_duration(type):
+    elif pa.types.is_timestamp(ty):
+        min_int64 = -(2**63)
+        max_int64 = 2**63 - 1
+        min_datetime = datetime.datetime.fromtimestamp(min_int64 // 10**9)
+        max_datetime = datetime.datetime.fromtimestamp(max_int64 // 10**9)
+        try:
+            offset_hours = int(ty.tz)
+            tz = pytz.FixedOffset(offset_hours * 60)
+        except ValueError:
+            tz = pytz.timezone(ty.tz)
+        value = st.datetimes(timezones=st.just(tz), min_value=min_datetime,
+                             max_value=max_datetime)
+    elif pa.types.is_duration(ty):
         value = st.timedeltas()
-    elif pa.types.is_binary(type) or pa.types.is_large_binary(type):
+    elif pa.types.is_binary(ty) or pa.types.is_large_binary(ty):
         value = st.binary()
-    elif pa.types.is_string(type) or pa.types.is_large_string(type):
+    elif pa.types.is_string(ty) or pa.types.is_large_string(ty):
         value = st.text()
-    elif pa.types.is_decimal(type):
-        # TODO(kszucs): properly limit the precision
-        # value = st.decimals(places=type.scale, allow_infinity=False)
-        h.reject()
+    elif pa.types.is_fixed_size_binary(ty):
+        value = st.binary(min_size=ty.byte_width, max_size=ty.byte_width)
+    elif pa.types.is_list(ty):
+        value = _pylist(ty.value_type, size=size, nullable=nullable)
+    elif pa.types.is_large_list(ty):
+        value = _pylist(ty.value_type, size=size, nullable=nullable)
+    elif pa.types.is_fixed_size_list(ty):
+        value = _pylist(ty.value_type, size=ty.list_size, nullable=nullable)
+    elif pa.types.is_dictionary(ty):
+        values = _pylist(ty.value_type, size=size, nullable=nullable)
+        return pa.array(draw(values), type=ty)
+    elif pa.types.is_map(ty):
+        value = _pymap(ty.key_type, ty.item_type, size=_default_array_sizes,
+                       nullable=nullable)
+    elif pa.types.is_struct(ty):
+        h.assume(len(ty) > 0)
+        fields, child_arrays = [], []
+        for field in ty:
+            fields.append(field)
+            child_arrays.append(draw(arrays(field.type, size=size)))
+        return pa.StructArray.from_arrays(child_arrays, fields=fields)
     else:
-        raise NotImplementedError(type)
+        raise NotImplementedError(ty)
 
+    if nullable:
+        value = st.one_of(st.none(), value)
     values = st.lists(value, min_size=size, max_size=size)
-    return pa.array(draw(values), type=type)
+
+    return pa.array(draw(values), type=ty)
 
 
 @st.composite
@@ -266,3 +355,60 @@ def tables(draw, type, rows=None, max_fields=None):
 all_chunked_arrays = chunked_arrays(all_types)
 all_record_batches = record_batches(all_types)
 all_tables = tables(all_types)
+
+
+# Define the same rules as above for pandas tests by excluding certain types
+# from the generation because of known issues.
+
+pandas_compatible_primitive_types = st.one_of(
+    null_type,
+    bool_type,
+    integer_types,
+    st.sampled_from([pa.float32(), pa.float64()]),
+    decimal128_type,
+    date_types,
+    time_types,
+    # Need to exclude timestamp and duration types otherwise hypothesis
+    # discovers ARROW-10210
+    # timestamp_types,
+    # duration_types
+    binary_type,
+    string_type,
+    large_binary_type,
+    large_string_type,
+)
+
+# Need to exclude floating point types otherwise hypothesis discovers
+# ARROW-10211
+pandas_compatible_dictionary_value_types = st.one_of(
+    bool_type,
+    integer_types,
+    binary_type,
+    string_type,
+    fixed_size_binary_type,
+)
+
+
+def pandas_compatible_list_types(
+    item_strategy=pandas_compatible_primitive_types
+):
+    # Need to exclude fixed size list type otherwise hypothesis discovers
+    # ARROW-10194
+    return (
+        st.builds(pa.list_, item_strategy) |
+        st.builds(pa.large_list, item_strategy)
+    )
+
+
+pandas_compatible_types = st.deferred(
+    lambda: st.one_of(
+        pandas_compatible_primitive_types,
+        pandas_compatible_list_types(pandas_compatible_primitive_types),
+        struct_types(pandas_compatible_primitive_types),
+        dictionary_types(
+            value_strategy=pandas_compatible_dictionary_value_types
+        ),
+        pandas_compatible_list_types(pandas_compatible_types),
+        struct_types(pandas_compatible_types)
+    )
+)
diff --git a/python/pyarrow/tests/test_adhoc_memory_leak.py b/python/pyarrow/tests/test_adhoc_memory_leak.py
index d95444d2d46..cd381cf427d 100644
--- a/python/pyarrow/tests/test_adhoc_memory_leak.py
+++ b/python/pyarrow/tests/test_adhoc_memory_leak.py
@@ -32,7 +32,7 @@
 @pytest.mark.pandas
 def test_deserialize_pandas_arrow_7956():
     df = pd.DataFrame({'a': np.arange(10000),
-                       'b': [pd.util.testing.rands(5) for _ in range(10000)]})
+                       'b': [test_util.rands(5) for _ in range(10000)]})
 
     def action():
         df_bytes = pa.ipc.serialize_pandas(df).to_pybytes()
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index a1bfc961dc3..17d52188602 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -32,6 +32,7 @@
     import pickle5
 except ImportError:
     pickle5 = None
+import pytz
 
 import pyarrow as pa
 import pyarrow.tests.strategies as past
@@ -309,6 +310,8 @@ def test_nulls(ty):
 def test_array_from_scalar():
     today = datetime.date.today()
     now = datetime.datetime.now()
+    now_utc = now.replace(tzinfo=pytz.utc)
+    now_with_tz = now_utc.astimezone(pytz.timezone('US/Eastern'))
     oneday = datetime.timedelta(days=1)
 
     cases = [
@@ -326,6 +329,14 @@ def test_array_from_scalar():
         (pa.scalar(True), 11, pa.array([True] * 11)),
         (today, 2, pa.array([today] * 2)),
         (now, 10, pa.array([now] * 10)),
+        (
+            now_with_tz,
+            2,
+            pa.array(
+                [now_utc] * 2,
+                type=pa.timestamp('us', tz=pytz.timezone('US/Eastern'))
+            )
+        ),
         (now.time(), 9, pa.array([now.time()] * 9)),
         (oneday, 4, pa.array([oneday] * 4)),
         (False, 9, pa.array([False] * 9)),
@@ -341,8 +352,8 @@ def test_array_from_scalar():
     for value, size, expected in cases:
         arr = pa.repeat(value, size)
         assert len(arr) == size
+        assert arr.type.equals(expected.type)
         assert arr.equals(expected)
-
         if expected.type == pa.null():
             assert arr.null_count == size
         else:
@@ -375,6 +386,10 @@ def test_array_getitem():
         with pytest.raises(IndexError):
             arr[idx]
 
+    # check that numpy scalars are supported
+    for idx in range(-len(arr), len(arr)):
+        assert arr[np.int32(idx)].as_py() == lst[idx]
+
 
 def test_array_slice():
     arr = pa.array(range(10))
@@ -392,6 +407,8 @@ def test_array_slice():
 
     # Slice past end of array
     assert len(arr.slice(len(arr))) == 0
+    assert len(arr.slice(len(arr) + 2)) == 0
+    assert len(arr.slice(len(arr) + 2, 100)) == 0
 
     with pytest.raises(IndexError):
         arr.slice(-1)
@@ -975,6 +992,28 @@ def check_result(result, expected_field_names, expected_type_codes,
         arr = pa.UnionArray.from_sparse(logical_types, [binary, int64[1:]])
 
 
+def test_union_array_to_pylist_with_nulls():
+    # ARROW-9556
+    arr = pa.UnionArray.from_sparse(
+        pa.array([0, 1, 0, 0, 1], type=pa.int8()),
+        [
+            pa.array([0.0, 1.1, None, 3.3, 4.4]),
+            pa.array([True, None, False, True, False]),
+        ]
+    )
+    assert arr.to_pylist() == [0.0, None, None, 3.3, False]
+
+    arr = pa.UnionArray.from_dense(
+        pa.array([0, 1, 0, 0, 0, 1, 1], type=pa.int8()),
+        pa.array([0, 0, 1, 2, 3, 1, 2], type=pa.int32()),
+        [
+            pa.array([0.0, 1.1, None, 3.3]),
+            pa.array([True, None, False])
+        ]
+    )
+    assert arr.to_pylist() == [0.0, True, 1.1, None, 3.3, None, False]
+
+
 def test_union_array_slice():
     # ARROW-2314
     arr = pa.UnionArray.from_sparse(pa.array([0, 0, 1, 1], type=pa.int8()),
@@ -1244,7 +1283,7 @@ def test_decimal_to_int_non_integer():
 
     for case in non_integer_cases:
         # test safe casting raises
-        msg_regexp = 'Rescaling decimal value would cause data loss'
+        msg_regexp = 'Rescaling Decimal128 value would cause data loss'
         with pytest.raises(pa.ArrowInvalid, match=msg_regexp):
             _check_cast_case(case)
 
@@ -1263,8 +1302,8 @@ def test_decimal_to_decimal():
     )
     assert result.equals(expected)
 
-    with pytest.raises(pa.ArrowInvalid,
-                       match='Rescaling decimal value would cause data loss'):
+    msg_regexp = 'Rescaling Decimal128 value would cause data loss'
+    with pytest.raises(pa.ArrowInvalid, match=msg_regexp):
         result = arr.cast(pa.decimal128(9, 1))
 
     result = arr.cast(pa.decimal128(9, 1), safe=False)
@@ -1804,6 +1843,15 @@ def test_array_from_numpy_datetimeD():
     assert result.equals(expected)
 
 
+def test_array_from_naive_datetimes():
+    arr = pa.array([
+        None,
+        datetime.datetime(2017, 4, 4, 12, 11, 10),
+        datetime.datetime(2018, 1, 1, 0, 2, 0)
+    ])
+    assert arr.type == pa.timestamp('us', tz=None)
+
+
 @pytest.mark.parametrize(('dtype', 'type'), [
     ('datetime64[s]', pa.timestamp('s')),
     ('datetime64[ms]', pa.timestamp('ms')),
@@ -2098,12 +2146,12 @@ def test_buffers_nested():
     assert bytearray(null_bitmap)[0] == 0b00000101
     # The child buffers: 'a'
     null_bitmap = buffers[1].to_pybytes()
-    assert bytearray(null_bitmap)[0] == 0b00000001
+    assert bytearray(null_bitmap)[0] == 0b00000011
     values = buffers[2].to_pybytes()
     assert struct.unpack('bxx', values) == (42,)
     # The child buffers: 'b'
     null_bitmap = buffers[3].to_pybytes()
-    assert bytearray(null_bitmap)[0] == 0b00000100
+    assert bytearray(null_bitmap)[0] == 0b00000110
     values = buffers[4].to_pybytes()
     assert struct.unpack('4xh', values) == (43,)
 
@@ -2471,9 +2519,20 @@ def test_numpy_binary_overflow_to_chunked():
 
 @pytest.mark.large_memory
 def test_list_child_overflow_to_chunked():
-    vals = [['x' * 1024]] * ((2 << 20) + 1)
-    with pytest.raises(ValueError, match="overflowed"):
-        pa.array(vals)
+    kilobyte_string = 'x' * 1024
+    two_mega = 2**21
+
+    vals = [[kilobyte_string]] * (two_mega - 1)
+    arr = pa.array(vals)
+    assert isinstance(arr, pa.Array)
+    assert len(arr) == two_mega - 1
+
+    vals = [[kilobyte_string]] * two_mega
+    arr = pa.array(vals)
+    assert isinstance(arr, pa.ChunkedArray)
+    assert len(arr) == two_mega
+    assert len(arr.chunk(0)) == two_mega - 1
+    assert len(arr.chunk(1)) == 1
 
 
 def test_infer_type_masked():
@@ -2573,6 +2632,17 @@ def test_concat_array_different_types():
         pa.concat_arrays([pa.array([1]), pa.array([2.])])
 
 
+def test_concat_array_invalid_type():
+    # ARROW-9920 - do not segfault on non-array input
+
+    with pytest.raises(TypeError, match="should contain Array objects"):
+        pa.concat_arrays([None])
+
+    arr = pa.chunked_array([[0, 1], [3, 4]])
+    with pytest.raises(TypeError, match="should contain Array objects"):
+        pa.concat_arrays(arr)
+
+
 @pytest.mark.pandas
 def test_to_pandas_timezone():
     # https://issues.apache.org/jira/browse/ARROW-6652
diff --git a/python/pyarrow/tests/test_cffi.py b/python/pyarrow/tests/test_cffi.py
index bcf3c723950..5505e571645 100644
--- a/python/pyarrow/tests/test_cffi.py
+++ b/python/pyarrow/tests/test_cffi.py
@@ -26,6 +26,13 @@
 
 import pytest
 
+try:
+    import pandas as pd
+    import pandas.testing as tm
+except ImportError:
+    pd = tm = None
+
+
 needs_cffi = pytest.mark.skipif(ffi is None,
                                 reason="test needs cffi package installed")
 
@@ -36,6 +43,34 @@
 assert_array_released = pytest.raises(
     ValueError, match="Cannot import released ArrowArray")
 
+assert_stream_released = pytest.raises(
+    ValueError, match="Cannot import released ArrowArrayStream")
+
+
+def make_schema():
+    return pa.schema([('ints', pa.list_(pa.int32()))],
+                     metadata={b'key1': b'value1'})
+
+
+def make_batch():
+    return pa.record_batch([[[1], [2, 42]]], make_schema())
+
+
+def make_batches():
+    schema = make_schema()
+    return [
+        pa.record_batch([[[1], [2, 42]]], schema),
+        pa.record_batch([[None, [], [5, 6]]], schema),
+    ]
+
+
+def make_serialized(schema, batches):
+    with pa.BufferOutputStream() as sink:
+        with pa.ipc.new_stream(sink, schema) as out:
+            for batch in batches:
+                out.write(batch)
+        return sink.getvalue()
+
 
 @needs_cffi
 def test_export_import_type():
@@ -120,10 +155,6 @@ def test_export_import_schema():
     c_schema = ffi.new("struct ArrowSchema*")
     ptr_schema = int(ffi.cast("uintptr_t", c_schema))
 
-    def make_schema():
-        return pa.schema([('ints', pa.list_(pa.int32()))],
-                         metadata={b'key1': b'value1'})
-
     gc.collect()  # Make sure no Arrow data dangles in a ref cycle
     old_allocated = pa.total_allocated_bytes()
 
@@ -156,13 +187,6 @@ def test_export_import_batch():
     c_array = ffi.new("struct ArrowArray*")
     ptr_array = int(ffi.cast("uintptr_t", c_array))
 
-    def make_schema():
-        return pa.schema([('ints', pa.list_(pa.int32()))],
-                         metadata={b'key1': b'value1'})
-
-    def make_batch():
-        return pa.record_batch([[[1], [2, 42]]], make_schema())
-
     gc.collect()  # Make sure no Arrow data dangles in a ref cycle
     old_allocated = pa.total_allocated_bytes()
 
@@ -172,7 +196,7 @@ def make_batch():
     py_value = batch.to_pydict()
     batch._export_to_c(ptr_array)
     assert pa.total_allocated_bytes() > old_allocated
-    # Delete recreate C++ object from exported pointer
+    # Delete and recreate C++ object from exported pointer
     del batch
     batch_new = pa.RecordBatch._import_from_c(ptr_array, schema)
     assert batch_new.to_pydict() == py_value
@@ -192,8 +216,6 @@ def make_batch():
     del batch
     batch_new = pa.RecordBatch._import_from_c(ptr_array, ptr_schema)
     assert batch_new.to_pydict() == py_value
-    print(batch_new.schema)
-    print(make_schema())
     assert batch_new.schema == make_schema()
     assert pa.total_allocated_bytes() > old_allocated
     del batch_new
@@ -211,3 +233,63 @@ def make_batch():
     # Now released
     with assert_schema_released:
         pa.RecordBatch._import_from_c(ptr_array, ptr_schema)
+
+
+def _export_import_batch_reader(ptr_stream, reader_factory):
+    # Prepare input
+    batches = make_batches()
+    schema = batches[0].schema
+
+    reader = reader_factory(schema, batches)
+    reader._export_to_c(ptr_stream)
+    # Delete and recreate C++ object from exported pointer
+    del reader, batches
+
+    reader_new = pa.ipc.RecordBatchReader._import_from_c(ptr_stream)
+    assert reader_new.schema == schema
+    got_batches = list(reader_new)
+    del reader_new
+    assert got_batches == make_batches()
+
+    # Test read_pandas()
+    if pd is not None:
+        batches = make_batches()
+        schema = batches[0].schema
+        expected_df = pa.Table.from_batches(batches).to_pandas()
+
+        reader = reader_factory(schema, batches)
+        reader._export_to_c(ptr_stream)
+        del reader, batches
+
+        reader_new = pa.ipc.RecordBatchReader._import_from_c(ptr_stream)
+        got_df = reader_new.read_pandas()
+        del reader_new
+        tm.assert_frame_equal(expected_df, got_df)
+
+
+def make_ipc_stream_reader(schema, batches):
+    return pa.ipc.open_stream(make_serialized(schema, batches))
+
+
+def make_py_record_batch_reader(schema, batches):
+    return pa.ipc.RecordBatchReader.from_batches(schema, batches)
+
+
+@needs_cffi
+@pytest.mark.parametrize('reader_factory',
+                         [make_ipc_stream_reader,
+                          make_py_record_batch_reader])
+def test_export_import_batch_reader(reader_factory):
+    c_stream = ffi.new("struct ArrowArrayStream*")
+    ptr_stream = int(ffi.cast("uintptr_t", c_stream))
+
+    gc.collect()  # Make sure no Arrow data dangles in a ref cycle
+    old_allocated = pa.total_allocated_bytes()
+
+    _export_import_batch_reader(ptr_stream, reader_factory)
+
+    assert pa.total_allocated_bytes() == old_allocated
+
+    # Now released
+    with assert_stream_released:
+        pa.ipc.RecordBatchReader._import_from_c(ptr_stream)
diff --git a/python/pyarrow/tests/test_compute.py b/python/pyarrow/tests/test_compute.py
index bdc057d707b..048245cf871 100644
--- a/python/pyarrow/tests/test_compute.py
+++ b/python/pyarrow/tests/test_compute.py
@@ -15,9 +15,15 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from datetime import datetime
 from functools import lru_cache
-import numpy as np
+import inspect
+import pickle
 import pytest
+import random
+import textwrap
+
+import numpy as np
 
 import pyarrow as pa
 import pyarrow.compute as pc
@@ -50,6 +56,13 @@
     if hasattr(func, '__arrow_compute_function__')]
 
 
+exported_option_classes = [
+    cls for (name, cls) in sorted(pc.__dict__.items())
+    if (isinstance(cls, type) and
+        cls is not pc.FunctionOptions and
+        issubclass(cls, pc.FunctionOptions))]
+
+
 numerical_arrow_types = [
     pa.int8(),
     pa.int16(),
@@ -70,13 +83,113 @@ def test_exported_functions():
     functions = exported_functions
     assert len(functions) >= 10
     for func in functions:
-        args = [None] * func.__arrow_compute_function__['arity']
+        args = [object()] * func.__arrow_compute_function__['arity']
         with pytest.raises(TypeError,
                            match="Got unexpected argument type "
-                                 "<class 'NoneType'> for compute function"):
+                                 "<class 'object'> for compute function"):
             func(*args)
 
 
+def test_exported_option_classes():
+    classes = exported_option_classes
+    assert len(classes) >= 10
+    for cls in classes:
+        # Option classes must have an introspectable constructor signature,
+        # and that signature should not have any *args or **kwargs.
+        sig = inspect.signature(cls)
+        for param in sig.parameters.values():
+            assert param.kind not in (param.VAR_POSITIONAL,
+                                      param.VAR_KEYWORD)
+
+
+def test_list_functions():
+    assert len(pc.list_functions()) > 10
+    assert "add" in pc.list_functions()
+
+
+def _check_get_function(name, expected_func_cls, expected_ker_cls,
+                        min_num_kernels=1):
+    func = pc.get_function(name)
+    assert isinstance(func, expected_func_cls)
+    n = func.num_kernels
+    assert n >= min_num_kernels
+    assert n == len(func.kernels)
+    assert all(isinstance(ker, expected_ker_cls) for ker in func.kernels)
+
+
+def test_get_function_scalar():
+    _check_get_function("add", pc.ScalarFunction, pc.ScalarKernel, 8)
+
+
+def test_get_function_vector():
+    _check_get_function("unique", pc.VectorFunction, pc.VectorKernel, 8)
+
+
+def test_get_function_aggregate():
+    _check_get_function("mean", pc.ScalarAggregateFunction,
+                        pc.ScalarAggregateKernel, 8)
+
+
+def test_call_function_with_memory_pool():
+    arr = pa.array(["foo", "bar", "baz"])
+    indices = np.array([2, 2, 1])
+    result1 = arr.take(indices)
+    result2 = pc.call_function('take', [arr, indices],
+                               memory_pool=pa.default_memory_pool())
+    expected = pa.array(["baz", "baz", "bar"])
+    assert result1.equals(expected)
+    assert result2.equals(expected)
+
+    result3 = pc.take(arr, indices, memory_pool=pa.default_memory_pool())
+    assert result3.equals(expected)
+
+
+def test_pickle_functions():
+    # Pickle registered functions
+    for name in pc.list_functions():
+        func = pc.get_function(name)
+        reconstructed = pickle.loads(pickle.dumps(func))
+        assert type(reconstructed) is type(func)
+        assert reconstructed.name == func.name
+        assert reconstructed.arity == func.arity
+        assert reconstructed.num_kernels == func.num_kernels
+
+
+def test_pickle_global_functions():
+    # Pickle global wrappers (manual or automatic) of registered functions
+    for name in pc.list_functions():
+        func = getattr(pc, name)
+        reconstructed = pickle.loads(pickle.dumps(func))
+        assert reconstructed is func
+
+
+def test_function_attributes():
+    # Sanity check attributes of registered functions
+    for name in pc.list_functions():
+        func = pc.get_function(name)
+        assert isinstance(func, pc.Function)
+        assert func.name == name
+        kernels = func.kernels
+        assert func.num_kernels == len(kernels)
+        assert all(isinstance(ker, pc.Kernel) for ker in kernels)
+        assert func.arity >= 1  # no varargs functions for now
+        repr(func)
+        for ker in kernels:
+            repr(ker)
+
+
+def test_input_type_conversion():
+    # Automatic array conversion from Python
+    arr = pc.add([1, 2], [4, None])
+    assert arr.to_pylist() == [5, None]
+    # Automatic scalar conversion from Python
+    arr = pc.add([1, 2], 4)
+    assert arr.to_pylist() == [5, 6]
+    # Other scalar type
+    assert pc.equal(["foo", "bar", None],
+                    "foo").to_pylist() == [True, False, None]
+
+
 @pytest.mark.parametrize('arrow_type', numerical_arrow_types)
 def test_sum_array(arrow_type):
     arr = pa.array([1, 2, 3, 4], type=arrow_type)
@@ -109,6 +222,36 @@ def test_sum_chunked_array(arrow_type):
     assert pc.sum(arr).as_py() is None  # noqa: E711
 
 
+def test_mode_array():
+    # ARROW-9917
+    arr = pa.array([1, 1, 3, 4, 3, 5], type='int64')
+    expected = {"mode": 1, "count": 2}
+    assert pc.mode(arr).as_py() == {"mode": 1, "count": 2}
+
+    arr = pa.array([], type='int64')
+    expected = {"mode": None, "count": None}
+    assert pc.mode(arr).as_py() == expected
+
+
+def test_mode_chunked_array():
+    # ARROW-9917
+    arr = pa.chunked_array([pa.array([1, 1, 3, 4, 3, 5], type='int64')])
+    expected = {"mode": 1, "count": 2}
+    assert pc.mode(arr).as_py() == expected
+
+    arr = pa.chunked_array((), type='int64')
+    expected = {"mode": None, "count": None}
+    assert arr.num_chunks == 0
+    assert pc.mode(arr).as_py() == expected
+
+
+def test_variance():
+    data = [1, 2, 3, 4, 5, 6, 7, 8]
+    assert pc.variance(data).as_py() == 5.25
+    assert pc.variance(data, ddof=0).as_py() == 5.25
+    assert pc.variance(data, ddof=1).as_py() == 6.0
+
+
 def test_match_substring():
     arr = pa.array(["ab", "abc", "ba", None])
     result = pc.match_substring(arr, "ab")
@@ -116,6 +259,133 @@ def test_match_substring():
     assert expected.equals(result)
 
 
+def test_split_pattern():
+    arr = pa.array(["-foo---bar--", "---foo---b"])
+    result = pc.split_pattern(arr, pattern="---")
+    expected = pa.array([["-foo", "bar--"], ["", "foo", "b"]])
+    assert expected.equals(result)
+
+    result = pc.split_pattern(arr, pattern="---", max_splits=1)
+    expected = pa.array([["-foo", "bar--"], ["", "foo---b"]])
+    assert expected.equals(result)
+
+    result = pc.split_pattern(arr, pattern="---", max_splits=1, reverse=True)
+    expected = pa.array([["-foo", "bar--"], ["---foo", "b"]])
+    assert expected.equals(result)
+
+
+def test_split_whitespace_utf8():
+    arr = pa.array(["foo bar", " foo  \u3000\tb"])
+    result = pc.utf8_split_whitespace(arr)
+    expected = pa.array([["foo", "bar"], ["", "foo", "b"]])
+    assert expected.equals(result)
+
+    result = pc.utf8_split_whitespace(arr, max_splits=1)
+    expected = pa.array([["foo", "bar"], ["", "foo  \u3000\tb"]])
+    assert expected.equals(result)
+
+    result = pc.utf8_split_whitespace(arr, max_splits=1, reverse=True)
+    expected = pa.array([["foo", "bar"], [" foo", "b"]])
+    assert expected.equals(result)
+
+
+def test_split_whitespace_ascii():
+    arr = pa.array(["foo bar", " foo  \u3000\tb"])
+    result = pc.ascii_split_whitespace(arr)
+    expected = pa.array([["foo", "bar"], ["", "foo", "\u3000", "b"]])
+    assert expected.equals(result)
+
+    result = pc.ascii_split_whitespace(arr, max_splits=1)
+    expected = pa.array([["foo", "bar"], ["", "foo  \u3000\tb"]])
+    assert expected.equals(result)
+
+    result = pc.ascii_split_whitespace(arr, max_splits=1, reverse=True)
+    expected = pa.array([["foo", "bar"], [" foo  \u3000", "b"]])
+    assert expected.equals(result)
+
+
+def test_min_max():
+    # An example generated function wrapper with possible options
+    data = [4, 5, 6, None, 1]
+    s = pc.min_max(data)
+    assert s.as_py() == {'min': 1, 'max': 6}
+    s = pc.min_max(data, options=pc.MinMaxOptions())
+    assert s.as_py() == {'min': 1, 'max': 6}
+    s = pc.min_max(data, options=pc.MinMaxOptions(null_handling='skip'))
+    assert s.as_py() == {'min': 1, 'max': 6}
+    s = pc.min_max(data, options=pc.MinMaxOptions(null_handling='emit_null'))
+    assert s.as_py() == {'min': None, 'max': None}
+
+    # Options as dict of kwargs
+    s = pc.min_max(data, options={'null_handling': 'emit_null'})
+    assert s.as_py() == {'min': None, 'max': None}
+    # Options as named functions arguments
+    s = pc.min_max(data, null_handling='emit_null')
+    assert s.as_py() == {'min': None, 'max': None}
+
+    # Both options and named arguments
+    with pytest.raises(TypeError):
+        s = pc.min_max(data, options=pc.MinMaxOptions(),
+                       null_handling='emit_null')
+
+    # Wrong options type
+    options = pc.TakeOptions()
+    with pytest.raises(TypeError):
+        s = pc.min_max(data, options=options)
+
+    # Missing argument
+    with pytest.raises(
+            TypeError,
+            match=r"min_max\(\) missing 1 required positional argument"):
+        s = pc.min_max()
+
+
+def test_is_valid():
+    # An example generated function wrapper without options
+    data = [4, 5, None]
+    assert pc.is_valid(data).to_pylist() == [True, True, False]
+
+    with pytest.raises(TypeError):
+        pc.is_valid(data, options=None)
+
+
+def test_generated_docstrings():
+    assert pc.min_max.__doc__ == textwrap.dedent("""\
+        Compute the minimum and maximum values of a numeric array.
+
+        Null values are ignored by default.
+        This can be changed through MinMaxOptions.
+
+        Parameters
+        ----------
+        array : Array-like
+            Argument to compute function
+        memory_pool : pyarrow.MemoryPool, optional
+            If not passed, will allocate memory from the default memory pool.
+        options : pyarrow.compute.MinMaxOptions, optional
+            Parameters altering compute function semantics
+        **kwargs: optional
+            Parameters for MinMaxOptions constructor.  Either `options`
+            or `**kwargs` can be passed, but not both at the same time.
+        """)
+    assert pc.add.__doc__ == textwrap.dedent("""\
+        Add the arguments element-wise.
+
+        Results will wrap around on integer overflow.
+        Use function "add_checked" if you want overflow
+        to return an error.
+
+        Parameters
+        ----------
+        x : Array-like or scalar-like
+            Argument to compute function
+        y : Array-like or scalar-like
+            Argument to compute function
+        memory_pool : pyarrow.MemoryPool, optional
+            If not passed, will allocate memory from the default memory pool.
+        """)
+
+
 # We use isprintable to find about codepoints that Python doesn't know, but
 # utf8proc does (or in a future version of Python the other way around).
 # These codepoints cannot be compared between Arrow and the Python
@@ -234,11 +504,13 @@ def test_string_py_compat_boolean(function_name, variant):
         # the issues we know of, we skip
         if i in ignore:
             continue
+        # Compare results with the equivalent Python predicate
+        # (except "is_space" where functions are known to be incompatible)
         c = chr(i)
-        if hasattr(pc, arrow_name):
+        if hasattr(pc, arrow_name) and function_name != 'is_space':
             ar = pa.array([c])
-            assert getattr(pc, arrow_name)(
-                ar)[0].as_py() == getattr(c, py_name)()
+            arrow_func = getattr(pc, arrow_name)
+            assert arrow_func(ar)[0].as_py() == getattr(c, py_name)()
 
 
 @pytest.mark.parametrize(('ty', 'values'), all_array_types)
@@ -322,17 +594,6 @@ def test_take_on_chunked_array():
     assert result.equals(expected)
 
 
-def test_call_function_with_memory_pool():
-    arr = pa.array(["foo", "bar", "baz"])
-    indices = np.array([2, 2, 1])
-    result1 = arr.take(indices)
-    result2 = pc.call_function('take', [arr, indices],
-                               memory_pool=pa.default_memory_pool())
-    expected = pa.array(["baz", "baz", "bar"])
-    assert result1.equals(expected)
-    assert result2.equals(expected)
-
-
 @pytest.mark.parametrize('ordered', [False, True])
 def test_take_dictionary(ordered):
     arr = pa.DictionaryArray.from_arrays([0, 1, 2, 0, 1, 2], ['a', 'b', 'c'],
@@ -344,6 +605,20 @@ def test_take_dictionary(ordered):
     assert result.type.ordered is ordered
 
 
+def test_take_null_type():
+    # ARROW-10027
+    arr = pa.array([None] * 10)
+    chunked_arr = pa.chunked_array([[None] * 5] * 2)
+    batch = pa.record_batch([arr], names=['a'])
+    table = pa.table({'a': arr})
+
+    indices = pa.array([1, 3, 7, None])
+    assert len(arr.take(indices)) == 4
+    assert len(chunked_arr.take(indices)) == 4
+    assert len(batch.take(indices).column(0)) == 4
+    assert len(table.take(indices).column(0)) == 4
+
+
 @pytest.mark.parametrize(('ty', 'values'), all_array_types)
 def test_filter(ty, values):
     arr = pa.array(values, type=ty)
@@ -439,6 +714,20 @@ def test_filter_errors():
             obj.filter(mask)
 
 
+def test_filter_null_type():
+    # ARROW-10027
+    arr = pa.array([None] * 10)
+    chunked_arr = pa.chunked_array([[None] * 5] * 2)
+    batch = pa.record_batch([arr], names=['a'])
+    table = pa.table({'a': arr})
+
+    mask = pa.array([True, False] * 5)
+    assert len(arr.filter(mask)) == 5
+    assert len(chunked_arr.filter(mask)) == 5
+    assert len(batch.filter(mask).column(0)) == 5
+    assert len(table.filter(mask).column(0)) == 5
+
+
 @pytest.mark.parametrize("typ", ["array", "chunked_array"])
 def test_compare_array(typ):
     if typ == "array":
@@ -562,7 +851,7 @@ def test_is_null():
 def test_fill_null():
     arr = pa.array([1, 2, None, 4], type=pa.int8())
     fill_value = pa.array([5], type=pa.int8())
-    with pytest.raises(TypeError):
+    with pytest.raises(pa.ArrowInvalid, match="tried to convert to int"):
         arr.fill_null(fill_value)
 
     arr = pa.array([None, None, None, None], type=pa.null())
@@ -620,3 +909,63 @@ def test_fill_null_chunked_array(arrow_type):
 
     result = arr.fill_null(pa.scalar(5, type='int8'))
     assert result.equals(expected)
+
+
+def test_logical():
+    a = pa.array([True, False, False, None])
+    b = pa.array([True, True, False, True])
+
+    assert pc.and_(a, b) == pa.array([True, False, False, None])
+    assert pc.and_kleene(a, b) == pa.array([True, False, False, None])
+
+    assert pc.or_(a, b) == pa.array([True, True, False, None])
+    assert pc.or_kleene(a, b) == pa.array([True, True, False, True])
+
+    assert pc.xor(a, b) == pa.array([False, True, False, None])
+
+    assert pc.invert(a) == pa.array([False, True, True, None])
+
+
+def test_cast():
+    arr = pa.array([2**63 - 1], type='int64')
+
+    with pytest.raises(pa.ArrowInvalid):
+        pc.cast(arr, 'int32')
+
+    assert pc.cast(arr, 'int32', safe=False) == pa.array([-1], type='int32')
+
+    arr = pa.array([datetime(2010, 1, 1), datetime(2015, 1, 1)])
+    expected = pa.array([1262304000000, 1420070400000], type='timestamp[ms]')
+    assert pc.cast(arr, 'timestamp[ms]') == expected
+
+
+def test_strptime():
+    arr = pa.array(["5/1/2020", None, "12/13/1900"])
+
+    got = pc.strptime(arr, format='%m/%d/%Y', unit='s')
+    expected = pa.array([datetime(2020, 5, 1), None, datetime(1900, 12, 13)],
+                        type=pa.timestamp('s'))
+    assert got == expected
+
+
+def test_count():
+    arr = pa.array([1, 2, 3, None, None])
+    assert pc.count(arr).as_py() == 3
+    assert pc.count(arr, count_mode='count_non_null').as_py() == 3
+    assert pc.count(arr, count_mode='count_null').as_py() == 2
+
+    with pytest.raises(ValueError, match="'zzz' is not a valid count_mode"):
+        pc.count(arr, count_mode='zzz')
+
+
+def test_partition_nth():
+    data = list(range(100, 140))
+    random.shuffle(data)
+    pivot = 10
+    indices = pc.partition_nth_indices(data, pivot=pivot).to_pylist()
+    assert len(indices) == len(data)
+    assert sorted(indices) == list(range(len(data)))
+    assert all(data[indices[i]] <= data[indices[pivot]]
+               for i in range(pivot))
+    assert all(data[indices[i]] >= data[indices[pivot]]
+               for i in range(pivot, len(data)))
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index 2bdb64005f4..d150833723b 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -15,20 +15,20 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import pytest
-
-from pyarrow.pandas_compat import _pandas_api  # noqa
-import pyarrow as pa
-
 import collections
 import datetime
 import decimal
 import itertools
 import math
-import traceback
 
+import hypothesis as h
 import numpy as np
 import pytz
+import pytest
+
+from pyarrow.pandas_compat import _pandas_api  # noqa
+import pyarrow as pa
+import pyarrow.tests.strategies as past
 
 
 int_type_pairs = [
@@ -382,18 +382,21 @@ def test_sequence_custom_integers(seq):
 @parametrize_with_iterable_types
 def test_broken_integers(seq):
     data = [MyBrokenInt()]
-    with pytest.raises(ZeroDivisionError) as exc_info:
+    with pytest.raises(pa.ArrowInvalid, match="tried to convert to int"):
         pa.array(seq(data), type=pa.int64())
-    # Original traceback is kept
-    tb_lines = traceback.format_tb(exc_info.tb)
-    assert "# MARKER" in tb_lines[-1]
 
 
 def test_numpy_scalars_mixed_type():
     # ARROW-4324
     data = [np.int32(10), np.float32(0.5)]
     arr = pa.array(data)
-    expected = pa.array([10, 0.5], type='float64')
+    expected = pa.array([10, 0.5], type="float64")
+    assert arr.equals(expected)
+
+    # ARROW-9490
+    data = [np.int8(10), np.float32(0.5)]
+    arr = pa.array(data)
+    expected = pa.array([10, 0.5], type="float32")
     assert arr.equals(expected)
 
 
@@ -566,7 +569,7 @@ def test_ndarray_nested_numpy_double(from_pandas, inner_seq):
         inner_seq([1., 2., 3.]),
         inner_seq([np.nan]),
         None
-    ])
+    ], dtype=object)
     arr = pa.array(data, from_pandas=from_pandas)
     assert len(arr) == 4
     assert arr.null_count == 1
@@ -623,6 +626,50 @@ def test_multidimensional_ndarray_as_nested_list():
     assert result.equals(expected)
 
 
+@pytest.mark.parametrize(('data', 'value_type'), [
+    ([True, False], pa.bool_()),
+    ([None, None], pa.null()),
+    ([1, 2, None], pa.int8()),
+    ([1, 2., 3., None], pa.float32()),
+    ([datetime.date.today(), None], pa.date32()),
+    ([None, datetime.date.today()], pa.date64()),
+    ([datetime.time(1, 1, 1), None], pa.time32('s')),
+    ([None, datetime.time(2, 2, 2)], pa.time64('us')),
+    ([datetime.datetime.now(), None], pa.timestamp('us')),
+    ([datetime.timedelta(seconds=10)], pa.duration('s')),
+    ([b"a", b"b"], pa.binary()),
+    ([b"aaa", b"bbb", b"ccc"], pa.binary(3)),
+    ([b"a", b"b", b"c"], pa.large_binary()),
+    (["a", "b", "c"], pa.string()),
+    (["a", "b", "c"], pa.large_string()),
+    (
+        [{"a": 1, "b": 2}, None, {"a": 5, "b": None}],
+        pa.struct([('a', pa.int8()), ('b', pa.int16())])
+    )
+])
+def test_list_array_from_object_ndarray(data, value_type):
+    ty = pa.list_(value_type)
+    ndarray = np.array(data, dtype=object)
+    arr = pa.array([ndarray], type=ty)
+    assert arr.type.equals(ty)
+    assert arr.to_pylist() == [data]
+
+
+@pytest.mark.parametrize(('data', 'value_type'), [
+    ([[1, 2], [3]], pa.list_(pa.int64())),
+    ([[1, 2], [3, 4]], pa.list_(pa.int64(), 2)),
+    ([[1], [2, 3]], pa.large_list(pa.int64()))
+])
+def test_nested_list_array_from_object_ndarray(data, value_type):
+    ndarray = np.empty(len(data), dtype=object)
+    ndarray[:] = [np.array(item, dtype=object) for item in data]
+
+    ty = pa.list_(value_type)
+    arr = pa.array([ndarray], type=ty)
+    assert arr.type.equals(ty)
+    assert arr.to_pylist() == [data]
+
+
 def test_array_ignore_nan_from_pandas():
     # See ARROW-4324, this reverts logic that was introduced in
     # ARROW-2240
@@ -760,6 +807,15 @@ def test_fixed_size_bytes_does_not_accept_varying_lengths():
         pa.array(data, type=pa.binary(4))
 
 
+def test_fixed_size_binary_length_check():
+    # ARROW-10193
+    data = [b'\x19h\r\x9e\x00\x00\x00\x00\x01\x9b\x9fA']
+    assert len(data[0]) == 12
+    ty = pa.binary(12)
+    arr = pa.array(data, type=ty)
+    assert arr.to_pylist() == data
+
+
 def test_sequence_date():
     data = [datetime.date(2000, 1, 1), None, datetime.date(1970, 1, 1),
             datetime.date(2040, 2, 26)]
@@ -797,6 +853,70 @@ def test_date32_overflow():
         pa.array(data3, type=pa.date32())
 
 
+@pytest.mark.parametrize(('time_type', 'unit', 'int_type'), [
+    (pa.time32, 's', 'int32'),
+    (pa.time32, 'ms', 'int32'),
+    (pa.time64, 'us', 'int64'),
+    (pa.time64, 'ns', 'int64'),
+])
+def test_sequence_time_with_timezone(time_type, unit, int_type):
+    def expected_integer_value(t):
+        # only use with utc time object because it doesn't adjust with the
+        # offset
+        units = ['s', 'ms', 'us', 'ns']
+        multiplier = 10**(units.index(unit) * 3)
+        if t is None:
+            return None
+        seconds = (
+            t.hour * 3600 +
+            t.minute * 60 +
+            t.second +
+            t.microsecond * 10**-6
+        )
+        return int(seconds * multiplier)
+
+    def expected_time_value(t):
+        # only use with utc time object because it doesn't adjust with the
+        # time objects tzdata
+        if unit == 's':
+            return t.replace(microsecond=0)
+        elif unit == 'ms':
+            return t.replace(microsecond=(t.microsecond // 1000) * 1000)
+        else:
+            return t
+
+    # only timezone naive times are supported in arrow
+    data = [
+        datetime.time(8, 23, 34, 123456),
+        datetime.time(5, 0, 0, 1000),
+        None,
+        datetime.time(1, 11, 56, 432539),
+        datetime.time(23, 10, 0, 437699)
+    ]
+
+    ty = time_type(unit)
+    arr = pa.array(data, type=ty)
+    assert len(arr) == 5
+    assert arr.type == ty
+    assert arr.null_count == 1
+
+    # test that the underlying integers are UTC values
+    values = arr.cast(int_type)
+    expected = list(map(expected_integer_value, data))
+    assert values.to_pylist() == expected
+
+    # test that the scalars are datetime.time objects with UTC timezone
+    assert arr[0].as_py() == expected_time_value(data[0])
+    assert arr[1].as_py() == expected_time_value(data[1])
+    assert arr[2].as_py() is None
+    assert arr[3].as_py() == expected_time_value(data[3])
+    assert arr[4].as_py() == expected_time_value(data[4])
+
+    def tz(hours, minutes=0):
+        offset = datetime.timedelta(hours=hours, minutes=minutes)
+        return datetime.timezone(offset)
+
+
 def test_sequence_timestamp():
     data = [
         datetime.datetime(2007, 7, 13, 1, 23, 34, 123456),
@@ -817,6 +937,191 @@ def test_sequence_timestamp():
                                                46, 57, 437699)
 
 
+@pytest.mark.parametrize('timezone', [
+    None,
+    'UTC',
+    'Etc/GMT-1',
+    'Europe/Budapest',
+])
+@pytest.mark.parametrize('unit', [
+    's',
+    'ms',
+    'us',
+    'ns'
+])
+def test_sequence_timestamp_with_timezone(timezone, unit):
+    def expected_integer_value(dt):
+        units = ['s', 'ms', 'us', 'ns']
+        multiplier = 10**(units.index(unit) * 3)
+        if dt is None:
+            return None
+        else:
+            # avoid float precision issues
+            ts = decimal.Decimal(str(dt.timestamp()))
+            return int(ts * multiplier)
+
+    def expected_datetime_value(dt):
+        if dt is None:
+            return None
+
+        if unit == 's':
+            dt = dt.replace(microsecond=0)
+        elif unit == 'ms':
+            dt = dt.replace(microsecond=(dt.microsecond // 1000) * 1000)
+
+        # adjust the timezone
+        if timezone is None:
+            # make datetime timezone unaware
+            return dt.replace(tzinfo=None)
+        else:
+            # convert to the expected timezone
+            return dt.astimezone(pytz.timezone(timezone))
+
+    data = [
+        datetime.datetime(2007, 7, 13, 8, 23, 34, 123456),  # naive
+        pytz.utc.localize(
+            datetime.datetime(2008, 1, 5, 5, 0, 0, 1000)
+        ),
+        None,
+        pytz.timezone('US/Eastern').localize(
+            datetime.datetime(2006, 1, 13, 12, 34, 56, 432539)
+        ),
+        pytz.timezone('Europe/Moscow').localize(
+            datetime.datetime(2010, 8, 13, 5, 0, 0, 437699)
+        ),
+    ]
+    utcdata = [
+        pytz.utc.localize(data[0]),
+        data[1],
+        None,
+        data[3].astimezone(pytz.utc),
+        data[4].astimezone(pytz.utc),
+    ]
+
+    ty = pa.timestamp(unit, tz=timezone)
+    arr = pa.array(data, type=ty)
+    assert len(arr) == 5
+    assert arr.type == ty
+    assert arr.null_count == 1
+
+    # test that the underlying integers are UTC values
+    values = arr.cast('int64')
+    expected = list(map(expected_integer_value, utcdata))
+    assert values.to_pylist() == expected
+
+    # test that the scalars are datetimes with the correct timezone
+    for i in range(len(arr)):
+        assert arr[i].as_py() == expected_datetime_value(utcdata[i])
+
+
+@pytest.mark.parametrize('timezone', [
+    None,
+    'UTC',
+    'Etc/GMT-1',
+    'Europe/Budapest',
+])
+def test_pyarrow_ignore_timezone_environment_variable(monkeypatch, timezone):
+    # note that any non-empty value will evaluate to true
+    monkeypatch.setenv("PYARROW_IGNORE_TIMEZONE", "1")
+    data = [
+        datetime.datetime(2007, 7, 13, 8, 23, 34, 123456),  # naive
+        pytz.utc.localize(
+            datetime.datetime(2008, 1, 5, 5, 0, 0, 1000)
+        ),
+        pytz.timezone('US/Eastern').localize(
+            datetime.datetime(2006, 1, 13, 12, 34, 56, 432539)
+        ),
+        pytz.timezone('Europe/Moscow').localize(
+            datetime.datetime(2010, 8, 13, 5, 0, 0, 437699)
+        ),
+    ]
+
+    expected = [dt.replace(tzinfo=None) for dt in data]
+    if timezone is not None:
+        tzinfo = pytz.timezone(timezone)
+        expected = [tzinfo.fromutc(dt) for dt in expected]
+
+    ty = pa.timestamp('us', tz=timezone)
+    arr = pa.array(data, type=ty)
+    assert arr.to_pylist() == expected
+
+
+def test_sequence_timestamp_with_timezone_inference():
+    data = [
+        datetime.datetime(2007, 7, 13, 8, 23, 34, 123456),  # naive
+        pytz.utc.localize(
+            datetime.datetime(2008, 1, 5, 5, 0, 0, 1000)
+        ),
+        None,
+        pytz.timezone('US/Eastern').localize(
+            datetime.datetime(2006, 1, 13, 12, 34, 56, 432539)
+        ),
+        pytz.timezone('Europe/Moscow').localize(
+            datetime.datetime(2010, 8, 13, 5, 0, 0, 437699)
+        ),
+    ]
+    expected = [
+        pa.timestamp('us', tz=None),
+        pa.timestamp('us', tz='UTC'),
+        pa.timestamp('us', tz=None),
+        pa.timestamp('us', tz='US/Eastern'),
+        pa.timestamp('us', tz='Europe/Moscow')
+    ]
+    for dt, expected_type in zip(data, expected):
+        prepended = [dt] + data
+        arr = pa.array(prepended)
+        assert arr.type == expected_type
+
+
+@pytest.mark.pandas
+def test_sequence_timestamp_from_mixed_builtin_and_pandas_datetimes():
+    import pandas as pd
+
+    data = [
+        pd.Timestamp(1184307814123456123, tz=pytz.timezone('US/Eastern'),
+                     unit='ns'),
+        datetime.datetime(2007, 7, 13, 8, 23, 34, 123456),  # naive
+        pytz.utc.localize(
+            datetime.datetime(2008, 1, 5, 5, 0, 0, 1000)
+        ),
+        None,
+    ]
+    utcdata = [
+        data[0].astimezone(pytz.utc),
+        pytz.utc.localize(data[1]),
+        data[2].astimezone(pytz.utc),
+        None,
+    ]
+
+    arr = pa.array(data)
+    assert arr.type == pa.timestamp('us', tz='US/Eastern')
+
+    values = arr.cast('int64')
+    expected = [int(dt.timestamp() * 10**6) if dt else None for dt in utcdata]
+    assert values.to_pylist() == expected
+
+
+def test_sequence_timestamp_out_of_bounds_nanosecond():
+    # https://issues.apache.org/jira/browse/ARROW-9768
+    # datetime outside of range supported for nanosecond resolution
+    data = [datetime.datetime(2262, 4, 12)]
+    with pytest.raises(ValueError, match="out of bounds"):
+        pa.array(data, type=pa.timestamp('ns'))
+
+    # with microsecond resolution it works fine
+    arr = pa.array(data, type=pa.timestamp('us'))
+    assert arr.to_pylist() == data
+
+    # case where the naive is within bounds, but converted to UTC not
+    tz = datetime.timezone(datetime.timedelta(hours=-1))
+    data = [datetime.datetime(2262, 4, 11, 23, tzinfo=tz)]
+    with pytest.raises(ValueError, match="out of bounds"):
+        pa.array(data, type=pa.timestamp('ns'))
+
+    arr = pa.array(data, type=pa.timestamp('us'))
+    assert arr.to_pylist()[0] == datetime.datetime(2262, 4, 12)
+
+
 def test_sequence_numpy_timestamp():
     data = [
         np.datetime64(datetime.datetime(2007, 7, 13, 1, 23, 34, 123456)),
@@ -837,34 +1142,6 @@ def test_sequence_numpy_timestamp():
                                                46, 57, 437699)
 
 
-def test_sequence_timestamp_with_unit():
-    data = [
-        datetime.datetime(2007, 7, 13, 1, 23, 34, 123456),
-    ]
-
-    s = pa.timestamp('s')
-    ms = pa.timestamp('ms')
-    us = pa.timestamp('us')
-
-    arr_s = pa.array(data, type=s)
-    assert len(arr_s) == 1
-    assert arr_s.type == s
-    assert arr_s[0].as_py() == datetime.datetime(2007, 7, 13, 1,
-                                                 23, 34, 0)
-
-    arr_ms = pa.array(data, type=ms)
-    assert len(arr_ms) == 1
-    assert arr_ms.type == ms
-    assert arr_ms[0].as_py() == datetime.datetime(2007, 7, 13, 1,
-                                                  23, 34, 123000)
-
-    arr_us = pa.array(data, type=us)
-    assert len(arr_us) == 1
-    assert arr_us.type == us
-    assert arr_us[0].as_py() == datetime.datetime(2007, 7, 13, 1,
-                                                  23, 34, 123456)
-
-
 class MyDate(datetime.date):
     pass
 
@@ -1148,61 +1425,62 @@ def test_sequence_mixed_types_with_specified_type_fails():
 
 def test_sequence_decimal():
     data = [decimal.Decimal('1234.183'), decimal.Decimal('8094.234')]
-    type = pa.decimal128(precision=7, scale=3)
-    arr = pa.array(data, type=type)
-    assert arr.to_pylist() == data
+    for type in [pa.decimal128, pa.decimal256]:
+        arr = pa.array(data, type=type(precision=7, scale=3))
+        assert arr.to_pylist() == data
 
 
 def test_sequence_decimal_different_precisions():
     data = [
         decimal.Decimal('1234234983.183'), decimal.Decimal('80943244.234')
     ]
-    type = pa.decimal128(precision=13, scale=3)
-    arr = pa.array(data, type=type)
-    assert arr.to_pylist() == data
+    for type in [pa.decimal128, pa.decimal256]:
+        arr = pa.array(data, type=type(precision=13, scale=3))
+        assert arr.to_pylist() == data
 
 
 def test_sequence_decimal_no_scale():
     data = [decimal.Decimal('1234234983'), decimal.Decimal('8094324')]
-    type = pa.decimal128(precision=10)
-    arr = pa.array(data, type=type)
-    assert arr.to_pylist() == data
+    for type in [pa.decimal128, pa.decimal256]:
+        arr = pa.array(data, type=type(precision=10))
+        assert arr.to_pylist() == data
 
 
 def test_sequence_decimal_negative():
     data = [decimal.Decimal('-1234.234983'), decimal.Decimal('-8.094324')]
-    type = pa.decimal128(precision=10, scale=6)
-    arr = pa.array(data, type=type)
-    assert arr.to_pylist() == data
+    for type in [pa.decimal128, pa.decimal256]:
+        arr = pa.array(data, type=type(precision=10, scale=6))
+        assert arr.to_pylist() == data
 
 
 def test_sequence_decimal_no_whole_part():
     data = [decimal.Decimal('-.4234983'), decimal.Decimal('.0103943')]
-    type = pa.decimal128(precision=7, scale=7)
-    arr = pa.array(data, type=type)
-    assert arr.to_pylist() == data
+    for type in [pa.decimal128, pa.decimal256]:
+        arr = pa.array(data, type=type(precision=7, scale=7))
+        assert arr.to_pylist() == data
 
 
 def test_sequence_decimal_large_integer():
     data = [decimal.Decimal('-394029506937548693.42983'),
             decimal.Decimal('32358695912932.01033')]
-    type = pa.decimal128(precision=23, scale=5)
-    arr = pa.array(data, type=type)
-    assert arr.to_pylist() == data
+    for type in [pa.decimal128, pa.decimal256]:
+        arr = pa.array(data, type=type(precision=23, scale=5))
+        assert arr.to_pylist() == data
 
 
 def test_sequence_decimal_from_integers():
     data = [0, 1, -39402950693754869342983]
     expected = [decimal.Decimal(x) for x in data]
+    # TODO: update this test after scaling implementation.
     type = pa.decimal128(precision=28, scale=5)
     arr = pa.array(data, type=type)
     assert arr.to_pylist() == expected
 
 
 def test_sequence_decimal_too_high_precision():
-    # ARROW-6989 python decimal created from float has too high precision
+    # ARROW-6989 python decimal has too high precision
     with pytest.raises(ValueError, match="precision out of range"):
-        pa.array([decimal.Decimal(123.234)])
+        pa.array([decimal.Decimal('1' * 80)])
 
 
 def test_range_types():
@@ -1322,6 +1600,108 @@ def test_struct_from_tuples():
             pa.array([tup], type=ty)
 
 
+def test_struct_from_list_of_pairs():
+    ty = pa.struct([
+        pa.field('a', pa.int32()),
+        pa.field('b', pa.string()),
+        pa.field('c', pa.bool_())
+    ])
+    data = [
+        [('a', 5), ('b', 'foo'), ('c', True)],
+        [('a', 6), ('b', 'bar'), ('c', False)],
+        None
+    ]
+    arr = pa.array(data, type=ty)
+    assert arr.to_pylist() == [
+        {'a': 5, 'b': 'foo', 'c': True},
+        {'a': 6, 'b': 'bar', 'c': False},
+        None
+    ]
+
+    # test with duplicated field names
+    ty = pa.struct([
+        pa.field('a', pa.int32()),
+        pa.field('a', pa.string()),
+        pa.field('b', pa.bool_())
+    ])
+    data = [
+        [('a', 5), ('a', 'foo'), ('b', True)],
+        [('a', 6), ('a', 'bar'), ('b', False)],
+    ]
+    arr = pa.array(data, type=ty)
+    with pytest.raises(KeyError):
+        # TODO(kszucs): ARROW-9997
+        arr.to_pylist()
+
+    # test with empty elements
+    ty = pa.struct([
+        pa.field('a', pa.int32()),
+        pa.field('b', pa.string()),
+        pa.field('c', pa.bool_())
+    ])
+    data = [
+        [],
+        [('a', 5), ('b', 'foo'), ('c', True)],
+        [('a', 2), ('b', 'baz')],
+        [('a', 1), ('b', 'bar'), ('c', False), ('d', 'julia')],
+    ]
+    expected = [
+        {'a': None, 'b': None, 'c': None},
+        {'a': 5, 'b': 'foo', 'c': True},
+        {'a': 2, 'b': 'baz', 'c': None},
+        {'a': 1, 'b': 'bar', 'c': False},
+    ]
+    arr = pa.array(data, type=ty)
+    assert arr.to_pylist() == expected
+
+
+def test_struct_from_list_of_pairs_errors():
+    ty = pa.struct([
+        pa.field('a', pa.int32()),
+        pa.field('b', pa.string()),
+        pa.field('c', pa.bool_())
+    ])
+
+    # test that it raises if the key doesn't match the expected field name
+    data = [
+        [],
+        [('a', 5), ('c', True), ('b', None)],
+    ]
+    msg = "The expected field name is `b` but `c` was given"
+    with pytest.raises(ValueError, match=msg):
+        pa.array(data, type=ty)
+
+    # test various errors both at the first position and after because of key
+    # type inference
+    template = (
+        r"Could not convert {} with type {}: was expecting tuple of "
+        r"\(key, value\) pair"
+    )
+    cases = [
+        tuple(),  # empty key-value pair
+        tuple('a',),  # missing value
+        tuple('unknown-key',),  # not known field name
+        'string',  # not a tuple
+    ]
+    for key_value_pair in cases:
+        msg = template.format(
+            str(key_value_pair).replace('(', r'\(').replace(')', r'\)'),
+            type(key_value_pair).__name__
+        )
+
+        with pytest.raises(TypeError, match=msg):
+            pa.array([
+                [key_value_pair],
+                [('a', 5), ('b', 'foo'), ('c', None)],
+            ], type=ty)
+
+        with pytest.raises(TypeError, match=msg):
+            pa.array([
+                [('a', 5), ('b', 'foo'), ('c', None)],
+                [key_value_pair],
+            ], type=ty)
+
+
 def test_struct_from_mixed_sequence():
     # It is forbidden to mix dicts and tuples when initializing a struct array
     ty = pa.struct([pa.field('a', pa.int32()),
@@ -1339,6 +1719,7 @@ def test_struct_from_dicts_inference():
                                pa.field('c', pa.bool_())])
     data = [{'a': 5, 'b': 'foo', 'c': True},
             {'a': 6, 'b': 'bar', 'c': False}]
+
     arr = pa.array(data)
     check_struct_type(arr.type, expected_type)
     assert arr.to_pylist() == data
@@ -1352,6 +1733,7 @@ def test_struct_from_dicts_inference():
                 None,
                 {'a': None, 'b': None, 'c': None},
                 {'a': None, 'b': 'bar', 'c': None}]
+
     arr = pa.array(data)
     data_as_ndarray = np.empty(len(data), dtype=object)
     data_as_ndarray[:] = data
@@ -1370,6 +1752,7 @@ def test_struct_from_dicts_inference():
             {'a': {'aa': None, 'ab': False}, 'b': None},
             {'a': None, 'b': 'bar'}]
     arr = pa.array(data)
+
     assert arr.to_pylist() == data
 
     # Edge cases
@@ -1421,17 +1804,6 @@ def test_decimal_array_with_none_and_nan():
     assert array.to_pylist() == [decimal.Decimal('1.2340'), None, None, None]
 
 
-@pytest.mark.parametrize('tz,name', [
-    (pytz.FixedOffset(90), '+01:30'),
-    (pytz.FixedOffset(-90), '-01:30'),
-    (pytz.utc, 'UTC'),
-    (pytz.timezone('America/New_York'), 'America/New_York')
-])
-def test_timezone_string(tz, name):
-    assert pa.lib.tzinfo_to_string(tz) == name
-    assert pa.lib.string_to_tzinfo(name) == tz
-
-
 def test_map_from_dicts():
     data = [[{'key': b'a', 'value': 1}, {'key': b'b', 'value': 2}],
             [{'key': b'c', 'value': 3}],
@@ -1459,7 +1831,7 @@ def test_map_from_dicts():
 
     # Invalid dictionary types
     for entry in [[{'key': '1', 'value': 5}], [{'key': {'value': 2}}]]:
-        with pytest.raises(TypeError, match="integer is required"):
+        with pytest.raises(pa.ArrowInvalid, match="tried to convert to int"):
             pa.array([entry], type=pa.map_('i4', 'i4'))
 
 
@@ -1484,3 +1856,284 @@ def test_map_from_tuples():
     for entry in [[(5,)], [()], [('5', 'foo', True)]]:
         with pytest.raises(ValueError, match="(?i)tuple size"):
             pa.array([entry], type=pa.map_('i4', 'i4'))
+
+
+def test_dictionary_from_boolean():
+    typ = pa.dictionary(pa.int8(), value_type=pa.bool_())
+    a = pa.array([False, False, True, False, True], type=typ)
+    assert isinstance(a.type, pa.DictionaryType)
+    assert a.type.equals(typ)
+
+    expected_indices = pa.array([0, 0, 1, 0, 1], type=pa.int8())
+    expected_dictionary = pa.array([False, True], type=pa.bool_())
+    assert a.indices.equals(expected_indices)
+    assert a.dictionary.equals(expected_dictionary)
+
+
+@pytest.mark.parametrize('value_type', [
+    pa.int8(),
+    pa.int16(),
+    pa.int32(),
+    pa.int64(),
+    pa.uint8(),
+    pa.uint16(),
+    pa.uint32(),
+    pa.uint64(),
+    pa.float32(),
+    pa.float64(),
+])
+def test_dictionary_from_integers(value_type):
+    typ = pa.dictionary(pa.int8(), value_type=value_type)
+    a = pa.array([1, 2, 1, 1, 2, 3], type=typ)
+    assert isinstance(a.type, pa.DictionaryType)
+    assert a.type.equals(typ)
+
+    expected_indices = pa.array([0, 1, 0, 0, 1, 2], type=pa.int8())
+    expected_dictionary = pa.array([1, 2, 3], type=value_type)
+    assert a.indices.equals(expected_indices)
+    assert a.dictionary.equals(expected_dictionary)
+
+
+@pytest.mark.parametrize('input_index_type', [
+    pa.int8(),
+    pa.int16(),
+    pa.int32(),
+    pa.int64()
+])
+def test_dictionary_index_type(input_index_type):
+    # dictionary array is constructed using adaptive index type builder,
+    # but the input index type is considered as the minimal width type to use
+
+    typ = pa.dictionary(input_index_type, value_type=pa.int64())
+    arr = pa.array(range(10), type=typ)
+    assert arr.type.equals(typ)
+
+
+def test_dictionary_is_always_adaptive():
+    # dictionary array is constructed using adaptive index type builder,
+    # meaning that the output index type may be wider than the given index type
+    # since it depends on the input data
+    typ = pa.dictionary(pa.int8(), value_type=pa.int64())
+
+    a = pa.array(range(2**7), type=typ)
+    expected = pa.dictionary(pa.int8(), pa.int64())
+    assert a.type.equals(expected)
+
+    a = pa.array(range(2**7 + 1), type=typ)
+    expected = pa.dictionary(pa.int16(), pa.int64())
+    assert a.type.equals(expected)
+
+
+def test_dictionary_from_strings():
+    for value_type in [pa.binary(), pa.string()]:
+        typ = pa.dictionary(pa.int8(), value_type)
+        a = pa.array(["", "a", "bb", "a", "bb", "ccc"], type=typ)
+
+        assert isinstance(a.type, pa.DictionaryType)
+
+        expected_indices = pa.array([0, 1, 2, 1, 2, 3], type=pa.int8())
+        expected_dictionary = pa.array(["", "a", "bb", "ccc"], type=value_type)
+        assert a.indices.equals(expected_indices)
+        assert a.dictionary.equals(expected_dictionary)
+
+    # fixed size binary type
+    typ = pa.dictionary(pa.int8(), pa.binary(3))
+    a = pa.array(["aaa", "aaa", "bbb", "ccc", "bbb"], type=typ)
+    assert isinstance(a.type, pa.DictionaryType)
+
+    expected_indices = pa.array([0, 0, 1, 2, 1], type=pa.int8())
+    expected_dictionary = pa.array(["aaa", "bbb", "ccc"], type=pa.binary(3))
+    assert a.indices.equals(expected_indices)
+    assert a.dictionary.equals(expected_dictionary)
+
+
+@pytest.mark.parametrize(('unit', 'expected'), [
+    ('s', datetime.timedelta(seconds=-2147483000)),
+    ('ms', datetime.timedelta(milliseconds=-2147483000)),
+    ('us', datetime.timedelta(microseconds=-2147483000)),
+    ('ns', datetime.timedelta(microseconds=-2147483))
+])
+def test_duration_array_roundtrip_corner_cases(unit, expected):
+    # Corner case discovered by hypothesis: there were implicit conversions to
+    # unsigned values resulting wrong values with wrong signs.
+    ty = pa.duration(unit)
+    arr = pa.array([-2147483000], type=ty)
+    restored = pa.array(arr.to_pylist(), type=ty)
+    assert arr.equals(restored)
+
+    expected_list = [expected]
+    if unit == 'ns':
+        # if pandas is available then a pandas Timedelta is returned
+        try:
+            import pandas as pd
+        except ImportError:
+            pass
+        else:
+            expected_list = [pd.Timedelta(-2147483000, unit='ns')]
+
+    assert restored.to_pylist() == expected_list
+
+
+@pytest.mark.pandas
+def test_roundtrip_nanosecond_resolution_pandas_temporal_objects():
+    # corner case discovered by hypothesis: preserving the nanoseconds on
+    # conversion from a list of Timedelta and Timestamp objects
+    import pandas as pd
+
+    ty = pa.duration('ns')
+    arr = pa.array([9223371273709551616], type=ty)
+    data = arr.to_pylist()
+    assert isinstance(data[0], pd.Timedelta)
+    restored = pa.array(data, type=ty)
+    assert arr.equals(restored)
+    assert restored.to_pylist() == [
+        pd.Timedelta(9223371273709551616, unit='ns')
+    ]
+
+    ty = pa.timestamp('ns')
+    arr = pa.array([9223371273709551616], type=ty)
+    data = arr.to_pylist()
+    assert isinstance(data[0], pd.Timestamp)
+    restored = pa.array(data, type=ty)
+    assert arr.equals(restored)
+    assert restored.to_pylist() == [
+        pd.Timestamp(9223371273709551616, unit='ns')
+    ]
+
+    ty = pa.timestamp('ns', tz='US/Eastern')
+    value = 1604119893000000000
+    arr = pa.array([value], type=ty)
+    data = arr.to_pylist()
+    assert isinstance(data[0], pd.Timestamp)
+    restored = pa.array(data, type=ty)
+    assert arr.equals(restored)
+    assert restored.to_pylist() == [
+        pd.Timestamp(value, unit='ns').tz_localize(
+            "UTC").tz_convert('US/Eastern')
+    ]
+
+
+@h.given(past.all_arrays)
+def test_array_to_pylist_roundtrip(arr):
+    seq = arr.to_pylist()
+    restored = pa.array(seq, type=arr.type)
+    assert restored.equals(arr)
+
+
+@pytest.mark.large_memory
+def test_auto_chunking_binary_like():
+    # single chunk
+    v1 = b'x' * 100000000
+    v2 = b'x' * 147483646
+
+    # single chunk
+    one_chunk_data = [v1] * 20 + [b'', None, v2]
+    arr = pa.array(one_chunk_data, type=pa.binary())
+    assert isinstance(arr, pa.Array)
+    assert len(arr) == 23
+    assert arr[20].as_py() == b''
+    assert arr[21].as_py() is None
+    assert arr[22].as_py() == v2
+
+    # two chunks
+    two_chunk_data = one_chunk_data + [b'two']
+    arr = pa.array(two_chunk_data, type=pa.binary())
+    assert isinstance(arr, pa.ChunkedArray)
+    assert arr.num_chunks == 2
+    assert len(arr.chunk(0)) == 23
+    assert len(arr.chunk(1)) == 1
+    assert arr.chunk(0)[20].as_py() == b''
+    assert arr.chunk(0)[21].as_py() is None
+    assert arr.chunk(0)[22].as_py() == v2
+    assert arr.chunk(1).to_pylist() == [b'two']
+
+    # three chunks
+    three_chunk_data = one_chunk_data * 2 + [b'three', b'three']
+    arr = pa.array(three_chunk_data, type=pa.binary())
+    assert isinstance(arr, pa.ChunkedArray)
+    assert arr.num_chunks == 3
+    assert len(arr.chunk(0)) == 23
+    assert len(arr.chunk(1)) == 23
+    assert len(arr.chunk(2)) == 2
+    for i in range(2):
+        assert arr.chunk(i)[20].as_py() == b''
+        assert arr.chunk(i)[21].as_py() is None
+        assert arr.chunk(i)[22].as_py() == v2
+    assert arr.chunk(2).to_pylist() == [b'three', b'three']
+
+
+@pytest.mark.large_memory
+def test_auto_chunking_list_of_binary():
+    # ARROW-6281
+    vals = [['x' * 1024]] * ((2 << 20) + 1)
+    arr = pa.array(vals)
+    assert isinstance(arr, pa.ChunkedArray)
+    assert arr.num_chunks == 2
+    assert len(arr.chunk(0)) == 2**21 - 1
+    assert len(arr.chunk(1)) == 2
+    assert arr.chunk(1).to_pylist() == [['x' * 1024]] * 2
+
+
+@pytest.mark.slow
+@pytest.mark.large_memory
+def test_auto_chunking_list_like():
+    item = np.ones((2**28,), dtype='uint8')
+    data = [item] * (2**3 - 1)
+    arr = pa.array(data, type=pa.list_(pa.uint8()))
+    assert isinstance(arr, pa.Array)
+    assert len(arr) == 7
+
+    item = np.ones((2**28,), dtype='uint8')
+    data = [item] * 2**3
+    arr = pa.array(data, type=pa.list_(pa.uint8()))
+    assert isinstance(arr, pa.ChunkedArray)
+    assert arr.num_chunks == 2
+    assert len(arr.chunk(0)) == 7
+    assert len(arr.chunk(1)) == 1
+    assert arr.chunk(1)[0].as_py() == list(item)
+
+
+@pytest.mark.slow
+@pytest.mark.large_memory
+def test_auto_chunking_map_type():
+    # takes ~20 minutes locally
+    ty = pa.map_(pa.int8(), pa.int8())
+    item = [(1, 1)] * 2**28
+    data = [item] * 2**3
+    arr = pa.array(data, type=ty)
+    assert isinstance(arr, pa.ChunkedArray)
+    assert len(arr.chunk(0)) == 7
+    assert len(arr.chunk(1)) == 1
+
+
+@pytest.mark.large_memory
+@pytest.mark.parametrize(('ty', 'char'), [
+    (pa.string(), 'x'),
+    (pa.binary(), b'x'),
+])
+def test_nested_auto_chunking(ty, char):
+    v1 = char * 100000000
+    v2 = char * 147483646
+
+    struct_type = pa.struct([
+        pa.field('bool', pa.bool_()),
+        pa.field('integer', pa.int64()),
+        pa.field('string-like', ty),
+    ])
+
+    data = [{'bool': True, 'integer': 1, 'string-like': v1}] * 20
+    data.append({'bool': True, 'integer': 1, 'string-like': v2})
+    arr = pa.array(data, type=struct_type)
+    assert isinstance(arr, pa.Array)
+
+    data.append({'bool': True, 'integer': 1, 'string-like': char})
+    arr = pa.array(data, type=struct_type)
+    assert isinstance(arr, pa.ChunkedArray)
+    assert arr.num_chunks == 2
+    assert len(arr.chunk(0)) == 21
+    assert len(arr.chunk(1)) == 1
+    assert arr.chunk(1)[0].as_py() == {
+        'bool': True,
+        'integer': 1,
+        'string-like': char
+    }
diff --git a/python/pyarrow/tests/test_csv.py b/python/pyarrow/tests/test_csv.py
index 582f04bba3d..e95a9fff167 100644
--- a/python/pyarrow/tests/test_csv.py
+++ b/python/pyarrow/tests/test_csv.py
@@ -16,7 +16,7 @@
 # under the License.
 
 import bz2
-from datetime import datetime
+from datetime import date, datetime
 from decimal import Decimal
 import gc
 import gzip
@@ -534,6 +534,30 @@ def test_timestamp_parsers(self):
             'b': [datetime(1980, 1, 1), datetime(1980, 1, 2)],
         }
 
+    def test_dates(self):
+        # Dates are inferred as timestamps by default
+        rows = b"a,b\n1970-01-01,1970-01-02\n1971-01-01,1971-01-02\n"
+        table = self.read_bytes(rows)
+        schema = pa.schema([('a', pa.timestamp('s')),
+                            ('b', pa.timestamp('s'))])
+        assert table.schema == schema
+        assert table.to_pydict() == {
+            'a': [datetime(1970, 1, 1), datetime(1971, 1, 1)],
+            'b': [datetime(1970, 1, 2), datetime(1971, 1, 2)],
+        }
+
+        # Can ask for date types explicitly
+        opts = ConvertOptions()
+        opts.column_types = {'a': pa.date32(), 'b': pa.date64()}
+        table = self.read_bytes(rows, convert_options=opts)
+        schema = pa.schema([('a', pa.date32()),
+                            ('b', pa.date64())])
+        assert table.schema == schema
+        assert table.to_pydict() == {
+            'a': [date(1970, 1, 1), date(1971, 1, 1)],
+            'b': [date(1970, 1, 2), date(1971, 1, 2)],
+        }
+
     def test_auto_dict_encode(self):
         opts = ConvertOptions(auto_dict_encode=True)
         rows = "a,b\nab,1\ncdé,2\ncdé,3\nab,4".encode()
@@ -683,6 +707,38 @@ def test_column_types(self):
         assert "In CSV column #1: " in err
         assert "CSV conversion error to float: invalid value 'XXX'" in err
 
+    def test_column_types_dict(self):
+        # Ask for dict-encoded column types in ConvertOptions
+        column_types = [
+            ('a', pa.dictionary(pa.int32(), pa.utf8())),
+            ('b', pa.dictionary(pa.int32(), pa.int64())),
+            ('c', pa.dictionary(pa.int32(), pa.decimal128(11, 2))),
+            ('d', pa.dictionary(pa.int32(), pa.large_utf8()))]
+
+        opts = ConvertOptions(column_types=dict(column_types))
+        rows = (b"a,b,c,d\n"
+                b"abc,123456,1.0,zz\n"
+                b"defg,123456,0.5,xx\n"
+                b"abc,N/A,1.0,xx\n")
+        table = self.read_bytes(rows, convert_options=opts)
+
+        schema = pa.schema(column_types)
+        expected = {
+            'a': ["abc", "defg", "abc"],
+            'b': [123456, 123456, None],
+            'c': [Decimal("1.00"), Decimal("0.50"), Decimal("1.00")],
+            'd': ["zz", "xx", "xx"],
+        }
+        assert table.schema == schema
+        assert table.to_pydict() == expected
+
+        # Unsupported index type
+        column_types[0] = ('a', pa.dictionary(pa.int8(), pa.utf8()))
+
+        opts = ConvertOptions(column_types=dict(column_types))
+        with pytest.raises(NotImplementedError):
+            table = self.read_bytes(rows, convert_options=opts)
+
     def test_column_types_with_column_names(self):
         # When both `column_names` and `column_types` are given, names
         # in `column_types` should refer to names in `column_names`
diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py
index 1270614fb7f..92a77d91c83 100644
--- a/python/pyarrow/tests/test_dataset.py
+++ b/python/pyarrow/tests/test_dataset.py
@@ -27,6 +27,7 @@
 import pyarrow as pa
 import pyarrow.csv
 import pyarrow.fs as fs
+from pyarrow.tests.util import change_cwd
 
 try:
     import pandas as pd
@@ -43,16 +44,6 @@
 pytestmark = pytest.mark.dataset
 
 
-@contextlib.contextmanager
-def change_cwd(path):
-    curdir = os.getcwd()
-    os.chdir(str(path))
-    try:
-        yield
-    finally:
-        os.chdir(curdir)
-
-
 def _generate_data(n):
     import datetime
     import itertools
@@ -178,9 +169,9 @@ def multisourcefs(request):
         with mockfs.open_output_stream(path) as out:
             pq.write_table(_table_from_pandas(chunk), out)
 
-    # create one with schema partitioning by week and color
+    # create one with schema partitioning by weekday and color
     mockfs.create_dir('schema')
-    for part, chunk in df_b.groupby([df_b.date.dt.week, df_b.color]):
+    for part, chunk in df_b.groupby([df_b.date.dt.dayofweek, df_b.color]):
         folder = 'schema/{}/{}'.format(*part)
         path = '{}/chunk.parquet'.format(folder)
         mockfs.create_dir(folder)
@@ -236,7 +227,7 @@ def test_filesystem_dataset(mockfs):
 
     dataset_from_fragments = ds.FileSystemDataset(
         fragments, schema=schema, format=file_format,
-        root_partition=root_partition,
+        filesystem=mockfs, root_partition=root_partition,
     )
     dataset_from_paths = ds.FileSystemDataset.from_paths(
         paths, schema=schema, format=file_format, filesystem=mockfs,
@@ -255,20 +246,22 @@ def test_filesystem_dataset(mockfs):
             assert fragment.path == path
             assert isinstance(fragment.format, ds.ParquetFileFormat)
             assert isinstance(fragment, ds.ParquetFileFragment)
-            assert fragment.row_groups is None
+            assert fragment.row_groups == [0]
+            assert fragment.num_row_groups == 1
 
             row_group_fragments = list(fragment.split_by_row_group())
-            assert len(row_group_fragments) == 1
+            assert fragment.num_row_groups == len(row_group_fragments) == 1
             assert isinstance(row_group_fragments[0], ds.ParquetFileFragment)
             assert row_group_fragments[0].path == path
-            assert row_group_fragments[0].row_groups == [ds.RowGroupInfo(0)]
+            assert row_group_fragments[0].row_groups == [0]
+            assert row_group_fragments[0].num_row_groups == 1
 
         fragments = list(dataset.get_fragments(filter=ds.field("const") == 0))
         assert len(fragments) == 2
 
     # the root_partition keyword has a default
     dataset = ds.FileSystemDataset(
-        fragments, schema=schema, format=file_format
+        fragments, schema=schema, format=file_format, filesystem=mockfs
     )
     assert dataset.partition_expression.equals(ds.scalar(True))
 
@@ -285,8 +278,8 @@ def test_filesystem_dataset(mockfs):
         ds.FileSystemDataset(fragments, file_format, schema)
     # validation of root_partition
     with pytest.raises(TypeError, match="incorrect type"):
-        ds.FileSystemDataset(fragments, schema=schema, format=file_format,
-                             root_partition=1)
+        ds.FileSystemDataset(fragments, schema=schema,
+                             format=file_format, root_partition=1)
     # missing required argument in from_paths
     with pytest.raises(TypeError, match="incorrect type"):
         ds.FileSystemDataset.from_paths(fragments, format=file_format)
@@ -605,14 +598,15 @@ def test_make_fragment(multisourcefs):
 
     for path in dataset.files:
         fragment = parquet_format.make_fragment(path, multisourcefs)
+        assert fragment.row_groups == [0]
+
         row_group_fragment = parquet_format.make_fragment(path, multisourcefs,
                                                           row_groups=[0])
         for f in [fragment, row_group_fragment]:
             assert isinstance(f, ds.ParquetFileFragment)
             assert f.path == path
             assert isinstance(f.filesystem, type(multisourcefs))
-        assert fragment.row_groups is None
-        assert row_group_fragment.row_groups == [ds.RowGroupInfo(0)]
+        assert row_group_fragment.row_groups == [0]
 
 
 def test_make_csv_fragment_from_buffer():
@@ -813,13 +807,14 @@ def test_fragments_parquet_row_groups(tempdir):
 
     # list and scan row group fragments
     row_group_fragments = list(fragment.split_by_row_group())
-    assert len(row_group_fragments) == 2
+    assert len(row_group_fragments) == fragment.num_row_groups == 2
     result = row_group_fragments[0].to_table(schema=dataset.schema)
     assert result.column_names == ['f1', 'f2', 'part']
     assert len(result) == 2
     assert result.equals(table.slice(0, 2))
 
     assert row_group_fragments[0].row_groups is not None
+    assert row_group_fragments[0].num_row_groups == 1
     assert row_group_fragments[0].row_groups[0].statistics == {
         'f1': {'min': 0, 'max': 1},
         'f2': {'min': 1, 'max': 1},
@@ -832,6 +827,44 @@ def test_fragments_parquet_row_groups(tempdir):
     assert len(result) == 1
 
 
+@pytest.mark.parquet
+def test_fragments_parquet_num_row_groups(tempdir):
+    import pyarrow.parquet as pq
+
+    table = pa.table({'a': range(8)})
+    pq.write_table(table, tempdir / "test.parquet", row_group_size=2)
+    dataset = ds.dataset(tempdir / "test.parquet", format="parquet")
+    original_fragment = list(dataset.get_fragments())[0]
+
+    # create fragment with subset of row groups
+    fragment = original_fragment.format.make_fragment(
+        original_fragment.path, original_fragment.filesystem,
+        row_groups=[1, 3])
+    assert fragment.num_row_groups == 2
+    # ensure that parsing metadata preserves correct number of row groups
+    fragment.ensure_complete_metadata()
+    assert fragment.num_row_groups == 2
+    assert len(fragment.row_groups) == 2
+
+
+@pytest.mark.pandas
+@pytest.mark.parquet
+def test_fragments_parquet_row_groups_dictionary(tempdir):
+    import pandas as pd
+
+    df = pd.DataFrame(dict(col1=['a', 'b'], col2=[1, 2]))
+    df['col1'] = df['col1'].astype("category")
+
+    import pyarrow.parquet as pq
+    pq.write_table(pa.table(df), tempdir / "test_filter_dictionary.parquet")
+
+    import pyarrow.dataset as ds
+    dataset = ds.dataset(tempdir / 'test_filter_dictionary.parquet')
+    result = dataset.to_table(filter=ds.field("col1") == "a")
+
+    assert (df.iloc[0] == result.to_pandas()).all().all()
+
+
 @pytest.mark.pandas
 @pytest.mark.parquet
 def test_fragments_parquet_ensure_metadata(tempdir, open_logging_fs):
@@ -842,10 +875,9 @@ def test_fragments_parquet_ensure_metadata(tempdir, open_logging_fs):
     fragment = list(dataset.get_fragments())[0]
 
     # with default discovery, no metadata loaded
-    assert fragment.row_groups is None
     with assert_opens([fragment.path]):
         fragment.ensure_complete_metadata()
-    assert fragment.row_groups is not None
+    assert fragment.row_groups == [0, 1]
 
     # second time -> use cached / no file IO
     with assert_opens([]):
@@ -855,13 +887,7 @@ def test_fragments_parquet_ensure_metadata(tempdir, open_logging_fs):
     new_fragment = fragment.format.make_fragment(
         fragment.path, fragment.filesystem, row_groups=[0, 1]
     )
-    assert new_fragment.row_groups is not None
-    assert len(new_fragment.row_groups) == 2
-    row_group = new_fragment.row_groups[0]
-    assert row_group.id == 0
-    # no initialized statistics
-    assert row_group.num_rows == -1
-    assert row_group.statistics is None
+    assert new_fragment.row_groups == fragment.row_groups
 
     # collect metadata
     new_fragment.ensure_complete_metadata()
@@ -870,13 +896,13 @@ def test_fragments_parquet_ensure_metadata(tempdir, open_logging_fs):
     assert row_group.num_rows == 2
     assert row_group.statistics is not None
 
-    # pickling preserves row group ids but not statistics
+    # pickling preserves row group ids
     pickled_fragment = pickle.loads(pickle.dumps(new_fragment))
-    assert pickled_fragment.row_groups is not None
-    row_group = pickled_fragment.row_groups[0]
-    assert row_group.id == 0
-    assert row_group.num_rows == -1
-    assert row_group.statistics is None
+    with assert_opens([fragment.path]):
+        assert pickled_fragment.row_groups == [0, 1]
+        row_group = pickled_fragment.row_groups[0]
+        assert row_group.id == 0
+        assert row_group.statistics is not None
 
 
 def _create_dataset_all_types(tempdir, chunk_size=None):
@@ -1054,10 +1080,84 @@ def test_fragments_parquet_row_groups_reconstruct(tempdir):
         fragment.path, fragment.filesystem,
         partition_expression=fragment.partition_expression,
         row_groups={2})
-    with pytest.raises(IndexError, match="Trying to scan row group 2"):
+    with pytest.raises(IndexError, match="references row group 2"):
         new_fragment.to_table()
 
 
+@pytest.mark.pandas
+@pytest.mark.parquet
+def test_fragments_parquet_subset_ids(tempdir, open_logging_fs):
+    fs, assert_opens = open_logging_fs
+    table, dataset = _create_dataset_for_fragments(tempdir, chunk_size=1,
+                                                   filesystem=fs)
+    fragment = list(dataset.get_fragments())[0]
+
+    # select with row group ids
+    subfrag = fragment.subset(row_group_ids=[0, 3])
+    with assert_opens([]):
+        assert subfrag.num_row_groups == 2
+        assert subfrag.row_groups == [0, 3]
+        assert subfrag.row_groups[0].statistics is not None
+
+    # check correct scan result of subset
+    result = subfrag.to_table()
+    assert result.to_pydict() == {"f1": [0, 3], "f2": [1, 1]}
+
+    # empty list of ids
+    subfrag = fragment.subset(row_group_ids=[])
+    assert subfrag.num_row_groups == 0
+    assert subfrag.row_groups == []
+    result = subfrag.to_table(schema=dataset.schema)
+    assert result.num_rows == 0
+    assert result.equals(table[:0])
+
+
+@pytest.mark.pandas
+@pytest.mark.parquet
+def test_fragments_parquet_subset_filter(tempdir, open_logging_fs):
+    fs, assert_opens = open_logging_fs
+    table, dataset = _create_dataset_for_fragments(tempdir, chunk_size=1,
+                                                   filesystem=fs)
+    fragment = list(dataset.get_fragments())[0]
+
+    # select with filter
+    subfrag = fragment.subset(ds.field("f1") >= 1)
+    with assert_opens([]):
+        assert subfrag.num_row_groups == 3
+        assert len(subfrag.row_groups) == 3
+        assert subfrag.row_groups[0].statistics is not None
+
+    # check correct scan result of subset
+    result = subfrag.to_table()
+    assert result.to_pydict() == {"f1": [1, 2, 3], "f2": [1, 1, 1]}
+
+    # filter that results in empty selection
+    subfrag = fragment.subset(ds.field("f1") > 5)
+    assert subfrag.num_row_groups == 0
+    assert subfrag.row_groups == []
+    result = subfrag.to_table(schema=dataset.schema)
+    assert result.num_rows == 0
+    assert result.equals(table[:0])
+
+    # passing schema to ensure filter on partition expression works
+    subfrag = fragment.subset(ds.field("part") == "a", schema=dataset.schema)
+    assert subfrag.num_row_groups == 4
+
+
+@pytest.mark.pandas
+@pytest.mark.parquet
+def test_fragments_parquet_subset_invalid(tempdir):
+    _, dataset = _create_dataset_for_fragments(tempdir, chunk_size=1)
+    fragment = list(dataset.get_fragments())[0]
+
+    # passing none or both of filter / row_group_ids
+    with pytest.raises(ValueError):
+        fragment.subset(ds.field("f1") >= 1, row_group_ids=[1, 2])
+
+    with pytest.raises(ValueError):
+        fragment.subset()
+
+
 def test_partitioning_factory(mockfs):
     paths_or_selector = fs.FileSelector('subdir', recursive=True)
     format = ds.ParquetFileFormat()
@@ -1086,33 +1186,23 @@ def test_partitioning_factory(mockfs):
     assert isinstance(hive_partitioning_factory, ds.PartitioningFactory)
 
 
-def test_partitioning_factory_dictionary(mockfs):
+@pytest.mark.parametrize('infer_dictionary', [False, True])
+def test_partitioning_factory_dictionary(mockfs, infer_dictionary):
     paths_or_selector = fs.FileSelector('subdir', recursive=True)
     format = ds.ParquetFileFormat()
     options = ds.FileSystemFactoryOptions('subdir')
 
-    max_size_to_inferred_type = {
-        0: pa.string(),
-        1: pa.string(),
-        2: pa.dictionary(pa.int32(), pa.string()),
-        64: pa.dictionary(pa.int32(), pa.string()),
-        None: pa.dictionary(pa.int32(), pa.string()),
-    }
+    options.partitioning_factory = ds.DirectoryPartitioning.discover(
+        ['group', 'key'], infer_dictionary=infer_dictionary)
 
-    for max_size, expected_type in max_size_to_inferred_type.items():
-        options.partitioning_factory = ds.DirectoryPartitioning.discover(
-            ['group', 'key'],
-            max_partition_dictionary_size=max_size)
-
-        factory = ds.FileSystemDatasetFactory(
-            mockfs, paths_or_selector, format, options)
+    factory = ds.FileSystemDatasetFactory(
+        mockfs, paths_or_selector, format, options)
 
-        inferred_schema = factory.inspect()
+    inferred_schema = factory.inspect()
+    if infer_dictionary:
+        expected_type = pa.dictionary(pa.int32(), pa.string())
         assert inferred_schema.field('key').type == expected_type
 
-        if expected_type == pa.string():
-            continue
-
         table = factory.finish().to_table().combine_chunks()
         actual = table.column('key').chunk(0)
         expected = pa.array(['xxx'] * 5 + ['yyy'] * 5).dictionary_encode()
@@ -1123,6 +1213,8 @@ def test_partitioning_factory_dictionary(mockfs):
         actual = table.column('key').chunk(0)
         expected = expected.slice(0, 5)
         assert actual.equals(expected)
+    else:
+        assert inferred_schema.field('key').type == pa.string()
 
 
 def test_partitioning_function():
@@ -1498,33 +1590,34 @@ def test_open_dataset_partitioned_dictionary_type(tempdir, partitioning,
     import pyarrow.parquet as pq
     table = pa.table({'a': range(9), 'b': [0.] * 4 + [1.] * 5})
 
+    if partitioning == "directory":
+        partitioning = ds.DirectoryPartitioning.discover(
+            ["part1", "part2"], infer_dictionary=True)
+        fmt = "{0}/{1}"
+    else:
+        partitioning = ds.HivePartitioning.discover(infer_dictionary=True)
+        fmt = "part1={0}/part2={1}"
+
     basepath = tempdir / "dataset"
     basepath.mkdir()
 
     part_keys1, part_keys2 = partition_keys
     for part1 in part_keys1:
         for part2 in part_keys2:
-            if partitioning == 'directory':
-                fmt = "{0}/{1}"
-            else:
-                fmt = "part1={0}/part2={1}"
             path = basepath / fmt.format(part1, part2)
             path.mkdir(parents=True)
             pq.write_table(table, path / "test.parquet")
 
-    if partitioning == "directory":
-        part = ds.DirectoryPartitioning.discover(
-            ["part1", "part2"], max_partition_dictionary_size=None)
-    else:
-        part = ds.HivePartitioning.discover(max_partition_dictionary_size=None)
+    dataset = ds.dataset(str(basepath), partitioning=partitioning)
 
-    dataset = ds.dataset(str(basepath), partitioning=part)
-
-    dict_type = pa.dictionary(pa.int32(), pa.string())
-    part_type1 = dict_type if isinstance(part_keys1[0], str) else pa.int32()
-    part_type2 = dict_type if isinstance(part_keys2[0], str) else pa.int32()
+    def dict_type(key):
+        value_type = pa.string() if isinstance(key, str) else pa.int32()
+        return pa.dictionary(pa.int32(), value_type)
     expected_schema = table.schema.append(
-        pa.field("part1", part_type1)).append(pa.field("part2", part_type2))
+        pa.field("part1", dict_type(part_keys1[0]))
+    ).append(
+        pa.field("part2", dict_type(part_keys2[0]))
+    )
     assert dataset.schema.equals(expected_schema)
 
 
@@ -2061,6 +2154,11 @@ def test_parquet_dataset_lazy_filtering(tempdir, open_logging_fs):
     with assert_opens([]):
         fragments[0].split_by_row_group(ds.field("f1") > 15)
 
+    # ensuring metadata of splitted fragment should also not open any file
+    with assert_opens([]):
+        rg_fragments = fragments[0].split_by_row_group()
+        rg_fragments[0].ensure_complete_metadata()
+
     # FIXME(bkietz) on Windows this results in FileNotFoundErrors.
     # but actually scanning does open files
     # with assert_opens([f.path for f in fragments]):
@@ -2123,3 +2221,272 @@ def test_dataset_project_only_partition_columns(tempdir):
     part_only = dataset.to_table(columns=['part'], use_threads=False)
 
     assert all_cols.column('part').equals(part_only.column('part'))
+
+
+@pytest.mark.parquet
+@pytest.mark.pandas
+def test_dataset_project_null_column(tempdir):
+    import pandas as pd
+    df = pd.DataFrame({"col": np.array([None, None, None], dtype='object')})
+
+    f = tempdir / "test_dataset_project_null_column.parquet"
+    df.to_parquet(f, engine="pyarrow")
+
+    dataset = ds.dataset(f, format="parquet",
+                         schema=pa.schema([("col", pa.int64())]))
+    expected = pa.table({'col': pa.array([None, None, None], pa.int64())})
+    assert dataset.to_table().equals(expected)
+
+
+def _check_dataset_roundtrip(dataset, base_dir, expected_files,
+                             base_dir_path=None, partitioning=None):
+    base_dir_path = base_dir_path or base_dir
+
+    ds.write_dataset(dataset, base_dir, format="feather",
+                     partitioning=partitioning, use_threads=False)
+
+    # check that all files are present
+    file_paths = list(base_dir_path.rglob("*"))
+    assert set(file_paths) == set(expected_files)
+
+    # check that reading back in as dataset gives the same result
+    dataset2 = ds.dataset(
+        base_dir_path, format="feather", partitioning=partitioning)
+    assert dataset2.to_table().equals(dataset.to_table())
+
+
+@pytest.mark.parquet
+def test_write_dataset(tempdir):
+    # manually create a written dataset and read as dataset object
+    directory = tempdir / 'single-file'
+    directory.mkdir()
+    _ = _create_single_file(directory)
+    dataset = ds.dataset(directory)
+
+    # full string path
+    target = tempdir / 'single-file-target'
+    expected_files = [target / "part-0.feather"]
+    _check_dataset_roundtrip(dataset, str(target), expected_files, target)
+
+    # pathlib path object
+    target = tempdir / 'single-file-target2'
+    expected_files = [target / "part-0.feather"]
+    _check_dataset_roundtrip(dataset, target, expected_files, target)
+
+    # TODO
+    # # relative path
+    # target = tempdir / 'single-file-target3'
+    # expected_files = [target / "part-0.ipc"]
+    # _check_dataset_roundtrip(
+    #     dataset, './single-file-target3', expected_files, target)
+
+    # Directory of files
+    directory = tempdir / 'single-directory'
+    directory.mkdir()
+    _ = _create_directory_of_files(directory)
+    dataset = ds.dataset(directory)
+
+    target = tempdir / 'single-directory-target'
+    expected_files = [target / "part-0.feather"]
+    _check_dataset_roundtrip(dataset, str(target), expected_files, target)
+
+
+@pytest.mark.parquet
+@pytest.mark.pandas
+def test_write_dataset_partitioned(tempdir):
+    directory = tempdir / "partitioned"
+    _ = _create_parquet_dataset_partitioned(directory)
+    partitioning = ds.partitioning(flavor="hive")
+    dataset = ds.dataset(directory, partitioning=partitioning)
+
+    # hive partitioning
+    target = tempdir / 'partitioned-hive-target'
+    expected_paths = [
+        target / "part=a", target / "part=a" / "part-0.feather",
+        target / "part=b", target / "part=b" / "part-1.feather"
+    ]
+    partitioning_schema = ds.partitioning(
+        pa.schema([("part", pa.string())]), flavor="hive")
+    _check_dataset_roundtrip(
+        dataset, str(target), expected_paths, target,
+        partitioning=partitioning_schema)
+
+    # directory partitioning
+    target = tempdir / 'partitioned-dir-target'
+    expected_paths = [
+        target / "a", target / "a" / "part-0.feather",
+        target / "b", target / "b" / "part-1.feather"
+    ]
+    partitioning_schema = ds.partitioning(
+        pa.schema([("part", pa.string())]))
+    _check_dataset_roundtrip(
+        dataset, str(target), expected_paths, target,
+        partitioning=partitioning_schema)
+
+
+@pytest.mark.parquet
+@pytest.mark.pandas
+def test_write_dataset_use_threads(tempdir):
+    directory = tempdir / "partitioned"
+    _ = _create_parquet_dataset_partitioned(directory)
+    dataset = ds.dataset(directory, partitioning="hive")
+
+    partitioning = ds.partitioning(
+        pa.schema([("part", pa.string())]), flavor="hive")
+
+    target1 = tempdir / 'partitioned1'
+    ds.write_dataset(
+        dataset, target1, format="feather", partitioning=partitioning,
+        use_threads=True
+    )
+    target2 = tempdir / 'partitioned2'
+    ds.write_dataset(
+        dataset, target2, format="feather", partitioning=partitioning,
+        use_threads=False
+    )
+
+    # check that reading in gives same result
+    result1 = ds.dataset(target1, format="feather", partitioning=partitioning)
+    result2 = ds.dataset(target2, format="feather", partitioning=partitioning)
+    assert result1.to_table().equals(result2.to_table())
+
+
+def test_write_table(tempdir):
+    table = pa.table([
+        pa.array(range(20)), pa.array(np.random.randn(20)),
+        pa.array(np.repeat(['a', 'b'], 10))
+    ], names=["f1", "f2", "part"])
+
+    base_dir = tempdir / 'single'
+    ds.write_dataset(table, base_dir,
+                     basename_template='dat_{i}.arrow', format="feather")
+    # check that all files are present
+    file_paths = list(base_dir.rglob("*"))
+    expected_paths = [base_dir / "dat_0.arrow"]
+    assert set(file_paths) == set(expected_paths)
+    # check Table roundtrip
+    result = ds.dataset(base_dir, format="ipc").to_table()
+    assert result.equals(table)
+
+    # with partitioning
+    base_dir = tempdir / 'partitioned'
+    partitioning = ds.partitioning(
+        pa.schema([("part", pa.string())]), flavor="hive")
+    ds.write_dataset(table, base_dir, format="feather",
+                     basename_template='dat_{i}.arrow',
+                     partitioning=partitioning)
+    file_paths = list(base_dir.rglob("*"))
+    expected_paths = [
+        base_dir / "part=a", base_dir / "part=a" / "dat_0.arrow",
+        base_dir / "part=b", base_dir / "part=b" / "dat_1.arrow"
+    ]
+    assert set(file_paths) == set(expected_paths)
+    result = ds.dataset(base_dir, format="ipc", partitioning=partitioning)
+    assert result.to_table().equals(table)
+
+
+def test_write_table_multiple_fragments(tempdir):
+    table = pa.table([
+        pa.array(range(10)), pa.array(np.random.randn(10)),
+        pa.array(np.repeat(['a', 'b'], 5))
+    ], names=["f1", "f2", "part"])
+    table = pa.concat_tables([table]*2)
+
+    # Table with multiple batches written as single Fragment by default
+    base_dir = tempdir / 'single'
+    ds.write_dataset(table, base_dir, format="feather")
+    assert set(base_dir.rglob("*")) == set([base_dir / "part-0.feather"])
+    assert ds.dataset(base_dir, format="ipc").to_table().equals(table)
+
+    # Same for single-element list of Table
+    base_dir = tempdir / 'single-list'
+    ds.write_dataset([table], base_dir, format="feather")
+    assert set(base_dir.rglob("*")) == set([base_dir / "part-0.feather"])
+    assert ds.dataset(base_dir, format="ipc").to_table().equals(table)
+
+    # Provide list of batches to write multiple fragments
+    base_dir = tempdir / 'multiple'
+    ds.write_dataset(table.to_batches(), base_dir, format="feather")
+    assert set(base_dir.rglob("*")) == set(
+        [base_dir / "part-0.feather"])
+    assert ds.dataset(base_dir, format="ipc").to_table().equals(table)
+
+    # Provide list of tables to write multiple fragments
+    base_dir = tempdir / 'multiple-table'
+    ds.write_dataset([table, table], base_dir, format="feather")
+    assert set(base_dir.rglob("*")) == set(
+        [base_dir / "part-0.feather"])
+    assert ds.dataset(base_dir, format="ipc").to_table().equals(
+        pa.concat_tables([table]*2)
+    )
+
+
+@pytest.mark.parquet
+def test_write_dataset_parquet(tempdir):
+    import pyarrow.parquet as pq
+
+    table = pa.table([
+        pa.array(range(20)), pa.array(np.random.randn(20)),
+        pa.array(np.repeat(['a', 'b'], 10))
+    ], names=["f1", "f2", "part"])
+
+    # using default "parquet" format string
+
+    base_dir = tempdir / 'parquet_dataset'
+    ds.write_dataset(table, base_dir, format="parquet")
+    # check that all files are present
+    file_paths = list(base_dir.rglob("*"))
+    expected_paths = [base_dir / "part-0.parquet"]
+    assert set(file_paths) == set(expected_paths)
+    # check Table roundtrip
+    result = ds.dataset(base_dir, format="parquet").to_table()
+    assert result.equals(table)
+
+    # using custom options
+    for version in ["1.0", "2.0"]:
+        format = ds.ParquetFileFormat()
+        opts = format.make_write_options(version=version)
+        base_dir = tempdir / 'parquet_dataset_version{0}'.format(version)
+        ds.write_dataset(table, base_dir, format=format, file_options=opts)
+        meta = pq.read_metadata(base_dir / "part-0.parquet")
+        assert meta.format_version == version
+
+
+@pytest.mark.parquet
+@pytest.mark.pandas
+def test_write_dataset_arrow_schema_metadata(tempdir):
+    # ensure we serialize ARROW schema in the parquet metadata, to have a
+    # correct roundtrip (e.g. preserve non-UTC timezone)
+    import pyarrow.parquet as pq
+
+    table = pa.table({"a": [pd.Timestamp("2012-01-01", tz="Europe/Brussels")]})
+    assert table["a"].type.tz == "Europe/Brussels"
+
+    ds.write_dataset(table, tempdir, format="parquet")
+    result = pq.read_table(tempdir / "part-0.parquet")
+    assert result["a"].type.tz == "Europe/Brussels"
+
+
+def test_write_dataset_schema_metadata(tempdir):
+    # ensure that schema metadata gets written
+    from pyarrow import feather
+
+    table = pa.table({'a': [1, 2, 3]})
+    table = table.replace_schema_metadata({b'key': b'value'})
+    ds.write_dataset(table, tempdir, format="feather")
+
+    schema = feather.read_table(tempdir / "part-0.feather").schema
+    assert schema.metadata == {b'key': b'value'}
+
+
+@pytest.mark.parquet
+def test_write_dataset_schema_metadata_parquet(tempdir):
+    # ensure that schema metadata gets written
+    import pyarrow.parquet as pq
+
+    table = pa.table({'a': [1, 2, 3]})
+    table = table.replace_schema_metadata({b'key': b'value'})
+    ds.write_dataset(table, tempdir, format="parquet")
+
+    schema = pq.read_table(tempdir / "part-0.parquet").schema
+    assert schema.metadata == {b'key': b'value'}
diff --git a/python/pyarrow/tests/test_extension_type.py b/python/pyarrow/tests/test_extension_type.py
index a3ef336a22f..1ef70e79696 100644
--- a/python/pyarrow/tests/test_extension_type.py
+++ b/python/pyarrow/tests/test_extension_type.py
@@ -56,6 +56,26 @@ def __reduce__(self):
         return ParamExtType, (self.width,)
 
 
+class MyStructType(pa.PyExtensionType):
+    storage_type = pa.struct([('left', pa.int64()),
+                              ('right', pa.int64())])
+
+    def __init__(self):
+        pa.PyExtensionType.__init__(self, self.storage_type)
+
+    def __reduce__(self):
+        return MyStructType, ()
+
+
+class MyListType(pa.PyExtensionType):
+
+    def __init__(self, storage_type):
+        pa.PyExtensionType.__init__(self, storage_type)
+
+    def __reduce__(self):
+        return MyListType, (self.storage_type,)
+
+
 def ipc_write_batch(batch):
     stream = pa.BufferOutputStream()
     writer = pa.RecordBatchStreamWriter(stream, batch.schema)
@@ -74,6 +94,18 @@ def test_ext_type_basics():
     assert ty.extension_name == "arrow.py_extension_type"
 
 
+def test_ext_type_str():
+    ty = IntegerType()
+    expected = "extension<arrow.py_extension_type<IntegerType>>"
+    assert str(ty) == expected
+    assert pa.DataType.__str__(ty) == expected
+
+
+def test_ext_type_repr():
+    ty = IntegerType()
+    assert repr(ty) == "IntegerType(DataType(int64))"
+
+
 def test_ext_type__lifetime():
     ty = UuidType()
     wr = weakref.ref(ty)
@@ -424,8 +456,8 @@ def test_generic_ext_type_register(registered_period_type):
 
 
 @pytest.mark.parquet
-def test_parquet(tmpdir, registered_period_type):
-    # Parquet support for extension types
+def test_parquet_period(tmpdir, registered_period_type):
+    # Parquet support for primitive extension types
     period_type, period_class = registered_period_type
     storage = pa.array([1, 2, 3, 4], pa.int64())
     arr = pa.ExtensionArray.from_storage(period_type, storage)
@@ -433,7 +465,7 @@ def test_parquet(tmpdir, registered_period_type):
 
     import pyarrow.parquet as pq
 
-    filename = tmpdir / 'extension_type.parquet'
+    filename = tmpdir / 'period_extension_type.parquet'
     pq.write_table(table, filename)
 
     # Stored in parquet as storage type but with extension metadata saved
@@ -469,6 +501,100 @@ def test_parquet(tmpdir, registered_period_type):
     }
 
 
+@pytest.mark.parquet
+def test_parquet_extension_with_nested_storage(tmpdir):
+    # Parquet support for extension types with nested storage type
+    import pyarrow.parquet as pq
+
+    struct_array = pa.StructArray.from_arrays(
+        [pa.array([0, 1], type="int64"), pa.array([4, 5], type="int64")],
+        names=["left", "right"])
+    list_array = pa.array([[1, 2, 3], [4, 5]], type=pa.list_(pa.int32()))
+
+    mystruct_array = pa.ExtensionArray.from_storage(MyStructType(),
+                                                    struct_array)
+    mylist_array = pa.ExtensionArray.from_storage(
+        MyListType(list_array.type), list_array)
+
+    orig_table = pa.table({'structs': mystruct_array,
+                           'lists': mylist_array})
+    filename = tmpdir / 'nested_extension_storage.parquet'
+    pq.write_table(orig_table, filename)
+
+    table = pq.read_table(filename)
+    assert table.column('structs').type == mystruct_array.type
+    assert table.column('lists').type == mylist_array.type
+    assert table == orig_table
+
+
+@pytest.mark.parquet
+def test_parquet_nested_extension(tmpdir):
+    # Parquet support for extension types nested in struct or list
+    import pyarrow.parquet as pq
+
+    ext_type = IntegerType()
+    storage = pa.array([4, 5, 6, 7], type=pa.int64())
+    ext_array = pa.ExtensionArray.from_storage(ext_type, storage)
+
+    # Struct of extensions
+    struct_array = pa.StructArray.from_arrays(
+        [storage, ext_array],
+        names=['ints', 'exts'])
+
+    orig_table = pa.table({'structs': struct_array})
+    filename = tmpdir / 'struct_of_ext.parquet'
+    pq.write_table(orig_table, filename)
+
+    table = pq.read_table(filename)
+    assert table.column(0).type == struct_array.type
+    assert table == orig_table
+
+    # List of extensions
+    list_array = pa.ListArray.from_arrays([0, 1, None, 3], ext_array)
+
+    orig_table = pa.table({'lists': list_array})
+    filename = tmpdir / 'list_of_ext.parquet'
+    pq.write_table(orig_table, filename)
+
+    table = pq.read_table(filename)
+    assert table.column(0).type == list_array.type
+    assert table == orig_table
+
+    # Large list of extensions
+    list_array = pa.LargeListArray.from_arrays([0, 1, None, 3], ext_array)
+
+    orig_table = pa.table({'lists': list_array})
+    filename = tmpdir / 'list_of_ext.parquet'
+    pq.write_table(orig_table, filename)
+
+    table = pq.read_table(filename)
+    assert table.column(0).type == list_array.type
+    assert table == orig_table
+
+
+@pytest.mark.parquet
+def test_parquet_extension_nested_in_extension(tmpdir):
+    # Parquet support for extension<list<extension>>
+    import pyarrow.parquet as pq
+
+    inner_ext_type = IntegerType()
+    inner_storage = pa.array([4, 5, 6, 7], type=pa.int64())
+    inner_ext_array = pa.ExtensionArray.from_storage(inner_ext_type,
+                                                     inner_storage)
+
+    list_array = pa.ListArray.from_arrays([0, 1, None, 3], inner_ext_array)
+    mylist_array = pa.ExtensionArray.from_storage(
+        MyListType(list_array.type), list_array)
+
+    orig_table = pa.table({'lists': mylist_array})
+    filename = tmpdir / 'ext_of_list_of_ext.parquet'
+    pq.write_table(orig_table, filename)
+
+    table = pq.read_table(filename)
+    assert table.column(0).type == mylist_array.type
+    assert table == orig_table
+
+
 def test_to_numpy():
     period_type = PeriodType('D')
     storage = pa.array([1, 2, 3, 4], pa.int64())
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index f183b028050..b3348269e4a 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -123,24 +123,27 @@ def f():
     pytest.raises(exc, f)
 
 
-@pytest.mark.pandas
 def test_dataset(version):
     num_values = (100, 100)
     num_files = 5
     paths = [random_path() for i in range(num_files)]
-    df = pd.DataFrame(np.random.randn(*num_values),
-                      columns=['col_' + str(i)
-                               for i in range(num_values[1])])
+    data = {
+        "col_" + str(i): np.random.randn(num_values[0])
+        for i in range(num_values[1])
+    }
+    table = pa.table(data)
 
     TEST_FILES.extend(paths)
     for index, path in enumerate(paths):
-        rows = (index * (num_values[0] // num_files),
-                (index + 1) * (num_values[0] // num_files))
+        rows = (
+            index * (num_values[0] // num_files),
+            (index + 1) * (num_values[0] // num_files),
+        )
 
-        write_feather(df.iloc[rows[0]:rows[1]], path, version=version)
+        write_feather(table[rows[0]: rows[1]], path, version=version)
 
-    data = FeatherDataset(paths).read_pandas()
-    assert_frame_equal(data, df)
+    data = FeatherDataset(paths).read_table()
+    assert data.equals(table)
 
 
 @pytest.mark.pandas
@@ -165,21 +168,17 @@ def test_read_table(version):
     TEST_FILES.append(path)
 
     values = np.random.randint(0, 100, size=num_values)
+    columns = ['col_' + str(i) for i in range(100)]
+    table = pa.Table.from_arrays(values, columns)
 
-    df = pd.DataFrame(values, columns=['col_' + str(i)
-                                       for i in range(100)])
-    write_feather(df, path, version=version)
-
-    data = pd.DataFrame(values,
-                        columns=['col_' + str(i) for i in range(100)])
-    table = pa.Table.from_pandas(data)
+    write_feather(table, path, version=version)
 
     result = read_table(path)
-    assert_frame_equal(table.to_pandas(), result.to_pandas())
+    assert result.equals(table)
 
     # Test without memory mapping
     result = read_table(path, memory_map=False)
-    assert_frame_equal(table.to_pandas(), result.to_pandas())
+    assert result.equals(table)
 
     result = read_feather(path, memory_map=False)
     assert_frame_equal(table.to_pandas(), result)
@@ -206,18 +205,15 @@ def test_float_nulls(version):
         expected_cols.append(values)
 
     table = pa.table(arrays, names=dtypes)
-    write_feather(table, path, version=version)
-
-    ex_frame = pd.DataFrame(dict(zip(dtypes, expected_cols)),
-                            columns=dtypes)
+    _check_arrow_roundtrip(table)
 
-    result = read_feather(path)
-    assert_frame_equal(result, ex_frame)
+    df = table.to_pandas()
+    _check_pandas_roundtrip(df, version=version)
 
 
 @pytest.mark.pandas
 def test_integer_no_nulls(version):
-    data = {}
+    data, arr = {}, []
 
     numpy_dtypes = ['i1', 'i2', 'i4', 'i8',
                     'u1', 'u2', 'u4', 'u8']
@@ -226,10 +222,14 @@ def test_integer_no_nulls(version):
     for dtype in numpy_dtypes:
         values = np.random.randint(0, 100, size=num_values)
         data[dtype] = values.astype(dtype)
+        arr.append(values.astype(dtype))
 
     df = pd.DataFrame(data)
     _check_pandas_roundtrip(df, version=version)
 
+    table = pa.table(arr, names=numpy_dtypes)
+    _check_arrow_roundtrip(table)
+
 
 @pytest.mark.pandas
 def test_platform_numpy_integers(version):
@@ -268,13 +268,10 @@ def test_integer_with_nulls(version):
         expected_cols.append(expected)
 
     table = pa.table(arrays, names=int_dtypes)
-    write_feather(table, path, version=version)
-
-    ex_frame = pd.DataFrame(dict(zip(int_dtypes, expected_cols)),
-                            columns=int_dtypes)
+    _check_arrow_roundtrip(table)
 
-    result = read_feather(path)
-    assert_frame_equal(result, ex_frame)
+    df = table.to_pandas()
+    _check_pandas_roundtrip(df, version=version)
 
 
 @pytest.mark.pandas
@@ -300,41 +297,32 @@ def test_boolean_nulls(version):
     values = np.random.randint(0, 10, size=num_values) < 5
 
     table = pa.table([pa.array(values, mask=mask)], names=['bools'])
-    write_feather(table, path, version=version)
-
-    expected = values.astype(object)
-    expected[mask] = None
-
-    ex_frame = pd.DataFrame({'bools': expected})
+    _check_arrow_roundtrip(table)
 
-    result = read_feather(path)
-    assert_frame_equal(result, ex_frame)
+    df = table.to_pandas()
+    _check_pandas_roundtrip(df, version=version)
 
 
-@pytest.mark.pandas
 def test_buffer_bounds_error(version):
     # ARROW-1676
     path = random_path()
     TEST_FILES.append(path)
 
     for i in range(16, 256):
-        values = pa.array([None] + list(range(i)), type=pa.float64())
-
-        write_feather(pa.table([values], names=['arr']), path,
-                      version=version)
-        result = read_feather(path)
-        expected = pd.DataFrame({'arr': values.to_pandas()})
-        assert_frame_equal(result, expected)
-
-        _check_pandas_roundtrip(expected, version=version)
+        table = pa.Table.from_arrays(
+            [pa.array([None] + list(range(i)), type=pa.float64())],
+            names=["arr"]
+        )
+        _check_arrow_roundtrip(table)
 
 
-@pytest.mark.pandas
 def test_boolean_object_nulls(version):
     repeats = 100
-    arr = np.array([False, None, True] * repeats, dtype=object)
-    df = pd.DataFrame({'bools': arr})
-    _check_pandas_roundtrip(df, version=version)
+    table = pa.Table.from_arrays(
+        [np.array([False, None, True] * repeats, dtype=object)],
+        names=["arr"]
+    )
+    _check_arrow_roundtrip(table)
 
 
 @pytest.mark.pandas
@@ -511,7 +499,6 @@ def test_read_columns(version):
                             columns=['boo', 'woo'])
 
 
-@pytest.mark.pandas
 def test_overwritten_file(version):
     path = random_path()
     TEST_FILES.append(path)
@@ -520,10 +507,12 @@ def test_overwritten_file(version):
     np.random.seed(0)
 
     values = np.random.randint(0, 10, size=num_values)
-    write_feather(pd.DataFrame({'ints': values}), path, version=version)
 
-    df = pd.DataFrame({'ints': values[0: num_values//2]})
-    _check_pandas_roundtrip(df, path=path, version=version)
+    table = pa.table({'ints': values})
+    write_feather(table, path)
+
+    table = pa.table({'more_ints': values[0:num_values//2]})
+    _check_arrow_roundtrip(table, path=path)
 
 
 @pytest.mark.pandas
@@ -710,9 +699,8 @@ def test_chunked_binary_error_message():
 def test_feather_without_pandas(tempdir, version):
     # ARROW-8345
     table = pa.table([pa.array([1, 2, 3])], names=['f0'])
-    write_feather(table, str(tempdir / "data.feather"), version=version)
-    result = read_table(str(tempdir / "data.feather"))
-    assert result.equals(table)
+    path = str(tempdir / "data.feather")
+    _check_arrow_roundtrip(table, path)
 
 
 @pytest.mark.pandas
@@ -739,9 +727,11 @@ def test_read_column_duplicated_selection(tempdir, version):
     path = str(tempdir / "data.feather")
     write_feather(table, path, version=version)
 
+    expected = pa.table([[1, 2, 3], [4, 5, 6], [1, 2, 3]],
+                        names=['a', 'b', 'a'])
     for col_selection in [['a', 'b', 'a'], [0, 1, 0]]:
         result = read_table(path, columns=col_selection)
-        assert result.column_names == ['a', 'b', 'a']
+        assert result.equals(expected)
 
 
 def test_read_column_duplicated_in_file(tempdir):
diff --git a/python/pyarrow/tests/test_filesystem.py b/python/pyarrow/tests/test_filesystem.py
index 4a6606ff51a..b859a4353ee 100644
--- a/python/pyarrow/tests/test_filesystem.py
+++ b/python/pyarrow/tests/test_filesystem.py
@@ -15,8 +15,38 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import sys
+
+import pyarrow as pa
 from pyarrow import filesystem
 
+import pytest
+
+
+def test_filesystem_deprecated():
+    with pytest.warns(DeprecationWarning):
+        filesystem.LocalFileSystem()
+
+    with pytest.warns(DeprecationWarning):
+        filesystem.LocalFileSystem.get_instance()
+
+
+@pytest.mark.skipif(sys.version_info < (3, 7),
+                    reason="getattr needs Python 3.7")
+def test_filesystem_deprecated_toplevel():
+
+    with pytest.warns(DeprecationWarning):
+        pa.localfs
+
+    with pytest.warns(DeprecationWarning):
+        pa.FileSystem
+
+    with pytest.warns(DeprecationWarning):
+        pa.LocalFileSystem
+
+    with pytest.warns(DeprecationWarning):
+        pa.HadoopFileSystem
+
 
 def test_resolve_uri():
     uri = "file:///home/user/myfile.parquet"
diff --git a/python/pyarrow/tests/test_flight.py b/python/pyarrow/tests/test_flight.py
index 50e993dce87..3a5f5778f62 100644
--- a/python/pyarrow/tests/test_flight.py
+++ b/python/pyarrow/tests/test_flight.py
@@ -952,6 +952,8 @@ def test_http_basic_unauth():
             list(client.do_action(action))
 
 
+@pytest.mark.skipif(os.name == 'nt',
+                    reason="ARROW-10013: gRPC on Windows corrupts peer()")
 def test_http_basic_auth():
     """Test a Python implementation of HTTP basic authentication."""
     with EchoStreamFlightServer(auth_handler=basic_auth_handler) as server:
@@ -1040,6 +1042,22 @@ def test_tls_do_get():
         assert data.equals(table)
 
 
+@pytest.mark.requires_testing_data
+def test_tls_disable_server_verification():
+    """Try a simple do_get call over TLS with server verification disabled."""
+    table = simple_ints_table()
+    certs = example_tls_certs()
+
+    with ConstantFlightServer(tls_certificates=certs["certificates"]) as s:
+        try:
+            client = FlightClient(('localhost', s.port),
+                                  disable_server_verification=True)
+        except NotImplementedError:
+            pytest.skip('disable_server_verification feature is not available')
+        data = client.do_get(flight.Ticket(b'ints')).read_all()
+        assert data.equals(table)
+
+
 @pytest.mark.requires_testing_data
 def test_tls_override_hostname():
     """Check that incorrectly overriding the hostname fails."""
diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py
index 50657ea966f..4f926b4897c 100644
--- a/python/pyarrow/tests/test_fs.py
+++ b/python/pyarrow/tests/test_fs.py
@@ -16,6 +16,7 @@
 # under the License.
 
 from datetime import datetime, timezone, timedelta
+from distutils.version import LooseVersion
 import gzip
 import pathlib
 import pickle
@@ -349,6 +350,8 @@ def py_fsspec_memoryfs(request, tempdir):
 @pytest.fixture
 def py_fsspec_s3fs(request, s3_connection, s3_server):
     s3fs = pytest.importorskip("s3fs")
+    if sys.version_info < (3, 7) and s3fs.__version__ >= LooseVersion("0.5"):
+        pytest.skip("s3fs>=0.5 version is async and requires Python >= 3.7")
 
     host, port, access_key, secret_key = s3_connection
     bucket = 'pyarrow-filesystem/'
@@ -601,8 +604,6 @@ class Path:
 
 
 def test_get_file_info(fs, pathfn):
-    skip_fsspec_s3fs(fs)  # s3fs doesn't create nested directories
-
     aaa = pathfn('a/aa/aaa/')
     bb = pathfn('a/bb')
     c = pathfn('c.txt')
@@ -619,6 +620,7 @@ def test_get_file_info(fs, pathfn):
     assert aaa_info.path == aaa
     assert 'aaa' in repr(aaa_info)
     assert aaa_info.extension == ''
+    assert aaa_info.type == FileType.Directory
     assert 'FileType.Directory' in repr(aaa_info)
     assert aaa_info.size is None
     check_mtime_or_absent(aaa_info)
@@ -629,7 +631,7 @@ def test_get_file_info(fs, pathfn):
     assert bb_info.type == FileType.File
     assert 'FileType.File' in repr(bb_info)
     assert bb_info.size == 0
-    if fs.type_name != "py::fsspec+memory":
+    if fs.type_name not in ["py::fsspec+memory", "py::fsspec+s3"]:
         check_mtime(bb_info)
 
     assert c_info.path == str(c)
@@ -638,7 +640,7 @@ def test_get_file_info(fs, pathfn):
     assert c_info.type == FileType.File
     assert 'FileType.File' in repr(c_info)
     assert c_info.size == 4
-    if fs.type_name != "py::fsspec+memory":
+    if fs.type_name not in ["py::fsspec+memory", "py::fsspec+s3"]:
         check_mtime(c_info)
 
     assert zzz_info.path == str(zzz)
@@ -650,14 +652,19 @@ def test_get_file_info(fs, pathfn):
     assert 'FileType.NotFound' in repr(zzz_info)
     check_mtime_absent(zzz_info)
 
+    # with single path
+    aaa_info2 = fs.get_file_info(aaa)
+    assert aaa_info.path == aaa_info2.path
+    assert aaa_info.type == aaa_info2.type
 
-def test_get_file_info_with_selector(fs, pathfn):
-    skip_fsspec_s3fs(fs)
 
+def test_get_file_info_with_selector(fs, pathfn):
     base_dir = pathfn('selector-dir/')
     file_a = pathfn('selector-dir/test_file_a')
     file_b = pathfn('selector-dir/test_file_b')
     dir_a = pathfn('selector-dir/test_dir_a')
+    file_c = pathfn('selector-dir/test_dir_a/test_file_c')
+    dir_b = pathfn('selector-dir/test_dir_b')
 
     try:
         fs.create_dir(base_dir)
@@ -666,34 +673,51 @@ def test_get_file_info_with_selector(fs, pathfn):
         with fs.open_output_stream(file_b):
             pass
         fs.create_dir(dir_a)
+        with fs.open_output_stream(file_c):
+            pass
+        fs.create_dir(dir_b)
 
+        # recursive selector
         selector = FileSelector(base_dir, allow_not_found=False,
                                 recursive=True)
         assert selector.base_dir == base_dir
 
         infos = fs.get_file_info(selector)
-        assert len(infos) == 3
+        if fs.type_name == "py::fsspec+s3":
+            # s3fs only lists directories if they are not empty
+            assert len(infos) == 4
+        else:
+            assert len(infos) == 5
 
         for info in infos:
-            if info.path.endswith(file_a):
+            if (info.path.endswith(file_a) or info.path.endswith(file_b) or
+                    info.path.endswith(file_c)):
                 assert info.type == FileType.File
-            elif info.path.endswith(file_b):
-                assert info.type == FileType.File
-            elif info.path.rstrip("/").endswith(dir_a):
+            elif (info.path.rstrip("/").endswith(dir_a) or
+                  info.path.rstrip("/").endswith(dir_b)):
                 assert info.type == FileType.Directory
             else:
                 raise ValueError('unexpected path {}'.format(info.path))
             check_mtime_or_absent(info)
+
+        # non-recursive selector -> not selecting the nested file_c
+        selector = FileSelector(base_dir, recursive=False)
+
+        infos = fs.get_file_info(selector)
+        if fs.type_name == "py::fsspec+s3":
+            # s3fs only lists directories if they are not empty
+            assert len(infos) == 3
+        else:
+            assert len(infos) == 4
+
     finally:
-        fs.delete_file(file_a)
-        fs.delete_file(file_b)
-        fs.delete_dir(dir_a)
         fs.delete_dir(base_dir)
 
 
 def test_create_dir(fs, pathfn):
-    skip_fsspec_s3fs(fs)  # create_dir doesn't create dir, so delete dir fails
-
+    # s3fs fails deleting dir fails if it is empty
+    # (https://github.com/dask/s3fs/issues/317)
+    skip_fsspec_s3fs(fs)
     d = pathfn('test-directory/')
 
     with pytest.raises(pa.ArrowIOError):
@@ -780,10 +804,6 @@ def test_copy_file(fs, pathfn, allow_copy_file):
 
 
 def test_move_directory(fs, pathfn, allow_move_dir):
-    if fs.type_name == "py::fsspec+memory":
-        # https://github.com/intake/filesystem_spec/issues/316
-        pytest.xfail(reason='Not working with in-memory fsspec')
-
     # move directory (doesn't work with S3)
     s = pathfn('source-dir/')
     t = pathfn('target-dir/')
@@ -801,10 +821,6 @@ def test_move_directory(fs, pathfn, allow_move_dir):
 
 
 def test_move_file(fs, pathfn):
-    if fs.type_name == "py::fsspec+memory":
-        # https://issues.apache.org/jira/browse/ARROW-9621
-        # https://github.com/intake/filesystem_spec/issues/367
-        pytest.xfail(reason='Not working with in-memory fsspec')
     s = pathfn('test-move-source-file')
     t = pathfn('test-move-target-file')
 
@@ -983,12 +999,21 @@ def test_mockfs_mtime_roundtrip(mockfs):
 
 
 @pytest.mark.s3
-def test_s3_options():
+def test_s3_options(monkeypatch):
     from pyarrow.fs import S3FileSystem
 
+    # Avoid wait for unavailable metadata server in ARN role example below
+    monkeypatch.setenv("AWS_EC2_METADATA_DISABLED", "true")
+
     fs = S3FileSystem(access_key='access', secret_key='secret',
-                      region='us-east-1', scheme='https',
-                      endpoint_override='localhost:8999')
+                      session_token='token', region='us-east-2',
+                      scheme='https', endpoint_override='localhost:8999')
+    assert isinstance(fs, S3FileSystem)
+    assert fs.region == 'us-east-2'
+    assert pickle.loads(pickle.dumps(fs)) == fs
+
+    fs = S3FileSystem(role_arn='role', session_name='session',
+                      external_id='id', load_frequency=100)
     assert isinstance(fs, S3FileSystem)
     assert pickle.loads(pickle.dumps(fs)) == fs
 
@@ -996,6 +1021,14 @@ def test_s3_options():
         S3FileSystem(access_key='access')
     with pytest.raises(ValueError):
         S3FileSystem(secret_key='secret')
+    with pytest.raises(ValueError):
+        S3FileSystem(access_key='access', session_token='token')
+    with pytest.raises(ValueError):
+        S3FileSystem(secret_key='secret', session_token='token')
+    with pytest.raises(ValueError):
+        S3FileSystem(
+            access_key='access', secret_key='secret', role_arn='arn'
+        )
 
 
 @pytest.mark.hdfs
@@ -1298,5 +1331,31 @@ def test_s3_real_aws():
     # This is a minimal integration check for ARROW-9261 and similar issues.
     from pyarrow.fs import S3FileSystem
     fs = S3FileSystem(anonymous=True)
+    assert fs.region == 'us-east-1'  # default region
+
+    fs = S3FileSystem(anonymous=True, region='us-east-2')
     entries = fs.get_file_info(FileSelector('ursa-labs-taxi-data'))
     assert len(entries) > 0
+
+
+@pytest.mark.s3
+def test_s3_real_aws_region_selection():
+    # Taken from a registry of open S3-hosted datasets
+    # at https://github.com/awslabs/open-data-registry
+    fs, path = FileSystem.from_uri('s3://mf-nwp-models/README.txt')
+    assert fs.region == 'eu-west-1'
+    with fs.open_input_stream(path) as f:
+        assert b"Meteo-France Atmospheric models on AWS" in f.read(50)
+
+    # Passing an explicit region disables auto-selection
+    fs, path = FileSystem.from_uri(
+        's3://mf-nwp-models/README.txt?region=us-east-2')
+    assert fs.region == 'us-east-2'
+    # Reading from the wrong region may still work for public buckets...
+
+    # Non-existent bucket (hopefully, otherwise need to fix this test)
+    with pytest.raises(IOError, match="Bucket '.*' not found"):
+        FileSystem.from_uri('s3://x-arrow-non-existent-bucket')
+    fs, path = FileSystem.from_uri(
+        's3://x-arrow-non-existent-bucket?region=us-east-3')
+    assert fs.region == 'us-east-3'
diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
index 2110be59069..b7a270924e3 100644
--- a/python/pyarrow/tests/test_hdfs.py
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -46,7 +46,8 @@ def hdfs_test_client():
         raise ValueError('Env variable ARROW_HDFS_TEST_PORT was not '
                          'an integer')
 
-    return pa.hdfs.connect(host, port, user)
+    with pytest.warns(DeprecationWarning):
+        return pa.hdfs.connect(host, port, user)
 
 
 @pytest.mark.hdfs
@@ -320,7 +321,8 @@ def test_read_multiple_parquet_files_with_uri(self):
 
         expected = self._write_multiple_hdfs_pq_files(tmpdir)
         path = _get_hdfs_uri(tmpdir)
-        result = pq.read_table(path)
+        # TODO for URI it should not be needed to pass this argument
+        result = pq.read_table(path, use_legacy_dataset=True)
 
         _pandas_api.assert_frame_equal(result.to_pandas()
                                        .sort_values(by='index')
@@ -344,7 +346,9 @@ def test_read_write_parquet_files_with_uri(self):
 
         pq.write_table(table, path, filesystem=self.hdfs)
 
-        result = pq.read_table(path, filesystem=self.hdfs).to_pandas()
+        result = pq.read_table(
+            path, filesystem=self.hdfs, use_legacy_dataset=True
+        ).to_pandas()
 
         _pandas_api.assert_frame_equal(result, df)
 
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index 948709d9376..69dc1355d28 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -573,7 +573,7 @@ def test_compress_decompress(compression):
     INPUT_SIZE = 10000
     test_data = (np.random.randint(0, 255, size=INPUT_SIZE)
                  .astype(np.uint8)
-                 .tostring())
+                 .tobytes())
     test_buf = pa.py_buffer(test_data)
 
     compressed_buf = pa.compress(test_buf, codec=compression)
@@ -977,6 +977,22 @@ def test_native_file_modes(tmpdir):
         assert f.seekable()
 
 
+def test_native_file_permissions(tmpdir):
+    # ARROW-10124: permissions of created files should follow umask
+    cur_umask = os.umask(0o002)
+    os.umask(cur_umask)
+
+    path = os.path.join(str(tmpdir), guid())
+    with pa.OSFile(path, mode='w'):
+        pass
+    assert os.stat(path).st_mode & 0o777 == 0o666 & ~cur_umask
+
+    path = os.path.join(str(tmpdir), guid())
+    with pa.memory_map(path, 'w'):
+        pass
+    assert os.stat(path).st_mode & 0o777 == 0o666 & ~cur_umask
+
+
 def test_native_file_raises_ValueError_after_close(tmpdir):
     path = os.path.join(str(tmpdir), guid())
     with open(path, 'wb') as f:
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 44f8499e834..62bb1e708a5 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -15,11 +15,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from collections import UserList
 import io
 import pytest
 import socket
 import sys
 import threading
+import weakref
 
 import numpy as np
 
@@ -763,6 +765,7 @@ def test_serialize_pandas_no_preserve_index():
     assert_frame_equal(result, df)
 
 
+@pytest.mark.filterwarnings("ignore:'pyarrow:DeprecationWarning")
 def test_serialize_with_pandas_objects():
     df = pd.DataFrame({'a': [1, 2, 3]}, index=[1, 2, 3])
     s = pd.Series([1, 2, 3, 4])
@@ -858,3 +861,36 @@ def test_write_empty_ipc_file():
         table = reader.read_all()
     assert len(table) == 0
     assert table.schema.equals(schema)
+
+
+def test_py_record_batch_reader():
+    def make_schema():
+        return pa.schema([('field', pa.int64())])
+
+    def make_batches():
+        schema = make_schema()
+        batch1 = pa.record_batch([[1, 2, 3]], schema=schema)
+        batch2 = pa.record_batch([[4, 5]], schema=schema)
+        return [batch1, batch2]
+
+    # With iterable
+    batches = UserList(make_batches())  # weakrefable
+    wr = weakref.ref(batches)
+
+    with pa.ipc.RecordBatchReader.from_batches(make_schema(),
+                                               batches) as reader:
+        batches = None
+        assert wr() is not None
+        assert list(reader) == make_batches()
+        assert wr() is None
+
+    # With iterator
+    batches = iter(UserList(make_batches()))  # weakrefable
+    wr = weakref.ref(batches)
+
+    with pa.ipc.RecordBatchReader.from_batches(make_schema(),
+                                               batches) as reader:
+        batches = None
+        assert wr() is not None
+        assert list(reader) == make_batches()
+        assert wr() is None
diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py
index b023e394ef9..f46e75fc3ac 100644
--- a/python/pyarrow/tests/test_pandas.py
+++ b/python/pyarrow/tests/test_pandas.py
@@ -22,7 +22,7 @@
 import sys
 
 from collections import OrderedDict
-from datetime import date, datetime, time, timedelta
+from datetime import date, datetime, time, timedelta, timezone
 from distutils.version import LooseVersion
 
 import hypothesis as h
@@ -35,6 +35,7 @@
 
 from pyarrow.pandas_compat import get_logical_type, _pandas_api
 from pyarrow.tests.util import random_ascii, rands
+import pyarrow.tests.strategies as past
 
 import pyarrow as pa
 try:
@@ -998,9 +999,13 @@ def test_python_datetime_with_pytz_timezone(self, tz):
     def test_python_datetime_with_timezone_tzinfo(self):
         from datetime import timezone
 
-        values = [datetime(2018, 1, 1, 12, 23, 45, tzinfo=pytz.utc)]
-        df = pd.DataFrame({'datetime': values})
-        _check_pandas_roundtrip(df)
+        if LooseVersion(pd.__version__) > "0.25.0":
+            # older pandas versions fail on datetime.timezone.utc (as in input)
+            # vs pytz.UTC (as in result)
+            values = [datetime(2018, 1, 1, 12, 23, 45, tzinfo=timezone.utc)]
+            # also test with index to ensure both paths roundtrip (ARROW-9962)
+            df = pd.DataFrame({'datetime': values}, index=values)
+            _check_pandas_roundtrip(df, preserve_index=True)
 
         # datetime.timezone is going to be pytz.FixedOffset
         hours = 1
@@ -1008,9 +1013,9 @@ def test_python_datetime_with_timezone_tzinfo(self):
         tz_pytz = pytz.FixedOffset(hours * 60)
         values = [datetime(2018, 1, 1, 12, 23, 45, tzinfo=tz_timezone)]
         values_exp = [datetime(2018, 1, 1, 12, 23, 45, tzinfo=tz_pytz)]
-        df = pd.DataFrame({'datetime': values})
-        df_exp = pd.DataFrame({'datetime': values_exp})
-        _check_pandas_roundtrip(df, expected=df_exp)
+        df = pd.DataFrame({'datetime': values}, index=values)
+        df_exp = pd.DataFrame({'datetime': values_exp}, index=values_exp)
+        _check_pandas_roundtrip(df, expected=df_exp, preserve_index=True)
 
     def test_python_datetime_subclass(self):
 
@@ -1497,6 +1502,22 @@ def test_bytes_exceed_2gb(self):
         table = pa.Table.from_pandas(df)
         assert table[0].num_chunks == 2
 
+    @pytest.mark.large_memory
+    @pytest.mark.parametrize('char', ['x', b'x'])
+    def test_auto_chunking_pandas_series_of_strings(self, char):
+        # ARROW-2367
+        v1 = char * 100000000
+        v2 = char * 147483646
+
+        df = pd.DataFrame({
+            'strings': [[v1]] * 20 + [[v2]] + [[b'x']]
+        })
+        arr = pa.array(df['strings'], from_pandas=True)
+        assert isinstance(arr, pa.ChunkedArray)
+        assert arr.num_chunks == 2
+        assert len(arr.chunk(0)) == 21
+        assert len(arr.chunk(1)) == 1
+
     def test_fixed_size_bytes(self):
         values = [b'foo', None, bytearray(b'bar'), None, None, b'hey']
         df = pd.DataFrame({'strings': values})
@@ -1772,7 +1793,7 @@ def test_column_of_arrays_to_py(self):
             np.arange(5, dtype=dtype),
             None,
             np.arange(1, dtype=dtype)
-        ])
+        ], dtype=object)
         type_ = pa.list_(pa.int8())
         parr = pa.array(arr, type=type_)
 
@@ -2069,7 +2090,7 @@ def test_nested_large_list(self):
                       type=pa.large_list(pa.large_list(pa.int64())))
              .to_pandas())
         tm.assert_series_equal(
-            s, pd.Series([[[1, 2, 3], [4]], None]),
+            s, pd.Series([[[1, 2, 3], [4]], None], dtype=object),
             check_names=False)
 
     def test_large_binary_list(self):
@@ -2087,6 +2108,89 @@ def test_large_binary_list(self):
                 s, pd.Series([["aa", "bb"], None, ["cc"], []]),
                 check_names=False)
 
+    def test_list_of_dictionary(self):
+        child = pa.array(["foo", "bar", None, "foo"]).dictionary_encode()
+        arr = pa.ListArray.from_arrays([0, 1, 3, 3, 4], child)
+
+        # Expected a Series of lists
+        expected = pd.Series(arr.to_pylist())
+        tm.assert_series_equal(arr.to_pandas(), expected)
+
+        # Same but with nulls
+        arr = arr.take([0, 1, None, 3])
+        expected[2] = None
+        tm.assert_series_equal(arr.to_pandas(), expected)
+
+    @pytest.mark.slow
+    @pytest.mark.large_memory
+    def test_auto_chunking_on_list_overflow(self):
+        # ARROW-9976
+        n = 2**24
+        df = pd.DataFrame.from_dict({
+            "a": list(np.zeros((n, 2**7), dtype='uint8')),
+            "b": range(n)
+        })
+        table = pa.Table.from_pandas(df)
+
+        column_a = table[0]
+        assert column_a.num_chunks == 2
+        assert len(column_a.chunk(0)) == 2**24 - 1
+        assert len(column_a.chunk(1)) == 1
+
+    def test_map_array_roundtrip(self):
+        data = [[(b'a', 1), (b'b', 2)],
+                [(b'c', 3)],
+                [(b'd', 4), (b'e', 5), (b'f', 6)],
+                [(b'g', 7)]]
+
+        df = pd.DataFrame({"map": data})
+        schema = pa.schema([("map", pa.map_(pa.binary(), pa.int32()))])
+
+        _check_pandas_roundtrip(df, schema=schema)
+
+    def test_map_array_chunked(self):
+        data1 = [[(b'a', 1), (b'b', 2)],
+                 [(b'c', 3)],
+                 [(b'd', 4), (b'e', 5), (b'f', 6)],
+                 [(b'g', 7)]]
+        data2 = [[(k, v * 2) for k, v in row] for row in data1]
+
+        arr1 = pa.array(data1, type=pa.map_(pa.binary(), pa.int32()))
+        arr2 = pa.array(data2, type=pa.map_(pa.binary(), pa.int32()))
+        arr = pa.chunked_array([arr1, arr2])
+
+        expected = pd.Series(data1 + data2)
+        actual = arr.to_pandas()
+        tm.assert_series_equal(actual, expected, check_names=False)
+
+    def test_map_array_with_nulls(self):
+        data = [[(b'a', 1), (b'b', 2)],
+                None,
+                [(b'd', 4), (b'e', 5), (b'f', None)],
+                [(b'g', 7)]]
+
+        # None value in item array causes upcast to float
+        expected = [[(k, float(v) if v is not None else None) for k, v in row]
+                    if row is not None else None for row in data]
+        expected = pd.Series(expected)
+
+        arr = pa.array(data, type=pa.map_(pa.binary(), pa.int32()))
+        actual = arr.to_pandas()
+        tm.assert_series_equal(actual, expected, check_names=False)
+
+    def test_map_array_dictionary_encoded(self):
+        offsets = pa.array([0, 3, 5])
+        items = pa.array(['a', 'b', 'c', 'a', 'd']).dictionary_encode()
+        keys = pa.array(list(range(len(items))))
+        arr = pa.MapArray.from_arrays(offsets, keys, items)
+
+        # Dictionary encoded values converted to dense
+        expected = pd.Series(
+            [[(0, 'a'), (1, 'b'), (2, 'c')], [(3, 'a'), (4, 'd')]])
+
+        actual = arr.to_pandas()
+        tm.assert_series_equal(actual, expected, check_names=False)
+
 
 class TestConvertStructTypes:
     """
@@ -2265,6 +2369,24 @@ def test_from_tuples(self):
             df, expected=expected_df, schema=expected_schema,
             expected_schema=expected_schema)
 
+    def test_struct_of_dictionary(self):
+        names = ['ints', 'strs']
+        children = [pa.array([456, 789, 456]).dictionary_encode(),
+                    pa.array(["foo", "foo", None]).dictionary_encode()]
+        arr = pa.StructArray.from_arrays(children, names=names)
+
+        # Expected a Series of {field name: field value} dicts
+        rows_as_tuples = zip(*(child.to_pylist() for child in children))
+        rows_as_dicts = [dict(zip(names, row)) for row in rows_as_tuples]
+
+        expected = pd.Series(rows_as_dicts)
+        tm.assert_series_equal(arr.to_pandas(), expected)
+
+        # Same but with nulls
+        arr = arr.take([0, None, 2])
+        expected[1] = None
+        tm.assert_series_equal(arr.to_pandas(), expected)
+
 
 class TestZeroCopyConversion:
     """
@@ -2447,19 +2569,22 @@ def test_category_zero_chunks(self):
             expected = pd.DataFrame({'a': expected})
             tm.assert_frame_equal(result, expected)
 
-    def test_mixed_types_fails(self):
-        data = pd.DataFrame({'a': ['a', 1, 2.0]})
-        with pytest.raises(pa.ArrowTypeError):
-            pa.Table.from_pandas(data)
-
-        data = pd.DataFrame({'a': [1, True]})
-        with pytest.raises(pa.ArrowTypeError):
-            pa.Table.from_pandas(data)
-
-        data = pd.DataFrame({'a': ['a', 1, 2.0]})
-        expected_msg = 'Conversion failed for column a'
-        with pytest.raises(pa.ArrowTypeError, match=expected_msg):
-            pa.Table.from_pandas(data)
+    @pytest.mark.parametrize(
+        "data,error_type",
+        [
+            ({"a": ["a", 1, 2.0]}, pa.ArrowTypeError),
+            ({"a": ["a", 1, 2.0]}, pa.ArrowTypeError),
+            ({"a": [1, True]}, pa.ArrowTypeError),
+            ({"a": [True, "a"]}, pa.ArrowInvalid),
+            ({"a": [1, "a"]}, pa.ArrowInvalid),
+            ({"a": [1.0, "a"]}, pa.ArrowInvalid),
+        ],
+    )
+    def test_mixed_types_fails(self, data, error_type):
+        df = pd.DataFrame(data)
+        msg = "Conversion failed for column a with type object"
+        with pytest.raises(error_type, match=msg):
+            pa.Table.from_pandas(df)
 
     def test_strided_data_import(self):
         cases = []
@@ -2592,7 +2717,11 @@ def test_safe_unsafe_casts(self):
 
     def test_error_sparse(self):
         # ARROW-2818
-        df = pd.DataFrame({'a': pd.SparseArray([1, np.nan, 3])})
+        try:
+            df = pd.DataFrame({'a': pd.arrays.SparseArray([1, np.nan, 3])})
+        except AttributeError:
+            # pandas.arrays module introduced in pandas 0.24
+            df = pd.DataFrame({'a': pd.SparseArray([1, np.nan, 3])})
         with pytest.raises(TypeError, match="Sparse pandas data"):
             pa.Table.from_pandas(df)
 
@@ -2644,10 +2773,13 @@ def test_roundtrip_with_bytes_unicode(columns):
 
 
 def _check_serialize_components_roundtrip(pd_obj):
-    ctx = pa.default_serialization_context()
+    with pytest.warns(DeprecationWarning):
+        ctx = pa.default_serialization_context()
 
-    components = ctx.serialize(pd_obj).to_components()
-    deserialized = ctx.deserialize_components(components)
+    with pytest.warns(DeprecationWarning):
+        components = ctx.serialize(pd_obj).to_components()
+    with pytest.warns(DeprecationWarning):
+        deserialized = ctx.deserialize_components(components)
 
     if isinstance(pd_obj, pd.DataFrame):
         tm.assert_frame_equal(pd_obj, deserialized)
@@ -2714,6 +2846,17 @@ class A:
             pa.Table.from_pandas(df)
 
 
+# ----------------------------------------------------------------------
+# Hypothesis tests
+
+
+@h.given(past.arrays(past.pandas_compatible_types))
+def test_array_to_pandas_roundtrip(arr):
+    s = arr.to_pandas()
+    restored = pa.array(s, type=arr.type, from_pandas=True)
+    assert restored.equals(arr)
+
+
 # ----------------------------------------------------------------------
 # Test object deduplication in to_pandas
 
@@ -3327,13 +3470,31 @@ def test_cast_timestamp_unit():
     assert result.equals(expected)
 
 
-def test_struct_with_timestamp_tz():
+def test_nested_with_timestamp_tz_round_trip():
+    ts = pd.Timestamp.now()
+    ts_dt = ts.to_pydatetime()
+    arr = pa.array([ts_dt], type=pa.timestamp('us', tz='America/New_York'))
+    struct = pa.StructArray.from_arrays([arr, arr], ['start', 'stop'])
+
+    result = struct.to_pandas()
+    restored = pa.array(result)
+    assert restored.equals(struct)
+
+
+def test_nested_with_timestamp_tz():
     # ARROW-7723
     ts = pd.Timestamp.now()
+    ts_dt = ts.to_pydatetime()
 
     # XXX: Ensure that this data does not get promoted to nanoseconds (and thus
     # integers) to preserve behavior in 0.15.1
     for unit in ['s', 'ms', 'us']:
+        if unit in ['s', 'ms']:
+            # This is used for verifying timezone conversion to micros are not
+            # important
+            def truncate(x): return x.replace(microsecond=0)
+        else:
+            def truncate(x): return x
         arr = pa.array([ts], type=pa.timestamp(unit))
         arr2 = pa.array([ts], type=pa.timestamp(unit, tz='America/New_York'))
 
@@ -3342,20 +3503,30 @@ def test_struct_with_timestamp_tz():
 
         result = arr3.to_pandas()
         assert isinstance(result[0]['start'], datetime)
+        assert result[0]['start'].tzinfo is None
         assert isinstance(result[0]['stop'], datetime)
+        assert result[0]['stop'].tzinfo is None
 
         result = arr4.to_pandas()
         assert isinstance(result[0]['start'], datetime)
+        assert result[0]['start'].tzinfo is not None
+        utc_dt = result[0]['start'].astimezone(timezone.utc)
+        assert truncate(utc_dt).replace(tzinfo=None) == truncate(ts_dt)
         assert isinstance(result[0]['stop'], datetime)
+        assert result[0]['stop'].tzinfo is not None
 
         # same conversion for table
         result = pa.table({'a': arr3}).to_pandas()
         assert isinstance(result['a'][0]['start'], datetime)
+        assert result['a'][0]['start'].tzinfo is None
         assert isinstance(result['a'][0]['stop'], datetime)
+        assert result['a'][0]['stop'].tzinfo is None
 
         result = pa.table({'a': arr4}).to_pandas()
         assert isinstance(result['a'][0]['start'], datetime)
+        assert result['a'][0]['start'].tzinfo is not None
         assert isinstance(result['a'][0]['stop'], datetime)
+        assert result['a'][0]['stop'].tzinfo is not None
 
 
 # ----------------------------------------------------------------------
@@ -3503,11 +3674,10 @@ def test_dictionary_from_pandas_specified_type():
     assert result.type.equals(typ)
     assert result.to_pylist() == ['a', 'b']
 
-    # mismatching values type -> raise error (for now a deprecation warning)
+    # mismatching values type -> raise error
     typ = pa.dictionary(index_type=pa.int8(), value_type=pa.int64())
-    with pytest.warns(FutureWarning):
+    with pytest.raises(pa.ArrowInvalid):
         result = pa.array(cat, type=typ)
-    assert result.to_pylist() == ['a', 'b']
 
     # mismatching order -> raise error (for now a deprecation warning)
     typ = pa.dictionary(
@@ -3983,6 +4153,20 @@ def test_metadata_compat_missing_field_name():
     tm.assert_frame_equal(result, expected, check_like=True)
 
 
+def test_metadata_index_name_not_json_serializable():
+    name = np.int64(6)  # not json serializable by default
+    table = pa.table(pd.DataFrame(index=pd.RangeIndex(0, 4, name=name)))
+    metadata = table.schema.pandas_metadata
+    assert metadata['index_columns'][0]['name'] == '6'
+
+
+def test_metadata_index_name_is_json_serializable():
+    name = 6  # json serializable by default
+    table = pa.table(pd.DataFrame(index=pd.RangeIndex(0, 4, name=name)))
+    metadata = table.schema.pandas_metadata
+    assert metadata['index_columns'][0]['name'] == 6
+
+
 def make_df_with_timestamps():
     # Some of the milliseconds timestamps deliberately don't fit in the range
     # that is possible with nanosecond timestamps.
@@ -4032,19 +4216,25 @@ def test_timestamp_as_object_out_of_range():
 
 
 @pytest.mark.parametrize("resolution", ["s", "ms", "us"])
+@pytest.mark.parametrize("tz", [None, "America/New_York"])
 # One datetime outside nanosecond range, one inside nanosecond range:
 @pytest.mark.parametrize("dt", [datetime(1553, 1, 1), datetime(2020, 1, 1)])
-def test_timestamp_as_object_non_nanosecond(resolution, dt):
+def test_timestamp_as_object_non_nanosecond(resolution, tz, dt):
     # Timestamps can be converted Arrow and reloaded into Pandas with no loss
     # of information if the timestamp_as_object option is True.
-    arr = pa.array([dt], type=pa.timestamp(resolution))
-    result = arr.to_pandas(timestamp_as_object=True)
-    assert result.dtype == object
-    assert isinstance(result[0], datetime)
-    assert result[0] == dt
-
+    arr = pa.array([dt], type=pa.timestamp(resolution, tz=tz))
     table = pa.table({'a': arr})
-    result = table.to_pandas(timestamp_as_object=True)['a']
-    assert result.dtype == object
-    assert isinstance(result[0], datetime)
-    assert result[0] == dt
+
+    for result in [
+        arr.to_pandas(timestamp_as_object=True),
+        table.to_pandas(timestamp_as_object=True)['a']
+    ]:
+        assert result.dtype == object
+        assert isinstance(result[0], datetime)
+        if tz:
+            assert result[0].tzinfo is not None
+            expected = result[0].tzinfo.fromutc(dt)
+        else:
+            assert result[0].tzinfo is None
+            expected = dt
+        assert result[0] == expected
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index b87cd55e320..2ff94b9d8d9 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -31,6 +31,7 @@
 from pyarrow.tests import util
 from pyarrow.util import guid
 from pyarrow.filesystem import LocalFileSystem, FileSystem
+from pyarrow import fs
 
 
 try:
@@ -496,6 +497,36 @@ def test_multiple_path_types(tempdir, use_legacy_dataset):
     tm.assert_frame_equal(df, df_read)
 
 
+@pytest.mark.dataset
+@parametrize_legacy_dataset
+@pytest.mark.parametrize("filesystem", [
+    None, fs.LocalFileSystem(), LocalFileSystem._get_instance()
+])
+def test_relative_paths(tempdir, use_legacy_dataset, filesystem):
+    # reading and writing from relative paths
+    table = pa.table({"a": [1, 2, 3]})
+
+    # reading
+    pq.write_table(table, str(tempdir / "data.parquet"))
+    with util.change_cwd(tempdir):
+        result = pq.read_table("data.parquet", filesystem=filesystem,
+                               use_legacy_dataset=use_legacy_dataset)
+    assert result.equals(table)
+
+    # writing
+    with util.change_cwd(tempdir):
+        pq.write_table(table, "data2.parquet", filesystem=filesystem)
+    result = pq.read_table(tempdir / "data2.parquet")
+    assert result.equals(table)
+
+
+@parametrize_legacy_dataset
+def test_read_non_existing_file(use_legacy_dataset):
+    # ensure we have a proper error message
+    with pytest.raises(FileNotFoundError):
+        pq.read_table('i-am-not-existing.parquet')
+
+
 # TODO(dataset) duplicate column selection actually gives duplicate columns now
 @pytest.mark.pandas
 @parametrize_legacy_dataset_not_supported
@@ -662,14 +693,6 @@ def test_pandas_can_write_nested_data(tempdir):
     # This succeeds under V2
     _write_table(arrow_table, imos)
 
-    # Under V1 it fails.
-    with pytest.raises(ValueError):
-        import os
-        os.environ['ARROW_PARQUET_WRITER_ENGINE'] = 'V1'
-        imos = pa.BufferOutputStream()
-        _write_table(arrow_table, imos)
-    del os.environ['ARROW_PARQUET_WRITER_ENGINE']
-
 
 @pytest.mark.pandas
 @parametrize_legacy_dataset
@@ -696,10 +719,6 @@ def test_pandas_parquet_pyfile_roundtrip(tempdir, use_legacy_dataset):
     tm.assert_frame_equal(df, df_read)
 
 
-# ARROW-9424: LZ4 support is currently disabled
-SUPPORTED_COMPRESSIONS = ['NONE', 'SNAPPY', 'GZIP', 'ZSTD']
-
-
 @pytest.mark.pandas
 @parametrize_legacy_dataset
 def test_pandas_parquet_configuration_options(tempdir, use_legacy_dataset):
@@ -737,7 +756,7 @@ def test_pandas_parquet_configuration_options(tempdir, use_legacy_dataset):
         df_read = table_read.to_pandas()
         tm.assert_frame_equal(df, df_read)
 
-    for compression in SUPPORTED_COMPRESSIONS:
+    for compression in ['NONE', 'SNAPPY', 'GZIP', 'LZ4', 'ZSTD']:
         if (compression != 'NONE' and
                 not pa.lib.Codec.is_available(compression)):
             continue
@@ -749,13 +768,6 @@ def test_pandas_parquet_configuration_options(tempdir, use_legacy_dataset):
         tm.assert_frame_equal(df, df_read)
 
 
-# ARROW-9424: LZ4 support is currently disabled
-def test_lz4_compression_disabled():
-    table = pa.table([pa.array([1, 2, 3, 4, 5])], names=['f0'])
-    with pytest.raises(IOError):
-        pq.write_table(table, pa.BufferOutputStream(), compression='lz4')
-
-
 def make_sample_file(table_or_df):
     if isinstance(table_or_df, pa.Table):
         a_table = table_or_df
@@ -826,6 +838,11 @@ def test_compression_level(use_legacy_dataset):
                      compression_level=5,
                      use_legacy_dataset=use_legacy_dataset)
 
+    # Check that the user can provide a compression per column
+    _check_roundtrip(table, expected=table,
+                     compression={'a': "gzip", 'b': "snappy"},
+                     use_legacy_dataset=use_legacy_dataset)
+
     # Check that the user can provide a compression level per column
     _check_roundtrip(table, expected=table, compression="gzip",
                      compression_level={'a': 2, 'b': 3},
@@ -837,9 +854,8 @@ def test_compression_level(use_legacy_dataset):
     # level.
     # GZIP (zlib) allows for specifying a compression level but as of up
     # to version 1.2.11 the valid range is [-1, 9].
-    invalid_combinations = [("snappy", 4), ("gzip", -1337),
+    invalid_combinations = [("snappy", 4), ("lz4", 5), ("gzip", -1337),
                             ("None", 444), ("lzo", 14)]
-    # ARROW-9424: lz4 is disabled for now ("lz4", 5),
     buf = io.BytesIO()
     for (codec, level) in invalid_combinations:
         with pytest.raises((ValueError, OSError)):
@@ -1702,13 +1718,13 @@ def test_partition_set_dictionary_type():
 @pytest.mark.pandas
 @parametrize_legacy_dataset
 def test_read_partitioned_directory(tempdir, use_legacy_dataset):
-    fs = LocalFileSystem.get_instance()
+    fs = LocalFileSystem._get_instance()
     _partition_test_for_filesystem(fs, tempdir, use_legacy_dataset)
 
 
 @pytest.mark.pandas
 def test_create_parquet_dataset_multi_threaded(tempdir):
-    fs = LocalFileSystem.get_instance()
+    fs = LocalFileSystem._get_instance()
     base_path = tempdir
 
     _partition_test_for_filesystem(fs, base_path)
@@ -1728,7 +1744,7 @@ def test_create_parquet_dataset_multi_threaded(tempdir):
 def test_read_partitioned_columns_selection(tempdir, use_legacy_dataset):
     # ARROW-3861 - do not include partition columns in resulting table when
     # `columns` keyword was passed without those columns
-    fs = LocalFileSystem.get_instance()
+    fs = LocalFileSystem._get_instance()
     base_path = tempdir
     _partition_test_for_filesystem(fs, base_path)
 
@@ -1747,7 +1763,7 @@ def test_read_partitioned_columns_selection(tempdir, use_legacy_dataset):
 @pytest.mark.pandas
 @parametrize_legacy_dataset
 def test_filters_equivalency(tempdir, use_legacy_dataset):
-    fs = LocalFileSystem.get_instance()
+    fs = LocalFileSystem._get_instance()
     base_path = tempdir
 
     integer_keys = [0, 1]
@@ -1835,7 +1851,7 @@ def test_filters_equivalency(tempdir, use_legacy_dataset):
 @pytest.mark.pandas
 @parametrize_legacy_dataset
 def test_filters_cutoff_exclusive_integer(tempdir, use_legacy_dataset):
-    fs = LocalFileSystem.get_instance()
+    fs = LocalFileSystem._get_instance()
     base_path = tempdir
 
     integer_keys = [0, 1, 2, 3, 4]
@@ -1877,7 +1893,7 @@ def test_filters_cutoff_exclusive_integer(tempdir, use_legacy_dataset):
     reason='Loss of type information in creation of categoricals.'
 )
 def test_filters_cutoff_exclusive_datetime(tempdir, use_legacy_dataset):
-    fs = LocalFileSystem.get_instance()
+    fs = LocalFileSystem._get_instance()
     base_path = tempdir
 
     date_keys = [
@@ -1922,7 +1938,7 @@ def test_filters_cutoff_exclusive_datetime(tempdir, use_legacy_dataset):
 @pytest.mark.pandas
 @parametrize_legacy_dataset
 def test_filters_inclusive_integer(tempdir, use_legacy_dataset):
-    fs = LocalFileSystem.get_instance()
+    fs = LocalFileSystem._get_instance()
     base_path = tempdir
 
     integer_keys = [0, 1, 2, 3, 4]
@@ -1958,7 +1974,7 @@ def test_filters_inclusive_integer(tempdir, use_legacy_dataset):
 @pytest.mark.pandas
 @parametrize_legacy_dataset
 def test_filters_inclusive_set(tempdir, use_legacy_dataset):
-    fs = LocalFileSystem.get_instance()
+    fs = LocalFileSystem._get_instance()
     base_path = tempdir
 
     integer_keys = [0, 1]
@@ -1996,7 +2012,7 @@ def test_filters_inclusive_set(tempdir, use_legacy_dataset):
 @pytest.mark.pandas
 @parametrize_legacy_dataset
 def test_filters_invalid_pred_op(tempdir, use_legacy_dataset):
-    fs = LocalFileSystem.get_instance()
+    fs = LocalFileSystem._get_instance()
     base_path = tempdir
 
     integer_keys = [0, 1, 2, 3, 4]
@@ -2044,7 +2060,7 @@ def test_filters_invalid_pred_op(tempdir, use_legacy_dataset):
 def test_filters_invalid_column(tempdir, use_legacy_dataset):
     # ARROW-5572 - raise error on invalid name in filter specification
     # works with new dataset / xfail with legacy implementation
-    fs = LocalFileSystem.get_instance()
+    fs = LocalFileSystem._get_instance()
     base_path = tempdir
 
     integer_keys = [0, 1, 2, 3, 4]
@@ -2069,7 +2085,7 @@ def test_filters_invalid_column(tempdir, use_legacy_dataset):
 @parametrize_legacy_dataset
 def test_filters_read_table(tempdir, use_legacy_dataset):
     # test that filters keyword is passed through in read_table
-    fs = LocalFileSystem.get_instance()
+    fs = LocalFileSystem._get_instance()
     base_path = tempdir
 
     integer_keys = [0, 1, 2, 3, 4]
@@ -2106,7 +2122,7 @@ def test_filters_read_table(tempdir, use_legacy_dataset):
 def test_partition_keys_with_underscores(tempdir, use_legacy_dataset):
     # ARROW-5666 - partition field values with underscores preserve underscores
     # xfail with legacy dataset -> they get interpreted as integers
-    fs = LocalFileSystem.get_instance()
+    fs = LocalFileSystem._get_instance()
     base_path = tempdir
 
     string_keys = ["2019_2", "2019_3"]
@@ -2152,7 +2168,7 @@ def s3_bucket(request, s3_connection, s3_server):
 
 
 @pytest.fixture
-def s3_example(s3_connection, s3_server, s3_bucket):
+def s3_example_s3fs(s3_connection, s3_server, s3_bucket):
     s3fs = pytest.importorskip('s3fs')
 
     host, port, access_key, secret_key = s3_connection
@@ -2164,31 +2180,70 @@ def s3_example(s3_connection, s3_server, s3_bucket):
         }
     )
 
-    test_dir = guid()
-    bucket_uri = 's3://{}/{}'.format(s3_bucket, test_dir)
+    test_path = '{}/{}'.format(s3_bucket, guid())
+
+    fs.mkdir(test_path)
+    yield fs, test_path
+    try:
+        fs.rm(test_path, recursive=True)
+    except FileNotFoundError:
+        pass
+
+
+@parametrize_legacy_dataset
+def test_read_s3fs(s3_example_s3fs, use_legacy_dataset):
+    fs, path = s3_example_s3fs
+    path = path + "/test.parquet"
+    table = pa.table({"a": [1, 2, 3]})
+    _write_table(table, path, filesystem=fs)
+
+    result = _read_table(
+        path, filesystem=fs, use_legacy_dataset=use_legacy_dataset
+    )
+    assert result.equals(table)
+
 
-    fs.mkdir(bucket_uri)
-    yield fs, bucket_uri
-    fs.rm(bucket_uri, recursive=True)
+@parametrize_legacy_dataset
+def test_read_directory_s3fs(s3_example_s3fs, use_legacy_dataset):
+    fs, directory = s3_example_s3fs
+    path = directory + "/test.parquet"
+    table = pa.table({"a": [1, 2, 3]})
+    _write_table(table, path, filesystem=fs)
+
+    result = _read_table(
+        directory, filesystem=fs, use_legacy_dataset=use_legacy_dataset
+    )
+    assert result.equals(table)
 
 
 @pytest.mark.pandas
 @pytest.mark.s3
 @parametrize_legacy_dataset
-def test_read_partitioned_directory_s3fs(s3_example, use_legacy_dataset):
+def test_read_partitioned_directory_s3fs_wrapper(
+    s3_example_s3fs, use_legacy_dataset
+):
     from pyarrow.filesystem import S3FSWrapper
 
-    fs, bucket_uri = s3_example
+    fs, path = s3_example_s3fs
     wrapper = S3FSWrapper(fs)
-    _partition_test_for_filesystem(wrapper, bucket_uri)
+    _partition_test_for_filesystem(wrapper, path)
 
     # Check that we can auto-wrap
     dataset = pq.ParquetDataset(
-        bucket_uri, filesystem=fs, use_legacy_dataset=use_legacy_dataset
+        path, filesystem=fs, use_legacy_dataset=use_legacy_dataset
     )
     dataset.read()
 
 
+@pytest.mark.pandas
+@parametrize_legacy_dataset
+def test_read_partitioned_directory_s3fs(s3_example_s3fs, use_legacy_dataset):
+    fs, path = s3_example_s3fs
+    _partition_test_for_filesystem(
+        fs, path, use_legacy_dataset=use_legacy_dataset
+    )
+
+
 def _partition_test_for_filesystem(fs, base_path, use_legacy_dataset=True):
     foo_keys = [0, 1]
     bar_keys = ['a', 'b', 'c']
@@ -2218,9 +2273,7 @@ def _partition_test_for_filesystem(fs, base_path, use_legacy_dataset=True):
                    .reset_index(drop=True)
                    .reindex(columns=result_df.columns))
 
-    if use_legacy_dataset:
-        # integer partition field not dictionary encoded with new API
-        expected_df['foo'] = pd.Categorical(df['foo'], categories=foo_keys)
+    expected_df['foo'] = pd.Categorical(df['foo'], categories=foo_keys)
     expected_df['bar'] = pd.Categorical(df['bar'], categories=bar_keys)
 
     assert (result_df.columns == ['index', 'values', 'foo', 'bar']).all()
@@ -2234,32 +2287,34 @@ def _generate_partition_directories(fs, base_dir, partition_spec, df):
     # part_table : a pyarrow.Table to write to each partition
     DEPTH = len(partition_spec)
 
+    pathsep = getattr(fs, "pathsep", getattr(fs, "sep", "/"))
+
     def _visit_level(base_dir, level, part_keys):
         name, values = partition_spec[level]
         for value in values:
             this_part_keys = part_keys + [(name, value)]
 
-            level_dir = fs._path_join(
+            level_dir = pathsep.join([
                 str(base_dir),
                 '{}={}'.format(name, value)
-            )
+            ])
             fs.mkdir(level_dir)
 
             if level == DEPTH - 1:
                 # Generate example data
-                file_path = fs._path_join(level_dir, guid())
+                file_path = pathsep.join([level_dir, guid()])
                 filtered_df = _filter_partition(df, this_part_keys)
                 part_table = pa.Table.from_pandas(filtered_df)
                 with fs.open(file_path, 'wb') as f:
                     _write_table(part_table, f)
                 assert fs.exists(file_path)
 
-                file_success = fs._path_join(level_dir, '_SUCCESS')
+                file_success = pathsep.join([level_dir, '_SUCCESS'])
                 with fs.open(file_success, 'wb') as f:
                     pass
             else:
                 _visit_level(level_dir, level + 1, this_part_keys)
-                file_success = fs._path_join(level_dir, '_SUCCESS')
+                file_success = pathsep.join([level_dir, '_SUCCESS'])
                 with fs.open(file_success, 'wb') as f:
                     pass
 
@@ -2299,13 +2354,13 @@ def _test_read_common_metadata_files(fs, base_path):
 
 @pytest.mark.pandas
 def test_read_common_metadata_files(tempdir):
-    fs = LocalFileSystem.get_instance()
+    fs = LocalFileSystem._get_instance()
     _test_read_common_metadata_files(fs, tempdir)
 
 
 @pytest.mark.pandas
 def test_read_metadata_files(tempdir):
-    fs = LocalFileSystem.get_instance()
+    fs = LocalFileSystem._get_instance()
 
     N = 100
     df = pd.DataFrame({
@@ -2414,7 +2469,7 @@ def read_multiple_files(paths, columns=None, use_threads=True, **kwargs):
         result2 = read_multiple_files(paths, metadata=metadata)
         assert result2.equals(expected)
 
-        result3 = pa.localfs.read_parquet(dirpath, schema=metadata.schema)
+        result3 = pq.ParquetDataset(dirpath, schema=metadata.schema).read()
         assert result3.equals(expected)
     else:
         with pytest.raises(ValueError, match="no longer supported"):
@@ -2424,14 +2479,18 @@ def read_multiple_files(paths, columns=None, use_threads=True, **kwargs):
     to_read = [0, 2, 6, result.num_columns - 1]
 
     col_names = [result.field(i).name for i in to_read]
-    out = pa.localfs.read_parquet(dirpath, columns=col_names)
+    out = pq.read_table(
+        dirpath, columns=col_names, use_legacy_dataset=use_legacy_dataset
+    )
     expected = pa.Table.from_arrays([result.column(i) for i in to_read],
                                     names=col_names,
                                     metadata=result.schema.metadata)
     assert out.equals(expected)
 
     # Read with multiple threads
-    pa.localfs.read_parquet(dirpath, use_threads=True)
+    pq.read_table(
+        dirpath, use_threads=True, use_legacy_dataset=use_legacy_dataset
+    )
 
     # Test failure modes with non-uniform metadata
     bad_apple = _test_dataframe(size, seed=i).iloc[:, :4]
@@ -2691,6 +2750,7 @@ def test_ignore_custom_prefixes(tempdir, use_legacy_dataset):
         pa.array(part).dictionary_encode(),
     ], names=['index', '_part'])
 
+    # TODO use_legacy_dataset ARROW-10247
     pq.write_to_dataset(table, str(tempdir), partition_cols=['_part'])
 
     private_duplicate = tempdir / '_private_duplicate'
@@ -2827,9 +2887,10 @@ def _test_write_to_dataset_with_partitions(base_path,
     output_table = pa.Table.from_pandas(output_df, schema=schema, safe=False,
                                         preserve_index=False)
     pq.write_to_dataset(output_table, base_path, partition_by,
-                        filesystem=filesystem)
+                        filesystem=filesystem,
+                        use_legacy_dataset=use_legacy_dataset)
 
-    metadata_path = os.path.join(base_path, '_common_metadata')
+    metadata_path = os.path.join(str(base_path), '_common_metadata')
 
     if filesystem is not None:
         with filesystem.open(metadata_path, 'wb') as f:
@@ -2880,14 +2941,14 @@ def _test_write_to_dataset_no_partitions(base_path,
     output_table = pa.Table.from_pandas(output_df)
 
     if filesystem is None:
-        filesystem = LocalFileSystem.get_instance()
+        filesystem = LocalFileSystem._get_instance()
 
     # Without partitions, append files to root_path
     n = 5
     for i in range(n):
         pq.write_to_dataset(output_table, base_path,
                             filesystem=filesystem)
-    output_files = [file for file in filesystem.ls(base_path)
+    output_files = [file for file in filesystem.ls(str(base_path))
                     if file.endswith(".parquet")]
     assert len(output_files) == n
 
@@ -2935,11 +2996,64 @@ def test_write_to_dataset_with_partitions_and_index_name(
 @pytest.mark.pandas
 @parametrize_legacy_dataset
 def test_write_to_dataset_no_partitions(tempdir, use_legacy_dataset):
-    _test_write_to_dataset_no_partitions(str(tempdir))
+    _test_write_to_dataset_no_partitions(str(tempdir), use_legacy_dataset)
+
+
+@pytest.mark.pandas
+@parametrize_legacy_dataset
+def test_write_to_dataset_pathlib(tempdir, use_legacy_dataset):
+    _test_write_to_dataset_with_partitions(
+        tempdir / "test1", use_legacy_dataset)
+    _test_write_to_dataset_no_partitions(
+        tempdir / "test2", use_legacy_dataset)
+
+
+# Those tests are failing - see ARROW-10370
+# @pytest.mark.pandas
+# @pytest.mark.s3
+# @parametrize_legacy_dataset
+# def test_write_to_dataset_pathlib_nonlocal(
+#     tempdir, s3_example_s3fs, use_legacy_dataset
+# ):
+#    # pathlib paths are only accepted for local files
+#    fs, _ = s3_example_s3fs
+
+#    with pytest.raises(TypeError, match="path-like objects are only allowed"):
+#         _test_write_to_dataset_with_partitions(
+#             tempdir / "test1", use_legacy_dataset, filesystem=fs)
+
+#    with pytest.raises(TypeError, match="path-like objects are only allowed"):
+#         _test_write_to_dataset_no_partitions(
+#             tempdir / "test2", use_legacy_dataset, filesystem=fs)
+
+
+@pytest.mark.pandas
+@parametrize_legacy_dataset
+def test_write_to_dataset_with_partitions_s3fs(
+    s3_example_s3fs, use_legacy_dataset
+):
+    fs, path = s3_example_s3fs
+
+    _test_write_to_dataset_with_partitions(
+        path, use_legacy_dataset, filesystem=fs)
+
+
+@pytest.mark.pandas
+@parametrize_legacy_dataset
+def test_write_to_dataset_no_partitions_s3fs(
+    s3_example_s3fs, use_legacy_dataset
+):
+    fs, path = s3_example_s3fs
+
+    _test_write_to_dataset_no_partitions(
+        path, use_legacy_dataset, filesystem=fs)
 
 
 @pytest.mark.pandas
-def test_write_to_dataset_with_partitions_and_custom_filenames(tempdir):
+@parametrize_legacy_dataset_not_supported
+def test_write_to_dataset_with_partitions_and_custom_filenames(
+    tempdir, use_legacy_dataset
+):
     output_df = pd.DataFrame({'group1': list('aaabbbbccc'),
                               'group2': list('eefeffgeee'),
                               'num': list(range(10)),
@@ -2954,7 +3068,8 @@ def partition_filename_callback(keys):
         return "{}-{}.parquet".format(*keys)
 
     pq.write_to_dataset(output_table, path,
-                        partition_by, partition_filename_callback)
+                        partition_by, partition_filename_callback,
+                        use_legacy_dataset=use_legacy_dataset)
 
     dataset = pq.ParquetDataset(path)
 
@@ -2983,13 +3098,18 @@ def test_write_to_dataset_pandas_preserve_extensiondtypes(
     df['col'] = df['col'].astype("Int64")
     table = pa.table(df)
 
-    pq.write_to_dataset(table, str(tempdir / "case1"), partition_cols=['part'])
+    pq.write_to_dataset(
+        table, str(tempdir / "case1"), partition_cols=['part'],
+        use_legacy_dataset=use_legacy_dataset
+    )
     result = pq.read_table(
         str(tempdir / "case1"), use_legacy_dataset=use_legacy_dataset
     ).to_pandas()
     tm.assert_frame_equal(result[["col"]], df[["col"]])
 
-    pq.write_to_dataset(table, str(tempdir / "case2"))
+    pq.write_to_dataset(
+        table, str(tempdir / "case2"), use_legacy_dataset=use_legacy_dataset
+    )
     result = pq.read_table(
         str(tempdir / "case2"), use_legacy_dataset=use_legacy_dataset
     ).to_pandas()
@@ -3013,13 +3133,18 @@ def test_write_to_dataset_pandas_preserve_index(tempdir, use_legacy_dataset):
     df_cat = df[["col", "part"]].copy()
     df_cat["part"] = df_cat["part"].astype("category")
 
-    pq.write_to_dataset(table, str(tempdir / "case1"), partition_cols=['part'])
+    pq.write_to_dataset(
+        table, str(tempdir / "case1"), partition_cols=['part'],
+        use_legacy_dataset=use_legacy_dataset
+    )
     result = pq.read_table(
         str(tempdir / "case1"), use_legacy_dataset=use_legacy_dataset
     ).to_pandas()
     tm.assert_frame_equal(result, df_cat)
 
-    pq.write_to_dataset(table, str(tempdir / "case2"))
+    pq.write_to_dataset(
+        table, str(tempdir / "case2"), use_legacy_dataset=use_legacy_dataset
+    )
     result = pq.read_table(
         str(tempdir / "case2"), use_legacy_dataset=use_legacy_dataset
     ).to_pandas()
@@ -3276,7 +3401,7 @@ def test_backwards_compatible_column_metadata_handling(
 # TODO(dataset) support pickling
 def _make_dataset_for_pickling(tempdir, N=100):
     path = tempdir / 'data.parquet'
-    fs = LocalFileSystem.get_instance()
+    fs = LocalFileSystem._get_instance()
 
     df = pd.DataFrame({
         'index': np.arange(N),
@@ -3510,6 +3635,104 @@ def test_parquet_file_pass_directory_instead_of_file(tempdir):
         pq.ParquetFile(path)
 
 
+@pytest.mark.pandas
+@pytest.mark.parametrize("filesystem", [
+    None,
+    LocalFileSystem._get_instance(),
+    fs.LocalFileSystem(),
+])
+def test_parquet_writer_filesystem_local(tempdir, filesystem):
+    df = _test_dataframe(100)
+    table = pa.Table.from_pandas(df, preserve_index=False)
+    path = str(tempdir / 'data.parquet')
+
+    with pq.ParquetWriter(
+        path, table.schema, filesystem=filesystem, version='2.0'
+    ) as writer:
+        writer.write_table(table)
+
+    result = _read_table(path).to_pandas()
+    tm.assert_frame_equal(result, df)
+
+
+@pytest.fixture
+def s3_example_fs(s3_connection, s3_server):
+    from pyarrow.fs import FileSystem
+
+    host, port, access_key, secret_key = s3_connection
+    uri = (
+        "s3://{}:{}@mybucket/data.parquet?scheme=http&endpoint_override={}:{}"
+        .format(access_key, secret_key, host, port)
+    )
+    fs, path = FileSystem.from_uri(uri)
+
+    fs.create_dir("mybucket")
+
+    yield fs, uri, path
+
+
+@pytest.mark.pandas
+@pytest.mark.s3
+def test_parquet_writer_filesystem_s3(s3_example_fs):
+    df = _test_dataframe(100)
+    table = pa.Table.from_pandas(df, preserve_index=False)
+
+    fs, uri, path = s3_example_fs
+
+    with pq.ParquetWriter(
+        path, table.schema, filesystem=fs, version='2.0'
+    ) as writer:
+        writer.write_table(table)
+
+    result = _read_table(uri).to_pandas()
+    tm.assert_frame_equal(result, df)
+
+
+@pytest.mark.pandas
+@pytest.mark.s3
+def test_parquet_writer_filesystem_s3_uri(s3_example_fs):
+    df = _test_dataframe(100)
+    table = pa.Table.from_pandas(df, preserve_index=False)
+
+    fs, uri, path = s3_example_fs
+
+    with pq.ParquetWriter(uri, table.schema, version='2.0') as writer:
+        writer.write_table(table)
+
+    result = _read_table(path, filesystem=fs).to_pandas()
+    tm.assert_frame_equal(result, df)
+
+
+@pytest.mark.pandas
+def test_parquet_writer_filesystem_s3fs(s3_example_s3fs):
+    df = _test_dataframe(100)
+    table = pa.Table.from_pandas(df, preserve_index=False)
+
+    fs, directory = s3_example_s3fs
+    path = directory + "/test.parquet"
+
+    with pq.ParquetWriter(
+        path, table.schema, filesystem=fs, version='2.0'
+    ) as writer:
+        writer.write_table(table)
+
+    result = _read_table(path, filesystem=fs).to_pandas()
+    tm.assert_frame_equal(result, df)
+
+
+@pytest.mark.pandas
+def test_parquet_writer_filesystem_buffer_raises():
+    df = _test_dataframe(100)
+    table = pa.Table.from_pandas(df, preserve_index=False)
+    filesystem = fs.LocalFileSystem()
+
+    # Should raise ValueError when filesystem is passed with file-like object
+    with pytest.raises(ValueError, match="specified path is file-like"):
+        pq.ParquetWriter(
+            pa.BufferOutputStream(), table.schema, filesystem=filesystem
+        )
+
+
 @pytest.mark.pandas
 @parametrize_legacy_dataset
 def test_parquet_writer_with_caller_provided_filesystem(use_legacy_dataset):
@@ -3652,6 +3875,7 @@ def test_dataset_read_dictionary(tempdir, use_legacy_dataset):
     path = tempdir / "ARROW-3325-dataset"
     t1 = pa.table([[util.rands(10) for i in range(5)] * 10], names=['f0'])
     t2 = pa.table([[util.rands(10) for i in range(5)] * 10], names=['f0'])
+    # TODO pass use_legacy_dataset (need to fix unique names)
     pq.write_to_dataset(t1, root_path=str(path))
     pq.write_to_dataset(t2, root_path=str(path))
 
@@ -3706,7 +3930,8 @@ def test_direct_read_dictionary_subfield(use_legacy_dataset):
 
 
 @pytest.mark.pandas
-def test_write_to_dataset_metadata(tempdir):
+@parametrize_legacy_dataset
+def test_write_to_dataset_metadata(tempdir, use_legacy_dataset):
     path = tempdir / "ARROW-1983-dataset"
 
     # create and write a test dataset
@@ -3718,9 +3943,18 @@ def test_write_to_dataset_metadata(tempdir):
     table = pa.Table.from_pandas(df)
 
     metadata_list = []
+    if not use_legacy_dataset:
+        # New dataset implementation does not yet support metadata_collector
+        with pytest.raises(ValueError):
+            pq.write_to_dataset(table, root_path=str(path),
+                                partition_cols=['one', 'two'],
+                                metadata_collector=metadata_list,
+                                use_legacy_dataset=use_legacy_dataset)
+        return
     pq.write_to_dataset(table, root_path=str(path),
                         partition_cols=['one', 'two'],
-                        metadata_collector=metadata_list)
+                        metadata_collector=metadata_list,
+                        use_legacy_dataset=use_legacy_dataset)
 
     # open the dataset and collect metadata from pieces:
     dataset = pq.ParquetDataset(path)
@@ -4045,6 +4279,7 @@ def test_filter_before_validate_schema(tempdir, use_legacy_dataset):
 @pytest.mark.pandas
 @pytest.mark.fastparquet
 @pytest.mark.filterwarnings("ignore:RangeIndex:FutureWarning")
+@pytest.mark.filterwarnings("ignore:tostring:DeprecationWarning:fastparquet")
 def test_fastparquet_cross_compatibility(tempdir):
     fp = pytest.importorskip('fastparquet')
 
@@ -4174,3 +4409,33 @@ def test_dataset_partitioning(tempdir):
     with pytest.raises(ValueError):
         pq.ParquetDataset(
             str(root_path), partitioning=part, use_legacy_dataset=True)
+
+
+@pytest.mark.dataset
+def test_parquet_dataset_new_filesystem(tempdir):
+    # Ensure we can pass new FileSystem object to ParquetDataset
+    # (use new implementation automatically without specifying
+    #  use_legacy_dataset=False)
+    table = pa.table({'a': [1, 2, 3]})
+    pq.write_table(table, tempdir / 'data.parquet')
+    # don't use simple LocalFileSystem (as that gets mapped to legacy one)
+    filesystem = fs.SubTreeFileSystem(str(tempdir), fs.LocalFileSystem())
+    dataset = pq.ParquetDataset('.', filesystem=filesystem)
+    result = dataset.read()
+    assert result.equals(table)
+
+
+def test_parquet_dataset_partitions_piece_path_with_fsspec(tempdir):
+    # ARROW-10462 ensure that on Windows we properly use posix-style paths
+    # as used by fsspec
+    fsspec = pytest.importorskip("fsspec")
+    filesystem = fsspec.filesystem('file')
+    table = pa.table({'a': [1, 2, 3]})
+    pq.write_table(table, tempdir / 'data.parquet')
+
+    # pass a posix-style path (using "/" also on Windows)
+    path = str(tempdir).replace("\\", "/")
+    dataset = pq.ParquetDataset(path, filesystem=filesystem)
+    # ensure the piece path is also posix-style
+    expected = path + "/data.parquet"
+    assert dataset.pieces[0].path == expected
diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py
index c5747359a5e..3c00c29bb36 100644
--- a/python/pyarrow/tests/test_plasma.py
+++ b/python/pyarrow/tests/test_plasma.py
@@ -307,6 +307,8 @@ def test_put_and_get(self):
             [result] = self.plasma_client.get([object_id], timeout_ms=0)
             assert result == pa.plasma.ObjectNotAvailable
 
+    @pytest.mark.filterwarnings(
+        "ignore:'pyarrow.deserialize':DeprecationWarning")
     def test_put_and_get_raw_buffer(self):
         temp_id = random_object_id()
         use_meta = b"RAW"
@@ -338,6 +340,8 @@ def deserialize_or_output(data_tuple):
             result = deserialize_or_output(result)
             assert result == pa.plasma.ObjectNotAvailable
 
+    @pytest.mark.filterwarnings(
+        "ignore:'serialization_context':DeprecationWarning")
     def test_put_and_get_serialization_context(self):
 
         class CustomType:
@@ -349,7 +353,7 @@ def __init__(self, val):
         with pytest.raises(pa.ArrowSerializationError):
             self.plasma_client.put(val)
 
-        serialization_context = pa.SerializationContext()
+        serialization_context = pa.lib.SerializationContext()
         serialization_context.register_type(CustomType, 20*"\x00")
 
         object_id = self.plasma_client.put(
diff --git a/python/pyarrow/tests/test_scalars.py b/python/pyarrow/tests/test_scalars.py
index 091ae38e6e4..f516afdf2fe 100644
--- a/python/pyarrow/tests/test_scalars.py
+++ b/python/pyarrow/tests/test_scalars.py
@@ -43,6 +43,8 @@
     (np.float16(1.0), pa.float16(), pa.HalfFloatScalar, pa.HalfFloatValue),
     (1.0, pa.float32(), pa.FloatScalar, pa.FloatValue),
     (decimal.Decimal("1.123"), None, pa.Decimal128Scalar, pa.Decimal128Value),
+    (decimal.Decimal("1.1234567890123456789012345678901234567890"),
+     None, pa.Decimal256Scalar, pa.Decimal256Value),
     ("string", None, pa.StringScalar, pa.StringValue),
     (b"bytes", None, pa.BinaryScalar, pa.BinaryValue),
     ("largestring", pa.large_string(), pa.LargeStringScalar,
@@ -176,7 +178,7 @@ def test_numerics():
     assert s.as_py() == 0.5
 
 
-def test_decimal():
+def test_decimal128():
     v = decimal.Decimal("1.123")
     s = pa.scalar(v)
     assert isinstance(s, pa.Decimal128Scalar)
@@ -194,6 +196,25 @@ def test_decimal():
     assert s.as_py() == v
 
 
+def test_decimal256():
+    v = decimal.Decimal("1234567890123456789012345678901234567890.123")
+    s = pa.scalar(v)
+    assert isinstance(s, pa.Decimal256Scalar)
+    assert s.as_py() == v
+    assert s.type == pa.decimal256(43, 3)
+
+    v = decimal.Decimal("1.1234")
+    with pytest.raises(pa.ArrowInvalid):
+        pa.scalar(v, type=pa.decimal256(4, scale=3))
+    # TODO: Add the following after implementing Decimal256 scaling.
+    # with pytest.raises(pa.ArrowInvalid):
+    #     pa.scalar(v, type=pa.decimal256(5, scale=3))
+
+    s = pa.scalar(v, type=pa.decimal256(5, scale=4))
+    assert isinstance(s, pa.Decimal256Scalar)
+    assert s.as_py() == v
+
+
 def test_date():
     # ARROW-5125
     d1 = datetime.date(3200, 1, 1)
@@ -531,27 +552,28 @@ def test_map():
 
 
 def test_dictionary():
-    indices = [2, 1, 2, 0]
-    dictionary = ['foo', 'bar', 'baz']
+    indices = pa.array([2, None, 1, 2, 0, None])
+    dictionary = pa.array(['foo', 'bar', 'baz'])
 
     arr = pa.DictionaryArray.from_arrays(indices, dictionary)
-    expected = ['baz', 'bar', 'baz', 'foo']
+    expected = ['baz', None, 'bar', 'baz', 'foo', None]
+    assert arr.to_pylist() == expected
 
     for j, (i, v) in enumerate(zip(indices, expected)):
         s = arr[j]
 
         assert s.as_py() == v
         assert s.value.as_py() == v
-        assert s.index.as_py() == i
-        assert s.dictionary.to_pylist() == dictionary
+        assert s.index.equals(i)
+        assert s.dictionary.equals(dictionary)
 
         with pytest.warns(FutureWarning):
-            assert s.index_value.as_py() == i
+            assert s.index_value.equals(i)
         with pytest.warns(FutureWarning):
             assert s.dictionary_value.as_py() == v
 
-    with pytest.raises(pa.ArrowNotImplementedError):
-        pickle.loads(pickle.dumps(s))
+        restored = pickle.loads(pickle.dumps(s))
+        assert restored.equals(s)
 
 
 def test_union():
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index 5adcabbc918..da67aaa19aa 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -70,6 +70,7 @@ def test_type_to_pandas_dtype():
         (pa.string(), np.object_),
         (pa.list_(pa.int8()), np.object_),
         # (pa.list_(pa.int8(), 2), np.object_),  # TODO needs pandas conversion
+        (pa.map_(pa.int64(), pa.float64()), np.object_),
     ]
     for arrow_type, numpy_type in cases:
         assert arrow_type.to_pandas_dtype() == numpy_type
@@ -603,6 +604,7 @@ def test_type_schema_pickling():
         pa.timestamp('ms'),
         pa.timestamp('ns'),
         pa.decimal128(12, 2),
+        pa.decimal256(76, 38),
         pa.field('a', 'string', metadata={b'foo': b'bar'})
     ]
 
@@ -623,15 +625,22 @@ def test_type_schema_pickling():
 
 
 def test_empty_table():
-    schema = pa.schema([
+    schema1 = pa.schema([
         pa.field('f0', pa.int64()),
         pa.field('f1', pa.dictionary(pa.int32(), pa.string())),
         pa.field('f2', pa.list_(pa.list_(pa.int64()))),
     ])
-    table = schema.empty_table()
-    assert isinstance(table, pa.Table)
-    assert table.num_rows == 0
-    assert table.schema == schema
+    # test it preserves field nullability
+    schema2 = pa.schema([
+        pa.field('a', pa.int64(), nullable=False),
+        pa.field('b', pa.int64())
+    ])
+
+    for schema in [schema1, schema2]:
+        table = schema.empty_table()
+        assert isinstance(table, pa.Table)
+        assert table.num_rows == 0
+        assert table.schema == schema
 
 
 @pytest.mark.pandas
@@ -700,3 +709,14 @@ def test_schema_merge():
 
     with pytest.raises(pa.ArrowInvalid):
         pa.unify_schemas([b, d])
+
+
+def test_undecodable_metadata():
+    # ARROW-10214: undecodable metadata shouldn't fail repr()
+    data1 = b'abcdef\xff\x00'
+    data2 = b'ghijkl\xff\x00'
+    schema = pa.schema(
+        [pa.field('ints', pa.int16(), metadata={'key': data1})],
+        metadata={'key': data2})
+    assert 'abcdef' in str(schema)
+    assert 'ghijkl' in str(schema)
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index f1e0cf42563..456f42c92fa 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -52,6 +52,11 @@
     sparse = None
 
 
+# ignore all serialization deprecation warnings in this file, we test that the
+# warnings are actually raised in test_serialization_deprecated.py
+pytestmark = pytest.mark.filterwarnings("ignore:'pyarrow:DeprecationWarning")
+
+
 def assert_equal(obj1, obj2):
     if torch is not None and torch.is_tensor(obj1) and torch.is_tensor(obj2):
         if obj1.is_sparse:
@@ -233,7 +238,8 @@ class CustomError(Exception):
 
 
 def make_serialization_context():
-    context = pa.default_serialization_context()
+    with pytest.warns(DeprecationWarning):
+        context = pa.default_serialization_context()
 
     context.register_type(Foo, "Foo")
     context.register_type(Bar, "Bar")
diff --git a/python/pyarrow/tests/test_serialization_deprecated.py b/python/pyarrow/tests/test_serialization_deprecated.py
new file mode 100644
index 00000000000..f818d56b4d3
--- /dev/null
+++ b/python/pyarrow/tests/test_serialization_deprecated.py
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sys
+
+import pytest
+
+import pyarrow as pa
+
+
+def test_serialization_deprecated():
+    with pytest.warns(DeprecationWarning):
+        ser = pa.serialize(1)
+
+    with pytest.warns(DeprecationWarning):
+        pa.deserialize(ser.to_buffer())
+
+    f = pa.BufferOutputStream()
+    with pytest.warns(DeprecationWarning):
+        pa.serialize_to(12, f)
+
+    buf = f.getvalue()
+    f = pa.BufferReader(buf)
+    with pytest.warns(DeprecationWarning):
+        pa.read_serialized(f).deserialize()
+
+    with pytest.warns(DeprecationWarning):
+        pa.default_serialization_context()
+
+    context = pa.lib.SerializationContext()
+    with pytest.warns(DeprecationWarning):
+        pa.register_default_serialization_handlers(context)
+
+
+@pytest.mark.skipif(sys.version_info < (3, 7),
+                    reason="getattr needs Python 3.7")
+def test_serialization_deprecated_toplevel():
+    with pytest.warns(DeprecationWarning):
+        pa.SerializedPyObject()
+
+    with pytest.warns(DeprecationWarning):
+        pa.SerializationContext()
diff --git a/python/pyarrow/tests/test_strategies.py b/python/pyarrow/tests/test_strategies.py
index f4249df2462..14fc949928c 100644
--- a/python/pyarrow/tests/test_strategies.py
+++ b/python/pyarrow/tests/test_strategies.py
@@ -41,6 +41,11 @@ def test_arrays(array):
     assert isinstance(array, pa.lib.Array)
 
 
+@h.given(past.arrays(past.primitive_types, nullable=False))
+def test_array_nullability(array):
+    assert array.null_count == 0
+
+
 @h.given(past.all_chunked_arrays)
 def test_chunked_arrays(chunked_array):
     assert isinstance(chunked_array, pa.lib.ChunkedArray)
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 4012a919548..5eb31fbb726 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -165,6 +165,8 @@ def test_chunked_array_getitem():
         data[6]
     with pytest.raises(IndexError):
         data[-7]
+    # Ensure this works with numpy scalars
+    assert data[np.int32(1)].as_py() == 2
 
     data_slice = data[2:4]
     assert data_slice.to_pylist() == [3, 4]
@@ -177,6 +179,30 @@ def test_chunked_array_getitem():
     assert data_slice.to_pylist() == []
 
 
+def test_chunked_array_slice():
+    data = [
+        pa.array([1, 2, 3]),
+        pa.array([4, 5, 6])
+    ]
+    data = pa.chunked_array(data)
+
+    data_slice = data.slice(len(data))
+    assert data_slice.type == data.type
+    assert data_slice.to_pylist() == []
+
+    data_slice = data.slice(len(data) + 10)
+    assert data_slice.type == data.type
+    assert data_slice.to_pylist() == []
+
+    table = pa.Table.from_arrays([data], names=["a"])
+    table_slice = table.slice(len(table))
+    assert len(table_slice) == 0
+
+    table = pa.Table.from_arrays([data], names=["a"])
+    table_slice = table.slice(len(table) + 10)
+    assert len(table_slice) == 0
+
+
 def test_chunked_array_iter():
     data = [
         pa.array([0]),
diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py
index 1f905f3be43..e5c11415c05 100644
--- a/python/pyarrow/tests/test_types.py
+++ b/python/pyarrow/tests/test_types.py
@@ -17,11 +17,15 @@
 
 from collections import OrderedDict
 from collections.abc import Iterator
+import datetime
+import sys
 
 import pickle
 import pytest
+import pytz
 import hypothesis as h
 import hypothesis.strategies as st
+import hypothesis.extra.pytz as tzst
 import weakref
 
 import numpy as np
@@ -49,6 +53,7 @@ def get_many_types():
         pa.float32(),
         pa.float64(),
         pa.decimal128(19, 4),
+        pa.decimal256(76, 38),
         pa.string(),
         pa.binary(),
         pa.binary(10),
@@ -120,8 +125,21 @@ def test_null_field_may_not_be_non_nullable():
 
 
 def test_is_decimal():
-    assert types.is_decimal(pa.decimal128(19, 4))
-    assert not types.is_decimal(pa.int32())
+    decimal128 = pa.decimal128(19, 4)
+    decimal256 = pa.decimal256(76, 38)
+    int32 = pa.int32()
+
+    assert types.is_decimal(decimal128)
+    assert types.is_decimal(decimal256)
+    assert not types.is_decimal(int32)
+
+    assert types.is_decimal128(decimal128)
+    assert not types.is_decimal128(decimal256)
+    assert not types.is_decimal128(int32)
+
+    assert not types.is_decimal256(decimal128)
+    assert types.is_decimal256(decimal256)
+    assert not types.is_decimal256(int32)
 
 
 def test_is_list():
@@ -252,6 +270,130 @@ def test_is_primitive():
     assert not types.is_primitive(pa.list_(pa.int32()))
 
 
+@pytest.mark.parametrize(('tz', 'expected'), [
+    (pytz.utc, 'UTC'),
+    (pytz.timezone('Europe/Paris'), 'Europe/Paris'),
+    # StaticTzInfo.tzname returns with '-09' so we need to infer the timezone's
+    # name from the tzinfo.zone attribute
+    (pytz.timezone('Etc/GMT-9'), 'Etc/GMT-9'),
+    (pytz.FixedOffset(180), '+03:00'),
+    (datetime.timezone.utc, 'UTC' if sys.version_info >= (3, 6) else '+00:00'),
+    (datetime.timezone(datetime.timedelta(hours=1, minutes=30)), '+01:30')
+])
+def test_tzinfo_to_string(tz, expected):
+    assert pa.lib.tzinfo_to_string(tz) == expected
+
+
+def test_tzinfo_to_string_errors():
+    msg = "Not an instance of datetime.tzinfo"
+    with pytest.raises(TypeError):
+        pa.lib.tzinfo_to_string("Europe/Budapest")
+
+    if sys.version_info >= (3, 8):
+        # before 3.8 it was only possible to create timezone objects with whole
+        # number of minutes
+        tz = datetime.timezone(datetime.timedelta(hours=1, seconds=30))
+        msg = "Offset must represent whole number of minutes"
+        with pytest.raises(ValueError, match=msg):
+            pa.lib.tzinfo_to_string(tz)
+
+
+@h.given(tzst.timezones())
+def test_pytz_timezone_roundtrip(tz):
+    timezone_string = pa.lib.tzinfo_to_string(tz)
+    timezone_tzinfo = pa.lib.string_to_tzinfo(timezone_string)
+    assert timezone_tzinfo == tz
+
+
+def test_convert_custom_tzinfo_objects_to_string():
+    class CorrectTimezone1(datetime.tzinfo):
+        """
+        Conversion is using utcoffset()
+        """
+
+        def tzname(self, dt):
+            return None
+
+        def utcoffset(self, dt):
+            return datetime.timedelta(hours=-3, minutes=30)
+
+    class CorrectTimezone2(datetime.tzinfo):
+        """
+        Conversion is using tzname()
+        """
+
+        def tzname(self, dt):
+            return "+03:00"
+
+        def utcoffset(self, dt):
+            return datetime.timedelta(hours=3)
+
+    class BuggyTimezone1(datetime.tzinfo):
+        """
+        Unable to infer name or offset
+        """
+
+        def tzname(self, dt):
+            return None
+
+        def utcoffset(self, dt):
+            return None
+
+    class BuggyTimezone2(datetime.tzinfo):
+        """
+        Wrong offset type
+        """
+
+        def tzname(self, dt):
+            return None
+
+        def utcoffset(self, dt):
+            return "one hour"
+
+    class BuggyTimezone3(datetime.tzinfo):
+        """
+        Wrong timezone name type
+        """
+
+        def tzname(self, dt):
+            return 240
+
+        def utcoffset(self, dt):
+            return None
+
+    assert pa.lib.tzinfo_to_string(CorrectTimezone1()) == "-02:30"
+    assert pa.lib.tzinfo_to_string(CorrectTimezone2()) == "+03:00"
+
+    msg = (r"Object returned by tzinfo.utcoffset\(None\) is not an instance "
+           r"of datetime.timedelta")
+    for wrong in [BuggyTimezone1(), BuggyTimezone2(), BuggyTimezone3()]:
+        with pytest.raises(ValueError, match=msg):
+            pa.lib.tzinfo_to_string(wrong)
+
+
+@pytest.mark.parametrize(('string', 'expected'), [
+    ('UTC', pytz.utc),
+    ('Europe/Paris', pytz.timezone('Europe/Paris')),
+    ('+03:00', pytz.FixedOffset(180)),
+    ('+01:30', pytz.FixedOffset(90)),
+    ('-02:00', pytz.FixedOffset(-120))
+])
+def test_string_to_tzinfo(string, expected):
+    result = pa.lib.string_to_tzinfo(string)
+    assert result == expected
+
+
+@pytest.mark.parametrize('tz,name', [
+    (pytz.FixedOffset(90), '+01:30'),
+    (pytz.FixedOffset(-90), '-01:30'),
+    (pytz.utc, 'UTC'),
+    (pytz.timezone('America/New_York'), 'America/New_York')
+])
+def test_timezone_string_roundtrip(tz, name):
+    assert pa.lib.tzinfo_to_string(tz) == name
+    assert pa.lib.string_to_tzinfo(name) == tz
+
+
 def test_timestamp():
     for unit in ('s', 'ms', 'us', 'ns'):
         for tz in (None, 'UTC', 'Europe/Paris'):
@@ -567,6 +709,7 @@ def test_bit_width():
                          (pa.uint32(), 32),
                          (pa.float16(), 16),
                          (pa.decimal128(19, 4), 128),
+                         (pa.decimal256(76, 38), 256),
                          (pa.binary(42), 42 * 8)]:
         assert ty.bit_width == expected
     for ty in [pa.binary(), pa.string(), pa.list_(pa.int16())]:
@@ -584,6 +727,10 @@ def test_decimal_properties():
     assert ty.byte_width == 16
     assert ty.precision == 19
     assert ty.scale == 4
+    ty = pa.decimal256(76, 38)
+    assert ty.byte_width == 32
+    assert ty.precision == 76
+    assert ty.scale == 38
 
 
 def test_decimal_overflow():
@@ -591,7 +738,13 @@ def test_decimal_overflow():
     pa.decimal128(38, 0)
     for i in (0, -1, 39):
         with pytest.raises(ValueError):
-            pa.decimal128(39, 0)
+            pa.decimal128(i, 0)
+
+    pa.decimal256(1, 0)
+    pa.decimal256(76, 0)
+    for i in (0, -1, 77):
+        with pytest.raises(ValueError):
+            pa.decimal256(i, 0)
 
 
 def test_type_equality_operators():
diff --git a/python/pyarrow/tests/util.py b/python/pyarrow/tests/util.py
index 50844d24d42..b0357de23ba 100644
--- a/python/pyarrow/tests/util.py
+++ b/python/pyarrow/tests/util.py
@@ -210,3 +210,13 @@ def changed_environ(name, value):
             del os.environ[name]
         else:
             os.environ[name] = orig_value
+
+
+@contextlib.contextmanager
+def change_cwd(path):
+    curdir = os.getcwd()
+    os.chdir(str(path))
+    try:
+        yield
+    finally:
+        os.chdir(curdir)
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index edd96227b6a..b337f2428c7 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -46,7 +46,8 @@ cdef dict _pandas_type_map = {
     _Type_FIXED_SIZE_BINARY: np.object_,
     _Type_STRING: np.object_,
     _Type_LIST: np.object_,
-    _Type_DECIMAL: np.object_,
+    _Type_MAP: np.object_,
+    _Type_DECIMAL128: np.object_,
 }
 
 cdef dict _pep3118_type_map = {
@@ -154,7 +155,7 @@ cdef class DataType(_Weakrefable):
         return self.type.layout().buffers.size()
 
     def __str__(self):
-        return frombytes(self.type.ToString())
+        return frombytes(self.type.ToString(), safe=True)
 
     def __hash__(self):
         return hash(str(self))
@@ -623,6 +624,33 @@ cdef class Decimal128Type(FixedSizeBinaryType):
         return self.decimal128_type.scale()
 
 
+cdef class Decimal256Type(FixedSizeBinaryType):
+    """
+    Concrete class for Decimal256 data types.
+    """
+
+    cdef void init(self, const shared_ptr[CDataType]& type) except *:
+        FixedSizeBinaryType.init(self, type)
+        self.decimal256_type = <const CDecimal256Type*> type.get()
+
+    def __reduce__(self):
+        return decimal256, (self.precision, self.scale)
+
+    @property
+    def precision(self):
+        """
+        The decimal precision, in number of decimal digits (an integer).
+        """
+        return self.decimal256_type.precision()
+
+    @property
+    def scale(self):
+        """
+        The decimal scale (an integer).
+        """
+        return self.decimal256_type.scale()
+
+
 cdef class BaseExtensionType(DataType):
     """
     Concrete base class for extension types.
@@ -697,6 +725,10 @@ cdef class ExtensionType(BaseExtensionType):
         else:
             return NotImplemented
 
+    def __repr__(self):
+        fmt = '{0.__class__.__name__}({1})'
+        return fmt.format(self, repr(self.storage_type))
+
     def __arrow_ext_serialize__(self):
         """
         Serialized representation of metadata to reconstruct the type object.
@@ -884,7 +916,7 @@ cdef class KeyValueMetadata(_Metadata, Mapping):
         return str(self)
 
     def __str__(self):
-        return frombytes(self.metadata.ToString())
+        return frombytes(self.metadata.ToString(), safe=True)
 
     def __eq__(self, other):
         try:
@@ -1008,7 +1040,8 @@ cdef class Field(_Weakrefable):
         return field, (self.name, self.type, self.nullable, self.metadata)
 
     def __str__(self):
-        return 'pyarrow.Field<{0}>'.format(frombytes(self.field.ToString()))
+        return 'pyarrow.Field<{0}>'.format(
+            frombytes(self.field.ToString(), safe=True))
 
     def __repr__(self):
         return self.__str__()
@@ -1254,16 +1287,8 @@ cdef class Schema(_Weakrefable):
         -------
         table: pyarrow.Table
         """
-        arrays = []
-        names = []
-        for field in self:
-            arrays.append(_empty_array(field.type))
-            names.append(field.name)
-        return Table.from_arrays(
-            arrays=arrays,
-            names=names,
-            metadata=self.metadata
-        )
+        arrays = [_empty_array(field.type) for field in self]
+        return Table.from_arrays(arrays, schema=self)
 
     def equals(self, Schema other not None, bint check_metadata=False):
         """
@@ -1506,7 +1531,7 @@ cdef class Schema(_Weakrefable):
 
         return pyarrow_wrap_schema(c_schema)
 
-    def serialize(self, DictionaryMemo dictionary_memo=None, memory_pool=None):
+    def serialize(self, memory_pool=None):
         """
         Write Schema to Buffer as encapsulated IPC message
 
@@ -1514,10 +1539,6 @@ cdef class Schema(_Weakrefable):
         ----------
         memory_pool : MemoryPool, default None
             Uses default memory pool if not specified
-        dictionary_memo : DictionaryMemo, optional
-            If schema contains dictionaries, must pass a
-            DictionaryMemo to be able to deserialize RecordBatch
-            objects
 
         Returns
         -------
@@ -1526,17 +1547,10 @@ cdef class Schema(_Weakrefable):
         cdef:
             shared_ptr[CBuffer] buffer
             CMemoryPool* pool = maybe_unbox_memory_pool(memory_pool)
-            CDictionaryMemo temp_memo
-            CDictionaryMemo* arg_dict_memo
-
-        if dictionary_memo is not None:
-            arg_dict_memo = dictionary_memo.memo
-        else:
-            arg_dict_memo = &temp_memo
 
         with nogil:
             buffer = GetResultValue(SerializeSchema(deref(self.schema),
-                                                    arg_dict_memo, pool))
+                                                    pool))
         return pyarrow_wrap_buffer(buffer)
 
     def remove_metadata(self):
@@ -1589,7 +1603,7 @@ cdef class Schema(_Weakrefable):
                 )
             )
 
-        return frombytes(result)
+        return frombytes(result, safe=True)
 
     def _export_to_c(self, uintptr_t out_ptr):
         """
@@ -1816,9 +1830,6 @@ cdef timeunit_to_string(TimeUnit unit):
         return 'ns'
 
 
-_FIXED_OFFSET_RE = re.compile(r'([+-])(0[0-9]|1[0-9]|2[0-3]):([0-5][0-9])$')
-
-
 def tzinfo_to_string(tz):
     """
     Converts a time zone object into a string indicating the name of a time
@@ -1837,32 +1848,7 @@ def tzinfo_to_string(tz):
       name : str
         Time zone name
     """
-    import pytz
-    import datetime
-
-    def fixed_offset_to_string(offset):
-        seconds = int(offset.utcoffset(None).total_seconds())
-        sign = '+' if seconds >= 0 else '-'
-        minutes, seconds = divmod(abs(seconds), 60)
-        hours, minutes = divmod(minutes, 60)
-        if seconds > 0:
-            raise ValueError('Offset must represent whole number of minutes')
-        return '{}{:02d}:{:02d}'.format(sign, hours, minutes)
-
-    if tz is pytz.utc:
-        return tz.zone  # ARROW-4055
-    elif isinstance(tz, pytz.tzinfo.BaseTzInfo):
-        return tz.zone
-    elif isinstance(tz, pytz._FixedOffset):
-        return fixed_offset_to_string(tz)
-    elif isinstance(tz, datetime.tzinfo):
-        if isinstance(tz, datetime.timezone):
-            return fixed_offset_to_string(tz)
-        else:
-            raise ValueError('Unable to convert timezone `{}` to string'
-                             .format(tz))
-    else:
-        raise TypeError('Must be an instance of `datetime.tzinfo`')
+    return frombytes(GetResultValue(TzinfoToString(<PyObject*>tz)))
 
 
 def string_to_tzinfo(name):
@@ -1884,14 +1870,8 @@ def string_to_tzinfo(name):
       tz : datetime.tzinfo
         Time zone object
     """
-    import pytz
-    m = _FIXED_OFFSET_RE.match(name)
-    if m:
-        sign = 1 if m.group(1) == '+' else -1
-        hours, minutes = map(int, m.group(2, 3))
-        return pytz.FixedOffset(sign * (hours * 60 + minutes))
-    else:
-        return pytz.timezone(name)
+    cdef PyObject* tz = GetResultValue(StringToTzinfo(name.encode('utf-8')))
+    return PyObject_to_object(tz)
 
 
 def timestamp(unit, tz=None):
@@ -2140,6 +2120,26 @@ cpdef DataType decimal128(int precision, int scale=0):
     return pyarrow_wrap_data_type(decimal_type)
 
 
+cpdef DataType decimal256(int precision, int scale=0):
+    """
+    Create decimal type with precision and scale and 256bit width.
+
+    Parameters
+    ----------
+    precision : int
+    scale : int
+
+    Returns
+    -------
+    decimal_type : Decimal256Type
+    """
+    cdef shared_ptr[CDataType] decimal_type
+    if precision < 1 or precision > 76:
+        raise ValueError("precision should be between 1 and 76")
+    decimal_type.reset(new CDecimal256Type(precision, scale))
+    return pyarrow_wrap_data_type(decimal_type)
+
+
 def string():
     """
     Create UTF8 variable-length string type.
diff --git a/python/pyarrow/types.py b/python/pyarrow/types.py
index 66791543fec..708e2bc4643 100644
--- a/python/pyarrow/types.py
+++ b/python/pyarrow/types.py
@@ -31,6 +31,7 @@
                            lib.Type_UINT64}
 _INTEGER_TYPES = _SIGNED_INTEGER_TYPES | _UNSIGNED_INTEGER_TYPES
 _FLOATING_TYPES = {lib.Type_HALF_FLOAT, lib.Type_FLOAT, lib.Type_DOUBLE}
+_DECIMAL_TYPES = {lib.Type_DECIMAL128, lib.Type_DECIMAL256}
 _DATE_TYPES = {lib.Type_DATE32, lib.Type_DATE64}
 _TIME_TYPES = {lib.Type_TIME32, lib.Type_TIME64}
 _TEMPORAL_TYPES = {lib.Type_TIMESTAMP,
@@ -325,7 +326,21 @@ def is_decimal(t):
     """
     Return True if value is an instance of a decimal type.
     """
-    return t.id == lib.Type_DECIMAL
+    return t.id in _DECIMAL_TYPES
+
+
+def is_decimal128(t):
+    """
+    Return True if value is an instance of a decimal type.
+    """
+    return t.id == lib.Type_DECIMAL128
+
+
+def is_decimal256(t):
+    """
+    Return True if value is an instance of a decimal type.
+    """
+    return t.id == lib.Type_DECIMAL256
 
 
 def is_dictionary(t):
diff --git a/python/pyarrow/util.py b/python/pyarrow/util.py
index 690ba3f1fc1..e91294a3a1b 100644
--- a/python/pyarrow/util.py
+++ b/python/pyarrow/util.py
@@ -24,6 +24,11 @@
 import warnings
 
 
+_DEPR_MSG = (
+    "pyarrow.{} is deprecated as of {}, please use pyarrow.{} instead."
+)
+
+
 def implements(f):
     def decorator(g):
         g.__doc__ = f.__doc__
@@ -32,8 +37,7 @@ def decorator(g):
 
 
 def _deprecate_api(old_name, new_name, api, next_version):
-    msg = ('pyarrow.{} is deprecated as of {}, please use pyarrow.{} instead'
-           .format(old_name, next_version, new_name))
+    msg = _DEPR_MSG.format(old_name, next_version, new_name)
 
     def wrapper(*args, **kwargs):
         warnings.warn(msg, FutureWarning)
@@ -46,13 +50,12 @@ def _deprecate_class(old_name, new_class, next_version,
     """
     Raise warning if a deprecated class is used in an isinstance check.
     """
-    msg = 'pyarrow.{} is deprecated as of {}, please use pyarrow.{} instead'
-
     class _DeprecatedMeta(type):
         def __instancecheck__(self, other):
             warnings.warn(
-                msg.format(old_name, next_version, new_class.__name__),
-                FutureWarning
+                _DEPR_MSG.format(old_name, next_version, new_class.__name__),
+                FutureWarning,
+                stacklevel=2
             )
             return isinstance(other, new_class)
 
diff --git a/python/scripts/test_leak.py b/python/scripts/test_leak.py
index 83aac134969..f2bbe8d051b 100644
--- a/python/scripts/test_leak.py
+++ b/python/scripts/test_leak.py
@@ -20,7 +20,7 @@
 import pyarrow as pa
 import numpy as np
 import pandas as pd
-import pandas.util.testing as tm
+from pyarrow.tests.util import rands
 import memory_profiler
 import gc
 import io
@@ -85,7 +85,7 @@ def test_leak3():
                        for i in range(50)})
     table = pa.Table.from_pandas(df, preserve_index=False)
 
-    writer = pq.ParquetWriter('leak_test_' + tm.rands(5) + '.parquet',
+    writer = pq.ParquetWriter('leak_test_' + rands(5) + '.parquet',
                               table.schema)
 
     def func():
diff --git a/python/setup.py b/python/setup.py
index 0353db570d4..d803f7684d5 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -121,7 +121,11 @@ def run(self):
                       'bundle generated Cython C++ code '
                       '(used for code coverage)'),
                      ('bundle-arrow-cpp', None,
-                      'bundle the Arrow C++ libraries')] +
+                      'bundle the Arrow C++ libraries'),
+                     ('bundle-arrow-cpp-headers', None,
+                      'bundle the Arrow C++ headers'),
+                     ('bundle-plasma-executable', None,
+                      'bundle the plasma-store-server executable')] +
                     _build_ext.user_options)
 
     def initialize_options(self):
@@ -175,6 +179,10 @@ def initialize_options(self):
             os.environ.get('PYARROW_BUNDLE_CYTHON_CPP', '0'))
         self.bundle_boost = strtobool(
             os.environ.get('PYARROW_BUNDLE_BOOST', '0'))
+        self.bundle_arrow_cpp_headers = strtobool(
+            os.environ.get('PYARROW_BUNDLE_ARROW_CPP_HEADERS', '1'))
+        self.bundle_plasma_executable = strtobool(
+            os.environ.get('PYARROW_BUNDLE_PLASMA_EXECUTABLE', '1'))
 
     CYTHON_MODULE_NAMES = [
         'lib',
@@ -225,6 +233,7 @@ def _run_cmake(self):
 
             cmake_options = [
                 '-DPYTHON_EXECUTABLE=%s' % sys.executable,
+                '-DPython3_EXECUTABLE=%s' % sys.executable,
                 static_lib_option,
             ]
 
@@ -302,11 +311,12 @@ def append_cmake_bool(value, varname):
             else:
                 build_prefix = self.build_type
 
-            print('Bundling includes: ' + pjoin(build_prefix, 'include'))
-            if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')):
-                shutil.rmtree(pjoin(build_lib, 'pyarrow', 'include'))
-            shutil.move(pjoin(build_prefix, 'include'),
-                        pjoin(build_lib, 'pyarrow'))
+            if self.bundle_arrow_cpp or self.bundle_arrow_cpp_headers:
+                print('Bundling includes: ' + pjoin(build_prefix, 'include'))
+                if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')):
+                    shutil.rmtree(pjoin(build_lib, 'pyarrow', 'include'))
+                shutil.move(pjoin(build_prefix, 'include'),
+                            pjoin(build_lib, 'pyarrow'))
 
             # Move the built C-extension to the place expected by the Python
             # build
@@ -344,7 +354,7 @@ def append_cmake_bool(value, varname):
             if self.bundle_arrow_cpp:
                 self._bundle_arrow_cpp(build_prefix, build_lib)
 
-            if self.with_plasma:
+            if self.with_plasma and self.bundle_plasma_executable:
                 # Move the plasma store
                 source = os.path.join(self.build_type, "plasma-store-server")
                 target = os.path.join(build_lib,
@@ -517,7 +527,7 @@ def _move_shared_libs_unix(build_prefix, build_lib, lib_name):
 
 # If the event of not running from a git clone (e.g. from a git archive
 # or a Python sdist), see if we can set the version number ourselves
-default_version = '2.0.0-SNAPSHOT'
+default_version = '3.0.0-SNAPSHOT'
 if (not os.path.exists('../.git') and
         not os.environ.get('SETUPTOOLS_SCM_PRETEND_VERSION')):
     if os.path.exists('PKG-INFO'):
diff --git a/r/.Rbuildignore b/r/.Rbuildignore
index 6830c9019cd..91a8d741a8e 100644
--- a/r/.Rbuildignore
+++ b/r/.Rbuildignore
@@ -9,6 +9,7 @@ Dockerfile
 .*\.tar\.gz
 ^windows
 ^libarrow
+^revdep
 clang_format.sh
 ^cran-comments\.md$
 ^arrow_.*.tar.gz$
diff --git a/r/.gitignore b/r/.gitignore
index 5fda6334d16..e5ab1197071 100644
--- a/r/.gitignore
+++ b/r/.gitignore
@@ -13,6 +13,7 @@ src/Makevars
 src/Makevars.win
 windows/
 libarrow/
+revdep/
 vignettes/nyc-taxi/
 arrow_*.tar.gz
 arrow_*.tgz
diff --git a/r/DESCRIPTION b/r/DESCRIPTION
index cd11e469ef0..b54616e6a25 100644
--- a/r/DESCRIPTION
+++ b/r/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: arrow
 Title: Integration to 'Apache' 'Arrow'
-Version: 1.0.0.9000
+Version: 2.0.0.9000
 Authors@R: c(
     person("Romain", "Fran\u00e7ois", email = "romain@rstudio.com", role = c("aut"), comment = c(ORCID = "0000-0002-2444-4226")),
     person("Jeroen", "Ooms", email = "jeroen@berkeley.edu", role = c("aut")),
@@ -16,22 +16,19 @@ Description: 'Apache' 'Arrow' <https://arrow.apache.org/> is a cross-language
     package provides an interface to the 'Arrow C++' library.
 Depends: R (>= 3.1)
 License: Apache License (>= 2.0)
-URL: https://github.com/apache/arrow/, https://arrow.apache.org/docs/r
+URL: https://github.com/apache/arrow/, https://arrow.apache.org/docs/r/
 BugReports: https://issues.apache.org/jira/projects/ARROW/issues
 Encoding: UTF-8
 Language: en-US
 LazyData: true
-SystemRequirements: C++11
+SystemRequirements: C++11; for AWS S3 support on Linux, libcurl and openssl (optional)
 Biarch: true
-LinkingTo:
-    Rcpp (>= 1.0.1)
 Imports:
     assertthat,
-    bit64,
+    bit64 (>= 0.9-7),
     methods,
     purrr,
     R6,
-    Rcpp (>= 1.0.1),
     rlang,
     tidyselect,
     utils,
@@ -50,6 +47,7 @@ Suggests:
     rmarkdown,
     testthat,
     tibble
+LinkingTo: cpp11
 Collate:
     'enums.R'
     'arrow-package.R'
@@ -65,8 +63,12 @@ Collate:
     'compute.R'
     'config.R'
     'csv.R'
-    'dataset-write.R'
     'dataset.R'
+    'dataset-factory.R'
+    'dataset-format.R'
+    'dataset-partition.R'
+    'dataset-scan.R'
+    'dataset-write.R'
     'deprecated.R'
     'dictionary.R'
     'record-batch.R'
@@ -76,6 +78,7 @@ Collate:
     'feather.R'
     'field.R'
     'filesystem.R'
+    'flight.R'
     'install-arrow.R'
     'ipc_stream.R'
     'json.R'
diff --git a/r/NAMESPACE b/r/NAMESPACE
index be3d5b43780..9bb3e150179 100644
--- a/r/NAMESPACE
+++ b/r/NAMESPACE
@@ -3,6 +3,7 @@
 S3method("!=",ArrowObject)
 S3method("$",RecordBatch)
 S3method("$",Schema)
+S3method("$",SubTreeFileSystem)
 S3method("$",Table)
 S3method("==",ArrowObject)
 S3method("[",Array)
@@ -22,6 +23,7 @@ S3method(Ops,array_expression)
 S3method(all,equal.ArrowObject)
 S3method(as.character,Array)
 S3method(as.character,ChunkedArray)
+S3method(as.character,FileFormat)
 S3method(as.character,Scalar)
 S3method(as.data.frame,RecordBatch)
 S3method(as.data.frame,Table)
@@ -33,6 +35,7 @@ S3method(as.integer,Array)
 S3method(as.integer,ChunkedArray)
 S3method(as.integer,Scalar)
 S3method(as.list,RecordBatch)
+S3method(as.list,Schema)
 S3method(as.list,Table)
 S3method(as.raw,Buffer)
 S3method(as.vector,Array)
@@ -61,13 +64,20 @@ S3method(length,Array)
 S3method(length,ChunkedArray)
 S3method(length,Scalar)
 S3method(length,Schema)
+S3method(match_arrow,Array)
+S3method(match_arrow,ChunkedArray)
+S3method(match_arrow,default)
 S3method(max,Array)
+S3method(max,ChunkedArray)
 S3method(mean,Array)
 S3method(mean,ChunkedArray)
 S3method(mean,Scalar)
 S3method(min,Array)
+S3method(min,ChunkedArray)
 S3method(names,Dataset)
+S3method(names,FeatherReader)
 S3method(names,RecordBatch)
+S3method(names,Scanner)
 S3method(names,ScannerBuilder)
 S3method(names,Schema)
 S3method(names,Table)
@@ -93,6 +103,8 @@ S3method(tail,arrow_dplyr_query)
 S3method(type,Array)
 S3method(type,ChunkedArray)
 S3method(type,default)
+S3method(unique,Array)
+S3method(unique,ChunkedArray)
 S3method(vec_ptype_abbr,arrow_fixed_size_binary)
 S3method(vec_ptype_abbr,arrow_fixed_size_list)
 S3method(vec_ptype_abbr,arrow_large_list)
@@ -137,6 +149,7 @@ export(FixedSizeListArray)
 export(FixedSizeListType)
 export(HivePartitioning)
 export(HivePartitioningFactory)
+export(InMemoryDataset)
 export(IpcFileFormat)
 export(JsonParseOptions)
 export(JsonReadOptions)
@@ -172,9 +185,11 @@ export(StructScalar)
 export(SubTreeFileSystem)
 export(Table)
 export(TimeUnit)
+export(TimestampParser)
 export(Type)
 export(UnionDataset)
 export(arrow_available)
+export(arrow_with_s3)
 export(binary)
 export(bool)
 export(boolean)
@@ -183,6 +198,7 @@ export(cast_options)
 export(chunked_array)
 export(codec_is_available)
 export(contains)
+export(copy_files)
 export(cpu_count)
 export(dataset_factory)
 export(date32)
@@ -195,6 +211,8 @@ export(everything)
 export(field)
 export(fixed_size_binary)
 export(fixed_size_list_of)
+export(flight_connect)
+export(flight_get)
 export(float)
 export(float16)
 export(float32)
@@ -212,7 +230,9 @@ export(large_list_of)
 export(large_utf8)
 export(last_col)
 export(list_of)
+export(load_flight_server)
 export(map_batches)
+export(match_arrow)
 export(matches)
 export(mmap_create)
 export(mmap_open)
@@ -220,6 +240,7 @@ export(null)
 export(num_range)
 export(one_of)
 export(open_dataset)
+export(push_data)
 export(read_arrow)
 export(read_csv_arrow)
 export(read_delim_arrow)
@@ -231,6 +252,7 @@ export(read_parquet)
 export(read_schema)
 export(read_tsv_arrow)
 export(record_batch)
+export(s3_bucket)
 export(schema)
 export(set_cpu_count)
 export(starts_with)
@@ -253,13 +275,13 @@ export(write_ipc_stream)
 export(write_parquet)
 export(write_to_raw)
 importFrom(R6,R6Class)
-importFrom(Rcpp,sourceCpp)
 importFrom(assertthat,assert_that)
 importFrom(assertthat,is.string)
 importFrom(bit64,print.integer64)
 importFrom(bit64,str.integer64)
 importFrom(methods,as)
 importFrom(purrr,as_mapper)
+importFrom(purrr,keep)
 importFrom(purrr,map)
 importFrom(purrr,map2)
 importFrom(purrr,map_chr)
@@ -275,6 +297,8 @@ importFrom(rlang,enquos)
 importFrom(rlang,env)
 importFrom(rlang,env_bind)
 importFrom(rlang,eval_tidy)
+importFrom(rlang,exec)
+importFrom(rlang,is_bare_character)
 importFrom(rlang,is_false)
 importFrom(rlang,is_integerish)
 importFrom(rlang,list2)
@@ -297,8 +321,10 @@ importFrom(tidyselect,vars_rename)
 importFrom(tidyselect,vars_select)
 importFrom(utils,head)
 importFrom(utils,install.packages)
+importFrom(utils,modifyList)
 importFrom(utils,tail)
 importFrom(vctrs,s3_register)
+importFrom(vctrs,vec_cast)
 importFrom(vctrs,vec_ptype_abbr)
 importFrom(vctrs,vec_ptype_full)
 importFrom(vctrs,vec_size)
diff --git a/r/NEWS.md b/r/NEWS.md
index 4ea15d78112..15c66ae364a 100644
--- a/r/NEWS.md
+++ b/r/NEWS.md
@@ -17,12 +17,60 @@
   under the License.
 -->
 
-# arrow 1.0.0.9000
+# arrow 2.0.0.9000
+
+# arrow 2.0.0
+
+## Datasets
+
+* `write_dataset()` to Feather or Parquet files with partitioning. See the end of `vignette("dataset", package = "arrow")` for discussion and examples.
+* Datasets now have `head()`, `tail()`, and take (`[`) methods. `head()` is optimized but the others  may not be performant.
+* `collect()` gains an `as_data_frame` argument, default `TRUE` but when `FALSE` allows you to evaluate the accumulated `select` and `filter` query but keep the result in Arrow, not an R `data.frame`
+* `read_csv_arrow()` supports specifying column types, both with a `Schema` and with the compact string representation for types used in the `readr` package. It also has gained a `timestamp_parsers` argument that lets you express a set of `strptime` parse strings that will be tried to convert columns designated as `Timestamp` type.
+
+## AWS S3 support
+
+* S3 support is now enabled in binary macOS and Windows (Rtools40 only, i.e. R >= 4.0) packages. To enable it on Linux, you need the additional system dependencies `libcurl` and `openssl`, as well as a sufficiently modern compiler. See `vignette("install", package = "arrow")` for details.
+* File readers and writers (`read_parquet()`, `write_feather()`, et al.), as well as `open_dataset()` and `write_dataset()`, allow you to access resources on S3 (or on file systems that emulate S3) either by providing an `s3://` URI or by providing a `FileSystem$path()`. See `vignette("fs", package = "arrow")` for examples.
+* `copy_files()` allows you to recursively copy directories of files from one file system to another, such as from S3 to your local machine.
+
+## Flight RPC
+
+[Flight](https://arrow.apache.org/blog/2019/10/13/introducing-arrow-flight/)
+is a general-purpose client-server framework for high performance
+transport of large datasets over network interfaces.
+The `arrow` R package now provides methods for connecting to Flight RPC servers
+to send and receive data. See `vignette("flight", package = "arrow")` for an overview.
+
+## Computation
+
+* Comparison (`==`, `>`, etc.) and boolean (`&`, `|`, `!`) operations, along with `is.na`, `%in%` and `match` (called `match_arrow()`), on Arrow Arrays and ChunkedArrays are now implemented in the C++ library.
+* Aggregation methods `min()`, `max()`, and `unique()` are implemented for Arrays and ChunkedArrays.
+* `dplyr` filter expressions on Arrow Tables and RecordBatches are now evaluated in the C++ library, rather than by pulling data into R and evaluating. This yields significant performance improvements.
+* `dim()` (`nrow`) for dplyr queries on Table/RecordBatch is now supported
+
+## Packaging and installation
+
+* `arrow` now depends on [`cpp11`](https://cpp11.r-lib.org/), which brings more robust UTF-8 handling and faster compilation
+* The Linux build script now succeeds on older versions of R
+* MacOS binary packages now ship with zstandard compression enabled
+
+## Bug fixes and other enhancements
+
+* Automatic conversion of Arrow `Int64` type when all values fit with an R 32-bit integer now correctly inspects all chunks in a ChunkedArray, and this conversion can be disabled (so that `Int64` always yields a `bit64::integer64` vector) by setting `options(arrow.int64_downcast = FALSE)`.
+* In addition to the data.frame column metadata preserved in round trip, added in 1.0.0, now attributes of the data.frame itself are also preserved in Arrow schema metadata.
+* File writers now respect the system umask setting
+* `ParquetFileReader` has additional methods for accessing individual columns or row groups from the file
+* Various segfaults fixed: invalid input in `ParquetFileWriter`; invalid `ArrowObject` pointer from a saved R object; converting deeply nested structs from Arrow to R
+* The `properties` and `arrow_properties` arguments to `write_parquet()` are deprecated
+
+# arrow 1.0.1
 
 ## Bug fixes
 
 * Filtering a Dataset that has multiple partition keys using an `%in%` expression now faithfully returns all relevant rows
 * Datasets can now have path segments in the root directory that start with `.` or `_`; files and subdirectories starting with those prefixes are still ignored
+* `open_dataset("~/path")` now correctly expands the path
 * The `version` option to `write_parquet()` is now correctly implemented
 * An UBSAN failure in the `parquet-cpp` library has been fixed
 * For bundled Linux builds, the logic for finding `cmake` is more robust, and you can now specify a `/path/to/cmake` by setting the `CMAKE` environment variable
diff --git a/r/R/arrow-package.R b/r/R/arrow-package.R
index 45715a0ed12..bb32efd33d8 100644
--- a/r/R/arrow-package.R
+++ b/r/R/arrow-package.R
@@ -16,16 +16,15 @@
 # under the License.
 
 #' @importFrom R6 R6Class
-#' @importFrom purrr as_mapper map map2 map_chr map_dfr map_int map_lgl
+#' @importFrom purrr as_mapper map map2 map_chr map_dfr map_int map_lgl keep
 #' @importFrom assertthat assert_that is.string
-#' @importFrom rlang list2 %||% is_false abort dots_n warn enquo quo_is_null enquos is_integerish quos eval_tidy new_data_mask syms env env_bind as_label set_names
-#' @importFrom Rcpp sourceCpp
+#' @importFrom rlang list2 %||% is_false abort dots_n warn enquo quo_is_null enquos is_integerish quos eval_tidy new_data_mask syms env env_bind as_label set_names exec is_bare_character
 #' @importFrom tidyselect vars_select
 #' @useDynLib arrow, .registration = TRUE
 #' @keywords internal
 "_PACKAGE"
 
-#' @importFrom vctrs s3_register vec_size
+#' @importFrom vctrs s3_register vec_size vec_cast
 .onLoad <- function(...) {
   dplyr_methods <- paste0(
     "dplyr::",
@@ -54,18 +53,28 @@
 
 #' Is the C++ Arrow library available?
 #'
-#' You won't generally need to call this function, but it's here in case it
-#' helps for development purposes.
+#' You won't generally need to call these function, but they're made available
+#' for diagnostic purposes.
 #' @return `TRUE` or `FALSE` depending on whether the package was installed
-#' with the Arrow C++ library. If `FALSE`, you'll need to install the C++
-#' library and then reinstall the R package. See [install_arrow()] for help.
+#' with the Arrow C++ library (check with `arrow_available()`) or with S3
+#' support enabled (check with `arrow_with_s3()`).
 #' @export
 #' @examples
 #' arrow_available()
+#' arrow_with_s3()
+#' @seealso If either of these are `FALSE`, see
+#' `vignette("install", package = "arrow")` for guidance on reinstalling the
+#' package.
 arrow_available <- function() {
   .Call(`_arrow_available`)
 }
 
+#' @rdname arrow_available
+#' @export
+arrow_with_s3 <- function() {
+  .Call(`_s3_available`)
+}
+
 option_use_threads <- function() {
   !is_false(getOption("arrow.use_threads"))
 }
diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R
index e11b2d24c63..cf83d695a67 100644
--- a/r/R/arrowExports.R
+++ b/r/R/arrowExports.R
@@ -312,6 +312,10 @@ csv___ParseOptions__initialize <- function(options){
     .Call(`_arrow_csv___ParseOptions__initialize` , options)
 }
 
+csv___ReadOptions__column_names <- function(options){
+    .Call(`_arrow_csv___ReadOptions__column_names` , options)
+}
+
 csv___ConvertOptions__initialize <- function(options){
     .Call(`_arrow_csv___ConvertOptions__initialize` , options)
 }
@@ -324,6 +328,22 @@ csv___TableReader__Read <- function(table_reader){
     .Call(`_arrow_csv___TableReader__Read` , table_reader)
 }
 
+TimestampParser__kind <- function(parser){
+    .Call(`_arrow_TimestampParser__kind` , parser)
+}
+
+TimestampParser__format <- function(parser){
+    .Call(`_arrow_TimestampParser__format` , parser)
+}
+
+TimestampParser__MakeStrptime <- function(format){
+    .Call(`_arrow_TimestampParser__MakeStrptime` , format)
+}
+
+TimestampParser__MakeISO8601 <- function(){
+    .Call(`_arrow_TimestampParser__MakeISO8601` )
+}
+
 dataset___Dataset__NewScan <- function(ds){
     .Call(`_arrow_dataset___Dataset__NewScan` , ds)
 }
@@ -344,6 +364,10 @@ dataset___UnionDataset__create <- function(datasets, schm){
     .Call(`_arrow_dataset___UnionDataset__create` , datasets, schm)
 }
 
+dataset___InMemoryDataset__create <- function(table){
+    .Call(`_arrow_dataset___InMemoryDataset__create` , table)
+}
+
 dataset___UnionDataset__children <- function(ds){
     .Call(`_arrow_dataset___UnionDataset__children` , ds)
 }
@@ -352,6 +376,10 @@ dataset___FileSystemDataset__format <- function(dataset){
     .Call(`_arrow_dataset___FileSystemDataset__format` , dataset)
 }
 
+dataset___FileSystemDataset__filesystem <- function(dataset){
+    .Call(`_arrow_dataset___FileSystemDataset__filesystem` , dataset)
+}
+
 dataset___FileSystemDataset__files <- function(dataset){
     .Call(`_arrow_dataset___FileSystemDataset__files` , dataset)
 }
@@ -388,10 +416,30 @@ dataset___FileFormat__type_name <- function(format){
     .Call(`_arrow_dataset___FileFormat__type_name` , format)
 }
 
+dataset___FileFormat__DefaultWriteOptions <- function(fmt){
+    .Call(`_arrow_dataset___FileFormat__DefaultWriteOptions` , fmt)
+}
+
 dataset___ParquetFileFormat__Make <- function(use_buffered_stream, buffer_size, dict_columns){
     .Call(`_arrow_dataset___ParquetFileFormat__Make` , use_buffered_stream, buffer_size, dict_columns)
 }
 
+dataset___FileWriteOptions__type_name <- function(options){
+    .Call(`_arrow_dataset___FileWriteOptions__type_name` , options)
+}
+
+dataset___ParquetFileWriteOptions__update <- function(options, writer_props, arrow_writer_props){
+    invisible(.Call(`_arrow_dataset___ParquetFileWriteOptions__update` , options, writer_props, arrow_writer_props))
+}
+
+dataset___IpcFileWriteOptions__update2 <- function(ipc_options, use_legacy_format, codec, metadata_version){
+    invisible(.Call(`_arrow_dataset___IpcFileWriteOptions__update2` , ipc_options, use_legacy_format, codec, metadata_version))
+}
+
+dataset___IpcFileWriteOptions__update1 <- function(ipc_options, use_legacy_format, metadata_version){
+    invisible(.Call(`_arrow_dataset___IpcFileWriteOptions__update1` , ipc_options, use_legacy_format, metadata_version))
+}
+
 dataset___IpcFileFormat__Make <- function(){
     .Call(`_arrow_dataset___IpcFileFormat__Make` )
 }
@@ -452,12 +500,16 @@ dataset___Scanner__Scan <- function(scanner){
     .Call(`_arrow_dataset___Scanner__Scan` , scanner)
 }
 
+dataset___Scanner__schema <- function(sc){
+    .Call(`_arrow_dataset___Scanner__schema` , sc)
+}
+
 dataset___ScanTask__get_batches <- function(scan_task){
     .Call(`_arrow_dataset___ScanTask__get_batches` , scan_task)
 }
 
-dataset___Dataset__Write <- function(ds, schema, format, filesystem, path, partitioning){
-    invisible(.Call(`_arrow_dataset___Dataset__Write` , ds, schema, format, filesystem, path, partitioning))
+dataset___Dataset__Write <- function(file_write_options, filesystem, base_dir, partitioning, basename_template, scanner){
+    invisible(.Call(`_arrow_dataset___Dataset__Write` , file_write_options, filesystem, base_dir, partitioning, basename_template, scanner))
 }
 
 shared_ptr_is_null <- function(xp){
@@ -576,8 +628,8 @@ fixed_size_list__ <- function(x, list_size){
     .Call(`_arrow_fixed_size_list__` , x, list_size)
 }
 
-struct_ <- function(fields){
-    .Call(`_arrow_struct_` , fields)
+struct__ <- function(fields){
+    .Call(`_arrow_struct__` , fields)
 }
 
 DataType__ToString <- function(type){
@@ -912,16 +964,28 @@ fs___SubTreeFileSystem__create <- function(base_path, base_fs){
     .Call(`_arrow_fs___SubTreeFileSystem__create` , base_path, base_fs)
 }
 
+fs___SubTreeFileSystem__base_fs <- function(file_system){
+    .Call(`_arrow_fs___SubTreeFileSystem__base_fs` , file_system)
+}
+
+fs___SubTreeFileSystem__base_path <- function(file_system){
+    .Call(`_arrow_fs___SubTreeFileSystem__base_path` , file_system)
+}
+
 fs___FileSystemFromUri <- function(path){
     .Call(`_arrow_fs___FileSystemFromUri` , path)
 }
 
-fs___EnsureS3Initialized <- function(){
-    invisible(.Call(`_arrow_fs___EnsureS3Initialized` ))
+fs___CopyFiles <- function(source_fs, source_sel, destination_fs, destination_base_dir, chunk_size, use_threads){
+    invisible(.Call(`_arrow_fs___CopyFiles` , source_fs, source_sel, destination_fs, destination_base_dir, chunk_size, use_threads))
 }
 
-fs___S3FileSystem__create <- function(){
-    .Call(`_arrow_fs___S3FileSystem__create` )
+fs___S3FileSystem__create <- function(anonymous, access_key, secret_key, session_token, role_arn, session_name, external_id, load_frequency, region, endpoint_override, scheme, background_writes){
+    .Call(`_arrow_fs___S3FileSystem__create` , anonymous, access_key, secret_key, session_token, role_arn, session_name, external_id, load_frequency, region, endpoint_override, scheme, background_writes)
+}
+
+fs___S3FileSystem__region <- function(fs){
+    .Call(`_arrow_fs___S3FileSystem__region` , fs)
 }
 
 io___Readable__Read <- function(x, nbytes){
@@ -1012,12 +1076,12 @@ io___BufferOutputStream__Write <- function(stream, bytes){
     invisible(.Call(`_arrow_io___BufferOutputStream__Write` , stream, bytes))
 }
 
-json___ReadOptions__initialize <- function(options){
-    .Call(`_arrow_json___ReadOptions__initialize` , options)
+json___ReadOptions__initialize <- function(use_threads, block_size){
+    .Call(`_arrow_json___ReadOptions__initialize` , use_threads, block_size)
 }
 
-json___ParseOptions__initialize <- function(options){
-    .Call(`_arrow_json___ParseOptions__initialize` , options)
+json___ParseOptions__initialize <- function(newlines_in_values){
+    .Call(`_arrow_json___ParseOptions__initialize` , newlines_in_values)
 }
 
 json___TableReader__Make <- function(input, read_options, parse_options){
@@ -1120,10 +1184,38 @@ parquet___arrow___FileReader__ReadTable2 <- function(reader, column_indices){
     .Call(`_arrow_parquet___arrow___FileReader__ReadTable2` , reader, column_indices)
 }
 
+parquet___arrow___FileReader__ReadRowGroup1 <- function(reader, i){
+    .Call(`_arrow_parquet___arrow___FileReader__ReadRowGroup1` , reader, i)
+}
+
+parquet___arrow___FileReader__ReadRowGroup2 <- function(reader, i, column_indices){
+    .Call(`_arrow_parquet___arrow___FileReader__ReadRowGroup2` , reader, i, column_indices)
+}
+
+parquet___arrow___FileReader__ReadRowGroups1 <- function(reader, row_groups){
+    .Call(`_arrow_parquet___arrow___FileReader__ReadRowGroups1` , reader, row_groups)
+}
+
+parquet___arrow___FileReader__ReadRowGroups2 <- function(reader, row_groups, column_indices){
+    .Call(`_arrow_parquet___arrow___FileReader__ReadRowGroups2` , reader, row_groups, column_indices)
+}
+
 parquet___arrow___FileReader__num_rows <- function(reader){
     .Call(`_arrow_parquet___arrow___FileReader__num_rows` , reader)
 }
 
+parquet___arrow___FileReader__num_columns <- function(reader){
+    .Call(`_arrow_parquet___arrow___FileReader__num_columns` , reader)
+}
+
+parquet___arrow___FileReader__num_row_groups <- function(reader){
+    .Call(`_arrow_parquet___arrow___FileReader__num_row_groups` , reader)
+}
+
+parquet___arrow___FileReader__ReadColumn <- function(reader, i){
+    .Call(`_arrow_parquet___arrow___FileReader__ReadColumn` , reader, i)
+}
+
 parquet___ArrowWriterProperties___create <- function(allow_truncated_timestamps, use_deprecated_int96_timestamps, timestamp_unit){
     .Call(`_arrow_parquet___ArrowWriterProperties___create` , allow_truncated_timestamps, use_deprecated_int96_timestamps, timestamp_unit)
 }
@@ -1212,12 +1304,12 @@ ExportSchema <- function(schema, ptr){
     invisible(.Call(`_arrow_ExportSchema` , schema, ptr))
 }
 
-ExportArray <- function(array, ptr, schema_ptr){
-    invisible(.Call(`_arrow_ExportArray` , array, ptr, schema_ptr))
+ExportArray <- function(array, array_ptr, schema_ptr){
+    invisible(.Call(`_arrow_ExportArray` , array, array_ptr, schema_ptr))
 }
 
-ExportRecordBatch <- function(batch, ptr, schema_ptr){
-    invisible(.Call(`_arrow_ExportRecordBatch` , batch, ptr, schema_ptr))
+ExportRecordBatch <- function(batch, array_ptr, schema_ptr){
+    invisible(.Call(`_arrow_ExportRecordBatch` , batch, array_ptr, schema_ptr))
 }
 
 RecordBatch__num_columns <- function(x){
@@ -1248,12 +1340,8 @@ RecordBatch__GetColumnByName <- function(batch, name){
     .Call(`_arrow_RecordBatch__GetColumnByName` , batch, name)
 }
 
-RecordBatch__select <- function(batch, indices){
-    .Call(`_arrow_RecordBatch__select` , batch, indices)
-}
-
-RecordBatch__from_dataframe <- function(tbl){
-    .Call(`_arrow_RecordBatch__from_dataframe` , tbl)
+RecordBatch__SelectColumns <- function(batch, indices){
+    .Call(`_arrow_RecordBatch__SelectColumns` , batch, indices)
 }
 
 RecordBatch__Equals <- function(self, other, check_metadata){
@@ -1440,10 +1528,6 @@ arrow__UnifySchemas <- function(schemas){
     .Call(`_arrow_arrow__UnifySchemas` , schemas)
 }
 
-Table__from_dataframe <- function(tbl){
-    .Call(`_arrow_Table__from_dataframe` , tbl)
-}
-
 Table__num_columns <- function(x){
     .Call(`_arrow_Table__num_columns` , x)
 }
@@ -1500,8 +1584,16 @@ Table__GetColumnByName <- function(table, name){
     .Call(`_arrow_Table__GetColumnByName` , table, name)
 }
 
-Table__select <- function(table, indices){
-    .Call(`_arrow_Table__select` , table, indices)
+Table__SelectColumns <- function(table, indices){
+    .Call(`_arrow_Table__SelectColumns` , table, indices)
+}
+
+all_record_batches <- function(lst){
+    .Call(`_arrow_all_record_batches` , lst)
+}
+
+Table__from_record_batches <- function(batches, schema_sxp){
+    .Call(`_arrow_Table__from_record_batches` , batches, schema_sxp)
 }
 
 Table__from_dots <- function(lst, schema_sxp){
diff --git a/r/R/compute.R b/r/R/compute.R
index 60a1a46976c..691d59e3666 100644
--- a/r/R/compute.R
+++ b/r/R/compute.R
@@ -57,19 +57,23 @@ min.Array <- function(..., na.rm = FALSE) {
   scalar_aggregate("min_max", ..., na.rm = na.rm)$GetFieldByName("min")
 }
 
+#' @export
+min.ChunkedArray <- min.Array
+
 #' @export
 max.Array <- function(..., na.rm = FALSE) {
   scalar_aggregate("min_max", ..., na.rm = na.rm)$GetFieldByName("max")
 }
 
+#' @export
+max.ChunkedArray <- max.Array
+
 scalar_aggregate <- function(FUN, ..., na.rm = FALSE) {
   a <- collect_arrays_from_dots(list(...))
-  if (!na.rm && a$null_count > 0) {
-    if (FUN %in% c("mean", "sum")) {
-      # Arrow sum/mean function always drops NAs so handle that here
-      # https://issues.apache.org/jira/browse/ARROW-9054
-      return(Scalar$create(NA_real_))
-    }
+  if (!na.rm && a$null_count > 0 && (FUN %in% c("mean", "sum"))) {
+    # Arrow sum/mean function always drops NAs so handle that here
+    # https://issues.apache.org/jira/browse/ARROW-9054
+    return(Scalar$create(NA_real_))
   }
 
   Scalar$create(call_function(FUN, a, options = list(na.rm = na.rm)))
@@ -94,6 +98,46 @@ collect_arrays_from_dots <- function(dots) {
   ChunkedArray$create(!!!arrays)
 }
 
+#' @export
+unique.Array <- function(x, incomparables = FALSE, ...) {
+  Array$create(call_function("unique", x))
+}
+
+#' @export
+unique.ChunkedArray <- unique.Array
+
+#' `match` for Arrow objects
+#'
+#' `base::match()` is not a generic, so we can't just define Arrow methods for
+#' it. This function exposes the analogous function in the Arrow C++ library.
+#'
+#' @param x `Array` or `ChunkedArray`
+#' @param table `Array`, `ChunkedArray`, or R vector lookup table.
+#' @param ... additional arguments, ignored
+#' @return An `int32`-type `Array` of the same length as `x` with the
+#' (0-based) indexes into `table`.
+#' @export
+match_arrow <- function(x, table, ...) UseMethod("match_arrow")
+
+#' @export
+match_arrow.default <- function(x, table, ...) match(x, table, ...)
+
+#' @export
+match_arrow.Array <- function(x, table, ...) {
+  if (!inherits(table, c("Array", "ChunkedArray"))) {
+    table <- Array$create(table)
+  }
+  Array$create(call_function("index_in_meta_binary", x, table))
+}
+
+#' @export
+match_arrow.ChunkedArray <- function(x, table, ...) {
+  if (!inherits(table, c("Array", "ChunkedArray"))) {
+    table <- Array$create(table)
+  }
+  shared_ptr(ChunkedArray, call_function("index_in_meta_binary", x, table))
+}
+
 CastOptions <- R6Class("CastOptions", inherit = ArrowObject)
 
 #' Cast options
diff --git a/r/R/csv.R b/r/R/csv.R
index e145a907e28..24a0f11312f 100644
--- a/r/R/csv.R
+++ b/r/R/csv.R
@@ -32,7 +32,53 @@
 #' `parse_options`, `convert_options`, or `read_options` arguments, or you can
 #' use [CsvTableReader] directly for lower-level access.
 #'
-#' @param file A character file name, `raw` vector, or an Arrow input stream.
+#' @section Specifying column types and names:
+#'
+#' By default, the CSV reader will infer the column names and data types from the file, but there
+#' are a few ways you can specify them directly.
+#'
+#' One way is to provide an Arrow [Schema] in the `schema` argument,
+#' which is an ordered map of column name to type.
+#' When provided, it satisfies both the `col_names` and `col_types` arguments.
+#' This is good if you know all of this information up front.
+#'
+#' You can also pass a `Schema` to the `col_types` argument. If you do this,
+#' column names will still be inferred from the file unless you also specify
+#' `col_names`. In either case, the column names in the `Schema` must match the
+#' data's column names, whether they are explicitly provided or inferred. That
+#' said, this `Schema` does not have to reference all columns: those omitted
+#' will have their types inferred.
+#'
+#' Alternatively, you can declare column types by providing the compact string representation
+#' that `readr` uses to the `col_types` argument. This means you provide a
+#' single string, one character per column, where the characters map to Arrow
+#' types analogously to the `readr` type mapping:
+#'
+#' * "c": `utf8()`
+#' * "i": `int32()`
+#' * "n": `float64()`
+#' * "d": `float64()`
+#' * "l": `bool()`
+#' * "f": `dictionary()`
+#' * "D": `date32()`
+#' * "T": `time32()`
+#' * "t": `timestamp()`
+#' * "_": `null()`
+#' * "-": `null()`
+#' * "?": infer the type from the data
+#'
+#' If you use the compact string representation for `col_types`, you must also
+#' specify `col_names`.
+#'
+#' Regardless of how types are specified, all columns with a `null()` type will
+#' be dropped.
+#'
+#' Note that if you are specifying column names, whether by `schema` or
+#' `col_names`, and the CSV file has a header row that would otherwise be used
+#' to idenfity column names, you'll need to add `skip = 1` to skip that row.
+#'
+#' @param file A character file name or URI, `raw` vector, an Arrow input stream,
+#' or a `FileSystem` with path (`SubTreeFileSystem`).
 #' If a file name, a memory-mapped Arrow [InputStream] will be opened and
 #' closed when finished; compression will be detected from the file extension
 #' and handled automatically. If an input stream is provided, it will be left
@@ -46,10 +92,14 @@
 #' characters? This is more general than `escape_double` as backslashes
 #' can be used to escape the delimiter character, the quote character, or
 #' to add special characters like `\\n`.
+#' @param schema [Schema] that describes the table. If provided, it will be
+#' used to satisfy both `col_names` and `col_types`.
 #' @param col_names If `TRUE`, the first row of the input will be used as the
 #' column names and will not be included in the data frame. If `FALSE`, column
 #' names will be generated by Arrow, starting with "f0", "f1", ..., "fN".
 #' Alternatively, you can specify a character vector of column names.
+#' @param col_types A compact string representation of the column types, or
+#' `NULL` (the default) to infer types from the data.
 #' @param col_select A character vector of column names to keep, as in the
 #' "select" argument to `data.table::fread()`, or a
 #' [tidy selection specification][tidyselect::vars_select()]
@@ -63,6 +113,12 @@
 #' `TRUE`, blank rows will not be represented at all. If `FALSE`, they will be
 #' filled with missings.
 #' @param skip Number of lines to skip before reading data.
+#' @param timestamp_parsers User-defined timestamp parsers. If more than one
+#' parser is specified, the CSV conversion logic will try parsing values
+#' starting from the beginning of this vector. Possible values are:
+#'  - `NULL`: the default, which uses the ISO-8601 parser
+#'  - a character vector of [strptime][base::strptime()] parse strings
+#'  - a list of [TimestampParser] objects
 #' @param parse_options see [file reader options][CsvReadOptions].
 #' If given, this overrides any
 #' parsing options provided in other arguments (e.g. `delim`, `quote`, etc.).
@@ -88,8 +144,9 @@ read_delim_arrow <- function(file,
                              quote = '"',
                              escape_double = TRUE,
                              escape_backslash = FALSE,
+                             schema = NULL,
                              col_names = TRUE,
-                             # col_types = TRUE,
+                             col_types = NULL,
                              col_select = NULL,
                              na = c("", "NA"),
                              quoted_na = TRUE,
@@ -98,8 +155,12 @@ read_delim_arrow <- function(file,
                              parse_options = NULL,
                              convert_options = NULL,
                              read_options = NULL,
-                             as_data_frame = TRUE) {
-
+                             as_data_frame = TRUE,
+                             timestamp_parsers = NULL) {
+  if (inherits(schema, "Schema")) {
+    col_names <- names(schema)
+    col_types <- schema
+  }
   if (is.null(parse_options)) {
     parse_options <- readr_to_csv_parse_options(
       delim,
@@ -109,13 +170,17 @@ read_delim_arrow <- function(file,
       skip_empty_rows
     )
   }
-
   if (is.null(read_options)) {
     read_options <- readr_to_csv_read_options(skip, col_names)
   }
   if (is.null(convert_options)) {
-    # TODO: col_types (needs wiring in CsvConvertOptions)
-    convert_options <- readr_to_csv_convert_options(na, quoted_na)
+    convert_options <- readr_to_csv_convert_options(
+      na,
+      quoted_na,
+      col_types = col_types,
+      col_names = read_options$column_names,
+      timestamp_parsers = timestamp_parsers
+    )
   }
 
   if (!inherits(file, "InputStream")) {
@@ -129,7 +194,13 @@ read_delim_arrow <- function(file,
     convert_options = convert_options
   )
 
-  tab <- reader$Read()$select(!!enquo(col_select))
+  tab <- reader$Read()
+
+  # TODO: move this into convert_options using include_columns
+  col_select <- enquo(col_select)
+  if (!quo_is_null(col_select)) {
+    tab <- tab[vars_select(names(tab), !!col_select)]
+  }
 
   if (isTRUE(as_data_frame)) {
     tab <- as.data.frame(tab)
@@ -144,8 +215,9 @@ read_csv_arrow <- function(file,
                            quote = '"',
                            escape_double = TRUE,
                            escape_backslash = FALSE,
+                           schema = NULL,
                            col_names = TRUE,
-                           # col_types = TRUE,
+                           col_types = NULL,
                            col_select = NULL,
                            na = c("", "NA"),
                            quoted_na = TRUE,
@@ -154,7 +226,8 @@ read_csv_arrow <- function(file,
                            parse_options = NULL,
                            convert_options = NULL,
                            read_options = NULL,
-                           as_data_frame = TRUE) {
+                           as_data_frame = TRUE,
+                           timestamp_parsers = NULL) {
 
   mc <- match.call()
   mc$delim <- ","
@@ -168,8 +241,9 @@ read_tsv_arrow <- function(file,
                            quote = '"',
                            escape_double = TRUE,
                            escape_backslash = FALSE,
+                           schema = NULL,
                            col_names = TRUE,
-                           # col_types = TRUE,
+                           col_types = NULL,
                            col_select = NULL,
                            na = c("", "NA"),
                            quoted_na = TRUE,
@@ -178,7 +252,8 @@ read_tsv_arrow <- function(file,
                            parse_options = NULL,
                            convert_options = NULL,
                            read_options = NULL,
-                           as_data_frame = TRUE) {
+                           as_data_frame = TRUE,
+                           timestamp_parsers = NULL) {
 
   mc <- match.call()
   mc$delim <- "\t"
@@ -201,7 +276,7 @@ read_tsv_arrow <- function(file,
 #' The `CsvTableReader$create()` and `JsonTableReader$create()` factory methods
 #' take the following arguments:
 #'
-#' - `file` A character path to a local file, or an Arrow input stream
+#' - `file` An Arrow [InputStream]
 #' - `convert_options` (CSV only), `parse_options`, `read_options`: see
 #'    [CsvReadOptions]
 #' - `...` additional parameters.
@@ -222,7 +297,7 @@ CsvTableReader$create <- function(file,
                                   parse_options = CsvParseOptions$create(),
                                   convert_options = CsvConvertOptions$create(),
                                   ...) {
-  file <- make_readable_file(file)
+  assert_is(file, "InputStream")
   shared_ptr(
     CsvTableReader,
     csv___TableReader__Make(file, read_options, parse_options, convert_options)
@@ -236,7 +311,7 @@ CsvTableReader$create <- function(file,
 #' @usage NULL
 #' @format NULL
 #' @description `CsvReadOptions`, `CsvParseOptions`, `CsvConvertOptions`,
-#' `JsonReadOptions`, and `JsonParseOptions` are containers for various
+#' `JsonReadOptions`, `JsonParseOptions`, and `TimestampParser` are containers for various
 #' file reading options. See their usage in [read_csv_arrow()] and
 #' [read_json_arrow()], respectively.
 #'
@@ -284,19 +359,48 @@ CsvTableReader$create <- function(file,
 #' - `strings_can_be_null` Logical: can string / binary columns have
 #'    null values? Similar to the `quoted_na` argument to `readr::read_csv()`.
 #'    (default `FALSE`)
+#' - `true_values` character vector of recognized spellings for `TRUE` values
+#' - `false_values` character vector of recognized spellings for `FALSE` values
+#' - `col_types` A `Schema` or `NULL` to infer types
+#' - `auto_dict_encode` Logical: Whether to try to automatically
+#'    dictionary-encode string / binary data (think `stringsAsFactors`). Default `FALSE`.
+#'    This setting is ignored for non-inferred columns (those in `col_types`).
+#' - `auto_dict_max_cardinality` If `auto_dict_encode`, string/binary columns
+#'    are dictionary-encoded up to this number of unique values (default 50),
+#'    after which it switches to regular encoding.
+#' - `include_columns` If non-empty, indicates the names of columns from the
+#'    CSV file that should be actually read and converted (in the vector's order).
+#' - `include_missing_columns` Logical: if `include_columns` is provided, should
+#'    columns named in it but not found in the data be included as a column of
+#'    type `null()`? The default (`FALSE`) means that the reader will instead
+#'    raise an error.
+#' - `timestamp_parsers` User-defined timestamp parsers. If more than one
+#'    parser is specified, the CSV conversion logic will try parsing values
+#'    starting from the beginning of this vector. Possible values are
+#'    (a) `NULL`, the default, which uses the ISO-8601 parser;
+#'    (b) a character vector of [strptime][base::strptime()] parse strings; or
+#'    (c) a list of [TimestampParser] objects.
 #'
-#' @section Methods:
+#' `TimestampParser$create()` takes an optional `format` string argument.
+#' See [`strptime()`][base::strptime()] for example syntax.
+#' The default is to use an ISO-8601 format parser.
+#' @section Active bindings:
 #'
-#' These classes have no implemented methods. They are containers for the
-#' options.
+#' - `column_names`: from `CsvReadOptions`
 #'
 #' @export
-CsvReadOptions <- R6Class("CsvReadOptions", inherit = ArrowObject)
+CsvReadOptions <- R6Class("CsvReadOptions",
+  inherit = ArrowObject,
+  active = list(
+    column_names = function() csv___ReadOptions__column_names(self)
+  )
+)
 CsvReadOptions$create <- function(use_threads = option_use_threads(),
                                   block_size = 1048576L,
                                   skip_rows = 0L,
                                   column_names = character(0),
                                   autogenerate_column_names = FALSE) {
+
   shared_ptr(CsvReadOptions, csv___ReadOptions__initialize(
     list(
       use_threads = use_threads,
@@ -308,7 +412,7 @@ CsvReadOptions$create <- function(use_threads = option_use_threads(),
   ))
 }
 
-readr_to_csv_read_options <- function(skip, col_names) {
+readr_to_csv_read_options <- function(skip, col_names, col_types) {
   if (isTRUE(col_names)) {
     # C++ default to parse is 0-length string array
     col_names <- character(0)
@@ -368,6 +472,25 @@ readr_to_csv_parse_options <- function(delim = ",",
   )
 }
 
+#' @rdname CsvReadOptions
+#' @usage NULL
+#' @format NULL
+#' @docType class
+#' @export
+TimestampParser <- R6Class("TimestampParser", inherit = ArrowObject,
+  public = list(
+    kind = function() TimestampParser__kind(self),
+    format = function() TimestampParser__format(self)
+  )
+)
+TimestampParser$create <- function(format = NULL) {
+  if (is.null(format)) {
+    shared_ptr(TimestampParser, TimestampParser__MakeISO8601())
+  } else {
+    shared_ptr(TimestampParser, TimestampParser__MakeStrptime(format))
+  }
+}
+
 #' @rdname CsvReadOptions
 #' @usage NULL
 #' @format NULL
@@ -376,23 +499,93 @@ readr_to_csv_parse_options <- function(delim = ",",
 CsvConvertOptions <- R6Class("CsvConvertOptions", inherit = ArrowObject)
 CsvConvertOptions$create <- function(check_utf8 = TRUE,
                                      null_values = c("", "NA"),
-                                     strings_can_be_null = FALSE) {
-  # TODO: there are more conversion options available:
-  # // Optional per-column types (disabling type inference on those columns)
-  # std::unordered_map<std::string, std::shared_ptr<DataType>> column_types;
-  # // Recognized spellings for boolean values
-  # std::vector<std::string> true_values;
-  # std::vector<std::string> false_values;
+                                     true_values = c("T", "true", "TRUE"),
+                                     false_values= c("F", "false", "FALSE"),
+                                     strings_can_be_null = FALSE,
+                                     col_types = NULL,
+                                     auto_dict_encode = FALSE,
+                                     auto_dict_max_cardinality = 50L,
+                                     include_columns = character(),
+                                     include_missing_columns = FALSE,
+                                     timestamp_parsers = NULL) {
+
+  if (!is.null(col_types) && !inherits(col_types, "Schema")) {
+    abort(c(
+      "Unsupported `col_types` specification.",
+      i = "`col_types` must be NULL, or a <Schema>."
+    ))
+  }
 
   shared_ptr(CsvConvertOptions, csv___ConvertOptions__initialize(
     list(
       check_utf8 = check_utf8,
       null_values = null_values,
-      strings_can_be_null = strings_can_be_null
+      strings_can_be_null = strings_can_be_null,
+      col_types = col_types,
+      true_values = true_values,
+      false_values = false_values,
+      auto_dict_encode = auto_dict_encode,
+      auto_dict_max_cardinality = auto_dict_max_cardinality,
+      include_columns = include_columns,
+      include_missing_columns = include_missing_columns,
+      timestamp_parsers = timestamp_parsers
     )
   ))
 }
 
-readr_to_csv_convert_options <- function(na, quoted_na) {
-    CsvConvertOptions$create(null_values = na, strings_can_be_null = quoted_na)
+readr_to_csv_convert_options <- function(na,
+                                         quoted_na,
+                                         col_types = NULL,
+                                         col_names = NULL,
+                                         timestamp_parsers = NULL) {
+  include_columns <- character()
+
+  if (is.character(col_types)) {
+    if (length(col_types) != 1L) {
+      abort("`col_types` is a character vector that is not of size 1")
+    }
+    n <- nchar(col_types)
+    specs <- substring(col_types, seq_len(n), seq_len(n))
+    if (!is_bare_character(col_names, n)) {
+      abort("Compact specification for `col_types` requires `col_names`")
+    }
+
+    col_types <- set_names(nm = col_names, map2(specs, col_names, ~{
+      switch(.x,
+             "c" = utf8(),
+             "i" = int32(),
+             "n" = float64(),
+             "d" = float64(),
+             "l" = bool(),
+             "f" = dictionary(),
+             "D" = date32(),
+             "T" = time32(),
+             "t" = timestamp(),
+             "_" = null(),
+             "-" = null(),
+             "?" = NULL,
+             abort("Unsupported compact specification: '", .x,"' for column '", .y, "'")
+      )
+    }))
+    # To "guess" types, omit them from col_types
+    col_types <- keep(col_types, ~!is.null(.x))
+    col_types <- schema(!!!col_types)
+  }
+
+  if (!is.null(col_types)) {
+    assert_is(col_types, "Schema")
+    # If any columns are null(), drop them
+    # (by specifying the other columns in include_columns)
+    nulls <- map_lgl(col_types$fields, ~.$type$Equals(null()))
+    if (any(nulls)) {
+      include_columns <- setdiff(col_names, names(col_types)[nulls])
+    }
+  }
+  CsvConvertOptions$create(
+    null_values = na,
+    strings_can_be_null = quoted_na,
+    col_types = col_types,
+    timestamp_parsers = timestamp_parsers,
+    include_columns = include_columns
+  )
 }
diff --git a/r/R/dataset-factory.R b/r/R/dataset-factory.R
new file mode 100644
index 00000000000..8b3439c419d
--- /dev/null
+++ b/r/R/dataset-factory.R
@@ -0,0 +1,147 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#' @include dataset.R
+
+#' @usage NULL
+#' @format NULL
+#' @rdname Dataset
+#' @export
+DatasetFactory <- R6Class("DatasetFactory", inherit = ArrowObject,
+  public = list(
+    Finish = function(schema = NULL, unify_schemas = FALSE) {
+      if (is.null(schema)) {
+        ptr <- dataset___DatasetFactory__Finish1(self, unify_schemas)
+      } else {
+        ptr <- dataset___DatasetFactory__Finish2(self, schema)
+      }
+      shared_ptr(Dataset, ptr)$..dispatch()
+    },
+    Inspect = function(unify_schemas = FALSE) {
+      shared_ptr(Schema, dataset___DatasetFactory__Inspect(self, unify_schemas))
+    }
+  )
+)
+DatasetFactory$create <- function(x,
+                                  filesystem = NULL,
+                                  format = c("parquet", "arrow", "ipc", "feather", "csv", "tsv", "text"),
+                                  partitioning = NULL,
+                                  ...) {
+  if (is_list_of(x, "DatasetFactory")) {
+    return(shared_ptr(DatasetFactory, dataset___UnionDatasetFactory__Make(x)))
+  }
+
+  path_and_fs <- get_path_and_filesystem(x, filesystem)
+  selector <- FileSelector$create(path_and_fs$path, allow_not_found = FALSE, recursive = TRUE)
+
+  if (is.character(format)) {
+    format <- FileFormat$create(match.arg(format), ...)
+  } else {
+    assert_is(format, "FileFormat")
+  }
+
+  if (!is.null(partitioning)) {
+    if (inherits(partitioning, "Schema")) {
+      partitioning <- DirectoryPartitioning$create(partitioning)
+    } else if (is.character(partitioning)) {
+      # These are the column/field names, and we should autodetect their types
+      partitioning <- DirectoryPartitioningFactory$create(partitioning)
+    }
+  }
+  FileSystemDatasetFactory$create(path_and_fs$fs, selector, format, partitioning)
+}
+
+#' Create a DatasetFactory
+#'
+#' A [Dataset] can constructed using one or more [DatasetFactory]s.
+#' This function helps you construct a `DatasetFactory` that you can pass to
+#' [open_dataset()].
+#'
+#' If you would only have a single `DatasetFactory` (for example, you have a
+#' single directory containing Parquet files), you can call `open_dataset()`
+#' directly. Use `dataset_factory()` when you
+#' want to combine different directories, file systems, or file formats.
+#'
+#' @param x A string file x containing data files, or
+#' a list of `DatasetFactory` objects whose datasets should be
+#' grouped. If this argument is specified it will be used to construct a
+#' `UnionDatasetFactory` and other arguments will be ignored.
+#' @param filesystem A [FileSystem] object; if omitted, the `FileSystem` will
+#' be detected from `x`
+#' @param format A [FileFormat] object, or a string identifier of the format of
+#' the files in `x`. Currently supported values:
+#' * "parquet"
+#' * "ipc"/"arrow"/"feather", all aliases for each other; for Feather, note that
+#'   only version 2 files are supported
+#' * "csv"/"text", aliases for the same thing (because comma is the default
+#'   delimiter for text files
+#' * "tsv", equivalent to passing `format = "text", delimiter = "\t"`
+#'
+#' Default is "parquet", unless a `delimiter` is also specified, in which case
+#' it is assumed to be "text".
+#' @param partitioning One of
+#'   * A `Schema`, in which case the file paths relative to `sources` will be
+#'    parsed, and path segments will be matched with the schema fields. For
+#'    example, `schema(year = int16(), month = int8())` would create partitions
+#'    for file paths like "2019/01/file.parquet", "2019/02/file.parquet", etc.
+#'   * A character vector that defines the field names corresponding to those
+#'    path segments (that is, you're providing the names that would correspond
+#'    to a `Schema` but the types will be autodetected)
+#'   * A `HivePartitioning` or `HivePartitioningFactory`, as returned
+#'    by [hive_partition()] which parses explicit or autodetected fields from
+#'    Hive-style path segments
+#'   * `NULL` for no partitioning
+#' @param ... Additional format-specific options, passed to
+#' `FileFormat$create()`. For CSV options, note that you can specify them either
+#' with the Arrow C++ library naming ("delimiter", "quoting", etc.) or the
+#' `readr`-style naming used in [read_csv_arrow()] ("delim", "quote", etc.)
+#' @return A `DatasetFactory` object. Pass this to [open_dataset()],
+#' in a list potentially with other `DatasetFactory` objects, to create
+#' a `Dataset`.
+#' @export
+dataset_factory <- DatasetFactory$create
+
+#' @usage NULL
+#' @format NULL
+#' @rdname Dataset
+#' @export
+FileSystemDatasetFactory <- R6Class("FileSystemDatasetFactory",
+  inherit = DatasetFactory
+)
+FileSystemDatasetFactory$create <- function(filesystem,
+                                            selector,
+                                            format,
+                                            partitioning = NULL) {
+  assert_is(filesystem, "FileSystem")
+  assert_is(selector, "FileSelector")
+  assert_is(format, "FileFormat")
+
+  if (is.null(partitioning)) {
+    ptr <- dataset___FileSystemDatasetFactory__Make1(filesystem, selector, format)
+  } else if (inherits(partitioning, "PartitioningFactory")) {
+    ptr <- dataset___FileSystemDatasetFactory__Make3(filesystem, selector, format, partitioning)
+  } else if (inherits(partitioning, "Partitioning")) {
+    ptr <- dataset___FileSystemDatasetFactory__Make2(filesystem, selector, format, partitioning)
+  } else {
+    stop(
+      "Expected 'partitioning' to be NULL, PartitioningFactory or Partitioning",
+      call. = FALSE
+    )
+  }
+
+  shared_ptr(FileSystemDatasetFactory, ptr)
+}
diff --git a/r/R/dataset-format.R b/r/R/dataset-format.R
new file mode 100644
index 00000000000..8300e415e2c
--- /dev/null
+++ b/r/R/dataset-format.R
@@ -0,0 +1,168 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#' Dataset file formats
+#'
+#' @description
+#' A `FileFormat` holds information about how to read and parse the files
+#' included in a `Dataset`. There are subclasses corresponding to the supported
+#' file formats (`ParquetFileFormat` and `IpcFileFormat`).
+#'
+#' @section Factory:
+#' `FileFormat$create()` takes the following arguments:
+#' * `format`: A string identifier of the file format. Currently supported values:
+#'   * "parquet"
+#'   * "ipc"/"arrow"/"feather", all aliases for each other; for Feather, note that
+#'     only version 2 files are supported
+#'   * "csv"/"text", aliases for the same thing (because comma is the default
+#'     delimiter for text files
+#'   * "tsv", equivalent to passing `format = "text", delimiter = "\t"`
+#' * `...`: Additional format-specific options
+#'
+#'   `format = "parquet"``:
+#'   * `use_buffered_stream`: Read files through buffered input streams rather than
+#'                            loading entire row groups at once. This may be enabled
+#'                            to reduce memory overhead. Disabled by default.
+#'   * `buffer_size`: Size of buffered stream, if enabled. Default is 8KB.
+#'   * `dict_columns`: Names of columns which should be read as dictionaries.
+#'
+#'   `format = "text"`: see [CsvReadOptions]. Note that you can specify them either
+#'   with the Arrow C++ library naming ("delimiter", "quoting", etc.) or the
+#'   `readr`-style naming used in [read_csv_arrow()] ("delim", "quote", etc.)
+#'
+#' It returns the appropriate subclass of `FileFormat` (e.g. `ParquetFileFormat`)
+#' @rdname FileFormat
+#' @name FileFormat
+#' @export
+FileFormat <- R6Class("FileFormat", inherit = ArrowObject,
+  public = list(
+    ..dispatch = function() {
+      type <- self$type
+      if (type == "parquet") {
+        shared_ptr(ParquetFileFormat, self$pointer())
+      } else if (type == "ipc") {
+        shared_ptr(IpcFileFormat, self$pointer())
+      } else if (type == "csv") {
+        shared_ptr(CsvFileFormat, self$pointer())
+      } else {
+        self
+      }
+    }
+  ),
+  active = list(
+    # @description
+    # Return the `FileFormat`'s type
+    type = function() dataset___FileFormat__type_name(self)
+  )
+)
+FileFormat$create <- function(format, ...) {
+  opt_names <- names(list(...))
+  if (format %in% c("csv", "text") || any(opt_names %in% c("delim", "delimiter"))) {
+    CsvFileFormat$create(...)
+  } else if (format == c("tsv")) {
+    CsvFileFormat$create(delimiter = "\t", ...)
+  } else if (format == "parquet") {
+    ParquetFileFormat$create(...)
+  } else if (format %in% c("ipc", "arrow", "feather")) { # These are aliases for the same thing
+    shared_ptr(IpcFileFormat, dataset___IpcFileFormat__Make())
+  } else {
+    stop("Unsupported file format: ", format, call. = FALSE)
+  }
+}
+
+#' @export
+as.character.FileFormat <- function(x, ...) {
+  out <- x$type
+  # Slight hack: special case IPC -> feather, otherwise is just the type_name
+  ifelse(out == "ipc", "feather", out)
+}
+
+#' @usage NULL
+#' @format NULL
+#' @rdname FileFormat
+#' @export
+ParquetFileFormat <- R6Class("ParquetFileFormat", inherit = FileFormat)
+ParquetFileFormat$create <- function(use_buffered_stream = FALSE,
+                                     buffer_size = 8196,
+                                     dict_columns = character(0)) {
+  shared_ptr(ParquetFileFormat, dataset___ParquetFileFormat__Make(
+    use_buffered_stream, buffer_size, dict_columns))
+}
+
+#' @usage NULL
+#' @format NULL
+#' @rdname FileFormat
+#' @export
+IpcFileFormat <- R6Class("IpcFileFormat", inherit = FileFormat)
+
+#' @usage NULL
+#' @format NULL
+#' @rdname FileFormat
+#' @export
+CsvFileFormat <- R6Class("CsvFileFormat", inherit = FileFormat)
+CsvFileFormat$create <- function(..., opts = csv_file_format_parse_options(...)) {
+  shared_ptr(CsvFileFormat, dataset___CsvFileFormat__Make(opts))
+}
+
+csv_file_format_parse_options <- function(...) {
+  # Support both the readr spelling of options and the arrow spelling
+  readr_opts <- c("delim", "quote", "escape_double", "escape_backslash", "skip_empty_rows")
+  if (any(readr_opts %in% names(list(...)))) {
+    readr_to_csv_parse_options(...)
+  } else {
+    CsvParseOptions$create(...)
+  }
+}
+
+#' Format-specific write options
+#'
+#' @description
+#' A `FileWriteOptions` holds write options specific to a `FileFormat`.
+FileWriteOptions <- R6Class("FileWriteOptions", inherit = ArrowObject,
+  public = list(
+    update = function(...) {
+      if (self$type == "parquet") {
+        dataset___ParquetFileWriteOptions__update(self,
+            ParquetWriterProperties$create(...),
+            ParquetArrowWriterProperties$create(...))
+      } else if (self$type == "ipc") {
+        args <- list(...)
+        if (is.null(args$codec)) {
+          dataset___IpcFileWriteOptions__update1(self,
+              get_ipc_use_legacy_format(args$use_legacy_format),
+              get_ipc_metadata_version(args$metadata_version))
+        } else {
+          dataset___IpcFileWriteOptions__update2(self,
+              get_ipc_use_legacy_format(args$use_legacy_format),
+              args$codec,
+              get_ipc_metadata_version(args$metadata_version))
+        }
+      }
+      invisible(self)
+    }
+  ),
+  active = list(
+    type = function() dataset___FileWriteOptions__type_name(self)
+  )
+)
+FileWriteOptions$create <- function(format, ...) {
+  if (!inherits(format, "FileFormat")) {
+    format <- FileFormat$create(format)
+  }
+  options <- shared_ptr(FileWriteOptions, dataset___FileFormat__DefaultWriteOptions(format))
+  options$update(...)
+}
diff --git a/r/R/dataset-partition.R b/r/R/dataset-partition.R
new file mode 100644
index 00000000000..be47406a6d1
--- /dev/null
+++ b/r/R/dataset-partition.R
@@ -0,0 +1,113 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#' Define Partitioning for a Dataset
+#'
+#' @description
+#' Pass a `Partitioning` object to a [FileSystemDatasetFactory]'s `$create()`
+#' method to indicate how the file's paths should be interpreted to define
+#' partitioning.
+#'
+#' `DirectoryPartitioning` describes how to interpret raw path segments, in
+#' order. For example, `schema(year = int16(), month = int8())` would define
+#' partitions for file paths like "2019/01/file.parquet",
+#' "2019/02/file.parquet", etc.
+#'
+#' `HivePartitioning` is for Hive-style partitioning, which embeds field
+#' names and values in path segments, such as
+#' "/year=2019/month=2/data.parquet". Because fields are named in the path
+#' segments, order does not matter.
+#'
+#' `PartitioningFactory` subclasses instruct the `DatasetFactory` to detect
+#' partition features from the file paths.
+#' @section Factory:
+#' Both `DirectoryPartitioning$create()` and `HivePartitioning$create()`
+#' methods take a [Schema] as a single input argument. The helper
+#' function [`hive_partition(...)`][hive_partition] is shorthand for
+#' `HivePartitioning$create(schema(...))`.
+#'
+#' With `DirectoryPartitioningFactory$create()`, you can provide just the
+#' names of the path segments (in our example, `c("year", "month")`), and
+#' the `DatasetFactory` will infer the data types for those partition variables.
+#' `HivePartitioningFactory$create()` takes no arguments: both variable names
+#' and their types can be inferred from the file paths. `hive_partition()` with
+#' no arguments returns a `HivePartitioningFactory`.
+#' @name Partitioning
+#' @rdname Partitioning
+#' @export
+Partitioning <- R6Class("Partitioning", inherit = ArrowObject)
+#' @usage NULL
+#' @format NULL
+#' @rdname Partitioning
+#' @export
+DirectoryPartitioning <- R6Class("DirectoryPartitioning", inherit = Partitioning)
+DirectoryPartitioning$create <- function(schema) {
+  shared_ptr(DirectoryPartitioning, dataset___DirectoryPartitioning(schema))
+}
+
+#' @usage NULL
+#' @format NULL
+#' @rdname Partitioning
+#' @export
+HivePartitioning <- R6Class("HivePartitioning", inherit = Partitioning)
+HivePartitioning$create <- function(schema) {
+  shared_ptr(HivePartitioning, dataset___HivePartitioning(schema))
+}
+
+#' Construct Hive partitioning
+#'
+#' Hive partitioning embeds field names and values in path segments, such as
+#' "/year=2019/month=2/data.parquet".
+#'
+#' Because fields are named in the path segments, order of fields passed to
+#' `hive_partition()` does not matter.
+#' @param ... named list of [data types][data-type], passed to [schema()]
+#' @return A [HivePartitioning][Partitioning], or a `HivePartitioningFactory` if
+#' calling `hive_partition()` with no arguments.
+#' @examples
+#' \donttest{
+#' hive_partition(year = int16(), month = int8())
+#' }
+#' @export
+hive_partition <- function(...) {
+  schm <- schema(...)
+  if (length(schm) == 0) {
+    HivePartitioningFactory$create()
+  } else {
+    HivePartitioning$create(schm)
+  }
+}
+
+PartitioningFactory <- R6Class("PartitioningFactory", inherit = ArrowObject)
+
+#' @usage NULL
+#' @format NULL
+#' @rdname Partitioning
+#' @export
+DirectoryPartitioningFactory <- R6Class("DirectoryPartitioningFactory ", inherit = PartitioningFactory)
+DirectoryPartitioningFactory$create <- function(x) {
+  shared_ptr(DirectoryPartitioningFactory, dataset___DirectoryPartitioning__MakeFactory(x))
+}
+
+#' @usage NULL
+#' @format NULL
+#' @rdname Partitioning
+#' @export
+HivePartitioningFactory <- R6Class("HivePartitioningFactory", inherit = PartitioningFactory)
+HivePartitioningFactory$create <- function() {
+  shared_ptr(HivePartitioningFactory, dataset___HivePartitioning__MakeFactory())
+}
diff --git a/r/R/dataset-scan.R b/r/R/dataset-scan.R
new file mode 100644
index 00000000000..e9017825782
--- /dev/null
+++ b/r/R/dataset-scan.R
@@ -0,0 +1,180 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#' Scan the contents of a dataset
+#'
+#' @description
+#' A `Scanner` iterates over a [Dataset]'s fragments and returns data
+#' according to given row filtering and column projection. A `ScannerBuilder`
+#' can help create one.
+#'
+#' @section Factory:
+#' `Scanner$create()` wraps the `ScannerBuilder` interface to make a `Scanner`.
+#' It takes the following arguments:
+#'
+#' * `dataset`: A `Dataset` or `arrow_dplyr_query` object, as returned by the
+#'    `dplyr` methods on `Dataset`.
+#' * `projection`: A character vector of column names to select
+#' * `filter`: A `Expression` to filter the scanned rows by, or `TRUE` (default)
+#'    to keep all rows.
+#' * `use_threads`: logical: should scanning use multithreading? Default `TRUE`
+#' * `...`: Additional arguments, currently ignored
+#' @section Methods:
+#' `ScannerBuilder` has the following methods:
+#'
+#' - `$Project(cols)`: Indicate that the scan should only return columns given
+#' by `cols`, a character vector of column names
+#' - `$Filter(expr)`: Filter rows by an [Expression].
+#' - `$UseThreads(threads)`: logical: should the scan use multithreading?
+#' The method's default input is `TRUE`, but you must call the method to enable
+#' multithreading because the scanner default is `FALSE`.
+#' - `$BatchSize(batch_size)`: integer: Maximum row count of scanned record
+#' batches, default is 32K. If scanned record batches are overflowing memory
+#' then this method can be called to reduce their size.
+#' - `$schema`: Active binding, returns the [Schema] of the Dataset
+#' - `$Finish()`: Returns a `Scanner`
+#'
+#' `Scanner` currently has a single method, `$ToTable()`, which evaluates the
+#' query and returns an Arrow [Table].
+#' @rdname Scanner
+#' @name Scanner
+#' @export
+Scanner <- R6Class("Scanner", inherit = ArrowObject,
+  public = list(
+    ToTable = function() shared_ptr(Table, dataset___Scanner__ToTable(self)),
+    Scan = function() map(dataset___Scanner__Scan(self), shared_ptr, class = ScanTask)
+  ),
+  active = list(
+    schema = function() shared_ptr(Schema, dataset___Scanner__schema(self))
+  )
+)
+Scanner$create <- function(dataset,
+                           projection = NULL,
+                           filter = TRUE,
+                           use_threads = option_use_threads(),
+                           batch_size = NULL,
+                           ...) {
+  if (inherits(dataset, "arrow_dplyr_query")) {
+    return(Scanner$create(
+      dataset$.data,
+      dataset$selected_columns,
+      dataset$filtered_rows,
+      use_threads,
+      ...
+    ))
+  }
+  if (inherits(dataset, c("data.frame", "RecordBatch", "Table"))) {
+    dataset <- InMemoryDataset$create(dataset)
+  }
+  assert_is(dataset, "Dataset")
+
+  scanner_builder <- dataset$NewScan()
+  if (use_threads) {
+    scanner_builder$UseThreads()
+  }
+  if (!is.null(projection)) {
+    scanner_builder$Project(projection)
+  }
+  if (!isTRUE(filter)) {
+    scanner_builder$Filter(filter)
+  }
+  if (is_integerish(batch_size)) {
+    scanner_builder$BatchSize(batch_size)
+  }
+  scanner_builder$Finish()
+}
+
+#' @export
+names.Scanner <- function(x) names(x$schema)
+
+ScanTask <- R6Class("ScanTask", inherit = ArrowObject,
+  public = list(
+    Execute = function() map(dataset___ScanTask__get_batches(self), shared_ptr, class = RecordBatch)
+  )
+)
+
+#' Apply a function to a stream of RecordBatches
+#'
+#' As an alternative to calling `collect()` on a `Dataset` query, you can
+#' use this function to access the stream of `RecordBatch`es in the `Dataset`.
+#' This lets you aggregate on each chunk and pull the intermediate results into
+#' a `data.frame` for further aggregation, even if you couldn't fit the whole
+#' `Dataset` result in memory.
+#'
+#' This is experimental and not recommended for production use.
+#'
+#' @param X A `Dataset` or `arrow_dplyr_query` object, as returned by the
+#' `dplyr` methods on `Dataset`.
+#' @param FUN A function or `purrr`-style lambda expression to apply to each
+#' batch
+#' @param ... Additional arguments passed to `FUN`
+#' @param .data.frame logical: collect the resulting chunks into a single
+#' `data.frame`? Default `TRUE`
+#' @export
+map_batches <- function(X, FUN, ..., .data.frame = TRUE) {
+  if (.data.frame) {
+    lapply <- map_dfr
+  }
+  scanner <- Scanner$create(ensure_group_vars(X))
+  FUN <- as_mapper(FUN)
+  # message("Making ScanTasks")
+  lapply(scanner$Scan(), function(scan_task) {
+    # This outer lapply could be parallelized
+    # message("Making Batches")
+    lapply(scan_task$Execute(), function(batch) {
+      # message("Processing Batch")
+      # This inner lapply cannot be parallelized
+      # TODO: wrap batch in arrow_dplyr_query with X$selected_columns and X$group_by_vars
+      # if X is arrow_dplyr_query, if some other arg (.dplyr?) == TRUE
+      FUN(batch, ...)
+    })
+  })
+}
+
+#' @usage NULL
+#' @format NULL
+#' @rdname Scanner
+#' @export
+ScannerBuilder <- R6Class("ScannerBuilder", inherit = ArrowObject,
+  public = list(
+    Project = function(cols) {
+      assert_is(cols, "character")
+      dataset___ScannerBuilder__Project(self, cols)
+      self
+    },
+    Filter = function(expr) {
+      assert_is(expr, "Expression")
+      dataset___ScannerBuilder__Filter(self, expr)
+      self
+    },
+    UseThreads = function(threads = option_use_threads()) {
+      dataset___ScannerBuilder__UseThreads(self, threads)
+      self
+    },
+    BatchSize = function(batch_size) {
+      dataset___ScannerBuilder__BatchSize(self, batch_size)
+      self
+    },
+    Finish = function() unique_ptr(Scanner, dataset___ScannerBuilder__Finish(self))
+  ),
+  active = list(
+    schema = function() shared_ptr(Schema, dataset___ScannerBuilder__schema(self))
+  )
+)
+
+#' @export
+names.ScannerBuilder <- function(x) names(x$schema)
diff --git a/r/R/dataset-write.R b/r/R/dataset-write.R
index 8baca1e52bb..c5c92926715 100644
--- a/r/R/dataset-write.R
+++ b/r/R/dataset-write.R
@@ -21,62 +21,73 @@
 #' binary storage formats, and by specifying relevant partitioning, you can
 #' make it much faster to read and query.
 #'
-#' @param dataset [Dataset] or `arrow_dplyr_query`. If a `arrow_dplyr_query`,
-#' note that `select()` or `filter()` queries are not currently supported.
-#' @param path string path to a directory to write to (directory will be
-#' created if it does not exist)
-#' @param format file format to write the dataset to. Currently only "feather"
-#' (aka "ipc") is supported.
+#' @param dataset [Dataset], [RecordBatch], [Table], `arrow_dplyr_query`, or
+#' `data.frame`. If an `arrow_dplyr_query` or `grouped_df`,
+#' `schema` and `partitioning` will be taken from the result of any `select()`
+#' and `group_by()` operations done on the dataset. `filter()` queries will be
+#' applied to restrict written rows.
+#' Note that `select()`-ed columns may not be renamed.
+#' @param path string path, URI, or `SubTreeFileSystem` referencing a directory
+#' to write to (directory will be created if it does not exist)
+#' @param format file format to write the dataset to. Currently supported
+#' formats are "feather" (aka "ipc") and "parquet". Default is to write to the
+#' same format as `dataset`.
 #' @param partitioning `Partitioning` or a character vector of columns to
 #' use as partition keys (to be written as path segments). Default is to
 #' use the current `group_by()` columns.
+#' @param basename_template string template for the names of files to be written.
+#' Must contain `"{i}"`, which will be replaced with an autoincremented
+#' integer to generate basenames of datafiles. For example, `"part-{i}.feather"`
+#' will yield `"part-0.feather", ...`.
 #' @param hive_style logical: write partition segments as Hive-style
 #' (`key1=value1/key2=value2/file.ext`) or as just bare values. Default is `TRUE`.
-#' @param ... additional arguments, passed to `dataset$write()`
+#' @param ... additional format-specific arguments. For available Parquet
+#' options, see [write_parquet()]. The available Feather options are
+#' - `use_legacy_format` logical: write data formatted so that Arrow libraries
+#'   versions 0.14 and lower can read it. Default is `FALSE`. You can also
+#'   enable this by setting the environment variable `ARROW_PRE_0_15_IPC_FORMAT=1`.
+#' - `metadata_version`: A string like "V5" or the equivalent integer indicating
+#'   the Arrow IPC MetadataVersion. Default (NULL) will use the latest version,
+#'   unless the environment variable `ARROW_PRE_1_0_METADATA_VERSION=1`, in
+#'   which case it will be V4.
+#' - `codec`: A [Codec] which will be used to compress body buffers of written
+#'   files. Default (NULL) will not compress body buffers.
 #' @return The input `dataset`, invisibly
 #' @export
 write_dataset <- function(dataset,
                           path,
-                          format = dataset$format$type,
+                          format = dataset$format,
                           partitioning = dplyr::group_vars(dataset),
+                          basename_template = paste0("part-{i}.", as.character(format)),
                           hive_style = TRUE,
                           ...) {
   if (inherits(dataset, "arrow_dplyr_query")) {
-    force(partitioning) # get the group_vars before we drop the object
-    # TODO: Write a filtered/projected dataset
-    if (!isTRUE(dataset$filtered_rows)) {
-      stop("Writing a filtered dataset is not yet supported", call. = FALSE)
+    # We can select a subset of columns but we can't rename them
+    if (!all(dataset$selected_columns == names(dataset$selected_columns))) {
+      stop("Renaming columns when writing a dataset is not yet supported", call. = FALSE)
     }
-    if (!identical(dataset$selected_columns, set_names(names(dataset$.data)))) {
-      # TODO: actually, we can do this?
-      stop("TODO", call. = FALSE)
-    }
-    dataset <- dataset$.data
-  }
-  if (!inherits(dataset, "Dataset")) {
-    stop("'dataset' must be a Dataset", call. = FALSE)
-    # TODO: This does not exist yet (in the R bindings at least)
-    # dataset <- InMemoryDataset$create(dataset)
-  }
-
-  if (!inherits(format, "FileFormat")) {
-    format <- FileFormat$create(format, ...)
-  }
-  if (!inherits(format, "IpcFileFormat")) {
-    stop(
-      "Unsupported format; datasets currently can only be written to IPC/Feather format",
-      call. = FALSE
-    )
+    # partitioning vars need to be in the `select` schema
+    dataset <- ensure_group_vars(dataset)
+  } else if (inherits(dataset, "grouped_df")) {
+    force(partitioning)
+    # Drop the grouping metadata before writing; we've already consumed it
+    # now to construct `partitioning` and don't want it in the metadata$r
+    dataset <- dplyr::ungroup(dataset)
   }
 
+  scanner <- Scanner$create(dataset)
   if (!inherits(partitioning, "Partitioning")) {
-    # TODO: tidyselect?
-    partition_schema <- dataset$schema[partitioning]
+    partition_schema <- scanner$schema[partitioning]
     if (isTRUE(hive_style)) {
       partitioning <- HivePartitioning$create(partition_schema)
     } else {
       partitioning <- DirectoryPartitioning$create(partition_schema)
     }
   }
-  dataset$write(path, format = format, partitioning = partitioning, ...)
+
+  path_and_fs <- get_path_and_filesystem(path)
+  options <- FileWriteOptions$create(format, table = scanner, ...)
+
+  dataset___Dataset__Write(options, path_and_fs$fs, path_and_fs$path,
+                           partitioning, basename_template, scanner)
 }
diff --git a/r/R/dataset.R b/r/R/dataset.R
index ab49b70c4f1..7b1d6609295 100644
--- a/r/R/dataset.R
+++ b/r/R/dataset.R
@@ -133,9 +133,6 @@ open_dataset <- function(sources,
 #'   may also replace the dataset's schema by using `ds$schema <- new_schema`.
 #'   This method currently supports only adding, removing, or reordering
 #'   fields in the schema: you cannot alter or cast the field types.
-#' - `$write(path, filesystem, schema, format, partitioning)`: writes the
-#'   dataset to `path` in the `format` file format, partitioned by `partitioning`,
-#'   and invisibly returns `self`. See [write_dataset()].
 #'
 #' `FileSystemDataset` has the following methods:
 #' - `$files`: Active binding, returns the files of the `FileSystemDataset`
@@ -162,20 +159,7 @@ Dataset <- R6Class("Dataset", inherit = ArrowObject,
     # Start a new scan of the data
     # @return A [ScannerBuilder]
     NewScan = function() unique_ptr(ScannerBuilder, dataset___Dataset__NewScan(self)),
-    ToString = function() self$schema$ToString(),
-    write = function(path, filesystem = NULL, schema = self$schema, format, partitioning) {
-      if (!inherits(filesystem, "FileSystem")) {
-        if (grepl("://", path)) {
-          fs_from_uri <- FileSystem$from_uri(path)
-          filesystem <- fs_from_uri$fs
-          path <- fs_from_uri$path
-        } else {
-          filesystem <- LocalFileSystem$create()
-        }
-      }
-      dataset___Dataset__Write(self, schema, format, filesystem, path, partitioning)
-      invisible(self)
-    }
+    ToString = function() self$schema$ToString()
   ),
   active = list(
     schema = function(schema) {
@@ -199,15 +183,6 @@ Dataset <- R6Class("Dataset", inherit = ArrowObject,
 )
 Dataset$create <- open_dataset
 
-#' @export
-names.Dataset <- function(x) names(x$schema)
-
-#' @export
-dim.Dataset <- function(x) c(x$num_rows, x$num_cols)
-
-#' @export
-c.Dataset <- function(...) Dataset$create(list(...))
-
 #' @name FileSystemDataset
 #' @rdname Dataset
 #' @export
@@ -239,6 +214,11 @@ FileSystemDataset <- R6Class("FileSystemDataset", inherit = Dataset,
     format = function() {
       shared_ptr(FileFormat, dataset___FileSystemDataset__format(self))$..dispatch()
     },
+    # @description
+    # Return the filesystem of files in this `Dataset`
+    filesystem = function() {
+      shared_ptr(FileSystem, dataset___FileSystemDataset__filesystem(self))$..dispatch()
+    },
     num_rows = function() {
       if (inherits(self$format, "ParquetFileFormat")) {
         # It's generally fast enough to skim the files directly
@@ -269,372 +249,26 @@ UnionDataset <- R6Class("UnionDataset", inherit = Dataset,
   )
 )
 
-#' @usage NULL
-#' @format NULL
+#' @name InMemoryDataset
 #' @rdname Dataset
 #' @export
-DatasetFactory <- R6Class("DatasetFactory", inherit = ArrowObject,
-  public = list(
-    Finish = function(schema = NULL, unify_schemas = FALSE) {
-      if (is.null(schema)) {
-        ptr <- dataset___DatasetFactory__Finish1(self, unify_schemas)
-      } else {
-        ptr <- dataset___DatasetFactory__Finish2(self, schema)
-      }
-      shared_ptr(Dataset, ptr)$..dispatch()
-    },
-    Inspect = function(unify_schemas = FALSE) {
-      shared_ptr(Schema, dataset___DatasetFactory__Inspect(self, unify_schemas))
-    }
-  )
-)
-DatasetFactory$create <- function(x,
-                                  filesystem = NULL,
-                                  format = c("parquet", "arrow", "ipc", "feather", "csv", "tsv", "text"),
-                                  partitioning = NULL,
-                                  ...) {
-  if (is_list_of(x, "DatasetFactory")) {
-    return(shared_ptr(DatasetFactory, dataset___UnionDatasetFactory__Make(x)))
+InMemoryDataset <- R6Class("InMemoryDataset", inherit = Dataset)
+InMemoryDataset$create <- function(x) {
+  if (!inherits(x, "Table")) {
+    x <- Table$create(x)
   }
-  if (!is.string(x)) {
-    stop("'x' must be a string or a list of DatasetFactory", call. = FALSE)
-  }
-
-  if (!inherits(filesystem, "FileSystem")) {
-    if (grepl("://", x)) {
-      fs_from_uri <- FileSystem$from_uri(x)
-      filesystem <- fs_from_uri$fs
-      x <- fs_from_uri$path
-    } else {
-      filesystem <- LocalFileSystem$create()
-    }
-  }
-  selector <- FileSelector$create(x, allow_not_found = FALSE, recursive = TRUE)
-
-  if (is.character(format)) {
-    format <- FileFormat$create(match.arg(format), ...)
-  } else {
-    assert_is(format, "FileFormat")
-  }
-
-  if (!is.null(partitioning)) {
-    if (inherits(partitioning, "Schema")) {
-      partitioning <- DirectoryPartitioning$create(partitioning)
-    } else if (is.character(partitioning)) {
-      # These are the column/field names, and we should autodetect their types
-      partitioning <- DirectoryPartitioningFactory$create(partitioning)
-    }
-  }
-  FileSystemDatasetFactory$create(filesystem, selector, format, partitioning)
+  shared_ptr(InMemoryDataset, dataset___InMemoryDataset__create(x))
 }
 
 
-#' Create a DatasetFactory
-#'
-#' A [Dataset] can constructed using one or more [DatasetFactory]s.
-#' This function helps you construct a `DatasetFactory` that you can pass to
-#' [open_dataset()].
-#'
-#' If you would only have a single `DatasetFactory` (for example, you have a
-#' single directory containing Parquet files), you can call `open_dataset()`
-#' directly. Use `dataset_factory()` when you
-#' want to combine different directories, file systems, or file formats.
-#'
-#' @param x A string file x containing data files, or
-#' a list of `DatasetFactory` objects whose datasets should be
-#' grouped. If this argument is specified it will be used to construct a
-#' `UnionDatasetFactory` and other arguments will be ignored.
-#' @param filesystem A [FileSystem] object; if omitted, the `FileSystem` will
-#' be detected from `x`
-#' @param format A [FileFormat] object, or a string identifier of the format of
-#' the files in `x`. Currently supported values:
-#' * "parquet"
-#' * "ipc"/"arrow"/"feather", all aliases for each other; for Feather, note that
-#'   only version 2 files are supported
-#' * "csv"/"text", aliases for the same thing (because comma is the default
-#'   delimiter for text files
-#' * "tsv", equivalent to passing `format = "text", delimiter = "\t"`
-#'
-#' Default is "parquet", unless a `delimiter` is also specified, in which case
-#' it is assumed to be "text".
-#' @param partitioning One of
-#'   * A `Schema`, in which case the file paths relative to `sources` will be
-#'    parsed, and path segments will be matched with the schema fields. For
-#'    example, `schema(year = int16(), month = int8())` would create partitions
-#'    for file paths like "2019/01/file.parquet", "2019/02/file.parquet", etc.
-#'   * A character vector that defines the field names corresponding to those
-#'    path segments (that is, you're providing the names that would correspond
-#'    to a `Schema` but the types will be autodetected)
-#'   * A `HivePartitioning` or `HivePartitioningFactory`, as returned
-#'    by [hive_partition()] which parses explicit or autodetected fields from
-#'    Hive-style path segments
-#'   * `NULL` for no partitioning
-#' @param ... Additional format-specific options, passed to
-#' `FileFormat$create()`. For CSV options, note that you can specify them either
-#' with the Arrow C++ library naming ("delimiter", "quoting", etc.) or the
-#' `readr`-style naming used in [read_csv_arrow()] ("delim", "quote", etc.)
-#' @return A `DatasetFactory` object. Pass this to [open_dataset()],
-#' in a list potentially with other `DatasetFactory` objects, to create
-#' a `Dataset`.
-#' @export
-dataset_factory <- DatasetFactory$create
-
-#' @usage NULL
-#' @format NULL
-#' @rdname Dataset
-#' @export
-FileSystemDatasetFactory <- R6Class("FileSystemDatasetFactory",
-  inherit = DatasetFactory
-)
-FileSystemDatasetFactory$create <- function(filesystem,
-                                            selector,
-                                            format,
-                                            partitioning = NULL) {
-  assert_is(filesystem, "FileSystem")
-  assert_is(selector, "FileSelector")
-  assert_is(format, "FileFormat")
-
-  if (is.null(partitioning)) {
-    ptr <- dataset___FileSystemDatasetFactory__Make1(filesystem, selector, format)
-  } else if (inherits(partitioning, "PartitioningFactory")) {
-    ptr <- dataset___FileSystemDatasetFactory__Make3(filesystem, selector, format, partitioning)
-  } else if (inherits(partitioning, "Partitioning")) {
-    ptr <- dataset___FileSystemDatasetFactory__Make2(filesystem, selector, format, partitioning)
-  } else {
-    stop(
-      "Expected 'partitioning' to be NULL, PartitioningFactory or Partitioning",
-      call. = FALSE
-    )
-  }
-
-  shared_ptr(FileSystemDatasetFactory, ptr)
-}
-
-#' Dataset file formats
-#'
-#' @description
-#' A `FileFormat` holds information about how to read and parse the files
-#' included in a `Dataset`. There are subclasses corresponding to the supported
-#' file formats (`ParquetFileFormat` and `IpcFileFormat`).
-#'
-#' @section Factory:
-#' `FileFormat$create()` takes the following arguments:
-#' * `format`: A string identifier of the file format. Currently supported values:
-#'   * "parquet"
-#'   * "ipc"/"arrow"/"feather", all aliases for each other; for Feather, note that
-#'     only version 2 files are supported
-#'   * "csv"/"text", aliases for the same thing (because comma is the default
-#'     delimiter for text files
-#'   * "tsv", equivalent to passing `format = "text", delimiter = "\t"`
-#' * `...`: Additional format-specific options
-#'
-#'   `format = "parquet"``:
-#'   * `use_buffered_stream`: Read files through buffered input streams rather than
-#'                            loading entire row groups at once. This may be enabled
-#'                            to reduce memory overhead. Disabled by default.
-#'   * `buffer_size`: Size of buffered stream, if enabled. Default is 8KB.
-#'   * `dict_columns`: Names of columns which should be read as dictionaries.
-#'
-#'   `format = "text"`: see [CsvReadOptions]. Note that you can specify them either
-#'   with the Arrow C++ library naming ("delimiter", "quoting", etc.) or the
-#'   `readr`-style naming used in [read_csv_arrow()] ("delim", "quote", etc.)
-#'
-#' It returns the appropriate subclass of `FileFormat` (e.g. `ParquetFileFormat`)
-#' @rdname FileFormat
-#' @name FileFormat
-#' @export
-FileFormat <- R6Class("FileFormat", inherit = ArrowObject,
-  public = list(
-    ..dispatch = function() {
-      type <- self$type
-      if (type == "parquet") {
-        shared_ptr(ParquetFileFormat, self$pointer())
-      } else if (type == "ipc") {
-        shared_ptr(IpcFileFormat, self$pointer())
-      } else if (type == "csv") {
-        shared_ptr(CsvFileFormat, self$pointer())
-      } else {
-        self
-      }
-    }
-  ),
-  active = list(
-    # @description
-    # Return the `FileFormat`'s type
-    type = function() dataset___FileFormat__type_name(self)
-  )
-)
-FileFormat$create <- function(format, ...) {
-  opt_names <- names(list(...))
-  if (format %in% c("csv", "text") || any(opt_names %in% c("delim", "delimiter"))) {
-    CsvFileFormat$create(...)
-  } else if (format == c("tsv")) {
-    CsvFileFormat$create(delimiter = "\t", ...)
-  } else if (format == "parquet") {
-    ParquetFileFormat$create(...)
-  } else if (format %in% c("ipc", "arrow", "feather")) { # These are aliases for the same thing
-    shared_ptr(IpcFileFormat, dataset___IpcFileFormat__Make())
-  } else {
-    stop("Unsupported file format: ", format, call. = FALSE)
-  }
-}
-
-#' @usage NULL
-#' @format NULL
-#' @rdname FileFormat
-#' @export
-ParquetFileFormat <- R6Class("ParquetFileFormat", inherit = FileFormat)
-ParquetFileFormat$create <- function(use_buffered_stream = FALSE,
-                                     buffer_size = 8196,
-                                     dict_columns = character(0)) {
-  shared_ptr(ParquetFileFormat, dataset___ParquetFileFormat__Make(
-    use_buffered_stream, buffer_size, dict_columns))
-}
-
-#' @usage NULL
-#' @format NULL
-#' @rdname FileFormat
 #' @export
-IpcFileFormat <- R6Class("IpcFileFormat", inherit = FileFormat)
+names.Dataset <- function(x) names(x$schema)
 
-#' @usage NULL
-#' @format NULL
-#' @rdname FileFormat
 #' @export
-CsvFileFormat <- R6Class("CsvFileFormat", inherit = FileFormat)
-CsvFileFormat$create <- function(..., opts = csv_file_format_parse_options(...)) {
-  shared_ptr(CsvFileFormat, dataset___CsvFileFormat__Make(opts))
-}
-
-csv_file_format_parse_options <- function(...) {
-  # Support both the readr spelling of options and the arrow spelling
-  readr_opts <- c("delim", "quote", "escape_double", "escape_backslash", "skip_empty_rows")
-  if (any(readr_opts %in% names(list(...)))) {
-    readr_to_csv_parse_options(...)
-  } else {
-    CsvParseOptions$create(...)
-  }
-}
-
-#' Scan the contents of a dataset
-#'
-#' @description
-#' A `Scanner` iterates over a [Dataset]'s fragments and returns data
-#' according to given row filtering and column projection. A `ScannerBuilder`
-#' can help create one.
-#'
-#' @section Factory:
-#' `Scanner$create()` wraps the `ScannerBuilder` interface to make a `Scanner`.
-#' It takes the following arguments:
-#'
-#' * `dataset`: A `Dataset` or `arrow_dplyr_query` object, as returned by the
-#'    `dplyr` methods on `Dataset`.
-#' * `projection`: A character vector of column names to select
-#' * `filter`: A `Expression` to filter the scanned rows by, or `TRUE` (default)
-#'    to keep all rows.
-#' * `use_threads`: logical: should scanning use multithreading? Default `TRUE`
-#' * `...`: Additional arguments, currently ignored
-#' @section Methods:
-#' `ScannerBuilder` has the following methods:
-#'
-#' - `$Project(cols)`: Indicate that the scan should only return columns given
-#' by `cols`, a character vector of column names
-#' - `$Filter(expr)`: Filter rows by an [Expression].
-#' - `$UseThreads(threads)`: logical: should the scan use multithreading?
-#' The method's default input is `TRUE`, but you must call the method to enable
-#' multithreading because the scanner default is `FALSE`.
-#' - `$BatchSize(batch_size)`: integer: Maximum row count of scanned record
-#' batches, default is 32K. If scanned record batches are overflowing memory
-#' then this method can be called to reduce their size.
-#' - `$schema`: Active binding, returns the [Schema] of the Dataset
-#' - `$Finish()`: Returns a `Scanner`
-#'
-#' `Scanner` currently has a single method, `$ToTable()`, which evaluates the
-#' query and returns an Arrow [Table].
-#' @rdname Scanner
-#' @name Scanner
-#' @export
-Scanner <- R6Class("Scanner", inherit = ArrowObject,
-  public = list(
-    ToTable = function() shared_ptr(Table, dataset___Scanner__ToTable(self)),
-    Scan = function() map(dataset___Scanner__Scan(self), shared_ptr, class = ScanTask)
-  )
-)
-Scanner$create <- function(dataset,
-                           projection = NULL,
-                           filter = TRUE,
-                           use_threads = option_use_threads(),
-                           batch_size = NULL,
-                           ...) {
-  if (inherits(dataset, "arrow_dplyr_query") && inherits(dataset$.data, "Dataset")) {
-    return(Scanner$create(
-      dataset$.data,
-      dataset$selected_columns,
-      dataset$filtered_rows,
-      use_threads,
-      ...
-    ))
-  }
-  assert_is(dataset, "Dataset")
-  scanner_builder <- dataset$NewScan()
-  if (use_threads) {
-    scanner_builder$UseThreads()
-  }
-  if (!is.null(projection)) {
-    scanner_builder$Project(projection)
-  }
-  if (!isTRUE(filter)) {
-    scanner_builder$Filter(filter)
-  }
-  if (is_integerish(batch_size)) {
-    scanner_builder$BatchSize(batch_size)
-  }
-  scanner_builder$Finish()
-}
-
-ScanTask <- R6Class("ScanTask", inherit = ArrowObject,
-  public = list(
-    Execute = function() map(dataset___ScanTask__get_batches(self), shared_ptr, class = RecordBatch)
-  )
-)
+dim.Dataset <- function(x) c(x$num_rows, x$num_cols)
 
-#' Apply a function to a stream of RecordBatches
-#'
-#' As an alternative to calling `collect()` on a `Dataset` query, you can
-#' use this function to access the stream of `RecordBatch`es in the `Dataset`.
-#' This lets you aggregate on each chunk and pull the intermediate results into
-#' a `data.frame` for further aggregation, even if you couldn't fit the whole
-#' `Dataset` result in memory.
-#'
-#' This is experimental and not recommended for production use.
-#'
-#' @param X A `Dataset` or `arrow_dplyr_query` object, as returned by the
-#' `dplyr` methods on `Dataset`.
-#' @param FUN A function or `purrr`-style lambda expression to apply to each
-#' batch
-#' @param ... Additional arguments passed to `FUN`
-#' @param .data.frame logical: collect the resulting chunks into a single
-#' `data.frame`? Default `TRUE`
 #' @export
-map_batches <- function(X, FUN, ..., .data.frame = TRUE) {
-  if (.data.frame) {
-    lapply <- map_dfr
-  }
-  scanner <- Scanner$create(ensure_group_vars(X))
-  FUN <- as_mapper(FUN)
-  # message("Making ScanTasks")
-  lapply(scanner$Scan(), function(scan_task) {
-    # This outer lapply could be parallelized
-    # message("Making Batches")
-    lapply(scan_task$Execute(), function(batch) {
-      # message("Processing Batch")
-      # This inner lapply cannot be parallelized
-      # TODO: wrap batch in arrow_dplyr_query with X$selected_columns and X$group_by_vars
-      # if X is arrow_dplyr_query, if some other arg (.dplyr?) == TRUE
-      FUN(batch, ...)
-    })
-  })
-}
+c.Dataset <- function(...) Dataset$create(list(...))
 
 #' @export
 head.Dataset <- function(x, n = 6L, ...) {
@@ -703,134 +337,3 @@ take_dataset_rows <- function(x, i) {
   # Now sort
   tab$Take(result_order - 1L)
 }
-
-#' @usage NULL
-#' @format NULL
-#' @rdname Scanner
-#' @export
-ScannerBuilder <- R6Class("ScannerBuilder", inherit = ArrowObject,
-  public = list(
-    Project = function(cols) {
-      assert_is(cols, "character")
-      dataset___ScannerBuilder__Project(self, cols)
-      self
-    },
-    Filter = function(expr) {
-      assert_is(expr, "Expression")
-      dataset___ScannerBuilder__Filter(self, expr)
-      self
-    },
-    UseThreads = function(threads = option_use_threads()) {
-      dataset___ScannerBuilder__UseThreads(self, threads)
-      self
-    },
-    BatchSize = function(batch_size) {
-      dataset___ScannerBuilder__BatchSize(self, batch_size)
-      self
-    },
-    Finish = function() unique_ptr(Scanner, dataset___ScannerBuilder__Finish(self))
-  ),
-  active = list(
-    schema = function() shared_ptr(Schema, dataset___ScannerBuilder__schema(self))
-  )
-)
-
-#' @export
-names.ScannerBuilder <- function(x) names(x$schema)
-
-#' Define Partitioning for a Dataset
-#'
-#' @description
-#' Pass a `Partitioning` object to a [FileSystemDatasetFactory]'s `$create()`
-#' method to indicate how the file's paths should be interpreted to define
-#' partitioning.
-#'
-#' `DirectoryPartitioning` describes how to interpret raw path segments, in
-#' order. For example, `schema(year = int16(), month = int8())` would define
-#' partitions for file paths like "2019/01/file.parquet",
-#' "2019/02/file.parquet", etc.
-#'
-#' `HivePartitioning` is for Hive-style partitioning, which embeds field
-#' names and values in path segments, such as
-#' "/year=2019/month=2/data.parquet". Because fields are named in the path
-#' segments, order does not matter.
-#'
-#' `PartitioningFactory` subclasses instruct the `DatasetFactory` to detect
-#' partition features from the file paths.
-#' @section Factory:
-#' Both `DirectoryPartitioning$create()` and `HivePartitioning$create()`
-#' methods take a [Schema] as a single input argument. The helper
-#' function [`hive_partition(...)`][hive_partition] is shorthand for
-#' `HivePartitioning$create(schema(...))`.
-#'
-#' With `DirectoryPartitioningFactory$create()`, you can provide just the
-#' names of the path segments (in our example, `c("year", "month")`), and
-#' the `DatasetFactory` will infer the data types for those partition variables.
-#' `HivePartitioningFactory$create()` takes no arguments: both variable names
-#' and their types can be inferred from the file paths. `hive_partition()` with
-#' no arguments returns a `HivePartitioningFactory`.
-#' @name Partitioning
-#' @rdname Partitioning
-#' @export
-Partitioning <- R6Class("Partitioning", inherit = ArrowObject)
-#' @usage NULL
-#' @format NULL
-#' @rdname Partitioning
-#' @export
-DirectoryPartitioning <- R6Class("DirectoryPartitioning", inherit = Partitioning)
-DirectoryPartitioning$create <- function(schema) {
-  shared_ptr(DirectoryPartitioning, dataset___DirectoryPartitioning(schema))
-}
-
-#' @usage NULL
-#' @format NULL
-#' @rdname Partitioning
-#' @export
-HivePartitioning <- R6Class("HivePartitioning", inherit = Partitioning)
-HivePartitioning$create <- function(schema) {
-  shared_ptr(HivePartitioning, dataset___HivePartitioning(schema))
-}
-
-#' Construct Hive partitioning
-#'
-#' Hive partitioning embeds field names and values in path segments, such as
-#' "/year=2019/month=2/data.parquet".
-#'
-#' Because fields are named in the path segments, order of fields passed to
-#' `hive_partition()` does not matter.
-#' @param ... named list of [data types][data-type], passed to [schema()]
-#' @return A [HivePartitioning][Partitioning], or a `HivePartitioningFactory` if
-#' calling `hive_partition()` with no arguments.
-#' @examples
-#' \donttest{
-#' hive_partition(year = int16(), month = int8())
-#' }
-#' @export
-hive_partition <- function(...) {
-  schm <- schema(...)
-  if (length(schm) == 0) {
-    HivePartitioningFactory$create()
-  } else {
-    HivePartitioning$create(schm)
-  }
-}
-
-PartitioningFactory <- R6Class("PartitioningFactory", inherit = ArrowObject)
-
-#' @usage NULL
-#' @format NULL
-#' @rdname Partitioning
-#' @export
-DirectoryPartitioningFactory <- R6Class("DirectoryPartitioningFactory ", inherit = PartitioningFactory)
-DirectoryPartitioningFactory$create <- function(x) {
-  shared_ptr(DirectoryPartitioningFactory, dataset___DirectoryPartitioning__MakeFactory(x))
-}
-
-#' @usage NULL
-#' @format NULL
-#' @rdname Partitioning
-#' @export
-HivePartitioningFactory <- R6Class("HivePartitioningFactory", inherit = PartitioningFactory)
-HivePartitioningFactory$create <- function() {
-  shared_ptr(HivePartitioningFactory, dataset___HivePartitioning__MakeFactory())
-}
diff --git a/r/R/enums.R b/r/R/enums.R
index 05905710231..14910bc92e0 100644
--- a/r/R/enums.R
+++ b/r/R/enums.R
@@ -66,18 +66,19 @@ Type <- enum("Type::type",
   INTERVAL_MONTHS = 21L,
   INTERVAL_DAY_TIME = 22L,
   DECIMAL = 23L,
-  LIST = 24L,
-  STRUCT = 25L,
-  SPARSE_UNION = 26L,
-  DENSE_UNION = 27L,
-  DICTIONARY = 28L,
-  MAP = 29L,
-  EXTENSION = 30L,
-  FIXED_SIZE_LIST = 31L,
-  DURATION = 32L,
-  LARGE_STRING = 33L,
-  LARGE_BINARY = 34L,
-  LARGE_LIST = 35L
+  DECIMAL256 = 24L,
+  LIST = 25L,
+  STRUCT = 26L,
+  SPARSE_UNION = 27L,
+  DENSE_UNION = 28L,
+  DICTIONARY = 29L,
+  MAP = 30L,
+  EXTENSION = 31L,
+  FIXED_SIZE_LIST = 32L,
+  DURATION = 33L,
+  LARGE_STRING = 34L,
+  LARGE_BINARY = 35L,
+  LARGE_LIST = 36L
 )
 
 #' @rdname enums
diff --git a/r/R/feather.R b/r/R/feather.R
index 9b8dc8c5121..52f8b59ece6 100644
--- a/r/R/feather.R
+++ b/r/R/feather.R
@@ -24,7 +24,8 @@
 #' and the version 2 specification, which is the Apache Arrow IPC file format.
 #'
 #' @param x `data.frame`, [RecordBatch], or [Table]
-#' @param sink A string file path or [OutputStream]
+#' @param sink A string file path, URI, or [OutputStream], or path in a file
+#' system (`SubTreeFileSystem`)
 #' @param version integer Feather file version. Version 2 is the current.
 #' Version 1 is the more limited legacy format.
 #' @param chunk_size For V2 files, the number of rows that each chunk of data
@@ -105,11 +106,10 @@ write_feather <- function(x,
   }
   assert_is(x, "Table")
 
-  if (is.string(sink)) {
-    sink <- FileOutputStream$create(sink)
+  if (!inherits(sink, "OutputStream")) {
+    sink <- make_output_stream(sink)
     on.exit(sink$close())
   }
-  assert_is(sink, "OutputStream")
   ipc___WriteFeather__Table(sink, x, version, chunk_size, compression, compression_level)
   invisible(x_out)
 }
@@ -142,16 +142,15 @@ write_feather <- function(x,
 #' df <- read_feather(tf, col_select = starts_with("d"))
 #' }
 read_feather <- function(file, col_select = NULL, as_data_frame = TRUE, ...) {
-  if (!inherits(file, "InputStream")) {
+  if (!inherits(file, "RandomAccessFile")) {
     file <- make_readable_file(file)
     on.exit(file$close())
   }
   reader <- FeatherReader$create(file, ...)
 
-  all_columns <- ipc___feather___Reader__column_names(reader)
   col_select <- enquo(col_select)
   columns <- if (!quo_is_null(col_select)) {
-    vars_select(all_columns, !!col_select)
+    vars_select(names(reader), !!col_select)
   }
 
   out <- reader$Read(columns)
@@ -198,10 +197,14 @@ FeatherReader <- R6Class("FeatherReader", inherit = ArrowObject,
   ),
   active = list(
     # versions are officially 2 for V1 and 3 for V2 :shrug:
-    version = function() ipc___feather___Reader__version(self) - 1L
+    version = function() ipc___feather___Reader__version(self) - 1L,
+    column_names = function() ipc___feather___Reader__column_names(self)
   )
 )
 
+#' @export
+names.FeatherReader <- function(x) x$column_names
+
 FeatherReader$create <- function(file, mmap = TRUE, ...) {
   assert_is(file, "RandomAccessFile")
   shared_ptr(FeatherReader, ipc___feather___Reader__Open(file))
diff --git a/r/R/filesystem.R b/r/R/filesystem.R
index f0e123ac4cd..f8b32b8b312 100644
--- a/r/R/filesystem.R
+++ b/r/R/filesystem.R
@@ -122,11 +122,39 @@ FileSelector$create <- function(base_dir, allow_not_found = FALSE, recursive = F
 #'
 #' @section Factory:
 #'
-#' The `$create()` factory methods instantiate the `FileSystem` object and
-#' take the following arguments, depending on the subclass:
+#' `LocalFileSystem$create()` returns the object and takes no arguments.
 #'
-#' - no argument is needed for instantiating a `LocalFileSystem`
-#' - `base_path` and `base_fs` for instantiating a `SubTreeFileSystem`
+#' `SubTreeFileSystem$create()` takes the following arguments:
+#'
+#' - `base_path`, a string path
+#' - `base_fs`, a `FileSystem` object
+#'
+#' `S3FileSystem$create()` optionally takes arguments:
+#'
+#' - `anonymous`: logical, default `FALSE`. If true, will not attempt to look up
+#'    credentials using standard AWS configuration methods.
+#' - `access_key`, `secret_key`: authentication credentials. If one is provided,
+#'    the other must be as well. If both are provided, they will override any
+#'    AWS configuration set at the environment level.
+#' - `session_token`: optional string for authentication along with
+#'    `access_key` and `secret_key`
+#' - `role_arn`: string AWS ARN of an AccessRole. If provided instead of `access_key` and
+#'    `secret_key`, temporary credentials will be fetched by assuming this role.
+#' - `session_name`: optional string identifier for the assumed role session.
+#' - `external_id`: optional unique string identifier that might be required
+#'    when you assume a role in another account.
+#' - `load_frequency`: integer, frequency (in seconds) with which temporary
+#'    credentials from an assumed role session will be refreshed. Default is
+#'    900 (i.e. 15 minutes)
+#' - `region`: AWS region to connect to. If omitted, the AWS library will
+#'    provide a sensible default based on client configuration, falling back
+#'    to "us-east-1" if no other alternatives are found.
+#' - `endpoint_override`: If non-empty, override region with a connect string
+#'    such as "localhost:9000". This is useful for connecting to file systems
+#'    that emulate S3.
+#' - `scheme`: S3 connection transport (default "https")
+#' - `background_writes`: logical, whether `OutputStream` writes will be issued
+#'    in the background, without blocking (default `TRUE`)
 #'
 #' @section Methods:
 #'
@@ -157,6 +185,15 @@ FileSelector$create <- function(base_dir, allow_not_found = FALSE, recursive = F
 #' - `$OpenAppendStream(path)`: Open an [output stream][OutputStream] for
 #'    appending.
 #'
+#' @section Active bindings:
+#'
+#' - `$type_name`: string filesystem type name, such as "local", "s3", etc.
+#' - `$region`: string AWS region, for `S3FileSystem` and `SubTreeFileSystem`
+#'    containing a `S3FileSystem`
+#' - `$base_fs`: for `SubTreeFileSystem`, the `FileSystem` it contains
+#' - `$base_path`: for `SubTreeFileSystem`, the path in `$base_fs` which is considered
+#'    root in this `SubTreeFileSystem`.
+#'
 #' @usage NULL
 #' @format NULL
 #' @docType class
@@ -228,13 +265,26 @@ FileSystem <- R6Class("FileSystem", inherit = ArrowObject,
       shared_ptr(InputStream, fs___FileSystem__OpenInputStream(self, clean_path_rel(path)))
     },
     OpenInputFile = function(path) {
-      shared_ptr(InputStream, fs___FileSystem__OpenInputFile(self, clean_path_rel(path)))
+      shared_ptr(RandomAccessFile, fs___FileSystem__OpenInputFile(self, clean_path_rel(path)))
     },
     OpenOutputStream = function(path) {
       shared_ptr(OutputStream, fs___FileSystem__OpenOutputStream(self, clean_path_rel(path)))
     },
     OpenAppendStream = function(path) {
       shared_ptr(OutputStream, fs___FileSystem__OpenAppendStream(self, clean_path_rel(path)))
+    },
+
+    # Friendlier R user interface
+    path = function(x) SubTreeFileSystem$create(x, self),
+    cd = function(x) SubTreeFileSystem$create(x, self),
+    ls = function(path = "", ...) {
+      selector <- FileSelector$create(path, ...) # ... for recursive = TRUE
+      infos <- self$GetFileInfo(selector)
+      map_chr(infos, ~.$path)
+      # TODO: add full.names argument like base::dir() (default right now is TRUE)
+      # TODO: see fs package for glob/regexp filtering
+      # TODO: verbose method that shows other attributes as df
+      # TODO: print methods for FileInfo, SubTreeFileSystem, S3FileSystem
     }
   ),
   active = list(
@@ -242,11 +292,34 @@ FileSystem <- R6Class("FileSystem", inherit = ArrowObject,
   )
 )
 FileSystem$from_uri <- function(uri) {
+  assert_that(is.string(uri))
   out <- fs___FileSystemFromUri(uri)
   out$fs <- shared_ptr(FileSystem, out$fs)$..dispatch()
   out
 }
 
+get_path_and_filesystem <- function(x, filesystem = NULL) {
+  # Wrapper around FileSystem$from_uri that handles local paths
+  # and an optional explicit filesystem
+  if (inherits(x, "SubTreeFileSystem")) {
+    return(list(fs = x$base_fs, path = x$base_path))
+  }
+  assert_that(is.string(x))
+  if (is_url(x)) {
+    if (!is.null(filesystem)) {
+      # Stop? Can't have URL (which yields a fs) and another fs
+    }
+    FileSystem$from_uri(x)
+  } else {
+    list(
+      fs = filesystem %||% LocalFileSystem$create(),
+      path = clean_path_abs(x)
+    )
+  }
+}
+
+is_url <- function(x) is.string(x) && grepl("://", x)
+
 #' @usage NULL
 #' @format NULL
 #' @rdname FileSystem
@@ -259,25 +332,158 @@ LocalFileSystem$create <- function() {
 #' @usage NULL
 #' @format NULL
 #' @rdname FileSystem
+#' @importFrom utils modifyList
 #' @export
-S3FileSystem <- R6Class("S3FileSystem", inherit = FileSystem)
-S3FileSystem$create <- function() {
-  fs___EnsureS3Initialized()
-  shared_ptr(S3FileSystem, fs___S3FileSystem__create())
+S3FileSystem <- R6Class("S3FileSystem", inherit = FileSystem,
+  active = list(
+    region = function() fs___S3FileSystem__region(self)
+  )
+)
+S3FileSystem$create <- function(anonymous = FALSE, ...) {
+  args <- list2(...)
+  if (anonymous) {
+    invalid_args <- intersect(c("access_key", "secret_key", "session_token", "role_arn", "session_name", "external_id", "load_frequency"), names(args))
+    if (length(invalid_args)) {
+      stop("Cannot specify ", oxford_paste(invalid_args), " when anonymous = TRUE", call. = FALSE)
+    }
+  } else {
+    keys_present <- length(intersect(c("access_key", "secret_key"), names(args)))
+    if (keys_present == 1) {
+      stop("Key authentication requires both access_key and secret_key", call. = FALSE)
+    }
+    if ("session_token" %in% names(args) && keys_present != 2) {
+      stop(
+        "In order to initialize a session with temporary credentials, ",
+        "both secret_key and access_key must be provided ",
+        "in addition to session_token.",
+        call. = FALSE
+      )
+    }
+    arn <- "role_arn" %in% names(args)
+    if (keys_present == 2 && arn) {
+      stop("Cannot provide both key authentication and role_arn", call. = FALSE)
+    }
+    arn_extras <- intersect(c("session_name", "external_id", "load_frequency"), names(args))
+    if (length(arn_extras) > 0 && !arn) {
+      stop("Cannot specify ", oxford_paste(arn_extras), " without providing a role_arn string", call. = FALSE)
+    }
+  }
+  args <- c(modifyList(default_s3_options, args), anonymous = anonymous)
+  shared_ptr(S3FileSystem, exec(fs___S3FileSystem__create, !!!args))
 }
 
-arrow_with_s3 <- function() {
-  .Call(`_s3_available`)
+default_s3_options <- list(
+  access_key = "",
+  secret_key = "",
+  session_token = "",
+  role_arn = "",
+  session_name = "",
+  external_id = "",
+  load_frequency = 900L,
+  region = "",
+  endpoint_override = "",
+  scheme = "",
+  background_writes = TRUE
+)
+
+#' Connect to an AWS S3 bucket
+#'
+#' `s3_bucket()` is a convenience function to create an `S3FileSystem` object
+#' that automatically detects the bucket's AWS region and holding onto the its
+#' relative path.
+#'
+#' @param bucket string S3 bucket name or path
+#' @param ... Additional connection options, passed to `S3FileSystem$create()`
+#' @return A `SubTreeFileSystem` containing an `S3FileSystem` and the bucket's
+#' relative path. Note that this function's success does not guarantee that you
+#' are authorized to access the bucket's contents.
+#' @examples
+#' if (arrow_with_s3()) {
+#'   bucket <- s3_bucket("ursa-labs-taxi-data")
+#' }
+#' @export
+s3_bucket <- function(bucket, ...) {
+  assert_that(is.string(bucket))
+  args <- list2(...)
+
+  # Use FileSystemFromUri to detect the bucket's region
+  if (!is_url(bucket)) {
+    bucket <- paste0("s3://", bucket)
+  }
+  fs_and_path <- FileSystem$from_uri(bucket)
+  fs <- fs_and_path$fs
+  # If there are no additional S3Options, we can use that filesystem
+  # Otherwise, take the region that was detected and make a new fs with the args
+  if (length(args)) {
+    args$region <- fs$region
+    fs <- exec(S3FileSystem$create, !!!args)
+  }
+  # Return a subtree pointing at that bucket path
+  SubTreeFileSystem$create(fs_and_path$path, fs)
 }
 
 #' @usage NULL
 #' @format NULL
 #' @rdname FileSystem
 #' @export
-SubTreeFileSystem <- R6Class("SubTreeFileSystem", inherit = FileSystem)
-SubTreeFileSystem$create <- function(base_path, base_fs) {
-  xp <- fs___SubTreeFileSystem__create(clean_path_rel(base_path), base_fs)
-  shared_ptr(SubTreeFileSystem, xp)
+SubTreeFileSystem <- R6Class("SubTreeFileSystem", inherit = FileSystem,
+  active = list(
+    base_fs = function() {
+      shared_ptr(FileSystem, fs___SubTreeFileSystem__base_fs(self))$..dispatch()
+    },
+    base_path = function() fs___SubTreeFileSystem__base_path(self)
+  )
+)
+SubTreeFileSystem$create <- function(base_path, base_fs = NULL) {
+  fs_and_path <- get_path_and_filesystem(base_path, base_fs)
+  shared_ptr(
+    SubTreeFileSystem,
+    fs___SubTreeFileSystem__create(fs_and_path$path, fs_and_path$fs)
+  )
+}
+
+#' @export
+`$.SubTreeFileSystem` <- function(x, name, ...) {
+  # This is to allow delegating methods/properties to the base_fs
+  assert_that(is.string(name))
+  if (name %in% ls(x)) {
+    get(name, x)
+  } else {
+    get(name, x$base_fs)
+  }
+}
+
+#' Copy files between FileSystems
+#'
+#' @param from A string path to a local directory or file, a URI, or a
+#' `SubTreeFileSystem`. Files will be copied recursively from this path.
+#' @param to A string path to a local directory or file, a URI, or a
+#' `SubTreeFileSystem`. Directories will be created as necessary
+#' @param chunk_size The maximum size of block to read before flushing
+#' to the destination file. A larger chunk_size will use more memory while
+#' copying but may help accommodate high latency FileSystems.
+#' @return Nothing: called for side effects in the file system
+#' @export
+#' @examples
+#' \dontrun{
+#' # Copy an S3 bucket's files to a local directory:
+#' copy_files("s3://your-bucket-name", "local-directory")
+#' # Using a FileSystem object
+#' copy_files(s3_bucket("your-bucket-name"), "local-directory")
+#' # Or go the other way, from local to S3
+#' copy_files("local-directory", s3_bucket("your-bucket-name"))
+#' }
+copy_files <- function(from, to, chunk_size = 1024L * 1024L) {
+  from <- get_path_and_filesystem(from)
+  to <- get_path_and_filesystem(to)
+  invisible(fs___CopyFiles(
+    from$fs,
+    FileSelector$create(from$path, recursive = TRUE),
+    to$fs,
+    to$path,
+    chunk_size,
+    option_use_threads()
+  ))
 }
 
 clean_path_abs <- function(path) {
diff --git a/r/R/flight.R b/r/R/flight.R
new file mode 100644
index 00000000000..7721661faf2
--- /dev/null
+++ b/r/R/flight.R
@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#' Load a Python Flight server
+#'
+#' @param name string Python module name
+#' @param path file system path where the Python module is found. Default is
+#' to look in the `inst/` directory for included modules.
+#' @export
+load_flight_server <- function(name, path = system.file(package = "arrow")) {
+  reticulate::import_from_path(name, path)
+}
+
+#' Connect to a Flight server
+#'
+#' @param host string hostname to connect to
+#' @param port integer port to connect on
+#' @param scheme URL scheme, default is "grpc+tcp"
+#' @return A `pyarrow.flight.FlightClient`.
+#' @export
+flight_connect <- function(host = "localhost", port, scheme = "grpc+tcp") {
+  pa <- reticulate::import("pyarrow")
+  location <- paste0(scheme, "://", host, ":", port)
+  pa$flight$FlightClient(location)
+}
+
+#' Send data to a Flight server
+#'
+#' @param client `pyarrow.flight.FlightClient`, as returned by [flight_connect()]
+#' @param data `data.frame` or [RecordBatch] to upload
+#' @param path string identifier to store the data under
+#' @return `client`, invisibly.
+#' @export
+push_data <- function(client, data, path) {
+  if (inherits(data, "data.frame")) {
+    data <- record_batch(data)
+  }
+  # TODO: this is only RecordBatch; handle Table
+  py_data <- reticulate::r_to_py(data)
+  writer <- client$do_put(descriptor_for_path(path), py_data$schema)[[1]]
+  writer$write_batch(py_data)
+  writer$close()
+  invisible(client)
+}
+
+#' Get data from a Flight server
+#'
+#' @param client `pyarrow.flight.FlightClient`, as returned by [flight_connect()]
+#' @param path string identifier under which the data is stored
+#' @return A [RecordBatch]
+#' @export
+flight_get <- function(client, path) {
+  info <- client$get_flight_info(descriptor_for_path(path))
+  # Hack: assume a single ticket, on the same server as client is already connected
+  ticket <- info$endpoints[[1]]$ticket
+  reader <- client$do_get(ticket)
+  # Next hack: assume a single record batch
+  # TODO: read_all() instead? Or read all chunks and build Table in R?
+  chunk <- reader$read_chunk()
+  # Drop $app_metadata and just return the data
+  chunk$data
+}
+
+descriptor_for_path <- function(path) {
+  pa <- reticulate::import("pyarrow")
+  pa$flight$FlightDescriptor$for_path(path)
+}
diff --git a/r/R/install-arrow.R b/r/R/install-arrow.R
index e3fa2ba2a2a..a3514ec4140 100644
--- a/r/R/install-arrow.R
+++ b/r/R/install-arrow.R
@@ -55,8 +55,12 @@ install_arrow <- function(nightly = FALSE,
   conda <- isTRUE(grepl("conda", R.Version()$platform))
 
   if (sysname %in% c("windows", "darwin", "linux")) {
-    if (conda && !nightly) {
-      system("conda install -y -c conda-forge --strict-channel-priority r-arrow")
+    if (conda) {
+      if (nightly) {
+        system("conda install -y -c arrow-nightlies -c conda-forge --strict-channel-priority r-arrow")
+      } else {
+        system("conda install -y -c conda-forge --strict-channel-priority r-arrow")
+      }
     } else {
       Sys.setenv(
         LIBARROW_DOWNLOAD = "true",
@@ -90,12 +94,12 @@ arrow_repos <- function(repos = getOption("repos"), nightly = FALSE) {
     # Set the default/CDN
     repos <- "https://cloud.r-project.org/"
   }
-  bintray <- getOption("arrow.dev.repo", "https://dl.bintray.com/ursalabs/arrow-r")
+  dev_repo <- getOption("arrow.dev_repo", "https://arrow-r-nightly.s3.amazonaws.com")
   # Remove it if it's there (so nightly=FALSE won't accidentally pull from it)
-  repos <- setdiff(repos, bintray)
+  repos <- setdiff(repos, dev_repo)
   if (nightly) {
     # Add it first
-    repos <- c(bintray, repos)
+    repos <- c(dev_repo, repos)
   }
   repos
 }
diff --git a/r/R/io.R b/r/R/io.R
index c14c5ce1abc..b4dbbeb6a5f 100644
--- a/r/R/io.R
+++ b/r/R/io.R
@@ -224,15 +224,29 @@ mmap_open <- function(path, mode = c("read", "write", "readwrite")) {
 #' with this compression codec, either a [Codec] or the string name of one.
 #' If `NULL` (default) and `file` is a string file name, the function will try
 #' to infer compression from the file extension.
+#' @param filesystem If not `NULL`, `file` will be opened via the
+#' `filesystem$OpenInputFile()` filesystem method, rather than the `io` module's
+#' `MemoryMappedFile` or `ReadableFile` constructors.
 #' @return An `InputStream` or a subclass of one.
 #' @keywords internal
-make_readable_file <- function(file, mmap = TRUE, compression = NULL) {
+make_readable_file <- function(file, mmap = TRUE, compression = NULL, filesystem = NULL) {
+  if (inherits(file, "SubTreeFileSystem")) {
+    filesystem <- file$base_fs
+    file <- file$base_path
+  }
   if (is.string(file)) {
+    if (is_url(file)) {
+      fs_and_path <- FileSystem$from_uri(file)
+      filesystem <- fs_and_path$fs
+      file <- fs_and_path$path
+    }
     if (is.null(compression)) {
       # Infer compression from the file path
       compression <- detect_compression(file)
     }
-    if (isTRUE(mmap)) {
+    if (!is.null(filesystem)) {
+      file <- filesystem$OpenInputFile(file)
+    } else if (isTRUE(mmap)) {
       file <- mmap_open(file)
     } else {
       file <- ReadableFile$create(file)
@@ -247,6 +261,23 @@ make_readable_file <- function(file, mmap = TRUE, compression = NULL) {
   file
 }
 
+make_output_stream <- function(x, filesystem = NULL) {
+  if (inherits(x, "SubTreeFileSystem")) {
+    filesystem <- x$base_fs
+    x <- x$base_path
+  } else if (is_url(x)) {
+    fs_and_path <- FileSystem$from_uri(x)
+    filesystem = fs_and_path$fs
+    x <- fs_and_path$path
+  }
+  assert_that(is.string(x))
+  if (is.null(filesystem)) {
+    FileOutputStream$create(x)
+  } else {
+    filesystem$OpenOutputStream(x)
+  }
+}
+
 detect_compression <- function(path) {
   assert_that(is.string(path))
   switch(tools::file_ext(path),
diff --git a/r/R/ipc_stream.R b/r/R/ipc_stream.R
index 0c728b26b53..4f506f3332b 100644
--- a/r/R/ipc_stream.R
+++ b/r/R/ipc_stream.R
@@ -40,11 +40,10 @@ write_ipc_stream <- function(x, sink, ...) {
   if (is.data.frame(x)) {
     x <- Table$create(x)
   }
-  if (is.string(sink)) {
-    sink <- FileOutputStream$create(sink)
+  if (!inherits(sink, "OutputStream")) {
+    sink <- make_output_stream(sink)
     on.exit(sink$close())
   }
-  assert_is(sink, "OutputStream")
 
   writer <- RecordBatchStreamWriter$create(sink, x$schema)
   writer$write(x)
@@ -82,10 +81,11 @@ write_to_raw <- function(x, format = c("stream", "file")) {
 #' `read_arrow()`, a wrapper around `read_ipc_stream()` and `read_feather()`,
 #' is deprecated. You should explicitly choose
 #' the function that will read the desired IPC format (stream or file) since
-#' a file or `InputStream` may contain either. 
+#' a file or `InputStream` may contain either.
 #'
-#' @param file A character file name, `raw` vector, or an Arrow input stream.
-#' If a file name, a memory-mapped Arrow [InputStream] will be opened and
+#' @param file A character file name or URI, `raw` vector, an Arrow input stream,
+#' or a `FileSystem` with path (`SubTreeFileSystem`).
+#' If a file name or URI, an Arrow [InputStream] will be opened and
 #' closed when finished. If an input stream is provided, it will be left
 #' open.
 #' @param as_data_frame Should the function return a `data.frame` (default) or
diff --git a/r/R/json.R b/r/R/json.R
index ea340e3e657..1cc39fa42d0 100644
--- a/r/R/json.R
+++ b/r/R/json.R
@@ -35,8 +35,20 @@
 #'   ', tf, useBytes=TRUE)
 #'   df <- read_json_arrow(tf)
 #' }
-read_json_arrow <- function(file, col_select = NULL, as_data_frame = TRUE, ...) {
-  tab <- JsonTableReader$create(file, ...)$Read()$select(!!enquo(col_select))
+read_json_arrow <- function(file,
+                            col_select = NULL,
+                            as_data_frame = TRUE,
+                            ...) {
+  if (!inherits(file, "InputStream")) {
+    file <- make_readable_file(file)
+    on.exit(file$close())
+  }
+  tab <- JsonTableReader$create(file, ...)$Read()
+
+  col_select <- enquo(col_select)
+  if (!quo_is_null(col_select)) {
+    tab <- tab[vars_select(names(tab), !!col_select)]
+  }
 
   if (isTRUE(as_data_frame)) {
     tab <- as.data.frame(tab)
@@ -59,8 +71,7 @@ JsonTableReader$create <- function(file,
                                    read_options = JsonReadOptions$create(),
                                    parse_options = JsonParseOptions$create(),
                                    ...) {
-
-  file <- make_readable_file(file)
+  assert_is(file, "InputStream")
   shared_ptr(
     JsonTableReader,
     json___TableReader__Make(file, read_options, parse_options)
@@ -74,12 +85,7 @@ JsonTableReader$create <- function(file,
 #' @export
 JsonReadOptions <- R6Class("JsonReadOptions", inherit = ArrowObject)
 JsonReadOptions$create <- function(use_threads = option_use_threads(), block_size = 1048576L) {
-  shared_ptr(JsonReadOptions, json___ReadOptions__initialize(
-    list(
-      use_threads = use_threads,
-      block_size = block_size
-    )
-  ))
+  shared_ptr(JsonReadOptions, json___ReadOptions__initialize(use_threads, block_size))
 }
 
 #' @rdname CsvReadOptions
@@ -89,9 +95,5 @@ JsonReadOptions$create <- function(use_threads = option_use_threads(), block_siz
 #' @export
 JsonParseOptions <- R6Class("JsonParseOptions", inherit = ArrowObject)
 JsonParseOptions$create <- function(newlines_in_values = FALSE) {
-  shared_ptr(JsonParseOptions, json___ParseOptions__initialize(
-    list(
-      newlines_in_values = newlines_in_values
-    )
-  ))
+  shared_ptr(JsonParseOptions, json___ParseOptions__initialize(newlines_in_values))
 }
diff --git a/r/R/memory-pool.R b/r/R/memory-pool.R
index d830e3b2d47..dfd3a48cb72 100644
--- a/r/R/memory-pool.R
+++ b/r/R/memory-pool.R
@@ -25,10 +25,12 @@
 #'
 #' @section Methods:
 #'
-#' TODO
+#' - `bytes_allocated()`
+#' - `max_memory()`
 #'
 #' @rdname MemoryPool
 #' @name MemoryPool
+#' @keywords internal
 MemoryPool <- R6Class("MemoryPool",
   inherit = ArrowObject,
   public = list(
@@ -44,6 +46,7 @@ MemoryPool <- R6Class("MemoryPool",
 #'
 #' @return the default [arrow::MemoryPool][MemoryPool]
 #' @export
+#' @keywords internal
 default_memory_pool <- function() {
   shared_ptr(MemoryPool, MemoryPool__default())
 }
diff --git a/r/R/parquet.R b/r/R/parquet.R
index b44cf189805..1bc67427b48 100644
--- a/r/R/parquet.R
+++ b/r/R/parquet.R
@@ -45,7 +45,18 @@ read_parquet <- function(file,
     on.exit(file$close())
   }
   reader <- ParquetFileReader$create(file, props = props, ...)
-  tab <- reader$ReadTable(!!enquo(col_select))
+
+  col_select <- enquo(col_select)
+  if (!quo_is_null(col_select)) {
+    # infer which columns to keep from schema
+    schema <- reader$GetSchema()
+    names <- names(schema)
+    indices <- match(vars_select(names, !!col_select), names) - 1L
+    tab <- reader$ReadTable(indices)
+  } else {
+    # read all columns
+    tab <- reader$ReadTable()
+  }
 
   if (as_data_frame) {
     tab <- as.data.frame(tab)
@@ -58,10 +69,16 @@ read_parquet <- function(file,
 #' [Parquet](https://parquet.apache.org/) is a columnar storage file format.
 #' This function enables you to write Parquet files from R.
 #'
-#' @param x An [arrow::Table][Table], or an object convertible to it.
-#' @param sink an [arrow::io::OutputStream][OutputStream] or a string which is interpreted as a file path
-#' @param chunk_size chunk size in number of rows. If NULL, the total number of rows is used.
+#' Due to features of the format, Parquet files cannot be appended to.
+#' If you want to use the Parquet format but also want the ability to extend
+#' your dataset, you can write to additional Parquet files and then treat
+#' the whole directory of files as a [Dataset] you can query.
+#' See `vignette("dataset", package = "arrow")` for examples of this.
 #'
+#' @param x `data.frame`, [RecordBatch], or [Table]
+#' @param sink A string file path, URI, or [OutputStream], or path in a file
+#' system (`SubTreeFileSystem`)
+#' @param chunk_size chunk size in number of rows. If NULL, the total number of rows is used.
 #' @param version parquet version, "1.0" or "2.0". Default "1.0". Numeric values
 #'   are coerced to character.
 #' @param compression compression algorithm. Default "snappy". See details.
@@ -70,23 +87,18 @@ read_parquet <- function(file,
 #' @param write_statistics Specify if we should write statistics. Default `TRUE`
 #' @param data_page_size Set a target threshold for the approximate encoded
 #'    size of data pages within a column chunk (in bytes). Default 1 MiB.
-#' @param properties properties for parquet writer, derived from arguments
-#'   `version`, `compression`, `compression_level`, `use_dictionary`,
-#'   `write_statistics` and `data_page_size`. You should not specify any of
-#'    these arguments if you also provide a `properties` argument, as they will
-#'    be ignored.
-#'
 #' @param use_deprecated_int96_timestamps Write timestamps to INT96 Parquet format. Default `FALSE`.
 #' @param coerce_timestamps Cast timestamps a particular resolution. Can be
 #'   `NULL`, "ms" or "us". Default `NULL` (no casting)
 #' @param allow_truncated_timestamps Allow loss of data when coercing timestamps to a
 #'    particular resolution. E.g. if microsecond or nanosecond data is lost when coercing
 #'    to "ms", do not raise an exception
-#'
-#' @param arrow_properties arrow specific writer properties, derived from arguments
-#'   `use_deprecated_int96_timestamps`, `coerce_timestamps` and `allow_truncated_timestamps`
-#'    You should not specify any of these arguments if you also provide a `properties`
-#'    argument, as they will be ignored.
+#' @param properties A `ParquetWriterProperties` object, used instead of the options
+#'    enumerated in this function's signature. Providing `properties` as an argument
+#'    is deprecated; if you need to assemble `ParquetWriterProperties` outside
+#'    of `write_parquet()`, use `ParquetFileWriter` instead.
+#' @param arrow_properties A `ParquetArrowWriterProperties` object. Like
+#'    `properties`, this argument is deprecated.
 #'
 #' @details The parameters `compression`, `compression_level`, `use_dictionary` and
 #'   `write_statistics` support various patterns:
@@ -126,59 +138,73 @@ write_parquet <- function(x,
                           chunk_size = NULL,
                           # writer properties
                           version = NULL,
-                          compression = NULL,
+                          compression = default_parquet_compression(),
                           compression_level = NULL,
                           use_dictionary = NULL,
                           write_statistics = NULL,
                           data_page_size = NULL,
-                          properties = ParquetWriterProperties$create(
-                            x,
-                            version = version,
-                            compression = compression,
-                            compression_level = compression_level,
-                            use_dictionary = use_dictionary,
-                            write_statistics = write_statistics,
-                            data_page_size = data_page_size
-                          ),
                           # arrow writer properties
                           use_deprecated_int96_timestamps = FALSE,
                           coerce_timestamps = NULL,
                           allow_truncated_timestamps = FALSE,
-                          arrow_properties = ParquetArrowWriterProperties$create(
-                            use_deprecated_int96_timestamps = use_deprecated_int96_timestamps,
-                            coerce_timestamps = coerce_timestamps,
-                            allow_truncated_timestamps = allow_truncated_timestamps
-                          )) {
+                          properties = NULL,
+                          arrow_properties = NULL) {
   x_out <- x
   if (is.data.frame(x)) {
     x <- Table$create(x)
   }
 
-  if (is.string(sink)) {
-    sink <- FileOutputStream$create(sink)
+  if (!inherits(sink, "OutputStream")) {
+    sink <- make_output_stream(sink)
     on.exit(sink$close())
-  } else if (!inherits(sink, "OutputStream")) {
-    abort("sink must be a file path or an OutputStream")
   }
 
-  schema <- x$schema
-  # Match the pyarrow default (overriding the C++ default)
-  if (is.null(compression) && codec_is_available("snappy")) {
-    compression <- "snappy"
+  # Deprecation warnings
+  if (!is.null(properties)) {
+    warning("Providing 'properties' is deprecated. If you need to assemble properties outside this function, use ParquetFileWriter instead.")
   }
-  # Note: `properties` and `arrow_properties` are not actually $create()-ed
-  # until the next line, so the compression change is applied.
-  writer <- ParquetFileWriter$create(schema, sink, properties = properties, arrow_properties = arrow_properties)
+  if (!is.null(arrow_properties)) {
+    warning("Providing 'arrow_properties' is deprecated. If you need to assemble arrow_properties outside this function, use ParquetFileWriter instead.")
+  }
+
+  writer <- ParquetFileWriter$create(
+    x$schema,
+    sink,
+    properties = properties %||% ParquetWriterProperties$create(
+      x,
+      version = version,
+      compression = compression,
+      compression_level = compression_level,
+      use_dictionary = use_dictionary,
+      write_statistics = write_statistics,
+      data_page_size = data_page_size
+    ),
+    arrow_properties = arrow_properties %||% ParquetArrowWriterProperties$create(
+      use_deprecated_int96_timestamps = use_deprecated_int96_timestamps,
+      coerce_timestamps = coerce_timestamps,
+      allow_truncated_timestamps = allow_truncated_timestamps
+    )
+  )
   writer$WriteTable(x, chunk_size = chunk_size %||% x$num_rows)
   writer$Close()
 
   invisible(x_out)
 }
 
+default_parquet_compression <- function() {
+  # Match the pyarrow default (overriding the C++ default)
+  if (codec_is_available("snappy")) {
+    "snappy"
+  } else {
+    NULL
+  }
+}
+
 ParquetArrowWriterProperties <- R6Class("ParquetArrowWriterProperties", inherit = ArrowObject)
 ParquetArrowWriterProperties$create <- function(use_deprecated_int96_timestamps = FALSE,
                                                 coerce_timestamps = NULL,
-                                                allow_truncated_timestamps = FALSE) {
+                                                allow_truncated_timestamps = FALSE,
+                                                ...) {
   if (is.null(coerce_timestamps)) {
     timestamp_unit <- -1L # null sentinel value
   } else {
@@ -268,7 +294,8 @@ ParquetWriterPropertiesBuilder <- R6Class("ParquetWriterPropertiesBuilder", inhe
       )
     },
     set_compression_level = function(table, compression_level){
-      assert_that(is_integerish(compression_level))
+      # cast to integer but keep names
+      compression_level <- set_names(as.integer(compression_level), names(compression_level))
       private$.set(table, compression_level,
         parquet___ArrowWriterProperties___Builder__set_compression_levels
       )
@@ -315,11 +342,12 @@ ParquetWriterPropertiesBuilder <- R6Class("ParquetWriterPropertiesBuilder", inhe
 
 ParquetWriterProperties$create <- function(table,
                                            version = NULL,
-                                           compression = NULL,
+                                           compression = default_parquet_compression(),
                                            compression_level = NULL,
                                            use_dictionary = NULL,
                                            write_statistics = NULL,
-                                           data_page_size = NULL) {
+                                           data_page_size = NULL,
+                                           ...) {
   builder <- shared_ptr(
     ParquetWriterPropertiesBuilder,
     parquet___WriterProperties___Builder__create()
@@ -359,9 +387,16 @@ ParquetWriterProperties$create <- function(table,
 #' takes the following arguments:
 #'
 #' - `schema` A [Schema]
-#' - `sink` An [arrow::io::OutputStream][OutputStream] or a string which is interpreted as a file path
+#' - `sink` An [arrow::io::OutputStream][OutputStream]
 #' - `properties` An instance of [ParquetWriterProperties]
 #' - `arrow_properties` An instance of `ParquetArrowWriterProperties`
+#'
+#' @section Methods:
+#'
+#' - `WriteTable` Write a [Table] to `sink`
+#' - `Close` Close the writer. Note: does not close the `sink`.
+#'   [arrow::io::OutputStream][OutputStream] has its own `close()` method.
+#'
 #' @export
 #' @include arrow-package.R
 ParquetFileWriter <- R6Class("ParquetFileWriter", inherit = ArrowObject,
@@ -376,6 +411,7 @@ ParquetFileWriter$create <- function(schema,
                                      sink,
                                      properties = ParquetWriterProperties$create(),
                                      arrow_properties = ParquetArrowWriterProperties$create()) {
+  assert_is(sink, "OutputStream")
   shared_ptr(
     ParquetFileWriter,
     parquet___arrow___ParquetFileWriter__Open(schema, sink, properties, arrow_properties)
@@ -404,10 +440,20 @@ ParquetFileWriter$create <- function(schema,
 #'
 #' @section Methods:
 #'
-#' - `$ReadTable(col_select)`: get an `arrow::Table` from the file, possibly
-#'    with columns filtered by a character vector of column names or a
-#'    `tidyselect` specification.
+#' - `$ReadTable(column_indices)`: get an `arrow::Table` from the file. The optional
+#'    `column_indices=` argument is a 0-based integer vector indicating which columns to retain.
+#' - `$ReadRowGroup(i, column_indices)`: get an `arrow::Table` by reading the `i`th row group (0-based).
+#'    The optional `column_indices=` argument is a 0-based integer vector indicating which columns to retain.
+#' - `$ReadRowGroups(row_groups, column_indices)`: get an `arrow::Table` by reading several row groups (0-based integers).
+#'    The optional `column_indices=` argument is a 0-based integer vector indicating which columns to retain.
 #' - `$GetSchema()`: get the `arrow::Schema` of the data in the file
+#' - `$ReadColumn(i)`: read the `i`th column (0-based) as a [ChunkedArray].
+#'
+#' @section Active bindings:
+#'
+#' - `$num_rows`: number of rows.
+#' - `$num_columns`: number of columns.
+#' - `$num_row_groups`: number of row groups.
 #'
 #' @export
 #' @examples
@@ -417,7 +463,7 @@ ParquetFileWriter$create <- function(schema,
 #' pq$GetSchema()
 #' if (codec_is_available("snappy")) {
 #'   # This file has compressed data columns
-#'   tab <- pq$ReadTable(starts_with("c"))
+#'   tab <- pq$ReadTable()
 #'   tab$schema
 #' }
 #' }
@@ -427,19 +473,45 @@ ParquetFileReader <- R6Class("ParquetFileReader",
   active = list(
     num_rows = function() {
       as.integer(parquet___arrow___FileReader__num_rows(self))
+    },
+    num_columns = function() {
+      parquet___arrow___FileReader__num_columns(self)
+    },
+    num_row_groups = function() {
+      parquet___arrow___FileReader__num_row_groups(self)
     }
   ),
   public = list(
-    ReadTable = function(col_select = NULL) {
-      col_select <- enquo(col_select)
-      if (quo_is_null(col_select)) {
+    ReadTable = function(column_indices = NULL) {
+      if (is.null(column_indices)) {
         shared_ptr(Table, parquet___arrow___FileReader__ReadTable1(self))
       } else {
-        all_vars <- shared_ptr(Schema, parquet___arrow___FileReader__GetSchema(self))$names
-        indices <- match(vars_select(all_vars, !!col_select), all_vars) - 1L
-        shared_ptr(Table, parquet___arrow___FileReader__ReadTable2(self, indices))
+        column_indices <- vec_cast(column_indices, integer())
+        shared_ptr(Table, parquet___arrow___FileReader__ReadTable2(self, column_indices))
+      }
+    },
+    ReadRowGroup = function(i, column_indices = NULL) {
+      i <- vec_cast(i, integer())
+      if (is.null(column_indices)) {
+        shared_ptr(Table, parquet___arrow___FileReader__ReadRowGroup1(self, i))
+      } else {
+        column_indices <- vec_cast(column_indices, integer())
+        shared_ptr(Table, parquet___arrow___FileReader__ReadRowGroup2(self, i, column_indices))
       }
     },
+    ReadRowGroups = function(row_groups, column_indices = NULL) {
+      row_groups <- vec_cast(row_groups, integer())
+      if (is.null(column_indices)) {
+        shared_ptr(Table, parquet___arrow___FileReader__ReadRowGroups1(self, row_groups))
+      } else {
+        column_indices <- vec_cast(column_indices, integer())
+        shared_ptr(Table, parquet___arrow___FileReader__ReadRowGroups2(self, row_groups, column_indices))
+      }
+    },
+    ReadColumn = function(i) {
+      i <- vec_cast(i, integer())
+      shared_ptr(ChunkedArray, parquet___arrow___FileReader__ReadColumn(self, i))
+    },
     GetSchema = function() {
       shared_ptr(Schema, parquet___arrow___FileReader__GetSchema(self))
     }
diff --git a/r/R/record-batch-writer.R b/r/R/record-batch-writer.R
index 4c4d0bb8703..8b51603110d 100644
--- a/r/R/record-batch-writer.R
+++ b/r/R/record-batch-writer.R
@@ -39,7 +39,7 @@
 #' - `sink` An `OutputStream`
 #' - `schema` A [Schema] for the data to be written
 #' - `use_legacy_format` logical: write data formatted so that Arrow libraries
-#'   versions 0.14 and lower can read it? Default is `FALSE`. You can also
+#'   versions 0.14 and lower can read it. Default is `FALSE`. You can also
 #'   enable this by setting the environment variable `ARROW_PRE_0_15_IPC_FORMAT=1`.
 #' - `metadata_version`: A string like "V5" or the equivalent integer indicating
 #'   the Arrow IPC MetadataVersion. Default (NULL) will use the latest version,
@@ -130,7 +130,6 @@ RecordBatchStreamWriter$create <- function(sink,
       call. = FALSE
     )
   }
-  use_legacy_format <- use_legacy_format %||% identical(Sys.getenv("ARROW_PRE_0_15_IPC_FORMAT"), "1")
   assert_is(sink, "OutputStream")
   assert_is(schema, "Schema")
 
@@ -138,7 +137,7 @@ RecordBatchStreamWriter$create <- function(sink,
     ipc___RecordBatchStreamWriter__Open(
       sink,
       schema,
-      isTRUE(use_legacy_format),
+      get_ipc_use_legacy_format(use_legacy_format),
       get_ipc_metadata_version(metadata_version)
     )
   )
@@ -160,7 +159,6 @@ RecordBatchFileWriter$create <- function(sink,
       call. = FALSE
     )
   }
-  use_legacy_format <- use_legacy_format %||% identical(Sys.getenv("ARROW_PRE_0_15_IPC_FORMAT"), "1")
   assert_is(sink, "OutputStream")
   assert_is(schema, "Schema")
 
@@ -168,7 +166,7 @@ RecordBatchFileWriter$create <- function(sink,
     ipc___RecordBatchFileWriter__Open(
       sink,
       schema,
-      isTRUE(use_legacy_format),
+      get_ipc_use_legacy_format(use_legacy_format),
       get_ipc_metadata_version(metadata_version)
     )
   )
@@ -196,3 +194,7 @@ get_ipc_metadata_version <- function(x) {
   }
   out
 }
+
+get_ipc_use_legacy_format <- function(x) {
+  isTRUE(x %||% identical(Sys.getenv("ARROW_PRE_0_15_IPC_FORMAT"), "1"))
+}
diff --git a/r/R/record-batch.R b/r/R/record-batch.R
index 712000a97ab..1b5fcbba24e 100644
--- a/r/R/record-batch.R
+++ b/r/R/record-batch.R
@@ -48,9 +48,7 @@
 #' - `$names()`: Get all column names (called by `names(batch)`)
 #' - `$GetColumnByName(name)`: Extract an `Array` by string name
 #' - `$RemoveColumn(i)`: Drops a column from the batch by integer position
-#' - `$select(spec)`: Return a new record batch with a selection of columns.
-#'    This supports the usual `character`, `numeric`, and `logical` selection
-#'    methods as well as "tidy select" expressions.
+#' - `$selectColumns(indices)`: Return a new record batch with a selection of columns, expressed as 0-based integers.
 #' - `$Slice(offset, length = NULL)`: Create a zero-copy view starting at the
 #'    indicated integer offset and going for the given length, or to the end
 #'    of the table if `NULL`, the default.
@@ -84,21 +82,12 @@ RecordBatch <- R6Class("RecordBatch", inherit = ArrowObject,
       assert_that(is.string(name))
       shared_ptr(Array, RecordBatch__GetColumnByName(self, name))
     },
-    select = function(spec) {
-      spec <- enquo(spec)
-      if (quo_is_null(spec)) {
-        self
-      } else {
-        all_vars <- self$names()
-        vars <- vars_select(all_vars, !!spec)
-        indices <- match(vars, all_vars)
-        shared_ptr(RecordBatch, RecordBatch__select(self, indices))
-      }
+    SelectColumns = function(indices) {
+      shared_ptr(RecordBatch, RecordBatch__SelectColumns(self, indices))
     },
     RemoveColumn = function(i){
       shared_ptr(RecordBatch, RecordBatch__RemoveColumn(self, i))
     },
-
     Slice = function(offset, length = NULL) {
       if (is.null(length)) {
         shared_ptr(RecordBatch, RecordBatch__Slice1(self, offset))
@@ -218,7 +207,16 @@ names.RecordBatch <- function(x) x$names()
   if (!missing(j)) {
     # Selecting columns is cheaper than filtering rows, so do it first.
     # That way, if we're filtering too, we have fewer arrays to filter/slice/take
-    x <- x$select(j)
+    if (is_integerish(j)) {
+      if (all(j < 0)) {
+        # in R, negative j means "everything but j"
+        j <- setdiff(seq_len(x$num_columns), -1 * j)
+      }
+      x <- x$SelectColumns(as.integer(j) - 1L)
+    } else if (is.character(j)) {
+      x <- x$SelectColumns(match(j, names(x)) - 1L)
+    }
+
     if (drop && ncol(x) == 1L) {
       x <- x$column(0)
     }
@@ -278,7 +276,14 @@ as.data.frame.RecordBatch <- function(x, row.names = NULL, optional = FALSE, ...
 apply_arrow_r_metadata <- function(x, r_metadata) {
   tryCatch({
     if (!is.null(r_metadata$attributes)) {
-      attributes(x) <- r_metadata$attributes
+      attributes(x)[names(r_metadata$attributes)] <- r_metadata$attributes
+      if (inherits(x, "POSIXlt")) {
+        # We store POSIXlt as a StructArray, which is translated back to R
+        # as a data.frame, but while data frames have a row.names = c(NA, nrow(x))
+        # attribute, POSIXlt does not, so since this is now no longer an object
+        # of class data.frame, remove the extraneous attribute
+        attr(x, "row.names") <- NULL
+      }
     }
 
     columns_metadata <- r_metadata$columns
diff --git a/r/R/schema.R b/r/R/schema.R
index ddbf30fe93b..4bba8b87f47 100644
--- a/r/R/schema.R
+++ b/r/R/schema.R
@@ -84,10 +84,7 @@ Schema <- R6Class("Schema",
   ),
   active = list(
     names = function() {
-      out <- Schema__field_names(self)
-      # Hack: Rcpp should set the encoding
-      Encoding(out) <- "UTF-8"
-      out
+      Schema__field_names(self)
     },
     num_fields = function() Schema__num_fields(self),
     fields = function() map(Schema__fields(self), shared_ptr, class = Field),
@@ -185,6 +182,9 @@ length.Schema <- function(x) x$num_fields
   }
 }
 
+#' @export
+as.list.Schema <- function(x, ...) x$fields
+
 #' read a Schema from a stream
 #'
 #' @param stream a `Message`, `InputStream`, or `Buffer`
diff --git a/r/R/struct.R b/r/R/struct.R
index 219fb28ab2a..feda966f56d 100644
--- a/r/R/struct.R
+++ b/r/R/struct.R
@@ -24,7 +24,7 @@ StructType <- R6Class("StructType",
     GetFieldIndex = function(name) StructType__GetFieldIndex(self, name)
   )
 )
-StructType$create <- function(...) shared_ptr(StructType, struct_(.fields(list(...))))
+StructType$create <- function(...) shared_ptr(StructType, struct__(.fields(list(...))))
 
 #' @rdname data-type
 #' @export
diff --git a/r/R/table.R b/r/R/table.R
index 1391eee4a7d..6c87a3c3318 100644
--- a/r/R/table.R
+++ b/r/R/table.R
@@ -55,9 +55,7 @@
 #' - `$ColumnNames()`: Get all column names (called by `names(tab)`)
 #' - `$GetColumnByName(name)`: Extract a `ChunkedArray` by string name
 #' - `$field(i)`: Extract a `Field` from the table schema by integer position
-#' - `$select(spec)`: Return a new table with a selection of columns.
-#'    This supports the usual `character`, `numeric`, and `logical` selection
-#'    methods as well as "tidy select" expressions.
+#' - `$SelectColumns(indices)`: Return new `Table` with specified columns, expressed as 0-based integers.
 #' - `$Slice(offset, length = NULL)`: Create a zero-copy view starting at the
 #'    indicated integer offset and going for the given length, or to the end
 #'    of the table if `NULL`, the default.
@@ -115,16 +113,8 @@ Table <- R6Class("Table", inherit = ArrowObject,
       shared_ptr(Table, Table__cast(self, target_schema, options))
     },
 
-    select = function(spec) {
-      spec <- enquo(spec)
-      if (quo_is_null(spec)) {
-        self
-      } else {
-        all_vars <- self$ColumnNames()
-        vars <- vars_select(all_vars, !!spec)
-        indices <- match(vars, all_vars)
-        shared_ptr(Table, Table__select(self, indices))
-      }
+    SelectColumns = function(indices) {
+      shared_ptr(Table, Table__SelectColumns(self, indices))
     },
 
     Slice = function(offset, length = NULL) {
@@ -185,6 +175,41 @@ Table <- R6Class("Table", inherit = ArrowObject,
   )
 )
 
+arrow_attributes <- function(x, only_top_level = FALSE) {
+  att <- attributes(x)
+
+  removed_attributes <- character()
+  if (identical(class(x), c("tbl_df", "tbl", "data.frame"))) {
+    removed_attributes <- c("class", "row.names", "names")
+  } else if (inherits(x, "data.frame")) {
+    removed_attributes <- c("row.names", "names")
+  } else if (inherits(x, "factor")) {
+    removed_attributes <- c("class", "levels")
+  } else if (inherits(x, "integer64") || inherits(x, "Date")) {
+    removed_attributes <- c("class")
+  } else if (inherits(x, "POSIXct")) {
+    removed_attributes <- c("class", "tzone")
+  } else if (inherits(x, "hms") || inherits(x, "difftime")) {
+    removed_attributes <- c("class", "units")
+  }
+
+  att <- att[setdiff(names(att), removed_attributes)]
+  if (isTRUE(only_top_level)) {
+    return(att)
+  }
+
+  if (is.data.frame(x)) {
+    columns <- map(x, arrow_attributes)
+    if (length(att) || !all(map_lgl(columns, is.null))) {
+      list(attributes = att, columns = columns)
+    }
+  } else if (length(att)) {
+    list(attributes = att, columns = NULL)
+  } else {
+    NULL
+  }
+}
+
 Table$create <- function(..., schema = NULL) {
   dots <- list2(...)
   # making sure there are always names
@@ -192,7 +217,11 @@ Table$create <- function(..., schema = NULL) {
     names(dots) <- rep_len("", length(dots))
   }
   stopifnot(length(dots) > 0)
-  shared_ptr(Table, Table__from_dots(dots, schema))
+  if (all_record_batches(dots)) {
+    shared_ptr(Table, Table__from_record_batches(dots, schema))
+  } else {
+    shared_ptr(Table, Table__from_dots(dots, schema))
+  }
 }
 
 #' @export
diff --git a/r/README.md b/r/README.md
index 1d560195676..dc2c7e0616f 100644
--- a/r/README.md
+++ b/r/README.md
@@ -60,10 +60,10 @@ install_arrow()
 ## Installing a development version
 
 Development versions of the package (binary and source) are built daily and hosted at
-<https://dl.bintray.com/ursalabs/arrow-r/>. To install from there:
+<https://arrow-r-nightly.s3.amazonaws.com>. To install from there:
 
 ``` r
-install.packages("arrow", repos = "https://dl.bintray.com/ursalabs/arrow-r")
+install.packages("arrow", repos = "https://arrow-r-nightly.s3.amazonaws.com")
 ```
 
 Or
@@ -72,6 +72,12 @@ Or
 install_arrow(nightly = TRUE)
 ```
 
+Conda users can install `arrow` nightlies from our nightlies channel using:
+
+```
+conda install -c arrow-nightlies -c conda-forge --strict-channel-priority r-arrow
+```
+
 These daily package builds are not official Apache releases and are not
 recommended for production use. They may be useful for testing bug fixes
 and new features under active development.
@@ -91,7 +97,7 @@ brew install apache-arrow --HEAD
 ```
 
 On Windows, you can download a .zip file with the arrow dependencies from the
-[nightly bintray repository](https://dl.bintray.com/ursalabs/arrow-r/libarrow/bin/windows/),
+[nightly repository](https://dl.bintray.com/ursalabs/arrow-r/libarrow/bin/windows/),
 and then set the `RWINLIB_LOCAL` environment variable to point to that
 zip file before installing the `arrow` R package. Version numbers in that
 repository correspond to dates, and you will likely want the most recent.
@@ -102,12 +108,49 @@ elsewhere, you’ll need to build it from source too.
 
 First, install the C++ library. See the [developer
 guide](https://arrow.apache.org/docs/developers/cpp/building.html) for details.
+It's recommended to make a `build` directory inside of the `cpp` directory of
+the Arrow git repository (it is git-ignored). Assuming you are inside `cpp/build`,
+you'll first call `cmake` to configure the build and then `make install`.
+For the R package, you'll need to enable several features in the C++ library
+using `-D` flags:
+
+```
+cmake
+  -DARROW_COMPUTE=ON \
+  -DARROW_CSV=ON \
+  -DARROW_DATASET=ON \
+  -DARROW_FILESYSTEM=ON \
+  -DARROW_JEMALLOC=ON \
+  -DARROW_JSON=ON \
+  -DARROW_PARQUET=ON \
+  -DCMAKE_BUILD_TYPE=release \
+  ..
+```
+
+where `..` is the path to the `cpp/` directory when you're in `cpp/build`.
+
+If you want to enable support for compression libraries, add some or all of these:
+
+```
+  -DARROW_WITH_BROTLI=ON \
+  -DARROW_WITH_BZ2=ON \
+  -DARROW_WITH_LZ4=ON \
+  -DARROW_WITH_SNAPPY=ON \
+  -DARROW_WITH_ZLIB=ON \
+  -DARROW_WITH_ZSTD=ON \
+```
+
+Other flags that may be useful:
+
+* `-DARROW_EXTRA_ERROR_CONTEXT=ON` makes errors coming from the C++ library point to files and line numbers
+* `-DARROW_INSTALL_NAME_RPATH=OFF` may be needed on macOS if there are problems at link time
+* `-DBOOST_SOURCE=BUNDLED`, for example, or any other dependency `*_SOURCE`, if you have a system version of a C++ dependency that doesn't work correctly with Arrow. This tells the build to compile its own version of the dependency from source.
 
 Note that after any change to the C++ library, you must reinstall it and
 run `make clean` or `git clean -fdx .` to remove any cached object code
 in the `r/src/` directory before reinstalling the R package. This is
 only necessary if you make changes to the C++ library source; you do not
-need to manually purge object files if you are only editing R or Rcpp
+need to manually purge object files if you are only editing R or C++
 code inside `r/`.
 
 Once you’ve built the C++ library, you can install the R package and its
@@ -120,7 +163,7 @@ R -e 'install.packages(c("devtools", "roxygen2", "pkgdown", "covr")); devtools::
 R CMD INSTALL .
 ```
 
-If you need to set any compilation flags while building the Rcpp
+If you need to set any compilation flags while building the C++
 extensions, you can use the `ARROW_R_CXXFLAGS` environment variable. For
 example, if you are using `perf` to profile the R extensions, you may
 need to set
@@ -149,9 +192,9 @@ For any other build/configuration challenges, see the [C++ developer
 guide](https://arrow.apache.org/docs/developers/cpp/building.html) and
 `vignette("install", package = "arrow")`.
 
-### Editing Rcpp code
+### Editing C++ code
 
-The `arrow` package uses some customized tools on top of `Rcpp` to
+The `arrow` package uses some customized tools on top of `cpp11` to
 prepare its C++ code in `src/`. If you change C++ code in the R package,
 you will need to set the `ARROW_R_DEV` environment variable to `TRUE`
 (optionally, add it to your`~/.Renviron` file to persist across
@@ -172,6 +215,27 @@ isn’t found, you can explicitly provide the path to it like
 this by installing LLVM via Homebrew and running the script as
 `CLANG_FORMAT=$(brew --prefix llvm@8)/bin/clang-format ./lint.sh`
 
+### Running tests
+
+Some tests are conditionally enabled based on the availability of certain
+features in the package build (S3 support, compression libraries, etc.).
+Others are generally skipped by default but can be enabled with environment
+variables or other settings:
+
+* All tests are skipped on Linux if the package builds without the C++ libarrow.
+  To make the build fail if libarrow is not available (as in, to test that
+  the C++ build was successful), set `TEST_R_WITH_ARROW=TRUE`
+* Some tests are disabled unless `ARROW_R_DEV=TRUE`
+* Tests that require allocating >2GB of memory to test Large types are disabled
+  unless `ARROW_LARGE_MEMORY_TESTS=TRUE`
+* Integration tests against a real S3 bucket are disabled unless credentials
+  are set in `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`; these are available
+  on request
+* S3 tests using [MinIO](https://min.io/) locally are enabled if the
+  `minio server` process is found running. If you're running MinIO with custom
+  settings, you can set `MINIO_ACCESS_KEY`, `MINIO_SECRET_KEY`, and
+  `MINIO_PORT` to override the defaults.
+
 ### Useful functions
 
 Within an R session, these can help with package development:
diff --git a/r/_pkgdown.yml b/r/_pkgdown.yml
index ff48eefd69e..d946f31dc26 100644
--- a/r/_pkgdown.yml
+++ b/r/_pkgdown.yml
@@ -61,6 +61,7 @@ reference:
 - title: Multi-file datasets
   contents:
   - open_dataset
+  - write_dataset
   - dataset_factory
   - hive_partition
   - Dataset
@@ -68,6 +69,7 @@ reference:
   - Expression
   - Scanner
   - FileFormat
+  - FileWriteOptions
   - map_batches
 - title: Reading and writing files
   contents:
@@ -116,6 +118,19 @@ reference:
   - DictionaryType
   - FixedWidthType
   - cast_options
+- title: Flight
+  contents:
+  - load_flight_server
+  - flight_connect
+  - push_data
+  - flight_get
+- title: File systems
+  contents:
+  - s3_bucket
+  - FileSystem
+  - FileInfo
+  - FileSelector
+  - copy_files
 - title: Input/Output
   contents:
   - InputStream
@@ -127,11 +142,6 @@ reference:
   - compression
   - Codec
   - codec_is_available
-  - MemoryPool
-  - default_memory_pool
-  - FileSystem
-  - FileInfo
-  - FileSelector
 - title: Configuration
   contents:
   - cpu_count
diff --git a/r/configure b/r/configure
index ecaeff77d6c..21bad6b1aa2 100755
--- a/r/configure
+++ b/r/configure
@@ -43,7 +43,7 @@ VERSION=`grep ^Version DESCRIPTION | sed s/Version:\ //`
 UNAME=`uname -s`
 
 # generate code
-if [ "$ARROW_R_DEV" = "true" ]; then
+if [ "$ARROW_R_DEV" = "true" ] && [ -f "data-raw/codegen.R" ]; then
   echo "*** Generating code with data-raw/codegen.R"
   ${R_HOME}/bin/Rscript data-raw/codegen.R
 fi
@@ -98,6 +98,8 @@ else
       if [ "$FORCE_AUTOBREW" != "true" ] && [ "`command -v brew`" ] && [ "`brew ls --versions ${PKG_BREW_NAME}`" != "" ]; then
         echo "*** Using Homebrew ${PKG_BREW_NAME}"
         BREWDIR=`brew --prefix`
+        PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS -larrow_bundled_dependencies"
+        PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include"
       else
         echo "*** Downloading ${PKG_BREW_NAME}"
         if [ -f "autobrew" ]; then
@@ -112,9 +114,8 @@ else
         if [ $? -ne 0 ]; then
           echo "Failed to retrieve binary for ${PKG_BREW_NAME}"
         fi
+        # autobrew sets `PKG_LIBS` and `PKG_CFLAGS`
       fi
-      PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include"
-      PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS -larrow_bundled_dependencies"
     elif [ "$UNAME" = "Linux" ]; then
       # Set some default values/backwards compatibility
       if [ "${LIBARROW_DOWNLOAD}" = "" ] && [ "${NOT_CRAN}" != "" ]; then
@@ -178,6 +179,10 @@ if [ $? -eq 0 ] || [ "$UNAME" = "Darwin" ]; then
   grep 'set(ARROW_S3 "ON")' $LIB_DIR/cmake/arrow/ArrowOptions.cmake >/dev/null 2>&1
   if [ $? -eq 0 ]; then
     PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_S3"
+    if [ "$BUNDLED_LIBS" != "" ]; then
+      # We're depending on openssl/curl from the system, so they're not in the bundled deps
+      PKG_LIBS="$PKG_LIBS -lssl -lcrypto -lcurl"
+    fi
   fi
   echo "PKG_CFLAGS=$PKG_CFLAGS"
   echo "PKG_LIBS=$PKG_LIBS"
diff --git a/r/configure.win b/r/configure.win
index 5caf8e24663..886205fdafa 100644
--- a/r/configure.win
+++ b/r/configure.win
@@ -44,11 +44,22 @@ else
 fi
 OPENSSL_LIBS="-lcrypto -lcrypt32"
 MIMALLOC_LIBS="-lbcrypt -lpsapi"
+AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-identity-management -laws-cpp-sdk-cognito-identity -laws-cpp-sdk-sts -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -lUserenv -lversion -lws2_32 -lBcrypt -lWininet -lwinhttp"
 
 # NOTE: If you make changes to the libraries below, you should also change
 # ci/scripts/r_windows_build.sh and ci/scripts/PKGBUILD
 PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_DS_STATIC -DARROW_R_WITH_ARROW"
-PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lsnappy -lz -lzstd -llz4 ${MIMALLOC_LIBS} ${OPENSSL_LIBS} -lws2_32"
+PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lsnappy -lz -lzstd -llz4 ${MIMALLOC_LIBS} ${OPENSSL_LIBS}"
+
+# S3 support only for Rtools40 (i.e. R >= 4.0)
+"${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" -e 'R.version$major >= 4' | grep TRUE >/dev/null 2>&1
+if [ $? -eq 0 ]; then
+  PKG_CFLAGS="${PKG_CFLAGS} -DARROW_R_WITH_S3"
+  PKG_LIBS="${PKG_LIBS} ${AWS_LIBS}"
+else
+  # It seems that order matters
+  PKG_LIBS="${PKG_LIBS} -lws2_32"
+fi
 
 # Set any user-defined CXXFLAGS
 if [ "$ARROW_R_CXXFLAGS" ]; then
diff --git a/r/data-raw/codegen.R b/r/data-raw/codegen.R
index a7f3581aa8d..ede4043c46f 100644
--- a/r/data-raw/codegen.R
+++ b/r/data-raw/codegen.R
@@ -18,10 +18,10 @@
 # This file is used to generate code in the files
 # src/arrowExports.cpp and R/arrowExports.R
 #
-# This is similar to what Rcpp::compileAttributes() would do,
+# This is similar to what compileAttributes() would do,
 # with some arrow specific changes.
 #
-# Functions are decorated with [[arrow::export]] instead of [[Rcpp::export]]
+# Functions are decorated with [[arrow::export]]
 # and the generated code adds a layer of protection so that
 # the arrow package can be installed even when libarrow is not
 #
@@ -45,9 +45,6 @@ suppressPackageStartupMessages({
 get_exported_functions <- function(decorations, export_tag) {
   out <- decorations %>%
     filter(decoration %in% paste0(export_tag, "::export")) %>%
-    # the three lines below can be expressed with rap()
-    # more concisely
-    # rap(            ~ decor:::parse_cpp_function(context))
     mutate(functions = map(context, decor:::parse_cpp_function)) %>%
     { vec_cbind(., vec_rbind(!!!pull(., functions))) } %>%
     select(-functions) %>%
@@ -67,7 +64,7 @@ wrap_call <- function(name, return_type, args) {
   if(return_type == "void") {
     glue::glue("\t{call};\n\treturn R_NilValue;", .trim = FALSE)
   } else {
-    glue::glue("\treturn Rcpp::wrap({call});")
+    glue::glue("\treturn cpp11::as_sexp({call});")
   }
 }
 
@@ -81,13 +78,13 @@ cpp_functions_definitions <- arrow_exports %>%
     // {basename(file)}
     #if defined(ARROW_R_WITH_{toupper(decoration)})
     {return_type} {name}({real_params});
-    RcppExport SEXP _arrow_{name}({sexp_params}){{
-    BEGIN_RCPP
+    extern "C" SEXP _arrow_{name}({sexp_params}){{
+    BEGIN_CPP11
     {input_params}{return_line}{wrap_call(name, return_type, args)}
-    END_RCPP
+    END_CPP11
     }}
     #else
-    RcppExport SEXP _arrow_{name}({sexp_params}){{
+    extern "C" SEXP _arrow_{name}({sexp_params}){{
     \tRf_error("Cannot call {name}(). Please use arrow::install_arrow() to install required runtime libraries. ");
     }}
     #endif
@@ -96,7 +93,7 @@ cpp_functions_definitions <- arrow_exports %>%
       sep = "\n",
       real_params = glue_collapse_data(args, "{type} {name}"),
       sexp_params = glue_collapse_data(args, "SEXP {name}_sexp"),
-      input_params = glue_collapse_data(args, "\tRcpp::traits::input_parameter<{type}>::type {name}({name}_sexp);", sep = "\n"),
+      input_params = glue_collapse_data(args, "\tarrow::r::Input<{type}>::type {name}({name}_sexp);", sep = "\n"),
       return_line = if(nrow(args)) "\n" else ""
     )
   }) %>%
@@ -111,10 +108,10 @@ cpp_functions_registration <- arrow_exports %>%
 
 writeLines(con = "src/arrowExports.cpp", glue::glue('
 // Generated by using data-raw/codegen.R -> do not edit by hand
-#include "./arrow_exports.h"
-#include <Rcpp.h>
+#include <cpp11.hpp>
+#include <cpp11/declarations.hpp>
 
-using namespace Rcpp;
+#include "./arrow_exports.h"
 
 {cpp_functions_definitions}
 
@@ -145,7 +142,7 @@ static const R_CallMethodDef CallEntries[] = {{
 \t\t{{NULL, NULL, 0}}
 }};
 
-RcppExport void R_init_arrow(DllInfo* dll){{
+extern "C" void R_init_arrow(DllInfo* dll){{
   R_registerRoutines(dll, NULL, CallEntries, NULL, NULL);
   R_useDynamicSymbols(dll, FALSE);
 }}
diff --git a/r/inst/build_arrow_static.sh b/r/inst/build_arrow_static.sh
index 86c1ee9ab5d..a57aa70a068 100755
--- a/r/inst/build_arrow_static.sh
+++ b/r/inst/build_arrow_static.sh
@@ -63,6 +63,7 @@ ${CMAKE} -DARROW_BOOST_USE_SHARED=OFF \
     -DARROW_JEMALLOC=${ARROW_JEMALLOC:-ON} \
     -DARROW_JSON=ON \
     -DARROW_PARQUET=ON \
+    -DARROW_S3=${ARROW_S3:-$ARROW_DEFAULT_PARAM} \
     -DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI:-$ARROW_DEFAULT_PARAM} \
     -DARROW_WITH_BZ2=${ARROW_WITH_BZ2:-$ARROW_DEFAULT_PARAM} \
     -DARROW_WITH_LZ4=${ARROW_WITH_LZ4:-$ARROW_DEFAULT_PARAM} \
@@ -76,7 +77,6 @@ ${CMAKE} -DARROW_BOOST_USE_SHARED=OFF \
     -DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON \
     -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON \
     -DCMAKE_UNITY_BUILD=ON \
-    -DOPENSSL_USE_STATIC_LIBS=ON \
     ${EXTRA_CMAKE_FLAGS} \
     -G ${CMAKE_GENERATOR:-"Unix Makefiles"} \
     ${SOURCE_DIR}
diff --git a/r/inst/demo_flight_server.py b/r/inst/demo_flight_server.py
new file mode 100644
index 00000000000..0c81aa912a1
--- /dev/null
+++ b/r/inst/demo_flight_server.py
@@ -0,0 +1,120 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+    An example Flight Python server.
+    See https://github.com/apache/arrow/blob/master/python/examples/flight/server.py
+"""
+
+import ast
+import threading
+import time
+
+import pyarrow
+import pyarrow.flight
+
+
+class DemoFlightServer(pyarrow.flight.FlightServerBase):
+    def __init__(self, host="localhost", port=5005):
+        if isinstance(port, float):
+            # Because R is looser with integer vs. float
+            port = int(port)
+        location = "grpc+tcp://{}:{}".format(host, port)
+        super(DemoFlightServer, self).__init__(location)
+        self.flights = {}
+        self.host = host
+
+    @classmethod
+    def descriptor_to_key(self, descriptor):
+        return (descriptor.descriptor_type.value, descriptor.command,
+                tuple(descriptor.path or tuple()))
+
+    def _make_flight_info(self, key, descriptor, table):
+        location = pyarrow.flight.Location.for_grpc_tcp(self.host, self.port)
+        endpoints = [pyarrow.flight.FlightEndpoint(repr(key), [location]), ]
+
+        mock_sink = pyarrow.MockOutputStream()
+        stream_writer = pyarrow.RecordBatchStreamWriter(
+            mock_sink, table.schema)
+        stream_writer.write_table(table)
+        stream_writer.close()
+        data_size = mock_sink.size()
+
+        return pyarrow.flight.FlightInfo(table.schema,
+                                         descriptor, endpoints,
+                                         table.num_rows, data_size)
+
+    def list_flights(self, context, criteria):
+        print("list_flights")
+        for key, table in self.flights.items():
+            if key[1] is not None:
+                descriptor = \
+                    pyarrow.flight.FlightDescriptor.for_command(key[1])
+            else:
+                descriptor = pyarrow.flight.FlightDescriptor.for_path(*key[2])
+
+            yield self._make_flight_info(key, descriptor, table)
+
+    def get_flight_info(self, context, descriptor):
+        print("get_flight_info")
+        key = DemoFlightServer.descriptor_to_key(descriptor)
+        if key in self.flights:
+            table = self.flights[key]
+            return self._make_flight_info(key, descriptor, table)
+        raise KeyError('Flight not found.')
+
+    def do_put(self, context, descriptor, reader, writer):
+        print("do_put")
+        key = DemoFlightServer.descriptor_to_key(descriptor)
+        print(key)
+        self.flights[key] = reader.read_all()
+        print(self.flights[key])
+
+    def do_get(self, context, ticket):
+        print("do_get")
+        key = ast.literal_eval(ticket.ticket.decode())
+        if key not in self.flights:
+            return None
+        return pyarrow.flight.RecordBatchStream(self.flights[key])
+
+    def list_actions(self, context):
+        print("list_actions")
+        return [
+            ("clear", "Clear the stored flights."),
+            ("shutdown", "Shut down this server."),
+        ]
+
+    def do_action(self, context, action):
+        print("do_action")
+        if action.type == "clear":
+            raise NotImplementedError(
+                "{} is not implemented.".format(action.type))
+        elif action.type == "healthcheck":
+            pass
+        elif action.type == "shutdown":
+            yield pyarrow.flight.Result(pyarrow.py_buffer(b'Shutdown!'))
+            # Shut down on background thread to avoid blocking current
+            # request
+            threading.Thread(target=self._shutdown).start()
+        else:
+            raise KeyError("Unknown action {!r}".format(action.type))
+
+    def _shutdown(self):
+        """Shut down after a delay."""
+        print("Server is shutting down...")
+        time.sleep(2)
+        self.shutdown()
diff --git a/r/man/CsvReadOptions.Rd b/r/man/CsvReadOptions.Rd
index 81c6be4fb3f..805330768b3 100644
--- a/r/man/CsvReadOptions.Rd
+++ b/r/man/CsvReadOptions.Rd
@@ -4,13 +4,14 @@
 \name{CsvReadOptions}
 \alias{CsvReadOptions}
 \alias{CsvParseOptions}
+\alias{TimestampParser}
 \alias{CsvConvertOptions}
 \alias{JsonReadOptions}
 \alias{JsonParseOptions}
 \title{File reader options}
 \description{
 \code{CsvReadOptions}, \code{CsvParseOptions}, \code{CsvConvertOptions},
-\code{JsonReadOptions}, and \code{JsonParseOptions} are containers for various
+\code{JsonReadOptions}, \code{JsonParseOptions}, and \code{TimestampParser} are containers for various
 file reading options. See their usage in \code{\link[=read_csv_arrow]{read_csv_arrow()}} and
 \code{\link[=read_json_arrow]{read_json_arrow()}}, respectively.
 }
@@ -62,13 +63,38 @@ Analogous to the \code{na.strings} argument to
 \item \code{strings_can_be_null} Logical: can string / binary columns have
 null values? Similar to the \code{quoted_na} argument to \code{readr::read_csv()}.
 (default \code{FALSE})
-}
+\item \code{true_values} character vector of recognized spellings for \code{TRUE} values
+\item \code{false_values} character vector of recognized spellings for \code{FALSE} values
+\item \code{col_types} A \code{Schema} or \code{NULL} to infer types
+\item \code{auto_dict_encode} Logical: Whether to try to automatically
+dictionary-encode string / binary data (think \code{stringsAsFactors}). Default \code{FALSE}.
+This setting is ignored for non-inferred columns (those in \code{col_types}).
+\item \code{auto_dict_max_cardinality} If \code{auto_dict_encode}, string/binary columns
+are dictionary-encoded up to this number of unique values (default 50),
+after which it switches to regular encoding.
+\item \code{include_columns} If non-empty, indicates the names of columns from the
+CSV file that should be actually read and converted (in the vector's order).
+\item \code{include_missing_columns} Logical: if \code{include_columns} is provided, should
+columns named in it but not found in the data be included as a column of
+type \code{null()}? The default (\code{FALSE}) means that the reader will instead
+raise an error.
+\item \code{timestamp_parsers} User-defined timestamp parsers. If more than one
+parser is specified, the CSV conversion logic will try parsing values
+starting from the beginning of this vector. Possible values are
+(a) \code{NULL}, the default, which uses the ISO-8601 parser;
+(b) a character vector of \link[base:strptime]{strptime} parse strings; or
+(c) a list of \link{TimestampParser} objects.
 }
 
-\section{Methods}{
+\code{TimestampParser$create()} takes an optional \code{format} string argument.
+See \code{\link[base:strptime]{strptime()}} for example syntax.
+The default is to use an ISO-8601 format parser.
+}
 
+\section{Active bindings}{
 
-These classes have no implemented methods. They are containers for the
-options.
+\itemize{
+\item \code{column_names}: from \code{CsvReadOptions}
+}
 }
 
diff --git a/r/man/CsvTableReader.Rd b/r/man/CsvTableReader.Rd
index 8343653b862..1afa9d02016 100644
--- a/r/man/CsvTableReader.Rd
+++ b/r/man/CsvTableReader.Rd
@@ -16,7 +16,7 @@ and JSON table readers. See their usage in \code{\link[=read_csv_arrow]{read_csv
 The \code{CsvTableReader$create()} and \code{JsonTableReader$create()} factory methods
 take the following arguments:
 \itemize{
-\item \code{file} A character path to a local file, or an Arrow input stream
+\item \code{file} An Arrow \link{InputStream}
 \item \code{convert_options} (CSV only), \code{parse_options}, \code{read_options}: see
 \link{CsvReadOptions}
 \item \code{...} additional parameters.
diff --git a/r/man/Dataset.Rd b/r/man/Dataset.Rd
index 1f8ce96f456..3c9a314195d 100644
--- a/r/man/Dataset.Rd
+++ b/r/man/Dataset.Rd
@@ -1,9 +1,10 @@
 % Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/dataset.R
+% Please edit documentation in R/dataset.R, R/dataset-factory.R
 \name{Dataset}
 \alias{Dataset}
 \alias{FileSystemDataset}
 \alias{UnionDataset}
+\alias{InMemoryDataset}
 \alias{DatasetFactory}
 \alias{FileSystemDatasetFactory}
 \title{Multi-file datasets}
@@ -60,9 +61,6 @@ A \code{Dataset} has the following methods:
 may also replace the dataset's schema by using \code{ds$schema <- new_schema}.
 This method currently supports only adding, removing, or reordering
 fields in the schema: you cannot alter or cast the field types.
-\item \verb{$write(path, filesystem, schema, format, partitioning)}: writes the
-dataset to \code{path} in the \code{format} file format, partitioned by \code{partitioning},
-and invisibly returns \code{self}. See \code{\link[=write_dataset]{write_dataset()}}.
 }
 
 \code{FileSystemDataset} has the following methods:
diff --git a/r/man/FileFormat.Rd b/r/man/FileFormat.Rd
index b8959bcc89b..c17e98bce55 100644
--- a/r/man/FileFormat.Rd
+++ b/r/man/FileFormat.Rd
@@ -1,5 +1,5 @@
 % Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/dataset.R
+% Please edit documentation in R/dataset-format.R
 \name{FileFormat}
 \alias{FileFormat}
 \alias{ParquetFileFormat}
diff --git a/r/man/FileSystem.Rd b/r/man/FileSystem.Rd
index a675c5bfb4e..2f3dcff670b 100644
--- a/r/man/FileSystem.Rd
+++ b/r/man/FileSystem.Rd
@@ -16,11 +16,40 @@ to another implementation after prepending a fixed base path
 \section{Factory}{
 
 
-The \verb{$create()} factory methods instantiate the \code{FileSystem} object and
-take the following arguments, depending on the subclass:
+\code{LocalFileSystem$create()} returns the object and takes no arguments.
+
+\code{SubTreeFileSystem$create()} takes the following arguments:
 \itemize{
-\item no argument is needed for instantiating a \code{LocalFileSystem}
-\item \code{base_path} and \code{base_fs} for instantiating a \code{SubTreeFileSystem}
+\item \code{base_path}, a string path
+\item \code{base_fs}, a \code{FileSystem} object
+}
+
+\code{S3FileSystem$create()} optionally takes arguments:
+\itemize{
+\item \code{anonymous}: logical, default \code{FALSE}. If true, will not attempt to look up
+credentials using standard AWS configuration methods.
+\item \code{access_key}, \code{secret_key}: authentication credentials. If one is provided,
+the other must be as well. If both are provided, they will override any
+AWS configuration set at the environment level.
+\item \code{session_token}: optional string for authentication along with
+\code{access_key} and \code{secret_key}
+\item \code{role_arn}: string AWS ARN of an AccessRole. If provided instead of \code{access_key} and
+\code{secret_key}, temporary credentials will be fetched by assuming this role.
+\item \code{session_name}: optional string identifier for the assumed role session.
+\item \code{external_id}: optional unique string identifier that might be required
+when you assume a role in another account.
+\item \code{load_frequency}: integer, frequency (in seconds) with which temporary
+credentials from an assumed role session will be refreshed. Default is
+900 (i.e. 15 minutes)
+\item \code{region}: AWS region to connect to. If omitted, the AWS library will
+provide a sensible default based on client configuration, falling back
+to "us-east-1" if no other alternatives are found.
+\item \code{endpoint_override}: If non-empty, override region with a connect string
+such as "localhost:9000". This is useful for connecting to file systems
+that emulate S3.
+\item \code{scheme}: S3 connection transport (default "https")
+\item \code{background_writes}: logical, whether \code{OutputStream} writes will be issued
+in the background, without blocking (default \code{TRUE})
 }
 }
 
@@ -56,3 +85,15 @@ appending.
 }
 }
 
+\section{Active bindings}{
+
+\itemize{
+\item \verb{$type_name}: string filesystem type name, such as "local", "s3", etc.
+\item \verb{$region}: string AWS region, for \code{S3FileSystem} and \code{SubTreeFileSystem}
+containing a \code{S3FileSystem}
+\item \verb{$base_fs}: for \code{SubTreeFileSystem}, the \code{FileSystem} it contains
+\item \verb{$base_path}: for \code{SubTreeFileSystem}, the path in \verb{$base_fs} which is considered
+root in this \code{SubTreeFileSystem}.
+}
+}
+
diff --git a/r/man/FileWriteOptions.Rd b/r/man/FileWriteOptions.Rd
new file mode 100644
index 00000000000..661393c8e0d
--- /dev/null
+++ b/r/man/FileWriteOptions.Rd
@@ -0,0 +1,8 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/dataset-format.R
+\name{FileWriteOptions}
+\alias{FileWriteOptions}
+\title{Format-specific write options}
+\description{
+A \code{FileWriteOptions} holds write options specific to a \code{FileFormat}.
+}
diff --git a/r/man/MemoryPool.Rd b/r/man/MemoryPool.Rd
index 8bffc76caab..9b16c45165b 100644
--- a/r/man/MemoryPool.Rd
+++ b/r/man/MemoryPool.Rd
@@ -9,7 +9,10 @@ class arrow::MemoryPool
 }
 \section{Methods}{
 
-
-TODO
+\itemize{
+\item \code{bytes_allocated()}
+\item \code{max_memory()}
+}
 }
 
+\keyword{internal}
diff --git a/r/man/ParquetFileReader.Rd b/r/man/ParquetFileReader.Rd
index 1e9f78f4d04..828ba696e28 100644
--- a/r/man/ParquetFileReader.Rd
+++ b/r/man/ParquetFileReader.Rd
@@ -24,10 +24,23 @@ takes the following arguments:
 \section{Methods}{
 
 \itemize{
-\item \verb{$ReadTable(col_select)}: get an \code{arrow::Table} from the file, possibly
-with columns filtered by a character vector of column names or a
-\code{tidyselect} specification.
+\item \verb{$ReadTable(column_indices)}: get an \code{arrow::Table} from the file. The optional
+\verb{column_indices=} argument is a 0-based integer vector indicating which columns to retain.
+\item \verb{$ReadRowGroup(i, column_indices)}: get an \code{arrow::Table} by reading the \code{i}th row group (0-based).
+The optional \verb{column_indices=} argument is a 0-based integer vector indicating which columns to retain.
+\item \verb{$ReadRowGroups(row_groups, column_indices)}: get an \code{arrow::Table} by reading several row groups (0-based integers).
+The optional \verb{column_indices=} argument is a 0-based integer vector indicating which columns to retain.
 \item \verb{$GetSchema()}: get the \code{arrow::Schema} of the data in the file
+\item \verb{$ReadColumn(i)}: read the \code{i}th column (0-based) as a \link{ChunkedArray}.
+}
+}
+
+\section{Active bindings}{
+
+\itemize{
+\item \verb{$num_rows}: number of rows.
+\item \verb{$num_columns}: number of columns.
+\item \verb{$num_row_groups}: number of row groups.
 }
 }
 
@@ -38,7 +51,7 @@ pq <- ParquetFileReader$create(f)
 pq$GetSchema()
 if (codec_is_available("snappy")) {
   # This file has compressed data columns
-  tab <- pq$ReadTable(starts_with("c"))
+  tab <- pq$ReadTable()
   tab$schema
 }
 }
diff --git a/r/man/ParquetFileWriter.Rd b/r/man/ParquetFileWriter.Rd
index c53ae321a8c..f36e85ab6c4 100644
--- a/r/man/ParquetFileWriter.Rd
+++ b/r/man/ParquetFileWriter.Rd
@@ -14,9 +14,18 @@ The \code{ParquetFileWriter$create()} factory method instantiates the object and
 takes the following arguments:
 \itemize{
 \item \code{schema} A \link{Schema}
-\item \code{sink} An \link[=OutputStream]{arrow::io::OutputStream} or a string which is interpreted as a file path
+\item \code{sink} An \link[=OutputStream]{arrow::io::OutputStream}
 \item \code{properties} An instance of \link{ParquetWriterProperties}
 \item \code{arrow_properties} An instance of \code{ParquetArrowWriterProperties}
 }
 }
 
+\section{Methods}{
+
+\itemize{
+\item \code{WriteTable} Write a \link{Table} to \code{sink}
+\item \code{Close} Close the writer. Note: does not close the \code{sink}.
+\link[=OutputStream]{arrow::io::OutputStream} has its own \code{close()} method.
+}
+}
+
diff --git a/r/man/Partitioning.Rd b/r/man/Partitioning.Rd
index dc12832c97f..342b66455fe 100644
--- a/r/man/Partitioning.Rd
+++ b/r/man/Partitioning.Rd
@@ -1,5 +1,5 @@
 % Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/dataset.R
+% Please edit documentation in R/dataset-partition.R
 \name{Partitioning}
 \alias{Partitioning}
 \alias{DirectoryPartitioning}
diff --git a/r/man/RecordBatch.Rd b/r/man/RecordBatch.Rd
index 40c34968d63..c9cdb343ef8 100644
--- a/r/man/RecordBatch.Rd
+++ b/r/man/RecordBatch.Rd
@@ -48,9 +48,7 @@ the following R6 methods that map onto the underlying C++ methods:
 \item \verb{$names()}: Get all column names (called by \code{names(batch)})
 \item \verb{$GetColumnByName(name)}: Extract an \code{Array} by string name
 \item \verb{$RemoveColumn(i)}: Drops a column from the batch by integer position
-\item \verb{$select(spec)}: Return a new record batch with a selection of columns.
-This supports the usual \code{character}, \code{numeric}, and \code{logical} selection
-methods as well as "tidy select" expressions.
+\item \verb{$selectColumns(indices)}: Return a new record batch with a selection of columns, expressed as 0-based integers.
 \item \verb{$Slice(offset, length = NULL)}: Create a zero-copy view starting at the
 indicated integer offset and going for the given length, or to the end
 of the table if \code{NULL}, the default.
diff --git a/r/man/RecordBatchWriter.Rd b/r/man/RecordBatchWriter.Rd
index 0422da6caae..038653b9e24 100644
--- a/r/man/RecordBatchWriter.Rd
+++ b/r/man/RecordBatchWriter.Rd
@@ -23,7 +23,7 @@ factory methods instantiate the object and take the following arguments:
 \item \code{sink} An \code{OutputStream}
 \item \code{schema} A \link{Schema} for the data to be written
 \item \code{use_legacy_format} logical: write data formatted so that Arrow libraries
-versions 0.14 and lower can read it? Default is \code{FALSE}. You can also
+versions 0.14 and lower can read it. Default is \code{FALSE}. You can also
 enable this by setting the environment variable \code{ARROW_PRE_0_15_IPC_FORMAT=1}.
 \item \code{metadata_version}: A string like "V5" or the equivalent integer indicating
 the Arrow IPC MetadataVersion. Default (NULL) will use the latest version,
diff --git a/r/man/Scanner.Rd b/r/man/Scanner.Rd
index 2b82d2029a4..36b6e7f21f7 100644
--- a/r/man/Scanner.Rd
+++ b/r/man/Scanner.Rd
@@ -1,5 +1,5 @@
 % Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/dataset.R
+% Please edit documentation in R/dataset-scan.R
 \name{Scanner}
 \alias{Scanner}
 \alias{ScannerBuilder}
diff --git a/r/man/Table.Rd b/r/man/Table.Rd
index 2014a30be9e..18c7da12393 100644
--- a/r/man/Table.Rd
+++ b/r/man/Table.Rd
@@ -46,9 +46,7 @@ the following R6 methods that map onto the underlying C++ methods:
 \item \verb{$ColumnNames()}: Get all column names (called by \code{names(tab)})
 \item \verb{$GetColumnByName(name)}: Extract a \code{ChunkedArray} by string name
 \item \verb{$field(i)}: Extract a \code{Field} from the table schema by integer position
-\item \verb{$select(spec)}: Return a new table with a selection of columns.
-This supports the usual \code{character}, \code{numeric}, and \code{logical} selection
-methods as well as "tidy select" expressions.
+\item \verb{$SelectColumns(indices)}: Return new \code{Table} with specified columns, expressed as 0-based integers.
 \item \verb{$Slice(offset, length = NULL)}: Create a zero-copy view starting at the
 indicated integer offset and going for the given length, or to the end
 of the table if \code{NULL}, the default.
diff --git a/r/man/arrow-package.Rd b/r/man/arrow-package.Rd
index 0f90d55fb00..0c19402a045 100644
--- a/r/man/arrow-package.Rd
+++ b/r/man/arrow-package.Rd
@@ -16,7 +16,7 @@
 Useful links:
 \itemize{
   \item \url{https://github.com/apache/arrow/}
-  \item \url{https://arrow.apache.org/docs/r}
+  \item \url{https://arrow.apache.org/docs/r/}
   \item Report bugs at \url{https://issues.apache.org/jira/projects/ARROW/issues}
 }
 
diff --git a/r/man/arrow_available.Rd b/r/man/arrow_available.Rd
index 9e696d530d0..bca7e684654 100644
--- a/r/man/arrow_available.Rd
+++ b/r/man/arrow_available.Rd
@@ -2,19 +2,28 @@
 % Please edit documentation in R/arrow-package.R
 \name{arrow_available}
 \alias{arrow_available}
+\alias{arrow_with_s3}
 \title{Is the C++ Arrow library available?}
 \usage{
 arrow_available()
+
+arrow_with_s3()
 }
 \value{
 \code{TRUE} or \code{FALSE} depending on whether the package was installed
-with the Arrow C++ library. If \code{FALSE}, you'll need to install the C++
-library and then reinstall the R package. See \code{\link[=install_arrow]{install_arrow()}} for help.
+with the Arrow C++ library (check with \code{arrow_available()}) or with S3
+support enabled (check with \code{arrow_with_s3()}).
 }
 \description{
-You won't generally need to call this function, but it's here in case it
-helps for development purposes.
+You won't generally need to call these function, but they're made available
+for diagnostic purposes.
 }
 \examples{
 arrow_available()
+arrow_with_s3()
+}
+\seealso{
+If either of these are \code{FALSE}, see
+\code{vignette("install", package = "arrow")} for guidance on reinstalling the
+package.
 }
diff --git a/r/man/copy_files.Rd b/r/man/copy_files.Rd
new file mode 100644
index 00000000000..65edf56cb48
--- /dev/null
+++ b/r/man/copy_files.Rd
@@ -0,0 +1,35 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/filesystem.R
+\name{copy_files}
+\alias{copy_files}
+\title{Copy files between FileSystems}
+\usage{
+copy_files(from, to, chunk_size = 1024L * 1024L)
+}
+\arguments{
+\item{from}{A string path to a local directory or file, a URI, or a
+\code{SubTreeFileSystem}. Files will be copied recursively from this path.}
+
+\item{to}{A string path to a local directory or file, a URI, or a
+\code{SubTreeFileSystem}. Directories will be created as necessary}
+
+\item{chunk_size}{The maximum size of block to read before flushing
+to the destination file. A larger chunk_size will use more memory while
+copying but may help accommodate high latency FileSystems.}
+}
+\value{
+Nothing: called for side effects in the file system
+}
+\description{
+Copy files between FileSystems
+}
+\examples{
+\dontrun{
+# Copy an S3 bucket's files to a local directory:
+copy_files("s3://your-bucket-name", "local-directory")
+# Using a FileSystem object
+copy_files(s3_bucket("your-bucket-name"), "local-directory")
+# Or go the other way, from local to S3
+copy_files("local-directory", s3_bucket("your-bucket-name"))
+}
+}
diff --git a/r/man/dataset_factory.Rd b/r/man/dataset_factory.Rd
index efc967b36e2..3216c5b0a16 100644
--- a/r/man/dataset_factory.Rd
+++ b/r/man/dataset_factory.Rd
@@ -1,5 +1,5 @@
 % Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/dataset.R
+% Please edit documentation in R/dataset-factory.R
 \name{dataset_factory}
 \alias{dataset_factory}
 \title{Create a DatasetFactory}
diff --git a/r/man/default_memory_pool.Rd b/r/man/default_memory_pool.Rd
index 859b40631af..51dde979be5 100644
--- a/r/man/default_memory_pool.Rd
+++ b/r/man/default_memory_pool.Rd
@@ -12,3 +12,4 @@ the default \link[=MemoryPool]{arrow::MemoryPool}
 \description{
 default \link[=MemoryPool]{arrow::MemoryPool}
 }
+\keyword{internal}
diff --git a/r/man/enums.Rd b/r/man/enums.Rd
index 67890fbb4a5..e4cb2d85469 100644
--- a/r/man/enums.Rd
+++ b/r/man/enums.Rd
@@ -19,7 +19,7 @@ An object of class \code{TimeUnit::type} (inherits from \code{arrow-enum}) of le
 
 An object of class \code{DateUnit} (inherits from \code{arrow-enum}) of length 2.
 
-An object of class \code{Type::type} (inherits from \code{arrow-enum}) of length 36.
+An object of class \code{Type::type} (inherits from \code{arrow-enum}) of length 37.
 
 An object of class \code{StatusCode} (inherits from \code{arrow-enum}) of length 17.
 
diff --git a/r/man/flight_connect.Rd b/r/man/flight_connect.Rd
new file mode 100644
index 00000000000..9da7fad7530
--- /dev/null
+++ b/r/man/flight_connect.Rd
@@ -0,0 +1,21 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/flight.R
+\name{flight_connect}
+\alias{flight_connect}
+\title{Connect to a Flight server}
+\usage{
+flight_connect(host = "localhost", port, scheme = "grpc+tcp")
+}
+\arguments{
+\item{host}{string hostname to connect to}
+
+\item{port}{integer port to connect on}
+
+\item{scheme}{URL scheme, default is "grpc+tcp"}
+}
+\value{
+A \code{pyarrow.flight.FlightClient}.
+}
+\description{
+Connect to a Flight server
+}
diff --git a/r/man/flight_get.Rd b/r/man/flight_get.Rd
new file mode 100644
index 00000000000..f427b39b1fb
--- /dev/null
+++ b/r/man/flight_get.Rd
@@ -0,0 +1,19 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/flight.R
+\name{flight_get}
+\alias{flight_get}
+\title{Get data from a Flight server}
+\usage{
+flight_get(client, path)
+}
+\arguments{
+\item{client}{\code{pyarrow.flight.FlightClient}, as returned by \code{\link[=flight_connect]{flight_connect()}}}
+
+\item{path}{string identifier under which the data is stored}
+}
+\value{
+A \link{RecordBatch}
+}
+\description{
+Get data from a Flight server
+}
diff --git a/r/man/hive_partition.Rd b/r/man/hive_partition.Rd
index c5c7f2c94d9..16729150bee 100644
--- a/r/man/hive_partition.Rd
+++ b/r/man/hive_partition.Rd
@@ -1,5 +1,5 @@
 % Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/dataset.R
+% Please edit documentation in R/dataset-partition.R
 \name{hive_partition}
 \alias{hive_partition}
 \title{Construct Hive partitioning}
diff --git a/r/man/load_flight_server.Rd b/r/man/load_flight_server.Rd
new file mode 100644
index 00000000000..7e2000a9ca2
--- /dev/null
+++ b/r/man/load_flight_server.Rd
@@ -0,0 +1,17 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/flight.R
+\name{load_flight_server}
+\alias{load_flight_server}
+\title{Load a Python Flight server}
+\usage{
+load_flight_server(name, path = system.file(package = "arrow"))
+}
+\arguments{
+\item{name}{string Python module name}
+
+\item{path}{file system path where the Python module is found. Default is
+to look in the \verb{inst/} directory for included modules.}
+}
+\description{
+Load a Python Flight server
+}
diff --git a/r/man/make_readable_file.Rd b/r/man/make_readable_file.Rd
index 11d302c0b04..fe2e2982612 100644
--- a/r/man/make_readable_file.Rd
+++ b/r/man/make_readable_file.Rd
@@ -4,7 +4,7 @@
 \alias{make_readable_file}
 \title{Handle a range of possible input sources}
 \usage{
-make_readable_file(file, mmap = TRUE, compression = NULL)
+make_readable_file(file, mmap = TRUE, compression = NULL, filesystem = NULL)
 }
 \arguments{
 \item{file}{A character file name, \code{raw} vector, or an Arrow input stream}
@@ -15,6 +15,10 @@ make_readable_file(file, mmap = TRUE, compression = NULL)
 with this compression codec, either a \link{Codec} or the string name of one.
 If \code{NULL} (default) and \code{file} is a string file name, the function will try
 to infer compression from the file extension.}
+
+\item{filesystem}{If not \code{NULL}, \code{file} will be opened via the
+\code{filesystem$OpenInputFile()} filesystem method, rather than the \code{io} module's
+\code{MemoryMappedFile} or \code{ReadableFile} constructors.}
 }
 \value{
 An \code{InputStream} or a subclass of one.
diff --git a/r/man/map_batches.Rd b/r/man/map_batches.Rd
index 67d97a8f655..08e7b86c057 100644
--- a/r/man/map_batches.Rd
+++ b/r/man/map_batches.Rd
@@ -1,5 +1,5 @@
 % Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/dataset.R
+% Please edit documentation in R/dataset-scan.R
 \name{map_batches}
 \alias{map_batches}
 \title{Apply a function to a stream of RecordBatches}
diff --git a/r/man/match_arrow.Rd b/r/man/match_arrow.Rd
new file mode 100644
index 00000000000..9097797ad76
--- /dev/null
+++ b/r/man/match_arrow.Rd
@@ -0,0 +1,23 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/compute.R
+\name{match_arrow}
+\alias{match_arrow}
+\title{\code{match} for Arrow objects}
+\usage{
+match_arrow(x, table, ...)
+}
+\arguments{
+\item{x}{\code{Array} or \code{ChunkedArray}}
+
+\item{table}{\code{Array}, \code{ChunkedArray}, or R vector lookup table.}
+
+\item{...}{additional arguments, ignored}
+}
+\value{
+An \code{int32}-type \code{Array} of the same length as \code{x} with the
+(0-based) indexes into \code{table}.
+}
+\description{
+\code{base::match()} is not a generic, so we can't just define Arrow methods for
+it. This function exposes the analogous function in the Arrow C++ library.
+}
diff --git a/r/man/push_data.Rd b/r/man/push_data.Rd
new file mode 100644
index 00000000000..8fb65c7cd72
--- /dev/null
+++ b/r/man/push_data.Rd
@@ -0,0 +1,21 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/flight.R
+\name{push_data}
+\alias{push_data}
+\title{Send data to a Flight server}
+\usage{
+push_data(client, data, path)
+}
+\arguments{
+\item{client}{\code{pyarrow.flight.FlightClient}, as returned by \code{\link[=flight_connect]{flight_connect()}}}
+
+\item{data}{\code{data.frame} or \link{RecordBatch} to upload}
+
+\item{path}{string identifier to store the data under}
+}
+\value{
+\code{client}, invisibly.
+}
+\description{
+Send data to a Flight server
+}
diff --git a/r/man/read_delim_arrow.Rd b/r/man/read_delim_arrow.Rd
index 124abdcb912..f676b9fc75d 100644
--- a/r/man/read_delim_arrow.Rd
+++ b/r/man/read_delim_arrow.Rd
@@ -12,7 +12,9 @@ read_delim_arrow(
   quote = "\\"",
   escape_double = TRUE,
   escape_backslash = FALSE,
+  schema = NULL,
   col_names = TRUE,
+  col_types = NULL,
   col_select = NULL,
   na = c("", "NA"),
   quoted_na = TRUE,
@@ -21,7 +23,8 @@ read_delim_arrow(
   parse_options = NULL,
   convert_options = NULL,
   read_options = NULL,
-  as_data_frame = TRUE
+  as_data_frame = TRUE,
+  timestamp_parsers = NULL
 )
 
 read_csv_arrow(
@@ -29,7 +32,9 @@ read_csv_arrow(
   quote = "\\"",
   escape_double = TRUE,
   escape_backslash = FALSE,
+  schema = NULL,
   col_names = TRUE,
+  col_types = NULL,
   col_select = NULL,
   na = c("", "NA"),
   quoted_na = TRUE,
@@ -38,7 +43,8 @@ read_csv_arrow(
   parse_options = NULL,
   convert_options = NULL,
   read_options = NULL,
-  as_data_frame = TRUE
+  as_data_frame = TRUE,
+  timestamp_parsers = NULL
 )
 
 read_tsv_arrow(
@@ -46,7 +52,9 @@ read_tsv_arrow(
   quote = "\\"",
   escape_double = TRUE,
   escape_backslash = FALSE,
+  schema = NULL,
   col_names = TRUE,
+  col_types = NULL,
   col_select = NULL,
   na = c("", "NA"),
   quoted_na = TRUE,
@@ -55,11 +63,13 @@ read_tsv_arrow(
   parse_options = NULL,
   convert_options = NULL,
   read_options = NULL,
-  as_data_frame = TRUE
+  as_data_frame = TRUE,
+  timestamp_parsers = NULL
 )
 }
 \arguments{
-\item{file}{A character file name, \code{raw} vector, or an Arrow input stream.
+\item{file}{A character file name or URI, \code{raw} vector, an Arrow input stream,
+or a \code{FileSystem} with path (\code{SubTreeFileSystem}).
 If a file name, a memory-mapped Arrow \link{InputStream} will be opened and
 closed when finished; compression will be detected from the file extension
 and handled automatically. If an input stream is provided, it will be left
@@ -78,11 +88,17 @@ characters? This is more general than \code{escape_double} as backslashes
 can be used to escape the delimiter character, the quote character, or
 to add special characters like \verb{\\\\n}.}
 
+\item{schema}{\link{Schema} that describes the table. If provided, it will be
+used to satisfy both \code{col_names} and \code{col_types}.}
+
 \item{col_names}{If \code{TRUE}, the first row of the input will be used as the
 column names and will not be included in the data frame. If \code{FALSE}, column
 names will be generated by Arrow, starting with "f0", "f1", ..., "fN".
 Alternatively, you can specify a character vector of column names.}
 
+\item{col_types}{A compact string representation of the column types, or
+\code{NULL} (the default) to infer types from the data.}
+
 \item{col_select}{A character vector of column names to keep, as in the
 "select" argument to \code{data.table::fread()}, or a
 \link[tidyselect:vars_select]{tidy selection specification}
@@ -111,6 +127,15 @@ parsing options provided in other arguments (e.g. \code{delim}, \code{quote}, et
 
 \item{as_data_frame}{Should the function return a \code{data.frame} (default) or
 an Arrow \link{Table}?}
+
+\item{timestamp_parsers}{User-defined timestamp parsers. If more than one
+parser is specified, the CSV conversion logic will try parsing values
+starting from the beginning of this vector. Possible values are:
+\itemize{
+\item \code{NULL}: the default, which uses the ISO-8601 parser
+\item a character vector of \link[base:strptime]{strptime} parse strings
+\item a list of \link{TimestampParser} objects
+}}
 }
 \value{
 A \code{data.frame}, or a Table if \code{as_data_frame = FALSE}.
@@ -132,6 +157,54 @@ equivalent in \code{readr::read_csv()}, you can either provide them in the
 \code{parse_options}, \code{convert_options}, or \code{read_options} arguments, or you can
 use \link{CsvTableReader} directly for lower-level access.
 }
+\section{Specifying column types and names}{
+
+
+By default, the CSV reader will infer the column names and data types from the file, but there
+are a few ways you can specify them directly.
+
+One way is to provide an Arrow \link{Schema} in the \code{schema} argument,
+which is an ordered map of column name to type.
+When provided, it satisfies both the \code{col_names} and \code{col_types} arguments.
+This is good if you know all of this information up front.
+
+You can also pass a \code{Schema} to the \code{col_types} argument. If you do this,
+column names will still be inferred from the file unless you also specify
+\code{col_names}. In either case, the column names in the \code{Schema} must match the
+data's column names, whether they are explicitly provided or inferred. That
+said, this \code{Schema} does not have to reference all columns: those omitted
+will have their types inferred.
+
+Alternatively, you can declare column types by providing the compact string representation
+that \code{readr} uses to the \code{col_types} argument. This means you provide a
+single string, one character per column, where the characters map to Arrow
+types analogously to the \code{readr} type mapping:
+\itemize{
+\item "c": \code{utf8()}
+\item "i": \code{int32()}
+\item "n": \code{float64()}
+\item "d": \code{float64()}
+\item "l": \code{bool()}
+\item "f": \code{dictionary()}
+\item "D": \code{date32()}
+\item "T": \code{time32()}
+\item "t": \code{timestamp()}
+\item "_": \code{null()}
+\item "-": \code{null()}
+\item "?": infer the type from the data
+}
+
+If you use the compact string representation for \code{col_types}, you must also
+specify \code{col_names}.
+
+Regardless of how types are specified, all columns with a \code{null()} type will
+be dropped.
+
+Note that if you are specifying column names, whether by \code{schema} or
+\code{col_names}, and the CSV file has a header row that would otherwise be used
+to idenfity column names, you'll need to add \code{skip = 1} to skip that row.
+}
+
 \examples{
 \donttest{
   tf <- tempfile()
diff --git a/r/man/read_feather.Rd b/r/man/read_feather.Rd
index f507edb456e..c5467c3a22f 100644
--- a/r/man/read_feather.Rd
+++ b/r/man/read_feather.Rd
@@ -7,8 +7,9 @@
 read_feather(file, col_select = NULL, as_data_frame = TRUE, ...)
 }
 \arguments{
-\item{file}{A character file name, \code{raw} vector, or an Arrow input stream.
-If a file name, a memory-mapped Arrow \link{InputStream} will be opened and
+\item{file}{A character file name or URI, \code{raw} vector, an Arrow input stream,
+or a \code{FileSystem} with path (\code{SubTreeFileSystem}).
+If a file name or URI, an Arrow \link{InputStream} will be opened and
 closed when finished. If an input stream is provided, it will be left
 open.}
 
diff --git a/r/man/read_ipc_stream.Rd b/r/man/read_ipc_stream.Rd
index 1cc969b922e..d4dd7831421 100644
--- a/r/man/read_ipc_stream.Rd
+++ b/r/man/read_ipc_stream.Rd
@@ -10,8 +10,9 @@ read_arrow(file, ...)
 read_ipc_stream(file, as_data_frame = TRUE, ...)
 }
 \arguments{
-\item{file}{A character file name, \code{raw} vector, or an Arrow input stream.
-If a file name, a memory-mapped Arrow \link{InputStream} will be opened and
+\item{file}{A character file name or URI, \code{raw} vector, an Arrow input stream,
+or a \code{FileSystem} with path (\code{SubTreeFileSystem}).
+If a file name or URI, an Arrow \link{InputStream} will be opened and
 closed when finished. If an input stream is provided, it will be left
 open.}
 
diff --git a/r/man/read_json_arrow.Rd b/r/man/read_json_arrow.Rd
index 37fff64daf0..8894a5d389e 100644
--- a/r/man/read_json_arrow.Rd
+++ b/r/man/read_json_arrow.Rd
@@ -7,7 +7,8 @@
 read_json_arrow(file, col_select = NULL, as_data_frame = TRUE, ...)
 }
 \arguments{
-\item{file}{A character file name, \code{raw} vector, or an Arrow input stream.
+\item{file}{A character file name or URI, \code{raw} vector, an Arrow input stream,
+or a \code{FileSystem} with path (\code{SubTreeFileSystem}).
 If a file name, a memory-mapped Arrow \link{InputStream} will be opened and
 closed when finished; compression will be detected from the file extension
 and handled automatically. If an input stream is provided, it will be left
diff --git a/r/man/read_parquet.Rd b/r/man/read_parquet.Rd
index 6bd7335c40d..ec36734e599 100644
--- a/r/man/read_parquet.Rd
+++ b/r/man/read_parquet.Rd
@@ -13,8 +13,9 @@ read_parquet(
 )
 }
 \arguments{
-\item{file}{A character file name, \code{raw} vector, or an Arrow input stream.
-If a file name, a memory-mapped Arrow \link{InputStream} will be opened and
+\item{file}{A character file name or URI, \code{raw} vector, an Arrow input stream,
+or a \code{FileSystem} with path (\code{SubTreeFileSystem}).
+If a file name or URI, an Arrow \link{InputStream} will be opened and
 closed when finished. If an input stream is provided, it will be left
 open.}
 
diff --git a/r/man/s3_bucket.Rd b/r/man/s3_bucket.Rd
new file mode 100644
index 00000000000..7791e9bc5f2
--- /dev/null
+++ b/r/man/s3_bucket.Rd
@@ -0,0 +1,28 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/filesystem.R
+\name{s3_bucket}
+\alias{s3_bucket}
+\title{Connect to an AWS S3 bucket}
+\usage{
+s3_bucket(bucket, ...)
+}
+\arguments{
+\item{bucket}{string S3 bucket name or path}
+
+\item{...}{Additional connection options, passed to \code{S3FileSystem$create()}}
+}
+\value{
+A \code{SubTreeFileSystem} containing an \code{S3FileSystem} and the bucket's
+relative path. Note that this function's success does not guarantee that you
+are authorized to access the bucket's contents.
+}
+\description{
+\code{s3_bucket()} is a convenience function to create an \code{S3FileSystem} object
+that automatically detects the bucket's AWS region and holding onto the its
+relative path.
+}
+\examples{
+if (arrow_with_s3()) {
+  bucket <- s3_bucket("ursa-labs-taxi-data")
+}
+}
diff --git a/r/man/write_dataset.Rd b/r/man/write_dataset.Rd
index 54bd3539f3c..755c4c92803 100644
--- a/r/man/write_dataset.Rd
+++ b/r/man/write_dataset.Rd
@@ -7,30 +7,53 @@
 write_dataset(
   dataset,
   path,
-  format = dataset$format$type,
+  format = dataset$format,
   partitioning = dplyr::group_vars(dataset),
+  basename_template = paste0("part-{i}.", as.character(format)),
   hive_style = TRUE,
   ...
 )
 }
 \arguments{
-\item{dataset}{\link{Dataset} or \code{arrow_dplyr_query}. If a \code{arrow_dplyr_query},
-note that \code{select()} or \code{filter()} queries are not currently supported.}
+\item{dataset}{\link{Dataset}, \link{RecordBatch}, \link{Table}, \code{arrow_dplyr_query}, or
+\code{data.frame}. If an \code{arrow_dplyr_query} or \code{grouped_df},
+\code{schema} and \code{partitioning} will be taken from the result of any \code{select()}
+and \code{group_by()} operations done on the dataset. \code{filter()} queries will be
+applied to restrict written rows.
+Note that \code{select()}-ed columns may not be renamed.}
 
-\item{path}{string path to a directory to write to (directory will be
-created if it does not exist)}
+\item{path}{string path, URI, or \code{SubTreeFileSystem} referencing a directory
+to write to (directory will be created if it does not exist)}
 
-\item{format}{file format to write the dataset to. Currently only "feather"
-(aka "ipc") is supported.}
+\item{format}{file format to write the dataset to. Currently supported
+formats are "feather" (aka "ipc") and "parquet". Default is to write to the
+same format as \code{dataset}.}
 
 \item{partitioning}{\code{Partitioning} or a character vector of columns to
 use as partition keys (to be written as path segments). Default is to
 use the current \code{group_by()} columns.}
 
+\item{basename_template}{string template for the names of files to be written.
+Must contain \code{"{i}"}, which will be replaced with an autoincremented
+integer to generate basenames of datafiles. For example, \code{"part-{i}.feather"}
+will yield \verb{"part-0.feather", ...}.}
+
 \item{hive_style}{logical: write partition segments as Hive-style
 (\code{key1=value1/key2=value2/file.ext}) or as just bare values. Default is \code{TRUE}.}
 
-\item{...}{additional arguments, passed to \code{dataset$write()}}
+\item{...}{additional format-specific arguments. For available Parquet
+options, see \code{\link[=write_parquet]{write_parquet()}}. The available Feather options are
+\itemize{
+\item \code{use_legacy_format} logical: write data formatted so that Arrow libraries
+versions 0.14 and lower can read it. Default is \code{FALSE}. You can also
+enable this by setting the environment variable \code{ARROW_PRE_0_15_IPC_FORMAT=1}.
+\item \code{metadata_version}: A string like "V5" or the equivalent integer indicating
+the Arrow IPC MetadataVersion. Default (NULL) will use the latest version,
+unless the environment variable \code{ARROW_PRE_1_0_METADATA_VERSION=1}, in
+which case it will be V4.
+\item \code{codec}: A \link{Codec} which will be used to compress body buffers of written
+files. Default (NULL) will not compress body buffers.
+}}
 }
 \value{
 The input \code{dataset}, invisibly
diff --git a/r/man/write_feather.Rd b/r/man/write_feather.Rd
index e9639480a5d..277c8197475 100644
--- a/r/man/write_feather.Rd
+++ b/r/man/write_feather.Rd
@@ -16,7 +16,8 @@ write_feather(
 \arguments{
 \item{x}{\code{data.frame}, \link{RecordBatch}, or \link{Table}}
 
-\item{sink}{A string file path or \link{OutputStream}}
+\item{sink}{A string file path, URI, or \link{OutputStream}, or path in a file
+system (\code{SubTreeFileSystem})}
 
 \item{version}{integer Feather file version. Version 2 is the current.
 Version 1 is the more limited legacy format.}
diff --git a/r/man/write_ipc_stream.Rd b/r/man/write_ipc_stream.Rd
index 2bf4fdd2430..4f742ce9178 100644
--- a/r/man/write_ipc_stream.Rd
+++ b/r/man/write_ipc_stream.Rd
@@ -12,7 +12,8 @@ write_ipc_stream(x, sink, ...)
 \arguments{
 \item{x}{\code{data.frame}, \link{RecordBatch}, or \link{Table}}
 
-\item{sink}{A string file path or \link{OutputStream}}
+\item{sink}{A string file path, URI, or \link{OutputStream}, or path in a file
+system (\code{SubTreeFileSystem})}
 
 \item{...}{extra parameters passed to \code{write_feather()}.}
 }
diff --git a/r/man/write_parquet.Rd b/r/man/write_parquet.Rd
index e005dfbd684..d0e4f24dc46 100644
--- a/r/man/write_parquet.Rd
+++ b/r/man/write_parquet.Rd
@@ -9,27 +9,23 @@ write_parquet(
   sink,
   chunk_size = NULL,
   version = NULL,
-  compression = NULL,
+  compression = default_parquet_compression(),
   compression_level = NULL,
   use_dictionary = NULL,
   write_statistics = NULL,
   data_page_size = NULL,
-  properties = ParquetWriterProperties$create(x, version = version, compression =
-    compression, compression_level = compression_level, use_dictionary = use_dictionary,
-    write_statistics = write_statistics, data_page_size = data_page_size),
   use_deprecated_int96_timestamps = FALSE,
   coerce_timestamps = NULL,
   allow_truncated_timestamps = FALSE,
- 
-    arrow_properties = ParquetArrowWriterProperties$create(use_deprecated_int96_timestamps
-    = use_deprecated_int96_timestamps, coerce_timestamps = coerce_timestamps,
-    allow_truncated_timestamps = allow_truncated_timestamps)
+  properties = NULL,
+  arrow_properties = NULL
 )
 }
 \arguments{
-\item{x}{An \link[=Table]{arrow::Table}, or an object convertible to it.}
+\item{x}{\code{data.frame}, \link{RecordBatch}, or \link{Table}}
 
-\item{sink}{an \link[=OutputStream]{arrow::io::OutputStream} or a string which is interpreted as a file path}
+\item{sink}{A string file path, URI, or \link{OutputStream}, or path in a file
+system (\code{SubTreeFileSystem})}
 
 \item{chunk_size}{chunk size in number of rows. If NULL, the total number of rows is used.}
 
@@ -47,12 +43,6 @@ are coerced to character.}
 \item{data_page_size}{Set a target threshold for the approximate encoded
 size of data pages within a column chunk (in bytes). Default 1 MiB.}
 
-\item{properties}{properties for parquet writer, derived from arguments
-\code{version}, \code{compression}, \code{compression_level}, \code{use_dictionary},
-\code{write_statistics} and \code{data_page_size}. You should not specify any of
-these arguments if you also provide a \code{properties} argument, as they will
-be ignored.}
-
 \item{use_deprecated_int96_timestamps}{Write timestamps to INT96 Parquet format. Default \code{FALSE}.}
 
 \item{coerce_timestamps}{Cast timestamps a particular resolution. Can be
@@ -62,10 +52,13 @@ be ignored.}
 particular resolution. E.g. if microsecond or nanosecond data is lost when coercing
 to "ms", do not raise an exception}
 
-\item{arrow_properties}{arrow specific writer properties, derived from arguments
-\code{use_deprecated_int96_timestamps}, \code{coerce_timestamps} and \code{allow_truncated_timestamps}
-You should not specify any of these arguments if you also provide a \code{properties}
-argument, as they will be ignored.}
+\item{properties}{A \code{ParquetWriterProperties} object, used instead of the options
+enumerated in this function's signature. Providing \code{properties} as an argument
+is deprecated; if you need to assemble \code{ParquetWriterProperties} outside
+of \code{write_parquet()}, use \code{ParquetFileWriter} instead.}
+
+\item{arrow_properties}{A \code{ParquetArrowWriterProperties} object. Like
+\code{properties}, this argument is deprecated.}
 }
 \value{
 the input \code{x} invisibly.
@@ -75,6 +68,12 @@ the input \code{x} invisibly.
 This function enables you to write Parquet files from R.
 }
 \details{
+Due to features of the format, Parquet files cannot be appended to.
+If you want to use the Parquet format but also want the ability to extend
+your dataset, you can write to additional Parquet files and then treat
+the whole directory of files as a \link{Dataset} you can query.
+See \code{vignette("dataset", package = "arrow")} for examples of this.
+
 The parameters \code{compression}, \code{compression_level}, \code{use_dictionary} and
 \code{write_statistics} support various patterns:
 \itemize{
diff --git a/r/src/Makevars.in b/r/src/Makevars.in
index 9a9191dca86..7e20f9561ed 100644
--- a/r/src/Makevars.in
+++ b/r/src/Makevars.in
@@ -15,7 +15,12 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# when cpp11 is vendored:
+# PKG_CPPFLAGS=@cflags@ -I../inst/include/
+
+# when it is not:
 PKG_CPPFLAGS=@cflags@
+
 # `-fvisibility=hidden` does not play well with UBSAN:
 # https://bugs.llvm.org/show_bug.cgi?id=39191
 # https://www.mail-archive.com/gcc-bugs@gcc.gnu.org/msg534862.html
diff --git a/r/src/array.cpp b/r/src/array.cpp
index 5879dc91675..844ac2a704d 100644
--- a/r/src/array.cpp
+++ b/r/src/array.cpp
@@ -17,48 +17,45 @@
 
 #include "./arrow_types.h"
 
-using Rcpp::LogicalVector;
-using Rcpp::no_init;
-
 #if defined(ARROW_R_WITH_ARROW)
 
 #include <arrow/array.h>
 #include <arrow/util/bitmap_reader.h>
 
-void arrow::r::validate_slice_offset(int offset, int len) {
+void arrow::r::validate_slice_offset(R_xlen_t offset, int64_t len) {
   if (offset == NA_INTEGER) {
-    Rcpp::stop("Slice 'offset' cannot be NA");
+    cpp11::stop("Slice 'offset' cannot be NA");
   }
   if (offset < 0) {
-    Rcpp::stop("Slice 'offset' cannot be negative");
+    cpp11::stop("Slice 'offset' cannot be negative");
   }
   if (offset > len) {
-    Rcpp::stop("Slice 'offset' greater than array length");
+    cpp11::stop("Slice 'offset' greater than array length");
   }
 }
 
-void arrow::r::validate_slice_length(int length, int available) {
+void arrow::r::validate_slice_length(R_xlen_t length, int64_t available) {
   if (length == NA_INTEGER) {
-    Rcpp::stop("Slice 'length' cannot be NA");
+    cpp11::stop("Slice 'length' cannot be NA");
   }
   if (length < 0) {
-    Rcpp::stop("Slice 'length' cannot be negative");
+    cpp11::stop("Slice 'length' cannot be negative");
   }
   if (length > available) {
-    Rcpp::warning("Slice 'length' greater than available length");
+    cpp11::warning("Slice 'length' greater than available length");
   }
 }
 
 // [[arrow::export]]
 std::shared_ptr<arrow::Array> Array__Slice1(const std::shared_ptr<arrow::Array>& array,
-                                            int offset) {
+                                            R_xlen_t offset) {
   arrow::r::validate_slice_offset(offset, array->length());
   return array->Slice(offset);
 }
 
 // [[arrow::export]]
 std::shared_ptr<arrow::Array> Array__Slice2(const std::shared_ptr<arrow::Array>& array,
-                                            int offset, int length) {
+                                            R_xlen_t offset, R_xlen_t length) {
   arrow::r::validate_slice_offset(offset, array->length());
   arrow::r::validate_slice_length(length, array->length() - offset);
   return array->Slice(offset, length);
@@ -66,21 +63,21 @@ std::shared_ptr<arrow::Array> Array__Slice2(const std::shared_ptr<arrow::Array>&
 
 void arrow::r::validate_index(int i, int len) {
   if (i == NA_INTEGER) {
-    Rcpp::stop("'i' cannot be NA");
+    cpp11::stop("'i' cannot be NA");
   }
   if (i < 0 || i >= len) {
-    Rcpp::stop("subscript out of bounds");
+    cpp11::stop("subscript out of bounds");
   }
 }
 
 // [[arrow::export]]
-bool Array__IsNull(const std::shared_ptr<arrow::Array>& x, int i) {
+bool Array__IsNull(const std::shared_ptr<arrow::Array>& x, R_xlen_t i) {
   arrow::r::validate_index(i, x->length());
   return x->IsNull(i);
 }
 
 // [[arrow::export]]
-bool Array__IsValid(const std::shared_ptr<arrow::Array>& x, int i) {
+bool Array__IsValid(const std::shared_ptr<arrow::Array>& x, R_xlen_t i) {
   arrow::r::validate_index(i, x->length());
   return x->IsValid(i);
 }
@@ -129,16 +126,16 @@ std::shared_ptr<arrow::ArrayData> Array__data(
 
 // [[arrow::export]]
 bool Array__RangeEquals(const std::shared_ptr<arrow::Array>& self,
-                        const std::shared_ptr<arrow::Array>& other, int start_idx,
-                        int end_idx, int other_start_idx) {
+                        const std::shared_ptr<arrow::Array>& other, R_xlen_t start_idx,
+                        R_xlen_t end_idx, R_xlen_t other_start_idx) {
   if (start_idx == NA_INTEGER) {
-    Rcpp::stop("'start_idx' cannot be NA");
+    cpp11::stop("'start_idx' cannot be NA");
   }
   if (end_idx == NA_INTEGER) {
-    Rcpp::stop("'end_idx' cannot be NA");
+    cpp11::stop("'end_idx' cannot be NA");
   }
   if (other_start_idx == NA_INTEGER) {
-    Rcpp::stop("'other_start_idx' cannot be NA");
+    cpp11::stop("'other_start_idx' cannot be NA");
   }
   return self->RangeEquals(*other, start_idx, end_idx, other_start_idx);
 }
@@ -245,17 +242,17 @@ int64_t FixedSizeListArray__value_offset(
 }
 
 // [[arrow::export]]
-Rcpp::IntegerVector ListArray__raw_value_offsets(
+cpp11::writable::integers ListArray__raw_value_offsets(
     const std::shared_ptr<arrow::ListArray>& array) {
   auto offsets = array->raw_value_offsets();
-  return Rcpp::IntegerVector(offsets, offsets + array->length());
+  return cpp11::writable::integers(offsets, offsets + array->length());
 }
 
 // [[arrow::export]]
-Rcpp::IntegerVector LargeListArray__raw_value_offsets(
+cpp11::writable::integers LargeListArray__raw_value_offsets(
     const std::shared_ptr<arrow::LargeListArray>& array) {
   auto offsets = array->raw_value_offsets();
-  return Rcpp::IntegerVector(offsets, offsets + array->length());
+  return cpp11::writable::integers(offsets, offsets + array->length());
 }
 
 #endif
diff --git a/r/src/array_from_vector.cpp b/r/src/array_from_vector.cpp
index 4711c4b730a..c2d22868535 100644
--- a/r/src/array_from_vector.cpp
+++ b/r/src/array_from_vector.cpp
@@ -170,16 +170,13 @@ struct VectorToArrayConverter {
     R_xlen_t n = XLENGTH(x);
     RETURN_NOT_OK(builder->Reserve(n));
     for (R_xlen_t i = 0; i < n; i++) {
-      SEXP s = STRING_ELT(x, i);
-      if (s == NA_STRING) {
+      SEXP si = STRING_ELT(x, i);
+      if (si == NA_STRING) {
         RETURN_NOT_OK(binary_builder->AppendNull());
         continue;
-      } else {
-        // Make sure we're ingesting UTF-8
-        s = Rf_mkCharCE(Rf_translateCharUTF8(s), CE_UTF8);
       }
-
-      RETURN_NOT_OK(binary_builder->Append(CHAR(s), LENGTH(s)));
+      std::string s = cpp11::r_string(si);
+      RETURN_NOT_OK(binary_builder->Append(s.c_str(), s.size()));
     }
 
     return Status::OK();
@@ -324,7 +321,7 @@ struct VectorToArrayConverter {
 
   static std::shared_ptr<Array> Visit(SEXP x, const std::shared_ptr<DataType>& type) {
     std::unique_ptr<ArrayBuilder> builder;
-    StopIfNotOk(MakeBuilder(arrow::default_memory_pool(), type, &builder));
+    StopIfNotOk(MakeBuilder(gc_memory_pool(), type, &builder));
 
     VectorToArrayConverter converter{x, builder.get()};
     StopIfNotOk(arrow::VisitTypeInline(*type, &converter));
@@ -339,13 +336,13 @@ struct VectorToArrayConverter {
 };
 
 template <typename Type>
-std::shared_ptr<Array> MakeFactorArrayImpl(Rcpp::IntegerVector_ factor,
+std::shared_ptr<Array> MakeFactorArrayImpl(cpp11::integers factor,
                                            const std::shared_ptr<arrow::DataType>& type) {
   using value_type = typename arrow::TypeTraits<Type>::ArrayType::value_type;
   auto n = factor.size();
 
   std::shared_ptr<Buffer> indices_buffer =
-      ValueOrStop(AllocateBuffer(n * sizeof(value_type)));
+      ValueOrStop(AllocateBuffer(n * sizeof(value_type), gc_memory_pool()));
 
   std::vector<std::shared_ptr<Buffer>> buffers{nullptr, indices_buffer};
 
@@ -360,7 +357,8 @@ std::shared_ptr<Array> MakeFactorArrayImpl(Rcpp::IntegerVector_ factor,
 
   if (i < n) {
     // there are NA's so we need a null buffer
-    auto null_buffer = ValueOrStop(AllocateBuffer(BitUtil::BytesForBits(n)));
+    auto null_buffer =
+        ValueOrStop(AllocateBuffer(BitUtil::BytesForBits(n), gc_memory_pool()));
     internal::FirstTimeBitmapWriter null_bitmap_writer(null_buffer->mutable_data(), 0, n);
 
     // catch up
@@ -393,7 +391,7 @@ std::shared_ptr<Array> MakeFactorArrayImpl(Rcpp::IntegerVector_ factor,
   return ValueOrStop(DictionaryArray::FromArrays(type, array_indices, dict));
 }
 
-std::shared_ptr<Array> MakeFactorArray(Rcpp::IntegerVector_ factor,
+std::shared_ptr<Array> MakeFactorArray(cpp11::integers factor,
                                        const std::shared_ptr<arrow::DataType>& type) {
   const auto& dict_type = checked_cast<const arrow::DictionaryType&>(*type);
   switch (dict_type.index_type()->id()) {
@@ -406,9 +404,11 @@ std::shared_ptr<Array> MakeFactorArray(Rcpp::IntegerVector_ factor,
     case Type::INT64:
       return MakeFactorArrayImpl<arrow::Int64Type>(factor, type);
     default:
-      Rcpp::stop(tfm::format("Cannot convert to dictionary with index_type %s",
-                             dict_type.index_type()->ToString()));
+      break;
   }
+
+  cpp11::stop("Cannot convert to dictionary with index_type '%s'",
+              dict_type.index_type()->ToString().c_str());
 }
 
 std::shared_ptr<Array> MakeStructArray(SEXP df, const std::shared_ptr<DataType>& type) {
@@ -586,9 +586,8 @@ struct Unbox<Type, enable_if_integer<Type>> {
         break;
     }
 
-    return Status::Invalid(
-        tfm::format("Cannot convert R vector of type %s to integer Arrow array",
-                    Rcpp::type2name(obj)));
+    return Status::Invalid("Cannot convert R vector of type <", Rf_type2char(TYPEOF(obj)),
+                           "> to integer Arrow array");
   }
 
   template <typename T>
@@ -1064,42 +1063,40 @@ class FixedSizeBinaryVectorConverter : public VectorConverter {
   FixedSizeBinaryBuilder* typed_builder_;
 };
 
-template <typename Builder>
+template <typename StringBuilder>
 class StringVectorConverter : public VectorConverter {
  public:
   ~StringVectorConverter() {}
 
   Status Init(ArrayBuilder* builder) {
-    typed_builder_ = checked_cast<Builder*>(builder);
+    typed_builder_ = checked_cast<StringBuilder*>(builder);
     return Status::OK();
   }
 
   Status Ingest(SEXP obj) {
     ARROW_RETURN_IF(TYPEOF(obj) != STRSXP,
                     Status::RError("Expecting a character vector"));
-    R_xlen_t n = XLENGTH(obj);
 
-    // Reserve enough space before appending
-    int64_t size = 0;
-    for (R_xlen_t i = 0; i < n; i++) {
-      SEXP string_i = STRING_ELT(obj, i);
-      if (string_i != NA_STRING) {
-        size += XLENGTH(Rf_mkCharCE(Rf_translateCharUTF8(string_i), CE_UTF8));
-      }
+    cpp11::strings s(arrow::r::utf8_strings(obj));
+    RETURN_NOT_OK(typed_builder_->Reserve(s.size()));
+
+    // we know all the R strings are utf8 already, so we can get
+    // a definite size and then use UnsafeAppend*()
+    int64_t total_length = 0;
+    for (cpp11::r_string si : s) {
+      total_length += cpp11::is_na(si) ? 0 : si.size();
     }
-    RETURN_NOT_OK(typed_builder_->Reserve(size));
+    RETURN_NOT_OK(typed_builder_->ReserveData(total_length));
 
     // append
-    for (R_xlen_t i = 0; i < n; i++) {
-      SEXP string_i = STRING_ELT(obj, i);
-      if (string_i == NA_STRING) {
-        RETURN_NOT_OK(typed_builder_->AppendNull());
+    for (cpp11::r_string si : s) {
+      if (si == NA_STRING) {
+        typed_builder_->UnsafeAppendNull();
       } else {
-        // Make sure we're ingesting UTF-8
-        string_i = Rf_mkCharCE(Rf_translateCharUTF8(string_i), CE_UTF8);
-        RETURN_NOT_OK(typed_builder_->Append(CHAR(string_i), XLENGTH(string_i)));
+        typed_builder_->UnsafeAppend(CHAR(si), si.size());
       }
     }
+
     return Status::OK();
   }
 
@@ -1108,7 +1105,7 @@ class StringVectorConverter : public VectorConverter {
   }
 
  private:
-  Builder* typed_builder_;
+  StringBuilder* typed_builder_;
 };
 
 #define NUMERIC_CONVERTER(TYPE_ENUM, TYPE)                                               \
@@ -1191,17 +1188,16 @@ std::shared_ptr<arrow::DataType> InferArrowTypeFromFactor(SEXP factor) {
 
 template <int VectorType>
 std::shared_ptr<arrow::DataType> InferArrowTypeFromVector(SEXP x) {
-  Rcpp::stop("Unknown vector type: ", VectorType);
+  cpp11::stop("Unknown vector type: ", VectorType);
 }
 
 template <>
 std::shared_ptr<arrow::DataType> InferArrowTypeFromVector<ENVSXP>(SEXP x) {
   if (Rf_inherits(x, "Array")) {
-    Rcpp::ConstReferenceSmartPtrInputParameter<std::shared_ptr<arrow::Array>> array(x);
-    return static_cast<std::shared_ptr<arrow::Array>>(array)->type();
+    return cpp11::as_cpp<std::shared_ptr<arrow::Array>>(x)->type();
   }
 
-  Rcpp::stop("Unrecognized vector instance for type ENVSXP");
+  cpp11::stop("Unrecognized vector instance for type ENVSXP");
 }
 
 template <>
@@ -1250,32 +1246,30 @@ std::shared_ptr<arrow::DataType> InferArrowTypeFromVector<REALSXP>(SEXP x) {
 
 template <>
 std::shared_ptr<arrow::DataType> InferArrowTypeFromVector<STRSXP>(SEXP x) {
-  // See how big the character vector is
-  R_xlen_t n = XLENGTH(x);
-  int64_t size = 0;
-  for (R_xlen_t i = 0; i < n; i++) {
-    SEXP string_i = STRING_ELT(x, i);
-    if (string_i != NA_STRING) {
-      size += XLENGTH(Rf_mkCharCE(Rf_translateCharUTF8(string_i), CE_UTF8));
-    }
-    if (size > arrow::kBinaryMemoryLimit) {
-      // Exceeds 2GB capacity of utf8 type, so use large
-      return large_utf8();
+  return cpp11::unwind_protect([&] {
+    R_xlen_t n = XLENGTH(x);
+
+    int64_t size = 0;
+
+    for (R_xlen_t i = 0; i < n; i++) {
+      size += arrow::r::unsafe::r_string_size(STRING_ELT(x, i));
+      if (size > arrow::kBinaryMemoryLimit) {
+        // Exceeds 2GB capacity of utf8 type, so use large
+        return large_utf8();
+      }
     }
-  }
 
-  return utf8();
+    return utf8();
+  });
 }
 
-static inline std::shared_ptr<arrow::DataType> InferArrowTypeFromDataFrame(SEXP x) {
-  R_xlen_t n = XLENGTH(x);
-  SEXP names = Rf_getAttrib(x, R_NamesSymbol);
+static inline std::shared_ptr<arrow::DataType> InferArrowTypeFromDataFrame(
+    cpp11::list x) {
+  R_xlen_t n = x.size();
+  cpp11::strings names(x.attr(R_NamesSymbol));
   std::vector<std::shared_ptr<arrow::Field>> fields(n);
   for (R_xlen_t i = 0; i < n; i++) {
-    // Make sure we're ingesting UTF-8
-    const auto* field_name =
-        CHAR(Rf_mkCharCE(Rf_translateCharUTF8(STRING_ELT(names, i)), CE_UTF8));
-    fields[i] = arrow::field(field_name, InferArrowType(VECTOR_ELT(x, i)));
+    fields[i] = arrow::field(names[i], InferArrowType(x[i]));
   }
   return arrow::struct_(std::move(fields));
 }
@@ -1290,7 +1284,7 @@ std::shared_ptr<arrow::DataType> InferArrowTypeFromVector<VECSXP>(SEXP x) {
       SEXP byte_width = Rf_getAttrib(x, symbols::byte_width);
       if (Rf_isNull(byte_width) || TYPEOF(byte_width) != INTSXP ||
           XLENGTH(byte_width) != 1) {
-        Rcpp::stop("malformed arrow_fixed_size_binary object");
+        cpp11::stop("malformed arrow_fixed_size_binary object");
       }
       return arrow::fixed_size_binary(INTEGER(byte_width)[0]);
     }
@@ -1306,7 +1300,7 @@ std::shared_ptr<arrow::DataType> InferArrowTypeFromVector<VECSXP>(SEXP x) {
     SEXP ptype = Rf_getAttrib(x, symbols::ptype);
     if (Rf_isNull(ptype)) {
       if (XLENGTH(x) == 0) {
-        Rcpp::stop(
+        cpp11::stop(
             "Requires at least one element to infer the values' type of a list vector");
       }
 
@@ -1337,7 +1331,7 @@ std::shared_ptr<arrow::DataType> InferArrowType(SEXP x) {
       break;
   }
 
-  Rcpp::stop("Cannot infer type from vector");
+  cpp11::stop("Cannot infer type from vector");
 }
 
 // in some situations we can just use the memory of the R object in an RBuffer
@@ -1360,21 +1354,22 @@ bool can_reuse_memory(SEXP x, const std::shared_ptr<arrow::DataType>& type) {
 
 // this is only used on some special cases when the arrow Array can just use the memory of
 // the R object, via an RBuffer, hence be zero copy
-template <int RTYPE, typename Type>
+template <int RTYPE, typename RVector, typename Type>
 std::shared_ptr<Array> MakeSimpleArray(SEXP x) {
   using value_type = typename arrow::TypeTraits<Type>::ArrayType::value_type;
-  Rcpp::Vector<RTYPE, Rcpp::NoProtectStorage> vec(x);
+  RVector vec(x);
   auto n = vec.size();
-  auto p_vec_start = reinterpret_cast<value_type*>(vec.begin());
+  auto p_vec_start = reinterpret_cast<value_type*>(DATAPTR(vec));
   auto p_vec_end = p_vec_start + n;
   std::vector<std::shared_ptr<Buffer>> buffers{nullptr,
-                                               std::make_shared<RBuffer<RTYPE>>(vec)};
+                                               std::make_shared<RBuffer<RVector>>(vec)};
 
   int null_count = 0;
 
   auto first_na = std::find_if(p_vec_start, p_vec_end, is_na<value_type>);
   if (first_na < p_vec_end) {
-    auto null_bitmap = ValueOrStop(AllocateBuffer(BitUtil::BytesForBits(n)));
+    auto null_bitmap =
+        ValueOrStop(AllocateBuffer(BitUtil::BytesForBits(n), gc_memory_pool()));
     internal::FirstTimeBitmapWriter bitmap_writer(null_bitmap->mutable_data(), 0, n);
 
     // first loop to clear all the bits before the first NA
@@ -1410,16 +1405,16 @@ std::shared_ptr<arrow::Array> Array__from_vector_reuse_memory(SEXP x) {
   auto type = TYPEOF(x);
 
   if (type == INTSXP) {
-    return MakeSimpleArray<INTSXP, Int32Type>(x);
+    return MakeSimpleArray<INTSXP, cpp11::integers, Int32Type>(x);
   } else if (type == REALSXP && Rf_inherits(x, "integer64")) {
-    return MakeSimpleArray<REALSXP, Int64Type>(x);
+    return MakeSimpleArray<REALSXP, cpp11::doubles, Int64Type>(x);
   } else if (type == REALSXP) {
-    return MakeSimpleArray<REALSXP, DoubleType>(x);
+    return MakeSimpleArray<REALSXP, cpp11::doubles, DoubleType>(x);
   } else if (type == RAWSXP) {
-    return MakeSimpleArray<RAWSXP, UInt8Type>(x);
+    return MakeSimpleArray<RAWSXP, cpp11::raws, UInt8Type>(x);
   }
 
-  Rcpp::stop("Unreachable: you might need to fix can_reuse_memory()");
+  cpp11::stop("Unreachable: you might need to fix can_reuse_memory()");
 }
 
 bool CheckCompatibleFactor(SEXP obj, const std::shared_ptr<arrow::DataType>& type) {
@@ -1450,25 +1445,28 @@ arrow::Status CheckCompatibleStruct(SEXP obj,
   // the columns themselves are not checked against the
   // types of the fields, because Array__from_vector will error
   // when not compatible.
-  SEXP names = Rf_getAttrib(obj, R_NamesSymbol);
-  SEXP name_i;
-  for (int i = 0; i < num_fields; i++) {
-    name_i = Rf_mkCharCE(Rf_translateCharUTF8(STRING_ELT(names, i)), CE_UTF8);
-    if (type->field(i)->name() != CHAR(name_i)) {
-      return Status::RError("Field name in position ", i, " (", type->field(i)->name(),
-                            ") does not match the name of the column of the data frame (",
-                            CHAR(name_i), ")");
+  cpp11::strings names = Rf_getAttrib(obj, R_NamesSymbol);
+
+  return cpp11::unwind_protect([&] {
+    for (int i = 0; i < num_fields; i++) {
+      const char* name_i = arrow::r::unsafe::utf8_string(names[i]);
+      auto field_name = type->field(i)->name();
+      if (field_name != name_i) {
+        return Status::RError(
+            "Field name in position ", i, " (", field_name,
+            ") does not match the name of the column of the data frame (", name_i, ")");
+      }
     }
-  }
 
-  return Status::OK();
+    return Status::OK();
+  });
 }
 
 std::shared_ptr<arrow::Array> Array__from_vector(
     SEXP x, const std::shared_ptr<arrow::DataType>& type, bool type_inferred) {
   // short circuit if `x` is already an Array
   if (Rf_inherits(x, "Array")) {
-    return Rcpp::ConstReferenceSmartPtrInputParameter<std::shared_ptr<arrow::Array>>(x);
+    return cpp11::as_cpp<std::shared_ptr<arrow::Array>>(x);
   }
 
   // special case when we can just use the data from the R vector
@@ -1487,7 +1485,7 @@ std::shared_ptr<arrow::Array> Array__from_vector(
       return arrow::r::MakeFactorArray(x, type);
     }
 
-    Rcpp::stop("Object incompatible with dictionary type");
+    cpp11::stop("Object incompatible with dictionary type");
   }
 
   if (type->id() == Type::LIST || type->id() == Type::LARGE_LIST ||
@@ -1516,7 +1514,7 @@ std::shared_ptr<arrow::Array> Array__from_vector(
 
   // Create ArrayBuilder for type
   std::unique_ptr<arrow::ArrayBuilder> type_builder;
-  StopIfNotOk(arrow::MakeBuilder(arrow::default_memory_pool(), type, &type_builder));
+  StopIfNotOk(arrow::MakeBuilder(gc_memory_pool(), type, &type_builder));
   StopIfNotOk(converter->Init(type_builder.get()));
 
   // ingest R data and grab the result array
@@ -1543,14 +1541,14 @@ std::shared_ptr<arrow::Array> Array__from_vector(SEXP x, SEXP s_type) {
   if (type_inferred) {
     type = arrow::r::InferArrowType(x);
   } else {
-    type = arrow::r::extract<arrow::DataType>(s_type);
+    type = cpp11::as_cpp<std::shared_ptr<arrow::DataType>>(s_type);
   }
 
   return arrow::r::Array__from_vector(x, type, type_inferred);
 }
 
 // [[arrow::export]]
-std::shared_ptr<arrow::ChunkedArray> ChunkedArray__from_list(Rcpp::List chunks,
+std::shared_ptr<arrow::ChunkedArray> ChunkedArray__from_list(cpp11::list chunks,
                                                              SEXP s_type) {
   std::vector<std::shared_ptr<arrow::Array>> vec;
 
@@ -1562,17 +1560,17 @@ std::shared_ptr<arrow::ChunkedArray> ChunkedArray__from_list(Rcpp::List chunks,
   std::shared_ptr<arrow::DataType> type;
   if (type_inferred) {
     if (n == 0) {
-      Rcpp::stop("type must be specified for empty list");
+      cpp11::stop("type must be specified for empty list");
     }
     type = arrow::r::InferArrowType(VECTOR_ELT(chunks, 0));
   } else {
-    type = arrow::r::extract<arrow::DataType>(s_type);
+    type = cpp11::as_cpp<std::shared_ptr<arrow::DataType>>(s_type);
   }
 
   if (n == 0) {
     std::shared_ptr<arrow::Array> array;
     std::unique_ptr<arrow::ArrayBuilder> type_builder;
-    StopIfNotOk(arrow::MakeBuilder(arrow::default_memory_pool(), type, &type_builder));
+    StopIfNotOk(arrow::MakeBuilder(gc_memory_pool(), type, &type_builder));
     StopIfNotOk(type_builder->Finish(&array));
     vec.push_back(array);
   } else {
@@ -1580,11 +1578,10 @@ std::shared_ptr<arrow::ChunkedArray> ChunkedArray__from_list(Rcpp::List chunks,
     // because we might have inferred the type from the first element of the list
     //
     // this only really matters for dictionary arrays
-    vec.push_back(
-        arrow::r::Array__from_vector(VECTOR_ELT(chunks, 0), type, type_inferred));
+    vec.push_back(arrow::r::Array__from_vector(chunks[0], type, type_inferred));
 
     for (R_xlen_t i = 1; i < n; i++) {
-      vec.push_back(arrow::r::Array__from_vector(VECTOR_ELT(chunks, i), type, false));
+      vec.push_back(arrow::r::Array__from_vector(chunks[i], type, false));
     }
   }
 
diff --git a/r/src/array_to_vector.cpp b/r/src/array_to_vector.cpp
index 8be2d02384f..c9331b9f92d 100644
--- a/r/src/array_to_vector.cpp
+++ b/r/src/array_to_vector.cpp
@@ -35,13 +35,6 @@ using internal::IntegersCanFit;
 
 namespace r {
 
-using Rcpp::default_value;
-using Rcpp::IntegerVector;
-using Rcpp::LogicalVector_;
-using Rcpp::no_init;
-using Rcpp::Shield;
-using Rcpp::StringVector_;
-
 class Converter {
  public:
   explicit Converter(ArrayVector arrays) : arrays_(std::move(arrays)) {}
@@ -92,7 +85,7 @@ class Converter {
   //
   // for each array, add a task to the task group
   //
-  // The task group is Finish() iun the caller
+  // The task group is Finish() in the caller
   void IngestParallel(SEXP data, const std::shared_ptr<arrow::internal::TaskGroup>& tg) {
     R_xlen_t k = 0, i = 0;
     for (const auto& array : arrays_) {
@@ -111,105 +104,148 @@ class Converter {
   ArrayVector arrays_;
 };
 
-// data[start:(start+n)] = NA
-template <int RTYPE>
-Status AllNull_Ingest(SEXP data, R_xlen_t start, R_xlen_t n) {
-  auto p_data = Rcpp::internal::r_vector_start<RTYPE>(data) + start;
-  std::fill_n(p_data, n, Rcpp::default_value<RTYPE>());
-  return Status::OK();
-}
-
-// ingest the data from `array` into a slice of `data`
-//
-// each element goes through `lambda` when some conversion is needed
-template <int RTYPE, typename array_value_type, typename Lambda>
-Status SomeNull_Ingest(SEXP data, R_xlen_t start, R_xlen_t n,
-                       const array_value_type* p_values,
-                       const std::shared_ptr<arrow::Array>& array, Lambda lambda) {
-  if (!p_values) {
-    return Status::Invalid("Invalid data buffer");
-  }
-  auto p_data = Rcpp::internal::r_vector_start<RTYPE>(data) + start;
-
-  if (array->null_count()) {
-    arrow::internal::BitmapReader bitmap_reader(array->null_bitmap()->data(),
-                                                array->offset(), n);
-    for (R_xlen_t i = 0; i < n; i++, bitmap_reader.Next(), ++p_data, ++p_values) {
-      *p_data = bitmap_reader.IsSet() ? lambda(*p_values) : default_value<RTYPE>();
-    }
-  } else {
-    std::transform(p_values, p_values + n, p_data, lambda);
-  }
-
-  return Status::OK();
-}
-
-template <typename Lambda>
-Status IngestSome(const std::shared_ptr<arrow::Array>& array, R_xlen_t n, Lambda lambda) {
+template <typename SetNonNull, typename SetNull>
+Status IngestSome(const std::shared_ptr<arrow::Array>& array, R_xlen_t n,
+                  SetNonNull&& set_non_null, SetNull&& set_null) {
   if (array->null_count()) {
     internal::BitmapReader bitmap_reader(array->null_bitmap()->data(), array->offset(),
                                          n);
 
     for (R_xlen_t i = 0; i < n; i++, bitmap_reader.Next()) {
-      if (bitmap_reader.IsSet()) RETURN_NOT_OK(lambda(i));
+      if (bitmap_reader.IsSet()) {
+        RETURN_NOT_OK(set_non_null(i));
+      } else {
+        RETURN_NOT_OK(set_null(i));
+      }
     }
 
   } else {
     for (R_xlen_t i = 0; i < n; i++) {
-      RETURN_NOT_OK(lambda(i));
+      RETURN_NOT_OK(set_non_null(i));
     }
   }
 
   return Status::OK();
 }
 
+template <typename SetNonNull>
+Status IngestSome(const std::shared_ptr<arrow::Array>& array, R_xlen_t n,
+                  SetNonNull&& set_non_null) {
+  auto nothing = [](R_xlen_t i) { return Status::OK(); };
+  return IngestSome(array, n, std::forward<SetNonNull>(set_non_null), nothing);
+}
+
 // Allocate + Ingest
 SEXP ArrayVector__as_vector(R_xlen_t n, const std::shared_ptr<DataType>& type,
                             const ArrayVector& arrays) {
   auto converter = Converter::Make(type, arrays);
-  Shield<SEXP> data(converter->Allocate(n));
+  SEXP data = PROTECT(converter->Allocate(n));
   StopIfNotOk(converter->IngestSerial(data));
+  UNPROTECT(1);
   return data;
 }
 
-template <int RTYPE>
-class Converter_SimpleArray : public Converter {
-  using Vector = Rcpp::Vector<RTYPE, Rcpp::NoProtectStorage>;
-  using value_type = typename Vector::stored_type;
+template <typename Type>
+class Converter_Int : public Converter {
+  using value_type = typename TypeTraits<Type>::ArrayType::value_type;
 
  public:
-  explicit Converter_SimpleArray(const ArrayVector& arrays) : Converter(arrays) {}
+  explicit Converter_Int(const ArrayVector& arrays) : Converter(arrays) {}
 
-  SEXP Allocate(R_xlen_t n) const { return Vector(no_init(n)); }
+  SEXP Allocate(R_xlen_t n) const { return Rf_allocVector(INTSXP, n); }
 
   Status Ingest_all_nulls(SEXP data, R_xlen_t start, R_xlen_t n) const {
-    return AllNull_Ingest<RTYPE>(data, start, n);
+    std::fill_n(INTEGER(data) + start, n, NA_INTEGER);
+    return Status::OK();
   }
 
   Status Ingest_some_nulls(SEXP data, const std::shared_ptr<arrow::Array>& array,
                            R_xlen_t start, R_xlen_t n, size_t chunk_index) const {
     auto p_values = array->data()->GetValues<value_type>(1);
-    auto echo = [](value_type value) { return value; };
-    return SomeNull_Ingest<RTYPE, value_type>(data, start, n, p_values, array, echo);
+    if (!p_values) {
+      return Status::Invalid("Invalid data buffer");
+    }
+    auto p_data = INTEGER(data) + start;
+    auto ingest_one = [&](R_xlen_t i) {
+      p_data[i] = static_cast<int>(p_values[i]);
+      return Status::OK();
+    };
+    auto null_one = [&](R_xlen_t i) {
+      p_data[i] = NA_INTEGER;
+      return Status::OK();
+    };
+
+    return IngestSome(array, n, ingest_one, null_one);
   }
 };
 
-class Converter_Date32 : public Converter_SimpleArray<REALSXP> {
+template <typename Type>
+class Converter_Double : public Converter {
+  using value_type = typename TypeTraits<Type>::ArrayType::value_type;
+
  public:
-  explicit Converter_Date32(const ArrayVector& arrays)
-      : Converter_SimpleArray<REALSXP>(arrays) {}
+  explicit Converter_Double(const ArrayVector& arrays) : Converter(arrays) {}
+
+  SEXP Allocate(R_xlen_t n) const { return Rf_allocVector(REALSXP, n); }
+
+  Status Ingest_all_nulls(SEXP data, R_xlen_t start, R_xlen_t n) const {
+    std::fill_n(REAL(data) + start, n, NA_REAL);
+    return Status::OK();
+  }
+
+  Status Ingest_some_nulls(SEXP data, const std::shared_ptr<arrow::Array>& array,
+                           R_xlen_t start, R_xlen_t n, size_t chunk_index) const {
+    auto p_values = array->data()->GetValues<value_type>(1);
+    if (!p_values) {
+      return Status::Invalid("Invalid data buffer");
+    }
+    auto p_data = REAL(data) + start;
+    auto ingest_one = [&](R_xlen_t i) {
+      p_data[i] = static_cast<value_type>(p_values[i]);
+      return Status::OK();
+    };
+    auto null_one = [&](R_xlen_t i) {
+      p_data[i] = NA_REAL;
+      return Status::OK();
+    };
+
+    return IngestSome(array, n, ingest_one, null_one);
+  }
+};
+
+class Converter_Date32 : public Converter {
+ public:
+  explicit Converter_Date32(const ArrayVector& arrays) : Converter(arrays) {}
 
   SEXP Allocate(R_xlen_t n) const {
-    Rcpp::NumericVector data(no_init(n));
-    data.attr("class") = "Date";
+    SEXP data = PROTECT(Rf_allocVector(REALSXP, n));
+    Rf_classgets(data, Rf_mkString("Date"));
+    UNPROTECT(1);
     return data;
   }
 
+  Status Ingest_all_nulls(SEXP data, R_xlen_t start, R_xlen_t n) const {
+    std::fill_n(REAL(data) + start, n, NA_REAL);
+    return Status::OK();
+  }
+
   Status Ingest_some_nulls(SEXP data, const std::shared_ptr<arrow::Array>& array,
                            R_xlen_t start, R_xlen_t n, size_t chunk_index) const {
-    auto convert = [](int days) { return static_cast<double>(days); };
-    return SomeNull_Ingest<REALSXP, int>(data, start, n, array->data()->GetValues<int>(1),
-                                         array, convert);
+    auto p_values = array->data()->GetValues<int>(1);
+    if (!p_values) {
+      return Status::Invalid("Invalid data buffer");
+    }
+    auto p_data = REAL(data) + start;
+    auto ingest_one = [&](R_xlen_t i) {
+      p_data[i] = static_cast<double>(p_values[i]);
+      return Status::OK();
+    };
+    auto null_one = [&](R_xlen_t i) {
+      p_data[i] = NA_REAL;
+      return Status::OK();
+    };
+
+    return IngestSome(array, n, ingest_one, null_one);
   }
 };
 
@@ -218,10 +254,13 @@ struct Converter_String : public Converter {
  public:
   explicit Converter_String(const ArrayVector& arrays) : Converter(arrays) {}
 
-  SEXP Allocate(R_xlen_t n) const { return StringVector_(no_init(n)); }
+  SEXP Allocate(R_xlen_t n) const { return Rf_allocVector(STRSXP, n); }
 
   Status Ingest_all_nulls(SEXP data, R_xlen_t start, R_xlen_t n) const {
-    return AllNull_Ingest<STRSXP>(data, start, n);
+    for (R_xlen_t i = 0; i < n; i++) {
+      SET_STRING_ELT(data, i + start, NA_STRING);
+    }
+    return Status::OK();
   }
 
   Status Ingest_some_nulls(SEXP data, const std::shared_ptr<arrow::Array>& array,
@@ -253,18 +292,21 @@ struct Converter_String : public Converter {
       // need to watch for nulls
       arrow::internal::BitmapReader null_reader(array->null_bitmap_data(),
                                                 array->offset(), n);
-      for (int i = 0; i < n; i++, null_reader.Next()) {
-        if (null_reader.IsSet()) {
-          SET_STRING_ELT(data, start + i, r_string(string_array->GetString(i)));
-        } else {
-          SET_STRING_ELT(data, start + i, NA_STRING);
+      cpp11::unwind_protect([&] {
+        for (int i = 0; i < n; i++, null_reader.Next()) {
+          if (null_reader.IsSet()) {
+            SET_STRING_ELT(data, start + i, r_string_from_view(string_array->GetView(i)));
+          } else {
+            SET_STRING_ELT(data, start + i, NA_STRING);
+          }
         }
-      }
-
+      });
     } else {
-      for (int i = 0; i < n; i++) {
-        SET_STRING_ELT(data, start + i, r_string(string_array->GetString(i)));
-      }
+      cpp11::unwind_protect([&] {
+        for (int i = 0; i < n; i++) {
+          SET_STRING_ELT(data, start + i, r_string_from_view(string_array->GetView(i)));
+        }
+      });
     }
 
     return Status::OK();
@@ -272,7 +314,8 @@ struct Converter_String : public Converter {
 
   bool Parallel() const { return false; }
 
-  inline SEXP r_string(const arrow::util::string_view& view) const {
+ private:
+  static SEXP r_string_from_view(const arrow::util::string_view& view) {
     return Rf_mkCharLenCE(view.data(), view.size(), CE_UTF8);
   }
 };
@@ -281,35 +324,35 @@ class Converter_Boolean : public Converter {
  public:
   explicit Converter_Boolean(const ArrayVector& arrays) : Converter(arrays) {}
 
-  SEXP Allocate(R_xlen_t n) const { return LogicalVector_(no_init(n)); }
+  SEXP Allocate(R_xlen_t n) const { return Rf_allocVector(LGLSXP, n); }
 
   Status Ingest_all_nulls(SEXP data, R_xlen_t start, R_xlen_t n) const {
-    return AllNull_Ingest<LGLSXP>(data, start, n);
+    std::fill_n(LOGICAL(data) + start, n, NA_LOGICAL);
+    return Status::OK();
   }
 
   Status Ingest_some_nulls(SEXP data, const std::shared_ptr<arrow::Array>& array,
                            R_xlen_t start, R_xlen_t n, size_t chunk_index) const {
-    auto p_data = Rcpp::internal::r_vector_start<LGLSXP>(data) + start;
+    auto p_data = LOGICAL(data) + start;
     auto p_bools = array->data()->GetValues<uint8_t>(1, 0);
     if (!p_bools) {
       return Status::Invalid("Invalid data buffer");
     }
 
     arrow::internal::BitmapReader data_reader(p_bools, array->offset(), n);
-    if (array->null_count()) {
-      arrow::internal::BitmapReader null_reader(array->null_bitmap()->data(),
-                                                array->offset(), n);
+    auto ingest_one = [&](R_xlen_t i) {
+      p_data[i] = data_reader.IsSet();
+      data_reader.Next();
+      return Status::OK();
+    };
 
-      for (R_xlen_t i = 0; i < n; i++, data_reader.Next(), null_reader.Next(), ++p_data) {
-        *p_data = null_reader.IsSet() ? data_reader.IsSet() : NA_LOGICAL;
-      }
-    } else {
-      for (R_xlen_t i = 0; i < n; i++, data_reader.Next(), ++p_data) {
-        *p_data = data_reader.IsSet();
-      }
-    }
+    auto null_one = [&](R_xlen_t i) {
+      data_reader.Next();
+      p_data[i] = NA_LOGICAL;
+      return Status::OK();
+    };
 
-    return Status::OK();
+    return IngestSome(array, n, ingest_one, null_one);
   }
 };
 
@@ -347,7 +390,7 @@ class Converter_Binary : public Converter {
       SEXP raw = PROTECT(Rf_allocVector(RAWSXP, ni));
       std::copy(value, value + ni, RAW(raw));
 
-      SET_VECTOR_ELT(data, i, raw);
+      SET_VECTOR_ELT(data, i + start, raw);
       UNPROTECT(1);
 
       return Status::OK();
@@ -355,6 +398,8 @@ class Converter_Binary : public Converter {
 
     return IngestSome(array, n, ingest_one);
   }
+
+  virtual bool Parallel() const { return false; }
 };
 
 class Converter_FixedSizeBinary : public Converter {
@@ -385,7 +430,7 @@ class Converter_FixedSizeBinary : public Converter {
       SEXP raw = PROTECT(Rf_allocVector(RAWSXP, byte_width));
       std::copy(value, value + byte_width, RAW(raw));
 
-      SET_VECTOR_ELT(data, i, raw);
+      SET_VECTOR_ELT(data, i + start, raw);
       UNPROTECT(1);
 
       return Status::OK();
@@ -394,6 +439,8 @@ class Converter_FixedSizeBinary : public Converter {
     return IngestSome(array, n, ingest_one);
   }
 
+  virtual bool Parallel() const { return false; }
+
  private:
   int byte_width_;
 };
@@ -437,8 +484,8 @@ class Converter_Dictionary : public Converter {
           // TODO: also add int64, uint32, uint64 downcasts, if possible
           break;
         default:
-          Rcpp::stop("Cannot convert Dictionary Array of type `%s` to R",
-                     dict_array.type()->ToString());
+          cpp11::stop("Cannot convert Dictionary Array of type `%s` to R",
+                      dict_array.type()->ToString().c_str());
       }
 
       dictionary_ = dict_array.dictionary();
@@ -446,7 +493,7 @@ class Converter_Dictionary : public Converter {
   }
 
   SEXP Allocate(R_xlen_t n) const {
-    IntegerVector data(no_init(n));
+    cpp11::writable::integers data(n);
     data.attr("levels") = GetLevels();
     if (GetOrdered()) {
       Rf_classgets(data, arrow::r::data::classes_ordered);
@@ -456,8 +503,11 @@ class Converter_Dictionary : public Converter {
     return data;
   }
 
+  virtual bool Parallel() const { return false; }
+
   Status Ingest_all_nulls(SEXP data, R_xlen_t start, R_xlen_t n) const {
-    return AllNull_Ingest<INTSXP>(data, start, n);
+    std::fill_n(INTEGER(data) + start, n, NA_INTEGER);
+    return Status::OK();
   }
 
   Status Ingest_some_nulls(SEXP data, const std::shared_ptr<arrow::Array>& array,
@@ -495,20 +545,31 @@ class Converter_Dictionary : public Converter {
     auto indices = checked_cast<const DictionaryArray&>(*array).indices();
     auto raw_indices = indices->data()->GetValues<index_type>(1);
 
+    auto p_data = INTEGER(data) + start;
+    auto null_one = [&](R_xlen_t i) {
+      p_data[i] = NA_INTEGER;
+      return Status::OK();
+    };
+
     // convert the 0-based indices from the arrow Array
     // to 1-based indices used in R factors
     if (need_unification_) {
       // transpose the indices before converting
       auto transposed =
           reinterpret_cast<const int32_t*>(arrays_transpose_[chunk_index]->data());
-      auto transpose_convert = [=](index_type i) { return transposed[i] + 1; };
 
-      return SomeNull_Ingest<INTSXP>(data, start, n, raw_indices, indices,
-                                     transpose_convert);
-    } else {
-      auto convert = [](index_type i) { return static_cast<int>(i) + 1; };
+      auto ingest_one = [&](R_xlen_t i) {
+        p_data[i] = transposed[raw_indices[i]] + 1;
+        return Status::OK();
+      };
 
-      return SomeNull_Ingest<INTSXP>(data, start, n, raw_indices, indices, convert);
+      return IngestSome(array, n, ingest_one, null_one);
+    } else {
+      auto ingest_one = [&](R_xlen_t i) {
+        p_data[i] = static_cast<int>(raw_indices[i]) + 1;
+        return Status::OK();
+      };
+      return IngestSome(array, n, ingest_one, null_one);
     }
   }
 
@@ -537,10 +598,9 @@ class Converter_Dictionary : public Converter {
     // Alternative: preserve the logical type of the dictionary values
     // (e.g. if dict is timestamp, return a POSIXt R vector, not factor)
     if (dictionary_->type_id() != Type::STRING) {
-      Rcpp::warning(
-          "Coercing dictionary values from type %s to R character factor levels",
-          dictionary_->type()->ToString());
+      cpp11::warning("Coercing dictionary values to R character factor levels");
     }
+
     SEXP vec = PROTECT(ArrayVector__as_vector(dictionary_->length(), dictionary_->type(),
                                               {dictionary_}));
     SEXP strings_vec = PROTECT(Rf_coerceVector(vec, STRSXP));
@@ -564,21 +624,17 @@ class Converter_Struct : public Converter {
     // allocate a data frame column to host each array
     auto first_array = checked_cast<const arrow::StructArray*>(this->arrays_[0].get());
     auto type = first_array->struct_type();
-    int nf = first_array->num_fields();
-    Rcpp::List out(nf);
-    Rcpp::CharacterVector colnames(nf);
-    for (int i = 0; i < nf; i++) {
-      out[i] = converters[i]->Allocate(n);
-      colnames[i] = Rcpp::String(type->field(i)->name(), CE_UTF8);
-    }
+    auto out =
+        arrow::r::to_r_list(converters, [n](const std::shared_ptr<Converter>& converter) {
+          return converter->Allocate(n);
+        });
+    auto colnames = arrow::r::to_r_strings(
+        type->fields(),
+        [](const std::shared_ptr<Field>& field) { return field->name(); });
+    out.attr(symbols::row_names) = arrow::r::short_row_names(n);
+    out.attr(R_NamesSymbol) = colnames;
+    out.attr(R_ClassSymbol) = arrow::r::data::classes_tbl_df;
 
-    IntegerVector rn(2);
-    rn[0] = NA_INTEGER;
-    rn[1] = -n;
-    Rf_setAttrib(out, symbols::row_names, rn);
-    Rf_setAttrib(out, R_NamesSymbol, colnames);
-    Rf_setAttrib(out, R_ClassSymbol,
-                 Rcpp::CharacterVector::create("tbl_df", "tbl", "data.frame"));
     return out;
   }
 
@@ -595,7 +651,7 @@ class Converter_Struct : public Converter {
     auto struct_array = checked_cast<const arrow::StructArray*>(array.get());
     int nf = converters.size();
     // Flatten() deals with merging of nulls
-    auto arrays = ValueOrStop(struct_array->Flatten(default_memory_pool()));
+    auto arrays = ValueOrStop(struct_array->Flatten(gc_memory_pool()));
     for (int i = 0; i < nf; i++) {
       StopIfNotOk(converters[i]->Ingest_some_nulls(VECTOR_ELT(data, i), arrays[i], start,
                                                    n, chunk_index));
@@ -604,6 +660,15 @@ class Converter_Struct : public Converter {
     return Status::OK();
   }
 
+  virtual bool Parallel() const {
+    // this can only run in parallel if all the
+    // inner converters can
+    for (const auto& converter : converters) {
+      if (!converter->Parallel()) return false;
+    }
+    return true;
+  }
+
  private:
   std::vector<std::shared_ptr<Converter>> converters;
 };
@@ -615,49 +680,29 @@ class Converter_Date64 : public Converter {
   explicit Converter_Date64(const ArrayVector& arrays) : Converter(arrays) {}
 
   SEXP Allocate(R_xlen_t n) const {
-    Rcpp::NumericVector data(no_init(n));
+    cpp11::writable::doubles data(n);
     Rf_classgets(data, arrow::r::data::classes_POSIXct);
     return data;
   }
 
   Status Ingest_all_nulls(SEXP data, R_xlen_t start, R_xlen_t n) const {
-    return AllNull_Ingest<REALSXP>(data, start, n);
-  }
-
-  Status Ingest_some_nulls(SEXP data, const std::shared_ptr<arrow::Array>& array,
-                           R_xlen_t start, R_xlen_t n, size_t chunk_index) const {
-    auto convert = [](int64_t ms) { return static_cast<double>(ms / 1000); };
-    return SomeNull_Ingest<REALSXP, int64_t>(
-        data, start, n, array->data()->GetValues<int64_t>(1), array, convert);
-  }
-};
-
-template <int RTYPE, typename Type>
-class Converter_Promotion : public Converter {
-  using r_stored_type = typename Rcpp::Vector<RTYPE>::stored_type;
-  using value_type = typename TypeTraits<Type>::ArrayType::value_type;
-
- public:
-  explicit Converter_Promotion(const ArrayVector& arrays) : Converter(arrays) {}
-
-  SEXP Allocate(R_xlen_t n) const {
-    return Rcpp::Vector<RTYPE, Rcpp::NoProtectStorage>(no_init(n));
-  }
-
-  Status Ingest_all_nulls(SEXP data, R_xlen_t start, R_xlen_t n) const {
-    return AllNull_Ingest<RTYPE>(data, start, n);
+    std::fill_n(REAL(data) + start, n, NA_REAL);
+    return Status::OK();
   }
 
   Status Ingest_some_nulls(SEXP data, const std::shared_ptr<arrow::Array>& array,
                            R_xlen_t start, R_xlen_t n, size_t chunk_index) const {
-    auto convert = [](value_type value) { return static_cast<r_stored_type>(value); };
-    return SomeNull_Ingest<RTYPE, value_type>(
-        data, start, n, array->data()->GetValues<value_type>(1), array, convert);
-  }
-
- private:
-  static r_stored_type value_convert(value_type value) {
-    return static_cast<r_stored_type>(value);
+    auto p_data = REAL(data) + start;
+    auto p_values = array->data()->GetValues<int64_t>(1);
+    auto ingest_one = [&](R_xlen_t i) {
+      p_data[i] = static_cast<double>(p_values[i] / 1000);
+      return Status::OK();
+    };
+    auto null_one = [&](R_xlen_t i) {
+      p_data[i] = NA_REAL;
+      return Status::OK();
+    };
+    return IngestSome(array, n, ingest_one, null_one);
   }
 };
 
@@ -667,25 +712,34 @@ class Converter_Time : public Converter {
   explicit Converter_Time(const ArrayVector& arrays) : Converter(arrays) {}
 
   SEXP Allocate(R_xlen_t n) const {
-    Rcpp::NumericVector data(no_init(n));
-    data.attr("class") = Rcpp::CharacterVector::create("hms", "difftime");
+    cpp11::writable::doubles data(n);
+    data.attr("class") = cpp11::writable::strings({"hms", "difftime"});
+
     // hms difftime is always stored as "seconds"
-    data.attr("units") = Rcpp::CharacterVector::create("secs");
+    data.attr("units") = cpp11::writable::strings({"secs"});
     return data;
   }
 
   Status Ingest_all_nulls(SEXP data, R_xlen_t start, R_xlen_t n) const {
-    return AllNull_Ingest<REALSXP>(data, start, n);
+    std::fill_n(REAL(data) + start, n, NA_REAL);
+    return Status::OK();
   }
 
   Status Ingest_some_nulls(SEXP data, const std::shared_ptr<arrow::Array>& array,
                            R_xlen_t start, R_xlen_t n, size_t chunk_index) const {
     int multiplier = TimeUnit_multiplier(array);
-    auto convert = [=](value_type value) {
-      return static_cast<double>(value) / multiplier;
+
+    auto p_data = REAL(data) + start;
+    auto p_values = array->data()->GetValues<value_type>(1);
+    auto ingest_one = [&](R_xlen_t i) {
+      p_data[i] = static_cast<double>(p_values[i]) / multiplier;
+      return Status::OK();
+    };
+    auto null_one = [&](R_xlen_t i) {
+      p_data[i] = NA_REAL;
+      return Status::OK();
     };
-    return SomeNull_Ingest<REALSXP, value_type>(
-        data, start, n, array->data()->GetValues<value_type>(1), array, convert);
+    return IngestSome(array, n, ingest_one, null_one);
   }
 
  private:
@@ -713,7 +767,7 @@ class Converter_Timestamp : public Converter_Time<value_type, TimestampType> {
       : Converter_Time<value_type, TimestampType>(arrays) {}
 
   SEXP Allocate(R_xlen_t n) const {
-    Rcpp::NumericVector data(no_init(n));
+    cpp11::writable::doubles data(n);
     Rf_classgets(data, arrow::r::data::classes_POSIXct);
     auto array = checked_cast<const TimestampArray*>(this->arrays_[0].get());
     auto array_type = checked_cast<const TimestampType*>(array->type().get());
@@ -729,32 +783,28 @@ class Converter_Decimal : public Converter {
  public:
   explicit Converter_Decimal(const ArrayVector& arrays) : Converter(arrays) {}
 
-  SEXP Allocate(R_xlen_t n) const { return Rcpp::NumericVector_(no_init(n)); }
+  SEXP Allocate(R_xlen_t n) const { return Rf_allocVector(REALSXP, n); }
 
   Status Ingest_all_nulls(SEXP data, R_xlen_t start, R_xlen_t n) const {
-    return AllNull_Ingest<REALSXP>(data, start, n);
+    std::fill_n(REAL(data) + start, n, NA_REAL);
+    return Status::OK();
   }
 
   Status Ingest_some_nulls(SEXP data, const std::shared_ptr<arrow::Array>& array,
                            R_xlen_t start, R_xlen_t n, size_t chunk_index) const {
-    auto p_data = Rcpp::internal::r_vector_start<REALSXP>(data) + start;
+    auto p_data = REAL(data) + start;
     const auto& decimals_arr = checked_cast<const arrow::Decimal128Array&>(*array);
 
-    if (array->null_count()) {
-      internal::BitmapReader bitmap_reader(array->null_bitmap()->data(), array->offset(),
-                                           n);
-
-      for (R_xlen_t i = 0; i < n; i++, bitmap_reader.Next(), ++p_data) {
-        *p_data = bitmap_reader.IsSet() ? std::stod(decimals_arr.FormatValue(i).c_str())
-                                        : NA_REAL;
-      }
-    } else {
-      for (R_xlen_t i = 0; i < n; i++, ++p_data) {
-        *p_data = std::stod(decimals_arr.FormatValue(i).c_str());
-      }
-    }
+    auto ingest_one = [&](R_xlen_t i) {
+      p_data[i] = std::stod(decimals_arr.FormatValue(i).c_str());
+      return Status::OK();
+    };
+    auto null_one = [&](R_xlen_t i) {
+      p_data[i] = NA_REAL;
+      return Status::OK();
+    };
 
-    return Status::OK();
+    return IngestSome(array, n, ingest_one, null_one);
   }
 };
 
@@ -769,23 +819,20 @@ class Converter_List : public Converter {
       : Converter(arrays), value_type_(value_type) {}
 
   SEXP Allocate(R_xlen_t n) const {
-    Rcpp::List res(no_init(n));
-    if (std::is_same<ListArrayType, ListArray>::value) {
-      Rf_setAttrib(res, R_ClassSymbol, arrow::r::data::classes_arrow_list);
-    } else {
-      Rf_setAttrib(res, R_ClassSymbol, arrow::r::data::classes_arrow_large_list);
-    }
+    cpp11::writable::list res(n);
+    res.attr(R_ClassSymbol) = std::is_same<ListArrayType, ListArray>::value
+                                  ? arrow::r::data::classes_arrow_list
+                                  : arrow::r::data::classes_arrow_large_list;
 
     // Build an empty array to match value_type
     std::unique_ptr<arrow::ArrayBuilder> builder;
-    StopIfNotOk(arrow::MakeBuilder(arrow::default_memory_pool(), value_type_, &builder));
+    StopIfNotOk(arrow::MakeBuilder(gc_memory_pool(), value_type_, &builder));
 
     std::shared_ptr<arrow::Array> array;
     StopIfNotOk(builder->Finish(&array));
 
     // convert to an R object to store as the list' ptype
-    SEXP ptype = Array__as_vector(array);
-    Rf_setAttrib(res, arrow::r::symbols::ptype, ptype);
+    res.attr(arrow::r::symbols::ptype) = Array__as_vector(array);
 
     return res;
   }
@@ -824,20 +871,19 @@ class Converter_FixedSizeList : public Converter {
       : Converter(arrays), value_type_(value_type), list_size_(list_size) {}
 
   SEXP Allocate(R_xlen_t n) const {
-    Rcpp::List res(no_init(n));
+    cpp11::writable::list res(n);
     Rf_classgets(res, arrow::r::data::classes_arrow_fixed_size_list);
-    Rf_setAttrib(res, arrow::r::symbols::list_size, Rf_ScalarInteger(list_size_));
+    res.attr(arrow::r::symbols::list_size) = Rf_ScalarInteger(list_size_);
 
     // Build an empty array to match value_type
     std::unique_ptr<arrow::ArrayBuilder> builder;
-    StopIfNotOk(arrow::MakeBuilder(arrow::default_memory_pool(), value_type_, &builder));
+    StopIfNotOk(arrow::MakeBuilder(gc_memory_pool(), value_type_, &builder));
 
     std::shared_ptr<arrow::Array> array;
     StopIfNotOk(builder->Finish(&array));
 
     // convert to an R object to store as the list' ptype
-    SEXP ptype = Array__as_vector(array);
-    Rf_setAttrib(res, arrow::r::symbols::ptype, ptype);
+    res.attr(arrow::r::symbols::ptype) = Array__as_vector(array);
 
     return res;
   }
@@ -868,7 +914,7 @@ class Converter_Int64 : public Converter {
   explicit Converter_Int64(const ArrayVector& arrays) : Converter(arrays) {}
 
   SEXP Allocate(R_xlen_t n) const {
-    Rcpp::NumericVector data(no_init(n));
+    cpp11::writable::doubles data(n);
     data.attr("class") = "integer64";
     return data;
   }
@@ -907,8 +953,10 @@ class Converter_Null : public Converter {
   explicit Converter_Null(const ArrayVector& arrays) : Converter(arrays) {}
 
   SEXP Allocate(R_xlen_t n) const {
-    Rcpp::LogicalVector data(n, NA_LOGICAL);
-    data.attr("class") = "vctrs_unspecified";
+    SEXP data = PROTECT(Rf_allocVector(LGLSXP, n));
+    std::fill_n(LOGICAL(data), n, NA_LOGICAL);
+    Rf_classgets(data, Rf_mkString("vctrs_unspecified"));
+    UNPROTECT(1);
     return data;
   }
 
@@ -923,14 +971,25 @@ class Converter_Null : public Converter {
 };
 
 bool ArraysCanFitInteger(ArrayVector arrays) {
-  bool out = false;
+  bool all_can_fit = true;
   auto i32 = arrow::int32();
   for (const auto& array : arrays) {
-    if (!out) {
-      out = arrow::IntegersCanFit(arrow::Datum(array), *i32).ok();
+    if (all_can_fit) {
+      all_can_fit = arrow::IntegersCanFit(arrow::Datum(array), *i32).ok();
     }
   }
-  return out;
+  return all_can_fit;
+}
+
+bool GetBoolOption(const std::string& name, bool default_) {
+  SEXP getOption = Rf_install("getOption");
+  cpp11::sexp call = Rf_lang2(getOption, Rf_mkString(name.c_str()));
+  cpp11::sexp res = Rf_eval(call, R_BaseEnv);
+  if (TYPEOF(res) == LGLSXP) {
+    return LOGICAL(res)[0] == TRUE;
+  } else {
+    return default_;
+  }
 }
 
 std::shared_ptr<Converter> Converter::Make(const std::shared_ptr<DataType>& type,
@@ -944,10 +1003,11 @@ std::shared_ptr<Converter> Converter::Make(const std::shared_ptr<DataType>& type
   switch (type->id()) {
     // direct support
     case Type::INT32:
-      return std::make_shared<arrow::r::Converter_SimpleArray<INTSXP>>(std::move(arrays));
+      return std::make_shared<arrow::r::Converter_Int<arrow::Int32Type>>(
+          std::move(arrays));
 
     case Type::DOUBLE:
-      return std::make_shared<arrow::r::Converter_SimpleArray<REALSXP>>(
+      return std::make_shared<arrow::r::Converter_Double<arrow::DoubleType>>(
           std::move(arrays));
 
       // need to handle 1-bit case
@@ -987,47 +1047,46 @@ std::shared_ptr<Converter> Converter::Make(const std::shared_ptr<DataType>& type
 
       // promotions to integer vector
     case Type::INT8:
-      return std::make_shared<arrow::r::Converter_Promotion<INTSXP, arrow::Int8Type>>(
+      return std::make_shared<arrow::r::Converter_Int<arrow::Int8Type>>(
           std::move(arrays));
 
     case Type::UINT8:
-      return std::make_shared<arrow::r::Converter_Promotion<INTSXP, arrow::UInt8Type>>(
+      return std::make_shared<arrow::r::Converter_Int<arrow::UInt8Type>>(
           std::move(arrays));
 
     case Type::INT16:
-      return std::make_shared<arrow::r::Converter_Promotion<INTSXP, arrow::Int16Type>>(
+      return std::make_shared<arrow::r::Converter_Int<arrow::Int16Type>>(
           std::move(arrays));
 
     case Type::UINT16:
-      return std::make_shared<arrow::r::Converter_Promotion<INTSXP, arrow::UInt16Type>>(
+      return std::make_shared<arrow::r::Converter_Int<arrow::UInt16Type>>(
           std::move(arrays));
 
       // promotions to numeric vector, if they don't fit into int32
     case Type::UINT32:
       if (ArraysCanFitInteger(arrays)) {
-        return std::make_shared<arrow::r::Converter_Promotion<INTSXP, arrow::UInt32Type>>(
+        return std::make_shared<arrow::r::Converter_Int<arrow::UInt32Type>>(
             std::move(arrays));
       } else {
-        return std::make_shared<
-            arrow::r::Converter_Promotion<REALSXP, arrow::UInt32Type>>(std::move(arrays));
+        return std::make_shared<arrow::r::Converter_Double<arrow::UInt32Type>>(
+            std::move(arrays));
       }
 
     case Type::UINT64:
       if (ArraysCanFitInteger(arrays)) {
-        return std::make_shared<arrow::r::Converter_Promotion<INTSXP, arrow::UInt64Type>>(
+        return std::make_shared<arrow::r::Converter_Int<arrow::UInt64Type>>(
             std::move(arrays));
       } else {
-        return std::make_shared<
-            arrow::r::Converter_Promotion<REALSXP, arrow::UInt64Type>>(std::move(arrays));
+        return std::make_shared<arrow::r::Converter_Double<arrow::UInt64Type>>(
+            std::move(arrays));
       }
 
     case Type::HALF_FLOAT:
-      return std::make_shared<
-          arrow::r::Converter_Promotion<REALSXP, arrow::HalfFloatType>>(
+      return std::make_shared<arrow::r::Converter_Double<arrow::HalfFloatType>>(
           std::move(arrays));
 
     case Type::FLOAT:
-      return std::make_shared<arrow::r::Converter_Promotion<REALSXP, arrow::FloatType>>(
+      return std::make_shared<arrow::r::Converter_Double<arrow::FloatType>>(
           std::move(arrays));
 
       // time32 and time64
@@ -1041,9 +1100,9 @@ std::shared_ptr<Converter> Converter::Make(const std::shared_ptr<DataType>& type
       return std::make_shared<arrow::r::Converter_Timestamp<int64_t>>(std::move(arrays));
 
     case Type::INT64:
-      // Prefer integer if it fits
-      if (ArraysCanFitInteger(arrays)) {
-        return std::make_shared<arrow::r::Converter_Promotion<INTSXP, arrow::Int64Type>>(
+      // Prefer integer if it fits, unless option arrow.int64_downcast is `false`
+      if (GetBoolOption("arrow.int64_downcast", true) && ArraysCanFitInteger(arrays)) {
+        return std::make_shared<arrow::r::Converter_Int<arrow::Int64Type>>(
             std::move(arrays));
       } else {
         return std::make_shared<arrow::r::Converter_Int64>(std::move(arrays));
@@ -1079,29 +1138,27 @@ std::shared_ptr<Converter> Converter::Make(const std::shared_ptr<DataType>& type
       break;
   }
 
-  Rcpp::stop(tfm::format("cannot handle Array of type %s", type->name()));
-  return nullptr;
+  cpp11::stop("cannot handle Array of type ", type->name().c_str());
 }
 
-Rcpp::List to_dataframe_serial(
-    int64_t nr, int64_t nc, const Rcpp::CharacterVector& names,
+cpp11::writable::list to_dataframe_serial(
+    int64_t nr, int64_t nc, const cpp11::writable::strings& names,
     const std::vector<std::shared_ptr<Converter>>& converters) {
-  Rcpp::List tbl(nc);
-
+  cpp11::writable::list tbl(nc);
   for (int i = 0; i < nc; i++) {
     SEXP column = tbl[i] = converters[i]->Allocate(nr);
     StopIfNotOk(converters[i]->IngestSerial(column));
   }
-  tbl.attr("names") = names;
-  tbl.attr("class") = Rcpp::CharacterVector::create("tbl_df", "tbl", "data.frame");
-  tbl.attr("row.names") = Rcpp::IntegerVector::create(NA_INTEGER, -nr);
+  tbl.attr(R_NamesSymbol) = names;
+  tbl.attr(R_ClassSymbol) = arrow::r::data::classes_tbl_df;
+  tbl.attr(R_RowNamesSymbol) = arrow::r::short_row_names(nr);
   return tbl;
 }
 
-Rcpp::List to_dataframe_parallel(
-    int64_t nr, int64_t nc, const Rcpp::CharacterVector& names,
+cpp11::writable::list to_dataframe_parallel(
+    int64_t nr, int64_t nc, const cpp11::writable::strings& names,
     const std::vector<std::shared_ptr<Converter>>& converters) {
-  Rcpp::List tbl(nc);
+  cpp11::writable::list tbl(nc);
 
   // task group to ingest data in parallel
   auto tg = arrow::internal::TaskGroup::MakeThreaded(arrow::internal::GetCpuThreadPool());
@@ -1133,9 +1190,9 @@ Rcpp::List to_dataframe_parallel(
 
   StopIfNotOk(status);
 
-  tbl.attr("names") = names;
-  tbl.attr("class") = Rcpp::CharacterVector::create("tbl_df", "tbl", "data.frame");
-  tbl.attr("row.names") = IntegerVector::create(NA_INTEGER, -nr);
+  tbl.attr(R_NamesSymbol) = names;
+  tbl.attr(R_ClassSymbol) = arrow::r::data::classes_tbl_df;
+  tbl.attr(R_RowNamesSymbol) = arrow::r::short_row_names(nr);
 
   return tbl;
 }
@@ -1155,16 +1212,16 @@ SEXP ChunkedArray__as_vector(const std::shared_ptr<arrow::ChunkedArray>& chunked
 }
 
 // [[arrow::export]]
-Rcpp::List RecordBatch__to_dataframe(const std::shared_ptr<arrow::RecordBatch>& batch,
-                                     bool use_threads) {
+cpp11::writable::list RecordBatch__to_dataframe(
+    const std::shared_ptr<arrow::RecordBatch>& batch, bool use_threads) {
   int64_t nc = batch->num_columns();
   int64_t nr = batch->num_rows();
-  Rcpp::CharacterVector names(nc);
+  cpp11::writable::strings names(nc);
   std::vector<arrow::ArrayVector> arrays(nc);
   std::vector<std::shared_ptr<arrow::r::Converter>> converters(nc);
 
-  for (int64_t i = 0; i < nc; i++) {
-    names[i] = Rcpp::String(batch->column_name(i), CE_UTF8);
+  for (R_xlen_t i = 0; i < nc; i++) {
+    names[i] = batch->column_name(i);
     arrays[i] = {batch->column(i)};
     converters[i] = arrow::r::Converter::Make(batch->column(i)->type(), arrays[i]);
   }
@@ -1177,17 +1234,17 @@ Rcpp::List RecordBatch__to_dataframe(const std::shared_ptr<arrow::RecordBatch>&
 }
 
 // [[arrow::export]]
-Rcpp::List Table__to_dataframe(const std::shared_ptr<arrow::Table>& table,
-                               bool use_threads) {
+cpp11::writable::list Table__to_dataframe(const std::shared_ptr<arrow::Table>& table,
+                                          bool use_threads) {
   int64_t nc = table->num_columns();
   int64_t nr = table->num_rows();
-  Rcpp::CharacterVector names(nc);
+  cpp11::writable::strings names(nc);
   std::vector<std::shared_ptr<arrow::r::Converter>> converters(nc);
 
-  for (int64_t i = 0; i < nc; i++) {
+  for (R_xlen_t i = 0; i < nc; i++) {
     converters[i] =
         arrow::r::Converter::Make(table->column(i)->type(), table->column(i)->chunks());
-    names[i] = Rcpp::String(table->field(i)->name(), CE_UTF8);
+    names[i] = table->field(i)->name();
   }
 
   if (use_threads) {
diff --git a/r/src/arraydata.cpp b/r/src/arraydata.cpp
index c1194d76eeb..a8ee60f36a0 100644
--- a/r/src/arraydata.cpp
+++ b/r/src/arraydata.cpp
@@ -17,9 +17,6 @@
 
 #include "./arrow_types.h"
 
-using Rcpp::List;
-using Rcpp::wrap;
-
 #if defined(ARROW_R_WITH_ARROW)
 #include <arrow/array/data.h>
 
@@ -45,8 +42,8 @@ int ArrayData__get_offset(const std::shared_ptr<arrow::ArrayData>& x) {
 }
 
 // [[arrow::export]]
-List ArrayData__buffers(const std::shared_ptr<arrow::ArrayData>& x) {
-  return wrap(x->buffers);
+cpp11::list ArrayData__buffers(const std::shared_ptr<arrow::ArrayData>& x) {
+  return cpp11::as_sexp(x->buffers);
 }
 
 #endif
diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp
index 17f97cb2c96..cadfc8c0745 100644
--- a/r/src/arrowExports.cpp
+++ b/r/src/arrowExports.cpp
@@ -1,70 +1,70 @@
 // Generated by using data-raw/codegen.R -> do not edit by hand
-#include "./arrow_exports.h"
-#include <Rcpp.h>
+#include <cpp11.hpp>
+#include <cpp11/declarations.hpp>
 
-using namespace Rcpp;
+#include "./arrow_exports.h"
 
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::Array> Array__Slice1(const std::shared_ptr<arrow::Array>& array, int offset);
-RcppExport SEXP _arrow_Array__Slice1(SEXP array_sexp, SEXP offset_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type array(array_sexp);
-	Rcpp::traits::input_parameter<int>::type offset(offset_sexp);
-	return Rcpp::wrap(Array__Slice1(array, offset));
-END_RCPP
+std::shared_ptr<arrow::Array> Array__Slice1(const std::shared_ptr<arrow::Array>& array, R_xlen_t offset);
+extern "C" SEXP _arrow_Array__Slice1(SEXP array_sexp, SEXP offset_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type array(array_sexp);
+	arrow::r::Input<R_xlen_t>::type offset(offset_sexp);
+	return cpp11::as_sexp(Array__Slice1(array, offset));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__Slice1(SEXP array_sexp, SEXP offset_sexp){
+extern "C" SEXP _arrow_Array__Slice1(SEXP array_sexp, SEXP offset_sexp){
 	Rf_error("Cannot call Array__Slice1(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::Array> Array__Slice2(const std::shared_ptr<arrow::Array>& array, int offset, int length);
-RcppExport SEXP _arrow_Array__Slice2(SEXP array_sexp, SEXP offset_sexp, SEXP length_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type array(array_sexp);
-	Rcpp::traits::input_parameter<int>::type offset(offset_sexp);
-	Rcpp::traits::input_parameter<int>::type length(length_sexp);
-	return Rcpp::wrap(Array__Slice2(array, offset, length));
-END_RCPP
+std::shared_ptr<arrow::Array> Array__Slice2(const std::shared_ptr<arrow::Array>& array, R_xlen_t offset, R_xlen_t length);
+extern "C" SEXP _arrow_Array__Slice2(SEXP array_sexp, SEXP offset_sexp, SEXP length_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type array(array_sexp);
+	arrow::r::Input<R_xlen_t>::type offset(offset_sexp);
+	arrow::r::Input<R_xlen_t>::type length(length_sexp);
+	return cpp11::as_sexp(Array__Slice2(array, offset, length));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__Slice2(SEXP array_sexp, SEXP offset_sexp, SEXP length_sexp){
+extern "C" SEXP _arrow_Array__Slice2(SEXP array_sexp, SEXP offset_sexp, SEXP length_sexp){
 	Rf_error("Cannot call Array__Slice2(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
-bool Array__IsNull(const std::shared_ptr<arrow::Array>& x, int i);
-RcppExport SEXP _arrow_Array__IsNull(SEXP x_sexp, SEXP i_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type x(x_sexp);
-	Rcpp::traits::input_parameter<int>::type i(i_sexp);
-	return Rcpp::wrap(Array__IsNull(x, i));
-END_RCPP
+bool Array__IsNull(const std::shared_ptr<arrow::Array>& x, R_xlen_t i);
+extern "C" SEXP _arrow_Array__IsNull(SEXP x_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type x(x_sexp);
+	arrow::r::Input<R_xlen_t>::type i(i_sexp);
+	return cpp11::as_sexp(Array__IsNull(x, i));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__IsNull(SEXP x_sexp, SEXP i_sexp){
+extern "C" SEXP _arrow_Array__IsNull(SEXP x_sexp, SEXP i_sexp){
 	Rf_error("Cannot call Array__IsNull(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
-bool Array__IsValid(const std::shared_ptr<arrow::Array>& x, int i);
-RcppExport SEXP _arrow_Array__IsValid(SEXP x_sexp, SEXP i_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type x(x_sexp);
-	Rcpp::traits::input_parameter<int>::type i(i_sexp);
-	return Rcpp::wrap(Array__IsValid(x, i));
-END_RCPP
+bool Array__IsValid(const std::shared_ptr<arrow::Array>& x, R_xlen_t i);
+extern "C" SEXP _arrow_Array__IsValid(SEXP x_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type x(x_sexp);
+	arrow::r::Input<R_xlen_t>::type i(i_sexp);
+	return cpp11::as_sexp(Array__IsValid(x, i));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__IsValid(SEXP x_sexp, SEXP i_sexp){
+extern "C" SEXP _arrow_Array__IsValid(SEXP x_sexp, SEXP i_sexp){
 	Rf_error("Cannot call Array__IsValid(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -72,14 +72,14 @@ RcppExport SEXP _arrow_Array__IsValid(SEXP x_sexp, SEXP i_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int Array__length(const std::shared_ptr<arrow::Array>& x);
-RcppExport SEXP _arrow_Array__length(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type x(x_sexp);
-	return Rcpp::wrap(Array__length(x));
-END_RCPP
+extern "C" SEXP _arrow_Array__length(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type x(x_sexp);
+	return cpp11::as_sexp(Array__length(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__length(SEXP x_sexp){
+extern "C" SEXP _arrow_Array__length(SEXP x_sexp){
 	Rf_error("Cannot call Array__length(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -87,14 +87,14 @@ RcppExport SEXP _arrow_Array__length(SEXP x_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int Array__offset(const std::shared_ptr<arrow::Array>& x);
-RcppExport SEXP _arrow_Array__offset(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type x(x_sexp);
-	return Rcpp::wrap(Array__offset(x));
-END_RCPP
+extern "C" SEXP _arrow_Array__offset(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type x(x_sexp);
+	return cpp11::as_sexp(Array__offset(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__offset(SEXP x_sexp){
+extern "C" SEXP _arrow_Array__offset(SEXP x_sexp){
 	Rf_error("Cannot call Array__offset(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -102,14 +102,14 @@ RcppExport SEXP _arrow_Array__offset(SEXP x_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int Array__null_count(const std::shared_ptr<arrow::Array>& x);
-RcppExport SEXP _arrow_Array__null_count(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type x(x_sexp);
-	return Rcpp::wrap(Array__null_count(x));
-END_RCPP
+extern "C" SEXP _arrow_Array__null_count(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type x(x_sexp);
+	return cpp11::as_sexp(Array__null_count(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__null_count(SEXP x_sexp){
+extern "C" SEXP _arrow_Array__null_count(SEXP x_sexp){
 	Rf_error("Cannot call Array__null_count(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -117,14 +117,14 @@ RcppExport SEXP _arrow_Array__null_count(SEXP x_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Array__type(const std::shared_ptr<arrow::Array>& x);
-RcppExport SEXP _arrow_Array__type(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type x(x_sexp);
-	return Rcpp::wrap(Array__type(x));
-END_RCPP
+extern "C" SEXP _arrow_Array__type(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type x(x_sexp);
+	return cpp11::as_sexp(Array__type(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__type(SEXP x_sexp){
+extern "C" SEXP _arrow_Array__type(SEXP x_sexp){
 	Rf_error("Cannot call Array__type(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -132,14 +132,14 @@ RcppExport SEXP _arrow_Array__type(SEXP x_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::string Array__ToString(const std::shared_ptr<arrow::Array>& x);
-RcppExport SEXP _arrow_Array__ToString(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type x(x_sexp);
-	return Rcpp::wrap(Array__ToString(x));
-END_RCPP
+extern "C" SEXP _arrow_Array__ToString(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type x(x_sexp);
+	return cpp11::as_sexp(Array__ToString(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__ToString(SEXP x_sexp){
+extern "C" SEXP _arrow_Array__ToString(SEXP x_sexp){
 	Rf_error("Cannot call Array__ToString(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -147,14 +147,14 @@ RcppExport SEXP _arrow_Array__ToString(SEXP x_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 arrow::Type::type Array__type_id(const std::shared_ptr<arrow::Array>& x);
-RcppExport SEXP _arrow_Array__type_id(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type x(x_sexp);
-	return Rcpp::wrap(Array__type_id(x));
-END_RCPP
+extern "C" SEXP _arrow_Array__type_id(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type x(x_sexp);
+	return cpp11::as_sexp(Array__type_id(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__type_id(SEXP x_sexp){
+extern "C" SEXP _arrow_Array__type_id(SEXP x_sexp){
 	Rf_error("Cannot call Array__type_id(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -162,15 +162,15 @@ RcppExport SEXP _arrow_Array__type_id(SEXP x_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool Array__Equals(const std::shared_ptr<arrow::Array>& lhs, const std::shared_ptr<arrow::Array>& rhs);
-RcppExport SEXP _arrow_Array__Equals(SEXP lhs_sexp, SEXP rhs_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type lhs(lhs_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type rhs(rhs_sexp);
-	return Rcpp::wrap(Array__Equals(lhs, rhs));
-END_RCPP
+extern "C" SEXP _arrow_Array__Equals(SEXP lhs_sexp, SEXP rhs_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type lhs(lhs_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type rhs(rhs_sexp);
+	return cpp11::as_sexp(Array__Equals(lhs, rhs));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__Equals(SEXP lhs_sexp, SEXP rhs_sexp){
+extern "C" SEXP _arrow_Array__Equals(SEXP lhs_sexp, SEXP rhs_sexp){
 	Rf_error("Cannot call Array__Equals(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -178,15 +178,15 @@ RcppExport SEXP _arrow_Array__Equals(SEXP lhs_sexp, SEXP rhs_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool Array__ApproxEquals(const std::shared_ptr<arrow::Array>& lhs, const std::shared_ptr<arrow::Array>& rhs);
-RcppExport SEXP _arrow_Array__ApproxEquals(SEXP lhs_sexp, SEXP rhs_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type lhs(lhs_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type rhs(rhs_sexp);
-	return Rcpp::wrap(Array__ApproxEquals(lhs, rhs));
-END_RCPP
+extern "C" SEXP _arrow_Array__ApproxEquals(SEXP lhs_sexp, SEXP rhs_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type lhs(lhs_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type rhs(rhs_sexp);
+	return cpp11::as_sexp(Array__ApproxEquals(lhs, rhs));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__ApproxEquals(SEXP lhs_sexp, SEXP rhs_sexp){
+extern "C" SEXP _arrow_Array__ApproxEquals(SEXP lhs_sexp, SEXP rhs_sexp){
 	Rf_error("Cannot call Array__ApproxEquals(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -194,33 +194,33 @@ RcppExport SEXP _arrow_Array__ApproxEquals(SEXP lhs_sexp, SEXP rhs_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::ArrayData> Array__data(const std::shared_ptr<arrow::Array>& array);
-RcppExport SEXP _arrow_Array__data(SEXP array_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type array(array_sexp);
-	return Rcpp::wrap(Array__data(array));
-END_RCPP
+extern "C" SEXP _arrow_Array__data(SEXP array_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type array(array_sexp);
+	return cpp11::as_sexp(Array__data(array));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__data(SEXP array_sexp){
+extern "C" SEXP _arrow_Array__data(SEXP array_sexp){
 	Rf_error("Cannot call Array__data(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
-bool Array__RangeEquals(const std::shared_ptr<arrow::Array>& self, const std::shared_ptr<arrow::Array>& other, int start_idx, int end_idx, int other_start_idx);
-RcppExport SEXP _arrow_Array__RangeEquals(SEXP self_sexp, SEXP other_sexp, SEXP start_idx_sexp, SEXP end_idx_sexp, SEXP other_start_idx_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type self(self_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type other(other_sexp);
-	Rcpp::traits::input_parameter<int>::type start_idx(start_idx_sexp);
-	Rcpp::traits::input_parameter<int>::type end_idx(end_idx_sexp);
-	Rcpp::traits::input_parameter<int>::type other_start_idx(other_start_idx_sexp);
-	return Rcpp::wrap(Array__RangeEquals(self, other, start_idx, end_idx, other_start_idx));
-END_RCPP
+bool Array__RangeEquals(const std::shared_ptr<arrow::Array>& self, const std::shared_ptr<arrow::Array>& other, R_xlen_t start_idx, R_xlen_t end_idx, R_xlen_t other_start_idx);
+extern "C" SEXP _arrow_Array__RangeEquals(SEXP self_sexp, SEXP other_sexp, SEXP start_idx_sexp, SEXP end_idx_sexp, SEXP other_start_idx_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type self(self_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type other(other_sexp);
+	arrow::r::Input<R_xlen_t>::type start_idx(start_idx_sexp);
+	arrow::r::Input<R_xlen_t>::type end_idx(end_idx_sexp);
+	arrow::r::Input<R_xlen_t>::type other_start_idx(other_start_idx_sexp);
+	return cpp11::as_sexp(Array__RangeEquals(self, other, start_idx, end_idx, other_start_idx));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__RangeEquals(SEXP self_sexp, SEXP other_sexp, SEXP start_idx_sexp, SEXP end_idx_sexp, SEXP other_start_idx_sexp){
+extern "C" SEXP _arrow_Array__RangeEquals(SEXP self_sexp, SEXP other_sexp, SEXP start_idx_sexp, SEXP end_idx_sexp, SEXP other_start_idx_sexp){
 	Rf_error("Cannot call Array__RangeEquals(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -228,15 +228,15 @@ RcppExport SEXP _arrow_Array__RangeEquals(SEXP self_sexp, SEXP other_sexp, SEXP
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Array> Array__View(const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& type);
-RcppExport SEXP _arrow_Array__View(SEXP array_sexp, SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type array(array_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DataType>&>::type type(type_sexp);
-	return Rcpp::wrap(Array__View(array, type));
-END_RCPP
+extern "C" SEXP _arrow_Array__View(SEXP array_sexp, SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type array(array_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::DataType>&>::type type(type_sexp);
+	return cpp11::as_sexp(Array__View(array, type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__View(SEXP array_sexp, SEXP type_sexp){
+extern "C" SEXP _arrow_Array__View(SEXP array_sexp, SEXP type_sexp){
 	Rf_error("Cannot call Array__View(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -244,15 +244,15 @@ RcppExport SEXP _arrow_Array__View(SEXP array_sexp, SEXP type_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void Array__Validate(const std::shared_ptr<arrow::Array>& array);
-RcppExport SEXP _arrow_Array__Validate(SEXP array_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type array(array_sexp);
+extern "C" SEXP _arrow_Array__Validate(SEXP array_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type array(array_sexp);
 	Array__Validate(array);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__Validate(SEXP array_sexp){
+extern "C" SEXP _arrow_Array__Validate(SEXP array_sexp){
 	Rf_error("Cannot call Array__Validate(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -260,14 +260,14 @@ RcppExport SEXP _arrow_Array__Validate(SEXP array_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Array> DictionaryArray__indices(const std::shared_ptr<arrow::DictionaryArray>& array);
-RcppExport SEXP _arrow_DictionaryArray__indices(SEXP array_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DictionaryArray>&>::type array(array_sexp);
-	return Rcpp::wrap(DictionaryArray__indices(array));
-END_RCPP
+extern "C" SEXP _arrow_DictionaryArray__indices(SEXP array_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::DictionaryArray>&>::type array(array_sexp);
+	return cpp11::as_sexp(DictionaryArray__indices(array));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_DictionaryArray__indices(SEXP array_sexp){
+extern "C" SEXP _arrow_DictionaryArray__indices(SEXP array_sexp){
 	Rf_error("Cannot call DictionaryArray__indices(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -275,14 +275,14 @@ RcppExport SEXP _arrow_DictionaryArray__indices(SEXP array_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Array> DictionaryArray__dictionary(const std::shared_ptr<arrow::DictionaryArray>& array);
-RcppExport SEXP _arrow_DictionaryArray__dictionary(SEXP array_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DictionaryArray>&>::type array(array_sexp);
-	return Rcpp::wrap(DictionaryArray__dictionary(array));
-END_RCPP
+extern "C" SEXP _arrow_DictionaryArray__dictionary(SEXP array_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::DictionaryArray>&>::type array(array_sexp);
+	return cpp11::as_sexp(DictionaryArray__dictionary(array));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_DictionaryArray__dictionary(SEXP array_sexp){
+extern "C" SEXP _arrow_DictionaryArray__dictionary(SEXP array_sexp){
 	Rf_error("Cannot call DictionaryArray__dictionary(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -290,15 +290,15 @@ RcppExport SEXP _arrow_DictionaryArray__dictionary(SEXP array_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Array> StructArray__field(const std::shared_ptr<arrow::StructArray>& array, int i);
-RcppExport SEXP _arrow_StructArray__field(SEXP array_sexp, SEXP i_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::StructArray>&>::type array(array_sexp);
-	Rcpp::traits::input_parameter<int>::type i(i_sexp);
-	return Rcpp::wrap(StructArray__field(array, i));
-END_RCPP
+extern "C" SEXP _arrow_StructArray__field(SEXP array_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::StructArray>&>::type array(array_sexp);
+	arrow::r::Input<int>::type i(i_sexp);
+	return cpp11::as_sexp(StructArray__field(array, i));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_StructArray__field(SEXP array_sexp, SEXP i_sexp){
+extern "C" SEXP _arrow_StructArray__field(SEXP array_sexp, SEXP i_sexp){
 	Rf_error("Cannot call StructArray__field(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -306,15 +306,15 @@ RcppExport SEXP _arrow_StructArray__field(SEXP array_sexp, SEXP i_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Array> StructArray__GetFieldByName(const std::shared_ptr<arrow::StructArray>& array, const std::string& name);
-RcppExport SEXP _arrow_StructArray__GetFieldByName(SEXP array_sexp, SEXP name_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::StructArray>&>::type array(array_sexp);
-	Rcpp::traits::input_parameter<const std::string&>::type name(name_sexp);
-	return Rcpp::wrap(StructArray__GetFieldByName(array, name));
-END_RCPP
+extern "C" SEXP _arrow_StructArray__GetFieldByName(SEXP array_sexp, SEXP name_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::StructArray>&>::type array(array_sexp);
+	arrow::r::Input<const std::string&>::type name(name_sexp);
+	return cpp11::as_sexp(StructArray__GetFieldByName(array, name));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_StructArray__GetFieldByName(SEXP array_sexp, SEXP name_sexp){
+extern "C" SEXP _arrow_StructArray__GetFieldByName(SEXP array_sexp, SEXP name_sexp){
 	Rf_error("Cannot call StructArray__GetFieldByName(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -322,14 +322,14 @@ RcppExport SEXP _arrow_StructArray__GetFieldByName(SEXP array_sexp, SEXP name_se
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 arrow::ArrayVector StructArray__Flatten(const std::shared_ptr<arrow::StructArray>& array);
-RcppExport SEXP _arrow_StructArray__Flatten(SEXP array_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::StructArray>&>::type array(array_sexp);
-	return Rcpp::wrap(StructArray__Flatten(array));
-END_RCPP
+extern "C" SEXP _arrow_StructArray__Flatten(SEXP array_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::StructArray>&>::type array(array_sexp);
+	return cpp11::as_sexp(StructArray__Flatten(array));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_StructArray__Flatten(SEXP array_sexp){
+extern "C" SEXP _arrow_StructArray__Flatten(SEXP array_sexp){
 	Rf_error("Cannot call StructArray__Flatten(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -337,14 +337,14 @@ RcppExport SEXP _arrow_StructArray__Flatten(SEXP array_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> ListArray__value_type(const std::shared_ptr<arrow::ListArray>& array);
-RcppExport SEXP _arrow_ListArray__value_type(SEXP array_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ListArray>&>::type array(array_sexp);
-	return Rcpp::wrap(ListArray__value_type(array));
-END_RCPP
+extern "C" SEXP _arrow_ListArray__value_type(SEXP array_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ListArray>&>::type array(array_sexp);
+	return cpp11::as_sexp(ListArray__value_type(array));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ListArray__value_type(SEXP array_sexp){
+extern "C" SEXP _arrow_ListArray__value_type(SEXP array_sexp){
 	Rf_error("Cannot call ListArray__value_type(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -352,14 +352,14 @@ RcppExport SEXP _arrow_ListArray__value_type(SEXP array_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> LargeListArray__value_type(const std::shared_ptr<arrow::LargeListArray>& array);
-RcppExport SEXP _arrow_LargeListArray__value_type(SEXP array_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::LargeListArray>&>::type array(array_sexp);
-	return Rcpp::wrap(LargeListArray__value_type(array));
-END_RCPP
+extern "C" SEXP _arrow_LargeListArray__value_type(SEXP array_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::LargeListArray>&>::type array(array_sexp);
+	return cpp11::as_sexp(LargeListArray__value_type(array));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_LargeListArray__value_type(SEXP array_sexp){
+extern "C" SEXP _arrow_LargeListArray__value_type(SEXP array_sexp){
 	Rf_error("Cannot call LargeListArray__value_type(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -367,14 +367,14 @@ RcppExport SEXP _arrow_LargeListArray__value_type(SEXP array_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Array> ListArray__values(const std::shared_ptr<arrow::ListArray>& array);
-RcppExport SEXP _arrow_ListArray__values(SEXP array_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ListArray>&>::type array(array_sexp);
-	return Rcpp::wrap(ListArray__values(array));
-END_RCPP
+extern "C" SEXP _arrow_ListArray__values(SEXP array_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ListArray>&>::type array(array_sexp);
+	return cpp11::as_sexp(ListArray__values(array));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ListArray__values(SEXP array_sexp){
+extern "C" SEXP _arrow_ListArray__values(SEXP array_sexp){
 	Rf_error("Cannot call ListArray__values(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -382,14 +382,14 @@ RcppExport SEXP _arrow_ListArray__values(SEXP array_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Array> LargeListArray__values(const std::shared_ptr<arrow::LargeListArray>& array);
-RcppExport SEXP _arrow_LargeListArray__values(SEXP array_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::LargeListArray>&>::type array(array_sexp);
-	return Rcpp::wrap(LargeListArray__values(array));
-END_RCPP
+extern "C" SEXP _arrow_LargeListArray__values(SEXP array_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::LargeListArray>&>::type array(array_sexp);
+	return cpp11::as_sexp(LargeListArray__values(array));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_LargeListArray__values(SEXP array_sexp){
+extern "C" SEXP _arrow_LargeListArray__values(SEXP array_sexp){
 	Rf_error("Cannot call LargeListArray__values(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -397,15 +397,15 @@ RcppExport SEXP _arrow_LargeListArray__values(SEXP array_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int32_t ListArray__value_length(const std::shared_ptr<arrow::ListArray>& array, int64_t i);
-RcppExport SEXP _arrow_ListArray__value_length(SEXP array_sexp, SEXP i_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ListArray>&>::type array(array_sexp);
-	Rcpp::traits::input_parameter<int64_t>::type i(i_sexp);
-	return Rcpp::wrap(ListArray__value_length(array, i));
-END_RCPP
+extern "C" SEXP _arrow_ListArray__value_length(SEXP array_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ListArray>&>::type array(array_sexp);
+	arrow::r::Input<int64_t>::type i(i_sexp);
+	return cpp11::as_sexp(ListArray__value_length(array, i));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ListArray__value_length(SEXP array_sexp, SEXP i_sexp){
+extern "C" SEXP _arrow_ListArray__value_length(SEXP array_sexp, SEXP i_sexp){
 	Rf_error("Cannot call ListArray__value_length(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -413,15 +413,15 @@ RcppExport SEXP _arrow_ListArray__value_length(SEXP array_sexp, SEXP i_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int64_t LargeListArray__value_length(const std::shared_ptr<arrow::LargeListArray>& array, int64_t i);
-RcppExport SEXP _arrow_LargeListArray__value_length(SEXP array_sexp, SEXP i_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::LargeListArray>&>::type array(array_sexp);
-	Rcpp::traits::input_parameter<int64_t>::type i(i_sexp);
-	return Rcpp::wrap(LargeListArray__value_length(array, i));
-END_RCPP
+extern "C" SEXP _arrow_LargeListArray__value_length(SEXP array_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::LargeListArray>&>::type array(array_sexp);
+	arrow::r::Input<int64_t>::type i(i_sexp);
+	return cpp11::as_sexp(LargeListArray__value_length(array, i));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_LargeListArray__value_length(SEXP array_sexp, SEXP i_sexp){
+extern "C" SEXP _arrow_LargeListArray__value_length(SEXP array_sexp, SEXP i_sexp){
 	Rf_error("Cannot call LargeListArray__value_length(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -429,15 +429,15 @@ RcppExport SEXP _arrow_LargeListArray__value_length(SEXP array_sexp, SEXP i_sexp
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int64_t FixedSizeListArray__value_length(const std::shared_ptr<arrow::FixedSizeListArray>& array, int64_t i);
-RcppExport SEXP _arrow_FixedSizeListArray__value_length(SEXP array_sexp, SEXP i_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::FixedSizeListArray>&>::type array(array_sexp);
-	Rcpp::traits::input_parameter<int64_t>::type i(i_sexp);
-	return Rcpp::wrap(FixedSizeListArray__value_length(array, i));
-END_RCPP
+extern "C" SEXP _arrow_FixedSizeListArray__value_length(SEXP array_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::FixedSizeListArray>&>::type array(array_sexp);
+	arrow::r::Input<int64_t>::type i(i_sexp);
+	return cpp11::as_sexp(FixedSizeListArray__value_length(array, i));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_FixedSizeListArray__value_length(SEXP array_sexp, SEXP i_sexp){
+extern "C" SEXP _arrow_FixedSizeListArray__value_length(SEXP array_sexp, SEXP i_sexp){
 	Rf_error("Cannot call FixedSizeListArray__value_length(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -445,15 +445,15 @@ RcppExport SEXP _arrow_FixedSizeListArray__value_length(SEXP array_sexp, SEXP i_
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int32_t ListArray__value_offset(const std::shared_ptr<arrow::ListArray>& array, int64_t i);
-RcppExport SEXP _arrow_ListArray__value_offset(SEXP array_sexp, SEXP i_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ListArray>&>::type array(array_sexp);
-	Rcpp::traits::input_parameter<int64_t>::type i(i_sexp);
-	return Rcpp::wrap(ListArray__value_offset(array, i));
-END_RCPP
+extern "C" SEXP _arrow_ListArray__value_offset(SEXP array_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ListArray>&>::type array(array_sexp);
+	arrow::r::Input<int64_t>::type i(i_sexp);
+	return cpp11::as_sexp(ListArray__value_offset(array, i));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ListArray__value_offset(SEXP array_sexp, SEXP i_sexp){
+extern "C" SEXP _arrow_ListArray__value_offset(SEXP array_sexp, SEXP i_sexp){
 	Rf_error("Cannot call ListArray__value_offset(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -461,15 +461,15 @@ RcppExport SEXP _arrow_ListArray__value_offset(SEXP array_sexp, SEXP i_sexp){
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int64_t LargeListArray__value_offset(const std::shared_ptr<arrow::LargeListArray>& array, int64_t i);
-RcppExport SEXP _arrow_LargeListArray__value_offset(SEXP array_sexp, SEXP i_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::LargeListArray>&>::type array(array_sexp);
-	Rcpp::traits::input_parameter<int64_t>::type i(i_sexp);
-	return Rcpp::wrap(LargeListArray__value_offset(array, i));
-END_RCPP
+extern "C" SEXP _arrow_LargeListArray__value_offset(SEXP array_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::LargeListArray>&>::type array(array_sexp);
+	arrow::r::Input<int64_t>::type i(i_sexp);
+	return cpp11::as_sexp(LargeListArray__value_offset(array, i));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_LargeListArray__value_offset(SEXP array_sexp, SEXP i_sexp){
+extern "C" SEXP _arrow_LargeListArray__value_offset(SEXP array_sexp, SEXP i_sexp){
 	Rf_error("Cannot call LargeListArray__value_offset(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -477,45 +477,45 @@ RcppExport SEXP _arrow_LargeListArray__value_offset(SEXP array_sexp, SEXP i_sexp
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int64_t FixedSizeListArray__value_offset(const std::shared_ptr<arrow::FixedSizeListArray>& array, int64_t i);
-RcppExport SEXP _arrow_FixedSizeListArray__value_offset(SEXP array_sexp, SEXP i_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::FixedSizeListArray>&>::type array(array_sexp);
-	Rcpp::traits::input_parameter<int64_t>::type i(i_sexp);
-	return Rcpp::wrap(FixedSizeListArray__value_offset(array, i));
-END_RCPP
+extern "C" SEXP _arrow_FixedSizeListArray__value_offset(SEXP array_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::FixedSizeListArray>&>::type array(array_sexp);
+	arrow::r::Input<int64_t>::type i(i_sexp);
+	return cpp11::as_sexp(FixedSizeListArray__value_offset(array, i));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_FixedSizeListArray__value_offset(SEXP array_sexp, SEXP i_sexp){
+extern "C" SEXP _arrow_FixedSizeListArray__value_offset(SEXP array_sexp, SEXP i_sexp){
 	Rf_error("Cannot call FixedSizeListArray__value_offset(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
-Rcpp::IntegerVector ListArray__raw_value_offsets(const std::shared_ptr<arrow::ListArray>& array);
-RcppExport SEXP _arrow_ListArray__raw_value_offsets(SEXP array_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ListArray>&>::type array(array_sexp);
-	return Rcpp::wrap(ListArray__raw_value_offsets(array));
-END_RCPP
+cpp11::writable::integers ListArray__raw_value_offsets(const std::shared_ptr<arrow::ListArray>& array);
+extern "C" SEXP _arrow_ListArray__raw_value_offsets(SEXP array_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ListArray>&>::type array(array_sexp);
+	return cpp11::as_sexp(ListArray__raw_value_offsets(array));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ListArray__raw_value_offsets(SEXP array_sexp){
+extern "C" SEXP _arrow_ListArray__raw_value_offsets(SEXP array_sexp){
 	Rf_error("Cannot call ListArray__raw_value_offsets(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // array.cpp
 #if defined(ARROW_R_WITH_ARROW)
-Rcpp::IntegerVector LargeListArray__raw_value_offsets(const std::shared_ptr<arrow::LargeListArray>& array);
-RcppExport SEXP _arrow_LargeListArray__raw_value_offsets(SEXP array_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::LargeListArray>&>::type array(array_sexp);
-	return Rcpp::wrap(LargeListArray__raw_value_offsets(array));
-END_RCPP
+cpp11::writable::integers LargeListArray__raw_value_offsets(const std::shared_ptr<arrow::LargeListArray>& array);
+extern "C" SEXP _arrow_LargeListArray__raw_value_offsets(SEXP array_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::LargeListArray>&>::type array(array_sexp);
+	return cpp11::as_sexp(LargeListArray__raw_value_offsets(array));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_LargeListArray__raw_value_offsets(SEXP array_sexp){
+extern "C" SEXP _arrow_LargeListArray__raw_value_offsets(SEXP array_sexp){
 	Rf_error("Cannot call LargeListArray__raw_value_offsets(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -523,14 +523,14 @@ RcppExport SEXP _arrow_LargeListArray__raw_value_offsets(SEXP array_sexp){
 // array_from_vector.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Array__infer_type(SEXP x);
-RcppExport SEXP _arrow_Array__infer_type(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<SEXP>::type x(x_sexp);
-	return Rcpp::wrap(Array__infer_type(x));
-END_RCPP
+extern "C" SEXP _arrow_Array__infer_type(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<SEXP>::type x(x_sexp);
+	return cpp11::as_sexp(Array__infer_type(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__infer_type(SEXP x_sexp){
+extern "C" SEXP _arrow_Array__infer_type(SEXP x_sexp){
 	Rf_error("Cannot call Array__infer_type(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -538,31 +538,31 @@ RcppExport SEXP _arrow_Array__infer_type(SEXP x_sexp){
 // array_from_vector.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Array> Array__from_vector(SEXP x, SEXP s_type);
-RcppExport SEXP _arrow_Array__from_vector(SEXP x_sexp, SEXP s_type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<SEXP>::type x(x_sexp);
-	Rcpp::traits::input_parameter<SEXP>::type s_type(s_type_sexp);
-	return Rcpp::wrap(Array__from_vector(x, s_type));
-END_RCPP
+extern "C" SEXP _arrow_Array__from_vector(SEXP x_sexp, SEXP s_type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<SEXP>::type x(x_sexp);
+	arrow::r::Input<SEXP>::type s_type(s_type_sexp);
+	return cpp11::as_sexp(Array__from_vector(x, s_type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__from_vector(SEXP x_sexp, SEXP s_type_sexp){
+extern "C" SEXP _arrow_Array__from_vector(SEXP x_sexp, SEXP s_type_sexp){
 	Rf_error("Cannot call Array__from_vector(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // array_from_vector.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::ChunkedArray> ChunkedArray__from_list(Rcpp::List chunks, SEXP s_type);
-RcppExport SEXP _arrow_ChunkedArray__from_list(SEXP chunks_sexp, SEXP s_type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<Rcpp::List>::type chunks(chunks_sexp);
-	Rcpp::traits::input_parameter<SEXP>::type s_type(s_type_sexp);
-	return Rcpp::wrap(ChunkedArray__from_list(chunks, s_type));
-END_RCPP
+std::shared_ptr<arrow::ChunkedArray> ChunkedArray__from_list(cpp11::list chunks, SEXP s_type);
+extern "C" SEXP _arrow_ChunkedArray__from_list(SEXP chunks_sexp, SEXP s_type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<cpp11::list>::type chunks(chunks_sexp);
+	arrow::r::Input<SEXP>::type s_type(s_type_sexp);
+	return cpp11::as_sexp(ChunkedArray__from_list(chunks, s_type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ChunkedArray__from_list(SEXP chunks_sexp, SEXP s_type_sexp){
+extern "C" SEXP _arrow_ChunkedArray__from_list(SEXP chunks_sexp, SEXP s_type_sexp){
 	Rf_error("Cannot call ChunkedArray__from_list(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -570,16 +570,16 @@ RcppExport SEXP _arrow_ChunkedArray__from_list(SEXP chunks_sexp, SEXP s_type_sex
 // array_from_vector.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Array> DictionaryArray__FromArrays(const std::shared_ptr<arrow::DataType>& type, const std::shared_ptr<arrow::Array>& indices, const std::shared_ptr<arrow::Array>& dict);
-RcppExport SEXP _arrow_DictionaryArray__FromArrays(SEXP type_sexp, SEXP indices_sexp, SEXP dict_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DataType>&>::type type(type_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type indices(indices_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type dict(dict_sexp);
-	return Rcpp::wrap(DictionaryArray__FromArrays(type, indices, dict));
-END_RCPP
+extern "C" SEXP _arrow_DictionaryArray__FromArrays(SEXP type_sexp, SEXP indices_sexp, SEXP dict_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::DataType>&>::type type(type_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type indices(indices_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type dict(dict_sexp);
+	return cpp11::as_sexp(DictionaryArray__FromArrays(type, indices, dict));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_DictionaryArray__FromArrays(SEXP type_sexp, SEXP indices_sexp, SEXP dict_sexp){
+extern "C" SEXP _arrow_DictionaryArray__FromArrays(SEXP type_sexp, SEXP indices_sexp, SEXP dict_sexp){
 	Rf_error("Cannot call DictionaryArray__FromArrays(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -587,14 +587,14 @@ RcppExport SEXP _arrow_DictionaryArray__FromArrays(SEXP type_sexp, SEXP indices_
 // array_to_vector.cpp
 #if defined(ARROW_R_WITH_ARROW)
 SEXP Array__as_vector(const std::shared_ptr<arrow::Array>& array);
-RcppExport SEXP _arrow_Array__as_vector(SEXP array_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type array(array_sexp);
-	return Rcpp::wrap(Array__as_vector(array));
-END_RCPP
+extern "C" SEXP _arrow_Array__as_vector(SEXP array_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type array(array_sexp);
+	return cpp11::as_sexp(Array__as_vector(array));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__as_vector(SEXP array_sexp){
+extern "C" SEXP _arrow_Array__as_vector(SEXP array_sexp){
 	Rf_error("Cannot call Array__as_vector(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -602,46 +602,46 @@ RcppExport SEXP _arrow_Array__as_vector(SEXP array_sexp){
 // array_to_vector.cpp
 #if defined(ARROW_R_WITH_ARROW)
 SEXP ChunkedArray__as_vector(const std::shared_ptr<arrow::ChunkedArray>& chunked_array);
-RcppExport SEXP _arrow_ChunkedArray__as_vector(SEXP chunked_array_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
-	return Rcpp::wrap(ChunkedArray__as_vector(chunked_array));
-END_RCPP
+extern "C" SEXP _arrow_ChunkedArray__as_vector(SEXP chunked_array_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
+	return cpp11::as_sexp(ChunkedArray__as_vector(chunked_array));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ChunkedArray__as_vector(SEXP chunked_array_sexp){
+extern "C" SEXP _arrow_ChunkedArray__as_vector(SEXP chunked_array_sexp){
 	Rf_error("Cannot call ChunkedArray__as_vector(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // array_to_vector.cpp
 #if defined(ARROW_R_WITH_ARROW)
-Rcpp::List RecordBatch__to_dataframe(const std::shared_ptr<arrow::RecordBatch>& batch, bool use_threads);
-RcppExport SEXP _arrow_RecordBatch__to_dataframe(SEXP batch_sexp, SEXP use_threads_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
-	Rcpp::traits::input_parameter<bool>::type use_threads(use_threads_sexp);
-	return Rcpp::wrap(RecordBatch__to_dataframe(batch, use_threads));
-END_RCPP
+cpp11::writable::list RecordBatch__to_dataframe(const std::shared_ptr<arrow::RecordBatch>& batch, bool use_threads);
+extern "C" SEXP _arrow_RecordBatch__to_dataframe(SEXP batch_sexp, SEXP use_threads_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
+	arrow::r::Input<bool>::type use_threads(use_threads_sexp);
+	return cpp11::as_sexp(RecordBatch__to_dataframe(batch, use_threads));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_RecordBatch__to_dataframe(SEXP batch_sexp, SEXP use_threads_sexp){
+extern "C" SEXP _arrow_RecordBatch__to_dataframe(SEXP batch_sexp, SEXP use_threads_sexp){
 	Rf_error("Cannot call RecordBatch__to_dataframe(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // array_to_vector.cpp
 #if defined(ARROW_R_WITH_ARROW)
-Rcpp::List Table__to_dataframe(const std::shared_ptr<arrow::Table>& table, bool use_threads);
-RcppExport SEXP _arrow_Table__to_dataframe(SEXP table_sexp, SEXP use_threads_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
-	Rcpp::traits::input_parameter<bool>::type use_threads(use_threads_sexp);
-	return Rcpp::wrap(Table__to_dataframe(table, use_threads));
-END_RCPP
+cpp11::writable::list Table__to_dataframe(const std::shared_ptr<arrow::Table>& table, bool use_threads);
+extern "C" SEXP _arrow_Table__to_dataframe(SEXP table_sexp, SEXP use_threads_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
+	arrow::r::Input<bool>::type use_threads(use_threads_sexp);
+	return cpp11::as_sexp(Table__to_dataframe(table, use_threads));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Table__to_dataframe(SEXP table_sexp, SEXP use_threads_sexp){
+extern "C" SEXP _arrow_Table__to_dataframe(SEXP table_sexp, SEXP use_threads_sexp){
 	Rf_error("Cannot call Table__to_dataframe(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -649,14 +649,14 @@ RcppExport SEXP _arrow_Table__to_dataframe(SEXP table_sexp, SEXP use_threads_sex
 // arraydata.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> ArrayData__get_type(const std::shared_ptr<arrow::ArrayData>& x);
-RcppExport SEXP _arrow_ArrayData__get_type(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ArrayData>&>::type x(x_sexp);
-	return Rcpp::wrap(ArrayData__get_type(x));
-END_RCPP
+extern "C" SEXP _arrow_ArrayData__get_type(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ArrayData>&>::type x(x_sexp);
+	return cpp11::as_sexp(ArrayData__get_type(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ArrayData__get_type(SEXP x_sexp){
+extern "C" SEXP _arrow_ArrayData__get_type(SEXP x_sexp){
 	Rf_error("Cannot call ArrayData__get_type(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -664,14 +664,14 @@ RcppExport SEXP _arrow_ArrayData__get_type(SEXP x_sexp){
 // arraydata.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int ArrayData__get_length(const std::shared_ptr<arrow::ArrayData>& x);
-RcppExport SEXP _arrow_ArrayData__get_length(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ArrayData>&>::type x(x_sexp);
-	return Rcpp::wrap(ArrayData__get_length(x));
-END_RCPP
+extern "C" SEXP _arrow_ArrayData__get_length(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ArrayData>&>::type x(x_sexp);
+	return cpp11::as_sexp(ArrayData__get_length(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ArrayData__get_length(SEXP x_sexp){
+extern "C" SEXP _arrow_ArrayData__get_length(SEXP x_sexp){
 	Rf_error("Cannot call ArrayData__get_length(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -679,14 +679,14 @@ RcppExport SEXP _arrow_ArrayData__get_length(SEXP x_sexp){
 // arraydata.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int ArrayData__get_null_count(const std::shared_ptr<arrow::ArrayData>& x);
-RcppExport SEXP _arrow_ArrayData__get_null_count(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ArrayData>&>::type x(x_sexp);
-	return Rcpp::wrap(ArrayData__get_null_count(x));
-END_RCPP
+extern "C" SEXP _arrow_ArrayData__get_null_count(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ArrayData>&>::type x(x_sexp);
+	return cpp11::as_sexp(ArrayData__get_null_count(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ArrayData__get_null_count(SEXP x_sexp){
+extern "C" SEXP _arrow_ArrayData__get_null_count(SEXP x_sexp){
 	Rf_error("Cannot call ArrayData__get_null_count(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -694,29 +694,29 @@ RcppExport SEXP _arrow_ArrayData__get_null_count(SEXP x_sexp){
 // arraydata.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int ArrayData__get_offset(const std::shared_ptr<arrow::ArrayData>& x);
-RcppExport SEXP _arrow_ArrayData__get_offset(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ArrayData>&>::type x(x_sexp);
-	return Rcpp::wrap(ArrayData__get_offset(x));
-END_RCPP
+extern "C" SEXP _arrow_ArrayData__get_offset(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ArrayData>&>::type x(x_sexp);
+	return cpp11::as_sexp(ArrayData__get_offset(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ArrayData__get_offset(SEXP x_sexp){
+extern "C" SEXP _arrow_ArrayData__get_offset(SEXP x_sexp){
 	Rf_error("Cannot call ArrayData__get_offset(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // arraydata.cpp
 #if defined(ARROW_R_WITH_ARROW)
-List ArrayData__buffers(const std::shared_ptr<arrow::ArrayData>& x);
-RcppExport SEXP _arrow_ArrayData__buffers(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ArrayData>&>::type x(x_sexp);
-	return Rcpp::wrap(ArrayData__buffers(x));
-END_RCPP
+cpp11::list ArrayData__buffers(const std::shared_ptr<arrow::ArrayData>& x);
+extern "C" SEXP _arrow_ArrayData__buffers(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ArrayData>&>::type x(x_sexp);
+	return cpp11::as_sexp(ArrayData__buffers(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ArrayData__buffers(SEXP x_sexp){
+extern "C" SEXP _arrow_ArrayData__buffers(SEXP x_sexp){
 	Rf_error("Cannot call ArrayData__buffers(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -724,14 +724,14 @@ RcppExport SEXP _arrow_ArrayData__buffers(SEXP x_sexp){
 // buffer.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool Buffer__is_mutable(const std::shared_ptr<arrow::Buffer>& buffer);
-RcppExport SEXP _arrow_Buffer__is_mutable(SEXP buffer_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Buffer>&>::type buffer(buffer_sexp);
-	return Rcpp::wrap(Buffer__is_mutable(buffer));
-END_RCPP
+extern "C" SEXP _arrow_Buffer__is_mutable(SEXP buffer_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Buffer>&>::type buffer(buffer_sexp);
+	return cpp11::as_sexp(Buffer__is_mutable(buffer));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Buffer__is_mutable(SEXP buffer_sexp){
+extern "C" SEXP _arrow_Buffer__is_mutable(SEXP buffer_sexp){
 	Rf_error("Cannot call Buffer__is_mutable(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -739,15 +739,15 @@ RcppExport SEXP _arrow_Buffer__is_mutable(SEXP buffer_sexp){
 // buffer.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void Buffer__ZeroPadding(const std::shared_ptr<arrow::Buffer>& buffer);
-RcppExport SEXP _arrow_Buffer__ZeroPadding(SEXP buffer_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Buffer>&>::type buffer(buffer_sexp);
+extern "C" SEXP _arrow_Buffer__ZeroPadding(SEXP buffer_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Buffer>&>::type buffer(buffer_sexp);
 	Buffer__ZeroPadding(buffer);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Buffer__ZeroPadding(SEXP buffer_sexp){
+extern "C" SEXP _arrow_Buffer__ZeroPadding(SEXP buffer_sexp){
 	Rf_error("Cannot call Buffer__ZeroPadding(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -755,14 +755,14 @@ RcppExport SEXP _arrow_Buffer__ZeroPadding(SEXP buffer_sexp){
 // buffer.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int64_t Buffer__capacity(const std::shared_ptr<arrow::Buffer>& buffer);
-RcppExport SEXP _arrow_Buffer__capacity(SEXP buffer_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Buffer>&>::type buffer(buffer_sexp);
-	return Rcpp::wrap(Buffer__capacity(buffer));
-END_RCPP
+extern "C" SEXP _arrow_Buffer__capacity(SEXP buffer_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Buffer>&>::type buffer(buffer_sexp);
+	return cpp11::as_sexp(Buffer__capacity(buffer));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Buffer__capacity(SEXP buffer_sexp){
+extern "C" SEXP _arrow_Buffer__capacity(SEXP buffer_sexp){
 	Rf_error("Cannot call Buffer__capacity(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -770,14 +770,14 @@ RcppExport SEXP _arrow_Buffer__capacity(SEXP buffer_sexp){
 // buffer.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int64_t Buffer__size(const std::shared_ptr<arrow::Buffer>& buffer);
-RcppExport SEXP _arrow_Buffer__size(SEXP buffer_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Buffer>&>::type buffer(buffer_sexp);
-	return Rcpp::wrap(Buffer__size(buffer));
-END_RCPP
+extern "C" SEXP _arrow_Buffer__size(SEXP buffer_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Buffer>&>::type buffer(buffer_sexp);
+	return cpp11::as_sexp(Buffer__size(buffer));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Buffer__size(SEXP buffer_sexp){
+extern "C" SEXP _arrow_Buffer__size(SEXP buffer_sexp){
 	Rf_error("Cannot call Buffer__size(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -785,29 +785,29 @@ RcppExport SEXP _arrow_Buffer__size(SEXP buffer_sexp){
 // buffer.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Buffer> r___RBuffer__initialize(SEXP x);
-RcppExport SEXP _arrow_r___RBuffer__initialize(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<SEXP>::type x(x_sexp);
-	return Rcpp::wrap(r___RBuffer__initialize(x));
-END_RCPP
+extern "C" SEXP _arrow_r___RBuffer__initialize(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<SEXP>::type x(x_sexp);
+	return cpp11::as_sexp(r___RBuffer__initialize(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_r___RBuffer__initialize(SEXP x_sexp){
+extern "C" SEXP _arrow_r___RBuffer__initialize(SEXP x_sexp){
 	Rf_error("Cannot call r___RBuffer__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // buffer.cpp
 #if defined(ARROW_R_WITH_ARROW)
-Rcpp::RawVector Buffer__data(const std::shared_ptr<arrow::Buffer>& buffer);
-RcppExport SEXP _arrow_Buffer__data(SEXP buffer_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Buffer>&>::type buffer(buffer_sexp);
-	return Rcpp::wrap(Buffer__data(buffer));
-END_RCPP
+cpp11::writable::raws Buffer__data(const std::shared_ptr<arrow::Buffer>& buffer);
+extern "C" SEXP _arrow_Buffer__data(SEXP buffer_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Buffer>&>::type buffer(buffer_sexp);
+	return cpp11::as_sexp(Buffer__data(buffer));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Buffer__data(SEXP buffer_sexp){
+extern "C" SEXP _arrow_Buffer__data(SEXP buffer_sexp){
 	Rf_error("Cannot call Buffer__data(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -815,15 +815,15 @@ RcppExport SEXP _arrow_Buffer__data(SEXP buffer_sexp){
 // buffer.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool Buffer__Equals(const std::shared_ptr<arrow::Buffer>& x, const std::shared_ptr<arrow::Buffer>& y);
-RcppExport SEXP _arrow_Buffer__Equals(SEXP x_sexp, SEXP y_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Buffer>&>::type x(x_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Buffer>&>::type y(y_sexp);
-	return Rcpp::wrap(Buffer__Equals(x, y));
-END_RCPP
+extern "C" SEXP _arrow_Buffer__Equals(SEXP x_sexp, SEXP y_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Buffer>&>::type x(x_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Buffer>&>::type y(y_sexp);
+	return cpp11::as_sexp(Buffer__Equals(x, y));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Buffer__Equals(SEXP x_sexp, SEXP y_sexp){
+extern "C" SEXP _arrow_Buffer__Equals(SEXP x_sexp, SEXP y_sexp){
 	Rf_error("Cannot call Buffer__Equals(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -831,14 +831,14 @@ RcppExport SEXP _arrow_Buffer__Equals(SEXP x_sexp, SEXP y_sexp){
 // chunkedarray.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int ChunkedArray__length(const std::shared_ptr<arrow::ChunkedArray>& chunked_array);
-RcppExport SEXP _arrow_ChunkedArray__length(SEXP chunked_array_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
-	return Rcpp::wrap(ChunkedArray__length(chunked_array));
-END_RCPP
+extern "C" SEXP _arrow_ChunkedArray__length(SEXP chunked_array_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
+	return cpp11::as_sexp(ChunkedArray__length(chunked_array));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ChunkedArray__length(SEXP chunked_array_sexp){
+extern "C" SEXP _arrow_ChunkedArray__length(SEXP chunked_array_sexp){
 	Rf_error("Cannot call ChunkedArray__length(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -846,14 +846,14 @@ RcppExport SEXP _arrow_ChunkedArray__length(SEXP chunked_array_sexp){
 // chunkedarray.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int ChunkedArray__null_count(const std::shared_ptr<arrow::ChunkedArray>& chunked_array);
-RcppExport SEXP _arrow_ChunkedArray__null_count(SEXP chunked_array_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
-	return Rcpp::wrap(ChunkedArray__null_count(chunked_array));
-END_RCPP
+extern "C" SEXP _arrow_ChunkedArray__null_count(SEXP chunked_array_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
+	return cpp11::as_sexp(ChunkedArray__null_count(chunked_array));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ChunkedArray__null_count(SEXP chunked_array_sexp){
+extern "C" SEXP _arrow_ChunkedArray__null_count(SEXP chunked_array_sexp){
 	Rf_error("Cannot call ChunkedArray__null_count(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -861,14 +861,14 @@ RcppExport SEXP _arrow_ChunkedArray__null_count(SEXP chunked_array_sexp){
 // chunkedarray.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int ChunkedArray__num_chunks(const std::shared_ptr<arrow::ChunkedArray>& chunked_array);
-RcppExport SEXP _arrow_ChunkedArray__num_chunks(SEXP chunked_array_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
-	return Rcpp::wrap(ChunkedArray__num_chunks(chunked_array));
-END_RCPP
+extern "C" SEXP _arrow_ChunkedArray__num_chunks(SEXP chunked_array_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
+	return cpp11::as_sexp(ChunkedArray__num_chunks(chunked_array));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ChunkedArray__num_chunks(SEXP chunked_array_sexp){
+extern "C" SEXP _arrow_ChunkedArray__num_chunks(SEXP chunked_array_sexp){
 	Rf_error("Cannot call ChunkedArray__num_chunks(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -876,30 +876,30 @@ RcppExport SEXP _arrow_ChunkedArray__num_chunks(SEXP chunked_array_sexp){
 // chunkedarray.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Array> ChunkedArray__chunk(const std::shared_ptr<arrow::ChunkedArray>& chunked_array, int i);
-RcppExport SEXP _arrow_ChunkedArray__chunk(SEXP chunked_array_sexp, SEXP i_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
-	Rcpp::traits::input_parameter<int>::type i(i_sexp);
-	return Rcpp::wrap(ChunkedArray__chunk(chunked_array, i));
-END_RCPP
+extern "C" SEXP _arrow_ChunkedArray__chunk(SEXP chunked_array_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
+	arrow::r::Input<int>::type i(i_sexp);
+	return cpp11::as_sexp(ChunkedArray__chunk(chunked_array, i));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ChunkedArray__chunk(SEXP chunked_array_sexp, SEXP i_sexp){
+extern "C" SEXP _arrow_ChunkedArray__chunk(SEXP chunked_array_sexp, SEXP i_sexp){
 	Rf_error("Cannot call ChunkedArray__chunk(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // chunkedarray.cpp
 #if defined(ARROW_R_WITH_ARROW)
-List ChunkedArray__chunks(const std::shared_ptr<arrow::ChunkedArray>& chunked_array);
-RcppExport SEXP _arrow_ChunkedArray__chunks(SEXP chunked_array_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
-	return Rcpp::wrap(ChunkedArray__chunks(chunked_array));
-END_RCPP
+cpp11::list ChunkedArray__chunks(const std::shared_ptr<arrow::ChunkedArray>& chunked_array);
+extern "C" SEXP _arrow_ChunkedArray__chunks(SEXP chunked_array_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
+	return cpp11::as_sexp(ChunkedArray__chunks(chunked_array));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ChunkedArray__chunks(SEXP chunked_array_sexp){
+extern "C" SEXP _arrow_ChunkedArray__chunks(SEXP chunked_array_sexp){
 	Rf_error("Cannot call ChunkedArray__chunks(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -907,47 +907,47 @@ RcppExport SEXP _arrow_ChunkedArray__chunks(SEXP chunked_array_sexp){
 // chunkedarray.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> ChunkedArray__type(const std::shared_ptr<arrow::ChunkedArray>& chunked_array);
-RcppExport SEXP _arrow_ChunkedArray__type(SEXP chunked_array_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
-	return Rcpp::wrap(ChunkedArray__type(chunked_array));
-END_RCPP
+extern "C" SEXP _arrow_ChunkedArray__type(SEXP chunked_array_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
+	return cpp11::as_sexp(ChunkedArray__type(chunked_array));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ChunkedArray__type(SEXP chunked_array_sexp){
+extern "C" SEXP _arrow_ChunkedArray__type(SEXP chunked_array_sexp){
 	Rf_error("Cannot call ChunkedArray__type(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // chunkedarray.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::ChunkedArray> ChunkedArray__Slice1(const std::shared_ptr<arrow::ChunkedArray>& chunked_array, int offset);
-RcppExport SEXP _arrow_ChunkedArray__Slice1(SEXP chunked_array_sexp, SEXP offset_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
-	Rcpp::traits::input_parameter<int>::type offset(offset_sexp);
-	return Rcpp::wrap(ChunkedArray__Slice1(chunked_array, offset));
-END_RCPP
+std::shared_ptr<arrow::ChunkedArray> ChunkedArray__Slice1(const std::shared_ptr<arrow::ChunkedArray>& chunked_array, R_xlen_t offset);
+extern "C" SEXP _arrow_ChunkedArray__Slice1(SEXP chunked_array_sexp, SEXP offset_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
+	arrow::r::Input<R_xlen_t>::type offset(offset_sexp);
+	return cpp11::as_sexp(ChunkedArray__Slice1(chunked_array, offset));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ChunkedArray__Slice1(SEXP chunked_array_sexp, SEXP offset_sexp){
+extern "C" SEXP _arrow_ChunkedArray__Slice1(SEXP chunked_array_sexp, SEXP offset_sexp){
 	Rf_error("Cannot call ChunkedArray__Slice1(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // chunkedarray.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::ChunkedArray> ChunkedArray__Slice2(const std::shared_ptr<arrow::ChunkedArray>& chunked_array, int offset, int length);
-RcppExport SEXP _arrow_ChunkedArray__Slice2(SEXP chunked_array_sexp, SEXP offset_sexp, SEXP length_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
-	Rcpp::traits::input_parameter<int>::type offset(offset_sexp);
-	Rcpp::traits::input_parameter<int>::type length(length_sexp);
-	return Rcpp::wrap(ChunkedArray__Slice2(chunked_array, offset, length));
-END_RCPP
+std::shared_ptr<arrow::ChunkedArray> ChunkedArray__Slice2(const std::shared_ptr<arrow::ChunkedArray>& chunked_array, R_xlen_t offset, R_xlen_t length);
+extern "C" SEXP _arrow_ChunkedArray__Slice2(SEXP chunked_array_sexp, SEXP offset_sexp, SEXP length_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
+	arrow::r::Input<R_xlen_t>::type offset(offset_sexp);
+	arrow::r::Input<R_xlen_t>::type length(length_sexp);
+	return cpp11::as_sexp(ChunkedArray__Slice2(chunked_array, offset, length));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ChunkedArray__Slice2(SEXP chunked_array_sexp, SEXP offset_sexp, SEXP length_sexp){
+extern "C" SEXP _arrow_ChunkedArray__Slice2(SEXP chunked_array_sexp, SEXP offset_sexp, SEXP length_sexp){
 	Rf_error("Cannot call ChunkedArray__Slice2(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -955,15 +955,15 @@ RcppExport SEXP _arrow_ChunkedArray__Slice2(SEXP chunked_array_sexp, SEXP offset
 // chunkedarray.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::ChunkedArray> ChunkedArray__View(const std::shared_ptr<arrow::ChunkedArray>& array, const std::shared_ptr<arrow::DataType>& type);
-RcppExport SEXP _arrow_ChunkedArray__View(SEXP array_sexp, SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ChunkedArray>&>::type array(array_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DataType>&>::type type(type_sexp);
-	return Rcpp::wrap(ChunkedArray__View(array, type));
-END_RCPP
+extern "C" SEXP _arrow_ChunkedArray__View(SEXP array_sexp, SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ChunkedArray>&>::type array(array_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::DataType>&>::type type(type_sexp);
+	return cpp11::as_sexp(ChunkedArray__View(array, type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ChunkedArray__View(SEXP array_sexp, SEXP type_sexp){
+extern "C" SEXP _arrow_ChunkedArray__View(SEXP array_sexp, SEXP type_sexp){
 	Rf_error("Cannot call ChunkedArray__View(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -971,15 +971,15 @@ RcppExport SEXP _arrow_ChunkedArray__View(SEXP array_sexp, SEXP type_sexp){
 // chunkedarray.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void ChunkedArray__Validate(const std::shared_ptr<arrow::ChunkedArray>& chunked_array);
-RcppExport SEXP _arrow_ChunkedArray__Validate(SEXP chunked_array_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
+extern "C" SEXP _arrow_ChunkedArray__Validate(SEXP chunked_array_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
 	ChunkedArray__Validate(chunked_array);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ChunkedArray__Validate(SEXP chunked_array_sexp){
+extern "C" SEXP _arrow_ChunkedArray__Validate(SEXP chunked_array_sexp){
 	Rf_error("Cannot call ChunkedArray__Validate(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -987,15 +987,15 @@ RcppExport SEXP _arrow_ChunkedArray__Validate(SEXP chunked_array_sexp){
 // chunkedarray.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool ChunkedArray__Equals(const std::shared_ptr<arrow::ChunkedArray>& x, const std::shared_ptr<arrow::ChunkedArray>& y);
-RcppExport SEXP _arrow_ChunkedArray__Equals(SEXP x_sexp, SEXP y_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ChunkedArray>&>::type x(x_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ChunkedArray>&>::type y(y_sexp);
-	return Rcpp::wrap(ChunkedArray__Equals(x, y));
-END_RCPP
+extern "C" SEXP _arrow_ChunkedArray__Equals(SEXP x_sexp, SEXP y_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ChunkedArray>&>::type x(x_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::ChunkedArray>&>::type y(y_sexp);
+	return cpp11::as_sexp(ChunkedArray__Equals(x, y));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ChunkedArray__Equals(SEXP x_sexp, SEXP y_sexp){
+extern "C" SEXP _arrow_ChunkedArray__Equals(SEXP x_sexp, SEXP y_sexp){
 	Rf_error("Cannot call ChunkedArray__Equals(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1003,30 +1003,30 @@ RcppExport SEXP _arrow_ChunkedArray__Equals(SEXP x_sexp, SEXP y_sexp){
 // chunkedarray.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::string ChunkedArray__ToString(const std::shared_ptr<arrow::ChunkedArray>& x);
-RcppExport SEXP _arrow_ChunkedArray__ToString(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ChunkedArray>&>::type x(x_sexp);
-	return Rcpp::wrap(ChunkedArray__ToString(x));
-END_RCPP
+extern "C" SEXP _arrow_ChunkedArray__ToString(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ChunkedArray>&>::type x(x_sexp);
+	return cpp11::as_sexp(ChunkedArray__ToString(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ChunkedArray__ToString(SEXP x_sexp){
+extern "C" SEXP _arrow_ChunkedArray__ToString(SEXP x_sexp){
 	Rf_error("Cannot call ChunkedArray__ToString(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // compression.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::util::Codec> util___Codec__Create(arrow::Compression::type codec, int compression_level);
-RcppExport SEXP _arrow_util___Codec__Create(SEXP codec_sexp, SEXP compression_level_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<arrow::Compression::type>::type codec(codec_sexp);
-	Rcpp::traits::input_parameter<int>::type compression_level(compression_level_sexp);
-	return Rcpp::wrap(util___Codec__Create(codec, compression_level));
-END_RCPP
+std::shared_ptr<arrow::util::Codec> util___Codec__Create(arrow::Compression::type codec, R_xlen_t compression_level);
+extern "C" SEXP _arrow_util___Codec__Create(SEXP codec_sexp, SEXP compression_level_sexp){
+BEGIN_CPP11
+	arrow::r::Input<arrow::Compression::type>::type codec(codec_sexp);
+	arrow::r::Input<R_xlen_t>::type compression_level(compression_level_sexp);
+	return cpp11::as_sexp(util___Codec__Create(codec, compression_level));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_util___Codec__Create(SEXP codec_sexp, SEXP compression_level_sexp){
+extern "C" SEXP _arrow_util___Codec__Create(SEXP codec_sexp, SEXP compression_level_sexp){
 	Rf_error("Cannot call util___Codec__Create(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1034,14 +1034,14 @@ RcppExport SEXP _arrow_util___Codec__Create(SEXP codec_sexp, SEXP compression_le
 // compression.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::string util___Codec__name(const std::shared_ptr<arrow::util::Codec>& codec);
-RcppExport SEXP _arrow_util___Codec__name(SEXP codec_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::util::Codec>&>::type codec(codec_sexp);
-	return Rcpp::wrap(util___Codec__name(codec));
-END_RCPP
+extern "C" SEXP _arrow_util___Codec__name(SEXP codec_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::util::Codec>&>::type codec(codec_sexp);
+	return cpp11::as_sexp(util___Codec__name(codec));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_util___Codec__name(SEXP codec_sexp){
+extern "C" SEXP _arrow_util___Codec__name(SEXP codec_sexp){
 	Rf_error("Cannot call util___Codec__name(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1049,14 +1049,14 @@ RcppExport SEXP _arrow_util___Codec__name(SEXP codec_sexp){
 // compression.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool util___Codec__IsAvailable(arrow::Compression::type codec);
-RcppExport SEXP _arrow_util___Codec__IsAvailable(SEXP codec_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<arrow::Compression::type>::type codec(codec_sexp);
-	return Rcpp::wrap(util___Codec__IsAvailable(codec));
-END_RCPP
+extern "C" SEXP _arrow_util___Codec__IsAvailable(SEXP codec_sexp){
+BEGIN_CPP11
+	arrow::r::Input<arrow::Compression::type>::type codec(codec_sexp);
+	return cpp11::as_sexp(util___Codec__IsAvailable(codec));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_util___Codec__IsAvailable(SEXP codec_sexp){
+extern "C" SEXP _arrow_util___Codec__IsAvailable(SEXP codec_sexp){
 	Rf_error("Cannot call util___Codec__IsAvailable(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1064,15 +1064,15 @@ RcppExport SEXP _arrow_util___Codec__IsAvailable(SEXP codec_sexp){
 // compression.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::io::CompressedOutputStream> io___CompressedOutputStream__Make(const std::shared_ptr<arrow::util::Codec>& codec, const std::shared_ptr<arrow::io::OutputStream>& raw);
-RcppExport SEXP _arrow_io___CompressedOutputStream__Make(SEXP codec_sexp, SEXP raw_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::util::Codec>&>::type codec(codec_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::OutputStream>&>::type raw(raw_sexp);
-	return Rcpp::wrap(io___CompressedOutputStream__Make(codec, raw));
-END_RCPP
+extern "C" SEXP _arrow_io___CompressedOutputStream__Make(SEXP codec_sexp, SEXP raw_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::util::Codec>&>::type codec(codec_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::io::OutputStream>&>::type raw(raw_sexp);
+	return cpp11::as_sexp(io___CompressedOutputStream__Make(codec, raw));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___CompressedOutputStream__Make(SEXP codec_sexp, SEXP raw_sexp){
+extern "C" SEXP _arrow_io___CompressedOutputStream__Make(SEXP codec_sexp, SEXP raw_sexp){
 	Rf_error("Cannot call io___CompressedOutputStream__Make(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1080,15 +1080,15 @@ RcppExport SEXP _arrow_io___CompressedOutputStream__Make(SEXP codec_sexp, SEXP r
 // compression.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::io::CompressedInputStream> io___CompressedInputStream__Make(const std::shared_ptr<arrow::util::Codec>& codec, const std::shared_ptr<arrow::io::InputStream>& raw);
-RcppExport SEXP _arrow_io___CompressedInputStream__Make(SEXP codec_sexp, SEXP raw_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::util::Codec>&>::type codec(codec_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::InputStream>&>::type raw(raw_sexp);
-	return Rcpp::wrap(io___CompressedInputStream__Make(codec, raw));
-END_RCPP
+extern "C" SEXP _arrow_io___CompressedInputStream__Make(SEXP codec_sexp, SEXP raw_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::util::Codec>&>::type codec(codec_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::io::InputStream>&>::type raw(raw_sexp);
+	return cpp11::as_sexp(io___CompressedInputStream__Make(codec, raw));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___CompressedInputStream__Make(SEXP codec_sexp, SEXP raw_sexp){
+extern "C" SEXP _arrow_io___CompressedInputStream__Make(SEXP codec_sexp, SEXP raw_sexp){
 	Rf_error("Cannot call io___CompressedInputStream__Make(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1096,16 +1096,16 @@ RcppExport SEXP _arrow_io___CompressedInputStream__Make(SEXP codec_sexp, SEXP ra
 // compute.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::compute::CastOptions> compute___CastOptions__initialize(bool allow_int_overflow, bool allow_time_truncate, bool allow_float_truncate);
-RcppExport SEXP _arrow_compute___CastOptions__initialize(SEXP allow_int_overflow_sexp, SEXP allow_time_truncate_sexp, SEXP allow_float_truncate_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<bool>::type allow_int_overflow(allow_int_overflow_sexp);
-	Rcpp::traits::input_parameter<bool>::type allow_time_truncate(allow_time_truncate_sexp);
-	Rcpp::traits::input_parameter<bool>::type allow_float_truncate(allow_float_truncate_sexp);
-	return Rcpp::wrap(compute___CastOptions__initialize(allow_int_overflow, allow_time_truncate, allow_float_truncate));
-END_RCPP
+extern "C" SEXP _arrow_compute___CastOptions__initialize(SEXP allow_int_overflow_sexp, SEXP allow_time_truncate_sexp, SEXP allow_float_truncate_sexp){
+BEGIN_CPP11
+	arrow::r::Input<bool>::type allow_int_overflow(allow_int_overflow_sexp);
+	arrow::r::Input<bool>::type allow_time_truncate(allow_time_truncate_sexp);
+	arrow::r::Input<bool>::type allow_float_truncate(allow_float_truncate_sexp);
+	return cpp11::as_sexp(compute___CastOptions__initialize(allow_int_overflow, allow_time_truncate, allow_float_truncate));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_compute___CastOptions__initialize(SEXP allow_int_overflow_sexp, SEXP allow_time_truncate_sexp, SEXP allow_float_truncate_sexp){
+extern "C" SEXP _arrow_compute___CastOptions__initialize(SEXP allow_int_overflow_sexp, SEXP allow_time_truncate_sexp, SEXP allow_float_truncate_sexp){
 	Rf_error("Cannot call compute___CastOptions__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1113,16 +1113,16 @@ RcppExport SEXP _arrow_compute___CastOptions__initialize(SEXP allow_int_overflow
 // compute.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Array> Array__cast(const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& target_type, const std::shared_ptr<arrow::compute::CastOptions>& options);
-RcppExport SEXP _arrow_Array__cast(SEXP array_sexp, SEXP target_type_sexp, SEXP options_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type array(array_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DataType>&>::type target_type(target_type_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::compute::CastOptions>&>::type options(options_sexp);
-	return Rcpp::wrap(Array__cast(array, target_type, options));
-END_RCPP
+extern "C" SEXP _arrow_Array__cast(SEXP array_sexp, SEXP target_type_sexp, SEXP options_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type array(array_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::DataType>&>::type target_type(target_type_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::compute::CastOptions>&>::type options(options_sexp);
+	return cpp11::as_sexp(Array__cast(array, target_type, options));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__cast(SEXP array_sexp, SEXP target_type_sexp, SEXP options_sexp){
+extern "C" SEXP _arrow_Array__cast(SEXP array_sexp, SEXP target_type_sexp, SEXP options_sexp){
 	Rf_error("Cannot call Array__cast(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1130,16 +1130,16 @@ RcppExport SEXP _arrow_Array__cast(SEXP array_sexp, SEXP target_type_sexp, SEXP
 // compute.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::ChunkedArray> ChunkedArray__cast(const std::shared_ptr<arrow::ChunkedArray>& chunked_array, const std::shared_ptr<arrow::DataType>& target_type, const std::shared_ptr<arrow::compute::CastOptions>& options);
-RcppExport SEXP _arrow_ChunkedArray__cast(SEXP chunked_array_sexp, SEXP target_type_sexp, SEXP options_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DataType>&>::type target_type(target_type_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::compute::CastOptions>&>::type options(options_sexp);
-	return Rcpp::wrap(ChunkedArray__cast(chunked_array, target_type, options));
-END_RCPP
+extern "C" SEXP _arrow_ChunkedArray__cast(SEXP chunked_array_sexp, SEXP target_type_sexp, SEXP options_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ChunkedArray>&>::type chunked_array(chunked_array_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::DataType>&>::type target_type(target_type_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::compute::CastOptions>&>::type options(options_sexp);
+	return cpp11::as_sexp(ChunkedArray__cast(chunked_array, target_type, options));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ChunkedArray__cast(SEXP chunked_array_sexp, SEXP target_type_sexp, SEXP options_sexp){
+extern "C" SEXP _arrow_ChunkedArray__cast(SEXP chunked_array_sexp, SEXP target_type_sexp, SEXP options_sexp){
 	Rf_error("Cannot call ChunkedArray__cast(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1147,16 +1147,16 @@ RcppExport SEXP _arrow_ChunkedArray__cast(SEXP chunked_array_sexp, SEXP target_t
 // compute.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::RecordBatch> RecordBatch__cast(const std::shared_ptr<arrow::RecordBatch>& batch, const std::shared_ptr<arrow::Schema>& schema, const std::shared_ptr<arrow::compute::CastOptions>& options);
-RcppExport SEXP _arrow_RecordBatch__cast(SEXP batch_sexp, SEXP schema_sexp, SEXP options_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::compute::CastOptions>&>::type options(options_sexp);
-	return Rcpp::wrap(RecordBatch__cast(batch, schema, options));
-END_RCPP
+extern "C" SEXP _arrow_RecordBatch__cast(SEXP batch_sexp, SEXP schema_sexp, SEXP options_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::compute::CastOptions>&>::type options(options_sexp);
+	return cpp11::as_sexp(RecordBatch__cast(batch, schema, options));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_RecordBatch__cast(SEXP batch_sexp, SEXP schema_sexp, SEXP options_sexp){
+extern "C" SEXP _arrow_RecordBatch__cast(SEXP batch_sexp, SEXP schema_sexp, SEXP options_sexp){
 	Rf_error("Cannot call RecordBatch__cast(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1164,78 +1164,93 @@ RcppExport SEXP _arrow_RecordBatch__cast(SEXP batch_sexp, SEXP schema_sexp, SEXP
 // compute.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Table> Table__cast(const std::shared_ptr<arrow::Table>& table, const std::shared_ptr<arrow::Schema>& schema, const std::shared_ptr<arrow::compute::CastOptions>& options);
-RcppExport SEXP _arrow_Table__cast(SEXP table_sexp, SEXP schema_sexp, SEXP options_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::compute::CastOptions>&>::type options(options_sexp);
-	return Rcpp::wrap(Table__cast(table, schema, options));
-END_RCPP
+extern "C" SEXP _arrow_Table__cast(SEXP table_sexp, SEXP schema_sexp, SEXP options_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::compute::CastOptions>&>::type options(options_sexp);
+	return cpp11::as_sexp(Table__cast(table, schema, options));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Table__cast(SEXP table_sexp, SEXP schema_sexp, SEXP options_sexp){
+extern "C" SEXP _arrow_Table__cast(SEXP table_sexp, SEXP schema_sexp, SEXP options_sexp){
 	Rf_error("Cannot call Table__cast(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // compute.cpp
 #if defined(ARROW_R_WITH_ARROW)
-SEXP compute__CallFunction(std::string func_name, List_ args, List_ options);
-RcppExport SEXP _arrow_compute__CallFunction(SEXP func_name_sexp, SEXP args_sexp, SEXP options_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<std::string>::type func_name(func_name_sexp);
-	Rcpp::traits::input_parameter<List_>::type args(args_sexp);
-	Rcpp::traits::input_parameter<List_>::type options(options_sexp);
-	return Rcpp::wrap(compute__CallFunction(func_name, args, options));
-END_RCPP
+SEXP compute__CallFunction(std::string func_name, cpp11::list args, cpp11::list options);
+extern "C" SEXP _arrow_compute__CallFunction(SEXP func_name_sexp, SEXP args_sexp, SEXP options_sexp){
+BEGIN_CPP11
+	arrow::r::Input<std::string>::type func_name(func_name_sexp);
+	arrow::r::Input<cpp11::list>::type args(args_sexp);
+	arrow::r::Input<cpp11::list>::type options(options_sexp);
+	return cpp11::as_sexp(compute__CallFunction(func_name, args, options));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_compute__CallFunction(SEXP func_name_sexp, SEXP args_sexp, SEXP options_sexp){
+extern "C" SEXP _arrow_compute__CallFunction(SEXP func_name_sexp, SEXP args_sexp, SEXP options_sexp){
 	Rf_error("Cannot call compute__CallFunction(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // csv.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::csv::ReadOptions> csv___ReadOptions__initialize(List_ options);
-RcppExport SEXP _arrow_csv___ReadOptions__initialize(SEXP options_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<List_>::type options(options_sexp);
-	return Rcpp::wrap(csv___ReadOptions__initialize(options));
-END_RCPP
+std::shared_ptr<arrow::csv::ReadOptions> csv___ReadOptions__initialize(cpp11::list options);
+extern "C" SEXP _arrow_csv___ReadOptions__initialize(SEXP options_sexp){
+BEGIN_CPP11
+	arrow::r::Input<cpp11::list>::type options(options_sexp);
+	return cpp11::as_sexp(csv___ReadOptions__initialize(options));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_csv___ReadOptions__initialize(SEXP options_sexp){
+extern "C" SEXP _arrow_csv___ReadOptions__initialize(SEXP options_sexp){
 	Rf_error("Cannot call csv___ReadOptions__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // csv.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::csv::ParseOptions> csv___ParseOptions__initialize(List_ options);
-RcppExport SEXP _arrow_csv___ParseOptions__initialize(SEXP options_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<List_>::type options(options_sexp);
-	return Rcpp::wrap(csv___ParseOptions__initialize(options));
-END_RCPP
+std::shared_ptr<arrow::csv::ParseOptions> csv___ParseOptions__initialize(cpp11::list options);
+extern "C" SEXP _arrow_csv___ParseOptions__initialize(SEXP options_sexp){
+BEGIN_CPP11
+	arrow::r::Input<cpp11::list>::type options(options_sexp);
+	return cpp11::as_sexp(csv___ParseOptions__initialize(options));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_csv___ParseOptions__initialize(SEXP options_sexp){
+extern "C" SEXP _arrow_csv___ParseOptions__initialize(SEXP options_sexp){
 	Rf_error("Cannot call csv___ParseOptions__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // csv.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::csv::ConvertOptions> csv___ConvertOptions__initialize(List_ options);
-RcppExport SEXP _arrow_csv___ConvertOptions__initialize(SEXP options_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<List_>::type options(options_sexp);
-	return Rcpp::wrap(csv___ConvertOptions__initialize(options));
-END_RCPP
+SEXP csv___ReadOptions__column_names(const std::shared_ptr<arrow::csv::ReadOptions>& options);
+extern "C" SEXP _arrow_csv___ReadOptions__column_names(SEXP options_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::csv::ReadOptions>&>::type options(options_sexp);
+	return cpp11::as_sexp(csv___ReadOptions__column_names(options));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_csv___ConvertOptions__initialize(SEXP options_sexp){
+extern "C" SEXP _arrow_csv___ReadOptions__column_names(SEXP options_sexp){
+	Rf_error("Cannot call csv___ReadOptions__column_names(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// csv.cpp
+#if defined(ARROW_R_WITH_ARROW)
+std::shared_ptr<arrow::csv::ConvertOptions> csv___ConvertOptions__initialize(cpp11::list options);
+extern "C" SEXP _arrow_csv___ConvertOptions__initialize(SEXP options_sexp){
+BEGIN_CPP11
+	arrow::r::Input<cpp11::list>::type options(options_sexp);
+	return cpp11::as_sexp(csv___ConvertOptions__initialize(options));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_csv___ConvertOptions__initialize(SEXP options_sexp){
 	Rf_error("Cannot call csv___ConvertOptions__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1243,17 +1258,17 @@ RcppExport SEXP _arrow_csv___ConvertOptions__initialize(SEXP options_sexp){
 // csv.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::csv::TableReader> csv___TableReader__Make(const std::shared_ptr<arrow::io::InputStream>& input, const std::shared_ptr<arrow::csv::ReadOptions>& read_options, const std::shared_ptr<arrow::csv::ParseOptions>& parse_options, const std::shared_ptr<arrow::csv::ConvertOptions>& convert_options);
-RcppExport SEXP _arrow_csv___TableReader__Make(SEXP input_sexp, SEXP read_options_sexp, SEXP parse_options_sexp, SEXP convert_options_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::InputStream>&>::type input(input_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::csv::ReadOptions>&>::type read_options(read_options_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::csv::ParseOptions>&>::type parse_options(parse_options_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::csv::ConvertOptions>&>::type convert_options(convert_options_sexp);
-	return Rcpp::wrap(csv___TableReader__Make(input, read_options, parse_options, convert_options));
-END_RCPP
+extern "C" SEXP _arrow_csv___TableReader__Make(SEXP input_sexp, SEXP read_options_sexp, SEXP parse_options_sexp, SEXP convert_options_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::InputStream>&>::type input(input_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::csv::ReadOptions>&>::type read_options(read_options_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::csv::ParseOptions>&>::type parse_options(parse_options_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::csv::ConvertOptions>&>::type convert_options(convert_options_sexp);
+	return cpp11::as_sexp(csv___TableReader__Make(input, read_options, parse_options, convert_options));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_csv___TableReader__Make(SEXP input_sexp, SEXP read_options_sexp, SEXP parse_options_sexp, SEXP convert_options_sexp){
+extern "C" SEXP _arrow_csv___TableReader__Make(SEXP input_sexp, SEXP read_options_sexp, SEXP parse_options_sexp, SEXP convert_options_sexp){
 	Rf_error("Cannot call csv___TableReader__Make(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1261,29 +1276,88 @@ RcppExport SEXP _arrow_csv___TableReader__Make(SEXP input_sexp, SEXP read_option
 // csv.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Table> csv___TableReader__Read(const std::shared_ptr<arrow::csv::TableReader>& table_reader);
-RcppExport SEXP _arrow_csv___TableReader__Read(SEXP table_reader_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::csv::TableReader>&>::type table_reader(table_reader_sexp);
-	return Rcpp::wrap(csv___TableReader__Read(table_reader));
-END_RCPP
+extern "C" SEXP _arrow_csv___TableReader__Read(SEXP table_reader_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::csv::TableReader>&>::type table_reader(table_reader_sexp);
+	return cpp11::as_sexp(csv___TableReader__Read(table_reader));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_csv___TableReader__Read(SEXP table_reader_sexp){
+extern "C" SEXP _arrow_csv___TableReader__Read(SEXP table_reader_sexp){
 	Rf_error("Cannot call csv___TableReader__Read(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
+// csv.cpp
+#if defined(ARROW_R_WITH_ARROW)
+std::string TimestampParser__kind(const std::shared_ptr<arrow::TimestampParser>& parser);
+extern "C" SEXP _arrow_TimestampParser__kind(SEXP parser_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::TimestampParser>&>::type parser(parser_sexp);
+	return cpp11::as_sexp(TimestampParser__kind(parser));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_TimestampParser__kind(SEXP parser_sexp){
+	Rf_error("Cannot call TimestampParser__kind(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// csv.cpp
+#if defined(ARROW_R_WITH_ARROW)
+std::string TimestampParser__format(const std::shared_ptr<arrow::TimestampParser>& parser);
+extern "C" SEXP _arrow_TimestampParser__format(SEXP parser_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::TimestampParser>&>::type parser(parser_sexp);
+	return cpp11::as_sexp(TimestampParser__format(parser));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_TimestampParser__format(SEXP parser_sexp){
+	Rf_error("Cannot call TimestampParser__format(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// csv.cpp
+#if defined(ARROW_R_WITH_ARROW)
+std::shared_ptr<arrow::TimestampParser> TimestampParser__MakeStrptime(std::string format);
+extern "C" SEXP _arrow_TimestampParser__MakeStrptime(SEXP format_sexp){
+BEGIN_CPP11
+	arrow::r::Input<std::string>::type format(format_sexp);
+	return cpp11::as_sexp(TimestampParser__MakeStrptime(format));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_TimestampParser__MakeStrptime(SEXP format_sexp){
+	Rf_error("Cannot call TimestampParser__MakeStrptime(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// csv.cpp
+#if defined(ARROW_R_WITH_ARROW)
+std::shared_ptr<arrow::TimestampParser> TimestampParser__MakeISO8601();
+extern "C" SEXP _arrow_TimestampParser__MakeISO8601(){
+BEGIN_CPP11
+	return cpp11::as_sexp(TimestampParser__MakeISO8601());
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_TimestampParser__MakeISO8601(){
+	Rf_error("Cannot call TimestampParser__MakeISO8601(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::ScannerBuilder> dataset___Dataset__NewScan(const std::shared_ptr<ds::Dataset>& ds);
-RcppExport SEXP _arrow_dataset___Dataset__NewScan(SEXP ds_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Dataset>&>::type ds(ds_sexp);
-	return Rcpp::wrap(dataset___Dataset__NewScan(ds));
-END_RCPP
+extern "C" SEXP _arrow_dataset___Dataset__NewScan(SEXP ds_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::Dataset>&>::type ds(ds_sexp);
+	return cpp11::as_sexp(dataset___Dataset__NewScan(ds));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___Dataset__NewScan(SEXP ds_sexp){
+extern "C" SEXP _arrow_dataset___Dataset__NewScan(SEXP ds_sexp){
 	Rf_error("Cannot call dataset___Dataset__NewScan(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1291,14 +1365,14 @@ RcppExport SEXP _arrow_dataset___Dataset__NewScan(SEXP ds_sexp){
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Schema> dataset___Dataset__schema(const std::shared_ptr<ds::Dataset>& dataset);
-RcppExport SEXP _arrow_dataset___Dataset__schema(SEXP dataset_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Dataset>&>::type dataset(dataset_sexp);
-	return Rcpp::wrap(dataset___Dataset__schema(dataset));
-END_RCPP
+extern "C" SEXP _arrow_dataset___Dataset__schema(SEXP dataset_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::Dataset>&>::type dataset(dataset_sexp);
+	return cpp11::as_sexp(dataset___Dataset__schema(dataset));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___Dataset__schema(SEXP dataset_sexp){
+extern "C" SEXP _arrow_dataset___Dataset__schema(SEXP dataset_sexp){
 	Rf_error("Cannot call dataset___Dataset__schema(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1306,14 +1380,14 @@ RcppExport SEXP _arrow_dataset___Dataset__schema(SEXP dataset_sexp){
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::string dataset___Dataset__type_name(const std::shared_ptr<ds::Dataset>& dataset);
-RcppExport SEXP _arrow_dataset___Dataset__type_name(SEXP dataset_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Dataset>&>::type dataset(dataset_sexp);
-	return Rcpp::wrap(dataset___Dataset__type_name(dataset));
-END_RCPP
+extern "C" SEXP _arrow_dataset___Dataset__type_name(SEXP dataset_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::Dataset>&>::type dataset(dataset_sexp);
+	return cpp11::as_sexp(dataset___Dataset__type_name(dataset));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___Dataset__type_name(SEXP dataset_sexp){
+extern "C" SEXP _arrow_dataset___Dataset__type_name(SEXP dataset_sexp){
 	Rf_error("Cannot call dataset___Dataset__type_name(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1321,15 +1395,15 @@ RcppExport SEXP _arrow_dataset___Dataset__type_name(SEXP dataset_sexp){
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::Dataset> dataset___Dataset__ReplaceSchema(const std::shared_ptr<ds::Dataset>& dataset, const std::shared_ptr<arrow::Schema>& schm);
-RcppExport SEXP _arrow_dataset___Dataset__ReplaceSchema(SEXP dataset_sexp, SEXP schm_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Dataset>&>::type dataset(dataset_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schm(schm_sexp);
-	return Rcpp::wrap(dataset___Dataset__ReplaceSchema(dataset, schm));
-END_RCPP
+extern "C" SEXP _arrow_dataset___Dataset__ReplaceSchema(SEXP dataset_sexp, SEXP schm_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::Dataset>&>::type dataset(dataset_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type schm(schm_sexp);
+	return cpp11::as_sexp(dataset___Dataset__ReplaceSchema(dataset, schm));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___Dataset__ReplaceSchema(SEXP dataset_sexp, SEXP schm_sexp){
+extern "C" SEXP _arrow_dataset___Dataset__ReplaceSchema(SEXP dataset_sexp, SEXP schm_sexp){
 	Rf_error("Cannot call dataset___Dataset__ReplaceSchema(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1337,30 +1411,45 @@ RcppExport SEXP _arrow_dataset___Dataset__ReplaceSchema(SEXP dataset_sexp, SEXP
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::UnionDataset> dataset___UnionDataset__create(const ds::DatasetVector& datasets, const std::shared_ptr<arrow::Schema>& schm);
-RcppExport SEXP _arrow_dataset___UnionDataset__create(SEXP datasets_sexp, SEXP schm_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const ds::DatasetVector&>::type datasets(datasets_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schm(schm_sexp);
-	return Rcpp::wrap(dataset___UnionDataset__create(datasets, schm));
-END_RCPP
+extern "C" SEXP _arrow_dataset___UnionDataset__create(SEXP datasets_sexp, SEXP schm_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const ds::DatasetVector&>::type datasets(datasets_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type schm(schm_sexp);
+	return cpp11::as_sexp(dataset___UnionDataset__create(datasets, schm));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___UnionDataset__create(SEXP datasets_sexp, SEXP schm_sexp){
+extern "C" SEXP _arrow_dataset___UnionDataset__create(SEXP datasets_sexp, SEXP schm_sexp){
 	Rf_error("Cannot call dataset___UnionDataset__create(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
+// dataset.cpp
+#if defined(ARROW_R_WITH_ARROW)
+std::shared_ptr<ds::InMemoryDataset> dataset___InMemoryDataset__create(const std::shared_ptr<arrow::Table>& table);
+extern "C" SEXP _arrow_dataset___InMemoryDataset__create(SEXP table_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
+	return cpp11::as_sexp(dataset___InMemoryDataset__create(table));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_dataset___InMemoryDataset__create(SEXP table_sexp){
+	Rf_error("Cannot call dataset___InMemoryDataset__create(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 ds::DatasetVector dataset___UnionDataset__children(const std::shared_ptr<ds::UnionDataset>& ds);
-RcppExport SEXP _arrow_dataset___UnionDataset__children(SEXP ds_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::UnionDataset>&>::type ds(ds_sexp);
-	return Rcpp::wrap(dataset___UnionDataset__children(ds));
-END_RCPP
+extern "C" SEXP _arrow_dataset___UnionDataset__children(SEXP ds_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::UnionDataset>&>::type ds(ds_sexp);
+	return cpp11::as_sexp(dataset___UnionDataset__children(ds));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___UnionDataset__children(SEXP ds_sexp){
+extern "C" SEXP _arrow_dataset___UnionDataset__children(SEXP ds_sexp){
 	Rf_error("Cannot call dataset___UnionDataset__children(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1368,29 +1457,44 @@ RcppExport SEXP _arrow_dataset___UnionDataset__children(SEXP ds_sexp){
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::FileFormat> dataset___FileSystemDataset__format(const std::shared_ptr<ds::FileSystemDataset>& dataset);
-RcppExport SEXP _arrow_dataset___FileSystemDataset__format(SEXP dataset_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::FileSystemDataset>&>::type dataset(dataset_sexp);
-	return Rcpp::wrap(dataset___FileSystemDataset__format(dataset));
-END_RCPP
+extern "C" SEXP _arrow_dataset___FileSystemDataset__format(SEXP dataset_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::FileSystemDataset>&>::type dataset(dataset_sexp);
+	return cpp11::as_sexp(dataset___FileSystemDataset__format(dataset));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___FileSystemDataset__format(SEXP dataset_sexp){
+extern "C" SEXP _arrow_dataset___FileSystemDataset__format(SEXP dataset_sexp){
 	Rf_error("Cannot call dataset___FileSystemDataset__format(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
+// dataset.cpp
+#if defined(ARROW_R_WITH_ARROW)
+std::shared_ptr<fs::FileSystem> dataset___FileSystemDataset__filesystem(const std::shared_ptr<ds::FileSystemDataset>& dataset);
+extern "C" SEXP _arrow_dataset___FileSystemDataset__filesystem(SEXP dataset_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::FileSystemDataset>&>::type dataset(dataset_sexp);
+	return cpp11::as_sexp(dataset___FileSystemDataset__filesystem(dataset));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_dataset___FileSystemDataset__filesystem(SEXP dataset_sexp){
+	Rf_error("Cannot call dataset___FileSystemDataset__filesystem(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::vector<std::string> dataset___FileSystemDataset__files(const std::shared_ptr<ds::FileSystemDataset>& dataset);
-RcppExport SEXP _arrow_dataset___FileSystemDataset__files(SEXP dataset_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::FileSystemDataset>&>::type dataset(dataset_sexp);
-	return Rcpp::wrap(dataset___FileSystemDataset__files(dataset));
-END_RCPP
+extern "C" SEXP _arrow_dataset___FileSystemDataset__files(SEXP dataset_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::FileSystemDataset>&>::type dataset(dataset_sexp);
+	return cpp11::as_sexp(dataset___FileSystemDataset__files(dataset));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___FileSystemDataset__files(SEXP dataset_sexp){
+extern "C" SEXP _arrow_dataset___FileSystemDataset__files(SEXP dataset_sexp){
 	Rf_error("Cannot call dataset___FileSystemDataset__files(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1398,15 +1502,15 @@ RcppExport SEXP _arrow_dataset___FileSystemDataset__files(SEXP dataset_sexp){
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::Dataset> dataset___DatasetFactory__Finish1(const std::shared_ptr<ds::DatasetFactory>& factory, bool unify_schemas);
-RcppExport SEXP _arrow_dataset___DatasetFactory__Finish1(SEXP factory_sexp, SEXP unify_schemas_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::DatasetFactory>&>::type factory(factory_sexp);
-	Rcpp::traits::input_parameter<bool>::type unify_schemas(unify_schemas_sexp);
-	return Rcpp::wrap(dataset___DatasetFactory__Finish1(factory, unify_schemas));
-END_RCPP
+extern "C" SEXP _arrow_dataset___DatasetFactory__Finish1(SEXP factory_sexp, SEXP unify_schemas_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::DatasetFactory>&>::type factory(factory_sexp);
+	arrow::r::Input<bool>::type unify_schemas(unify_schemas_sexp);
+	return cpp11::as_sexp(dataset___DatasetFactory__Finish1(factory, unify_schemas));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___DatasetFactory__Finish1(SEXP factory_sexp, SEXP unify_schemas_sexp){
+extern "C" SEXP _arrow_dataset___DatasetFactory__Finish1(SEXP factory_sexp, SEXP unify_schemas_sexp){
 	Rf_error("Cannot call dataset___DatasetFactory__Finish1(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1414,15 +1518,15 @@ RcppExport SEXP _arrow_dataset___DatasetFactory__Finish1(SEXP factory_sexp, SEXP
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::Dataset> dataset___DatasetFactory__Finish2(const std::shared_ptr<ds::DatasetFactory>& factory, const std::shared_ptr<arrow::Schema>& schema);
-RcppExport SEXP _arrow_dataset___DatasetFactory__Finish2(SEXP factory_sexp, SEXP schema_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::DatasetFactory>&>::type factory(factory_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
-	return Rcpp::wrap(dataset___DatasetFactory__Finish2(factory, schema));
-END_RCPP
+extern "C" SEXP _arrow_dataset___DatasetFactory__Finish2(SEXP factory_sexp, SEXP schema_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::DatasetFactory>&>::type factory(factory_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
+	return cpp11::as_sexp(dataset___DatasetFactory__Finish2(factory, schema));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___DatasetFactory__Finish2(SEXP factory_sexp, SEXP schema_sexp){
+extern "C" SEXP _arrow_dataset___DatasetFactory__Finish2(SEXP factory_sexp, SEXP schema_sexp){
 	Rf_error("Cannot call dataset___DatasetFactory__Finish2(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1430,15 +1534,15 @@ RcppExport SEXP _arrow_dataset___DatasetFactory__Finish2(SEXP factory_sexp, SEXP
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Schema> dataset___DatasetFactory__Inspect(const std::shared_ptr<ds::DatasetFactory>& factory, bool unify_schemas);
-RcppExport SEXP _arrow_dataset___DatasetFactory__Inspect(SEXP factory_sexp, SEXP unify_schemas_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::DatasetFactory>&>::type factory(factory_sexp);
-	Rcpp::traits::input_parameter<bool>::type unify_schemas(unify_schemas_sexp);
-	return Rcpp::wrap(dataset___DatasetFactory__Inspect(factory, unify_schemas));
-END_RCPP
+extern "C" SEXP _arrow_dataset___DatasetFactory__Inspect(SEXP factory_sexp, SEXP unify_schemas_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::DatasetFactory>&>::type factory(factory_sexp);
+	arrow::r::Input<bool>::type unify_schemas(unify_schemas_sexp);
+	return cpp11::as_sexp(dataset___DatasetFactory__Inspect(factory, unify_schemas));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___DatasetFactory__Inspect(SEXP factory_sexp, SEXP unify_schemas_sexp){
+extern "C" SEXP _arrow_dataset___DatasetFactory__Inspect(SEXP factory_sexp, SEXP unify_schemas_sexp){
 	Rf_error("Cannot call dataset___DatasetFactory__Inspect(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1446,14 +1550,14 @@ RcppExport SEXP _arrow_dataset___DatasetFactory__Inspect(SEXP factory_sexp, SEXP
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::DatasetFactory> dataset___UnionDatasetFactory__Make(const std::vector<std::shared_ptr<ds::DatasetFactory>>& children);
-RcppExport SEXP _arrow_dataset___UnionDatasetFactory__Make(SEXP children_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::vector<std::shared_ptr<ds::DatasetFactory>>&>::type children(children_sexp);
-	return Rcpp::wrap(dataset___UnionDatasetFactory__Make(children));
-END_RCPP
+extern "C" SEXP _arrow_dataset___UnionDatasetFactory__Make(SEXP children_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::vector<std::shared_ptr<ds::DatasetFactory>>&>::type children(children_sexp);
+	return cpp11::as_sexp(dataset___UnionDatasetFactory__Make(children));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___UnionDatasetFactory__Make(SEXP children_sexp){
+extern "C" SEXP _arrow_dataset___UnionDatasetFactory__Make(SEXP children_sexp){
 	Rf_error("Cannot call dataset___UnionDatasetFactory__Make(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1461,17 +1565,17 @@ RcppExport SEXP _arrow_dataset___UnionDatasetFactory__Make(SEXP children_sexp){
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::DatasetFactory> dataset___FileSystemDatasetFactory__Make2(const std::shared_ptr<fs::FileSystem>& fs, const std::shared_ptr<fs::FileSelector>& selector, const std::shared_ptr<ds::FileFormat>& format, const std::shared_ptr<ds::Partitioning>& partitioning);
-RcppExport SEXP _arrow_dataset___FileSystemDatasetFactory__Make2(SEXP fs_sexp, SEXP selector_sexp, SEXP format_sexp, SEXP partitioning_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSystem>&>::type fs(fs_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSelector>&>::type selector(selector_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::FileFormat>&>::type format(format_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Partitioning>&>::type partitioning(partitioning_sexp);
-	return Rcpp::wrap(dataset___FileSystemDatasetFactory__Make2(fs, selector, format, partitioning));
-END_RCPP
+extern "C" SEXP _arrow_dataset___FileSystemDatasetFactory__Make2(SEXP fs_sexp, SEXP selector_sexp, SEXP format_sexp, SEXP partitioning_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type fs(fs_sexp);
+	arrow::r::Input<const std::shared_ptr<fs::FileSelector>&>::type selector(selector_sexp);
+	arrow::r::Input<const std::shared_ptr<ds::FileFormat>&>::type format(format_sexp);
+	arrow::r::Input<const std::shared_ptr<ds::Partitioning>&>::type partitioning(partitioning_sexp);
+	return cpp11::as_sexp(dataset___FileSystemDatasetFactory__Make2(fs, selector, format, partitioning));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___FileSystemDatasetFactory__Make2(SEXP fs_sexp, SEXP selector_sexp, SEXP format_sexp, SEXP partitioning_sexp){
+extern "C" SEXP _arrow_dataset___FileSystemDatasetFactory__Make2(SEXP fs_sexp, SEXP selector_sexp, SEXP format_sexp, SEXP partitioning_sexp){
 	Rf_error("Cannot call dataset___FileSystemDatasetFactory__Make2(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1479,16 +1583,16 @@ RcppExport SEXP _arrow_dataset___FileSystemDatasetFactory__Make2(SEXP fs_sexp, S
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::DatasetFactory> dataset___FileSystemDatasetFactory__Make1(const std::shared_ptr<fs::FileSystem>& fs, const std::shared_ptr<fs::FileSelector>& selector, const std::shared_ptr<ds::FileFormat>& format);
-RcppExport SEXP _arrow_dataset___FileSystemDatasetFactory__Make1(SEXP fs_sexp, SEXP selector_sexp, SEXP format_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSystem>&>::type fs(fs_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSelector>&>::type selector(selector_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::FileFormat>&>::type format(format_sexp);
-	return Rcpp::wrap(dataset___FileSystemDatasetFactory__Make1(fs, selector, format));
-END_RCPP
+extern "C" SEXP _arrow_dataset___FileSystemDatasetFactory__Make1(SEXP fs_sexp, SEXP selector_sexp, SEXP format_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type fs(fs_sexp);
+	arrow::r::Input<const std::shared_ptr<fs::FileSelector>&>::type selector(selector_sexp);
+	arrow::r::Input<const std::shared_ptr<ds::FileFormat>&>::type format(format_sexp);
+	return cpp11::as_sexp(dataset___FileSystemDatasetFactory__Make1(fs, selector, format));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___FileSystemDatasetFactory__Make1(SEXP fs_sexp, SEXP selector_sexp, SEXP format_sexp){
+extern "C" SEXP _arrow_dataset___FileSystemDatasetFactory__Make1(SEXP fs_sexp, SEXP selector_sexp, SEXP format_sexp){
 	Rf_error("Cannot call dataset___FileSystemDatasetFactory__Make1(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1496,17 +1600,17 @@ RcppExport SEXP _arrow_dataset___FileSystemDatasetFactory__Make1(SEXP fs_sexp, S
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::DatasetFactory> dataset___FileSystemDatasetFactory__Make3(const std::shared_ptr<fs::FileSystem>& fs, const std::shared_ptr<fs::FileSelector>& selector, const std::shared_ptr<ds::FileFormat>& format, const std::shared_ptr<ds::PartitioningFactory>& factory);
-RcppExport SEXP _arrow_dataset___FileSystemDatasetFactory__Make3(SEXP fs_sexp, SEXP selector_sexp, SEXP format_sexp, SEXP factory_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSystem>&>::type fs(fs_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSelector>&>::type selector(selector_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::FileFormat>&>::type format(format_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::PartitioningFactory>&>::type factory(factory_sexp);
-	return Rcpp::wrap(dataset___FileSystemDatasetFactory__Make3(fs, selector, format, factory));
-END_RCPP
+extern "C" SEXP _arrow_dataset___FileSystemDatasetFactory__Make3(SEXP fs_sexp, SEXP selector_sexp, SEXP format_sexp, SEXP factory_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type fs(fs_sexp);
+	arrow::r::Input<const std::shared_ptr<fs::FileSelector>&>::type selector(selector_sexp);
+	arrow::r::Input<const std::shared_ptr<ds::FileFormat>&>::type format(format_sexp);
+	arrow::r::Input<const std::shared_ptr<ds::PartitioningFactory>&>::type factory(factory_sexp);
+	return cpp11::as_sexp(dataset___FileSystemDatasetFactory__Make3(fs, selector, format, factory));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___FileSystemDatasetFactory__Make3(SEXP fs_sexp, SEXP selector_sexp, SEXP format_sexp, SEXP factory_sexp){
+extern "C" SEXP _arrow_dataset___FileSystemDatasetFactory__Make3(SEXP fs_sexp, SEXP selector_sexp, SEXP format_sexp, SEXP factory_sexp){
 	Rf_error("Cannot call dataset___FileSystemDatasetFactory__Make3(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1514,45 +1618,130 @@ RcppExport SEXP _arrow_dataset___FileSystemDatasetFactory__Make3(SEXP fs_sexp, S
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::string dataset___FileFormat__type_name(const std::shared_ptr<ds::FileFormat>& format);
-RcppExport SEXP _arrow_dataset___FileFormat__type_name(SEXP format_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::FileFormat>&>::type format(format_sexp);
-	return Rcpp::wrap(dataset___FileFormat__type_name(format));
-END_RCPP
+extern "C" SEXP _arrow_dataset___FileFormat__type_name(SEXP format_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::FileFormat>&>::type format(format_sexp);
+	return cpp11::as_sexp(dataset___FileFormat__type_name(format));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___FileFormat__type_name(SEXP format_sexp){
+extern "C" SEXP _arrow_dataset___FileFormat__type_name(SEXP format_sexp){
 	Rf_error("Cannot call dataset___FileFormat__type_name(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<ds::ParquetFileFormat> dataset___ParquetFileFormat__Make(bool use_buffered_stream, int64_t buffer_size, CharacterVector dict_columns);
-RcppExport SEXP _arrow_dataset___ParquetFileFormat__Make(SEXP use_buffered_stream_sexp, SEXP buffer_size_sexp, SEXP dict_columns_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<bool>::type use_buffered_stream(use_buffered_stream_sexp);
-	Rcpp::traits::input_parameter<int64_t>::type buffer_size(buffer_size_sexp);
-	Rcpp::traits::input_parameter<CharacterVector>::type dict_columns(dict_columns_sexp);
-	return Rcpp::wrap(dataset___ParquetFileFormat__Make(use_buffered_stream, buffer_size, dict_columns));
-END_RCPP
+std::shared_ptr<ds::FileWriteOptions> dataset___FileFormat__DefaultWriteOptions(const std::shared_ptr<ds::FileFormat>& fmt);
+extern "C" SEXP _arrow_dataset___FileFormat__DefaultWriteOptions(SEXP fmt_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::FileFormat>&>::type fmt(fmt_sexp);
+	return cpp11::as_sexp(dataset___FileFormat__DefaultWriteOptions(fmt));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___ParquetFileFormat__Make(SEXP use_buffered_stream_sexp, SEXP buffer_size_sexp, SEXP dict_columns_sexp){
+extern "C" SEXP _arrow_dataset___FileFormat__DefaultWriteOptions(SEXP fmt_sexp){
+	Rf_error("Cannot call dataset___FileFormat__DefaultWriteOptions(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// dataset.cpp
+#if defined(ARROW_R_WITH_ARROW)
+std::shared_ptr<ds::ParquetFileFormat> dataset___ParquetFileFormat__Make(bool use_buffered_stream, int64_t buffer_size, cpp11::strings dict_columns);
+extern "C" SEXP _arrow_dataset___ParquetFileFormat__Make(SEXP use_buffered_stream_sexp, SEXP buffer_size_sexp, SEXP dict_columns_sexp){
+BEGIN_CPP11
+	arrow::r::Input<bool>::type use_buffered_stream(use_buffered_stream_sexp);
+	arrow::r::Input<int64_t>::type buffer_size(buffer_size_sexp);
+	arrow::r::Input<cpp11::strings>::type dict_columns(dict_columns_sexp);
+	return cpp11::as_sexp(dataset___ParquetFileFormat__Make(use_buffered_stream, buffer_size, dict_columns));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_dataset___ParquetFileFormat__Make(SEXP use_buffered_stream_sexp, SEXP buffer_size_sexp, SEXP dict_columns_sexp){
 	Rf_error("Cannot call dataset___ParquetFileFormat__Make(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
+// dataset.cpp
+#if defined(ARROW_R_WITH_ARROW)
+std::string dataset___FileWriteOptions__type_name(const std::shared_ptr<ds::FileWriteOptions>& options);
+extern "C" SEXP _arrow_dataset___FileWriteOptions__type_name(SEXP options_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::FileWriteOptions>&>::type options(options_sexp);
+	return cpp11::as_sexp(dataset___FileWriteOptions__type_name(options));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_dataset___FileWriteOptions__type_name(SEXP options_sexp){
+	Rf_error("Cannot call dataset___FileWriteOptions__type_name(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// dataset.cpp
+#if defined(ARROW_R_WITH_ARROW)
+void dataset___ParquetFileWriteOptions__update(const std::shared_ptr<ds::ParquetFileWriteOptions>& options, const std::shared_ptr<parquet::WriterProperties>& writer_props, const std::shared_ptr<parquet::ArrowWriterProperties>& arrow_writer_props);
+extern "C" SEXP _arrow_dataset___ParquetFileWriteOptions__update(SEXP options_sexp, SEXP writer_props_sexp, SEXP arrow_writer_props_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::ParquetFileWriteOptions>&>::type options(options_sexp);
+	arrow::r::Input<const std::shared_ptr<parquet::WriterProperties>&>::type writer_props(writer_props_sexp);
+	arrow::r::Input<const std::shared_ptr<parquet::ArrowWriterProperties>&>::type arrow_writer_props(arrow_writer_props_sexp);
+	dataset___ParquetFileWriteOptions__update(options, writer_props, arrow_writer_props);
+	return R_NilValue;
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_dataset___ParquetFileWriteOptions__update(SEXP options_sexp, SEXP writer_props_sexp, SEXP arrow_writer_props_sexp){
+	Rf_error("Cannot call dataset___ParquetFileWriteOptions__update(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// dataset.cpp
+#if defined(ARROW_R_WITH_ARROW)
+void dataset___IpcFileWriteOptions__update2(const std::shared_ptr<ds::IpcFileWriteOptions>& ipc_options, bool use_legacy_format, const std::shared_ptr<arrow::util::Codec>& codec, arrow::ipc::MetadataVersion metadata_version);
+extern "C" SEXP _arrow_dataset___IpcFileWriteOptions__update2(SEXP ipc_options_sexp, SEXP use_legacy_format_sexp, SEXP codec_sexp, SEXP metadata_version_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::IpcFileWriteOptions>&>::type ipc_options(ipc_options_sexp);
+	arrow::r::Input<bool>::type use_legacy_format(use_legacy_format_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::util::Codec>&>::type codec(codec_sexp);
+	arrow::r::Input<arrow::ipc::MetadataVersion>::type metadata_version(metadata_version_sexp);
+	dataset___IpcFileWriteOptions__update2(ipc_options, use_legacy_format, codec, metadata_version);
+	return R_NilValue;
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_dataset___IpcFileWriteOptions__update2(SEXP ipc_options_sexp, SEXP use_legacy_format_sexp, SEXP codec_sexp, SEXP metadata_version_sexp){
+	Rf_error("Cannot call dataset___IpcFileWriteOptions__update2(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// dataset.cpp
+#if defined(ARROW_R_WITH_ARROW)
+void dataset___IpcFileWriteOptions__update1(const std::shared_ptr<ds::IpcFileWriteOptions>& ipc_options, bool use_legacy_format, arrow::ipc::MetadataVersion metadata_version);
+extern "C" SEXP _arrow_dataset___IpcFileWriteOptions__update1(SEXP ipc_options_sexp, SEXP use_legacy_format_sexp, SEXP metadata_version_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::IpcFileWriteOptions>&>::type ipc_options(ipc_options_sexp);
+	arrow::r::Input<bool>::type use_legacy_format(use_legacy_format_sexp);
+	arrow::r::Input<arrow::ipc::MetadataVersion>::type metadata_version(metadata_version_sexp);
+	dataset___IpcFileWriteOptions__update1(ipc_options, use_legacy_format, metadata_version);
+	return R_NilValue;
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_dataset___IpcFileWriteOptions__update1(SEXP ipc_options_sexp, SEXP use_legacy_format_sexp, SEXP metadata_version_sexp){
+	Rf_error("Cannot call dataset___IpcFileWriteOptions__update1(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::IpcFileFormat> dataset___IpcFileFormat__Make();
-RcppExport SEXP _arrow_dataset___IpcFileFormat__Make(){
-BEGIN_RCPP
-	return Rcpp::wrap(dataset___IpcFileFormat__Make());
-END_RCPP
+extern "C" SEXP _arrow_dataset___IpcFileFormat__Make(){
+BEGIN_CPP11
+	return cpp11::as_sexp(dataset___IpcFileFormat__Make());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___IpcFileFormat__Make(){
+extern "C" SEXP _arrow_dataset___IpcFileFormat__Make(){
 	Rf_error("Cannot call dataset___IpcFileFormat__Make(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1560,14 +1749,14 @@ RcppExport SEXP _arrow_dataset___IpcFileFormat__Make(){
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::CsvFileFormat> dataset___CsvFileFormat__Make(const std::shared_ptr<arrow::csv::ParseOptions>& parse_options);
-RcppExport SEXP _arrow_dataset___CsvFileFormat__Make(SEXP parse_options_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::csv::ParseOptions>&>::type parse_options(parse_options_sexp);
-	return Rcpp::wrap(dataset___CsvFileFormat__Make(parse_options));
-END_RCPP
+extern "C" SEXP _arrow_dataset___CsvFileFormat__Make(SEXP parse_options_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::csv::ParseOptions>&>::type parse_options(parse_options_sexp);
+	return cpp11::as_sexp(dataset___CsvFileFormat__Make(parse_options));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___CsvFileFormat__Make(SEXP parse_options_sexp){
+extern "C" SEXP _arrow_dataset___CsvFileFormat__Make(SEXP parse_options_sexp){
 	Rf_error("Cannot call dataset___CsvFileFormat__Make(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1575,14 +1764,14 @@ RcppExport SEXP _arrow_dataset___CsvFileFormat__Make(SEXP parse_options_sexp){
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::Partitioning> dataset___DirectoryPartitioning(const std::shared_ptr<arrow::Schema>& schm);
-RcppExport SEXP _arrow_dataset___DirectoryPartitioning(SEXP schm_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schm(schm_sexp);
-	return Rcpp::wrap(dataset___DirectoryPartitioning(schm));
-END_RCPP
+extern "C" SEXP _arrow_dataset___DirectoryPartitioning(SEXP schm_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type schm(schm_sexp);
+	return cpp11::as_sexp(dataset___DirectoryPartitioning(schm));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___DirectoryPartitioning(SEXP schm_sexp){
+extern "C" SEXP _arrow_dataset___DirectoryPartitioning(SEXP schm_sexp){
 	Rf_error("Cannot call dataset___DirectoryPartitioning(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1590,14 +1779,14 @@ RcppExport SEXP _arrow_dataset___DirectoryPartitioning(SEXP schm_sexp){
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::PartitioningFactory> dataset___DirectoryPartitioning__MakeFactory(const std::vector<std::string>& field_names);
-RcppExport SEXP _arrow_dataset___DirectoryPartitioning__MakeFactory(SEXP field_names_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::vector<std::string>&>::type field_names(field_names_sexp);
-	return Rcpp::wrap(dataset___DirectoryPartitioning__MakeFactory(field_names));
-END_RCPP
+extern "C" SEXP _arrow_dataset___DirectoryPartitioning__MakeFactory(SEXP field_names_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::vector<std::string>&>::type field_names(field_names_sexp);
+	return cpp11::as_sexp(dataset___DirectoryPartitioning__MakeFactory(field_names));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___DirectoryPartitioning__MakeFactory(SEXP field_names_sexp){
+extern "C" SEXP _arrow_dataset___DirectoryPartitioning__MakeFactory(SEXP field_names_sexp){
 	Rf_error("Cannot call dataset___DirectoryPartitioning__MakeFactory(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1605,14 +1794,14 @@ RcppExport SEXP _arrow_dataset___DirectoryPartitioning__MakeFactory(SEXP field_n
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::Partitioning> dataset___HivePartitioning(const std::shared_ptr<arrow::Schema>& schm);
-RcppExport SEXP _arrow_dataset___HivePartitioning(SEXP schm_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schm(schm_sexp);
-	return Rcpp::wrap(dataset___HivePartitioning(schm));
-END_RCPP
+extern "C" SEXP _arrow_dataset___HivePartitioning(SEXP schm_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type schm(schm_sexp);
+	return cpp11::as_sexp(dataset___HivePartitioning(schm));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___HivePartitioning(SEXP schm_sexp){
+extern "C" SEXP _arrow_dataset___HivePartitioning(SEXP schm_sexp){
 	Rf_error("Cannot call dataset___HivePartitioning(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1620,13 +1809,13 @@ RcppExport SEXP _arrow_dataset___HivePartitioning(SEXP schm_sexp){
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::PartitioningFactory> dataset___HivePartitioning__MakeFactory();
-RcppExport SEXP _arrow_dataset___HivePartitioning__MakeFactory(){
-BEGIN_RCPP
-	return Rcpp::wrap(dataset___HivePartitioning__MakeFactory());
-END_RCPP
+extern "C" SEXP _arrow_dataset___HivePartitioning__MakeFactory(){
+BEGIN_CPP11
+	return cpp11::as_sexp(dataset___HivePartitioning__MakeFactory());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___HivePartitioning__MakeFactory(){
+extern "C" SEXP _arrow_dataset___HivePartitioning__MakeFactory(){
 	Rf_error("Cannot call dataset___HivePartitioning__MakeFactory(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1634,16 +1823,16 @@ RcppExport SEXP _arrow_dataset___HivePartitioning__MakeFactory(){
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void dataset___ScannerBuilder__Project(const std::shared_ptr<ds::ScannerBuilder>& sb, const std::vector<std::string>& cols);
-RcppExport SEXP _arrow_dataset___ScannerBuilder__Project(SEXP sb_sexp, SEXP cols_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::ScannerBuilder>&>::type sb(sb_sexp);
-	Rcpp::traits::input_parameter<const std::vector<std::string>&>::type cols(cols_sexp);
+extern "C" SEXP _arrow_dataset___ScannerBuilder__Project(SEXP sb_sexp, SEXP cols_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::ScannerBuilder>&>::type sb(sb_sexp);
+	arrow::r::Input<const std::vector<std::string>&>::type cols(cols_sexp);
 	dataset___ScannerBuilder__Project(sb, cols);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___ScannerBuilder__Project(SEXP sb_sexp, SEXP cols_sexp){
+extern "C" SEXP _arrow_dataset___ScannerBuilder__Project(SEXP sb_sexp, SEXP cols_sexp){
 	Rf_error("Cannot call dataset___ScannerBuilder__Project(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1651,16 +1840,16 @@ RcppExport SEXP _arrow_dataset___ScannerBuilder__Project(SEXP sb_sexp, SEXP cols
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void dataset___ScannerBuilder__Filter(const std::shared_ptr<ds::ScannerBuilder>& sb, const std::shared_ptr<ds::Expression>& expr);
-RcppExport SEXP _arrow_dataset___ScannerBuilder__Filter(SEXP sb_sexp, SEXP expr_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::ScannerBuilder>&>::type sb(sb_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type expr(expr_sexp);
+extern "C" SEXP _arrow_dataset___ScannerBuilder__Filter(SEXP sb_sexp, SEXP expr_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::ScannerBuilder>&>::type sb(sb_sexp);
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type expr(expr_sexp);
 	dataset___ScannerBuilder__Filter(sb, expr);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___ScannerBuilder__Filter(SEXP sb_sexp, SEXP expr_sexp){
+extern "C" SEXP _arrow_dataset___ScannerBuilder__Filter(SEXP sb_sexp, SEXP expr_sexp){
 	Rf_error("Cannot call dataset___ScannerBuilder__Filter(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1668,16 +1857,16 @@ RcppExport SEXP _arrow_dataset___ScannerBuilder__Filter(SEXP sb_sexp, SEXP expr_
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void dataset___ScannerBuilder__UseThreads(const std::shared_ptr<ds::ScannerBuilder>& sb, bool threads);
-RcppExport SEXP _arrow_dataset___ScannerBuilder__UseThreads(SEXP sb_sexp, SEXP threads_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::ScannerBuilder>&>::type sb(sb_sexp);
-	Rcpp::traits::input_parameter<bool>::type threads(threads_sexp);
+extern "C" SEXP _arrow_dataset___ScannerBuilder__UseThreads(SEXP sb_sexp, SEXP threads_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::ScannerBuilder>&>::type sb(sb_sexp);
+	arrow::r::Input<bool>::type threads(threads_sexp);
 	dataset___ScannerBuilder__UseThreads(sb, threads);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___ScannerBuilder__UseThreads(SEXP sb_sexp, SEXP threads_sexp){
+extern "C" SEXP _arrow_dataset___ScannerBuilder__UseThreads(SEXP sb_sexp, SEXP threads_sexp){
 	Rf_error("Cannot call dataset___ScannerBuilder__UseThreads(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1685,16 +1874,16 @@ RcppExport SEXP _arrow_dataset___ScannerBuilder__UseThreads(SEXP sb_sexp, SEXP t
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void dataset___ScannerBuilder__BatchSize(const std::shared_ptr<ds::ScannerBuilder>& sb, int64_t batch_size);
-RcppExport SEXP _arrow_dataset___ScannerBuilder__BatchSize(SEXP sb_sexp, SEXP batch_size_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::ScannerBuilder>&>::type sb(sb_sexp);
-	Rcpp::traits::input_parameter<int64_t>::type batch_size(batch_size_sexp);
+extern "C" SEXP _arrow_dataset___ScannerBuilder__BatchSize(SEXP sb_sexp, SEXP batch_size_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::ScannerBuilder>&>::type sb(sb_sexp);
+	arrow::r::Input<int64_t>::type batch_size(batch_size_sexp);
 	dataset___ScannerBuilder__BatchSize(sb, batch_size);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___ScannerBuilder__BatchSize(SEXP sb_sexp, SEXP batch_size_sexp){
+extern "C" SEXP _arrow_dataset___ScannerBuilder__BatchSize(SEXP sb_sexp, SEXP batch_size_sexp){
 	Rf_error("Cannot call dataset___ScannerBuilder__BatchSize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1702,14 +1891,14 @@ RcppExport SEXP _arrow_dataset___ScannerBuilder__BatchSize(SEXP sb_sexp, SEXP ba
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Schema> dataset___ScannerBuilder__schema(const std::shared_ptr<ds::ScannerBuilder>& sb);
-RcppExport SEXP _arrow_dataset___ScannerBuilder__schema(SEXP sb_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::ScannerBuilder>&>::type sb(sb_sexp);
-	return Rcpp::wrap(dataset___ScannerBuilder__schema(sb));
-END_RCPP
+extern "C" SEXP _arrow_dataset___ScannerBuilder__schema(SEXP sb_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::ScannerBuilder>&>::type sb(sb_sexp);
+	return cpp11::as_sexp(dataset___ScannerBuilder__schema(sb));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___ScannerBuilder__schema(SEXP sb_sexp){
+extern "C" SEXP _arrow_dataset___ScannerBuilder__schema(SEXP sb_sexp){
 	Rf_error("Cannot call dataset___ScannerBuilder__schema(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1717,14 +1906,14 @@ RcppExport SEXP _arrow_dataset___ScannerBuilder__schema(SEXP sb_sexp){
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::Scanner> dataset___ScannerBuilder__Finish(const std::shared_ptr<ds::ScannerBuilder>& sb);
-RcppExport SEXP _arrow_dataset___ScannerBuilder__Finish(SEXP sb_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::ScannerBuilder>&>::type sb(sb_sexp);
-	return Rcpp::wrap(dataset___ScannerBuilder__Finish(sb));
-END_RCPP
+extern "C" SEXP _arrow_dataset___ScannerBuilder__Finish(SEXP sb_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::ScannerBuilder>&>::type sb(sb_sexp);
+	return cpp11::as_sexp(dataset___ScannerBuilder__Finish(sb));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___ScannerBuilder__Finish(SEXP sb_sexp){
+extern "C" SEXP _arrow_dataset___ScannerBuilder__Finish(SEXP sb_sexp){
 	Rf_error("Cannot call dataset___ScannerBuilder__Finish(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1732,14 +1921,14 @@ RcppExport SEXP _arrow_dataset___ScannerBuilder__Finish(SEXP sb_sexp){
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Table> dataset___Scanner__ToTable(const std::shared_ptr<ds::Scanner>& scanner);
-RcppExport SEXP _arrow_dataset___Scanner__ToTable(SEXP scanner_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Scanner>&>::type scanner(scanner_sexp);
-	return Rcpp::wrap(dataset___Scanner__ToTable(scanner));
-END_RCPP
+extern "C" SEXP _arrow_dataset___Scanner__ToTable(SEXP scanner_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::Scanner>&>::type scanner(scanner_sexp);
+	return cpp11::as_sexp(dataset___Scanner__ToTable(scanner));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___Scanner__ToTable(SEXP scanner_sexp){
+extern "C" SEXP _arrow_dataset___Scanner__ToTable(SEXP scanner_sexp){
 	Rf_error("Cannot call dataset___Scanner__ToTable(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1747,15 +1936,15 @@ RcppExport SEXP _arrow_dataset___Scanner__ToTable(SEXP scanner_sexp){
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Table> dataset___Scanner__head(const std::shared_ptr<ds::Scanner>& scanner, int n);
-RcppExport SEXP _arrow_dataset___Scanner__head(SEXP scanner_sexp, SEXP n_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Scanner>&>::type scanner(scanner_sexp);
-	Rcpp::traits::input_parameter<int>::type n(n_sexp);
-	return Rcpp::wrap(dataset___Scanner__head(scanner, n));
-END_RCPP
+extern "C" SEXP _arrow_dataset___Scanner__head(SEXP scanner_sexp, SEXP n_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::Scanner>&>::type scanner(scanner_sexp);
+	arrow::r::Input<int>::type n(n_sexp);
+	return cpp11::as_sexp(dataset___Scanner__head(scanner, n));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___Scanner__head(SEXP scanner_sexp, SEXP n_sexp){
+extern "C" SEXP _arrow_dataset___Scanner__head(SEXP scanner_sexp, SEXP n_sexp){
 	Rf_error("Cannot call dataset___Scanner__head(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1763,50 +1952,65 @@ RcppExport SEXP _arrow_dataset___Scanner__head(SEXP scanner_sexp, SEXP n_sexp){
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::vector<std::shared_ptr<ds::ScanTask>> dataset___Scanner__Scan(const std::shared_ptr<ds::Scanner>& scanner);
-RcppExport SEXP _arrow_dataset___Scanner__Scan(SEXP scanner_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Scanner>&>::type scanner(scanner_sexp);
-	return Rcpp::wrap(dataset___Scanner__Scan(scanner));
-END_RCPP
+extern "C" SEXP _arrow_dataset___Scanner__Scan(SEXP scanner_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::Scanner>&>::type scanner(scanner_sexp);
+	return cpp11::as_sexp(dataset___Scanner__Scan(scanner));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___Scanner__Scan(SEXP scanner_sexp){
+extern "C" SEXP _arrow_dataset___Scanner__Scan(SEXP scanner_sexp){
 	Rf_error("Cannot call dataset___Scanner__Scan(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
+// dataset.cpp
+#if defined(ARROW_R_WITH_ARROW)
+std::shared_ptr<arrow::Schema> dataset___Scanner__schema(const std::shared_ptr<ds::Scanner>& sc);
+extern "C" SEXP _arrow_dataset___Scanner__schema(SEXP sc_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::Scanner>&>::type sc(sc_sexp);
+	return cpp11::as_sexp(dataset___Scanner__schema(sc));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_dataset___Scanner__schema(SEXP sc_sexp){
+	Rf_error("Cannot call dataset___Scanner__schema(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::vector<std::shared_ptr<arrow::RecordBatch>> dataset___ScanTask__get_batches(const std::shared_ptr<ds::ScanTask>& scan_task);
-RcppExport SEXP _arrow_dataset___ScanTask__get_batches(SEXP scan_task_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::ScanTask>&>::type scan_task(scan_task_sexp);
-	return Rcpp::wrap(dataset___ScanTask__get_batches(scan_task));
-END_RCPP
+extern "C" SEXP _arrow_dataset___ScanTask__get_batches(SEXP scan_task_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::ScanTask>&>::type scan_task(scan_task_sexp);
+	return cpp11::as_sexp(dataset___ScanTask__get_batches(scan_task));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___ScanTask__get_batches(SEXP scan_task_sexp){
+extern "C" SEXP _arrow_dataset___ScanTask__get_batches(SEXP scan_task_sexp){
 	Rf_error("Cannot call dataset___ScanTask__get_batches(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // dataset.cpp
 #if defined(ARROW_R_WITH_ARROW)
-void dataset___Dataset__Write(const std::shared_ptr<ds::Dataset>& ds, const std::shared_ptr<arrow::Schema>& schema, const std::shared_ptr<ds::FileFormat>& format, const std::shared_ptr<fs::FileSystem>& filesystem, std::string path, const std::shared_ptr<ds::Partitioning>& partitioning);
-RcppExport SEXP _arrow_dataset___Dataset__Write(SEXP ds_sexp, SEXP schema_sexp, SEXP format_sexp, SEXP filesystem_sexp, SEXP path_sexp, SEXP partitioning_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Dataset>&>::type ds(ds_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::FileFormat>&>::type format(format_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSystem>&>::type filesystem(filesystem_sexp);
-	Rcpp::traits::input_parameter<std::string>::type path(path_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Partitioning>&>::type partitioning(partitioning_sexp);
-	dataset___Dataset__Write(ds, schema, format, filesystem, path, partitioning);
+void dataset___Dataset__Write(const std::shared_ptr<ds::FileWriteOptions>& file_write_options, const std::shared_ptr<fs::FileSystem>& filesystem, std::string base_dir, const std::shared_ptr<ds::Partitioning>& partitioning, std::string basename_template, const std::shared_ptr<ds::Scanner>& scanner);
+extern "C" SEXP _arrow_dataset___Dataset__Write(SEXP file_write_options_sexp, SEXP filesystem_sexp, SEXP base_dir_sexp, SEXP partitioning_sexp, SEXP basename_template_sexp, SEXP scanner_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::FileWriteOptions>&>::type file_write_options(file_write_options_sexp);
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type filesystem(filesystem_sexp);
+	arrow::r::Input<std::string>::type base_dir(base_dir_sexp);
+	arrow::r::Input<const std::shared_ptr<ds::Partitioning>&>::type partitioning(partitioning_sexp);
+	arrow::r::Input<std::string>::type basename_template(basename_template_sexp);
+	arrow::r::Input<const std::shared_ptr<ds::Scanner>&>::type scanner(scanner_sexp);
+	dataset___Dataset__Write(file_write_options, filesystem, base_dir, partitioning, basename_template, scanner);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___Dataset__Write(SEXP ds_sexp, SEXP schema_sexp, SEXP format_sexp, SEXP filesystem_sexp, SEXP path_sexp, SEXP partitioning_sexp){
+extern "C" SEXP _arrow_dataset___Dataset__Write(SEXP file_write_options_sexp, SEXP filesystem_sexp, SEXP base_dir_sexp, SEXP partitioning_sexp, SEXP basename_template_sexp, SEXP scanner_sexp){
 	Rf_error("Cannot call dataset___Dataset__Write(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1814,14 +2018,14 @@ RcppExport SEXP _arrow_dataset___Dataset__Write(SEXP ds_sexp, SEXP schema_sexp,
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool shared_ptr_is_null(SEXP xp);
-RcppExport SEXP _arrow_shared_ptr_is_null(SEXP xp_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<SEXP>::type xp(xp_sexp);
-	return Rcpp::wrap(shared_ptr_is_null(xp));
-END_RCPP
+extern "C" SEXP _arrow_shared_ptr_is_null(SEXP xp_sexp){
+BEGIN_CPP11
+	arrow::r::Input<SEXP>::type xp(xp_sexp);
+	return cpp11::as_sexp(shared_ptr_is_null(xp));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_shared_ptr_is_null(SEXP xp_sexp){
+extern "C" SEXP _arrow_shared_ptr_is_null(SEXP xp_sexp){
 	Rf_error("Cannot call shared_ptr_is_null(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1829,14 +2033,14 @@ RcppExport SEXP _arrow_shared_ptr_is_null(SEXP xp_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool unique_ptr_is_null(SEXP xp);
-RcppExport SEXP _arrow_unique_ptr_is_null(SEXP xp_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<SEXP>::type xp(xp_sexp);
-	return Rcpp::wrap(unique_ptr_is_null(xp));
-END_RCPP
+extern "C" SEXP _arrow_unique_ptr_is_null(SEXP xp_sexp){
+BEGIN_CPP11
+	arrow::r::Input<SEXP>::type xp(xp_sexp);
+	return cpp11::as_sexp(unique_ptr_is_null(xp));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_unique_ptr_is_null(SEXP xp_sexp){
+extern "C" SEXP _arrow_unique_ptr_is_null(SEXP xp_sexp){
 	Rf_error("Cannot call unique_ptr_is_null(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1844,13 +2048,13 @@ RcppExport SEXP _arrow_unique_ptr_is_null(SEXP xp_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Int8__initialize();
-RcppExport SEXP _arrow_Int8__initialize(){
-BEGIN_RCPP
-	return Rcpp::wrap(Int8__initialize());
-END_RCPP
+extern "C" SEXP _arrow_Int8__initialize(){
+BEGIN_CPP11
+	return cpp11::as_sexp(Int8__initialize());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Int8__initialize(){
+extern "C" SEXP _arrow_Int8__initialize(){
 	Rf_error("Cannot call Int8__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1858,13 +2062,13 @@ RcppExport SEXP _arrow_Int8__initialize(){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Int16__initialize();
-RcppExport SEXP _arrow_Int16__initialize(){
-BEGIN_RCPP
-	return Rcpp::wrap(Int16__initialize());
-END_RCPP
+extern "C" SEXP _arrow_Int16__initialize(){
+BEGIN_CPP11
+	return cpp11::as_sexp(Int16__initialize());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Int16__initialize(){
+extern "C" SEXP _arrow_Int16__initialize(){
 	Rf_error("Cannot call Int16__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1872,13 +2076,13 @@ RcppExport SEXP _arrow_Int16__initialize(){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Int32__initialize();
-RcppExport SEXP _arrow_Int32__initialize(){
-BEGIN_RCPP
-	return Rcpp::wrap(Int32__initialize());
-END_RCPP
+extern "C" SEXP _arrow_Int32__initialize(){
+BEGIN_CPP11
+	return cpp11::as_sexp(Int32__initialize());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Int32__initialize(){
+extern "C" SEXP _arrow_Int32__initialize(){
 	Rf_error("Cannot call Int32__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1886,13 +2090,13 @@ RcppExport SEXP _arrow_Int32__initialize(){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Int64__initialize();
-RcppExport SEXP _arrow_Int64__initialize(){
-BEGIN_RCPP
-	return Rcpp::wrap(Int64__initialize());
-END_RCPP
+extern "C" SEXP _arrow_Int64__initialize(){
+BEGIN_CPP11
+	return cpp11::as_sexp(Int64__initialize());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Int64__initialize(){
+extern "C" SEXP _arrow_Int64__initialize(){
 	Rf_error("Cannot call Int64__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1900,13 +2104,13 @@ RcppExport SEXP _arrow_Int64__initialize(){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> UInt8__initialize();
-RcppExport SEXP _arrow_UInt8__initialize(){
-BEGIN_RCPP
-	return Rcpp::wrap(UInt8__initialize());
-END_RCPP
+extern "C" SEXP _arrow_UInt8__initialize(){
+BEGIN_CPP11
+	return cpp11::as_sexp(UInt8__initialize());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_UInt8__initialize(){
+extern "C" SEXP _arrow_UInt8__initialize(){
 	Rf_error("Cannot call UInt8__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1914,13 +2118,13 @@ RcppExport SEXP _arrow_UInt8__initialize(){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> UInt16__initialize();
-RcppExport SEXP _arrow_UInt16__initialize(){
-BEGIN_RCPP
-	return Rcpp::wrap(UInt16__initialize());
-END_RCPP
+extern "C" SEXP _arrow_UInt16__initialize(){
+BEGIN_CPP11
+	return cpp11::as_sexp(UInt16__initialize());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_UInt16__initialize(){
+extern "C" SEXP _arrow_UInt16__initialize(){
 	Rf_error("Cannot call UInt16__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1928,13 +2132,13 @@ RcppExport SEXP _arrow_UInt16__initialize(){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> UInt32__initialize();
-RcppExport SEXP _arrow_UInt32__initialize(){
-BEGIN_RCPP
-	return Rcpp::wrap(UInt32__initialize());
-END_RCPP
+extern "C" SEXP _arrow_UInt32__initialize(){
+BEGIN_CPP11
+	return cpp11::as_sexp(UInt32__initialize());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_UInt32__initialize(){
+extern "C" SEXP _arrow_UInt32__initialize(){
 	Rf_error("Cannot call UInt32__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1942,13 +2146,13 @@ RcppExport SEXP _arrow_UInt32__initialize(){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> UInt64__initialize();
-RcppExport SEXP _arrow_UInt64__initialize(){
-BEGIN_RCPP
-	return Rcpp::wrap(UInt64__initialize());
-END_RCPP
+extern "C" SEXP _arrow_UInt64__initialize(){
+BEGIN_CPP11
+	return cpp11::as_sexp(UInt64__initialize());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_UInt64__initialize(){
+extern "C" SEXP _arrow_UInt64__initialize(){
 	Rf_error("Cannot call UInt64__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1956,13 +2160,13 @@ RcppExport SEXP _arrow_UInt64__initialize(){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Float16__initialize();
-RcppExport SEXP _arrow_Float16__initialize(){
-BEGIN_RCPP
-	return Rcpp::wrap(Float16__initialize());
-END_RCPP
+extern "C" SEXP _arrow_Float16__initialize(){
+BEGIN_CPP11
+	return cpp11::as_sexp(Float16__initialize());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Float16__initialize(){
+extern "C" SEXP _arrow_Float16__initialize(){
 	Rf_error("Cannot call Float16__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1970,13 +2174,13 @@ RcppExport SEXP _arrow_Float16__initialize(){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Float32__initialize();
-RcppExport SEXP _arrow_Float32__initialize(){
-BEGIN_RCPP
-	return Rcpp::wrap(Float32__initialize());
-END_RCPP
+extern "C" SEXP _arrow_Float32__initialize(){
+BEGIN_CPP11
+	return cpp11::as_sexp(Float32__initialize());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Float32__initialize(){
+extern "C" SEXP _arrow_Float32__initialize(){
 	Rf_error("Cannot call Float32__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1984,13 +2188,13 @@ RcppExport SEXP _arrow_Float32__initialize(){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Float64__initialize();
-RcppExport SEXP _arrow_Float64__initialize(){
-BEGIN_RCPP
-	return Rcpp::wrap(Float64__initialize());
-END_RCPP
+extern "C" SEXP _arrow_Float64__initialize(){
+BEGIN_CPP11
+	return cpp11::as_sexp(Float64__initialize());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Float64__initialize(){
+extern "C" SEXP _arrow_Float64__initialize(){
 	Rf_error("Cannot call Float64__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -1998,13 +2202,13 @@ RcppExport SEXP _arrow_Float64__initialize(){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Boolean__initialize();
-RcppExport SEXP _arrow_Boolean__initialize(){
-BEGIN_RCPP
-	return Rcpp::wrap(Boolean__initialize());
-END_RCPP
+extern "C" SEXP _arrow_Boolean__initialize(){
+BEGIN_CPP11
+	return cpp11::as_sexp(Boolean__initialize());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Boolean__initialize(){
+extern "C" SEXP _arrow_Boolean__initialize(){
 	Rf_error("Cannot call Boolean__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2012,13 +2216,13 @@ RcppExport SEXP _arrow_Boolean__initialize(){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Utf8__initialize();
-RcppExport SEXP _arrow_Utf8__initialize(){
-BEGIN_RCPP
-	return Rcpp::wrap(Utf8__initialize());
-END_RCPP
+extern "C" SEXP _arrow_Utf8__initialize(){
+BEGIN_CPP11
+	return cpp11::as_sexp(Utf8__initialize());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Utf8__initialize(){
+extern "C" SEXP _arrow_Utf8__initialize(){
 	Rf_error("Cannot call Utf8__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2026,13 +2230,13 @@ RcppExport SEXP _arrow_Utf8__initialize(){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> LargeUtf8__initialize();
-RcppExport SEXP _arrow_LargeUtf8__initialize(){
-BEGIN_RCPP
-	return Rcpp::wrap(LargeUtf8__initialize());
-END_RCPP
+extern "C" SEXP _arrow_LargeUtf8__initialize(){
+BEGIN_CPP11
+	return cpp11::as_sexp(LargeUtf8__initialize());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_LargeUtf8__initialize(){
+extern "C" SEXP _arrow_LargeUtf8__initialize(){
 	Rf_error("Cannot call LargeUtf8__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2040,13 +2244,13 @@ RcppExport SEXP _arrow_LargeUtf8__initialize(){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Binary__initialize();
-RcppExport SEXP _arrow_Binary__initialize(){
-BEGIN_RCPP
-	return Rcpp::wrap(Binary__initialize());
-END_RCPP
+extern "C" SEXP _arrow_Binary__initialize(){
+BEGIN_CPP11
+	return cpp11::as_sexp(Binary__initialize());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Binary__initialize(){
+extern "C" SEXP _arrow_Binary__initialize(){
 	Rf_error("Cannot call Binary__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2054,13 +2258,13 @@ RcppExport SEXP _arrow_Binary__initialize(){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> LargeBinary__initialize();
-RcppExport SEXP _arrow_LargeBinary__initialize(){
-BEGIN_RCPP
-	return Rcpp::wrap(LargeBinary__initialize());
-END_RCPP
+extern "C" SEXP _arrow_LargeBinary__initialize(){
+BEGIN_CPP11
+	return cpp11::as_sexp(LargeBinary__initialize());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_LargeBinary__initialize(){
+extern "C" SEXP _arrow_LargeBinary__initialize(){
 	Rf_error("Cannot call LargeBinary__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2068,13 +2272,13 @@ RcppExport SEXP _arrow_LargeBinary__initialize(){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Date32__initialize();
-RcppExport SEXP _arrow_Date32__initialize(){
-BEGIN_RCPP
-	return Rcpp::wrap(Date32__initialize());
-END_RCPP
+extern "C" SEXP _arrow_Date32__initialize(){
+BEGIN_CPP11
+	return cpp11::as_sexp(Date32__initialize());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Date32__initialize(){
+extern "C" SEXP _arrow_Date32__initialize(){
 	Rf_error("Cannot call Date32__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2082,13 +2286,13 @@ RcppExport SEXP _arrow_Date32__initialize(){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Date64__initialize();
-RcppExport SEXP _arrow_Date64__initialize(){
-BEGIN_RCPP
-	return Rcpp::wrap(Date64__initialize());
-END_RCPP
+extern "C" SEXP _arrow_Date64__initialize(){
+BEGIN_CPP11
+	return cpp11::as_sexp(Date64__initialize());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Date64__initialize(){
+extern "C" SEXP _arrow_Date64__initialize(){
 	Rf_error("Cannot call Date64__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2096,13 +2300,13 @@ RcppExport SEXP _arrow_Date64__initialize(){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Null__initialize();
-RcppExport SEXP _arrow_Null__initialize(){
-BEGIN_RCPP
-	return Rcpp::wrap(Null__initialize());
-END_RCPP
+extern "C" SEXP _arrow_Null__initialize(){
+BEGIN_CPP11
+	return cpp11::as_sexp(Null__initialize());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Null__initialize(){
+extern "C" SEXP _arrow_Null__initialize(){
 	Rf_error("Cannot call Null__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2110,30 +2314,30 @@ RcppExport SEXP _arrow_Null__initialize(){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Decimal128Type__initialize(int32_t precision, int32_t scale);
-RcppExport SEXP _arrow_Decimal128Type__initialize(SEXP precision_sexp, SEXP scale_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<int32_t>::type precision(precision_sexp);
-	Rcpp::traits::input_parameter<int32_t>::type scale(scale_sexp);
-	return Rcpp::wrap(Decimal128Type__initialize(precision, scale));
-END_RCPP
+extern "C" SEXP _arrow_Decimal128Type__initialize(SEXP precision_sexp, SEXP scale_sexp){
+BEGIN_CPP11
+	arrow::r::Input<int32_t>::type precision(precision_sexp);
+	arrow::r::Input<int32_t>::type scale(scale_sexp);
+	return cpp11::as_sexp(Decimal128Type__initialize(precision, scale));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Decimal128Type__initialize(SEXP precision_sexp, SEXP scale_sexp){
+extern "C" SEXP _arrow_Decimal128Type__initialize(SEXP precision_sexp, SEXP scale_sexp){
 	Rf_error("Cannot call Decimal128Type__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::DataType> FixedSizeBinary__initialize(int32_t byte_width);
-RcppExport SEXP _arrow_FixedSizeBinary__initialize(SEXP byte_width_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<int32_t>::type byte_width(byte_width_sexp);
-	return Rcpp::wrap(FixedSizeBinary__initialize(byte_width));
-END_RCPP
+std::shared_ptr<arrow::DataType> FixedSizeBinary__initialize(R_xlen_t byte_width);
+extern "C" SEXP _arrow_FixedSizeBinary__initialize(SEXP byte_width_sexp){
+BEGIN_CPP11
+	arrow::r::Input<R_xlen_t>::type byte_width(byte_width_sexp);
+	return cpp11::as_sexp(FixedSizeBinary__initialize(byte_width));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_FixedSizeBinary__initialize(SEXP byte_width_sexp){
+extern "C" SEXP _arrow_FixedSizeBinary__initialize(SEXP byte_width_sexp){
 	Rf_error("Cannot call FixedSizeBinary__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2141,15 +2345,15 @@ RcppExport SEXP _arrow_FixedSizeBinary__initialize(SEXP byte_width_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Timestamp__initialize(arrow::TimeUnit::type unit, const std::string& timezone);
-RcppExport SEXP _arrow_Timestamp__initialize(SEXP unit_sexp, SEXP timezone_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<arrow::TimeUnit::type>::type unit(unit_sexp);
-	Rcpp::traits::input_parameter<const std::string&>::type timezone(timezone_sexp);
-	return Rcpp::wrap(Timestamp__initialize(unit, timezone));
-END_RCPP
+extern "C" SEXP _arrow_Timestamp__initialize(SEXP unit_sexp, SEXP timezone_sexp){
+BEGIN_CPP11
+	arrow::r::Input<arrow::TimeUnit::type>::type unit(unit_sexp);
+	arrow::r::Input<const std::string&>::type timezone(timezone_sexp);
+	return cpp11::as_sexp(Timestamp__initialize(unit, timezone));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Timestamp__initialize(SEXP unit_sexp, SEXP timezone_sexp){
+extern "C" SEXP _arrow_Timestamp__initialize(SEXP unit_sexp, SEXP timezone_sexp){
 	Rf_error("Cannot call Timestamp__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2157,14 +2361,14 @@ RcppExport SEXP _arrow_Timestamp__initialize(SEXP unit_sexp, SEXP timezone_sexp)
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Time32__initialize(arrow::TimeUnit::type unit);
-RcppExport SEXP _arrow_Time32__initialize(SEXP unit_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<arrow::TimeUnit::type>::type unit(unit_sexp);
-	return Rcpp::wrap(Time32__initialize(unit));
-END_RCPP
+extern "C" SEXP _arrow_Time32__initialize(SEXP unit_sexp){
+BEGIN_CPP11
+	arrow::r::Input<arrow::TimeUnit::type>::type unit(unit_sexp);
+	return cpp11::as_sexp(Time32__initialize(unit));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Time32__initialize(SEXP unit_sexp){
+extern "C" SEXP _arrow_Time32__initialize(SEXP unit_sexp){
 	Rf_error("Cannot call Time32__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2172,14 +2376,14 @@ RcppExport SEXP _arrow_Time32__initialize(SEXP unit_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Time64__initialize(arrow::TimeUnit::type unit);
-RcppExport SEXP _arrow_Time64__initialize(SEXP unit_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<arrow::TimeUnit::type>::type unit(unit_sexp);
-	return Rcpp::wrap(Time64__initialize(unit));
-END_RCPP
+extern "C" SEXP _arrow_Time64__initialize(SEXP unit_sexp){
+BEGIN_CPP11
+	arrow::r::Input<arrow::TimeUnit::type>::type unit(unit_sexp);
+	return cpp11::as_sexp(Time64__initialize(unit));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Time64__initialize(SEXP unit_sexp){
+extern "C" SEXP _arrow_Time64__initialize(SEXP unit_sexp){
 	Rf_error("Cannot call Time64__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2187,14 +2391,14 @@ RcppExport SEXP _arrow_Time64__initialize(SEXP unit_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 SEXP list__(SEXP x);
-RcppExport SEXP _arrow_list__(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<SEXP>::type x(x_sexp);
-	return Rcpp::wrap(list__(x));
-END_RCPP
+extern "C" SEXP _arrow_list__(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<SEXP>::type x(x_sexp);
+	return cpp11::as_sexp(list__(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_list__(SEXP x_sexp){
+extern "C" SEXP _arrow_list__(SEXP x_sexp){
 	Rf_error("Cannot call list__(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2202,14 +2406,14 @@ RcppExport SEXP _arrow_list__(SEXP x_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 SEXP large_list__(SEXP x);
-RcppExport SEXP _arrow_large_list__(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<SEXP>::type x(x_sexp);
-	return Rcpp::wrap(large_list__(x));
-END_RCPP
+extern "C" SEXP _arrow_large_list__(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<SEXP>::type x(x_sexp);
+	return cpp11::as_sexp(large_list__(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_large_list__(SEXP x_sexp){
+extern "C" SEXP _arrow_large_list__(SEXP x_sexp){
 	Rf_error("Cannot call large_list__(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2217,45 +2421,45 @@ RcppExport SEXP _arrow_large_list__(SEXP x_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 SEXP fixed_size_list__(SEXP x, int list_size);
-RcppExport SEXP _arrow_fixed_size_list__(SEXP x_sexp, SEXP list_size_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<SEXP>::type x(x_sexp);
-	Rcpp::traits::input_parameter<int>::type list_size(list_size_sexp);
-	return Rcpp::wrap(fixed_size_list__(x, list_size));
-END_RCPP
+extern "C" SEXP _arrow_fixed_size_list__(SEXP x_sexp, SEXP list_size_sexp){
+BEGIN_CPP11
+	arrow::r::Input<SEXP>::type x(x_sexp);
+	arrow::r::Input<int>::type list_size(list_size_sexp);
+	return cpp11::as_sexp(fixed_size_list__(x, list_size));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fixed_size_list__(SEXP x_sexp, SEXP list_size_sexp){
+extern "C" SEXP _arrow_fixed_size_list__(SEXP x_sexp, SEXP list_size_sexp){
 	Rf_error("Cannot call fixed_size_list__(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::DataType> struct_(List fields);
-RcppExport SEXP _arrow_struct_(SEXP fields_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<List>::type fields(fields_sexp);
-	return Rcpp::wrap(struct_(fields));
-END_RCPP
+std::shared_ptr<arrow::DataType> struct__(const std::vector<std::shared_ptr<arrow::Field>>& fields);
+extern "C" SEXP _arrow_struct__(SEXP fields_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::vector<std::shared_ptr<arrow::Field>>&>::type fields(fields_sexp);
+	return cpp11::as_sexp(struct__(fields));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_struct_(SEXP fields_sexp){
-	Rf_error("Cannot call struct_(). Please use arrow::install_arrow() to install required runtime libraries. ");
+extern "C" SEXP _arrow_struct__(SEXP fields_sexp){
+	Rf_error("Cannot call struct__(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::string DataType__ToString(const std::shared_ptr<arrow::DataType>& type);
-RcppExport SEXP _arrow_DataType__ToString(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DataType>&>::type type(type_sexp);
-	return Rcpp::wrap(DataType__ToString(type));
-END_RCPP
+extern "C" SEXP _arrow_DataType__ToString(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::DataType>&>::type type(type_sexp);
+	return cpp11::as_sexp(DataType__ToString(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_DataType__ToString(SEXP type_sexp){
+extern "C" SEXP _arrow_DataType__ToString(SEXP type_sexp){
 	Rf_error("Cannot call DataType__ToString(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2263,14 +2467,14 @@ RcppExport SEXP _arrow_DataType__ToString(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::string DataType__name(const std::shared_ptr<arrow::DataType>& type);
-RcppExport SEXP _arrow_DataType__name(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DataType>&>::type type(type_sexp);
-	return Rcpp::wrap(DataType__name(type));
-END_RCPP
+extern "C" SEXP _arrow_DataType__name(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::DataType>&>::type type(type_sexp);
+	return cpp11::as_sexp(DataType__name(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_DataType__name(SEXP type_sexp){
+extern "C" SEXP _arrow_DataType__name(SEXP type_sexp){
 	Rf_error("Cannot call DataType__name(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2278,15 +2482,15 @@ RcppExport SEXP _arrow_DataType__name(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool DataType__Equals(const std::shared_ptr<arrow::DataType>& lhs, const std::shared_ptr<arrow::DataType>& rhs);
-RcppExport SEXP _arrow_DataType__Equals(SEXP lhs_sexp, SEXP rhs_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DataType>&>::type lhs(lhs_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DataType>&>::type rhs(rhs_sexp);
-	return Rcpp::wrap(DataType__Equals(lhs, rhs));
-END_RCPP
+extern "C" SEXP _arrow_DataType__Equals(SEXP lhs_sexp, SEXP rhs_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::DataType>&>::type lhs(lhs_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::DataType>&>::type rhs(rhs_sexp);
+	return cpp11::as_sexp(DataType__Equals(lhs, rhs));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_DataType__Equals(SEXP lhs_sexp, SEXP rhs_sexp){
+extern "C" SEXP _arrow_DataType__Equals(SEXP lhs_sexp, SEXP rhs_sexp){
 	Rf_error("Cannot call DataType__Equals(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2294,29 +2498,29 @@ RcppExport SEXP _arrow_DataType__Equals(SEXP lhs_sexp, SEXP rhs_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int DataType__num_children(const std::shared_ptr<arrow::DataType>& type);
-RcppExport SEXP _arrow_DataType__num_children(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DataType>&>::type type(type_sexp);
-	return Rcpp::wrap(DataType__num_children(type));
-END_RCPP
+extern "C" SEXP _arrow_DataType__num_children(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::DataType>&>::type type(type_sexp);
+	return cpp11::as_sexp(DataType__num_children(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_DataType__num_children(SEXP type_sexp){
+extern "C" SEXP _arrow_DataType__num_children(SEXP type_sexp){
 	Rf_error("Cannot call DataType__num_children(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
-List DataType__children_pointer(const std::shared_ptr<arrow::DataType>& type);
-RcppExport SEXP _arrow_DataType__children_pointer(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DataType>&>::type type(type_sexp);
-	return Rcpp::wrap(DataType__children_pointer(type));
-END_RCPP
+cpp11::writable::list DataType__children_pointer(const std::shared_ptr<arrow::DataType>& type);
+extern "C" SEXP _arrow_DataType__children_pointer(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::DataType>&>::type type(type_sexp);
+	return cpp11::as_sexp(DataType__children_pointer(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_DataType__children_pointer(SEXP type_sexp){
+extern "C" SEXP _arrow_DataType__children_pointer(SEXP type_sexp){
 	Rf_error("Cannot call DataType__children_pointer(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2324,14 +2528,14 @@ RcppExport SEXP _arrow_DataType__children_pointer(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 arrow::Type::type DataType__id(const std::shared_ptr<arrow::DataType>& type);
-RcppExport SEXP _arrow_DataType__id(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DataType>&>::type type(type_sexp);
-	return Rcpp::wrap(DataType__id(type));
-END_RCPP
+extern "C" SEXP _arrow_DataType__id(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::DataType>&>::type type(type_sexp);
+	return cpp11::as_sexp(DataType__id(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_DataType__id(SEXP type_sexp){
+extern "C" SEXP _arrow_DataType__id(SEXP type_sexp){
 	Rf_error("Cannot call DataType__id(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2339,14 +2543,14 @@ RcppExport SEXP _arrow_DataType__id(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::string ListType__ToString(const std::shared_ptr<arrow::ListType>& type);
-RcppExport SEXP _arrow_ListType__ToString(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ListType>&>::type type(type_sexp);
-	return Rcpp::wrap(ListType__ToString(type));
-END_RCPP
+extern "C" SEXP _arrow_ListType__ToString(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ListType>&>::type type(type_sexp);
+	return cpp11::as_sexp(ListType__ToString(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ListType__ToString(SEXP type_sexp){
+extern "C" SEXP _arrow_ListType__ToString(SEXP type_sexp){
 	Rf_error("Cannot call ListType__ToString(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2354,14 +2558,14 @@ RcppExport SEXP _arrow_ListType__ToString(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int FixedWidthType__bit_width(const std::shared_ptr<arrow::FixedWidthType>& type);
-RcppExport SEXP _arrow_FixedWidthType__bit_width(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::FixedWidthType>&>::type type(type_sexp);
-	return Rcpp::wrap(FixedWidthType__bit_width(type));
-END_RCPP
+extern "C" SEXP _arrow_FixedWidthType__bit_width(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::FixedWidthType>&>::type type(type_sexp);
+	return cpp11::as_sexp(FixedWidthType__bit_width(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_FixedWidthType__bit_width(SEXP type_sexp){
+extern "C" SEXP _arrow_FixedWidthType__bit_width(SEXP type_sexp){
 	Rf_error("Cannot call FixedWidthType__bit_width(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2369,14 +2573,14 @@ RcppExport SEXP _arrow_FixedWidthType__bit_width(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 arrow::DateUnit DateType__unit(const std::shared_ptr<arrow::DateType>& type);
-RcppExport SEXP _arrow_DateType__unit(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DateType>&>::type type(type_sexp);
-	return Rcpp::wrap(DateType__unit(type));
-END_RCPP
+extern "C" SEXP _arrow_DateType__unit(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::DateType>&>::type type(type_sexp);
+	return cpp11::as_sexp(DateType__unit(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_DateType__unit(SEXP type_sexp){
+extern "C" SEXP _arrow_DateType__unit(SEXP type_sexp){
 	Rf_error("Cannot call DateType__unit(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2384,14 +2588,14 @@ RcppExport SEXP _arrow_DateType__unit(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 arrow::TimeUnit::type TimeType__unit(const std::shared_ptr<arrow::TimeType>& type);
-RcppExport SEXP _arrow_TimeType__unit(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::TimeType>&>::type type(type_sexp);
-	return Rcpp::wrap(TimeType__unit(type));
-END_RCPP
+extern "C" SEXP _arrow_TimeType__unit(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::TimeType>&>::type type(type_sexp);
+	return cpp11::as_sexp(TimeType__unit(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_TimeType__unit(SEXP type_sexp){
+extern "C" SEXP _arrow_TimeType__unit(SEXP type_sexp){
 	Rf_error("Cannot call TimeType__unit(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2399,14 +2603,14 @@ RcppExport SEXP _arrow_TimeType__unit(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int32_t DecimalType__precision(const std::shared_ptr<arrow::DecimalType>& type);
-RcppExport SEXP _arrow_DecimalType__precision(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DecimalType>&>::type type(type_sexp);
-	return Rcpp::wrap(DecimalType__precision(type));
-END_RCPP
+extern "C" SEXP _arrow_DecimalType__precision(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::DecimalType>&>::type type(type_sexp);
+	return cpp11::as_sexp(DecimalType__precision(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_DecimalType__precision(SEXP type_sexp){
+extern "C" SEXP _arrow_DecimalType__precision(SEXP type_sexp){
 	Rf_error("Cannot call DecimalType__precision(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2414,14 +2618,14 @@ RcppExport SEXP _arrow_DecimalType__precision(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int32_t DecimalType__scale(const std::shared_ptr<arrow::DecimalType>& type);
-RcppExport SEXP _arrow_DecimalType__scale(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DecimalType>&>::type type(type_sexp);
-	return Rcpp::wrap(DecimalType__scale(type));
-END_RCPP
+extern "C" SEXP _arrow_DecimalType__scale(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::DecimalType>&>::type type(type_sexp);
+	return cpp11::as_sexp(DecimalType__scale(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_DecimalType__scale(SEXP type_sexp){
+extern "C" SEXP _arrow_DecimalType__scale(SEXP type_sexp){
 	Rf_error("Cannot call DecimalType__scale(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2429,14 +2633,14 @@ RcppExport SEXP _arrow_DecimalType__scale(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::string TimestampType__timezone(const std::shared_ptr<arrow::TimestampType>& type);
-RcppExport SEXP _arrow_TimestampType__timezone(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::TimestampType>&>::type type(type_sexp);
-	return Rcpp::wrap(TimestampType__timezone(type));
-END_RCPP
+extern "C" SEXP _arrow_TimestampType__timezone(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::TimestampType>&>::type type(type_sexp);
+	return cpp11::as_sexp(TimestampType__timezone(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_TimestampType__timezone(SEXP type_sexp){
+extern "C" SEXP _arrow_TimestampType__timezone(SEXP type_sexp){
 	Rf_error("Cannot call TimestampType__timezone(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2444,14 +2648,14 @@ RcppExport SEXP _arrow_TimestampType__timezone(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 arrow::TimeUnit::type TimestampType__unit(const std::shared_ptr<arrow::TimestampType>& type);
-RcppExport SEXP _arrow_TimestampType__unit(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::TimestampType>&>::type type(type_sexp);
-	return Rcpp::wrap(TimestampType__unit(type));
-END_RCPP
+extern "C" SEXP _arrow_TimestampType__unit(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::TimestampType>&>::type type(type_sexp);
+	return cpp11::as_sexp(TimestampType__unit(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_TimestampType__unit(SEXP type_sexp){
+extern "C" SEXP _arrow_TimestampType__unit(SEXP type_sexp){
 	Rf_error("Cannot call TimestampType__unit(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2459,16 +2663,16 @@ RcppExport SEXP _arrow_TimestampType__unit(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> DictionaryType__initialize(const std::shared_ptr<arrow::DataType>& index_type, const std::shared_ptr<arrow::DataType>& value_type, bool ordered);
-RcppExport SEXP _arrow_DictionaryType__initialize(SEXP index_type_sexp, SEXP value_type_sexp, SEXP ordered_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DataType>&>::type index_type(index_type_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DataType>&>::type value_type(value_type_sexp);
-	Rcpp::traits::input_parameter<bool>::type ordered(ordered_sexp);
-	return Rcpp::wrap(DictionaryType__initialize(index_type, value_type, ordered));
-END_RCPP
+extern "C" SEXP _arrow_DictionaryType__initialize(SEXP index_type_sexp, SEXP value_type_sexp, SEXP ordered_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::DataType>&>::type index_type(index_type_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::DataType>&>::type value_type(value_type_sexp);
+	arrow::r::Input<bool>::type ordered(ordered_sexp);
+	return cpp11::as_sexp(DictionaryType__initialize(index_type, value_type, ordered));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_DictionaryType__initialize(SEXP index_type_sexp, SEXP value_type_sexp, SEXP ordered_sexp){
+extern "C" SEXP _arrow_DictionaryType__initialize(SEXP index_type_sexp, SEXP value_type_sexp, SEXP ordered_sexp){
 	Rf_error("Cannot call DictionaryType__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2476,14 +2680,14 @@ RcppExport SEXP _arrow_DictionaryType__initialize(SEXP index_type_sexp, SEXP val
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> DictionaryType__index_type(const std::shared_ptr<arrow::DictionaryType>& type);
-RcppExport SEXP _arrow_DictionaryType__index_type(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DictionaryType>&>::type type(type_sexp);
-	return Rcpp::wrap(DictionaryType__index_type(type));
-END_RCPP
+extern "C" SEXP _arrow_DictionaryType__index_type(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::DictionaryType>&>::type type(type_sexp);
+	return cpp11::as_sexp(DictionaryType__index_type(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_DictionaryType__index_type(SEXP type_sexp){
+extern "C" SEXP _arrow_DictionaryType__index_type(SEXP type_sexp){
 	Rf_error("Cannot call DictionaryType__index_type(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2491,14 +2695,14 @@ RcppExport SEXP _arrow_DictionaryType__index_type(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> DictionaryType__value_type(const std::shared_ptr<arrow::DictionaryType>& type);
-RcppExport SEXP _arrow_DictionaryType__value_type(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DictionaryType>&>::type type(type_sexp);
-	return Rcpp::wrap(DictionaryType__value_type(type));
-END_RCPP
+extern "C" SEXP _arrow_DictionaryType__value_type(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::DictionaryType>&>::type type(type_sexp);
+	return cpp11::as_sexp(DictionaryType__value_type(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_DictionaryType__value_type(SEXP type_sexp){
+extern "C" SEXP _arrow_DictionaryType__value_type(SEXP type_sexp){
 	Rf_error("Cannot call DictionaryType__value_type(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2506,14 +2710,14 @@ RcppExport SEXP _arrow_DictionaryType__value_type(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::string DictionaryType__name(const std::shared_ptr<arrow::DictionaryType>& type);
-RcppExport SEXP _arrow_DictionaryType__name(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DictionaryType>&>::type type(type_sexp);
-	return Rcpp::wrap(DictionaryType__name(type));
-END_RCPP
+extern "C" SEXP _arrow_DictionaryType__name(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::DictionaryType>&>::type type(type_sexp);
+	return cpp11::as_sexp(DictionaryType__name(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_DictionaryType__name(SEXP type_sexp){
+extern "C" SEXP _arrow_DictionaryType__name(SEXP type_sexp){
 	Rf_error("Cannot call DictionaryType__name(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2521,14 +2725,14 @@ RcppExport SEXP _arrow_DictionaryType__name(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool DictionaryType__ordered(const std::shared_ptr<arrow::DictionaryType>& type);
-RcppExport SEXP _arrow_DictionaryType__ordered(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DictionaryType>&>::type type(type_sexp);
-	return Rcpp::wrap(DictionaryType__ordered(type));
-END_RCPP
+extern "C" SEXP _arrow_DictionaryType__ordered(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::DictionaryType>&>::type type(type_sexp);
+	return cpp11::as_sexp(DictionaryType__ordered(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_DictionaryType__ordered(SEXP type_sexp){
+extern "C" SEXP _arrow_DictionaryType__ordered(SEXP type_sexp){
 	Rf_error("Cannot call DictionaryType__ordered(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2536,15 +2740,15 @@ RcppExport SEXP _arrow_DictionaryType__ordered(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Field> StructType__GetFieldByName(const std::shared_ptr<arrow::StructType>& type, const std::string& name);
-RcppExport SEXP _arrow_StructType__GetFieldByName(SEXP type_sexp, SEXP name_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::StructType>&>::type type(type_sexp);
-	Rcpp::traits::input_parameter<const std::string&>::type name(name_sexp);
-	return Rcpp::wrap(StructType__GetFieldByName(type, name));
-END_RCPP
+extern "C" SEXP _arrow_StructType__GetFieldByName(SEXP type_sexp, SEXP name_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::StructType>&>::type type(type_sexp);
+	arrow::r::Input<const std::string&>::type name(name_sexp);
+	return cpp11::as_sexp(StructType__GetFieldByName(type, name));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_StructType__GetFieldByName(SEXP type_sexp, SEXP name_sexp){
+extern "C" SEXP _arrow_StructType__GetFieldByName(SEXP type_sexp, SEXP name_sexp){
 	Rf_error("Cannot call StructType__GetFieldByName(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2552,15 +2756,15 @@ RcppExport SEXP _arrow_StructType__GetFieldByName(SEXP type_sexp, SEXP name_sexp
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int StructType__GetFieldIndex(const std::shared_ptr<arrow::StructType>& type, const std::string& name);
-RcppExport SEXP _arrow_StructType__GetFieldIndex(SEXP type_sexp, SEXP name_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::StructType>&>::type type(type_sexp);
-	Rcpp::traits::input_parameter<const std::string&>::type name(name_sexp);
-	return Rcpp::wrap(StructType__GetFieldIndex(type, name));
-END_RCPP
+extern "C" SEXP _arrow_StructType__GetFieldIndex(SEXP type_sexp, SEXP name_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::StructType>&>::type type(type_sexp);
+	arrow::r::Input<const std::string&>::type name(name_sexp);
+	return cpp11::as_sexp(StructType__GetFieldIndex(type, name));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_StructType__GetFieldIndex(SEXP type_sexp, SEXP name_sexp){
+extern "C" SEXP _arrow_StructType__GetFieldIndex(SEXP type_sexp, SEXP name_sexp){
 	Rf_error("Cannot call StructType__GetFieldIndex(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2568,14 +2772,14 @@ RcppExport SEXP _arrow_StructType__GetFieldIndex(SEXP type_sexp, SEXP name_sexp)
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Field> ListType__value_field(const std::shared_ptr<arrow::ListType>& type);
-RcppExport SEXP _arrow_ListType__value_field(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ListType>&>::type type(type_sexp);
-	return Rcpp::wrap(ListType__value_field(type));
-END_RCPP
+extern "C" SEXP _arrow_ListType__value_field(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ListType>&>::type type(type_sexp);
+	return cpp11::as_sexp(ListType__value_field(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ListType__value_field(SEXP type_sexp){
+extern "C" SEXP _arrow_ListType__value_field(SEXP type_sexp){
 	Rf_error("Cannot call ListType__value_field(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2583,14 +2787,14 @@ RcppExport SEXP _arrow_ListType__value_field(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> ListType__value_type(const std::shared_ptr<arrow::ListType>& type);
-RcppExport SEXP _arrow_ListType__value_type(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ListType>&>::type type(type_sexp);
-	return Rcpp::wrap(ListType__value_type(type));
-END_RCPP
+extern "C" SEXP _arrow_ListType__value_type(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ListType>&>::type type(type_sexp);
+	return cpp11::as_sexp(ListType__value_type(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ListType__value_type(SEXP type_sexp){
+extern "C" SEXP _arrow_ListType__value_type(SEXP type_sexp){
 	Rf_error("Cannot call ListType__value_type(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2598,14 +2802,14 @@ RcppExport SEXP _arrow_ListType__value_type(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Field> LargeListType__value_field(const std::shared_ptr<arrow::LargeListType>& type);
-RcppExport SEXP _arrow_LargeListType__value_field(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::LargeListType>&>::type type(type_sexp);
-	return Rcpp::wrap(LargeListType__value_field(type));
-END_RCPP
+extern "C" SEXP _arrow_LargeListType__value_field(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::LargeListType>&>::type type(type_sexp);
+	return cpp11::as_sexp(LargeListType__value_field(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_LargeListType__value_field(SEXP type_sexp){
+extern "C" SEXP _arrow_LargeListType__value_field(SEXP type_sexp){
 	Rf_error("Cannot call LargeListType__value_field(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2613,14 +2817,14 @@ RcppExport SEXP _arrow_LargeListType__value_field(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> LargeListType__value_type(const std::shared_ptr<arrow::LargeListType>& type);
-RcppExport SEXP _arrow_LargeListType__value_type(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::LargeListType>&>::type type(type_sexp);
-	return Rcpp::wrap(LargeListType__value_type(type));
-END_RCPP
+extern "C" SEXP _arrow_LargeListType__value_type(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::LargeListType>&>::type type(type_sexp);
+	return cpp11::as_sexp(LargeListType__value_type(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_LargeListType__value_type(SEXP type_sexp){
+extern "C" SEXP _arrow_LargeListType__value_type(SEXP type_sexp){
 	Rf_error("Cannot call LargeListType__value_type(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2628,14 +2832,14 @@ RcppExport SEXP _arrow_LargeListType__value_type(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Field> FixedSizeListType__value_field(const std::shared_ptr<arrow::FixedSizeListType>& type);
-RcppExport SEXP _arrow_FixedSizeListType__value_field(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::FixedSizeListType>&>::type type(type_sexp);
-	return Rcpp::wrap(FixedSizeListType__value_field(type));
-END_RCPP
+extern "C" SEXP _arrow_FixedSizeListType__value_field(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::FixedSizeListType>&>::type type(type_sexp);
+	return cpp11::as_sexp(FixedSizeListType__value_field(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_FixedSizeListType__value_field(SEXP type_sexp){
+extern "C" SEXP _arrow_FixedSizeListType__value_field(SEXP type_sexp){
 	Rf_error("Cannot call FixedSizeListType__value_field(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2643,14 +2847,14 @@ RcppExport SEXP _arrow_FixedSizeListType__value_field(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> FixedSizeListType__value_type(const std::shared_ptr<arrow::FixedSizeListType>& type);
-RcppExport SEXP _arrow_FixedSizeListType__value_type(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::FixedSizeListType>&>::type type(type_sexp);
-	return Rcpp::wrap(FixedSizeListType__value_type(type));
-END_RCPP
+extern "C" SEXP _arrow_FixedSizeListType__value_type(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::FixedSizeListType>&>::type type(type_sexp);
+	return cpp11::as_sexp(FixedSizeListType__value_type(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_FixedSizeListType__value_type(SEXP type_sexp){
+extern "C" SEXP _arrow_FixedSizeListType__value_type(SEXP type_sexp){
 	Rf_error("Cannot call FixedSizeListType__value_type(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2658,14 +2862,14 @@ RcppExport SEXP _arrow_FixedSizeListType__value_type(SEXP type_sexp){
 // datatype.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int FixedSizeListType__list_size(const std::shared_ptr<arrow::FixedSizeListType>& type);
-RcppExport SEXP _arrow_FixedSizeListType__list_size(SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::FixedSizeListType>&>::type type(type_sexp);
-	return Rcpp::wrap(FixedSizeListType__list_size(type));
-END_RCPP
+extern "C" SEXP _arrow_FixedSizeListType__list_size(SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::FixedSizeListType>&>::type type(type_sexp);
+	return cpp11::as_sexp(FixedSizeListType__list_size(type));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_FixedSizeListType__list_size(SEXP type_sexp){
+extern "C" SEXP _arrow_FixedSizeListType__list_size(SEXP type_sexp){
 	Rf_error("Cannot call FixedSizeListType__list_size(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2673,14 +2877,14 @@ RcppExport SEXP _arrow_FixedSizeListType__list_size(SEXP type_sexp){
 // expression.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::Expression> dataset___expr__field_ref(std::string name);
-RcppExport SEXP _arrow_dataset___expr__field_ref(SEXP name_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<std::string>::type name(name_sexp);
-	return Rcpp::wrap(dataset___expr__field_ref(name));
-END_RCPP
+extern "C" SEXP _arrow_dataset___expr__field_ref(SEXP name_sexp){
+BEGIN_CPP11
+	arrow::r::Input<std::string>::type name(name_sexp);
+	return cpp11::as_sexp(dataset___expr__field_ref(name));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___expr__field_ref(SEXP name_sexp){
+extern "C" SEXP _arrow_dataset___expr__field_ref(SEXP name_sexp){
 	Rf_error("Cannot call dataset___expr__field_ref(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2688,15 +2892,15 @@ RcppExport SEXP _arrow_dataset___expr__field_ref(SEXP name_sexp){
 // expression.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::Expression> dataset___expr__equal(const std::shared_ptr<ds::Expression>& lhs, const std::shared_ptr<ds::Expression>& rhs);
-RcppExport SEXP _arrow_dataset___expr__equal(SEXP lhs_sexp, SEXP rhs_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type rhs(rhs_sexp);
-	return Rcpp::wrap(dataset___expr__equal(lhs, rhs));
-END_RCPP
+extern "C" SEXP _arrow_dataset___expr__equal(SEXP lhs_sexp, SEXP rhs_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type rhs(rhs_sexp);
+	return cpp11::as_sexp(dataset___expr__equal(lhs, rhs));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___expr__equal(SEXP lhs_sexp, SEXP rhs_sexp){
+extern "C" SEXP _arrow_dataset___expr__equal(SEXP lhs_sexp, SEXP rhs_sexp){
 	Rf_error("Cannot call dataset___expr__equal(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2704,15 +2908,15 @@ RcppExport SEXP _arrow_dataset___expr__equal(SEXP lhs_sexp, SEXP rhs_sexp){
 // expression.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::Expression> dataset___expr__not_equal(const std::shared_ptr<ds::Expression>& lhs, const std::shared_ptr<ds::Expression>& rhs);
-RcppExport SEXP _arrow_dataset___expr__not_equal(SEXP lhs_sexp, SEXP rhs_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type rhs(rhs_sexp);
-	return Rcpp::wrap(dataset___expr__not_equal(lhs, rhs));
-END_RCPP
+extern "C" SEXP _arrow_dataset___expr__not_equal(SEXP lhs_sexp, SEXP rhs_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type rhs(rhs_sexp);
+	return cpp11::as_sexp(dataset___expr__not_equal(lhs, rhs));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___expr__not_equal(SEXP lhs_sexp, SEXP rhs_sexp){
+extern "C" SEXP _arrow_dataset___expr__not_equal(SEXP lhs_sexp, SEXP rhs_sexp){
 	Rf_error("Cannot call dataset___expr__not_equal(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2720,15 +2924,15 @@ RcppExport SEXP _arrow_dataset___expr__not_equal(SEXP lhs_sexp, SEXP rhs_sexp){
 // expression.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::Expression> dataset___expr__greater(const std::shared_ptr<ds::Expression>& lhs, const std::shared_ptr<ds::Expression>& rhs);
-RcppExport SEXP _arrow_dataset___expr__greater(SEXP lhs_sexp, SEXP rhs_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type rhs(rhs_sexp);
-	return Rcpp::wrap(dataset___expr__greater(lhs, rhs));
-END_RCPP
+extern "C" SEXP _arrow_dataset___expr__greater(SEXP lhs_sexp, SEXP rhs_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type rhs(rhs_sexp);
+	return cpp11::as_sexp(dataset___expr__greater(lhs, rhs));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___expr__greater(SEXP lhs_sexp, SEXP rhs_sexp){
+extern "C" SEXP _arrow_dataset___expr__greater(SEXP lhs_sexp, SEXP rhs_sexp){
 	Rf_error("Cannot call dataset___expr__greater(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2736,15 +2940,15 @@ RcppExport SEXP _arrow_dataset___expr__greater(SEXP lhs_sexp, SEXP rhs_sexp){
 // expression.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::Expression> dataset___expr__greater_equal(const std::shared_ptr<ds::Expression>& lhs, const std::shared_ptr<ds::Expression>& rhs);
-RcppExport SEXP _arrow_dataset___expr__greater_equal(SEXP lhs_sexp, SEXP rhs_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type rhs(rhs_sexp);
-	return Rcpp::wrap(dataset___expr__greater_equal(lhs, rhs));
-END_RCPP
+extern "C" SEXP _arrow_dataset___expr__greater_equal(SEXP lhs_sexp, SEXP rhs_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type rhs(rhs_sexp);
+	return cpp11::as_sexp(dataset___expr__greater_equal(lhs, rhs));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___expr__greater_equal(SEXP lhs_sexp, SEXP rhs_sexp){
+extern "C" SEXP _arrow_dataset___expr__greater_equal(SEXP lhs_sexp, SEXP rhs_sexp){
 	Rf_error("Cannot call dataset___expr__greater_equal(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2752,15 +2956,15 @@ RcppExport SEXP _arrow_dataset___expr__greater_equal(SEXP lhs_sexp, SEXP rhs_sex
 // expression.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::Expression> dataset___expr__less(const std::shared_ptr<ds::Expression>& lhs, const std::shared_ptr<ds::Expression>& rhs);
-RcppExport SEXP _arrow_dataset___expr__less(SEXP lhs_sexp, SEXP rhs_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type rhs(rhs_sexp);
-	return Rcpp::wrap(dataset___expr__less(lhs, rhs));
-END_RCPP
+extern "C" SEXP _arrow_dataset___expr__less(SEXP lhs_sexp, SEXP rhs_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type rhs(rhs_sexp);
+	return cpp11::as_sexp(dataset___expr__less(lhs, rhs));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___expr__less(SEXP lhs_sexp, SEXP rhs_sexp){
+extern "C" SEXP _arrow_dataset___expr__less(SEXP lhs_sexp, SEXP rhs_sexp){
 	Rf_error("Cannot call dataset___expr__less(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2768,15 +2972,15 @@ RcppExport SEXP _arrow_dataset___expr__less(SEXP lhs_sexp, SEXP rhs_sexp){
 // expression.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::Expression> dataset___expr__less_equal(const std::shared_ptr<ds::Expression>& lhs, const std::shared_ptr<ds::Expression>& rhs);
-RcppExport SEXP _arrow_dataset___expr__less_equal(SEXP lhs_sexp, SEXP rhs_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type rhs(rhs_sexp);
-	return Rcpp::wrap(dataset___expr__less_equal(lhs, rhs));
-END_RCPP
+extern "C" SEXP _arrow_dataset___expr__less_equal(SEXP lhs_sexp, SEXP rhs_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type rhs(rhs_sexp);
+	return cpp11::as_sexp(dataset___expr__less_equal(lhs, rhs));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___expr__less_equal(SEXP lhs_sexp, SEXP rhs_sexp){
+extern "C" SEXP _arrow_dataset___expr__less_equal(SEXP lhs_sexp, SEXP rhs_sexp){
 	Rf_error("Cannot call dataset___expr__less_equal(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2784,15 +2988,15 @@ RcppExport SEXP _arrow_dataset___expr__less_equal(SEXP lhs_sexp, SEXP rhs_sexp){
 // expression.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::Expression> dataset___expr__in(const std::shared_ptr<ds::Expression>& lhs, const std::shared_ptr<arrow::Array>& rhs);
-RcppExport SEXP _arrow_dataset___expr__in(SEXP lhs_sexp, SEXP rhs_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type rhs(rhs_sexp);
-	return Rcpp::wrap(dataset___expr__in(lhs, rhs));
-END_RCPP
+extern "C" SEXP _arrow_dataset___expr__in(SEXP lhs_sexp, SEXP rhs_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type rhs(rhs_sexp);
+	return cpp11::as_sexp(dataset___expr__in(lhs, rhs));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___expr__in(SEXP lhs_sexp, SEXP rhs_sexp){
+extern "C" SEXP _arrow_dataset___expr__in(SEXP lhs_sexp, SEXP rhs_sexp){
 	Rf_error("Cannot call dataset___expr__in(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2800,15 +3004,15 @@ RcppExport SEXP _arrow_dataset___expr__in(SEXP lhs_sexp, SEXP rhs_sexp){
 // expression.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::Expression> dataset___expr__and(const std::shared_ptr<ds::Expression>& lhs, const std::shared_ptr<ds::Expression>& rhs);
-RcppExport SEXP _arrow_dataset___expr__and(SEXP lhs_sexp, SEXP rhs_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type rhs(rhs_sexp);
-	return Rcpp::wrap(dataset___expr__and(lhs, rhs));
-END_RCPP
+extern "C" SEXP _arrow_dataset___expr__and(SEXP lhs_sexp, SEXP rhs_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type rhs(rhs_sexp);
+	return cpp11::as_sexp(dataset___expr__and(lhs, rhs));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___expr__and(SEXP lhs_sexp, SEXP rhs_sexp){
+extern "C" SEXP _arrow_dataset___expr__and(SEXP lhs_sexp, SEXP rhs_sexp){
 	Rf_error("Cannot call dataset___expr__and(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2816,15 +3020,15 @@ RcppExport SEXP _arrow_dataset___expr__and(SEXP lhs_sexp, SEXP rhs_sexp){
 // expression.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::Expression> dataset___expr__or(const std::shared_ptr<ds::Expression>& lhs, const std::shared_ptr<ds::Expression>& rhs);
-RcppExport SEXP _arrow_dataset___expr__or(SEXP lhs_sexp, SEXP rhs_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type rhs(rhs_sexp);
-	return Rcpp::wrap(dataset___expr__or(lhs, rhs));
-END_RCPP
+extern "C" SEXP _arrow_dataset___expr__or(SEXP lhs_sexp, SEXP rhs_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type rhs(rhs_sexp);
+	return cpp11::as_sexp(dataset___expr__or(lhs, rhs));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___expr__or(SEXP lhs_sexp, SEXP rhs_sexp){
+extern "C" SEXP _arrow_dataset___expr__or(SEXP lhs_sexp, SEXP rhs_sexp){
 	Rf_error("Cannot call dataset___expr__or(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2832,14 +3036,14 @@ RcppExport SEXP _arrow_dataset___expr__or(SEXP lhs_sexp, SEXP rhs_sexp){
 // expression.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::Expression> dataset___expr__not(const std::shared_ptr<ds::Expression>& lhs);
-RcppExport SEXP _arrow_dataset___expr__not(SEXP lhs_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
-	return Rcpp::wrap(dataset___expr__not(lhs));
-END_RCPP
+extern "C" SEXP _arrow_dataset___expr__not(SEXP lhs_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
+	return cpp11::as_sexp(dataset___expr__not(lhs));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___expr__not(SEXP lhs_sexp){
+extern "C" SEXP _arrow_dataset___expr__not(SEXP lhs_sexp){
 	Rf_error("Cannot call dataset___expr__not(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2847,14 +3051,14 @@ RcppExport SEXP _arrow_dataset___expr__not(SEXP lhs_sexp){
 // expression.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::Expression> dataset___expr__is_valid(const std::shared_ptr<ds::Expression>& lhs);
-RcppExport SEXP _arrow_dataset___expr__is_valid(SEXP lhs_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
-	return Rcpp::wrap(dataset___expr__is_valid(lhs));
-END_RCPP
+extern "C" SEXP _arrow_dataset___expr__is_valid(SEXP lhs_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type lhs(lhs_sexp);
+	return cpp11::as_sexp(dataset___expr__is_valid(lhs));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___expr__is_valid(SEXP lhs_sexp){
+extern "C" SEXP _arrow_dataset___expr__is_valid(SEXP lhs_sexp){
 	Rf_error("Cannot call dataset___expr__is_valid(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2862,14 +3066,14 @@ RcppExport SEXP _arrow_dataset___expr__is_valid(SEXP lhs_sexp){
 // expression.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<ds::Expression> dataset___expr__scalar(const std::shared_ptr<arrow::Scalar>& x);
-RcppExport SEXP _arrow_dataset___expr__scalar(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Scalar>&>::type x(x_sexp);
-	return Rcpp::wrap(dataset___expr__scalar(x));
-END_RCPP
+extern "C" SEXP _arrow_dataset___expr__scalar(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Scalar>&>::type x(x_sexp);
+	return cpp11::as_sexp(dataset___expr__scalar(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___expr__scalar(SEXP x_sexp){
+extern "C" SEXP _arrow_dataset___expr__scalar(SEXP x_sexp){
 	Rf_error("Cannot call dataset___expr__scalar(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2877,14 +3081,14 @@ RcppExport SEXP _arrow_dataset___expr__scalar(SEXP x_sexp){
 // expression.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::string dataset___expr__ToString(const std::shared_ptr<ds::Expression>& x);
-RcppExport SEXP _arrow_dataset___expr__ToString(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<ds::Expression>&>::type x(x_sexp);
-	return Rcpp::wrap(dataset___expr__ToString(x));
-END_RCPP
+extern "C" SEXP _arrow_dataset___expr__ToString(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<ds::Expression>&>::type x(x_sexp);
+	return cpp11::as_sexp(dataset___expr__ToString(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_dataset___expr__ToString(SEXP x_sexp){
+extern "C" SEXP _arrow_dataset___expr__ToString(SEXP x_sexp){
 	Rf_error("Cannot call dataset___expr__ToString(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2892,20 +3096,20 @@ RcppExport SEXP _arrow_dataset___expr__ToString(SEXP x_sexp){
 // feather.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void ipc___WriteFeather__Table(const std::shared_ptr<arrow::io::OutputStream>& stream, const std::shared_ptr<arrow::Table>& table, int version, int chunk_size, arrow::Compression::type compression, int compression_level);
-RcppExport SEXP _arrow_ipc___WriteFeather__Table(SEXP stream_sexp, SEXP table_sexp, SEXP version_sexp, SEXP chunk_size_sexp, SEXP compression_sexp, SEXP compression_level_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::OutputStream>&>::type stream(stream_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
-	Rcpp::traits::input_parameter<int>::type version(version_sexp);
-	Rcpp::traits::input_parameter<int>::type chunk_size(chunk_size_sexp);
-	Rcpp::traits::input_parameter<arrow::Compression::type>::type compression(compression_sexp);
-	Rcpp::traits::input_parameter<int>::type compression_level(compression_level_sexp);
+extern "C" SEXP _arrow_ipc___WriteFeather__Table(SEXP stream_sexp, SEXP table_sexp, SEXP version_sexp, SEXP chunk_size_sexp, SEXP compression_sexp, SEXP compression_level_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::OutputStream>&>::type stream(stream_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
+	arrow::r::Input<int>::type version(version_sexp);
+	arrow::r::Input<int>::type chunk_size(chunk_size_sexp);
+	arrow::r::Input<arrow::Compression::type>::type compression(compression_sexp);
+	arrow::r::Input<int>::type compression_level(compression_level_sexp);
 	ipc___WriteFeather__Table(stream, table, version, chunk_size, compression, compression_level);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___WriteFeather__Table(SEXP stream_sexp, SEXP table_sexp, SEXP version_sexp, SEXP chunk_size_sexp, SEXP compression_sexp, SEXP compression_level_sexp){
+extern "C" SEXP _arrow_ipc___WriteFeather__Table(SEXP stream_sexp, SEXP table_sexp, SEXP version_sexp, SEXP chunk_size_sexp, SEXP compression_sexp, SEXP compression_level_sexp){
 	Rf_error("Cannot call ipc___WriteFeather__Table(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2913,14 +3117,14 @@ RcppExport SEXP _arrow_ipc___WriteFeather__Table(SEXP stream_sexp, SEXP table_se
 // feather.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int ipc___feather___Reader__version(const std::shared_ptr<arrow::ipc::feather::Reader>& reader);
-RcppExport SEXP _arrow_ipc___feather___Reader__version(SEXP reader_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ipc::feather::Reader>&>::type reader(reader_sexp);
-	return Rcpp::wrap(ipc___feather___Reader__version(reader));
-END_RCPP
+extern "C" SEXP _arrow_ipc___feather___Reader__version(SEXP reader_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ipc::feather::Reader>&>::type reader(reader_sexp);
+	return cpp11::as_sexp(ipc___feather___Reader__version(reader));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___feather___Reader__version(SEXP reader_sexp){
+extern "C" SEXP _arrow_ipc___feather___Reader__version(SEXP reader_sexp){
 	Rf_error("Cannot call ipc___feather___Reader__version(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2928,15 +3132,15 @@ RcppExport SEXP _arrow_ipc___feather___Reader__version(SEXP reader_sexp){
 // feather.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Table> ipc___feather___Reader__Read(const std::shared_ptr<arrow::ipc::feather::Reader>& reader, SEXP columns);
-RcppExport SEXP _arrow_ipc___feather___Reader__Read(SEXP reader_sexp, SEXP columns_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ipc::feather::Reader>&>::type reader(reader_sexp);
-	Rcpp::traits::input_parameter<SEXP>::type columns(columns_sexp);
-	return Rcpp::wrap(ipc___feather___Reader__Read(reader, columns));
-END_RCPP
+extern "C" SEXP _arrow_ipc___feather___Reader__Read(SEXP reader_sexp, SEXP columns_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ipc::feather::Reader>&>::type reader(reader_sexp);
+	arrow::r::Input<SEXP>::type columns(columns_sexp);
+	return cpp11::as_sexp(ipc___feather___Reader__Read(reader, columns));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___feather___Reader__Read(SEXP reader_sexp, SEXP columns_sexp){
+extern "C" SEXP _arrow_ipc___feather___Reader__Read(SEXP reader_sexp, SEXP columns_sexp){
 	Rf_error("Cannot call ipc___feather___Reader__Read(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2944,29 +3148,29 @@ RcppExport SEXP _arrow_ipc___feather___Reader__Read(SEXP reader_sexp, SEXP colum
 // feather.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::ipc::feather::Reader> ipc___feather___Reader__Open(const std::shared_ptr<arrow::io::RandomAccessFile>& stream);
-RcppExport SEXP _arrow_ipc___feather___Reader__Open(SEXP stream_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::RandomAccessFile>&>::type stream(stream_sexp);
-	return Rcpp::wrap(ipc___feather___Reader__Open(stream));
-END_RCPP
+extern "C" SEXP _arrow_ipc___feather___Reader__Open(SEXP stream_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::RandomAccessFile>&>::type stream(stream_sexp);
+	return cpp11::as_sexp(ipc___feather___Reader__Open(stream));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___feather___Reader__Open(SEXP stream_sexp){
+extern "C" SEXP _arrow_ipc___feather___Reader__Open(SEXP stream_sexp){
 	Rf_error("Cannot call ipc___feather___Reader__Open(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // feather.cpp
 #if defined(ARROW_R_WITH_ARROW)
-Rcpp::CharacterVector ipc___feather___Reader__column_names(const std::shared_ptr<arrow::ipc::feather::Reader>& reader);
-RcppExport SEXP _arrow_ipc___feather___Reader__column_names(SEXP reader_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ipc::feather::Reader>&>::type reader(reader_sexp);
-	return Rcpp::wrap(ipc___feather___Reader__column_names(reader));
-END_RCPP
+cpp11::writable::strings ipc___feather___Reader__column_names(const std::shared_ptr<arrow::ipc::feather::Reader>& reader);
+extern "C" SEXP _arrow_ipc___feather___Reader__column_names(SEXP reader_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ipc::feather::Reader>&>::type reader(reader_sexp);
+	return cpp11::as_sexp(ipc___feather___Reader__column_names(reader));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___feather___Reader__column_names(SEXP reader_sexp){
+extern "C" SEXP _arrow_ipc___feather___Reader__column_names(SEXP reader_sexp){
 	Rf_error("Cannot call ipc___feather___Reader__column_names(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2974,16 +3178,16 @@ RcppExport SEXP _arrow_ipc___feather___Reader__column_names(SEXP reader_sexp){
 // field.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Field> Field__initialize(const std::string& name, const std::shared_ptr<arrow::DataType>& field, bool nullable);
-RcppExport SEXP _arrow_Field__initialize(SEXP name_sexp, SEXP field_sexp, SEXP nullable_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::string&>::type name(name_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DataType>&>::type field(field_sexp);
-	Rcpp::traits::input_parameter<bool>::type nullable(nullable_sexp);
-	return Rcpp::wrap(Field__initialize(name, field, nullable));
-END_RCPP
+extern "C" SEXP _arrow_Field__initialize(SEXP name_sexp, SEXP field_sexp, SEXP nullable_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::string&>::type name(name_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::DataType>&>::type field(field_sexp);
+	arrow::r::Input<bool>::type nullable(nullable_sexp);
+	return cpp11::as_sexp(Field__initialize(name, field, nullable));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Field__initialize(SEXP name_sexp, SEXP field_sexp, SEXP nullable_sexp){
+extern "C" SEXP _arrow_Field__initialize(SEXP name_sexp, SEXP field_sexp, SEXP nullable_sexp){
 	Rf_error("Cannot call Field__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -2991,14 +3195,14 @@ RcppExport SEXP _arrow_Field__initialize(SEXP name_sexp, SEXP field_sexp, SEXP n
 // field.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::string Field__ToString(const std::shared_ptr<arrow::Field>& field);
-RcppExport SEXP _arrow_Field__ToString(SEXP field_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Field>&>::type field(field_sexp);
-	return Rcpp::wrap(Field__ToString(field));
-END_RCPP
+extern "C" SEXP _arrow_Field__ToString(SEXP field_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Field>&>::type field(field_sexp);
+	return cpp11::as_sexp(Field__ToString(field));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Field__ToString(SEXP field_sexp){
+extern "C" SEXP _arrow_Field__ToString(SEXP field_sexp){
 	Rf_error("Cannot call Field__ToString(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3006,14 +3210,14 @@ RcppExport SEXP _arrow_Field__ToString(SEXP field_sexp){
 // field.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::string Field__name(const std::shared_ptr<arrow::Field>& field);
-RcppExport SEXP _arrow_Field__name(SEXP field_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Field>&>::type field(field_sexp);
-	return Rcpp::wrap(Field__name(field));
-END_RCPP
+extern "C" SEXP _arrow_Field__name(SEXP field_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Field>&>::type field(field_sexp);
+	return cpp11::as_sexp(Field__name(field));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Field__name(SEXP field_sexp){
+extern "C" SEXP _arrow_Field__name(SEXP field_sexp){
 	Rf_error("Cannot call Field__name(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3021,15 +3225,15 @@ RcppExport SEXP _arrow_Field__name(SEXP field_sexp){
 // field.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool Field__Equals(const std::shared_ptr<arrow::Field>& field, const std::shared_ptr<arrow::Field>& other);
-RcppExport SEXP _arrow_Field__Equals(SEXP field_sexp, SEXP other_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Field>&>::type field(field_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Field>&>::type other(other_sexp);
-	return Rcpp::wrap(Field__Equals(field, other));
-END_RCPP
+extern "C" SEXP _arrow_Field__Equals(SEXP field_sexp, SEXP other_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Field>&>::type field(field_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Field>&>::type other(other_sexp);
+	return cpp11::as_sexp(Field__Equals(field, other));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Field__Equals(SEXP field_sexp, SEXP other_sexp){
+extern "C" SEXP _arrow_Field__Equals(SEXP field_sexp, SEXP other_sexp){
 	Rf_error("Cannot call Field__Equals(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3037,14 +3241,14 @@ RcppExport SEXP _arrow_Field__Equals(SEXP field_sexp, SEXP other_sexp){
 // field.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool Field__nullable(const std::shared_ptr<arrow::Field>& field);
-RcppExport SEXP _arrow_Field__nullable(SEXP field_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Field>&>::type field(field_sexp);
-	return Rcpp::wrap(Field__nullable(field));
-END_RCPP
+extern "C" SEXP _arrow_Field__nullable(SEXP field_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Field>&>::type field(field_sexp);
+	return cpp11::as_sexp(Field__nullable(field));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Field__nullable(SEXP field_sexp){
+extern "C" SEXP _arrow_Field__nullable(SEXP field_sexp){
 	Rf_error("Cannot call Field__nullable(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3052,14 +3256,14 @@ RcppExport SEXP _arrow_Field__nullable(SEXP field_sexp){
 // field.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Field__type(const std::shared_ptr<arrow::Field>& field);
-RcppExport SEXP _arrow_Field__type(SEXP field_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Field>&>::type field(field_sexp);
-	return Rcpp::wrap(Field__type(field));
-END_RCPP
+extern "C" SEXP _arrow_Field__type(SEXP field_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Field>&>::type field(field_sexp);
+	return cpp11::as_sexp(Field__type(field));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Field__type(SEXP field_sexp){
+extern "C" SEXP _arrow_Field__type(SEXP field_sexp){
 	Rf_error("Cannot call Field__type(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3067,14 +3271,14 @@ RcppExport SEXP _arrow_Field__type(SEXP field_sexp){
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 fs::FileType fs___FileInfo__type(const std::shared_ptr<fs::FileInfo>& x);
-RcppExport SEXP _arrow_fs___FileInfo__type(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileInfo>&>::type x(x_sexp);
-	return Rcpp::wrap(fs___FileInfo__type(x));
-END_RCPP
+extern "C" SEXP _arrow_fs___FileInfo__type(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileInfo>&>::type x(x_sexp);
+	return cpp11::as_sexp(fs___FileInfo__type(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileInfo__type(SEXP x_sexp){
+extern "C" SEXP _arrow_fs___FileInfo__type(SEXP x_sexp){
 	Rf_error("Cannot call fs___FileInfo__type(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3082,16 +3286,16 @@ RcppExport SEXP _arrow_fs___FileInfo__type(SEXP x_sexp){
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void fs___FileInfo__set_type(const std::shared_ptr<fs::FileInfo>& x, fs::FileType type);
-RcppExport SEXP _arrow_fs___FileInfo__set_type(SEXP x_sexp, SEXP type_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileInfo>&>::type x(x_sexp);
-	Rcpp::traits::input_parameter<fs::FileType>::type type(type_sexp);
+extern "C" SEXP _arrow_fs___FileInfo__set_type(SEXP x_sexp, SEXP type_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileInfo>&>::type x(x_sexp);
+	arrow::r::Input<fs::FileType>::type type(type_sexp);
 	fs___FileInfo__set_type(x, type);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileInfo__set_type(SEXP x_sexp, SEXP type_sexp){
+extern "C" SEXP _arrow_fs___FileInfo__set_type(SEXP x_sexp, SEXP type_sexp){
 	Rf_error("Cannot call fs___FileInfo__set_type(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3099,14 +3303,14 @@ RcppExport SEXP _arrow_fs___FileInfo__set_type(SEXP x_sexp, SEXP type_sexp){
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::string fs___FileInfo__path(const std::shared_ptr<fs::FileInfo>& x);
-RcppExport SEXP _arrow_fs___FileInfo__path(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileInfo>&>::type x(x_sexp);
-	return Rcpp::wrap(fs___FileInfo__path(x));
-END_RCPP
+extern "C" SEXP _arrow_fs___FileInfo__path(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileInfo>&>::type x(x_sexp);
+	return cpp11::as_sexp(fs___FileInfo__path(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileInfo__path(SEXP x_sexp){
+extern "C" SEXP _arrow_fs___FileInfo__path(SEXP x_sexp){
 	Rf_error("Cannot call fs___FileInfo__path(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3114,16 +3318,16 @@ RcppExport SEXP _arrow_fs___FileInfo__path(SEXP x_sexp){
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void fs___FileInfo__set_path(const std::shared_ptr<fs::FileInfo>& x, const std::string& path);
-RcppExport SEXP _arrow_fs___FileInfo__set_path(SEXP x_sexp, SEXP path_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileInfo>&>::type x(x_sexp);
-	Rcpp::traits::input_parameter<const std::string&>::type path(path_sexp);
+extern "C" SEXP _arrow_fs___FileInfo__set_path(SEXP x_sexp, SEXP path_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileInfo>&>::type x(x_sexp);
+	arrow::r::Input<const std::string&>::type path(path_sexp);
 	fs___FileInfo__set_path(x, path);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileInfo__set_path(SEXP x_sexp, SEXP path_sexp){
+extern "C" SEXP _arrow_fs___FileInfo__set_path(SEXP x_sexp, SEXP path_sexp){
 	Rf_error("Cannot call fs___FileInfo__set_path(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3131,14 +3335,14 @@ RcppExport SEXP _arrow_fs___FileInfo__set_path(SEXP x_sexp, SEXP path_sexp){
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int64_t fs___FileInfo__size(const std::shared_ptr<fs::FileInfo>& x);
-RcppExport SEXP _arrow_fs___FileInfo__size(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileInfo>&>::type x(x_sexp);
-	return Rcpp::wrap(fs___FileInfo__size(x));
-END_RCPP
+extern "C" SEXP _arrow_fs___FileInfo__size(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileInfo>&>::type x(x_sexp);
+	return cpp11::as_sexp(fs___FileInfo__size(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileInfo__size(SEXP x_sexp){
+extern "C" SEXP _arrow_fs___FileInfo__size(SEXP x_sexp){
 	Rf_error("Cannot call fs___FileInfo__size(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3146,16 +3350,16 @@ RcppExport SEXP _arrow_fs___FileInfo__size(SEXP x_sexp){
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void fs___FileInfo__set_size(const std::shared_ptr<fs::FileInfo>& x, int64_t size);
-RcppExport SEXP _arrow_fs___FileInfo__set_size(SEXP x_sexp, SEXP size_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileInfo>&>::type x(x_sexp);
-	Rcpp::traits::input_parameter<int64_t>::type size(size_sexp);
+extern "C" SEXP _arrow_fs___FileInfo__set_size(SEXP x_sexp, SEXP size_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileInfo>&>::type x(x_sexp);
+	arrow::r::Input<int64_t>::type size(size_sexp);
 	fs___FileInfo__set_size(x, size);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileInfo__set_size(SEXP x_sexp, SEXP size_sexp){
+extern "C" SEXP _arrow_fs___FileInfo__set_size(SEXP x_sexp, SEXP size_sexp){
 	Rf_error("Cannot call fs___FileInfo__set_size(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3163,14 +3367,14 @@ RcppExport SEXP _arrow_fs___FileInfo__set_size(SEXP x_sexp, SEXP size_sexp){
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::string fs___FileInfo__base_name(const std::shared_ptr<fs::FileInfo>& x);
-RcppExport SEXP _arrow_fs___FileInfo__base_name(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileInfo>&>::type x(x_sexp);
-	return Rcpp::wrap(fs___FileInfo__base_name(x));
-END_RCPP
+extern "C" SEXP _arrow_fs___FileInfo__base_name(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileInfo>&>::type x(x_sexp);
+	return cpp11::as_sexp(fs___FileInfo__base_name(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileInfo__base_name(SEXP x_sexp){
+extern "C" SEXP _arrow_fs___FileInfo__base_name(SEXP x_sexp){
 	Rf_error("Cannot call fs___FileInfo__base_name(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3178,14 +3382,14 @@ RcppExport SEXP _arrow_fs___FileInfo__base_name(SEXP x_sexp){
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::string fs___FileInfo__extension(const std::shared_ptr<fs::FileInfo>& x);
-RcppExport SEXP _arrow_fs___FileInfo__extension(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileInfo>&>::type x(x_sexp);
-	return Rcpp::wrap(fs___FileInfo__extension(x));
-END_RCPP
+extern "C" SEXP _arrow_fs___FileInfo__extension(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileInfo>&>::type x(x_sexp);
+	return cpp11::as_sexp(fs___FileInfo__extension(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileInfo__extension(SEXP x_sexp){
+extern "C" SEXP _arrow_fs___FileInfo__extension(SEXP x_sexp){
 	Rf_error("Cannot call fs___FileInfo__extension(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3193,14 +3397,14 @@ RcppExport SEXP _arrow_fs___FileInfo__extension(SEXP x_sexp){
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 SEXP fs___FileInfo__mtime(const std::shared_ptr<fs::FileInfo>& x);
-RcppExport SEXP _arrow_fs___FileInfo__mtime(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileInfo>&>::type x(x_sexp);
-	return Rcpp::wrap(fs___FileInfo__mtime(x));
-END_RCPP
+extern "C" SEXP _arrow_fs___FileInfo__mtime(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileInfo>&>::type x(x_sexp);
+	return cpp11::as_sexp(fs___FileInfo__mtime(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileInfo__mtime(SEXP x_sexp){
+extern "C" SEXP _arrow_fs___FileInfo__mtime(SEXP x_sexp){
 	Rf_error("Cannot call fs___FileInfo__mtime(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3208,16 +3412,16 @@ RcppExport SEXP _arrow_fs___FileInfo__mtime(SEXP x_sexp){
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void fs___FileInfo__set_mtime(const std::shared_ptr<fs::FileInfo>& x, SEXP time);
-RcppExport SEXP _arrow_fs___FileInfo__set_mtime(SEXP x_sexp, SEXP time_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileInfo>&>::type x(x_sexp);
-	Rcpp::traits::input_parameter<SEXP>::type time(time_sexp);
+extern "C" SEXP _arrow_fs___FileInfo__set_mtime(SEXP x_sexp, SEXP time_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileInfo>&>::type x(x_sexp);
+	arrow::r::Input<SEXP>::type time(time_sexp);
 	fs___FileInfo__set_mtime(x, time);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileInfo__set_mtime(SEXP x_sexp, SEXP time_sexp){
+extern "C" SEXP _arrow_fs___FileInfo__set_mtime(SEXP x_sexp, SEXP time_sexp){
 	Rf_error("Cannot call fs___FileInfo__set_mtime(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3225,14 +3429,14 @@ RcppExport SEXP _arrow_fs___FileInfo__set_mtime(SEXP x_sexp, SEXP time_sexp){
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::string fs___FileSelector__base_dir(const std::shared_ptr<fs::FileSelector>& selector);
-RcppExport SEXP _arrow_fs___FileSelector__base_dir(SEXP selector_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSelector>&>::type selector(selector_sexp);
-	return Rcpp::wrap(fs___FileSelector__base_dir(selector));
-END_RCPP
+extern "C" SEXP _arrow_fs___FileSelector__base_dir(SEXP selector_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSelector>&>::type selector(selector_sexp);
+	return cpp11::as_sexp(fs___FileSelector__base_dir(selector));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileSelector__base_dir(SEXP selector_sexp){
+extern "C" SEXP _arrow_fs___FileSelector__base_dir(SEXP selector_sexp){
 	Rf_error("Cannot call fs___FileSelector__base_dir(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3240,14 +3444,14 @@ RcppExport SEXP _arrow_fs___FileSelector__base_dir(SEXP selector_sexp){
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool fs___FileSelector__allow_not_found(const std::shared_ptr<fs::FileSelector>& selector);
-RcppExport SEXP _arrow_fs___FileSelector__allow_not_found(SEXP selector_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSelector>&>::type selector(selector_sexp);
-	return Rcpp::wrap(fs___FileSelector__allow_not_found(selector));
-END_RCPP
+extern "C" SEXP _arrow_fs___FileSelector__allow_not_found(SEXP selector_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSelector>&>::type selector(selector_sexp);
+	return cpp11::as_sexp(fs___FileSelector__allow_not_found(selector));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileSelector__allow_not_found(SEXP selector_sexp){
+extern "C" SEXP _arrow_fs___FileSelector__allow_not_found(SEXP selector_sexp){
 	Rf_error("Cannot call fs___FileSelector__allow_not_found(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3255,14 +3459,14 @@ RcppExport SEXP _arrow_fs___FileSelector__allow_not_found(SEXP selector_sexp){
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool fs___FileSelector__recursive(const std::shared_ptr<fs::FileSelector>& selector);
-RcppExport SEXP _arrow_fs___FileSelector__recursive(SEXP selector_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSelector>&>::type selector(selector_sexp);
-	return Rcpp::wrap(fs___FileSelector__recursive(selector));
-END_RCPP
+extern "C" SEXP _arrow_fs___FileSelector__recursive(SEXP selector_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSelector>&>::type selector(selector_sexp);
+	return cpp11::as_sexp(fs___FileSelector__recursive(selector));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileSelector__recursive(SEXP selector_sexp){
+extern "C" SEXP _arrow_fs___FileSelector__recursive(SEXP selector_sexp){
 	Rf_error("Cannot call fs___FileSelector__recursive(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3270,16 +3474,16 @@ RcppExport SEXP _arrow_fs___FileSelector__recursive(SEXP selector_sexp){
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<fs::FileSelector> fs___FileSelector__create(const std::string& base_dir, bool allow_not_found, bool recursive);
-RcppExport SEXP _arrow_fs___FileSelector__create(SEXP base_dir_sexp, SEXP allow_not_found_sexp, SEXP recursive_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::string&>::type base_dir(base_dir_sexp);
-	Rcpp::traits::input_parameter<bool>::type allow_not_found(allow_not_found_sexp);
-	Rcpp::traits::input_parameter<bool>::type recursive(recursive_sexp);
-	return Rcpp::wrap(fs___FileSelector__create(base_dir, allow_not_found, recursive));
-END_RCPP
+extern "C" SEXP _arrow_fs___FileSelector__create(SEXP base_dir_sexp, SEXP allow_not_found_sexp, SEXP recursive_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::string&>::type base_dir(base_dir_sexp);
+	arrow::r::Input<bool>::type allow_not_found(allow_not_found_sexp);
+	arrow::r::Input<bool>::type recursive(recursive_sexp);
+	return cpp11::as_sexp(fs___FileSelector__create(base_dir, allow_not_found, recursive));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileSelector__create(SEXP base_dir_sexp, SEXP allow_not_found_sexp, SEXP recursive_sexp){
+extern "C" SEXP _arrow_fs___FileSelector__create(SEXP base_dir_sexp, SEXP allow_not_found_sexp, SEXP recursive_sexp){
 	Rf_error("Cannot call fs___FileSelector__create(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3287,15 +3491,15 @@ RcppExport SEXP _arrow_fs___FileSelector__create(SEXP base_dir_sexp, SEXP allow_
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::vector<std::shared_ptr<fs::FileInfo>> fs___FileSystem__GetTargetInfos_Paths(const std::shared_ptr<fs::FileSystem>& file_system, const std::vector<std::string>& paths);
-RcppExport SEXP _arrow_fs___FileSystem__GetTargetInfos_Paths(SEXP file_system_sexp, SEXP paths_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
-	Rcpp::traits::input_parameter<const std::vector<std::string>&>::type paths(paths_sexp);
-	return Rcpp::wrap(fs___FileSystem__GetTargetInfos_Paths(file_system, paths));
-END_RCPP
+extern "C" SEXP _arrow_fs___FileSystem__GetTargetInfos_Paths(SEXP file_system_sexp, SEXP paths_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
+	arrow::r::Input<const std::vector<std::string>&>::type paths(paths_sexp);
+	return cpp11::as_sexp(fs___FileSystem__GetTargetInfos_Paths(file_system, paths));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileSystem__GetTargetInfos_Paths(SEXP file_system_sexp, SEXP paths_sexp){
+extern "C" SEXP _arrow_fs___FileSystem__GetTargetInfos_Paths(SEXP file_system_sexp, SEXP paths_sexp){
 	Rf_error("Cannot call fs___FileSystem__GetTargetInfos_Paths(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3303,15 +3507,15 @@ RcppExport SEXP _arrow_fs___FileSystem__GetTargetInfos_Paths(SEXP file_system_se
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::vector<std::shared_ptr<fs::FileInfo>> fs___FileSystem__GetTargetInfos_FileSelector(const std::shared_ptr<fs::FileSystem>& file_system, const std::shared_ptr<fs::FileSelector>& selector);
-RcppExport SEXP _arrow_fs___FileSystem__GetTargetInfos_FileSelector(SEXP file_system_sexp, SEXP selector_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSelector>&>::type selector(selector_sexp);
-	return Rcpp::wrap(fs___FileSystem__GetTargetInfos_FileSelector(file_system, selector));
-END_RCPP
+extern "C" SEXP _arrow_fs___FileSystem__GetTargetInfos_FileSelector(SEXP file_system_sexp, SEXP selector_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
+	arrow::r::Input<const std::shared_ptr<fs::FileSelector>&>::type selector(selector_sexp);
+	return cpp11::as_sexp(fs___FileSystem__GetTargetInfos_FileSelector(file_system, selector));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileSystem__GetTargetInfos_FileSelector(SEXP file_system_sexp, SEXP selector_sexp){
+extern "C" SEXP _arrow_fs___FileSystem__GetTargetInfos_FileSelector(SEXP file_system_sexp, SEXP selector_sexp){
 	Rf_error("Cannot call fs___FileSystem__GetTargetInfos_FileSelector(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3319,17 +3523,17 @@ RcppExport SEXP _arrow_fs___FileSystem__GetTargetInfos_FileSelector(SEXP file_sy
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void fs___FileSystem__CreateDir(const std::shared_ptr<fs::FileSystem>& file_system, const std::string& path, bool recursive);
-RcppExport SEXP _arrow_fs___FileSystem__CreateDir(SEXP file_system_sexp, SEXP path_sexp, SEXP recursive_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
-	Rcpp::traits::input_parameter<const std::string&>::type path(path_sexp);
-	Rcpp::traits::input_parameter<bool>::type recursive(recursive_sexp);
+extern "C" SEXP _arrow_fs___FileSystem__CreateDir(SEXP file_system_sexp, SEXP path_sexp, SEXP recursive_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
+	arrow::r::Input<const std::string&>::type path(path_sexp);
+	arrow::r::Input<bool>::type recursive(recursive_sexp);
 	fs___FileSystem__CreateDir(file_system, path, recursive);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileSystem__CreateDir(SEXP file_system_sexp, SEXP path_sexp, SEXP recursive_sexp){
+extern "C" SEXP _arrow_fs___FileSystem__CreateDir(SEXP file_system_sexp, SEXP path_sexp, SEXP recursive_sexp){
 	Rf_error("Cannot call fs___FileSystem__CreateDir(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3337,16 +3541,16 @@ RcppExport SEXP _arrow_fs___FileSystem__CreateDir(SEXP file_system_sexp, SEXP pa
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void fs___FileSystem__DeleteDir(const std::shared_ptr<fs::FileSystem>& file_system, const std::string& path);
-RcppExport SEXP _arrow_fs___FileSystem__DeleteDir(SEXP file_system_sexp, SEXP path_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
-	Rcpp::traits::input_parameter<const std::string&>::type path(path_sexp);
+extern "C" SEXP _arrow_fs___FileSystem__DeleteDir(SEXP file_system_sexp, SEXP path_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
+	arrow::r::Input<const std::string&>::type path(path_sexp);
 	fs___FileSystem__DeleteDir(file_system, path);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileSystem__DeleteDir(SEXP file_system_sexp, SEXP path_sexp){
+extern "C" SEXP _arrow_fs___FileSystem__DeleteDir(SEXP file_system_sexp, SEXP path_sexp){
 	Rf_error("Cannot call fs___FileSystem__DeleteDir(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3354,16 +3558,16 @@ RcppExport SEXP _arrow_fs___FileSystem__DeleteDir(SEXP file_system_sexp, SEXP pa
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void fs___FileSystem__DeleteDirContents(const std::shared_ptr<fs::FileSystem>& file_system, const std::string& path);
-RcppExport SEXP _arrow_fs___FileSystem__DeleteDirContents(SEXP file_system_sexp, SEXP path_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
-	Rcpp::traits::input_parameter<const std::string&>::type path(path_sexp);
+extern "C" SEXP _arrow_fs___FileSystem__DeleteDirContents(SEXP file_system_sexp, SEXP path_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
+	arrow::r::Input<const std::string&>::type path(path_sexp);
 	fs___FileSystem__DeleteDirContents(file_system, path);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileSystem__DeleteDirContents(SEXP file_system_sexp, SEXP path_sexp){
+extern "C" SEXP _arrow_fs___FileSystem__DeleteDirContents(SEXP file_system_sexp, SEXP path_sexp){
 	Rf_error("Cannot call fs___FileSystem__DeleteDirContents(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3371,16 +3575,16 @@ RcppExport SEXP _arrow_fs___FileSystem__DeleteDirContents(SEXP file_system_sexp,
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void fs___FileSystem__DeleteFile(const std::shared_ptr<fs::FileSystem>& file_system, const std::string& path);
-RcppExport SEXP _arrow_fs___FileSystem__DeleteFile(SEXP file_system_sexp, SEXP path_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
-	Rcpp::traits::input_parameter<const std::string&>::type path(path_sexp);
+extern "C" SEXP _arrow_fs___FileSystem__DeleteFile(SEXP file_system_sexp, SEXP path_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
+	arrow::r::Input<const std::string&>::type path(path_sexp);
 	fs___FileSystem__DeleteFile(file_system, path);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileSystem__DeleteFile(SEXP file_system_sexp, SEXP path_sexp){
+extern "C" SEXP _arrow_fs___FileSystem__DeleteFile(SEXP file_system_sexp, SEXP path_sexp){
 	Rf_error("Cannot call fs___FileSystem__DeleteFile(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3388,16 +3592,16 @@ RcppExport SEXP _arrow_fs___FileSystem__DeleteFile(SEXP file_system_sexp, SEXP p
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void fs___FileSystem__DeleteFiles(const std::shared_ptr<fs::FileSystem>& file_system, const std::vector<std::string>& paths);
-RcppExport SEXP _arrow_fs___FileSystem__DeleteFiles(SEXP file_system_sexp, SEXP paths_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
-	Rcpp::traits::input_parameter<const std::vector<std::string>&>::type paths(paths_sexp);
+extern "C" SEXP _arrow_fs___FileSystem__DeleteFiles(SEXP file_system_sexp, SEXP paths_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
+	arrow::r::Input<const std::vector<std::string>&>::type paths(paths_sexp);
 	fs___FileSystem__DeleteFiles(file_system, paths);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileSystem__DeleteFiles(SEXP file_system_sexp, SEXP paths_sexp){
+extern "C" SEXP _arrow_fs___FileSystem__DeleteFiles(SEXP file_system_sexp, SEXP paths_sexp){
 	Rf_error("Cannot call fs___FileSystem__DeleteFiles(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3405,17 +3609,17 @@ RcppExport SEXP _arrow_fs___FileSystem__DeleteFiles(SEXP file_system_sexp, SEXP
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void fs___FileSystem__Move(const std::shared_ptr<fs::FileSystem>& file_system, const std::string& src, const std::string& dest);
-RcppExport SEXP _arrow_fs___FileSystem__Move(SEXP file_system_sexp, SEXP src_sexp, SEXP dest_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
-	Rcpp::traits::input_parameter<const std::string&>::type src(src_sexp);
-	Rcpp::traits::input_parameter<const std::string&>::type dest(dest_sexp);
+extern "C" SEXP _arrow_fs___FileSystem__Move(SEXP file_system_sexp, SEXP src_sexp, SEXP dest_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
+	arrow::r::Input<const std::string&>::type src(src_sexp);
+	arrow::r::Input<const std::string&>::type dest(dest_sexp);
 	fs___FileSystem__Move(file_system, src, dest);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileSystem__Move(SEXP file_system_sexp, SEXP src_sexp, SEXP dest_sexp){
+extern "C" SEXP _arrow_fs___FileSystem__Move(SEXP file_system_sexp, SEXP src_sexp, SEXP dest_sexp){
 	Rf_error("Cannot call fs___FileSystem__Move(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3423,17 +3627,17 @@ RcppExport SEXP _arrow_fs___FileSystem__Move(SEXP file_system_sexp, SEXP src_sex
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void fs___FileSystem__CopyFile(const std::shared_ptr<fs::FileSystem>& file_system, const std::string& src, const std::string& dest);
-RcppExport SEXP _arrow_fs___FileSystem__CopyFile(SEXP file_system_sexp, SEXP src_sexp, SEXP dest_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
-	Rcpp::traits::input_parameter<const std::string&>::type src(src_sexp);
-	Rcpp::traits::input_parameter<const std::string&>::type dest(dest_sexp);
+extern "C" SEXP _arrow_fs___FileSystem__CopyFile(SEXP file_system_sexp, SEXP src_sexp, SEXP dest_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
+	arrow::r::Input<const std::string&>::type src(src_sexp);
+	arrow::r::Input<const std::string&>::type dest(dest_sexp);
 	fs___FileSystem__CopyFile(file_system, src, dest);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileSystem__CopyFile(SEXP file_system_sexp, SEXP src_sexp, SEXP dest_sexp){
+extern "C" SEXP _arrow_fs___FileSystem__CopyFile(SEXP file_system_sexp, SEXP src_sexp, SEXP dest_sexp){
 	Rf_error("Cannot call fs___FileSystem__CopyFile(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3441,15 +3645,15 @@ RcppExport SEXP _arrow_fs___FileSystem__CopyFile(SEXP file_system_sexp, SEXP src
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::io::InputStream> fs___FileSystem__OpenInputStream(const std::shared_ptr<fs::FileSystem>& file_system, const std::string& path);
-RcppExport SEXP _arrow_fs___FileSystem__OpenInputStream(SEXP file_system_sexp, SEXP path_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
-	Rcpp::traits::input_parameter<const std::string&>::type path(path_sexp);
-	return Rcpp::wrap(fs___FileSystem__OpenInputStream(file_system, path));
-END_RCPP
+extern "C" SEXP _arrow_fs___FileSystem__OpenInputStream(SEXP file_system_sexp, SEXP path_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
+	arrow::r::Input<const std::string&>::type path(path_sexp);
+	return cpp11::as_sexp(fs___FileSystem__OpenInputStream(file_system, path));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileSystem__OpenInputStream(SEXP file_system_sexp, SEXP path_sexp){
+extern "C" SEXP _arrow_fs___FileSystem__OpenInputStream(SEXP file_system_sexp, SEXP path_sexp){
 	Rf_error("Cannot call fs___FileSystem__OpenInputStream(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3457,15 +3661,15 @@ RcppExport SEXP _arrow_fs___FileSystem__OpenInputStream(SEXP file_system_sexp, S
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::io::RandomAccessFile> fs___FileSystem__OpenInputFile(const std::shared_ptr<fs::FileSystem>& file_system, const std::string& path);
-RcppExport SEXP _arrow_fs___FileSystem__OpenInputFile(SEXP file_system_sexp, SEXP path_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
-	Rcpp::traits::input_parameter<const std::string&>::type path(path_sexp);
-	return Rcpp::wrap(fs___FileSystem__OpenInputFile(file_system, path));
-END_RCPP
+extern "C" SEXP _arrow_fs___FileSystem__OpenInputFile(SEXP file_system_sexp, SEXP path_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
+	arrow::r::Input<const std::string&>::type path(path_sexp);
+	return cpp11::as_sexp(fs___FileSystem__OpenInputFile(file_system, path));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileSystem__OpenInputFile(SEXP file_system_sexp, SEXP path_sexp){
+extern "C" SEXP _arrow_fs___FileSystem__OpenInputFile(SEXP file_system_sexp, SEXP path_sexp){
 	Rf_error("Cannot call fs___FileSystem__OpenInputFile(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3473,15 +3677,15 @@ RcppExport SEXP _arrow_fs___FileSystem__OpenInputFile(SEXP file_system_sexp, SEX
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::io::OutputStream> fs___FileSystem__OpenOutputStream(const std::shared_ptr<fs::FileSystem>& file_system, const std::string& path);
-RcppExport SEXP _arrow_fs___FileSystem__OpenOutputStream(SEXP file_system_sexp, SEXP path_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
-	Rcpp::traits::input_parameter<const std::string&>::type path(path_sexp);
-	return Rcpp::wrap(fs___FileSystem__OpenOutputStream(file_system, path));
-END_RCPP
+extern "C" SEXP _arrow_fs___FileSystem__OpenOutputStream(SEXP file_system_sexp, SEXP path_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
+	arrow::r::Input<const std::string&>::type path(path_sexp);
+	return cpp11::as_sexp(fs___FileSystem__OpenOutputStream(file_system, path));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileSystem__OpenOutputStream(SEXP file_system_sexp, SEXP path_sexp){
+extern "C" SEXP _arrow_fs___FileSystem__OpenOutputStream(SEXP file_system_sexp, SEXP path_sexp){
 	Rf_error("Cannot call fs___FileSystem__OpenOutputStream(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3489,15 +3693,15 @@ RcppExport SEXP _arrow_fs___FileSystem__OpenOutputStream(SEXP file_system_sexp,
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::io::OutputStream> fs___FileSystem__OpenAppendStream(const std::shared_ptr<fs::FileSystem>& file_system, const std::string& path);
-RcppExport SEXP _arrow_fs___FileSystem__OpenAppendStream(SEXP file_system_sexp, SEXP path_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
-	Rcpp::traits::input_parameter<const std::string&>::type path(path_sexp);
-	return Rcpp::wrap(fs___FileSystem__OpenAppendStream(file_system, path));
-END_RCPP
+extern "C" SEXP _arrow_fs___FileSystem__OpenAppendStream(SEXP file_system_sexp, SEXP path_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
+	arrow::r::Input<const std::string&>::type path(path_sexp);
+	return cpp11::as_sexp(fs___FileSystem__OpenAppendStream(file_system, path));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileSystem__OpenAppendStream(SEXP file_system_sexp, SEXP path_sexp){
+extern "C" SEXP _arrow_fs___FileSystem__OpenAppendStream(SEXP file_system_sexp, SEXP path_sexp){
 	Rf_error("Cannot call fs___FileSystem__OpenAppendStream(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3505,14 +3709,14 @@ RcppExport SEXP _arrow_fs___FileSystem__OpenAppendStream(SEXP file_system_sexp,
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::string fs___FileSystem__type_name(const std::shared_ptr<fs::FileSystem>& file_system);
-RcppExport SEXP _arrow_fs___FileSystem__type_name(SEXP file_system_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
-	return Rcpp::wrap(fs___FileSystem__type_name(file_system));
-END_RCPP
+extern "C" SEXP _arrow_fs___FileSystem__type_name(SEXP file_system_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type file_system(file_system_sexp);
+	return cpp11::as_sexp(fs___FileSystem__type_name(file_system));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileSystem__type_name(SEXP file_system_sexp){
+extern "C" SEXP _arrow_fs___FileSystem__type_name(SEXP file_system_sexp){
 	Rf_error("Cannot call fs___FileSystem__type_name(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3520,13 +3724,13 @@ RcppExport SEXP _arrow_fs___FileSystem__type_name(SEXP file_system_sexp){
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<fs::LocalFileSystem> fs___LocalFileSystem__create();
-RcppExport SEXP _arrow_fs___LocalFileSystem__create(){
-BEGIN_RCPP
-	return Rcpp::wrap(fs___LocalFileSystem__create());
-END_RCPP
+extern "C" SEXP _arrow_fs___LocalFileSystem__create(){
+BEGIN_CPP11
+	return cpp11::as_sexp(fs___LocalFileSystem__create());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___LocalFileSystem__create(){
+extern "C" SEXP _arrow_fs___LocalFileSystem__create(){
 	Rf_error("Cannot call fs___LocalFileSystem__create(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3534,75 +3738,138 @@ RcppExport SEXP _arrow_fs___LocalFileSystem__create(){
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<fs::SubTreeFileSystem> fs___SubTreeFileSystem__create(const std::string& base_path, const std::shared_ptr<fs::FileSystem>& base_fs);
-RcppExport SEXP _arrow_fs___SubTreeFileSystem__create(SEXP base_path_sexp, SEXP base_fs_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::string&>::type base_path(base_path_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<fs::FileSystem>&>::type base_fs(base_fs_sexp);
-	return Rcpp::wrap(fs___SubTreeFileSystem__create(base_path, base_fs));
-END_RCPP
+extern "C" SEXP _arrow_fs___SubTreeFileSystem__create(SEXP base_path_sexp, SEXP base_fs_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::string&>::type base_path(base_path_sexp);
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type base_fs(base_fs_sexp);
+	return cpp11::as_sexp(fs___SubTreeFileSystem__create(base_path, base_fs));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___SubTreeFileSystem__create(SEXP base_path_sexp, SEXP base_fs_sexp){
+extern "C" SEXP _arrow_fs___SubTreeFileSystem__create(SEXP base_path_sexp, SEXP base_fs_sexp){
 	Rf_error("Cannot call fs___SubTreeFileSystem__create(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // filesystem.cpp
 #if defined(ARROW_R_WITH_ARROW)
-Rcpp::List fs___FileSystemFromUri(const std::string& path);
-RcppExport SEXP _arrow_fs___FileSystemFromUri(SEXP path_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::string&>::type path(path_sexp);
-	return Rcpp::wrap(fs___FileSystemFromUri(path));
-END_RCPP
+std::shared_ptr<fs::FileSystem> fs___SubTreeFileSystem__base_fs(const std::shared_ptr<fs::SubTreeFileSystem>& file_system);
+extern "C" SEXP _arrow_fs___SubTreeFileSystem__base_fs(SEXP file_system_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::SubTreeFileSystem>&>::type file_system(file_system_sexp);
+	return cpp11::as_sexp(fs___SubTreeFileSystem__base_fs(file_system));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_fs___SubTreeFileSystem__base_fs(SEXP file_system_sexp){
+	Rf_error("Cannot call fs___SubTreeFileSystem__base_fs(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// filesystem.cpp
+#if defined(ARROW_R_WITH_ARROW)
+std::string fs___SubTreeFileSystem__base_path(const std::shared_ptr<fs::SubTreeFileSystem>& file_system);
+extern "C" SEXP _arrow_fs___SubTreeFileSystem__base_path(SEXP file_system_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::SubTreeFileSystem>&>::type file_system(file_system_sexp);
+	return cpp11::as_sexp(fs___SubTreeFileSystem__base_path(file_system));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___FileSystemFromUri(SEXP path_sexp){
+extern "C" SEXP _arrow_fs___SubTreeFileSystem__base_path(SEXP file_system_sexp){
+	Rf_error("Cannot call fs___SubTreeFileSystem__base_path(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// filesystem.cpp
+#if defined(ARROW_R_WITH_ARROW)
+cpp11::writable::list fs___FileSystemFromUri(const std::string& path);
+extern "C" SEXP _arrow_fs___FileSystemFromUri(SEXP path_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::string&>::type path(path_sexp);
+	return cpp11::as_sexp(fs___FileSystemFromUri(path));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_fs___FileSystemFromUri(SEXP path_sexp){
 	Rf_error("Cannot call fs___FileSystemFromUri(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // filesystem.cpp
-#if defined(ARROW_R_WITH_S3)
-void fs___EnsureS3Initialized();
-RcppExport SEXP _arrow_fs___EnsureS3Initialized(){
-BEGIN_RCPP
-	fs___EnsureS3Initialized();
+#if defined(ARROW_R_WITH_ARROW)
+void fs___CopyFiles(const std::shared_ptr<fs::FileSystem>& source_fs, const std::shared_ptr<fs::FileSelector>& source_sel, const std::shared_ptr<fs::FileSystem>& destination_fs, const std::string& destination_base_dir, int64_t chunk_size, bool use_threads);
+extern "C" SEXP _arrow_fs___CopyFiles(SEXP source_fs_sexp, SEXP source_sel_sexp, SEXP destination_fs_sexp, SEXP destination_base_dir_sexp, SEXP chunk_size_sexp, SEXP use_threads_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type source_fs(source_fs_sexp);
+	arrow::r::Input<const std::shared_ptr<fs::FileSelector>&>::type source_sel(source_sel_sexp);
+	arrow::r::Input<const std::shared_ptr<fs::FileSystem>&>::type destination_fs(destination_fs_sexp);
+	arrow::r::Input<const std::string&>::type destination_base_dir(destination_base_dir_sexp);
+	arrow::r::Input<int64_t>::type chunk_size(chunk_size_sexp);
+	arrow::r::Input<bool>::type use_threads(use_threads_sexp);
+	fs___CopyFiles(source_fs, source_sel, destination_fs, destination_base_dir, chunk_size, use_threads);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___EnsureS3Initialized(){
-	Rf_error("Cannot call fs___EnsureS3Initialized(). Please use arrow::install_arrow() to install required runtime libraries. ");
+extern "C" SEXP _arrow_fs___CopyFiles(SEXP source_fs_sexp, SEXP source_sel_sexp, SEXP destination_fs_sexp, SEXP destination_base_dir_sexp, SEXP chunk_size_sexp, SEXP use_threads_sexp){
+	Rf_error("Cannot call fs___CopyFiles(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// filesystem.cpp
+#if defined(ARROW_R_WITH_S3)
+std::shared_ptr<fs::S3FileSystem> fs___S3FileSystem__create(bool anonymous, std::string access_key, std::string secret_key, std::string session_token, std::string role_arn, std::string session_name, std::string external_id, int load_frequency, std::string region, std::string endpoint_override, std::string scheme, bool background_writes);
+extern "C" SEXP _arrow_fs___S3FileSystem__create(SEXP anonymous_sexp, SEXP access_key_sexp, SEXP secret_key_sexp, SEXP session_token_sexp, SEXP role_arn_sexp, SEXP session_name_sexp, SEXP external_id_sexp, SEXP load_frequency_sexp, SEXP region_sexp, SEXP endpoint_override_sexp, SEXP scheme_sexp, SEXP background_writes_sexp){
+BEGIN_CPP11
+	arrow::r::Input<bool>::type anonymous(anonymous_sexp);
+	arrow::r::Input<std::string>::type access_key(access_key_sexp);
+	arrow::r::Input<std::string>::type secret_key(secret_key_sexp);
+	arrow::r::Input<std::string>::type session_token(session_token_sexp);
+	arrow::r::Input<std::string>::type role_arn(role_arn_sexp);
+	arrow::r::Input<std::string>::type session_name(session_name_sexp);
+	arrow::r::Input<std::string>::type external_id(external_id_sexp);
+	arrow::r::Input<int>::type load_frequency(load_frequency_sexp);
+	arrow::r::Input<std::string>::type region(region_sexp);
+	arrow::r::Input<std::string>::type endpoint_override(endpoint_override_sexp);
+	arrow::r::Input<std::string>::type scheme(scheme_sexp);
+	arrow::r::Input<bool>::type background_writes(background_writes_sexp);
+	return cpp11::as_sexp(fs___S3FileSystem__create(anonymous, access_key, secret_key, session_token, role_arn, session_name, external_id, load_frequency, region, endpoint_override, scheme, background_writes));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_fs___S3FileSystem__create(SEXP anonymous_sexp, SEXP access_key_sexp, SEXP secret_key_sexp, SEXP session_token_sexp, SEXP role_arn_sexp, SEXP session_name_sexp, SEXP external_id_sexp, SEXP load_frequency_sexp, SEXP region_sexp, SEXP endpoint_override_sexp, SEXP scheme_sexp, SEXP background_writes_sexp){
+	Rf_error("Cannot call fs___S3FileSystem__create(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // filesystem.cpp
 #if defined(ARROW_R_WITH_S3)
-std::shared_ptr<fs::S3FileSystem> fs___S3FileSystem__create();
-RcppExport SEXP _arrow_fs___S3FileSystem__create(){
-BEGIN_RCPP
-	return Rcpp::wrap(fs___S3FileSystem__create());
-END_RCPP
+std::string fs___S3FileSystem__region(const std::shared_ptr<fs::S3FileSystem>& fs);
+extern "C" SEXP _arrow_fs___S3FileSystem__region(SEXP fs_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<fs::S3FileSystem>&>::type fs(fs_sexp);
+	return cpp11::as_sexp(fs___S3FileSystem__region(fs));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_fs___S3FileSystem__create(){
-	Rf_error("Cannot call fs___S3FileSystem__create(). Please use arrow::install_arrow() to install required runtime libraries. ");
+extern "C" SEXP _arrow_fs___S3FileSystem__region(SEXP fs_sexp){
+	Rf_error("Cannot call fs___S3FileSystem__region(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Buffer> io___Readable__Read(const std::shared_ptr<arrow::io::Readable>& x, int64_t nbytes);
-RcppExport SEXP _arrow_io___Readable__Read(SEXP x_sexp, SEXP nbytes_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::Readable>&>::type x(x_sexp);
-	Rcpp::traits::input_parameter<int64_t>::type nbytes(nbytes_sexp);
-	return Rcpp::wrap(io___Readable__Read(x, nbytes));
-END_RCPP
+extern "C" SEXP _arrow_io___Readable__Read(SEXP x_sexp, SEXP nbytes_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::Readable>&>::type x(x_sexp);
+	arrow::r::Input<int64_t>::type nbytes(nbytes_sexp);
+	return cpp11::as_sexp(io___Readable__Read(x, nbytes));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___Readable__Read(SEXP x_sexp, SEXP nbytes_sexp){
+extern "C" SEXP _arrow_io___Readable__Read(SEXP x_sexp, SEXP nbytes_sexp){
 	Rf_error("Cannot call io___Readable__Read(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3610,15 +3877,15 @@ RcppExport SEXP _arrow_io___Readable__Read(SEXP x_sexp, SEXP nbytes_sexp){
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void io___InputStream__Close(const std::shared_ptr<arrow::io::InputStream>& x);
-RcppExport SEXP _arrow_io___InputStream__Close(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::InputStream>&>::type x(x_sexp);
+extern "C" SEXP _arrow_io___InputStream__Close(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::InputStream>&>::type x(x_sexp);
 	io___InputStream__Close(x);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___InputStream__Close(SEXP x_sexp){
+extern "C" SEXP _arrow_io___InputStream__Close(SEXP x_sexp){
 	Rf_error("Cannot call io___InputStream__Close(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3626,15 +3893,15 @@ RcppExport SEXP _arrow_io___InputStream__Close(SEXP x_sexp){
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void io___OutputStream__Close(const std::shared_ptr<arrow::io::OutputStream>& x);
-RcppExport SEXP _arrow_io___OutputStream__Close(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::OutputStream>&>::type x(x_sexp);
+extern "C" SEXP _arrow_io___OutputStream__Close(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::OutputStream>&>::type x(x_sexp);
 	io___OutputStream__Close(x);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___OutputStream__Close(SEXP x_sexp){
+extern "C" SEXP _arrow_io___OutputStream__Close(SEXP x_sexp){
 	Rf_error("Cannot call io___OutputStream__Close(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3642,14 +3909,14 @@ RcppExport SEXP _arrow_io___OutputStream__Close(SEXP x_sexp){
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int64_t io___RandomAccessFile__GetSize(const std::shared_ptr<arrow::io::RandomAccessFile>& x);
-RcppExport SEXP _arrow_io___RandomAccessFile__GetSize(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::RandomAccessFile>&>::type x(x_sexp);
-	return Rcpp::wrap(io___RandomAccessFile__GetSize(x));
-END_RCPP
+extern "C" SEXP _arrow_io___RandomAccessFile__GetSize(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::RandomAccessFile>&>::type x(x_sexp);
+	return cpp11::as_sexp(io___RandomAccessFile__GetSize(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___RandomAccessFile__GetSize(SEXP x_sexp){
+extern "C" SEXP _arrow_io___RandomAccessFile__GetSize(SEXP x_sexp){
 	Rf_error("Cannot call io___RandomAccessFile__GetSize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3657,14 +3924,14 @@ RcppExport SEXP _arrow_io___RandomAccessFile__GetSize(SEXP x_sexp){
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool io___RandomAccessFile__supports_zero_copy(const std::shared_ptr<arrow::io::RandomAccessFile>& x);
-RcppExport SEXP _arrow_io___RandomAccessFile__supports_zero_copy(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::RandomAccessFile>&>::type x(x_sexp);
-	return Rcpp::wrap(io___RandomAccessFile__supports_zero_copy(x));
-END_RCPP
+extern "C" SEXP _arrow_io___RandomAccessFile__supports_zero_copy(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::RandomAccessFile>&>::type x(x_sexp);
+	return cpp11::as_sexp(io___RandomAccessFile__supports_zero_copy(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___RandomAccessFile__supports_zero_copy(SEXP x_sexp){
+extern "C" SEXP _arrow_io___RandomAccessFile__supports_zero_copy(SEXP x_sexp){
 	Rf_error("Cannot call io___RandomAccessFile__supports_zero_copy(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3672,16 +3939,16 @@ RcppExport SEXP _arrow_io___RandomAccessFile__supports_zero_copy(SEXP x_sexp){
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void io___RandomAccessFile__Seek(const std::shared_ptr<arrow::io::RandomAccessFile>& x, int64_t position);
-RcppExport SEXP _arrow_io___RandomAccessFile__Seek(SEXP x_sexp, SEXP position_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::RandomAccessFile>&>::type x(x_sexp);
-	Rcpp::traits::input_parameter<int64_t>::type position(position_sexp);
+extern "C" SEXP _arrow_io___RandomAccessFile__Seek(SEXP x_sexp, SEXP position_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::RandomAccessFile>&>::type x(x_sexp);
+	arrow::r::Input<int64_t>::type position(position_sexp);
 	io___RandomAccessFile__Seek(x, position);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___RandomAccessFile__Seek(SEXP x_sexp, SEXP position_sexp){
+extern "C" SEXP _arrow_io___RandomAccessFile__Seek(SEXP x_sexp, SEXP position_sexp){
 	Rf_error("Cannot call io___RandomAccessFile__Seek(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3689,14 +3956,14 @@ RcppExport SEXP _arrow_io___RandomAccessFile__Seek(SEXP x_sexp, SEXP position_se
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int64_t io___RandomAccessFile__Tell(const std::shared_ptr<arrow::io::RandomAccessFile>& x);
-RcppExport SEXP _arrow_io___RandomAccessFile__Tell(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::RandomAccessFile>&>::type x(x_sexp);
-	return Rcpp::wrap(io___RandomAccessFile__Tell(x));
-END_RCPP
+extern "C" SEXP _arrow_io___RandomAccessFile__Tell(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::RandomAccessFile>&>::type x(x_sexp);
+	return cpp11::as_sexp(io___RandomAccessFile__Tell(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___RandomAccessFile__Tell(SEXP x_sexp){
+extern "C" SEXP _arrow_io___RandomAccessFile__Tell(SEXP x_sexp){
 	Rf_error("Cannot call io___RandomAccessFile__Tell(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3704,14 +3971,14 @@ RcppExport SEXP _arrow_io___RandomAccessFile__Tell(SEXP x_sexp){
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Buffer> io___RandomAccessFile__Read0(const std::shared_ptr<arrow::io::RandomAccessFile>& x);
-RcppExport SEXP _arrow_io___RandomAccessFile__Read0(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::RandomAccessFile>&>::type x(x_sexp);
-	return Rcpp::wrap(io___RandomAccessFile__Read0(x));
-END_RCPP
+extern "C" SEXP _arrow_io___RandomAccessFile__Read0(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::RandomAccessFile>&>::type x(x_sexp);
+	return cpp11::as_sexp(io___RandomAccessFile__Read0(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___RandomAccessFile__Read0(SEXP x_sexp){
+extern "C" SEXP _arrow_io___RandomAccessFile__Read0(SEXP x_sexp){
 	Rf_error("Cannot call io___RandomAccessFile__Read0(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3719,16 +3986,16 @@ RcppExport SEXP _arrow_io___RandomAccessFile__Read0(SEXP x_sexp){
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Buffer> io___RandomAccessFile__ReadAt(const std::shared_ptr<arrow::io::RandomAccessFile>& x, int64_t position, int64_t nbytes);
-RcppExport SEXP _arrow_io___RandomAccessFile__ReadAt(SEXP x_sexp, SEXP position_sexp, SEXP nbytes_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::RandomAccessFile>&>::type x(x_sexp);
-	Rcpp::traits::input_parameter<int64_t>::type position(position_sexp);
-	Rcpp::traits::input_parameter<int64_t>::type nbytes(nbytes_sexp);
-	return Rcpp::wrap(io___RandomAccessFile__ReadAt(x, position, nbytes));
-END_RCPP
+extern "C" SEXP _arrow_io___RandomAccessFile__ReadAt(SEXP x_sexp, SEXP position_sexp, SEXP nbytes_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::RandomAccessFile>&>::type x(x_sexp);
+	arrow::r::Input<int64_t>::type position(position_sexp);
+	arrow::r::Input<int64_t>::type nbytes(nbytes_sexp);
+	return cpp11::as_sexp(io___RandomAccessFile__ReadAt(x, position, nbytes));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___RandomAccessFile__ReadAt(SEXP x_sexp, SEXP position_sexp, SEXP nbytes_sexp){
+extern "C" SEXP _arrow_io___RandomAccessFile__ReadAt(SEXP x_sexp, SEXP position_sexp, SEXP nbytes_sexp){
 	Rf_error("Cannot call io___RandomAccessFile__ReadAt(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3736,15 +4003,15 @@ RcppExport SEXP _arrow_io___RandomAccessFile__ReadAt(SEXP x_sexp, SEXP position_
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::io::MemoryMappedFile> io___MemoryMappedFile__Create(const std::string& path, int64_t size);
-RcppExport SEXP _arrow_io___MemoryMappedFile__Create(SEXP path_sexp, SEXP size_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::string&>::type path(path_sexp);
-	Rcpp::traits::input_parameter<int64_t>::type size(size_sexp);
-	return Rcpp::wrap(io___MemoryMappedFile__Create(path, size));
-END_RCPP
+extern "C" SEXP _arrow_io___MemoryMappedFile__Create(SEXP path_sexp, SEXP size_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::string&>::type path(path_sexp);
+	arrow::r::Input<int64_t>::type size(size_sexp);
+	return cpp11::as_sexp(io___MemoryMappedFile__Create(path, size));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___MemoryMappedFile__Create(SEXP path_sexp, SEXP size_sexp){
+extern "C" SEXP _arrow_io___MemoryMappedFile__Create(SEXP path_sexp, SEXP size_sexp){
 	Rf_error("Cannot call io___MemoryMappedFile__Create(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3752,15 +4019,15 @@ RcppExport SEXP _arrow_io___MemoryMappedFile__Create(SEXP path_sexp, SEXP size_s
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::io::MemoryMappedFile> io___MemoryMappedFile__Open(const std::string& path, arrow::io::FileMode::type mode);
-RcppExport SEXP _arrow_io___MemoryMappedFile__Open(SEXP path_sexp, SEXP mode_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::string&>::type path(path_sexp);
-	Rcpp::traits::input_parameter<arrow::io::FileMode::type>::type mode(mode_sexp);
-	return Rcpp::wrap(io___MemoryMappedFile__Open(path, mode));
-END_RCPP
+extern "C" SEXP _arrow_io___MemoryMappedFile__Open(SEXP path_sexp, SEXP mode_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::string&>::type path(path_sexp);
+	arrow::r::Input<arrow::io::FileMode::type>::type mode(mode_sexp);
+	return cpp11::as_sexp(io___MemoryMappedFile__Open(path, mode));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___MemoryMappedFile__Open(SEXP path_sexp, SEXP mode_sexp){
+extern "C" SEXP _arrow_io___MemoryMappedFile__Open(SEXP path_sexp, SEXP mode_sexp){
 	Rf_error("Cannot call io___MemoryMappedFile__Open(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3768,16 +4035,16 @@ RcppExport SEXP _arrow_io___MemoryMappedFile__Open(SEXP path_sexp, SEXP mode_sex
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void io___MemoryMappedFile__Resize(const std::shared_ptr<arrow::io::MemoryMappedFile>& x, int64_t size);
-RcppExport SEXP _arrow_io___MemoryMappedFile__Resize(SEXP x_sexp, SEXP size_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::MemoryMappedFile>&>::type x(x_sexp);
-	Rcpp::traits::input_parameter<int64_t>::type size(size_sexp);
+extern "C" SEXP _arrow_io___MemoryMappedFile__Resize(SEXP x_sexp, SEXP size_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::MemoryMappedFile>&>::type x(x_sexp);
+	arrow::r::Input<int64_t>::type size(size_sexp);
 	io___MemoryMappedFile__Resize(x, size);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___MemoryMappedFile__Resize(SEXP x_sexp, SEXP size_sexp){
+extern "C" SEXP _arrow_io___MemoryMappedFile__Resize(SEXP x_sexp, SEXP size_sexp){
 	Rf_error("Cannot call io___MemoryMappedFile__Resize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3785,14 +4052,14 @@ RcppExport SEXP _arrow_io___MemoryMappedFile__Resize(SEXP x_sexp, SEXP size_sexp
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::io::ReadableFile> io___ReadableFile__Open(const std::string& path);
-RcppExport SEXP _arrow_io___ReadableFile__Open(SEXP path_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::string&>::type path(path_sexp);
-	return Rcpp::wrap(io___ReadableFile__Open(path));
-END_RCPP
+extern "C" SEXP _arrow_io___ReadableFile__Open(SEXP path_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::string&>::type path(path_sexp);
+	return cpp11::as_sexp(io___ReadableFile__Open(path));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___ReadableFile__Open(SEXP path_sexp){
+extern "C" SEXP _arrow_io___ReadableFile__Open(SEXP path_sexp){
 	Rf_error("Cannot call io___ReadableFile__Open(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3800,14 +4067,14 @@ RcppExport SEXP _arrow_io___ReadableFile__Open(SEXP path_sexp){
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::io::BufferReader> io___BufferReader__initialize(const std::shared_ptr<arrow::Buffer>& buffer);
-RcppExport SEXP _arrow_io___BufferReader__initialize(SEXP buffer_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Buffer>&>::type buffer(buffer_sexp);
-	return Rcpp::wrap(io___BufferReader__initialize(buffer));
-END_RCPP
+extern "C" SEXP _arrow_io___BufferReader__initialize(SEXP buffer_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Buffer>&>::type buffer(buffer_sexp);
+	return cpp11::as_sexp(io___BufferReader__initialize(buffer));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___BufferReader__initialize(SEXP buffer_sexp){
+extern "C" SEXP _arrow_io___BufferReader__initialize(SEXP buffer_sexp){
 	Rf_error("Cannot call io___BufferReader__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3815,16 +4082,16 @@ RcppExport SEXP _arrow_io___BufferReader__initialize(SEXP buffer_sexp){
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void io___Writable__write(const std::shared_ptr<arrow::io::Writable>& stream, const std::shared_ptr<arrow::Buffer>& buf);
-RcppExport SEXP _arrow_io___Writable__write(SEXP stream_sexp, SEXP buf_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::Writable>&>::type stream(stream_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Buffer>&>::type buf(buf_sexp);
+extern "C" SEXP _arrow_io___Writable__write(SEXP stream_sexp, SEXP buf_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::Writable>&>::type stream(stream_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Buffer>&>::type buf(buf_sexp);
 	io___Writable__write(stream, buf);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___Writable__write(SEXP stream_sexp, SEXP buf_sexp){
+extern "C" SEXP _arrow_io___Writable__write(SEXP stream_sexp, SEXP buf_sexp){
 	Rf_error("Cannot call io___Writable__write(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3832,14 +4099,14 @@ RcppExport SEXP _arrow_io___Writable__write(SEXP stream_sexp, SEXP buf_sexp){
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int64_t io___OutputStream__Tell(const std::shared_ptr<arrow::io::OutputStream>& stream);
-RcppExport SEXP _arrow_io___OutputStream__Tell(SEXP stream_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::OutputStream>&>::type stream(stream_sexp);
-	return Rcpp::wrap(io___OutputStream__Tell(stream));
-END_RCPP
+extern "C" SEXP _arrow_io___OutputStream__Tell(SEXP stream_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::OutputStream>&>::type stream(stream_sexp);
+	return cpp11::as_sexp(io___OutputStream__Tell(stream));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___OutputStream__Tell(SEXP stream_sexp){
+extern "C" SEXP _arrow_io___OutputStream__Tell(SEXP stream_sexp){
 	Rf_error("Cannot call io___OutputStream__Tell(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3847,14 +4114,14 @@ RcppExport SEXP _arrow_io___OutputStream__Tell(SEXP stream_sexp){
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::io::FileOutputStream> io___FileOutputStream__Open(const std::string& path);
-RcppExport SEXP _arrow_io___FileOutputStream__Open(SEXP path_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::string&>::type path(path_sexp);
-	return Rcpp::wrap(io___FileOutputStream__Open(path));
-END_RCPP
+extern "C" SEXP _arrow_io___FileOutputStream__Open(SEXP path_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::string&>::type path(path_sexp);
+	return cpp11::as_sexp(io___FileOutputStream__Open(path));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___FileOutputStream__Open(SEXP path_sexp){
+extern "C" SEXP _arrow_io___FileOutputStream__Open(SEXP path_sexp){
 	Rf_error("Cannot call io___FileOutputStream__Open(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3862,14 +4129,14 @@ RcppExport SEXP _arrow_io___FileOutputStream__Open(SEXP path_sexp){
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::io::BufferOutputStream> io___BufferOutputStream__Create(int64_t initial_capacity);
-RcppExport SEXP _arrow_io___BufferOutputStream__Create(SEXP initial_capacity_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<int64_t>::type initial_capacity(initial_capacity_sexp);
-	return Rcpp::wrap(io___BufferOutputStream__Create(initial_capacity));
-END_RCPP
+extern "C" SEXP _arrow_io___BufferOutputStream__Create(SEXP initial_capacity_sexp){
+BEGIN_CPP11
+	arrow::r::Input<int64_t>::type initial_capacity(initial_capacity_sexp);
+	return cpp11::as_sexp(io___BufferOutputStream__Create(initial_capacity));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___BufferOutputStream__Create(SEXP initial_capacity_sexp){
+extern "C" SEXP _arrow_io___BufferOutputStream__Create(SEXP initial_capacity_sexp){
 	Rf_error("Cannot call io___BufferOutputStream__Create(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3877,14 +4144,14 @@ RcppExport SEXP _arrow_io___BufferOutputStream__Create(SEXP initial_capacity_sex
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int64_t io___BufferOutputStream__capacity(const std::shared_ptr<arrow::io::BufferOutputStream>& stream);
-RcppExport SEXP _arrow_io___BufferOutputStream__capacity(SEXP stream_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::BufferOutputStream>&>::type stream(stream_sexp);
-	return Rcpp::wrap(io___BufferOutputStream__capacity(stream));
-END_RCPP
+extern "C" SEXP _arrow_io___BufferOutputStream__capacity(SEXP stream_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::BufferOutputStream>&>::type stream(stream_sexp);
+	return cpp11::as_sexp(io___BufferOutputStream__capacity(stream));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___BufferOutputStream__capacity(SEXP stream_sexp){
+extern "C" SEXP _arrow_io___BufferOutputStream__capacity(SEXP stream_sexp){
 	Rf_error("Cannot call io___BufferOutputStream__capacity(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3892,14 +4159,14 @@ RcppExport SEXP _arrow_io___BufferOutputStream__capacity(SEXP stream_sexp){
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Buffer> io___BufferOutputStream__Finish(const std::shared_ptr<arrow::io::BufferOutputStream>& stream);
-RcppExport SEXP _arrow_io___BufferOutputStream__Finish(SEXP stream_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::BufferOutputStream>&>::type stream(stream_sexp);
-	return Rcpp::wrap(io___BufferOutputStream__Finish(stream));
-END_RCPP
+extern "C" SEXP _arrow_io___BufferOutputStream__Finish(SEXP stream_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::BufferOutputStream>&>::type stream(stream_sexp);
+	return cpp11::as_sexp(io___BufferOutputStream__Finish(stream));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___BufferOutputStream__Finish(SEXP stream_sexp){
+extern "C" SEXP _arrow_io___BufferOutputStream__Finish(SEXP stream_sexp){
 	Rf_error("Cannot call io___BufferOutputStream__Finish(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3907,61 +4174,62 @@ RcppExport SEXP _arrow_io___BufferOutputStream__Finish(SEXP stream_sexp){
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int64_t io___BufferOutputStream__Tell(const std::shared_ptr<arrow::io::BufferOutputStream>& stream);
-RcppExport SEXP _arrow_io___BufferOutputStream__Tell(SEXP stream_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::BufferOutputStream>&>::type stream(stream_sexp);
-	return Rcpp::wrap(io___BufferOutputStream__Tell(stream));
-END_RCPP
+extern "C" SEXP _arrow_io___BufferOutputStream__Tell(SEXP stream_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::BufferOutputStream>&>::type stream(stream_sexp);
+	return cpp11::as_sexp(io___BufferOutputStream__Tell(stream));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___BufferOutputStream__Tell(SEXP stream_sexp){
+extern "C" SEXP _arrow_io___BufferOutputStream__Tell(SEXP stream_sexp){
 	Rf_error("Cannot call io___BufferOutputStream__Tell(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // io.cpp
 #if defined(ARROW_R_WITH_ARROW)
-void io___BufferOutputStream__Write(const std::shared_ptr<arrow::io::BufferOutputStream>& stream, RawVector_ bytes);
-RcppExport SEXP _arrow_io___BufferOutputStream__Write(SEXP stream_sexp, SEXP bytes_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::BufferOutputStream>&>::type stream(stream_sexp);
-	Rcpp::traits::input_parameter<RawVector_>::type bytes(bytes_sexp);
+void io___BufferOutputStream__Write(const std::shared_ptr<arrow::io::BufferOutputStream>& stream, cpp11::raws bytes);
+extern "C" SEXP _arrow_io___BufferOutputStream__Write(SEXP stream_sexp, SEXP bytes_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::BufferOutputStream>&>::type stream(stream_sexp);
+	arrow::r::Input<cpp11::raws>::type bytes(bytes_sexp);
 	io___BufferOutputStream__Write(stream, bytes);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_io___BufferOutputStream__Write(SEXP stream_sexp, SEXP bytes_sexp){
+extern "C" SEXP _arrow_io___BufferOutputStream__Write(SEXP stream_sexp, SEXP bytes_sexp){
 	Rf_error("Cannot call io___BufferOutputStream__Write(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // json.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::json::ReadOptions> json___ReadOptions__initialize(List_ options);
-RcppExport SEXP _arrow_json___ReadOptions__initialize(SEXP options_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<List_>::type options(options_sexp);
-	return Rcpp::wrap(json___ReadOptions__initialize(options));
-END_RCPP
+std::shared_ptr<arrow::json::ReadOptions> json___ReadOptions__initialize(bool use_threads, int block_size);
+extern "C" SEXP _arrow_json___ReadOptions__initialize(SEXP use_threads_sexp, SEXP block_size_sexp){
+BEGIN_CPP11
+	arrow::r::Input<bool>::type use_threads(use_threads_sexp);
+	arrow::r::Input<int>::type block_size(block_size_sexp);
+	return cpp11::as_sexp(json___ReadOptions__initialize(use_threads, block_size));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_json___ReadOptions__initialize(SEXP options_sexp){
+extern "C" SEXP _arrow_json___ReadOptions__initialize(SEXP use_threads_sexp, SEXP block_size_sexp){
 	Rf_error("Cannot call json___ReadOptions__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // json.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::json::ParseOptions> json___ParseOptions__initialize(List_ options);
-RcppExport SEXP _arrow_json___ParseOptions__initialize(SEXP options_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<List_>::type options(options_sexp);
-	return Rcpp::wrap(json___ParseOptions__initialize(options));
-END_RCPP
+std::shared_ptr<arrow::json::ParseOptions> json___ParseOptions__initialize(bool newlines_in_values);
+extern "C" SEXP _arrow_json___ParseOptions__initialize(SEXP newlines_in_values_sexp){
+BEGIN_CPP11
+	arrow::r::Input<bool>::type newlines_in_values(newlines_in_values_sexp);
+	return cpp11::as_sexp(json___ParseOptions__initialize(newlines_in_values));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_json___ParseOptions__initialize(SEXP options_sexp){
+extern "C" SEXP _arrow_json___ParseOptions__initialize(SEXP newlines_in_values_sexp){
 	Rf_error("Cannot call json___ParseOptions__initialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3969,16 +4237,16 @@ RcppExport SEXP _arrow_json___ParseOptions__initialize(SEXP options_sexp){
 // json.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::json::TableReader> json___TableReader__Make(const std::shared_ptr<arrow::io::InputStream>& input, const std::shared_ptr<arrow::json::ReadOptions>& read_options, const std::shared_ptr<arrow::json::ParseOptions>& parse_options);
-RcppExport SEXP _arrow_json___TableReader__Make(SEXP input_sexp, SEXP read_options_sexp, SEXP parse_options_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::InputStream>&>::type input(input_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::json::ReadOptions>&>::type read_options(read_options_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::json::ParseOptions>&>::type parse_options(parse_options_sexp);
-	return Rcpp::wrap(json___TableReader__Make(input, read_options, parse_options));
-END_RCPP
+extern "C" SEXP _arrow_json___TableReader__Make(SEXP input_sexp, SEXP read_options_sexp, SEXP parse_options_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::InputStream>&>::type input(input_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::json::ReadOptions>&>::type read_options(read_options_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::json::ParseOptions>&>::type parse_options(parse_options_sexp);
+	return cpp11::as_sexp(json___TableReader__Make(input, read_options, parse_options));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_json___TableReader__Make(SEXP input_sexp, SEXP read_options_sexp, SEXP parse_options_sexp){
+extern "C" SEXP _arrow_json___TableReader__Make(SEXP input_sexp, SEXP read_options_sexp, SEXP parse_options_sexp){
 	Rf_error("Cannot call json___TableReader__Make(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -3986,14 +4254,14 @@ RcppExport SEXP _arrow_json___TableReader__Make(SEXP input_sexp, SEXP read_optio
 // json.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Table> json___TableReader__Read(const std::shared_ptr<arrow::json::TableReader>& table_reader);
-RcppExport SEXP _arrow_json___TableReader__Read(SEXP table_reader_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::json::TableReader>&>::type table_reader(table_reader_sexp);
-	return Rcpp::wrap(json___TableReader__Read(table_reader));
-END_RCPP
+extern "C" SEXP _arrow_json___TableReader__Read(SEXP table_reader_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::json::TableReader>&>::type table_reader(table_reader_sexp);
+	return cpp11::as_sexp(json___TableReader__Read(table_reader));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_json___TableReader__Read(SEXP table_reader_sexp){
+extern "C" SEXP _arrow_json___TableReader__Read(SEXP table_reader_sexp){
 	Rf_error("Cannot call json___TableReader__Read(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4001,13 +4269,13 @@ RcppExport SEXP _arrow_json___TableReader__Read(SEXP table_reader_sexp){
 // memorypool.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::MemoryPool> MemoryPool__default();
-RcppExport SEXP _arrow_MemoryPool__default(){
-BEGIN_RCPP
-	return Rcpp::wrap(MemoryPool__default());
-END_RCPP
+extern "C" SEXP _arrow_MemoryPool__default(){
+BEGIN_CPP11
+	return cpp11::as_sexp(MemoryPool__default());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_MemoryPool__default(){
+extern "C" SEXP _arrow_MemoryPool__default(){
 	Rf_error("Cannot call MemoryPool__default(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4015,14 +4283,14 @@ RcppExport SEXP _arrow_MemoryPool__default(){
 // memorypool.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int MemoryPool__bytes_allocated(const std::shared_ptr<arrow::MemoryPool>& pool);
-RcppExport SEXP _arrow_MemoryPool__bytes_allocated(SEXP pool_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::MemoryPool>&>::type pool(pool_sexp);
-	return Rcpp::wrap(MemoryPool__bytes_allocated(pool));
-END_RCPP
+extern "C" SEXP _arrow_MemoryPool__bytes_allocated(SEXP pool_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::MemoryPool>&>::type pool(pool_sexp);
+	return cpp11::as_sexp(MemoryPool__bytes_allocated(pool));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_MemoryPool__bytes_allocated(SEXP pool_sexp){
+extern "C" SEXP _arrow_MemoryPool__bytes_allocated(SEXP pool_sexp){
 	Rf_error("Cannot call MemoryPool__bytes_allocated(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4030,14 +4298,14 @@ RcppExport SEXP _arrow_MemoryPool__bytes_allocated(SEXP pool_sexp){
 // memorypool.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int MemoryPool__max_memory(const std::shared_ptr<arrow::MemoryPool>& pool);
-RcppExport SEXP _arrow_MemoryPool__max_memory(SEXP pool_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::MemoryPool>&>::type pool(pool_sexp);
-	return Rcpp::wrap(MemoryPool__max_memory(pool));
-END_RCPP
+extern "C" SEXP _arrow_MemoryPool__max_memory(SEXP pool_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::MemoryPool>&>::type pool(pool_sexp);
+	return cpp11::as_sexp(MemoryPool__max_memory(pool));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_MemoryPool__max_memory(SEXP pool_sexp){
+extern "C" SEXP _arrow_MemoryPool__max_memory(SEXP pool_sexp){
 	Rf_error("Cannot call MemoryPool__max_memory(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4045,14 +4313,14 @@ RcppExport SEXP _arrow_MemoryPool__max_memory(SEXP pool_sexp){
 // message.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int64_t ipc___Message__body_length(const std::unique_ptr<arrow::ipc::Message>& message);
-RcppExport SEXP _arrow_ipc___Message__body_length(SEXP message_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::unique_ptr<arrow::ipc::Message>&>::type message(message_sexp);
-	return Rcpp::wrap(ipc___Message__body_length(message));
-END_RCPP
+extern "C" SEXP _arrow_ipc___Message__body_length(SEXP message_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::unique_ptr<arrow::ipc::Message>&>::type message(message_sexp);
+	return cpp11::as_sexp(ipc___Message__body_length(message));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___Message__body_length(SEXP message_sexp){
+extern "C" SEXP _arrow_ipc___Message__body_length(SEXP message_sexp){
 	Rf_error("Cannot call ipc___Message__body_length(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4060,14 +4328,14 @@ RcppExport SEXP _arrow_ipc___Message__body_length(SEXP message_sexp){
 // message.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Buffer> ipc___Message__metadata(const std::unique_ptr<arrow::ipc::Message>& message);
-RcppExport SEXP _arrow_ipc___Message__metadata(SEXP message_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::unique_ptr<arrow::ipc::Message>&>::type message(message_sexp);
-	return Rcpp::wrap(ipc___Message__metadata(message));
-END_RCPP
+extern "C" SEXP _arrow_ipc___Message__metadata(SEXP message_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::unique_ptr<arrow::ipc::Message>&>::type message(message_sexp);
+	return cpp11::as_sexp(ipc___Message__metadata(message));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___Message__metadata(SEXP message_sexp){
+extern "C" SEXP _arrow_ipc___Message__metadata(SEXP message_sexp){
 	Rf_error("Cannot call ipc___Message__metadata(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4075,14 +4343,14 @@ RcppExport SEXP _arrow_ipc___Message__metadata(SEXP message_sexp){
 // message.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Buffer> ipc___Message__body(const std::unique_ptr<arrow::ipc::Message>& message);
-RcppExport SEXP _arrow_ipc___Message__body(SEXP message_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::unique_ptr<arrow::ipc::Message>&>::type message(message_sexp);
-	return Rcpp::wrap(ipc___Message__body(message));
-END_RCPP
+extern "C" SEXP _arrow_ipc___Message__body(SEXP message_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::unique_ptr<arrow::ipc::Message>&>::type message(message_sexp);
+	return cpp11::as_sexp(ipc___Message__body(message));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___Message__body(SEXP message_sexp){
+extern "C" SEXP _arrow_ipc___Message__body(SEXP message_sexp){
 	Rf_error("Cannot call ipc___Message__body(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4090,14 +4358,14 @@ RcppExport SEXP _arrow_ipc___Message__body(SEXP message_sexp){
 // message.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int64_t ipc___Message__Verify(const std::unique_ptr<arrow::ipc::Message>& message);
-RcppExport SEXP _arrow_ipc___Message__Verify(SEXP message_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::unique_ptr<arrow::ipc::Message>&>::type message(message_sexp);
-	return Rcpp::wrap(ipc___Message__Verify(message));
-END_RCPP
+extern "C" SEXP _arrow_ipc___Message__Verify(SEXP message_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::unique_ptr<arrow::ipc::Message>&>::type message(message_sexp);
+	return cpp11::as_sexp(ipc___Message__Verify(message));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___Message__Verify(SEXP message_sexp){
+extern "C" SEXP _arrow_ipc___Message__Verify(SEXP message_sexp){
 	Rf_error("Cannot call ipc___Message__Verify(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4105,14 +4373,14 @@ RcppExport SEXP _arrow_ipc___Message__Verify(SEXP message_sexp){
 // message.cpp
 #if defined(ARROW_R_WITH_ARROW)
 arrow::ipc::MessageType ipc___Message__type(const std::unique_ptr<arrow::ipc::Message>& message);
-RcppExport SEXP _arrow_ipc___Message__type(SEXP message_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::unique_ptr<arrow::ipc::Message>&>::type message(message_sexp);
-	return Rcpp::wrap(ipc___Message__type(message));
-END_RCPP
+extern "C" SEXP _arrow_ipc___Message__type(SEXP message_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::unique_ptr<arrow::ipc::Message>&>::type message(message_sexp);
+	return cpp11::as_sexp(ipc___Message__type(message));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___Message__type(SEXP message_sexp){
+extern "C" SEXP _arrow_ipc___Message__type(SEXP message_sexp){
 	Rf_error("Cannot call ipc___Message__type(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4120,15 +4388,15 @@ RcppExport SEXP _arrow_ipc___Message__type(SEXP message_sexp){
 // message.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool ipc___Message__Equals(const std::unique_ptr<arrow::ipc::Message>& x, const std::unique_ptr<arrow::ipc::Message>& y);
-RcppExport SEXP _arrow_ipc___Message__Equals(SEXP x_sexp, SEXP y_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::unique_ptr<arrow::ipc::Message>&>::type x(x_sexp);
-	Rcpp::traits::input_parameter<const std::unique_ptr<arrow::ipc::Message>&>::type y(y_sexp);
-	return Rcpp::wrap(ipc___Message__Equals(x, y));
-END_RCPP
+extern "C" SEXP _arrow_ipc___Message__Equals(SEXP x_sexp, SEXP y_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::unique_ptr<arrow::ipc::Message>&>::type x(x_sexp);
+	arrow::r::Input<const std::unique_ptr<arrow::ipc::Message>&>::type y(y_sexp);
+	return cpp11::as_sexp(ipc___Message__Equals(x, y));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___Message__Equals(SEXP x_sexp, SEXP y_sexp){
+extern "C" SEXP _arrow_ipc___Message__Equals(SEXP x_sexp, SEXP y_sexp){
 	Rf_error("Cannot call ipc___Message__Equals(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4136,15 +4404,15 @@ RcppExport SEXP _arrow_ipc___Message__Equals(SEXP x_sexp, SEXP y_sexp){
 // message.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::RecordBatch> ipc___ReadRecordBatch__Message__Schema(const std::unique_ptr<arrow::ipc::Message>& message, const std::shared_ptr<arrow::Schema>& schema);
-RcppExport SEXP _arrow_ipc___ReadRecordBatch__Message__Schema(SEXP message_sexp, SEXP schema_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::unique_ptr<arrow::ipc::Message>&>::type message(message_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
-	return Rcpp::wrap(ipc___ReadRecordBatch__Message__Schema(message, schema));
-END_RCPP
+extern "C" SEXP _arrow_ipc___ReadRecordBatch__Message__Schema(SEXP message_sexp, SEXP schema_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::unique_ptr<arrow::ipc::Message>&>::type message(message_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
+	return cpp11::as_sexp(ipc___ReadRecordBatch__Message__Schema(message, schema));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___ReadRecordBatch__Message__Schema(SEXP message_sexp, SEXP schema_sexp){
+extern "C" SEXP _arrow_ipc___ReadRecordBatch__Message__Schema(SEXP message_sexp, SEXP schema_sexp){
 	Rf_error("Cannot call ipc___ReadRecordBatch__Message__Schema(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4152,14 +4420,14 @@ RcppExport SEXP _arrow_ipc___ReadRecordBatch__Message__Schema(SEXP message_sexp,
 // message.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Schema> ipc___ReadSchema_InputStream(const std::shared_ptr<arrow::io::InputStream>& stream);
-RcppExport SEXP _arrow_ipc___ReadSchema_InputStream(SEXP stream_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::InputStream>&>::type stream(stream_sexp);
-	return Rcpp::wrap(ipc___ReadSchema_InputStream(stream));
-END_RCPP
+extern "C" SEXP _arrow_ipc___ReadSchema_InputStream(SEXP stream_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::InputStream>&>::type stream(stream_sexp);
+	return cpp11::as_sexp(ipc___ReadSchema_InputStream(stream));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___ReadSchema_InputStream(SEXP stream_sexp){
+extern "C" SEXP _arrow_ipc___ReadSchema_InputStream(SEXP stream_sexp){
 	Rf_error("Cannot call ipc___ReadSchema_InputStream(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4167,14 +4435,14 @@ RcppExport SEXP _arrow_ipc___ReadSchema_InputStream(SEXP stream_sexp){
 // message.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Schema> ipc___ReadSchema_Message(const std::unique_ptr<arrow::ipc::Message>& message);
-RcppExport SEXP _arrow_ipc___ReadSchema_Message(SEXP message_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::unique_ptr<arrow::ipc::Message>&>::type message(message_sexp);
-	return Rcpp::wrap(ipc___ReadSchema_Message(message));
-END_RCPP
+extern "C" SEXP _arrow_ipc___ReadSchema_Message(SEXP message_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::unique_ptr<arrow::ipc::Message>&>::type message(message_sexp);
+	return cpp11::as_sexp(ipc___ReadSchema_Message(message));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___ReadSchema_Message(SEXP message_sexp){
+extern "C" SEXP _arrow_ipc___ReadSchema_Message(SEXP message_sexp){
 	Rf_error("Cannot call ipc___ReadSchema_Message(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4182,14 +4450,14 @@ RcppExport SEXP _arrow_ipc___ReadSchema_Message(SEXP message_sexp){
 // message.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::ipc::MessageReader> ipc___MessageReader__Open(const std::shared_ptr<arrow::io::InputStream>& stream);
-RcppExport SEXP _arrow_ipc___MessageReader__Open(SEXP stream_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::InputStream>&>::type stream(stream_sexp);
-	return Rcpp::wrap(ipc___MessageReader__Open(stream));
-END_RCPP
+extern "C" SEXP _arrow_ipc___MessageReader__Open(SEXP stream_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::InputStream>&>::type stream(stream_sexp);
+	return cpp11::as_sexp(ipc___MessageReader__Open(stream));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___MessageReader__Open(SEXP stream_sexp){
+extern "C" SEXP _arrow_ipc___MessageReader__Open(SEXP stream_sexp){
 	Rf_error("Cannot call ipc___MessageReader__Open(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4197,14 +4465,14 @@ RcppExport SEXP _arrow_ipc___MessageReader__Open(SEXP stream_sexp){
 // message.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::ipc::Message> ipc___MessageReader__ReadNextMessage(const std::unique_ptr<arrow::ipc::MessageReader>& reader);
-RcppExport SEXP _arrow_ipc___MessageReader__ReadNextMessage(SEXP reader_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::unique_ptr<arrow::ipc::MessageReader>&>::type reader(reader_sexp);
-	return Rcpp::wrap(ipc___MessageReader__ReadNextMessage(reader));
-END_RCPP
+extern "C" SEXP _arrow_ipc___MessageReader__ReadNextMessage(SEXP reader_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::unique_ptr<arrow::ipc::MessageReader>&>::type reader(reader_sexp);
+	return cpp11::as_sexp(ipc___MessageReader__ReadNextMessage(reader));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___MessageReader__ReadNextMessage(SEXP reader_sexp){
+extern "C" SEXP _arrow_ipc___MessageReader__ReadNextMessage(SEXP reader_sexp){
 	Rf_error("Cannot call ipc___MessageReader__ReadNextMessage(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4212,14 +4480,14 @@ RcppExport SEXP _arrow_ipc___MessageReader__ReadNextMessage(SEXP reader_sexp){
 // message.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::ipc::Message> ipc___ReadMessage(const std::shared_ptr<arrow::io::InputStream>& stream);
-RcppExport SEXP _arrow_ipc___ReadMessage(SEXP stream_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::InputStream>&>::type stream(stream_sexp);
-	return Rcpp::wrap(ipc___ReadMessage(stream));
-END_RCPP
+extern "C" SEXP _arrow_ipc___ReadMessage(SEXP stream_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::InputStream>&>::type stream(stream_sexp);
+	return cpp11::as_sexp(ipc___ReadMessage(stream));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___ReadMessage(SEXP stream_sexp){
+extern "C" SEXP _arrow_ipc___ReadMessage(SEXP stream_sexp){
 	Rf_error("Cannot call ipc___ReadMessage(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4227,14 +4495,14 @@ RcppExport SEXP _arrow_ipc___ReadMessage(SEXP stream_sexp){
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<parquet::ArrowReaderProperties> parquet___arrow___ArrowReaderProperties__Make(bool use_threads);
-RcppExport SEXP _arrow_parquet___arrow___ArrowReaderProperties__Make(SEXP use_threads_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<bool>::type use_threads(use_threads_sexp);
-	return Rcpp::wrap(parquet___arrow___ArrowReaderProperties__Make(use_threads));
-END_RCPP
+extern "C" SEXP _arrow_parquet___arrow___ArrowReaderProperties__Make(SEXP use_threads_sexp){
+BEGIN_CPP11
+	arrow::r::Input<bool>::type use_threads(use_threads_sexp);
+	return cpp11::as_sexp(parquet___arrow___ArrowReaderProperties__Make(use_threads));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___arrow___ArrowReaderProperties__Make(SEXP use_threads_sexp){
+extern "C" SEXP _arrow_parquet___arrow___ArrowReaderProperties__Make(SEXP use_threads_sexp){
 	Rf_error("Cannot call parquet___arrow___ArrowReaderProperties__Make(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4242,16 +4510,16 @@ RcppExport SEXP _arrow_parquet___arrow___ArrowReaderProperties__Make(SEXP use_th
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void parquet___arrow___ArrowReaderProperties__set_use_threads(const std::shared_ptr<parquet::ArrowReaderProperties>& properties, bool use_threads);
-RcppExport SEXP _arrow_parquet___arrow___ArrowReaderProperties__set_use_threads(SEXP properties_sexp, SEXP use_threads_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::ArrowReaderProperties>&>::type properties(properties_sexp);
-	Rcpp::traits::input_parameter<bool>::type use_threads(use_threads_sexp);
+extern "C" SEXP _arrow_parquet___arrow___ArrowReaderProperties__set_use_threads(SEXP properties_sexp, SEXP use_threads_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::ArrowReaderProperties>&>::type properties(properties_sexp);
+	arrow::r::Input<bool>::type use_threads(use_threads_sexp);
 	parquet___arrow___ArrowReaderProperties__set_use_threads(properties, use_threads);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___arrow___ArrowReaderProperties__set_use_threads(SEXP properties_sexp, SEXP use_threads_sexp){
+extern "C" SEXP _arrow_parquet___arrow___ArrowReaderProperties__set_use_threads(SEXP properties_sexp, SEXP use_threads_sexp){
 	Rf_error("Cannot call parquet___arrow___ArrowReaderProperties__set_use_threads(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4259,15 +4527,15 @@ RcppExport SEXP _arrow_parquet___arrow___ArrowReaderProperties__set_use_threads(
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool parquet___arrow___ArrowReaderProperties__get_use_threads(const std::shared_ptr<parquet::ArrowReaderProperties>& properties, bool use_threads);
-RcppExport SEXP _arrow_parquet___arrow___ArrowReaderProperties__get_use_threads(SEXP properties_sexp, SEXP use_threads_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::ArrowReaderProperties>&>::type properties(properties_sexp);
-	Rcpp::traits::input_parameter<bool>::type use_threads(use_threads_sexp);
-	return Rcpp::wrap(parquet___arrow___ArrowReaderProperties__get_use_threads(properties, use_threads));
-END_RCPP
+extern "C" SEXP _arrow_parquet___arrow___ArrowReaderProperties__get_use_threads(SEXP properties_sexp, SEXP use_threads_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::ArrowReaderProperties>&>::type properties(properties_sexp);
+	arrow::r::Input<bool>::type use_threads(use_threads_sexp);
+	return cpp11::as_sexp(parquet___arrow___ArrowReaderProperties__get_use_threads(properties, use_threads));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___arrow___ArrowReaderProperties__get_use_threads(SEXP properties_sexp, SEXP use_threads_sexp){
+extern "C" SEXP _arrow_parquet___arrow___ArrowReaderProperties__get_use_threads(SEXP properties_sexp, SEXP use_threads_sexp){
 	Rf_error("Cannot call parquet___arrow___ArrowReaderProperties__get_use_threads(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4275,15 +4543,15 @@ RcppExport SEXP _arrow_parquet___arrow___ArrowReaderProperties__get_use_threads(
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool parquet___arrow___ArrowReaderProperties__get_read_dictionary(const std::shared_ptr<parquet::ArrowReaderProperties>& properties, int column_index);
-RcppExport SEXP _arrow_parquet___arrow___ArrowReaderProperties__get_read_dictionary(SEXP properties_sexp, SEXP column_index_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::ArrowReaderProperties>&>::type properties(properties_sexp);
-	Rcpp::traits::input_parameter<int>::type column_index(column_index_sexp);
-	return Rcpp::wrap(parquet___arrow___ArrowReaderProperties__get_read_dictionary(properties, column_index));
-END_RCPP
+extern "C" SEXP _arrow_parquet___arrow___ArrowReaderProperties__get_read_dictionary(SEXP properties_sexp, SEXP column_index_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::ArrowReaderProperties>&>::type properties(properties_sexp);
+	arrow::r::Input<int>::type column_index(column_index_sexp);
+	return cpp11::as_sexp(parquet___arrow___ArrowReaderProperties__get_read_dictionary(properties, column_index));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___arrow___ArrowReaderProperties__get_read_dictionary(SEXP properties_sexp, SEXP column_index_sexp){
+extern "C" SEXP _arrow_parquet___arrow___ArrowReaderProperties__get_read_dictionary(SEXP properties_sexp, SEXP column_index_sexp){
 	Rf_error("Cannot call parquet___arrow___ArrowReaderProperties__get_read_dictionary(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4291,17 +4559,17 @@ RcppExport SEXP _arrow_parquet___arrow___ArrowReaderProperties__get_read_diction
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void parquet___arrow___ArrowReaderProperties__set_read_dictionary(const std::shared_ptr<parquet::ArrowReaderProperties>& properties, int column_index, bool read_dict);
-RcppExport SEXP _arrow_parquet___arrow___ArrowReaderProperties__set_read_dictionary(SEXP properties_sexp, SEXP column_index_sexp, SEXP read_dict_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::ArrowReaderProperties>&>::type properties(properties_sexp);
-	Rcpp::traits::input_parameter<int>::type column_index(column_index_sexp);
-	Rcpp::traits::input_parameter<bool>::type read_dict(read_dict_sexp);
+extern "C" SEXP _arrow_parquet___arrow___ArrowReaderProperties__set_read_dictionary(SEXP properties_sexp, SEXP column_index_sexp, SEXP read_dict_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::ArrowReaderProperties>&>::type properties(properties_sexp);
+	arrow::r::Input<int>::type column_index(column_index_sexp);
+	arrow::r::Input<bool>::type read_dict(read_dict_sexp);
 	parquet___arrow___ArrowReaderProperties__set_read_dictionary(properties, column_index, read_dict);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___arrow___ArrowReaderProperties__set_read_dictionary(SEXP properties_sexp, SEXP column_index_sexp, SEXP read_dict_sexp){
+extern "C" SEXP _arrow_parquet___arrow___ArrowReaderProperties__set_read_dictionary(SEXP properties_sexp, SEXP column_index_sexp, SEXP read_dict_sexp){
 	Rf_error("Cannot call parquet___arrow___ArrowReaderProperties__set_read_dictionary(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4309,15 +4577,15 @@ RcppExport SEXP _arrow_parquet___arrow___ArrowReaderProperties__set_read_diction
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<parquet::arrow::FileReader> parquet___arrow___FileReader__OpenFile(const std::shared_ptr<arrow::io::RandomAccessFile>& file, const std::shared_ptr<parquet::ArrowReaderProperties>& props);
-RcppExport SEXP _arrow_parquet___arrow___FileReader__OpenFile(SEXP file_sexp, SEXP props_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::RandomAccessFile>&>::type file(file_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::ArrowReaderProperties>&>::type props(props_sexp);
-	return Rcpp::wrap(parquet___arrow___FileReader__OpenFile(file, props));
-END_RCPP
+extern "C" SEXP _arrow_parquet___arrow___FileReader__OpenFile(SEXP file_sexp, SEXP props_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::RandomAccessFile>&>::type file(file_sexp);
+	arrow::r::Input<const std::shared_ptr<parquet::ArrowReaderProperties>&>::type props(props_sexp);
+	return cpp11::as_sexp(parquet___arrow___FileReader__OpenFile(file, props));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___arrow___FileReader__OpenFile(SEXP file_sexp, SEXP props_sexp){
+extern "C" SEXP _arrow_parquet___arrow___FileReader__OpenFile(SEXP file_sexp, SEXP props_sexp){
 	Rf_error("Cannot call parquet___arrow___FileReader__OpenFile(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4325,14 +4593,14 @@ RcppExport SEXP _arrow_parquet___arrow___FileReader__OpenFile(SEXP file_sexp, SE
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Table> parquet___arrow___FileReader__ReadTable1(const std::shared_ptr<parquet::arrow::FileReader>& reader);
-RcppExport SEXP _arrow_parquet___arrow___FileReader__ReadTable1(SEXP reader_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::arrow::FileReader>&>::type reader(reader_sexp);
-	return Rcpp::wrap(parquet___arrow___FileReader__ReadTable1(reader));
-END_RCPP
+extern "C" SEXP _arrow_parquet___arrow___FileReader__ReadTable1(SEXP reader_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::arrow::FileReader>&>::type reader(reader_sexp);
+	return cpp11::as_sexp(parquet___arrow___FileReader__ReadTable1(reader));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___arrow___FileReader__ReadTable1(SEXP reader_sexp){
+extern "C" SEXP _arrow_parquet___arrow___FileReader__ReadTable1(SEXP reader_sexp){
 	Rf_error("Cannot call parquet___arrow___FileReader__ReadTable1(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4340,47 +4608,159 @@ RcppExport SEXP _arrow_parquet___arrow___FileReader__ReadTable1(SEXP reader_sexp
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Table> parquet___arrow___FileReader__ReadTable2(const std::shared_ptr<parquet::arrow::FileReader>& reader, const std::vector<int>& column_indices);
-RcppExport SEXP _arrow_parquet___arrow___FileReader__ReadTable2(SEXP reader_sexp, SEXP column_indices_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::arrow::FileReader>&>::type reader(reader_sexp);
-	Rcpp::traits::input_parameter<const std::vector<int>&>::type column_indices(column_indices_sexp);
-	return Rcpp::wrap(parquet___arrow___FileReader__ReadTable2(reader, column_indices));
-END_RCPP
+extern "C" SEXP _arrow_parquet___arrow___FileReader__ReadTable2(SEXP reader_sexp, SEXP column_indices_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::arrow::FileReader>&>::type reader(reader_sexp);
+	arrow::r::Input<const std::vector<int>&>::type column_indices(column_indices_sexp);
+	return cpp11::as_sexp(parquet___arrow___FileReader__ReadTable2(reader, column_indices));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___arrow___FileReader__ReadTable2(SEXP reader_sexp, SEXP column_indices_sexp){
+extern "C" SEXP _arrow_parquet___arrow___FileReader__ReadTable2(SEXP reader_sexp, SEXP column_indices_sexp){
 	Rf_error("Cannot call parquet___arrow___FileReader__ReadTable2(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
+// parquet.cpp
+#if defined(ARROW_R_WITH_ARROW)
+std::shared_ptr<arrow::Table> parquet___arrow___FileReader__ReadRowGroup1(const std::shared_ptr<parquet::arrow::FileReader>& reader, int i);
+extern "C" SEXP _arrow_parquet___arrow___FileReader__ReadRowGroup1(SEXP reader_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::arrow::FileReader>&>::type reader(reader_sexp);
+	arrow::r::Input<int>::type i(i_sexp);
+	return cpp11::as_sexp(parquet___arrow___FileReader__ReadRowGroup1(reader, i));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_parquet___arrow___FileReader__ReadRowGroup1(SEXP reader_sexp, SEXP i_sexp){
+	Rf_error("Cannot call parquet___arrow___FileReader__ReadRowGroup1(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// parquet.cpp
+#if defined(ARROW_R_WITH_ARROW)
+std::shared_ptr<arrow::Table> parquet___arrow___FileReader__ReadRowGroup2(const std::shared_ptr<parquet::arrow::FileReader>& reader, int i, const std::vector<int>& column_indices);
+extern "C" SEXP _arrow_parquet___arrow___FileReader__ReadRowGroup2(SEXP reader_sexp, SEXP i_sexp, SEXP column_indices_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::arrow::FileReader>&>::type reader(reader_sexp);
+	arrow::r::Input<int>::type i(i_sexp);
+	arrow::r::Input<const std::vector<int>&>::type column_indices(column_indices_sexp);
+	return cpp11::as_sexp(parquet___arrow___FileReader__ReadRowGroup2(reader, i, column_indices));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_parquet___arrow___FileReader__ReadRowGroup2(SEXP reader_sexp, SEXP i_sexp, SEXP column_indices_sexp){
+	Rf_error("Cannot call parquet___arrow___FileReader__ReadRowGroup2(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// parquet.cpp
+#if defined(ARROW_R_WITH_ARROW)
+std::shared_ptr<arrow::Table> parquet___arrow___FileReader__ReadRowGroups1(const std::shared_ptr<parquet::arrow::FileReader>& reader, const std::vector<int>& row_groups);
+extern "C" SEXP _arrow_parquet___arrow___FileReader__ReadRowGroups1(SEXP reader_sexp, SEXP row_groups_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::arrow::FileReader>&>::type reader(reader_sexp);
+	arrow::r::Input<const std::vector<int>&>::type row_groups(row_groups_sexp);
+	return cpp11::as_sexp(parquet___arrow___FileReader__ReadRowGroups1(reader, row_groups));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_parquet___arrow___FileReader__ReadRowGroups1(SEXP reader_sexp, SEXP row_groups_sexp){
+	Rf_error("Cannot call parquet___arrow___FileReader__ReadRowGroups1(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// parquet.cpp
+#if defined(ARROW_R_WITH_ARROW)
+std::shared_ptr<arrow::Table> parquet___arrow___FileReader__ReadRowGroups2(const std::shared_ptr<parquet::arrow::FileReader>& reader, const std::vector<int>& row_groups, const std::vector<int>& column_indices);
+extern "C" SEXP _arrow_parquet___arrow___FileReader__ReadRowGroups2(SEXP reader_sexp, SEXP row_groups_sexp, SEXP column_indices_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::arrow::FileReader>&>::type reader(reader_sexp);
+	arrow::r::Input<const std::vector<int>&>::type row_groups(row_groups_sexp);
+	arrow::r::Input<const std::vector<int>&>::type column_indices(column_indices_sexp);
+	return cpp11::as_sexp(parquet___arrow___FileReader__ReadRowGroups2(reader, row_groups, column_indices));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_parquet___arrow___FileReader__ReadRowGroups2(SEXP reader_sexp, SEXP row_groups_sexp, SEXP column_indices_sexp){
+	Rf_error("Cannot call parquet___arrow___FileReader__ReadRowGroups2(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int64_t parquet___arrow___FileReader__num_rows(const std::shared_ptr<parquet::arrow::FileReader>& reader);
-RcppExport SEXP _arrow_parquet___arrow___FileReader__num_rows(SEXP reader_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::arrow::FileReader>&>::type reader(reader_sexp);
-	return Rcpp::wrap(parquet___arrow___FileReader__num_rows(reader));
-END_RCPP
+extern "C" SEXP _arrow_parquet___arrow___FileReader__num_rows(SEXP reader_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::arrow::FileReader>&>::type reader(reader_sexp);
+	return cpp11::as_sexp(parquet___arrow___FileReader__num_rows(reader));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___arrow___FileReader__num_rows(SEXP reader_sexp){
+extern "C" SEXP _arrow_parquet___arrow___FileReader__num_rows(SEXP reader_sexp){
 	Rf_error("Cannot call parquet___arrow___FileReader__num_rows(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
+// parquet.cpp
+#if defined(ARROW_R_WITH_ARROW)
+int parquet___arrow___FileReader__num_columns(const std::shared_ptr<parquet::arrow::FileReader>& reader);
+extern "C" SEXP _arrow_parquet___arrow___FileReader__num_columns(SEXP reader_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::arrow::FileReader>&>::type reader(reader_sexp);
+	return cpp11::as_sexp(parquet___arrow___FileReader__num_columns(reader));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_parquet___arrow___FileReader__num_columns(SEXP reader_sexp){
+	Rf_error("Cannot call parquet___arrow___FileReader__num_columns(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// parquet.cpp
+#if defined(ARROW_R_WITH_ARROW)
+int parquet___arrow___FileReader__num_row_groups(const std::shared_ptr<parquet::arrow::FileReader>& reader);
+extern "C" SEXP _arrow_parquet___arrow___FileReader__num_row_groups(SEXP reader_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::arrow::FileReader>&>::type reader(reader_sexp);
+	return cpp11::as_sexp(parquet___arrow___FileReader__num_row_groups(reader));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_parquet___arrow___FileReader__num_row_groups(SEXP reader_sexp){
+	Rf_error("Cannot call parquet___arrow___FileReader__num_row_groups(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// parquet.cpp
+#if defined(ARROW_R_WITH_ARROW)
+std::shared_ptr<arrow::ChunkedArray> parquet___arrow___FileReader__ReadColumn(const std::shared_ptr<parquet::arrow::FileReader>& reader, int i);
+extern "C" SEXP _arrow_parquet___arrow___FileReader__ReadColumn(SEXP reader_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::arrow::FileReader>&>::type reader(reader_sexp);
+	arrow::r::Input<int>::type i(i_sexp);
+	return cpp11::as_sexp(parquet___arrow___FileReader__ReadColumn(reader, i));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_parquet___arrow___FileReader__ReadColumn(SEXP reader_sexp, SEXP i_sexp){
+	Rf_error("Cannot call parquet___arrow___FileReader__ReadColumn(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<parquet::ArrowWriterProperties> parquet___ArrowWriterProperties___create(bool allow_truncated_timestamps, bool use_deprecated_int96_timestamps, int timestamp_unit);
-RcppExport SEXP _arrow_parquet___ArrowWriterProperties___create(SEXP allow_truncated_timestamps_sexp, SEXP use_deprecated_int96_timestamps_sexp, SEXP timestamp_unit_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<bool>::type allow_truncated_timestamps(allow_truncated_timestamps_sexp);
-	Rcpp::traits::input_parameter<bool>::type use_deprecated_int96_timestamps(use_deprecated_int96_timestamps_sexp);
-	Rcpp::traits::input_parameter<int>::type timestamp_unit(timestamp_unit_sexp);
-	return Rcpp::wrap(parquet___ArrowWriterProperties___create(allow_truncated_timestamps, use_deprecated_int96_timestamps, timestamp_unit));
-END_RCPP
+extern "C" SEXP _arrow_parquet___ArrowWriterProperties___create(SEXP allow_truncated_timestamps_sexp, SEXP use_deprecated_int96_timestamps_sexp, SEXP timestamp_unit_sexp){
+BEGIN_CPP11
+	arrow::r::Input<bool>::type allow_truncated_timestamps(allow_truncated_timestamps_sexp);
+	arrow::r::Input<bool>::type use_deprecated_int96_timestamps(use_deprecated_int96_timestamps_sexp);
+	arrow::r::Input<int>::type timestamp_unit(timestamp_unit_sexp);
+	return cpp11::as_sexp(parquet___ArrowWriterProperties___create(allow_truncated_timestamps, use_deprecated_int96_timestamps, timestamp_unit));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___ArrowWriterProperties___create(SEXP allow_truncated_timestamps_sexp, SEXP use_deprecated_int96_timestamps_sexp, SEXP timestamp_unit_sexp){
+extern "C" SEXP _arrow_parquet___ArrowWriterProperties___create(SEXP allow_truncated_timestamps_sexp, SEXP use_deprecated_int96_timestamps_sexp, SEXP timestamp_unit_sexp){
 	Rf_error("Cannot call parquet___ArrowWriterProperties___create(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4388,13 +4768,13 @@ RcppExport SEXP _arrow_parquet___ArrowWriterProperties___create(SEXP allow_trunc
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<parquet::WriterPropertiesBuilder> parquet___WriterProperties___Builder__create();
-RcppExport SEXP _arrow_parquet___WriterProperties___Builder__create(){
-BEGIN_RCPP
-	return Rcpp::wrap(parquet___WriterProperties___Builder__create());
-END_RCPP
+extern "C" SEXP _arrow_parquet___WriterProperties___Builder__create(){
+BEGIN_CPP11
+	return cpp11::as_sexp(parquet___WriterProperties___Builder__create());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___WriterProperties___Builder__create(){
+extern "C" SEXP _arrow_parquet___WriterProperties___Builder__create(){
 	Rf_error("Cannot call parquet___WriterProperties___Builder__create(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4402,88 +4782,88 @@ RcppExport SEXP _arrow_parquet___WriterProperties___Builder__create(){
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void parquet___WriterProperties___Builder__version(const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder, const parquet::ParquetVersion::type& version);
-RcppExport SEXP _arrow_parquet___WriterProperties___Builder__version(SEXP builder_sexp, SEXP version_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::WriterPropertiesBuilder>&>::type builder(builder_sexp);
-	Rcpp::traits::input_parameter<const parquet::ParquetVersion::type&>::type version(version_sexp);
+extern "C" SEXP _arrow_parquet___WriterProperties___Builder__version(SEXP builder_sexp, SEXP version_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::WriterPropertiesBuilder>&>::type builder(builder_sexp);
+	arrow::r::Input<const parquet::ParquetVersion::type&>::type version(version_sexp);
 	parquet___WriterProperties___Builder__version(builder, version);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___WriterProperties___Builder__version(SEXP builder_sexp, SEXP version_sexp){
+extern "C" SEXP _arrow_parquet___WriterProperties___Builder__version(SEXP builder_sexp, SEXP version_sexp){
 	Rf_error("Cannot call parquet___WriterProperties___Builder__version(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
-void parquet___ArrowWriterProperties___Builder__set_compressions(const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder, const std::vector<std::string>& paths, const Rcpp::IntegerVector& types);
-RcppExport SEXP _arrow_parquet___ArrowWriterProperties___Builder__set_compressions(SEXP builder_sexp, SEXP paths_sexp, SEXP types_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::WriterPropertiesBuilder>&>::type builder(builder_sexp);
-	Rcpp::traits::input_parameter<const std::vector<std::string>&>::type paths(paths_sexp);
-	Rcpp::traits::input_parameter<const Rcpp::IntegerVector&>::type types(types_sexp);
+void parquet___ArrowWriterProperties___Builder__set_compressions(const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder, const std::vector<std::string>& paths, cpp11::integers types);
+extern "C" SEXP _arrow_parquet___ArrowWriterProperties___Builder__set_compressions(SEXP builder_sexp, SEXP paths_sexp, SEXP types_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::WriterPropertiesBuilder>&>::type builder(builder_sexp);
+	arrow::r::Input<const std::vector<std::string>&>::type paths(paths_sexp);
+	arrow::r::Input<cpp11::integers>::type types(types_sexp);
 	parquet___ArrowWriterProperties___Builder__set_compressions(builder, paths, types);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___ArrowWriterProperties___Builder__set_compressions(SEXP builder_sexp, SEXP paths_sexp, SEXP types_sexp){
+extern "C" SEXP _arrow_parquet___ArrowWriterProperties___Builder__set_compressions(SEXP builder_sexp, SEXP paths_sexp, SEXP types_sexp){
 	Rf_error("Cannot call parquet___ArrowWriterProperties___Builder__set_compressions(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
-void parquet___ArrowWriterProperties___Builder__set_compression_levels(const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder, const std::vector<std::string>& paths, const Rcpp::IntegerVector& levels);
-RcppExport SEXP _arrow_parquet___ArrowWriterProperties___Builder__set_compression_levels(SEXP builder_sexp, SEXP paths_sexp, SEXP levels_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::WriterPropertiesBuilder>&>::type builder(builder_sexp);
-	Rcpp::traits::input_parameter<const std::vector<std::string>&>::type paths(paths_sexp);
-	Rcpp::traits::input_parameter<const Rcpp::IntegerVector&>::type levels(levels_sexp);
+void parquet___ArrowWriterProperties___Builder__set_compression_levels(const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder, const std::vector<std::string>& paths, cpp11::integers levels);
+extern "C" SEXP _arrow_parquet___ArrowWriterProperties___Builder__set_compression_levels(SEXP builder_sexp, SEXP paths_sexp, SEXP levels_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::WriterPropertiesBuilder>&>::type builder(builder_sexp);
+	arrow::r::Input<const std::vector<std::string>&>::type paths(paths_sexp);
+	arrow::r::Input<cpp11::integers>::type levels(levels_sexp);
 	parquet___ArrowWriterProperties___Builder__set_compression_levels(builder, paths, levels);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___ArrowWriterProperties___Builder__set_compression_levels(SEXP builder_sexp, SEXP paths_sexp, SEXP levels_sexp){
+extern "C" SEXP _arrow_parquet___ArrowWriterProperties___Builder__set_compression_levels(SEXP builder_sexp, SEXP paths_sexp, SEXP levels_sexp){
 	Rf_error("Cannot call parquet___ArrowWriterProperties___Builder__set_compression_levels(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
-void parquet___ArrowWriterProperties___Builder__set_use_dictionary(const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder, const std::vector<std::string>& paths, const Rcpp::LogicalVector& use_dictionary);
-RcppExport SEXP _arrow_parquet___ArrowWriterProperties___Builder__set_use_dictionary(SEXP builder_sexp, SEXP paths_sexp, SEXP use_dictionary_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::WriterPropertiesBuilder>&>::type builder(builder_sexp);
-	Rcpp::traits::input_parameter<const std::vector<std::string>&>::type paths(paths_sexp);
-	Rcpp::traits::input_parameter<const Rcpp::LogicalVector&>::type use_dictionary(use_dictionary_sexp);
+void parquet___ArrowWriterProperties___Builder__set_use_dictionary(const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder, const std::vector<std::string>& paths, cpp11::logicals use_dictionary);
+extern "C" SEXP _arrow_parquet___ArrowWriterProperties___Builder__set_use_dictionary(SEXP builder_sexp, SEXP paths_sexp, SEXP use_dictionary_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::WriterPropertiesBuilder>&>::type builder(builder_sexp);
+	arrow::r::Input<const std::vector<std::string>&>::type paths(paths_sexp);
+	arrow::r::Input<cpp11::logicals>::type use_dictionary(use_dictionary_sexp);
 	parquet___ArrowWriterProperties___Builder__set_use_dictionary(builder, paths, use_dictionary);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___ArrowWriterProperties___Builder__set_use_dictionary(SEXP builder_sexp, SEXP paths_sexp, SEXP use_dictionary_sexp){
+extern "C" SEXP _arrow_parquet___ArrowWriterProperties___Builder__set_use_dictionary(SEXP builder_sexp, SEXP paths_sexp, SEXP use_dictionary_sexp){
 	Rf_error("Cannot call parquet___ArrowWriterProperties___Builder__set_use_dictionary(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
-void parquet___ArrowWriterProperties___Builder__set_write_statistics(const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder, const std::vector<std::string>& paths, const Rcpp::LogicalVector& write_statistics);
-RcppExport SEXP _arrow_parquet___ArrowWriterProperties___Builder__set_write_statistics(SEXP builder_sexp, SEXP paths_sexp, SEXP write_statistics_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::WriterPropertiesBuilder>&>::type builder(builder_sexp);
-	Rcpp::traits::input_parameter<const std::vector<std::string>&>::type paths(paths_sexp);
-	Rcpp::traits::input_parameter<const Rcpp::LogicalVector&>::type write_statistics(write_statistics_sexp);
+void parquet___ArrowWriterProperties___Builder__set_write_statistics(const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder, const std::vector<std::string>& paths, cpp11::logicals write_statistics);
+extern "C" SEXP _arrow_parquet___ArrowWriterProperties___Builder__set_write_statistics(SEXP builder_sexp, SEXP paths_sexp, SEXP write_statistics_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::WriterPropertiesBuilder>&>::type builder(builder_sexp);
+	arrow::r::Input<const std::vector<std::string>&>::type paths(paths_sexp);
+	arrow::r::Input<cpp11::logicals>::type write_statistics(write_statistics_sexp);
 	parquet___ArrowWriterProperties___Builder__set_write_statistics(builder, paths, write_statistics);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___ArrowWriterProperties___Builder__set_write_statistics(SEXP builder_sexp, SEXP paths_sexp, SEXP write_statistics_sexp){
+extern "C" SEXP _arrow_parquet___ArrowWriterProperties___Builder__set_write_statistics(SEXP builder_sexp, SEXP paths_sexp, SEXP write_statistics_sexp){
 	Rf_error("Cannot call parquet___ArrowWriterProperties___Builder__set_write_statistics(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4491,16 +4871,16 @@ RcppExport SEXP _arrow_parquet___ArrowWriterProperties___Builder__set_write_stat
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void parquet___ArrowWriterProperties___Builder__data_page_size(const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder, int64_t data_page_size);
-RcppExport SEXP _arrow_parquet___ArrowWriterProperties___Builder__data_page_size(SEXP builder_sexp, SEXP data_page_size_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::WriterPropertiesBuilder>&>::type builder(builder_sexp);
-	Rcpp::traits::input_parameter<int64_t>::type data_page_size(data_page_size_sexp);
+extern "C" SEXP _arrow_parquet___ArrowWriterProperties___Builder__data_page_size(SEXP builder_sexp, SEXP data_page_size_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::WriterPropertiesBuilder>&>::type builder(builder_sexp);
+	arrow::r::Input<int64_t>::type data_page_size(data_page_size_sexp);
 	parquet___ArrowWriterProperties___Builder__data_page_size(builder, data_page_size);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___ArrowWriterProperties___Builder__data_page_size(SEXP builder_sexp, SEXP data_page_size_sexp){
+extern "C" SEXP _arrow_parquet___ArrowWriterProperties___Builder__data_page_size(SEXP builder_sexp, SEXP data_page_size_sexp){
 	Rf_error("Cannot call parquet___ArrowWriterProperties___Builder__data_page_size(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4508,14 +4888,14 @@ RcppExport SEXP _arrow_parquet___ArrowWriterProperties___Builder__data_page_size
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<parquet::WriterProperties> parquet___WriterProperties___Builder__build(const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder);
-RcppExport SEXP _arrow_parquet___WriterProperties___Builder__build(SEXP builder_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::WriterPropertiesBuilder>&>::type builder(builder_sexp);
-	return Rcpp::wrap(parquet___WriterProperties___Builder__build(builder));
-END_RCPP
+extern "C" SEXP _arrow_parquet___WriterProperties___Builder__build(SEXP builder_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::WriterPropertiesBuilder>&>::type builder(builder_sexp);
+	return cpp11::as_sexp(parquet___WriterProperties___Builder__build(builder));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___WriterProperties___Builder__build(SEXP builder_sexp){
+extern "C" SEXP _arrow_parquet___WriterProperties___Builder__build(SEXP builder_sexp){
 	Rf_error("Cannot call parquet___WriterProperties___Builder__build(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4523,17 +4903,17 @@ RcppExport SEXP _arrow_parquet___WriterProperties___Builder__build(SEXP builder_
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<parquet::arrow::FileWriter> parquet___arrow___ParquetFileWriter__Open(const std::shared_ptr<arrow::Schema>& schema, const std::shared_ptr<arrow::io::OutputStream>& sink, const std::shared_ptr<parquet::WriterProperties>& properties, const std::shared_ptr<parquet::ArrowWriterProperties>& arrow_properties);
-RcppExport SEXP _arrow_parquet___arrow___ParquetFileWriter__Open(SEXP schema_sexp, SEXP sink_sexp, SEXP properties_sexp, SEXP arrow_properties_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::OutputStream>&>::type sink(sink_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::WriterProperties>&>::type properties(properties_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::ArrowWriterProperties>&>::type arrow_properties(arrow_properties_sexp);
-	return Rcpp::wrap(parquet___arrow___ParquetFileWriter__Open(schema, sink, properties, arrow_properties));
-END_RCPP
+extern "C" SEXP _arrow_parquet___arrow___ParquetFileWriter__Open(SEXP schema_sexp, SEXP sink_sexp, SEXP properties_sexp, SEXP arrow_properties_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::io::OutputStream>&>::type sink(sink_sexp);
+	arrow::r::Input<const std::shared_ptr<parquet::WriterProperties>&>::type properties(properties_sexp);
+	arrow::r::Input<const std::shared_ptr<parquet::ArrowWriterProperties>&>::type arrow_properties(arrow_properties_sexp);
+	return cpp11::as_sexp(parquet___arrow___ParquetFileWriter__Open(schema, sink, properties, arrow_properties));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___arrow___ParquetFileWriter__Open(SEXP schema_sexp, SEXP sink_sexp, SEXP properties_sexp, SEXP arrow_properties_sexp){
+extern "C" SEXP _arrow_parquet___arrow___ParquetFileWriter__Open(SEXP schema_sexp, SEXP sink_sexp, SEXP properties_sexp, SEXP arrow_properties_sexp){
 	Rf_error("Cannot call parquet___arrow___ParquetFileWriter__Open(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4541,17 +4921,17 @@ RcppExport SEXP _arrow_parquet___arrow___ParquetFileWriter__Open(SEXP schema_sex
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void parquet___arrow___FileWriter__WriteTable(const std::shared_ptr<parquet::arrow::FileWriter>& writer, const std::shared_ptr<arrow::Table>& table, int64_t chunk_size);
-RcppExport SEXP _arrow_parquet___arrow___FileWriter__WriteTable(SEXP writer_sexp, SEXP table_sexp, SEXP chunk_size_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::arrow::FileWriter>&>::type writer(writer_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
-	Rcpp::traits::input_parameter<int64_t>::type chunk_size(chunk_size_sexp);
+extern "C" SEXP _arrow_parquet___arrow___FileWriter__WriteTable(SEXP writer_sexp, SEXP table_sexp, SEXP chunk_size_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::arrow::FileWriter>&>::type writer(writer_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
+	arrow::r::Input<int64_t>::type chunk_size(chunk_size_sexp);
 	parquet___arrow___FileWriter__WriteTable(writer, table, chunk_size);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___arrow___FileWriter__WriteTable(SEXP writer_sexp, SEXP table_sexp, SEXP chunk_size_sexp){
+extern "C" SEXP _arrow_parquet___arrow___FileWriter__WriteTable(SEXP writer_sexp, SEXP table_sexp, SEXP chunk_size_sexp){
 	Rf_error("Cannot call parquet___arrow___FileWriter__WriteTable(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4559,15 +4939,15 @@ RcppExport SEXP _arrow_parquet___arrow___FileWriter__WriteTable(SEXP writer_sexp
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void parquet___arrow___FileWriter__Close(const std::shared_ptr<parquet::arrow::FileWriter>& writer);
-RcppExport SEXP _arrow_parquet___arrow___FileWriter__Close(SEXP writer_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::arrow::FileWriter>&>::type writer(writer_sexp);
+extern "C" SEXP _arrow_parquet___arrow___FileWriter__Close(SEXP writer_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::arrow::FileWriter>&>::type writer(writer_sexp);
 	parquet___arrow___FileWriter__Close(writer);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___arrow___FileWriter__Close(SEXP writer_sexp){
+extern "C" SEXP _arrow_parquet___arrow___FileWriter__Close(SEXP writer_sexp){
 	Rf_error("Cannot call parquet___arrow___FileWriter__Close(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4575,18 +4955,18 @@ RcppExport SEXP _arrow_parquet___arrow___FileWriter__Close(SEXP writer_sexp){
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void parquet___arrow___WriteTable(const std::shared_ptr<arrow::Table>& table, const std::shared_ptr<arrow::io::OutputStream>& sink, const std::shared_ptr<parquet::WriterProperties>& properties, const std::shared_ptr<parquet::ArrowWriterProperties>& arrow_properties);
-RcppExport SEXP _arrow_parquet___arrow___WriteTable(SEXP table_sexp, SEXP sink_sexp, SEXP properties_sexp, SEXP arrow_properties_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::OutputStream>&>::type sink(sink_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::WriterProperties>&>::type properties(properties_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::ArrowWriterProperties>&>::type arrow_properties(arrow_properties_sexp);
+extern "C" SEXP _arrow_parquet___arrow___WriteTable(SEXP table_sexp, SEXP sink_sexp, SEXP properties_sexp, SEXP arrow_properties_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::io::OutputStream>&>::type sink(sink_sexp);
+	arrow::r::Input<const std::shared_ptr<parquet::WriterProperties>&>::type properties(properties_sexp);
+	arrow::r::Input<const std::shared_ptr<parquet::ArrowWriterProperties>&>::type arrow_properties(arrow_properties_sexp);
 	parquet___arrow___WriteTable(table, sink, properties, arrow_properties);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___arrow___WriteTable(SEXP table_sexp, SEXP sink_sexp, SEXP properties_sexp, SEXP arrow_properties_sexp){
+extern "C" SEXP _arrow_parquet___arrow___WriteTable(SEXP table_sexp, SEXP sink_sexp, SEXP properties_sexp, SEXP arrow_properties_sexp){
 	Rf_error("Cannot call parquet___arrow___WriteTable(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4594,176 +4974,176 @@ RcppExport SEXP _arrow_parquet___arrow___WriteTable(SEXP table_sexp, SEXP sink_s
 // parquet.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Schema> parquet___arrow___FileReader__GetSchema(const std::shared_ptr<parquet::arrow::FileReader>& reader);
-RcppExport SEXP _arrow_parquet___arrow___FileReader__GetSchema(SEXP reader_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<parquet::arrow::FileReader>&>::type reader(reader_sexp);
-	return Rcpp::wrap(parquet___arrow___FileReader__GetSchema(reader));
-END_RCPP
+extern "C" SEXP _arrow_parquet___arrow___FileReader__GetSchema(SEXP reader_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<parquet::arrow::FileReader>&>::type reader(reader_sexp);
+	return cpp11::as_sexp(parquet___arrow___FileReader__GetSchema(reader));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_parquet___arrow___FileReader__GetSchema(SEXP reader_sexp){
+extern "C" SEXP _arrow_parquet___arrow___FileReader__GetSchema(SEXP reader_sexp){
 	Rf_error("Cannot call parquet___arrow___FileReader__GetSchema(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // py-to-r.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::Array> ImportArray(uintptr_t array, uintptr_t schema);
-RcppExport SEXP _arrow_ImportArray(SEXP array_sexp, SEXP schema_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<uintptr_t>::type array(array_sexp);
-	Rcpp::traits::input_parameter<uintptr_t>::type schema(schema_sexp);
-	return Rcpp::wrap(ImportArray(array, schema));
-END_RCPP
+std::shared_ptr<arrow::Array> ImportArray(arrow::r::Pointer<struct ArrowArray> array, arrow::r::Pointer<struct ArrowSchema> schema);
+extern "C" SEXP _arrow_ImportArray(SEXP array_sexp, SEXP schema_sexp){
+BEGIN_CPP11
+	arrow::r::Input<arrow::r::Pointer<struct ArrowArray>>::type array(array_sexp);
+	arrow::r::Input<arrow::r::Pointer<struct ArrowSchema>>::type schema(schema_sexp);
+	return cpp11::as_sexp(ImportArray(array, schema));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ImportArray(SEXP array_sexp, SEXP schema_sexp){
+extern "C" SEXP _arrow_ImportArray(SEXP array_sexp, SEXP schema_sexp){
 	Rf_error("Cannot call ImportArray(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // py-to-r.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::RecordBatch> ImportRecordBatch(uintptr_t array, uintptr_t schema);
-RcppExport SEXP _arrow_ImportRecordBatch(SEXP array_sexp, SEXP schema_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<uintptr_t>::type array(array_sexp);
-	Rcpp::traits::input_parameter<uintptr_t>::type schema(schema_sexp);
-	return Rcpp::wrap(ImportRecordBatch(array, schema));
-END_RCPP
+std::shared_ptr<arrow::RecordBatch> ImportRecordBatch(arrow::r::Pointer<struct ArrowArray> array, arrow::r::Pointer<struct ArrowSchema> schema);
+extern "C" SEXP _arrow_ImportRecordBatch(SEXP array_sexp, SEXP schema_sexp){
+BEGIN_CPP11
+	arrow::r::Input<arrow::r::Pointer<struct ArrowArray>>::type array(array_sexp);
+	arrow::r::Input<arrow::r::Pointer<struct ArrowSchema>>::type schema(schema_sexp);
+	return cpp11::as_sexp(ImportRecordBatch(array, schema));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ImportRecordBatch(SEXP array_sexp, SEXP schema_sexp){
+extern "C" SEXP _arrow_ImportRecordBatch(SEXP array_sexp, SEXP schema_sexp){
 	Rf_error("Cannot call ImportRecordBatch(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // py-to-r.cpp
 #if defined(ARROW_R_WITH_ARROW)
-uintptr_t allocate_arrow_schema();
-RcppExport SEXP _arrow_allocate_arrow_schema(){
-BEGIN_RCPP
-	return Rcpp::wrap(allocate_arrow_schema());
-END_RCPP
+arrow::r::Pointer<struct ArrowSchema> allocate_arrow_schema();
+extern "C" SEXP _arrow_allocate_arrow_schema(){
+BEGIN_CPP11
+	return cpp11::as_sexp(allocate_arrow_schema());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_allocate_arrow_schema(){
+extern "C" SEXP _arrow_allocate_arrow_schema(){
 	Rf_error("Cannot call allocate_arrow_schema(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // py-to-r.cpp
 #if defined(ARROW_R_WITH_ARROW)
-void delete_arrow_schema(uintptr_t ptr);
-RcppExport SEXP _arrow_delete_arrow_schema(SEXP ptr_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<uintptr_t>::type ptr(ptr_sexp);
+void delete_arrow_schema(arrow::r::Pointer<struct ArrowSchema> ptr);
+extern "C" SEXP _arrow_delete_arrow_schema(SEXP ptr_sexp){
+BEGIN_CPP11
+	arrow::r::Input<arrow::r::Pointer<struct ArrowSchema>>::type ptr(ptr_sexp);
 	delete_arrow_schema(ptr);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_delete_arrow_schema(SEXP ptr_sexp){
+extern "C" SEXP _arrow_delete_arrow_schema(SEXP ptr_sexp){
 	Rf_error("Cannot call delete_arrow_schema(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // py-to-r.cpp
 #if defined(ARROW_R_WITH_ARROW)
-uintptr_t allocate_arrow_array();
-RcppExport SEXP _arrow_allocate_arrow_array(){
-BEGIN_RCPP
-	return Rcpp::wrap(allocate_arrow_array());
-END_RCPP
+arrow::r::Pointer<struct ArrowArray> allocate_arrow_array();
+extern "C" SEXP _arrow_allocate_arrow_array(){
+BEGIN_CPP11
+	return cpp11::as_sexp(allocate_arrow_array());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_allocate_arrow_array(){
+extern "C" SEXP _arrow_allocate_arrow_array(){
 	Rf_error("Cannot call allocate_arrow_array(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // py-to-r.cpp
 #if defined(ARROW_R_WITH_ARROW)
-void delete_arrow_array(uintptr_t ptr);
-RcppExport SEXP _arrow_delete_arrow_array(SEXP ptr_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<uintptr_t>::type ptr(ptr_sexp);
+void delete_arrow_array(arrow::r::Pointer<struct ArrowArray> ptr);
+extern "C" SEXP _arrow_delete_arrow_array(SEXP ptr_sexp){
+BEGIN_CPP11
+	arrow::r::Input<arrow::r::Pointer<struct ArrowArray>>::type ptr(ptr_sexp);
 	delete_arrow_array(ptr);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_delete_arrow_array(SEXP ptr_sexp){
+extern "C" SEXP _arrow_delete_arrow_array(SEXP ptr_sexp){
 	Rf_error("Cannot call delete_arrow_array(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // py-to-r.cpp
 #if defined(ARROW_R_WITH_ARROW)
-void ExportType(const std::shared_ptr<arrow::DataType>& type, uintptr_t ptr);
-RcppExport SEXP _arrow_ExportType(SEXP type_sexp, SEXP ptr_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DataType>&>::type type(type_sexp);
-	Rcpp::traits::input_parameter<uintptr_t>::type ptr(ptr_sexp);
+void ExportType(const std::shared_ptr<arrow::DataType>& type, arrow::r::Pointer<struct ArrowSchema> ptr);
+extern "C" SEXP _arrow_ExportType(SEXP type_sexp, SEXP ptr_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::DataType>&>::type type(type_sexp);
+	arrow::r::Input<arrow::r::Pointer<struct ArrowSchema>>::type ptr(ptr_sexp);
 	ExportType(type, ptr);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ExportType(SEXP type_sexp, SEXP ptr_sexp){
+extern "C" SEXP _arrow_ExportType(SEXP type_sexp, SEXP ptr_sexp){
 	Rf_error("Cannot call ExportType(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // py-to-r.cpp
 #if defined(ARROW_R_WITH_ARROW)
-void ExportSchema(const std::shared_ptr<arrow::Schema>& schema, uintptr_t ptr);
-RcppExport SEXP _arrow_ExportSchema(SEXP schema_sexp, SEXP ptr_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
-	Rcpp::traits::input_parameter<uintptr_t>::type ptr(ptr_sexp);
+void ExportSchema(const std::shared_ptr<arrow::Schema>& schema, arrow::r::Pointer<struct ArrowSchema> ptr);
+extern "C" SEXP _arrow_ExportSchema(SEXP schema_sexp, SEXP ptr_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
+	arrow::r::Input<arrow::r::Pointer<struct ArrowSchema>>::type ptr(ptr_sexp);
 	ExportSchema(schema, ptr);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ExportSchema(SEXP schema_sexp, SEXP ptr_sexp){
+extern "C" SEXP _arrow_ExportSchema(SEXP schema_sexp, SEXP ptr_sexp){
 	Rf_error("Cannot call ExportSchema(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // py-to-r.cpp
 #if defined(ARROW_R_WITH_ARROW)
-void ExportArray(const std::shared_ptr<arrow::Array>& array, uintptr_t ptr, uintptr_t schema_ptr);
-RcppExport SEXP _arrow_ExportArray(SEXP array_sexp, SEXP ptr_sexp, SEXP schema_ptr_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type array(array_sexp);
-	Rcpp::traits::input_parameter<uintptr_t>::type ptr(ptr_sexp);
-	Rcpp::traits::input_parameter<uintptr_t>::type schema_ptr(schema_ptr_sexp);
-	ExportArray(array, ptr, schema_ptr);
+void ExportArray(const std::shared_ptr<arrow::Array>& array, arrow::r::Pointer<struct ArrowArray> array_ptr, arrow::r::Pointer<struct ArrowSchema> schema_ptr);
+extern "C" SEXP _arrow_ExportArray(SEXP array_sexp, SEXP array_ptr_sexp, SEXP schema_ptr_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type array(array_sexp);
+	arrow::r::Input<arrow::r::Pointer<struct ArrowArray>>::type array_ptr(array_ptr_sexp);
+	arrow::r::Input<arrow::r::Pointer<struct ArrowSchema>>::type schema_ptr(schema_ptr_sexp);
+	ExportArray(array, array_ptr, schema_ptr);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ExportArray(SEXP array_sexp, SEXP ptr_sexp, SEXP schema_ptr_sexp){
+extern "C" SEXP _arrow_ExportArray(SEXP array_sexp, SEXP array_ptr_sexp, SEXP schema_ptr_sexp){
 	Rf_error("Cannot call ExportArray(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // py-to-r.cpp
 #if defined(ARROW_R_WITH_ARROW)
-void ExportRecordBatch(const std::shared_ptr<arrow::RecordBatch>& batch, uintptr_t ptr, uintptr_t schema_ptr);
-RcppExport SEXP _arrow_ExportRecordBatch(SEXP batch_sexp, SEXP ptr_sexp, SEXP schema_ptr_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
-	Rcpp::traits::input_parameter<uintptr_t>::type ptr(ptr_sexp);
-	Rcpp::traits::input_parameter<uintptr_t>::type schema_ptr(schema_ptr_sexp);
-	ExportRecordBatch(batch, ptr, schema_ptr);
+void ExportRecordBatch(const std::shared_ptr<arrow::RecordBatch>& batch, arrow::r::Pointer<ArrowArray> array_ptr, arrow::r::Pointer<ArrowSchema> schema_ptr);
+extern "C" SEXP _arrow_ExportRecordBatch(SEXP batch_sexp, SEXP array_ptr_sexp, SEXP schema_ptr_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
+	arrow::r::Input<arrow::r::Pointer<ArrowArray>>::type array_ptr(array_ptr_sexp);
+	arrow::r::Input<arrow::r::Pointer<ArrowSchema>>::type schema_ptr(schema_ptr_sexp);
+	ExportRecordBatch(batch, array_ptr, schema_ptr);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ExportRecordBatch(SEXP batch_sexp, SEXP ptr_sexp, SEXP schema_ptr_sexp){
+extern "C" SEXP _arrow_ExportRecordBatch(SEXP batch_sexp, SEXP array_ptr_sexp, SEXP schema_ptr_sexp){
 	Rf_error("Cannot call ExportRecordBatch(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4771,14 +5151,14 @@ RcppExport SEXP _arrow_ExportRecordBatch(SEXP batch_sexp, SEXP ptr_sexp, SEXP sc
 // recordbatch.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int RecordBatch__num_columns(const std::shared_ptr<arrow::RecordBatch>& x);
-RcppExport SEXP _arrow_RecordBatch__num_columns(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatch>&>::type x(x_sexp);
-	return Rcpp::wrap(RecordBatch__num_columns(x));
-END_RCPP
+extern "C" SEXP _arrow_RecordBatch__num_columns(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatch>&>::type x(x_sexp);
+	return cpp11::as_sexp(RecordBatch__num_columns(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_RecordBatch__num_columns(SEXP x_sexp){
+extern "C" SEXP _arrow_RecordBatch__num_columns(SEXP x_sexp){
 	Rf_error("Cannot call RecordBatch__num_columns(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4786,14 +5166,14 @@ RcppExport SEXP _arrow_RecordBatch__num_columns(SEXP x_sexp){
 // recordbatch.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int RecordBatch__num_rows(const std::shared_ptr<arrow::RecordBatch>& x);
-RcppExport SEXP _arrow_RecordBatch__num_rows(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatch>&>::type x(x_sexp);
-	return Rcpp::wrap(RecordBatch__num_rows(x));
-END_RCPP
+extern "C" SEXP _arrow_RecordBatch__num_rows(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatch>&>::type x(x_sexp);
+	return cpp11::as_sexp(RecordBatch__num_rows(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_RecordBatch__num_rows(SEXP x_sexp){
+extern "C" SEXP _arrow_RecordBatch__num_rows(SEXP x_sexp){
 	Rf_error("Cannot call RecordBatch__num_rows(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4801,30 +5181,30 @@ RcppExport SEXP _arrow_RecordBatch__num_rows(SEXP x_sexp){
 // recordbatch.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Schema> RecordBatch__schema(const std::shared_ptr<arrow::RecordBatch>& x);
-RcppExport SEXP _arrow_RecordBatch__schema(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatch>&>::type x(x_sexp);
-	return Rcpp::wrap(RecordBatch__schema(x));
-END_RCPP
+extern "C" SEXP _arrow_RecordBatch__schema(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatch>&>::type x(x_sexp);
+	return cpp11::as_sexp(RecordBatch__schema(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_RecordBatch__schema(SEXP x_sexp){
+extern "C" SEXP _arrow_RecordBatch__schema(SEXP x_sexp){
 	Rf_error("Cannot call RecordBatch__schema(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // recordbatch.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::RecordBatch> RecordBatch__ReplaceSchemaMetadata(const std::shared_ptr<arrow::RecordBatch>& x, Rcpp::CharacterVector metadata);
-RcppExport SEXP _arrow_RecordBatch__ReplaceSchemaMetadata(SEXP x_sexp, SEXP metadata_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatch>&>::type x(x_sexp);
-	Rcpp::traits::input_parameter<Rcpp::CharacterVector>::type metadata(metadata_sexp);
-	return Rcpp::wrap(RecordBatch__ReplaceSchemaMetadata(x, metadata));
-END_RCPP
+std::shared_ptr<arrow::RecordBatch> RecordBatch__ReplaceSchemaMetadata(const std::shared_ptr<arrow::RecordBatch>& x, cpp11::strings metadata);
+extern "C" SEXP _arrow_RecordBatch__ReplaceSchemaMetadata(SEXP x_sexp, SEXP metadata_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatch>&>::type x(x_sexp);
+	arrow::r::Input<cpp11::strings>::type metadata(metadata_sexp);
+	return cpp11::as_sexp(RecordBatch__ReplaceSchemaMetadata(x, metadata));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_RecordBatch__ReplaceSchemaMetadata(SEXP x_sexp, SEXP metadata_sexp){
+extern "C" SEXP _arrow_RecordBatch__ReplaceSchemaMetadata(SEXP x_sexp, SEXP metadata_sexp){
 	Rf_error("Cannot call RecordBatch__ReplaceSchemaMetadata(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4832,30 +5212,30 @@ RcppExport SEXP _arrow_RecordBatch__ReplaceSchemaMetadata(SEXP x_sexp, SEXP meta
 // recordbatch.cpp
 #if defined(ARROW_R_WITH_ARROW)
 arrow::ArrayVector RecordBatch__columns(const std::shared_ptr<arrow::RecordBatch>& batch);
-RcppExport SEXP _arrow_RecordBatch__columns(SEXP batch_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
-	return Rcpp::wrap(RecordBatch__columns(batch));
-END_RCPP
+extern "C" SEXP _arrow_RecordBatch__columns(SEXP batch_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
+	return cpp11::as_sexp(RecordBatch__columns(batch));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_RecordBatch__columns(SEXP batch_sexp){
+extern "C" SEXP _arrow_RecordBatch__columns(SEXP batch_sexp){
 	Rf_error("Cannot call RecordBatch__columns(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // recordbatch.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::Array> RecordBatch__column(const std::shared_ptr<arrow::RecordBatch>& batch, int i);
-RcppExport SEXP _arrow_RecordBatch__column(SEXP batch_sexp, SEXP i_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
-	Rcpp::traits::input_parameter<int>::type i(i_sexp);
-	return Rcpp::wrap(RecordBatch__column(batch, i));
-END_RCPP
+std::shared_ptr<arrow::Array> RecordBatch__column(const std::shared_ptr<arrow::RecordBatch>& batch, R_xlen_t i);
+extern "C" SEXP _arrow_RecordBatch__column(SEXP batch_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
+	arrow::r::Input<R_xlen_t>::type i(i_sexp);
+	return cpp11::as_sexp(RecordBatch__column(batch, i));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_RecordBatch__column(SEXP batch_sexp, SEXP i_sexp){
+extern "C" SEXP _arrow_RecordBatch__column(SEXP batch_sexp, SEXP i_sexp){
 	Rf_error("Cannot call RecordBatch__column(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -4863,158 +5243,143 @@ RcppExport SEXP _arrow_RecordBatch__column(SEXP batch_sexp, SEXP i_sexp){
 // recordbatch.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Array> RecordBatch__GetColumnByName(const std::shared_ptr<arrow::RecordBatch>& batch, const std::string& name);
-RcppExport SEXP _arrow_RecordBatch__GetColumnByName(SEXP batch_sexp, SEXP name_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
-	Rcpp::traits::input_parameter<const std::string&>::type name(name_sexp);
-	return Rcpp::wrap(RecordBatch__GetColumnByName(batch, name));
-END_RCPP
+extern "C" SEXP _arrow_RecordBatch__GetColumnByName(SEXP batch_sexp, SEXP name_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
+	arrow::r::Input<const std::string&>::type name(name_sexp);
+	return cpp11::as_sexp(RecordBatch__GetColumnByName(batch, name));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_RecordBatch__GetColumnByName(SEXP batch_sexp, SEXP name_sexp){
+extern "C" SEXP _arrow_RecordBatch__GetColumnByName(SEXP batch_sexp, SEXP name_sexp){
 	Rf_error("Cannot call RecordBatch__GetColumnByName(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // recordbatch.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::RecordBatch> RecordBatch__select(const std::shared_ptr<arrow::RecordBatch>& batch, const Rcpp::IntegerVector& indices);
-RcppExport SEXP _arrow_RecordBatch__select(SEXP batch_sexp, SEXP indices_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
-	Rcpp::traits::input_parameter<const Rcpp::IntegerVector&>::type indices(indices_sexp);
-	return Rcpp::wrap(RecordBatch__select(batch, indices));
-END_RCPP
+std::shared_ptr<arrow::RecordBatch> RecordBatch__SelectColumns(const std::shared_ptr<arrow::RecordBatch>& batch, cpp11::integers indices);
+extern "C" SEXP _arrow_RecordBatch__SelectColumns(SEXP batch_sexp, SEXP indices_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
+	arrow::r::Input<cpp11::integers>::type indices(indices_sexp);
+	return cpp11::as_sexp(RecordBatch__SelectColumns(batch, indices));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_RecordBatch__select(SEXP batch_sexp, SEXP indices_sexp){
-	Rf_error("Cannot call RecordBatch__select(). Please use arrow::install_arrow() to install required runtime libraries. ");
-}
-#endif
-
-// recordbatch.cpp
-#if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::RecordBatch> RecordBatch__from_dataframe(Rcpp::DataFrame tbl);
-RcppExport SEXP _arrow_RecordBatch__from_dataframe(SEXP tbl_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<Rcpp::DataFrame>::type tbl(tbl_sexp);
-	return Rcpp::wrap(RecordBatch__from_dataframe(tbl));
-END_RCPP
-}
-#else
-RcppExport SEXP _arrow_RecordBatch__from_dataframe(SEXP tbl_sexp){
-	Rf_error("Cannot call RecordBatch__from_dataframe(). Please use arrow::install_arrow() to install required runtime libraries. ");
+extern "C" SEXP _arrow_RecordBatch__SelectColumns(SEXP batch_sexp, SEXP indices_sexp){
+	Rf_error("Cannot call RecordBatch__SelectColumns(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // recordbatch.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool RecordBatch__Equals(const std::shared_ptr<arrow::RecordBatch>& self, const std::shared_ptr<arrow::RecordBatch>& other, bool check_metadata);
-RcppExport SEXP _arrow_RecordBatch__Equals(SEXP self_sexp, SEXP other_sexp, SEXP check_metadata_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatch>&>::type self(self_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatch>&>::type other(other_sexp);
-	Rcpp::traits::input_parameter<bool>::type check_metadata(check_metadata_sexp);
-	return Rcpp::wrap(RecordBatch__Equals(self, other, check_metadata));
-END_RCPP
+extern "C" SEXP _arrow_RecordBatch__Equals(SEXP self_sexp, SEXP other_sexp, SEXP check_metadata_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatch>&>::type self(self_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatch>&>::type other(other_sexp);
+	arrow::r::Input<bool>::type check_metadata(check_metadata_sexp);
+	return cpp11::as_sexp(RecordBatch__Equals(self, other, check_metadata));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_RecordBatch__Equals(SEXP self_sexp, SEXP other_sexp, SEXP check_metadata_sexp){
+extern "C" SEXP _arrow_RecordBatch__Equals(SEXP self_sexp, SEXP other_sexp, SEXP check_metadata_sexp){
 	Rf_error("Cannot call RecordBatch__Equals(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // recordbatch.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::RecordBatch> RecordBatch__RemoveColumn(const std::shared_ptr<arrow::RecordBatch>& batch, int i);
-RcppExport SEXP _arrow_RecordBatch__RemoveColumn(SEXP batch_sexp, SEXP i_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
-	Rcpp::traits::input_parameter<int>::type i(i_sexp);
-	return Rcpp::wrap(RecordBatch__RemoveColumn(batch, i));
-END_RCPP
+std::shared_ptr<arrow::RecordBatch> RecordBatch__RemoveColumn(const std::shared_ptr<arrow::RecordBatch>& batch, R_xlen_t i);
+extern "C" SEXP _arrow_RecordBatch__RemoveColumn(SEXP batch_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
+	arrow::r::Input<R_xlen_t>::type i(i_sexp);
+	return cpp11::as_sexp(RecordBatch__RemoveColumn(batch, i));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_RecordBatch__RemoveColumn(SEXP batch_sexp, SEXP i_sexp){
+extern "C" SEXP _arrow_RecordBatch__RemoveColumn(SEXP batch_sexp, SEXP i_sexp){
 	Rf_error("Cannot call RecordBatch__RemoveColumn(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // recordbatch.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::string RecordBatch__column_name(const std::shared_ptr<arrow::RecordBatch>& batch, int i);
-RcppExport SEXP _arrow_RecordBatch__column_name(SEXP batch_sexp, SEXP i_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
-	Rcpp::traits::input_parameter<int>::type i(i_sexp);
-	return Rcpp::wrap(RecordBatch__column_name(batch, i));
-END_RCPP
+std::string RecordBatch__column_name(const std::shared_ptr<arrow::RecordBatch>& batch, R_xlen_t i);
+extern "C" SEXP _arrow_RecordBatch__column_name(SEXP batch_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
+	arrow::r::Input<R_xlen_t>::type i(i_sexp);
+	return cpp11::as_sexp(RecordBatch__column_name(batch, i));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_RecordBatch__column_name(SEXP batch_sexp, SEXP i_sexp){
+extern "C" SEXP _arrow_RecordBatch__column_name(SEXP batch_sexp, SEXP i_sexp){
 	Rf_error("Cannot call RecordBatch__column_name(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // recordbatch.cpp
 #if defined(ARROW_R_WITH_ARROW)
-Rcpp::CharacterVector RecordBatch__names(const std::shared_ptr<arrow::RecordBatch>& batch);
-RcppExport SEXP _arrow_RecordBatch__names(SEXP batch_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
-	return Rcpp::wrap(RecordBatch__names(batch));
-END_RCPP
+cpp11::writable::strings RecordBatch__names(const std::shared_ptr<arrow::RecordBatch>& batch);
+extern "C" SEXP _arrow_RecordBatch__names(SEXP batch_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
+	return cpp11::as_sexp(RecordBatch__names(batch));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_RecordBatch__names(SEXP batch_sexp){
+extern "C" SEXP _arrow_RecordBatch__names(SEXP batch_sexp){
 	Rf_error("Cannot call RecordBatch__names(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // recordbatch.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::RecordBatch> RecordBatch__Slice1(const std::shared_ptr<arrow::RecordBatch>& self, int offset);
-RcppExport SEXP _arrow_RecordBatch__Slice1(SEXP self_sexp, SEXP offset_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatch>&>::type self(self_sexp);
-	Rcpp::traits::input_parameter<int>::type offset(offset_sexp);
-	return Rcpp::wrap(RecordBatch__Slice1(self, offset));
-END_RCPP
+std::shared_ptr<arrow::RecordBatch> RecordBatch__Slice1(const std::shared_ptr<arrow::RecordBatch>& self, R_xlen_t offset);
+extern "C" SEXP _arrow_RecordBatch__Slice1(SEXP self_sexp, SEXP offset_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatch>&>::type self(self_sexp);
+	arrow::r::Input<R_xlen_t>::type offset(offset_sexp);
+	return cpp11::as_sexp(RecordBatch__Slice1(self, offset));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_RecordBatch__Slice1(SEXP self_sexp, SEXP offset_sexp){
+extern "C" SEXP _arrow_RecordBatch__Slice1(SEXP self_sexp, SEXP offset_sexp){
 	Rf_error("Cannot call RecordBatch__Slice1(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // recordbatch.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::RecordBatch> RecordBatch__Slice2(const std::shared_ptr<arrow::RecordBatch>& self, int offset, int length);
-RcppExport SEXP _arrow_RecordBatch__Slice2(SEXP self_sexp, SEXP offset_sexp, SEXP length_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatch>&>::type self(self_sexp);
-	Rcpp::traits::input_parameter<int>::type offset(offset_sexp);
-	Rcpp::traits::input_parameter<int>::type length(length_sexp);
-	return Rcpp::wrap(RecordBatch__Slice2(self, offset, length));
-END_RCPP
+std::shared_ptr<arrow::RecordBatch> RecordBatch__Slice2(const std::shared_ptr<arrow::RecordBatch>& self, R_xlen_t offset, R_xlen_t length);
+extern "C" SEXP _arrow_RecordBatch__Slice2(SEXP self_sexp, SEXP offset_sexp, SEXP length_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatch>&>::type self(self_sexp);
+	arrow::r::Input<R_xlen_t>::type offset(offset_sexp);
+	arrow::r::Input<R_xlen_t>::type length(length_sexp);
+	return cpp11::as_sexp(RecordBatch__Slice2(self, offset, length));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_RecordBatch__Slice2(SEXP self_sexp, SEXP offset_sexp, SEXP length_sexp){
+extern "C" SEXP _arrow_RecordBatch__Slice2(SEXP self_sexp, SEXP offset_sexp, SEXP length_sexp){
 	Rf_error("Cannot call RecordBatch__Slice2(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // recordbatch.cpp
 #if defined(ARROW_R_WITH_ARROW)
-Rcpp::RawVector ipc___SerializeRecordBatch__Raw(const std::shared_ptr<arrow::RecordBatch>& batch);
-RcppExport SEXP _arrow_ipc___SerializeRecordBatch__Raw(SEXP batch_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
-	return Rcpp::wrap(ipc___SerializeRecordBatch__Raw(batch));
-END_RCPP
+cpp11::raws ipc___SerializeRecordBatch__Raw(const std::shared_ptr<arrow::RecordBatch>& batch);
+extern "C" SEXP _arrow_ipc___SerializeRecordBatch__Raw(SEXP batch_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
+	return cpp11::as_sexp(ipc___SerializeRecordBatch__Raw(batch));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___SerializeRecordBatch__Raw(SEXP batch_sexp){
+extern "C" SEXP _arrow_ipc___SerializeRecordBatch__Raw(SEXP batch_sexp){
 	Rf_error("Cannot call ipc___SerializeRecordBatch__Raw(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5022,15 +5387,15 @@ RcppExport SEXP _arrow_ipc___SerializeRecordBatch__Raw(SEXP batch_sexp){
 // recordbatch.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::RecordBatch> ipc___ReadRecordBatch__InputStream__Schema(const std::shared_ptr<arrow::io::InputStream>& stream, const std::shared_ptr<arrow::Schema>& schema);
-RcppExport SEXP _arrow_ipc___ReadRecordBatch__InputStream__Schema(SEXP stream_sexp, SEXP schema_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::InputStream>&>::type stream(stream_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
-	return Rcpp::wrap(ipc___ReadRecordBatch__InputStream__Schema(stream, schema));
-END_RCPP
+extern "C" SEXP _arrow_ipc___ReadRecordBatch__InputStream__Schema(SEXP stream_sexp, SEXP schema_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::InputStream>&>::type stream(stream_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
+	return cpp11::as_sexp(ipc___ReadRecordBatch__InputStream__Schema(stream, schema));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___ReadRecordBatch__InputStream__Schema(SEXP stream_sexp, SEXP schema_sexp){
+extern "C" SEXP _arrow_ipc___ReadRecordBatch__InputStream__Schema(SEXP stream_sexp, SEXP schema_sexp){
 	Rf_error("Cannot call ipc___ReadRecordBatch__InputStream__Schema(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5038,15 +5403,15 @@ RcppExport SEXP _arrow_ipc___ReadRecordBatch__InputStream__Schema(SEXP stream_se
 // recordbatch.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::RecordBatch> RecordBatch__from_arrays(SEXP schema_sxp, SEXP lst);
-RcppExport SEXP _arrow_RecordBatch__from_arrays(SEXP schema_sxp_sexp, SEXP lst_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<SEXP>::type schema_sxp(schema_sxp_sexp);
-	Rcpp::traits::input_parameter<SEXP>::type lst(lst_sexp);
-	return Rcpp::wrap(RecordBatch__from_arrays(schema_sxp, lst));
-END_RCPP
+extern "C" SEXP _arrow_RecordBatch__from_arrays(SEXP schema_sxp_sexp, SEXP lst_sexp){
+BEGIN_CPP11
+	arrow::r::Input<SEXP>::type schema_sxp(schema_sxp_sexp);
+	arrow::r::Input<SEXP>::type lst(lst_sexp);
+	return cpp11::as_sexp(RecordBatch__from_arrays(schema_sxp, lst));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_RecordBatch__from_arrays(SEXP schema_sxp_sexp, SEXP lst_sexp){
+extern "C" SEXP _arrow_RecordBatch__from_arrays(SEXP schema_sxp_sexp, SEXP lst_sexp){
 	Rf_error("Cannot call RecordBatch__from_arrays(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5054,14 +5419,14 @@ RcppExport SEXP _arrow_RecordBatch__from_arrays(SEXP schema_sxp_sexp, SEXP lst_s
 // recordbatchreader.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Schema> RecordBatchReader__schema(const std::shared_ptr<arrow::RecordBatchReader>& reader);
-RcppExport SEXP _arrow_RecordBatchReader__schema(SEXP reader_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatchReader>&>::type reader(reader_sexp);
-	return Rcpp::wrap(RecordBatchReader__schema(reader));
-END_RCPP
+extern "C" SEXP _arrow_RecordBatchReader__schema(SEXP reader_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatchReader>&>::type reader(reader_sexp);
+	return cpp11::as_sexp(RecordBatchReader__schema(reader));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_RecordBatchReader__schema(SEXP reader_sexp){
+extern "C" SEXP _arrow_RecordBatchReader__schema(SEXP reader_sexp){
 	Rf_error("Cannot call RecordBatchReader__schema(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5069,14 +5434,14 @@ RcppExport SEXP _arrow_RecordBatchReader__schema(SEXP reader_sexp){
 // recordbatchreader.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::RecordBatch> RecordBatchReader__ReadNext(const std::shared_ptr<arrow::RecordBatchReader>& reader);
-RcppExport SEXP _arrow_RecordBatchReader__ReadNext(SEXP reader_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatchReader>&>::type reader(reader_sexp);
-	return Rcpp::wrap(RecordBatchReader__ReadNext(reader));
-END_RCPP
+extern "C" SEXP _arrow_RecordBatchReader__ReadNext(SEXP reader_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatchReader>&>::type reader(reader_sexp);
+	return cpp11::as_sexp(RecordBatchReader__ReadNext(reader));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_RecordBatchReader__ReadNext(SEXP reader_sexp){
+extern "C" SEXP _arrow_RecordBatchReader__ReadNext(SEXP reader_sexp){
 	Rf_error("Cannot call RecordBatchReader__ReadNext(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5084,14 +5449,14 @@ RcppExport SEXP _arrow_RecordBatchReader__ReadNext(SEXP reader_sexp){
 // recordbatchreader.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::RecordBatchReader> ipc___RecordBatchStreamReader__Open(const std::shared_ptr<arrow::io::InputStream>& stream);
-RcppExport SEXP _arrow_ipc___RecordBatchStreamReader__Open(SEXP stream_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::InputStream>&>::type stream(stream_sexp);
-	return Rcpp::wrap(ipc___RecordBatchStreamReader__Open(stream));
-END_RCPP
+extern "C" SEXP _arrow_ipc___RecordBatchStreamReader__Open(SEXP stream_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::InputStream>&>::type stream(stream_sexp);
+	return cpp11::as_sexp(ipc___RecordBatchStreamReader__Open(stream));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___RecordBatchStreamReader__Open(SEXP stream_sexp){
+extern "C" SEXP _arrow_ipc___RecordBatchStreamReader__Open(SEXP stream_sexp){
 	Rf_error("Cannot call ipc___RecordBatchStreamReader__Open(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5099,14 +5464,14 @@ RcppExport SEXP _arrow_ipc___RecordBatchStreamReader__Open(SEXP stream_sexp){
 // recordbatchreader.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::vector<std::shared_ptr<arrow::RecordBatch>> ipc___RecordBatchStreamReader__batches(const std::shared_ptr<arrow::ipc::RecordBatchStreamReader>& reader);
-RcppExport SEXP _arrow_ipc___RecordBatchStreamReader__batches(SEXP reader_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ipc::RecordBatchStreamReader>&>::type reader(reader_sexp);
-	return Rcpp::wrap(ipc___RecordBatchStreamReader__batches(reader));
-END_RCPP
+extern "C" SEXP _arrow_ipc___RecordBatchStreamReader__batches(SEXP reader_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ipc::RecordBatchStreamReader>&>::type reader(reader_sexp);
+	return cpp11::as_sexp(ipc___RecordBatchStreamReader__batches(reader));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___RecordBatchStreamReader__batches(SEXP reader_sexp){
+extern "C" SEXP _arrow_ipc___RecordBatchStreamReader__batches(SEXP reader_sexp){
 	Rf_error("Cannot call ipc___RecordBatchStreamReader__batches(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5114,14 +5479,14 @@ RcppExport SEXP _arrow_ipc___RecordBatchStreamReader__batches(SEXP reader_sexp){
 // recordbatchreader.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Schema> ipc___RecordBatchFileReader__schema(const std::shared_ptr<arrow::ipc::RecordBatchFileReader>& reader);
-RcppExport SEXP _arrow_ipc___RecordBatchFileReader__schema(SEXP reader_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ipc::RecordBatchFileReader>&>::type reader(reader_sexp);
-	return Rcpp::wrap(ipc___RecordBatchFileReader__schema(reader));
-END_RCPP
+extern "C" SEXP _arrow_ipc___RecordBatchFileReader__schema(SEXP reader_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ipc::RecordBatchFileReader>&>::type reader(reader_sexp);
+	return cpp11::as_sexp(ipc___RecordBatchFileReader__schema(reader));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___RecordBatchFileReader__schema(SEXP reader_sexp){
+extern "C" SEXP _arrow_ipc___RecordBatchFileReader__schema(SEXP reader_sexp){
 	Rf_error("Cannot call ipc___RecordBatchFileReader__schema(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5129,14 +5494,14 @@ RcppExport SEXP _arrow_ipc___RecordBatchFileReader__schema(SEXP reader_sexp){
 // recordbatchreader.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int ipc___RecordBatchFileReader__num_record_batches(const std::shared_ptr<arrow::ipc::RecordBatchFileReader>& reader);
-RcppExport SEXP _arrow_ipc___RecordBatchFileReader__num_record_batches(SEXP reader_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ipc::RecordBatchFileReader>&>::type reader(reader_sexp);
-	return Rcpp::wrap(ipc___RecordBatchFileReader__num_record_batches(reader));
-END_RCPP
+extern "C" SEXP _arrow_ipc___RecordBatchFileReader__num_record_batches(SEXP reader_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ipc::RecordBatchFileReader>&>::type reader(reader_sexp);
+	return cpp11::as_sexp(ipc___RecordBatchFileReader__num_record_batches(reader));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___RecordBatchFileReader__num_record_batches(SEXP reader_sexp){
+extern "C" SEXP _arrow_ipc___RecordBatchFileReader__num_record_batches(SEXP reader_sexp){
 	Rf_error("Cannot call ipc___RecordBatchFileReader__num_record_batches(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5144,15 +5509,15 @@ RcppExport SEXP _arrow_ipc___RecordBatchFileReader__num_record_batches(SEXP read
 // recordbatchreader.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::RecordBatch> ipc___RecordBatchFileReader__ReadRecordBatch(const std::shared_ptr<arrow::ipc::RecordBatchFileReader>& reader, int i);
-RcppExport SEXP _arrow_ipc___RecordBatchFileReader__ReadRecordBatch(SEXP reader_sexp, SEXP i_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ipc::RecordBatchFileReader>&>::type reader(reader_sexp);
-	Rcpp::traits::input_parameter<int>::type i(i_sexp);
-	return Rcpp::wrap(ipc___RecordBatchFileReader__ReadRecordBatch(reader, i));
-END_RCPP
+extern "C" SEXP _arrow_ipc___RecordBatchFileReader__ReadRecordBatch(SEXP reader_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ipc::RecordBatchFileReader>&>::type reader(reader_sexp);
+	arrow::r::Input<int>::type i(i_sexp);
+	return cpp11::as_sexp(ipc___RecordBatchFileReader__ReadRecordBatch(reader, i));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___RecordBatchFileReader__ReadRecordBatch(SEXP reader_sexp, SEXP i_sexp){
+extern "C" SEXP _arrow_ipc___RecordBatchFileReader__ReadRecordBatch(SEXP reader_sexp, SEXP i_sexp){
 	Rf_error("Cannot call ipc___RecordBatchFileReader__ReadRecordBatch(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5160,14 +5525,14 @@ RcppExport SEXP _arrow_ipc___RecordBatchFileReader__ReadRecordBatch(SEXP reader_
 // recordbatchreader.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::ipc::RecordBatchFileReader> ipc___RecordBatchFileReader__Open(const std::shared_ptr<arrow::io::RandomAccessFile>& file);
-RcppExport SEXP _arrow_ipc___RecordBatchFileReader__Open(SEXP file_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::RandomAccessFile>&>::type file(file_sexp);
-	return Rcpp::wrap(ipc___RecordBatchFileReader__Open(file));
-END_RCPP
+extern "C" SEXP _arrow_ipc___RecordBatchFileReader__Open(SEXP file_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::RandomAccessFile>&>::type file(file_sexp);
+	return cpp11::as_sexp(ipc___RecordBatchFileReader__Open(file));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___RecordBatchFileReader__Open(SEXP file_sexp){
+extern "C" SEXP _arrow_ipc___RecordBatchFileReader__Open(SEXP file_sexp){
 	Rf_error("Cannot call ipc___RecordBatchFileReader__Open(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5175,14 +5540,14 @@ RcppExport SEXP _arrow_ipc___RecordBatchFileReader__Open(SEXP file_sexp){
 // recordbatchreader.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Table> Table__from_RecordBatchFileReader(const std::shared_ptr<arrow::ipc::RecordBatchFileReader>& reader);
-RcppExport SEXP _arrow_Table__from_RecordBatchFileReader(SEXP reader_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ipc::RecordBatchFileReader>&>::type reader(reader_sexp);
-	return Rcpp::wrap(Table__from_RecordBatchFileReader(reader));
-END_RCPP
+extern "C" SEXP _arrow_Table__from_RecordBatchFileReader(SEXP reader_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ipc::RecordBatchFileReader>&>::type reader(reader_sexp);
+	return cpp11::as_sexp(Table__from_RecordBatchFileReader(reader));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Table__from_RecordBatchFileReader(SEXP reader_sexp){
+extern "C" SEXP _arrow_Table__from_RecordBatchFileReader(SEXP reader_sexp){
 	Rf_error("Cannot call Table__from_RecordBatchFileReader(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5190,14 +5555,14 @@ RcppExport SEXP _arrow_Table__from_RecordBatchFileReader(SEXP reader_sexp){
 // recordbatchreader.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Table> Table__from_RecordBatchStreamReader(const std::shared_ptr<arrow::ipc::RecordBatchStreamReader>& reader);
-RcppExport SEXP _arrow_Table__from_RecordBatchStreamReader(SEXP reader_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ipc::RecordBatchStreamReader>&>::type reader(reader_sexp);
-	return Rcpp::wrap(Table__from_RecordBatchStreamReader(reader));
-END_RCPP
+extern "C" SEXP _arrow_Table__from_RecordBatchStreamReader(SEXP reader_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ipc::RecordBatchStreamReader>&>::type reader(reader_sexp);
+	return cpp11::as_sexp(Table__from_RecordBatchStreamReader(reader));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Table__from_RecordBatchStreamReader(SEXP reader_sexp){
+extern "C" SEXP _arrow_Table__from_RecordBatchStreamReader(SEXP reader_sexp){
 	Rf_error("Cannot call Table__from_RecordBatchStreamReader(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5205,14 +5570,14 @@ RcppExport SEXP _arrow_Table__from_RecordBatchStreamReader(SEXP reader_sexp){
 // recordbatchreader.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::vector<std::shared_ptr<arrow::RecordBatch>> ipc___RecordBatchFileReader__batches(const std::shared_ptr<arrow::ipc::RecordBatchFileReader>& reader);
-RcppExport SEXP _arrow_ipc___RecordBatchFileReader__batches(SEXP reader_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ipc::RecordBatchFileReader>&>::type reader(reader_sexp);
-	return Rcpp::wrap(ipc___RecordBatchFileReader__batches(reader));
-END_RCPP
+extern "C" SEXP _arrow_ipc___RecordBatchFileReader__batches(SEXP reader_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ipc::RecordBatchFileReader>&>::type reader(reader_sexp);
+	return cpp11::as_sexp(ipc___RecordBatchFileReader__batches(reader));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___RecordBatchFileReader__batches(SEXP reader_sexp){
+extern "C" SEXP _arrow_ipc___RecordBatchFileReader__batches(SEXP reader_sexp){
 	Rf_error("Cannot call ipc___RecordBatchFileReader__batches(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5220,16 +5585,16 @@ RcppExport SEXP _arrow_ipc___RecordBatchFileReader__batches(SEXP reader_sexp){
 // recordbatchwriter.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void ipc___RecordBatchWriter__WriteRecordBatch(const std::shared_ptr<arrow::ipc::RecordBatchWriter>& batch_writer, const std::shared_ptr<arrow::RecordBatch>& batch);
-RcppExport SEXP _arrow_ipc___RecordBatchWriter__WriteRecordBatch(SEXP batch_writer_sexp, SEXP batch_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ipc::RecordBatchWriter>&>::type batch_writer(batch_writer_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
+extern "C" SEXP _arrow_ipc___RecordBatchWriter__WriteRecordBatch(SEXP batch_writer_sexp, SEXP batch_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ipc::RecordBatchWriter>&>::type batch_writer(batch_writer_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::RecordBatch>&>::type batch(batch_sexp);
 	ipc___RecordBatchWriter__WriteRecordBatch(batch_writer, batch);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___RecordBatchWriter__WriteRecordBatch(SEXP batch_writer_sexp, SEXP batch_sexp){
+extern "C" SEXP _arrow_ipc___RecordBatchWriter__WriteRecordBatch(SEXP batch_writer_sexp, SEXP batch_sexp){
 	Rf_error("Cannot call ipc___RecordBatchWriter__WriteRecordBatch(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5237,16 +5602,16 @@ RcppExport SEXP _arrow_ipc___RecordBatchWriter__WriteRecordBatch(SEXP batch_writ
 // recordbatchwriter.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void ipc___RecordBatchWriter__WriteTable(const std::shared_ptr<arrow::ipc::RecordBatchWriter>& batch_writer, const std::shared_ptr<arrow::Table>& table);
-RcppExport SEXP _arrow_ipc___RecordBatchWriter__WriteTable(SEXP batch_writer_sexp, SEXP table_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ipc::RecordBatchWriter>&>::type batch_writer(batch_writer_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
+extern "C" SEXP _arrow_ipc___RecordBatchWriter__WriteTable(SEXP batch_writer_sexp, SEXP table_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ipc::RecordBatchWriter>&>::type batch_writer(batch_writer_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
 	ipc___RecordBatchWriter__WriteTable(batch_writer, table);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___RecordBatchWriter__WriteTable(SEXP batch_writer_sexp, SEXP table_sexp){
+extern "C" SEXP _arrow_ipc___RecordBatchWriter__WriteTable(SEXP batch_writer_sexp, SEXP table_sexp){
 	Rf_error("Cannot call ipc___RecordBatchWriter__WriteTable(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5254,15 +5619,15 @@ RcppExport SEXP _arrow_ipc___RecordBatchWriter__WriteTable(SEXP batch_writer_sex
 // recordbatchwriter.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void ipc___RecordBatchWriter__Close(const std::shared_ptr<arrow::ipc::RecordBatchWriter>& batch_writer);
-RcppExport SEXP _arrow_ipc___RecordBatchWriter__Close(SEXP batch_writer_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::ipc::RecordBatchWriter>&>::type batch_writer(batch_writer_sexp);
+extern "C" SEXP _arrow_ipc___RecordBatchWriter__Close(SEXP batch_writer_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::ipc::RecordBatchWriter>&>::type batch_writer(batch_writer_sexp);
 	ipc___RecordBatchWriter__Close(batch_writer);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___RecordBatchWriter__Close(SEXP batch_writer_sexp){
+extern "C" SEXP _arrow_ipc___RecordBatchWriter__Close(SEXP batch_writer_sexp){
 	Rf_error("Cannot call ipc___RecordBatchWriter__Close(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5270,17 +5635,17 @@ RcppExport SEXP _arrow_ipc___RecordBatchWriter__Close(SEXP batch_writer_sexp){
 // recordbatchwriter.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::ipc::RecordBatchWriter> ipc___RecordBatchFileWriter__Open(const std::shared_ptr<arrow::io::OutputStream>& stream, const std::shared_ptr<arrow::Schema>& schema, bool use_legacy_format, arrow::ipc::MetadataVersion metadata_version);
-RcppExport SEXP _arrow_ipc___RecordBatchFileWriter__Open(SEXP stream_sexp, SEXP schema_sexp, SEXP use_legacy_format_sexp, SEXP metadata_version_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::OutputStream>&>::type stream(stream_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
-	Rcpp::traits::input_parameter<bool>::type use_legacy_format(use_legacy_format_sexp);
-	Rcpp::traits::input_parameter<arrow::ipc::MetadataVersion>::type metadata_version(metadata_version_sexp);
-	return Rcpp::wrap(ipc___RecordBatchFileWriter__Open(stream, schema, use_legacy_format, metadata_version));
-END_RCPP
+extern "C" SEXP _arrow_ipc___RecordBatchFileWriter__Open(SEXP stream_sexp, SEXP schema_sexp, SEXP use_legacy_format_sexp, SEXP metadata_version_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::OutputStream>&>::type stream(stream_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
+	arrow::r::Input<bool>::type use_legacy_format(use_legacy_format_sexp);
+	arrow::r::Input<arrow::ipc::MetadataVersion>::type metadata_version(metadata_version_sexp);
+	return cpp11::as_sexp(ipc___RecordBatchFileWriter__Open(stream, schema, use_legacy_format, metadata_version));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___RecordBatchFileWriter__Open(SEXP stream_sexp, SEXP schema_sexp, SEXP use_legacy_format_sexp, SEXP metadata_version_sexp){
+extern "C" SEXP _arrow_ipc___RecordBatchFileWriter__Open(SEXP stream_sexp, SEXP schema_sexp, SEXP use_legacy_format_sexp, SEXP metadata_version_sexp){
 	Rf_error("Cannot call ipc___RecordBatchFileWriter__Open(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5288,17 +5653,17 @@ RcppExport SEXP _arrow_ipc___RecordBatchFileWriter__Open(SEXP stream_sexp, SEXP
 // recordbatchwriter.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::ipc::RecordBatchWriter> ipc___RecordBatchStreamWriter__Open(const std::shared_ptr<arrow::io::OutputStream>& stream, const std::shared_ptr<arrow::Schema>& schema, bool use_legacy_format, arrow::ipc::MetadataVersion metadata_version);
-RcppExport SEXP _arrow_ipc___RecordBatchStreamWriter__Open(SEXP stream_sexp, SEXP schema_sexp, SEXP use_legacy_format_sexp, SEXP metadata_version_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::io::OutputStream>&>::type stream(stream_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
-	Rcpp::traits::input_parameter<bool>::type use_legacy_format(use_legacy_format_sexp);
-	Rcpp::traits::input_parameter<arrow::ipc::MetadataVersion>::type metadata_version(metadata_version_sexp);
-	return Rcpp::wrap(ipc___RecordBatchStreamWriter__Open(stream, schema, use_legacy_format, metadata_version));
-END_RCPP
+extern "C" SEXP _arrow_ipc___RecordBatchStreamWriter__Open(SEXP stream_sexp, SEXP schema_sexp, SEXP use_legacy_format_sexp, SEXP metadata_version_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::io::OutputStream>&>::type stream(stream_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
+	arrow::r::Input<bool>::type use_legacy_format(use_legacy_format_sexp);
+	arrow::r::Input<arrow::ipc::MetadataVersion>::type metadata_version(metadata_version_sexp);
+	return cpp11::as_sexp(ipc___RecordBatchStreamWriter__Open(stream, schema, use_legacy_format, metadata_version));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_ipc___RecordBatchStreamWriter__Open(SEXP stream_sexp, SEXP schema_sexp, SEXP use_legacy_format_sexp, SEXP metadata_version_sexp){
+extern "C" SEXP _arrow_ipc___RecordBatchStreamWriter__Open(SEXP stream_sexp, SEXP schema_sexp, SEXP use_legacy_format_sexp, SEXP metadata_version_sexp){
 	Rf_error("Cannot call ipc___RecordBatchStreamWriter__Open(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5306,15 +5671,15 @@ RcppExport SEXP _arrow_ipc___RecordBatchStreamWriter__Open(SEXP stream_sexp, SEX
 // scalar.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Scalar> Array__GetScalar(const std::shared_ptr<arrow::Array>& x, int64_t i);
-RcppExport SEXP _arrow_Array__GetScalar(SEXP x_sexp, SEXP i_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Array>&>::type x(x_sexp);
-	Rcpp::traits::input_parameter<int64_t>::type i(i_sexp);
-	return Rcpp::wrap(Array__GetScalar(x, i));
-END_RCPP
+extern "C" SEXP _arrow_Array__GetScalar(SEXP x_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Array>&>::type x(x_sexp);
+	arrow::r::Input<int64_t>::type i(i_sexp);
+	return cpp11::as_sexp(Array__GetScalar(x, i));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Array__GetScalar(SEXP x_sexp, SEXP i_sexp){
+extern "C" SEXP _arrow_Array__GetScalar(SEXP x_sexp, SEXP i_sexp){
 	Rf_error("Cannot call Array__GetScalar(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5322,14 +5687,14 @@ RcppExport SEXP _arrow_Array__GetScalar(SEXP x_sexp, SEXP i_sexp){
 // scalar.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::string Scalar__ToString(const std::shared_ptr<arrow::Scalar>& s);
-RcppExport SEXP _arrow_Scalar__ToString(SEXP s_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Scalar>&>::type s(s_sexp);
-	return Rcpp::wrap(Scalar__ToString(s));
-END_RCPP
+extern "C" SEXP _arrow_Scalar__ToString(SEXP s_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Scalar>&>::type s(s_sexp);
+	return cpp11::as_sexp(Scalar__ToString(s));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Scalar__ToString(SEXP s_sexp){
+extern "C" SEXP _arrow_Scalar__ToString(SEXP s_sexp){
 	Rf_error("Cannot call Scalar__ToString(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5337,15 +5702,15 @@ RcppExport SEXP _arrow_Scalar__ToString(SEXP s_sexp){
 // scalar.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Scalar> Scalar__CastTo(const std::shared_ptr<arrow::Scalar>& s, const std::shared_ptr<arrow::DataType>& t);
-RcppExport SEXP _arrow_Scalar__CastTo(SEXP s_sexp, SEXP t_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Scalar>&>::type s(s_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::DataType>&>::type t(t_sexp);
-	return Rcpp::wrap(Scalar__CastTo(s, t));
-END_RCPP
+extern "C" SEXP _arrow_Scalar__CastTo(SEXP s_sexp, SEXP t_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Scalar>&>::type s(s_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::DataType>&>::type t(t_sexp);
+	return cpp11::as_sexp(Scalar__CastTo(s, t));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Scalar__CastTo(SEXP s_sexp, SEXP t_sexp){
+extern "C" SEXP _arrow_Scalar__CastTo(SEXP s_sexp, SEXP t_sexp){
 	Rf_error("Cannot call Scalar__CastTo(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5353,15 +5718,15 @@ RcppExport SEXP _arrow_Scalar__CastTo(SEXP s_sexp, SEXP t_sexp){
 // scalar.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Scalar> StructScalar__field(const std::shared_ptr<arrow::StructScalar>& s, int i);
-RcppExport SEXP _arrow_StructScalar__field(SEXP s_sexp, SEXP i_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::StructScalar>&>::type s(s_sexp);
-	Rcpp::traits::input_parameter<int>::type i(i_sexp);
-	return Rcpp::wrap(StructScalar__field(s, i));
-END_RCPP
+extern "C" SEXP _arrow_StructScalar__field(SEXP s_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::StructScalar>&>::type s(s_sexp);
+	arrow::r::Input<int>::type i(i_sexp);
+	return cpp11::as_sexp(StructScalar__field(s, i));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_StructScalar__field(SEXP s_sexp, SEXP i_sexp){
+extern "C" SEXP _arrow_StructScalar__field(SEXP s_sexp, SEXP i_sexp){
 	Rf_error("Cannot call StructScalar__field(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5369,15 +5734,15 @@ RcppExport SEXP _arrow_StructScalar__field(SEXP s_sexp, SEXP i_sexp){
 // scalar.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Scalar> StructScalar__GetFieldByName(const std::shared_ptr<arrow::StructScalar>& s, const std::string& name);
-RcppExport SEXP _arrow_StructScalar__GetFieldByName(SEXP s_sexp, SEXP name_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::StructScalar>&>::type s(s_sexp);
-	Rcpp::traits::input_parameter<const std::string&>::type name(name_sexp);
-	return Rcpp::wrap(StructScalar__GetFieldByName(s, name));
-END_RCPP
+extern "C" SEXP _arrow_StructScalar__GetFieldByName(SEXP s_sexp, SEXP name_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::StructScalar>&>::type s(s_sexp);
+	arrow::r::Input<const std::string&>::type name(name_sexp);
+	return cpp11::as_sexp(StructScalar__GetFieldByName(s, name));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_StructScalar__GetFieldByName(SEXP s_sexp, SEXP name_sexp){
+extern "C" SEXP _arrow_StructScalar__GetFieldByName(SEXP s_sexp, SEXP name_sexp){
 	Rf_error("Cannot call StructScalar__GetFieldByName(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5385,14 +5750,14 @@ RcppExport SEXP _arrow_StructScalar__GetFieldByName(SEXP s_sexp, SEXP name_sexp)
 // scalar.cpp
 #if defined(ARROW_R_WITH_ARROW)
 SEXP Scalar__as_vector(const std::shared_ptr<arrow::Scalar>& scalar);
-RcppExport SEXP _arrow_Scalar__as_vector(SEXP scalar_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Scalar>&>::type scalar(scalar_sexp);
-	return Rcpp::wrap(Scalar__as_vector(scalar));
-END_RCPP
+extern "C" SEXP _arrow_Scalar__as_vector(SEXP scalar_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Scalar>&>::type scalar(scalar_sexp);
+	return cpp11::as_sexp(Scalar__as_vector(scalar));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Scalar__as_vector(SEXP scalar_sexp){
+extern "C" SEXP _arrow_Scalar__as_vector(SEXP scalar_sexp){
 	Rf_error("Cannot call Scalar__as_vector(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5400,14 +5765,14 @@ RcppExport SEXP _arrow_Scalar__as_vector(SEXP scalar_sexp){
 // scalar.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool Scalar__is_valid(const std::shared_ptr<arrow::Scalar>& s);
-RcppExport SEXP _arrow_Scalar__is_valid(SEXP s_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Scalar>&>::type s(s_sexp);
-	return Rcpp::wrap(Scalar__is_valid(s));
-END_RCPP
+extern "C" SEXP _arrow_Scalar__is_valid(SEXP s_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Scalar>&>::type s(s_sexp);
+	return cpp11::as_sexp(Scalar__is_valid(s));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Scalar__is_valid(SEXP s_sexp){
+extern "C" SEXP _arrow_Scalar__is_valid(SEXP s_sexp){
 	Rf_error("Cannot call Scalar__is_valid(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5415,29 +5780,29 @@ RcppExport SEXP _arrow_Scalar__is_valid(SEXP s_sexp){
 // scalar.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::DataType> Scalar__type(const std::shared_ptr<arrow::Scalar>& s);
-RcppExport SEXP _arrow_Scalar__type(SEXP s_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Scalar>&>::type s(s_sexp);
-	return Rcpp::wrap(Scalar__type(s));
-END_RCPP
+extern "C" SEXP _arrow_Scalar__type(SEXP s_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Scalar>&>::type s(s_sexp);
+	return cpp11::as_sexp(Scalar__type(s));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Scalar__type(SEXP s_sexp){
+extern "C" SEXP _arrow_Scalar__type(SEXP s_sexp){
 	Rf_error("Cannot call Scalar__type(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // schema.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::Schema> schema_(Rcpp::List fields);
-RcppExport SEXP _arrow_schema_(SEXP fields_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<Rcpp::List>::type fields(fields_sexp);
-	return Rcpp::wrap(schema_(fields));
-END_RCPP
+std::shared_ptr<arrow::Schema> schema_(const std::vector<std::shared_ptr<arrow::Field>>& fields);
+extern "C" SEXP _arrow_schema_(SEXP fields_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::vector<std::shared_ptr<arrow::Field>>&>::type fields(fields_sexp);
+	return cpp11::as_sexp(schema_(fields));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_schema_(SEXP fields_sexp){
+extern "C" SEXP _arrow_schema_(SEXP fields_sexp){
 	Rf_error("Cannot call schema_(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5445,14 +5810,14 @@ RcppExport SEXP _arrow_schema_(SEXP fields_sexp){
 // schema.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::string Schema__ToString(const std::shared_ptr<arrow::Schema>& s);
-RcppExport SEXP _arrow_Schema__ToString(SEXP s_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type s(s_sexp);
-	return Rcpp::wrap(Schema__ToString(s));
-END_RCPP
+extern "C" SEXP _arrow_Schema__ToString(SEXP s_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type s(s_sexp);
+	return cpp11::as_sexp(Schema__ToString(s));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Schema__ToString(SEXP s_sexp){
+extern "C" SEXP _arrow_Schema__ToString(SEXP s_sexp){
 	Rf_error("Cannot call Schema__ToString(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5460,14 +5825,14 @@ RcppExport SEXP _arrow_Schema__ToString(SEXP s_sexp){
 // schema.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int Schema__num_fields(const std::shared_ptr<arrow::Schema>& s);
-RcppExport SEXP _arrow_Schema__num_fields(SEXP s_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type s(s_sexp);
-	return Rcpp::wrap(Schema__num_fields(s));
-END_RCPP
+extern "C" SEXP _arrow_Schema__num_fields(SEXP s_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type s(s_sexp);
+	return cpp11::as_sexp(Schema__num_fields(s));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Schema__num_fields(SEXP s_sexp){
+extern "C" SEXP _arrow_Schema__num_fields(SEXP s_sexp){
 	Rf_error("Cannot call Schema__num_fields(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5475,15 +5840,15 @@ RcppExport SEXP _arrow_Schema__num_fields(SEXP s_sexp){
 // schema.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Field> Schema__field(const std::shared_ptr<arrow::Schema>& s, int i);
-RcppExport SEXP _arrow_Schema__field(SEXP s_sexp, SEXP i_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type s(s_sexp);
-	Rcpp::traits::input_parameter<int>::type i(i_sexp);
-	return Rcpp::wrap(Schema__field(s, i));
-END_RCPP
+extern "C" SEXP _arrow_Schema__field(SEXP s_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type s(s_sexp);
+	arrow::r::Input<int>::type i(i_sexp);
+	return cpp11::as_sexp(Schema__field(s, i));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Schema__field(SEXP s_sexp, SEXP i_sexp){
+extern "C" SEXP _arrow_Schema__field(SEXP s_sexp, SEXP i_sexp){
 	Rf_error("Cannot call Schema__field(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5491,15 +5856,15 @@ RcppExport SEXP _arrow_Schema__field(SEXP s_sexp, SEXP i_sexp){
 // schema.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Field> Schema__GetFieldByName(const std::shared_ptr<arrow::Schema>& s, std::string x);
-RcppExport SEXP _arrow_Schema__GetFieldByName(SEXP s_sexp, SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type s(s_sexp);
-	Rcpp::traits::input_parameter<std::string>::type x(x_sexp);
-	return Rcpp::wrap(Schema__GetFieldByName(s, x));
-END_RCPP
+extern "C" SEXP _arrow_Schema__GetFieldByName(SEXP s_sexp, SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type s(s_sexp);
+	arrow::r::Input<std::string>::type x(x_sexp);
+	return cpp11::as_sexp(Schema__GetFieldByName(s, x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Schema__GetFieldByName(SEXP s_sexp, SEXP x_sexp){
+extern "C" SEXP _arrow_Schema__GetFieldByName(SEXP s_sexp, SEXP x_sexp){
 	Rf_error("Cannot call Schema__GetFieldByName(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5507,14 +5872,14 @@ RcppExport SEXP _arrow_Schema__GetFieldByName(SEXP s_sexp, SEXP x_sexp){
 // schema.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::vector<std::shared_ptr<arrow::Field>> Schema__fields(const std::shared_ptr<arrow::Schema>& schema);
-RcppExport SEXP _arrow_Schema__fields(SEXP schema_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
-	return Rcpp::wrap(Schema__fields(schema));
-END_RCPP
+extern "C" SEXP _arrow_Schema__fields(SEXP schema_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
+	return cpp11::as_sexp(Schema__fields(schema));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Schema__fields(SEXP schema_sexp){
+extern "C" SEXP _arrow_Schema__fields(SEXP schema_sexp){
 	Rf_error("Cannot call Schema__fields(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5522,14 +5887,14 @@ RcppExport SEXP _arrow_Schema__fields(SEXP schema_sexp){
 // schema.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::vector<std::string> Schema__field_names(const std::shared_ptr<arrow::Schema>& schema);
-RcppExport SEXP _arrow_Schema__field_names(SEXP schema_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
-	return Rcpp::wrap(Schema__field_names(schema));
-END_RCPP
+extern "C" SEXP _arrow_Schema__field_names(SEXP schema_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
+	return cpp11::as_sexp(Schema__field_names(schema));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Schema__field_names(SEXP schema_sexp){
+extern "C" SEXP _arrow_Schema__field_names(SEXP schema_sexp){
 	Rf_error("Cannot call Schema__field_names(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5537,60 +5902,60 @@ RcppExport SEXP _arrow_Schema__field_names(SEXP schema_sexp){
 // schema.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool Schema__HasMetadata(const std::shared_ptr<arrow::Schema>& schema);
-RcppExport SEXP _arrow_Schema__HasMetadata(SEXP schema_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
-	return Rcpp::wrap(Schema__HasMetadata(schema));
-END_RCPP
+extern "C" SEXP _arrow_Schema__HasMetadata(SEXP schema_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
+	return cpp11::as_sexp(Schema__HasMetadata(schema));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Schema__HasMetadata(SEXP schema_sexp){
+extern "C" SEXP _arrow_Schema__HasMetadata(SEXP schema_sexp){
 	Rf_error("Cannot call Schema__HasMetadata(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // schema.cpp
 #if defined(ARROW_R_WITH_ARROW)
-Rcpp::List Schema__metadata(const std::shared_ptr<arrow::Schema>& schema);
-RcppExport SEXP _arrow_Schema__metadata(SEXP schema_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
-	return Rcpp::wrap(Schema__metadata(schema));
-END_RCPP
+cpp11::writable::list Schema__metadata(const std::shared_ptr<arrow::Schema>& schema);
+extern "C" SEXP _arrow_Schema__metadata(SEXP schema_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
+	return cpp11::as_sexp(Schema__metadata(schema));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Schema__metadata(SEXP schema_sexp){
+extern "C" SEXP _arrow_Schema__metadata(SEXP schema_sexp){
 	Rf_error("Cannot call Schema__metadata(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // schema.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::Schema> Schema__WithMetadata(const std::shared_ptr<arrow::Schema>& schema, Rcpp::CharacterVector metadata);
-RcppExport SEXP _arrow_Schema__WithMetadata(SEXP schema_sexp, SEXP metadata_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
-	Rcpp::traits::input_parameter<Rcpp::CharacterVector>::type metadata(metadata_sexp);
-	return Rcpp::wrap(Schema__WithMetadata(schema, metadata));
-END_RCPP
+std::shared_ptr<arrow::Schema> Schema__WithMetadata(const std::shared_ptr<arrow::Schema>& schema, cpp11::strings metadata);
+extern "C" SEXP _arrow_Schema__WithMetadata(SEXP schema_sexp, SEXP metadata_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
+	arrow::r::Input<cpp11::strings>::type metadata(metadata_sexp);
+	return cpp11::as_sexp(Schema__WithMetadata(schema, metadata));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Schema__WithMetadata(SEXP schema_sexp, SEXP metadata_sexp){
+extern "C" SEXP _arrow_Schema__WithMetadata(SEXP schema_sexp, SEXP metadata_sexp){
 	Rf_error("Cannot call Schema__WithMetadata(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // schema.cpp
 #if defined(ARROW_R_WITH_ARROW)
-Rcpp::RawVector Schema__serialize(const std::shared_ptr<arrow::Schema>& schema);
-RcppExport SEXP _arrow_Schema__serialize(SEXP schema_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
-	return Rcpp::wrap(Schema__serialize(schema));
-END_RCPP
+cpp11::writable::raws Schema__serialize(const std::shared_ptr<arrow::Schema>& schema);
+extern "C" SEXP _arrow_Schema__serialize(SEXP schema_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
+	return cpp11::as_sexp(Schema__serialize(schema));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Schema__serialize(SEXP schema_sexp){
+extern "C" SEXP _arrow_Schema__serialize(SEXP schema_sexp){
 	Rf_error("Cannot call Schema__serialize(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5598,16 +5963,16 @@ RcppExport SEXP _arrow_Schema__serialize(SEXP schema_sexp){
 // schema.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool Schema__Equals(const std::shared_ptr<arrow::Schema>& schema, const std::shared_ptr<arrow::Schema>& other, bool check_metadata);
-RcppExport SEXP _arrow_Schema__Equals(SEXP schema_sexp, SEXP other_sexp, SEXP check_metadata_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Schema>&>::type other(other_sexp);
-	Rcpp::traits::input_parameter<bool>::type check_metadata(check_metadata_sexp);
-	return Rcpp::wrap(Schema__Equals(schema, other, check_metadata));
-END_RCPP
+extern "C" SEXP _arrow_Schema__Equals(SEXP schema_sexp, SEXP other_sexp, SEXP check_metadata_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Schema>&>::type other(other_sexp);
+	arrow::r::Input<bool>::type check_metadata(check_metadata_sexp);
+	return cpp11::as_sexp(Schema__Equals(schema, other, check_metadata));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Schema__Equals(SEXP schema_sexp, SEXP other_sexp, SEXP check_metadata_sexp){
+extern "C" SEXP _arrow_Schema__Equals(SEXP schema_sexp, SEXP other_sexp, SEXP check_metadata_sexp){
 	Rf_error("Cannot call Schema__Equals(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5615,44 +5980,29 @@ RcppExport SEXP _arrow_Schema__Equals(SEXP schema_sexp, SEXP other_sexp, SEXP ch
 // schema.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Schema> arrow__UnifySchemas(const std::vector<std::shared_ptr<arrow::Schema>>& schemas);
-RcppExport SEXP _arrow_arrow__UnifySchemas(SEXP schemas_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::vector<std::shared_ptr<arrow::Schema>>&>::type schemas(schemas_sexp);
-	return Rcpp::wrap(arrow__UnifySchemas(schemas));
-END_RCPP
+extern "C" SEXP _arrow_arrow__UnifySchemas(SEXP schemas_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::vector<std::shared_ptr<arrow::Schema>>&>::type schemas(schemas_sexp);
+	return cpp11::as_sexp(arrow__UnifySchemas(schemas));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_arrow__UnifySchemas(SEXP schemas_sexp){
+extern "C" SEXP _arrow_arrow__UnifySchemas(SEXP schemas_sexp){
 	Rf_error("Cannot call arrow__UnifySchemas(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
-// table.cpp
-#if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::Table> Table__from_dataframe(DataFrame tbl);
-RcppExport SEXP _arrow_Table__from_dataframe(SEXP tbl_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<DataFrame>::type tbl(tbl_sexp);
-	return Rcpp::wrap(Table__from_dataframe(tbl));
-END_RCPP
-}
-#else
-RcppExport SEXP _arrow_Table__from_dataframe(SEXP tbl_sexp){
-	Rf_error("Cannot call Table__from_dataframe(). Please use arrow::install_arrow() to install required runtime libraries. ");
-}
-#endif
-
 // table.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int Table__num_columns(const std::shared_ptr<arrow::Table>& x);
-RcppExport SEXP _arrow_Table__num_columns(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type x(x_sexp);
-	return Rcpp::wrap(Table__num_columns(x));
-END_RCPP
+extern "C" SEXP _arrow_Table__num_columns(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type x(x_sexp);
+	return cpp11::as_sexp(Table__num_columns(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Table__num_columns(SEXP x_sexp){
+extern "C" SEXP _arrow_Table__num_columns(SEXP x_sexp){
 	Rf_error("Cannot call Table__num_columns(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5660,14 +6010,14 @@ RcppExport SEXP _arrow_Table__num_columns(SEXP x_sexp){
 // table.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int Table__num_rows(const std::shared_ptr<arrow::Table>& x);
-RcppExport SEXP _arrow_Table__num_rows(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type x(x_sexp);
-	return Rcpp::wrap(Table__num_rows(x));
-END_RCPP
+extern "C" SEXP _arrow_Table__num_rows(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type x(x_sexp);
+	return cpp11::as_sexp(Table__num_rows(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Table__num_rows(SEXP x_sexp){
+extern "C" SEXP _arrow_Table__num_rows(SEXP x_sexp){
 	Rf_error("Cannot call Table__num_rows(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5675,62 +6025,62 @@ RcppExport SEXP _arrow_Table__num_rows(SEXP x_sexp){
 // table.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Schema> Table__schema(const std::shared_ptr<arrow::Table>& x);
-RcppExport SEXP _arrow_Table__schema(SEXP x_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type x(x_sexp);
-	return Rcpp::wrap(Table__schema(x));
-END_RCPP
+extern "C" SEXP _arrow_Table__schema(SEXP x_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type x(x_sexp);
+	return cpp11::as_sexp(Table__schema(x));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Table__schema(SEXP x_sexp){
+extern "C" SEXP _arrow_Table__schema(SEXP x_sexp){
 	Rf_error("Cannot call Table__schema(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // table.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::Table> Table__ReplaceSchemaMetadata(const std::shared_ptr<arrow::Table>& x, Rcpp::CharacterVector metadata);
-RcppExport SEXP _arrow_Table__ReplaceSchemaMetadata(SEXP x_sexp, SEXP metadata_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type x(x_sexp);
-	Rcpp::traits::input_parameter<Rcpp::CharacterVector>::type metadata(metadata_sexp);
-	return Rcpp::wrap(Table__ReplaceSchemaMetadata(x, metadata));
-END_RCPP
+std::shared_ptr<arrow::Table> Table__ReplaceSchemaMetadata(const std::shared_ptr<arrow::Table>& x, cpp11::strings metadata);
+extern "C" SEXP _arrow_Table__ReplaceSchemaMetadata(SEXP x_sexp, SEXP metadata_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type x(x_sexp);
+	arrow::r::Input<cpp11::strings>::type metadata(metadata_sexp);
+	return cpp11::as_sexp(Table__ReplaceSchemaMetadata(x, metadata));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Table__ReplaceSchemaMetadata(SEXP x_sexp, SEXP metadata_sexp){
+extern "C" SEXP _arrow_Table__ReplaceSchemaMetadata(SEXP x_sexp, SEXP metadata_sexp){
 	Rf_error("Cannot call Table__ReplaceSchemaMetadata(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // table.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::ChunkedArray> Table__column(const std::shared_ptr<arrow::Table>& table, int i);
-RcppExport SEXP _arrow_Table__column(SEXP table_sexp, SEXP i_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
-	Rcpp::traits::input_parameter<int>::type i(i_sexp);
-	return Rcpp::wrap(Table__column(table, i));
-END_RCPP
+std::shared_ptr<arrow::ChunkedArray> Table__column(const std::shared_ptr<arrow::Table>& table, R_xlen_t i);
+extern "C" SEXP _arrow_Table__column(SEXP table_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
+	arrow::r::Input<R_xlen_t>::type i(i_sexp);
+	return cpp11::as_sexp(Table__column(table, i));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Table__column(SEXP table_sexp, SEXP i_sexp){
+extern "C" SEXP _arrow_Table__column(SEXP table_sexp, SEXP i_sexp){
 	Rf_error("Cannot call Table__column(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // table.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::Field> Table__field(const std::shared_ptr<arrow::Table>& table, int i);
-RcppExport SEXP _arrow_Table__field(SEXP table_sexp, SEXP i_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
-	Rcpp::traits::input_parameter<int>::type i(i_sexp);
-	return Rcpp::wrap(Table__field(table, i));
-END_RCPP
+std::shared_ptr<arrow::Field> Table__field(const std::shared_ptr<arrow::Table>& table, R_xlen_t i);
+extern "C" SEXP _arrow_Table__field(SEXP table_sexp, SEXP i_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
+	arrow::r::Input<R_xlen_t>::type i(i_sexp);
+	return cpp11::as_sexp(Table__field(table, i));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Table__field(SEXP table_sexp, SEXP i_sexp){
+extern "C" SEXP _arrow_Table__field(SEXP table_sexp, SEXP i_sexp){
 	Rf_error("Cannot call Table__field(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5738,14 +6088,14 @@ RcppExport SEXP _arrow_Table__field(SEXP table_sexp, SEXP i_sexp){
 // table.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::vector<std::shared_ptr<arrow::ChunkedArray>> Table__columns(const std::shared_ptr<arrow::Table>& table);
-RcppExport SEXP _arrow_Table__columns(SEXP table_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
-	return Rcpp::wrap(Table__columns(table));
-END_RCPP
+extern "C" SEXP _arrow_Table__columns(SEXP table_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
+	return cpp11::as_sexp(Table__columns(table));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Table__columns(SEXP table_sexp){
+extern "C" SEXP _arrow_Table__columns(SEXP table_sexp){
 	Rf_error("Cannot call Table__columns(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5753,47 +6103,47 @@ RcppExport SEXP _arrow_Table__columns(SEXP table_sexp){
 // table.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::vector<std::string> Table__ColumnNames(const std::shared_ptr<arrow::Table>& table);
-RcppExport SEXP _arrow_Table__ColumnNames(SEXP table_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
-	return Rcpp::wrap(Table__ColumnNames(table));
-END_RCPP
+extern "C" SEXP _arrow_Table__ColumnNames(SEXP table_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
+	return cpp11::as_sexp(Table__ColumnNames(table));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Table__ColumnNames(SEXP table_sexp){
+extern "C" SEXP _arrow_Table__ColumnNames(SEXP table_sexp){
 	Rf_error("Cannot call Table__ColumnNames(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // table.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::Table> Table__Slice1(const std::shared_ptr<arrow::Table>& table, int offset);
-RcppExport SEXP _arrow_Table__Slice1(SEXP table_sexp, SEXP offset_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
-	Rcpp::traits::input_parameter<int>::type offset(offset_sexp);
-	return Rcpp::wrap(Table__Slice1(table, offset));
-END_RCPP
+std::shared_ptr<arrow::Table> Table__Slice1(const std::shared_ptr<arrow::Table>& table, R_xlen_t offset);
+extern "C" SEXP _arrow_Table__Slice1(SEXP table_sexp, SEXP offset_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
+	arrow::r::Input<R_xlen_t>::type offset(offset_sexp);
+	return cpp11::as_sexp(Table__Slice1(table, offset));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Table__Slice1(SEXP table_sexp, SEXP offset_sexp){
+extern "C" SEXP _arrow_Table__Slice1(SEXP table_sexp, SEXP offset_sexp){
 	Rf_error("Cannot call Table__Slice1(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // table.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::Table> Table__Slice2(const std::shared_ptr<arrow::Table>& table, int offset, int length);
-RcppExport SEXP _arrow_Table__Slice2(SEXP table_sexp, SEXP offset_sexp, SEXP length_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
-	Rcpp::traits::input_parameter<int>::type offset(offset_sexp);
-	Rcpp::traits::input_parameter<int>::type length(length_sexp);
-	return Rcpp::wrap(Table__Slice2(table, offset, length));
-END_RCPP
+std::shared_ptr<arrow::Table> Table__Slice2(const std::shared_ptr<arrow::Table>& table, R_xlen_t offset, R_xlen_t length);
+extern "C" SEXP _arrow_Table__Slice2(SEXP table_sexp, SEXP offset_sexp, SEXP length_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
+	arrow::r::Input<R_xlen_t>::type offset(offset_sexp);
+	arrow::r::Input<R_xlen_t>::type length(length_sexp);
+	return cpp11::as_sexp(Table__Slice2(table, offset, length));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Table__Slice2(SEXP table_sexp, SEXP offset_sexp, SEXP length_sexp){
+extern "C" SEXP _arrow_Table__Slice2(SEXP table_sexp, SEXP offset_sexp, SEXP length_sexp){
 	Rf_error("Cannot call Table__Slice2(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5801,16 +6151,16 @@ RcppExport SEXP _arrow_Table__Slice2(SEXP table_sexp, SEXP offset_sexp, SEXP len
 // table.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool Table__Equals(const std::shared_ptr<arrow::Table>& lhs, const std::shared_ptr<arrow::Table>& rhs, bool check_metadata);
-RcppExport SEXP _arrow_Table__Equals(SEXP lhs_sexp, SEXP rhs_sexp, SEXP check_metadata_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type lhs(lhs_sexp);
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type rhs(rhs_sexp);
-	Rcpp::traits::input_parameter<bool>::type check_metadata(check_metadata_sexp);
-	return Rcpp::wrap(Table__Equals(lhs, rhs, check_metadata));
-END_RCPP
+extern "C" SEXP _arrow_Table__Equals(SEXP lhs_sexp, SEXP rhs_sexp, SEXP check_metadata_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type lhs(lhs_sexp);
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type rhs(rhs_sexp);
+	arrow::r::Input<bool>::type check_metadata(check_metadata_sexp);
+	return cpp11::as_sexp(Table__Equals(lhs, rhs, check_metadata));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Table__Equals(SEXP lhs_sexp, SEXP rhs_sexp, SEXP check_metadata_sexp){
+extern "C" SEXP _arrow_Table__Equals(SEXP lhs_sexp, SEXP rhs_sexp, SEXP check_metadata_sexp){
 	Rf_error("Cannot call Table__Equals(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5818,14 +6168,14 @@ RcppExport SEXP _arrow_Table__Equals(SEXP lhs_sexp, SEXP rhs_sexp, SEXP check_me
 // table.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool Table__Validate(const std::shared_ptr<arrow::Table>& table);
-RcppExport SEXP _arrow_Table__Validate(SEXP table_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
-	return Rcpp::wrap(Table__Validate(table));
-END_RCPP
+extern "C" SEXP _arrow_Table__Validate(SEXP table_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
+	return cpp11::as_sexp(Table__Validate(table));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Table__Validate(SEXP table_sexp){
+extern "C" SEXP _arrow_Table__Validate(SEXP table_sexp){
 	Rf_error("Cannot call Table__Validate(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5833,14 +6183,14 @@ RcppExport SEXP _arrow_Table__Validate(SEXP table_sexp){
 // table.cpp
 #if defined(ARROW_R_WITH_ARROW)
 bool Table__ValidateFull(const std::shared_ptr<arrow::Table>& table);
-RcppExport SEXP _arrow_Table__ValidateFull(SEXP table_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
-	return Rcpp::wrap(Table__ValidateFull(table));
-END_RCPP
+extern "C" SEXP _arrow_Table__ValidateFull(SEXP table_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
+	return cpp11::as_sexp(Table__ValidateFull(table));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Table__ValidateFull(SEXP table_sexp){
+extern "C" SEXP _arrow_Table__ValidateFull(SEXP table_sexp){
 	Rf_error("Cannot call Table__ValidateFull(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5848,47 +6198,78 @@ RcppExport SEXP _arrow_Table__ValidateFull(SEXP table_sexp){
 // table.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::ChunkedArray> Table__GetColumnByName(const std::shared_ptr<arrow::Table>& table, const std::string& name);
-RcppExport SEXP _arrow_Table__GetColumnByName(SEXP table_sexp, SEXP name_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
-	Rcpp::traits::input_parameter<const std::string&>::type name(name_sexp);
-	return Rcpp::wrap(Table__GetColumnByName(table, name));
-END_RCPP
+extern "C" SEXP _arrow_Table__GetColumnByName(SEXP table_sexp, SEXP name_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
+	arrow::r::Input<const std::string&>::type name(name_sexp);
+	return cpp11::as_sexp(Table__GetColumnByName(table, name));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Table__GetColumnByName(SEXP table_sexp, SEXP name_sexp){
+extern "C" SEXP _arrow_Table__GetColumnByName(SEXP table_sexp, SEXP name_sexp){
 	Rf_error("Cannot call Table__GetColumnByName(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // table.cpp
 #if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<arrow::Table> Table__select(const std::shared_ptr<arrow::Table>& table, const Rcpp::IntegerVector& indices);
-RcppExport SEXP _arrow_Table__select(SEXP table_sexp, SEXP indices_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
-	Rcpp::traits::input_parameter<const Rcpp::IntegerVector&>::type indices(indices_sexp);
-	return Rcpp::wrap(Table__select(table, indices));
-END_RCPP
+std::shared_ptr<arrow::Table> Table__SelectColumns(const std::shared_ptr<arrow::Table>& table, const std::vector<int>& indices);
+extern "C" SEXP _arrow_Table__SelectColumns(SEXP table_sexp, SEXP indices_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::shared_ptr<arrow::Table>&>::type table(table_sexp);
+	arrow::r::Input<const std::vector<int>&>::type indices(indices_sexp);
+	return cpp11::as_sexp(Table__SelectColumns(table, indices));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_Table__SelectColumns(SEXP table_sexp, SEXP indices_sexp){
+	Rf_error("Cannot call Table__SelectColumns(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// table.cpp
+#if defined(ARROW_R_WITH_ARROW)
+bool all_record_batches(SEXP lst);
+extern "C" SEXP _arrow_all_record_batches(SEXP lst_sexp){
+BEGIN_CPP11
+	arrow::r::Input<SEXP>::type lst(lst_sexp);
+	return cpp11::as_sexp(all_record_batches(lst));
+END_CPP11
+}
+#else
+extern "C" SEXP _arrow_all_record_batches(SEXP lst_sexp){
+	Rf_error("Cannot call all_record_batches(). Please use arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// table.cpp
+#if defined(ARROW_R_WITH_ARROW)
+std::shared_ptr<arrow::Table> Table__from_record_batches(const std::vector<std::shared_ptr<arrow::RecordBatch>>& batches, SEXP schema_sxp);
+extern "C" SEXP _arrow_Table__from_record_batches(SEXP batches_sexp, SEXP schema_sxp_sexp){
+BEGIN_CPP11
+	arrow::r::Input<const std::vector<std::shared_ptr<arrow::RecordBatch>>&>::type batches(batches_sexp);
+	arrow::r::Input<SEXP>::type schema_sxp(schema_sxp_sexp);
+	return cpp11::as_sexp(Table__from_record_batches(batches, schema_sxp));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Table__select(SEXP table_sexp, SEXP indices_sexp){
-	Rf_error("Cannot call Table__select(). Please use arrow::install_arrow() to install required runtime libraries. ");
+extern "C" SEXP _arrow_Table__from_record_batches(SEXP batches_sexp, SEXP schema_sxp_sexp){
+	Rf_error("Cannot call Table__from_record_batches(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
 
 // table.cpp
 #if defined(ARROW_R_WITH_ARROW)
 std::shared_ptr<arrow::Table> Table__from_dots(SEXP lst, SEXP schema_sxp);
-RcppExport SEXP _arrow_Table__from_dots(SEXP lst_sexp, SEXP schema_sxp_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<SEXP>::type lst(lst_sexp);
-	Rcpp::traits::input_parameter<SEXP>::type schema_sxp(schema_sxp_sexp);
-	return Rcpp::wrap(Table__from_dots(lst, schema_sxp));
-END_RCPP
+extern "C" SEXP _arrow_Table__from_dots(SEXP lst_sexp, SEXP schema_sxp_sexp){
+BEGIN_CPP11
+	arrow::r::Input<SEXP>::type lst(lst_sexp);
+	arrow::r::Input<SEXP>::type schema_sxp(schema_sxp_sexp);
+	return cpp11::as_sexp(Table__from_dots(lst, schema_sxp));
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_Table__from_dots(SEXP lst_sexp, SEXP schema_sxp_sexp){
+extern "C" SEXP _arrow_Table__from_dots(SEXP lst_sexp, SEXP schema_sxp_sexp){
 	Rf_error("Cannot call Table__from_dots(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5896,13 +6277,13 @@ RcppExport SEXP _arrow_Table__from_dots(SEXP lst_sexp, SEXP schema_sxp_sexp){
 // threadpool.cpp
 #if defined(ARROW_R_WITH_ARROW)
 int GetCpuThreadPoolCapacity();
-RcppExport SEXP _arrow_GetCpuThreadPoolCapacity(){
-BEGIN_RCPP
-	return Rcpp::wrap(GetCpuThreadPoolCapacity());
-END_RCPP
+extern "C" SEXP _arrow_GetCpuThreadPoolCapacity(){
+BEGIN_CPP11
+	return cpp11::as_sexp(GetCpuThreadPoolCapacity());
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_GetCpuThreadPoolCapacity(){
+extern "C" SEXP _arrow_GetCpuThreadPoolCapacity(){
 	Rf_error("Cannot call GetCpuThreadPoolCapacity(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -5910,15 +6291,15 @@ RcppExport SEXP _arrow_GetCpuThreadPoolCapacity(){
 // threadpool.cpp
 #if defined(ARROW_R_WITH_ARROW)
 void SetCpuThreadPoolCapacity(int threads);
-RcppExport SEXP _arrow_SetCpuThreadPoolCapacity(SEXP threads_sexp){
-BEGIN_RCPP
-	Rcpp::traits::input_parameter<int>::type threads(threads_sexp);
+extern "C" SEXP _arrow_SetCpuThreadPoolCapacity(SEXP threads_sexp){
+BEGIN_CPP11
+	arrow::r::Input<int>::type threads(threads_sexp);
 	SetCpuThreadPoolCapacity(threads);
 	return R_NilValue;
-END_RCPP
+END_CPP11
 }
 #else
-RcppExport SEXP _arrow_SetCpuThreadPoolCapacity(SEXP threads_sexp){
+extern "C" SEXP _arrow_SetCpuThreadPoolCapacity(SEXP threads_sexp){
 	Rf_error("Cannot call SetCpuThreadPoolCapacity(). Please use arrow::install_arrow() to install required runtime libraries. ");
 }
 #endif
@@ -6025,16 +6406,23 @@ static const R_CallMethodDef CallEntries[] = {
 		{ "_arrow_compute__CallFunction", (DL_FUNC) &_arrow_compute__CallFunction, 3}, 
 		{ "_arrow_csv___ReadOptions__initialize", (DL_FUNC) &_arrow_csv___ReadOptions__initialize, 1}, 
 		{ "_arrow_csv___ParseOptions__initialize", (DL_FUNC) &_arrow_csv___ParseOptions__initialize, 1}, 
+		{ "_arrow_csv___ReadOptions__column_names", (DL_FUNC) &_arrow_csv___ReadOptions__column_names, 1}, 
 		{ "_arrow_csv___ConvertOptions__initialize", (DL_FUNC) &_arrow_csv___ConvertOptions__initialize, 1}, 
 		{ "_arrow_csv___TableReader__Make", (DL_FUNC) &_arrow_csv___TableReader__Make, 4}, 
 		{ "_arrow_csv___TableReader__Read", (DL_FUNC) &_arrow_csv___TableReader__Read, 1}, 
+		{ "_arrow_TimestampParser__kind", (DL_FUNC) &_arrow_TimestampParser__kind, 1}, 
+		{ "_arrow_TimestampParser__format", (DL_FUNC) &_arrow_TimestampParser__format, 1}, 
+		{ "_arrow_TimestampParser__MakeStrptime", (DL_FUNC) &_arrow_TimestampParser__MakeStrptime, 1}, 
+		{ "_arrow_TimestampParser__MakeISO8601", (DL_FUNC) &_arrow_TimestampParser__MakeISO8601, 0}, 
 		{ "_arrow_dataset___Dataset__NewScan", (DL_FUNC) &_arrow_dataset___Dataset__NewScan, 1}, 
 		{ "_arrow_dataset___Dataset__schema", (DL_FUNC) &_arrow_dataset___Dataset__schema, 1}, 
 		{ "_arrow_dataset___Dataset__type_name", (DL_FUNC) &_arrow_dataset___Dataset__type_name, 1}, 
 		{ "_arrow_dataset___Dataset__ReplaceSchema", (DL_FUNC) &_arrow_dataset___Dataset__ReplaceSchema, 2}, 
 		{ "_arrow_dataset___UnionDataset__create", (DL_FUNC) &_arrow_dataset___UnionDataset__create, 2}, 
+		{ "_arrow_dataset___InMemoryDataset__create", (DL_FUNC) &_arrow_dataset___InMemoryDataset__create, 1}, 
 		{ "_arrow_dataset___UnionDataset__children", (DL_FUNC) &_arrow_dataset___UnionDataset__children, 1}, 
 		{ "_arrow_dataset___FileSystemDataset__format", (DL_FUNC) &_arrow_dataset___FileSystemDataset__format, 1}, 
+		{ "_arrow_dataset___FileSystemDataset__filesystem", (DL_FUNC) &_arrow_dataset___FileSystemDataset__filesystem, 1}, 
 		{ "_arrow_dataset___FileSystemDataset__files", (DL_FUNC) &_arrow_dataset___FileSystemDataset__files, 1}, 
 		{ "_arrow_dataset___DatasetFactory__Finish1", (DL_FUNC) &_arrow_dataset___DatasetFactory__Finish1, 2}, 
 		{ "_arrow_dataset___DatasetFactory__Finish2", (DL_FUNC) &_arrow_dataset___DatasetFactory__Finish2, 2}, 
@@ -6044,7 +6432,12 @@ static const R_CallMethodDef CallEntries[] = {
 		{ "_arrow_dataset___FileSystemDatasetFactory__Make1", (DL_FUNC) &_arrow_dataset___FileSystemDatasetFactory__Make1, 3}, 
 		{ "_arrow_dataset___FileSystemDatasetFactory__Make3", (DL_FUNC) &_arrow_dataset___FileSystemDatasetFactory__Make3, 4}, 
 		{ "_arrow_dataset___FileFormat__type_name", (DL_FUNC) &_arrow_dataset___FileFormat__type_name, 1}, 
+		{ "_arrow_dataset___FileFormat__DefaultWriteOptions", (DL_FUNC) &_arrow_dataset___FileFormat__DefaultWriteOptions, 1}, 
 		{ "_arrow_dataset___ParquetFileFormat__Make", (DL_FUNC) &_arrow_dataset___ParquetFileFormat__Make, 3}, 
+		{ "_arrow_dataset___FileWriteOptions__type_name", (DL_FUNC) &_arrow_dataset___FileWriteOptions__type_name, 1}, 
+		{ "_arrow_dataset___ParquetFileWriteOptions__update", (DL_FUNC) &_arrow_dataset___ParquetFileWriteOptions__update, 3}, 
+		{ "_arrow_dataset___IpcFileWriteOptions__update2", (DL_FUNC) &_arrow_dataset___IpcFileWriteOptions__update2, 4}, 
+		{ "_arrow_dataset___IpcFileWriteOptions__update1", (DL_FUNC) &_arrow_dataset___IpcFileWriteOptions__update1, 3}, 
 		{ "_arrow_dataset___IpcFileFormat__Make", (DL_FUNC) &_arrow_dataset___IpcFileFormat__Make, 0}, 
 		{ "_arrow_dataset___CsvFileFormat__Make", (DL_FUNC) &_arrow_dataset___CsvFileFormat__Make, 1}, 
 		{ "_arrow_dataset___DirectoryPartitioning", (DL_FUNC) &_arrow_dataset___DirectoryPartitioning, 1}, 
@@ -6060,6 +6453,7 @@ static const R_CallMethodDef CallEntries[] = {
 		{ "_arrow_dataset___Scanner__ToTable", (DL_FUNC) &_arrow_dataset___Scanner__ToTable, 1}, 
 		{ "_arrow_dataset___Scanner__head", (DL_FUNC) &_arrow_dataset___Scanner__head, 2}, 
 		{ "_arrow_dataset___Scanner__Scan", (DL_FUNC) &_arrow_dataset___Scanner__Scan, 1}, 
+		{ "_arrow_dataset___Scanner__schema", (DL_FUNC) &_arrow_dataset___Scanner__schema, 1}, 
 		{ "_arrow_dataset___ScanTask__get_batches", (DL_FUNC) &_arrow_dataset___ScanTask__get_batches, 1}, 
 		{ "_arrow_dataset___Dataset__Write", (DL_FUNC) &_arrow_dataset___Dataset__Write, 6}, 
 		{ "_arrow_shared_ptr_is_null", (DL_FUNC) &_arrow_shared_ptr_is_null, 1}, 
@@ -6091,7 +6485,7 @@ static const R_CallMethodDef CallEntries[] = {
 		{ "_arrow_list__", (DL_FUNC) &_arrow_list__, 1}, 
 		{ "_arrow_large_list__", (DL_FUNC) &_arrow_large_list__, 1}, 
 		{ "_arrow_fixed_size_list__", (DL_FUNC) &_arrow_fixed_size_list__, 2}, 
-		{ "_arrow_struct_", (DL_FUNC) &_arrow_struct_, 1}, 
+		{ "_arrow_struct__", (DL_FUNC) &_arrow_struct__, 1}, 
 		{ "_arrow_DataType__ToString", (DL_FUNC) &_arrow_DataType__ToString, 1}, 
 		{ "_arrow_DataType__name", (DL_FUNC) &_arrow_DataType__name, 1}, 
 		{ "_arrow_DataType__Equals", (DL_FUNC) &_arrow_DataType__Equals, 2}, 
@@ -6175,9 +6569,12 @@ static const R_CallMethodDef CallEntries[] = {
 		{ "_arrow_fs___FileSystem__type_name", (DL_FUNC) &_arrow_fs___FileSystem__type_name, 1}, 
 		{ "_arrow_fs___LocalFileSystem__create", (DL_FUNC) &_arrow_fs___LocalFileSystem__create, 0}, 
 		{ "_arrow_fs___SubTreeFileSystem__create", (DL_FUNC) &_arrow_fs___SubTreeFileSystem__create, 2}, 
+		{ "_arrow_fs___SubTreeFileSystem__base_fs", (DL_FUNC) &_arrow_fs___SubTreeFileSystem__base_fs, 1}, 
+		{ "_arrow_fs___SubTreeFileSystem__base_path", (DL_FUNC) &_arrow_fs___SubTreeFileSystem__base_path, 1}, 
 		{ "_arrow_fs___FileSystemFromUri", (DL_FUNC) &_arrow_fs___FileSystemFromUri, 1}, 
-		{ "_arrow_fs___EnsureS3Initialized", (DL_FUNC) &_arrow_fs___EnsureS3Initialized, 0}, 
-		{ "_arrow_fs___S3FileSystem__create", (DL_FUNC) &_arrow_fs___S3FileSystem__create, 0}, 
+		{ "_arrow_fs___CopyFiles", (DL_FUNC) &_arrow_fs___CopyFiles, 6}, 
+		{ "_arrow_fs___S3FileSystem__create", (DL_FUNC) &_arrow_fs___S3FileSystem__create, 12}, 
+		{ "_arrow_fs___S3FileSystem__region", (DL_FUNC) &_arrow_fs___S3FileSystem__region, 1}, 
 		{ "_arrow_io___Readable__Read", (DL_FUNC) &_arrow_io___Readable__Read, 2}, 
 		{ "_arrow_io___InputStream__Close", (DL_FUNC) &_arrow_io___InputStream__Close, 1}, 
 		{ "_arrow_io___OutputStream__Close", (DL_FUNC) &_arrow_io___OutputStream__Close, 1}, 
@@ -6200,7 +6597,7 @@ static const R_CallMethodDef CallEntries[] = {
 		{ "_arrow_io___BufferOutputStream__Finish", (DL_FUNC) &_arrow_io___BufferOutputStream__Finish, 1}, 
 		{ "_arrow_io___BufferOutputStream__Tell", (DL_FUNC) &_arrow_io___BufferOutputStream__Tell, 1}, 
 		{ "_arrow_io___BufferOutputStream__Write", (DL_FUNC) &_arrow_io___BufferOutputStream__Write, 2}, 
-		{ "_arrow_json___ReadOptions__initialize", (DL_FUNC) &_arrow_json___ReadOptions__initialize, 1}, 
+		{ "_arrow_json___ReadOptions__initialize", (DL_FUNC) &_arrow_json___ReadOptions__initialize, 2}, 
 		{ "_arrow_json___ParseOptions__initialize", (DL_FUNC) &_arrow_json___ParseOptions__initialize, 1}, 
 		{ "_arrow_json___TableReader__Make", (DL_FUNC) &_arrow_json___TableReader__Make, 3}, 
 		{ "_arrow_json___TableReader__Read", (DL_FUNC) &_arrow_json___TableReader__Read, 1}, 
@@ -6227,7 +6624,14 @@ static const R_CallMethodDef CallEntries[] = {
 		{ "_arrow_parquet___arrow___FileReader__OpenFile", (DL_FUNC) &_arrow_parquet___arrow___FileReader__OpenFile, 2}, 
 		{ "_arrow_parquet___arrow___FileReader__ReadTable1", (DL_FUNC) &_arrow_parquet___arrow___FileReader__ReadTable1, 1}, 
 		{ "_arrow_parquet___arrow___FileReader__ReadTable2", (DL_FUNC) &_arrow_parquet___arrow___FileReader__ReadTable2, 2}, 
+		{ "_arrow_parquet___arrow___FileReader__ReadRowGroup1", (DL_FUNC) &_arrow_parquet___arrow___FileReader__ReadRowGroup1, 2}, 
+		{ "_arrow_parquet___arrow___FileReader__ReadRowGroup2", (DL_FUNC) &_arrow_parquet___arrow___FileReader__ReadRowGroup2, 3}, 
+		{ "_arrow_parquet___arrow___FileReader__ReadRowGroups1", (DL_FUNC) &_arrow_parquet___arrow___FileReader__ReadRowGroups1, 2}, 
+		{ "_arrow_parquet___arrow___FileReader__ReadRowGroups2", (DL_FUNC) &_arrow_parquet___arrow___FileReader__ReadRowGroups2, 3}, 
 		{ "_arrow_parquet___arrow___FileReader__num_rows", (DL_FUNC) &_arrow_parquet___arrow___FileReader__num_rows, 1}, 
+		{ "_arrow_parquet___arrow___FileReader__num_columns", (DL_FUNC) &_arrow_parquet___arrow___FileReader__num_columns, 1}, 
+		{ "_arrow_parquet___arrow___FileReader__num_row_groups", (DL_FUNC) &_arrow_parquet___arrow___FileReader__num_row_groups, 1}, 
+		{ "_arrow_parquet___arrow___FileReader__ReadColumn", (DL_FUNC) &_arrow_parquet___arrow___FileReader__ReadColumn, 2}, 
 		{ "_arrow_parquet___ArrowWriterProperties___create", (DL_FUNC) &_arrow_parquet___ArrowWriterProperties___create, 3}, 
 		{ "_arrow_parquet___WriterProperties___Builder__create", (DL_FUNC) &_arrow_parquet___WriterProperties___Builder__create, 0}, 
 		{ "_arrow_parquet___WriterProperties___Builder__version", (DL_FUNC) &_arrow_parquet___WriterProperties___Builder__version, 2}, 
@@ -6259,8 +6663,7 @@ static const R_CallMethodDef CallEntries[] = {
 		{ "_arrow_RecordBatch__columns", (DL_FUNC) &_arrow_RecordBatch__columns, 1}, 
 		{ "_arrow_RecordBatch__column", (DL_FUNC) &_arrow_RecordBatch__column, 2}, 
 		{ "_arrow_RecordBatch__GetColumnByName", (DL_FUNC) &_arrow_RecordBatch__GetColumnByName, 2}, 
-		{ "_arrow_RecordBatch__select", (DL_FUNC) &_arrow_RecordBatch__select, 2}, 
-		{ "_arrow_RecordBatch__from_dataframe", (DL_FUNC) &_arrow_RecordBatch__from_dataframe, 1}, 
+		{ "_arrow_RecordBatch__SelectColumns", (DL_FUNC) &_arrow_RecordBatch__SelectColumns, 2}, 
 		{ "_arrow_RecordBatch__Equals", (DL_FUNC) &_arrow_RecordBatch__Equals, 3}, 
 		{ "_arrow_RecordBatch__RemoveColumn", (DL_FUNC) &_arrow_RecordBatch__RemoveColumn, 2}, 
 		{ "_arrow_RecordBatch__column_name", (DL_FUNC) &_arrow_RecordBatch__column_name, 2}, 
@@ -6307,7 +6710,6 @@ static const R_CallMethodDef CallEntries[] = {
 		{ "_arrow_Schema__serialize", (DL_FUNC) &_arrow_Schema__serialize, 1}, 
 		{ "_arrow_Schema__Equals", (DL_FUNC) &_arrow_Schema__Equals, 3}, 
 		{ "_arrow_arrow__UnifySchemas", (DL_FUNC) &_arrow_arrow__UnifySchemas, 1}, 
-		{ "_arrow_Table__from_dataframe", (DL_FUNC) &_arrow_Table__from_dataframe, 1}, 
 		{ "_arrow_Table__num_columns", (DL_FUNC) &_arrow_Table__num_columns, 1}, 
 		{ "_arrow_Table__num_rows", (DL_FUNC) &_arrow_Table__num_rows, 1}, 
 		{ "_arrow_Table__schema", (DL_FUNC) &_arrow_Table__schema, 1}, 
@@ -6322,14 +6724,16 @@ static const R_CallMethodDef CallEntries[] = {
 		{ "_arrow_Table__Validate", (DL_FUNC) &_arrow_Table__Validate, 1}, 
 		{ "_arrow_Table__ValidateFull", (DL_FUNC) &_arrow_Table__ValidateFull, 1}, 
 		{ "_arrow_Table__GetColumnByName", (DL_FUNC) &_arrow_Table__GetColumnByName, 2}, 
-		{ "_arrow_Table__select", (DL_FUNC) &_arrow_Table__select, 2}, 
+		{ "_arrow_Table__SelectColumns", (DL_FUNC) &_arrow_Table__SelectColumns, 2}, 
+		{ "_arrow_all_record_batches", (DL_FUNC) &_arrow_all_record_batches, 1}, 
+		{ "_arrow_Table__from_record_batches", (DL_FUNC) &_arrow_Table__from_record_batches, 2}, 
 		{ "_arrow_Table__from_dots", (DL_FUNC) &_arrow_Table__from_dots, 2}, 
 		{ "_arrow_GetCpuThreadPoolCapacity", (DL_FUNC) &_arrow_GetCpuThreadPoolCapacity, 0}, 
 		{ "_arrow_SetCpuThreadPoolCapacity", (DL_FUNC) &_arrow_SetCpuThreadPoolCapacity, 1}, 
 		{NULL, NULL, 0}
 };
 
-RcppExport void R_init_arrow(DllInfo* dll){
+extern "C" void R_init_arrow(DllInfo* dll){
   R_registerRoutines(dll, NULL, CallEntries, NULL, NULL);
   R_useDynamicSymbols(dll, FALSE);
 }
diff --git a/r/src/arrow_cpp11.h b/r/src/arrow_cpp11.h
new file mode 100644
index 00000000000..859b0491cd0
--- /dev/null
+++ b/r/src/arrow_cpp11.h
@@ -0,0 +1,326 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstring>  // for strlen
+#include <limits>
+#include <memory>
+#include <utility>
+#include <vector>
+#undef Free
+
+#include "./nameof.h"
+
+namespace cpp11 {
+
+template <typename T>
+SEXP as_sexp(const std::shared_ptr<T>& ptr);
+
+template <typename T>
+SEXP as_sexp(const std::vector<std::shared_ptr<T>>& vec);
+
+}  // namespace cpp11
+
+// TODO: move this include up once we can resolve this issue in cpp11
+//       https://github.com/apache/arrow/pull/7819#discussion_r471664878
+#include <cpp11.hpp>
+
+// borrowed from enc package
+// because R does not make these macros available (i.e. from Defn.h)
+#define UTF8_MASK (1 << 3)
+#define ASCII_MASK (1 << 6)
+
+#define IS_ASCII(x) (LEVELS(x) & ASCII_MASK)
+#define IS_UTF8(x) (LEVELS(x) & UTF8_MASK)
+
+namespace arrow {
+namespace r {
+
+template <typename T>
+struct Pointer {
+  Pointer() : ptr_(new T()) {}
+  explicit Pointer(SEXP x)
+      : ptr_(reinterpret_cast<T*>(static_cast<uintptr_t>(REAL(x)[0]))) {}
+
+  inline operator SEXP() const {
+    return Rf_ScalarReal(static_cast<double>(reinterpret_cast<uintptr_t>(ptr_)));
+  }
+
+  inline operator T*() const { return ptr_; }
+
+  inline void finalize() { delete ptr_; }
+
+  T* ptr_;
+};
+
+// until cpp11 has a similar class
+class complexs {
+ public:
+  using value_type = Rcomplex;
+
+  explicit complexs(SEXP x) : data_(x) {}
+
+  inline R_xlen_t size() const { return XLENGTH(data_); }
+
+  inline operator SEXP() const { return data_; }
+
+ private:
+  cpp11::sexp data_;
+};
+
+// functions that need to be called from an unwind_protect()
+namespace unsafe {
+
+inline const char* utf8_string(SEXP s) {
+  if (!IS_UTF8(s) && !IS_ASCII(s)) {
+    return Rf_translateCharUTF8(s);
+  } else {
+    return CHAR(s);
+  }
+}
+
+inline R_xlen_t r_string_size(SEXP s) {
+  if (s == NA_STRING) {
+    return 0;
+  } else if (IS_ASCII(s) || IS_UTF8(s)) {
+    return XLENGTH(s);
+  } else {
+    return strlen(Rf_translateCharUTF8(s));
+  }
+}
+
+}  // namespace unsafe
+
+inline SEXP utf8_strings(SEXP x) {
+  return cpp11::unwind_protect([x] {
+    R_xlen_t n = XLENGTH(x);
+    for (R_xlen_t i = 0; i < n; i++) {
+      SEXP s = STRING_ELT(x, i);
+      if (s != NA_STRING && !IS_UTF8(s) && !IS_ASCII(s)) {
+        SET_STRING_ELT(x, i, Rf_mkCharCE(Rf_translateCharUTF8(s), CE_UTF8));
+      }
+    }
+    return x;
+  });
+}
+
+struct symbols {
+  static SEXP units;
+  static SEXP tzone;
+  static SEXP xp;
+  static SEXP dot_Internal;
+  static SEXP inspect;
+  static SEXP row_names;
+  static SEXP serialize_arrow_r_metadata;
+  static SEXP as_list;
+  static SEXP ptype;
+  static SEXP byte_width;
+  static SEXP list_size;
+  static SEXP arrow_attributes;
+};
+
+struct data {
+  static SEXP classes_POSIXct;
+  static SEXP classes_metadata_r;
+  static SEXP classes_vctrs_list_of;
+  static SEXP classes_tbl_df;
+
+  static SEXP classes_arrow_binary;
+  static SEXP classes_arrow_large_binary;
+  static SEXP classes_arrow_fixed_size_binary;
+
+  static SEXP classes_arrow_list;
+  static SEXP classes_arrow_large_list;
+  static SEXP classes_arrow_fixed_size_list;
+
+  static SEXP classes_factor;
+  static SEXP classes_ordered;
+
+  static SEXP names_metadata;
+  static SEXP empty_raw;
+};
+
+struct ns {
+  static SEXP arrow;
+};
+
+template <typename Pointer>
+Pointer r6_to_pointer(SEXP self) {
+  if (!Rf_inherits(self, "ArrowObject")) {
+    std::string type_name = arrow::util::nameof<
+        cpp11::decay_t<typename std::remove_pointer<Pointer>::type>>();
+    cpp11::stop("Invalid R object for %s, must be an ArrowObject", type_name.c_str());
+  }
+  void* p = R_ExternalPtrAddr(Rf_findVarInFrame(self, arrow::r::symbols::xp));
+  if (p == nullptr) {
+    SEXP klass = Rf_getAttrib(self, R_ClassSymbol);
+    cpp11::stop("Invalid <%s>, external pointer to null", CHAR(STRING_ELT(klass, 0)));
+  }
+  return reinterpret_cast<Pointer>(p);
+}
+
+// T is either std::shared_ptr<U> or std::unique_ptr<U>
+// e.g. T = std::shared_ptr<arrow::Array>
+template <typename T>
+class ExternalPtrInput {
+ public:
+  explicit ExternalPtrInput(SEXP self) : ptr_(r6_to_pointer<const T*>(self)) {}
+
+  operator const T&() const { return *ptr_; }
+
+ private:
+  const T* ptr_;
+};
+
+template <typename T>
+class VectorExternalPtrInput {
+ public:
+  explicit VectorExternalPtrInput(SEXP self) : vec_(XLENGTH(self)) {
+    R_xlen_t i = 0;
+    for (auto& element : vec_) {
+      element = *r6_to_pointer<const T*>(VECTOR_ELT(self, i++));
+    }
+  }
+  operator const std::vector<T>&() const { return vec_; }
+
+ private:
+  std::vector<T> vec_;
+};
+
+template <typename T>
+class DefaultInput {
+ public:
+  explicit DefaultInput(SEXP from) : from_(from) {}
+
+  operator T() const { return cpp11::as_cpp<T>(from_); }
+
+ private:
+  SEXP from_;
+};
+
+template <typename T>
+class ConstReferenceInput {
+ public:
+  explicit ConstReferenceInput(SEXP from) : obj_(cpp11::as_cpp<T>(from)) {}
+
+  using const_reference = const T&;
+  operator const_reference() const { return obj_; }
+
+ private:
+  T obj_;
+};
+
+template <typename T>
+struct Input {
+  using type = DefaultInput<T>;
+};
+
+template <typename T>
+struct Input<const T&> {
+  using type = ConstReferenceInput<typename std::decay<T>::type>;
+};
+
+template <typename T>
+struct Input<const std::shared_ptr<T>&> {
+  using type = ExternalPtrInput<std::shared_ptr<T>>;
+};
+
+template <typename T>
+struct Input<const std::unique_ptr<T>&> {
+  using type = ExternalPtrInput<std::unique_ptr<T>>;
+};
+
+template <typename T>
+struct Input<const std::vector<std::shared_ptr<T>>&> {
+  using type = VectorExternalPtrInput<std::shared_ptr<T>>;
+};
+
+template <typename Rvector, typename T, typename ToVectorElement>
+Rvector to_r_vector(const std::vector<std::shared_ptr<T>>& x,
+                    ToVectorElement&& to_element) {
+  R_xlen_t n = x.size();
+  Rvector out(n);
+  for (R_xlen_t i = 0; i < n; i++) {
+    out[i] = to_element(x[i]);
+  }
+  return out;
+}
+
+template <typename T, typename ToString>
+cpp11::writable::strings to_r_strings(const std::vector<std::shared_ptr<T>>& x,
+                                      ToString&& to_string) {
+  return to_r_vector<cpp11::writable::strings>(x, std::forward<ToString>(to_string));
+}
+
+template <typename T>
+cpp11::writable::list to_r_list(const std::vector<std::shared_ptr<T>>& x) {
+  auto as_sexp = [](const std::shared_ptr<T>& t) { return cpp11::as_sexp(t); };
+  return to_r_vector<cpp11::writable::list>(x, as_sexp);
+}
+
+template <typename T, typename ToListElement>
+cpp11::writable::list to_r_list(const std::vector<std::shared_ptr<T>>& x,
+                                ToListElement&& to_element) {
+  auto as_sexp = [&](const std::shared_ptr<T>& t) {
+    return cpp11::as_sexp(to_element(t));
+  };
+  return to_r_vector<cpp11::writable::list>(x, as_sexp);
+}
+
+inline cpp11::writable::integers short_row_names(int n) { return {NA_INTEGER, -n}; }
+
+template <typename T>
+std::vector<T> from_r_list(cpp11::list args) {
+  std::vector<T> vec;
+  R_xlen_t n = args.size();
+  for (R_xlen_t i = 0; i < n; i++) {
+    vec.push_back(cpp11::as_cpp<T>(args[i]));
+  }
+  return vec;
+}
+
+bool GetBoolOption(const std::string& name, bool default_);
+
+}  // namespace r
+}  // namespace arrow
+
+namespace cpp11 {
+
+template <typename T>
+using enable_if_shared_ptr = typename std::enable_if<
+    std::is_same<std::shared_ptr<typename T::element_type>, T>::value, T>::type;
+
+template <typename T>
+enable_if_shared_ptr<T> as_cpp(SEXP from) {
+  return arrow::r::ExternalPtrInput<T>(from);
+}
+
+template <typename T>
+SEXP as_sexp(const std::shared_ptr<T>& ptr) {
+  return cpp11::external_pointer<std::shared_ptr<T>>(new std::shared_ptr<T>(ptr));
+}
+
+template <typename T>
+SEXP as_sexp(const std::vector<std::shared_ptr<T>>& vec) {
+  return arrow::r::to_r_list(vec);
+}
+
+template <typename E>
+enable_if_enum<E, SEXP> as_sexp(E e) {
+  return as_sexp(static_cast<int>(e));
+}
+
+}  // namespace cpp11
diff --git a/r/src/arrow_exports.h b/r/src/arrow_exports.h
index f568d2a14b9..c4cc0ff6ede 100644
--- a/r/src/arrow_exports.h
+++ b/r/src/arrow_exports.h
@@ -19,7 +19,7 @@
 
 #pragma once
 
-#include "./arrow_rcpp.h"
+#include "./arrow_cpp11.h"
 
 #if defined(ARROW_R_WITH_ARROW)
 #include <arrow/dataset/type_fwd.h>
@@ -29,6 +29,7 @@
 #include <arrow/status.h>
 #include <arrow/type_fwd.h>
 #include <arrow/util/compression.h>
+#include <arrow/util/value_parsing.h>
 
 namespace arrow {
 
@@ -83,15 +84,4 @@ class FileWriter;
 }  // namespace arrow
 }  // namespace parquet
 
-RCPP_EXPOSED_ENUM_NODECL(arrow::Type::type)
-RCPP_EXPOSED_ENUM_NODECL(arrow::DateUnit)
-RCPP_EXPOSED_ENUM_NODECL(arrow::ipc::MetadataVersion)
-RCPP_EXPOSED_ENUM_NODECL(arrow::TimeUnit::type)
-RCPP_EXPOSED_ENUM_NODECL(arrow::StatusCode)
-RCPP_EXPOSED_ENUM_NODECL(arrow::io::FileMode::type)
-RCPP_EXPOSED_ENUM_NODECL(arrow::ipc::MessageType)
-RCPP_EXPOSED_ENUM_NODECL(arrow::Compression::type)
-RCPP_EXPOSED_ENUM_NODECL(arrow::fs::FileType)
-RCPP_EXPOSED_ENUM_NODECL(parquet::ParquetVersion::type)
-
 #endif
diff --git a/r/src/arrow_rcpp.h b/r/src/arrow_rcpp.h
deleted file mode 100644
index a0a05815c01..00000000000
--- a/r/src/arrow_rcpp.h
+++ /dev/null
@@ -1,186 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <RcppCommon.h>
-
-#include <limits>
-#include <memory>
-#include <utility>
-#include <vector>
-#undef Free
-
-namespace arrow {
-namespace r {
-struct symbols {
-  static SEXP units;
-  static SEXP tzone;
-  static SEXP xp;
-  static SEXP dot_Internal;
-  static SEXP inspect;
-  static SEXP row_names;
-  static SEXP serialize_arrow_r_metadata;
-  static SEXP as_list;
-  static SEXP ptype;
-  static SEXP byte_width;
-  static SEXP list_size;
-};
-
-struct data {
-  static SEXP classes_POSIXct;
-  static SEXP classes_metadata_r;
-  static SEXP classes_vctrs_list_of;
-
-  static SEXP classes_arrow_binary;
-  static SEXP classes_arrow_large_binary;
-  static SEXP classes_arrow_fixed_size_binary;
-
-  static SEXP classes_arrow_list;
-  static SEXP classes_arrow_large_list;
-  static SEXP classes_arrow_fixed_size_list;
-
-  static SEXP classes_factor;
-  static SEXP classes_ordered;
-
-  static SEXP names_metadata;
-  static SEXP empty_raw;
-};
-
-struct ns {
-  static SEXP arrow;
-};
-
-}  // namespace r
-}  // namespace arrow
-
-namespace Rcpp {
-namespace internal {
-
-template <typename Pointer>
-Pointer r6_to_smart_pointer(SEXP self) {
-  return reinterpret_cast<Pointer>(
-      R_ExternalPtrAddr(Rf_findVarInFrame(self, arrow::r::symbols::xp)));
-}
-
-}  // namespace internal
-
-template <typename T>
-class ConstReferenceSmartPtrInputParameter {
- public:
-  using const_reference = const T&;
-
-  explicit ConstReferenceSmartPtrInputParameter(SEXP self)
-      : ptr(internal::r6_to_smart_pointer<const T*>(self)) {}
-
-  inline operator const_reference() { return *ptr; }
-
- private:
-  const T* ptr;
-};
-
-template <typename T>
-class ConstReferenceVectorSmartPtrInputParameter {
- public:
-  using const_reference = const std::vector<T>&;
-
-  explicit ConstReferenceVectorSmartPtrInputParameter(SEXP self) : vec() {
-    R_xlen_t n = XLENGTH(self);
-    for (R_xlen_t i = 0; i < n; i++) {
-      vec.push_back(*internal::r6_to_smart_pointer<const T*>(VECTOR_ELT(self, i)));
-    }
-  }
-
-  inline operator const_reference() { return vec; }
-
- private:
-  std::vector<T> vec;
-};
-
-namespace traits {
-
-template <typename T>
-struct input_parameter<const std::shared_ptr<T>&> {
-  typedef typename Rcpp::ConstReferenceSmartPtrInputParameter<std::shared_ptr<T>> type;
-};
-
-template <typename T>
-struct input_parameter<const std::unique_ptr<T>&> {
-  typedef typename Rcpp::ConstReferenceSmartPtrInputParameter<std::unique_ptr<T>> type;
-};
-
-template <typename T>
-struct input_parameter<const std::vector<std::shared_ptr<T>>&> {
-  typedef typename Rcpp::ConstReferenceVectorSmartPtrInputParameter<std::shared_ptr<T>>
-      type;
-};
-
-struct wrap_type_shared_ptr_tag {};
-struct wrap_type_unique_ptr_tag {};
-
-template <typename T>
-struct wrap_type_traits<std::shared_ptr<T>> {
-  using wrap_category = wrap_type_shared_ptr_tag;
-};
-
-template <typename T>
-struct wrap_type_traits<std::unique_ptr<T>> {
-  using wrap_category = wrap_type_unique_ptr_tag;
-};
-
-}  // namespace traits
-
-namespace internal {
-
-template <typename T>
-inline SEXP wrap_dispatch(const T& x, Rcpp::traits::wrap_type_shared_ptr_tag);
-
-template <typename T>
-inline SEXP wrap_dispatch(const T& x, Rcpp::traits::wrap_type_unique_ptr_tag);
-
-}  // namespace internal
-}  // namespace Rcpp
-
-#include <Rcpp.h>
-
-namespace Rcpp {
-namespace internal {
-
-template <typename T>
-inline SEXP wrap_dispatch(const T& x, Rcpp::traits::wrap_type_shared_ptr_tag) {
-  return Rcpp::XPtr<std::shared_ptr<typename T::element_type>>(
-      new std::shared_ptr<typename T::element_type>(x));
-}
-
-template <typename T>
-inline SEXP wrap_dispatch(const T& x, Rcpp::traits::wrap_type_unique_ptr_tag) {
-  return Rcpp::XPtr<std::unique_ptr<typename T::element_type>>(
-      new std::unique_ptr<typename T::element_type>(const_cast<T&>(x).release()));
-}
-
-}  // namespace internal
-
-}  // namespace Rcpp
-
-namespace Rcpp {
-using NumericVector_ = Rcpp::Vector<REALSXP, Rcpp::NoProtectStorage>;
-using IntegerVector_ = Rcpp::Vector<INTSXP, Rcpp::NoProtectStorage>;
-using LogicalVector_ = Rcpp::Vector<LGLSXP, Rcpp::NoProtectStorage>;
-using StringVector_ = Rcpp::Vector<STRSXP, Rcpp::NoProtectStorage>;
-using CharacterVector_ = StringVector_;
-using RawVector_ = Rcpp::Vector<RAWSXP, Rcpp::NoProtectStorage>;
-using List_ = Rcpp::Vector<VECSXP, Rcpp::NoProtectStorage>;
-
-}  // namespace Rcpp
diff --git a/r/src/arrow_types.h b/r/src/arrow_types.h
index 7221bf93a04..4a1845a16bc 100644
--- a/r/src/arrow_types.h
+++ b/r/src/arrow_types.h
@@ -17,29 +17,14 @@
 
 #pragma once
 
-#include "./arrow_rcpp.h"
+#include <cpp11/R.hpp>
 
-template <typename T>
-struct NoDelete {
-  inline void operator()(T* ptr) {}
-};
-
-namespace Rcpp {
-
-template <int RTYPE>
-inline constexpr typename Rcpp::Vector<RTYPE>::stored_type default_value() {
-  return Rcpp::Vector<RTYPE>::get_na();
-}
-template <>
-inline constexpr Rbyte default_value<RAWSXP>() {
-  return 0;
-}
-
-}  // namespace Rcpp
+#include "./arrow_cpp11.h"
 
 #if defined(ARROW_R_WITH_ARROW)
 
 #include <arrow/buffer.h>  // for RBuffer definition below
+#include <arrow/c/bridge.h>
 #include <arrow/result.h>
 #include <arrow/type_fwd.h>
 
@@ -48,19 +33,17 @@ inline constexpr Rbyte default_value<RAWSXP>() {
 #include <utility>
 #include <vector>
 
-RCPP_EXPOSED_ENUM_NODECL(arrow::StatusCode)
-
 SEXP ChunkedArray__as_vector(const std::shared_ptr<arrow::ChunkedArray>& chunked_array);
 SEXP Array__as_vector(const std::shared_ptr<arrow::Array>& array);
 std::shared_ptr<arrow::Array> Array__from_vector(SEXP x, SEXP type);
 std::shared_ptr<arrow::RecordBatch> RecordBatch__from_arrays(SEXP, SEXP);
-std::shared_ptr<arrow::RecordBatch> RecordBatch__from_dataframe(Rcpp::DataFrame tbl);
+arrow::MemoryPool* gc_memory_pool();
 
 namespace arrow {
 
 static inline void StopIfNotOk(const Status& status) {
   if (!(status.ok())) {
-    Rcpp::stop(status.ToString());
+    cpp11::stop(status.ToString());
   }
 }
 
@@ -74,68 +57,54 @@ namespace r {
 
 std::shared_ptr<arrow::DataType> InferArrowType(SEXP x);
 
-template <typename T>
-inline std::shared_ptr<T> extract(SEXP x) {
-  return Rcpp::ConstReferenceSmartPtrInputParameter<std::shared_ptr<T>>(x);
-}
-
 Status count_fields(SEXP lst, int* out);
 
 std::shared_ptr<arrow::Array> Array__from_vector(
     SEXP x, const std::shared_ptr<arrow::DataType>& type, bool type_inferred);
 
-template <typename T>
-std::vector<std::shared_ptr<T>> List_to_shared_ptr_vector(SEXP x) {
-  std::vector<std::shared_ptr<T>> vec;
-  R_xlen_t n = Rf_xlength(x);
-  for (R_xlen_t i = 0; i < n; i++) {
-    Rcpp::ConstReferenceSmartPtrInputParameter<std::shared_ptr<T>> ptr(VECTOR_ELT(x, i));
-    vec.push_back(ptr);
-  }
-  return vec;
-}
-
 void inspect(SEXP obj);
 
 // the integer64 sentinel
 constexpr int64_t NA_INT64 = std::numeric_limits<int64_t>::min();
 
-template <int RTYPE, typename Vec = Rcpp::Vector<RTYPE>>
+template <typename RVector>
 class RBuffer : public MutableBuffer {
  public:
-  explicit RBuffer(Vec vec)
-      : MutableBuffer(reinterpret_cast<uint8_t*>(vec.begin()),
-                      vec.size() * sizeof(typename Vec::stored_type)),
+  explicit RBuffer(RVector vec)
+      : MutableBuffer(reinterpret_cast<uint8_t*>(DATAPTR(vec)),
+                      vec.size() * sizeof(typename RVector::value_type),
+                      arrow::CPUDevice::memory_manager(gc_memory_pool())),
         vec_(vec) {}
 
  private:
   // vec_ holds the memory
-  Vec vec_;
+  RVector vec_;
 };
 
 std::shared_ptr<arrow::DataType> InferArrowTypeFromFactor(SEXP);
 
-void validate_slice_offset(int offset, int len);
+void validate_slice_offset(R_xlen_t offset, int64_t len);
 
-void validate_slice_length(int length, int available);
+void validate_slice_length(R_xlen_t length, int64_t available);
 
 void validate_index(int i, int len);
 
 template <typename Lambda>
-void TraverseDots(SEXP dots, int num_fields, Lambda lambda) {
-  SEXP names = Rf_getAttrib(dots, R_NamesSymbol);
+void TraverseDots(cpp11::list dots, int num_fields, Lambda lambda) {
+  cpp11::strings names(dots.attr(R_NamesSymbol));
 
   for (R_xlen_t i = 0, j = 0; j < num_fields; i++) {
-    SEXP name_i = STRING_ELT(names, i);
-    SEXP x_i = VECTOR_ELT(dots, i);
-
-    if (LENGTH(name_i) == 0) {
-      SEXP names_x_i = Rf_getAttrib(x_i, R_NamesSymbol);
-      for (R_xlen_t k = 0; k < XLENGTH(x_i); k++, j++) {
-        lambda(j, VECTOR_ELT(x_i, k), STRING_ELT(names_x_i, k));
+    auto name_i = names[i];
+
+    if (name_i.size() == 0) {
+      cpp11::list x_i = dots[i];
+      cpp11::strings names_x_i(x_i.attr(R_NamesSymbol));
+      R_xlen_t n_i = x_i.size();
+      for (R_xlen_t k = 0; k < n_i; k++, j++) {
+        lambda(j, x_i[k], names_x_i[k]);
       }
     } else {
-      lambda(j, x_i, name_i);
+      lambda(j, dots[i], name_i);
       j++;
     }
   }
diff --git a/r/src/buffer.cpp b/r/src/buffer.cpp
index 09ab39a5f98..2814677343d 100644
--- a/r/src/buffer.cpp
+++ b/r/src/buffer.cpp
@@ -43,23 +43,23 @@ int64_t Buffer__size(const std::shared_ptr<arrow::Buffer>& buffer) {
 std::shared_ptr<arrow::Buffer> r___RBuffer__initialize(SEXP x) {
   switch (TYPEOF(x)) {
     case RAWSXP:
-      return std::make_shared<arrow::r::RBuffer<RAWSXP>>(x);
+      return std::make_shared<arrow::r::RBuffer<cpp11::raws>>(x);
     case REALSXP:
-      return std::make_shared<arrow::r::RBuffer<REALSXP>>(x);
+      return std::make_shared<arrow::r::RBuffer<cpp11::doubles>>(x);
     case INTSXP:
-      return std::make_shared<arrow::r::RBuffer<INTSXP>>(x);
+      return std::make_shared<arrow::r::RBuffer<cpp11::integers>>(x);
     case CPLXSXP:
-      return std::make_shared<arrow::r::RBuffer<CPLXSXP>>(x);
+      return std::make_shared<arrow::r::RBuffer<arrow::r::complexs>>(
+          arrow::r::complexs(x));
     default:
-      Rcpp::stop(
-          tfm::format("R object of type %s not supported", Rf_type2char(TYPEOF(x))));
+      break;
   }
-  return nullptr;
+  cpp11::stop("R object of type <%s> not supported", Rf_type2char(TYPEOF(x)));
 }
 
 // [[arrow::export]]
-Rcpp::RawVector Buffer__data(const std::shared_ptr<arrow::Buffer>& buffer) {
-  return Rcpp::RawVector(buffer->data(), buffer->data() + buffer->size());
+cpp11::writable::raws Buffer__data(const std::shared_ptr<arrow::Buffer>& buffer) {
+  return cpp11::writable::raws(buffer->data(), buffer->data() + buffer->size());
 }
 
 // [[arrow::export]]
diff --git a/r/src/chunkedarray.cpp b/r/src/chunkedarray.cpp
index a451739f323..f52f20ee0de 100644
--- a/r/src/chunkedarray.cpp
+++ b/r/src/chunkedarray.cpp
@@ -17,9 +17,6 @@
 
 #include "./arrow_types.h"
 
-using Rcpp::List;
-using Rcpp::wrap;
-
 #if defined(ARROW_R_WITH_ARROW)
 
 #include <arrow/chunked_array.h>
@@ -47,8 +44,9 @@ std::shared_ptr<arrow::Array> ChunkedArray__chunk(
 }
 
 // [[arrow::export]]
-List ChunkedArray__chunks(const std::shared_ptr<arrow::ChunkedArray>& chunked_array) {
-  return wrap(chunked_array->chunks());
+cpp11::list ChunkedArray__chunks(
+    const std::shared_ptr<arrow::ChunkedArray>& chunked_array) {
+  return cpp11::as_sexp(chunked_array->chunks());
 }
 
 // [[arrow::export]]
@@ -59,14 +57,15 @@ std::shared_ptr<arrow::DataType> ChunkedArray__type(
 
 // [[arrow::export]]
 std::shared_ptr<arrow::ChunkedArray> ChunkedArray__Slice1(
-    const std::shared_ptr<arrow::ChunkedArray>& chunked_array, int offset) {
+    const std::shared_ptr<arrow::ChunkedArray>& chunked_array, R_xlen_t offset) {
   arrow::r::validate_slice_offset(offset, chunked_array->length());
   return chunked_array->Slice(offset);
 }
 
 // [[arrow::export]]
 std::shared_ptr<arrow::ChunkedArray> ChunkedArray__Slice2(
-    const std::shared_ptr<arrow::ChunkedArray>& chunked_array, int offset, int length) {
+    const std::shared_ptr<arrow::ChunkedArray>& chunked_array, R_xlen_t offset,
+    R_xlen_t length) {
   arrow::r::validate_slice_offset(offset, chunked_array->length());
   arrow::r::validate_slice_length(length, chunked_array->length() - offset);
   return chunked_array->Slice(offset, length);
diff --git a/r/src/compression.cpp b/r/src/compression.cpp
index 038390b71b4..18c63e4fd19 100644
--- a/r/src/compression.cpp
+++ b/r/src/compression.cpp
@@ -21,11 +21,9 @@
 #include <arrow/io/compressed.h>
 #include <arrow/util/compression.h>
 
-RCPP_EXPOSED_ENUM_NODECL(arrow::Compression::type)
-
 // [[arrow::export]]
 std::shared_ptr<arrow::util::Codec> util___Codec__Create(arrow::Compression::type codec,
-                                                         int compression_level) {
+                                                         R_xlen_t compression_level) {
   return ValueOrStop(arrow::util::Codec::Create(codec, compression_level));
 }
 
@@ -43,14 +41,16 @@ bool util___Codec__IsAvailable(arrow::Compression::type codec) {
 std::shared_ptr<arrow::io::CompressedOutputStream> io___CompressedOutputStream__Make(
     const std::shared_ptr<arrow::util::Codec>& codec,
     const std::shared_ptr<arrow::io::OutputStream>& raw) {
-  return ValueOrStop(arrow::io::CompressedOutputStream::Make(codec.get(), raw));
+  return ValueOrStop(
+      arrow::io::CompressedOutputStream::Make(codec.get(), raw, gc_memory_pool()));
 }
 
 // [[arrow::export]]
 std::shared_ptr<arrow::io::CompressedInputStream> io___CompressedInputStream__Make(
     const std::shared_ptr<arrow::util::Codec>& codec,
     const std::shared_ptr<arrow::io::InputStream>& raw) {
-  return ValueOrStop(arrow::io::CompressedInputStream::Make(codec.get(), raw));
+  return ValueOrStop(
+      arrow::io::CompressedInputStream::Make(codec.get(), raw, gc_memory_pool()));
 }
 
 #endif
diff --git a/r/src/compute.cpp b/r/src/compute.cpp
index 8d185220dd0..3c288c93455 100644
--- a/r/src/compute.cpp
+++ b/r/src/compute.cpp
@@ -23,7 +23,10 @@
 #include <arrow/record_batch.h>
 #include <arrow/table.h>
 
-using Rcpp::List_;
+arrow::compute::ExecContext* gc_context() {
+  static arrow::compute::ExecContext context(gc_memory_pool());
+  return &context;
+}
 
 // [[arrow::export]]
 std::shared_ptr<arrow::compute::CastOptions> compute___CastOptions__initialize(
@@ -40,7 +43,7 @@ std::shared_ptr<arrow::Array> Array__cast(
     const std::shared_ptr<arrow::Array>& array,
     const std::shared_ptr<arrow::DataType>& target_type,
     const std::shared_ptr<arrow::compute::CastOptions>& options) {
-  return ValueOrStop(arrow::compute::Cast(*array, target_type, *options));
+  return ValueOrStop(arrow::compute::Cast(*array, target_type, *options, gc_context()));
 }
 
 // [[arrow::export]]
@@ -49,7 +52,8 @@ std::shared_ptr<arrow::ChunkedArray> ChunkedArray__cast(
     const std::shared_ptr<arrow::DataType>& target_type,
     const std::shared_ptr<arrow::compute::CastOptions>& options) {
   arrow::Datum value(chunked_array);
-  arrow::Datum out = ValueOrStop(arrow::compute::Cast(value, target_type, *options));
+  arrow::Datum out =
+      ValueOrStop(arrow::compute::Cast(value, target_type, *options, gc_context()));
   return out.chunked_array();
 }
 
@@ -86,13 +90,15 @@ std::shared_ptr<arrow::Table> Table__cast(
 template <typename T>
 std::shared_ptr<T> MaybeUnbox(const char* class_name, SEXP x) {
   if (Rf_inherits(x, "ArrowObject") && Rf_inherits(x, class_name)) {
-    Rcpp::ConstReferenceSmartPtrInputParameter<std::shared_ptr<T>> obj(x);
-    return static_cast<std::shared_ptr<T>>(obj);
+    return cpp11::as_cpp<std::shared_ptr<T>>(x);
   }
   return nullptr;
 }
 
-arrow::Datum to_datum(SEXP x) {
+namespace cpp11 {
+
+template <>
+arrow::Datum as_cpp<arrow::Datum>(SEXP x) {
   if (auto array = MaybeUnbox<arrow::Array>("Array", x)) {
     return array;
   }
@@ -115,40 +121,44 @@ arrow::Datum to_datum(SEXP x) {
 
   // This assumes that R objects have already been converted to Arrow objects;
   // that seems right but should we do the wrapping here too/instead?
-  Rcpp::stop("to_datum: Not implemented for type %s", Rf_type2char(TYPEOF(x)));
+  cpp11::stop("to_datum: Not implemented for type %s", Rf_type2char(TYPEOF(x)));
+  return arrow::Datum();
 }
+}  // namespace cpp11
 
 SEXP from_datum(arrow::Datum datum) {
   switch (datum.kind()) {
     case arrow::Datum::SCALAR:
-      return Rcpp::wrap(datum.scalar());
+      return cpp11::as_sexp(datum.scalar());
 
     case arrow::Datum::ARRAY:
-      return Rcpp::wrap(datum.make_array());
+      return cpp11::as_sexp(datum.make_array());
 
     case arrow::Datum::CHUNKED_ARRAY:
-      return Rcpp::wrap(datum.chunked_array());
+      return cpp11::as_sexp(datum.chunked_array());
 
     case arrow::Datum::RECORD_BATCH:
-      return Rcpp::wrap(datum.record_batch());
+      return cpp11::as_sexp(datum.record_batch());
 
     case arrow::Datum::TABLE:
-      return Rcpp::wrap(datum.table());
+      return cpp11::as_sexp(datum.table());
 
     default:
       break;
   }
 
   auto str = datum.ToString();
-  Rcpp::stop("from_datum: Not implemented for Datum %s", str.c_str());
+  cpp11::stop("from_datum: Not implemented for Datum %s", str.c_str());
+  return R_NilValue;
 }
 
 std::shared_ptr<arrow::compute::FunctionOptions> make_compute_options(
-    std::string func_name, List_ options) {
+    std::string func_name, cpp11::list options) {
   if (func_name == "filter") {
     using Options = arrow::compute::FilterOptions;
     auto out = std::make_shared<Options>(Options::Defaults());
-    if (!Rf_isNull(options["keep_na"]) && options["keep_na"]) {
+    SEXP keep_na = options["keep_na"];
+    if (!Rf_isNull(keep_na) && cpp11::as_cpp<bool>(keep_na)) {
       out->null_selection_behavior = Options::EMIT_NULL;
     }
     return out;
@@ -163,7 +173,8 @@ std::shared_ptr<arrow::compute::FunctionOptions> make_compute_options(
   if (func_name == "min_max") {
     using Options = arrow::compute::MinMaxOptions;
     auto out = std::make_shared<Options>(Options::Defaults());
-    out->null_handling = options["na.rm"] ? Options::SKIP : Options::OUTPUT_NULL;
+    out->null_handling =
+        cpp11::as_cpp<bool>(options["na.rm"]) ? Options::SKIP : Options::EMIT_NULL;
     return out;
   }
 
@@ -171,13 +182,11 @@ std::shared_ptr<arrow::compute::FunctionOptions> make_compute_options(
 }
 
 // [[arrow::export]]
-SEXP compute__CallFunction(std::string func_name, List_ args, List_ options) {
+SEXP compute__CallFunction(std::string func_name, cpp11::list args, cpp11::list options) {
   auto opts = make_compute_options(func_name, options);
-  std::vector<arrow::Datum> datum_args;
-  for (auto arg : args) {
-    datum_args.push_back(to_datum(arg));
-  }
-  auto out = ValueOrStop(arrow::compute::CallFunction(func_name, datum_args, opts.get()));
+  auto datum_args = arrow::r::from_r_list<arrow::Datum>(args);
+  auto out = ValueOrStop(
+      arrow::compute::CallFunction(func_name, datum_args, opts.get(), gc_context()));
   return from_datum(out);
 }
 
diff --git a/r/src/csv.cpp b/r/src/csv.cpp
index 9275c827c18..54d3abc3821 100644
--- a/r/src/csv.cpp
+++ b/r/src/csv.cpp
@@ -17,59 +17,120 @@
 
 #include "./arrow_types.h"
 
-using Rcpp::CharacterVector;
-using Rcpp::List_;
-
 #if defined(ARROW_R_WITH_ARROW)
 
 #include <arrow/csv/reader.h>
+#include <arrow/util/value_parsing.h>
 
 // [[arrow::export]]
-std::shared_ptr<arrow::csv::ReadOptions> csv___ReadOptions__initialize(List_ options) {
+std::shared_ptr<arrow::csv::ReadOptions> csv___ReadOptions__initialize(
+    cpp11::list options) {
   auto res =
       std::make_shared<arrow::csv::ReadOptions>(arrow::csv::ReadOptions::Defaults());
-  res->use_threads = options["use_threads"];
-  res->block_size = options["block_size"];
-  res->skip_rows = options["skip_rows"];
-  res->column_names = Rcpp::as<std::vector<std::string>>(options["column_names"]);
-  res->autogenerate_column_names = options["autogenerate_column_names"];
+  res->use_threads = cpp11::as_cpp<bool>(options["use_threads"]);
+  res->block_size = cpp11::as_cpp<int>(options["block_size"]);
+  res->skip_rows = cpp11::as_cpp<int>(options["skip_rows"]);
+  res->column_names = cpp11::as_cpp<std::vector<std::string>>(options["column_names"]);
+  res->autogenerate_column_names =
+      cpp11::as_cpp<bool>(options["autogenerate_column_names"]);
+
   return res;
 }
 
-inline char get_char(CharacterVector x) { return CHAR(STRING_ELT(x, 0))[0]; }
-
 // [[arrow::export]]
-std::shared_ptr<arrow::csv::ParseOptions> csv___ParseOptions__initialize(List_ options) {
+std::shared_ptr<arrow::csv::ParseOptions> csv___ParseOptions__initialize(
+    cpp11::list options) {
   auto res =
       std::make_shared<arrow::csv::ParseOptions>(arrow::csv::ParseOptions::Defaults());
-  res->delimiter = get_char(options["delimiter"]);
-  res->quoting = options["quoting"];
-  res->quote_char = get_char(options["quote_char"]);
-  res->double_quote = options["double_quote"];
-  res->escape_char = get_char(options["escape_char"]);
-  res->newlines_in_values = options["newlines_in_values"];
-  res->ignore_empty_lines = options["ignore_empty_lines"];
+  res->delimiter = cpp11::as_cpp<char>(options["delimiter"]);
+  res->quoting = cpp11::as_cpp<bool>(options["quoting"]);
+  res->quote_char = cpp11::as_cpp<char>(options["quote_char"]);
+  res->double_quote = cpp11::as_cpp<bool>(options["double_quote"]);
+  res->escape_char = cpp11::as_cpp<char>(options["escape_char"]);
+  res->newlines_in_values = cpp11::as_cpp<bool>(options["newlines_in_values"]);
+  res->ignore_empty_lines = cpp11::as_cpp<bool>(options["ignore_empty_lines"]);
   return res;
 }
 
+// [[arrow::export]]
+SEXP csv___ReadOptions__column_names(
+    const std::shared_ptr<arrow::csv::ReadOptions>& options) {
+  if (options->autogenerate_column_names) {
+    return R_NilValue;
+  }
+
+  return cpp11::as_sexp(options->column_names);
+}
+
 // [[arrow::export]]
 std::shared_ptr<arrow::csv::ConvertOptions> csv___ConvertOptions__initialize(
-    List_ options) {
+    cpp11::list options) {
   auto res = std::make_shared<arrow::csv::ConvertOptions>(
       arrow::csv::ConvertOptions::Defaults());
-  res->check_utf8 = options["check_utf8"];
+  res->check_utf8 = cpp11::as_cpp<bool>(options["check_utf8"]);
   // Recognized spellings for null values
-  res->null_values = Rcpp::as<std::vector<std::string>>(options["null_values"]);
+  res->null_values = cpp11::as_cpp<std::vector<std::string>>(options["null_values"]);
   // Whether string / binary columns can have null values.
   // If true, then strings in "null_values" are considered null for string columns.
   // If false, then all strings are valid string values.
-  res->strings_can_be_null = options["strings_can_be_null"];
-  // TODO: there are more conversion options available:
-  // // Optional per-column types (disabling type inference on those columns)
-  // std::unordered_map<std::string, std::shared_ptr<DataType>> column_types;
-  // // Recognized spellings for boolean values
-  // std::vector<std::string> true_values;
-  // std::vector<std::string> false_values;
+  res->strings_can_be_null = cpp11::as_cpp<bool>(options["strings_can_be_null"]);
+
+  res->true_values = cpp11::as_cpp<std::vector<std::string>>(options["true_values"]);
+  res->false_values = cpp11::as_cpp<std::vector<std::string>>(options["false_values"]);
+
+  SEXP col_types = options["col_types"];
+  if (Rf_inherits(col_types, "Schema")) {
+    auto schema = cpp11::as_cpp<std::shared_ptr<arrow::Schema>>(col_types);
+    std::unordered_map<std::string, std::shared_ptr<arrow::DataType>> column_types;
+    for (const auto& field : schema->fields()) {
+      column_types.insert(std::make_pair(field->name(), field->type()));
+    }
+    res->column_types = column_types;
+  }
+
+  res->auto_dict_encode = cpp11::as_cpp<bool>(options["auto_dict_encode"]);
+  res->auto_dict_max_cardinality =
+      cpp11::as_cpp<int>(options["auto_dict_max_cardinality"]);
+  res->include_columns =
+      cpp11::as_cpp<std::vector<std::string>>(options["include_columns"]);
+  res->include_missing_columns = cpp11::as_cpp<bool>(options["include_missing_columns"]);
+
+  SEXP op_timestamp_parsers = options["timestamp_parsers"];
+  if (!Rf_isNull(op_timestamp_parsers)) {
+    std::vector<std::shared_ptr<arrow::TimestampParser>> timestamp_parsers;
+
+    // if we have a character vector, convert to arrow::TimestampParser
+    if (TYPEOF(op_timestamp_parsers) == STRSXP) {
+      cpp11::strings s_timestamp_parsers(op_timestamp_parsers);
+      for (cpp11::r_string s : s_timestamp_parsers) {
+        timestamp_parsers.push_back(arrow::TimestampParser::MakeStrptime(s));
+      }
+
+    } else if (TYPEOF(op_timestamp_parsers) == VECSXP) {
+      cpp11::list lst_parsers(op_timestamp_parsers);
+
+      for (SEXP x : lst_parsers) {
+        // handle scalar string and TimestampParser instances
+        if (TYPEOF(x) == STRSXP && XLENGTH(x) == 1) {
+          timestamp_parsers.push_back(
+              arrow::TimestampParser::MakeStrptime(CHAR(STRING_ELT(x, 0))));
+        } else if (Rf_inherits(x, "TimestampParser")) {
+          timestamp_parsers.push_back(
+              cpp11::as_cpp<std::shared_ptr<arrow::TimestampParser>>(x));
+        } else {
+          cpp11::stop(
+              "unsupported timestamp parser, must be a scalar string or a "
+              "<TimestampParser> object");
+        }
+      }
+
+    } else {
+      cpp11::stop(
+          "unsupported timestamp parser, must be character vector of strptime "
+          "specifications, or a list of <TimestampParser> objects");
+    }
+    res->timestamp_parsers = timestamp_parsers;
+  }
 
   return res;
 }
@@ -80,9 +141,8 @@ std::shared_ptr<arrow::csv::TableReader> csv___TableReader__Make(
     const std::shared_ptr<arrow::csv::ReadOptions>& read_options,
     const std::shared_ptr<arrow::csv::ParseOptions>& parse_options,
     const std::shared_ptr<arrow::csv::ConvertOptions>& convert_options) {
-  return ValueOrStop(arrow::csv::TableReader::Make(arrow::default_memory_pool(), input,
-                                                   *read_options, *parse_options,
-                                                   *convert_options));
+  return ValueOrStop(arrow::csv::TableReader::Make(gc_memory_pool(), input, *read_options,
+                                                   *parse_options, *convert_options));
 }
 
 // [[arrow::export]]
@@ -91,4 +151,26 @@ std::shared_ptr<arrow::Table> csv___TableReader__Read(
   return ValueOrStop(table_reader->Read());
 }
 
+// [[arrow::export]]
+std::string TimestampParser__kind(const std::shared_ptr<arrow::TimestampParser>& parser) {
+  return parser->kind();
+}
+
+// [[arrow::export]]
+std::string TimestampParser__format(
+    const std::shared_ptr<arrow::TimestampParser>& parser) {
+  return parser->format();
+}
+
+// [[arrow::export]]
+std::shared_ptr<arrow::TimestampParser> TimestampParser__MakeStrptime(
+    std::string format) {
+  return arrow::TimestampParser::MakeStrptime(format);
+}
+
+// [[arrow::export]]
+std::shared_ptr<arrow::TimestampParser> TimestampParser__MakeISO8601() {
+  return arrow::TimestampParser::MakeISO8601();
+}
+
 #endif
diff --git a/r/src/dataset.cpp b/r/src/dataset.cpp
index 873524397a0..4327b80f187 100644
--- a/r/src/dataset.cpp
+++ b/r/src/dataset.cpp
@@ -21,21 +21,21 @@
 
 #include <arrow/dataset/api.h>
 #include <arrow/filesystem/filesystem.h>
+#include <arrow/ipc/writer.h>
 #include <arrow/table.h>
 #include <arrow/util/iterator.h>
 
 namespace ds = ::arrow::dataset;
 namespace fs = ::arrow::fs;
 
-using Rcpp::CharacterVector;
-using Rcpp::String;
-
 // Dataset, UnionDataset, FileSystemDataset
 
 // [[arrow::export]]
 std::shared_ptr<ds::ScannerBuilder> dataset___Dataset__NewScan(
     const std::shared_ptr<ds::Dataset>& ds) {
-  return ValueOrStop(ds->NewScan());
+  auto context = std::make_shared<ds::ScanContext>();
+  context->pool = gc_memory_pool();
+  return ValueOrStop(ds->NewScan(std::move(context)));
 }
 
 // [[arrow::export]]
@@ -62,6 +62,12 @@ std::shared_ptr<ds::UnionDataset> dataset___UnionDataset__create(
   return ValueOrStop(ds::UnionDataset::Make(schm, datasets));
 }
 
+// [[arrow::export]]
+std::shared_ptr<ds::InMemoryDataset> dataset___InMemoryDataset__create(
+    const std::shared_ptr<arrow::Table>& table) {
+  return std::make_shared<ds::InMemoryDataset>(table);
+}
+
 // [[arrow::export]]
 ds::DatasetVector dataset___UnionDataset__children(
     const std::shared_ptr<ds::UnionDataset>& ds) {
@@ -74,6 +80,12 @@ std::shared_ptr<ds::FileFormat> dataset___FileSystemDataset__format(
   return dataset->format();
 }
 
+// [[arrow::export]]
+std::shared_ptr<fs::FileSystem> dataset___FileSystemDataset__filesystem(
+    const std::shared_ptr<ds::FileSystemDataset>& dataset) {
+  return dataset->filesystem();
+}
+
 // [[arrow::export]]
 std::vector<std::string> dataset___FileSystemDataset__files(
     const std::shared_ptr<ds::FileSystemDataset>& dataset) {
@@ -161,15 +173,21 @@ std::string dataset___FileFormat__type_name(
   return format->type_name();
 }
 
+// [[arrow::export]]
+std::shared_ptr<ds::FileWriteOptions> dataset___FileFormat__DefaultWriteOptions(
+    const std::shared_ptr<ds::FileFormat>& fmt) {
+  return fmt->DefaultWriteOptions();
+}
+
 // [[arrow::export]]
 std::shared_ptr<ds::ParquetFileFormat> dataset___ParquetFileFormat__Make(
-    bool use_buffered_stream, int64_t buffer_size, CharacterVector dict_columns) {
+    bool use_buffered_stream, int64_t buffer_size, cpp11::strings dict_columns) {
   auto fmt = std::make_shared<ds::ParquetFileFormat>();
 
   fmt->reader_options.use_buffered_stream = use_buffered_stream;
   fmt->reader_options.buffer_size = buffer_size;
 
-  auto dict_columns_vector = Rcpp::as<std::vector<std::string>>(dict_columns);
+  auto dict_columns_vector = cpp11::as_cpp<std::vector<std::string>>(dict_columns);
   auto& d = fmt->reader_options.dict_columns;
   std::move(dict_columns_vector.begin(), dict_columns_vector.end(),
             std::inserter(d, d.end()));
@@ -177,6 +195,39 @@ std::shared_ptr<ds::ParquetFileFormat> dataset___ParquetFileFormat__Make(
   return fmt;
 }
 
+// [[arrow::export]]
+std::string dataset___FileWriteOptions__type_name(
+    const std::shared_ptr<ds::FileWriteOptions>& options) {
+  return options->type_name();
+}
+
+// [[arrow::export]]
+void dataset___ParquetFileWriteOptions__update(
+    const std::shared_ptr<ds::ParquetFileWriteOptions>& options,
+    const std::shared_ptr<parquet::WriterProperties>& writer_props,
+    const std::shared_ptr<parquet::ArrowWriterProperties>& arrow_writer_props) {
+  options->writer_properties = writer_props;
+  options->arrow_writer_properties = arrow_writer_props;
+}
+
+// [[arrow::export]]
+void dataset___IpcFileWriteOptions__update2(
+    const std::shared_ptr<ds::IpcFileWriteOptions>& ipc_options, bool use_legacy_format,
+    const std::shared_ptr<arrow::util::Codec>& codec,
+    arrow::ipc::MetadataVersion metadata_version) {
+  ipc_options->options->write_legacy_ipc_format = use_legacy_format;
+  ipc_options->options->codec = codec;
+  ipc_options->options->metadata_version = metadata_version;
+}
+
+// [[arrow::export]]
+void dataset___IpcFileWriteOptions__update1(
+    const std::shared_ptr<ds::IpcFileWriteOptions>& ipc_options, bool use_legacy_format,
+    arrow::ipc::MetadataVersion metadata_version) {
+  ipc_options->options->write_legacy_ipc_format = use_legacy_format;
+  ipc_options->options->metadata_version = metadata_version;
+}
+
 // [[arrow::export]]
 std::shared_ptr<ds::IpcFileFormat> dataset___IpcFileFormat__Make() {
   return std::make_shared<ds::IpcFileFormat>();
@@ -295,6 +346,12 @@ std::vector<std::shared_ptr<ds::ScanTask>> dataset___Scanner__Scan(
   return out;
 }
 
+// [[arrow::export]]
+std::shared_ptr<arrow::Schema> dataset___Scanner__schema(
+    const std::shared_ptr<ds::Scanner>& sc) {
+  return sc->schema();
+}
+
 // [[arrow::export]]
 std::vector<std::shared_ptr<arrow::RecordBatch>> dataset___ScanTask__get_batches(
     const std::shared_ptr<ds::ScanTask>& scan_task) {
@@ -310,18 +367,18 @@ std::vector<std::shared_ptr<arrow::RecordBatch>> dataset___ScanTask__get_batches
 }
 
 // [[arrow::export]]
-void dataset___Dataset__Write(const std::shared_ptr<ds::Dataset>& ds,
-                              const std::shared_ptr<arrow::Schema>& schema,
-                              const std::shared_ptr<ds::FileFormat>& format,
-                              const std::shared_ptr<fs::FileSystem>& filesystem,
-                              std::string path,
-                              const std::shared_ptr<ds::Partitioning>& partitioning) {
-  auto frags = ds->GetFragments();
-  auto ctx = std::make_shared<ds::ScanContext>();
-  ctx->use_threads = true;
-  StopIfNotOk(ds::FileSystemDataset::Write(schema, format, filesystem, path, partitioning,
-                                           ctx, std::move(frags)));
-  return;
+void dataset___Dataset__Write(
+    const std::shared_ptr<ds::FileWriteOptions>& file_write_options,
+    const std::shared_ptr<fs::FileSystem>& filesystem, std::string base_dir,
+    const std::shared_ptr<ds::Partitioning>& partitioning, std::string basename_template,
+    const std::shared_ptr<ds::Scanner>& scanner) {
+  ds::FileSystemDatasetWriteOptions opts;
+  opts.file_write_options = file_write_options;
+  opts.filesystem = filesystem;
+  opts.base_dir = base_dir;
+  opts.partitioning = partitioning;
+  opts.basename_template = basename_template;
+  StopIfNotOk(ds::FileSystemDataset::Write(opts, scanner));
 }
 
 #endif
diff --git a/r/src/datatype.cpp b/r/src/datatype.cpp
index 3d6d0c9a753..81d2bd40dd7 100644
--- a/r/src/datatype.cpp
+++ b/r/src/datatype.cpp
@@ -17,18 +17,9 @@
 
 #include "./arrow_types.h"
 
-using Rcpp::CharacterVector;
-using Rcpp::List;
-using Rcpp::stop;
-using Rcpp::wrap;
-
 #if defined(ARROW_R_WITH_ARROW)
 #include <arrow/type.h>
 
-RCPP_EXPOSED_ENUM_NODECL(arrow::Type::type)
-RCPP_EXPOSED_ENUM_NODECL(arrow::DateUnit)
-RCPP_EXPOSED_ENUM_NODECL(arrow::TimeUnit::type)
-
 // [[arrow::export]]
 bool shared_ptr_is_null(SEXP xp) {
   return reinterpret_cast<std::shared_ptr<void>*>(R_ExternalPtrAddr(xp))->get() ==
@@ -108,12 +99,12 @@ std::shared_ptr<arrow::DataType> Decimal128Type__initialize(int32_t precision,
 }
 
 // [[arrow::export]]
-std::shared_ptr<arrow::DataType> FixedSizeBinary__initialize(int32_t byte_width) {
+std::shared_ptr<arrow::DataType> FixedSizeBinary__initialize(R_xlen_t byte_width) {
   if (byte_width == NA_INTEGER) {
-    Rcpp::stop("'byte_width' cannot be NA");
+    cpp11::stop("'byte_width' cannot be NA");
   }
   if (byte_width < 1) {
-    Rcpp::stop("'byte_width' must be > 0");
+    cpp11::stop("'byte_width' must be > 0");
   }
   return arrow::fixed_size_binary(byte_width);
 }
@@ -137,54 +128,55 @@ std::shared_ptr<arrow::DataType> Time64__initialize(arrow::TimeUnit::type unit)
 // [[arrow::export]]
 SEXP list__(SEXP x) {
   if (Rf_inherits(x, "Field")) {
-    Rcpp::ConstReferenceSmartPtrInputParameter<std::shared_ptr<arrow::Field>> field(x);
-    return wrap(arrow::list(field));
+    auto field = cpp11::as_cpp<std::shared_ptr<arrow::Field>>(x);
+    return cpp11::as_sexp(arrow::list(field));
   }
 
   if (Rf_inherits(x, "DataType")) {
-    Rcpp::ConstReferenceSmartPtrInputParameter<std::shared_ptr<arrow::DataType>> type(x);
-    return wrap(arrow::list(type));
+    auto type = cpp11::as_cpp<std::shared_ptr<arrow::DataType>>(x);
+    return cpp11::as_sexp(arrow::list(type));
   }
 
-  stop("incompatible");
+  cpp11::stop("incompatible");
   return R_NilValue;
 }
 
 // [[arrow::export]]
 SEXP large_list__(SEXP x) {
   if (Rf_inherits(x, "Field")) {
-    Rcpp::ConstReferenceSmartPtrInputParameter<std::shared_ptr<arrow::Field>> field(x);
-    return wrap(arrow::large_list(field));
+    auto field = cpp11::as_cpp<std::shared_ptr<arrow::Field>>(x);
+    return cpp11::as_sexp(arrow::large_list(field));
   }
 
   if (Rf_inherits(x, "DataType")) {
-    Rcpp::ConstReferenceSmartPtrInputParameter<std::shared_ptr<arrow::DataType>> type(x);
-    return wrap(arrow::large_list(type));
+    auto type = cpp11::as_cpp<std::shared_ptr<arrow::DataType>>(x);
+    return cpp11::as_sexp(arrow::large_list(type));
   }
 
-  stop("incompatible");
+  cpp11::stop("incompatible");
   return R_NilValue;
 }
 
 // [[arrow::export]]
 SEXP fixed_size_list__(SEXP x, int list_size) {
   if (Rf_inherits(x, "Field")) {
-    Rcpp::ConstReferenceSmartPtrInputParameter<std::shared_ptr<arrow::Field>> field(x);
-    return wrap(arrow::fixed_size_list(field, list_size));
+    auto field = cpp11::as_cpp<std::shared_ptr<arrow::Field>>(x);
+    return cpp11::as_sexp(arrow::fixed_size_list(field, list_size));
   }
 
   if (Rf_inherits(x, "DataType")) {
-    Rcpp::ConstReferenceSmartPtrInputParameter<std::shared_ptr<arrow::DataType>> type(x);
-    return wrap(arrow::fixed_size_list(type, list_size));
+    auto type = cpp11::as_cpp<std::shared_ptr<arrow::DataType>>(x);
+    return cpp11::as_sexp(arrow::fixed_size_list(type, list_size));
   }
 
-  stop("incompatible");
+  cpp11::stop("incompatible");
   return R_NilValue;
 }
 
 // [[arrow::export]]
-std::shared_ptr<arrow::DataType> struct_(List fields) {
-  return arrow::struct_(arrow::r::List_to_shared_ptr_vector<arrow::Field>(fields));
+std::shared_ptr<arrow::DataType> struct__(
+    const std::vector<std::shared_ptr<arrow::Field>>& fields) {
+  return arrow::struct_(fields);
 }
 
 // [[arrow::export]]
@@ -209,8 +201,9 @@ int DataType__num_children(const std::shared_ptr<arrow::DataType>& type) {
 }
 
 // [[arrow::export]]
-List DataType__children_pointer(const std::shared_ptr<arrow::DataType>& type) {
-  return List(type->fields().begin(), type->fields().end());
+cpp11::writable::list DataType__children_pointer(
+    const std::shared_ptr<arrow::DataType>& type) {
+  return arrow::r::to_r_list(type->fields());
 }
 
 // [[arrow::export]]
diff --git a/r/src/feather.cpp b/r/src/feather.cpp
index b66da47965d..22bdd7acde0 100644
--- a/r/src/feather.cpp
+++ b/r/src/feather.cpp
@@ -65,7 +65,7 @@ std::shared_ptr<arrow::Table> ipc___feather___Reader__Read(
       StopIfNotOk(reader->Read(&table));
       break;
     default:
-      Rcpp::stop("incompatible column specification");
+      cpp11::stop("incompatible column specification");
       break;
   }
 
@@ -79,15 +79,11 @@ std::shared_ptr<arrow::ipc::feather::Reader> ipc___feather___Reader__Open(
 }
 
 // [[arrow::export]]
-Rcpp::CharacterVector ipc___feather___Reader__column_names(
+cpp11::writable::strings ipc___feather___Reader__column_names(
     const std::shared_ptr<arrow::ipc::feather::Reader>& reader) {
-  auto sch = reader->schema();
-  int64_t n = sch->num_fields();
-  Rcpp::CharacterVector out(n);
-  for (int i = 0; i < n; i++) {
-    out[i] = sch->field(i)->name();
-  }
-  return out;
+  return arrow::r::to_r_strings(
+      reader->schema()->fields(),
+      [](const std::shared_ptr<arrow::Field>& field) { return field->name(); });
 }
 
 #endif
diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp
index ccead59f6cb..53959804fe8 100644
--- a/r/src/filesystem.cpp
+++ b/r/src/filesystem.cpp
@@ -16,6 +16,7 @@
 // under the License.
 
 #include "./arrow_types.h"
+
 #if defined(ARROW_R_WITH_ARROW)
 
 #include <arrow/filesystem/filesystem.h>
@@ -23,8 +24,6 @@
 
 namespace fs = ::arrow::fs;
 
-RCPP_EXPOSED_ENUM_NODECL(arrow::fs::FileType)
-
 // FileInfo
 
 // [[arrow::export]]
@@ -223,11 +222,34 @@ std::shared_ptr<fs::SubTreeFileSystem> fs___SubTreeFileSystem__create(
 }
 
 // [[arrow::export]]
-Rcpp::List fs___FileSystemFromUri(const std::string& path) {
+std::shared_ptr<fs::FileSystem> fs___SubTreeFileSystem__base_fs(
+    const std::shared_ptr<fs::SubTreeFileSystem>& file_system) {
+  return file_system->base_fs();
+}
+
+// [[arrow::export]]
+std::string fs___SubTreeFileSystem__base_path(
+    const std::shared_ptr<fs::SubTreeFileSystem>& file_system) {
+  return file_system->base_path();
+}
+
+// [[arrow::export]]
+cpp11::writable::list fs___FileSystemFromUri(const std::string& path) {
+  using cpp11::literals::operator"" _nm;
+
   std::string out_path;
   auto file_system = ValueOrStop(fs::FileSystemFromUri(path, &out_path));
-  return Rcpp::List::create(Rcpp::Named("fs") = file_system,
-                            Rcpp::Named("path") = out_path);
+  return cpp11::writable::list({"fs"_nm = file_system, "path"_nm = out_path});
+}
+
+// [[arrow::export]]
+void fs___CopyFiles(const std::shared_ptr<fs::FileSystem>& source_fs,
+                    const std::shared_ptr<fs::FileSelector>& source_sel,
+                    const std::shared_ptr<fs::FileSystem>& destination_fs,
+                    const std::string& destination_base_dir,
+                    int64_t chunk_size = 1024 * 1024, bool use_threads = true) {
+  StopIfNotOk(fs::CopyFiles(source_fs, *source_sel, destination_fs, destination_base_dir,
+                            chunk_size, use_threads));
 }
 
 #endif
@@ -237,12 +259,48 @@ Rcpp::List fs___FileSystemFromUri(const std::string& path) {
 #include <arrow/filesystem/s3fs.h>
 
 // [[s3::export]]
-void fs___EnsureS3Initialized() { StopIfNotOk(fs::EnsureS3Initialized()); }
+std::shared_ptr<fs::S3FileSystem> fs___S3FileSystem__create(
+    bool anonymous = false, std::string access_key = "", std::string secret_key = "",
+    std::string session_token = "", std::string role_arn = "",
+    std::string session_name = "", std::string external_id = "", int load_frequency = 900,
+    std::string region = "", std::string endpoint_override = "", std::string scheme = "",
+    bool background_writes = true) {
+  fs::S3Options s3_opts;
+  // Handle auth (anonymous, keys, default)
+  // (validation/internal coherence handled in R)
+  if (anonymous) {
+    s3_opts = fs::S3Options::Anonymous();
+  } else if (access_key != "" && secret_key != "") {
+    s3_opts = fs::S3Options::FromAccessKey(access_key, secret_key, session_token);
+  } else if (role_arn != "") {
+    s3_opts = fs::S3Options::FromAssumeRole(role_arn, session_name, external_id,
+                                            load_frequency);
+  } else {
+    s3_opts = fs::S3Options::Defaults();
+  }
+
+  // Now handle the rest of the options
+  /// AWS region to connect to (default determined by AWS SDK)
+  if (region != "") {
+    s3_opts.region = region;
+  }
+  /// If non-empty, override region with a connect string such as "localhost:9000"
+  s3_opts.endpoint_override = endpoint_override;
+  /// S3 connection transport, default "https"
+  if (scheme != "") {
+    s3_opts.scheme = scheme;
+  }
+  /// Whether OutputStream writes will be issued in the background, without blocking
+  /// default true
+  s3_opts.background_writes = background_writes;
+
+  StopIfNotOk(fs::EnsureS3Initialized());
+  return ValueOrStop(fs::S3FileSystem::Make(s3_opts));
+}
 
 // [[s3::export]]
-std::shared_ptr<fs::S3FileSystem> fs___S3FileSystem__create() {
-  auto opts = fs::S3Options::Defaults();
-  return ValueOrStop(fs::S3FileSystem::Make(opts));
+std::string fs___S3FileSystem__region(const std::shared_ptr<fs::S3FileSystem>& fs) {
+  return fs->region();
 }
 
 #endif
diff --git a/r/src/io.cpp b/r/src/io.cpp
index 763a653bee5..6a912dd7815 100644
--- a/r/src/io.cpp
+++ b/r/src/io.cpp
@@ -17,14 +17,10 @@
 
 #include "./arrow_types.h"
 
-using Rcpp::RawVector_;
-
 #if defined(ARROW_R_WITH_ARROW)
 #include <arrow/io/file.h>
 #include <arrow/io/memory.h>
 
-RCPP_EXPOSED_ENUM_NODECL(arrow::io::FileMode::type)
-
 // ------ arrow::io::Readable
 
 // [[arrow::export]]
@@ -115,7 +111,7 @@ void io___MemoryMappedFile__Resize(const std::shared_ptr<arrow::io::MemoryMapped
 // [[arrow::export]]
 std::shared_ptr<arrow::io::ReadableFile> io___ReadableFile__Open(
     const std::string& path) {
-  return ValueOrStop(arrow::io::ReadableFile::Open(path));
+  return ValueOrStop(arrow::io::ReadableFile::Open(path, gc_memory_pool()));
 }
 
 // ------ arrow::io::BufferReader
@@ -154,8 +150,8 @@ std::shared_ptr<arrow::io::FileOutputStream> io___FileOutputStream__Open(
 // [[arrow::export]]
 std::shared_ptr<arrow::io::BufferOutputStream> io___BufferOutputStream__Create(
     int64_t initial_capacity) {
-  return ValueOrStop(arrow::io::BufferOutputStream::Create(initial_capacity,
-                                                           arrow::default_memory_pool()));
+  return ValueOrStop(
+      arrow::io::BufferOutputStream::Create(initial_capacity, gc_memory_pool()));
 }
 
 // [[arrow::export]]
@@ -178,8 +174,8 @@ int64_t io___BufferOutputStream__Tell(
 
 // [[arrow::export]]
 void io___BufferOutputStream__Write(
-    const std::shared_ptr<arrow::io::BufferOutputStream>& stream, RawVector_ bytes) {
-  StopIfNotOk(stream->Write(bytes.begin(), bytes.size()));
+    const std::shared_ptr<arrow::io::BufferOutputStream>& stream, cpp11::raws bytes) {
+  StopIfNotOk(stream->Write(RAW(bytes), bytes.size()));
 }
 
 #endif
diff --git a/r/src/json.cpp b/r/src/json.cpp
index edeeb343cca..87d40623f6b 100644
--- a/r/src/json.cpp
+++ b/r/src/json.cpp
@@ -20,24 +20,22 @@
 
 #include <arrow/json/reader.h>
 
-using Rcpp::CharacterVector;
-using Rcpp::List_;
-
 // [[arrow::export]]
-std::shared_ptr<arrow::json::ReadOptions> json___ReadOptions__initialize(List_ options) {
+std::shared_ptr<arrow::json::ReadOptions> json___ReadOptions__initialize(bool use_threads,
+                                                                         int block_size) {
   auto res =
       std::make_shared<arrow::json::ReadOptions>(arrow::json::ReadOptions::Defaults());
-  res->use_threads = options["use_threads"];
-  res->block_size = options["block_size"];
+  res->use_threads = use_threads;
+  res->block_size = block_size;
   return res;
 }
 
 // [[arrow::export]]
 std::shared_ptr<arrow::json::ParseOptions> json___ParseOptions__initialize(
-    List_ options) {
+    bool newlines_in_values) {
   auto res =
       std::make_shared<arrow::json::ParseOptions>(arrow::json::ParseOptions::Defaults());
-  res->newlines_in_values = options["newlines_in_values"];
+  res->newlines_in_values = newlines_in_values;
   return res;
 }
 
@@ -46,7 +44,7 @@ std::shared_ptr<arrow::json::TableReader> json___TableReader__Make(
     const std::shared_ptr<arrow::io::InputStream>& input,
     const std::shared_ptr<arrow::json::ReadOptions>& read_options,
     const std::shared_ptr<arrow::json::ParseOptions>& parse_options) {
-  return ValueOrStop(arrow::json::TableReader::Make(arrow::default_memory_pool(), input,
+  return ValueOrStop(arrow::json::TableReader::Make(gc_memory_pool(), input,
                                                     *read_options, *parse_options));
 }
 
diff --git a/r/src/memorypool.cpp b/r/src/memorypool.cpp
index 1d475caf235..05b79dc3929 100644
--- a/r/src/memorypool.cpp
+++ b/r/src/memorypool.cpp
@@ -18,11 +18,55 @@
 #include "./arrow_types.h"
 #if defined(ARROW_R_WITH_ARROW)
 #include <arrow/memory_pool.h>
+#include <arrow/util/mutex.h>
+
+class GcMemoryPool : public arrow::MemoryPool {
+ public:
+  GcMemoryPool() : pool_(arrow::default_memory_pool()) {}
+
+  arrow::Status Allocate(int64_t size, uint8_t** out) override {
+    return GcAndTryAgain([&] { return pool_->Allocate(size, out); });
+  }
+
+  arrow::Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) override {
+    return GcAndTryAgain([&] { return pool_->Reallocate(old_size, new_size, ptr); });
+  }
+
+  void Free(uint8_t* buffer, int64_t size) override { pool_->Free(buffer, size); }
+
+  int64_t bytes_allocated() const override { return pool_->bytes_allocated(); }
+
+  int64_t max_memory() const override { return pool_->max_memory(); }
+
+  std::string backend_name() const override { return pool_->backend_name() + "-gc"; }
+
+ private:
+  template <typename Call>
+  arrow::Status GcAndTryAgain(const Call& call) {
+    if (call().ok()) {
+      return arrow::Status::OK();
+    } else {
+      auto lock = mutex_.Lock();
+
+      // ARROW-10080: Allocation may fail spuriously since the garbage collector is lazy.
+      // Force it to run then try again in case any reusable allocations have been freed.
+      static cpp11::function gc = cpp11::package("base")["gc"];
+      gc();
+    }
+    return call();
+  }
+
+  arrow::util::Mutex mutex_;
+  arrow::MemoryPool* pool_;
+};
+
+static GcMemoryPool g_pool;
+
+arrow::MemoryPool* gc_memory_pool() { return &g_pool; }
 
 // [[arrow::export]]
 std::shared_ptr<arrow::MemoryPool> MemoryPool__default() {
-  return std::shared_ptr<arrow::MemoryPool>(arrow::default_memory_pool(),
-                                            NoDelete<arrow::MemoryPool>());
+  return std::shared_ptr<GcMemoryPool>(&g_pool, [](...) {});
 }
 
 // [[arrow::export]]
diff --git a/r/src/message.cpp b/r/src/message.cpp
index 8ec5a3faab0..fd50007d93d 100644
--- a/r/src/message.cpp
+++ b/r/src/message.cpp
@@ -21,8 +21,6 @@
 #include <arrow/ipc/reader.h>
 #include <arrow/ipc/writer.h>
 
-RCPP_EXPOSED_ENUM_NODECL(arrow::ipc::MessageType)
-
 // [[arrow::export]]
 int64_t ipc___Message__body_length(const std::unique_ptr<arrow::ipc::Message>& message) {
   return message->body_length();
diff --git a/r/src/nameof.h b/r/src/nameof.h
new file mode 100644
index 00000000000..a46ac762869
--- /dev/null
+++ b/r/src/nameof.h
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <string>
+
+namespace arrow {
+namespace util {
+namespace detail {
+
+#ifdef _MSC_VER
+#define ARROW_PRETTY_FUNCTION __FUNCSIG__
+#else
+#define ARROW_PRETTY_FUNCTION __PRETTY_FUNCTION__
+#endif
+
+template <typename T>
+const char* raw() {
+  return ARROW_PRETTY_FUNCTION;
+}
+
+template <typename T>
+size_t raw_sizeof() {
+  return sizeof(ARROW_PRETTY_FUNCTION);
+}
+
+#undef ARROW_PRETTY_FUNCTION
+
+constexpr bool starts_with(char const* haystack, char const* needle) {
+  return needle[0] == '\0' ||
+         (haystack[0] == needle[0] && starts_with(haystack + 1, needle + 1));
+}
+
+constexpr size_t search(char const* haystack, char const* needle) {
+  return haystack[0] == '\0' || starts_with(haystack, needle)
+             ? 0
+             : search(haystack + 1, needle) + 1;
+}
+
+const size_t typename_prefix = search(raw<void>(), "void");
+
+template <typename T>
+size_t struct_class_prefix() {
+#ifdef _MSC_VER
+  return starts_with(raw<T>() + typename_prefix, "struct ")
+             ? 7
+             : starts_with(raw<T>() + typename_prefix, "class ") ? 6 : 0;
+#else
+  return 0;
+#endif
+}
+
+template <typename T>
+size_t typename_length() {
+  // raw_sizeof<T>() - raw_sizeof<void>() == (length of T's name) - strlen("void")
+  // (length of T's name) == raw_sizeof<T>() - raw_sizeof<void>() + strlen("void")
+  return raw_sizeof<T>() - raw_sizeof<void>() + 4;
+}
+
+template <typename T>
+const char* typename_begin() {
+  return raw<T>() + struct_class_prefix<T>() + typename_prefix;
+}
+
+}  // namespace detail
+
+template <typename T>
+std::string nameof() {
+  return {detail::typename_begin<T>(), detail::typename_length<T>()};
+}
+
+}  // namespace util
+}  // namespace arrow
diff --git a/r/src/parquet.cpp b/r/src/parquet.cpp
index 626e58c23af..6f8db31410f 100644
--- a/r/src/parquet.cpp
+++ b/r/src/parquet.cpp
@@ -24,8 +24,6 @@
 #include <parquet/arrow/writer.h>
 #include <parquet/exception.h>
 
-RCPP_EXPOSED_ENUM_NODECL(parquet::ParquetVersion::type)
-
 // [[arrow::export]]
 std::shared_ptr<parquet::ArrowReaderProperties>
 parquet___arrow___ArrowReaderProperties__Make(bool use_threads) {
@@ -64,7 +62,8 @@ std::shared_ptr<parquet::arrow::FileReader> parquet___arrow___FileReader__OpenFi
   std::unique_ptr<parquet::arrow::FileReader> reader;
   parquet::arrow::FileReaderBuilder builder;
   PARQUET_THROW_NOT_OK(builder.Open(file));
-  PARQUET_THROW_NOT_OK(builder.properties(*props)->Build(&reader));
+  PARQUET_THROW_NOT_OK(
+      builder.memory_pool(gc_memory_pool())->properties(*props)->Build(&reader));
   return std::move(reader);
 }
 
@@ -85,12 +84,67 @@ std::shared_ptr<arrow::Table> parquet___arrow___FileReader__ReadTable2(
   return table;
 }
 
+// [[arrow::export]]
+std::shared_ptr<arrow::Table> parquet___arrow___FileReader__ReadRowGroup1(
+    const std::shared_ptr<parquet::arrow::FileReader>& reader, int i) {
+  std::shared_ptr<arrow::Table> table;
+  PARQUET_THROW_NOT_OK(reader->ReadRowGroup(i, &table));
+  return table;
+}
+
+// [[arrow::export]]
+std::shared_ptr<arrow::Table> parquet___arrow___FileReader__ReadRowGroup2(
+    const std::shared_ptr<parquet::arrow::FileReader>& reader, int i,
+    const std::vector<int>& column_indices) {
+  std::shared_ptr<arrow::Table> table;
+  PARQUET_THROW_NOT_OK(reader->ReadRowGroup(i, column_indices, &table));
+  return table;
+}
+
+// [[arrow::export]]
+std::shared_ptr<arrow::Table> parquet___arrow___FileReader__ReadRowGroups1(
+    const std::shared_ptr<parquet::arrow::FileReader>& reader,
+    const std::vector<int>& row_groups) {
+  std::shared_ptr<arrow::Table> table;
+  PARQUET_THROW_NOT_OK(reader->ReadRowGroups(row_groups, &table));
+  return table;
+}
+
+// [[arrow::export]]
+std::shared_ptr<arrow::Table> parquet___arrow___FileReader__ReadRowGroups2(
+    const std::shared_ptr<parquet::arrow::FileReader>& reader,
+    const std::vector<int>& row_groups, const std::vector<int>& column_indices) {
+  std::shared_ptr<arrow::Table> table;
+  PARQUET_THROW_NOT_OK(reader->ReadRowGroups(row_groups, column_indices, &table));
+  return table;
+}
+
 // [[arrow::export]]
 int64_t parquet___arrow___FileReader__num_rows(
     const std::shared_ptr<parquet::arrow::FileReader>& reader) {
   return reader->parquet_reader()->metadata()->num_rows();
 }
 
+// [[arrow::export]]
+int parquet___arrow___FileReader__num_columns(
+    const std::shared_ptr<parquet::arrow::FileReader>& reader) {
+  return reader->parquet_reader()->metadata()->num_columns();
+}
+
+// [[arrow::export]]
+int parquet___arrow___FileReader__num_row_groups(
+    const std::shared_ptr<parquet::arrow::FileReader>& reader) {
+  return reader->num_row_groups();
+}
+
+// [[arrow::export]]
+std::shared_ptr<arrow::ChunkedArray> parquet___arrow___FileReader__ReadColumn(
+    const std::shared_ptr<parquet::arrow::FileReader>& reader, int i) {
+  std::shared_ptr<arrow::ChunkedArray> array;
+  PARQUET_THROW_NOT_OK(reader->ReadColumn(i - 1, &array));
+  return array;
+}
+
 namespace parquet {
 
 class WriterPropertiesBuilder : public WriterProperties::Builder {
@@ -142,7 +196,7 @@ void parquet___WriterProperties___Builder__version(
 // [[arrow::export]]
 void parquet___ArrowWriterProperties___Builder__set_compressions(
     const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder,
-    const std::vector<std::string>& paths, const Rcpp::IntegerVector& types) {
+    const std::vector<std::string>& paths, cpp11::integers types) {
   auto n = types.size();
   if (n == 1) {
     builder->compression(static_cast<arrow::Compression::type>(types[0]));
@@ -156,7 +210,7 @@ void parquet___ArrowWriterProperties___Builder__set_compressions(
 // [[arrow::export]]
 void parquet___ArrowWriterProperties___Builder__set_compression_levels(
     const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder,
-    const std::vector<std::string>& paths, const Rcpp::IntegerVector& levels) {
+    const std::vector<std::string>& paths, cpp11::integers levels) {
   auto n = levels.size();
   if (n == 1) {
     builder->compression_level(levels[0]);
@@ -170,10 +224,10 @@ void parquet___ArrowWriterProperties___Builder__set_compression_levels(
 // [[arrow::export]]
 void parquet___ArrowWriterProperties___Builder__set_use_dictionary(
     const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder,
-    const std::vector<std::string>& paths, const Rcpp::LogicalVector& use_dictionary) {
+    const std::vector<std::string>& paths, cpp11::logicals use_dictionary) {
   auto n = use_dictionary.size();
   if (n == 1) {
-    if (use_dictionary[0]) {
+    if (use_dictionary[0] == TRUE) {
       builder->enable_dictionary();
     } else {
       builder->disable_dictionary();
@@ -181,7 +235,7 @@ void parquet___ArrowWriterProperties___Builder__set_use_dictionary(
   } else {
     builder->disable_dictionary();
     for (decltype(n) i = 0; i < n; i++) {
-      if (use_dictionary[i]) {
+      if (use_dictionary[i] == TRUE) {
         builder->enable_dictionary(paths[i]);
       } else {
         builder->disable_dictionary(paths[i]);
@@ -193,10 +247,10 @@ void parquet___ArrowWriterProperties___Builder__set_use_dictionary(
 // [[arrow::export]]
 void parquet___ArrowWriterProperties___Builder__set_write_statistics(
     const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder,
-    const std::vector<std::string>& paths, const Rcpp::LogicalVector& write_statistics) {
+    const std::vector<std::string>& paths, cpp11::logicals write_statistics) {
   auto n = write_statistics.size();
   if (n == 1) {
-    if (write_statistics[0]) {
+    if (write_statistics[0] == TRUE) {
       builder->enable_statistics();
     } else {
       builder->disable_statistics();
@@ -204,7 +258,7 @@ void parquet___ArrowWriterProperties___Builder__set_write_statistics(
   } else {
     builder->disable_statistics();
     for (decltype(n) i = 0; i < n; i++) {
-      if (write_statistics[i]) {
+      if (write_statistics[i] == TRUE) {
         builder->enable_statistics(paths[i]);
       } else {
         builder->disable_statistics(paths[i]);
@@ -233,9 +287,8 @@ std::shared_ptr<parquet::arrow::FileWriter> parquet___arrow___ParquetFileWriter_
     const std::shared_ptr<parquet::WriterProperties>& properties,
     const std::shared_ptr<parquet::ArrowWriterProperties>& arrow_properties) {
   std::unique_ptr<parquet::arrow::FileWriter> writer;
-  PARQUET_THROW_NOT_OK(
-      parquet::arrow::FileWriter::Open(*schema, arrow::default_memory_pool(), sink,
-                                       properties, arrow_properties, &writer));
+  PARQUET_THROW_NOT_OK(parquet::arrow::FileWriter::Open(
+      *schema, gc_memory_pool(), sink, properties, arrow_properties, &writer));
   return std::move(writer);
 }
 
@@ -258,9 +311,8 @@ void parquet___arrow___WriteTable(
     const std::shared_ptr<arrow::io::OutputStream>& sink,
     const std::shared_ptr<parquet::WriterProperties>& properties,
     const std::shared_ptr<parquet::ArrowWriterProperties>& arrow_properties) {
-  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, arrow::default_memory_pool(),
-                                                  sink, table->num_rows(), properties,
-                                                  arrow_properties));
+  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(
+      *table, gc_memory_pool(), sink, table->num_rows(), properties, arrow_properties));
 }
 
 // [[arrow::export]]
diff --git a/r/src/py-to-r.cpp b/r/src/py-to-r.cpp
index 91b5ebc617d..d2ff13bc2f2 100644
--- a/r/src/py-to-r.cpp
+++ b/r/src/py-to-r.cpp
@@ -18,60 +18,56 @@
 #include "./arrow_types.h"
 
 #if defined(ARROW_R_WITH_ARROW)
-#include <arrow/c/bridge.h>
 
 // [[arrow::export]]
-std::shared_ptr<arrow::Array> ImportArray(uintptr_t array, uintptr_t schema) {
-  return ValueOrStop(arrow::ImportArray(reinterpret_cast<struct ArrowArray*>(array),
-                                        reinterpret_cast<struct ArrowSchema*>(schema)));
+std::shared_ptr<arrow::Array> ImportArray(arrow::r::Pointer<struct ArrowArray> array,
+                                          arrow::r::Pointer<struct ArrowSchema> schema) {
+  return ValueOrStop(arrow::ImportArray(array, schema));
 }
 
 // [[arrow::export]]
-std::shared_ptr<arrow::RecordBatch> ImportRecordBatch(uintptr_t array, uintptr_t schema) {
-  return ValueOrStop(
-      arrow::ImportRecordBatch(reinterpret_cast<struct ArrowArray*>(array),
-                               reinterpret_cast<struct ArrowSchema*>(schema)));
+std::shared_ptr<arrow::RecordBatch> ImportRecordBatch(
+    arrow::r::Pointer<struct ArrowArray> array,
+    arrow::r::Pointer<struct ArrowSchema> schema) {
+  return ValueOrStop(arrow::ImportRecordBatch(array, schema));
 }
 
 // [[arrow::export]]
-uintptr_t allocate_arrow_schema() { return reinterpret_cast<uintptr_t>(new ArrowSchema); }
+arrow::r::Pointer<struct ArrowSchema> allocate_arrow_schema() { return {}; }
 
 // [[arrow::export]]
-void delete_arrow_schema(uintptr_t ptr) {
-  delete reinterpret_cast<struct ArrowSchema*>(ptr);
-}
+void delete_arrow_schema(arrow::r::Pointer<struct ArrowSchema> ptr) { ptr.finalize(); }
 
 // [[arrow::export]]
-uintptr_t allocate_arrow_array() { return reinterpret_cast<uintptr_t>(new ArrowArray); }
+arrow::r::Pointer<struct ArrowArray> allocate_arrow_array() { return {}; }
 
 // [[arrow::export]]
-void delete_arrow_array(uintptr_t ptr) {
-  delete reinterpret_cast<struct ArrowArray*>(ptr);
-}
+void delete_arrow_array(arrow::r::Pointer<struct ArrowArray> ptr) { ptr.finalize(); }
 
 // [[arrow::export]]
-void ExportType(const std::shared_ptr<arrow::DataType>& type, uintptr_t ptr) {
-  StopIfNotOk(arrow::ExportType(*type, reinterpret_cast<struct ArrowSchema*>(ptr)));
+void ExportType(const std::shared_ptr<arrow::DataType>& type,
+                arrow::r::Pointer<struct ArrowSchema> ptr) {
+  StopIfNotOk(arrow::ExportType(*type, ptr));
 }
 
 // [[arrow::export]]
-void ExportSchema(const std::shared_ptr<arrow::Schema>& schema, uintptr_t ptr) {
-  StopIfNotOk(arrow::ExportSchema(*schema, reinterpret_cast<struct ArrowSchema*>(ptr)));
+void ExportSchema(const std::shared_ptr<arrow::Schema>& schema,
+                  arrow::r::Pointer<struct ArrowSchema> ptr) {
+  StopIfNotOk(arrow::ExportSchema(*schema, ptr));
 }
 
 // [[arrow::export]]
-void ExportArray(const std::shared_ptr<arrow::Array>& array, uintptr_t ptr,
-                 uintptr_t schema_ptr) {
-  StopIfNotOk(arrow::ExportArray(*array, reinterpret_cast<struct ArrowArray*>(ptr),
-                                 reinterpret_cast<struct ArrowSchema*>(schema_ptr)));
+void ExportArray(const std::shared_ptr<arrow::Array>& array,
+                 arrow::r::Pointer<struct ArrowArray> array_ptr,
+                 arrow::r::Pointer<struct ArrowSchema> schema_ptr) {
+  StopIfNotOk(arrow::ExportArray(*array, array_ptr, schema_ptr));
 }
 
 // [[arrow::export]]
-void ExportRecordBatch(const std::shared_ptr<arrow::RecordBatch>& batch, uintptr_t ptr,
-                       uintptr_t schema_ptr) {
-  StopIfNotOk(
-      arrow::ExportRecordBatch(*batch, reinterpret_cast<struct ArrowArray*>(ptr),
-                               reinterpret_cast<struct ArrowSchema*>(schema_ptr)));
+void ExportRecordBatch(const std::shared_ptr<arrow::RecordBatch>& batch,
+                       arrow::r::Pointer<ArrowArray> array_ptr,
+                       arrow::r::Pointer<ArrowSchema> schema_ptr) {
+  StopIfNotOk(arrow::ExportRecordBatch(*batch, array_ptr, schema_ptr));
 }
 
 #endif
diff --git a/r/src/recordbatch.cpp b/r/src/recordbatch.cpp
index 922bae2217e..02b61f60633 100644
--- a/r/src/recordbatch.cpp
+++ b/r/src/recordbatch.cpp
@@ -44,9 +44,11 @@ std::shared_ptr<arrow::Schema> RecordBatch__schema(
 
 // [[arrow::export]]
 std::shared_ptr<arrow::RecordBatch> RecordBatch__ReplaceSchemaMetadata(
-    const std::shared_ptr<arrow::RecordBatch>& x, Rcpp::CharacterVector metadata) {
-  auto kv = std::shared_ptr<arrow::KeyValueMetadata>(new arrow::KeyValueMetadata(
-      metadata.names(), Rcpp::as<std::vector<std::string>>(metadata)));
+    const std::shared_ptr<arrow::RecordBatch>& x, cpp11::strings metadata) {
+  auto vec_metadata = cpp11::as_cpp<std::vector<std::string>>(metadata);
+  auto names_metadata = cpp11::as_cpp<std::vector<std::string>>(metadata.names());
+  auto kv = std::shared_ptr<arrow::KeyValueMetadata>(
+      new arrow::KeyValueMetadata(names_metadata, vec_metadata));
   return x->ReplaceSchemaMetadata(kv);
 }
 
@@ -63,7 +65,7 @@ arrow::ArrayVector RecordBatch__columns(
 
 // [[arrow::export]]
 std::shared_ptr<arrow::Array> RecordBatch__column(
-    const std::shared_ptr<arrow::RecordBatch>& batch, int i) {
+    const std::shared_ptr<arrow::RecordBatch>& batch, R_xlen_t i) {
   arrow::r::validate_index(i, batch->num_columns());
   return batch->column(i);
 }
@@ -75,9 +77,8 @@ std::shared_ptr<arrow::Array> RecordBatch__GetColumnByName(
 }
 
 // [[arrow::export]]
-std::shared_ptr<arrow::RecordBatch> RecordBatch__select(
-    const std::shared_ptr<arrow::RecordBatch>& batch,
-    const Rcpp::IntegerVector& indices) {
+std::shared_ptr<arrow::RecordBatch> RecordBatch__SelectColumns(
+    const std::shared_ptr<arrow::RecordBatch>& batch, cpp11::integers indices) {
   R_xlen_t n = indices.size();
   auto nrows = batch->num_rows();
 
@@ -85,7 +86,7 @@ std::shared_ptr<arrow::RecordBatch> RecordBatch__select(
   std::vector<std::shared_ptr<arrow::Array>> columns(n);
 
   for (R_xlen_t i = 0; i < n; i++) {
-    int pos = indices[i] - 1;
+    int pos = indices[i];
     fields[i] = batch->schema()->field(pos);
     columns[i] = batch->column(pos);
   }
@@ -94,24 +95,6 @@ std::shared_ptr<arrow::RecordBatch> RecordBatch__select(
   return arrow::RecordBatch::Make(schema, nrows, columns);
 }
 
-// [[arrow::export]]
-std::shared_ptr<arrow::RecordBatch> RecordBatch__from_dataframe(Rcpp::DataFrame tbl) {
-  Rcpp::CharacterVector names = tbl.names();
-
-  std::vector<std::shared_ptr<arrow::Field>> fields;
-  std::vector<std::shared_ptr<arrow::Array>> arrays;
-
-  for (int i = 0; i < tbl.size(); i++) {
-    SEXP x = tbl[i];
-    arrays.push_back(Array__from_vector(x, R_NilValue));
-    fields.push_back(
-        std::make_shared<arrow::Field>(std::string(names[i]), arrays[i]->type()));
-  }
-  auto schema = std::make_shared<arrow::Schema>(std::move(fields));
-
-  return arrow::RecordBatch::Make(schema, tbl.nrow(), std::move(arrays));
-}
-
 // [[arrow::export]]
 bool RecordBatch__Equals(const std::shared_ptr<arrow::RecordBatch>& self,
                          const std::shared_ptr<arrow::RecordBatch>& other,
@@ -121,23 +104,23 @@ bool RecordBatch__Equals(const std::shared_ptr<arrow::RecordBatch>& self,
 
 // [[arrow::export]]
 std::shared_ptr<arrow::RecordBatch> RecordBatch__RemoveColumn(
-    const std::shared_ptr<arrow::RecordBatch>& batch, int i) {
+    const std::shared_ptr<arrow::RecordBatch>& batch, R_xlen_t i) {
   arrow::r::validate_index(i, batch->num_columns());
   return ValueOrStop(batch->RemoveColumn(i));
 }
 
 // [[arrow::export]]
 std::string RecordBatch__column_name(const std::shared_ptr<arrow::RecordBatch>& batch,
-                                     int i) {
+                                     R_xlen_t i) {
   arrow::r::validate_index(i, batch->num_columns());
   return batch->column_name(i);
 }
 
 // [[arrow::export]]
-Rcpp::CharacterVector RecordBatch__names(
+cpp11::writable::strings RecordBatch__names(
     const std::shared_ptr<arrow::RecordBatch>& batch) {
   int n = batch->num_columns();
-  Rcpp::CharacterVector names(n);
+  cpp11::writable::strings names(n);
   for (int i = 0; i < n; i++) {
     names[i] = batch->column_name(i);
   }
@@ -146,31 +129,31 @@ Rcpp::CharacterVector RecordBatch__names(
 
 // [[arrow::export]]
 std::shared_ptr<arrow::RecordBatch> RecordBatch__Slice1(
-    const std::shared_ptr<arrow::RecordBatch>& self, int offset) {
+    const std::shared_ptr<arrow::RecordBatch>& self, R_xlen_t offset) {
   arrow::r::validate_slice_offset(offset, self->num_rows());
   return self->Slice(offset);
 }
 
 // [[arrow::export]]
 std::shared_ptr<arrow::RecordBatch> RecordBatch__Slice2(
-    const std::shared_ptr<arrow::RecordBatch>& self, int offset, int length) {
+    const std::shared_ptr<arrow::RecordBatch>& self, R_xlen_t offset, R_xlen_t length) {
   arrow::r::validate_slice_offset(offset, self->num_rows());
   arrow::r::validate_slice_length(length, self->num_rows() - offset);
   return self->Slice(offset, length);
 }
 
 // [[arrow::export]]
-Rcpp::RawVector ipc___SerializeRecordBatch__Raw(
+cpp11::raws ipc___SerializeRecordBatch__Raw(
     const std::shared_ptr<arrow::RecordBatch>& batch) {
   // how many bytes do we need ?
   int64_t size;
   StopIfNotOk(arrow::ipc::GetRecordBatchSize(*batch, &size));
 
   // allocate the result raw vector
-  Rcpp::RawVector out(Rcpp::no_init(size));
+  cpp11::writable::raws out(size);
 
   // serialize into the bytes of the raw vector
-  auto buffer = std::make_shared<arrow::r::RBuffer<RAWSXP, Rcpp::RawVector>>(out);
+  auto buffer = std::make_shared<arrow::r::RBuffer<cpp11::raws>>(out);
   arrow::io::FixedSizeBufferWriter stream(buffer);
   StopIfNotOk(arrow::ipc::SerializeRecordBatch(
       *batch, arrow::ipc::IpcWriteOptions::Defaults(), &stream));
@@ -185,6 +168,7 @@ std::shared_ptr<arrow::RecordBatch> ipc___ReadRecordBatch__InputStream__Schema(
     const std::shared_ptr<arrow::Schema>& schema) {
   // TODO: promote to function arg
   arrow::ipc::DictionaryMemo memo;
+  StopIfNotOk(memo.fields().AddSchemaFields(*schema));
   return ValueOrStop(arrow::ipc::ReadRecordBatch(
       schema, &memo, arrow::ipc::IpcReadOptions::Defaults(), stream.get()));
 }
@@ -237,37 +221,22 @@ std::shared_ptr<arrow::RecordBatch> RecordBatch__from_arrays__known_schema(
   StopIfNotOk(arrow::r::count_fields(lst, &num_fields));
 
   if (schema->num_fields() != num_fields) {
-    Rcpp::stop("incompatible. schema has %d fields, and %d arrays are supplied",
-               schema->num_fields(), num_fields);
+    cpp11::stop("incompatible. schema has %d fields, and %d arrays are supplied",
+                schema->num_fields(), num_fields);
   }
 
   // convert lst to a vector of arrow::Array
   std::vector<std::shared_ptr<arrow::Array>> arrays(num_fields);
-  SEXP names = Rf_getAttrib(lst, R_NamesSymbol);
 
-  auto fill_array = [&arrays, &schema](int j, SEXP x, SEXP name) {
-    name = Rf_mkCharCE(Rf_translateCharUTF8(name), CE_UTF8);
-    if (schema->field(j)->name() != CHAR(name)) {
-      Rcpp::stop("field at index %d has name '%s' != '%s'", j + 1,
-                 schema->field(j)->name(), CHAR(name));
+  auto fill_array = [&arrays, &schema](int j, SEXP x, std::string name) {
+    if (schema->field(j)->name() != name) {
+      cpp11::stop("field at index %d has name '%s' != '%s'", j + 1,
+                  schema->field(j)->name().c_str(), name.c_str());
     }
     arrays[j] = arrow::r::Array__from_vector(x, schema->field(j)->type(), false);
   };
 
-  for (R_xlen_t i = 0, j = 0; j < num_fields; i++) {
-    SEXP name_i = STRING_ELT(names, i);
-    SEXP x_i = VECTOR_ELT(lst, i);
-
-    if (LENGTH(name_i) == 0) {
-      SEXP names_x_i = Rf_getAttrib(x_i, R_NamesSymbol);
-      for (R_xlen_t k = 0; k < XLENGTH(x_i); k++, j++) {
-        fill_array(j, VECTOR_ELT(x_i, k), STRING_ELT(names_x_i, k));
-      }
-    } else {
-      fill_array(j, x_i, name_i);
-      j++;
-    }
-  }
+  arrow::r::TraverseDots(lst, num_fields, fill_array);
 
   int64_t num_rows = 0;
   StopIfNotOk(arrow::r::check_consistent_array_size(arrays, &num_rows));
@@ -280,7 +249,7 @@ namespace r {
 arrow::Status CollectRecordBatchArrays(
     SEXP lst, const std::shared_ptr<arrow::Schema>& schema, int num_fields, bool inferred,
     std::vector<std::shared_ptr<arrow::Array>>& arrays) {
-  auto extract_one_array = [&arrays, &schema, inferred](int j, SEXP x, SEXP name) {
+  auto extract_one_array = [&arrays, &schema, inferred](int j, SEXP x, cpp11::r_string) {
     arrays[j] = arrow::r::Array__from_vector(x, schema->field(j)->type(), inferred);
   };
   arrow::r::TraverseDots(lst, num_fields, extract_one_array);
diff --git a/r/src/recordbatchreader.cpp b/r/src/recordbatchreader.cpp
index 93b54251148..7ecb42002a9 100644
--- a/r/src/recordbatchreader.cpp
+++ b/r/src/recordbatchreader.cpp
@@ -78,7 +78,7 @@ int ipc___RecordBatchFileReader__num_record_batches(
 std::shared_ptr<arrow::RecordBatch> ipc___RecordBatchFileReader__ReadRecordBatch(
     const std::shared_ptr<arrow::ipc::RecordBatchFileReader>& reader, int i) {
   if (i < 0 && i >= reader->num_record_batches()) {
-    Rcpp::stop("Record batch index out of bounds");
+    cpp11::stop("Record batch index out of bounds");
   }
   return ValueOrStop(reader->ReadRecordBatch(i));
 }
diff --git a/r/src/recordbatchwriter.cpp b/r/src/recordbatchwriter.cpp
index 9d038dd42ba..4714c1d104a 100644
--- a/r/src/recordbatchwriter.cpp
+++ b/r/src/recordbatchwriter.cpp
@@ -20,8 +20,6 @@
 #if defined(ARROW_R_WITH_ARROW)
 #include <arrow/ipc/writer.h>
 
-RCPP_EXPOSED_ENUM_NODECL(arrow::ipc::MetadataVersion)
-
 // [[arrow::export]]
 void ipc___RecordBatchWriter__WriteRecordBatch(
     const std::shared_ptr<arrow::ipc::RecordBatchWriter>& batch_writer,
@@ -50,7 +48,7 @@ std::shared_ptr<arrow::ipc::RecordBatchWriter> ipc___RecordBatchFileWriter__Open
   auto options = arrow::ipc::IpcWriteOptions::Defaults();
   options.write_legacy_ipc_format = use_legacy_format;
   options.metadata_version = metadata_version;
-  return ValueOrStop(arrow::ipc::NewFileWriter(stream.get(), schema, options));
+  return ValueOrStop(arrow::ipc::MakeFileWriter(stream, schema, options));
 }
 
 // [[arrow::export]]
@@ -61,7 +59,7 @@ std::shared_ptr<arrow::ipc::RecordBatchWriter> ipc___RecordBatchStreamWriter__Op
   auto options = arrow::ipc::IpcWriteOptions::Defaults();
   options.write_legacy_ipc_format = use_legacy_format;
   options.metadata_version = metadata_version;
-  return ValueOrStop(NewStreamWriter(stream.get(), schema, options));
+  return ValueOrStop(MakeStreamWriter(stream, schema, options));
 }
 
 #endif
diff --git a/r/src/scalar.cpp b/r/src/scalar.cpp
index 013ff1e6026..d9a3b569c36 100644
--- a/r/src/scalar.cpp
+++ b/r/src/scalar.cpp
@@ -54,7 +54,7 @@ std::shared_ptr<arrow::Scalar> StructScalar__GetFieldByName(
 
 // [[arrow::export]]
 SEXP Scalar__as_vector(const std::shared_ptr<arrow::Scalar>& scalar) {
-  auto array = ValueOrStop(arrow::MakeArrayFromScalar(*scalar, 1));
+  auto array = ValueOrStop(arrow::MakeArrayFromScalar(*scalar, 1, gc_memory_pool()));
 
   // defined in array_to_vector.cpp
   SEXP Array__as_vector(const std::shared_ptr<arrow::Array>& array);
diff --git a/r/src/schema.cpp b/r/src/schema.cpp
index 15e7027b815..7cb9a02eeab 100644
--- a/r/src/schema.cpp
+++ b/r/src/schema.cpp
@@ -18,14 +18,14 @@
 #include "./arrow_types.h"
 
 #if defined(ARROW_R_WITH_ARROW)
-#include <arrow/ipc/reader.h>
 #include <arrow/ipc/writer.h>
 #include <arrow/type.h>
 #include <arrow/util/key_value_metadata.h>
 
 // [[arrow::export]]
-std::shared_ptr<arrow::Schema> schema_(Rcpp::List fields) {
-  return arrow::schema(arrow::r::List_to_shared_ptr_vector<arrow::Field>(fields));
+std::shared_ptr<arrow::Schema> schema_(
+    const std::vector<std::shared_ptr<arrow::Field>>& fields) {
+  return arrow::schema(fields);
 }
 
 // [[arrow::export]]
@@ -42,7 +42,7 @@ int Schema__num_fields(const std::shared_ptr<arrow::Schema>& s) {
 std::shared_ptr<arrow::Field> Schema__field(const std::shared_ptr<arrow::Schema>& s,
                                             int i) {
   if (i >= s->num_fields() || i < 0) {
-    Rcpp::stop("Invalid field index for schema.");
+    cpp11::stop("Invalid field index for schema.");
   }
 
   return s->field(i);
@@ -72,47 +72,44 @@ bool Schema__HasMetadata(const std::shared_ptr<arrow::Schema>& schema) {
 }
 
 // [[arrow::export]]
-Rcpp::List Schema__metadata(const std::shared_ptr<arrow::Schema>& schema) {
+cpp11::writable::list Schema__metadata(const std::shared_ptr<arrow::Schema>& schema) {
   auto meta = schema->metadata();
   int64_t n = 0;
   if (schema->HasMetadata()) {
     n = meta->size();
   }
 
-  Rcpp::List out(n);
+  cpp11::writable::list out(n);
   std::vector<std::string> names_out(n);
 
   for (int i = 0; i < n; i++) {
     auto key = meta->key(i);
-    out[i] = meta->value(i);
+    out[i] = cpp11::as_sexp(meta->value(i));
     if (key == "r") {
-      Rf_setAttrib(out[i], R_ClassSymbol, arrow::r::data::classes_metadata_r);
+      Rf_classgets(out[i], arrow::r::data::classes_metadata_r);
     }
     names_out[i] = key;
   }
-  out.attr("names") = names_out;
+  out.names() = names_out;
   return out;
 }
 
 // [[arrow::export]]
 std::shared_ptr<arrow::Schema> Schema__WithMetadata(
-    const std::shared_ptr<arrow::Schema>& schema, Rcpp::CharacterVector metadata) {
-  auto kv = std::shared_ptr<arrow::KeyValueMetadata>(new arrow::KeyValueMetadata(
-      metadata.names(), Rcpp::as<std::vector<std::string>>(metadata)));
-  return schema->WithMetadata(kv);
+    const std::shared_ptr<arrow::Schema>& schema, cpp11::strings metadata) {
+  auto values = cpp11::as_cpp<std::vector<std::string>>(metadata);
+  auto names = cpp11::as_cpp<std::vector<std::string>>(metadata.attr("names"));
+
+  auto kv =
+      std::make_shared<arrow::KeyValueMetadata>(std::move(names), std::move(values));
+  return schema->WithMetadata(std::move(kv));
 }
 
 // [[arrow::export]]
-Rcpp::RawVector Schema__serialize(const std::shared_ptr<arrow::Schema>& schema) {
-  arrow::ipc::DictionaryMemo empty_memo;
-  std::shared_ptr<arrow::Buffer> out =
-      ValueOrStop(arrow::ipc::SerializeSchema(*schema, &empty_memo));
-
+cpp11::writable::raws Schema__serialize(const std::shared_ptr<arrow::Schema>& schema) {
+  auto out = ValueOrStop(arrow::ipc::SerializeSchema(*schema));
   auto n = out->size();
-  Rcpp::RawVector vec(out->size());
-  std::copy_n(out->data(), n, vec.begin());
-
-  return vec;
+  return cpp11::writable::raws(out->data(), out->data() + n);
 }
 
 // [[arrow::export]]
diff --git a/r/src/symbols.cpp b/r/src/symbols.cpp
index f087f82ccad..abb9cd49576 100644
--- a/r/src/symbols.cpp
+++ b/r/src/symbols.cpp
@@ -30,6 +30,7 @@ SEXP symbols::as_list = Rf_install("as.list");
 SEXP symbols::ptype = Rf_install("ptype");
 SEXP symbols::byte_width = Rf_install("byte_width");
 SEXP symbols::list_size = Rf_install("list_size");
+SEXP symbols::arrow_attributes = Rf_install("arrow_attributes");
 
 // persistently protect `x` and return it
 SEXP precious(SEXP x) {
@@ -37,58 +38,46 @@ SEXP precious(SEXP x) {
   return x;
 }
 
-// return R string vector, e.g.
-// strings({"foo", "bar"}) returns a size 2 STRSXP
-SEXP strings(std::initializer_list<std::string> list) {
-  size_t n = list.size();
-  SEXP s = PROTECT(Rf_allocVector(STRSXP, n));
-
-  auto it = list.begin();
-  for (size_t i = 0; i < n; i++, ++it) {
-    SET_STRING_ELT(s, i, Rf_mkCharLen(it->c_str(), it->size()));
-  }
-
-  UNPROTECT(1);
-  return s;
-}
-
 // returns the namespace environment for package `name`
 SEXP r_namespace(std::string name) {
-  SEXP s_name = PROTECT(strings({name}));
+  SEXP s_name = PROTECT(cpp11::writable::strings({name}));
   SEXP ns = R_FindNamespace(s_name);
   UNPROTECT(1);
   return ns;
 }
-SEXP data::classes_POSIXct = precious(strings({"POSIXct", "POSIXt"}));
-SEXP data::classes_metadata_r = precious(strings({"arrow_r_metadata"}));
+SEXP data::classes_POSIXct = precious(cpp11::writable::strings({"POSIXct", "POSIXt"}));
+SEXP data::classes_metadata_r = precious(cpp11::writable::strings({"arrow_r_metadata"}));
 SEXP data::classes_vctrs_list_of =
-    precious(strings({"vctrs_list_of", "vctrs_vctr", "list"}));
+    precious(cpp11::writable::strings({"vctrs_list_of", "vctrs_vctr", "list"}));
+SEXP data::classes_tbl_df =
+    precious(cpp11::writable::strings({"tbl_df", "tbl", "data.frame"}));
 
 SEXP data::classes_arrow_binary =
-    precious(strings({"arrow_binary", "vctrs_vctr", "list"}));
+    precious(cpp11::writable::strings({"arrow_binary", "vctrs_vctr", "list"}));
 SEXP data::classes_arrow_large_binary =
-    precious(strings({"arrow_large_binary", "vctrs_vctr", "list"}));
+    precious(cpp11::writable::strings({"arrow_large_binary", "vctrs_vctr", "list"}));
 SEXP data::classes_arrow_fixed_size_binary =
-    precious(strings({"arrow_fixed_size_binary", "vctrs_vctr", "list"}));
-SEXP data::classes_factor = precious(strings({"factor"}));
-SEXP data::classes_ordered = precious(strings({"ordered", "factor"}));
+    precious(cpp11::writable::strings({"arrow_fixed_size_binary", "vctrs_vctr", "list"}));
+SEXP data::classes_factor = precious(cpp11::writable::strings({"factor"}));
+SEXP data::classes_ordered = precious(cpp11::writable::strings({"ordered", "factor"}));
 
-SEXP data::classes_arrow_list =
-    precious(strings({"arrow_list", "vctrs_list_of", "vctrs_vctr", "list"}));
-SEXP data::classes_arrow_large_list =
-    precious(strings({"arrow_large_list", "vctrs_list_of", "vctrs_vctr", "list"}));
-SEXP data::classes_arrow_fixed_size_list =
-    precious(strings({"arrow_fixed_size_list", "vctrs_list_of", "vctrs_vctr", "list"}));
+SEXP data::classes_arrow_list = precious(
+    cpp11::writable::strings({"arrow_list", "vctrs_list_of", "vctrs_vctr", "list"}));
+SEXP data::classes_arrow_large_list = precious(cpp11::writable::strings(
+    {"arrow_large_list", "vctrs_list_of", "vctrs_vctr", "list"}));
+SEXP data::classes_arrow_fixed_size_list = precious(cpp11::writable::strings(
+    {"arrow_fixed_size_list", "vctrs_list_of", "vctrs_vctr", "list"}));
 
-SEXP data::names_metadata = precious(strings({"attributes", "columns"}));
+SEXP data::names_metadata = precious(cpp11::writable::strings({"attributes", "columns"}));
 SEXP data::empty_raw = precious(Rf_allocVector(RAWSXP, 0));
 
 SEXP ns::arrow = precious(r_namespace("arrow"));
 
 void inspect(SEXP obj) {
-  Rcpp::Shield<SEXP> call_inspect(Rf_lang2(symbols::inspect, obj));
-  Rcpp::Shield<SEXP> call_internal(Rf_lang2(symbols::dot_Internal, call_inspect));
+  SEXP call_inspect = PROTECT(Rf_lang2(symbols::inspect, obj));
+  SEXP call_internal = PROTECT(Rf_lang2(symbols::dot_Internal, call_inspect));
   Rf_eval(call_internal, R_GlobalEnv);
+  UNPROTECT(2);
 }
 
 }  // namespace r
diff --git a/r/src/table.cpp b/r/src/table.cpp
index d9d1efcaa96..97da768aa59 100644
--- a/r/src/table.cpp
+++ b/r/src/table.cpp
@@ -22,15 +22,6 @@
 #include <arrow/table.h>
 #include <arrow/util/key_value_metadata.h>
 
-using Rcpp::DataFrame;
-
-// [[arrow::export]]
-std::shared_ptr<arrow::Table> Table__from_dataframe(DataFrame tbl) {
-  auto rb = RecordBatch__from_dataframe(tbl);
-
-  return ValueOrStop(arrow::Table::FromRecordBatches({std::move(rb)}));
-}
-
 // [[arrow::export]]
 int Table__num_columns(const std::shared_ptr<arrow::Table>& x) {
   return x->num_columns();
@@ -46,22 +37,24 @@ std::shared_ptr<arrow::Schema> Table__schema(const std::shared_ptr<arrow::Table>
 
 // [[arrow::export]]
 std::shared_ptr<arrow::Table> Table__ReplaceSchemaMetadata(
-    const std::shared_ptr<arrow::Table>& x, Rcpp::CharacterVector metadata) {
-  auto kv = std::shared_ptr<arrow::KeyValueMetadata>(new arrow::KeyValueMetadata(
-      metadata.names(), Rcpp::as<std::vector<std::string>>(metadata)));
+    const std::shared_ptr<arrow::Table>& x, cpp11::strings metadata) {
+  auto vec_metadata = cpp11::as_cpp<std::vector<std::string>>(metadata);
+  auto names_metadata = cpp11::as_cpp<std::vector<std::string>>(metadata.names());
+  auto kv = std::shared_ptr<arrow::KeyValueMetadata>(
+      new arrow::KeyValueMetadata(names_metadata, vec_metadata));
   return x->ReplaceSchemaMetadata(kv);
 }
 
 // [[arrow::export]]
 std::shared_ptr<arrow::ChunkedArray> Table__column(
-    const std::shared_ptr<arrow::Table>& table, int i) {
+    const std::shared_ptr<arrow::Table>& table, R_xlen_t i) {
   arrow::r::validate_index(i, table->num_columns());
   return table->column(i);
 }
 
 // [[arrow::export]]
 std::shared_ptr<arrow::Field> Table__field(const std::shared_ptr<arrow::Table>& table,
-                                           int i) {
+                                           R_xlen_t i) {
   arrow::r::validate_index(i, table->num_columns());
   return table->field(i);
 }
@@ -84,14 +77,14 @@ std::vector<std::string> Table__ColumnNames(const std::shared_ptr<arrow::Table>&
 
 // [[arrow::export]]
 std::shared_ptr<arrow::Table> Table__Slice1(const std::shared_ptr<arrow::Table>& table,
-                                            int offset) {
+                                            R_xlen_t offset) {
   arrow::r::validate_slice_offset(offset, table->num_rows());
   return table->Slice(offset);
 }
 
 // [[arrow::export]]
 std::shared_ptr<arrow::Table> Table__Slice2(const std::shared_ptr<arrow::Table>& table,
-                                            int offset, int length) {
+                                            R_xlen_t offset, R_xlen_t length) {
   arrow::r::validate_slice_offset(offset, table->num_rows());
   arrow::r::validate_slice_length(length, table->num_rows() - offset);
   return table->Slice(offset, length);
@@ -122,29 +115,9 @@ std::shared_ptr<arrow::ChunkedArray> Table__GetColumnByName(
 }
 
 // [[arrow::export]]
-std::shared_ptr<arrow::Table> Table__select(const std::shared_ptr<arrow::Table>& table,
-                                            const Rcpp::IntegerVector& indices) {
-  R_xlen_t n = indices.size();
-
-  std::vector<std::shared_ptr<arrow::Field>> fields(n);
-  std::vector<std::shared_ptr<arrow::ChunkedArray>> columns(n);
-
-  for (R_xlen_t i = 0; i < n; i++) {
-    int pos = indices[i] - 1;
-    fields[i] = table->schema()->field(pos);
-    columns[i] = table->column(pos);
-  }
-
-  auto schema = std::make_shared<arrow::Schema>(std::move(fields));
-  return arrow::Table::Make(schema, columns);
-}
-
-bool all_record_batches(SEXP lst) {
-  R_xlen_t n = XLENGTH(lst);
-  for (R_xlen_t i = 0; i < n; i++) {
-    if (!Rf_inherits(VECTOR_ELT(lst, i), "RecordBatch")) return false;
-  }
-  return true;
+std::shared_ptr<arrow::Table> Table__SelectColumns(
+    const std::shared_ptr<arrow::Table>& table, const std::vector<int>& indices) {
+  return ValueOrStop(table->SelectColumns(indices));
 }
 
 namespace arrow {
@@ -154,7 +127,7 @@ arrow::Status InferSchemaFromDots(SEXP lst, SEXP schema_sxp, int num_fields,
                                   std::shared_ptr<arrow::Schema>& schema) {
   // maybe a schema was given
   if (Rf_inherits(schema_sxp, "Schema")) {
-    schema = arrow::r::extract<arrow::Schema>(schema_sxp);
+    schema = cpp11::as_cpp<std::shared_ptr<arrow::Schema>>(schema_sxp);
     return arrow::Status::OK();
   }
 
@@ -165,17 +138,16 @@ arrow::Status InferSchemaFromDots(SEXP lst, SEXP schema_sxp, int num_fields,
   // infer the schema from the `...`
   std::vector<std::shared_ptr<arrow::Field>> fields(num_fields);
 
-  auto extract_one_field = [&fields](int j, SEXP x, SEXP name) {
-    // Make sure we're ingesting UTF-8
-    name = Rf_mkCharCE(Rf_translateCharUTF8(name), CE_UTF8);
+  auto extract_one_field = [&fields](int j, SEXP x, std::string name) {
     if (Rf_inherits(x, "ChunkedArray")) {
-      fields[j] =
-          arrow::field(CHAR(name), arrow::r::extract<arrow::ChunkedArray>(x)->type());
+      fields[j] = arrow::field(
+          name, cpp11::as_cpp<std::shared_ptr<arrow::ChunkedArray>>(x)->type());
     } else if (Rf_inherits(x, "Array")) {
-      fields[j] = arrow::field(CHAR(name), arrow::r::extract<arrow::Array>(x)->type());
+      fields[j] =
+          arrow::field(name, cpp11::as_cpp<std::shared_ptr<arrow::Array>>(x)->type());
     } else {
       // TODO: we just need the type at this point
-      fields[j] = arrow::field(CHAR(name), arrow::r::InferArrowType(x));
+      fields[j] = arrow::field(name, arrow::r::InferArrowType(x));
     }
   };
   arrow::r::TraverseDots(lst, num_fields, extract_one_field);
@@ -185,126 +157,61 @@ arrow::Status InferSchemaFromDots(SEXP lst, SEXP schema_sxp, int num_fields,
   return arrow::Status::OK();
 }
 
+SEXP arrow_attributes(SEXP x, bool only_top_level) {
+  SEXP call = PROTECT(
+      Rf_lang3(arrow::r::symbols::arrow_attributes, x, Rf_ScalarLogical(only_top_level)));
+  SEXP att = Rf_eval(call, arrow::r::ns::arrow);
+  UNPROTECT(1);
+  return att;
+}
+
 SEXP CollectColumnMetadata(SEXP lst, int num_fields, bool& has_metadata) {
   // Preallocate for the lambda to fill in
-  SEXP metadata_columns = PROTECT(Rf_allocVector(VECSXP, num_fields));
-  SEXP metadata_columns_names = PROTECT(Rf_allocVector(STRSXP, num_fields));
-  Rf_setAttrib(metadata_columns, R_NamesSymbol, metadata_columns_names);
+  cpp11::writable::list metadata_columns(num_fields);
+
+  cpp11::writable::strings metadata_columns_names(num_fields);
 
   auto extract_one_metadata = [&metadata_columns, &metadata_columns_names, &has_metadata](
-                                  int j, SEXP x, SEXP name) {
-    // Make sure we're ingesting UTF-8
-    name = Rf_mkCharCE(Rf_translateCharUTF8(name), CE_UTF8);
-    SET_STRING_ELT(metadata_columns_names, j, name);
+                                  int j, SEXP x, std::string name) {
+    metadata_columns_names[j] = name;
+
     // no metadata for arrow R6 objects
     if (Rf_inherits(x, "ArrowObject")) {
       return;
     }
+    metadata_columns[j] = arrow_attributes(x, false);
 
-    bool this_has_metadata = false;
-    SEXP att = ATTRIB(x);
-    if (!Rf_isNull(att) || Rf_inherits(x, "data.frame")) {
-      // Each field in columns is also: list(attributes=list(), columns=namedList(fields))
-      // Only nested types will have columns though
-      SEXP r_meta = PROTECT(Rf_allocVector(VECSXP, 2));
-      Rf_setAttrib(r_meta, R_NamesSymbol, arrow::r::data::names_metadata);
-      if (!Rf_isNull(att)) {
-        SEXP att_list =
-            PROTECT(Rf_eval(Rf_lang2(arrow::r::symbols::as_list, att), R_GlobalEnv));
-
-        // Pop off attributes that are already preserved in the Arrow types
-        std::vector<std::string> bad_fields = {};
-        auto class_attr = Rf_getAttrib(x, R_ClassSymbol);
-        if (Rf_length(class_attr) == 1) {
-          std::string class_name(CHAR(STRING_ELT(class_attr, 0)));
-          if (class_name == "factor") {
-            bad_fields = {"class", "levels"};
-          } else if (class_name == "Date" || class_name == "integer64") {
-            bad_fields = {"class"};
-          } else if (class_name == "data.frame") {
-            bad_fields = {"class", "names", "row.names"};  // TODO: preserve row names?
-          }
-        } else if (Rf_inherits(x, "tbl_df")) {
-          // TODO: what about subclass of tibble?
-          bad_fields = {"class", "names", "row.names"};
-        } else if (Rf_inherits(x, "data.frame")) {
-          bad_fields = {"names", "row.names"};  // TODO: preserve row names?
-        } else if (Rf_inherits(x, "POSIXct")) {
-          // Note that "tzone" is optional so it may not exist
-          bad_fields = {"class", "tzone"};
-        } else if (Rf_inherits(x, "hms") && Rf_inherits(x, "difftime")) {
-          bad_fields = {"class", "units"};
-        }
-        if (Rf_length(att_list) > (int)bad_fields.size()) {
-          // If the fields we should exclude are the only ones we have,
-          // there's nothing to do. Otherwise, set what we have.
-          SET_VECTOR_ELT(r_meta, 0, att_list);
-          this_has_metadata = true;
-          // TODO: We could do something like this to just drop those fields
-          // if (bad_fields.size() > 0) {
-          //   // Make a new list without them
-          //   R_xlen_t new_size = Rf_length(att_list) - bad_fields.size();
-          //   SEXP new_list = PROTECT(Rf_allocVector(VECSXP, new_size));
-          //   SEXP new_list_names = PROTECT(Rf_allocVector(STRSXP, new_size));
-          //   Rf_setAttrib(new_list, R_NamesSymbol, new_list_names);
-          //
-          //   SEXP att_list_names = Rf_getAttrib(att_list, R_NamesSymbol);
-          //   SEXP old_name;
-          //   R_xlen_t new_i = 0;
-          //   for (R_xlen_t name_i = 0; name_i < Rf_length(att_list_names); name_i++) {
-          //     old_name = STRING_ELT(att_list_names, name_i);
-          //     if (old_name not in bad_fields) {  // TODO, obviously
-          //       SET_VECTOR_ELT(new_list, new_i, VECTOR_ELT(att_list, name_i));
-          //       SET_STRING_ELT(new_list_names, new_i, old_name);
-          //       new_i++;
-          //     }
-          //   }
-          //   att_list = new_list;
-          //   UNPROTECT(2);
-          // }
-        }
-        UNPROTECT(1);
-      }
-      if (Rf_inherits(x, "data.frame")) {
-        int inner_num_fields;
-        StopIfNotOk(arrow::r::count_fields(x, &inner_num_fields));
-        SET_VECTOR_ELT(r_meta, 1,
-                       CollectColumnMetadata(x, inner_num_fields, has_metadata));
-        UNPROTECT(2);  // CollectColumnMetadata adds 2 PROTECTS
-        this_has_metadata = true;
-      }
-      if (this_has_metadata) {
-        SET_VECTOR_ELT(metadata_columns, j, r_meta);
-        has_metadata = true;
-      }
-      UNPROTECT(1);
+    if (!Rf_isNull(metadata_columns[j])) {
+      has_metadata = true;
     }
   };
-
   arrow::r::TraverseDots(lst, num_fields, extract_one_metadata);
+
+  metadata_columns.names() = metadata_columns_names;
   return metadata_columns;
 }
 
 arrow::Status AddMetadataFromDots(SEXP lst, int num_fields,
                                   std::shared_ptr<arrow::Schema>& schema) {
   // Preallocate the r_metadata object: list(attributes=list(), columns=namedList(fields))
-  SEXP metadata = PROTECT(Rf_allocVector(VECSXP, 2));
-  Rf_setAttrib(metadata, R_NamesSymbol, arrow::r::data::names_metadata);
+
+  cpp11::writable::list metadata(2);
+  metadata.names() = arrow::r::data::names_metadata;
 
   bool has_metadata = false;
-  // TODO: we want to keep any top-level data-frame attributes
-  // but the auto-splice code has stripped them out by the time we get here
-  // https://issues.apache.org/jira/browse/ARROW-9271
-  //
-  // SEXP att = ATTRIB(lst);
-  // if (!Rf_isNull(att)) {
-  //   SEXP att_list_call = PROTECT(Rf_lang2(arrow::r::symbols::as_list, att));
-  //   SET_VECTOR_ELT(metadata, 0, PROTECT(Rf_eval(att_list_call, R_GlobalEnv)));
-  //   UNPROTECT(2);
-  //   has_metadata = true;
-  // }
-  SET_VECTOR_ELT(metadata, 1, CollectColumnMetadata(lst, num_fields, has_metadata));
-  UNPROTECT(2);  // CollectColumnMetadata adds 2 PROTECTS
+
+  // "top level" attributes, only relevant if the first object is not named and a data
+  // frame
+  cpp11::strings names = Rf_getAttrib(lst, R_NamesSymbol);
+  if (names[0] == "" && Rf_inherits(VECTOR_ELT(lst, 0), "data.frame")) {
+    SEXP top_level = metadata[0] = arrow_attributes(VECTOR_ELT(lst, 0), true);
+    if (!Rf_isNull(top_level) && XLENGTH(top_level) > 0) {
+      has_metadata = true;
+    }
+  }
+
+  // recurse to get all columns metadata
+  metadata[1] = CollectColumnMetadata(lst, num_fields, has_metadata);
 
   if (has_metadata) {
     SEXP serialise_call =
@@ -316,7 +223,6 @@ arrow::Status AddMetadataFromDots(SEXP lst, int num_fields,
 
     UNPROTECT(2);
   }
-  UNPROTECT(1);
 
   return arrow::Status::OK();
 }
@@ -324,12 +230,13 @@ arrow::Status AddMetadataFromDots(SEXP lst, int num_fields,
 arrow::Status CollectTableColumns(
     SEXP lst, const std::shared_ptr<arrow::Schema>& schema, int num_fields, bool inferred,
     std::vector<std::shared_ptr<arrow::ChunkedArray>>& columns) {
-  auto extract_one_column = [&columns, &schema, inferred](int j, SEXP x, SEXP name) {
+  auto extract_one_column = [&columns, &schema, inferred](int j, SEXP x,
+                                                          cpp11::r_string) {
     if (Rf_inherits(x, "ChunkedArray")) {
-      columns[j] = arrow::r::extract<arrow::ChunkedArray>(x);
+      columns[j] = cpp11::as_cpp<std::shared_ptr<arrow::ChunkedArray>>(x);
     } else if (Rf_inherits(x, "Array")) {
-      columns[j] =
-          std::make_shared<arrow::ChunkedArray>(arrow::r::extract<arrow::Array>(x));
+      columns[j] = std::make_shared<arrow::ChunkedArray>(
+          cpp11::as_cpp<std::shared_ptr<arrow::Array>>(x));
     } else {
       auto array = arrow::r::Array__from_vector(x, schema->field(j)->type(), inferred);
       columns[j] = std::make_shared<arrow::ChunkedArray>(array);
@@ -343,23 +250,35 @@ arrow::Status CollectTableColumns(
 }  // namespace arrow
 
 // [[arrow::export]]
-std::shared_ptr<arrow::Table> Table__from_dots(SEXP lst, SEXP schema_sxp) {
-  bool infer_schema = !Rf_inherits(schema_sxp, "Schema");
+bool all_record_batches(SEXP lst) {
+  R_xlen_t n = XLENGTH(lst);
+  for (R_xlen_t i = 0; i < n; i++) {
+    if (!Rf_inherits(VECTOR_ELT(lst, i), "RecordBatch")) return false;
+  }
+  return true;
+}
 
-  if (all_record_batches(lst)) {
-    auto batches = arrow::r::List_to_shared_ptr_vector<arrow::RecordBatch>(lst);
-    std::shared_ptr<arrow::Table> tab;
+// [[arrow::export]]
+std::shared_ptr<arrow::Table> Table__from_record_batches(
+    const std::vector<std::shared_ptr<arrow::RecordBatch>>& batches, SEXP schema_sxp) {
+  bool infer_schema = !Rf_inherits(schema_sxp, "Schema");
 
-    if (infer_schema) {
-      tab = ValueOrStop(arrow::Table::FromRecordBatches(std::move(batches)));
-    } else {
-      auto schema = arrow::r::extract<arrow::Schema>(schema_sxp);
-      tab = ValueOrStop(arrow::Table::FromRecordBatches(schema, std::move(batches)));
-    }
+  std::shared_ptr<arrow::Table> tab;
 
-    return tab;
+  if (infer_schema) {
+    tab = ValueOrStop(arrow::Table::FromRecordBatches(std::move(batches)));
+  } else {
+    auto schema = cpp11::as_cpp<std::shared_ptr<arrow::Schema>>(schema_sxp);
+    tab = ValueOrStop(arrow::Table::FromRecordBatches(schema, std::move(batches)));
   }
 
+  return tab;
+}
+
+// [[arrow::export]]
+std::shared_ptr<arrow::Table> Table__from_dots(SEXP lst, SEXP schema_sxp) {
+  bool infer_schema = !Rf_inherits(schema_sxp, "Schema");
+
   int num_fields;
   StopIfNotOk(arrow::r::count_fields(lst, &num_fields));
 
diff --git a/r/tests/testthat/helper-roundtrip.R b/r/tests/testthat/helper-roundtrip.R
new file mode 100644
index 00000000000..16d002c9f2a
--- /dev/null
+++ b/r/tests/testthat/helper-roundtrip.R
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+expect_array_roundtrip <- function(x, type, as = NULL) {
+  a <- Array$create(x, type = as)
+  expect_type_equal(a$type, type)
+  expect_identical(length(a), length(x))
+  if (!inherits(type, c("ListType", "LargeListType"))) {
+    # TODO: revisit how missingness works with ListArrays
+    # R list objects don't handle missingness the same way as other vectors.
+    # Is there some vctrs thing we should do on the roundtrip back to R?
+    expect_equal(as.vector(is.na(a)), is.na(x))
+  }
+  expect_equivalent(as.vector(a), x)
+  # Make sure the storage mode is the same on roundtrip (esp. integer vs. numeric)
+  expect_identical(typeof(as.vector(a)), typeof(x))
+
+  if (length(x)) {
+    a_sliced <- a$Slice(1)
+    x_sliced <- x[-1]
+    expect_type_equal(a_sliced$type, type)
+    expect_identical(length(a_sliced), length(x_sliced))
+    if (!inherits(type, c("ListType", "LargeListType"))) {
+      expect_equal(as.vector(is.na(a_sliced)), is.na(x_sliced))
+    }
+    expect_equivalent(as.vector(a_sliced), x_sliced)
+  }
+  invisible(a)
+}
diff --git a/r/tests/testthat/helper-skip.R b/r/tests/testthat/helper-skip.R
index 94bfd49efad..366cc75fb27 100644
--- a/r/tests/testthat/helper-skip.R
+++ b/r/tests/testthat/helper-skip.R
@@ -43,3 +43,8 @@ skip_if_not_running_large_memory_tests <- function() {
     "environment variable ARROW_LARGE_MEMORY_TESTS"
   )
 }
+
+process_is_running <- function(x) {
+  cmd <- sprintf("ps aux | grep '%s' | grep -v grep", x)
+  tryCatch(system(cmd, ignore.stdout = TRUE) == 0, error = function(e) FALSE)
+}
diff --git a/r/tests/testthat/test-Array-errors.txt b/r/tests/testthat/test-Array-errors.txt
deleted file mode 100644
index 94a79703de5..00000000000
--- a/r/tests/testthat/test-Array-errors.txt
+++ /dev/null
@@ -1,25 +0,0 @@
-> Array$create(list(numeric(0)), list_of(bool()))
-Error in Array__from_vector(x, type): Unknown: Cannot convert list element 1 to an Array of type `bool` : Expecting a logical vector
-
-> Array$create(list(numeric(0)), list_of(int32()))
-Error in Array__from_vector(x, type): Unknown: Cannot convert list element 1 to an Array of type `int32` : Expecting an integer vector
-
-> Array$create(list(integer(0)), list_of(float64()))
-Error in Array__from_vector(x, type): Unknown: Cannot convert list element 1 to an Array of type `double` : Expecting a numeric vector
-
-> lgl <- logical(0)
-> int <- integer(0)
-> num <- numeric(0)
-> char <- character(0)
-> Array$create(list())
-Error in Array__from_vector(x, type): Requires at least one element to infer the values' type of a list vector
-
-> Array$create(list(lgl, lgl, int))
-Error in Array__from_vector(x, type): Unknown: Cannot convert list element 3 to an Array of type `bool` : Expecting a logical vector
-
-> Array$create(list(char, num, char))
-Error in Array__from_vector(x, type): Unknown: Cannot convert list element 2 to an Array of type `string` : Expecting a character vector
-
-> Array$create(list(int, int, num))
-Error in Array__from_vector(x, type): Unknown: Cannot convert list element 3 to an Array of type `int32` : Expecting an integer vector
-
diff --git a/r/tests/testthat/test-Array.R b/r/tests/testthat/test-Array.R
index ce1b5bef176..59c6dd9866a 100644
--- a/r/tests/testthat/test-Array.R
+++ b/r/tests/testthat/test-Array.R
@@ -17,33 +17,6 @@
 
 context("Array")
 
-expect_array_roundtrip <- function(x, type, as = NULL) {
-  a <- Array$create(x, type = as)
-  expect_type_equal(a$type, type)
-  expect_identical(length(a), length(x))
-  if (!inherits(type, c("ListType", "LargeListType"))) {
-    # TODO: revisit how missingness works with ListArrays
-    # R list objects don't handle missingness the same way as other vectors.
-    # Is there some vctrs thing we should do on the roundtrip back to R?
-    expect_equal(as.vector(is.na(a)), is.na(x))
-  }
-  expect_equivalent(as.vector(a), x)
-  # Make sure the storage mode is the same on roundtrip (esp. integer vs. numeric)
-  expect_identical(typeof(as.vector(a)), typeof(x))
-
-  if (length(x)) {
-    a_sliced <- a$Slice(1)
-    x_sliced <- x[-1]
-    expect_type_equal(a_sliced$type, type)
-    expect_identical(length(a_sliced), length(x_sliced))
-    if (!inherits(type, c("ListType", "LargeListType"))) {
-      expect_equal(as.vector(is.na(a_sliced)), is.na(x_sliced))
-    }
-    expect_equivalent(as.vector(a_sliced), x_sliced)
-  }
-  invisible(a)
-}
-
 test_that("Integer Array", {
   ints <- c(1:10, 1:10, 1:5)
   x <- expect_array_roundtrip(ints, int32())
@@ -131,14 +104,14 @@ test_that("Slice() and RangeEquals()", {
   expect_true(x$RangeEquals(z, 10, 15, 0))
 
   # Input validation
-  expect_error(x$Slice("ten"), class = "Rcpp::not_compatible")
+  expect_error(x$Slice("ten"))
   expect_error(x$Slice(NA_integer_), "Slice 'offset' cannot be NA")
   expect_error(x$Slice(NA), "Slice 'offset' cannot be NA")
-  expect_error(x$Slice(10, "ten"), class = "Rcpp::not_compatible")
+  expect_error(x$Slice(10, "ten"))
   expect_error(x$Slice(10, NA_integer_), "Slice 'length' cannot be NA")
   expect_error(x$Slice(NA_integer_, NA_integer_), "Slice 'offset' cannot be NA")
-  expect_error(x$Slice(c(10, 10)), class = "Rcpp::not_compatible")
-  expect_error(x$Slice(10, c(10, 10)), class = "Rcpp::not_compatible")
+  expect_error(x$Slice(c(10, 10)))
+  expect_error(x$Slice(10, c(10, 10)))
   expect_error(x$Slice(1000), "Slice 'offset' greater than array length")
   expect_error(x$Slice(-1), "Slice 'offset' cannot be negative")
   expect_error(z$Slice(10, 10), "Slice 'offset' greater than array length")
@@ -157,7 +130,7 @@ test_that("Slice() and RangeEquals()", {
   expect_error(x$RangeEquals(y, NA, 24), "'start_idx' cannot be NA")
   expect_error(x$RangeEquals(y, 10, NA), "'end_idx' cannot be NA")
   expect_error(x$RangeEquals(y, 10, 24, NA), "'other_start_idx' cannot be NA")
-  expect_error(x$RangeEquals(y, "ten", 24), class = "Rcpp::not_compatible")
+  expect_error(x$RangeEquals(y, "ten", 24))
   # TODO (if anyone uses RangeEquals)
   # expect_error(x$RangeEquals(y, 10, 2400, 0)) # does not error
   # expect_error(x$RangeEquals(y, 1000, 24, 0)) # does not error
@@ -186,10 +159,10 @@ test_that("Array supports NA", {
   expect_equal(as.vector(is.na(x_dbl)), c(rep(FALSE, 10), TRUE))
 
   # Input validation
-  expect_error(x_int$IsValid("ten"), class = "Rcpp::not_compatible")
-  expect_error(x_int$IsNull("ten"), class = "Rcpp::not_compatible")
-  expect_error(x_int$IsValid(c(10, 10)), class = "Rcpp::not_compatible")
-  expect_error(x_int$IsNull(c(10, 10)), class = "Rcpp::not_compatible")
+  expect_error(x_int$IsValid("ten"))
+  expect_error(x_int$IsNull("ten"))
+  expect_error(x_int$IsValid(c(10, 10)))
+  expect_error(x_int$IsNull(c(10, 10)))
   expect_error(x_int$IsValid(NA), "'i' cannot be NA")
   expect_error(x_int$IsNull(NA), "'i' cannot be NA")
   expect_error(x_int$IsValid(1000), "subscript out of bounds")
@@ -644,20 +617,18 @@ test_that("Array$create() handles vector -> fixed size list arrays", {
 })
 
 test_that("Array$create() should have helpful error", {
-  verify_output(test_path("test-Array-errors.txt"), {
-    Array$create(list(numeric(0)), list_of(bool()))
-    Array$create(list(numeric(0)), list_of(int32()))
-    Array$create(list(integer(0)), list_of(float64()))
-
-    lgl <- logical(0)
-    int <- integer(0)
-    num <- numeric(0)
-    char <- character(0)
-    Array$create(list())
-    Array$create(list(lgl, lgl, int))
-    Array$create(list(char, num, char))
-    Array$create(list(int, int, num))
-  })
+  expect_error(Array$create(list(numeric(0)), list_of(bool())), "Expecting a logical vector")
+  expect_error(Array$create(list(numeric(0)), list_of(int32())), "Expecting an integer vector")
+  expect_error(Array$create(list(integer(0)), list_of(float64())), "Expecting a numeric vector")
+
+  lgl <- logical(0)
+  int <- integer(0)
+  num <- numeric(0)
+  char <- character(0)
+  expect_error(Array$create(list()), "Requires at least one element to infer")
+  expect_error(Array$create(list(lgl, lgl, int)), "Expecting a logical vector")
+  expect_error(Array$create(list(char, num, char)), "Expecting a character vector")
+  expect_error(Array$create(list(int, int, num)), "Expecting an integer vector")
 })
 
 test_that("Array$View() (ARROW-6542)", {
@@ -755,8 +726,7 @@ test_that("Dictionary array: translate to R when dict isn't string", {
     expect_identical(
       as.vector(a),
       factor(c(3, 2, 2, 3, 1), labels = c("4.5", "3.2", "1.1"))
-    ),
-    "Coercing dictionary values from type double to R character factor levels"
+    )
   )
 })
 
@@ -779,3 +749,17 @@ test_that("Array$ApproxEquals", {
   expect_true(a$ApproxEquals(b))
   expect_false(a$ApproxEquals(vec))
 })
+
+test_that("auto int64 conversion to int can be disabled (ARROW-10093)", {
+  op <- options(arrow.int64_downcast = FALSE); on.exit(options(op))
+
+  a <- Array$create(1:10, int64())
+  expect_true(inherits(a$as_vector(), "integer64"))
+
+  batch <- RecordBatch$create(x = a)
+  expect_true(inherits(as.data.frame(batch)$x, "integer64"))
+
+  tab <- Table$create(x = a)
+  expect_true(inherits(as.data.frame(batch)$x, "integer64"))
+})
+
diff --git a/r/tests/testthat/test-RecordBatch.R b/r/tests/testthat/test-RecordBatch.R
index 3fceb4c5451..1a7f7eecc87 100644
--- a/r/tests/testthat/test-RecordBatch.R
+++ b/r/tests/testthat/test-RecordBatch.R
@@ -50,8 +50,8 @@ test_that("RecordBatch", {
   expect_error(batch$column_name(NA), "'i' cannot be NA")
   expect_error(batch$column_name(-1), "subscript out of bounds")
   expect_error(batch$column_name(1000), "subscript out of bounds")
-  expect_error(batch$column_name(1:2), class = "Rcpp::not_compatible")
-  expect_error(batch$column_name("one"), class = "Rcpp::not_compatible")
+  expect_error(batch$column_name(1:2))
+  expect_error(batch$column_name("one"))
 
   col_int <- batch$column(0)
   expect_true(inherits(col_int, 'Array'))
@@ -82,8 +82,8 @@ test_that("RecordBatch", {
   expect_error(batch$column(NA), "'i' cannot be NA")
   expect_error(batch$column(-1), "subscript out of bounds")
   expect_error(batch$column(1000), "subscript out of bounds")
-  expect_error(batch$column(1:2), class = "Rcpp::not_compatible")
-  expect_error(batch$column("one"), class = "Rcpp::not_compatible")
+  expect_error(batch$column(1:2))
+  expect_error(batch$column("one"))
 
   batch2 <- batch$RemoveColumn(0)
   expect_equal(
@@ -97,8 +97,8 @@ test_that("RecordBatch", {
   expect_error(batch$RemoveColumn(NA), "'i' cannot be NA")
   expect_error(batch$RemoveColumn(-1), "subscript out of bounds")
   expect_error(batch$RemoveColumn(1000), "subscript out of bounds")
-  expect_error(batch$RemoveColumn(1:2), class = "Rcpp::not_compatible")
-  expect_error(batch$RemoveColumn("one"), class = "Rcpp::not_compatible")
+  expect_error(batch$RemoveColumn(1:2))
+  expect_error(batch$RemoveColumn("one"))
 })
 
 test_that("RecordBatch S3 methods", {
@@ -117,14 +117,14 @@ test_that("RecordBatch$Slice", {
   expect_data_frame(batch4, tbl[6:7,])
 
   # Input validation
-  expect_error(batch$Slice("ten"), class = "Rcpp::not_compatible")
+  expect_error(batch$Slice("ten"))
   expect_error(batch$Slice(NA_integer_), "Slice 'offset' cannot be NA")
   expect_error(batch$Slice(NA), "Slice 'offset' cannot be NA")
-  expect_error(batch$Slice(10, "ten"), class = "Rcpp::not_compatible")
+  expect_error(batch$Slice(10, "ten"))
   expect_error(batch$Slice(10, NA_integer_), "Slice 'length' cannot be NA")
   expect_error(batch$Slice(NA_integer_, NA_integer_), "Slice 'offset' cannot be NA")
-  expect_error(batch$Slice(c(10, 10)), class = "Rcpp::not_compatible")
-  expect_error(batch$Slice(10, c(10, 10)), class = "Rcpp::not_compatible")
+  expect_error(batch$Slice(c(10, 10)))
+  expect_error(batch$Slice(10, c(10, 10)))
   expect_error(batch$Slice(1000), "Slice 'offset' greater than array length")
   expect_error(batch$Slice(-1), "Slice 'offset' cannot be negative")
   expect_error(batch4$Slice(10, 10), "Slice 'offset' greater than array length")
@@ -155,7 +155,7 @@ test_that("[[ and $ on RecordBatch", {
   expect_vector(batch[[4]], tbl$chr)
   expect_null(batch$qwerty)
   expect_null(batch[["asdf"]])
-  expect_error(batch[[c(4, 3)]], class = "Rcpp::not_compatible")
+  expect_error(batch[[c(4, 3)]])
   expect_error(batch[[NA]], "'i' must be character or numeric, not logical")
   expect_error(batch[[NULL]], "'i' must be character or numeric, not NULL")
   expect_error(batch[[c("asdf", "jkl;")]], 'name is not a string', fixed = TRUE)
diff --git a/r/tests/testthat/test-Table.R b/r/tests/testthat/test-Table.R
index b26fd8ca356..0758edd4509 100644
--- a/r/tests/testthat/test-Table.R
+++ b/r/tests/testthat/test-Table.R
@@ -80,14 +80,14 @@ test_that("Table $column and $field", {
   expect_error(tab$column(NA), "'i' cannot be NA")
   expect_error(tab$column(-1), "subscript out of bounds")
   expect_error(tab$column(1000), "subscript out of bounds")
-  expect_error(tab$column(1:2), class = "Rcpp::not_compatible")
-  expect_error(tab$column("one"), class = "Rcpp::not_compatible")
+  expect_error(tab$column(1:2))
+  expect_error(tab$column("one"))
 
   expect_error(tab$field(NA), "'i' cannot be NA")
   expect_error(tab$field(-1), "subscript out of bounds")
   expect_error(tab$field(1000), "subscript out of bounds")
-  expect_error(tab$field(1:2), class = "Rcpp::not_compatible")
-  expect_error(tab$field("one"), class = "Rcpp::not_compatible")
+  expect_error(tab$field(1:2))
+  expect_error(tab$field("one"))
 })
 
 test_that("[, [[, $ for Table", {
@@ -130,16 +130,16 @@ test_that("[, [[, $ for Table", {
   expect_null(tab[["asdf"]])
   # List-like column slicing
   expect_data_frame(tab[2:4], tbl[2:4])
-  expect_data_frame(tab[c(1, 0)], tbl[c(1, 0)])
+  expect_data_frame(tab[c(2, 1)], tbl[c(2, 1)])
+  expect_data_frame(tab[-3], tbl[-3])
 
-  expect_error(tab[[c(4, 3)]], class = "Rcpp::not_compatible")
+  expect_error(tab[[c(4, 3)]])
   expect_error(tab[[NA]], "'i' must be character or numeric, not logical")
   expect_error(tab[[NULL]], "'i' must be character or numeric, not NULL")
   expect_error(tab[[c("asdf", "jkl;")]], 'length(name) not equal to 1', fixed = TRUE)
-  expect_error(tab[-3], "Selections can't have negative value") # From tidyselect
-  expect_error(tab[-3:3], "Selections can't have negative value") # From tidyselect
-  expect_error(tab[1000]) # This is caught in vctrs, assert more specifically when it stabilizes
-  expect_error(tab[1:1000]) # same as ^
+  expect_error(tab[-3:3], "Invalid column index")
+  expect_error(tab[1000],  "Invalid column index")
+  expect_error(tab[1:1000], "Invalid column index")
 
   skip("Table with 0 cols doesn't know how many rows it should have")
   expect_data_frame(tab[0], tbl[0])
@@ -153,14 +153,14 @@ test_that("Table$Slice", {
   expect_data_frame(tab3, tbl[6:7,])
 
   # Input validation
-  expect_error(tab$Slice("ten"), class = "Rcpp::not_compatible")
+  expect_error(tab$Slice("ten"))
   expect_error(tab$Slice(NA_integer_), "Slice 'offset' cannot be NA")
   expect_error(tab$Slice(NA), "Slice 'offset' cannot be NA")
-  expect_error(tab$Slice(10, "ten"), class = "Rcpp::not_compatible")
+  expect_error(tab$Slice(10, "ten"))
   expect_error(tab$Slice(10, NA_integer_), "Slice 'length' cannot be NA")
   expect_error(tab$Slice(NA_integer_, NA_integer_), "Slice 'offset' cannot be NA")
-  expect_error(tab$Slice(c(10, 10)), class = "Rcpp::not_compatible")
-  expect_error(tab$Slice(10, c(10, 10)), class = "Rcpp::not_compatible")
+  expect_error(tab$Slice(c(10, 10)))
+  expect_error(tab$Slice(10, c(10, 10)))
   expect_error(tab$Slice(1000), "Slice 'offset' greater than array length")
   expect_error(tab$Slice(-1), "Slice 'offset' cannot be negative")
   expect_error(tab3$Slice(10, 10), "Slice 'offset' greater than array length")
@@ -349,3 +349,12 @@ test_that("Table unifies dictionary on conversion back to R (ARROW-8374)", {
 
   expect_identical(as.data.frame(tab), res)
 })
+
+test_that("Table$SelectColumns()", {
+  tab <- Table$create(x = 1:10, y = 1:10)
+
+  expect_equal(tab$SelectColumns(0L), Table$create(x = 1:10))
+
+  expect_error(tab$SelectColumns(2:4))
+  expect_error(tab$SelectColumns(""))
+})
diff --git a/r/tests/testthat/test-arrow.R b/r/tests/testthat/test-arrow.R
index 7ef25ac672e..2c901e1c96a 100644
--- a/r/tests/testthat/test-arrow.R
+++ b/r/tests/testthat/test-arrow.R
@@ -47,3 +47,28 @@ r_only({
     )
   })
 })
+
+test_that("arrow gracefully fails to load objects from other sessions (ARROW-10071)", {
+  a <- Array$create(1:10)
+  tf <- tempfile(); on.exit(unlink(tf))
+  saveRDS(a, tf)
+
+  b <- readRDS(tf)
+  expect_error(b$length(), "Invalid <Array>")
+})
+
+test_that("check for an ArrowObject in functions use std::shared_ptr", {
+  expect_error(Array__length(1), "Invalid R object")
+})
+
+test_that("MemoryPool calls gc() to free memory when allocation fails (ARROW-10080)", {
+  env <- new.env()
+  trace(gc, print = FALSE, tracer = function() {
+          env$gc_was_called <- TRUE
+        })
+  on.exit(untrace(gc))
+  # We expect this should fail because we don't have this much memory,
+  # but it should gc() and retry (and fail again)
+  expect_error(BufferOutputStream$create(2 ** 60))
+  expect_true(env$gc_was_called)
+})
diff --git a/r/tests/testthat/test-chunked-array.R b/r/tests/testthat/test-chunked-array.R
index b4695e28eed..3e2d2fd7637 100644
--- a/r/tests/testthat/test-chunked-array.R
+++ b/r/tests/testthat/test-chunked-array.R
@@ -65,18 +65,18 @@ test_that("ChunkedArray", {
 
   # input validation
   expect_error(x$chunk(14), "subscript out of bounds")
-  expect_error(x$chunk("one"), class = "Rcpp::not_compatible")
+  expect_error(x$chunk("one"))
   expect_error(x$chunk(NA_integer_), "'i' cannot be NA")
   expect_error(x$chunk(-1), "subscript out of bounds")
 
-  expect_error(x$Slice("ten"), class = "Rcpp::not_compatible")
+  expect_error(x$Slice("ten"))
   expect_error(x$Slice(NA_integer_), "Slice 'offset' cannot be NA")
   expect_error(x$Slice(NA), "Slice 'offset' cannot be NA")
-  expect_error(x$Slice(10, "ten"), class = "Rcpp::not_compatible")
+  expect_error(x$Slice(10, "ten"))
   expect_error(x$Slice(10, NA_integer_), "Slice 'length' cannot be NA")
   expect_error(x$Slice(NA_integer_, NA_integer_), "Slice 'offset' cannot be NA")
-  expect_error(x$Slice(c(10, 10)), class = "Rcpp::not_compatible")
-  expect_error(x$Slice(10, c(10, 10)), class = "Rcpp::not_compatible")
+  expect_error(x$Slice(c(10, 10)))
+  expect_error(x$Slice(10, c(10, 10)))
   expect_error(x$Slice(1000), "Slice 'offset' greater than array length")
   expect_error(x$Slice(-1), "Slice 'offset' cannot be negative")
   expect_error(z$Slice(10, 10), "Slice 'offset' greater than array length")
@@ -158,6 +158,13 @@ test_that("ChunkedArray supports POSIXct (ARROW-3716)", {
 test_that("ChunkedArray supports integer64 (ARROW-3716)", {
   x <- bit64::as.integer64(1:10) + MAX_INT
   expect_chunked_roundtrip(list(x, x), int64())
+  # Also with a first chunk that would downcast
+  zero <- Array$create(0L)$cast(int64())
+  expect_type_equal(zero, int64())
+  ca <- ChunkedArray$create(zero, x)
+  expect_type_equal(ca, int64())
+  expect_is(as.vector(ca), "integer64")
+  expect_identical(as.vector(ca), c(bit64::as.integer64(0L), x))
 })
 
 test_that("ChunkedArray supports difftime", {
diff --git a/r/tests/testthat/test-compute-aggregate.R b/r/tests/testthat/test-compute-aggregate.R
index 1e5f9a46b33..d80ff8c75b1 100644
--- a/r/tests/testthat/test-compute-aggregate.R
+++ b/r/tests/testthat/test-compute-aggregate.R
@@ -100,7 +100,7 @@ test_that("Bad input handling of call_function", {
   )
 })
 
-test_that("min/max.Array", {
+test_that("min.Array", {
   ints <- 1:4
   a <- Array$create(ints)
   expect_is(min(a), "Scalar")
@@ -121,3 +121,96 @@ test_that("min/max.Array", {
   # R is inconsistent here: typeof(min(NA)) == "integer", not "logical"
   expect_identical(as.vector(min(b)), as.logical(min(bools)))
 })
+
+test_that("max.Array", {
+  ints <- 1:4
+  a <- Array$create(ints)
+  expect_is(max(a), "Scalar")
+  expect_identical(as.vector(max(a)), max(ints))
+
+  floats <- c(1.3, 3, 2.4)
+  f <- Array$create(floats)
+  expect_identical(as.vector(max(f)), max(floats))
+
+  floats <- c(floats, NA)
+  na <- Array$create(floats)
+  expect_identical(as.vector(max(na)), max(floats))
+  expect_is(max(na, na.rm = TRUE), "Scalar")
+  expect_identical(as.vector(max(na, na.rm = TRUE)), max(floats, na.rm = TRUE))
+
+  bools <- c(TRUE, TRUE, FALSE)
+  b <- Array$create(bools)
+  # R is inconsistent here: typeof(max(NA)) == "integer", not "logical"
+  expect_identical(as.vector(max(b)), as.logical(max(bools)))
+})
+
+test_that("min.ChunkedArray", {
+  ints <- 1:4
+  a <- ChunkedArray$create(ints)
+  expect_is(min(a), "Scalar")
+  expect_identical(as.vector(min(a)), min(ints))
+
+  floats <- c(1.3, 3, 2.4)
+  f <- ChunkedArray$create(floats)
+  expect_identical(as.vector(min(f)), min(floats))
+
+  floats <- c(floats, NA)
+  na <- ChunkedArray$create(floats)
+  expect_identical(as.vector(min(na)), min(floats))
+  expect_is(min(na, na.rm = TRUE), "Scalar")
+  expect_identical(as.vector(min(na, na.rm = TRUE)), min(floats, na.rm = TRUE))
+
+  bools <- c(TRUE, TRUE, FALSE)
+  b <- ChunkedArray$create(bools)
+  # R is inconsistent here: typeof(min(NA)) == "integer", not "logical"
+  expect_identical(as.vector(min(b)), as.logical(min(bools)))
+})
+
+test_that("max.ChunkedArray", {
+  ints <- 1:4
+  a <- ChunkedArray$create(ints)
+  expect_is(max(a), "Scalar")
+  expect_identical(as.vector(max(a)), max(ints))
+
+  floats <- c(1.3, 3, 2.4)
+  f <- ChunkedArray$create(floats)
+  expect_identical(as.vector(max(f)), max(floats))
+
+  floats <- c(floats, NA)
+  na <- ChunkedArray$create(floats)
+  expect_identical(as.vector(max(na)), max(floats))
+  expect_is(max(na, na.rm = TRUE), "Scalar")
+  expect_identical(as.vector(max(na, na.rm = TRUE)), max(floats, na.rm = TRUE))
+
+  bools <- c(TRUE, TRUE, FALSE)
+  b <- ChunkedArray$create(bools)
+  # R is inconsistent here: typeof(max(NA)) == "integer", not "logical"
+  expect_identical(as.vector(max(b)), as.logical(max(bools)))
+})
+
+test_that("Edge cases", {
+  skip("ARROW-9054")
+  a <- Array$create(NA)
+  for (type in c(int32(), float64(), bool())) {
+    expect_equal(as.vector(sum(a$cast(type), na.rm = TRUE)), sum(NA, na.rm = TRUE))
+    expect_equal(as.vector(mean(a$cast(type), na.rm = TRUE)), mean(NA, na.rm = TRUE))
+    expect_equal(as.vector(min(a$cast(type), na.rm = TRUE)), min(NA, na.rm = TRUE))
+    expect_equal(as.vector(max(a$cast(type), na.rm = TRUE)), max(NA, na.rm = TRUE))
+  }
+})
+
+test_that("unique.Array", {
+  a <- Array$create(c(1, 4, 3, 1, 1, 3, 4))
+  expect_equal(unique(a), Array$create(c(1, 4, 3)))
+  ca <- ChunkedArray$create(a, a)
+  expect_equal(unique(ca), Array$create(c(1, 4, 3)))
+})
+
+test_that("match_arrow", {
+  a <- Array$create(c(1, 4, 3, 1, 1, 3, 4))
+  tab <- c(4, 3, 2, 1)
+  expect_equal(match_arrow(a, tab), Array$create(c(3L, 0L, 1L, 3L, 3L, 1L, 0L)))
+
+  ca <- ChunkedArray$create(c(1, 4, 3, 1, 1, 3, 4))
+  expect_equal(match_arrow(ca, tab), ChunkedArray$create(c(3L, 0L, 1L, 3L, 3L, 1L, 0L)))
+})
diff --git a/r/tests/testthat/test-csv.R b/r/tests/testthat/test-csv.R
index 2d85437904f..3de70b35471 100644
--- a/r/tests/testthat/test-csv.R
+++ b/r/tests/testthat/test-csv.R
@@ -174,3 +174,83 @@ test_that("read_csv_arrow() can detect compression from file name", {
   tab1 <- read_csv_arrow(tf)
   expect_equivalent(tbl, tab1)
 })
+
+test_that("read_csv_arrow(schema=)", {
+  tbl <- example_data[, "int"]
+  tf <- tempfile(); on.exit(unlink(tf))
+  write.csv(tbl, tf, row.names = FALSE)
+
+  df <- read_csv_arrow(tf, schema = schema(int = float64()), skip = 1)
+  expect_identical(df, tibble::tibble(int = as.numeric(tbl$int)))
+})
+
+test_that("read_csv_arrow(col_types = <Schema>)", {
+  tbl <- example_data[, "int"]
+  tf <- tempfile(); on.exit(unlink(tf))
+  write.csv(tbl, tf, row.names = FALSE)
+
+  df <- read_csv_arrow(tf, col_types = schema(int = float64()))
+  expect_identical(df, tibble::tibble(int = as.numeric(tbl$int)))
+})
+
+test_that("read_csv_arrow(col_types=string, col_names)", {
+  tbl <- example_data[, "int"]
+  tf <- tempfile(); on.exit(unlink(tf))
+  write.csv(tbl, tf, row.names = FALSE)
+
+  df <- read_csv_arrow(tf, col_names = "int", col_types = "d", skip = 1)
+  expect_identical(df, tibble::tibble(int = as.numeric(tbl$int)))
+
+  expect_error(read_csv_arrow(tf, col_types = c("i", "d")))
+  expect_error(read_csv_arrow(tf, col_types = "d"))
+  expect_error(read_csv_arrow(tf, col_types = "i", col_names = c("a", "b")))
+  expect_error(read_csv_arrow(tf, col_types = "y", col_names = "a"))
+})
+
+test_that("read_csv_arrow() can read timestamps", {
+  tbl <- tibble::tibble(time = as.POSIXct("2020-07-20 16:20", tz = "UTC"))
+  tf <- tempfile(); on.exit(unlink(tf))
+  write.csv(tbl, tf, row.names = FALSE)
+
+  df <- read_csv_arrow(tf, col_types = schema(time = timestamp(timezone = "UTC")))
+  expect_equal(tbl, df)
+
+  df <- read_csv_arrow(tf, col_types = "t", col_names = "time", skip = 1)
+  expect_equal(tbl, df, check.tzone = FALSE) # col_types = "t" makes timezone-naive timestamp
+})
+
+test_that("read_csv_arrow(timestamp_parsers=)", {
+  tf <- tempfile(); on.exit(unlink(tf))
+  tbl <- tibble::tibble(time = "23/09/2020")
+  write.csv(tbl, tf, row.names = FALSE)
+
+  df <- read_csv_arrow(
+    tf,
+    col_types = schema(time = timestamp(timezone = "UTC")),
+    timestamp_parsers = "%d/%m/%Y"
+  )
+  expect_equal(df$time, as.POSIXct(tbl$time, format = "%d/%m/%Y", tz = "UTC"))
+})
+
+test_that("Skipping columns with null()", {
+  tf <- tempfile(); on.exit(unlink(tf))
+  cols <- c("dbl", "lgl", "false", "chr")
+  tbl <- example_data[, cols]
+  write.csv(tbl, tf, row.names = FALSE)
+
+  df <- read_csv_arrow(tf, col_types = "d-_c", col_names = cols, skip = 1)
+  expect_identical(df, tbl[, c("dbl", "chr")])
+})
+
+test_that("Mix of guessing and declaring types", {
+  tf <- tempfile(); on.exit(unlink(tf))
+  cols <- c("dbl", "lgl", "false", "chr")
+  tbl <- example_data[, cols]
+  write.csv(tbl, tf, row.names = FALSE)
+
+  tab <- read_csv_arrow(tf, col_types = schema(dbl = float32()), as_data_frame = FALSE)
+  expect_equal(tab$schema, schema(dbl = float32(), lgl = bool(), false = bool(), chr = utf8()))
+
+  df <- read_csv_arrow(tf, col_types = "d-?c", col_names = cols, skip = 1)
+  expect_identical(df, tbl[, c("dbl", "false", "chr")])
+})
diff --git a/r/tests/testthat/test-data-type.R b/r/tests/testthat/test-data-type.R
index b5b652cf289..8ff3fc36713 100644
--- a/r/tests/testthat/test-data-type.R
+++ b/r/tests/testthat/test-data-type.R
@@ -408,6 +408,6 @@ test_that("FixedSizeBinary", {
   # input validation
   expect_error(fixed_size_binary(NA), "'byte_width' cannot be NA")
   expect_error(fixed_size_binary(-1), "'byte_width' must be > 0")
-  expect_error(fixed_size_binary("four"), class = "Rcpp::not_compatible")
-  expect_error(fixed_size_binary(c(2, 4)), class = "Rcpp::not_compatible")
+  expect_error(fixed_size_binary("four"))
+  expect_error(fixed_size_binary(c(2, 4)))
 })
diff --git a/r/tests/testthat/test-dataset.R b/r/tests/testthat/test-dataset.R
index b28b62c1324..73d654eb5a1 100644
--- a/r/tests/testthat/test-dataset.R
+++ b/r/tests/testthat/test-dataset.R
@@ -88,6 +88,8 @@ test_that("Setup (putting data in the dir)", {
 
 test_that("Simple interface for datasets", {
   ds <- open_dataset(dataset_dir, partitioning = schema(part = uint8()))
+  expect_is(ds$format, "ParquetFileFormat")
+  expect_is(ds$filesystem, "LocalFileSystem")
   expect_is(ds, "Dataset")
   expect_equivalent(
     ds %>%
@@ -120,13 +122,13 @@ test_that("Simple interface for datasets", {
   )
 })
 
-test_that("dim method returns the correct number of rows and columns",{
+test_that("dim method returns the correct number of rows and columns", {
   ds <- open_dataset(dataset_dir, partitioning = schema(part = uint8()))
   expect_identical(dim(ds), c(20L, 7L))
 })
 
 
-test_that("dim() correctly determine numbers of rows and columns on arrow_dplyr_query object",{
+test_that("dim() correctly determine numbers of rows and columns on arrow_dplyr_query object", {
   ds <- open_dataset(dataset_dir, partitioning = schema(part = uint8()))
 
   expect_warning(
@@ -175,7 +177,7 @@ test_that("dataset from URI", {
 test_that("Simple interface for datasets (custom ParquetFileFormat)", {
   ds <- open_dataset(dataset_dir, partitioning = schema(part = uint8()),
                      format = FileFormat$create("parquet", dict_columns = c("chr")))
-  expect_equivalent(ds$schema$GetFieldByName("chr")$type, dictionary())
+  expect_type_equal(ds$schema$GetFieldByName("chr")$type, dictionary())
 })
 
 test_that("Hive partitioning", {
@@ -223,6 +225,8 @@ test_that("Partitioning inference", {
 
 test_that("IPC/Feather format data", {
   ds <- open_dataset(ipc_dir, partitioning = "part", format = "feather")
+  expect_is(ds$format, "IpcFileFormat")
+  expect_is(ds$filesystem, "LocalFileSystem")
   expect_identical(names(ds), c(names(df1), "part"))
   expect_warning(
     expect_identical(dim(ds), c(NA, 7L))
@@ -249,6 +253,8 @@ test_that("IPC/Feather format data", {
 
 test_that("CSV dataset", {
   ds <- open_dataset(csv_dir, partitioning = "part", format = "csv")
+  expect_is(ds$format, "CsvFileFormat")
+  expect_is(ds$filesystem, "LocalFileSystem")
   expect_identical(names(ds), c(names(df1), "part"))
   expect_warning(
     expect_identical(dim(ds), c(NA, 7L))
@@ -365,7 +371,23 @@ test_that("Creating UnionDataset", {
   )
 
   # Confirm c() method error handling
-  expect_error(c(ds1, 42), "'x' must be a string or a list of DatasetFactory")
+  expect_error(c(ds1, 42), "string")
+})
+
+test_that("InMemoryDataset", {
+  ds <- InMemoryDataset$create(rbind(df1, df2))
+  expect_is(ds, "InMemoryDataset")
+  expect_equivalent(
+    ds %>%
+      select(chr, dbl) %>%
+      filter(dbl > 7 & dbl < 53L) %>%
+      collect() %>%
+      arrange(dbl),
+    rbind(
+      df1[8:10, c("chr", "dbl")],
+      df2[1:2, c("chr", "dbl")]
+    )
+  )
 })
 
 test_that("map_batches", {
@@ -737,6 +759,14 @@ test_that("Writing a dataset: CSV->IPC", {
       filter(integer > 6) %>%
       summarize(mean = mean(integer))
   )
+
+  # Check whether "int" is present in the files or just in the dirs
+  first <- read_feather(
+    dir(dst_dir, pattern = ".feather$", recursive = TRUE, full.names = TRUE)[1],
+    as_data_frame = FALSE
+  )
+  # It shouldn't be there
+  expect_false("int" %in% names(first))
 })
 
 test_that("Writing a dataset: Parquet->IPC", {
@@ -762,23 +792,86 @@ test_that("Writing a dataset: Parquet->IPC", {
   )
 })
 
+test_that("Writing a dataset: CSV->Parquet", {
+  skip_on_os("windows") # https://issues.apache.org/jira/browse/ARROW-9651
+  ds <- open_dataset(csv_dir, partitioning = "part", format = "csv")
+  dst_dir <- make_temp_dir()
+  write_dataset(ds, dst_dir, format = "parquet", partitioning = "int")
+  expect_true(dir.exists(dst_dir))
+  expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "=")))
+
+  new_ds <- open_dataset(dst_dir)
+
+  expect_equivalent(
+    new_ds %>%
+      select(string = chr, integer = int) %>%
+      filter(integer > 6 & integer < 11) %>%
+      collect() %>%
+      summarize(mean = mean(integer)),
+    df1 %>%
+      select(string = chr, integer = int) %>%
+      filter(integer > 6) %>%
+      summarize(mean = mean(integer))
+  )
+})
+
+test_that("Writing a dataset: Parquet->Parquet (default)", {
+  skip_on_os("windows") # https://issues.apache.org/jira/browse/ARROW-9651
+  ds <- open_dataset(hive_dir)
+  dst_dir <- make_temp_dir()
+  write_dataset(ds, dst_dir, partitioning = "int")
+  expect_true(dir.exists(dst_dir))
+  expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "=")))
+
+  new_ds <- open_dataset(dst_dir)
+
+  expect_equivalent(
+    new_ds %>%
+      select(string = chr, integer = int, group) %>%
+      filter(integer > 6 & group == 1) %>%
+      collect() %>%
+      summarize(mean = mean(integer)),
+    df1 %>%
+      select(string = chr, integer = int) %>%
+      filter(integer > 6) %>%
+      summarize(mean = mean(integer))
+  )
+})
+
 test_that("Dataset writing: dplyr methods", {
   skip_on_os("windows") # https://issues.apache.org/jira/browse/ARROW-9651
   ds <- open_dataset(hive_dir)
   dst_dir <- tempfile()
   # Specify partition vars by group_by
-  ds %>% group_by(int) %>% write_dataset(dst_dir, format = "feather")
+  ds %>%
+    group_by(int) %>%
+    write_dataset(dst_dir, format = "feather")
   expect_true(dir.exists(dst_dir))
   expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "=")))
 
   # select to specify schema
-  skip("TODO: select to specify schema")
-  ds %>% group_by(int) %>% select(lgl, chr) %>% write_dataset(dst_dir, format = "feather")
-  new_ds <- open_dataset(dst_dir, format = "feather")
+  dst_dir2 <- tempfile()
+  ds %>%
+    group_by(int) %>%
+    select(chr, dbl) %>%
+    write_dataset(dst_dir2, format = "feather")
+  new_ds <- open_dataset(dst_dir2, format = "feather")
+
+  expect_equivalent(
+    collect(new_ds) %>% arrange(int),
+    rbind(df1[c("chr", "dbl", "int")], df2[c("chr", "dbl", "int")])
+  )
+
+  # filter to restrict written rows
+  dst_dir3 <- tempfile()
+  ds %>%
+    filter(int == 4) %>%
+    write_dataset(dst_dir3, format = "feather")
+  new_ds <- open_dataset(dst_dir3, format = "feather")
 
   expect_equivalent(
-    collect(new_ds),
-    rbind(df1[c("lgl", "chr", "int")], df2[c("lgl", "chr", "int")])
+    new_ds %>% select(names(df1)) %>% collect(),
+    df1 %>% filter(int == 4)
   )
 })
 
@@ -797,22 +890,144 @@ test_that("Dataset writing: no partitioning", {
   dst_dir <- tempfile()
   write_dataset(ds, dst_dir, format = "feather", partitioning = NULL)
   expect_true(dir.exists(dst_dir))
-  expect_true(length(dir(dst_dir)) > 1)
+  expect_true(length(dir(dst_dir)) > 0)
+})
+
+test_that("Dataset writing: from data.frame", {
+  skip_on_os("windows") # https://issues.apache.org/jira/browse/ARROW-9651
+  dst_dir <- tempfile()
+  stacked <- rbind(df1, df2)
+  stacked %>%
+    group_by(int) %>%
+    write_dataset(dst_dir, format = "feather")
+  expect_true(dir.exists(dst_dir))
+  expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "=")))
+
+  new_ds <- open_dataset(dst_dir, format = "feather")
+
+  expect_equivalent(
+    new_ds %>%
+      select(string = chr, integer = int) %>%
+      filter(integer > 6 & integer < 11) %>%
+      collect() %>%
+      summarize(mean = mean(integer)),
+    df1 %>%
+      select(string = chr, integer = int) %>%
+      filter(integer > 6) %>%
+      summarize(mean = mean(integer))
+  )
+})
+
+test_that("Dataset writing: from RecordBatch", {
+  skip_on_os("windows") # https://issues.apache.org/jira/browse/ARROW-9651
+  dst_dir <- tempfile()
+  stacked <- record_batch(rbind(df1, df2))
+  stacked %>%
+    group_by(int) %>%
+    write_dataset(dst_dir, format = "feather")
+  expect_true(dir.exists(dst_dir))
+  expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "=")))
+
+  new_ds <- open_dataset(dst_dir, format = "feather")
+
+  expect_equivalent(
+    new_ds %>%
+      select(string = chr, integer = int) %>%
+      filter(integer > 6 & integer < 11) %>%
+      collect() %>%
+      summarize(mean = mean(integer)),
+    df1 %>%
+      select(string = chr, integer = int) %>%
+      filter(integer > 6) %>%
+      summarize(mean = mean(integer))
+  )
+})
+
+test_that("Writing a dataset: Ipc format options & compression", {
+  skip_on_os("windows") # https://issues.apache.org/jira/browse/ARROW-9651
+  ds <- open_dataset(csv_dir, partitioning = "part", format = "csv")
+  dst_dir <- make_temp_dir()
+
+  codec <- NULL
+  if (codec_is_available("zstd")) {
+    codec <- Codec$create("zstd")
+  }
+
+  write_dataset(ds, dst_dir, format = "feather", codec = codec)
+  expect_true(dir.exists(dst_dir))
+
+  new_ds <- open_dataset(dst_dir, format = "feather")
+  expect_equivalent(
+    new_ds %>%
+      select(string = chr, integer = int) %>%
+      filter(integer > 6 & integer < 11) %>%
+      collect() %>%
+      summarize(mean = mean(integer)),
+    df1 %>%
+      select(string = chr, integer = int) %>%
+      filter(integer > 6) %>%
+      summarize(mean = mean(integer))
+  )
+})
+
+test_that("Writing a dataset: Parquet format options", {
+  skip_on_os("windows") # https://issues.apache.org/jira/browse/ARROW-9651
+  ds <- open_dataset(csv_dir, partitioning = "part", format = "csv")
+  dst_dir <- make_temp_dir()
+  dst_dir_no_truncated_timestamps <- make_temp_dir()
+
+  # Use trace() to confirm that options are passed in
+  trace(
+    "parquet___ArrowWriterProperties___create",
+    tracer = quote(warning("allow_truncated_timestamps == ", allow_truncated_timestamps)),
+    print = FALSE,
+    where = write_dataset
+  )
+  expect_warning(
+    write_dataset(ds, dst_dir_no_truncated_timestamps, format = "parquet", partitioning = "int"),
+    "allow_truncated_timestamps == FALSE"
+  )
+  expect_warning(
+    write_dataset(ds, dst_dir, format = "parquet", partitioning = "int", allow_truncated_timestamps = TRUE),
+    "allow_truncated_timestamps == TRUE"
+  )
+  untrace("parquet___ArrowWriterProperties___create", where = write_dataset)
+
+  # Now confirm we can read back what we sent
+  expect_true(dir.exists(dst_dir))
+  expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "=")))
+
+  new_ds <- open_dataset(dst_dir)
+
+  expect_equivalent(
+    new_ds %>%
+      select(string = chr, integer = int) %>%
+      filter(integer > 6 & integer < 11) %>%
+      collect() %>%
+      summarize(mean = mean(integer)),
+    df1 %>%
+      select(string = chr, integer = int) %>%
+      filter(integer > 6) %>%
+      summarize(mean = mean(integer))
+  )
 })
 
 test_that("Dataset writing: unsupported features/input validation", {
-  expect_error(write_dataset(4), "'dataset' must be a Dataset")
+  expect_error(write_dataset(4), 'dataset must be a "Dataset"')
 
   ds <- open_dataset(hive_dir)
-
-  expect_error(write_dataset(ds, format = "csv"), "Unsupported format")
   expect_error(
-    filter(ds, int == 4) %>% write_dataset(ds),
-    "Writing a filtered dataset is not yet supported"
+    select(ds, integer = int) %>% write_dataset(ds),
+    "Renaming columns when writing a dataset is not yet supported"
   )
-
   expect_error(
     write_dataset(ds, partitioning = c("int", "NOTACOLUMN"), format = "ipc"),
     'Invalid field name: "NOTACOLUMN"'
   )
+  expect_error(
+    write_dataset(ds, tempfile(), basename_template = "something_without_i")
+  )
+  expect_error(
+    write_dataset(ds, tempfile(), basename_template = NULL)
+  )
 })
diff --git a/r/tests/testthat/test-filesystem.R b/r/tests/testthat/test-filesystem.R
index fc0e02f34a1..ff1f3feca3d 100644
--- a/r/tests/testthat/test-filesystem.R
+++ b/r/tests/testthat/test-filesystem.R
@@ -80,10 +80,14 @@ test_that("SubTreeFilesystem", {
   DESCRIPTION <- system.file("DESCRIPTION", package = "arrow")
   file.copy(DESCRIPTION, file.path(td, "DESCRIPTION"))
 
-  local_fs <- LocalFileSystem$create()
-  st_fs <- SubTreeFileSystem$create(td, local_fs)
+  st_fs <- SubTreeFileSystem$create(td)
   expect_is(st_fs, "SubTreeFileSystem")
   expect_is(st_fs, "FileSystem")
+  expect_is(st_fs$base_fs, "LocalFileSystem")
+
+  # FIXME windows has a trailing slash for one but not the other
+  # expect_identical(normalizePath(st_fs$base_path), normalizePath(td))
+
   st_fs$CreateDir("test")
   st_fs$CopyFile("DESCRIPTION", "DESC.txt")
   infos <- st_fs$GetFileInfo(c("DESCRIPTION", "test", "nope", "DESC.txt"))
@@ -93,6 +97,7 @@ test_that("SubTreeFilesystem", {
   expect_equal(infos[[4L]]$type, FileType$File)
   expect_equal(infos[[4L]]$extension(), "txt")
 
+  local_fs <- LocalFileSystem$create()
   local_fs$DeleteDirContents(td)
   infos <- st_fs$GetFileInfo(c("DESCRIPTION", "test", "nope", "DESC.txt"))
   expect_equal(infos[[1L]]$type, FileType$NotFound)
@@ -129,6 +134,14 @@ test_that("FileSystem$from_uri", {
   skip_if_not_available("s3")
   fs_and_path <- FileSystem$from_uri("s3://ursa-labs-taxi-data")
   expect_is(fs_and_path$fs, "S3FileSystem")
+  expect_identical(fs_and_path$fs$region, "us-east-2")
+})
+
+test_that("SubTreeFileSystem$create() with URI", {
+  skip_on_cran()
+  skip_if_not_available("s3")
+  fs <- SubTreeFileSystem$create("s3://ursa-labs-taxi-data")
+  expect_is(fs, "SubTreeFileSystem")
 })
 
 test_that("S3FileSystem", {
@@ -137,3 +150,14 @@ test_that("S3FileSystem", {
   s3fs <- S3FileSystem$create()
   expect_is(s3fs, "S3FileSystem")
 })
+
+test_that("s3_bucket", {
+  skip_on_cran()
+  skip_if_not_available("s3")
+  bucket <- s3_bucket("ursa-labs-r-test")
+  expect_is(bucket, "SubTreeFileSystem")
+  expect_is(bucket$base_fs, "S3FileSystem")
+  expect_identical(bucket$region, "us-west-2")
+  skip_on_os("windows") # FIXME
+  expect_identical(bucket$base_path, "ursa-labs-r-test/")
+})
diff --git a/r/tests/testthat/test-install-arrow.R b/r/tests/testthat/test-install-arrow.R
index cebaeeff4f7..8021e2ba10e 100644
--- a/r/tests/testthat/test-install-arrow.R
+++ b/r/tests/testthat/test-install-arrow.R
@@ -19,11 +19,16 @@ context("install_arrow()")
 
 r_only({
   test_that("arrow_repos", {
-    old <- options(repos=c(CRAN = "@CRAN@")) # Restore default
-    on.exit(options(old))
     cran <- "https://cloud.r-project.org/"
-    bt <- "https://dl.bintray.com/ursalabs/arrow-r"
+    bt <- "https://dl.bintray.com/ursalabs/fake_repo"
     other <- "https://cran.fiocruz.br/"
+
+    old <- options(
+      repos=c(CRAN = "@CRAN@"),  # Restore defaul
+      arrow.dev_repo = bt
+    )
+    on.exit(options(old))
+
     expect_identical(arrow_repos(), cran)
     expect_identical(arrow_repos(c(cran, bt)), cran)
     expect_identical(arrow_repos(c(bt, other)), other)
diff --git a/r/tests/testthat/test-metadata.R b/r/tests/testthat/test-metadata.R
index d599183ae85..1cd6fbc4599 100644
--- a/r/tests/testthat/test-metadata.R
+++ b/r/tests/testthat/test-metadata.R
@@ -61,6 +61,7 @@ test_that("Table R metadata", {
 test_that("R metadata is not stored for types that map to Arrow types (factor, Date, etc.)", {
   tab <- Table$create(example_data[1:6])
   expect_null(tab$metadata$r)
+
   expect_null(Table$create(example_with_times[1:3])$metadata$r)
 })
 
@@ -126,3 +127,10 @@ test_that("Date/time type roundtrip", {
   expect_is(rb$schema$posixlt$type, "StructType")
   expect_identical(as.data.frame(rb), example_with_times)
 })
+
+test_that("metadata keeps attribute of top level data frame", {
+  df <- structure(data.frame(x = 1, y = 2), foo = "bar")
+  tab <- Table$create(df)
+  expect_identical(attr(as.data.frame(tab), "foo"), "bar")
+  expect_identical(as.data.frame(tab), df)
+})
diff --git a/r/tests/testthat/test-parquet.R b/r/tests/testthat/test-parquet.R
index 7ddf4ccbb4c..a59f2eadb4e 100644
--- a/r/tests/testthat/test-parquet.R
+++ b/r/tests/testthat/test-parquet.R
@@ -191,3 +191,38 @@ test_that("write_parquet() handles version argument", {
     expect_error(write_parquet(df, tf, version = .x))
   })
 })
+
+test_that("ParquetFileWriter raises an error for non-OutputStream sink", {
+  sch = schema(a = float32())
+  # ARROW-9946
+  expect_error(
+    ParquetFileWriter$create(schema = sch, sink = tempfile()),
+    regex = "OutputStream"
+  )
+})
+
+test_that("ParquetFileReader $ReadRowGroup(s) methods", {
+  tab <- Table$create(x = 1:100)
+  tf <- tempfile(); on.exit(unlink(tf))
+  write_parquet(tab, tf, chunk_size = 10)
+
+  reader <- ParquetFileReader$create(tf)
+  expect_true(reader$ReadRowGroup(0) == Table$create(x = 1:10))
+  expect_true(reader$ReadRowGroup(9) == Table$create(x = 91:100))
+  expect_error(reader$ReadRowGroup(-1), "Some index in row_group_indices")
+  expect_error(reader$ReadRowGroup(111), "Some index in row_group_indices")
+  expect_error(reader$ReadRowGroup(c(1, 2)))
+  expect_error(reader$ReadRowGroup("a"))
+
+  expect_true(reader$ReadRowGroups(c(0, 1)) == Table$create(x = 1:20))
+  expect_error(reader$ReadRowGroups(c(0, 1, -2))) # although it gives a weird error
+  expect_error(reader$ReadRowGroups(c(0, 1, 31))) # ^^
+  expect_error(reader$ReadRowGroups(c("a", "b")))
+
+  ## -- with column_indices
+  expect_true(reader$ReadRowGroup(0, 0) == Table$create(x = 1:10))
+  expect_error(reader$ReadRowGroup(0, 1))
+
+  expect_true(reader$ReadRowGroups(c(0, 1), 0) == Table$create(x = 1:20))
+  expect_error(reader$ReadRowGroups(c(0, 1), 1))
+})
diff --git a/r/tests/testthat/test-python.R b/r/tests/testthat/test-python.R
index df910169052..6184f69c923 100644
--- a/r/tests/testthat/test-python.R
+++ b/r/tests/testthat/test-python.R
@@ -25,6 +25,8 @@ test_that("install_pyarrow", {
   # Bail out if virtualenv isn't available
   skip_if(inherits(venv, "try-error"))
   expect_error(install_pyarrow("arrow-test", nightly = TRUE), NA)
+  # Set this up for the following tests
+  reticulate::use_virtualenv("arrow-test")
 })
 
 test_that("Array from Python", {
diff --git a/r/tests/testthat/test-record-batch-reader.R b/r/tests/testthat/test-record-batch-reader.R
index e03664e82f7..d9c34068425 100644
--- a/r/tests/testthat/test-record-batch-reader.R
+++ b/r/tests/testthat/test-record-batch-reader.R
@@ -82,7 +82,7 @@ test_that("MetadataFormat", {
   Sys.setenv(ARROW_PRE_0_15_IPC_FORMAT = 1)
   expect_identical(get_ipc_metadata_version(NULL), 3L)
   Sys.setenv(ARROW_PRE_0_15_IPC_FORMAT = "")
-  
+
   expect_identical(get_ipc_metadata_version(NULL), 4L)
   Sys.setenv(ARROW_PRE_1_0_METADATA_VERSION = 1)
   expect_identical(get_ipc_metadata_version(NULL), 3L)
diff --git a/r/tests/testthat/test-s3-minio.R b/r/tests/testthat/test-s3-minio.R
new file mode 100644
index 00000000000..01d2d057e28
--- /dev/null
+++ b/r/tests/testthat/test-s3-minio.R
@@ -0,0 +1,191 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+context("S3 tests using local minio")
+
+if (arrow_with_s3() && process_is_running("minio server")) {
+  # Get minio config, with expected defaults
+  minio_key <- Sys.getenv("MINIO_ACCESS_KEY", "minioadmin")
+  minio_secret <- Sys.getenv("MINIO_SECRET_KEY", "minioadmin")
+  minio_port <- Sys.getenv("MINIO_PORT", "9000")
+
+  # Helper function for minio URIs
+  minio_uri <- function(...) {
+    template <- "s3://%s:%s@%s?scheme=http&endpoint_override=localhost%s%s"
+    sprintf(template, minio_key, minio_secret, minio_path(...), "%3A", minio_port)
+  }
+  minio_path <- function(...) paste(now, ..., sep = "/")
+
+  test_that("minio setup", {
+    # Create a "bucket" on minio for this test run, which we'll delete when done.
+    fs <- S3FileSystem$create(
+      access_key = minio_key,
+      secret_key = minio_secret,
+      scheme = "http",
+      endpoint_override = paste0("localhost:", minio_port)
+    )
+    expect_is(fs, "S3FileSystem")
+    now <- as.character(as.numeric(Sys.time()))
+    # If minio isn't running, this will hang for a few seconds and fail with a
+    # curl timeout, causing `run_these` to be set to FALSE and skipping the tests
+    fs$CreateDir(now)
+  })
+  # Clean up when we're all done
+  on.exit(fs$DeleteDir(now))
+
+  test_that("read/write Feather on minio", {
+    write_feather(example_data, minio_uri("test.feather"))
+    expect_identical(read_feather(minio_uri("test.feather")), example_data)
+  })
+
+  test_that("read/write Feather by filesystem, not URI", {
+    write_feather(example_data, fs$path(minio_path("test2.feather")))
+    expect_identical(
+      read_feather(fs$path(minio_path("test2.feather"))),
+      example_data
+    )
+  })
+
+  test_that("read/write stream", {
+    write_ipc_stream(example_data, fs$path(minio_path("test3.ipc")))
+    expect_identical(
+      read_ipc_stream(fs$path(minio_path("test3.ipc"))),
+      example_data
+    )
+  })
+
+  test_that("read/write Parquet on minio", {
+    write_parquet(example_data, fs$path(minio_uri("test.parquet")))
+    expect_identical(read_parquet(minio_uri("test.parquet")), example_data)
+  })
+
+  # Dataset test setup, cf. test-dataset.R
+  library(dplyr)
+  first_date <- lubridate::ymd_hms("2015-04-29 03:12:39")
+  df1 <- tibble(
+    int = 1:10,
+    dbl = as.numeric(1:10),
+    lgl = rep(c(TRUE, FALSE, NA, TRUE, FALSE), 2),
+    chr = letters[1:10],
+    fct = factor(LETTERS[1:10]),
+    ts = first_date + lubridate::days(1:10)
+  )
+
+  second_date <- lubridate::ymd_hms("2017-03-09 07:01:02")
+  df2 <- tibble(
+    int = 101:110,
+    dbl = as.numeric(51:60),
+    lgl = rep(c(TRUE, FALSE, NA, TRUE, FALSE), 2),
+    chr = letters[10:1],
+    fct = factor(LETTERS[10:1]),
+    ts = second_date + lubridate::days(10:1)
+  )
+
+  # This is also to set up the dataset tests
+  test_that("write_parquet with filesystem arg", {
+    fs$CreateDir(minio_path("hive_dir", "group=1", "other=xxx"))
+    fs$CreateDir(minio_path("hive_dir", "group=2", "other=yyy"))
+    expect_length(fs$ls(minio_path("hive_dir")), 2)
+    write_parquet(df1, fs$path(minio_path("hive_dir", "group=1", "other=xxx", "file1.parquet")))
+    write_parquet(df2, fs$path(minio_path("hive_dir", "group=2", "other=yyy", "file2.parquet")))
+    expect_identical(
+      read_parquet(fs$path(minio_path("hive_dir", "group=1", "other=xxx", "file1.parquet"))),
+      df1
+    )
+  })
+
+  test_that("open_dataset with fs", {
+    ds <- open_dataset(fs$path(minio_path("hive_dir")))
+    expect_identical(
+      ds %>% select(dbl, lgl) %>% collect(),
+      rbind(df1[, c("dbl", "lgl")], df2[, c("dbl", "lgl")])
+    )
+  })
+
+  test_that("write_dataset with fs", {
+    ds <- open_dataset(fs$path(minio_path("hive_dir")))
+    write_dataset(ds, fs$path(minio_path("new_dataset_dir")))
+    expect_length(fs$ls(minio_path("new_dataset_dir")), 1)
+  })
+
+  make_temp_dir <- function() {
+    path <- tempfile()
+    dir.create(path)
+    normalizePath(path, winslash = "/")
+  }
+
+  test_that("Let's test copy_files too", {
+    td <- make_temp_dir()
+    copy_files(minio_uri("hive_dir"), td)
+    expect_length(dir(td), 2)
+    ds <- open_dataset(td)
+    expect_identical(
+      ds %>% select(dbl, lgl) %>% collect(),
+      rbind(df1[, c("dbl", "lgl")], df2[, c("dbl", "lgl")])
+    )
+
+    # Let's copy the other way and use a SubTreeFileSystem rather than URI
+    copy_files(td, fs$path(minio_path("hive_dir2")))
+    ds2 <- open_dataset(fs$path(minio_path("hive_dir2")))
+    expect_identical(
+      ds2 %>% select(dbl, lgl) %>% collect(),
+      rbind(df1[, c("dbl", "lgl")], df2[, c("dbl", "lgl")])
+    )
+  })
+
+
+  test_that("S3FileSystem input validation", {
+    expect_error(
+      S3FileSystem$create(access_key = "foo"),
+      "Key authentication requires both access_key and secret_key"
+    )
+    expect_error(
+      S3FileSystem$create(secret_key = "foo"),
+      "Key authentication requires both access_key and secret_key"
+    )
+    expect_error(
+      S3FileSystem$create(session_token = "foo"),
+      paste0(
+        "In order to initialize a session with temporary credentials, ",
+        "both secret_key and access_key must be provided ",
+        "in addition to session_token."
+      )
+    )
+    expect_error(
+      S3FileSystem$create(access_key = "foo", secret_key = "asdf", anonymous = TRUE),
+      'Cannot specify "access_key" and "secret_key" when anonymous = TRUE'
+    )
+    expect_error(
+      S3FileSystem$create(access_key = "foo", secret_key = "asdf", role_arn = "qwer"),
+      "Cannot provide both key authentication and role_arn"
+    )
+    expect_error(
+      S3FileSystem$create(access_key = "foo", secret_key = "asdf", external_id = "qwer"),
+      'Cannot specify "external_id" without providing a role_arn string'
+    )
+    expect_error(
+      S3FileSystem$create(external_id = "foo"),
+      'Cannot specify "external_id" without providing a role_arn string'
+    )
+  })
+} else {
+  # Kinda hacky, let's put a skipped test here, just so we note that the tests
+  # didn't run
+  test_that("S3FileSystem tests with Minio", {
+    skip("Minio is not running")
+  })
+}
diff --git a/r/tests/testthat/test-s3.R b/r/tests/testthat/test-s3.R
new file mode 100644
index 00000000000..33c249547a6
--- /dev/null
+++ b/r/tests/testthat/test-s3.R
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+context("S3 integration tests")
+
+run_these <- tryCatch({
+  if (arrow_with_s3() &&
+      identical(tolower(Sys.getenv("ARROW_R_DEV")), "true") &&
+      !identical(Sys.getenv("AWS_ACCESS_KEY_ID"), "") &&
+      !identical(Sys.getenv("AWS_SECRET_ACCESS_KEY"), "")) {
+    # See if we have access to the test bucket
+    bucket <- s3_bucket("ursa-labs-r-test")
+    bucket$GetFileInfo("")
+    TRUE
+  } else {
+    FALSE
+  }
+}, error = function(e) FALSE)
+
+bucket_uri <- function(..., bucket = "s3://ursa-labs-r-test/%s?region=us-west-2") {
+  segments <- paste(..., sep = "/")
+  sprintf(bucket, segments)
+}
+
+if (run_these) {
+  now <- as.numeric(Sys.time())
+  on.exit(bucket$DeleteDir(now))
+
+  test_that("read/write Feather on S3", {
+    write_feather(example_data, bucket_uri(now, "test.feather"))
+    expect_identical(read_feather(bucket_uri(now, "test.feather")), example_data)
+  })
+
+  test_that("read/write Parquet on S3", {
+    write_parquet(example_data, bucket_uri(now, "test.parquet"))
+    expect_identical(read_parquet(bucket_uri(now, "test.parquet")), example_data)
+  })
+}
diff --git a/r/tools/autobrew b/r/tools/autobrew
index 7dcfffb22d1..07a68c50c7f 100644
--- a/r/tools/autobrew
+++ b/r/tools/autobrew
@@ -40,14 +40,15 @@ if [ -f "$LOCAL_FORMULA" ]; then
   $BREW deps -n "$LOCAL_FORMULA" 2>/dev/null
   BREW_DEPS=$($BREW deps -n "$LOCAL_FORMULA" 2>/dev/null)
   $BREW install --force-bottle $BREW_DEPS 2>&1 | perl -pe 's/Warning/Note/gi'
-  $BREW install --build-from-source --HEAD "$LOCAL_FORMULA" 2>&1 | perl -pe 's/Warning/Note/gi'
+  $BREW install -v --build-from-source --HEAD "$LOCAL_FORMULA" 2>&1 | perl -pe 's/Warning/Note/gi'
 else
   $BREW install --force-bottle $BREW_DEPS $PKG_BREW_NAME 2>&1 | perl -pe 's/Warning/Note/gi'
 fi
 
 # Hardcode this for my custom autobrew build
 rm -f $BREWDIR/lib/*.dylib
-PKG_LIBS="-L$BREWDIR/lib -lparquet -larrow_dataset -larrow -lthrift -llz4 -lsnappy"
+AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-identity-management -laws-cpp-sdk-cognito-identity -laws-cpp-sdk-sts -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -lpthread -lcurl"
+PKG_LIBS="-L$BREWDIR/lib -lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -llz4 -lsnappy -lzstd $AWS_LIBS"
 
 # Prevent CRAN builder from linking against old libs in /usr/local/lib
 for FILE in $BREWDIR/Cellar/*/*/lib/*.a; do
@@ -58,5 +59,7 @@ for FILE in $BREWDIR/Cellar/*/*/lib/*.a; do
   PKG_LIBS=`echo $PKG_LIBS | sed "s/-l$LIBNAME/-lbrew$LIBNAME/g"`
 done
 
+PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include"
+
 unset HOMEBREW_NO_ANALYTICS
 unset HOMEBREW_NO_AUTO_UPDATE
diff --git a/r/tools/linuxlibs.R b/r/tools/linuxlibs.R
index 65fe46b6850..814316de254 100644
--- a/r/tools/linuxlibs.R
+++ b/r/tools/linuxlibs.R
@@ -55,6 +55,12 @@ download_binary <- function(os = identify_os()) {
     binary_url <- paste0(arrow_repo, "bin/", os, "/arrow-", VERSION, ".zip")
     if (try_download(binary_url, libfile)) {
       cat(sprintf("*** Successfully retrieved C++ binaries for %s\n", os))
+      if (!identical(os, "centos-7")) {
+        # centos-7 uses gcc 4.8 so the binary doesn't have ARROW_S3=ON but the others do
+        # TODO: actually check for system requirements?
+        cat("**** Binary package requires libcurl and openssl\n")
+        cat("**** If installation fails, retry after installing those system requirements\n")
+      }
     } else {
       cat(sprintf("*** No C++ binaries found for %s\n", os))
       libfile <- NULL
@@ -256,7 +262,7 @@ find_local_source <- function(arrow_home = Sys.getenv("ARROW_HOME", "..")) {
 
 build_libarrow <- function(src_dir, dst_dir) {
   # We'll need to compile R bindings with these libs, so delete any .o files
-  system("rm src/*.o", ignore.stdout = quietly, ignore.stderr = quietly)
+  system("rm src/*.o", ignore.stdout = TRUE, ignore.stderr = TRUE)
   # Set up make for parallel building
   makeflags <- Sys.getenv("MAKEFLAGS")
   if (makeflags == "") {
@@ -298,10 +304,8 @@ build_libarrow <- function(src_dir, dst_dir) {
     # CXXFLAGS = R_CMD_config("CXX11FLAGS"), # We don't want the same debug symbols
     LDFLAGS = R_CMD_config("LDFLAGS")
   )
-  env_vars <- paste(
-    names(env_var_list), dQuote(env_var_list, FALSE),
-    sep = "=", collapse = " "
-  )
+  env_vars <- paste0(names(env_var_list), '="', env_var_list, '"', collapse = " ")
+  env_vars <- with_s3_support(env_vars)
   cat("**** arrow", ifelse(quietly, "", paste("with", env_vars)), "\n")
   status <- system(
     paste(env_vars, "inst/build_arrow_static.sh"),
@@ -369,6 +373,47 @@ cmake_version <- function(cmd = "cmake") {
   )
 }
 
+with_s3_support <- function(env_vars) {
+  arrow_s3 <- toupper(Sys.getenv("ARROW_S3")) == "ON" || tolower(Sys.getenv("LIBARROW_MINIMAL")) == "false"
+  if (arrow_s3) {
+    # User wants S3 support. Let's make sure they're not on gcc < 4.9
+    # and make sure that we have curl and openssl system libs
+    info <- system(paste(env_vars, "&& $CMAKE --system-information"), intern = TRUE)
+    info <- grep("^[A-Z_]* .*$", info, value = TRUE)
+    vals <- as.list(sub('^.*? "?(.*?)"?$', "\\1", info))
+    names(vals) <- sub("^(.*?) .*$", "\\1", info)
+    if (vals[["CMAKE_CXX_COMPILER_ID"]] == "GNU" &&
+        package_version(vals[["CMAKE_CXX_COMPILER_VERSION"]]) < 4.9) {
+      cat("**** S3 support not available for gcc < 4.9; building with ARROW_S3=OFF\n")
+      arrow_s3 <- FALSE
+    } else if (!cmake_find_package("CURL", NULL, env_vars)) {
+      cat("**** S3 support requires libcurl-devel (rpm) or libcurl4-openssl-dev (deb); building with ARROW_S3=OFF\n")
+      arrow_s3 <- FALSE
+    } else if (!cmake_find_package("OpenSSL", "1.0.2", env_vars)) {
+      cat("**** S3 support requires openssl-devel (rpm) or libssl-dev (deb), version >= 1.0.2; building with ARROW_S3=OFF\n")
+      arrow_s3 <- FALSE
+    }
+  }
+  paste(env_vars, ifelse(arrow_s3, "ARROW_S3=ON", "ARROW_S3=OFF"))
+}
+
+cmake_find_package <- function(pkg, version = NULL, env_vars) {
+  td <- tempfile()
+  dir.create(td)
+  options(.arrow.cleanup = c(getOption(".arrow.cleanup"), td))
+  find_package <- paste0("find_package(", pkg, " ", version, " REQUIRED)")
+  writeLines(find_package, file.path(td, "CMakeLists.txt"))
+  cmake_cmd <- paste0(
+    env_vars,
+    " && cd ", td,
+    " && $CMAKE",
+    " -DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON",
+    " -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON",
+    " ."
+  )
+  system(cmake_cmd, ignore.stdout = TRUE, ignore.stderr = TRUE) == 0
+}
+
 #####
 
 if (!file.exists(paste0(dst_dir, "/include/arrow/api.h"))) {
diff --git a/r/vignettes/dataset.Rmd b/r/vignettes/dataset.Rmd
index 2218d88f0fe..06653eef9d8 100644
--- a/r/vignettes/dataset.Rmd
+++ b/r/vignettes/dataset.Rmd
@@ -20,15 +20,35 @@ and what is on the immediate development roadmap.
 The [New York City taxi trip record data](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page)
 is widely used in big data exercises and competitions.
 For demonstration purposes, we have hosted a Parquet-formatted version
-of about 10 years of the trip data in a public S3 bucket.
+of about 10 years of the trip data in a public AWS S3 bucket.
 
 The total file size is around 37 gigabytes, even in the efficient Parquet file format.
 That's bigger than memory on most people's computers,
 so we can't just read it all in and stack it into a single data frame.
 
-In a future release, you'll be able to point your R session at S3 and query
-the dataset from there. For now, datasets need to be on your local file system.
-To download the files,
+In Windows and macOS binary packages, S3 support is included.
+On Linux when installing from source, S3 support is not enabled by default,
+and it has additional system requirements.
+See `vignette("install", package = "arrow")` for details.
+To see if your `arrow` installation has S3 support, run
+
+```{r}
+arrow::arrow_with_s3()
+```
+
+Even with S3 support enabled network, speed will be a bottleneck unless your
+machine is located in the same AWS region as the data. So, for this vignette,
+we assume that the NYC taxi dataset has been downloaded locally in a "nyc-taxi"
+directory.
+
+If your `arrow` build has S3 support, you can sync the data locally with:
+
+```{r, eval = FALSE}
+arrow::copy_files("s3://ursa-labs-taxi-data", "nyc-taxi")
+```
+
+If your `arrow` build doesn't have S3 support, you can download the files
+with some additional code:
 
 ```{r, eval = FALSE}
 bucket <- "https://ursa-labs-taxi-data.s3.us-east-2.amazonaws.com"
@@ -55,7 +75,7 @@ for (year in 2009:2019) {
 }
 ```
 
-Note that the vignette will not execute that code chunk: if you want to run
+Note that these download steps in the vignette are not executed: if you want to run
 with live data, you'll have to do it yourself separately.
 Given the size, if you're running this locally and don't have a fast connection,
 feel free to grab only a year or two of data.
@@ -87,8 +107,11 @@ ds <- open_dataset("nyc-taxi", partitioning = c("year", "month"))
 
 The default file format for `open_dataset()` is Parquet; if we had a directory
 of Arrow format files, we could include `format = "arrow"` in the call.
-Future versions will support more file formats, including CSV/delimited text data
-and JSON.
+Other supported formats include: "feather" (an alias for "arrow", as Feather v2
+is the Arrow file format), "csv", "tsv" (for tab-delimited), and "text" for
+generic text-delimited files. For text files, you can pass any parsing options
+("delim", "quote", etc.) to `open_dataset()` that you would otherwise pass to
+`read_csv_arrow()`.
 
 The `partitioning` argument lets us specify how the file paths provide information
 about how the dataset is chunked into different files. Our files in this example
@@ -257,7 +280,7 @@ rows match the filter. Relatedly, since Parquet files contain row groups with
 statistics on the data within, there may be entire chunks of data we can
 avoid scanning because they have no rows where `total_amount > 100`.
 
-## Going farther
+## More dataset options
 
 There are a few ways you can control the Dataset creation to adapt to special use cases.
 For one, you can specify a `schema` argument to declare the columns and their data types.
@@ -276,8 +299,91 @@ This would be useful, in our taxi dataset example, if you wanted to keep
 Another feature of Datasets is that they can be composed of multiple data sources.
 That is, you may have a directory of partitioned Parquet files in one location,
 and in another directory, files that haven't been partitioned.
-In the future, when there is support for cloud storage and other file formats,
-this would mean you could point to an S3 bucked of Parquet data and a directory
+Or, you could point to an S3 bucket of Parquet data and a directory
 of CSVs on the local file system and query them together as a single dataset.
 To create a multi-source dataset, provide a list of datasets to `open_dataset()`
 instead of a file path, or simply concatenate them like `big_dataset <- c(ds1, ds2)`.
+
+## Writing datasets
+
+As you can see, querying a large dataset can be made quite fast by storage in an
+efficient binary columnar format like Parquet or Feather and partitioning based on
+columns commonly used for filtering. However, we don't always get our data delivered
+to us that way. Sometimes we start with one giant CSV. Our first step in analyzing data
+is cleaning is up and reshaping it into a more usable form.
+
+The `write_dataset()` function allows you to take a Dataset or other tabular data object---an Arrow `Table` or `RecordBatch`, or an R `data.frame`---and write it to a different file format, partitioned into multiple files.
+
+Assume we have a version of the NYC Taxi data as CSV:
+
+```r
+ds <- open_dataset("nyc-taxi/csv/", format = "csv")
+```
+
+We can write it to a new location and translate the files to the Feather format
+by calling `write_dataset()` on it:
+
+```r
+write_dataset(ds, "nyc-taxi/feather", format = "feather")
+```
+
+Next, let's imagine that the "payment_type" column is something we often filter on,
+so we want to partition the data by that variable. By doing so we ensure that a filter like
+`payment_type == 3` will touch only a subset of files where payment_type is always 3.
+
+One natural way to express the columns you want to partition on is to use the
+`group_by()` method:
+
+```r
+ds %>%
+  group_by(payment_type) %>%
+  write_dataset("nyc-taxi/feather", format = "feather")
+```
+
+This will write files to a directory tree that looks like this:
+
+```r
+system("tree nyc-taxi/feather")
+
+# feather
+# ├── payment_type=1
+# │   └── part-5.feather
+# ├── payment_type=2
+# │   └── part-0.feather
+# ...
+# └── payment_type=5
+#     └── part-2.feather
+#
+# 5 directories, 25 files
+```
+
+Note that the directory names are `payment_type=1` and similar:
+this is the Hive-style partitioning described above. This means that when
+we call `open_dataset()` on this directory, we don't have to declare what the
+partitions are because they can be read from the file paths.
+(To instead write bare values for partition segments,
+i.e. `1` rather than `payment_type=1`, call `write_dataset()` with `hive_style = FALSE`.)
+
+Perhaps, though, `payment_type == 3` is the only data we ever care about,
+and we just want to drop the rest and have a smaller working set.
+For this, we can `filter()` them out when writing:
+
+```r
+ds %>%
+  filter(payment_type == 3) %>%
+  write_dataset("nyc-taxi/feather", format = "feather")
+```
+
+The other thing we can do when writing datasets is select a subset of and/or reorder
+columns. Suppose we never care about `vendor_id`, and being a string column,
+it can take up a lot of space when we read it in, so let's drop it:
+
+```r
+ds %>%
+  group_by(payment_type) %>%
+  select(-vendor_id) %>%
+  write_dataset("nyc-taxi/feather", format = "feather")
+```
+
+Note that while you can select a subset of columns,
+you cannot currently rename columns when writing.
diff --git a/r/vignettes/flight.Rmd b/r/vignettes/flight.Rmd
new file mode 100644
index 00000000000..202230fa0e1
--- /dev/null
+++ b/r/vignettes/flight.Rmd
@@ -0,0 +1,78 @@
+---
+title: "Connecting to Flight RPC Servers"
+output: rmarkdown::html_vignette
+vignette: >
+  %\VignetteIndexEntry{Connecting to Flight RPC Servers}
+  %\VignetteEngine{knitr::rmarkdown}
+  %\VignetteEncoding{UTF-8}
+---
+
+[**Flight**](https://arrow.apache.org/blog/2019/10/13/introducing-arrow-flight/)
+is a general-purpose client-server framework for high performance
+transport of large datasets over network interfaces, built as part of the
+[Apache Arrow](https://arrow.apache.org) project.
+The `arrow` package provides methods for connecting to Flight RPC servers
+to send and receive data.
+
+## Getting Started
+
+The `flight` functions in the package use `reticulate` to call methods in the
+`pyarrow` Python package. Before using them for the first time,
+you'll need to be sure you have `reticulate`, and you'll also need to
+install `pyarrow`:
+
+```r
+install.packages("reticulate")
+arrow::install_pyarrow()
+```
+
+See `vignette("python", package = "arrow")` for more details on setting up
+`pyarrow`.
+
+## Example
+
+The package includes methods for starting a Python-based Flight server, as well
+as methods for connecting to a Flight server running elsewhere.
+
+To illustrate both sides, in one process let's start a demo server:
+
+```r
+library(arrow)
+demo_server <- load_flight_server("demo_flight_server")
+server <- demo_server$DemoFlightServer(port = 8089)
+server$serve()
+```
+
+We'll leave that one running.
+
+In a different R process, let's connect to it and put some data in it.
+
+```r
+library(arrow)
+client <- flight_connect(port = 8089)
+# Upload some data to our server so there's something to demo
+push_data(client, iris, path = "test_data/iris")
+```
+
+Now, in a new R process, let's connect to the server and pull the data we
+put there:
+
+```r
+library(arrow)
+library(dplyr)
+client <- flight_connect(port = 8089)
+client %>%
+  flight_get("test_data/iris") %>%
+  group_by(Species) %>%
+  summarize(max_petal = max(Petal.Length))
+
+## # A tibble: 3 x 2
+##   Species    max_petal
+##   <fct>          <dbl>
+## 1 setosa           1.9
+## 2 versicolor       5.1
+## 3 virginica        6.9
+```
+
+Because `flight_get()` returns an Arrow data structure, we can directly pipe
+its result into a `dplyr` workflow.
diff --git a/r/vignettes/fs.Rmd b/r/vignettes/fs.Rmd
new file mode 100644
index 00000000000..5d699c49df0
--- /dev/null
+++ b/r/vignettes/fs.Rmd
@@ -0,0 +1,130 @@
+---
+title: "Working with Cloud Storage (S3)"
+output: rmarkdown::html_vignette
+vignette: >
+  %\VignetteIndexEntry{Working with Cloud Storage (S3)}
+  %\VignetteEngine{knitr::rmarkdown}
+  %\VignetteEncoding{UTF-8}
+---
+
+The Arrow C++ library includes a generic filesystem interface and specific
+implementations for some cloud storage systems. This setup allows various
+parts of the project to be able to read and write data with different storage
+backends. In the `arrow` R package, support has been enabled for AWS S3.
+This vignette provides an overview of working with S3 data using Arrow.
+
+> In Windows and macOS binary packages, S3 support is included. On Linux when installing from source, S3 support is not enabled by default, and it has additional system requirements. See `vignette("install", package = "arrow")` for details.
+
+## URIs
+
+File readers and writers (`read_parquet()`, `write_feather()`, et al.)
+accept an S3 URI as the source or destination file,
+as do `open_dataset()` and `write_dataset()`.
+An S3 URI looks like:
+
+```
+s3://[access_key:secret_key@]bucket/path[?region=]
+```
+
+For example, one of the NYC taxi data files used in `vignette("dataset", package = "arrow")` is found at
+
+```
+s3://ursa-labs-taxi-data/2019/06/data.parquet
+```
+
+Given this URI, we can pass it to `read_parquet()` just as if it were a local file path:
+
+```r
+df <- read_parquet("s3://ursa-labs-taxi-data/2019/06/data.parquet")
+```
+
+Note that this will be slower to read than if the file were local,
+though if you're running on a machine in the same AWS region as the file in S3,
+the cost of reading the data over the network should be much lower.
+
+## Creating a FileSystem object
+
+Another way to connect to S3 is to create a `FileSystem` object once and pass
+that to the read/write functions.
+`S3FileSystem` objects can be created with the `s3_bucket()` function, which
+automatically detects the bucket's AWS region. Additionally, the resulting
+`FileSystem` will consider paths relative to the bucket's path (so for example
+you don't need to prefix the bucket path when listing a directory).
+This may be convenient when dealing with
+long URIs, and it's necessary for some options and authentication methods
+that aren't supported in the URI format.
+
+With a `FileSystem` object, we can point to specific files in it with the `$path()` method.
+In the previous example, this would look like:
+
+```r
+bucket <- s3_bucket("ursa-labs-taxi-data")
+df <- read_parquet(bucket$path("2019/06/data.parquet"))
+```
+
+See the help for `FileSystem` for a list of options that `s3_bucket()` and `S3FileSystem$create()`
+can take. `region`, `scheme`, and `endpoint_override` can be encoded as query
+parameters in the URI (though `region` will be auto-detected in `s3_bucket()` or from the URI if omitted).
+`access_key` and `secret_key` can also be included,
+but other options are not supported in the URI.
+
+The object that `s3_bucket()` returns is technically a `SubTreeFileSystem`, which holds a path and a file system to which it corresponds. `SubTreeFileSystem`s can be useful for holding a reference to a subdirectory somewhere, on S3 or elsewhere.
+
+One way to get a subtree is to call the `$cd()` method on a `FileSystem`
+
+```r
+june2019 <- bucket$cd("2019/06")
+df <- read_parquet(june2019$path("data.parquet"))
+```
+
+`SubTreeFileSystem` can also be made from a URI:
+
+```r
+june2019 <- SubTreeFileSystem$create("s3://ursa-labs-taxi-data/2019/06")
+```
+
+## Authentication
+
+To access private S3 buckets, you need typically need two secret parameters:
+a `access_key`, which is like a user id,
+and `secret_key`, like a token.
+There are a few options for passing these credentials:
+
+1. Include them in the URI, like `s3://access_key:secret_key@bucket-name/path/to/file`. Be sure to [URL-encode](https://en.wikipedia.org/wiki/Percent-encoding) your secrets if they contain special characters like "/".
+
+2. Pass them as `access_key` and `secret_key` to `S3FileSystem$create()` or `s3_bucket()`
+
+3. Set them as environment variables named `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, respectively.
+
+4. Define them in a `~/.aws/credentials` file, according to the [AWS documentation](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html).
+
+You can also use an [AccessRole](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html)
+for temporary access by passing the `role_arn` identifier to `S3FileSystem$create()` or `s3_bucket()`.
+
+## File systems that emulate S3
+
+The `S3FileSystem` machinery enables you to work with any file system that
+provides an S3-compatible interface. For example, [MinIO](https://min.io/) is
+and object-storage server that emulates the S3 API. If you were to
+run `minio server` locally with its default settings, you could connect to
+it with `arrow` using `S3FileSystem` like this:
+
+```r
+minio <- S3FileSystem$create(
+  access_key = "minioadmin",
+  secret_key = "minioadmin",
+  scheme = "http",
+  endpoint_override = "localhost:9000"
+)
+```
+
+or, as a URI, it would be
+
+```
+s3://minioadmin:minioadmin@?scheme=http&endpoint_override=localhost%3A9000
+```
+
+(note the URL escaping of the `:` in `endpoint_override`).
+
+Among other applications, this can be useful for testing out code locally before
+running on a remote S3 bucket.
diff --git a/r/vignettes/install.Rmd b/r/vignettes/install.Rmd
index 1415a2d3f10..6edb1420549 100644
--- a/r/vignettes/install.Rmd
+++ b/r/vignettes/install.Rmd
@@ -38,10 +38,14 @@ Daily development builds, which are not official releases,
 can be installed from the Ursa Labs repository:
 
 ```r
-install.packages("arrow", repos = "https://dl.bintray.com/ursalabs/arrow-r")
+install.packages("arrow", repos = "https://arrow-r-nightly.s3.amazonaws.com")
 ```
 
-There currently are no daily `conda` builds.
+or for conda users via:
+
+```
+conda install -c arrow-nightlies -c conda-forge --strict-channel-priority r-arrow
+```
 
 You can also install the R package from a git checkout:
 
@@ -92,6 +96,27 @@ satisfy C++ dependencies.
 
 <!-- TODO: does remotes::install_github("apache/arrow/r") work now? -->
 
+## S3 support
+
+The `arrow` package allows you to work with data in AWS S3 or in other cloud
+storage system that emulate S3. However, support for working with S3 is not
+enabled in the default build, and it has additional system requirements. To
+enable it, set the environment variable `LIBARROW_MINIMAL=false` or
+`NOT_CRAN=true` to choose the full-featured build, or more selectively set
+`ARROW_S3=ON`. You also need the following system dependencies:
+
+* `gcc` >= 4.9 or `clang` >= 3.3; note that the default compiler on CentOS 7 is gcc 4.8.5, which is not sufficient
+* CURL: install `libcurl-devel` (rpm) or `libcurl4-openssl-dev` (deb)
+* OpenSSL >= 1.0.2: install `openssl-devel` (rpm) or `libssl-dev` (deb)
+
+The prebuilt C++ binaries come with S3 support enabled, so you will need to meet
+these system requirements in order to use them--the package will not install
+without them. If you're building everything from source, the install script
+will check for the presence of these dependencies and turn off S3 support in the
+build if the prerequisites are not met--installation will succeed but without
+S3 functionality. If afterwards you install the missing system requirements,
+you'll need to reinstall the package in order to enable S3 support.
+
 # How dependencies are resolved
 
 In order for the `arrow` R package to work, it needs the Arrow C++ library.
@@ -303,8 +328,8 @@ By default, these are all unset. All boolean variables are case-insensitive.
   `NOT_CRAN=true` as part of their workflow, without requiring additional
   environment variables to be set.
 * `ARROW_R_DEV`: If set to `true`, more verbose messaging will be printed
-  in the build script. This variable also is needed if you're modifying `Rcpp`
-  code in the package: see "Editing Rcpp code" in the README.
+  in the build script. This variable also is needed if you're modifying C++
+  code in the package: see "Editing C++ code" in the README.
 * `LIBARROW_DEBUG_DIR`: If the C++ library building from source fails (`cmake`),
   there may be messages telling you to check some log file in the build directory.
   However, when the library is built during R package installation,
diff --git a/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb b/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb
index 50c94e9fc1a..d2ba786a657 100644
--- a/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb
+++ b/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb
@@ -16,7 +16,7 @@
 # under the License.
 
 module ArrowCUDA
-  VERSION = "2.0.0-SNAPSHOT"
+  VERSION = "3.0.0-SNAPSHOT"
 
   module Version
     numbers, TAG = VERSION.split("-")
diff --git a/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb b/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb
index fd63fbf96f5..6b5ae254b52 100644
--- a/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb
+++ b/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb
@@ -16,7 +16,7 @@
 # under the License.
 
 module ArrowDataset
-  VERSION = "2.0.0-SNAPSHOT"
+  VERSION = "3.0.0-SNAPSHOT"
 
   module Version
     numbers, TAG = VERSION.split("-")
diff --git a/ruby/red-arrow/lib/arrow/version.rb b/ruby/red-arrow/lib/arrow/version.rb
index f36a08700f7..bdc2929c744 100644
--- a/ruby/red-arrow/lib/arrow/version.rb
+++ b/ruby/red-arrow/lib/arrow/version.rb
@@ -16,7 +16,7 @@
 # under the License.
 
 module Arrow
-  VERSION = "2.0.0-SNAPSHOT"
+  VERSION = "3.0.0-SNAPSHOT"
 
   module Version
     numbers, TAG = VERSION.split("-")
diff --git a/ruby/red-arrow/test/test-struct-array-builder.rb b/ruby/red-arrow/test/test-struct-array-builder.rb
index 0101719f14e..ab0aa5edffa 100644
--- a/ruby/red-arrow/test/test-struct-array-builder.rb
+++ b/ruby/red-arrow/test/test-struct-array-builder.rb
@@ -27,8 +27,8 @@ def setup
       @builder.append_value(nil)
       array = @builder.finish
       assert_equal([
-                     [nil],
-                     [nil],
+                     [false],
+                     [0],
                    ],
                    [
                      array.find_field(0).to_a,
@@ -87,8 +87,8 @@ def setup
       @builder.append_values([nil])
       array = @builder.finish
       assert_equal([
-                     [nil],
-                     [nil],
+                     [false],
+                     [0],
                    ],
                    [
                      array.find_field(0).to_a,
@@ -130,8 +130,8 @@ def setup
                              ])
       array = @builder.finish
       assert_equal([
-                     [nil, true, false],
-                     [nil, 1, 2],
+                     [false, true, false],
+                     [0, 1, 2],
                    ],
                    [
                      array.find_field(0).to_a,
@@ -152,8 +152,8 @@ def setup
                              ])
       array = @builder.finish
       assert_equal([
-                     [true, nil, true],
-                     [1, nil, 3],
+                     [true, false, true],
+                     [1, 0, 3],
                    ],
                    [
                      array.find_field(0).to_a,
diff --git a/ruby/red-arrow/test/test-struct-array.rb b/ruby/red-arrow/test/test-struct-array.rb
index 9ecdd98f4d4..2c01f33ef8d 100644
--- a/ruby/red-arrow/test/test-struct-array.rb
+++ b/ruby/red-arrow/test/test-struct-array.rb
@@ -27,8 +27,8 @@ class StructArrayTest < Test::Unit::TestCase
       ]
       array = Arrow::StructArray.new(data_type, values)
       assert_equal([
-                     [true, nil, false],
-                     [1, nil, 2],
+                     [true, false, false],
+                     [1, 0, 2],
                    ],
                    [
                      array.find_field(0).to_a,
diff --git a/ruby/red-gandiva/lib/gandiva/version.rb b/ruby/red-gandiva/lib/gandiva/version.rb
index 56a534c43dd..22258af0f67 100644
--- a/ruby/red-gandiva/lib/gandiva/version.rb
+++ b/ruby/red-gandiva/lib/gandiva/version.rb
@@ -16,7 +16,7 @@
 # under the License.
 
 module Gandiva
-  VERSION = "2.0.0-SNAPSHOT"
+  VERSION = "3.0.0-SNAPSHOT"
 
   module Version
     numbers, TAG = VERSION.split("-")
diff --git a/ruby/red-parquet/lib/parquet/arrow-table-savable.rb b/ruby/red-parquet/lib/parquet/arrow-table-savable.rb
index 0163b15ed82..70c5975273f 100644
--- a/ruby/red-parquet/lib/parquet/arrow-table-savable.rb
+++ b/ruby/red-parquet/lib/parquet/arrow-table-savable.rb
@@ -33,7 +33,7 @@ def save_as_parquet
           properties.__send__(set_method_name, value)
         end
       end
-      chunk_size = @options[:chunk_size] || 1024 # TODO
+      chunk_size = @options[:chunk_size] || @table.n_rows
       open_raw_output_stream do |output|
         ArrowFileWriter.open(@table.schema,
                              output,
diff --git a/ruby/red-parquet/lib/parquet/version.rb b/ruby/red-parquet/lib/parquet/version.rb
index 2e29bbb9462..c295e30a1d3 100644
--- a/ruby/red-parquet/lib/parquet/version.rb
+++ b/ruby/red-parquet/lib/parquet/version.rb
@@ -16,7 +16,7 @@
 # under the License.
 
 module Parquet
-  VERSION = "2.0.0-SNAPSHOT"
+  VERSION = "3.0.0-SNAPSHOT"
 
   module Version
     numbers, TAG = VERSION.split("-")
diff --git a/ruby/red-plasma/lib/plasma/version.rb b/ruby/red-plasma/lib/plasma/version.rb
index 74f297a12a4..dc6507f8e7d 100644
--- a/ruby/red-plasma/lib/plasma/version.rb
+++ b/ruby/red-plasma/lib/plasma/version.rb
@@ -16,7 +16,7 @@
 # under the License.
 
 module Plasma
-  VERSION = "2.0.0-SNAPSHOT"
+  VERSION = "3.0.0-SNAPSHOT"
 
   module Version
     numbers, TAG = VERSION.split("-")
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index 0cb529fb690..459fe8fd4ed 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -19,6 +19,8 @@
 members = [
         "arrow",
         "parquet",
+        "parquet_derive",
+        "parquet_derive_test",
         "datafusion",
         "arrow-flight",
         "integration-testing",
diff --git a/rust/README.md b/rust/README.md
index a9fcf8d656b..9265d2f2356 100644
--- a/rust/README.md
+++ b/rust/README.md
@@ -19,17 +19,40 @@
 
 # Native Rust implementation of Apache Arrow
 
-## The Rust implementation of Arrow consists of the following crates
+[![Coverage Status](https://codecov.io/gh/apache/arrow/rust/branch/master/graph/badge.svg)](https://codecov.io/gh/apache/arrow?branch=master)
+
+Welcome to the implementation of Arrow, the popular in-memory columnar format, in Rust.
+
+This part of the Arrow project is divided in 4 main components:
 
 | Crate     | Description | Documentation |
 |-----------|-------------|---------------|
-|Arrow      | Core functionality (memory layout, array builders, low level computations) | [(README)](arrow/README.md) |
-|Parquet    | Parquet support | [(README)](parquet/README.md) |
-|DataFusion | In-memory query engine with SQL support | [(README)](datafusion/README.md) |
+|Arrow        | Core functionality (memory layout, arrays, low level computations) | [(README)](arrow/README.md) |
+|Parquet      | Parquet support | [(README)](parquet/README.md) |
+|Arrow-flight | Arrow data between processes | [(README)](arrow-flight/README.md) |
+|DataFusion   | In-memory query engine with SQL support | [(README)](datafusion/README.md) |
+
+Independently, they support a vast array of functionality for in-memory computations.
+
+Together, they allow users to write an SQL query or a `DataFrame` (using `datafusion` crate), run it against a parquet file (using `parquet` crate), evaluate it in-memory using Arrow's columnar format (using the `arrow` crate), and send to another process (using `arrow-flight` crate).
+
+Generally speaking, the `arrow` crate offers the  functionality to develop code that uses Arrow arrays, and `datafusion` offers most operations typically found in SQL, with the notable exceptions of:
+
+* `join`
+* `window` functions
 
-## Prerequisites
+There are too many features to enumerate here, but some notable mentions:
 
-Before running tests and examples it is necessary to set up the local development environment.
+* `Arrow` implements all formats in the specification except certain dictionaries
+* `Arrow` supports SIMD operations to some of its vertical operations
+* `DataFusion` supports `async` execution
+* `DataFusion` supports user-defined functions, aggregates, and whole execution nodes
+
+You can find more details about each crate on their respective READMEs.
+
+## Developer's guide to Arrow Rust
+
+Before running tests and examples, it is necessary to set up the local development environment.
 
 ### Git Submodules
 
@@ -43,17 +66,25 @@ git submodule update --init
 
 This populates data in two git submodules:
 
-- `cpp/submodules/parquet_testing/data` (sourced from https://github.com/apache/parquet-testing.git)
-- `testing` (sourced from https://github.com/apache/arrow-testing)
+- `../cpp/submodules/parquet_testing/data` (sourced from https://github.com/apache/parquet-testing.git)
+- `../testing` (sourced from https://github.com/apache/arrow-testing)
+
+To run the tests of the whole crate, create two new environment variables to point to these directories as follows:
+
+```bash
+export PARQUET_TEST_DATA=../cpp/submodules/parquet-testing/data
+export ARROW_TEST_DATA=../testing/data
+```
 
-Create two new environment variables to point to these directories as follows:
+To run the tests of an individual crate within the project (e.g. in `datafusion/`), adjust the path
+accordingly:
 
 ```bash
-export PARQUET_TEST_DATA=/path/to/arrow/cpp/submodules/parquet-testing/data
-export ARROW_TEST_DATA=/path/to/arrow/testing/data/
+export PARQUET_TEST_DATA=../../cpp/submodules/parquet-testing/data
+export ARROW_TEST_DATA=../../testing/data
 ```
 
-It is now possible to run `cargo test` as usual.
+from here on, this is a pure Rust project and `cargo` can be used to run tests, benchmarks, docs and examples as usual.
 
 ## Code Formatting
 
@@ -66,6 +97,25 @@ and check for lint issues:
 cargo +stable fmt --all -- --check
 ```
 
+## Clippy Lints
+
+We recommend using `clippy` for checking lints during development. While we do not yet enforce `clippy` checks, we recommend not introducing new `clippy` errors or warnings.
+
+Run the following to check for clippy lints.
+
+```
+cargo clippy
+```
+
+If you use Visual Studio Code with the `rust-analyzer` plugin, you can enable `clippy` to run each time you save a file. See https://users.rust-lang.org/t/how-to-use-clippy-in-vs-code-with-rust-analyzer/41881.
+
+One of the concerns with `clippy` is that it often produces a lot of false positives, or that some recommendations may hurt readability. We do not have a policy of which lints are ignored, but if you disagree with a `clippy` lint, you may disable the lint and briefly justify it.
+
+Search for `allow(clippy::` in the codebase to identify lints that are ignored/allowed. We currently prefer ignoring lints on the lowest unit possible.
+* If you are introducing a line that returns a lint warning or error, you may disable the lint on that line.
+* If you have several lints on a function or module, you may disable the lint on the function or module.
+* If a lint is pervasive across multiple modules, you may disable it at the crate level.
+
 ## CI and Dockerized builds
 
 There are currently multiple CI systems that build the project and they all use the same docker image. It is possible to run the same build locally.
diff --git a/rust/arrow-flight/Cargo.toml b/rust/arrow-flight/Cargo.toml
index 829e3b5b413..abc9b1e5e7a 100644
--- a/rust/arrow-flight/Cargo.toml
+++ b/rust/arrow-flight/Cargo.toml
@@ -18,7 +18,7 @@
 [package]
 name = "arrow-flight"
 description = "Apache Arrow Flight"
-version = "2.0.0-SNAPSHOT"
+version = "3.0.0-SNAPSHOT"
 edition = "2018"
 authors = ["Apache Arrow <dev@arrow.apache.org>"]
 homepage = "https://github.com/apache/arrow"
@@ -26,8 +26,8 @@ repository = "https://github.com/apache/arrow"
 license = "Apache-2.0"
 
 [dependencies]
-arrow = { path = "../arrow", version = "2.0.0-SNAPSHOT" }
-tonic = "0.2"
+arrow = { path = "../arrow", version = "3.0.0-SNAPSHOT" }
+tonic = "0.3"
 bytes = "0.5"
 prost = "0.6"
 prost-derive = "0.6"
@@ -35,10 +35,10 @@ tokio = {version = "0.2", features = ["macros"]}
 futures = { version = "0.3", default-features = false, features = ["alloc"]}
 
 [build-dependencies]
-tonic-build = "0.2"
+tonic-build = "0.3"
 # Pin specific version of the tonic-build dependencies to avoid auto-generated
 # (and checked in) arrow.flight.protocol.rs from changing
-proc-macro2 = "=1.0.18"
+proc-macro2 = "=1.0.24"
 
 #[lib]
 #name = "flight"
diff --git a/rust/arrow-flight/src/arrow.flight.protocol.rs b/rust/arrow-flight/src/arrow.flight.protocol.rs
index aeb8f8851bd..871eb506b26 100644
--- a/rust/arrow-flight/src/arrow.flight.protocol.rs
+++ b/rust/arrow-flight/src/arrow.flight.protocol.rs
@@ -632,7 +632,6 @@ pub mod flight_service_server {
     #[doc = " accessed using the Arrow Flight Protocol. Additionally, a flight service"]
     #[doc = " can expose a set of actions that are available."]
     #[derive(Debug)]
-    #[doc(hidden)]
     pub struct FlightServiceServer<T: FlightService> {
         inner: _Inner<T>,
     }
@@ -687,7 +686,7 @@ pub mod flight_service_server {
                             >,
                         ) -> Self::Future {
                             let inner = self.0.clone();
-                            let fut = async move { inner.handshake(request).await };
+                            let fut = async move { (*inner).handshake(request).await };
                             Box::pin(fut)
                         }
                     }
@@ -725,7 +724,7 @@ pub mod flight_service_server {
                             request: tonic::Request<super::Criteria>,
                         ) -> Self::Future {
                             let inner = self.0.clone();
-                            let fut = async move { inner.list_flights(request).await };
+                            let fut = async move { (*inner).list_flights(request).await };
                             Box::pin(fut)
                         }
                     }
@@ -760,7 +759,8 @@ pub mod flight_service_server {
                             request: tonic::Request<super::FlightDescriptor>,
                         ) -> Self::Future {
                             let inner = self.0.clone();
-                            let fut = async move { inner.get_flight_info(request).await };
+                            let fut =
+                                async move { (*inner).get_flight_info(request).await };
                             Box::pin(fut)
                         }
                     }
@@ -795,7 +795,7 @@ pub mod flight_service_server {
                             request: tonic::Request<super::FlightDescriptor>,
                         ) -> Self::Future {
                             let inner = self.0.clone();
-                            let fut = async move { inner.get_schema(request).await };
+                            let fut = async move { (*inner).get_schema(request).await };
                             Box::pin(fut)
                         }
                     }
@@ -833,7 +833,7 @@ pub mod flight_service_server {
                             request: tonic::Request<super::Ticket>,
                         ) -> Self::Future {
                             let inner = self.0.clone();
-                            let fut = async move { inner.do_get(request).await };
+                            let fut = async move { (*inner).do_get(request).await };
                             Box::pin(fut)
                         }
                     }
@@ -871,7 +871,7 @@ pub mod flight_service_server {
                             request: tonic::Request<tonic::Streaming<super::FlightData>>,
                         ) -> Self::Future {
                             let inner = self.0.clone();
-                            let fut = async move { inner.do_put(request).await };
+                            let fut = async move { (*inner).do_put(request).await };
                             Box::pin(fut)
                         }
                     }
@@ -909,7 +909,7 @@ pub mod flight_service_server {
                             request: tonic::Request<tonic::Streaming<super::FlightData>>,
                         ) -> Self::Future {
                             let inner = self.0.clone();
-                            let fut = async move { inner.do_exchange(request).await };
+                            let fut = async move { (*inner).do_exchange(request).await };
                             Box::pin(fut)
                         }
                     }
@@ -947,7 +947,7 @@ pub mod flight_service_server {
                             request: tonic::Request<super::Action>,
                         ) -> Self::Future {
                             let inner = self.0.clone();
-                            let fut = async move { inner.do_action(request).await };
+                            let fut = async move { (*inner).do_action(request).await };
                             Box::pin(fut)
                         }
                     }
@@ -985,7 +985,7 @@ pub mod flight_service_server {
                             request: tonic::Request<super::Empty>,
                         ) -> Self::Future {
                             let inner = self.0.clone();
-                            let fut = async move { inner.list_actions(request).await };
+                            let fut = async move { (*inner).list_actions(request).await };
                             Box::pin(fut)
                         }
                     }
diff --git a/rust/arrow-flight/src/utils.rs b/rust/arrow-flight/src/utils.rs
index aa93cbfdc6e..ee19f34a7c5 100644
--- a/rust/arrow-flight/src/utils.rs
+++ b/rust/arrow-flight/src/utils.rs
@@ -23,41 +23,77 @@ use crate::{FlightData, SchemaResult};
 
 use arrow::datatypes::{Schema, SchemaRef};
 use arrow::error::{ArrowError, Result};
-use arrow::ipc::{convert, reader, writer};
+use arrow::ipc::{convert, reader, writer, writer::IpcWriteOptions};
 use arrow::record_batch::RecordBatch;
 
-/// Convert a `RecordBatch` to `FlightData` by getting the header and body as bytes
+/// Convert a `RecordBatch` to `FlightData` by converting the header and body to bytes
+///
+/// Note: This implicitly uses the default `IpcWriteOptions`. To configure options,
+/// use `flight_data_from_arrow_batch()`
 impl From<&RecordBatch> for FlightData {
     fn from(batch: &RecordBatch) -> Self {
-        let (header, body) = writer::record_batch_to_bytes(batch);
-        Self {
-            flight_descriptor: None,
-            app_metadata: vec![],
-            data_header: header,
-            data_body: body,
-        }
+        let options = IpcWriteOptions::default();
+        flight_data_from_arrow_batch(batch, &options)
+    }
+}
+
+/// Convert a `RecordBatch` to `FlightData` by converting the header and body to bytes
+pub fn flight_data_from_arrow_batch(
+    batch: &RecordBatch,
+    options: &IpcWriteOptions,
+) -> FlightData {
+    let data = writer::record_batch_to_bytes(batch, &options);
+    FlightData {
+        flight_descriptor: None,
+        app_metadata: vec![],
+        data_header: data.ipc_message,
+        data_body: data.arrow_data,
     }
 }
 
 /// Convert a `Schema` to `SchemaResult` by converting to an IPC message
+///
+/// Note: This implicitly uses the default `IpcWriteOptions`. To configure options,
+/// use `flight_schema_from_arrow_schema()`
 impl From<&Schema> for SchemaResult {
     fn from(schema: &Schema) -> Self {
-        Self {
-            schema: writer::schema_to_bytes(schema),
-        }
+        let options = IpcWriteOptions::default();
+        flight_schema_from_arrow_schema(schema, &options)
+    }
+}
+
+/// Convert a `Schema` to `SchemaResult` by converting to an IPC message
+pub fn flight_schema_from_arrow_schema(
+    schema: &Schema,
+    options: &IpcWriteOptions,
+) -> SchemaResult {
+    SchemaResult {
+        schema: writer::schema_to_bytes(schema, &options).ipc_message,
     }
 }
 
 /// Convert a `Schema` to `FlightData` by converting to an IPC message
+///
+/// Note: This implicitly uses the default `IpcWriteOptions`. To configure options,
+/// use `flight_data_from_arrow_schema()`
 impl From<&Schema> for FlightData {
     fn from(schema: &Schema) -> Self {
-        let schema = writer::schema_to_bytes(schema);
-        Self {
-            flight_descriptor: None,
-            app_metadata: vec![],
-            data_header: schema,
-            data_body: vec![],
-        }
+        let options = writer::IpcWriteOptions::default();
+        flight_data_from_arrow_schema(schema, &options)
+    }
+}
+
+/// Convert a `Schema` to `FlightData` by converting to an IPC message
+pub fn flight_data_from_arrow_schema(
+    schema: &Schema,
+    options: &IpcWriteOptions,
+) -> FlightData {
+    let schema = writer::schema_to_bytes(schema, &options);
+    FlightData {
+        flight_descriptor: None,
+        app_metadata: vec![],
+        data_header: schema.ipc_message,
+        data_body: vec![],
     }
 }
 
@@ -90,24 +126,32 @@ impl TryFrom<&SchemaResult> for Schema {
 }
 
 /// Convert a FlightData message to a RecordBatch
-pub fn flight_data_to_batch(
+pub fn flight_data_to_arrow_batch(
     data: &FlightData,
     schema: SchemaRef,
-) -> Result<Option<RecordBatch>> {
+) -> Option<Result<RecordBatch>> {
     // check that the data_header is a record batch message
     let message = arrow::ipc::get_root_as_message(&data.data_header[..]);
     let dictionaries_by_field = Vec::new();
-    let batch_header = message.header_as_record_batch().ok_or_else(|| {
-        ArrowError::ParseError(
-            "Unable to convert flight data header to a record batch".to_string(),
+
+    message
+        .header_as_record_batch()
+        .ok_or_else(|| {
+            ArrowError::ParseError(
+                "Unable to convert flight data header to a record batch".to_string(),
+            )
+        })
+        .map_or_else(
+            |err| Some(Err(err)),
+            |batch| {
+                Some(reader::read_record_batch(
+                    &data.data_body,
+                    batch,
+                    schema,
+                    &dictionaries_by_field,
+                ))
+            },
         )
-    })?;
-    reader::read_record_batch(
-        &data.data_body,
-        batch_header,
-        schema,
-        &dictionaries_by_field,
-    )
 }
 
-// TODO: add more explicit conversion that expoess flight descriptor and metadata options
+// TODO: add more explicit conversion that exposes flight descriptor and metadata options
diff --git a/rust/arrow/Cargo.toml b/rust/arrow/Cargo.toml
index e41557ae22e..71445768207 100644
--- a/rust/arrow/Cargo.toml
+++ b/rust/arrow/Cargo.toml
@@ -17,7 +17,7 @@
 
 [package]
 name = "arrow"
-version = "2.0.0-SNAPSHOT"
+version = "3.0.0-SNAPSHOT"
 description = "Rust implementation of Apache Arrow"
 homepage = "https://github.com/apache/arrow"
 repository = "https://github.com/apache/arrow"
@@ -39,13 +39,13 @@ path = "src/lib.rs"
 serde = { version = "1.0", features = ["rc"] }
 serde_derive = "1.0"
 serde_json = { version = "1.0", features = ["preserve_order"] }
-indexmap = "1.4"
+indexmap = "1.6"
 rand = "0.7"
 csv = "1.1"
 num = "0.3"
 regex = "1.3"
 lazy_static = "1.4"
-packed_simd = { version = "0.3", optional = true }
+packed_simd = { version = "0.3.4", optional = true, package = "packed_simd_2" }
 chrono = "0.4"
 flatbuffers = "0.6"
 hex = "0.4"
@@ -61,6 +61,10 @@ criterion = "0.3"
 flate2 = "1"
 tempfile = "3"
 
+[[bench]]
+name = "aggregate_kernels"
+harness = false
+
 [[bench]]
 name = "array_from_vec"
 harness = false
@@ -69,6 +73,10 @@ harness = false
 name = "builder"
 harness = false
 
+[[bench]]
+name = "buffer_bit_ops"
+harness = false
+
 [[bench]]
 name = "boolean_kernels"
 harness = false
@@ -97,6 +105,14 @@ harness = false
 name = "length_kernel"
 harness = false
 
+[[bench]]
+name = "sort_kernel"
+harness = false
+
 [[bench]]
 name = "csv_writer"
 harness = false
+
+[[bench]]
+name = "equal"
+harness = false
diff --git a/rust/arrow/README.md b/rust/arrow/README.md
index a7e668c9b9f..dc99f5f23e1 100644
--- a/rust/arrow/README.md
+++ b/rust/arrow/README.md
@@ -21,21 +21,24 @@
 
 [![Coverage Status](https://coveralls.io/repos/github/apache/arrow/badge.svg)](https://coveralls.io/github/apache/arrow)
 
-## Status
-
-This is a native Rust implementation of Apache Arrow. Currently the project
-is developed and tested against nightly Rust. The current status is:
-
-- [x] Primitive Arrays
-- [x] List Arrays
-- [x] Struct Arrays
-- [x] CSV Reader
-- [X] CSV Writer
-- [X] JSON Reader
-- [ ] Parquet Reader
-- [ ] Parquet Writer
-- [X] Arrow IPC
-- [ ] Interop tests with other implementations
+This crate contains a native Rust implementation of the [Arrow columnar format](https://arrow.apache.org/docs/format/Columnar.html). It uses nightly Rust.
+
+## Developer's guide
+
+Refer to [lib.rs](src/lib.rs) for an introduction to this crate and current functionality.
+
+### How to run the tests
+
+The tests of this crate depend on two environment variables to be defined.
+Assuming that you are in this crates' current directory:
+
+```bash
+export PARQUET_TEST_DATA=../../cpp/submodules/parquet-testing/data
+export ARROW_TEST_DATA=../../testing/data
+cargo test
+```
+
+runs all the tests.
 
 ## Examples
 
diff --git a/rust/arrow/benches/aggregate_kernels.rs b/rust/arrow/benches/aggregate_kernels.rs
new file mode 100644
index 00000000000..049a2844dff
--- /dev/null
+++ b/rust/arrow/benches/aggregate_kernels.rs
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[macro_use]
+extern crate criterion;
+use criterion::Criterion;
+
+use rand::Rng;
+use std::sync::Arc;
+
+extern crate arrow;
+
+use arrow::array::*;
+use arrow::compute::kernels::aggregate::*;
+
+fn create_array(size: usize, with_nulls: bool) -> ArrayRef {
+    // use random numbers to avoid spurious compiler optimizations wrt to branching
+    let mut rng = rand::thread_rng();
+    let mut builder = Float32Builder::new(size);
+
+    for _ in 0..size {
+        if with_nulls && rng.gen::<f32>() > 0.5 {
+            builder.append_null().unwrap();
+        } else {
+            builder.append_value(rng.gen()).unwrap();
+        }
+    }
+    Arc::new(builder.finish())
+}
+
+fn bench_sum(arr_a: &ArrayRef) {
+    let arr_a = arr_a.as_any().downcast_ref::<Float32Array>().unwrap();
+    criterion::black_box(sum(&arr_a).unwrap());
+}
+
+fn bench_min(arr_a: &ArrayRef) {
+    let arr_a = arr_a.as_any().downcast_ref::<Float32Array>().unwrap();
+    criterion::black_box(min(&arr_a).unwrap());
+}
+
+fn add_benchmark(c: &mut Criterion) {
+    let arr_a = create_array(512, false);
+
+    c.bench_function("sum 512", |b| b.iter(|| bench_sum(&arr_a)));
+    c.bench_function("min 512", |b| b.iter(|| bench_min(&arr_a)));
+
+    let arr_a = create_array(512, true);
+
+    c.bench_function("sum nulls 512", |b| b.iter(|| bench_sum(&arr_a)));
+    c.bench_function("min nulls 512", |b| b.iter(|| bench_min(&arr_a)));
+}
+
+criterion_group!(benches, add_benchmark);
+criterion_main!(benches);
diff --git a/rust/arrow/benches/arithmetic_kernels.rs b/rust/arrow/benches/arithmetic_kernels.rs
index b69687a92f3..93ad32bbfc0 100644
--- a/rust/arrow/benches/arithmetic_kernels.rs
+++ b/rust/arrow/benches/arithmetic_kernels.rs
@@ -19,64 +19,80 @@
 extern crate criterion;
 use criterion::Criterion;
 
+use rand::Rng;
 use std::sync::Arc;
 
 extern crate arrow;
 
 use arrow::array::*;
-use arrow::compute::kernels::aggregate::*;
 use arrow::compute::kernels::arithmetic::*;
 use arrow::compute::kernels::limit::*;
 
-fn create_array(size: usize) -> Float32Array {
+fn create_array(size: usize, with_nulls: bool) -> ArrayRef {
+    // use random numbers to avoid spurious compiler optimizations wrt to branching
+    let mut rng = rand::thread_rng();
     let mut builder = Float32Builder::new(size);
-    for _i in 0..size {
-        builder.append_value(1.0).unwrap();
+
+    for _ in 0..size {
+        if with_nulls && rng.gen::<f32>() > 0.5 {
+            builder.append_null().unwrap();
+        } else {
+            builder.append_value(rng.gen()).unwrap();
+        }
     }
-    builder.finish()
+    Arc::new(builder.finish())
 }
 
-fn bench_add(size: usize) {
-    let arr_a = create_array(size);
-    let arr_b = create_array(size);
-    criterion::black_box(add(&arr_a, &arr_b).unwrap());
+fn bench_add(arr_a: &ArrayRef, arr_b: &ArrayRef) {
+    let arr_a = arr_a.as_any().downcast_ref::<Float32Array>().unwrap();
+    let arr_b = arr_b.as_any().downcast_ref::<Float32Array>().unwrap();
+    criterion::black_box(add(arr_a, arr_b).unwrap());
 }
 
-fn bench_subtract(size: usize) {
-    let arr_a = create_array(size);
-    let arr_b = create_array(size);
+fn bench_subtract(arr_a: &ArrayRef, arr_b: &ArrayRef) {
+    let arr_a = arr_a.as_any().downcast_ref::<Float32Array>().unwrap();
+    let arr_b = arr_b.as_any().downcast_ref::<Float32Array>().unwrap();
     criterion::black_box(subtract(&arr_a, &arr_b).unwrap());
 }
 
-fn bench_multiply(size: usize) {
-    let arr_a = create_array(size);
-    let arr_b = create_array(size);
+fn bench_multiply(arr_a: &ArrayRef, arr_b: &ArrayRef) {
+    let arr_a = arr_a.as_any().downcast_ref::<Float32Array>().unwrap();
+    let arr_b = arr_b.as_any().downcast_ref::<Float32Array>().unwrap();
     criterion::black_box(multiply(&arr_a, &arr_b).unwrap());
 }
 
-fn bench_divide(size: usize) {
-    let arr_a = create_array(size);
-    let arr_b = create_array(size);
+fn bench_divide(arr_a: &ArrayRef, arr_b: &ArrayRef) {
+    let arr_a = arr_a.as_any().downcast_ref::<Float32Array>().unwrap();
+    let arr_b = arr_b.as_any().downcast_ref::<Float32Array>().unwrap();
     criterion::black_box(divide(&arr_a, &arr_b).unwrap());
 }
 
-fn bench_sum(size: usize) {
-    let arr_a = create_array(size);
-    criterion::black_box(sum(&arr_a).unwrap());
-}
-
-fn bench_limit(size: usize, max: usize) {
-    let arr_a: ArrayRef = Arc::new(create_array(size));
-    criterion::black_box(limit(&arr_a, max).unwrap());
+fn bench_limit(arr_a: &ArrayRef, max: usize) {
+    criterion::black_box(limit(arr_a, max).unwrap());
 }
 
 fn add_benchmark(c: &mut Criterion) {
-    c.bench_function("add 512", |b| b.iter(|| bench_add(512)));
-    c.bench_function("subtract 512", |b| b.iter(|| bench_subtract(512)));
-    c.bench_function("multiply 512", |b| b.iter(|| bench_multiply(512)));
-    c.bench_function("divide 512", |b| b.iter(|| bench_divide(512)));
-    c.bench_function("sum 512", |b| b.iter(|| bench_sum(512)));
-    c.bench_function("limit 512, 512", |b| b.iter(|| bench_limit(512, 512)));
+    let arr_a = create_array(512, false);
+    let arr_b = create_array(512, false);
+
+    c.bench_function("add 512", |b| b.iter(|| bench_add(&arr_a, &arr_b)));
+    c.bench_function("subtract 512", |b| {
+        b.iter(|| bench_subtract(&arr_a, &arr_b))
+    });
+    c.bench_function("multiply 512", |b| {
+        b.iter(|| bench_multiply(&arr_a, &arr_b))
+    });
+    c.bench_function("divide 512", |b| b.iter(|| bench_divide(&arr_a, &arr_b)));
+    c.bench_function("limit 512, 512", |b| b.iter(|| bench_limit(&arr_a, 512)));
+
+    let arr_a_nulls = create_array(512, false);
+    let arr_b_nulls = create_array(512, false);
+    c.bench_function("add_nulls_512", |b| {
+        b.iter(|| bench_add(&arr_a_nulls, &arr_b_nulls))
+    });
+    c.bench_function("divide_nulls_512", |b| {
+        b.iter(|| bench_divide(&arr_a_nulls, &arr_b_nulls))
+    });
 }
 
 criterion_group!(benches, add_benchmark);
diff --git a/rust/arrow/benches/array_from_vec.rs b/rust/arrow/benches/array_from_vec.rs
index 1918e61c913..41900b8831a 100644
--- a/rust/arrow/benches/array_from_vec.rs
+++ b/rust/arrow/benches/array_from_vec.rs
@@ -24,6 +24,7 @@ extern crate arrow;
 use arrow::array::*;
 use arrow::buffer::Buffer;
 use arrow::datatypes::*;
+use std::{convert::TryFrom, sync::Arc};
 
 fn array_from_vec(n: usize) {
     let mut v: Vec<u8> = Vec::with_capacity(n);
@@ -36,10 +37,84 @@ fn array_from_vec(n: usize) {
     criterion::black_box(Int32Array::from(arr_data));
 }
 
+fn array_string_from_vec(n: usize) {
+    let mut v: Vec<Option<&str>> = Vec::with_capacity(n);
+    for i in 0..n {
+        if i % 2 == 0 {
+            v.push(Some("hello world"));
+        } else {
+            v.push(None);
+        }
+    }
+    criterion::black_box(StringArray::from(v));
+}
+
+fn struct_array_values(
+    n: usize,
+) -> (
+    &'static str,
+    Vec<Option<&'static str>>,
+    &'static str,
+    Vec<Option<i32>>,
+) {
+    let mut strings: Vec<Option<&str>> = Vec::with_capacity(n);
+    let mut ints: Vec<Option<i32>> = Vec::with_capacity(n);
+    for _ in 0..n / 4 {
+        strings.extend_from_slice(&[Some("joe"), None, None, Some("mark")]);
+        ints.extend_from_slice(&[Some(1), Some(2), None, Some(4)]);
+    }
+    ("f1", strings, "f2", ints)
+}
+
+fn struct_array_from_vec(
+    field1: &str,
+    strings: &Vec<Option<&str>>,
+    field2: &str,
+    ints: &Vec<Option<i32>>,
+) {
+    let strings: ArrayRef = Arc::new(StringArray::from(strings.clone()));
+    let ints: ArrayRef = Arc::new(Int32Array::from(ints.clone()));
+
+    criterion::black_box(
+        StructArray::try_from(vec![(field1.clone(), strings), (field2.clone(), ints)])
+            .unwrap(),
+    );
+}
+
 fn criterion_benchmark(c: &mut Criterion) {
     c.bench_function("array_from_vec 128", |b| b.iter(|| array_from_vec(128)));
     c.bench_function("array_from_vec 256", |b| b.iter(|| array_from_vec(256)));
     c.bench_function("array_from_vec 512", |b| b.iter(|| array_from_vec(512)));
+
+    c.bench_function("array_string_from_vec 128", |b| {
+        b.iter(|| array_string_from_vec(128))
+    });
+    c.bench_function("array_string_from_vec 256", |b| {
+        b.iter(|| array_string_from_vec(256))
+    });
+    c.bench_function("array_string_from_vec 512", |b| {
+        b.iter(|| array_string_from_vec(512))
+    });
+
+    let (field1, strings, field2, ints) = struct_array_values(128);
+    c.bench_function("struct_array_from_vec 128", |b| {
+        b.iter(|| struct_array_from_vec(&field1, &strings, &field2, &ints))
+    });
+
+    let (field1, strings, field2, ints) = struct_array_values(256);
+    c.bench_function("struct_array_from_vec 256", |b| {
+        b.iter(|| struct_array_from_vec(&field1, &strings, &field2, &ints))
+    });
+
+    let (field1, strings, field2, ints) = struct_array_values(512);
+    c.bench_function("struct_array_from_vec 512", |b| {
+        b.iter(|| struct_array_from_vec(&field1, &strings, &field2, &ints))
+    });
+
+    let (field1, strings, field2, ints) = struct_array_values(1024);
+    c.bench_function("struct_array_from_vec 1024", |b| {
+        b.iter(|| struct_array_from_vec(&field1, &strings, &field2, &ints))
+    });
 }
 
 criterion_group!(benches, criterion_benchmark);
diff --git a/rust/arrow/benches/buffer_bit_ops.rs b/rust/arrow/benches/buffer_bit_ops.rs
new file mode 100644
index 00000000000..f905a0cf78c
--- /dev/null
+++ b/rust/arrow/benches/buffer_bit_ops.rs
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[macro_use]
+extern crate criterion;
+use criterion::Criterion;
+
+extern crate arrow;
+
+use arrow::buffer::{Buffer, MutableBuffer};
+
+///  Helper function to create arrays
+fn create_buffer(size: usize) -> Buffer {
+    let mut result = MutableBuffer::new(size).with_bitset(size, false);
+
+    for i in 0..size {
+        result.data_mut()[i] = 0b01010101;
+    }
+
+    result.freeze()
+}
+
+fn bench_buffer_and(left: &Buffer, right: &Buffer) {
+    criterion::black_box((left & right).unwrap());
+}
+
+fn bit_ops_benchmark(c: &mut Criterion) {
+    let left = create_buffer(512);
+    let right = create_buffer(512);
+    c.bench_function("buffer_bit_ops and", |b| {
+        b.iter(|| bench_buffer_and(&left, &right))
+    });
+}
+
+criterion_group!(benches, bit_ops_benchmark);
+criterion_main!(benches);
diff --git a/rust/arrow/benches/cast_kernels.rs b/rust/arrow/benches/cast_kernels.rs
index 3a2bb85a0bc..dbad552393f 100644
--- a/rust/arrow/benches/cast_kernels.rs
+++ b/rust/arrow/benches/cast_kernels.rs
@@ -29,100 +29,121 @@ use arrow::array::*;
 use arrow::compute::cast;
 use arrow::datatypes::*;
 
-// cast array from specified primitive array type to desired data type
-fn cast_array<FROM>(size: usize, to_type: DataType) -> ()
+fn build_array<FROM>(size: usize) -> ArrayRef
 where
     FROM: ArrowNumericType,
     Standard: Distribution<FROM::Native>,
-    PrimitiveArray<FROM>: std::convert::From<Vec<FROM::Native>>,
+    PrimitiveArray<FROM>: std::convert::From<Vec<Option<FROM::Native>>>,
 {
-    let array = Arc::new(PrimitiveArray::<FROM>::from(vec![
-        random::<FROM::Native>();
-        size
-    ])) as ArrayRef;
-    criterion::black_box(cast(&array, &to_type).unwrap());
+    let values = (0..size)
+        .map(|_| {
+            // 10% nulls, i.e. dense.
+            if random::<f64>() < 0.1 {
+                None
+            } else {
+                Some(random::<FROM::Native>())
+            }
+        })
+        .collect();
+
+    Arc::new(PrimitiveArray::<FROM>::from(values))
 }
 
-// cast timestamp array from specified primitive array type to desired data type
-fn cast_timestamp_array<FROM>(size: usize, to_type: DataType) -> ()
+fn build_timestamp_array<FROM>(size: usize) -> ArrayRef
 where
     FROM: ArrowTimestampType,
-    Standard: Distribution<i64>,
+    Standard: Distribution<FROM::Native>,
 {
-    let array = Arc::new(PrimitiveArray::<FROM>::from_vec(
-        vec![random::<i64>(); size],
-        None,
-    )) as ArrayRef;
-    criterion::black_box(cast(&array, &to_type).unwrap());
+    let values = (0..size)
+        .map(|_| {
+            if random::<f64>() < 0.5 {
+                None
+            } else {
+                Some(random::<i64>())
+            }
+        })
+        .collect::<Vec<Option<i64>>>();
+
+    Arc::new(PrimitiveArray::<FROM>::from_opt_vec(values, None))
+}
+
+// cast array from specified primitive array type to desired data type
+fn cast_array(array: &ArrayRef, to_type: DataType) {
+    criterion::black_box(cast(array, &to_type).unwrap());
 }
 
 fn add_benchmark(c: &mut Criterion) {
+    let i32_array = build_array::<Int32Type>(512);
+    let i64_array = build_array::<Int64Type>(512);
+    let f32_array = build_array::<Float32Type>(512);
+    let f64_array = build_array::<Float64Type>(512);
+    let date64_array = build_array::<Date64Type>(512);
+    let date32_array = build_array::<Date32Type>(512);
+    let time32s_array = build_array::<Time32SecondType>(512);
+    let time64ns_array = build_array::<Time64NanosecondType>(512);
+    let time_ns_array = build_timestamp_array::<TimestampNanosecondType>(512);
+    let time_ms_array = build_timestamp_array::<TimestampMillisecondType>(512);
+
     c.bench_function("cast int32 to int32 512", |b| {
-        b.iter(|| cast_array::<Int32Type>(512, DataType::Int32))
+        b.iter(|| cast_array(&i32_array, DataType::Int32))
     });
     c.bench_function("cast int32 to uint32 512", |b| {
-        b.iter(|| cast_array::<Int32Type>(512, DataType::UInt32))
+        b.iter(|| cast_array(&i32_array, DataType::UInt32))
     });
     c.bench_function("cast int32 to float32 512", |b| {
-        b.iter(|| cast_array::<Int32Type>(512, DataType::Float32))
+        b.iter(|| cast_array(&i32_array, DataType::Float32))
     });
     c.bench_function("cast int32 to float64 512", |b| {
-        b.iter(|| cast_array::<Int32Type>(512, DataType::Float64))
+        b.iter(|| cast_array(&i32_array, DataType::Float64))
     });
     c.bench_function("cast int32 to int64 512", |b| {
-        b.iter(|| cast_array::<Int32Type>(512, DataType::Int64))
+        b.iter(|| cast_array(&i32_array, DataType::Int64))
     });
     c.bench_function("cast float32 to int32 512", |b| {
-        b.iter(|| cast_array::<Float32Type>(512, DataType::Int32))
+        b.iter(|| cast_array(&f32_array, DataType::Int32))
     });
     c.bench_function("cast float64 to float32 512", |b| {
-        b.iter(|| cast_array::<Float64Type>(512, DataType::Float32))
+        b.iter(|| cast_array(&f64_array, DataType::Float32))
     });
     c.bench_function("cast float64 to uint64 512", |b| {
-        b.iter(|| cast_array::<Float64Type>(512, DataType::UInt64))
+        b.iter(|| cast_array(&f64_array, DataType::UInt64))
     });
     c.bench_function("cast int64 to int32 512", |b| {
-        b.iter(|| cast_array::<Int64Type>(512, DataType::Int32))
+        b.iter(|| cast_array(&i64_array, DataType::Int32))
     });
     c.bench_function("cast date64 to date32 512", |b| {
-        b.iter(|| cast_array::<Date64Type>(512, DataType::Date32(DateUnit::Day)))
+        b.iter(|| cast_array(&date64_array, DataType::Date32(DateUnit::Day)))
     });
     c.bench_function("cast date32 to date64 512", |b| {
-        b.iter(|| cast_array::<Date32Type>(512, DataType::Date64(DateUnit::Millisecond)))
+        b.iter(|| cast_array(&date32_array, DataType::Date64(DateUnit::Millisecond)))
     });
     c.bench_function("cast time32s to time32ms 512", |b| {
-        b.iter(|| {
-            cast_array::<Time32SecondType>(512, DataType::Time32(TimeUnit::Millisecond))
-        })
+        b.iter(|| cast_array(&time32s_array, DataType::Time32(TimeUnit::Millisecond)))
     });
     c.bench_function("cast time32s to time64us 512", |b| {
-        b.iter(|| {
-            cast_array::<Time32SecondType>(512, DataType::Time64(TimeUnit::Microsecond))
-        })
+        b.iter(|| cast_array(&time32s_array, DataType::Time64(TimeUnit::Microsecond)))
     });
     c.bench_function("cast time64ns to time32s 512", |b| {
-        b.iter(|| {
-            cast_array::<Time64NanosecondType>(512, DataType::Time32(TimeUnit::Second))
-        })
+        b.iter(|| cast_array(&time64ns_array, DataType::Time32(TimeUnit::Second)))
     });
     c.bench_function("cast timestamp_ns to timestamp_s 512", |b| {
         b.iter(|| {
-            cast_timestamp_array::<TimestampNanosecondType>(
-                512,
+            cast_array(
+                &time_ns_array,
                 DataType::Timestamp(TimeUnit::Nanosecond, None),
             )
         })
     });
     c.bench_function("cast timestamp_ms to timestamp_ns 512", |b| {
         b.iter(|| {
-            cast_timestamp_array::<TimestampMillisecondType>(
-                512,
+            cast_array(
+                &time_ms_array,
                 DataType::Timestamp(TimeUnit::Nanosecond, None),
             )
         })
     });
     c.bench_function("cast timestamp_ms to i64 512", |b| {
-        b.iter(|| cast_timestamp_array::<TimestampMillisecondType>(512, DataType::Int64))
+        b.iter(|| cast_array(&time_ms_array, DataType::Int64))
     });
 }
 
diff --git a/rust/arrow/benches/csv_writer.rs b/rust/arrow/benches/csv_writer.rs
index f02ea444f09..9305f6ba820 100644
--- a/rust/arrow/benches/csv_writer.rs
+++ b/rust/arrow/benches/csv_writer.rs
@@ -65,7 +65,7 @@ fn criterion_benchmark(c: &mut Criterion) {
     c.bench(
         "record_batches_to_csv",
         Benchmark::new("record_batches_to_csv", move |b| {
-            b.iter(|| record_batches_to_csv())
+            b.iter(record_batches_to_csv)
         }),
     );
 }
diff --git a/rust/arrow/benches/equal.rs b/rust/arrow/benches/equal.rs
new file mode 100644
index 00000000000..a73b70e1011
--- /dev/null
+++ b/rust/arrow/benches/equal.rs
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[macro_use]
+extern crate criterion;
+use criterion::Criterion;
+
+use rand::distributions::Alphanumeric;
+use rand::Rng;
+use std::sync::Arc;
+
+extern crate arrow;
+
+use arrow::array::*;
+
+fn create_string_array(size: usize, with_nulls: bool) -> ArrayRef {
+    // use random numbers to avoid spurious compiler optimizations wrt to branching
+    let mut rng = rand::thread_rng();
+    let mut builder = StringBuilder::new(size);
+
+    for _ in 0..size {
+        if with_nulls && rng.gen::<f32>() > 0.5 {
+            builder.append_null().unwrap();
+        } else {
+            let string = rand::thread_rng()
+                .sample_iter(&Alphanumeric)
+                .take(10)
+                .collect::<String>();
+            builder.append_value(&string).unwrap();
+        }
+    }
+    Arc::new(builder.finish())
+}
+
+fn create_array(size: usize, with_nulls: bool) -> ArrayRef {
+    // use random numbers to avoid spurious compiler optimizations wrt to branching
+    let mut rng = rand::thread_rng();
+    let mut builder = Float32Builder::new(size);
+
+    for _ in 0..size {
+        if with_nulls && rng.gen::<f32>() > 0.5 {
+            builder.append_null().unwrap();
+        } else {
+            builder.append_value(rng.gen()).unwrap();
+        }
+    }
+    Arc::new(builder.finish())
+}
+
+fn bench_equal(arr_a: &ArrayRef) {
+    criterion::black_box(arr_a == arr_a);
+}
+
+fn add_benchmark(c: &mut Criterion) {
+    let arr_a = create_array(512, false);
+    c.bench_function("equal_512", |b| b.iter(|| bench_equal(&arr_a)));
+
+    let arr_a_nulls = create_array(512, true);
+    c.bench_function("equal_nulls_512", |b| b.iter(|| bench_equal(&arr_a_nulls)));
+
+    let arr_a = create_string_array(512, false);
+    c.bench_function("equal_string_512", |b| b.iter(|| bench_equal(&arr_a)));
+
+    let arr_a_nulls = create_string_array(512, true);
+    c.bench_function("equal_string_nulls_512", |b| {
+        b.iter(|| bench_equal(&arr_a_nulls))
+    });
+}
+
+criterion_group!(benches, add_benchmark);
+criterion_main!(benches);
diff --git a/rust/arrow/benches/length_kernel.rs b/rust/arrow/benches/length_kernel.rs
index 39223d2657a..cdc338acee4 100644
--- a/rust/arrow/benches/length_kernel.rs
+++ b/rust/arrow/benches/length_kernel.rs
@@ -42,7 +42,7 @@ fn bench_length() {
 }
 
 fn add_benchmark(c: &mut Criterion) {
-    c.bench_function("length", |b| b.iter(|| bench_length()));
+    c.bench_function("length", |b| b.iter(bench_length));
 }
 
 criterion_group!(benches, add_benchmark);
diff --git a/rust/arrow/benches/sort_kernel.rs b/rust/arrow/benches/sort_kernel.rs
new file mode 100644
index 00000000000..3d2ddb08760
--- /dev/null
+++ b/rust/arrow/benches/sort_kernel.rs
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[macro_use]
+extern crate criterion;
+use criterion::Criterion;
+
+use rand::Rng;
+use std::sync::Arc;
+
+extern crate arrow;
+
+use arrow::array::*;
+use arrow::compute::kernels::sort::{lexsort, SortColumn};
+
+fn create_array(size: usize, with_nulls: bool) -> ArrayRef {
+    // use random numbers to avoid spurious compiler optimizations wrt to branching
+    let mut rng = rand::thread_rng();
+    let mut builder = Float32Builder::new(size);
+
+    for _ in 0..size {
+        if with_nulls && rng.gen::<f32>() > 0.5 {
+            builder.append_null().unwrap();
+        } else {
+            builder.append_value(rng.gen()).unwrap();
+        }
+    }
+    Arc::new(builder.finish())
+}
+
+fn bench_sort(arr_a: &ArrayRef, array_b: &ArrayRef) {
+    let columns = vec![
+        SortColumn {
+            values: arr_a.clone(),
+            options: None,
+        },
+        SortColumn {
+            values: array_b.clone(),
+            options: None,
+        },
+    ];
+
+    criterion::black_box(lexsort(&columns).unwrap());
+}
+
+fn add_benchmark(c: &mut Criterion) {
+    let arr_a = create_array(2u64.pow(10) as usize, false);
+    let arr_b = create_array(2u64.pow(10) as usize, false);
+
+    c.bench_function("sort 2^10", |b| b.iter(|| bench_sort(&arr_a, &arr_b)));
+
+    let arr_a = create_array(2u64.pow(12) as usize, false);
+    let arr_b = create_array(2u64.pow(12) as usize, false);
+
+    c.bench_function("sort 2^12", |b| b.iter(|| bench_sort(&arr_a, &arr_b)));
+
+    let arr_a = create_array(2u64.pow(10) as usize, true);
+    let arr_b = create_array(2u64.pow(10) as usize, true);
+
+    c.bench_function("sort nulls 2^10", |b| b.iter(|| bench_sort(&arr_a, &arr_b)));
+
+    let arr_a = create_array(2u64.pow(12) as usize, true);
+    let arr_b = create_array(2u64.pow(12) as usize, true);
+
+    c.bench_function("sort nulls 2^12", |b| b.iter(|| bench_sort(&arr_a, &arr_b)));
+}
+
+criterion_group!(benches, add_benchmark);
+criterion_main!(benches);
diff --git a/rust/arrow/benches/take_kernels.rs b/rust/arrow/benches/take_kernels.rs
index ee420808348..99e695f8020 100644
--- a/rust/arrow/benches/take_kernels.rs
+++ b/rust/arrow/benches/take_kernels.rs
@@ -18,7 +18,8 @@
 #[macro_use]
 extern crate criterion;
 use criterion::Criterion;
-use rand::distributions::{Distribution, Standard};
+
+use rand::distributions::{Alphanumeric, Distribution, Standard};
 use rand::prelude::random;
 use rand::Rng;
 
@@ -40,6 +41,21 @@ where
     Arc::new(PrimitiveArray::<T>::from(vec![random::<T::Native>(); size])) as ArrayRef
 }
 
+fn create_strings(size: usize) -> ArrayRef {
+    let v = (0..size)
+        .map(|_| {
+            rand::thread_rng()
+                .sample_iter(&Alphanumeric)
+                .take(5)
+                .collect::<String>()
+        })
+        .collect::<Vec<_>>();
+
+    Arc::new(StringArray::from(
+        v.iter().map(|x| &**x).collect::<Vec<&str>>(),
+    ))
+}
+
 fn create_random_index(size: usize) -> UInt32Array {
     let mut rng = rand::thread_rng();
     let ints = Int32Array::from(vec![rng.gen_range(-24i32, size as i32); size]);
@@ -51,46 +67,41 @@ fn create_random_index(size: usize) -> UInt32Array {
     )
 }
 
-fn take_numeric<T>(size: usize, index_len: usize) -> ()
-where
-    T: ArrowNumericType,
-    Standard: Distribution<T::Native>,
-    PrimitiveArray<T>: std::convert::From<Vec<T::Native>>,
-    T::Native: num::NumCast,
-{
-    let array = create_numeric::<T>(size);
-    let index = create_random_index(index_len);
-    criterion::black_box(take(&array, &index, None).unwrap());
-}
-
-fn take_boolean(size: usize, index_len: usize) -> () {
-    let array = Arc::new(BooleanArray::from(vec![random::<bool>(); size])) as ArrayRef;
-    let index = create_random_index(index_len);
-    criterion::black_box(take(&array, &index, None).unwrap());
+fn bench_take(values: &ArrayRef, indices: &UInt32Array) {
+    criterion::black_box(take(&values, &indices, None).unwrap());
 }
 
 fn add_benchmark(c: &mut Criterion) {
-    c.bench_function("take u8 256", |b| {
-        b.iter(|| take_numeric::<UInt8Type>(256, 256))
-    });
-    c.bench_function("take u8 512", |b| {
-        b.iter(|| take_numeric::<UInt8Type>(512, 512))
-    });
-    c.bench_function("take u8 1024", |b| {
-        b.iter(|| take_numeric::<UInt8Type>(1024, 1024))
+    let values = create_numeric::<Int32Type>(512);
+    let indices = create_random_index(512);
+    c.bench_function("take i32 512", |b| b.iter(|| bench_take(&values, &indices)));
+    let values = create_numeric::<Int32Type>(1024);
+    let indices = create_random_index(1024);
+    c.bench_function("take i32 1024", |b| {
+        b.iter(|| bench_take(&values, &indices))
     });
-    c.bench_function("take i32 256", |b| {
-        b.iter(|| take_numeric::<Int32Type>(256, 256))
+
+    let values = Arc::new(BooleanArray::from(vec![random::<bool>(); 512])) as ArrayRef;
+    let indices = create_random_index(512);
+    c.bench_function("take bool 512", |b| {
+        b.iter(|| bench_take(&values, &indices))
     });
-    c.bench_function("take i32 512", |b| {
-        b.iter(|| take_numeric::<Int32Type>(512, 512))
+
+    let values = Arc::new(BooleanArray::from(vec![random::<bool>(); 1024])) as ArrayRef;
+    let indices = create_random_index(1024);
+    c.bench_function("take bool 1024", |b| {
+        b.iter(|| bench_take(&values, &indices))
     });
-    c.bench_function("take i32 1024", |b| {
-        b.iter(|| take_numeric::<Int32Type>(1024, 1024))
+
+    let values = create_strings(512);
+    let indices = create_random_index(512);
+    c.bench_function("take str 512", |b| b.iter(|| bench_take(&values, &indices)));
+
+    let values = create_strings(1024);
+    let indices = create_random_index(1024);
+    c.bench_function("take str 1024", |b| {
+        b.iter(|| bench_take(&values, &indices))
     });
-    c.bench_function("take bool 256", |b| b.iter(|| take_boolean(256, 256)));
-    c.bench_function("take bool 512", |b| b.iter(|| take_boolean(512, 512)));
-    c.bench_function("take bool 1024", |b| b.iter(|| take_boolean(1024, 1024)));
 }
 
 criterion_group!(benches, add_benchmark);
diff --git a/rust/arrow/examples/builders.rs b/rust/arrow/examples/builders.rs
index 6dd23cd2aa0..61cce0ed97a 100644
--- a/rust/arrow/examples/builders.rs
+++ b/rust/arrow/examples/builders.rs
@@ -99,11 +99,12 @@ fn main() {
     let value_offsets = Buffer::from(&[0, 3, 6, 8].to_byte_slice());
 
     // Construct a list array from the above two
-    let list_data_type = DataType::List(Box::new(DataType::Int32));
-    let list_data = ArrayData::builder(list_data_type.clone())
+    let list_data_type =
+        DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
+    let list_data = ArrayData::builder(list_data_type)
         .len(3)
-        .add_buffer(value_offsets.clone())
-        .add_child_data(value_data.clone())
+        .add_buffer(value_offsets)
+        .add_child_data(value_data)
         .build();
     let list_array = ListArray::from(list_data);
 
diff --git a/rust/arrow/examples/read_csv.rs b/rust/arrow/examples/read_csv.rs
index f826028ce65..8c8dfa07498 100644
--- a/rust/arrow/examples/read_csv.rs
+++ b/rust/arrow/examples/read_csv.rs
@@ -35,11 +35,11 @@ fn main() -> Result<()> {
 
     let file = File::open("test/data/uk_cities.csv").unwrap();
 
-    let mut csv = csv::Reader::new(file, Arc::new(schema), false, None, 1024, None);
+    let mut csv = csv::Reader::new(file, Arc::new(schema), false, None, 1024, None, None);
     let _batch = csv.next().unwrap().unwrap();
     #[cfg(feature = "prettyprint")]
     {
-        print_batches(&vec![_batch]).unwrap();
+        print_batches(&[_batch]).unwrap();
     }
     Ok(())
 }
diff --git a/rust/arrow/examples/read_csv_infer_schema.rs b/rust/arrow/examples/read_csv_infer_schema.rs
index 6ddfcc277fe..467c855b92e 100644
--- a/rust/arrow/examples/read_csv_infer_schema.rs
+++ b/rust/arrow/examples/read_csv_infer_schema.rs
@@ -32,7 +32,7 @@ fn main() -> Result<()> {
     let _batch = csv.next().unwrap().unwrap();
     #[cfg(feature = "prettyprint")]
     {
-        print_batches(&vec![_batch]).unwrap();
+        print_batches(&[_batch]).unwrap();
     }
     Ok(())
 }
diff --git a/rust/arrow/examples/tensor_builder.rs b/rust/arrow/examples/tensor_builder.rs
new file mode 100644
index 00000000000..19421715ceb
--- /dev/null
+++ b/rust/arrow/examples/tensor_builder.rs
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+///! Tensor builder example
+extern crate arrow;
+
+use arrow::array::*; //{BufferBuilderTrait, Int32BufferBuilder, Float32BufferBuilder};
+use arrow::buffer::Buffer;
+use arrow::datatypes::ToByteSlice;
+use arrow::error::Result;
+use arrow::tensor::{Float32Tensor, Int32Tensor};
+
+fn main() -> Result<()> {
+    // Building a tensor using the buffer builder for Int32
+    // The buffer builder will pad the appended numbers
+    // to match the required size for each buffer
+    let mut builder = Int32BufferBuilder::new(16);
+    for i in 0..16 {
+        builder.append(i).unwrap();
+    }
+    let buf = builder.finish();
+
+    // When building a tensor the buffer and shape are required
+    // The new function will estimate the expected stride for the
+    // storage data
+    let tensor = Int32Tensor::try_new(buf, Some(vec![2, 8]), None, None)?;
+    println!("Int32 Tensor");
+    println!("{:?}", tensor);
+
+    // Creating a tensor using float type buffer builder
+    let mut builder = Float32BufferBuilder::new(4);
+    builder.append(1.0).unwrap();
+    builder.append(2.0).unwrap();
+    builder.append(3.0).unwrap();
+    builder.append(4.0).unwrap();
+    let buf = builder.finish();
+
+    // When building the tensor the buffer and shape are necessary
+    // The new function will estimate the expected stride for the
+    // storage data
+    let tensor = Float32Tensor::try_new(buf, Some(vec![2, 2]), None, None)?;
+    println!("\nFloat32 Tensor");
+    println!("{:?}", tensor);
+
+    // In order to build a tensor from an array the function to_byte_slice add the
+    // required padding to the elements in the array.
+    let buf = Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7, 9, 10].to_byte_slice());
+    let tensor = Int32Tensor::try_new(buf, Some(vec![2, 5]), None, None)?;
+    println!("\nInt32 Tensor");
+    println!("{:?}", tensor);
+
+    Ok(())
+}
diff --git a/rust/arrow/src/array/array.rs b/rust/arrow/src/array/array.rs
index 302a28c0bec..ad80881a88f 100644
--- a/rust/arrow/src/array/array.rs
+++ b/rust/arrow/src/array/array.rs
@@ -16,37 +16,16 @@
 // under the License.
 
 use std::any::Any;
-use std::convert::{From, TryFrom};
 use std::fmt;
-use std::io::Write;
-use std::iter::{FromIterator, IntoIterator};
-use std::mem;
 use std::sync::Arc;
 
-use chrono::prelude::*;
-
+use super::ArrayDataRef;
 use super::*;
-use crate::array::builder::StringDictionaryBuilder;
-use crate::array::equal::JsonEqual;
-use crate::buffer::{Buffer, MutableBuffer};
-use crate::datatypes::DataType::Struct;
-use crate::datatypes::*;
-use crate::error::{ArrowError, Result};
-use crate::memory;
-use crate::util::bit_util;
-
-/// Number of seconds in a day
-const SECONDS_IN_DAY: i64 = 86_400;
-/// Number of milliseconds in a second
-const MILLISECONDS: i64 = 1_000;
-/// Number of microseconds in a second
-const MICROSECONDS: i64 = 1_000_000;
-/// Number of nanoseconds in a second
-const NANOSECONDS: i64 = 1_000_000_000;
+use crate::array::equal_json::JsonEqual;
 
 /// Trait for dealing with different types of array at runtime when the type of the
 /// array is not known in advance.
-pub trait Array: fmt::Debug + Send + Sync + ArrayEqual + JsonEqual {
+pub trait Array: fmt::Debug + Send + Sync + JsonEqual {
     /// Returns the array as [`Any`](std::any::Any) so that it can be
     /// downcasted to a specific implementation.
     ///
@@ -108,10 +87,10 @@ pub trait Array: fmt::Debug + Send + Sync + ArrayEqual + JsonEqual {
     /// // Make slice over the values [2, 3, 4]
     /// let array_slice = array.slice(1, 3);
     ///
-    /// assert!(array_slice.equals(&Int32Array::from(vec![2, 3, 4])));
+    /// assert_eq!(array_slice.as_ref(), &Int32Array::from(vec![2, 3, 4]));
     /// ```
     fn slice(&self, offset: usize, length: usize) -> ArrayRef {
-        make_array(slice_data(self.data(), offset, length))
+        make_array(Arc::new(self.data_ref().as_ref().slice(offset, length)))
     }
 
     /// Returns the length (i.e., number of elements) of this array.
@@ -126,7 +105,7 @@ pub trait Array: fmt::Debug + Send + Sync + ArrayEqual + JsonEqual {
     /// assert_eq!(array.len(), 5);
     /// ```
     fn len(&self) -> usize {
-        self.data().len()
+        self.data_ref().len()
     }
 
     /// Returns whether this array is empty.
@@ -141,7 +120,7 @@ pub trait Array: fmt::Debug + Send + Sync + ArrayEqual + JsonEqual {
     /// assert_eq!(array.is_empty(), false);
     /// ```
     fn is_empty(&self) -> bool {
-        self.data().is_empty()
+        self.data_ref().is_empty()
     }
 
     /// Returns the offset into the underlying data used by this array(-slice).
@@ -161,7 +140,7 @@ pub trait Array: fmt::Debug + Send + Sync + ArrayEqual + JsonEqual {
     /// assert_eq!(array_slice.offset(), 1);
     /// ```
     fn offset(&self) -> usize {
-        self.data().offset()
+        self.data_ref().offset()
     }
 
     /// Returns whether the element at `index` is null.
@@ -178,7 +157,7 @@ pub trait Array: fmt::Debug + Send + Sync + ArrayEqual + JsonEqual {
     /// assert_eq!(array.is_null(1), true);
     /// ```
     fn is_null(&self, index: usize) -> bool {
-        self.data().is_null(self.data().offset() + index)
+        self.data().is_null(index)
     }
 
     /// Returns whether the element at `index` is not null.
@@ -195,7 +174,7 @@ pub trait Array: fmt::Debug + Send + Sync + ArrayEqual + JsonEqual {
     /// assert_eq!(array.is_valid(1), false);
     /// ```
     fn is_valid(&self, index: usize) -> bool {
-        self.data().is_valid(self.data().offset() + index)
+        self.data().is_valid(index)
     }
 
     /// Returns the total number of null values in this array.
@@ -211,7 +190,7 @@ pub trait Array: fmt::Debug + Send + Sync + ArrayEqual + JsonEqual {
     /// assert_eq!(array.null_count(), 2);
     /// ```
     fn null_count(&self) -> usize {
-        self.data().null_count()
+        self.data_ref().null_count()
     }
 
     /// Returns the total number of bytes of memory occupied by the buffers owned by this array.
@@ -332,903 +311,19 @@ pub fn make_array(data: ArrayDataRef) -> ArrayRef {
     }
 }
 
-/// Creates a zero-copy slice of the array's data.
-///
-/// # Panics
-///
-/// Panics if `offset + length > data.len()`.
-fn slice_data(data: ArrayDataRef, mut offset: usize, length: usize) -> ArrayDataRef {
-    assert!((offset + length) <= data.len());
-
-    let mut new_data = data.as_ref().clone();
-    let len = std::cmp::min(new_data.len - offset, length);
-
-    offset += data.offset;
-    new_data.len = len;
-    new_data.offset = offset;
-
-    // Calculate the new null count based on the offset
-    new_data.null_count = if let Some(bitmap) = new_data.null_bitmap() {
-        let valid_bits = bitmap.bits.data();
-        len.checked_sub(bit_util::count_set_bits_offset(valid_bits, offset, length))
-            .unwrap()
-    } else {
-        0
-    };
-
-    Arc::new(new_data)
-}
-
-/// ----------------------------------------------------------------------------
-/// Implementations of different array types
-
-struct RawPtrBox<T> {
-    inner: *const T,
-}
-
-impl<T> RawPtrBox<T> {
-    fn new(inner: *const T) -> Self {
-        Self { inner }
-    }
-
-    fn get(&self) -> *const T {
-        self.inner
-    }
-}
-
-unsafe impl<T> Send for RawPtrBox<T> {}
-unsafe impl<T> Sync for RawPtrBox<T> {}
-
-fn as_aligned_pointer<T>(p: *const u8) -> *const T {
-    assert!(
-        memory::is_aligned(p, mem::align_of::<T>()),
-        "memory is not aligned"
-    );
-    p as *const T
-}
-
-/// Array whose elements are of primitive types.
-pub struct PrimitiveArray<T: ArrowPrimitiveType> {
-    data: ArrayDataRef,
-    /// Pointer to the value array. The lifetime of this must be <= to the value buffer
-    /// stored in `data`, so it's safe to store.
-    /// Also note that boolean arrays are bit-packed, so although the underlying pointer
-    /// is of type bool it should be cast back to u8 before being used.
-    /// i.e. `self.raw_values.get() as *const u8`
-    raw_values: RawPtrBox<T::Native>,
-}
-
-/// Common operations for primitive types, including numeric types and boolean type.
-pub trait PrimitiveArrayOps<T: ArrowPrimitiveType> {
-    fn values(&self) -> Buffer;
-    fn value(&self, i: usize) -> T::Native;
-}
-
-// This is necessary when caller wants to access `PrimitiveArrayOps`'s methods with
-// `ArrowPrimitiveType`. It doesn't have any implementation as the actual implementations
-// are delegated to that of `ArrowNumericType` and `BooleanType`.
-impl<T: ArrowPrimitiveType> PrimitiveArrayOps<T> for PrimitiveArray<T> {
-    default fn values(&self) -> Buffer {
-        unimplemented!()
-    }
-
-    default fn value(&self, _: usize) -> T::Native {
-        unimplemented!()
-    }
-}
-
-impl<T: ArrowNumericType> PrimitiveArrayOps<T> for PrimitiveArray<T> {
-    fn values(&self) -> Buffer {
-        self.values()
-    }
-
-    fn value(&self, i: usize) -> T::Native {
-        self.value(i)
-    }
-}
-
-impl PrimitiveArrayOps<BooleanType> for BooleanArray {
-    fn values(&self) -> Buffer {
-        self.values()
-    }
-
-    fn value(&self, i: usize) -> bool {
-        self.value(i)
-    }
-}
-
-impl<T: ArrowPrimitiveType> Array for PrimitiveArray<T> {
-    fn as_any(&self) -> &Any {
-        self
-    }
-
-    fn data(&self) -> ArrayDataRef {
-        self.data.clone()
-    }
-
-    fn data_ref(&self) -> &ArrayDataRef {
-        &self.data
-    }
-
-    /// Returns the total number of bytes of memory occupied by the buffers owned by this [PrimitiveArray].
-    fn get_buffer_memory_size(&self) -> usize {
-        self.data.get_buffer_memory_size()
-    }
-
-    /// Returns the total number of bytes of memory occupied physically by this [PrimitiveArray].
-    fn get_array_memory_size(&self) -> usize {
-        self.data.get_array_memory_size() + mem::size_of_val(self)
-    }
-}
-
-/// Implementation for primitive arrays with numeric types.
-/// Boolean arrays are bit-packed and so implemented separately.
-impl<T: ArrowNumericType> PrimitiveArray<T> {
-    pub fn new(length: usize, values: Buffer, null_count: usize, offset: usize) -> Self {
-        let array_data = ArrayData::builder(T::get_data_type())
-            .len(length)
-            .add_buffer(values)
-            .null_count(null_count)
-            .offset(offset)
-            .build();
-        PrimitiveArray::from(array_data)
-    }
-
-    /// Returns a `Buffer` holding all the values of this array.
-    ///
-    /// Note this doesn't take the offset of this array into account.
-    pub fn values(&self) -> Buffer {
-        self.data.buffers()[0].clone()
-    }
-
-    /// Returns the length of this array.
-    pub fn len(&self) -> usize {
-        self.data.len()
-    }
-
-    /// Returns whether this array is empty.
-    pub fn is_empty(&self) -> bool {
-        self.data.is_empty()
-    }
-
-    /// Returns a raw pointer to the values of this array.
-    pub fn raw_values(&self) -> *const T::Native {
-        unsafe { self.raw_values.get().add(self.data.offset()) }
-    }
-
-    /// Returns the primitive value at index `i`.
-    ///
-    /// Note this doesn't do any bound checking, for performance reason.
-    pub fn value(&self, i: usize) -> T::Native {
-        unsafe { *(self.raw_values().add(i)) }
-    }
-
-    /// Returns a slice for the given offset and length
-    ///
-    /// Note this doesn't do any bound checking, for performance reason.
-    pub fn value_slice(&self, offset: usize, len: usize) -> &[T::Native] {
-        let raw =
-            unsafe { std::slice::from_raw_parts(self.raw_values().add(offset), len) };
-        &raw[..]
-    }
-
-    // Returns a new primitive array builder
-    pub fn builder(capacity: usize) -> PrimitiveBuilder<T> {
-        PrimitiveBuilder::<T>::new(capacity)
-    }
-}
-
-impl<T: ArrowTemporalType + ArrowNumericType> PrimitiveArray<T>
-where
-    i64: std::convert::From<T::Native>,
-{
-    /// Returns value as a chrono `NaiveDateTime`, handling time resolution
-    ///
-    /// If a data type cannot be converted to `NaiveDateTime`, a `None` is returned.
-    /// A valid value is expected, thus the user should first check for validity.
-    pub fn value_as_datetime(&self, i: usize) -> Option<NaiveDateTime> {
-        let v = i64::from(self.value(i));
-        match self.data_type() {
-            DataType::Date32(_) => {
-                // convert days into seconds
-                Some(NaiveDateTime::from_timestamp(v as i64 * SECONDS_IN_DAY, 0))
-            }
-            DataType::Date64(_) => Some(NaiveDateTime::from_timestamp(
-                // extract seconds from milliseconds
-                v / MILLISECONDS,
-                // discard extracted seconds and convert milliseconds to nanoseconds
-                (v % MILLISECONDS * MICROSECONDS) as u32,
-            )),
-            DataType::Time32(_) | DataType::Time64(_) => None,
-            DataType::Timestamp(unit, _) => match unit {
-                TimeUnit::Second => Some(NaiveDateTime::from_timestamp(v, 0)),
-                TimeUnit::Millisecond => Some(NaiveDateTime::from_timestamp(
-                    // extract seconds from milliseconds
-                    v / MILLISECONDS,
-                    // discard extracted seconds and convert milliseconds to nanoseconds
-                    (v % MILLISECONDS * MICROSECONDS) as u32,
-                )),
-                TimeUnit::Microsecond => Some(NaiveDateTime::from_timestamp(
-                    // extract seconds from microseconds
-                    v / MICROSECONDS,
-                    // discard extracted seconds and convert microseconds to nanoseconds
-                    (v % MICROSECONDS * MILLISECONDS) as u32,
-                )),
-                TimeUnit::Nanosecond => Some(NaiveDateTime::from_timestamp(
-                    // extract seconds from nanoseconds
-                    v / NANOSECONDS,
-                    // discard extracted seconds
-                    (v % NANOSECONDS) as u32,
-                )),
-            },
-            // interval is not yet fully documented [ARROW-3097]
-            DataType::Interval(_) => None,
-            _ => None,
-        }
-    }
-
-    /// Returns value as a chrono `NaiveDate` by using `Self::datetime()`
-    ///
-    /// If a data type cannot be converted to `NaiveDate`, a `None` is returned
-    pub fn value_as_date(&self, i: usize) -> Option<NaiveDate> {
-        self.value_as_datetime(i).map(|datetime| datetime.date())
-    }
-
-    /// Returns a value as a chrono `NaiveTime`
-    ///
-    /// `Date32` and `Date64` return UTC midnight as they do not have time resolution
-    pub fn value_as_time(&self, i: usize) -> Option<NaiveTime> {
-        match self.data_type() {
-            DataType::Time32(unit) => {
-                // safe to immediately cast to u32 as `self.value(i)` is positive i32
-                let v = i64::from(self.value(i)) as u32;
-                match unit {
-                    TimeUnit::Second => {
-                        Some(NaiveTime::from_num_seconds_from_midnight(v, 0))
-                    }
-                    TimeUnit::Millisecond => {
-                        Some(NaiveTime::from_num_seconds_from_midnight(
-                            // extract seconds from milliseconds
-                            v / MILLISECONDS as u32,
-                            // discard extracted seconds and convert milliseconds to
-                            // nanoseconds
-                            v % MILLISECONDS as u32 * MICROSECONDS as u32,
-                        ))
-                    }
-                    _ => None,
-                }
-            }
-            DataType::Time64(unit) => {
-                let v = i64::from(self.value(i));
-                match unit {
-                    TimeUnit::Microsecond => {
-                        Some(NaiveTime::from_num_seconds_from_midnight(
-                            // extract seconds from microseconds
-                            (v / MICROSECONDS) as u32,
-                            // discard extracted seconds and convert microseconds to
-                            // nanoseconds
-                            (v % MICROSECONDS * MILLISECONDS) as u32,
-                        ))
-                    }
-                    TimeUnit::Nanosecond => {
-                        Some(NaiveTime::from_num_seconds_from_midnight(
-                            // extract seconds from nanoseconds
-                            (v / NANOSECONDS) as u32,
-                            // discard extracted seconds
-                            (v % NANOSECONDS) as u32,
-                        ))
-                    }
-                    _ => None,
-                }
-            }
-            DataType::Timestamp(_, _) => {
-                self.value_as_datetime(i).map(|datetime| datetime.time())
-            }
-            DataType::Date32(_) | DataType::Date64(_) => {
-                Some(NaiveTime::from_hms(0, 0, 0))
-            }
-            DataType::Interval(_) => None,
-            _ => None,
-        }
-    }
-}
-
-impl<T: ArrowPrimitiveType> fmt::Debug for PrimitiveArray<T> {
-    default fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "PrimitiveArray<{:?}>\n[\n", T::get_data_type())?;
-        print_long_array(self, f, |array, index, f| {
-            fmt::Debug::fmt(&array.value(index), f)
-        })?;
-        write!(f, "]")
-    }
-}
-
-impl<T: ArrowNumericType> fmt::Debug for PrimitiveArray<T> {
-    default fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "PrimitiveArray<{:?}>\n[\n", T::get_data_type())?;
-        print_long_array(self, f, |array, index, f| {
-            fmt::Debug::fmt(&array.value(index), f)
-        })?;
-        write!(f, "]")
-    }
-}
-
-impl<T: ArrowNumericType + ArrowTemporalType> fmt::Debug for PrimitiveArray<T>
-where
-    i64: std::convert::From<T::Native>,
-{
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "PrimitiveArray<{:?}>\n[\n", T::get_data_type())?;
-        print_long_array(self, f, |array, index, f| match T::get_data_type() {
-            DataType::Date32(_) | DataType::Date64(_) => {
-                match array.value_as_date(index) {
-                    Some(date) => write!(f, "{:?}", date),
-                    None => write!(f, "null"),
-                }
-            }
-            DataType::Time32(_) | DataType::Time64(_) => {
-                match array.value_as_time(index) {
-                    Some(time) => write!(f, "{:?}", time),
-                    None => write!(f, "null"),
-                }
-            }
-            DataType::Timestamp(_, _) => match array.value_as_datetime(index) {
-                Some(datetime) => write!(f, "{:?}", datetime),
-                None => write!(f, "null"),
-            },
-            _ => write!(f, "null"),
-        })?;
-        write!(f, "]")
-    }
-}
-
-/// Specific implementation for Boolean arrays due to bit-packing
-impl PrimitiveArray<BooleanType> {
-    pub fn new(length: usize, values: Buffer, null_count: usize, offset: usize) -> Self {
-        let array_data = ArrayData::builder(DataType::Boolean)
-            .len(length)
-            .add_buffer(values)
-            .null_count(null_count)
-            .offset(offset)
-            .build();
-        BooleanArray::from(array_data)
-    }
-
-    /// Returns a `Buffer` holds all the values of this array.
-    ///
-    /// Note this doesn't take account into the offset of this array.
-    pub fn values(&self) -> Buffer {
-        self.data.buffers()[0].clone()
-    }
-
-    /// Returns the boolean value at index `i`.
-    pub fn value(&self, i: usize) -> bool {
-        assert!(i < self.data.len());
-        let offset = i + self.offset();
-        unsafe { bit_util::get_bit_raw(self.raw_values.get() as *const u8, offset) }
-    }
-
-    // Returns a new primitive array builder
-    pub fn builder(capacity: usize) -> BooleanBuilder {
-        BooleanBuilder::new(capacity)
-    }
-}
-
-impl fmt::Debug for PrimitiveArray<BooleanType> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "PrimitiveArray<{:?}>\n[\n", BooleanType::get_data_type())?;
-        print_long_array(self, f, |array, index, f| {
-            fmt::Debug::fmt(&array.value(index), f)
-        })?;
-        write!(f, "]")
-    }
-}
-
-// TODO: the macro is needed here because we'd get "conflicting implementations" error
-// otherwise with both `From<Vec<T::Native>>` and `From<Vec<Option<T::Native>>>`.
-// We should revisit this in future.
-macro_rules! def_numeric_from_vec {
-    ( $ty:ident, $native_ty:ident, $ty_id:expr ) => {
-        impl From<Vec<$native_ty>> for PrimitiveArray<$ty> {
-            fn from(data: Vec<$native_ty>) -> Self {
-                let array_data = ArrayData::builder($ty_id)
-                    .len(data.len())
-                    .add_buffer(Buffer::from(data.to_byte_slice()))
-                    .build();
-                PrimitiveArray::from(array_data)
-            }
-        }
-
-        // Constructs a primitive array from a vector. Should only be used for testing.
-        impl From<Vec<Option<$native_ty>>> for PrimitiveArray<$ty> {
-            fn from(data: Vec<Option<$native_ty>>) -> Self {
-                let data_len = data.len();
-                let num_bytes = bit_util::ceil(data_len, 8);
-                let mut null_buf =
-                    MutableBuffer::new(num_bytes).with_bitset(num_bytes, false);
-                let mut val_buf =
-                    MutableBuffer::new(data_len * mem::size_of::<$native_ty>());
-
-                {
-                    let null = vec![0; mem::size_of::<$native_ty>()];
-                    let null_slice = null_buf.data_mut();
-                    for (i, v) in data.iter().enumerate() {
-                        if let Some(n) = v {
-                            bit_util::set_bit(null_slice, i);
-                            // unwrap() in the following should be safe here since we've
-                            // made sure enough space is allocated for the values.
-                            val_buf.write_all(&n.to_byte_slice()).unwrap();
-                        } else {
-                            val_buf.write_all(&null).unwrap();
-                        }
-                    }
-                }
-
-                let array_data = ArrayData::builder($ty_id)
-                    .len(data_len)
-                    .add_buffer(val_buf.freeze())
-                    .null_bit_buffer(null_buf.freeze())
-                    .build();
-                PrimitiveArray::from(array_data)
-            }
-        }
-    };
-}
-
-def_numeric_from_vec!(Int8Type, i8, DataType::Int8);
-def_numeric_from_vec!(Int16Type, i16, DataType::Int16);
-def_numeric_from_vec!(Int32Type, i32, DataType::Int32);
-def_numeric_from_vec!(Int64Type, i64, DataType::Int64);
-def_numeric_from_vec!(UInt8Type, u8, DataType::UInt8);
-def_numeric_from_vec!(UInt16Type, u16, DataType::UInt16);
-def_numeric_from_vec!(UInt32Type, u32, DataType::UInt32);
-def_numeric_from_vec!(UInt64Type, u64, DataType::UInt64);
-def_numeric_from_vec!(Float32Type, f32, DataType::Float32);
-def_numeric_from_vec!(Float64Type, f64, DataType::Float64);
-
-def_numeric_from_vec!(Date32Type, i32, DataType::Date32(DateUnit::Day));
-def_numeric_from_vec!(Date64Type, i64, DataType::Date64(DateUnit::Millisecond));
-def_numeric_from_vec!(Time32SecondType, i32, DataType::Time32(TimeUnit::Second));
-def_numeric_from_vec!(
-    Time32MillisecondType,
-    i32,
-    DataType::Time32(TimeUnit::Millisecond)
-);
-def_numeric_from_vec!(
-    Time64MicrosecondType,
-    i64,
-    DataType::Time64(TimeUnit::Microsecond)
-);
-def_numeric_from_vec!(
-    Time64NanosecondType,
-    i64,
-    DataType::Time64(TimeUnit::Nanosecond)
-);
-def_numeric_from_vec!(
-    IntervalYearMonthType,
-    i32,
-    DataType::Interval(IntervalUnit::YearMonth)
-);
-def_numeric_from_vec!(
-    IntervalDayTimeType,
-    i64,
-    DataType::Interval(IntervalUnit::DayTime)
-);
-def_numeric_from_vec!(
-    DurationSecondType,
-    i64,
-    DataType::Duration(TimeUnit::Second)
-);
-def_numeric_from_vec!(
-    DurationMillisecondType,
-    i64,
-    DataType::Duration(TimeUnit::Millisecond)
-);
-def_numeric_from_vec!(
-    DurationMicrosecondType,
-    i64,
-    DataType::Duration(TimeUnit::Microsecond)
-);
-def_numeric_from_vec!(
-    DurationNanosecondType,
-    i64,
-    DataType::Duration(TimeUnit::Nanosecond)
-);
-def_numeric_from_vec!(
-    TimestampMillisecondType,
-    i64,
-    DataType::Timestamp(TimeUnit::Millisecond, None)
-);
-def_numeric_from_vec!(
-    TimestampMicrosecondType,
-    i64,
-    DataType::Timestamp(TimeUnit::Microsecond, None)
-);
-
-impl<T: ArrowTimestampType> PrimitiveArray<T> {
-    /// Construct a timestamp array from a vec of i64 values and an optional timezone
-    pub fn from_vec(data: Vec<i64>, timezone: Option<Arc<String>>) -> Self {
-        let array_data =
-            ArrayData::builder(DataType::Timestamp(T::get_time_unit(), timezone))
-                .len(data.len())
-                .add_buffer(Buffer::from(data.to_byte_slice()))
-                .build();
-        PrimitiveArray::from(array_data)
-    }
-}
-
-impl<T: ArrowTimestampType> PrimitiveArray<T> {
-    /// Construct a timestamp array from a vec of Option<i64> values and an optional timezone
-    pub fn from_opt_vec(data: Vec<Option<i64>>, timezone: Option<Arc<String>>) -> Self {
-        // TODO: duplicated from def_numeric_from_vec! macro, it looks possible to convert to generic
-        let data_len = data.len();
-        let num_bytes = bit_util::ceil(data_len, 8);
-        let mut null_buf = MutableBuffer::new(num_bytes).with_bitset(num_bytes, false);
-        let mut val_buf = MutableBuffer::new(data_len * mem::size_of::<i64>());
-
-        {
-            let null = vec![0; mem::size_of::<i64>()];
-            let null_slice = null_buf.data_mut();
-            for (i, v) in data.iter().enumerate() {
-                if let Some(n) = v {
-                    bit_util::set_bit(null_slice, i);
-                    // unwrap() in the following should be safe here since we've
-                    // made sure enough space is allocated for the values.
-                    val_buf.write_all(&n.to_byte_slice()).unwrap();
-                } else {
-                    val_buf.write_all(&null).unwrap();
-                }
-            }
-        }
-
-        let array_data =
-            ArrayData::builder(DataType::Timestamp(T::get_time_unit(), timezone))
-                .len(data_len)
-                .add_buffer(val_buf.freeze())
-                .null_bit_buffer(null_buf.freeze())
-                .build();
-        PrimitiveArray::from(array_data)
-    }
-}
-
-/// Constructs a boolean array from a vector. Should only be used for testing.
-impl From<Vec<bool>> for BooleanArray {
-    fn from(data: Vec<bool>) -> Self {
-        let num_byte = bit_util::ceil(data.len(), 8);
-        let mut mut_buf = MutableBuffer::new(num_byte).with_bitset(num_byte, false);
-        {
-            let mut_slice = mut_buf.data_mut();
-            for (i, b) in data.iter().enumerate() {
-                if *b {
-                    bit_util::set_bit(mut_slice, i);
-                }
-            }
-        }
-        let array_data = ArrayData::builder(DataType::Boolean)
-            .len(data.len())
-            .add_buffer(mut_buf.freeze())
-            .build();
-        BooleanArray::from(array_data)
-    }
-}
-
-impl From<Vec<Option<bool>>> for BooleanArray {
-    fn from(data: Vec<Option<bool>>) -> Self {
-        let data_len = data.len();
-        let num_byte = bit_util::ceil(data_len, 8);
-        let mut null_buf = MutableBuffer::new(num_byte).with_bitset(num_byte, false);
-        let mut val_buf = MutableBuffer::new(num_byte).with_bitset(num_byte, false);
-
-        {
-            let null_slice = null_buf.data_mut();
-            let val_slice = val_buf.data_mut();
-
-            for (i, v) in data.iter().enumerate() {
-                if let Some(b) = v {
-                    bit_util::set_bit(null_slice, i);
-                    if *b {
-                        bit_util::set_bit(val_slice, i);
-                    }
-                }
-            }
-        }
-
-        let array_data = ArrayData::builder(DataType::Boolean)
-            .len(data_len)
-            .add_buffer(val_buf.freeze())
-            .null_bit_buffer(null_buf.freeze())
-            .build();
-        BooleanArray::from(array_data)
-    }
-}
-
-/// Constructs a `PrimitiveArray` from an array data reference.
-impl<T: ArrowPrimitiveType> From<ArrayDataRef> for PrimitiveArray<T> {
-    default fn from(data: ArrayDataRef) -> Self {
-        assert_eq!(
-            data.buffers().len(),
-            1,
-            "PrimitiveArray data should contain a single buffer only (values buffer)"
-        );
-        let raw_values = data.buffers()[0].raw_data();
-        assert!(
-            memory::is_aligned::<u8>(raw_values, mem::align_of::<T::Native>()),
-            "memory is not aligned"
-        );
-        Self {
-            data,
-            raw_values: RawPtrBox::new(raw_values as *const T::Native),
-        }
-    }
-}
-
-/// Common operations for List types, currently `ListArray`, `FixedSizeListArray`, `BinaryArray`
-/// `StringArray` and `DictionaryArray`
-pub trait ListArrayOps {
-    fn value_offset_at(&self, i: usize) -> i32;
-}
-
-impl ListArrayOps for ListArray {
-    fn value_offset_at(&self, i: usize) -> i32 {
-        self.value_offset_at(i)
-    }
-}
-
-impl ListArrayOps for FixedSizeListArray {
-    fn value_offset_at(&self, i: usize) -> i32 {
-        self.value_offset_at(i)
-    }
-}
-
-impl ListArrayOps for BinaryArray {
-    fn value_offset_at(&self, i: usize) -> i32 {
-        self.value_offset_at(i)
-    }
-}
-
-impl ListArrayOps for StringArray {
-    fn value_offset_at(&self, i: usize) -> i32 {
-        self.value_offset_at(i)
-    }
-}
-
-impl ListArrayOps for FixedSizeBinaryArray {
-    fn value_offset_at(&self, i: usize) -> i32 {
-        self.value_offset_at(i)
-    }
-}
-
-/// Common operations for large List types, currently `LargeListArray`, `LargeBinaryArray`
-///  and `LargeStringArray`
-pub trait LargeListArrayOps {
-    fn value_offset_at(&self, i: usize) -> i64;
-}
-
-impl LargeListArrayOps for LargeBinaryArray {
-    fn value_offset_at(&self, i: usize) -> i64 {
-        self.value_offset_at(i)
-    }
-}
-
-impl LargeListArrayOps for LargeStringArray {
-    fn value_offset_at(&self, i: usize) -> i64 {
-        self.value_offset_at(i)
-    }
-}
-
-impl LargeListArrayOps for LargeListArray {
-    fn value_offset_at(&self, i: usize) -> i64 {
-        self.value_offset_at(i)
-    }
-}
-
-/// A list array where each element is a variable-sized sequence of values with the same
-/// type.
-pub struct ListArray {
-    data: ArrayDataRef,
-    values: ArrayRef,
-    value_offsets: RawPtrBox<i32>,
-}
-
-/// A list array where each element is a variable-sized sequence of values with the same
-/// type.
-pub struct LargeListArray {
-    data: ArrayDataRef,
-    values: ArrayRef,
-    value_offsets: RawPtrBox<i64>,
-}
-
-impl ListArray {
-    /// Returns a reference to the values of this list.
-    pub fn values(&self) -> ArrayRef {
-        self.values.clone()
-    }
-
-    /// Returns a clone of the value type of this list.
-    pub fn value_type(&self) -> DataType {
-        self.values.data().data_type().clone()
-    }
-
-    /// Returns ith value of this list array.
-    pub fn value(&self, i: usize) -> ArrayRef {
-        self.values
-            .slice(self.value_offset(i) as usize, self.value_length(i) as usize)
-    }
-
-    /// Returns the offset for value at index `i`.
-    ///
-    /// Note this doesn't do any bound checking, for performance reason.
-    #[inline]
-    pub fn value_offset(&self, i: usize) -> i32 {
-        self.value_offset_at(self.data.offset() + i)
-    }
-
-    /// Returns the length for value at index `i`.
-    ///
-    /// Note this doesn't do any bound checking, for performance reason.
-    #[inline]
-    pub fn value_length(&self, mut i: usize) -> i32 {
-        i += self.data.offset();
-        self.value_offset_at(i + 1) - self.value_offset_at(i)
-    }
-
-    #[inline]
-    fn value_offset_at(&self, i: usize) -> i32 {
-        unsafe { *self.value_offsets.get().add(i) }
-    }
-}
-
-impl LargeListArray {
-    /// Returns a reference to the values of this list.
-    pub fn values(&self) -> ArrayRef {
-        self.values.clone()
-    }
-
-    /// Returns a clone of the value type of this list.
-    pub fn value_type(&self) -> DataType {
-        self.values.data().data_type().clone()
-    }
-
-    /// Returns ith value of this list array.
-    pub fn value(&self, i: usize) -> ArrayRef {
-        self.values
-            .slice(self.value_offset(i) as usize, self.value_length(i) as usize)
-    }
-
-    /// Returns the offset for value at index `i`.
-    ///
-    /// Note this doesn't do any bound checking, for performance reason.
-    #[inline]
-    pub fn value_offset(&self, i: usize) -> i64 {
-        self.value_offset_at(self.data.offset() + i)
-    }
-
-    /// Returns the length for value at index `i`.
-    ///
-    /// Note this doesn't do any bound checking, for performance reason.
-    #[inline]
-    pub fn value_length(&self, mut i: usize) -> i64 {
-        i += self.data.offset();
-        self.value_offset_at(i + 1) - self.value_offset_at(i)
-    }
-
-    #[inline]
-    fn value_offset_at(&self, i: usize) -> i64 {
-        unsafe { *self.value_offsets.get().add(i) }
-    }
-}
-
-/// Constructs a `ListArray` from an array data reference.
-impl From<ArrayDataRef> for ListArray {
-    fn from(data: ArrayDataRef) -> Self {
-        assert_eq!(
-            data.buffers().len(),
-            1,
-            "ListArray data should contain a single buffer only (value offsets)"
-        );
-        assert_eq!(
-            data.child_data().len(),
-            1,
-            "ListArray should contain a single child array (values array)"
-        );
-        let values = make_array(data.child_data()[0].clone());
-        let raw_value_offsets = data.buffers()[0].raw_data();
-        let value_offsets: *const i32 = as_aligned_pointer(raw_value_offsets);
-        unsafe {
-            assert_eq!(*value_offsets.offset(0), 0, "offsets do not start at zero");
-        }
-        Self {
-            data,
-            values,
-            value_offsets: RawPtrBox::new(value_offsets),
-        }
-    }
-}
-
-/// Constructs a `LargeListArray` from an array data reference.
-impl From<ArrayDataRef> for LargeListArray {
-    fn from(data: ArrayDataRef) -> Self {
-        assert_eq!(
-            data.buffers().len(),
-            1,
-            "LargeListArray data should contain a single buffer only (value offsets)"
-        );
-        assert_eq!(
-            data.child_data().len(),
-            1,
-            "LargeListArray should contain a single child array (values array)"
-        );
-        let values = make_array(data.child_data()[0].clone());
-        let raw_value_offsets = data.buffers()[0].raw_data();
-        let value_offsets: *const i64 = as_aligned_pointer(raw_value_offsets);
-        unsafe {
-            assert_eq!(*value_offsets.offset(0), 0, "offsets do not start at zero");
-        }
-        Self {
-            data,
-            values,
-            value_offsets: RawPtrBox::new(value_offsets),
-        }
-    }
-}
-
-impl Array for ListArray {
-    fn as_any(&self) -> &Any {
-        self
-    }
-
-    fn data(&self) -> ArrayDataRef {
-        self.data.clone()
-    }
-
-    fn data_ref(&self) -> &ArrayDataRef {
-        &self.data
-    }
-
-    /// Returns the total number of bytes of memory occupied by the buffers owned by this [ListArray].
-    fn get_buffer_memory_size(&self) -> usize {
-        self.data.get_buffer_memory_size()
-    }
-
-    /// Returns the total number of bytes of memory occupied physically by this [ListArray].
-    fn get_array_memory_size(&self) -> usize {
-        self.data.get_array_memory_size() + mem::size_of_val(self)
-    }
-}
-
-impl Array for LargeListArray {
-    fn as_any(&self) -> &Any {
-        self
-    }
-
-    fn data(&self) -> ArrayDataRef {
-        self.data.clone()
-    }
-
-    fn data_ref(&self) -> &ArrayDataRef {
-        &self.data
-    }
-
-    /// Returns the total number of bytes of memory occupied by the buffers owned by this [LargeListArray].
-    fn get_buffer_memory_size(&self) -> usize {
-        self.data.get_buffer_memory_size() + self.values().get_buffer_memory_size()
-    }
-
-    /// Returns the total number of bytes of memory occupied physically by this [LargeListArray].
-    fn get_array_memory_size(&self) -> usize {
-        self.data.get_array_memory_size()
-            + self.values().get_array_memory_size()
-            + mem::size_of_val(self)
-    }
-}
-
 // Helper function for printing potentially long arrays.
-fn print_long_array<A, F>(array: &A, f: &mut fmt::Formatter, print_item: F) -> fmt::Result
+pub(super) fn print_long_array<A, F>(
+    array: &A,
+    f: &mut fmt::Formatter,
+    print_item: F,
+) -> fmt::Result
 where
     A: Array,
     F: Fn(&A, usize, &mut fmt::Formatter) -> fmt::Result,
 {
-    for i in 0..std::cmp::min(10, array.len()) {
+    let head = std::cmp::min(10, array.len());
+
+    for i in 0..head {
         if array.is_null(i) {
             writeln!(f, "  null,")?;
         } else {
@@ -1241,7 +336,10 @@ where
         if array.len() > 20 {
             writeln!(f, "  ...{} elements...,", array.len() - 20)?;
         }
-        for i in array.len() - 10..array.len() {
+
+        let tail = std::cmp::max(head, array.len() - 10);
+
+        for i in tail..array.len() {
             if array.is_null(i) {
                 writeln!(f, "  null,")?;
             } else {
@@ -1253,2875 +351,3 @@ where
     }
     Ok(())
 }
-
-impl fmt::Debug for ListArray {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "ListArray\n[\n")?;
-        print_long_array(self, f, |array, index, f| {
-            fmt::Debug::fmt(&array.value(index), f)
-        })?;
-        write!(f, "]")
-    }
-}
-
-impl fmt::Debug for LargeListArray {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "LargeListArray\n[\n")?;
-        print_long_array(self, f, |array, index, f| {
-            fmt::Debug::fmt(&array.value(index), f)
-        })?;
-        write!(f, "]")
-    }
-}
-
-/// A list array where each element is a fixed-size sequence of values with the same
-/// type.
-pub struct FixedSizeListArray {
-    data: ArrayDataRef,
-    values: ArrayRef,
-    length: i32,
-}
-
-impl FixedSizeListArray {
-    /// Returns a reference to the values of this list.
-    pub fn values(&self) -> ArrayRef {
-        self.values.clone()
-    }
-
-    /// Returns a clone of the value type of this list.
-    pub fn value_type(&self) -> DataType {
-        self.values.data().data_type().clone()
-    }
-
-    /// Returns ith value of this list array.
-    pub fn value(&self, i: usize) -> ArrayRef {
-        self.values
-            .slice(self.value_offset(i) as usize, self.value_length() as usize)
-    }
-
-    /// Returns the offset for value at index `i`.
-    ///
-    /// Note this doesn't do any bound checking, for performance reason.
-    #[inline]
-    pub fn value_offset(&self, i: usize) -> i32 {
-        self.value_offset_at(self.data.offset() + i)
-    }
-
-    /// Returns the length for value at index `i`.
-    ///
-    /// Note this doesn't do any bound checking, for performance reason.
-    #[inline]
-    pub fn value_length(&self) -> i32 {
-        self.length
-    }
-
-    #[inline]
-    fn value_offset_at(&self, i: usize) -> i32 {
-        i as i32 * self.length
-    }
-}
-
-/// Constructs a `FixedSizeListArray` from an array data reference.
-impl From<ArrayDataRef> for FixedSizeListArray {
-    fn from(data: ArrayDataRef) -> Self {
-        assert_eq!(
-            data.buffers().len(),
-            0,
-            "FixedSizeListArray data should not contain a buffer for value offsets"
-        );
-        assert_eq!(
-            data.child_data().len(),
-            1,
-            "FixedSizeListArray should contain a single child array (values array)"
-        );
-        let values = make_array(data.child_data()[0].clone());
-        let length = match data.data_type() {
-            DataType::FixedSizeList(_, len) => {
-                // check that child data is multiple of length
-                assert_eq!(
-                    values.len() % *len as usize,
-                    0,
-                    "FixedSizeListArray child array length should be a multiple of {}",
-                    len
-                );
-                *len
-            }
-            _ => {
-                panic!("FixedSizeListArray data should contain a FixedSizeList data type")
-            }
-        };
-        Self {
-            data,
-            values,
-            length,
-        }
-    }
-}
-
-impl Array for FixedSizeListArray {
-    fn as_any(&self) -> &Any {
-        self
-    }
-
-    fn data(&self) -> ArrayDataRef {
-        self.data.clone()
-    }
-
-    fn data_ref(&self) -> &ArrayDataRef {
-        &self.data
-    }
-
-    /// Returns the total number of bytes of memory occupied by the buffers owned by this [FixedSizeListArray].
-    fn get_buffer_memory_size(&self) -> usize {
-        self.data.get_buffer_memory_size() + self.values().get_buffer_memory_size()
-    }
-
-    /// Returns the total number of bytes of memory occupied physically by this [FixedSizeListArray].
-    fn get_array_memory_size(&self) -> usize {
-        self.data.get_array_memory_size()
-            + self.values().get_array_memory_size()
-            + mem::size_of_val(self)
-    }
-}
-
-impl fmt::Debug for FixedSizeListArray {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "FixedSizeListArray<{}>\n[\n", self.value_length())?;
-        print_long_array(self, f, |array, index, f| {
-            fmt::Debug::fmt(&array.value(index), f)
-        })?;
-        write!(f, "]")
-    }
-}
-
-macro_rules! make_binary_type {
-    ($name:ident, $offset_ty:ty) => {
-        pub struct $name {
-            data: ArrayDataRef,
-            value_offsets: RawPtrBox<$offset_ty>,
-            value_data: RawPtrBox<u8>,
-        }
-
-        impl $name {
-            /// Returns the offset for the element at index `i`.
-            ///
-            /// Note this doesn't do any bound checking, for performance reason.
-            #[inline]
-            pub fn value_offset(&self, i: usize) -> $offset_ty {
-                self.value_offset_at(self.data.offset() + i)
-            }
-
-            /// Returns the length for the element at index `i`.
-            ///
-            /// Note this doesn't do any bound checking, for performance reason.
-            #[inline]
-            pub fn value_length(&self, mut i: usize) -> $offset_ty {
-                i += self.data.offset();
-                self.value_offset_at(i + 1) - self.value_offset_at(i)
-            }
-
-            /// Returns a clone of the value offset buffer
-            pub fn value_offsets(&self) -> Buffer {
-                self.data.buffers()[0].clone()
-            }
-
-            /// Returns a clone of the value data buffer
-            pub fn value_data(&self) -> Buffer {
-                self.data.buffers()[1].clone()
-            }
-
-            #[inline]
-            fn value_offset_at(&self, i: usize) -> $offset_ty {
-                unsafe { *self.value_offsets.get().add(i) }
-            }
-        }
-
-        impl Array for $name {
-            fn as_any(&self) -> &Any {
-                self
-            }
-
-            fn data(&self) -> ArrayDataRef {
-                self.data.clone()
-            }
-
-            fn data_ref(&self) -> &ArrayDataRef {
-                &self.data
-            }
-
-            /// Returns the total number of bytes of memory occupied by the buffers owned by this [$name].
-            fn get_buffer_memory_size(&self) -> usize {
-                self.data.get_buffer_memory_size()
-            }
-
-            /// Returns the total number of bytes of memory occupied physically by this [$name].
-            fn get_array_memory_size(&self) -> usize {
-                self.data.get_array_memory_size() + mem::size_of_val(self)
-            }
-        }
-    };
-}
-
-make_binary_type!(BinaryArray, i32);
-make_binary_type!(LargeBinaryArray, i64);
-make_binary_type!(StringArray, i32);
-make_binary_type!(LargeStringArray, i64);
-
-/// A type of `FixedSizeListArray` whose elements are binaries.
-pub struct FixedSizeBinaryArray {
-    data: ArrayDataRef,
-    value_data: RawPtrBox<u8>,
-    length: i32,
-}
-
-impl BinaryArray {
-    /// Returns the element at index `i` as a byte slice.
-    pub fn value(&self, i: usize) -> &[u8] {
-        assert!(i < self.data.len(), "BinaryArray out of bounds access");
-        let offset = i.checked_add(self.data.offset()).unwrap();
-        unsafe {
-            let pos = self.value_offset_at(offset);
-            std::slice::from_raw_parts(
-                self.value_data.get().offset(pos as isize),
-                (self.value_offset_at(offset + 1) - pos) as usize,
-            )
-        }
-    }
-
-    /// Returns a new binary array builder
-    pub fn builder(capacity: usize) -> BinaryBuilder {
-        BinaryBuilder::new(capacity)
-    }
-}
-
-impl LargeBinaryArray {
-    /// Returns the element at index `i` as a byte slice.
-    pub fn value(&self, i: usize) -> &[u8] {
-        assert!(i < self.data.len(), "LargeBinaryArray out of bounds access");
-        let offset = i.checked_add(self.data.offset()).unwrap();
-        unsafe {
-            let pos = self.value_offset_at(offset);
-            std::slice::from_raw_parts(
-                self.value_data.get().offset(pos as isize),
-                (self.value_offset_at(offset + 1) - pos) as usize,
-            )
-        }
-    }
-
-    /// Returns a new large binary array builder
-    pub fn builder(capacity: usize) -> LargeBinaryBuilder {
-        LargeBinaryBuilder::new(capacity)
-    }
-}
-
-impl StringArray {
-    /// Returns the element at index `i` as a string slice.
-    pub fn value(&self, i: usize) -> &str {
-        assert!(i < self.data.len(), "StringArray out of bounds access");
-        let offset = i.checked_add(self.data.offset()).unwrap();
-        unsafe {
-            let pos = self.value_offset_at(offset);
-            let slice = std::slice::from_raw_parts(
-                self.value_data.get().offset(pos as isize),
-                (self.value_offset_at(offset + 1) - pos) as usize,
-            );
-
-            std::str::from_utf8_unchecked(slice)
-        }
-    }
-
-    /// Returns a new string array builder
-    pub fn builder(capacity: usize) -> StringBuilder {
-        StringBuilder::new(capacity)
-    }
-}
-
-impl LargeStringArray {
-    /// Returns the element at index `i` as a string slice.
-    pub fn value(&self, i: usize) -> &str {
-        assert!(i < self.data.len(), "LargeStringArray out of bounds access");
-        let offset = i.checked_add(self.data.offset()).unwrap();
-        unsafe {
-            let pos = self.value_offset_at(offset);
-            let slice = std::slice::from_raw_parts(
-                self.value_data.get().offset(pos as isize),
-                (self.value_offset_at(offset + 1) - pos) as usize,
-            );
-
-            std::str::from_utf8_unchecked(slice)
-        }
-    }
-
-    // Returns a new large string array builder
-    pub fn builder(capacity: usize) -> LargeStringBuilder {
-        LargeStringBuilder::new(capacity)
-    }
-}
-
-impl FixedSizeBinaryArray {
-    /// Returns the element at index `i` as a byte slice.
-    pub fn value(&self, i: usize) -> &[u8] {
-        assert!(
-            i < self.data.len(),
-            "FixedSizeBinaryArray out of bounds access"
-        );
-        let offset = i.checked_add(self.data.offset()).unwrap();
-        unsafe {
-            let pos = self.value_offset_at(offset);
-            std::slice::from_raw_parts(
-                self.value_data.get().offset(pos as isize),
-                (self.value_offset_at(offset + 1) - pos) as usize,
-            )
-        }
-    }
-
-    /// Returns the offset for the element at index `i`.
-    ///
-    /// Note this doesn't do any bound checking, for performance reason.
-    #[inline]
-    pub fn value_offset(&self, i: usize) -> i32 {
-        self.value_offset_at(self.data.offset() + i)
-    }
-
-    /// Returns the length for an element.
-    ///
-    /// All elements have the same length as the array is a fixed size.
-    #[inline]
-    pub fn value_length(&self) -> i32 {
-        self.length
-    }
-
-    /// Returns a clone of the value data buffer
-    pub fn value_data(&self) -> Buffer {
-        self.data.buffers()[0].clone()
-    }
-
-    #[inline]
-    fn value_offset_at(&self, i: usize) -> i32 {
-        self.length * i as i32
-    }
-}
-
-impl From<ArrayDataRef> for BinaryArray {
-    fn from(data: ArrayDataRef) -> Self {
-        assert_eq!(
-            data.buffers().len(),
-            2,
-            "BinaryArray data should contain 2 buffers only (offsets and values)"
-        );
-        let raw_value_offsets = data.buffers()[0].raw_data();
-        let value_data = data.buffers()[1].raw_data();
-        Self {
-            data,
-            value_offsets: RawPtrBox::new(as_aligned_pointer::<i32>(raw_value_offsets)),
-            value_data: RawPtrBox::new(value_data),
-        }
-    }
-}
-
-impl From<ArrayDataRef> for LargeBinaryArray {
-    fn from(data: ArrayDataRef) -> Self {
-        assert_eq!(
-            data.buffers().len(),
-            2,
-            "LargeBinaryArray data should contain 2 buffers only (offsets and values)"
-        );
-        let raw_value_offsets = data.buffers()[0].raw_data();
-        let value_data = data.buffers()[1].raw_data();
-        Self {
-            data,
-            value_offsets: RawPtrBox::new(as_aligned_pointer::<i64>(raw_value_offsets)),
-            value_data: RawPtrBox::new(value_data),
-        }
-    }
-}
-
-impl From<ArrayDataRef> for StringArray {
-    fn from(data: ArrayDataRef) -> Self {
-        assert_eq!(
-            data.buffers().len(),
-            2,
-            "StringArray data should contain 2 buffers only (offsets and values)"
-        );
-        let raw_value_offsets = data.buffers()[0].raw_data();
-        let value_data = data.buffers()[1].raw_data();
-        Self {
-            data,
-            value_offsets: RawPtrBox::new(as_aligned_pointer::<i32>(raw_value_offsets)),
-            value_data: RawPtrBox::new(value_data),
-        }
-    }
-}
-
-impl From<ArrayDataRef> for LargeStringArray {
-    fn from(data: ArrayDataRef) -> Self {
-        assert_eq!(
-            data.buffers().len(),
-            2,
-            "LargeStringArray data should contain 2 buffers only (offsets and values)"
-        );
-        let raw_value_offsets = data.buffers()[0].raw_data();
-        let value_data = data.buffers()[1].raw_data();
-        Self {
-            data,
-            value_offsets: RawPtrBox::new(as_aligned_pointer::<i64>(raw_value_offsets)),
-            value_data: RawPtrBox::new(value_data),
-        }
-    }
-}
-
-impl From<ArrayDataRef> for FixedSizeBinaryArray {
-    fn from(data: ArrayDataRef) -> Self {
-        assert_eq!(
-            data.buffers().len(),
-            1,
-            "FixedSizeBinaryArray data should contain 1 buffer only (values)"
-        );
-        let value_data = data.buffers()[0].raw_data();
-        let length = match data.data_type() {
-            DataType::FixedSizeBinary(len) => *len,
-            _ => panic!("Expected data type to be FixedSizeBinary"),
-        };
-        Self {
-            data,
-            value_data: RawPtrBox::new(value_data),
-            length,
-        }
-    }
-}
-
-impl<'a> From<Vec<&'a str>> for StringArray {
-    fn from(v: Vec<&'a str>) -> Self {
-        let mut offsets = Vec::with_capacity(v.len() + 1);
-        let mut values = Vec::new();
-        let mut length_so_far = 0;
-        offsets.push(length_so_far);
-        for s in &v {
-            length_so_far += s.len() as i32;
-            offsets.push(length_so_far as i32);
-            values.extend_from_slice(s.as_bytes());
-        }
-        let array_data = ArrayData::builder(DataType::Utf8)
-            .len(v.len())
-            .add_buffer(Buffer::from(offsets.to_byte_slice()))
-            .add_buffer(Buffer::from(&values[..]))
-            .build();
-        StringArray::from(array_data)
-    }
-}
-
-impl<'a> From<Vec<&'a str>> for LargeStringArray {
-    fn from(v: Vec<&'a str>) -> Self {
-        let mut offsets = Vec::with_capacity(v.len() + 1);
-        let mut values = Vec::new();
-        let mut length_so_far = 0;
-        offsets.push(length_so_far);
-        for s in &v {
-            length_so_far += s.len() as i64;
-            offsets.push(length_so_far as i64);
-            values.extend_from_slice(s.as_bytes());
-        }
-        let array_data = ArrayData::builder(DataType::LargeUtf8)
-            .len(v.len())
-            .add_buffer(Buffer::from(offsets.to_byte_slice()))
-            .add_buffer(Buffer::from(&values[..]))
-            .build();
-        LargeStringArray::from(array_data)
-    }
-}
-
-impl From<Vec<&[u8]>> for BinaryArray {
-    fn from(v: Vec<&[u8]>) -> Self {
-        let mut offsets = Vec::with_capacity(v.len() + 1);
-        let mut values = Vec::new();
-        let mut length_so_far = 0;
-        offsets.push(length_so_far);
-        for s in &v {
-            length_so_far += s.len() as i32;
-            offsets.push(length_so_far as i32);
-            values.extend_from_slice(s);
-        }
-        let array_data = ArrayData::builder(DataType::Binary)
-            .len(v.len())
-            .add_buffer(Buffer::from(offsets.to_byte_slice()))
-            .add_buffer(Buffer::from(&values[..]))
-            .build();
-        BinaryArray::from(array_data)
-    }
-}
-
-impl From<Vec<&[u8]>> for LargeBinaryArray {
-    fn from(v: Vec<&[u8]>) -> Self {
-        let mut offsets = Vec::with_capacity(v.len() + 1);
-        let mut values = Vec::new();
-        let mut length_so_far = 0;
-        offsets.push(length_so_far);
-        for s in &v {
-            length_so_far += s.len() as i64;
-            offsets.push(length_so_far as i64);
-            values.extend_from_slice(s);
-        }
-        let array_data = ArrayData::builder(DataType::LargeBinary)
-            .len(v.len())
-            .add_buffer(Buffer::from(offsets.to_byte_slice()))
-            .add_buffer(Buffer::from(&values[..]))
-            .build();
-        LargeBinaryArray::from(array_data)
-    }
-}
-
-impl<'a> TryFrom<Vec<Option<&'a str>>> for StringArray {
-    type Error = ArrowError;
-
-    fn try_from(v: Vec<Option<&'a str>>) -> Result<Self> {
-        let mut builder = StringBuilder::new(v.len());
-        for val in v {
-            if let Some(s) = val {
-                builder.append_value(s)?;
-            } else {
-                builder.append(false)?;
-            }
-        }
-        Ok(builder.finish())
-    }
-}
-
-impl<'a> TryFrom<Vec<Option<&'a str>>> for LargeStringArray {
-    type Error = ArrowError;
-
-    fn try_from(v: Vec<Option<&'a str>>) -> Result<Self> {
-        let mut builder = LargeStringBuilder::new(v.len());
-        for val in v {
-            if let Some(s) = val {
-                builder.append_value(s)?;
-            } else {
-                builder.append(false)?;
-            }
-        }
-        Ok(builder.finish())
-    }
-}
-
-/// Creates a `BinaryArray` from `List<u8>` array
-impl From<ListArray> for BinaryArray {
-    fn from(v: ListArray) -> Self {
-        assert_eq!(
-            v.data().child_data()[0].child_data().len(),
-            0,
-            "BinaryArray can only be created from list array of u8 values \
-             (i.e. List<PrimitiveArray<u8>>)."
-        );
-        assert_eq!(
-            v.data().child_data()[0].data_type(),
-            &DataType::UInt8,
-            "BinaryArray can only be created from List<u8> arrays, mismatched data types."
-        );
-
-        let mut builder = ArrayData::builder(DataType::Binary)
-            .len(v.len())
-            .add_buffer(v.data().buffers()[0].clone())
-            .add_buffer(v.data().child_data()[0].buffers()[0].clone());
-        if let Some(bitmap) = v.data().null_bitmap() {
-            builder = builder
-                .null_count(v.data().null_count())
-                .null_bit_buffer(bitmap.bits.clone())
-        }
-
-        let data = builder.build();
-        Self::from(data)
-    }
-}
-
-/// Creates a `StringArray` from `List<u8>` array
-impl From<ListArray> for StringArray {
-    fn from(v: ListArray) -> Self {
-        assert_eq!(
-            v.data().child_data()[0].child_data().len(),
-            0,
-            "StringArray can only be created from list array of u8 values \
-             (i.e. List<PrimitiveArray<u8>>)."
-        );
-        assert_eq!(
-            v.data().child_data()[0].data_type(),
-            &DataType::UInt8,
-            "StringArray can only be created from List<u8> arrays, mismatched data types."
-        );
-
-        let mut builder = ArrayData::builder(DataType::Utf8)
-            .len(v.len())
-            .add_buffer(v.data().buffers()[0].clone())
-            .add_buffer(v.data().child_data()[0].buffers()[0].clone());
-        if let Some(bitmap) = v.data().null_bitmap() {
-            builder = builder
-                .null_count(v.data().null_count())
-                .null_bit_buffer(bitmap.bits.clone())
-        }
-
-        let data = builder.build();
-        Self::from(data)
-    }
-}
-
-/// Creates a `LargeBinaryArray` from `LargeList<u8>` array
-impl From<LargeListArray> for LargeBinaryArray {
-    fn from(v: LargeListArray) -> Self {
-        assert_eq!(
-            v.data().child_data()[0].child_data().len(),
-            0,
-            "LargeBinaryArray can only be created from list array of u8 values \
-             (i.e. LargeList<PrimitiveArray<u8>>)."
-        );
-        assert_eq!(
-            v.data().child_data()[0].data_type(),
-            &DataType::UInt8,
-            "LargeBinaryArray can only be created from LargeList<u8> arrays, mismatched data types."
-        );
-
-        let mut builder = ArrayData::builder(DataType::LargeBinary)
-            .len(v.len())
-            .add_buffer(v.data().buffers()[0].clone())
-            .add_buffer(v.data().child_data()[0].buffers()[0].clone());
-        if let Some(bitmap) = v.data().null_bitmap() {
-            builder = builder
-                .null_count(v.data().null_count())
-                .null_bit_buffer(bitmap.bits.clone())
-        }
-
-        let data = builder.build();
-        Self::from(data)
-    }
-}
-
-/// Creates a `LargeStringArray` from `LargeList<u8>` array
-impl From<LargeListArray> for LargeStringArray {
-    fn from(v: LargeListArray) -> Self {
-        assert_eq!(
-            v.data().child_data()[0].child_data().len(),
-            0,
-            "LargeStringArray can only be created from list array of u8 values \
-             (i.e. LargeList<PrimitiveArray<u8>>)."
-        );
-        assert_eq!(
-            v.data().child_data()[0].data_type(),
-            &DataType::UInt8,
-            "LargeStringArray can only be created from LargeList<u8> arrays, mismatched data types."
-        );
-
-        let mut builder = ArrayData::builder(DataType::LargeUtf8)
-            .len(v.len())
-            .add_buffer(v.data().buffers()[0].clone())
-            .add_buffer(v.data().child_data()[0].buffers()[0].clone());
-        if let Some(bitmap) = v.data().null_bitmap() {
-            builder = builder
-                .null_count(v.data().null_count())
-                .null_bit_buffer(bitmap.bits.clone())
-        }
-
-        let data = builder.build();
-        Self::from(data)
-    }
-}
-
-/// Creates a `FixedSizeBinaryArray` from `FixedSizeList<u8>` array
-impl From<FixedSizeListArray> for FixedSizeBinaryArray {
-    fn from(v: FixedSizeListArray) -> Self {
-        assert_eq!(
-            v.data().child_data()[0].child_data().len(),
-            0,
-            "FixedSizeBinaryArray can only be created from list array of u8 values \
-             (i.e. FixedSizeList<PrimitiveArray<u8>>)."
-        );
-        assert_eq!(
-            v.data().child_data()[0].data_type(),
-            &DataType::UInt8,
-            "FixedSizeBinaryArray can only be created from FixedSizeList<u8> arrays, mismatched data types."
-        );
-
-        let mut builder = ArrayData::builder(DataType::FixedSizeBinary(v.value_length()))
-            .len(v.len())
-            .add_buffer(v.data().child_data()[0].buffers()[0].clone());
-        if let Some(bitmap) = v.data().null_bitmap() {
-            builder = builder
-                .null_count(v.data().null_count())
-                .null_bit_buffer(bitmap.bits.clone())
-        }
-
-        let data = builder.build();
-        Self::from(data)
-    }
-}
-
-impl fmt::Debug for BinaryArray {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "BinaryArray\n[\n")?;
-        print_long_array(self, f, |array, index, f| {
-            fmt::Debug::fmt(&array.value(index), f)
-        })?;
-        write!(f, "]")
-    }
-}
-
-impl fmt::Debug for LargeBinaryArray {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "LargeBinaryArray\n[\n")?;
-        print_long_array(self, f, |array, index, f| {
-            fmt::Debug::fmt(&array.value(index), f)
-        })?;
-        write!(f, "]")
-    }
-}
-
-impl fmt::Debug for StringArray {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "StringArray\n[\n")?;
-        print_long_array(self, f, |array, index, f| {
-            fmt::Debug::fmt(&array.value(index), f)
-        })?;
-        write!(f, "]")
-    }
-}
-
-impl fmt::Debug for LargeStringArray {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "LargeStringArray\n[\n")?;
-        print_long_array(self, f, |array, index, f| {
-            fmt::Debug::fmt(&array.value(index), f)
-        })?;
-        write!(f, "]")
-    }
-}
-
-impl fmt::Debug for FixedSizeBinaryArray {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "FixedSizeBinaryArray<{}>\n[\n", self.value_length())?;
-        print_long_array(self, f, |array, index, f| {
-            fmt::Debug::fmt(&array.value(index), f)
-        })?;
-        write!(f, "]")
-    }
-}
-
-impl Array for FixedSizeBinaryArray {
-    fn as_any(&self) -> &Any {
-        self
-    }
-
-    fn data(&self) -> ArrayDataRef {
-        self.data.clone()
-    }
-
-    fn data_ref(&self) -> &ArrayDataRef {
-        &self.data
-    }
-
-    /// Returns the total number of bytes of memory occupied by the buffers owned by this [FixedSizeBinaryArray].
-    fn get_buffer_memory_size(&self) -> usize {
-        self.data.get_buffer_memory_size()
-    }
-
-    /// Returns the total number of bytes of memory occupied physically by this [FixedSizeBinaryArray].
-    fn get_array_memory_size(&self) -> usize {
-        self.data.get_array_memory_size() + mem::size_of_val(self)
-    }
-}
-
-/// A nested array type where each child (called *field*) is represented by a separate
-/// array.
-pub struct StructArray {
-    data: ArrayDataRef,
-    pub(crate) boxed_fields: Vec<ArrayRef>,
-}
-
-impl StructArray {
-    /// Returns the field at `pos`.
-    pub fn column(&self, pos: usize) -> &ArrayRef {
-        &self.boxed_fields[pos]
-    }
-
-    /// Return the number of fields in this struct array
-    pub fn num_columns(&self) -> usize {
-        self.boxed_fields.len()
-    }
-
-    /// Returns the fields of the struct array
-    pub fn columns(&self) -> Vec<&ArrayRef> {
-        self.boxed_fields.iter().collect()
-    }
-
-    /// Returns child array refs of the struct array
-    pub fn columns_ref(&self) -> Vec<ArrayRef> {
-        self.boxed_fields.clone()
-    }
-
-    /// Return field names in this struct array
-    pub fn column_names(&self) -> Vec<&str> {
-        match self.data.data_type() {
-            Struct(fields) => fields
-                .iter()
-                .map(|f| f.name().as_str())
-                .collect::<Vec<&str>>(),
-            _ => unreachable!("Struct array's data type is not struct!"),
-        }
-    }
-
-    /// Return child array whose field name equals to column_name
-    pub fn column_by_name(&self, column_name: &str) -> Option<&ArrayRef> {
-        self.column_names()
-            .iter()
-            .position(|c| c == &column_name)
-            .map(|pos| self.column(pos))
-    }
-}
-
-impl From<ArrayDataRef> for StructArray {
-    fn from(data: ArrayDataRef) -> Self {
-        let mut boxed_fields = vec![];
-        for cd in data.child_data() {
-            let child_data = if data.offset != 0 || data.len != cd.len {
-                slice_data(cd.clone(), data.offset, data.len)
-            } else {
-                cd.clone()
-            };
-            boxed_fields.push(make_array(child_data));
-        }
-        Self { data, boxed_fields }
-    }
-}
-
-impl Array for StructArray {
-    fn as_any(&self) -> &Any {
-        self
-    }
-
-    fn data(&self) -> ArrayDataRef {
-        self.data.clone()
-    }
-
-    fn data_ref(&self) -> &ArrayDataRef {
-        &self.data
-    }
-
-    /// Returns the length (i.e., number of elements) of this array
-    fn len(&self) -> usize {
-        self.data().len()
-    }
-
-    /// Returns the total number of bytes of memory occupied by the buffers owned by this [StructArray].
-    fn get_buffer_memory_size(&self) -> usize {
-        self.data.get_buffer_memory_size()
-    }
-
-    /// Returns the total number of bytes of memory occupied physically by this [StructArray].
-    fn get_array_memory_size(&self) -> usize {
-        self.data.get_array_memory_size() + mem::size_of_val(self)
-    }
-}
-
-impl From<Vec<(Field, ArrayRef)>> for StructArray {
-    fn from(v: Vec<(Field, ArrayRef)>) -> Self {
-        let (field_types, field_values): (Vec<_>, Vec<_>) = v.into_iter().unzip();
-
-        // Check the length of the child arrays
-        let length = field_values[0].len();
-        for i in 1..field_values.len() {
-            assert_eq!(
-                length,
-                field_values[i].len(),
-                "all child arrays of a StructArray must have the same length"
-            );
-            assert_eq!(
-                field_types[i].data_type(),
-                field_values[i].data().data_type(),
-                "the field data types must match the array data in a StructArray"
-            )
-        }
-
-        let data = ArrayData::builder(DataType::Struct(field_types))
-            .child_data(field_values.into_iter().map(|a| a.data()).collect())
-            .len(length)
-            .build();
-        Self::from(data)
-    }
-}
-
-impl fmt::Debug for StructArray {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "StructArray\n[\n")?;
-        for (child_index, name) in self.column_names().iter().enumerate() {
-            let column = self.column(child_index);
-            writeln!(
-                f,
-                "-- child {}: \"{}\" ({:?})",
-                child_index,
-                name,
-                column.data_type()
-            )?;
-            fmt::Debug::fmt(column, f)?;
-            writeln!(f)?;
-        }
-        write!(f, "]")
-    }
-}
-
-impl From<(Vec<(Field, ArrayRef)>, Buffer, usize)> for StructArray {
-    fn from(triple: (Vec<(Field, ArrayRef)>, Buffer, usize)) -> Self {
-        let (field_types, field_values): (Vec<_>, Vec<_>) = triple.0.into_iter().unzip();
-
-        // Check the length of the child arrays
-        let length = field_values[0].len();
-        for i in 1..field_values.len() {
-            assert_eq!(
-                length,
-                field_values[i].len(),
-                "all child arrays of a StructArray must have the same length"
-            );
-            assert_eq!(
-                field_types[i].data_type(),
-                field_values[i].data().data_type(),
-                "the field data types must match the array data in a StructArray"
-            )
-        }
-
-        let data = ArrayData::builder(DataType::Struct(field_types))
-            .null_bit_buffer(triple.1)
-            .child_data(field_values.into_iter().map(|a| a.data()).collect())
-            .len(length)
-            .null_count(triple.2)
-            .build();
-        Self::from(data)
-    }
-}
-
-/// A dictionary array where each element is a single value indexed by an integer key.
-/// This is mostly used to represent strings or a limited set of primitive types as integers,
-/// for example when doing NLP analysis or representing chromosomes by name.
-///
-/// Example **with nullable** data:
-///
-/// ```
-/// use arrow::array::DictionaryArray;
-/// use arrow::datatypes::Int8Type;
-/// let test = vec!["a", "a", "b", "c"];
-/// let array : DictionaryArray<Int8Type> = test.iter().map(|&x| if x == "b" {None} else {Some(x)}).collect();
-/// assert_eq!(array.keys().collect::<Vec<Option<i8>>>(), vec![Some(0), Some(0), None, Some(1)]);
-/// ```
-///
-/// Example **without nullable** data:
-///
-/// ```
-/// use arrow::array::DictionaryArray;
-/// use arrow::datatypes::Int8Type;
-/// let test = vec!["a", "a", "b", "c"];
-/// let array : DictionaryArray<Int8Type> = test.into_iter().collect();
-/// assert_eq!(array.keys().collect::<Vec<Option<i8>>>(), vec![Some(0), Some(0), Some(1), Some(2)]);
-/// ```
-pub struct DictionaryArray<K: ArrowPrimitiveType> {
-    /// Array of keys, much like a PrimitiveArray
-    data: ArrayDataRef,
-
-    /// Pointer to the key values.
-    raw_values: RawPtrBox<K::Native>,
-
-    /// Array of any values.
-    values: ArrayRef,
-
-    /// Values are ordered.
-    is_ordered: bool,
-}
-
-#[derive(Debug)]
-pub struct NullableIter<'a, T> {
-    data: &'a ArrayDataRef, // TODO: Use a pointer to the null bitmap.
-    ptr: *const T,
-    i: usize,
-    len: usize,
-}
-
-impl<'a, T> std::iter::Iterator for NullableIter<'a, T>
-where
-    T: Clone,
-{
-    type Item = Option<T>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        let i = self.i;
-        if i >= self.len {
-            None
-        } else if self.data.is_null(i) {
-            self.i += 1;
-            Some(None)
-        } else {
-            self.i += 1;
-            unsafe { Some(Some((&*self.ptr.add(i)).clone())) }
-        }
-    }
-
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        (self.len, Some(self.len))
-    }
-
-    fn nth(&mut self, n: usize) -> Option<Self::Item> {
-        let i = self.i;
-        if i + n >= self.len {
-            self.i = self.len;
-            None
-        } else if self.data.is_null(i + n) {
-            self.i += n + 1;
-            Some(None)
-        } else {
-            self.i += n + 1;
-            unsafe { Some(Some((&*self.ptr.add(i + n)).clone())) }
-        }
-    }
-}
-
-impl<'a, K: ArrowPrimitiveType> DictionaryArray<K> {
-    /// Return an iterator to the keys of this dictionary.
-    pub fn keys(&self) -> NullableIter<'_, K::Native> {
-        NullableIter::<'_, K::Native> {
-            data: &self.data,
-            ptr: unsafe { self.raw_values.get().add(self.data.offset()) },
-            i: 0,
-            len: self.data.len(),
-        }
-    }
-
-    /// Returns the lookup key by doing reverse dictionary lookup
-    pub fn lookup_key(&self, value: &str) -> Option<K::Native> {
-        let rd_buf: &StringArray =
-            self.values.as_any().downcast_ref::<StringArray>().unwrap();
-
-        (0..rd_buf.len())
-            .position(|i| rd_buf.value(i) == value)
-            .map(K::Native::from_usize)
-            .flatten()
-    }
-
-    /// Returns an `ArrayRef` to the dictionary values.
-    pub fn values(&self) -> ArrayRef {
-        self.values.clone()
-    }
-
-    /// Returns a clone of the value type of this list.
-    pub fn value_type(&self) -> DataType {
-        self.values.data().data_type().clone()
-    }
-
-    /// The length of the dictionary is the length of the keys array.
-    pub fn len(&self) -> usize {
-        self.data.len()
-    }
-
-    /// Whether this dictionary is empty
-    pub fn is_empty(&self) -> bool {
-        self.data.is_empty()
-    }
-
-    // Currently exists for compatibility purposes with Arrow IPC.
-    pub fn is_ordered(&self) -> bool {
-        self.is_ordered
-    }
-}
-
-/// Constructs a `DictionaryArray` from an array data reference.
-impl<T: ArrowPrimitiveType> From<ArrayDataRef> for DictionaryArray<T> {
-    fn from(data: ArrayDataRef) -> Self {
-        assert_eq!(
-            data.buffers().len(),
-            1,
-            "DictionaryArray data should contain a single buffer only (keys)."
-        );
-        assert_eq!(
-            data.child_data().len(),
-            1,
-            "DictionaryArray should contain a single child array (values)."
-        );
-
-        let raw_values = data.buffers()[0].raw_data();
-        let dtype: &DataType = data.data_type();
-        let values = make_array(data.child_data()[0].clone());
-        if let DataType::Dictionary(_, _) = dtype {
-            Self {
-                data,
-                raw_values: RawPtrBox::new(raw_values as *const T::Native),
-                values,
-                is_ordered: false,
-            }
-        } else {
-            panic!("DictionaryArray must have Dictionary data type.")
-        }
-    }
-}
-
-/// Constructs a `DictionaryArray` from an iterator of optional strings.
-impl<T: ArrowPrimitiveType + ArrowDictionaryKeyType> FromIterator<Option<&'static str>>
-    for DictionaryArray<T>
-{
-    fn from_iter<I: IntoIterator<Item = Option<&'static str>>>(iter: I) -> Self {
-        let it = iter.into_iter();
-        let (lower, _) = it.size_hint();
-        let key_builder = PrimitiveBuilder::<T>::new(lower);
-        let value_builder = StringBuilder::new(256);
-        let mut builder = StringDictionaryBuilder::new(key_builder, value_builder);
-        it.for_each(|i| {
-            if let Some(i) = i {
-                // Note: impl ... for Result<DictionaryArray<T>> fails with
-                // error[E0117]: only traits defined in the current crate can be implemented for arbitrary types
-                builder
-                    .append(i)
-                    .expect("Unable to append a value to a dictionary array.");
-            } else {
-                builder
-                    .append_null()
-                    .expect("Unable to append a null value to a dictionary array.");
-            }
-        });
-
-        builder.finish()
-    }
-}
-
-/// Constructs a `DictionaryArray` from an iterator of strings.
-impl<T: ArrowPrimitiveType + ArrowDictionaryKeyType> FromIterator<&'static str>
-    for DictionaryArray<T>
-{
-    fn from_iter<I: IntoIterator<Item = &'static str>>(iter: I) -> Self {
-        let it = iter.into_iter();
-        let (lower, _) = it.size_hint();
-        let key_builder = PrimitiveBuilder::<T>::new(lower);
-        let value_builder = StringBuilder::new(256);
-        let mut builder = StringDictionaryBuilder::new(key_builder, value_builder);
-        it.for_each(|i| {
-            builder
-                .append(i)
-                .expect("Unable to append a value to a dictionary array.");
-        });
-
-        builder.finish()
-    }
-}
-
-impl<T: ArrowPrimitiveType> Array for DictionaryArray<T> {
-    fn as_any(&self) -> &Any {
-        self
-    }
-
-    fn data(&self) -> ArrayDataRef {
-        self.data.clone()
-    }
-
-    fn data_ref(&self) -> &ArrayDataRef {
-        &self.data
-    }
-
-    /// Returns the total number of bytes of memory occupied by the buffers owned by this [DictionaryArray].
-    fn get_buffer_memory_size(&self) -> usize {
-        self.data.get_buffer_memory_size() + self.values().get_buffer_memory_size()
-    }
-
-    /// Returns the total number of bytes of memory occupied physically by this [DictionaryArray].
-    fn get_array_memory_size(&self) -> usize {
-        self.data.get_array_memory_size()
-            + self.values().get_array_memory_size()
-            + mem::size_of_val(self)
-    }
-}
-
-impl<T: ArrowPrimitiveType> fmt::Debug for DictionaryArray<T> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        const MAX_LEN: usize = 10;
-        let keys: Vec<_> = self.keys().take(MAX_LEN).collect();
-        let elipsis = if self.keys().count() > MAX_LEN {
-            "..."
-        } else {
-            ""
-        };
-        writeln!(
-            f,
-            "DictionaryArray {{keys: {:?}{} values: {:?}}}",
-            keys, elipsis, self.values
-        )
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    use std::sync::Arc;
-    use std::thread;
-
-    use crate::buffer::Buffer;
-    use crate::datatypes::{DataType, Field};
-    use crate::memory;
-
-    #[test]
-    fn test_primitive_array_from_vec() {
-        let buf = Buffer::from(&[0, 1, 2, 3, 4].to_byte_slice());
-        let buf2 = buf.clone();
-        let arr = Int32Array::new(5, buf, 0, 0);
-        let slice = unsafe { std::slice::from_raw_parts(arr.raw_values(), 5) };
-        assert_eq!(buf2, arr.values());
-        assert_eq!(&[0, 1, 2, 3, 4], slice);
-        assert_eq!(5, arr.len());
-        assert_eq!(0, arr.offset());
-        assert_eq!(0, arr.null_count());
-        for i in 0..5 {
-            assert!(!arr.is_null(i));
-            assert!(arr.is_valid(i));
-            assert_eq!(i as i32, arr.value(i));
-        }
-
-        assert_eq!(64, arr.get_buffer_memory_size());
-        let internals_of_primitive_array = 8 + 72; // RawPtrBox & Arc<ArrayData> combined.
-        assert_eq!(
-            arr.get_buffer_memory_size() + internals_of_primitive_array,
-            arr.get_array_memory_size()
-        );
-    }
-
-    #[test]
-    fn test_primitive_array_from_vec_option() {
-        // Test building a primitive array with null values
-        let arr = Int32Array::from(vec![Some(0), None, Some(2), None, Some(4)]);
-        assert_eq!(5, arr.len());
-        assert_eq!(0, arr.offset());
-        assert_eq!(2, arr.null_count());
-        for i in 0..5 {
-            if i % 2 == 0 {
-                assert!(!arr.is_null(i));
-                assert!(arr.is_valid(i));
-                assert_eq!(i as i32, arr.value(i));
-            } else {
-                assert!(arr.is_null(i));
-                assert!(!arr.is_valid(i));
-            }
-        }
-
-        assert_eq!(128, arr.get_buffer_memory_size());
-        let internals_of_primitive_array = 8 + 72 + 16; // RawPtrBox & Arc<ArrayData> and it's null_bitmap combined.
-        assert_eq!(
-            arr.get_buffer_memory_size() + internals_of_primitive_array,
-            arr.get_array_memory_size()
-        );
-    }
-
-    #[test]
-    fn test_date64_array_from_vec_option() {
-        // Test building a primitive array with null values
-        // we use Int32 and Int64 as a backing array, so all Int32 and Int64 conventions
-        // work
-        let arr: PrimitiveArray<Date64Type> =
-            vec![Some(1550902545147), None, Some(1550902545147)].into();
-        assert_eq!(3, arr.len());
-        assert_eq!(0, arr.offset());
-        assert_eq!(1, arr.null_count());
-        for i in 0..3 {
-            if i % 2 == 0 {
-                assert!(!arr.is_null(i));
-                assert!(arr.is_valid(i));
-                assert_eq!(1550902545147, arr.value(i));
-                // roundtrip to and from datetime
-                assert_eq!(
-                    1550902545147,
-                    arr.value_as_datetime(i).unwrap().timestamp_millis()
-                );
-            } else {
-                assert!(arr.is_null(i));
-                assert!(!arr.is_valid(i));
-            }
-        }
-    }
-
-    #[test]
-    fn test_time32_millisecond_array_from_vec() {
-        // 1:        00:00:00.001
-        // 37800005: 10:30:00.005
-        // 86399210: 23:59:59.210
-        let arr: PrimitiveArray<Time32MillisecondType> =
-            vec![1, 37_800_005, 86_399_210].into();
-        assert_eq!(3, arr.len());
-        assert_eq!(0, arr.offset());
-        assert_eq!(0, arr.null_count());
-        let formatted = vec!["00:00:00.001", "10:30:00.005", "23:59:59.210"];
-        for i in 0..3 {
-            // check that we can't create dates or datetimes from time instances
-            assert_eq!(None, arr.value_as_datetime(i));
-            assert_eq!(None, arr.value_as_date(i));
-            let time = arr.value_as_time(i).unwrap();
-            assert_eq!(formatted[i], time.format("%H:%M:%S%.3f").to_string());
-        }
-    }
-
-    #[test]
-    fn test_time64_nanosecond_array_from_vec() {
-        // Test building a primitive array with null values
-        // we use Int32 and Int64 as a backing array, so all Int32 and Int64 conventions
-        // work
-
-        // 1e6:        00:00:00.001
-        // 37800005e6: 10:30:00.005
-        // 86399210e6: 23:59:59.210
-        let arr: PrimitiveArray<Time64NanosecondType> =
-            vec![1_000_000, 37_800_005_000_000, 86_399_210_000_000].into();
-        assert_eq!(3, arr.len());
-        assert_eq!(0, arr.offset());
-        assert_eq!(0, arr.null_count());
-        let formatted = vec!["00:00:00.001", "10:30:00.005", "23:59:59.210"];
-        for i in 0..3 {
-            // check that we can't create dates or datetimes from time instances
-            assert_eq!(None, arr.value_as_datetime(i));
-            assert_eq!(None, arr.value_as_date(i));
-            let time = arr.value_as_time(i).unwrap();
-            assert_eq!(formatted[i], time.format("%H:%M:%S%.3f").to_string());
-        }
-    }
-
-    #[test]
-    fn test_interval_array_from_vec() {
-        // intervals are currently not treated specially, but are Int32 and Int64 arrays
-        let arr = IntervalYearMonthArray::from(vec![Some(1), None, Some(-5)]);
-        assert_eq!(3, arr.len());
-        assert_eq!(0, arr.offset());
-        assert_eq!(1, arr.null_count());
-        assert_eq!(1, arr.value(0));
-        assert!(arr.is_null(1));
-        assert_eq!(-5, arr.value(2));
-
-        // a day_time interval contains days and milliseconds, but we do not yet have accessors for the values
-        let arr = IntervalDayTimeArray::from(vec![Some(1), None, Some(-5)]);
-        assert_eq!(3, arr.len());
-        assert_eq!(0, arr.offset());
-        assert_eq!(1, arr.null_count());
-        assert_eq!(1, arr.value(0));
-        assert!(arr.is_null(1));
-        assert_eq!(-5, arr.value(2));
-    }
-
-    #[test]
-    fn test_duration_array_from_vec() {
-        let arr = DurationSecondArray::from(vec![Some(1), None, Some(-5)]);
-        assert_eq!(3, arr.len());
-        assert_eq!(0, arr.offset());
-        assert_eq!(1, arr.null_count());
-        assert_eq!(1, arr.value(0));
-        assert!(arr.is_null(1));
-        assert_eq!(-5, arr.value(2));
-
-        let arr = DurationMillisecondArray::from(vec![Some(1), None, Some(-5)]);
-        assert_eq!(3, arr.len());
-        assert_eq!(0, arr.offset());
-        assert_eq!(1, arr.null_count());
-        assert_eq!(1, arr.value(0));
-        assert!(arr.is_null(1));
-        assert_eq!(-5, arr.value(2));
-
-        let arr = DurationMicrosecondArray::from(vec![Some(1), None, Some(-5)]);
-        assert_eq!(3, arr.len());
-        assert_eq!(0, arr.offset());
-        assert_eq!(1, arr.null_count());
-        assert_eq!(1, arr.value(0));
-        assert!(arr.is_null(1));
-        assert_eq!(-5, arr.value(2));
-
-        let arr = DurationNanosecondArray::from(vec![Some(1), None, Some(-5)]);
-        assert_eq!(3, arr.len());
-        assert_eq!(0, arr.offset());
-        assert_eq!(1, arr.null_count());
-        assert_eq!(1, arr.value(0));
-        assert!(arr.is_null(1));
-        assert_eq!(-5, arr.value(2));
-    }
-
-    #[test]
-    fn test_timestamp_array_from_vec() {
-        let arr = TimestampSecondArray::from_vec(vec![1, -5], None);
-        assert_eq!(2, arr.len());
-        assert_eq!(0, arr.offset());
-        assert_eq!(0, arr.null_count());
-        assert_eq!(1, arr.value(0));
-        assert_eq!(-5, arr.value(1));
-
-        let arr = TimestampMillisecondArray::from_vec(vec![1, -5], None);
-        assert_eq!(2, arr.len());
-        assert_eq!(0, arr.offset());
-        assert_eq!(0, arr.null_count());
-        assert_eq!(1, arr.value(0));
-        assert_eq!(-5, arr.value(1));
-
-        let arr = TimestampMicrosecondArray::from_vec(vec![1, -5], None);
-        assert_eq!(2, arr.len());
-        assert_eq!(0, arr.offset());
-        assert_eq!(0, arr.null_count());
-        assert_eq!(1, arr.value(0));
-        assert_eq!(-5, arr.value(1));
-
-        let arr = TimestampNanosecondArray::from_vec(vec![1, -5], None);
-        assert_eq!(2, arr.len());
-        assert_eq!(0, arr.offset());
-        assert_eq!(0, arr.null_count());
-        assert_eq!(1, arr.value(0));
-        assert_eq!(-5, arr.value(1));
-    }
-
-    #[test]
-    fn test_primitive_array_slice() {
-        let arr = Int32Array::from(vec![
-            Some(0),
-            None,
-            Some(2),
-            None,
-            Some(4),
-            Some(5),
-            Some(6),
-            None,
-            None,
-        ]);
-        assert_eq!(9, arr.len());
-        assert_eq!(0, arr.offset());
-        assert_eq!(4, arr.null_count());
-
-        let arr2 = arr.slice(2, 5);
-        assert_eq!(5, arr2.len());
-        assert_eq!(2, arr2.offset());
-        assert_eq!(1, arr2.null_count());
-
-        for i in 0..arr2.len() {
-            assert_eq!(i == 1, arr2.is_null(i));
-            assert_eq!(i != 1, arr2.is_valid(i));
-        }
-
-        let arr3 = arr2.slice(2, 3);
-        assert_eq!(3, arr3.len());
-        assert_eq!(4, arr3.offset());
-        assert_eq!(0, arr3.null_count());
-
-        let int_arr = arr3.as_any().downcast_ref::<Int32Array>().unwrap();
-        assert_eq!(4, int_arr.value(0));
-        assert_eq!(5, int_arr.value(1));
-        assert_eq!(6, int_arr.value(2));
-    }
-
-    #[test]
-    fn test_value_slice_no_bounds_check() {
-        let arr = Int32Array::from(vec![2, 3, 4]);
-        let _slice = arr.value_slice(0, 4);
-    }
-
-    #[test]
-    fn test_int32_fmt_debug() {
-        let buf = Buffer::from(&[0, 1, 2, 3, 4].to_byte_slice());
-        let arr = Int32Array::new(5, buf, 0, 0);
-        assert_eq!(
-            "PrimitiveArray<Int32>\n[\n  0,\n  1,\n  2,\n  3,\n  4,\n]",
-            format!("{:?}", arr)
-        );
-    }
-
-    #[test]
-    fn test_int32_with_null_fmt_debug() {
-        let mut builder = Int32Array::builder(3);
-        builder.append_slice(&[0, 1]).unwrap();
-        builder.append_null().unwrap();
-        builder.append_slice(&[3, 4]).unwrap();
-        let arr = builder.finish();
-        assert_eq!(
-            "PrimitiveArray<Int32>\n[\n  0,\n  1,\n  null,\n  3,\n  4,\n]",
-            format!("{:?}", arr)
-        );
-    }
-
-    #[test]
-    fn test_boolean_fmt_debug() {
-        let buf = Buffer::from(&[true, false, false].to_byte_slice());
-        let arr = BooleanArray::new(3, buf, 0, 0);
-        assert_eq!(
-            "PrimitiveArray<Boolean>\n[\n  true,\n  false,\n  false,\n]",
-            format!("{:?}", arr)
-        );
-    }
-
-    #[test]
-    fn test_boolean_with_null_fmt_debug() {
-        let mut builder = BooleanArray::builder(3);
-        builder.append_value(true).unwrap();
-        builder.append_null().unwrap();
-        builder.append_value(false).unwrap();
-        let arr = builder.finish();
-        assert_eq!(
-            "PrimitiveArray<Boolean>\n[\n  true,\n  null,\n  false,\n]",
-            format!("{:?}", arr)
-        );
-    }
-
-    #[test]
-    fn test_timestamp_fmt_debug() {
-        let arr: PrimitiveArray<TimestampMillisecondType> =
-            TimestampMillisecondArray::from_vec(vec![1546214400000, 1546214400000], None);
-        assert_eq!(
-            "PrimitiveArray<Timestamp(Millisecond, None)>\n[\n  2018-12-31T00:00:00,\n  2018-12-31T00:00:00,\n]",
-            format!("{:?}", arr)
-        );
-    }
-
-    #[test]
-    fn test_date32_fmt_debug() {
-        let arr: PrimitiveArray<Date32Type> = vec![12356, 13548].into();
-        assert_eq!(
-            "PrimitiveArray<Date32(Day)>\n[\n  2003-10-31,\n  2007-02-04,\n]",
-            format!("{:?}", arr)
-        );
-    }
-
-    #[test]
-    fn test_time32second_fmt_debug() {
-        let arr: PrimitiveArray<Time32SecondType> = vec![7201, 60054].into();
-        assert_eq!(
-            "PrimitiveArray<Time32(Second)>\n[\n  02:00:01,\n  16:40:54,\n]",
-            format!("{:?}", arr)
-        );
-    }
-
-    #[test]
-    fn test_primitive_array_builder() {
-        // Test building a primitive array with ArrayData builder and offset
-        let buf = Buffer::from(&[0, 1, 2, 3, 4].to_byte_slice());
-        let buf2 = buf.clone();
-        let data = ArrayData::builder(DataType::Int32)
-            .len(5)
-            .offset(2)
-            .add_buffer(buf)
-            .build();
-        let arr = Int32Array::from(data);
-        assert_eq!(buf2, arr.values());
-        assert_eq!(5, arr.len());
-        assert_eq!(0, arr.null_count());
-        for i in 0..3 {
-            assert_eq!((i + 2) as i32, arr.value(i));
-        }
-    }
-
-    #[test]
-    #[should_panic(expected = "PrimitiveArray data should contain a single buffer only \
-                               (values buffer)")]
-    fn test_primitive_array_invalid_buffer_len() {
-        let data = ArrayData::builder(DataType::Int32).len(5).build();
-        Int32Array::from(data);
-    }
-
-    #[test]
-    fn test_boolean_array_new() {
-        // 00000010 01001000
-        let buf = Buffer::from([72_u8, 2_u8]);
-        let buf2 = buf.clone();
-        let arr = BooleanArray::new(10, buf, 0, 0);
-        assert_eq!(buf2, arr.values());
-        assert_eq!(10, arr.len());
-        assert_eq!(0, arr.offset());
-        assert_eq!(0, arr.null_count());
-        for i in 0..10 {
-            assert!(!arr.is_null(i));
-            assert!(arr.is_valid(i));
-            assert_eq!(i == 3 || i == 6 || i == 9, arr.value(i), "failed at {}", i)
-        }
-    }
-
-    #[test]
-    fn test_boolean_array_from_vec() {
-        let buf = Buffer::from([10_u8]);
-        let arr = BooleanArray::from(vec![false, true, false, true]);
-        assert_eq!(buf, arr.values());
-        assert_eq!(4, arr.len());
-        assert_eq!(0, arr.offset());
-        assert_eq!(0, arr.null_count());
-        for i in 0..4 {
-            assert!(!arr.is_null(i));
-            assert!(arr.is_valid(i));
-            assert_eq!(i == 1 || i == 3, arr.value(i), "failed at {}", i)
-        }
-    }
-
-    #[test]
-    fn test_boolean_array_from_vec_option() {
-        let buf = Buffer::from([10_u8]);
-        let arr = BooleanArray::from(vec![Some(false), Some(true), None, Some(true)]);
-        assert_eq!(buf, arr.values());
-        assert_eq!(4, arr.len());
-        assert_eq!(0, arr.offset());
-        assert_eq!(1, arr.null_count());
-        for i in 0..4 {
-            if i == 2 {
-                assert!(arr.is_null(i));
-                assert!(!arr.is_valid(i));
-            } else {
-                assert!(!arr.is_null(i));
-                assert!(arr.is_valid(i));
-                assert_eq!(i == 1 || i == 3, arr.value(i), "failed at {}", i)
-            }
-        }
-    }
-
-    #[test]
-    fn test_boolean_array_builder() {
-        // Test building a boolean array with ArrayData builder and offset
-        // 000011011
-        let buf = Buffer::from([27_u8]);
-        let buf2 = buf.clone();
-        let data = ArrayData::builder(DataType::Boolean)
-            .len(5)
-            .offset(2)
-            .add_buffer(buf)
-            .build();
-        let arr = BooleanArray::from(data);
-        assert_eq!(buf2, arr.values());
-        assert_eq!(5, arr.len());
-        assert_eq!(2, arr.offset());
-        assert_eq!(0, arr.null_count());
-        for i in 0..3 {
-            assert_eq!(i != 0, arr.value(i), "failed at {}", i);
-        }
-    }
-
-    #[test]
-    #[should_panic(expected = "PrimitiveArray data should contain a single buffer only \
-                               (values buffer)")]
-    fn test_boolean_array_invalid_buffer_len() {
-        let data = ArrayData::builder(DataType::Boolean).len(5).build();
-        BooleanArray::from(data);
-    }
-
-    #[test]
-    fn test_list_array() {
-        // Construct a value array
-        let value_data = ArrayData::builder(DataType::Int32)
-            .len(8)
-            .add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
-            .build();
-
-        // Construct a buffer for value offsets, for the nested array:
-        //  [[0, 1, 2], [3, 4, 5], [6, 7]]
-        let value_offsets = Buffer::from(&[0, 3, 6, 8].to_byte_slice());
-
-        // Construct a list array from the above two
-        let list_data_type = DataType::List(Box::new(DataType::Int32));
-        let list_data = ArrayData::builder(list_data_type.clone())
-            .len(3)
-            .add_buffer(value_offsets.clone())
-            .add_child_data(value_data.clone())
-            .build();
-        let list_array = ListArray::from(list_data);
-
-        let values = list_array.values();
-        assert_eq!(value_data, values.data());
-        assert_eq!(DataType::Int32, list_array.value_type());
-        assert_eq!(3, list_array.len());
-        assert_eq!(0, list_array.null_count());
-        assert_eq!(6, list_array.value_offset(2));
-        assert_eq!(2, list_array.value_length(2));
-        assert_eq!(
-            0,
-            list_array
-                .value(0)
-                .as_any()
-                .downcast_ref::<Int32Array>()
-                .unwrap()
-                .value(0)
-        );
-        for i in 0..3 {
-            assert!(list_array.is_valid(i));
-            assert!(!list_array.is_null(i));
-        }
-
-        // Now test with a non-zero offset
-        let list_data = ArrayData::builder(list_data_type)
-            .len(3)
-            .offset(1)
-            .add_buffer(value_offsets)
-            .add_child_data(value_data.clone())
-            .build();
-        let list_array = ListArray::from(list_data);
-
-        let values = list_array.values();
-        assert_eq!(value_data, values.data());
-        assert_eq!(DataType::Int32, list_array.value_type());
-        assert_eq!(3, list_array.len());
-        assert_eq!(0, list_array.null_count());
-        assert_eq!(6, list_array.value_offset(1));
-        assert_eq!(2, list_array.value_length(1));
-        assert_eq!(
-            3,
-            list_array
-                .value(0)
-                .as_any()
-                .downcast_ref::<Int32Array>()
-                .unwrap()
-                .value(0)
-        );
-    }
-
-    #[test]
-    fn test_large_list_array() {
-        // Construct a value array
-        let value_data = ArrayData::builder(DataType::Int32)
-            .len(8)
-            .add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
-            .build();
-
-        // Construct a buffer for value offsets, for the nested array:
-        //  [[0, 1, 2], [3, 4, 5], [6, 7]]
-        let value_offsets = Buffer::from(&[0i64, 3, 6, 8].to_byte_slice());
-
-        // Construct a list array from the above two
-        let list_data_type = DataType::LargeList(Box::new(DataType::Int32));
-        let list_data = ArrayData::builder(list_data_type.clone())
-            .len(3)
-            .add_buffer(value_offsets.clone())
-            .add_child_data(value_data.clone())
-            .build();
-        let list_array = LargeListArray::from(list_data);
-
-        let values = list_array.values();
-        assert_eq!(value_data, values.data());
-        assert_eq!(DataType::Int32, list_array.value_type());
-        assert_eq!(3, list_array.len());
-        assert_eq!(0, list_array.null_count());
-        assert_eq!(6, list_array.value_offset(2));
-        assert_eq!(2, list_array.value_length(2));
-        assert_eq!(
-            0,
-            list_array
-                .value(0)
-                .as_any()
-                .downcast_ref::<Int32Array>()
-                .unwrap()
-                .value(0)
-        );
-        for i in 0..3 {
-            assert!(list_array.is_valid(i));
-            assert!(!list_array.is_null(i));
-        }
-
-        // Now test with a non-zero offset
-        let list_data = ArrayData::builder(list_data_type)
-            .len(3)
-            .offset(1)
-            .add_buffer(value_offsets)
-            .add_child_data(value_data.clone())
-            .build();
-        let list_array = LargeListArray::from(list_data);
-
-        let values = list_array.values();
-        assert_eq!(value_data, values.data());
-        assert_eq!(DataType::Int32, list_array.value_type());
-        assert_eq!(3, list_array.len());
-        assert_eq!(0, list_array.null_count());
-        assert_eq!(6, list_array.value_offset(1));
-        assert_eq!(2, list_array.value_length(1));
-        assert_eq!(
-            3,
-            list_array
-                .value(0)
-                .as_any()
-                .downcast_ref::<Int32Array>()
-                .unwrap()
-                .value(0)
-        );
-    }
-
-    #[test]
-    fn test_dictionary_array() {
-        // Construct a value array
-        let value_data = ArrayData::builder(DataType::Int8)
-            .len(8)
-            .add_buffer(Buffer::from(
-                &[10_i8, 11, 12, 13, 14, 15, 16, 17].to_byte_slice(),
-            ))
-            .build();
-
-        // Construct a buffer for value offsets, for the nested array:
-        let keys = Buffer::from(&[2_i16, 3, 4].to_byte_slice());
-
-        // Construct a dictionary array from the above two
-        let key_type = DataType::Int16;
-        let value_type = DataType::Int8;
-        let dict_data_type =
-            DataType::Dictionary(Box::new(key_type), Box::new(value_type));
-        let dict_data = ArrayData::builder(dict_data_type.clone())
-            .len(3)
-            .add_buffer(keys.clone())
-            .add_child_data(value_data.clone())
-            .build();
-        let dict_array = Int16DictionaryArray::from(dict_data);
-
-        let values = dict_array.values();
-        assert_eq!(value_data, values.data());
-        assert_eq!(DataType::Int8, dict_array.value_type());
-        assert_eq!(3, dict_array.len());
-
-        // Null count only makes sense in terms of the component arrays.
-        assert_eq!(0, dict_array.null_count());
-        assert_eq!(0, dict_array.values().null_count());
-        assert_eq!(Some(Some(3)), dict_array.keys().nth(1));
-        assert_eq!(Some(Some(4)), dict_array.keys().nth(2));
-
-        assert_eq!(
-            dict_array.keys().collect::<Vec<Option<i16>>>(),
-            vec![Some(2), Some(3), Some(4)]
-        );
-
-        // Now test with a non-zero offset
-        let dict_data = ArrayData::builder(dict_data_type)
-            .len(2)
-            .offset(1)
-            .add_buffer(keys)
-            .add_child_data(value_data.clone())
-            .build();
-        let dict_array = Int16DictionaryArray::from(dict_data);
-
-        let values = dict_array.values();
-        assert_eq!(value_data, values.data());
-        assert_eq!(DataType::Int8, dict_array.value_type());
-        assert_eq!(2, dict_array.len());
-        assert_eq!(Some(Some(3)), dict_array.keys().nth(0));
-        assert_eq!(Some(Some(4)), dict_array.keys().nth(1));
-
-        assert_eq!(
-            dict_array.keys().collect::<Vec<Option<i16>>>(),
-            vec![Some(3), Some(4)]
-        );
-    }
-
-    #[test]
-    fn test_fixed_size_list_array() {
-        // Construct a value array
-        let value_data = ArrayData::builder(DataType::Int32)
-            .len(9)
-            .add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7, 8].to_byte_slice()))
-            .build();
-
-        // Construct a list array from the above two
-        let list_data_type = DataType::FixedSizeList(Box::new(DataType::Int32), 3);
-        let list_data = ArrayData::builder(list_data_type.clone())
-            .len(3)
-            .add_child_data(value_data.clone())
-            .build();
-        let list_array = FixedSizeListArray::from(list_data);
-
-        let values = list_array.values();
-        assert_eq!(value_data, values.data());
-        assert_eq!(DataType::Int32, list_array.value_type());
-        assert_eq!(3, list_array.len());
-        assert_eq!(0, list_array.null_count());
-        assert_eq!(6, list_array.value_offset(2));
-        assert_eq!(3, list_array.value_length());
-        assert_eq!(
-            0,
-            list_array
-                .value(0)
-                .as_any()
-                .downcast_ref::<Int32Array>()
-                .unwrap()
-                .value(0)
-        );
-        for i in 0..3 {
-            assert!(list_array.is_valid(i));
-            assert!(!list_array.is_null(i));
-        }
-
-        // Now test with a non-zero offset
-        let list_data = ArrayData::builder(list_data_type)
-            .len(3)
-            .offset(1)
-            .add_child_data(value_data.clone())
-            .build();
-        let list_array = FixedSizeListArray::from(list_data);
-
-        let values = list_array.values();
-        assert_eq!(value_data, values.data());
-        assert_eq!(DataType::Int32, list_array.value_type());
-        assert_eq!(3, list_array.len());
-        assert_eq!(0, list_array.null_count());
-        assert_eq!(
-            3,
-            list_array
-                .value(0)
-                .as_any()
-                .downcast_ref::<Int32Array>()
-                .unwrap()
-                .value(0)
-        );
-        assert_eq!(6, list_array.value_offset(1));
-        assert_eq!(3, list_array.value_length());
-    }
-
-    #[test]
-    #[should_panic(
-        expected = "FixedSizeListArray child array length should be a multiple of 3"
-    )]
-    fn test_fixed_size_list_array_unequal_children() {
-        // Construct a value array
-        let value_data = ArrayData::builder(DataType::Int32)
-            .len(8)
-            .add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
-            .build();
-
-        // Construct a list array from the above two
-        let list_data_type = DataType::FixedSizeList(Box::new(DataType::Int32), 3);
-        let list_data = ArrayData::builder(list_data_type.clone())
-            .len(3)
-            .add_child_data(value_data.clone())
-            .build();
-        FixedSizeListArray::from(list_data);
-    }
-
-    #[test]
-    fn test_list_array_slice() {
-        // Construct a value array
-        let value_data = ArrayData::builder(DataType::Int32)
-            .len(10)
-            .add_buffer(Buffer::from(
-                &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9].to_byte_slice(),
-            ))
-            .build();
-
-        // Construct a buffer for value offsets, for the nested array:
-        //  [[0, 1], null, null, [2, 3], [4, 5], null, [6, 7, 8], null, [9]]
-        let value_offsets =
-            Buffer::from(&[0, 2, 2, 2, 4, 6, 6, 9, 9, 10].to_byte_slice());
-        // 01011001 00000001
-        let mut null_bits: [u8; 2] = [0; 2];
-        bit_util::set_bit(&mut null_bits, 0);
-        bit_util::set_bit(&mut null_bits, 3);
-        bit_util::set_bit(&mut null_bits, 4);
-        bit_util::set_bit(&mut null_bits, 6);
-        bit_util::set_bit(&mut null_bits, 8);
-
-        // Construct a list array from the above two
-        let list_data_type = DataType::List(Box::new(DataType::Int32));
-        let list_data = ArrayData::builder(list_data_type.clone())
-            .len(9)
-            .add_buffer(value_offsets.clone())
-            .add_child_data(value_data.clone())
-            .null_bit_buffer(Buffer::from(null_bits))
-            .build();
-        let list_array = ListArray::from(list_data);
-
-        let values = list_array.values();
-        assert_eq!(value_data, values.data());
-        assert_eq!(DataType::Int32, list_array.value_type());
-        assert_eq!(9, list_array.len());
-        assert_eq!(4, list_array.null_count());
-        assert_eq!(2, list_array.value_offset(3));
-        assert_eq!(2, list_array.value_length(3));
-
-        let sliced_array = list_array.slice(1, 6);
-        assert_eq!(6, sliced_array.len());
-        assert_eq!(1, sliced_array.offset());
-        assert_eq!(3, sliced_array.null_count());
-
-        for i in 0..sliced_array.len() {
-            if bit_util::get_bit(&null_bits, sliced_array.offset() + i) {
-                assert!(sliced_array.is_valid(i));
-            } else {
-                assert!(sliced_array.is_null(i));
-            }
-        }
-
-        // Check offset and length for each non-null value.
-        let sliced_list_array =
-            sliced_array.as_any().downcast_ref::<ListArray>().unwrap();
-        assert_eq!(2, sliced_list_array.value_offset(2));
-        assert_eq!(2, sliced_list_array.value_length(2));
-        assert_eq!(4, sliced_list_array.value_offset(3));
-        assert_eq!(2, sliced_list_array.value_length(3));
-        assert_eq!(6, sliced_list_array.value_offset(5));
-        assert_eq!(3, sliced_list_array.value_length(5));
-    }
-
-    #[test]
-    fn test_large_list_array_slice() {
-        // Construct a value array
-        let value_data = ArrayData::builder(DataType::Int32)
-            .len(10)
-            .add_buffer(Buffer::from(
-                &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9].to_byte_slice(),
-            ))
-            .build();
-
-        // Construct a buffer for value offsets, for the nested array:
-        //  [[0, 1], null, null, [2, 3], [4, 5], null, [6, 7, 8], null, [9]]
-        let value_offsets =
-            Buffer::from(&[0i64, 2, 2, 2, 4, 6, 6, 9, 9, 10].to_byte_slice());
-        // 01011001 00000001
-        let mut null_bits: [u8; 2] = [0; 2];
-        bit_util::set_bit(&mut null_bits, 0);
-        bit_util::set_bit(&mut null_bits, 3);
-        bit_util::set_bit(&mut null_bits, 4);
-        bit_util::set_bit(&mut null_bits, 6);
-        bit_util::set_bit(&mut null_bits, 8);
-
-        // Construct a list array from the above two
-        let list_data_type = DataType::LargeList(Box::new(DataType::Int32));
-        let list_data = ArrayData::builder(list_data_type.clone())
-            .len(9)
-            .add_buffer(value_offsets.clone())
-            .add_child_data(value_data.clone())
-            .null_bit_buffer(Buffer::from(null_bits))
-            .build();
-        let list_array = LargeListArray::from(list_data);
-
-        let values = list_array.values();
-        assert_eq!(value_data, values.data());
-        assert_eq!(DataType::Int32, list_array.value_type());
-        assert_eq!(9, list_array.len());
-        assert_eq!(4, list_array.null_count());
-        assert_eq!(2, list_array.value_offset(3));
-        assert_eq!(2, list_array.value_length(3));
-
-        let sliced_array = list_array.slice(1, 6);
-        assert_eq!(6, sliced_array.len());
-        assert_eq!(1, sliced_array.offset());
-        assert_eq!(3, sliced_array.null_count());
-
-        for i in 0..sliced_array.len() {
-            if bit_util::get_bit(&null_bits, sliced_array.offset() + i) {
-                assert!(sliced_array.is_valid(i));
-            } else {
-                assert!(sliced_array.is_null(i));
-            }
-        }
-
-        // Check offset and length for each non-null value.
-        let sliced_list_array = sliced_array
-            .as_any()
-            .downcast_ref::<LargeListArray>()
-            .unwrap();
-        assert_eq!(2, sliced_list_array.value_offset(2));
-        assert_eq!(2, sliced_list_array.value_length(2));
-        assert_eq!(4, sliced_list_array.value_offset(3));
-        assert_eq!(2, sliced_list_array.value_length(3));
-        assert_eq!(6, sliced_list_array.value_offset(5));
-        assert_eq!(3, sliced_list_array.value_length(5));
-    }
-
-    #[test]
-    fn test_fixed_size_list_array_slice() {
-        // Construct a value array
-        let value_data = ArrayData::builder(DataType::Int32)
-            .len(10)
-            .add_buffer(Buffer::from(
-                &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9].to_byte_slice(),
-            ))
-            .build();
-
-        // Set null buts for the nested array:
-        //  [[0, 1], null, null, [6, 7], [8, 9]]
-        // 01011001 00000001
-        let mut null_bits: [u8; 1] = [0; 1];
-        bit_util::set_bit(&mut null_bits, 0);
-        bit_util::set_bit(&mut null_bits, 3);
-        bit_util::set_bit(&mut null_bits, 4);
-
-        // Construct a fixed size list array from the above two
-        let list_data_type = DataType::FixedSizeList(Box::new(DataType::Int32), 2);
-        let list_data = ArrayData::builder(list_data_type.clone())
-            .len(5)
-            .add_child_data(value_data.clone())
-            .null_bit_buffer(Buffer::from(null_bits))
-            .build();
-        let list_array = FixedSizeListArray::from(list_data);
-
-        let values = list_array.values();
-        assert_eq!(value_data, values.data());
-        assert_eq!(DataType::Int32, list_array.value_type());
-        assert_eq!(5, list_array.len());
-        assert_eq!(2, list_array.null_count());
-        assert_eq!(6, list_array.value_offset(3));
-        assert_eq!(2, list_array.value_length());
-
-        let sliced_array = list_array.slice(1, 4);
-        assert_eq!(4, sliced_array.len());
-        assert_eq!(1, sliced_array.offset());
-        assert_eq!(2, sliced_array.null_count());
-
-        for i in 0..sliced_array.len() {
-            if bit_util::get_bit(&null_bits, sliced_array.offset() + i) {
-                assert!(sliced_array.is_valid(i));
-            } else {
-                assert!(sliced_array.is_null(i));
-            }
-        }
-
-        // Check offset and length for each non-null value.
-        let sliced_list_array = sliced_array
-            .as_any()
-            .downcast_ref::<FixedSizeListArray>()
-            .unwrap();
-        assert_eq!(2, sliced_list_array.value_length());
-        assert_eq!(6, sliced_list_array.value_offset(2));
-        assert_eq!(8, sliced_list_array.value_offset(3));
-    }
-
-    #[test]
-    #[should_panic(
-        expected = "ListArray data should contain a single buffer only (value offsets)"
-    )]
-    fn test_list_array_invalid_buffer_len() {
-        let value_data = ArrayData::builder(DataType::Int32)
-            .len(8)
-            .add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
-            .build();
-        let list_data_type = DataType::List(Box::new(DataType::Int32));
-        let list_data = ArrayData::builder(list_data_type)
-            .len(3)
-            .add_child_data(value_data)
-            .build();
-        ListArray::from(list_data);
-    }
-
-    #[test]
-    #[should_panic(
-        expected = "ListArray should contain a single child array (values array)"
-    )]
-    fn test_list_array_invalid_child_array_len() {
-        let value_offsets = Buffer::from(&[0, 2, 5, 7].to_byte_slice());
-        let list_data_type = DataType::List(Box::new(DataType::Int32));
-        let list_data = ArrayData::builder(list_data_type)
-            .len(3)
-            .add_buffer(value_offsets)
-            .build();
-        ListArray::from(list_data);
-    }
-
-    #[test]
-    #[should_panic(expected = "offsets do not start at zero")]
-    fn test_list_array_invalid_value_offset_start() {
-        let value_data = ArrayData::builder(DataType::Int32)
-            .len(8)
-            .add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
-            .build();
-
-        let value_offsets = Buffer::from(&[2, 2, 5, 7].to_byte_slice());
-
-        let list_data_type = DataType::List(Box::new(DataType::Int32));
-        let list_data = ArrayData::builder(list_data_type.clone())
-            .len(3)
-            .add_buffer(value_offsets.clone())
-            .add_child_data(value_data.clone())
-            .build();
-        ListArray::from(list_data);
-    }
-
-    #[test]
-    fn test_binary_array() {
-        let values: [u8; 12] = [
-            b'h', b'e', b'l', b'l', b'o', b'p', b'a', b'r', b'q', b'u', b'e', b't',
-        ];
-        let offsets: [i32; 4] = [0, 5, 5, 12];
-
-        // Array data: ["hello", "", "parquet"]
-        let array_data = ArrayData::builder(DataType::Binary)
-            .len(3)
-            .add_buffer(Buffer::from(offsets.to_byte_slice()))
-            .add_buffer(Buffer::from(&values[..]))
-            .build();
-        let binary_array = BinaryArray::from(array_data);
-        assert_eq!(3, binary_array.len());
-        assert_eq!(0, binary_array.null_count());
-        assert_eq!([b'h', b'e', b'l', b'l', b'o'], binary_array.value(0));
-        assert_eq!([] as [u8; 0], binary_array.value(1));
-        assert_eq!(
-            [b'p', b'a', b'r', b'q', b'u', b'e', b't'],
-            binary_array.value(2)
-        );
-        assert_eq!(5, binary_array.value_offset(2));
-        assert_eq!(7, binary_array.value_length(2));
-        for i in 0..3 {
-            assert!(binary_array.is_valid(i));
-            assert!(!binary_array.is_null(i));
-        }
-
-        // Test binary array with offset
-        let array_data = ArrayData::builder(DataType::Binary)
-            .len(4)
-            .offset(1)
-            .add_buffer(Buffer::from(offsets.to_byte_slice()))
-            .add_buffer(Buffer::from(&values[..]))
-            .build();
-        let binary_array = BinaryArray::from(array_data);
-        assert_eq!(
-            [b'p', b'a', b'r', b'q', b'u', b'e', b't'],
-            binary_array.value(1)
-        );
-        assert_eq!(5, binary_array.value_offset(0));
-        assert_eq!(0, binary_array.value_length(0));
-        assert_eq!(5, binary_array.value_offset(1));
-        assert_eq!(7, binary_array.value_length(1));
-    }
-
-    #[test]
-    fn test_large_binary_array() {
-        let values: [u8; 12] = [
-            b'h', b'e', b'l', b'l', b'o', b'p', b'a', b'r', b'q', b'u', b'e', b't',
-        ];
-        let offsets: [i64; 4] = [0, 5, 5, 12];
-
-        // Array data: ["hello", "", "parquet"]
-        let array_data = ArrayData::builder(DataType::Binary)
-            .len(3)
-            .add_buffer(Buffer::from(offsets.to_byte_slice()))
-            .add_buffer(Buffer::from(&values[..]))
-            .build();
-        let binary_array = LargeBinaryArray::from(array_data);
-        assert_eq!(3, binary_array.len());
-        assert_eq!(0, binary_array.null_count());
-        assert_eq!([b'h', b'e', b'l', b'l', b'o'], binary_array.value(0));
-        assert_eq!([] as [u8; 0], binary_array.value(1));
-        assert_eq!(
-            [b'p', b'a', b'r', b'q', b'u', b'e', b't'],
-            binary_array.value(2)
-        );
-        assert_eq!(5, binary_array.value_offset(2));
-        assert_eq!(7, binary_array.value_length(2));
-        for i in 0..3 {
-            assert!(binary_array.is_valid(i));
-            assert!(!binary_array.is_null(i));
-        }
-
-        // Test binary array with offset
-        let array_data = ArrayData::builder(DataType::LargeBinary)
-            .len(4)
-            .offset(1)
-            .add_buffer(Buffer::from(offsets.to_byte_slice()))
-            .add_buffer(Buffer::from(&values[..]))
-            .build();
-        let binary_array = LargeBinaryArray::from(array_data);
-        assert_eq!(
-            [b'p', b'a', b'r', b'q', b'u', b'e', b't'],
-            binary_array.value(1)
-        );
-        assert_eq!(5, binary_array.value_offset(0));
-        assert_eq!(0, binary_array.value_length(0));
-        assert_eq!(5, binary_array.value_offset(1));
-        assert_eq!(7, binary_array.value_length(1));
-    }
-
-    #[test]
-    fn test_binary_array_from_list_array() {
-        let values: [u8; 12] = [
-            b'h', b'e', b'l', b'l', b'o', b'p', b'a', b'r', b'q', b'u', b'e', b't',
-        ];
-        let values_data = ArrayData::builder(DataType::UInt8)
-            .len(12)
-            .add_buffer(Buffer::from(&values[..]))
-            .build();
-        let offsets: [i32; 4] = [0, 5, 5, 12];
-
-        // Array data: ["hello", "", "parquet"]
-        let array_data1 = ArrayData::builder(DataType::Binary)
-            .len(3)
-            .add_buffer(Buffer::from(offsets.to_byte_slice()))
-            .add_buffer(Buffer::from(&values[..]))
-            .build();
-        let binary_array1 = BinaryArray::from(array_data1);
-
-        let array_data2 = ArrayData::builder(DataType::Binary)
-            .len(3)
-            .add_buffer(Buffer::from(offsets.to_byte_slice()))
-            .add_child_data(values_data)
-            .build();
-        let list_array = ListArray::from(array_data2);
-        let binary_array2 = BinaryArray::from(list_array);
-
-        assert_eq!(2, binary_array2.data().buffers().len());
-        assert_eq!(0, binary_array2.data().child_data().len());
-
-        assert_eq!(binary_array1.len(), binary_array2.len());
-        assert_eq!(binary_array1.null_count(), binary_array2.null_count());
-        for i in 0..binary_array1.len() {
-            assert_eq!(binary_array1.value(i), binary_array2.value(i));
-            assert_eq!(binary_array1.value_offset(i), binary_array2.value_offset(i));
-            assert_eq!(binary_array1.value_length(i), binary_array2.value_length(i));
-        }
-    }
-
-    #[test]
-    fn test_large_binary_array_from_list_array() {
-        let values: [u8; 12] = [
-            b'h', b'e', b'l', b'l', b'o', b'p', b'a', b'r', b'q', b'u', b'e', b't',
-        ];
-        let values_data = ArrayData::builder(DataType::UInt8)
-            .len(12)
-            .add_buffer(Buffer::from(&values[..]))
-            .build();
-        let offsets: [i64; 4] = [0, 5, 5, 12];
-
-        // Array data: ["hello", "", "parquet"]
-        let array_data1 = ArrayData::builder(DataType::LargeBinary)
-            .len(3)
-            .add_buffer(Buffer::from(offsets.to_byte_slice()))
-            .add_buffer(Buffer::from(&values[..]))
-            .build();
-        let binary_array1 = LargeBinaryArray::from(array_data1);
-
-        let array_data2 = ArrayData::builder(DataType::Binary)
-            .len(3)
-            .add_buffer(Buffer::from(offsets.to_byte_slice()))
-            .add_child_data(values_data)
-            .build();
-        let list_array = LargeListArray::from(array_data2);
-        let binary_array2 = LargeBinaryArray::from(list_array);
-
-        assert_eq!(2, binary_array2.data().buffers().len());
-        assert_eq!(0, binary_array2.data().child_data().len());
-
-        assert_eq!(binary_array1.len(), binary_array2.len());
-        assert_eq!(binary_array1.null_count(), binary_array2.null_count());
-        for i in 0..binary_array1.len() {
-            assert_eq!(binary_array1.value(i), binary_array2.value(i));
-            assert_eq!(binary_array1.value_offset(i), binary_array2.value_offset(i));
-            assert_eq!(binary_array1.value_length(i), binary_array2.value_length(i));
-        }
-    }
-
-    #[test]
-    fn test_string_array_from_u8_slice() {
-        let values: Vec<&str> = vec!["hello", "", "parquet"];
-
-        // Array data: ["hello", "", "parquet"]
-        let string_array = StringArray::from(values);
-
-        assert_eq!(3, string_array.len());
-        assert_eq!(0, string_array.null_count());
-        assert_eq!("hello", string_array.value(0));
-        assert_eq!("", string_array.value(1));
-        assert_eq!("parquet", string_array.value(2));
-        assert_eq!(5, string_array.value_offset(2));
-        assert_eq!(7, string_array.value_length(2));
-        for i in 0..3 {
-            assert!(string_array.is_valid(i));
-            assert!(!string_array.is_null(i));
-        }
-    }
-
-    #[test]
-    fn test_large_string_array_from_u8_slice() {
-        let values: Vec<&str> = vec!["hello", "", "parquet"];
-
-        // Array data: ["hello", "", "parquet"]
-        let string_array = LargeStringArray::from(values);
-
-        assert_eq!(3, string_array.len());
-        assert_eq!(0, string_array.null_count());
-        assert_eq!("hello", string_array.value(0));
-        assert_eq!("", string_array.value(1));
-        assert_eq!("parquet", string_array.value(2));
-        assert_eq!(5, string_array.value_offset(2));
-        assert_eq!(7, string_array.value_length(2));
-        for i in 0..3 {
-            assert!(string_array.is_valid(i));
-            assert!(!string_array.is_null(i));
-        }
-    }
-
-    #[test]
-    fn test_nested_string_array() {
-        let string_builder = StringBuilder::new(3);
-        let mut list_of_string_builder = ListBuilder::new(string_builder);
-
-        list_of_string_builder.values().append_value("foo").unwrap();
-        list_of_string_builder.values().append_value("bar").unwrap();
-        list_of_string_builder.append(true).unwrap();
-
-        list_of_string_builder
-            .values()
-            .append_value("foobar")
-            .unwrap();
-        list_of_string_builder.append(true).unwrap();
-        let list_of_strings = list_of_string_builder.finish();
-
-        assert_eq!(list_of_strings.len(), 2);
-
-        let first_slot = list_of_strings.value(0);
-        let first_list = first_slot.as_any().downcast_ref::<StringArray>().unwrap();
-        assert_eq!(first_list.len(), 2);
-        assert_eq!(first_list.value(0), "foo");
-        assert_eq!(first_list.value(1), "bar");
-
-        let second_slot = list_of_strings.value(1);
-        let second_list = second_slot.as_any().downcast_ref::<StringArray>().unwrap();
-        assert_eq!(second_list.len(), 1);
-        assert_eq!(second_list.value(0), "foobar");
-    }
-
-    #[test]
-    #[should_panic(
-        expected = "BinaryArray can only be created from List<u8> arrays, mismatched \
-                    data types."
-    )]
-    fn test_binary_array_from_incorrect_list_array_type() {
-        let values: [u32; 12] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
-        let values_data = ArrayData::builder(DataType::UInt32)
-            .len(12)
-            .add_buffer(Buffer::from(values[..].to_byte_slice()))
-            .build();
-        let offsets: [i32; 4] = [0, 5, 5, 12];
-
-        let array_data = ArrayData::builder(DataType::Utf8)
-            .len(3)
-            .add_buffer(Buffer::from(offsets.to_byte_slice()))
-            .add_child_data(values_data)
-            .build();
-        let list_array = ListArray::from(array_data);
-        BinaryArray::from(list_array);
-    }
-
-    #[test]
-    #[should_panic(
-        expected = "BinaryArray can only be created from list array of u8 values \
-                    (i.e. List<PrimitiveArray<u8>>)."
-    )]
-    fn test_binary_array_from_incorrect_list_array() {
-        let values: [u32; 12] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
-        let values_data = ArrayData::builder(DataType::UInt32)
-            .len(12)
-            .add_buffer(Buffer::from(values[..].to_byte_slice()))
-            .add_child_data(ArrayData::builder(DataType::Boolean).build())
-            .build();
-        let offsets: [i32; 4] = [0, 5, 5, 12];
-
-        let array_data = ArrayData::builder(DataType::Utf8)
-            .len(3)
-            .add_buffer(Buffer::from(offsets.to_byte_slice()))
-            .add_child_data(values_data)
-            .build();
-        let list_array = ListArray::from(array_data);
-        BinaryArray::from(list_array);
-    }
-
-    #[test]
-    fn test_fixed_size_binary_array() {
-        let values: [u8; 15] = *b"hellotherearrow";
-
-        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
-            .len(3)
-            .add_buffer(Buffer::from(&values[..]))
-            .build();
-        let fixed_size_binary_array = FixedSizeBinaryArray::from(array_data);
-        assert_eq!(3, fixed_size_binary_array.len());
-        assert_eq!(0, fixed_size_binary_array.null_count());
-        assert_eq!(
-            [b'h', b'e', b'l', b'l', b'o'],
-            fixed_size_binary_array.value(0)
-        );
-        assert_eq!(
-            [b't', b'h', b'e', b'r', b'e'],
-            fixed_size_binary_array.value(1)
-        );
-        assert_eq!(
-            [b'a', b'r', b'r', b'o', b'w'],
-            fixed_size_binary_array.value(2)
-        );
-        assert_eq!(5, fixed_size_binary_array.value_length());
-        assert_eq!(10, fixed_size_binary_array.value_offset(2));
-        for i in 0..3 {
-            assert!(fixed_size_binary_array.is_valid(i));
-            assert!(!fixed_size_binary_array.is_null(i));
-        }
-
-        // Test binary array with offset
-        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
-            .len(2)
-            .offset(1)
-            .add_buffer(Buffer::from(&values[..]))
-            .build();
-        let fixed_size_binary_array = FixedSizeBinaryArray::from(array_data);
-        assert_eq!(
-            [b't', b'h', b'e', b'r', b'e'],
-            fixed_size_binary_array.value(0)
-        );
-        assert_eq!(
-            [b'a', b'r', b'r', b'o', b'w'],
-            fixed_size_binary_array.value(1)
-        );
-        assert_eq!(2, fixed_size_binary_array.len());
-        assert_eq!(5, fixed_size_binary_array.value_offset(0));
-        assert_eq!(5, fixed_size_binary_array.value_length());
-        assert_eq!(10, fixed_size_binary_array.value_offset(1));
-    }
-
-    #[test]
-    #[should_panic(
-        expected = "FixedSizeBinaryArray can only be created from list array of u8 values \
-                    (i.e. FixedSizeList<PrimitiveArray<u8>>)."
-    )]
-    fn test_fixed_size_binary_array_from_incorrect_list_array() {
-        let values: [u32; 12] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
-        let values_data = ArrayData::builder(DataType::UInt32)
-            .len(12)
-            .add_buffer(Buffer::from(values[..].to_byte_slice()))
-            .add_child_data(ArrayData::builder(DataType::Boolean).build())
-            .build();
-
-        let array_data =
-            ArrayData::builder(DataType::FixedSizeList(Box::new(DataType::Binary), 4))
-                .len(3)
-                .add_child_data(values_data)
-                .build();
-        let list_array = FixedSizeListArray::from(array_data);
-        FixedSizeBinaryArray::from(list_array);
-    }
-
-    #[test]
-    #[should_panic(expected = "BinaryArray out of bounds access")]
-    fn test_binary_array_get_value_index_out_of_bound() {
-        let values: [u8; 12] =
-            [104, 101, 108, 108, 111, 112, 97, 114, 113, 117, 101, 116];
-        let offsets: [i32; 4] = [0, 5, 5, 12];
-        let array_data = ArrayData::builder(DataType::Binary)
-            .len(3)
-            .add_buffer(Buffer::from(offsets.to_byte_slice()))
-            .add_buffer(Buffer::from(&values[..]))
-            .build();
-        let binary_array = BinaryArray::from(array_data);
-        binary_array.value(4);
-    }
-
-    #[test]
-    #[should_panic(expected = "StringArray out of bounds access")]
-    fn test_string_array_get_value_index_out_of_bound() {
-        let values: [u8; 12] = [
-            b'h', b'e', b'l', b'l', b'o', b'p', b'a', b'r', b'q', b'u', b'e', b't',
-        ];
-        let offsets: [i32; 4] = [0, 5, 5, 12];
-        let array_data = ArrayData::builder(DataType::Utf8)
-            .len(3)
-            .add_buffer(Buffer::from(offsets.to_byte_slice()))
-            .add_buffer(Buffer::from(&values[..]))
-            .build();
-        let string_array = StringArray::from(array_data);
-        string_array.value(4);
-    }
-
-    #[test]
-    fn test_binary_array_fmt_debug() {
-        let values: [u8; 15] = *b"hellotherearrow";
-
-        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
-            .len(3)
-            .add_buffer(Buffer::from(&values[..]))
-            .build();
-        let arr = FixedSizeBinaryArray::from(array_data);
-        assert_eq!(
-            "FixedSizeBinaryArray<5>\n[\n  [104, 101, 108, 108, 111],\n  [116, 104, 101, 114, 101],\n  [97, 114, 114, 111, 119],\n]",
-            format!("{:?}", arr)
-        );
-    }
-
-    #[test]
-    fn test_string_array_fmt_debug() {
-        let arr: StringArray = vec!["hello", "arrow"].into();
-        assert_eq!(
-            "StringArray\n[\n  \"hello\",\n  \"arrow\",\n]",
-            format!("{:?}", arr)
-        );
-    }
-
-    #[test]
-    fn test_large_string_array_fmt_debug() {
-        let arr: LargeStringArray = vec!["hello", "arrow"].into();
-        assert_eq!(
-            "LargeStringArray\n[\n  \"hello\",\n  \"arrow\",\n]",
-            format!("{:?}", arr)
-        );
-    }
-
-    #[test]
-    fn test_struct_array_builder() {
-        let boolean_data = ArrayData::builder(DataType::Boolean)
-            .len(4)
-            .add_buffer(Buffer::from([false, false, true, true].to_byte_slice()))
-            .build();
-        let int_data = ArrayData::builder(DataType::Int64)
-            .len(4)
-            .add_buffer(Buffer::from([42, 28, 19, 31].to_byte_slice()))
-            .build();
-        let mut field_types = vec![];
-        field_types.push(Field::new("a", DataType::Boolean, false));
-        field_types.push(Field::new("b", DataType::Int64, false));
-        let struct_array_data = ArrayData::builder(DataType::Struct(field_types))
-            .len(4)
-            .add_child_data(boolean_data.clone())
-            .add_child_data(int_data.clone())
-            .build();
-        let struct_array = StructArray::from(struct_array_data);
-
-        assert_eq!(boolean_data, struct_array.column(0).data());
-        assert_eq!(int_data, struct_array.column(1).data());
-    }
-
-    #[test]
-    fn test_struct_array_from() {
-        let boolean_data = ArrayData::builder(DataType::Boolean)
-            .len(4)
-            .add_buffer(Buffer::from([12_u8]))
-            .build();
-        let int_data = ArrayData::builder(DataType::Int32)
-            .len(4)
-            .add_buffer(Buffer::from([42, 28, 19, 31].to_byte_slice()))
-            .build();
-        let struct_array = StructArray::from(vec![
-            (
-                Field::new("b", DataType::Boolean, false),
-                Arc::new(BooleanArray::from(vec![false, false, true, true]))
-                    as Arc<Array>,
-            ),
-            (
-                Field::new("c", DataType::Int32, false),
-                Arc::new(Int32Array::from(vec![42, 28, 19, 31])),
-            ),
-        ]);
-        assert_eq!(boolean_data, struct_array.column(0).data());
-        assert_eq!(int_data, struct_array.column(1).data());
-        assert_eq!(4, struct_array.len());
-        assert_eq!(0, struct_array.null_count());
-        assert_eq!(0, struct_array.offset());
-    }
-
-    #[test]
-    #[should_panic(
-        expected = "the field data types must match the array data in a StructArray"
-    )]
-    fn test_struct_array_from_mismatched_types() {
-        StructArray::from(vec![
-            (
-                Field::new("b", DataType::Int16, false),
-                Arc::new(BooleanArray::from(vec![false, false, true, true]))
-                    as Arc<Array>,
-            ),
-            (
-                Field::new("c", DataType::Utf8, false),
-                Arc::new(Int32Array::from(vec![42, 28, 19, 31])),
-            ),
-        ]);
-    }
-
-    #[test]
-    fn test_struct_array_slice() {
-        let boolean_data = ArrayData::builder(DataType::Boolean)
-            .len(5)
-            .add_buffer(Buffer::from([0b00010000]))
-            .null_bit_buffer(Buffer::from([0b00010001]))
-            .build();
-        let int_data = ArrayData::builder(DataType::Int32)
-            .len(5)
-            .add_buffer(Buffer::from([0, 28, 42, 0, 0].to_byte_slice()))
-            .null_bit_buffer(Buffer::from([0b00000110]))
-            .build();
-
-        let mut field_types = vec![];
-        field_types.push(Field::new("a", DataType::Boolean, false));
-        field_types.push(Field::new("b", DataType::Int32, false));
-        let struct_array_data = ArrayData::builder(DataType::Struct(field_types))
-            .len(5)
-            .add_child_data(boolean_data.clone())
-            .add_child_data(int_data.clone())
-            .null_bit_buffer(Buffer::from([0b00010111]))
-            .build();
-        let struct_array = StructArray::from(struct_array_data);
-
-        assert_eq!(5, struct_array.len());
-        assert_eq!(1, struct_array.null_count());
-        assert!(struct_array.is_valid(0));
-        assert!(struct_array.is_valid(1));
-        assert!(struct_array.is_valid(2));
-        assert!(struct_array.is_null(3));
-        assert!(struct_array.is_valid(4));
-        assert_eq!(boolean_data, struct_array.column(0).data());
-        assert_eq!(int_data, struct_array.column(1).data());
-
-        let c0 = struct_array.column(0);
-        let c0 = c0.as_any().downcast_ref::<BooleanArray>().unwrap();
-        assert_eq!(5, c0.len());
-        assert_eq!(3, c0.null_count());
-        assert!(c0.is_valid(0));
-        assert_eq!(false, c0.value(0));
-        assert!(c0.is_null(1));
-        assert!(c0.is_null(2));
-        assert!(c0.is_null(3));
-        assert!(c0.is_valid(4));
-        assert_eq!(true, c0.value(4));
-
-        let c1 = struct_array.column(1);
-        let c1 = c1.as_any().downcast_ref::<Int32Array>().unwrap();
-        assert_eq!(5, c1.len());
-        assert_eq!(3, c1.null_count());
-        assert!(c1.is_null(0));
-        assert!(c1.is_valid(1));
-        assert_eq!(28, c1.value(1));
-        assert!(c1.is_valid(2));
-        assert_eq!(42, c1.value(2));
-        assert!(c1.is_null(3));
-        assert!(c1.is_null(4));
-
-        let sliced_array = struct_array.slice(2, 3);
-        let sliced_array = sliced_array.as_any().downcast_ref::<StructArray>().unwrap();
-        assert_eq!(3, sliced_array.len());
-        assert_eq!(2, sliced_array.offset());
-        assert_eq!(1, sliced_array.null_count());
-        assert!(sliced_array.is_valid(0));
-        assert!(sliced_array.is_null(1));
-        assert!(sliced_array.is_valid(2));
-
-        let sliced_c0 = sliced_array.column(0);
-        let sliced_c0 = sliced_c0.as_any().downcast_ref::<BooleanArray>().unwrap();
-        assert_eq!(3, sliced_c0.len());
-        assert_eq!(2, sliced_c0.offset());
-        assert!(sliced_c0.is_null(0));
-        assert!(sliced_c0.is_null(1));
-        assert!(sliced_c0.is_valid(2));
-        assert_eq!(true, sliced_c0.value(2));
-
-        let sliced_c1 = sliced_array.column(1);
-        let sliced_c1 = sliced_c1.as_any().downcast_ref::<Int32Array>().unwrap();
-        assert_eq!(3, sliced_c1.len());
-        assert_eq!(2, sliced_c1.offset());
-        assert!(sliced_c1.is_valid(0));
-        assert_eq!(42, sliced_c1.value(0));
-        assert!(sliced_c1.is_null(1));
-        assert!(sliced_c1.is_null(2));
-    }
-
-    #[test]
-    #[should_panic(
-        expected = "all child arrays of a StructArray must have the same length"
-    )]
-    fn test_invalid_struct_child_array_lengths() {
-        StructArray::from(vec![
-            (
-                Field::new("b", DataType::Float32, false),
-                Arc::new(Float32Array::from(vec![1.1])) as Arc<Array>,
-            ),
-            (
-                Field::new("c", DataType::Float64, false),
-                Arc::new(Float64Array::from(vec![2.2, 3.3])),
-            ),
-        ]);
-    }
-
-    #[test]
-    #[should_panic(expected = "memory is not aligned")]
-    fn test_primitive_array_alignment() {
-        let ptr = memory::allocate_aligned(8);
-        let buf = unsafe { Buffer::from_raw_parts(ptr, 8, 8) };
-        let buf2 = buf.slice(1);
-        let array_data = ArrayData::builder(DataType::Int32).add_buffer(buf2).build();
-        Int32Array::from(array_data);
-    }
-
-    #[test]
-    #[should_panic(expected = "memory is not aligned")]
-    fn test_list_array_alignment() {
-        let ptr = memory::allocate_aligned(8);
-        let buf = unsafe { Buffer::from_raw_parts(ptr, 8, 8) };
-        let buf2 = buf.slice(1);
-
-        let values: [i32; 8] = [0; 8];
-        let value_data = ArrayData::builder(DataType::Int32)
-            .add_buffer(Buffer::from(values.to_byte_slice()))
-            .build();
-
-        let list_data_type = DataType::List(Box::new(DataType::Int32));
-        let list_data = ArrayData::builder(list_data_type.clone())
-            .add_buffer(buf2)
-            .add_child_data(value_data.clone())
-            .build();
-        ListArray::from(list_data);
-    }
-
-    #[test]
-    #[should_panic(expected = "memory is not aligned")]
-    fn test_binary_array_alignment() {
-        let ptr = memory::allocate_aligned(8);
-        let buf = unsafe { Buffer::from_raw_parts(ptr, 8, 8) };
-        let buf2 = buf.slice(1);
-
-        let values: [u8; 12] = [0; 12];
-
-        let array_data = ArrayData::builder(DataType::Binary)
-            .add_buffer(buf2)
-            .add_buffer(Buffer::from(&values[..]))
-            .build();
-        BinaryArray::from(array_data);
-    }
-
-    #[test]
-    fn test_access_array_concurrently() {
-        let a = Int32Array::from(vec![5, 6, 7, 8, 9]);
-        let ret = thread::spawn(move || a.value(3)).join();
-
-        assert!(ret.is_ok());
-        assert_eq!(8, ret.ok().unwrap());
-    }
-
-    #[test]
-    fn test_dictionary_array_fmt_debug() {
-        let key_builder = PrimitiveBuilder::<UInt8Type>::new(3);
-        let value_builder = PrimitiveBuilder::<UInt32Type>::new(2);
-        let mut builder = PrimitiveDictionaryBuilder::new(key_builder, value_builder);
-        builder.append(12345678).unwrap();
-        builder.append_null().unwrap();
-        builder.append(22345678).unwrap();
-        let array = builder.finish();
-        assert_eq!(
-            "DictionaryArray {keys: [Some(0), None, Some(1)] values: PrimitiveArray<UInt32>\n[\n  12345678,\n  22345678,\n]}\n",
-            format!("{:?}", array)
-        );
-
-        let key_builder = PrimitiveBuilder::<UInt8Type>::new(20);
-        let value_builder = PrimitiveBuilder::<UInt32Type>::new(2);
-        let mut builder = PrimitiveDictionaryBuilder::new(key_builder, value_builder);
-        for _ in 0..20 {
-            builder.append(1).unwrap();
-        }
-        let array = builder.finish();
-        assert_eq!(
-            "DictionaryArray {keys: [Some(0), Some(0), Some(0), Some(0), Some(0), Some(0), Some(0), Some(0), Some(0), Some(0)]... values: PrimitiveArray<UInt32>\n[\n  1,\n]}\n",
-            format!("{:?}", array)
-        );
-    }
-
-    #[test]
-    fn test_dictionary_array_from_iter() {
-        let test = vec!["a", "a", "b", "c"];
-        let array: DictionaryArray<Int8Type> = test
-            .iter()
-            .map(|&x| if x == "b" { None } else { Some(x) })
-            .collect();
-        assert_eq!(
-            "DictionaryArray {keys: [Some(0), Some(0), None, Some(1)] values: StringArray\n[\n  \"a\",\n  \"c\",\n]}\n",
-            format!("{:?}", array)
-        );
-
-        let array: DictionaryArray<Int8Type> = test.into_iter().collect();
-        assert_eq!(
-            "DictionaryArray {keys: [Some(0), Some(0), Some(1), Some(2)] values: StringArray\n[\n  \"a\",\n  \"b\",\n  \"c\",\n]}\n",
-            format!("{:?}", array)
-        );
-    }
-
-    #[test]
-    fn test_dictionary_array_reverse_lookup_key() {
-        let test = vec!["a", "a", "b", "c"];
-        let array: DictionaryArray<Int8Type> = test.into_iter().collect();
-
-        assert_eq!(array.lookup_key("c"), Some(2));
-
-        // Direction of building a dictionary is the iterator direction
-        let test = vec!["t3", "t3", "t2", "t2", "t1", "t3", "t4", "t1", "t0"];
-        let array: DictionaryArray<Int8Type> = test.into_iter().collect();
-
-        assert_eq!(array.lookup_key("t1"), Some(2));
-        assert_eq!(array.lookup_key("non-existent"), None);
-    }
-}
diff --git a/rust/arrow/src/array/array_binary.rs b/rust/arrow/src/array/array_binary.rs
new file mode 100644
index 00000000000..d7a3eb7217a
--- /dev/null
+++ b/rust/arrow/src/array/array_binary.rs
@@ -0,0 +1,797 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::convert::From;
+use std::fmt;
+use std::mem;
+use std::{any::Any, iter::FromIterator};
+
+use super::{
+    array::print_long_array, raw_pointer::as_aligned_pointer, raw_pointer::RawPtrBox,
+    Array, ArrayData, ArrayDataRef, FixedSizeListArray, GenericBinaryIter,
+    GenericListArray, LargeListArray, ListArray, OffsetSizeTrait,
+};
+use crate::util::bit_util;
+use crate::{buffer::Buffer, datatypes::ToByteSlice};
+use crate::{buffer::MutableBuffer, datatypes::DataType};
+
+/// Like OffsetSizeTrait, but specialized for Binary
+// This allow us to expose a constant datatype for the GenericBinaryArray
+pub trait BinaryOffsetSizeTrait: OffsetSizeTrait {
+    const DATA_TYPE: DataType;
+}
+
+impl BinaryOffsetSizeTrait for i32 {
+    const DATA_TYPE: DataType = DataType::Binary;
+}
+
+impl BinaryOffsetSizeTrait for i64 {
+    const DATA_TYPE: DataType = DataType::LargeBinary;
+}
+
+pub struct GenericBinaryArray<OffsetSize: BinaryOffsetSizeTrait> {
+    data: ArrayDataRef,
+    value_offsets: RawPtrBox<OffsetSize>,
+    value_data: RawPtrBox<u8>,
+}
+
+impl<OffsetSize: BinaryOffsetSizeTrait> GenericBinaryArray<OffsetSize> {
+    /// Returns the offset for the element at index `i`.
+    ///
+    /// Note this doesn't do any bound checking, for performance reason.
+    #[inline]
+    pub fn value_offset(&self, i: usize) -> OffsetSize {
+        self.value_offset_at(self.data.offset() + i)
+    }
+
+    /// Returns the length for the element at index `i`.
+    ///
+    /// Note this doesn't do any bound checking, for performance reason.
+    #[inline]
+    pub fn value_length(&self, mut i: usize) -> OffsetSize {
+        i += self.data.offset();
+        self.value_offset_at(i + 1) - self.value_offset_at(i)
+    }
+
+    /// Returns a clone of the value offset buffer
+    pub fn value_offsets(&self) -> Buffer {
+        self.data.buffers()[0].clone()
+    }
+
+    /// Returns a clone of the value data buffer
+    pub fn value_data(&self) -> Buffer {
+        self.data.buffers()[1].clone()
+    }
+
+    #[inline]
+    fn value_offset_at(&self, i: usize) -> OffsetSize {
+        unsafe { *self.value_offsets.get().add(i) }
+    }
+
+    /// Returns the element at index `i` as a byte slice.
+    pub fn value(&self, i: usize) -> &[u8] {
+        assert!(i < self.data.len(), "BinaryArray out of bounds access");
+        let offset = i.checked_add(self.data.offset()).unwrap();
+        unsafe {
+            let pos = self.value_offset_at(offset);
+            std::slice::from_raw_parts(
+                self.value_data.get().offset(pos.to_isize()),
+                (self.value_offset_at(offset + 1) - pos).to_usize().unwrap(),
+            )
+        }
+    }
+
+    /// Creates a [GenericBinaryArray] from a vector of byte slices
+    pub fn from_vec(v: Vec<&[u8]>) -> Self {
+        let mut offsets = Vec::with_capacity(v.len() + 1);
+        let mut values = Vec::new();
+        let mut length_so_far: OffsetSize = OffsetSize::zero();
+        offsets.push(length_so_far);
+        for s in &v {
+            length_so_far = length_so_far + OffsetSize::from_usize(s.len()).unwrap();
+            offsets.push(length_so_far);
+            values.extend_from_slice(s);
+        }
+        let array_data = ArrayData::builder(OffsetSize::DATA_TYPE)
+            .len(v.len())
+            .add_buffer(Buffer::from(offsets.to_byte_slice()))
+            .add_buffer(Buffer::from(&values[..]))
+            .build();
+        GenericBinaryArray::<OffsetSize>::from(array_data)
+    }
+
+    /// Creates a [GenericBinaryArray] from a vector of Optional (null) byte slices
+    pub fn from_opt_vec(v: Vec<Option<&[u8]>>) -> Self {
+        v.into_iter().collect()
+    }
+
+    fn from_list(v: GenericListArray<OffsetSize>) -> Self {
+        assert_eq!(
+            v.data_ref().child_data()[0].child_data().len(),
+            0,
+            "BinaryArray can only be created from list array of u8 values \
+             (i.e. List<PrimitiveArray<u8>>)."
+        );
+        assert_eq!(
+            v.data_ref().child_data()[0].data_type(),
+            &DataType::UInt8,
+            "BinaryArray can only be created from List<u8> arrays, mismatched data types."
+        );
+
+        let mut builder = ArrayData::builder(OffsetSize::DATA_TYPE)
+            .len(v.len())
+            .add_buffer(v.data_ref().buffers()[0].clone())
+            .add_buffer(v.data_ref().child_data()[0].buffers()[0].clone());
+        if let Some(bitmap) = v.data_ref().null_bitmap() {
+            builder = builder
+                .null_count(v.data_ref().null_count())
+                .null_bit_buffer(bitmap.bits.clone())
+        }
+
+        let data = builder.build();
+        Self::from(data)
+    }
+}
+
+impl<'a, T: BinaryOffsetSizeTrait> GenericBinaryArray<T> {
+    /// constructs a new iterator
+    pub fn iter(&'a self) -> GenericBinaryIter<'a, T> {
+        GenericBinaryIter::<'a, T>::new(&self)
+    }
+}
+
+impl<OffsetSize: BinaryOffsetSizeTrait> fmt::Debug for GenericBinaryArray<OffsetSize> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}BinaryArray\n[\n", OffsetSize::prefix())?;
+        print_long_array(self, f, |array, index, f| {
+            fmt::Debug::fmt(&array.value(index), f)
+        })?;
+        write!(f, "]")
+    }
+}
+
+impl<OffsetSize: BinaryOffsetSizeTrait> Array for GenericBinaryArray<OffsetSize> {
+    fn as_any(&self) -> &Any {
+        self
+    }
+
+    fn data(&self) -> ArrayDataRef {
+        self.data.clone()
+    }
+
+    fn data_ref(&self) -> &ArrayDataRef {
+        &self.data
+    }
+
+    /// Returns the total number of bytes of memory occupied by the buffers owned by this [$name].
+    fn get_buffer_memory_size(&self) -> usize {
+        self.data.get_buffer_memory_size()
+    }
+
+    /// Returns the total number of bytes of memory occupied physically by this [$name].
+    fn get_array_memory_size(&self) -> usize {
+        self.data.get_array_memory_size() + mem::size_of_val(self)
+    }
+}
+
+impl<OffsetSize: BinaryOffsetSizeTrait> From<ArrayDataRef>
+    for GenericBinaryArray<OffsetSize>
+{
+    fn from(data: ArrayDataRef) -> Self {
+        assert_eq!(
+            data.data_type(),
+            &<OffsetSize as BinaryOffsetSizeTrait>::DATA_TYPE,
+            "[Large]BinaryArray expects Datatype::[Large]Binary"
+        );
+        assert_eq!(
+            data.buffers().len(),
+            2,
+            "BinaryArray data should contain 2 buffers only (offsets and values)"
+        );
+        let raw_value_offsets = data.buffers()[0].raw_data();
+        let value_data = data.buffers()[1].raw_data();
+        Self {
+            data,
+            value_offsets: RawPtrBox::new(as_aligned_pointer::<OffsetSize>(
+                raw_value_offsets,
+            )),
+            value_data: RawPtrBox::new(value_data),
+        }
+    }
+}
+
+impl<Ptr, OffsetSize: BinaryOffsetSizeTrait> FromIterator<Option<Ptr>>
+    for GenericBinaryArray<OffsetSize>
+where
+    Ptr: AsRef<[u8]>,
+{
+    fn from_iter<I: IntoIterator<Item = Option<Ptr>>>(iter: I) -> Self {
+        let iter = iter.into_iter();
+        let (_, data_len) = iter.size_hint();
+        let data_len = data_len.expect("Iterator must be sized"); // panic if no upper bound.
+
+        let mut offsets = Vec::with_capacity(data_len + 1);
+        let mut values = Vec::new();
+        let mut null_buf = MutableBuffer::new_null(data_len);
+        let mut length_so_far: OffsetSize = OffsetSize::zero();
+        offsets.push(length_so_far);
+
+        {
+            let null_slice = null_buf.data_mut();
+
+            for (i, s) in iter.enumerate() {
+                if let Some(s) = s {
+                    let s = s.as_ref();
+                    bit_util::set_bit(null_slice, i);
+                    length_so_far =
+                        length_so_far + OffsetSize::from_usize(s.len()).unwrap();
+                    values.extend_from_slice(s);
+                }
+                // always add an element in offsets
+                offsets.push(length_so_far);
+            }
+        }
+
+        let array_data = ArrayData::builder(OffsetSize::DATA_TYPE)
+            .len(data_len)
+            .add_buffer(Buffer::from(offsets.to_byte_slice()))
+            .add_buffer(Buffer::from(&values[..]))
+            .null_bit_buffer(null_buf.freeze())
+            .build();
+        Self::from(array_data)
+    }
+}
+
+/// An array where each element is a byte whose maximum length is represented by a i32.
+pub type BinaryArray = GenericBinaryArray<i32>;
+
+/// An array where each element is a byte whose maximum length is represented by a i64.
+pub type LargeBinaryArray = GenericBinaryArray<i64>;
+
+impl<'a, T: BinaryOffsetSizeTrait> IntoIterator for &'a GenericBinaryArray<T> {
+    type Item = Option<&'a [u8]>;
+    type IntoIter = GenericBinaryIter<'a, T>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        GenericBinaryIter::<'a, T>::new(self)
+    }
+}
+
+impl From<Vec<&[u8]>> for BinaryArray {
+    fn from(v: Vec<&[u8]>) -> Self {
+        BinaryArray::from_vec(v)
+    }
+}
+
+impl From<Vec<Option<&[u8]>>> for BinaryArray {
+    fn from(v: Vec<Option<&[u8]>>) -> Self {
+        BinaryArray::from_opt_vec(v)
+    }
+}
+
+impl From<Vec<&[u8]>> for LargeBinaryArray {
+    fn from(v: Vec<&[u8]>) -> Self {
+        LargeBinaryArray::from_vec(v)
+    }
+}
+
+impl From<Vec<Option<&[u8]>>> for LargeBinaryArray {
+    fn from(v: Vec<Option<&[u8]>>) -> Self {
+        LargeBinaryArray::from_opt_vec(v)
+    }
+}
+
+impl From<ListArray> for BinaryArray {
+    fn from(v: ListArray) -> Self {
+        BinaryArray::from_list(v)
+    }
+}
+
+impl From<LargeListArray> for LargeBinaryArray {
+    fn from(v: LargeListArray) -> Self {
+        LargeBinaryArray::from_list(v)
+    }
+}
+
+/// A type of `FixedSizeListArray` whose elements are binaries.
+pub struct FixedSizeBinaryArray {
+    data: ArrayDataRef,
+    value_data: RawPtrBox<u8>,
+    length: i32,
+}
+
+impl FixedSizeBinaryArray {
+    /// Returns the element at index `i` as a byte slice.
+    pub fn value(&self, i: usize) -> &[u8] {
+        assert!(
+            i < self.data.len(),
+            "FixedSizeBinaryArray out of bounds access"
+        );
+        let offset = i.checked_add(self.data.offset()).unwrap();
+        unsafe {
+            let pos = self.value_offset_at(offset);
+            std::slice::from_raw_parts(
+                self.value_data.get().offset(pos as isize),
+                (self.value_offset_at(offset + 1) - pos) as usize,
+            )
+        }
+    }
+
+    /// Returns the offset for the element at index `i`.
+    ///
+    /// Note this doesn't do any bound checking, for performance reason.
+    #[inline]
+    pub fn value_offset(&self, i: usize) -> i32 {
+        self.value_offset_at(self.data.offset() + i)
+    }
+
+    /// Returns the length for an element.
+    ///
+    /// All elements have the same length as the array is a fixed size.
+    #[inline]
+    pub fn value_length(&self) -> i32 {
+        self.length
+    }
+
+    /// Returns a clone of the value data buffer
+    pub fn value_data(&self) -> Buffer {
+        self.data.buffers()[0].clone()
+    }
+
+    #[inline]
+    fn value_offset_at(&self, i: usize) -> i32 {
+        self.length * i as i32
+    }
+}
+
+impl From<ArrayDataRef> for FixedSizeBinaryArray {
+    fn from(data: ArrayDataRef) -> Self {
+        assert_eq!(
+            data.buffers().len(),
+            1,
+            "FixedSizeBinaryArray data should contain 1 buffer only (values)"
+        );
+        let value_data = data.buffers()[0].raw_data();
+        let length = match data.data_type() {
+            DataType::FixedSizeBinary(len) => *len,
+            _ => panic!("Expected data type to be FixedSizeBinary"),
+        };
+        Self {
+            data,
+            value_data: RawPtrBox::new(value_data),
+            length,
+        }
+    }
+}
+
+/// Creates a `FixedSizeBinaryArray` from `FixedSizeList<u8>` array
+impl From<FixedSizeListArray> for FixedSizeBinaryArray {
+    fn from(v: FixedSizeListArray) -> Self {
+        assert_eq!(
+            v.data_ref().child_data()[0].child_data().len(),
+            0,
+            "FixedSizeBinaryArray can only be created from list array of u8 values \
+             (i.e. FixedSizeList<PrimitiveArray<u8>>)."
+        );
+        assert_eq!(
+            v.data_ref().child_data()[0].data_type(),
+            &DataType::UInt8,
+            "FixedSizeBinaryArray can only be created from FixedSizeList<u8> arrays, mismatched data types."
+        );
+
+        let mut builder = ArrayData::builder(DataType::FixedSizeBinary(v.value_length()))
+            .len(v.len())
+            .add_buffer(v.data_ref().child_data()[0].buffers()[0].clone());
+        if let Some(bitmap) = v.data_ref().null_bitmap() {
+            builder = builder
+                .null_count(v.data_ref().null_count())
+                .null_bit_buffer(bitmap.bits.clone())
+        }
+
+        let data = builder.build();
+        Self::from(data)
+    }
+}
+
+impl fmt::Debug for FixedSizeBinaryArray {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "FixedSizeBinaryArray<{}>\n[\n", self.value_length())?;
+        print_long_array(self, f, |array, index, f| {
+            fmt::Debug::fmt(&array.value(index), f)
+        })?;
+        write!(f, "]")
+    }
+}
+
+impl Array for FixedSizeBinaryArray {
+    fn as_any(&self) -> &Any {
+        self
+    }
+
+    fn data(&self) -> ArrayDataRef {
+        self.data.clone()
+    }
+
+    fn data_ref(&self) -> &ArrayDataRef {
+        &self.data
+    }
+
+    /// Returns the total number of bytes of memory occupied by the buffers owned by this [FixedSizeBinaryArray].
+    fn get_buffer_memory_size(&self) -> usize {
+        self.data.get_buffer_memory_size()
+    }
+
+    /// Returns the total number of bytes of memory occupied physically by this [FixedSizeBinaryArray].
+    fn get_array_memory_size(&self) -> usize {
+        self.data.get_array_memory_size() + mem::size_of_val(self)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::datatypes::Field;
+
+    use super::*;
+
+    #[test]
+    fn test_binary_array() {
+        let values: [u8; 12] = [
+            b'h', b'e', b'l', b'l', b'o', b'p', b'a', b'r', b'q', b'u', b'e', b't',
+        ];
+        let offsets: [i32; 4] = [0, 5, 5, 12];
+
+        // Array data: ["hello", "", "parquet"]
+        let array_data = ArrayData::builder(DataType::Binary)
+            .len(3)
+            .add_buffer(Buffer::from(offsets.to_byte_slice()))
+            .add_buffer(Buffer::from(&values[..]))
+            .build();
+        let binary_array = BinaryArray::from(array_data);
+        assert_eq!(3, binary_array.len());
+        assert_eq!(0, binary_array.null_count());
+        assert_eq!([b'h', b'e', b'l', b'l', b'o'], binary_array.value(0));
+        assert_eq!([] as [u8; 0], binary_array.value(1));
+        assert_eq!(
+            [b'p', b'a', b'r', b'q', b'u', b'e', b't'],
+            binary_array.value(2)
+        );
+        assert_eq!(5, binary_array.value_offset(2));
+        assert_eq!(7, binary_array.value_length(2));
+        for i in 0..3 {
+            assert!(binary_array.is_valid(i));
+            assert!(!binary_array.is_null(i));
+        }
+
+        // Test binary array with offset
+        let array_data = ArrayData::builder(DataType::Binary)
+            .len(4)
+            .offset(1)
+            .add_buffer(Buffer::from(offsets.to_byte_slice()))
+            .add_buffer(Buffer::from(&values[..]))
+            .build();
+        let binary_array = BinaryArray::from(array_data);
+        assert_eq!(
+            [b'p', b'a', b'r', b'q', b'u', b'e', b't'],
+            binary_array.value(1)
+        );
+        assert_eq!(5, binary_array.value_offset(0));
+        assert_eq!(0, binary_array.value_length(0));
+        assert_eq!(5, binary_array.value_offset(1));
+        assert_eq!(7, binary_array.value_length(1));
+    }
+
+    #[test]
+    fn test_large_binary_array() {
+        let values: [u8; 12] = [
+            b'h', b'e', b'l', b'l', b'o', b'p', b'a', b'r', b'q', b'u', b'e', b't',
+        ];
+        let offsets: [i64; 4] = [0, 5, 5, 12];
+
+        // Array data: ["hello", "", "parquet"]
+        let array_data = ArrayData::builder(DataType::LargeBinary)
+            .len(3)
+            .add_buffer(Buffer::from(offsets.to_byte_slice()))
+            .add_buffer(Buffer::from(&values[..]))
+            .build();
+        let binary_array = LargeBinaryArray::from(array_data);
+        assert_eq!(3, binary_array.len());
+        assert_eq!(0, binary_array.null_count());
+        assert_eq!([b'h', b'e', b'l', b'l', b'o'], binary_array.value(0));
+        assert_eq!([] as [u8; 0], binary_array.value(1));
+        assert_eq!(
+            [b'p', b'a', b'r', b'q', b'u', b'e', b't'],
+            binary_array.value(2)
+        );
+        assert_eq!(5, binary_array.value_offset(2));
+        assert_eq!(7, binary_array.value_length(2));
+        for i in 0..3 {
+            assert!(binary_array.is_valid(i));
+            assert!(!binary_array.is_null(i));
+        }
+
+        // Test binary array with offset
+        let array_data = ArrayData::builder(DataType::LargeBinary)
+            .len(4)
+            .offset(1)
+            .add_buffer(Buffer::from(offsets.to_byte_slice()))
+            .add_buffer(Buffer::from(&values[..]))
+            .build();
+        let binary_array = LargeBinaryArray::from(array_data);
+        assert_eq!(
+            [b'p', b'a', b'r', b'q', b'u', b'e', b't'],
+            binary_array.value(1)
+        );
+        assert_eq!(5, binary_array.value_offset(0));
+        assert_eq!(0, binary_array.value_length(0));
+        assert_eq!(5, binary_array.value_offset(1));
+        assert_eq!(7, binary_array.value_length(1));
+    }
+
+    #[test]
+    fn test_binary_array_from_list_array() {
+        let values: [u8; 12] = [
+            b'h', b'e', b'l', b'l', b'o', b'p', b'a', b'r', b'q', b'u', b'e', b't',
+        ];
+        let values_data = ArrayData::builder(DataType::UInt8)
+            .len(12)
+            .add_buffer(Buffer::from(&values[..]))
+            .build();
+        let offsets: [i32; 4] = [0, 5, 5, 12];
+
+        // Array data: ["hello", "", "parquet"]
+        let array_data1 = ArrayData::builder(DataType::Binary)
+            .len(3)
+            .add_buffer(Buffer::from(offsets.to_byte_slice()))
+            .add_buffer(Buffer::from(&values[..]))
+            .build();
+        let binary_array1 = BinaryArray::from(array_data1);
+
+        let array_data2 = ArrayData::builder(DataType::Binary)
+            .len(3)
+            .add_buffer(Buffer::from(offsets.to_byte_slice()))
+            .add_child_data(values_data)
+            .build();
+        let list_array = ListArray::from(array_data2);
+        let binary_array2 = BinaryArray::from(list_array);
+
+        assert_eq!(2, binary_array2.data().buffers().len());
+        assert_eq!(0, binary_array2.data().child_data().len());
+
+        assert_eq!(binary_array1.len(), binary_array2.len());
+        assert_eq!(binary_array1.null_count(), binary_array2.null_count());
+        for i in 0..binary_array1.len() {
+            assert_eq!(binary_array1.value(i), binary_array2.value(i));
+            assert_eq!(binary_array1.value_offset(i), binary_array2.value_offset(i));
+            assert_eq!(binary_array1.value_length(i), binary_array2.value_length(i));
+        }
+    }
+
+    #[test]
+    fn test_large_binary_array_from_list_array() {
+        let values: [u8; 12] = [
+            b'h', b'e', b'l', b'l', b'o', b'p', b'a', b'r', b'q', b'u', b'e', b't',
+        ];
+        let values_data = ArrayData::builder(DataType::UInt8)
+            .len(12)
+            .add_buffer(Buffer::from(&values[..]))
+            .build();
+        let offsets: [i64; 4] = [0, 5, 5, 12];
+
+        // Array data: ["hello", "", "parquet"]
+        let array_data1 = ArrayData::builder(DataType::LargeBinary)
+            .len(3)
+            .add_buffer(Buffer::from(offsets.to_byte_slice()))
+            .add_buffer(Buffer::from(&values[..]))
+            .build();
+        let binary_array1 = LargeBinaryArray::from(array_data1);
+
+        let array_data2 = ArrayData::builder(DataType::Binary)
+            .len(3)
+            .add_buffer(Buffer::from(offsets.to_byte_slice()))
+            .add_child_data(values_data)
+            .build();
+        let list_array = LargeListArray::from(array_data2);
+        let binary_array2 = LargeBinaryArray::from(list_array);
+
+        assert_eq!(2, binary_array2.data().buffers().len());
+        assert_eq!(0, binary_array2.data().child_data().len());
+
+        assert_eq!(binary_array1.len(), binary_array2.len());
+        assert_eq!(binary_array1.null_count(), binary_array2.null_count());
+        for i in 0..binary_array1.len() {
+            assert_eq!(binary_array1.value(i), binary_array2.value(i));
+            assert_eq!(binary_array1.value_offset(i), binary_array2.value_offset(i));
+            assert_eq!(binary_array1.value_length(i), binary_array2.value_length(i));
+        }
+    }
+
+    fn test_generic_binary_array_from_opt_vec<T: BinaryOffsetSizeTrait>() {
+        let values: Vec<Option<&[u8]>> =
+            vec![Some(b"one"), Some(b"two"), None, Some(b""), Some(b"three")];
+        let array = GenericBinaryArray::<T>::from_opt_vec(values);
+        assert_eq!(array.len(), 5);
+        assert_eq!(array.value(0), b"one");
+        assert_eq!(array.value(1), b"two");
+        assert_eq!(array.value(3), b"");
+        assert_eq!(array.value(4), b"three");
+        assert_eq!(array.is_null(0), false);
+        assert_eq!(array.is_null(1), false);
+        assert_eq!(array.is_null(2), true);
+        assert_eq!(array.is_null(3), false);
+        assert_eq!(array.is_null(4), false);
+    }
+
+    #[test]
+    fn test_large_binary_array_from_opt_vec() {
+        test_generic_binary_array_from_opt_vec::<i64>()
+    }
+
+    #[test]
+    fn test_binary_array_from_opt_vec() {
+        test_generic_binary_array_from_opt_vec::<i32>()
+    }
+
+    #[test]
+    #[should_panic(
+        expected = "BinaryArray can only be created from List<u8> arrays, mismatched \
+                    data types."
+    )]
+    fn test_binary_array_from_incorrect_list_array_type() {
+        let values: [u32; 12] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
+        let values_data = ArrayData::builder(DataType::UInt32)
+            .len(12)
+            .add_buffer(Buffer::from(values[..].to_byte_slice()))
+            .build();
+        let offsets: [i32; 4] = [0, 5, 5, 12];
+
+        let array_data = ArrayData::builder(DataType::Utf8)
+            .len(3)
+            .add_buffer(Buffer::from(offsets.to_byte_slice()))
+            .add_child_data(values_data)
+            .build();
+        let list_array = ListArray::from(array_data);
+        BinaryArray::from(list_array);
+    }
+
+    #[test]
+    #[should_panic(
+        expected = "BinaryArray can only be created from list array of u8 values \
+                    (i.e. List<PrimitiveArray<u8>>)."
+    )]
+    fn test_binary_array_from_incorrect_list_array() {
+        let values: [u32; 12] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
+        let values_data = ArrayData::builder(DataType::UInt32)
+            .len(12)
+            .add_buffer(Buffer::from(values[..].to_byte_slice()))
+            .add_child_data(ArrayData::builder(DataType::Boolean).build())
+            .build();
+        let offsets: [i32; 4] = [0, 5, 5, 12];
+
+        let array_data = ArrayData::builder(DataType::Utf8)
+            .len(3)
+            .add_buffer(Buffer::from(offsets.to_byte_slice()))
+            .add_child_data(values_data)
+            .build();
+        let list_array = ListArray::from(array_data);
+        BinaryArray::from(list_array);
+    }
+
+    #[test]
+    fn test_fixed_size_binary_array() {
+        let values: [u8; 15] = *b"hellotherearrow";
+
+        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
+            .len(3)
+            .add_buffer(Buffer::from(&values[..]))
+            .build();
+        let fixed_size_binary_array = FixedSizeBinaryArray::from(array_data);
+        assert_eq!(3, fixed_size_binary_array.len());
+        assert_eq!(0, fixed_size_binary_array.null_count());
+        assert_eq!(
+            [b'h', b'e', b'l', b'l', b'o'],
+            fixed_size_binary_array.value(0)
+        );
+        assert_eq!(
+            [b't', b'h', b'e', b'r', b'e'],
+            fixed_size_binary_array.value(1)
+        );
+        assert_eq!(
+            [b'a', b'r', b'r', b'o', b'w'],
+            fixed_size_binary_array.value(2)
+        );
+        assert_eq!(5, fixed_size_binary_array.value_length());
+        assert_eq!(10, fixed_size_binary_array.value_offset(2));
+        for i in 0..3 {
+            assert!(fixed_size_binary_array.is_valid(i));
+            assert!(!fixed_size_binary_array.is_null(i));
+        }
+
+        // Test binary array with offset
+        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
+            .len(2)
+            .offset(1)
+            .add_buffer(Buffer::from(&values[..]))
+            .build();
+        let fixed_size_binary_array = FixedSizeBinaryArray::from(array_data);
+        assert_eq!(
+            [b't', b'h', b'e', b'r', b'e'],
+            fixed_size_binary_array.value(0)
+        );
+        assert_eq!(
+            [b'a', b'r', b'r', b'o', b'w'],
+            fixed_size_binary_array.value(1)
+        );
+        assert_eq!(2, fixed_size_binary_array.len());
+        assert_eq!(5, fixed_size_binary_array.value_offset(0));
+        assert_eq!(5, fixed_size_binary_array.value_length());
+        assert_eq!(10, fixed_size_binary_array.value_offset(1));
+    }
+
+    #[test]
+    #[should_panic(
+        expected = "FixedSizeBinaryArray can only be created from list array of u8 values \
+                    (i.e. FixedSizeList<PrimitiveArray<u8>>)."
+    )]
+    fn test_fixed_size_binary_array_from_incorrect_list_array() {
+        let values: [u32; 12] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
+        let values_data = ArrayData::builder(DataType::UInt32)
+            .len(12)
+            .add_buffer(Buffer::from(values[..].to_byte_slice()))
+            .add_child_data(ArrayData::builder(DataType::Boolean).build())
+            .build();
+
+        let array_data = ArrayData::builder(DataType::FixedSizeList(
+            Box::new(Field::new("item", DataType::Binary, false)),
+            4,
+        ))
+        .len(3)
+        .add_child_data(values_data)
+        .build();
+        let list_array = FixedSizeListArray::from(array_data);
+        FixedSizeBinaryArray::from(list_array);
+    }
+
+    #[test]
+    #[should_panic(expected = "BinaryArray out of bounds access")]
+    fn test_binary_array_get_value_index_out_of_bound() {
+        let values: [u8; 12] =
+            [104, 101, 108, 108, 111, 112, 97, 114, 113, 117, 101, 116];
+        let offsets: [i32; 4] = [0, 5, 5, 12];
+        let array_data = ArrayData::builder(DataType::Binary)
+            .len(3)
+            .add_buffer(Buffer::from(offsets.to_byte_slice()))
+            .add_buffer(Buffer::from(&values[..]))
+            .build();
+        let binary_array = BinaryArray::from(array_data);
+        binary_array.value(4);
+    }
+
+    #[test]
+    fn test_binary_array_fmt_debug() {
+        let values: [u8; 15] = *b"hellotherearrow";
+
+        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
+            .len(3)
+            .add_buffer(Buffer::from(&values[..]))
+            .build();
+        let arr = FixedSizeBinaryArray::from(array_data);
+        assert_eq!(
+            "FixedSizeBinaryArray<5>\n[\n  [104, 101, 108, 108, 111],\n  [116, 104, 101, 114, 101],\n  [97, 114, 114, 111, 119],\n]",
+            format!("{:?}", arr)
+        );
+    }
+}
diff --git a/rust/arrow/src/array/array_dictionary.rs b/rust/arrow/src/array/array_dictionary.rs
new file mode 100644
index 00000000000..3cca55440f4
--- /dev/null
+++ b/rust/arrow/src/array/array_dictionary.rs
@@ -0,0 +1,412 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt;
+use std::iter::IntoIterator;
+use std::mem;
+use std::{any::Any, sync::Arc};
+use std::{convert::From, iter::FromIterator};
+
+use super::{
+    make_array, Array, ArrayData, ArrayDataRef, ArrayRef, PrimitiveArray,
+    PrimitiveBuilder, StringArray, StringBuilder, StringDictionaryBuilder,
+};
+use crate::datatypes::ArrowNativeType;
+use crate::datatypes::{ArrowDictionaryKeyType, ArrowPrimitiveType, DataType};
+
+/// A dictionary array where each element is a single value indexed by an integer key.
+/// This is mostly used to represent strings or a limited set of primitive types as integers,
+/// for example when doing NLP analysis or representing chromosomes by name.
+///
+/// Example **with nullable** data:
+///
+/// ```
+/// use arrow::array::{DictionaryArray, Int8Array};
+/// use arrow::datatypes::Int8Type;
+/// let test = vec!["a", "a", "b", "c"];
+/// let array : DictionaryArray<Int8Type> = test.iter().map(|&x| if x == "b" {None} else {Some(x)}).collect();
+/// assert_eq!(array.keys(), &Int8Array::from(vec![Some(0), Some(0), None, Some(1)]));
+/// ```
+///
+/// Example **without nullable** data:
+///
+/// ```
+/// use arrow::array::{DictionaryArray, Int8Array};
+/// use arrow::datatypes::Int8Type;
+/// let test = vec!["a", "a", "b", "c"];
+/// let array : DictionaryArray<Int8Type> = test.into_iter().collect();
+/// assert_eq!(array.keys(), &Int8Array::from(vec![0, 0, 1, 2]));
+/// ```
+pub struct DictionaryArray<K: ArrowPrimitiveType> {
+    /// Data of this dictionary. Note that this is _not_ compatible with the C Data interface,
+    /// as, in the current implementation, `values` below are the first child of this struct.
+    data: ArrayDataRef,
+
+    /// The keys of this dictionary. These are constructed from the buffer and null bitmap
+    /// of `data`.
+    /// Also, note that these do not correspond to the true values of this array. Rather, they map
+    /// to the real values.
+    keys: PrimitiveArray<K>,
+
+    /// Array of dictionary values (can by any DataType).
+    values: ArrayRef,
+
+    /// Values are ordered.
+    is_ordered: bool,
+}
+
+impl<'a, K: ArrowPrimitiveType> DictionaryArray<K> {
+    /// Return an iterator to the keys of this dictionary.
+    pub fn keys(&self) -> &PrimitiveArray<K> {
+        &self.keys
+    }
+
+    /// Returns an array view of the keys of this dictionary
+    pub fn keys_array(&self) -> PrimitiveArray<K> {
+        let data = self.data_ref();
+        let keys_data = ArrayData::new(
+            K::DATA_TYPE,
+            data.len(),
+            Some(data.null_count()),
+            data.null_buffer().cloned(),
+            data.offset(),
+            data.buffers().to_vec(),
+            vec![],
+        );
+        PrimitiveArray::<K>::from(Arc::new(keys_data))
+    }
+
+    /// Returns the lookup key by doing reverse dictionary lookup
+    pub fn lookup_key(&self, value: &str) -> Option<K::Native> {
+        let rd_buf: &StringArray =
+            self.values.as_any().downcast_ref::<StringArray>().unwrap();
+
+        (0..rd_buf.len())
+            .position(|i| rd_buf.value(i) == value)
+            .map(K::Native::from_usize)
+            .flatten()
+    }
+
+    /// Returns an `ArrayRef` to the dictionary values.
+    pub fn values(&self) -> ArrayRef {
+        self.values.clone()
+    }
+
+    /// Returns a clone of the value type of this list.
+    pub fn value_type(&self) -> DataType {
+        self.values.data_ref().data_type().clone()
+    }
+
+    /// The length of the dictionary is the length of the keys array.
+    pub fn len(&self) -> usize {
+        self.keys.len()
+    }
+
+    /// Whether this dictionary is empty
+    pub fn is_empty(&self) -> bool {
+        self.keys.is_empty()
+    }
+
+    // Currently exists for compatibility purposes with Arrow IPC.
+    pub fn is_ordered(&self) -> bool {
+        self.is_ordered
+    }
+}
+
+/// Constructs a `DictionaryArray` from an array data reference.
+impl<T: ArrowPrimitiveType> From<ArrayDataRef> for DictionaryArray<T> {
+    fn from(data: ArrayDataRef) -> Self {
+        assert_eq!(
+            data.buffers().len(),
+            1,
+            "DictionaryArray data should contain a single buffer only (keys)."
+        );
+        assert_eq!(
+            data.child_data().len(),
+            1,
+            "DictionaryArray should contain a single child array (values)."
+        );
+
+        if let DataType::Dictionary(key_data_type, _) = data.data_type() {
+            if key_data_type.as_ref() != &T::DATA_TYPE {
+                panic!("DictionaryArray's data type must match.")
+            };
+            // create a zero-copy of the keys' data
+            let keys = PrimitiveArray::<T>::from(Arc::new(ArrayData::new(
+                T::DATA_TYPE,
+                data.len(),
+                Some(data.null_count()),
+                data.null_buffer().cloned(),
+                data.offset(),
+                data.buffers().to_vec(),
+                vec![],
+            )));
+            let values = make_array(data.child_data()[0].clone());
+            Self {
+                data,
+                keys,
+                values,
+                is_ordered: false,
+            }
+        } else {
+            panic!("DictionaryArray must have Dictionary data type.")
+        }
+    }
+}
+
+/// Constructs a `DictionaryArray` from an iterator of optional strings.
+impl<'a, T: ArrowPrimitiveType + ArrowDictionaryKeyType> FromIterator<Option<&'a str>>
+    for DictionaryArray<T>
+{
+    fn from_iter<I: IntoIterator<Item = Option<&'a str>>>(iter: I) -> Self {
+        let it = iter.into_iter();
+        let (lower, _) = it.size_hint();
+        let key_builder = PrimitiveBuilder::<T>::new(lower);
+        let value_builder = StringBuilder::new(256);
+        let mut builder = StringDictionaryBuilder::new(key_builder, value_builder);
+        it.for_each(|i| {
+            if let Some(i) = i {
+                // Note: impl ... for Result<DictionaryArray<T>> fails with
+                // error[E0117]: only traits defined in the current crate can be implemented for arbitrary types
+                builder
+                    .append(i)
+                    .expect("Unable to append a value to a dictionary array.");
+            } else {
+                builder
+                    .append_null()
+                    .expect("Unable to append a null value to a dictionary array.");
+            }
+        });
+
+        builder.finish()
+    }
+}
+
+/// Constructs a `DictionaryArray` from an iterator of strings.
+impl<'a, T: ArrowPrimitiveType + ArrowDictionaryKeyType> FromIterator<&'a str>
+    for DictionaryArray<T>
+{
+    fn from_iter<I: IntoIterator<Item = &'a str>>(iter: I) -> Self {
+        let it = iter.into_iter();
+        let (lower, _) = it.size_hint();
+        let key_builder = PrimitiveBuilder::<T>::new(lower);
+        let value_builder = StringBuilder::new(256);
+        let mut builder = StringDictionaryBuilder::new(key_builder, value_builder);
+        it.for_each(|i| {
+            builder
+                .append(i)
+                .expect("Unable to append a value to a dictionary array.");
+        });
+
+        builder.finish()
+    }
+}
+
+impl<T: ArrowPrimitiveType> Array for DictionaryArray<T> {
+    fn as_any(&self) -> &Any {
+        self
+    }
+
+    fn data(&self) -> ArrayDataRef {
+        self.data.clone()
+    }
+
+    fn data_ref(&self) -> &ArrayDataRef {
+        &self.data
+    }
+
+    fn get_buffer_memory_size(&self) -> usize {
+        // Since both `keys` and `values` derive (are references from) `data`, we only need to account for `data`.
+        self.data.get_buffer_memory_size()
+    }
+
+    fn get_array_memory_size(&self) -> usize {
+        self.data.get_array_memory_size()
+            + self.keys.get_array_memory_size()
+            + self.values.get_array_memory_size()
+            + mem::size_of_val(self)
+    }
+}
+
+impl<T: ArrowPrimitiveType> fmt::Debug for DictionaryArray<T> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(
+            f,
+            "DictionaryArray {{keys: {:?} values: {:?}}}",
+            self.keys, self.values
+        )
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::{
+        array::Int16Array,
+        datatypes::{Int32Type, Int8Type, UInt32Type, UInt8Type},
+    };
+    use crate::{
+        array::Int16DictionaryArray, array::PrimitiveDictionaryBuilder,
+        datatypes::DataType,
+    };
+    use crate::{buffer::Buffer, datatypes::ToByteSlice};
+
+    #[test]
+    fn test_dictionary_array() {
+        // Construct a value array
+        let value_data = ArrayData::builder(DataType::Int8)
+            .len(8)
+            .add_buffer(Buffer::from(
+                &[10_i8, 11, 12, 13, 14, 15, 16, 17].to_byte_slice(),
+            ))
+            .build();
+
+        // Construct a buffer for value offsets, for the nested array:
+        let keys = Buffer::from(&[2_i16, 3, 4].to_byte_slice());
+
+        // Construct a dictionary array from the above two
+        let key_type = DataType::Int16;
+        let value_type = DataType::Int8;
+        let dict_data_type =
+            DataType::Dictionary(Box::new(key_type), Box::new(value_type));
+        let dict_data = ArrayData::builder(dict_data_type.clone())
+            .len(3)
+            .add_buffer(keys.clone())
+            .add_child_data(value_data.clone())
+            .build();
+        let dict_array = Int16DictionaryArray::from(dict_data);
+
+        let values = dict_array.values();
+        assert_eq!(value_data, values.data());
+        assert_eq!(DataType::Int8, dict_array.value_type());
+        assert_eq!(3, dict_array.len());
+
+        // Null count only makes sense in terms of the component arrays.
+        assert_eq!(0, dict_array.null_count());
+        assert_eq!(0, dict_array.values().null_count());
+        assert_eq!(dict_array.keys(), &Int16Array::from(vec![2_i16, 3, 4]));
+
+        // Now test with a non-zero offset
+        let dict_data = ArrayData::builder(dict_data_type)
+            .len(2)
+            .offset(1)
+            .add_buffer(keys)
+            .add_child_data(value_data.clone())
+            .build();
+        let dict_array = Int16DictionaryArray::from(dict_data);
+
+        let values = dict_array.values();
+        assert_eq!(value_data, values.data());
+        assert_eq!(DataType::Int8, dict_array.value_type());
+        assert_eq!(2, dict_array.len());
+        assert_eq!(dict_array.keys(), &Int16Array::from(vec![3_i16, 4]));
+    }
+
+    #[test]
+    fn test_dictionary_array_fmt_debug() {
+        let key_builder = PrimitiveBuilder::<UInt8Type>::new(3);
+        let value_builder = PrimitiveBuilder::<UInt32Type>::new(2);
+        let mut builder = PrimitiveDictionaryBuilder::new(key_builder, value_builder);
+        builder.append(12345678).unwrap();
+        builder.append_null().unwrap();
+        builder.append(22345678).unwrap();
+        let array = builder.finish();
+        assert_eq!(
+            "DictionaryArray {keys: PrimitiveArray<UInt8>\n[\n  0,\n  null,\n  1,\n] values: PrimitiveArray<UInt32>\n[\n  12345678,\n  22345678,\n]}\n",
+            format!("{:?}", array)
+        );
+
+        let key_builder = PrimitiveBuilder::<UInt8Type>::new(20);
+        let value_builder = PrimitiveBuilder::<UInt32Type>::new(2);
+        let mut builder = PrimitiveDictionaryBuilder::new(key_builder, value_builder);
+        for _ in 0..20 {
+            builder.append(1).unwrap();
+        }
+        let array = builder.finish();
+        assert_eq!(
+            "DictionaryArray {keys: PrimitiveArray<UInt8>\n[\n  0,\n  0,\n  0,\n  0,\n  0,\n  0,\n  0,\n  0,\n  0,\n  0,\n  0,\n  0,\n  0,\n  0,\n  0,\n  0,\n  0,\n  0,\n  0,\n  0,\n] values: PrimitiveArray<UInt32>\n[\n  1,\n]}\n",
+            format!("{:?}", array)
+        );
+    }
+
+    #[test]
+    fn test_dictionary_array_from_iter() {
+        let test = vec!["a", "a", "b", "c"];
+        let array: DictionaryArray<Int8Type> = test
+            .iter()
+            .map(|&x| if x == "b" { None } else { Some(x) })
+            .collect();
+        assert_eq!(
+            "DictionaryArray {keys: PrimitiveArray<Int8>\n[\n  0,\n  0,\n  null,\n  1,\n] values: StringArray\n[\n  \"a\",\n  \"c\",\n]}\n",
+            format!("{:?}", array)
+        );
+
+        let array: DictionaryArray<Int8Type> = test.into_iter().collect();
+        assert_eq!(
+            "DictionaryArray {keys: PrimitiveArray<Int8>\n[\n  0,\n  0,\n  1,\n  2,\n] values: StringArray\n[\n  \"a\",\n  \"b\",\n  \"c\",\n]}\n",
+            format!("{:?}", array)
+        );
+    }
+
+    #[test]
+    fn test_dictionary_array_reverse_lookup_key() {
+        let test = vec!["a", "a", "b", "c"];
+        let array: DictionaryArray<Int8Type> = test.into_iter().collect();
+
+        assert_eq!(array.lookup_key("c"), Some(2));
+
+        // Direction of building a dictionary is the iterator direction
+        let test = vec!["t3", "t3", "t2", "t2", "t1", "t3", "t4", "t1", "t0"];
+        let array: DictionaryArray<Int8Type> = test.into_iter().collect();
+
+        assert_eq!(array.lookup_key("t1"), Some(2));
+        assert_eq!(array.lookup_key("non-existent"), None);
+    }
+
+    #[test]
+    fn test_dictionary_keys_as_primitive_array() {
+        let test = vec!["a", "b", "c", "a"];
+        let array: DictionaryArray<Int8Type> = test.into_iter().collect();
+
+        let keys = array.keys_array();
+        assert_eq!(&DataType::Int8, keys.data_type());
+        assert_eq!(0, keys.null_count());
+        assert_eq!(&[0, 1, 2, 0], keys.value_slice(0, keys.len()));
+    }
+
+    #[test]
+    fn test_dictionary_keys_as_primitive_array_with_null() {
+        let test = vec![Some("a"), None, Some("b"), None, None, Some("a")];
+        let array: DictionaryArray<Int32Type> = test.into_iter().collect();
+
+        let keys = array.keys_array();
+        assert_eq!(&DataType::Int32, keys.data_type());
+        assert_eq!(3, keys.null_count());
+
+        assert_eq!(true, keys.is_valid(0));
+        assert_eq!(false, keys.is_valid(1));
+        assert_eq!(true, keys.is_valid(2));
+        assert_eq!(false, keys.is_valid(3));
+        assert_eq!(false, keys.is_valid(4));
+        assert_eq!(true, keys.is_valid(5));
+
+        assert_eq!(0, keys.value(0));
+        assert_eq!(1, keys.value(2));
+        assert_eq!(0, keys.value(5));
+    }
+}
diff --git a/rust/arrow/src/array/array_list.rs b/rust/arrow/src/array/array_list.rs
new file mode 100644
index 00000000000..4eb8dc56640
--- /dev/null
+++ b/rust/arrow/src/array/array_list.rs
@@ -0,0 +1,815 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::any::Any;
+use std::convert::From;
+use std::fmt;
+use std::mem;
+
+use num::Num;
+
+use super::{
+    array::print_long_array, make_array, raw_pointer::as_aligned_pointer,
+    raw_pointer::RawPtrBox, Array, ArrayDataRef, ArrayRef,
+};
+use crate::datatypes::ArrowNativeType;
+use crate::datatypes::DataType;
+
+/// trait declaring an offset size, relevant for i32 vs i64 array types.
+pub trait OffsetSizeTrait: ArrowNativeType + Num + Ord {
+    fn prefix() -> &'static str;
+
+    fn to_isize(&self) -> isize;
+}
+
+impl OffsetSizeTrait for i32 {
+    fn prefix() -> &'static str {
+        ""
+    }
+
+    fn to_isize(&self) -> isize {
+        num::ToPrimitive::to_isize(self).unwrap()
+    }
+}
+
+impl OffsetSizeTrait for i64 {
+    fn prefix() -> &'static str {
+        "Large"
+    }
+
+    fn to_isize(&self) -> isize {
+        num::ToPrimitive::to_isize(self).unwrap()
+    }
+}
+
+pub struct GenericListArray<OffsetSize> {
+    data: ArrayDataRef,
+    values: ArrayRef,
+    value_offsets: RawPtrBox<OffsetSize>,
+}
+
+impl<OffsetSize: OffsetSizeTrait> GenericListArray<OffsetSize> {
+    /// Returns a reference to the values of this list.
+    pub fn values(&self) -> ArrayRef {
+        self.values.clone()
+    }
+
+    /// Returns a clone of the value type of this list.
+    pub fn value_type(&self) -> DataType {
+        self.values.data_ref().data_type().clone()
+    }
+
+    /// Returns ith value of this list array.
+    pub fn value(&self, i: usize) -> ArrayRef {
+        self.values.slice(
+            self.value_offset(i).to_usize().unwrap(),
+            self.value_length(i).to_usize().unwrap(),
+        )
+    }
+
+    /// Returns the offset for value at index `i`.
+    ///
+    /// Note this doesn't do any bound checking, for performance reason.
+    #[inline]
+    pub fn value_offset(&self, i: usize) -> OffsetSize {
+        self.value_offset_at(self.data.offset() + i)
+    }
+
+    /// Returns the length for value at index `i`.
+    ///
+    /// Note this doesn't do any bound checking, for performance reason.
+    #[inline]
+    pub fn value_length(&self, mut i: usize) -> OffsetSize {
+        i += self.data.offset();
+        self.value_offset_at(i + 1) - self.value_offset_at(i)
+    }
+
+    #[inline]
+    fn value_offset_at(&self, i: usize) -> OffsetSize {
+        unsafe { *self.value_offsets.get().add(i) }
+    }
+}
+
+impl<OffsetSize: OffsetSizeTrait> From<ArrayDataRef> for GenericListArray<OffsetSize> {
+    fn from(data: ArrayDataRef) -> Self {
+        assert_eq!(
+            data.buffers().len(),
+            1,
+            "ListArray data should contain a single buffer only (value offsets)"
+        );
+        assert_eq!(
+            data.child_data().len(),
+            1,
+            "ListArray should contain a single child array (values array)"
+        );
+        let values = make_array(data.child_data()[0].clone());
+        let raw_value_offsets = data.buffers()[0].raw_data();
+        let value_offsets: *const OffsetSize = as_aligned_pointer(raw_value_offsets);
+        unsafe {
+            assert!(
+                (*value_offsets.offset(0)).is_zero(),
+                "offsets do not start at zero"
+            );
+        }
+        Self {
+            data,
+            values,
+            value_offsets: RawPtrBox::new(value_offsets),
+        }
+    }
+}
+
+impl<OffsetSize: 'static + OffsetSizeTrait> Array for GenericListArray<OffsetSize> {
+    fn as_any(&self) -> &Any {
+        self
+    }
+
+    fn data(&self) -> ArrayDataRef {
+        self.data.clone()
+    }
+
+    fn data_ref(&self) -> &ArrayDataRef {
+        &self.data
+    }
+
+    /// Returns the total number of bytes of memory occupied by the buffers owned by this [ListArray].
+    fn get_buffer_memory_size(&self) -> usize {
+        self.data.get_buffer_memory_size()
+    }
+
+    /// Returns the total number of bytes of memory occupied physically by this [ListArray].
+    fn get_array_memory_size(&self) -> usize {
+        self.data.get_array_memory_size() + mem::size_of_val(self)
+    }
+}
+
+impl<OffsetSize: OffsetSizeTrait> fmt::Debug for GenericListArray<OffsetSize> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}ListArray\n[\n", OffsetSize::prefix())?;
+        print_long_array(self, f, |array, index, f| {
+            fmt::Debug::fmt(&array.value(index), f)
+        })?;
+        write!(f, "]")
+    }
+}
+
+/// A list array where each element is a variable-sized sequence of values with the same
+/// type whose memory offsets between elements are represented by a i32.
+pub type ListArray = GenericListArray<i32>;
+
+/// A list array where each element is a variable-sized sequence of values with the same
+/// type whose memory offsets between elements are represented by a i64.
+pub type LargeListArray = GenericListArray<i64>;
+
+/// A list array where each element is a fixed-size sequence of values with the same
+/// type whose maximum length is represented by a i32.
+pub struct FixedSizeListArray {
+    data: ArrayDataRef,
+    values: ArrayRef,
+    length: i32,
+}
+
+impl FixedSizeListArray {
+    /// Returns a reference to the values of this list.
+    pub fn values(&self) -> ArrayRef {
+        self.values.clone()
+    }
+
+    /// Returns a clone of the value type of this list.
+    pub fn value_type(&self) -> DataType {
+        self.values.data_ref().data_type().clone()
+    }
+
+    /// Returns ith value of this list array.
+    pub fn value(&self, i: usize) -> ArrayRef {
+        self.values
+            .slice(self.value_offset(i) as usize, self.value_length() as usize)
+    }
+
+    /// Returns the offset for value at index `i`.
+    ///
+    /// Note this doesn't do any bound checking, for performance reason.
+    #[inline]
+    pub fn value_offset(&self, i: usize) -> i32 {
+        self.value_offset_at(self.data.offset() + i)
+    }
+
+    /// Returns the length for value at index `i`.
+    ///
+    /// Note this doesn't do any bound checking, for performance reason.
+    #[inline]
+    pub const fn value_length(&self) -> i32 {
+        self.length
+    }
+
+    #[inline]
+    const fn value_offset_at(&self, i: usize) -> i32 {
+        i as i32 * self.length
+    }
+}
+
+impl From<ArrayDataRef> for FixedSizeListArray {
+    fn from(data: ArrayDataRef) -> Self {
+        assert_eq!(
+            data.buffers().len(),
+            0,
+            "FixedSizeListArray data should not contain a buffer for value offsets"
+        );
+        assert_eq!(
+            data.child_data().len(),
+            1,
+            "FixedSizeListArray should contain a single child array (values array)"
+        );
+        let values = make_array(data.child_data()[0].clone());
+        let length = match data.data_type() {
+            DataType::FixedSizeList(_, len) => {
+                // check that child data is multiple of length
+                assert_eq!(
+                    values.len() % *len as usize,
+                    0,
+                    "FixedSizeListArray child array length should be a multiple of {}",
+                    len
+                );
+                *len
+            }
+            _ => {
+                panic!("FixedSizeListArray data should contain a FixedSizeList data type")
+            }
+        };
+        Self {
+            data,
+            values,
+            length,
+        }
+    }
+}
+
+impl Array for FixedSizeListArray {
+    fn as_any(&self) -> &Any {
+        self
+    }
+
+    fn data(&self) -> ArrayDataRef {
+        self.data.clone()
+    }
+
+    fn data_ref(&self) -> &ArrayDataRef {
+        &self.data
+    }
+
+    /// Returns the total number of bytes of memory occupied by the buffers owned by this [FixedSizeListArray].
+    fn get_buffer_memory_size(&self) -> usize {
+        self.data.get_buffer_memory_size() + self.values().get_buffer_memory_size()
+    }
+
+    /// Returns the total number of bytes of memory occupied physically by this [FixedSizeListArray].
+    fn get_array_memory_size(&self) -> usize {
+        self.data.get_array_memory_size()
+            + self.values().get_array_memory_size()
+            + mem::size_of_val(self)
+    }
+}
+
+impl fmt::Debug for FixedSizeListArray {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "FixedSizeListArray<{}>\n[\n", self.value_length())?;
+        print_long_array(self, f, |array, index, f| {
+            fmt::Debug::fmt(&array.value(index), f)
+        })?;
+        write!(f, "]")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{
+        array::ArrayData,
+        array::Int32Array,
+        buffer::Buffer,
+        datatypes::{Field, ToByteSlice},
+        memory,
+        util::bit_util,
+    };
+
+    use super::*;
+
+    #[test]
+    fn test_list_array() {
+        // Construct a value array
+        let value_data = ArrayData::builder(DataType::Int32)
+            .len(8)
+            .add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
+            .build();
+
+        // Construct a buffer for value offsets, for the nested array:
+        //  [[0, 1, 2], [3, 4, 5], [6, 7]]
+        let value_offsets = Buffer::from(&[0, 3, 6, 8].to_byte_slice());
+
+        // Construct a list array from the above two
+        let list_data_type =
+            DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
+        let list_data = ArrayData::builder(list_data_type.clone())
+            .len(3)
+            .add_buffer(value_offsets.clone())
+            .add_child_data(value_data.clone())
+            .build();
+        let list_array = ListArray::from(list_data);
+
+        let values = list_array.values();
+        assert_eq!(value_data, values.data());
+        assert_eq!(DataType::Int32, list_array.value_type());
+        assert_eq!(3, list_array.len());
+        assert_eq!(0, list_array.null_count());
+        assert_eq!(6, list_array.value_offset(2));
+        assert_eq!(2, list_array.value_length(2));
+        assert_eq!(
+            0,
+            list_array
+                .value(0)
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap()
+                .value(0)
+        );
+        for i in 0..3 {
+            assert!(list_array.is_valid(i));
+            assert!(!list_array.is_null(i));
+        }
+
+        // Now test with a non-zero offset
+        let list_data = ArrayData::builder(list_data_type)
+            .len(3)
+            .offset(1)
+            .add_buffer(value_offsets)
+            .add_child_data(value_data.clone())
+            .build();
+        let list_array = ListArray::from(list_data);
+
+        let values = list_array.values();
+        assert_eq!(value_data, values.data());
+        assert_eq!(DataType::Int32, list_array.value_type());
+        assert_eq!(3, list_array.len());
+        assert_eq!(0, list_array.null_count());
+        assert_eq!(6, list_array.value_offset(1));
+        assert_eq!(2, list_array.value_length(1));
+        assert_eq!(
+            3,
+            list_array
+                .value(0)
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap()
+                .value(0)
+        );
+    }
+
+    #[test]
+    fn test_large_list_array() {
+        // Construct a value array
+        let value_data = ArrayData::builder(DataType::Int32)
+            .len(8)
+            .add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
+            .build();
+
+        // Construct a buffer for value offsets, for the nested array:
+        //  [[0, 1, 2], [3, 4, 5], [6, 7]]
+        let value_offsets = Buffer::from(&[0i64, 3, 6, 8].to_byte_slice());
+
+        // Construct a list array from the above two
+        let list_data_type =
+            DataType::LargeList(Box::new(Field::new("item", DataType::Int32, false)));
+        let list_data = ArrayData::builder(list_data_type.clone())
+            .len(3)
+            .add_buffer(value_offsets.clone())
+            .add_child_data(value_data.clone())
+            .build();
+        let list_array = LargeListArray::from(list_data);
+
+        let values = list_array.values();
+        assert_eq!(value_data, values.data());
+        assert_eq!(DataType::Int32, list_array.value_type());
+        assert_eq!(3, list_array.len());
+        assert_eq!(0, list_array.null_count());
+        assert_eq!(6, list_array.value_offset(2));
+        assert_eq!(2, list_array.value_length(2));
+        assert_eq!(
+            0,
+            list_array
+                .value(0)
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap()
+                .value(0)
+        );
+        for i in 0..3 {
+            assert!(list_array.is_valid(i));
+            assert!(!list_array.is_null(i));
+        }
+
+        // Now test with a non-zero offset
+        let list_data = ArrayData::builder(list_data_type)
+            .len(3)
+            .offset(1)
+            .add_buffer(value_offsets)
+            .add_child_data(value_data.clone())
+            .build();
+        let list_array = LargeListArray::from(list_data);
+
+        let values = list_array.values();
+        assert_eq!(value_data, values.data());
+        assert_eq!(DataType::Int32, list_array.value_type());
+        assert_eq!(3, list_array.len());
+        assert_eq!(0, list_array.null_count());
+        assert_eq!(6, list_array.value_offset(1));
+        assert_eq!(2, list_array.value_length(1));
+        assert_eq!(
+            3,
+            list_array
+                .value(0)
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap()
+                .value(0)
+        );
+    }
+
+    #[test]
+    fn test_fixed_size_list_array() {
+        // Construct a value array
+        let value_data = ArrayData::builder(DataType::Int32)
+            .len(9)
+            .add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7, 8].to_byte_slice()))
+            .build();
+
+        // Construct a list array from the above two
+        let list_data_type = DataType::FixedSizeList(
+            Box::new(Field::new("item", DataType::Int32, false)),
+            3,
+        );
+        let list_data = ArrayData::builder(list_data_type.clone())
+            .len(3)
+            .add_child_data(value_data.clone())
+            .build();
+        let list_array = FixedSizeListArray::from(list_data);
+
+        let values = list_array.values();
+        assert_eq!(value_data, values.data());
+        assert_eq!(DataType::Int32, list_array.value_type());
+        assert_eq!(3, list_array.len());
+        assert_eq!(0, list_array.null_count());
+        assert_eq!(6, list_array.value_offset(2));
+        assert_eq!(3, list_array.value_length());
+        assert_eq!(
+            0,
+            list_array
+                .value(0)
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap()
+                .value(0)
+        );
+        for i in 0..3 {
+            assert!(list_array.is_valid(i));
+            assert!(!list_array.is_null(i));
+        }
+
+        // Now test with a non-zero offset
+        let list_data = ArrayData::builder(list_data_type)
+            .len(3)
+            .offset(1)
+            .add_child_data(value_data.clone())
+            .build();
+        let list_array = FixedSizeListArray::from(list_data);
+
+        let values = list_array.values();
+        assert_eq!(value_data, values.data());
+        assert_eq!(DataType::Int32, list_array.value_type());
+        assert_eq!(3, list_array.len());
+        assert_eq!(0, list_array.null_count());
+        assert_eq!(
+            3,
+            list_array
+                .value(0)
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap()
+                .value(0)
+        );
+        assert_eq!(6, list_array.value_offset(1));
+        assert_eq!(3, list_array.value_length());
+    }
+
+    #[test]
+    #[should_panic(
+        expected = "FixedSizeListArray child array length should be a multiple of 3"
+    )]
+    fn test_fixed_size_list_array_unequal_children() {
+        // Construct a value array
+        let value_data = ArrayData::builder(DataType::Int32)
+            .len(8)
+            .add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
+            .build();
+
+        // Construct a list array from the above two
+        let list_data_type = DataType::FixedSizeList(
+            Box::new(Field::new("item", DataType::Int32, false)),
+            3,
+        );
+        let list_data = ArrayData::builder(list_data_type)
+            .len(3)
+            .add_child_data(value_data)
+            .build();
+        FixedSizeListArray::from(list_data);
+    }
+
+    #[test]
+    fn test_list_array_slice() {
+        // Construct a value array
+        let value_data = ArrayData::builder(DataType::Int32)
+            .len(10)
+            .add_buffer(Buffer::from(
+                &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9].to_byte_slice(),
+            ))
+            .build();
+
+        // Construct a buffer for value offsets, for the nested array:
+        //  [[0, 1], null, null, [2, 3], [4, 5], null, [6, 7, 8], null, [9]]
+        let value_offsets =
+            Buffer::from(&[0, 2, 2, 2, 4, 6, 6, 9, 9, 10].to_byte_slice());
+        // 01011001 00000001
+        let mut null_bits: [u8; 2] = [0; 2];
+        bit_util::set_bit(&mut null_bits, 0);
+        bit_util::set_bit(&mut null_bits, 3);
+        bit_util::set_bit(&mut null_bits, 4);
+        bit_util::set_bit(&mut null_bits, 6);
+        bit_util::set_bit(&mut null_bits, 8);
+
+        // Construct a list array from the above two
+        let list_data_type =
+            DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
+        let list_data = ArrayData::builder(list_data_type)
+            .len(9)
+            .add_buffer(value_offsets)
+            .add_child_data(value_data.clone())
+            .null_bit_buffer(Buffer::from(null_bits))
+            .build();
+        let list_array = ListArray::from(list_data);
+
+        let values = list_array.values();
+        assert_eq!(value_data, values.data());
+        assert_eq!(DataType::Int32, list_array.value_type());
+        assert_eq!(9, list_array.len());
+        assert_eq!(4, list_array.null_count());
+        assert_eq!(2, list_array.value_offset(3));
+        assert_eq!(2, list_array.value_length(3));
+
+        let sliced_array = list_array.slice(1, 6);
+        assert_eq!(6, sliced_array.len());
+        assert_eq!(1, sliced_array.offset());
+        assert_eq!(3, sliced_array.null_count());
+
+        for i in 0..sliced_array.len() {
+            if bit_util::get_bit(&null_bits, sliced_array.offset() + i) {
+                assert!(sliced_array.is_valid(i));
+            } else {
+                assert!(sliced_array.is_null(i));
+            }
+        }
+
+        // Check offset and length for each non-null value.
+        let sliced_list_array =
+            sliced_array.as_any().downcast_ref::<ListArray>().unwrap();
+        assert_eq!(2, sliced_list_array.value_offset(2));
+        assert_eq!(2, sliced_list_array.value_length(2));
+        assert_eq!(4, sliced_list_array.value_offset(3));
+        assert_eq!(2, sliced_list_array.value_length(3));
+        assert_eq!(6, sliced_list_array.value_offset(5));
+        assert_eq!(3, sliced_list_array.value_length(5));
+    }
+
+    #[test]
+    fn test_large_list_array_slice() {
+        // Construct a value array
+        let value_data = ArrayData::builder(DataType::Int32)
+            .len(10)
+            .add_buffer(Buffer::from(
+                &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9].to_byte_slice(),
+            ))
+            .build();
+
+        // Construct a buffer for value offsets, for the nested array:
+        //  [[0, 1], null, null, [2, 3], [4, 5], null, [6, 7, 8], null, [9]]
+        let value_offsets =
+            Buffer::from(&[0i64, 2, 2, 2, 4, 6, 6, 9, 9, 10].to_byte_slice());
+        // 01011001 00000001
+        let mut null_bits: [u8; 2] = [0; 2];
+        bit_util::set_bit(&mut null_bits, 0);
+        bit_util::set_bit(&mut null_bits, 3);
+        bit_util::set_bit(&mut null_bits, 4);
+        bit_util::set_bit(&mut null_bits, 6);
+        bit_util::set_bit(&mut null_bits, 8);
+
+        // Construct a list array from the above two
+        let list_data_type =
+            DataType::LargeList(Box::new(Field::new("item", DataType::Int32, false)));
+        let list_data = ArrayData::builder(list_data_type)
+            .len(9)
+            .add_buffer(value_offsets)
+            .add_child_data(value_data.clone())
+            .null_bit_buffer(Buffer::from(null_bits))
+            .build();
+        let list_array = LargeListArray::from(list_data);
+
+        let values = list_array.values();
+        assert_eq!(value_data, values.data());
+        assert_eq!(DataType::Int32, list_array.value_type());
+        assert_eq!(9, list_array.len());
+        assert_eq!(4, list_array.null_count());
+        assert_eq!(2, list_array.value_offset(3));
+        assert_eq!(2, list_array.value_length(3));
+
+        let sliced_array = list_array.slice(1, 6);
+        assert_eq!(6, sliced_array.len());
+        assert_eq!(1, sliced_array.offset());
+        assert_eq!(3, sliced_array.null_count());
+
+        for i in 0..sliced_array.len() {
+            if bit_util::get_bit(&null_bits, sliced_array.offset() + i) {
+                assert!(sliced_array.is_valid(i));
+            } else {
+                assert!(sliced_array.is_null(i));
+            }
+        }
+
+        // Check offset and length for each non-null value.
+        let sliced_list_array = sliced_array
+            .as_any()
+            .downcast_ref::<LargeListArray>()
+            .unwrap();
+        assert_eq!(2, sliced_list_array.value_offset(2));
+        assert_eq!(2, sliced_list_array.value_length(2));
+        assert_eq!(4, sliced_list_array.value_offset(3));
+        assert_eq!(2, sliced_list_array.value_length(3));
+        assert_eq!(6, sliced_list_array.value_offset(5));
+        assert_eq!(3, sliced_list_array.value_length(5));
+    }
+
+    #[test]
+    fn test_fixed_size_list_array_slice() {
+        // Construct a value array
+        let value_data = ArrayData::builder(DataType::Int32)
+            .len(10)
+            .add_buffer(Buffer::from(
+                &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9].to_byte_slice(),
+            ))
+            .build();
+
+        // Set null buts for the nested array:
+        //  [[0, 1], null, null, [6, 7], [8, 9]]
+        // 01011001 00000001
+        let mut null_bits: [u8; 1] = [0; 1];
+        bit_util::set_bit(&mut null_bits, 0);
+        bit_util::set_bit(&mut null_bits, 3);
+        bit_util::set_bit(&mut null_bits, 4);
+
+        // Construct a fixed size list array from the above two
+        let list_data_type = DataType::FixedSizeList(
+            Box::new(Field::new("item", DataType::Int32, false)),
+            2,
+        );
+        let list_data = ArrayData::builder(list_data_type)
+            .len(5)
+            .add_child_data(value_data.clone())
+            .null_bit_buffer(Buffer::from(null_bits))
+            .build();
+        let list_array = FixedSizeListArray::from(list_data);
+
+        let values = list_array.values();
+        assert_eq!(value_data, values.data());
+        assert_eq!(DataType::Int32, list_array.value_type());
+        assert_eq!(5, list_array.len());
+        assert_eq!(2, list_array.null_count());
+        assert_eq!(6, list_array.value_offset(3));
+        assert_eq!(2, list_array.value_length());
+
+        let sliced_array = list_array.slice(1, 4);
+        assert_eq!(4, sliced_array.len());
+        assert_eq!(1, sliced_array.offset());
+        assert_eq!(2, sliced_array.null_count());
+
+        for i in 0..sliced_array.len() {
+            if bit_util::get_bit(&null_bits, sliced_array.offset() + i) {
+                assert!(sliced_array.is_valid(i));
+            } else {
+                assert!(sliced_array.is_null(i));
+            }
+        }
+
+        // Check offset and length for each non-null value.
+        let sliced_list_array = sliced_array
+            .as_any()
+            .downcast_ref::<FixedSizeListArray>()
+            .unwrap();
+        assert_eq!(2, sliced_list_array.value_length());
+        assert_eq!(6, sliced_list_array.value_offset(2));
+        assert_eq!(8, sliced_list_array.value_offset(3));
+    }
+
+    #[test]
+    #[should_panic(
+        expected = "ListArray data should contain a single buffer only (value offsets)"
+    )]
+    fn test_list_array_invalid_buffer_len() {
+        let value_data = ArrayData::builder(DataType::Int32)
+            .len(8)
+            .add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
+            .build();
+        let list_data_type =
+            DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
+        let list_data = ArrayData::builder(list_data_type)
+            .len(3)
+            .add_child_data(value_data)
+            .build();
+        ListArray::from(list_data);
+    }
+
+    #[test]
+    #[should_panic(
+        expected = "ListArray should contain a single child array (values array)"
+    )]
+    fn test_list_array_invalid_child_array_len() {
+        let value_offsets = Buffer::from(&[0, 2, 5, 7].to_byte_slice());
+        let list_data_type =
+            DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
+        let list_data = ArrayData::builder(list_data_type)
+            .len(3)
+            .add_buffer(value_offsets)
+            .build();
+        ListArray::from(list_data);
+    }
+
+    #[test]
+    #[should_panic(expected = "offsets do not start at zero")]
+    fn test_list_array_invalid_value_offset_start() {
+        let value_data = ArrayData::builder(DataType::Int32)
+            .len(8)
+            .add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
+            .build();
+
+        let value_offsets = Buffer::from(&[2, 2, 5, 7].to_byte_slice());
+
+        let list_data_type =
+            DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
+        let list_data = ArrayData::builder(list_data_type)
+            .len(3)
+            .add_buffer(value_offsets)
+            .add_child_data(value_data)
+            .build();
+        ListArray::from(list_data);
+    }
+
+    #[test]
+    #[should_panic(expected = "memory is not aligned")]
+    fn test_primitive_array_alignment() {
+        let ptr = memory::allocate_aligned(8);
+        let buf = unsafe { Buffer::from_raw_parts(ptr, 8, 8) };
+        let buf2 = buf.slice(1);
+        let array_data = ArrayData::builder(DataType::Int32).add_buffer(buf2).build();
+        Int32Array::from(array_data);
+    }
+
+    #[test]
+    #[should_panic(expected = "memory is not aligned")]
+    fn test_list_array_alignment() {
+        let ptr = memory::allocate_aligned(8);
+        let buf = unsafe { Buffer::from_raw_parts(ptr, 8, 8) };
+        let buf2 = buf.slice(1);
+
+        let values: [i32; 8] = [0; 8];
+        let value_data = ArrayData::builder(DataType::Int32)
+            .add_buffer(Buffer::from(values.to_byte_slice()))
+            .build();
+
+        let list_data_type =
+            DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
+        let list_data = ArrayData::builder(list_data_type)
+            .add_buffer(buf2)
+            .add_child_data(value_data)
+            .build();
+        ListArray::from(list_data);
+    }
+}
diff --git a/rust/arrow/src/array/array_primitive.rs b/rust/arrow/src/array/array_primitive.rs
new file mode 100644
index 00000000000..ade18dbe329
--- /dev/null
+++ b/rust/arrow/src/array/array_primitive.rs
@@ -0,0 +1,989 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::any::Any;
+use std::borrow::Borrow;
+use std::convert::From;
+use std::fmt;
+use std::io::Write;
+use std::iter::{FromIterator, IntoIterator};
+use std::mem;
+use std::sync::Arc;
+
+use chrono::prelude::*;
+
+use super::array::print_long_array;
+use super::raw_pointer::RawPtrBox;
+use super::*;
+use crate::buffer::{Buffer, MutableBuffer};
+use crate::memory;
+use crate::util::bit_util;
+
+/// Number of seconds in a day
+const SECONDS_IN_DAY: i64 = 86_400;
+/// Number of milliseconds in a second
+const MILLISECONDS: i64 = 1_000;
+/// Number of microseconds in a second
+const MICROSECONDS: i64 = 1_000_000;
+/// Number of nanoseconds in a second
+const NANOSECONDS: i64 = 1_000_000_000;
+
+/// Array whose elements are of primitive types.
+pub struct PrimitiveArray<T: ArrowPrimitiveType> {
+    data: ArrayDataRef,
+    /// Pointer to the value array. The lifetime of this must be <= to the value buffer
+    /// stored in `data`, so it's safe to store.
+    /// Also note that boolean arrays are bit-packed, so although the underlying pointer
+    /// is of type bool it should be cast back to u8 before being used.
+    /// i.e. `self.raw_values.get() as *const u8`
+    raw_values: RawPtrBox<T::Native>,
+}
+
+impl<T: ArrowPrimitiveType> PrimitiveArray<T> {
+    /// Returns the length of this array.
+    pub fn len(&self) -> usize {
+        self.data.len()
+    }
+
+    /// Returns whether this array is empty.
+    pub fn is_empty(&self) -> bool {
+        self.data.is_empty()
+    }
+
+    /// Returns a raw pointer to the values of this array.
+    pub fn raw_values(&self) -> *const T::Native {
+        unsafe { self.raw_values.get().add(self.data.offset()) }
+    }
+
+    /// Returns a slice for the given offset and length
+    ///
+    /// Note this doesn't do any bound checking, for performance reason.
+    pub fn value_slice(&self, offset: usize, len: usize) -> &[T::Native] {
+        let raw =
+            unsafe { std::slice::from_raw_parts(self.raw_values().add(offset), len) };
+        &raw[..]
+    }
+
+    // Returns a new primitive array builder
+    pub fn builder(capacity: usize) -> PrimitiveBuilder<T> {
+        PrimitiveBuilder::<T>::new(capacity)
+    }
+
+    /// Returns a `Buffer` holding all the values of this array.
+    ///
+    /// Note this doesn't take the offset of this array into account.
+    pub fn values(&self) -> Buffer {
+        self.data.buffers()[0].clone()
+    }
+
+    /// Returns the primitive value at index `i`.
+    ///
+    /// Note this doesn't do any bound checking, for performance reason.
+    pub fn value(&self, i: usize) -> T::Native {
+        let offset = i + self.offset();
+        unsafe { T::index(self.raw_values.get(), offset) }
+    }
+}
+
+impl<T: ArrowPrimitiveType> Array for PrimitiveArray<T> {
+    fn as_any(&self) -> &Any {
+        self
+    }
+
+    fn data(&self) -> ArrayDataRef {
+        self.data.clone()
+    }
+
+    fn data_ref(&self) -> &ArrayDataRef {
+        &self.data
+    }
+
+    /// Returns the total number of bytes of memory occupied by the buffers owned by this [PrimitiveArray].
+    fn get_buffer_memory_size(&self) -> usize {
+        self.data.get_buffer_memory_size()
+    }
+
+    /// Returns the total number of bytes of memory occupied physically by this [PrimitiveArray].
+    fn get_array_memory_size(&self) -> usize {
+        self.data.get_array_memory_size() + mem::size_of_val(self)
+    }
+}
+
+fn as_datetime<T: ArrowPrimitiveType>(v: i64) -> Option<NaiveDateTime> {
+    match T::DATA_TYPE {
+        DataType::Date32(_) => {
+            // convert days into seconds
+            Some(NaiveDateTime::from_timestamp(v as i64 * SECONDS_IN_DAY, 0))
+        }
+        DataType::Date64(_) => Some(NaiveDateTime::from_timestamp(
+            // extract seconds from milliseconds
+            v / MILLISECONDS,
+            // discard extracted seconds and convert milliseconds to nanoseconds
+            (v % MILLISECONDS * MICROSECONDS) as u32,
+        )),
+        DataType::Time32(_) | DataType::Time64(_) => None,
+        DataType::Timestamp(unit, _) => match unit {
+            TimeUnit::Second => Some(NaiveDateTime::from_timestamp(v, 0)),
+            TimeUnit::Millisecond => Some(NaiveDateTime::from_timestamp(
+                // extract seconds from milliseconds
+                v / MILLISECONDS,
+                // discard extracted seconds and convert milliseconds to nanoseconds
+                (v % MILLISECONDS * MICROSECONDS) as u32,
+            )),
+            TimeUnit::Microsecond => Some(NaiveDateTime::from_timestamp(
+                // extract seconds from microseconds
+                v / MICROSECONDS,
+                // discard extracted seconds and convert microseconds to nanoseconds
+                (v % MICROSECONDS * MILLISECONDS) as u32,
+            )),
+            TimeUnit::Nanosecond => Some(NaiveDateTime::from_timestamp(
+                // extract seconds from nanoseconds
+                v / NANOSECONDS,
+                // discard extracted seconds
+                (v % NANOSECONDS) as u32,
+            )),
+        },
+        // interval is not yet fully documented [ARROW-3097]
+        DataType::Interval(_) => None,
+        _ => None,
+    }
+}
+
+fn as_date<T: ArrowPrimitiveType>(v: i64) -> Option<NaiveDate> {
+    as_datetime::<T>(v).map(|datetime| datetime.date())
+}
+
+fn as_time<T: ArrowPrimitiveType>(v: i64) -> Option<NaiveTime> {
+    match T::DATA_TYPE {
+        DataType::Time32(unit) => {
+            // safe to immediately cast to u32 as `self.value(i)` is positive i32
+            let v = v as u32;
+            match unit {
+                TimeUnit::Second => Some(NaiveTime::from_num_seconds_from_midnight(v, 0)),
+                TimeUnit::Millisecond => {
+                    Some(NaiveTime::from_num_seconds_from_midnight(
+                        // extract seconds from milliseconds
+                        v / MILLISECONDS as u32,
+                        // discard extracted seconds and convert milliseconds to
+                        // nanoseconds
+                        v % MILLISECONDS as u32 * MICROSECONDS as u32,
+                    ))
+                }
+                _ => None,
+            }
+        }
+        DataType::Time64(unit) => {
+            match unit {
+                TimeUnit::Microsecond => {
+                    Some(NaiveTime::from_num_seconds_from_midnight(
+                        // extract seconds from microseconds
+                        (v / MICROSECONDS) as u32,
+                        // discard extracted seconds and convert microseconds to
+                        // nanoseconds
+                        (v % MICROSECONDS * MILLISECONDS) as u32,
+                    ))
+                }
+                TimeUnit::Nanosecond => {
+                    Some(NaiveTime::from_num_seconds_from_midnight(
+                        // extract seconds from nanoseconds
+                        (v / NANOSECONDS) as u32,
+                        // discard extracted seconds
+                        (v % NANOSECONDS) as u32,
+                    ))
+                }
+                _ => None,
+            }
+        }
+        DataType::Timestamp(_, _) => as_datetime::<T>(v).map(|datetime| datetime.time()),
+        DataType::Date32(_) | DataType::Date64(_) => Some(NaiveTime::from_hms(0, 0, 0)),
+        DataType::Interval(_) => None,
+        _ => None,
+    }
+}
+
+impl<T: ArrowTemporalType + ArrowNumericType> PrimitiveArray<T>
+where
+    i64: std::convert::From<T::Native>,
+{
+    /// Returns value as a chrono `NaiveDateTime`, handling time resolution
+    ///
+    /// If a data type cannot be converted to `NaiveDateTime`, a `None` is returned.
+    /// A valid value is expected, thus the user should first check for validity.
+    pub fn value_as_datetime(&self, i: usize) -> Option<NaiveDateTime> {
+        as_datetime::<T>(i64::from(self.value(i)))
+    }
+
+    /// Returns value as a chrono `NaiveDate` by using `Self::datetime()`
+    ///
+    /// If a data type cannot be converted to `NaiveDate`, a `None` is returned
+    pub fn value_as_date(&self, i: usize) -> Option<NaiveDate> {
+        self.value_as_datetime(i).map(|datetime| datetime.date())
+    }
+
+    /// Returns a value as a chrono `NaiveTime`
+    ///
+    /// `Date32` and `Date64` return UTC midnight as they do not have time resolution
+    pub fn value_as_time(&self, i: usize) -> Option<NaiveTime> {
+        as_time::<T>(i64::from(self.value(i)))
+    }
+}
+
+impl<T: ArrowPrimitiveType> fmt::Debug for PrimitiveArray<T> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "PrimitiveArray<{:?}>\n[\n", T::DATA_TYPE)?;
+        print_long_array(self, f, |array, index, f| match T::DATA_TYPE {
+            DataType::Date32(_) | DataType::Date64(_) => {
+                let v = self.value(index).to_usize().unwrap() as i64;
+                match as_date::<T>(v) {
+                    Some(date) => write!(f, "{:?}", date),
+                    None => write!(f, "null"),
+                }
+            }
+            DataType::Time32(_) | DataType::Time64(_) => {
+                let v = self.value(index).to_usize().unwrap() as i64;
+                match as_time::<T>(v) {
+                    Some(time) => write!(f, "{:?}", time),
+                    None => write!(f, "null"),
+                }
+            }
+            DataType::Timestamp(_, _) => {
+                let v = self.value(index).to_usize().unwrap() as i64;
+                match as_datetime::<T>(v) {
+                    Some(datetime) => write!(f, "{:?}", datetime),
+                    None => write!(f, "null"),
+                }
+            }
+            _ => fmt::Debug::fmt(&array.value(index), f),
+        })?;
+        write!(f, "]")
+    }
+}
+
+impl<'a, T: ArrowPrimitiveType> IntoIterator for &'a PrimitiveArray<T> {
+    type Item = Option<<T as ArrowPrimitiveType>::Native>;
+    type IntoIter = PrimitiveIter<'a, T>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        PrimitiveIter::<'a, T>::new(self)
+    }
+}
+
+impl<'a, T: ArrowPrimitiveType> PrimitiveArray<T> {
+    /// constructs a new iterator
+    pub fn iter(&'a self) -> PrimitiveIter<'a, T> {
+        PrimitiveIter::<'a, T>::new(&self)
+    }
+}
+
+impl<T: ArrowPrimitiveType, Ptr: Borrow<Option<<T as ArrowPrimitiveType>::Native>>>
+    FromIterator<Ptr> for PrimitiveArray<T>
+{
+    fn from_iter<I: IntoIterator<Item = Ptr>>(iter: I) -> Self {
+        let iter = iter.into_iter();
+        let (_, data_len) = iter.size_hint();
+        let data_len = data_len.expect("Iterator must be sized"); // panic if no upper bound.
+
+        let num_bytes = bit_util::ceil(data_len, 8);
+        let mut null_buf = MutableBuffer::new(num_bytes).with_bitset(num_bytes, false);
+        let mut val_buf = MutableBuffer::new(
+            data_len * mem::size_of::<<T as ArrowPrimitiveType>::Native>(),
+        );
+
+        let null = vec![0; mem::size_of::<<T as ArrowPrimitiveType>::Native>()];
+
+        let null_slice = null_buf.data_mut();
+        iter.enumerate().for_each(|(i, item)| {
+            if let Some(a) = item.borrow() {
+                bit_util::set_bit(null_slice, i);
+                val_buf.write_all(a.to_byte_slice()).unwrap();
+            } else {
+                val_buf.write_all(&null).unwrap();
+            }
+        });
+
+        let data = ArrayData::new(
+            T::DATA_TYPE,
+            data_len,
+            None,
+            Some(null_buf.freeze()),
+            0,
+            vec![val_buf.freeze()],
+            vec![],
+        );
+        PrimitiveArray::from(Arc::new(data))
+    }
+}
+
+// TODO: the macro is needed here because we'd get "conflicting implementations" error
+// otherwise with both `From<Vec<T::Native>>` and `From<Vec<Option<T::Native>>>`.
+// We should revisit this in future.
+macro_rules! def_numeric_from_vec {
+    ( $ty:ident ) => {
+        impl From<Vec<<$ty as ArrowPrimitiveType>::Native>> for PrimitiveArray<$ty> {
+            fn from(data: Vec<<$ty as ArrowPrimitiveType>::Native>) -> Self {
+                let array_data = ArrayData::builder($ty::DATA_TYPE)
+                    .len(data.len())
+                    .add_buffer(Buffer::from(data.to_byte_slice()))
+                    .build();
+                PrimitiveArray::from(array_data)
+            }
+        }
+
+        // Constructs a primitive array from a vector. Should only be used for testing.
+        impl From<Vec<Option<<$ty as ArrowPrimitiveType>::Native>>>
+            for PrimitiveArray<$ty>
+        {
+            fn from(data: Vec<Option<<$ty as ArrowPrimitiveType>::Native>>) -> Self {
+                PrimitiveArray::from_iter(data.iter())
+            }
+        }
+    };
+}
+
+def_numeric_from_vec!(Int8Type);
+def_numeric_from_vec!(Int16Type);
+def_numeric_from_vec!(Int32Type);
+def_numeric_from_vec!(Int64Type);
+def_numeric_from_vec!(UInt8Type);
+def_numeric_from_vec!(UInt16Type);
+def_numeric_from_vec!(UInt32Type);
+def_numeric_from_vec!(UInt64Type);
+def_numeric_from_vec!(Float32Type);
+def_numeric_from_vec!(Float64Type);
+
+def_numeric_from_vec!(Date32Type);
+def_numeric_from_vec!(Date64Type);
+def_numeric_from_vec!(Time32SecondType);
+def_numeric_from_vec!(Time32MillisecondType);
+def_numeric_from_vec!(Time64MicrosecondType);
+def_numeric_from_vec!(Time64NanosecondType);
+def_numeric_from_vec!(IntervalYearMonthType);
+def_numeric_from_vec!(IntervalDayTimeType);
+def_numeric_from_vec!(DurationSecondType);
+def_numeric_from_vec!(DurationMillisecondType);
+def_numeric_from_vec!(DurationMicrosecondType);
+def_numeric_from_vec!(DurationNanosecondType);
+def_numeric_from_vec!(TimestampMillisecondType);
+def_numeric_from_vec!(TimestampMicrosecondType);
+
+impl<T: ArrowTimestampType> PrimitiveArray<T> {
+    /// Construct a timestamp array from a vec of i64 values and an optional timezone
+    pub fn from_vec(data: Vec<i64>, timezone: Option<Arc<String>>) -> Self {
+        let array_data =
+            ArrayData::builder(DataType::Timestamp(T::get_time_unit(), timezone))
+                .len(data.len())
+                .add_buffer(Buffer::from(data.to_byte_slice()))
+                .build();
+        PrimitiveArray::from(array_data)
+    }
+}
+
+impl<T: ArrowTimestampType> PrimitiveArray<T> {
+    /// Construct a timestamp array from a vec of Option<i64> values and an optional timezone
+    pub fn from_opt_vec(data: Vec<Option<i64>>, timezone: Option<Arc<String>>) -> Self {
+        // TODO: duplicated from def_numeric_from_vec! macro, it looks possible to convert to generic
+        let data_len = data.len();
+        let mut null_buf = MutableBuffer::new_null(data_len);
+        let mut val_buf = MutableBuffer::new(data_len * mem::size_of::<i64>());
+
+        {
+            let null = vec![0; mem::size_of::<i64>()];
+            let null_slice = null_buf.data_mut();
+            for (i, v) in data.iter().enumerate() {
+                if let Some(n) = v {
+                    bit_util::set_bit(null_slice, i);
+                    // unwrap() in the following should be safe here since we've
+                    // made sure enough space is allocated for the values.
+                    val_buf.write_all(&n.to_byte_slice()).unwrap();
+                } else {
+                    val_buf.write_all(&null).unwrap();
+                }
+            }
+        }
+
+        let array_data =
+            ArrayData::builder(DataType::Timestamp(T::get_time_unit(), timezone))
+                .len(data_len)
+                .add_buffer(val_buf.freeze())
+                .null_bit_buffer(null_buf.freeze())
+                .build();
+        PrimitiveArray::from(array_data)
+    }
+}
+
+/// Constructs a boolean array from a vector. Should only be used for testing.
+impl From<Vec<bool>> for BooleanArray {
+    fn from(data: Vec<bool>) -> Self {
+        let mut mut_buf = MutableBuffer::new_null(data.len());
+        {
+            let mut_slice = mut_buf.data_mut();
+            for (i, b) in data.iter().enumerate() {
+                if *b {
+                    bit_util::set_bit(mut_slice, i);
+                }
+            }
+        }
+        let array_data = ArrayData::builder(DataType::Boolean)
+            .len(data.len())
+            .add_buffer(mut_buf.freeze())
+            .build();
+        BooleanArray::from(array_data)
+    }
+}
+
+impl From<Vec<Option<bool>>> for BooleanArray {
+    fn from(data: Vec<Option<bool>>) -> Self {
+        let data_len = data.len();
+        let num_byte = bit_util::ceil(data_len, 8);
+        let mut null_buf = MutableBuffer::new_null(data.len());
+        let mut val_buf = MutableBuffer::new(num_byte).with_bitset(num_byte, false);
+
+        {
+            let null_slice = null_buf.data_mut();
+            let val_slice = val_buf.data_mut();
+
+            for (i, v) in data.iter().enumerate() {
+                if let Some(b) = v {
+                    bit_util::set_bit(null_slice, i);
+                    if *b {
+                        bit_util::set_bit(val_slice, i);
+                    }
+                }
+            }
+        }
+
+        let array_data = ArrayData::builder(DataType::Boolean)
+            .len(data_len)
+            .add_buffer(val_buf.freeze())
+            .null_bit_buffer(null_buf.freeze())
+            .build();
+        BooleanArray::from(array_data)
+    }
+}
+
+/// Constructs a `PrimitiveArray` from an array data reference.
+impl<T: ArrowPrimitiveType> From<ArrayDataRef> for PrimitiveArray<T> {
+    fn from(data: ArrayDataRef) -> Self {
+        assert_eq!(
+            data.buffers().len(),
+            1,
+            "PrimitiveArray data should contain a single buffer only (values buffer)"
+        );
+        let raw_values = data.buffers()[0].raw_data();
+        assert!(
+            memory::is_aligned::<u8>(raw_values, mem::align_of::<T::Native>()),
+            "memory is not aligned"
+        );
+        Self {
+            data,
+            raw_values: RawPtrBox::new(raw_values as *const T::Native),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use std::thread;
+
+    use crate::buffer::Buffer;
+    use crate::datatypes::DataType;
+
+    #[test]
+    fn test_primitive_array_from_vec() {
+        let buf = Buffer::from(&[0, 1, 2, 3, 4].to_byte_slice());
+        let arr = Int32Array::from(vec![0, 1, 2, 3, 4]);
+        let slice = unsafe { std::slice::from_raw_parts(arr.raw_values(), 5) };
+        assert_eq!(buf, arr.values());
+        assert_eq!(&[0, 1, 2, 3, 4], slice);
+        assert_eq!(5, arr.len());
+        assert_eq!(0, arr.offset());
+        assert_eq!(0, arr.null_count());
+        for i in 0..5 {
+            assert!(!arr.is_null(i));
+            assert!(arr.is_valid(i));
+            assert_eq!(i as i32, arr.value(i));
+        }
+
+        assert_eq!(64, arr.get_buffer_memory_size());
+        let internals_of_primitive_array = 8 + 72; // RawPtrBox & Arc<ArrayData> combined.
+        assert_eq!(
+            arr.get_buffer_memory_size() + internals_of_primitive_array,
+            arr.get_array_memory_size()
+        );
+    }
+
+    #[test]
+    fn test_primitive_array_from_vec_option() {
+        // Test building a primitive array with null values
+        let arr = Int32Array::from(vec![Some(0), None, Some(2), None, Some(4)]);
+        assert_eq!(5, arr.len());
+        assert_eq!(0, arr.offset());
+        assert_eq!(2, arr.null_count());
+        for i in 0..5 {
+            if i % 2 == 0 {
+                assert!(!arr.is_null(i));
+                assert!(arr.is_valid(i));
+                assert_eq!(i as i32, arr.value(i));
+            } else {
+                assert!(arr.is_null(i));
+                assert!(!arr.is_valid(i));
+            }
+        }
+
+        assert_eq!(128, arr.get_buffer_memory_size());
+        let internals_of_primitive_array = 8 + 72 + 16; // RawPtrBox & Arc<ArrayData> and it's null_bitmap combined.
+        assert_eq!(
+            arr.get_buffer_memory_size() + internals_of_primitive_array,
+            arr.get_array_memory_size()
+        );
+    }
+
+    #[test]
+    fn test_date64_array_from_vec_option() {
+        // Test building a primitive array with null values
+        // we use Int32 and Int64 as a backing array, so all Int32 and Int64 conventions
+        // work
+        let arr: PrimitiveArray<Date64Type> =
+            vec![Some(1550902545147), None, Some(1550902545147)].into();
+        assert_eq!(3, arr.len());
+        assert_eq!(0, arr.offset());
+        assert_eq!(1, arr.null_count());
+        for i in 0..3 {
+            if i % 2 == 0 {
+                assert!(!arr.is_null(i));
+                assert!(arr.is_valid(i));
+                assert_eq!(1550902545147, arr.value(i));
+                // roundtrip to and from datetime
+                assert_eq!(
+                    1550902545147,
+                    arr.value_as_datetime(i).unwrap().timestamp_millis()
+                );
+            } else {
+                assert!(arr.is_null(i));
+                assert!(!arr.is_valid(i));
+            }
+        }
+    }
+
+    #[test]
+    fn test_time32_millisecond_array_from_vec() {
+        // 1:        00:00:00.001
+        // 37800005: 10:30:00.005
+        // 86399210: 23:59:59.210
+        let arr: PrimitiveArray<Time32MillisecondType> =
+            vec![1, 37_800_005, 86_399_210].into();
+        assert_eq!(3, arr.len());
+        assert_eq!(0, arr.offset());
+        assert_eq!(0, arr.null_count());
+        let formatted = vec!["00:00:00.001", "10:30:00.005", "23:59:59.210"];
+        for i in 0..3 {
+            // check that we can't create dates or datetimes from time instances
+            assert_eq!(None, arr.value_as_datetime(i));
+            assert_eq!(None, arr.value_as_date(i));
+            let time = arr.value_as_time(i).unwrap();
+            assert_eq!(formatted[i], time.format("%H:%M:%S%.3f").to_string());
+        }
+    }
+
+    #[test]
+    fn test_time64_nanosecond_array_from_vec() {
+        // Test building a primitive array with null values
+        // we use Int32 and Int64 as a backing array, so all Int32 and Int64 conventions
+        // work
+
+        // 1e6:        00:00:00.001
+        // 37800005e6: 10:30:00.005
+        // 86399210e6: 23:59:59.210
+        let arr: PrimitiveArray<Time64NanosecondType> =
+            vec![1_000_000, 37_800_005_000_000, 86_399_210_000_000].into();
+        assert_eq!(3, arr.len());
+        assert_eq!(0, arr.offset());
+        assert_eq!(0, arr.null_count());
+        let formatted = vec!["00:00:00.001", "10:30:00.005", "23:59:59.210"];
+        for i in 0..3 {
+            // check that we can't create dates or datetimes from time instances
+            assert_eq!(None, arr.value_as_datetime(i));
+            assert_eq!(None, arr.value_as_date(i));
+            let time = arr.value_as_time(i).unwrap();
+            assert_eq!(formatted[i], time.format("%H:%M:%S%.3f").to_string());
+        }
+    }
+
+    #[test]
+    fn test_interval_array_from_vec() {
+        // intervals are currently not treated specially, but are Int32 and Int64 arrays
+        let arr = IntervalYearMonthArray::from(vec![Some(1), None, Some(-5)]);
+        assert_eq!(3, arr.len());
+        assert_eq!(0, arr.offset());
+        assert_eq!(1, arr.null_count());
+        assert_eq!(1, arr.value(0));
+        assert!(arr.is_null(1));
+        assert_eq!(-5, arr.value(2));
+
+        // a day_time interval contains days and milliseconds, but we do not yet have accessors for the values
+        let arr = IntervalDayTimeArray::from(vec![Some(1), None, Some(-5)]);
+        assert_eq!(3, arr.len());
+        assert_eq!(0, arr.offset());
+        assert_eq!(1, arr.null_count());
+        assert_eq!(1, arr.value(0));
+        assert!(arr.is_null(1));
+        assert_eq!(-5, arr.value(2));
+    }
+
+    #[test]
+    fn test_duration_array_from_vec() {
+        let arr = DurationSecondArray::from(vec![Some(1), None, Some(-5)]);
+        assert_eq!(3, arr.len());
+        assert_eq!(0, arr.offset());
+        assert_eq!(1, arr.null_count());
+        assert_eq!(1, arr.value(0));
+        assert!(arr.is_null(1));
+        assert_eq!(-5, arr.value(2));
+
+        let arr = DurationMillisecondArray::from(vec![Some(1), None, Some(-5)]);
+        assert_eq!(3, arr.len());
+        assert_eq!(0, arr.offset());
+        assert_eq!(1, arr.null_count());
+        assert_eq!(1, arr.value(0));
+        assert!(arr.is_null(1));
+        assert_eq!(-5, arr.value(2));
+
+        let arr = DurationMicrosecondArray::from(vec![Some(1), None, Some(-5)]);
+        assert_eq!(3, arr.len());
+        assert_eq!(0, arr.offset());
+        assert_eq!(1, arr.null_count());
+        assert_eq!(1, arr.value(0));
+        assert!(arr.is_null(1));
+        assert_eq!(-5, arr.value(2));
+
+        let arr = DurationNanosecondArray::from(vec![Some(1), None, Some(-5)]);
+        assert_eq!(3, arr.len());
+        assert_eq!(0, arr.offset());
+        assert_eq!(1, arr.null_count());
+        assert_eq!(1, arr.value(0));
+        assert!(arr.is_null(1));
+        assert_eq!(-5, arr.value(2));
+    }
+
+    #[test]
+    fn test_timestamp_array_from_vec() {
+        let arr = TimestampSecondArray::from_vec(vec![1, -5], None);
+        assert_eq!(2, arr.len());
+        assert_eq!(0, arr.offset());
+        assert_eq!(0, arr.null_count());
+        assert_eq!(1, arr.value(0));
+        assert_eq!(-5, arr.value(1));
+
+        let arr = TimestampMillisecondArray::from_vec(vec![1, -5], None);
+        assert_eq!(2, arr.len());
+        assert_eq!(0, arr.offset());
+        assert_eq!(0, arr.null_count());
+        assert_eq!(1, arr.value(0));
+        assert_eq!(-5, arr.value(1));
+
+        let arr = TimestampMicrosecondArray::from_vec(vec![1, -5], None);
+        assert_eq!(2, arr.len());
+        assert_eq!(0, arr.offset());
+        assert_eq!(0, arr.null_count());
+        assert_eq!(1, arr.value(0));
+        assert_eq!(-5, arr.value(1));
+
+        let arr = TimestampNanosecondArray::from_vec(vec![1, -5], None);
+        assert_eq!(2, arr.len());
+        assert_eq!(0, arr.offset());
+        assert_eq!(0, arr.null_count());
+        assert_eq!(1, arr.value(0));
+        assert_eq!(-5, arr.value(1));
+    }
+
+    #[test]
+    fn test_primitive_array_slice() {
+        let arr = Int32Array::from(vec![
+            Some(0),
+            None,
+            Some(2),
+            None,
+            Some(4),
+            Some(5),
+            Some(6),
+            None,
+            None,
+        ]);
+        assert_eq!(9, arr.len());
+        assert_eq!(0, arr.offset());
+        assert_eq!(4, arr.null_count());
+
+        let arr2 = arr.slice(2, 5);
+        assert_eq!(5, arr2.len());
+        assert_eq!(2, arr2.offset());
+        assert_eq!(1, arr2.null_count());
+
+        for i in 0..arr2.len() {
+            assert_eq!(i == 1, arr2.is_null(i));
+            assert_eq!(i != 1, arr2.is_valid(i));
+        }
+
+        let arr3 = arr2.slice(2, 3);
+        assert_eq!(3, arr3.len());
+        assert_eq!(4, arr3.offset());
+        assert_eq!(0, arr3.null_count());
+
+        let int_arr = arr3.as_any().downcast_ref::<Int32Array>().unwrap();
+        assert_eq!(4, int_arr.value(0));
+        assert_eq!(5, int_arr.value(1));
+        assert_eq!(6, int_arr.value(2));
+    }
+
+    #[test]
+    fn test_boolean_array_slice() {
+        let arr = BooleanArray::from(vec![
+            Some(true),
+            None,
+            Some(false),
+            None,
+            Some(true),
+            Some(false),
+            Some(true),
+            Some(false),
+            None,
+            Some(true),
+        ]);
+
+        assert_eq!(10, arr.len());
+        assert_eq!(0, arr.offset());
+        assert_eq!(3, arr.null_count());
+
+        let arr2 = arr.slice(3, 5);
+        assert_eq!(5, arr2.len());
+        assert_eq!(3, arr2.offset());
+        assert_eq!(1, arr2.null_count());
+
+        let bool_arr = arr2.as_any().downcast_ref::<BooleanArray>().unwrap();
+
+        assert_eq!(false, bool_arr.is_valid(0));
+
+        assert_eq!(true, bool_arr.is_valid(1));
+        assert_eq!(true, bool_arr.value(1));
+
+        assert_eq!(true, bool_arr.is_valid(2));
+        assert_eq!(false, bool_arr.value(2));
+
+        assert_eq!(true, bool_arr.is_valid(3));
+        assert_eq!(true, bool_arr.value(3));
+
+        assert_eq!(true, bool_arr.is_valid(4));
+        assert_eq!(false, bool_arr.value(4));
+    }
+
+    #[test]
+    fn test_value_slice_no_bounds_check() {
+        let arr = Int32Array::from(vec![2, 3, 4]);
+        let _slice = arr.value_slice(0, 4);
+    }
+
+    #[test]
+    fn test_int32_fmt_debug() {
+        let arr = Int32Array::from(vec![0, 1, 2, 3, 4]);
+        assert_eq!(
+            "PrimitiveArray<Int32>\n[\n  0,\n  1,\n  2,\n  3,\n  4,\n]",
+            format!("{:?}", arr)
+        );
+    }
+
+    #[test]
+    fn test_fmt_debug_up_to_20_elements() {
+        (1..=20).for_each(|i| {
+            let values = (0..i).collect::<Vec<i16>>();
+            let array_expected = format!(
+                "PrimitiveArray<Int16>\n[\n{}\n]",
+                values
+                    .iter()
+                    .map(|v| { format!("  {},", v) })
+                    .collect::<Vec<String>>()
+                    .join("\n")
+            );
+            let array = Int16Array::from(values);
+
+            assert_eq!(array_expected, format!("{:?}", array));
+        })
+    }
+
+    #[test]
+    fn test_int32_with_null_fmt_debug() {
+        let mut builder = Int32Array::builder(3);
+        builder.append_slice(&[0, 1]).unwrap();
+        builder.append_null().unwrap();
+        builder.append_slice(&[3, 4]).unwrap();
+        let arr = builder.finish();
+        assert_eq!(
+            "PrimitiveArray<Int32>\n[\n  0,\n  1,\n  null,\n  3,\n  4,\n]",
+            format!("{:?}", arr)
+        );
+    }
+
+    #[test]
+    fn test_boolean_fmt_debug() {
+        let arr = BooleanArray::from(vec![true, false, false]);
+        assert_eq!(
+            "PrimitiveArray<Boolean>\n[\n  true,\n  false,\n  false,\n]",
+            format!("{:?}", arr)
+        );
+    }
+
+    #[test]
+    fn test_boolean_with_null_fmt_debug() {
+        let mut builder = BooleanArray::builder(3);
+        builder.append_value(true).unwrap();
+        builder.append_null().unwrap();
+        builder.append_value(false).unwrap();
+        let arr = builder.finish();
+        assert_eq!(
+            "PrimitiveArray<Boolean>\n[\n  true,\n  null,\n  false,\n]",
+            format!("{:?}", arr)
+        );
+    }
+
+    #[test]
+    fn test_timestamp_fmt_debug() {
+        let arr: PrimitiveArray<TimestampMillisecondType> =
+            TimestampMillisecondArray::from_vec(vec![1546214400000, 1546214400000], None);
+        assert_eq!(
+            "PrimitiveArray<Timestamp(Millisecond, None)>\n[\n  2018-12-31T00:00:00,\n  2018-12-31T00:00:00,\n]",
+            format!("{:?}", arr)
+        );
+    }
+
+    #[test]
+    fn test_date32_fmt_debug() {
+        let arr: PrimitiveArray<Date32Type> = vec![12356, 13548].into();
+        assert_eq!(
+            "PrimitiveArray<Date32(Day)>\n[\n  2003-10-31,\n  2007-02-04,\n]",
+            format!("{:?}", arr)
+        );
+    }
+
+    #[test]
+    fn test_time32second_fmt_debug() {
+        let arr: PrimitiveArray<Time32SecondType> = vec![7201, 60054].into();
+        assert_eq!(
+            "PrimitiveArray<Time32(Second)>\n[\n  02:00:01,\n  16:40:54,\n]",
+            format!("{:?}", arr)
+        );
+    }
+
+    #[test]
+    fn test_primitive_array_builder() {
+        // Test building a primitive array with ArrayData builder and offset
+        let buf = Buffer::from(&[0, 1, 2, 3, 4].to_byte_slice());
+        let buf2 = buf.clone();
+        let data = ArrayData::builder(DataType::Int32)
+            .len(5)
+            .offset(2)
+            .add_buffer(buf)
+            .build();
+        let arr = Int32Array::from(data);
+        assert_eq!(buf2, arr.values());
+        assert_eq!(5, arr.len());
+        assert_eq!(0, arr.null_count());
+        for i in 0..3 {
+            assert_eq!((i + 2) as i32, arr.value(i));
+        }
+    }
+
+    #[test]
+    #[should_panic(expected = "PrimitiveArray data should contain a single buffer only \
+                               (values buffer)")]
+    fn test_primitive_array_invalid_buffer_len() {
+        let data = ArrayData::builder(DataType::Int32).len(5).build();
+        Int32Array::from(data);
+    }
+
+    #[test]
+    fn test_boolean_array_from_vec() {
+        let buf = Buffer::from([10_u8]);
+        let arr = BooleanArray::from(vec![false, true, false, true]);
+        assert_eq!(buf, arr.values());
+        assert_eq!(4, arr.len());
+        assert_eq!(0, arr.offset());
+        assert_eq!(0, arr.null_count());
+        for i in 0..4 {
+            assert!(!arr.is_null(i));
+            assert!(arr.is_valid(i));
+            assert_eq!(i == 1 || i == 3, arr.value(i), "failed at {}", i)
+        }
+    }
+
+    #[test]
+    fn test_boolean_array_from_vec_option() {
+        let buf = Buffer::from([10_u8]);
+        let arr = BooleanArray::from(vec![Some(false), Some(true), None, Some(true)]);
+        assert_eq!(buf, arr.values());
+        assert_eq!(4, arr.len());
+        assert_eq!(0, arr.offset());
+        assert_eq!(1, arr.null_count());
+        for i in 0..4 {
+            if i == 2 {
+                assert!(arr.is_null(i));
+                assert!(!arr.is_valid(i));
+            } else {
+                assert!(!arr.is_null(i));
+                assert!(arr.is_valid(i));
+                assert_eq!(i == 1 || i == 3, arr.value(i), "failed at {}", i)
+            }
+        }
+    }
+
+    #[test]
+    fn test_boolean_array_builder() {
+        // Test building a boolean array with ArrayData builder and offset
+        // 000011011
+        let buf = Buffer::from([27_u8]);
+        let buf2 = buf.clone();
+        let data = ArrayData::builder(DataType::Boolean)
+            .len(5)
+            .offset(2)
+            .add_buffer(buf)
+            .build();
+        let arr = BooleanArray::from(data);
+        assert_eq!(buf2, arr.values());
+        assert_eq!(5, arr.len());
+        assert_eq!(2, arr.offset());
+        assert_eq!(0, arr.null_count());
+        for i in 0..3 {
+            assert_eq!(i != 0, arr.value(i), "failed at {}", i);
+        }
+    }
+
+    #[test]
+    #[should_panic(expected = "PrimitiveArray data should contain a single buffer only \
+                               (values buffer)")]
+    fn test_boolean_array_invalid_buffer_len() {
+        let data = ArrayData::builder(DataType::Boolean).len(5).build();
+        BooleanArray::from(data);
+    }
+
+    #[test]
+    fn test_access_array_concurrently() {
+        let a = Int32Array::from(vec![5, 6, 7, 8, 9]);
+        let ret = thread::spawn(move || a.value(3)).join();
+
+        assert!(ret.is_ok());
+        assert_eq!(8, ret.ok().unwrap());
+    }
+}
diff --git a/rust/arrow/src/array/array_string.rs b/rust/arrow/src/array/array_string.rs
new file mode 100644
index 00000000000..ec79ac9010d
--- /dev/null
+++ b/rust/arrow/src/array/array_string.rs
@@ -0,0 +1,445 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::convert::From;
+use std::fmt;
+use std::mem;
+use std::{any::Any, iter::FromIterator};
+
+use super::{
+    array::print_long_array, raw_pointer::as_aligned_pointer, raw_pointer::RawPtrBox,
+    Array, ArrayData, ArrayDataRef, GenericListArray, GenericStringIter, LargeListArray,
+    ListArray, OffsetSizeTrait,
+};
+use crate::util::bit_util;
+use crate::{buffer::Buffer, datatypes::ToByteSlice};
+use crate::{buffer::MutableBuffer, datatypes::DataType};
+
+/// Like OffsetSizeTrait, but specialized for Strings
+// This allow us to expose a constant datatype for the GenericStringArray
+pub trait StringOffsetSizeTrait: OffsetSizeTrait {
+    const DATA_TYPE: DataType;
+}
+
+impl StringOffsetSizeTrait for i32 {
+    const DATA_TYPE: DataType = DataType::Utf8;
+}
+
+impl StringOffsetSizeTrait for i64 {
+    const DATA_TYPE: DataType = DataType::LargeUtf8;
+}
+
+/// Generic struct for \[Large\]StringArray
+pub struct GenericStringArray<OffsetSize: StringOffsetSizeTrait> {
+    data: ArrayDataRef,
+    value_offsets: RawPtrBox<OffsetSize>,
+    value_data: RawPtrBox<u8>,
+}
+
+impl<OffsetSize: StringOffsetSizeTrait> GenericStringArray<OffsetSize> {
+    /// Returns the offset for the element at index `i`.
+    ///
+    /// Note this doesn't do any bound checking, for performance reason.
+    #[inline]
+    pub fn value_offset(&self, i: usize) -> OffsetSize {
+        self.value_offset_at(self.data.offset() + i)
+    }
+
+    /// Returns the length for the element at index `i`.
+    ///
+    /// Note this doesn't do any bound checking, for performance reason.
+    #[inline]
+    pub fn value_length(&self, mut i: usize) -> OffsetSize {
+        i += self.data.offset();
+        self.value_offset_at(i + 1) - self.value_offset_at(i)
+    }
+
+    /// Returns a clone of the value offset buffer
+    pub fn value_offsets(&self) -> Buffer {
+        self.data.buffers()[0].clone()
+    }
+
+    /// Returns a clone of the value data buffer
+    pub fn value_data(&self) -> Buffer {
+        self.data.buffers()[1].clone()
+    }
+
+    #[inline]
+    fn value_offset_at(&self, i: usize) -> OffsetSize {
+        unsafe { *self.value_offsets.get().add(i) }
+    }
+
+    /// Returns the element at index `i` as &str
+    pub fn value(&self, i: usize) -> &str {
+        assert!(i < self.data.len(), "StringArray out of bounds access");
+        let offset = i.checked_add(self.data.offset()).unwrap();
+        unsafe {
+            let pos = self.value_offset_at(offset);
+            let slice = std::slice::from_raw_parts(
+                self.value_data.get().offset(pos.to_isize()),
+                (self.value_offset_at(offset + 1) - pos).to_usize().unwrap(),
+            );
+
+            std::str::from_utf8_unchecked(slice)
+        }
+    }
+
+    fn from_list(v: GenericListArray<OffsetSize>) -> Self {
+        assert_eq!(
+            v.data().child_data()[0].child_data().len(),
+            0,
+            "StringArray can only be created from list array of u8 values \
+             (i.e. List<PrimitiveArray<u8>>)."
+        );
+        assert_eq!(
+            v.data_ref().child_data()[0].data_type(),
+            &DataType::UInt8,
+            "StringArray can only be created from List<u8> arrays, mismatched data types."
+        );
+
+        let mut builder = ArrayData::builder(OffsetSize::DATA_TYPE)
+            .len(v.len())
+            .add_buffer(v.data_ref().buffers()[0].clone())
+            .add_buffer(v.data_ref().child_data()[0].buffers()[0].clone());
+        if let Some(bitmap) = v.data().null_bitmap() {
+            builder = builder
+                .null_count(v.data_ref().null_count())
+                .null_bit_buffer(bitmap.bits.clone())
+        }
+
+        let data = builder.build();
+        Self::from(data)
+    }
+
+    pub(crate) fn from_vec(v: Vec<&str>) -> Self {
+        let mut offsets = Vec::with_capacity(v.len() + 1);
+        let mut values = Vec::new();
+        let mut length_so_far = OffsetSize::zero();
+        offsets.push(length_so_far);
+        for s in &v {
+            length_so_far = length_so_far + OffsetSize::from_usize(s.len()).unwrap();
+            offsets.push(length_so_far);
+            values.extend_from_slice(s.as_bytes());
+        }
+        let array_data = ArrayData::builder(OffsetSize::DATA_TYPE)
+            .len(v.len())
+            .add_buffer(Buffer::from(offsets.to_byte_slice()))
+            .add_buffer(Buffer::from(&values[..]))
+            .build();
+        Self::from(array_data)
+    }
+
+    pub(crate) fn from_opt_vec(v: Vec<Option<&str>>) -> Self {
+        GenericStringArray::from_iter(v.into_iter())
+    }
+}
+
+impl<'a, Ptr, OffsetSize: StringOffsetSizeTrait> FromIterator<Option<Ptr>>
+    for GenericStringArray<OffsetSize>
+where
+    Ptr: AsRef<str>,
+{
+    fn from_iter<I: IntoIterator<Item = Option<Ptr>>>(iter: I) -> Self {
+        let iter = iter.into_iter();
+        let (_, data_len) = iter.size_hint();
+        let data_len = data_len.expect("Iterator must be sized"); // panic if no upper bound.
+
+        let mut offsets = Vec::with_capacity(data_len + 1);
+        let mut values = Vec::new();
+        let mut null_buf = MutableBuffer::new_null(data_len);
+        let mut length_so_far = OffsetSize::zero();
+        offsets.push(length_so_far);
+
+        for (i, s) in iter.enumerate() {
+            if let Some(s) = s {
+                let s = s.as_ref();
+                // set null bit
+                let null_slice = null_buf.data_mut();
+                bit_util::set_bit(null_slice, i);
+
+                length_so_far = length_so_far + OffsetSize::from_usize(s.len()).unwrap();
+                offsets.push(length_so_far);
+                values.extend_from_slice(s.as_bytes());
+            } else {
+                offsets.push(length_so_far);
+                values.extend_from_slice(b"");
+            }
+        }
+
+        let array_data = ArrayData::builder(OffsetSize::DATA_TYPE)
+            .len(data_len)
+            .add_buffer(Buffer::from(offsets.to_byte_slice()))
+            .add_buffer(Buffer::from(&values[..]))
+            .null_bit_buffer(null_buf.freeze())
+            .build();
+        Self::from(array_data)
+    }
+}
+
+impl<'a, T: StringOffsetSizeTrait> IntoIterator for &'a GenericStringArray<T> {
+    type Item = Option<&'a str>;
+    type IntoIter = GenericStringIter<'a, T>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        GenericStringIter::<'a, T>::new(self)
+    }
+}
+
+impl<'a, T: StringOffsetSizeTrait> GenericStringArray<T> {
+    /// constructs a new iterator
+    pub fn iter(&'a self) -> GenericStringIter<'a, T> {
+        GenericStringIter::<'a, T>::new(&self)
+    }
+}
+
+impl<OffsetSize: StringOffsetSizeTrait> fmt::Debug for GenericStringArray<OffsetSize> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}StringArray\n[\n", OffsetSize::prefix())?;
+        print_long_array(self, f, |array, index, f| {
+            fmt::Debug::fmt(&array.value(index), f)
+        })?;
+        write!(f, "]")
+    }
+}
+
+impl<OffsetSize: StringOffsetSizeTrait> Array for GenericStringArray<OffsetSize> {
+    fn as_any(&self) -> &Any {
+        self
+    }
+
+    fn data(&self) -> ArrayDataRef {
+        self.data.clone()
+    }
+
+    fn data_ref(&self) -> &ArrayDataRef {
+        &self.data
+    }
+
+    /// Returns the total number of bytes of memory occupied by the buffers owned by this [$name].
+    fn get_buffer_memory_size(&self) -> usize {
+        self.data.get_buffer_memory_size()
+    }
+
+    /// Returns the total number of bytes of memory occupied physically by this [$name].
+    fn get_array_memory_size(&self) -> usize {
+        self.data.get_array_memory_size() + mem::size_of_val(self)
+    }
+}
+
+impl<OffsetSize: StringOffsetSizeTrait> From<ArrayDataRef>
+    for GenericStringArray<OffsetSize>
+{
+    fn from(data: ArrayDataRef) -> Self {
+        assert_eq!(
+            data.data_type(),
+            &<OffsetSize as StringOffsetSizeTrait>::DATA_TYPE,
+            "[Large]StringArray expects Datatype::[Large]Utf8"
+        );
+        assert_eq!(
+            data.buffers().len(),
+            2,
+            "StringArray data should contain 2 buffers only (offsets and values)"
+        );
+        let raw_value_offsets = data.buffers()[0].raw_data();
+        let value_data = data.buffers()[1].raw_data();
+        Self {
+            data,
+            value_offsets: RawPtrBox::new(as_aligned_pointer::<OffsetSize>(
+                raw_value_offsets,
+            )),
+            value_data: RawPtrBox::new(value_data),
+        }
+    }
+}
+
+/// An array where each element is a variable-sized sequence of bytes representing a string
+/// whose maximum length (in bytes) is represented by a i32.
+pub type StringArray = GenericStringArray<i32>;
+
+/// An array where each element is a variable-sized sequence of bytes representing a string
+/// whose maximum length (in bytes) is represented by a i64.
+pub type LargeStringArray = GenericStringArray<i64>;
+
+impl From<ListArray> for StringArray {
+    fn from(v: ListArray) -> Self {
+        StringArray::from_list(v)
+    }
+}
+
+impl From<LargeListArray> for LargeStringArray {
+    fn from(v: LargeListArray) -> Self {
+        LargeStringArray::from_list(v)
+    }
+}
+
+impl From<Vec<&str>> for StringArray {
+    fn from(v: Vec<&str>) -> Self {
+        StringArray::from_vec(v)
+    }
+}
+
+impl From<Vec<&str>> for LargeStringArray {
+    fn from(v: Vec<&str>) -> Self {
+        LargeStringArray::from_vec(v)
+    }
+}
+
+impl From<Vec<Option<&str>>> for StringArray {
+    fn from(v: Vec<Option<&str>>) -> Self {
+        StringArray::from_opt_vec(v)
+    }
+}
+
+impl From<Vec<Option<&str>>> for LargeStringArray {
+    fn from(v: Vec<Option<&str>>) -> Self {
+        LargeStringArray::from_opt_vec(v)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::array::{ListBuilder, StringBuilder};
+
+    use super::*;
+
+    #[test]
+    fn test_string_array_from_u8_slice() {
+        let values: Vec<&str> = vec!["hello", "", "parquet"];
+
+        // Array data: ["hello", "", "parquet"]
+        let string_array = StringArray::from(values);
+
+        assert_eq!(3, string_array.len());
+        assert_eq!(0, string_array.null_count());
+        assert_eq!("hello", string_array.value(0));
+        assert_eq!("", string_array.value(1));
+        assert_eq!("parquet", string_array.value(2));
+        assert_eq!(5, string_array.value_offset(2));
+        assert_eq!(7, string_array.value_length(2));
+        for i in 0..3 {
+            assert!(string_array.is_valid(i));
+            assert!(!string_array.is_null(i));
+        }
+    }
+
+    #[test]
+    #[should_panic(expected = "[Large]StringArray expects Datatype::[Large]Utf8")]
+    fn test_string_array_from_int() {
+        let array = LargeStringArray::from(vec!["a", "b"]);
+        StringArray::from(array.data());
+    }
+
+    #[test]
+    fn test_large_string_array_from_u8_slice() {
+        let values: Vec<&str> = vec!["hello", "", "parquet"];
+
+        // Array data: ["hello", "", "parquet"]
+        let string_array = LargeStringArray::from(values);
+
+        assert_eq!(3, string_array.len());
+        assert_eq!(0, string_array.null_count());
+        assert_eq!("hello", string_array.value(0));
+        assert_eq!("", string_array.value(1));
+        assert_eq!("parquet", string_array.value(2));
+        assert_eq!(5, string_array.value_offset(2));
+        assert_eq!(7, string_array.value_length(2));
+        for i in 0..3 {
+            assert!(string_array.is_valid(i));
+            assert!(!string_array.is_null(i));
+        }
+    }
+
+    #[test]
+    fn test_nested_string_array() {
+        let string_builder = StringBuilder::new(3);
+        let mut list_of_string_builder = ListBuilder::new(string_builder);
+
+        list_of_string_builder.values().append_value("foo").unwrap();
+        list_of_string_builder.values().append_value("bar").unwrap();
+        list_of_string_builder.append(true).unwrap();
+
+        list_of_string_builder
+            .values()
+            .append_value("foobar")
+            .unwrap();
+        list_of_string_builder.append(true).unwrap();
+        let list_of_strings = list_of_string_builder.finish();
+
+        assert_eq!(list_of_strings.len(), 2);
+
+        let first_slot = list_of_strings.value(0);
+        let first_list = first_slot.as_any().downcast_ref::<StringArray>().unwrap();
+        assert_eq!(first_list.len(), 2);
+        assert_eq!(first_list.value(0), "foo");
+        assert_eq!(first_list.value(1), "bar");
+
+        let second_slot = list_of_strings.value(1);
+        let second_list = second_slot.as_any().downcast_ref::<StringArray>().unwrap();
+        assert_eq!(second_list.len(), 1);
+        assert_eq!(second_list.value(0), "foobar");
+    }
+
+    #[test]
+    #[should_panic(expected = "StringArray out of bounds access")]
+    fn test_string_array_get_value_index_out_of_bound() {
+        let values: [u8; 12] = [
+            b'h', b'e', b'l', b'l', b'o', b'p', b'a', b'r', b'q', b'u', b'e', b't',
+        ];
+        let offsets: [i32; 4] = [0, 5, 5, 12];
+        let array_data = ArrayData::builder(DataType::Utf8)
+            .len(3)
+            .add_buffer(Buffer::from(offsets.to_byte_slice()))
+            .add_buffer(Buffer::from(&values[..]))
+            .build();
+        let string_array = StringArray::from(array_data);
+        string_array.value(4);
+    }
+
+    #[test]
+    fn test_string_array_fmt_debug() {
+        let arr: StringArray = vec!["hello", "arrow"].into();
+        assert_eq!(
+            "StringArray\n[\n  \"hello\",\n  \"arrow\",\n]",
+            format!("{:?}", arr)
+        );
+    }
+
+    #[test]
+    fn test_large_string_array_fmt_debug() {
+        let arr: LargeStringArray = vec!["hello", "arrow"].into();
+        assert_eq!(
+            "LargeStringArray\n[\n  \"hello\",\n  \"arrow\",\n]",
+            format!("{:?}", arr)
+        );
+    }
+
+    fn test_string_array_from_iter() {
+        let data = vec![Some("hello"), None, Some("arrow")];
+        // from Vec<Option<&str>>
+        let array1 = StringArray::from(data.clone());
+        // from Iterator<Option<&str>>
+        let array2: StringArray = data.clone().into_iter().collect();
+        // from Iterator<Option<String>>
+        let array3: StringArray = data
+            .into_iter()
+            .map(|x| x.map(|s| format!("{}", s)))
+            .collect();
+
+        assert_eq!(array1, array2);
+        assert_eq!(array2, array3);
+    }
+}
diff --git a/rust/arrow/src/array/array_struct.rs b/rust/arrow/src/array/array_struct.rs
new file mode 100644
index 00000000000..7f190b83008
--- /dev/null
+++ b/rust/arrow/src/array/array_struct.rs
@@ -0,0 +1,535 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::convert::{From, TryFrom};
+use std::fmt;
+use std::iter::IntoIterator;
+use std::mem;
+use std::{any::Any, sync::Arc};
+
+use super::{make_array, Array, ArrayData, ArrayDataRef, ArrayRef};
+use crate::datatypes::DataType;
+use crate::{
+    buffer::{buffer_bin_or, Buffer},
+    datatypes::Field,
+};
+use crate::{
+    error::{ArrowError, Result},
+    util::bit_util,
+};
+
+/// A nested array type where each child (called *field*) is represented by a separate
+/// array.
+pub struct StructArray {
+    data: ArrayDataRef,
+    pub(crate) boxed_fields: Vec<ArrayRef>,
+}
+
+impl StructArray {
+    /// Returns the field at `pos`.
+    pub fn column(&self, pos: usize) -> &ArrayRef {
+        &self.boxed_fields[pos]
+    }
+
+    /// Return the number of fields in this struct array
+    pub fn num_columns(&self) -> usize {
+        self.boxed_fields.len()
+    }
+
+    /// Returns the fields of the struct array
+    pub fn columns(&self) -> Vec<&ArrayRef> {
+        self.boxed_fields.iter().collect()
+    }
+
+    /// Returns child array refs of the struct array
+    pub fn columns_ref(&self) -> Vec<ArrayRef> {
+        self.boxed_fields.clone()
+    }
+
+    /// Return field names in this struct array
+    pub fn column_names(&self) -> Vec<&str> {
+        match self.data.data_type() {
+            DataType::Struct(fields) => fields
+                .iter()
+                .map(|f| f.name().as_str())
+                .collect::<Vec<&str>>(),
+            _ => unreachable!("Struct array's data type is not struct!"),
+        }
+    }
+
+    /// Return child array whose field name equals to column_name
+    pub fn column_by_name(&self, column_name: &str) -> Option<&ArrayRef> {
+        self.column_names()
+            .iter()
+            .position(|c| c == &column_name)
+            .map(|pos| self.column(pos))
+    }
+}
+
+impl From<ArrayDataRef> for StructArray {
+    fn from(data: ArrayDataRef) -> Self {
+        let mut boxed_fields = vec![];
+        for cd in data.child_data() {
+            let child_data = if data.offset() != 0 || data.len() != cd.len() {
+                Arc::new(cd.slice(data.offset(), data.len()))
+            } else {
+                cd.clone()
+            };
+            boxed_fields.push(make_array(child_data));
+        }
+        Self { data, boxed_fields }
+    }
+}
+
+impl TryFrom<Vec<(&str, ArrayRef)>> for StructArray {
+    type Error = ArrowError;
+
+    /// builds a StructArray from a vector of names and arrays.
+    /// This errors if the values have a different length.
+    /// An entry is set to Null when all values are null.
+    fn try_from(values: Vec<(&str, ArrayRef)>) -> Result<Self> {
+        let values_len = values.len();
+
+        // these will be populated
+        let mut fields = Vec::with_capacity(values_len);
+        let mut child_data = Vec::with_capacity(values_len);
+
+        // len: the size of the arrays.
+        let mut len: Option<usize> = None;
+        // null: the null mask of the arrays.
+        let mut null: Option<Buffer> = None;
+        for (field_name, array) in values {
+            let child_datum = array.data();
+            let child_datum_len = child_datum.len();
+            if let Some(len) = len {
+                if len != child_datum_len {
+                    return Err(ArrowError::InvalidArgumentError(
+                        format!("Array of field \"{}\" has length {}, but previous elements have length {}.
+                        All arrays in every entry in a struct array must have the same length.", field_name, child_datum_len, len)
+                    ));
+                }
+            } else {
+                len = Some(child_datum_len)
+            }
+            child_data.push(child_datum.clone());
+            fields.push(Field::new(
+                field_name,
+                array.data_type().clone(),
+                child_datum.null_buffer().is_some(),
+            ));
+
+            if let Some(child_null_buffer) = child_datum.null_buffer() {
+                null = Some(if let Some(null_buffer) = &null {
+                    buffer_bin_or(null_buffer, 0, child_null_buffer, 0, child_datum_len)
+                } else {
+                    child_null_buffer.clone()
+                });
+            } else if null.is_some() {
+                // when one of the fields has no nulls, them there is no null in the array
+                null = None;
+            }
+        }
+        let len = len.unwrap();
+
+        let mut builder = ArrayData::builder(DataType::Struct(fields))
+            .len(len)
+            .child_data(child_data);
+        if let Some(null_buffer) = null {
+            let null_count = len - bit_util::count_set_bits(null_buffer.data());
+            builder = builder.null_count(null_count).null_bit_buffer(null_buffer);
+        }
+
+        Ok(StructArray::from(builder.build()))
+    }
+}
+
+impl Array for StructArray {
+    fn as_any(&self) -> &Any {
+        self
+    }
+
+    fn data(&self) -> ArrayDataRef {
+        self.data.clone()
+    }
+
+    fn data_ref(&self) -> &ArrayDataRef {
+        &self.data
+    }
+
+    /// Returns the length (i.e., number of elements) of this array
+    fn len(&self) -> usize {
+        self.data_ref().len()
+    }
+
+    /// Returns the total number of bytes of memory occupied by the buffers owned by this [StructArray].
+    fn get_buffer_memory_size(&self) -> usize {
+        self.data.get_buffer_memory_size()
+    }
+
+    /// Returns the total number of bytes of memory occupied physically by this [StructArray].
+    fn get_array_memory_size(&self) -> usize {
+        self.data.get_array_memory_size() + mem::size_of_val(self)
+    }
+}
+
+impl From<Vec<(Field, ArrayRef)>> for StructArray {
+    fn from(v: Vec<(Field, ArrayRef)>) -> Self {
+        let (field_types, field_values): (Vec<_>, Vec<_>) = v.into_iter().unzip();
+
+        // Check the length of the child arrays
+        let length = field_values[0].len();
+        for i in 1..field_values.len() {
+            assert_eq!(
+                length,
+                field_values[i].len(),
+                "all child arrays of a StructArray must have the same length"
+            );
+            assert_eq!(
+                field_types[i].data_type(),
+                field_values[i].data().data_type(),
+                "the field data types must match the array data in a StructArray"
+            )
+        }
+
+        let data = ArrayData::builder(DataType::Struct(field_types))
+            .child_data(field_values.into_iter().map(|a| a.data()).collect())
+            .len(length)
+            .build();
+        Self::from(data)
+    }
+}
+
+impl fmt::Debug for StructArray {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "StructArray\n[\n")?;
+        for (child_index, name) in self.column_names().iter().enumerate() {
+            let column = self.column(child_index);
+            writeln!(
+                f,
+                "-- child {}: \"{}\" ({:?})",
+                child_index,
+                name,
+                column.data_type()
+            )?;
+            fmt::Debug::fmt(column, f)?;
+            writeln!(f)?;
+        }
+        write!(f, "]")
+    }
+}
+
+impl From<(Vec<(Field, ArrayRef)>, Buffer, usize)> for StructArray {
+    fn from(triple: (Vec<(Field, ArrayRef)>, Buffer, usize)) -> Self {
+        let (field_types, field_values): (Vec<_>, Vec<_>) = triple.0.into_iter().unzip();
+
+        // Check the length of the child arrays
+        let length = field_values[0].len();
+        for i in 1..field_values.len() {
+            assert_eq!(
+                length,
+                field_values[i].len(),
+                "all child arrays of a StructArray must have the same length"
+            );
+            assert_eq!(
+                field_types[i].data_type(),
+                field_values[i].data().data_type(),
+                "the field data types must match the array data in a StructArray"
+            )
+        }
+
+        let data = ArrayData::builder(DataType::Struct(field_types))
+            .null_bit_buffer(triple.1)
+            .child_data(field_values.into_iter().map(|a| a.data()).collect())
+            .len(length)
+            .null_count(triple.2)
+            .build();
+        Self::from(data)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use std::sync::Arc;
+
+    use crate::datatypes::{DataType, Field};
+    use crate::{
+        array::BooleanArray, array::Float32Array, array::Float64Array, array::Int32Array,
+        array::StringArray, bitmap::Bitmap,
+    };
+    use crate::{buffer::Buffer, datatypes::ToByteSlice};
+
+    #[test]
+    fn test_struct_array_builder() {
+        let boolean_data = ArrayData::builder(DataType::Boolean)
+            .len(4)
+            .add_buffer(Buffer::from([false, false, true, true].to_byte_slice()))
+            .build();
+        let int_data = ArrayData::builder(DataType::Int64)
+            .len(4)
+            .add_buffer(Buffer::from([42, 28, 19, 31].to_byte_slice()))
+            .build();
+        let mut field_types = vec![];
+        field_types.push(Field::new("a", DataType::Boolean, false));
+        field_types.push(Field::new("b", DataType::Int64, false));
+        let struct_array_data = ArrayData::builder(DataType::Struct(field_types))
+            .len(4)
+            .add_child_data(boolean_data.clone())
+            .add_child_data(int_data.clone())
+            .build();
+        let struct_array = StructArray::from(struct_array_data);
+
+        assert_eq!(boolean_data, struct_array.column(0).data());
+        assert_eq!(int_data, struct_array.column(1).data());
+    }
+
+    #[test]
+    fn test_struct_array_from() {
+        let boolean_data = ArrayData::builder(DataType::Boolean)
+            .len(4)
+            .add_buffer(Buffer::from([12_u8]))
+            .build();
+        let int_data = ArrayData::builder(DataType::Int32)
+            .len(4)
+            .add_buffer(Buffer::from([42, 28, 19, 31].to_byte_slice()))
+            .build();
+        let struct_array = StructArray::from(vec![
+            (
+                Field::new("b", DataType::Boolean, false),
+                Arc::new(BooleanArray::from(vec![false, false, true, true]))
+                    as Arc<Array>,
+            ),
+            (
+                Field::new("c", DataType::Int32, false),
+                Arc::new(Int32Array::from(vec![42, 28, 19, 31])),
+            ),
+        ]);
+        assert_eq!(boolean_data, struct_array.column(0).data());
+        assert_eq!(int_data, struct_array.column(1).data());
+        assert_eq!(4, struct_array.len());
+        assert_eq!(0, struct_array.null_count());
+        assert_eq!(0, struct_array.offset());
+    }
+
+    /// validates that the in-memory representation follows [the spec](https://arrow.apache.org/docs/format/Columnar.html#struct-layout)
+    #[test]
+    fn test_struct_array_from_vec() {
+        let strings: ArrayRef = Arc::new(StringArray::from(vec![
+            Some("joe"),
+            None,
+            None,
+            Some("mark"),
+        ]));
+        let ints: ArrayRef =
+            Arc::new(Int32Array::from(vec![Some(1), Some(2), None, Some(4)]));
+
+        let arr =
+            StructArray::try_from(vec![("f1", strings.clone()), ("f2", ints.clone())])
+                .unwrap();
+
+        let struct_data = arr.data();
+        assert_eq!(4, struct_data.len());
+        assert_eq!(1, struct_data.null_count());
+        assert_eq!(
+            // 00001011
+            &Some(Bitmap::from(Buffer::from(&[11_u8]))),
+            struct_data.null_bitmap()
+        );
+
+        let expected_string_data = ArrayData::builder(DataType::Utf8)
+            .len(4)
+            .null_count(2)
+            .null_bit_buffer(Buffer::from(&[9_u8]))
+            .add_buffer(Buffer::from(&[0, 3, 3, 3, 7].to_byte_slice()))
+            .add_buffer(Buffer::from("joemark".as_bytes()))
+            .build();
+
+        let expected_int_data = ArrayData::builder(DataType::Int32)
+            .len(4)
+            .null_count(1)
+            .null_bit_buffer(Buffer::from(&[11_u8]))
+            .add_buffer(Buffer::from(&[1, 2, 0, 4].to_byte_slice()))
+            .build();
+
+        assert_eq!(expected_string_data, arr.column(0).data());
+
+        // TODO: implement equality for ArrayData
+        assert_eq!(expected_int_data.len(), arr.column(1).data().len());
+        assert_eq!(
+            expected_int_data.null_count(),
+            arr.column(1).data().null_count()
+        );
+        assert_eq!(
+            expected_int_data.null_bitmap(),
+            arr.column(1).data().null_bitmap()
+        );
+        let expected_value_buf = expected_int_data.buffers()[0].clone();
+        let actual_value_buf = arr.column(1).data().buffers()[0].clone();
+        for i in 0..expected_int_data.len() {
+            if !expected_int_data.is_null(i) {
+                assert_eq!(
+                    expected_value_buf.data()[i * 4..(i + 1) * 4],
+                    actual_value_buf.data()[i * 4..(i + 1) * 4]
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_struct_array_from_vec_error() {
+        let strings: ArrayRef = Arc::new(StringArray::from(vec![
+            Some("joe"),
+            None,
+            None,
+            // 3 elements, not 4
+        ]));
+        let ints: ArrayRef =
+            Arc::new(Int32Array::from(vec![Some(1), Some(2), None, Some(4)]));
+
+        let arr =
+            StructArray::try_from(vec![("f1", strings.clone()), ("f2", ints.clone())]);
+
+        match arr {
+            Err(ArrowError::InvalidArgumentError(e)) => {
+                assert!(e.starts_with("Array of field \"f2\" has length 4, but previous elements have length 3."));
+            }
+            _ => assert!(false, "This test got an unexpected error type"),
+        };
+    }
+
+    #[test]
+    #[should_panic(
+        expected = "the field data types must match the array data in a StructArray"
+    )]
+    fn test_struct_array_from_mismatched_types() {
+        StructArray::from(vec![
+            (
+                Field::new("b", DataType::Int16, false),
+                Arc::new(BooleanArray::from(vec![false, false, true, true]))
+                    as Arc<Array>,
+            ),
+            (
+                Field::new("c", DataType::Utf8, false),
+                Arc::new(Int32Array::from(vec![42, 28, 19, 31])),
+            ),
+        ]);
+    }
+
+    #[test]
+    fn test_struct_array_slice() {
+        let boolean_data = ArrayData::builder(DataType::Boolean)
+            .len(5)
+            .add_buffer(Buffer::from([0b00010000]))
+            .null_bit_buffer(Buffer::from([0b00010001]))
+            .build();
+        let int_data = ArrayData::builder(DataType::Int32)
+            .len(5)
+            .add_buffer(Buffer::from([0, 28, 42, 0, 0].to_byte_slice()))
+            .null_bit_buffer(Buffer::from([0b00000110]))
+            .build();
+
+        let mut field_types = vec![];
+        field_types.push(Field::new("a", DataType::Boolean, false));
+        field_types.push(Field::new("b", DataType::Int32, false));
+        let struct_array_data = ArrayData::builder(DataType::Struct(field_types))
+            .len(5)
+            .add_child_data(boolean_data.clone())
+            .add_child_data(int_data.clone())
+            .null_bit_buffer(Buffer::from([0b00010111]))
+            .build();
+        let struct_array = StructArray::from(struct_array_data);
+
+        assert_eq!(5, struct_array.len());
+        assert_eq!(1, struct_array.null_count());
+        assert!(struct_array.is_valid(0));
+        assert!(struct_array.is_valid(1));
+        assert!(struct_array.is_valid(2));
+        assert!(struct_array.is_null(3));
+        assert!(struct_array.is_valid(4));
+        assert_eq!(boolean_data, struct_array.column(0).data());
+        assert_eq!(int_data, struct_array.column(1).data());
+
+        let c0 = struct_array.column(0);
+        let c0 = c0.as_any().downcast_ref::<BooleanArray>().unwrap();
+        assert_eq!(5, c0.len());
+        assert_eq!(3, c0.null_count());
+        assert!(c0.is_valid(0));
+        assert_eq!(false, c0.value(0));
+        assert!(c0.is_null(1));
+        assert!(c0.is_null(2));
+        assert!(c0.is_null(3));
+        assert!(c0.is_valid(4));
+        assert_eq!(true, c0.value(4));
+
+        let c1 = struct_array.column(1);
+        let c1 = c1.as_any().downcast_ref::<Int32Array>().unwrap();
+        assert_eq!(5, c1.len());
+        assert_eq!(3, c1.null_count());
+        assert!(c1.is_null(0));
+        assert!(c1.is_valid(1));
+        assert_eq!(28, c1.value(1));
+        assert!(c1.is_valid(2));
+        assert_eq!(42, c1.value(2));
+        assert!(c1.is_null(3));
+        assert!(c1.is_null(4));
+
+        let sliced_array = struct_array.slice(2, 3);
+        let sliced_array = sliced_array.as_any().downcast_ref::<StructArray>().unwrap();
+        assert_eq!(3, sliced_array.len());
+        assert_eq!(2, sliced_array.offset());
+        assert_eq!(1, sliced_array.null_count());
+        assert!(sliced_array.is_valid(0));
+        assert!(sliced_array.is_null(1));
+        assert!(sliced_array.is_valid(2));
+
+        let sliced_c0 = sliced_array.column(0);
+        let sliced_c0 = sliced_c0.as_any().downcast_ref::<BooleanArray>().unwrap();
+        assert_eq!(3, sliced_c0.len());
+        assert_eq!(2, sliced_c0.offset());
+        assert!(sliced_c0.is_null(0));
+        assert!(sliced_c0.is_null(1));
+        assert!(sliced_c0.is_valid(2));
+        assert_eq!(true, sliced_c0.value(2));
+
+        let sliced_c1 = sliced_array.column(1);
+        let sliced_c1 = sliced_c1.as_any().downcast_ref::<Int32Array>().unwrap();
+        assert_eq!(3, sliced_c1.len());
+        assert_eq!(2, sliced_c1.offset());
+        assert!(sliced_c1.is_valid(0));
+        assert_eq!(42, sliced_c1.value(0));
+        assert!(sliced_c1.is_null(1));
+        assert!(sliced_c1.is_null(2));
+    }
+
+    #[test]
+    #[should_panic(
+        expected = "all child arrays of a StructArray must have the same length"
+    )]
+    fn test_invalid_struct_child_array_lengths() {
+        StructArray::from(vec![
+            (
+                Field::new("b", DataType::Float32, false),
+                Arc::new(Float32Array::from(vec![1.1])) as Arc<Array>,
+            ),
+            (
+                Field::new("c", DataType::Float64, false),
+                Arc::new(Float64Array::from(vec![2.2, 3.3])),
+            ),
+        ]);
+    }
+}
diff --git a/rust/arrow/src/array/builder.rs b/rust/arrow/src/array/builder.rs
index 272d87fa870..f1037c735fb 100644
--- a/rust/arrow/src/array/builder.rs
+++ b/rust/arrow/src/array/builder.rs
@@ -253,8 +253,18 @@ pub trait BufferBuilderTrait<T: ArrowPrimitiveType> {
 }
 
 impl<T: ArrowPrimitiveType> BufferBuilderTrait<T> for BufferBuilder<T> {
-    default fn new(capacity: usize) -> Self {
-        let buffer = MutableBuffer::new(capacity * mem::size_of::<T::Native>());
+    #[inline]
+    fn new(capacity: usize) -> Self {
+        let buffer = if T::DATA_TYPE == DataType::Boolean {
+            let byte_capacity = bit_util::ceil(capacity, 8);
+            let actual_capacity = bit_util::round_upto_multiple_of_64(byte_capacity);
+            let mut buffer = MutableBuffer::new(actual_capacity);
+            buffer.set_null_bits(0, actual_capacity);
+            buffer
+        } else {
+            MutableBuffer::new(capacity * mem::size_of::<T::Native>())
+        };
+
         Self {
             buffer,
             len: 0,
@@ -275,43 +285,112 @@ impl<T: ArrowPrimitiveType> BufferBuilderTrait<T> for BufferBuilder<T> {
         bit_capacity / T::get_bit_width()
     }
 
-    default fn advance(&mut self, i: usize) -> Result<()> {
-        let new_buffer_len = (self.len + i) * mem::size_of::<T::Native>();
+    #[inline]
+    fn advance(&mut self, i: usize) -> Result<()> {
+        let new_buffer_len = if T::DATA_TYPE == DataType::Boolean {
+            bit_util::ceil(self.len + i, 8)
+        } else {
+            (self.len + i) * mem::size_of::<T::Native>()
+        };
         self.buffer.resize(new_buffer_len)?;
         self.len += i;
         Ok(())
     }
 
-    default fn reserve(&mut self, n: usize) -> Result<()> {
+    #[inline]
+    fn reserve(&mut self, n: usize) -> Result<()> {
         let new_capacity = self.len + n;
-        let byte_capacity = mem::size_of::<T::Native>() * new_capacity;
-        self.buffer.reserve(byte_capacity)?;
+        if T::DATA_TYPE == DataType::Boolean {
+            if new_capacity > self.capacity() {
+                let new_byte_capacity = bit_util::ceil(new_capacity, 8);
+                let existing_capacity = self.buffer.capacity();
+                let new_capacity = self.buffer.reserve(new_byte_capacity)?;
+                self.buffer
+                    .set_null_bits(existing_capacity, new_capacity - existing_capacity);
+            }
+        } else {
+            let byte_capacity = mem::size_of::<T::Native>() * new_capacity;
+            self.buffer.reserve(byte_capacity)?;
+        }
         Ok(())
     }
 
-    default fn append(&mut self, v: T::Native) -> Result<()> {
+    #[inline]
+    fn append(&mut self, v: T::Native) -> Result<()> {
         self.reserve(1)?;
-        self.write_bytes(v.to_byte_slice(), 1)
+        if T::DATA_TYPE == DataType::Boolean {
+            if v != T::default_value() {
+                unsafe {
+                    bit_util::set_bit_raw(self.buffer.raw_data_mut(), self.len);
+                }
+            }
+            self.len += 1;
+        } else {
+            self.write_bytes(v.to_byte_slice(), 1)?;
+        }
+        Ok(())
     }
 
-    default fn append_n(&mut self, n: usize, v: T::Native) -> Result<()> {
+    #[inline]
+    fn append_n(&mut self, n: usize, v: T::Native) -> Result<()> {
         self.reserve(n)?;
-        for _ in 0..n {
-            self.write_bytes(v.to_byte_slice(), 1)?;
+        if T::DATA_TYPE == DataType::Boolean {
+            if n != 0 && v != T::default_value() {
+                unsafe {
+                    bit_util::set_bits_raw(
+                        self.buffer.raw_data_mut(),
+                        self.len,
+                        self.len + n,
+                    )
+                }
+            }
+            self.len += n;
+        } else {
+            for _ in 0..n {
+                self.write_bytes(v.to_byte_slice(), 1)?;
+            }
         }
         Ok(())
     }
 
-    default fn append_slice(&mut self, slice: &[T::Native]) -> Result<()> {
+    #[inline]
+    fn append_slice(&mut self, slice: &[T::Native]) -> Result<()> {
         let array_slots = slice.len();
         self.reserve(array_slots)?;
-        self.write_bytes(slice.to_byte_slice(), array_slots)
+
+        if T::DATA_TYPE == DataType::Boolean {
+            for v in slice {
+                if *v != T::default_value() {
+                    // For performance the `len` of the buffer is not
+                    // updated on each append but is updated in the
+                    // `freeze` method instead.
+                    unsafe {
+                        bit_util::set_bit_raw(self.buffer.raw_data_mut(), self.len);
+                    }
+                }
+                self.len += 1;
+            }
+            Ok(())
+        } else {
+            self.write_bytes(slice.to_byte_slice(), array_slots)
+        }
     }
 
-    default fn finish(&mut self) -> Buffer {
-        let buf = std::mem::replace(&mut self.buffer, MutableBuffer::new(0));
-        self.len = 0;
-        buf.freeze()
+    #[inline]
+    fn finish(&mut self) -> Buffer {
+        if T::DATA_TYPE == DataType::Boolean {
+            // `append` does not update the buffer's `len` so do it before `freeze` is called.
+            let new_buffer_len = bit_util::ceil(self.len, 8);
+            debug_assert!(new_buffer_len >= self.buffer.len());
+            let mut buf = std::mem::replace(&mut self.buffer, MutableBuffer::new(0));
+            self.len = 0;
+            buf.resize(new_buffer_len).unwrap();
+            buf.freeze()
+        } else {
+            let buf = std::mem::replace(&mut self.buffer, MutableBuffer::new(0));
+            self.len = 0;
+            buf.freeze()
+        }
     }
 }
 
@@ -334,89 +413,6 @@ impl<T: ArrowPrimitiveType> BufferBuilder<T> {
     }
 }
 
-impl BufferBuilderTrait<BooleanType> for BufferBuilder<BooleanType> {
-    fn new(capacity: usize) -> Self {
-        let byte_capacity = bit_util::ceil(capacity, 8);
-        let actual_capacity = bit_util::round_upto_multiple_of_64(byte_capacity);
-        let mut buffer = MutableBuffer::new(actual_capacity);
-        buffer.set_null_bits(0, actual_capacity);
-        Self {
-            buffer,
-            len: 0,
-            _marker: PhantomData,
-        }
-    }
-
-    fn advance(&mut self, i: usize) -> Result<()> {
-        let new_buffer_len = bit_util::ceil(self.len + i, 8);
-        self.buffer.resize(new_buffer_len)?;
-        self.len += i;
-        Ok(())
-    }
-
-    fn append(&mut self, v: bool) -> Result<()> {
-        self.reserve(1)?;
-        if v {
-            // For performance the `len` of the buffer is not updated on each append but
-            // is updated in the `freeze` method instead.
-            unsafe {
-                bit_util::set_bit_raw(self.buffer.raw_data_mut(), self.len);
-            }
-        }
-        self.len += 1;
-        Ok(())
-    }
-
-    fn append_n(&mut self, n: usize, v: bool) -> Result<()> {
-        self.reserve(n)?;
-        if n != 0 && v {
-            unsafe {
-                bit_util::set_bits_raw(self.buffer.raw_data_mut(), self.len, self.len + n)
-            }
-        }
-        self.len += n;
-        Ok(())
-    }
-
-    fn append_slice(&mut self, slice: &[bool]) -> Result<()> {
-        self.reserve(slice.len())?;
-        for v in slice {
-            if *v {
-                // For performance the `len` of the buffer is not
-                // updated on each append but is updated in the
-                // `freeze` method instead.
-                unsafe {
-                    bit_util::set_bit_raw(self.buffer.raw_data_mut(), self.len);
-                }
-            }
-            self.len += 1;
-        }
-        Ok(())
-    }
-
-    fn reserve(&mut self, n: usize) -> Result<()> {
-        let new_capacity = self.len + n;
-        if new_capacity > self.capacity() {
-            let new_byte_capacity = bit_util::ceil(new_capacity, 8);
-            let existing_capacity = self.buffer.capacity();
-            let new_capacity = self.buffer.reserve(new_byte_capacity)?;
-            self.buffer
-                .set_null_bits(existing_capacity, new_capacity - existing_capacity);
-        }
-        Ok(())
-    }
-
-    fn finish(&mut self) -> Buffer {
-        // `append` does not update the buffer's `len` so do it before `freeze` is called.
-        let new_buffer_len = bit_util::ceil(self.len, 8);
-        debug_assert!(new_buffer_len >= self.buffer.len());
-        let mut buf = std::mem::replace(&mut self.buffer, MutableBuffer::new(0));
-        self.len = 0;
-        buf.resize(new_buffer_len).unwrap();
-        buf.freeze()
-    }
-}
-
 /// Trait for dealing with different array builders at runtime
 pub trait ArrayBuilder: Any {
     /// Returns the number of array slots in the builder
@@ -535,7 +531,7 @@ impl<T: ArrowPrimitiveType> ArrayBuilder for PrimitiveBuilder<T> {
 
             for i in 0..len {
                 // account for offset as `ArrayData` does not
-                self.bitmap_builder.append(array.is_valid(offset + i))?;
+                self.bitmap_builder.append(array.is_valid(i))?;
             }
         }
         Ok(())
@@ -545,7 +541,7 @@ impl<T: ArrowPrimitiveType> ArrayBuilder for PrimitiveBuilder<T> {
     ///
     /// This is used for validating array data types in `append_data`
     fn data_type(&self) -> DataType {
-        T::get_data_type()
+        T::DATA_TYPE
     }
 
     /// Builds the array and reset this builder.
@@ -618,7 +614,7 @@ impl<T: ArrowPrimitiveType> PrimitiveBuilder<T> {
         let len = self.len();
         let null_bit_buffer = self.bitmap_builder.finish();
         let null_count = len - bit_util::count_set_bits(null_bit_buffer.data());
-        let mut builder = ArrayData::builder(T::get_data_type())
+        let mut builder = ArrayData::builder(T::DATA_TYPE)
             .len(len)
             .add_buffer(self.values_builder.finish());
         if null_count > 0 {
@@ -636,7 +632,7 @@ impl<T: ArrowPrimitiveType> PrimitiveBuilder<T> {
         let null_bit_buffer = self.bitmap_builder.finish();
         let null_count = len - bit_util::count_set_bits(null_bit_buffer.data());
         let data_type = DataType::Dictionary(
-            Box::new(T::get_data_type()),
+            Box::new(T::DATA_TYPE),
             Box::new(values.data_type().clone()),
         );
         let mut builder = ArrayData::builder(data_type)
@@ -765,8 +761,7 @@ where
                 .append_slice(adjusted_offsets.as_slice())?;
 
             for i in 0..len {
-                // account for offset as `ArrayData` does not
-                self.bitmap_builder.append(array.is_valid(offset + i))?;
+                self.bitmap_builder.append(array.is_valid(i))?;
             }
         }
 
@@ -779,7 +774,11 @@ where
     ///
     /// This is used for validating array data types in `append_data`
     fn data_type(&self) -> DataType {
-        DataType::List(Box::new(self.values_builder.data_type()))
+        DataType::List(Box::new(Field::new(
+            "item",
+            self.values_builder.data_type(),
+            true,
+        )))
     }
 
     /// Returns the builder as a mutable `Any` reference.
@@ -843,15 +842,19 @@ where
 
         let offset_buffer = self.offsets_builder.finish();
         let null_bit_buffer = self.bitmap_builder.finish();
+        let nulls = bit_util::count_set_bits(null_bit_buffer.data());
         self.offsets_builder.append(0).unwrap();
-        let data =
-            ArrayData::builder(DataType::List(Box::new(values_data.data_type().clone())))
-                .len(len)
-                .null_count(len - bit_util::count_set_bits(null_bit_buffer.data()))
-                .add_buffer(offset_buffer)
-                .add_child_data(values_data)
-                .null_bit_buffer(null_bit_buffer)
-                .build();
+        let data = ArrayData::builder(DataType::List(Box::new(Field::new(
+            "item",
+            values_data.data_type().clone(),
+            true, // TODO: find a consistent way of getting this
+        ))))
+        .len(len)
+        .null_count(len - nulls)
+        .add_buffer(offset_buffer)
+        .add_child_data(values_data)
+        .null_bit_buffer(null_bit_buffer)
+        .build();
 
         ListArray::from(data)
     }
@@ -970,8 +973,7 @@ where
                 .append_slice(adjusted_offsets.as_slice())?;
 
             for i in 0..len {
-                // account for offset as `ArrayData` does not
-                self.bitmap_builder.append(array.is_valid(offset + i))?;
+                self.bitmap_builder.append(array.is_valid(i))?;
             }
         }
 
@@ -984,7 +986,11 @@ where
     ///
     /// This is used for validating array data types in `append_data`
     fn data_type(&self) -> DataType {
-        DataType::LargeList(Box::new(self.values_builder.data_type()))
+        DataType::LargeList(Box::new(Field::new(
+            "item",
+            self.values_builder.data_type(),
+            true,
+        )))
     }
 
     /// Returns the builder as a mutable `Any` reference.
@@ -1048,12 +1054,15 @@ where
 
         let offset_buffer = self.offsets_builder.finish();
         let null_bit_buffer = self.bitmap_builder.finish();
+        let nulls = bit_util::count_set_bits(null_bit_buffer.data());
         self.offsets_builder.append(0).unwrap();
-        let data = ArrayData::builder(DataType::LargeList(Box::new(
+        let data = ArrayData::builder(DataType::LargeList(Box::new(Field::new(
+            "item",
             values_data.data_type().clone(),
-        )))
+            true,
+        ))))
         .len(len)
-        .null_count(len - bit_util::count_set_bits(null_bit_buffer.data()))
+        .null_count(len - nulls)
         .add_buffer(offset_buffer)
         .add_child_data(values_data)
         .null_bit_buffer(null_bit_buffer)
@@ -1145,8 +1154,7 @@ where
             let sliced = child_array.slice(first_offset, offset_at_len - first_offset);
             self.values().append_data(&[sliced.data()])?;
             for i in 0..len {
-                // account for offset as `ArrayData` does not
-                self.bitmap_builder.append(array.is_valid(offset + i))?;
+                self.bitmap_builder.append(array.is_valid(i))?;
             }
         }
 
@@ -1159,7 +1167,10 @@ where
     ///
     /// This is used for validating array data types in `append_data`
     fn data_type(&self) -> DataType {
-        DataType::FixedSizeList(Box::new(self.values_builder.data_type()), self.list_len)
+        DataType::FixedSizeList(
+            Box::new(Field::new("item", self.values_builder.data_type(), true)),
+            self.list_len,
+        )
     }
 
     /// Returns the builder as a mutable `Any` reference.
@@ -1234,12 +1245,13 @@ where
         }
 
         let null_bit_buffer = self.bitmap_builder.finish();
+        let nulls = bit_util::count_set_bits(null_bit_buffer.data());
         let data = ArrayData::builder(DataType::FixedSizeList(
-            Box::new(values_data.data_type().clone()),
+            Box::new(Field::new("item", values_data.data_type().clone(), true)),
             self.list_len,
         ))
         .len(len)
-        .null_count(len - bit_util::count_set_bits(null_bit_buffer.data()))
+        .null_count(len - nulls)
         .add_child_data(values_data)
         .null_bit_buffer(null_bit_buffer)
         .build();
@@ -1274,14 +1286,6 @@ pub struct FixedSizeBinaryBuilder {
     builder: FixedSizeListBuilder<UInt8Builder>,
 }
 
-pub trait BinaryArrayBuilder: ArrayBuilder {}
-
-impl BinaryArrayBuilder for BinaryBuilder {}
-impl BinaryArrayBuilder for StringBuilder {}
-impl BinaryArrayBuilder for LargeStringBuilder {}
-impl BinaryArrayBuilder for LargeBinaryBuilder {}
-impl BinaryArrayBuilder for FixedSizeBinaryBuilder {}
-
 impl ArrayBuilder for BinaryBuilder {
     /// Returns the builder as a non-mutable `Any` reference.
     fn as_any(&self) -> &Any {
@@ -1457,7 +1461,7 @@ fn append_binary_data(
                 )) as ArrayDataRef;
 
                 Arc::new(ArrayData::new(
-                    DataType::List(Box::new(DataType::UInt8)),
+                    DataType::List(Box::new(Field::new("item", DataType::UInt8, true))),
                     array.len(),
                     None,
                     array.null_buffer().cloned(),
@@ -1509,7 +1513,11 @@ fn append_large_binary_data(
                 )) as ArrayDataRef;
 
                 Arc::new(ArrayData::new(
-                    DataType::LargeList(Box::new(DataType::UInt8)),
+                    DataType::LargeList(Box::new(Field::new(
+                        "item",
+                        DataType::UInt8,
+                        true,
+                    ))),
                     array.len(),
                     None,
                     array.null_buffer().cloned(),
@@ -1607,7 +1615,10 @@ impl ArrayBuilder for FixedSizeBinaryBuilder {
                 vec![],
             )) as ArrayDataRef;
             let list_data = Arc::new(ArrayData::new(
-                DataType::FixedSizeList(Box::new(DataType::UInt8), self.builder.list_len),
+                DataType::FixedSizeList(
+                    Box::new(Field::new("item", DataType::UInt8, true)),
+                    self.builder.list_len,
+                ),
                 array.len(),
                 None,
                 array.null_buffer().cloned(),
@@ -1949,8 +1960,7 @@ impl ArrayBuilder for StructBuilder {
                 builder.append_data(&[sliced.data()])?;
             }
             for i in 0..len {
-                // account for offset as `ArrayData` does not
-                self.bitmap_builder.append(array.is_valid(offset + i))?;
+                self.bitmap_builder.append(array.is_valid(i))?;
             }
         }
 
@@ -1994,6 +2004,79 @@ impl ArrayBuilder for StructBuilder {
     }
 }
 
+/// Returns a builder with capacity `capacity` that corresponds to the datatype `DataType`
+/// This function is useful to construct arrays from an arbitrary vectors with known/expected
+/// schema.
+pub fn make_builder(datatype: &DataType, capacity: usize) -> Box<ArrayBuilder> {
+    match datatype {
+        DataType::Null => unimplemented!(),
+        DataType::Boolean => Box::new(BooleanBuilder::new(capacity)),
+        DataType::Int8 => Box::new(Int8Builder::new(capacity)),
+        DataType::Int16 => Box::new(Int16Builder::new(capacity)),
+        DataType::Int32 => Box::new(Int32Builder::new(capacity)),
+        DataType::Int64 => Box::new(Int64Builder::new(capacity)),
+        DataType::UInt8 => Box::new(UInt8Builder::new(capacity)),
+        DataType::UInt16 => Box::new(UInt16Builder::new(capacity)),
+        DataType::UInt32 => Box::new(UInt32Builder::new(capacity)),
+        DataType::UInt64 => Box::new(UInt64Builder::new(capacity)),
+        DataType::Float32 => Box::new(Float32Builder::new(capacity)),
+        DataType::Float64 => Box::new(Float64Builder::new(capacity)),
+        DataType::Binary => Box::new(BinaryBuilder::new(capacity)),
+        DataType::FixedSizeBinary(len) => {
+            Box::new(FixedSizeBinaryBuilder::new(capacity, *len))
+        }
+        DataType::Utf8 => Box::new(StringBuilder::new(capacity)),
+        DataType::Date32(DateUnit::Day) => Box::new(Date32Builder::new(capacity)),
+        DataType::Date64(DateUnit::Millisecond) => Box::new(Date64Builder::new(capacity)),
+        DataType::Time32(TimeUnit::Second) => {
+            Box::new(Time32SecondBuilder::new(capacity))
+        }
+        DataType::Time32(TimeUnit::Millisecond) => {
+            Box::new(Time32MillisecondBuilder::new(capacity))
+        }
+        DataType::Time64(TimeUnit::Microsecond) => {
+            Box::new(Time64MicrosecondBuilder::new(capacity))
+        }
+        DataType::Time64(TimeUnit::Nanosecond) => {
+            Box::new(Time64NanosecondBuilder::new(capacity))
+        }
+        DataType::Timestamp(TimeUnit::Second, _) => {
+            Box::new(TimestampSecondBuilder::new(capacity))
+        }
+        DataType::Timestamp(TimeUnit::Millisecond, _) => {
+            Box::new(TimestampMillisecondBuilder::new(capacity))
+        }
+        DataType::Timestamp(TimeUnit::Microsecond, _) => {
+            Box::new(TimestampMicrosecondBuilder::new(capacity))
+        }
+        DataType::Timestamp(TimeUnit::Nanosecond, _) => {
+            Box::new(TimestampNanosecondBuilder::new(capacity))
+        }
+        DataType::Interval(IntervalUnit::YearMonth) => {
+            Box::new(IntervalYearMonthBuilder::new(capacity))
+        }
+        DataType::Interval(IntervalUnit::DayTime) => {
+            Box::new(IntervalDayTimeBuilder::new(capacity))
+        }
+        DataType::Duration(TimeUnit::Second) => {
+            Box::new(DurationSecondBuilder::new(capacity))
+        }
+        DataType::Duration(TimeUnit::Millisecond) => {
+            Box::new(DurationMillisecondBuilder::new(capacity))
+        }
+        DataType::Duration(TimeUnit::Microsecond) => {
+            Box::new(DurationMicrosecondBuilder::new(capacity))
+        }
+        DataType::Duration(TimeUnit::Nanosecond) => {
+            Box::new(DurationNanosecondBuilder::new(capacity))
+        }
+        DataType::Struct(fields) => {
+            Box::new(StructBuilder::from_fields(fields.clone(), capacity))
+        }
+        t => panic!("Data type {:?} is not currently supported", t),
+    }
+}
+
 impl StructBuilder {
     pub fn new(fields: Vec<Field>, builders: Vec<Box<ArrayBuilder>>) -> Self {
         let mut field_anys = Vec::with_capacity(builders.len());
@@ -2020,86 +2103,12 @@ impl StructBuilder {
         }
     }
 
-    pub fn from_schema(schema: Schema, capacity: usize) -> Self {
-        let fields = schema.fields();
+    pub fn from_fields(fields: Vec<Field>, capacity: usize) -> Self {
         let mut builders = Vec::with_capacity(fields.len());
-        for f in schema.fields() {
-            builders.push(Self::from_field(f.clone(), capacity));
-        }
-        Self::new(schema.fields, builders)
-    }
-
-    fn from_field(f: Field, capacity: usize) -> Box<ArrayBuilder> {
-        match f.data_type() {
-            DataType::Null => unimplemented!(),
-            DataType::Boolean => Box::new(BooleanBuilder::new(capacity)),
-            DataType::Int8 => Box::new(Int8Builder::new(capacity)),
-            DataType::Int16 => Box::new(Int16Builder::new(capacity)),
-            DataType::Int32 => Box::new(Int32Builder::new(capacity)),
-            DataType::Int64 => Box::new(Int64Builder::new(capacity)),
-            DataType::UInt8 => Box::new(UInt8Builder::new(capacity)),
-            DataType::UInt16 => Box::new(UInt16Builder::new(capacity)),
-            DataType::UInt32 => Box::new(UInt32Builder::new(capacity)),
-            DataType::UInt64 => Box::new(UInt64Builder::new(capacity)),
-            DataType::Float32 => Box::new(Float32Builder::new(capacity)),
-            DataType::Float64 => Box::new(Float64Builder::new(capacity)),
-            DataType::Binary => Box::new(BinaryBuilder::new(capacity)),
-            DataType::FixedSizeBinary(len) => {
-                Box::new(FixedSizeBinaryBuilder::new(capacity, *len))
-            }
-            DataType::Utf8 => Box::new(StringBuilder::new(capacity)),
-            DataType::Date32(DateUnit::Day) => Box::new(Date32Builder::new(capacity)),
-            DataType::Date64(DateUnit::Millisecond) => {
-                Box::new(Date64Builder::new(capacity))
-            }
-            DataType::Time32(TimeUnit::Second) => {
-                Box::new(Time32SecondBuilder::new(capacity))
-            }
-            DataType::Time32(TimeUnit::Millisecond) => {
-                Box::new(Time32MillisecondBuilder::new(capacity))
-            }
-            DataType::Time64(TimeUnit::Microsecond) => {
-                Box::new(Time64MicrosecondBuilder::new(capacity))
-            }
-            DataType::Time64(TimeUnit::Nanosecond) => {
-                Box::new(Time64NanosecondBuilder::new(capacity))
-            }
-            DataType::Timestamp(TimeUnit::Second, _) => {
-                Box::new(TimestampSecondBuilder::new(capacity))
-            }
-            DataType::Timestamp(TimeUnit::Millisecond, _) => {
-                Box::new(TimestampMillisecondBuilder::new(capacity))
-            }
-            DataType::Timestamp(TimeUnit::Microsecond, _) => {
-                Box::new(TimestampMicrosecondBuilder::new(capacity))
-            }
-            DataType::Timestamp(TimeUnit::Nanosecond, _) => {
-                Box::new(TimestampNanosecondBuilder::new(capacity))
-            }
-            DataType::Interval(IntervalUnit::YearMonth) => {
-                Box::new(IntervalYearMonthBuilder::new(capacity))
-            }
-            DataType::Interval(IntervalUnit::DayTime) => {
-                Box::new(IntervalDayTimeBuilder::new(capacity))
-            }
-            DataType::Duration(TimeUnit::Second) => {
-                Box::new(DurationSecondBuilder::new(capacity))
-            }
-            DataType::Duration(TimeUnit::Millisecond) => {
-                Box::new(DurationMillisecondBuilder::new(capacity))
-            }
-            DataType::Duration(TimeUnit::Microsecond) => {
-                Box::new(DurationMicrosecondBuilder::new(capacity))
-            }
-            DataType::Duration(TimeUnit::Nanosecond) => {
-                Box::new(DurationNanosecondBuilder::new(capacity))
-            }
-            DataType::Struct(fields) => {
-                let schema = Schema::new(fields.clone());
-                Box::new(Self::from_schema(schema, capacity))
-            }
-            t => panic!("Data type {:?} is not currently supported", t),
+        for field in &fields {
+            builders.push(make_builder(field.data_type(), capacity));
         }
+        Self::new(fields, builders)
     }
 
     /// Returns a mutable reference to the child field builder at index `i`.
@@ -2234,7 +2243,7 @@ where
     ///
     /// This is used for validating array data types in `append_data`
     fn data_type(&self) -> DataType {
-        DataType::Dictionary(Box::new(K::get_data_type()), Box::new(V::get_data_type()))
+        DataType::Dictionary(Box::new(K::DATA_TYPE), Box::new(V::DATA_TYPE))
     }
 
     /// Builds the array and reset this builder.
@@ -2313,10 +2322,10 @@ where
     ///
     /// ```
     /// use arrow::datatypes::Int16Type;
-    /// use arrow::array::{StringArray, StringDictionaryBuilder, PrimitiveBuilder};
+    /// use arrow::array::{StringArray, StringDictionaryBuilder, PrimitiveBuilder, Int16Array};
     /// use std::convert::TryFrom;
     ///
-    /// let dictionary_values = StringArray::try_from(vec![None, Some("abc"), Some("def")]).unwrap();
+    /// let dictionary_values = StringArray::from(vec![None, Some("abc"), Some("def")]);
     ///
     /// let mut builder = StringDictionaryBuilder::new_with_dictionary(PrimitiveBuilder::<Int16Type>::new(3), &dictionary_values).unwrap();
     /// builder.append("def").unwrap();
@@ -2325,9 +2334,9 @@ where
     ///
     /// let dictionary_array = builder.finish();
     ///
-    /// let keys: Vec<Option<i16>> = dictionary_array.keys().collect();
+    /// let keys = dictionary_array.keys();
     ///
-    /// assert_eq!(keys, vec![Some(2), None, Some(1)]);
+    /// assert_eq!(keys, &Int16Array::from(vec![Some(2), None, Some(1)]));
     /// ```
     pub fn new_with_dictionary(
         keys_builder: PrimitiveBuilder<K>,
@@ -2399,7 +2408,7 @@ where
     ///
     /// This is used for validating array data types in `append_data`
     fn data_type(&self) -> DataType {
-        DataType::Dictionary(Box::new(K::get_data_type()), Box::new(DataType::Utf8))
+        DataType::Dictionary(Box::new(K::DATA_TYPE), Box::new(DataType::Utf8))
     }
 
     /// Builds the array and reset this builder.
@@ -2449,7 +2458,6 @@ mod tests {
 
     use crate::array::Array;
     use crate::bitmap::Bitmap;
-    use std::convert::TryFrom;
 
     #[test]
     fn test_builder_i32_empty() {
@@ -2532,8 +2540,8 @@ mod tests {
     #[test]
     fn test_append_slice() {
         let mut b = UInt8BufferBuilder::new(0);
-        b.append_slice("Hello, ".as_bytes()).unwrap();
-        b.append_slice("World!".as_bytes()).unwrap();
+        b.append_slice(b"Hello, ").unwrap();
+        b.append_slice(b"World!").unwrap();
         let buffer = b.finish();
         assert_eq!(13, buffer.len());
 
@@ -3100,9 +3108,7 @@ mod tests {
         builder.append_byte(b'd').unwrap();
         builder.append(true).unwrap();
 
-        let array = builder.finish();
-
-        let binary_array = BinaryArray::from(array);
+        let binary_array = builder.finish();
 
         assert_eq!(3, binary_array.len());
         assert_eq!(0, binary_array.null_count());
@@ -3131,9 +3137,7 @@ mod tests {
         builder.append_byte(b'd').unwrap();
         builder.append(true).unwrap();
 
-        let array = builder.finish();
-
-        let binary_array = LargeBinaryArray::from(array);
+        let binary_array = builder.finish();
 
         assert_eq!(3, binary_array.len());
         assert_eq!(0, binary_array.null_count());
@@ -3152,9 +3156,7 @@ mod tests {
         builder.append(true).unwrap();
         builder.append_value("world").unwrap();
 
-        let array = builder.finish();
-
-        let string_array = StringArray::from(array);
+        let string_array = builder.finish();
 
         assert_eq!(3, string_array.len());
         assert_eq!(0, string_array.null_count());
@@ -3211,9 +3213,7 @@ mod tests {
         builder.append(true).unwrap();
         builder.append_value("world").unwrap();
 
-        let array = builder.finish();
-
-        let string_array = StringArray::from(array);
+        let string_array = builder.finish();
 
         assert_eq!(3, string_array.len());
         assert_eq!(0, string_array.null_count());
@@ -3275,7 +3275,7 @@ mod tests {
             .null_count(2)
             .null_bit_buffer(Buffer::from(&[9_u8]))
             .add_buffer(Buffer::from(&[0, 3, 3, 3, 7].to_byte_slice()))
-            .add_buffer(Buffer::from("joemark".as_bytes()))
+            .add_buffer(Buffer::from(b"joemark"))
             .build();
 
         let expected_int_data = ArrayData::builder(DataType::Int32)
@@ -3382,7 +3382,7 @@ mod tests {
         let struct_type = DataType::Struct(sub_fields);
         fields.push(Field::new("f3", struct_type, false));
 
-        let mut builder = StructBuilder::from_schema(Schema::new(fields), 5);
+        let mut builder = StructBuilder::from_fields(fields, 5);
         assert_eq!(3, builder.num_fields());
         assert!(builder.field_builder::<Float32Builder>(0).is_some());
         assert!(builder.field_builder::<StringBuilder>(1).is_some());
@@ -3390,14 +3390,17 @@ mod tests {
     }
 
     #[test]
-    #[should_panic(expected = "Data type List(Int64) is not currently supported")]
+    #[should_panic(
+        expected = "Data type List(Field { name: \"item\", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false }) is not currently supported"
+    )]
     fn test_struct_array_builder_from_schema_unsupported_type() {
         let mut fields = Vec::new();
         fields.push(Field::new("f1", DataType::Int16, false));
-        let list_type = DataType::List(Box::new(DataType::Int64));
+        let list_type =
+            DataType::List(Box::new(Field::new("item", DataType::Int64, true)));
         fields.push(Field::new("f2", list_type, false));
 
-        let _ = StructBuilder::from_schema(Schema::new(fields), 5);
+        let _ = StructBuilder::from_fields(fields, 5);
     }
 
     #[test]
@@ -3423,8 +3426,10 @@ mod tests {
         builder.append(22345678).unwrap();
         let array = builder.finish();
 
-        // Keys are strongly typed.
-        let aks: Vec<_> = array.keys().collect();
+        assert_eq!(
+            array.keys(),
+            &UInt8Array::from(vec![Some(0), None, Some(1)])
+        );
 
         // Values are polymorphic and so require a downcast.
         let av = array.values();
@@ -3435,7 +3440,6 @@ mod tests {
         assert_eq!(array.is_null(1), true);
         assert_eq!(array.is_null(2), false);
 
-        assert_eq!(aks, vec![Some(0), None, Some(1)]);
         assert_eq!(avs, &[12345678, 22345678]);
     }
 
@@ -3451,22 +3455,22 @@ mod tests {
         builder.append("abc").unwrap();
         let array = builder.finish();
 
-        // Keys are strongly typed.
-        let aks: Vec<_> = array.keys().collect();
+        assert_eq!(
+            array.keys(),
+            &Int8Array::from(vec![Some(0), None, Some(1), Some(1), Some(0)])
+        );
 
         // Values are polymorphic and so require a downcast.
         let av = array.values();
         let ava: &StringArray = av.as_any().downcast_ref::<StringArray>().unwrap();
 
-        assert_eq!(aks, vec![Some(0), None, Some(1), Some(1), Some(0)]);
         assert_eq!(ava.value(0), "abc");
         assert_eq!(ava.value(1), "def");
     }
 
     #[test]
     fn test_string_dictionary_builder_with_existing_dictionary() {
-        let dictionary =
-            StringArray::try_from(vec![None, Some("def"), Some("abc")]).unwrap();
+        let dictionary = StringArray::from(vec![None, Some("def"), Some("abc")]);
 
         let key_builder = PrimitiveBuilder::<Int8Type>::new(6);
         let mut builder =
@@ -3480,14 +3484,15 @@ mod tests {
         builder.append("ghi").unwrap();
         let array = builder.finish();
 
-        // Keys are strongly typed.
-        let aks: Vec<_> = array.keys().collect();
+        assert_eq!(
+            array.keys(),
+            &Int8Array::from(vec![Some(2), None, Some(1), Some(1), Some(2), Some(3)])
+        );
 
         // Values are polymorphic and so require a downcast.
         let av = array.values();
         let ava: &StringArray = av.as_any().downcast_ref::<StringArray>().unwrap();
 
-        assert_eq!(aks, vec![Some(2), None, Some(1), Some(1), Some(2), Some(3)]);
         assert_eq!(ava.is_valid(0), false);
         assert_eq!(ava.value(1), "def");
         assert_eq!(ava.value(2), "abc");
@@ -3496,7 +3501,8 @@ mod tests {
 
     #[test]
     fn test_string_dictionary_builder_with_reserved_null_value() {
-        let dictionary = StringArray::try_from(vec![None]).unwrap();
+        let dictionary: Vec<Option<&str>> = vec![None];
+        let dictionary = StringArray::from(dictionary);
 
         let key_builder = PrimitiveBuilder::<Int16Type>::new(4);
         let mut builder =
@@ -3511,7 +3517,7 @@ mod tests {
         assert_eq!(array.is_null(1), true);
         assert_eq!(array.is_valid(1), false);
 
-        let keys: Int16Array = array.data().into();
+        let keys = array.keys_array();
 
         assert_eq!(keys.value(0), 1);
         assert_eq!(keys.is_null(1), true);
@@ -3548,7 +3554,7 @@ mod tests {
             array.slice(2, 0).data(),
         ])?;
         let finished = builder.finish();
-        let expected = Arc::new(Int32Array::from(vec![
+        let expected = Int32Array::from(vec![
             None,
             Some(1),
             None,
@@ -3557,14 +3563,15 @@ mod tests {
             None,
             Some(6),
             Some(7),
+            // array.data() end
             Some(3),
             None,
             None,
             Some(6),
-        ])) as ArrayRef;
+        ]);
         assert_eq!(finished.len(), expected.len());
         assert_eq!(finished.null_count(), expected.null_count());
-        assert!(finished.equals(&(*expected)));
+        assert_eq!(finished, expected);
 
         let mut builder = Float64Builder::new(64);
         builder.append_null()?;
@@ -3578,7 +3585,7 @@ mod tests {
             array.slice(2, 1).data(),
         ])?;
         let finished = builder.finish();
-        let expected = Arc::new(Float64Array::from(vec![
+        let expected = Float64Array::from(vec![
             None,
             Some(1.0),
             None,
@@ -3593,10 +3600,10 @@ mod tests {
             Some(6.0),
             Some(7.0),
             None,
-        ])) as ArrayRef;
+        ]);
         assert_eq!(finished.len(), expected.len());
         assert_eq!(finished.null_count(), expected.null_count());
-        assert!(finished.equals(&(*expected)));
+        assert_eq!(finished, expected);
         Ok(())
     }
 
@@ -3620,7 +3627,7 @@ mod tests {
             array.slice(2, 0).data(),
         ])?;
         let finished = builder.finish();
-        let expected = Arc::new(BooleanArray::from(vec![
+        let expected = BooleanArray::from(vec![
             None,
             Some(true),
             None,
@@ -3633,10 +3640,10 @@ mod tests {
             None,
             None,
             Some(false),
-        ])) as ArrayRef;
+        ]);
         assert_eq!(finished.len(), expected.len());
         assert_eq!(finished.null_count(), expected.null_count());
-        assert!(finished.equals(&(*expected)));
+        assert_eq!(finished, expected);
         Ok(())
     }
 
@@ -3689,7 +3696,7 @@ mod tests {
         let list_value_offsets =
             Buffer::from(&[0, 3, 5, 11, 13, 13, 15, 15, 17].to_byte_slice());
         let expected_list_data = ArrayData::new(
-            DataType::List(Box::new(DataType::Int64)),
+            DataType::List(Box::new(Field::new("item", DataType::Int64, true))),
             8,
             None,
             None,
@@ -3702,7 +3709,7 @@ mod tests {
             finished.data().buffers()[0].data(),
             expected_list.data().buffers()[0].data()
         );
-        assert!(expected_list.values().equals(&*finished.values()));
+        assert_eq!(&expected_list.values(), &finished.values());
         assert_eq!(expected_list.len(), finished.len());
 
         Ok(())
@@ -3775,7 +3782,7 @@ mod tests {
             &[0, 3, 5, 5, 13, 15, 15, 15, 19, 19, 19, 19, 23].to_byte_slice(),
         );
         let expected_list_data = ArrayData::new(
-            DataType::List(Box::new(DataType::Int64)),
+            DataType::List(Box::new(Field::new("item", DataType::Int64, true))),
             12,
             None,
             None,
@@ -3792,7 +3799,7 @@ mod tests {
             finished.data().child_data()[0].buffers()[0].data(),
             expected_list.data().child_data()[0].buffers()[0].data()
         );
-        assert!(expected_list.values().equals(&*finished.values()));
+        assert_eq!(&expected_list.values(), &finished.values());
         assert_eq!(expected_list.len(), finished.len());
 
         Ok(())
@@ -3807,17 +3814,17 @@ mod tests {
         builder.append(true)?;
         builder.append(false)?;
 
-        let string_array = StringArray::try_from(vec![
+        let string_array = StringArray::from(vec![
             Some("alpha"),
             Some("beta"),
             None,
             Some("gamma"),
             Some("delta"),
             None,
-        ])?;
+        ]);
         let list_value_offsets = Buffer::from(&[0, 2, 3, 6].to_byte_slice());
         let list_data = ArrayData::new(
-            DataType::List(Box::new(DataType::Utf8)),
+            DataType::List(Box::new(Field::new("item", DataType::Utf8, true))),
             3,
             None,
             None,
@@ -3833,7 +3840,7 @@ mod tests {
         ])?;
         let finished = builder.finish();
 
-        let expected_string_array = StringArray::try_from(vec![
+        let expected_string_array = StringArray::from(vec![
             Some("Hello"),
             Some("Arrow"),
             // list_array
@@ -3849,10 +3856,10 @@ mod tests {
             Some("delta"),
             None,
             // slice(0, 0) returns nothing
-        ])?;
+        ]);
         let list_value_offsets = Buffer::from(&[0, 2, 2, 4, 5, 8, 9, 12].to_byte_slice());
         let expected_list_data = ArrayData::new(
-            DataType::List(Box::new(DataType::Utf8)),
+            DataType::List(Box::new(Field::new("item", DataType::Utf8, true))),
             7,
             None,
             None, // is this correct?
@@ -3869,7 +3876,7 @@ mod tests {
             finished.data().child_data()[0].buffers()[0].data(),
             expected_list.data().child_data()[0].buffers()[0].data()
         );
-        assert!(expected_list.values().equals(&*finished.values()));
+        assert_eq!(&expected_list.values(), &finished.values());
         assert_eq!(expected_list.len(), finished.len());
 
         Ok(())
@@ -3940,7 +3947,10 @@ mod tests {
             Some(12),
         ]);
         let expected_list_data = ArrayData::new(
-            DataType::FixedSizeList(Box::new(DataType::UInt16), 2),
+            DataType::FixedSizeList(
+                Box::new(Field::new("item", DataType::UInt16, true)),
+                2,
+            ),
             12,
             None,
             None,
@@ -3950,7 +3960,7 @@ mod tests {
         );
         let expected_list =
             FixedSizeListArray::from(Arc::new(expected_list_data) as ArrayDataRef);
-        assert!(expected_list.values().equals(&*finished.values()));
+        assert_eq!(&expected_list.values(), &finished.values());
         assert_eq!(expected_list.len(), finished.len());
 
         Ok(())
@@ -4010,7 +4020,10 @@ mod tests {
             None,
         ]);
         let expected_list_data = ArrayData::new(
-            DataType::FixedSizeList(Box::new(DataType::UInt8), 2),
+            DataType::FixedSizeList(
+                Box::new(Field::new("item", DataType::UInt8, true)),
+                2,
+            ),
             12,
             None,
             None,
@@ -4021,7 +4034,7 @@ mod tests {
         let expected_list =
             FixedSizeListArray::from(Arc::new(expected_list_data) as ArrayDataRef);
         let expected_list = FixedSizeBinaryArray::from(expected_list);
-        // assert!(expected_list.values().equals(&*finished.values()));
+        // assert_eq!(expected_list.values(), finished.values());
         assert_eq!(expected_list.len(), finished.len());
 
         Ok(())
@@ -4095,10 +4108,10 @@ mod tests {
             true, true, true, false, true, false, true, false, true, false, true, false,
             true, false,
         ])) as ArrayRef;
-        let expected = Arc::new(StructArray::from(vec![(field1, f1), (field2, f2)]));
+        let expected = StructArray::from(vec![(field1, f1), (field2, f2)]);
         assert_eq!(arr2.data().child_data()[0], expected.data().child_data()[0]);
         assert_eq!(arr2.data().child_data()[1], expected.data().child_data()[1]);
-        assert!(arr2.equals(&*expected));
+        assert_eq!(arr2, expected);
 
         Ok(())
     }
diff --git a/rust/arrow/src/array/cast.rs b/rust/arrow/src/array/cast.rs
index 78ddec4f54f..56e5d3a64e7 100644
--- a/rust/arrow/src/array/cast.rs
+++ b/rust/arrow/src/array/cast.rs
@@ -30,6 +30,16 @@ where
         .expect("Unable to downcast to primitive array")
 }
 
+/// Force downcast ArrayRef to DictionaryArray<T>
+pub fn as_dictionary_array<T>(arr: &ArrayRef) -> &DictionaryArray<T>
+where
+    T: ArrowDictionaryKeyType,
+{
+    arr.as_any()
+        .downcast_ref::<DictionaryArray<T>>()
+        .expect("Unable to downcast to dictionary array")
+}
+
 macro_rules! array_downcast_fn {
     ($name: ident, $arrty: ty, $arrty_str:expr) => {
         #[doc = "Force downcast ArrayRef to "]
diff --git a/rust/arrow/src/array/data.rs b/rust/arrow/src/array/data.rs
index f1e32c57d98..e328c01ee7a 100644
--- a/rust/arrow/src/array/data.rs
+++ b/rust/arrow/src/array/data.rs
@@ -21,27 +21,37 @@
 use std::mem;
 use std::sync::Arc;
 
-use crate::bitmap::Bitmap;
 use crate::buffer::Buffer;
 use crate::datatypes::DataType;
 use crate::util::bit_util;
+use crate::{bitmap::Bitmap, datatypes::ArrowNativeType};
+
+#[inline]
+fn count_nulls(null_bit_buffer: Option<&Buffer>, offset: usize, len: usize) -> usize {
+    if let Some(ref buf) = null_bit_buffer {
+        len.checked_sub(bit_util::count_set_bits_offset(buf.data(), offset, len))
+            .unwrap()
+    } else {
+        0
+    }
+}
 
 /// An generic representation of Arrow array data which encapsulates common attributes and
 /// operations for Arrow array. Specific operations for different arrays types (e.g.,
 /// primitive, list, struct) are implemented in `Array`.
-#[derive(PartialEq, Debug, Clone)]
+#[derive(Debug, Clone)]
 pub struct ArrayData {
     /// The data type for this array data
     data_type: DataType,
 
     /// The number of elements in this array data
-    pub(crate) len: usize,
+    len: usize,
 
     /// The number of null elements in this array data
-    pub(crate) null_count: usize,
+    null_count: usize,
 
-    /// The offset into this array data
-    pub(crate) offset: usize,
+    /// The offset into this array data, in number of items
+    offset: usize,
 
     /// The buffers for this array data. Note that depending on the array types, this
     /// could hold different kinds of buffers (e.g., value buffer, value offset buffer)
@@ -70,18 +80,7 @@ impl ArrayData {
         child_data: Vec<ArrayDataRef>,
     ) -> Self {
         let null_count = match null_count {
-            None => {
-                if let Some(ref buf) = null_bit_buffer {
-                    len.checked_sub(bit_util::count_set_bits_offset(
-                        buf.data(),
-                        offset,
-                        len,
-                    ))
-                    .unwrap()
-                } else {
-                    0
-                }
-            }
+            None => count_nulls(null_bit_buffer.as_ref(), offset, len),
             Some(null_count) => null_count,
         };
         let null_bitmap = null_bit_buffer.map(Bitmap::from);
@@ -97,12 +96,14 @@ impl ArrayData {
     }
 
     /// Returns a builder to construct a `ArrayData` instance.
-    pub fn builder(data_type: DataType) -> ArrayDataBuilder {
+    #[inline]
+    pub const fn builder(data_type: DataType) -> ArrayDataBuilder {
         ArrayDataBuilder::new(data_type)
     }
 
     /// Returns a reference to the data type of this array data
-    pub fn data_type(&self) -> &DataType {
+    #[inline]
+    pub const fn data_type(&self) -> &DataType {
         &self.data_type
     }
 
@@ -119,13 +120,14 @@ impl ArrayData {
     /// Returns whether the element at index `i` is null
     pub fn is_null(&self, i: usize) -> bool {
         if let Some(ref b) = self.null_bitmap {
-            return !b.is_set(i);
+            return !b.is_set(self.offset + i);
         }
         false
     }
 
     /// Returns a reference to the null bitmap of this array data
-    pub fn null_bitmap(&self) -> &Option<Bitmap> {
+    #[inline]
+    pub const fn null_bitmap(&self) -> &Option<Bitmap> {
         &self.null_bitmap
     }
 
@@ -137,28 +139,32 @@ impl ArrayData {
     /// Returns whether the element at index `i` is not null
     pub fn is_valid(&self, i: usize) -> bool {
         if let Some(ref b) = self.null_bitmap {
-            return b.is_set(i);
+            return b.is_set(self.offset + i);
         }
         true
     }
 
     /// Returns the length (i.e., number of elements) of this array
-    pub fn len(&self) -> usize {
+    #[inline]
+    pub const fn len(&self) -> usize {
         self.len
     }
 
-    pub fn is_empty(&self) -> bool {
+    // Returns whether array data is empty
+    #[inline]
+    pub const fn is_empty(&self) -> bool {
         self.len == 0
     }
 
     /// Returns the offset of this array
     #[inline]
-    pub fn offset(&self) -> usize {
+    pub const fn offset(&self) -> usize {
         self.offset
     }
 
     /// Returns the total number of nulls in this array
-    pub fn null_count(&self) -> usize {
+    #[inline]
+    pub const fn null_count(&self) -> usize {
         self.null_count
     }
 
@@ -200,6 +206,96 @@ impl ArrayData {
 
         size
     }
+
+    /// Creates a zero-copy slice of itself. This creates a new [ArrayData]
+    /// with a different offset, len and a shifted null bitmap.
+    ///
+    /// # Panics
+    ///
+    /// Panics if `offset + length > self.len()`.
+    pub fn slice(&self, offset: usize, length: usize) -> ArrayData {
+        assert!((offset + length) <= self.len());
+
+        let mut new_data = self.clone();
+
+        new_data.len = length;
+        new_data.offset = offset + self.offset;
+
+        new_data.null_count =
+            count_nulls(new_data.null_buffer(), new_data.offset, new_data.len);
+
+        new_data
+    }
+
+    /// Returns the `buffer` as a slice of type `T` starting at self.offset
+    /// # Panics
+    /// This function panics if:
+    /// * the buffer is not byte-aligned with type T, or
+    /// * the datatype is `Boolean` (it corresponds to a bit-packed buffer where the offset is not applicable)
+    #[inline]
+    pub(super) fn buffer<T: ArrowNativeType>(&self, buffer: usize) -> &[T] {
+        let values = unsafe { self.buffers[buffer].data().align_to::<T>() };
+        if values.0.len() != 0 || values.2.len() != 0 {
+            panic!("The buffer is not byte-aligned with its interpretation")
+        };
+        assert_ne!(self.data_type, DataType::Boolean);
+        &values.1[self.offset..]
+    }
+}
+
+impl PartialEq for ArrayData {
+    fn eq(&self, other: &Self) -> bool {
+        assert_eq!(
+            self.data_type(),
+            other.data_type(),
+            "Data types not the same"
+        );
+        assert_eq!(self.len(), other.len(), "Lengths not the same");
+        // TODO: when adding tests for this, test that we can compare with arrays that have offsets
+        assert_eq!(self.offset(), other.offset(), "Offsets not the same");
+        assert_eq!(self.null_count(), other.null_count());
+        // compare buffers excluding padding
+        let self_buffers = self.buffers();
+        let other_buffers = other.buffers();
+        assert_eq!(self_buffers.len(), other_buffers.len());
+        self_buffers.iter().zip(other_buffers).for_each(|(s, o)| {
+            compare_buffer_regions(
+                s,
+                self.offset(), // TODO mul by data length
+                o,
+                other.offset(), // TODO mul by data len
+            );
+        });
+        // assert_eq!(self.buffers(), other.buffers());
+
+        assert_eq!(self.child_data(), other.child_data());
+        // null arrays can skip the null bitmap, thus only compare if there are no nulls
+        if self.null_count() != 0 || other.null_count() != 0 {
+            compare_buffer_regions(
+                self.null_buffer().unwrap(),
+                self.offset(),
+                other.null_buffer().unwrap(),
+                other.offset(),
+            )
+        }
+        true
+    }
+}
+
+/// A helper to compare buffer regions of 2 buffers.
+/// Compares the length of the shorter buffer.
+fn compare_buffer_regions(
+    left: &Buffer,
+    left_offset: usize,
+    right: &Buffer,
+    right_offset: usize,
+) {
+    // for convenience, we assume that the buffer lengths are only unequal if one has padding,
+    // so we take the shorter length so we can discard the padding from the longer length
+    let shorter_len = left.len().min(right.len());
+    let s_sliced = left.bit_slice(left_offset, shorter_len);
+    let o_sliced = right.bit_slice(right_offset, shorter_len);
+    assert_eq!(s_sliced, o_sliced);
 }
 
 /// Builder for `ArrayData` type
@@ -215,7 +311,8 @@ pub struct ArrayDataBuilder {
 }
 
 impl ArrayDataBuilder {
-    pub fn new(data_type: DataType) -> Self {
+    #[inline]
+    pub const fn new(data_type: DataType) -> Self {
         Self {
             data_type,
             len: 0,
@@ -227,12 +324,14 @@ impl ArrayDataBuilder {
         }
     }
 
-    pub fn len(mut self, n: usize) -> Self {
+    #[inline]
+    pub const fn len(mut self, n: usize) -> Self {
         self.len = n;
         self
     }
 
-    pub fn null_count(mut self, n: usize) -> Self {
+    #[inline]
+    pub const fn null_count(mut self, n: usize) -> Self {
         self.null_count = Some(n);
         self
     }
@@ -242,7 +341,8 @@ impl ArrayDataBuilder {
         self
     }
 
-    pub fn offset(mut self, n: usize) -> Self {
+    #[inline]
+    pub const fn offset(mut self, n: usize) -> Self {
         self.offset = n;
         self
     }
@@ -369,4 +469,27 @@ mod tests {
         assert!(arr_data.null_buffer().is_some());
         assert_eq!(&bit_v, arr_data.null_buffer().unwrap().data());
     }
+
+    #[test]
+    fn test_slice() {
+        let mut bit_v: [u8; 2] = [0; 2];
+        bit_util::set_bit(&mut bit_v, 0);
+        bit_util::set_bit(&mut bit_v, 3);
+        bit_util::set_bit(&mut bit_v, 10);
+        let data = ArrayData::builder(DataType::Int32)
+            .len(16)
+            .null_bit_buffer(Buffer::from(bit_v))
+            .build();
+        let data = data.as_ref();
+        let new_data = data.slice(1, 15);
+        assert_eq!(data.len() - 1, new_data.len());
+        assert_eq!(1, new_data.offset());
+        assert_eq!(data.null_count(), new_data.null_count());
+
+        // slice of a slice (removes one null)
+        let new_data = new_data.slice(1, 14);
+        assert_eq!(data.len() - 2, new_data.len());
+        assert_eq!(2, new_data.offset());
+        assert_eq!(data.null_count() - 1, new_data.null_count());
+    }
 }
diff --git a/rust/arrow/src/array/equal.rs b/rust/arrow/src/array/equal.rs
deleted file mode 100644
index 02707a5f104..00000000000
--- a/rust/arrow/src/array/equal.rs
+++ /dev/null
@@ -1,2674 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use super::*;
-use crate::datatypes::*;
-use crate::util::bit_util;
-use hex::FromHex;
-use serde_json::value::Value::{Null as JNull, Object, String as JString};
-use serde_json::Value;
-
-/// Trait for `Array` equality.
-pub trait ArrayEqual {
-    /// Returns true if this array is equal to the `other` array
-    fn equals(&self, other: &dyn Array) -> bool;
-
-    /// Returns true if the range [start_idx, end_idx) is equal to
-    /// [other_start_idx, other_start_idx + end_idx - start_idx) in the `other` array
-    fn range_equals(
-        &self,
-        other: &dyn Array,
-        start_idx: usize,
-        end_idx: usize,
-        other_start_idx: usize,
-    ) -> bool;
-}
-
-impl<T: ArrowPrimitiveType> ArrayEqual for PrimitiveArray<T> {
-    default fn equals(&self, other: &dyn Array) -> bool {
-        if !base_equal(&self.data(), &other.data()) {
-            return false;
-        }
-
-        let value_buf = self.data_ref().buffers()[0].clone();
-        let other_value_buf = other.data_ref().buffers()[0].clone();
-        let byte_width = T::get_bit_width() / 8;
-
-        if self.null_count() > 0 {
-            let values = value_buf.data();
-            let other_values = other_value_buf.data();
-
-            for i in 0..self.len() {
-                if self.is_valid(i) {
-                    let start = (i + self.offset()) * byte_width;
-                    let data = &values[start..(start + byte_width)];
-                    let other_start = (i + other.offset()) * byte_width;
-                    let other_data =
-                        &other_values[other_start..(other_start + byte_width)];
-                    if data != other_data {
-                        return false;
-                    }
-                }
-            }
-        } else {
-            let start = self.offset() * byte_width;
-            let other_start = other.offset() * byte_width;
-            let len = self.len() * byte_width;
-            let data = &value_buf.data()[start..(start + len)];
-            let other_data = &other_value_buf.data()[other_start..(other_start + len)];
-            if data != other_data {
-                return false;
-            }
-        }
-
-        true
-    }
-
-    default fn range_equals(
-        &self,
-        other: &dyn Array,
-        start_idx: usize,
-        end_idx: usize,
-        other_start_idx: usize,
-    ) -> bool {
-        assert!(other_start_idx + (end_idx - start_idx) <= other.len());
-        let other = other.as_any().downcast_ref::<PrimitiveArray<T>>().unwrap();
-
-        let mut j = other_start_idx;
-        for i in start_idx..end_idx {
-            let is_null = self.is_null(i);
-            let other_is_null = other.is_null(j);
-            if is_null != other_is_null || (!is_null && self.value(i) != other.value(j)) {
-                return false;
-            }
-            j += 1;
-        }
-
-        true
-    }
-}
-
-impl ArrayEqual for BooleanArray {
-    fn equals(&self, other: &dyn Array) -> bool {
-        if !base_equal(&self.data(), &other.data()) {
-            return false;
-        }
-
-        let values = self.data_ref().buffers()[0].data();
-        let other_values = other.data_ref().buffers()[0].data();
-
-        // TODO: we can do this more efficiently if all values are not-null
-        for i in 0..self.len() {
-            if self.is_valid(i)
-                && bit_util::get_bit(values, i + self.offset())
-                    != bit_util::get_bit(other_values, i + other.offset())
-            {
-                return false;
-            }
-        }
-
-        true
-    }
-}
-
-impl<T: ArrowNumericType> PartialEq for PrimitiveArray<T> {
-    fn eq(&self, other: &PrimitiveArray<T>) -> bool {
-        self.equals(other)
-    }
-}
-
-impl PartialEq for BooleanArray {
-    fn eq(&self, other: &BooleanArray) -> bool {
-        self.equals(other)
-    }
-}
-
-impl PartialEq for StringArray {
-    fn eq(&self, other: &Self) -> bool {
-        self.equals(other)
-    }
-}
-
-impl PartialEq for FixedSizeBinaryArray {
-    fn eq(&self, other: &Self) -> bool {
-        self.equals(other)
-    }
-}
-
-impl PartialEq for BinaryArray {
-    fn eq(&self, other: &Self) -> bool {
-        self.equals(other)
-    }
-}
-
-impl ArrayEqual for ListArray {
-    fn equals(&self, other: &dyn Array) -> bool {
-        if !base_equal(&self.data(), &other.data()) {
-            return false;
-        }
-
-        let other = other.as_any().downcast_ref::<ListArray>().unwrap();
-
-        if !value_offset_equal(self, other) {
-            return false;
-        }
-
-        if !self.values().range_equals(
-            &*other.values(),
-            self.value_offset(0) as usize,
-            self.value_offset(self.len()) as usize,
-            other.value_offset(0) as usize,
-        ) {
-            return false;
-        }
-
-        true
-    }
-
-    fn range_equals(
-        &self,
-        other: &dyn Array,
-        start_idx: usize,
-        end_idx: usize,
-        other_start_idx: usize,
-    ) -> bool {
-        assert!(other_start_idx + (end_idx - start_idx) <= other.len());
-        let other = other.as_any().downcast_ref::<ListArray>().unwrap();
-
-        let mut j = other_start_idx;
-        for i in start_idx..end_idx {
-            let is_null = self.is_null(i);
-            let other_is_null = other.is_null(j);
-
-            if is_null != other_is_null {
-                return false;
-            }
-
-            if is_null {
-                continue;
-            }
-
-            let start_offset = self.value_offset(i) as usize;
-            let end_offset = self.value_offset(i + 1) as usize;
-            let other_start_offset = other.value_offset(j) as usize;
-            let other_end_offset = other.value_offset(j + 1) as usize;
-
-            if end_offset - start_offset != other_end_offset - other_start_offset {
-                return false;
-            }
-
-            if !self.values().range_equals(
-                &*other.values(),
-                start_offset,
-                end_offset,
-                other_start_offset,
-            ) {
-                return false;
-            }
-
-            j += 1;
-        }
-
-        true
-    }
-}
-
-impl ArrayEqual for LargeListArray {
-    fn equals(&self, other: &dyn Array) -> bool {
-        if !base_equal(&self.data(), &other.data()) {
-            return false;
-        }
-
-        let other = other.as_any().downcast_ref::<LargeListArray>().unwrap();
-
-        if !large_value_offset_equal(self, other) {
-            return false;
-        }
-
-        if !self.values().range_equals(
-            &*other.values(),
-            self.value_offset(0) as usize,
-            self.value_offset(self.len()) as usize,
-            other.value_offset(0) as usize,
-        ) {
-            return false;
-        }
-
-        true
-    }
-
-    fn range_equals(
-        &self,
-        other: &dyn Array,
-        start_idx: usize,
-        end_idx: usize,
-        other_start_idx: usize,
-    ) -> bool {
-        assert!(other_start_idx + (end_idx - start_idx) <= other.len());
-        let other = other.as_any().downcast_ref::<LargeListArray>().unwrap();
-
-        let mut j = other_start_idx;
-        for i in start_idx..end_idx {
-            let is_null = self.is_null(i);
-            let other_is_null = other.is_null(j);
-
-            if is_null != other_is_null {
-                return false;
-            }
-
-            if is_null {
-                continue;
-            }
-
-            let start_offset = self.value_offset(i) as usize;
-            let end_offset = self.value_offset(i + 1) as usize;
-            let other_start_offset = other.value_offset(j) as usize;
-            let other_end_offset = other.value_offset(j + 1) as usize;
-
-            if end_offset - start_offset != other_end_offset - other_start_offset {
-                return false;
-            }
-
-            if !self.values().range_equals(
-                &*other.values(),
-                start_offset,
-                end_offset,
-                other_start_offset,
-            ) {
-                return false;
-            }
-
-            j += 1;
-        }
-
-        true
-    }
-}
-
-impl<T: ArrowPrimitiveType> ArrayEqual for DictionaryArray<T> {
-    fn equals(&self, other: &dyn Array) -> bool {
-        self.range_equals(other, 0, self.len(), 0)
-    }
-
-    default fn range_equals(
-        &self,
-        other: &dyn Array,
-        start_idx: usize,
-        end_idx: usize,
-        other_start_idx: usize,
-    ) -> bool {
-        assert!(other_start_idx + (end_idx - start_idx) <= other.len());
-        let other = other.as_any().downcast_ref::<DictionaryArray<T>>().unwrap();
-
-        let iter_a = self.keys().take(end_idx).skip(start_idx);
-        let iter_b = other.keys().skip(other_start_idx);
-
-        // For now, all the values must be the same
-        iter_a.eq(iter_b)
-            && self
-                .values()
-                .range_equals(&*other.values(), 0, other.values().len(), 0)
-    }
-}
-
-impl ArrayEqual for FixedSizeListArray {
-    fn equals(&self, other: &dyn Array) -> bool {
-        if !base_equal(&self.data(), &other.data()) {
-            return false;
-        }
-
-        let other = other.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
-
-        if !self.values().range_equals(
-            &*other.values(),
-            self.value_offset(0) as usize,
-            self.value_offset(self.len()) as usize,
-            other.value_offset(0) as usize,
-        ) {
-            return false;
-        }
-
-        true
-    }
-
-    fn range_equals(
-        &self,
-        other: &dyn Array,
-        start_idx: usize,
-        end_idx: usize,
-        other_start_idx: usize,
-    ) -> bool {
-        assert!(other_start_idx + (end_idx - start_idx) <= other.len());
-        let other = other.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
-
-        let mut j = other_start_idx;
-        for i in start_idx..end_idx {
-            let is_null = self.is_null(i);
-            let other_is_null = other.is_null(j);
-
-            if is_null != other_is_null {
-                return false;
-            }
-
-            if is_null {
-                continue;
-            }
-
-            let start_offset = self.value_offset(i) as usize;
-            let end_offset = self.value_offset(i + 1) as usize;
-            let other_start_offset = other.value_offset(j) as usize;
-            let other_end_offset = other.value_offset(j + 1) as usize;
-
-            if end_offset - start_offset != other_end_offset - other_start_offset {
-                return false;
-            }
-
-            if !self.values().range_equals(
-                &*other.values(),
-                start_offset,
-                end_offset,
-                other_start_offset,
-            ) {
-                return false;
-            }
-
-            j += 1;
-        }
-
-        true
-    }
-}
-
-impl ArrayEqual for BinaryArray {
-    fn equals(&self, other: &dyn Array) -> bool {
-        if !base_equal(&self.data(), &other.data()) {
-            return false;
-        }
-
-        let other = other.as_any().downcast_ref::<BinaryArray>().unwrap();
-
-        if !value_offset_equal(self, other) {
-            return false;
-        }
-
-        // TODO: handle null & length == 0 case?
-
-        let value_buf = self.value_data();
-        let other_value_buf = other.value_data();
-        let value_data = value_buf.data();
-        let other_value_data = other_value_buf.data();
-
-        if self.null_count() == 0 {
-            // No offset in both - just do memcmp
-            if self.offset() == 0 && other.offset() == 0 {
-                let len = self.value_offset(self.len()) as usize;
-                return value_data[..len] == other_value_data[..len];
-            } else {
-                let start = self.value_offset(0) as usize;
-                let other_start = other.value_offset(0) as usize;
-                let len = (self.value_offset(self.len()) - self.value_offset(0)) as usize;
-                return value_data[start..(start + len)]
-                    == other_value_data[other_start..(other_start + len)];
-            }
-        } else {
-            for i in 0..self.len() {
-                if self.is_null(i) {
-                    continue;
-                }
-
-                let start = self.value_offset(i) as usize;
-                let other_start = other.value_offset(i) as usize;
-                let len = self.value_length(i) as usize;
-                if value_data[start..(start + len)]
-                    != other_value_data[other_start..(other_start + len)]
-                {
-                    return false;
-                }
-            }
-        }
-
-        true
-    }
-
-    fn range_equals(
-        &self,
-        other: &dyn Array,
-        start_idx: usize,
-        end_idx: usize,
-        other_start_idx: usize,
-    ) -> bool {
-        assert!(other_start_idx + (end_idx - start_idx) <= other.len());
-        let other = other.as_any().downcast_ref::<BinaryArray>().unwrap();
-
-        let mut j = other_start_idx;
-        for i in start_idx..end_idx {
-            let is_null = self.is_null(i);
-            let other_is_null = other.is_null(j);
-
-            if is_null != other_is_null {
-                return false;
-            }
-
-            if is_null {
-                continue;
-            }
-
-            let start_offset = self.value_offset(i) as usize;
-            let end_offset = self.value_offset(i + 1) as usize;
-            let other_start_offset = other.value_offset(j) as usize;
-            let other_end_offset = other.value_offset(j + 1) as usize;
-
-            if end_offset - start_offset != other_end_offset - other_start_offset {
-                return false;
-            }
-
-            let value_buf = self.value_data();
-            let other_value_buf = other.value_data();
-            let value_data = value_buf.data();
-            let other_value_data = other_value_buf.data();
-
-            if end_offset - start_offset > 0 {
-                let len = end_offset - start_offset;
-                if value_data[start_offset..(start_offset + len)]
-                    != other_value_data[other_start_offset..(other_start_offset + len)]
-                {
-                    return false;
-                }
-            }
-
-            j += 1;
-        }
-
-        true
-    }
-}
-
-impl ArrayEqual for StringArray {
-    fn equals(&self, other: &dyn Array) -> bool {
-        if !base_equal(&self.data(), &other.data()) {
-            return false;
-        }
-
-        let other = other.as_any().downcast_ref::<StringArray>().unwrap();
-
-        if !value_offset_equal(self, other) {
-            return false;
-        }
-
-        // TODO: handle null & length == 0 case?
-
-        let value_buf = self.value_data();
-        let other_value_buf = other.value_data();
-        let value_data = value_buf.data();
-        let other_value_data = other_value_buf.data();
-
-        if self.null_count() == 0 {
-            // No offset in both - just do memcmp
-            if self.offset() == 0 && other.offset() == 0 {
-                let len = self.value_offset(self.len()) as usize;
-                return value_data[..len] == other_value_data[..len];
-            } else {
-                let start = self.value_offset(0) as usize;
-                let other_start = other.value_offset(0) as usize;
-                let len = (self.value_offset(self.len()) - self.value_offset(0)) as usize;
-                return value_data[start..(start + len)]
-                    == other_value_data[other_start..(other_start + len)];
-            }
-        } else {
-            for i in 0..self.len() {
-                if self.is_null(i) {
-                    continue;
-                }
-
-                let start = self.value_offset(i) as usize;
-                let other_start = other.value_offset(i) as usize;
-                let len = self.value_length(i) as usize;
-                if value_data[start..(start + len)]
-                    != other_value_data[other_start..(other_start + len)]
-                {
-                    return false;
-                }
-            }
-        }
-
-        true
-    }
-
-    fn range_equals(
-        &self,
-        other: &dyn Array,
-        start_idx: usize,
-        end_idx: usize,
-        other_start_idx: usize,
-    ) -> bool {
-        assert!(other_start_idx + (end_idx - start_idx) <= other.len());
-        let other = other.as_any().downcast_ref::<StringArray>().unwrap();
-
-        let mut j = other_start_idx;
-        for i in start_idx..end_idx {
-            let is_null = self.is_null(i);
-            let other_is_null = other.is_null(j);
-
-            if is_null != other_is_null {
-                return false;
-            }
-
-            if is_null {
-                continue;
-            }
-
-            let start_offset = self.value_offset(i) as usize;
-            let end_offset = self.value_offset(i + 1) as usize;
-            let other_start_offset = other.value_offset(j) as usize;
-            let other_end_offset = other.value_offset(j + 1) as usize;
-
-            if end_offset - start_offset != other_end_offset - other_start_offset {
-                return false;
-            }
-
-            let value_buf = self.value_data();
-            let other_value_buf = other.value_data();
-            let value_data = value_buf.data();
-            let other_value_data = other_value_buf.data();
-
-            if end_offset - start_offset > 0 {
-                let len = end_offset - start_offset;
-                if value_data[start_offset..(start_offset + len)]
-                    != other_value_data[other_start_offset..(other_start_offset + len)]
-                {
-                    return false;
-                }
-            }
-
-            j += 1;
-        }
-
-        true
-    }
-}
-
-impl ArrayEqual for LargeBinaryArray {
-    fn equals(&self, other: &dyn Array) -> bool {
-        if !base_equal(&self.data(), &other.data()) {
-            return false;
-        }
-
-        let other = other.as_any().downcast_ref::<LargeBinaryArray>().unwrap();
-
-        if !large_value_offset_equal(self, other) {
-            return false;
-        }
-
-        // TODO: handle null & length == 0 case?
-
-        let value_buf = self.value_data();
-        let other_value_buf = other.value_data();
-        let value_data = value_buf.data();
-        let other_value_data = other_value_buf.data();
-
-        if self.null_count() == 0 {
-            // No offset in both - just do memcmp
-            if self.offset() == 0 && other.offset() == 0 {
-                let len = self.value_offset(self.len()) as usize;
-                return value_data[..len] == other_value_data[..len];
-            } else {
-                let start = self.value_offset(0) as usize;
-                let other_start = other.value_offset(0) as usize;
-                let len = (self.value_offset(self.len()) - self.value_offset(0)) as usize;
-                return value_data[start..(start + len)]
-                    == other_value_data[other_start..(other_start + len)];
-            }
-        } else {
-            for i in 0..self.len() {
-                if self.is_null(i) {
-                    continue;
-                }
-
-                let start = self.value_offset(i) as usize;
-                let other_start = other.value_offset(i) as usize;
-                let len = self.value_length(i) as usize;
-                if value_data[start..(start + len)]
-                    != other_value_data[other_start..(other_start + len)]
-                {
-                    return false;
-                }
-            }
-        }
-
-        true
-    }
-
-    fn range_equals(
-        &self,
-        other: &dyn Array,
-        start_idx: usize,
-        end_idx: usize,
-        other_start_idx: usize,
-    ) -> bool {
-        assert!(other_start_idx + (end_idx - start_idx) <= other.len());
-        let other = other.as_any().downcast_ref::<LargeBinaryArray>().unwrap();
-
-        let mut j = other_start_idx;
-        for i in start_idx..end_idx {
-            let is_null = self.is_null(i);
-            let other_is_null = other.is_null(j);
-
-            if is_null != other_is_null {
-                return false;
-            }
-
-            if is_null {
-                continue;
-            }
-
-            let start_offset = self.value_offset(i) as usize;
-            let end_offset = self.value_offset(i + 1) as usize;
-            let other_start_offset = other.value_offset(j) as usize;
-            let other_end_offset = other.value_offset(j + 1) as usize;
-
-            if end_offset - start_offset != other_end_offset - other_start_offset {
-                return false;
-            }
-
-            let value_buf = self.value_data();
-            let other_value_buf = other.value_data();
-            let value_data = value_buf.data();
-            let other_value_data = other_value_buf.data();
-
-            if end_offset - start_offset > 0 {
-                let len = end_offset - start_offset;
-                if value_data[start_offset..(start_offset + len)]
-                    != other_value_data[other_start_offset..(other_start_offset + len)]
-                {
-                    return false;
-                }
-            }
-
-            j += 1;
-        }
-
-        true
-    }
-}
-
-impl ArrayEqual for LargeStringArray {
-    fn equals(&self, other: &dyn Array) -> bool {
-        if !base_equal(&self.data(), &other.data()) {
-            return false;
-        }
-
-        let other = other.as_any().downcast_ref::<LargeStringArray>().unwrap();
-
-        if !large_value_offset_equal(self, other) {
-            return false;
-        }
-
-        // TODO: handle null & length == 0 case?
-
-        let value_buf = self.value_data();
-        let other_value_buf = other.value_data();
-        let value_data = value_buf.data();
-        let other_value_data = other_value_buf.data();
-
-        if self.null_count() == 0 {
-            // No offset in both - just do memcmp
-            if self.offset() == 0 && other.offset() == 0 {
-                let len = self.value_offset(self.len()) as usize;
-                return value_data[..len] == other_value_data[..len];
-            } else {
-                let start = self.value_offset(0) as usize;
-                let other_start = other.value_offset(0) as usize;
-                let len = (self.value_offset(self.len()) - self.value_offset(0)) as usize;
-                return value_data[start..(start + len)]
-                    == other_value_data[other_start..(other_start + len)];
-            }
-        } else {
-            for i in 0..self.len() {
-                if self.is_null(i) {
-                    continue;
-                }
-
-                let start = self.value_offset(i) as usize;
-                let other_start = other.value_offset(i) as usize;
-                let len = self.value_length(i) as usize;
-                if value_data[start..(start + len)]
-                    != other_value_data[other_start..(other_start + len)]
-                {
-                    return false;
-                }
-            }
-        }
-
-        true
-    }
-
-    fn range_equals(
-        &self,
-        other: &dyn Array,
-        start_idx: usize,
-        end_idx: usize,
-        other_start_idx: usize,
-    ) -> bool {
-        assert!(other_start_idx + (end_idx - start_idx) <= other.len());
-        let other = other.as_any().downcast_ref::<LargeStringArray>().unwrap();
-
-        let mut j = other_start_idx;
-        for i in start_idx..end_idx {
-            let is_null = self.is_null(i);
-            let other_is_null = other.is_null(j);
-
-            if is_null != other_is_null {
-                return false;
-            }
-
-            if is_null {
-                continue;
-            }
-
-            let start_offset = self.value_offset(i) as usize;
-            let end_offset = self.value_offset(i + 1) as usize;
-            let other_start_offset = other.value_offset(j) as usize;
-            let other_end_offset = other.value_offset(j + 1) as usize;
-
-            if end_offset - start_offset != other_end_offset - other_start_offset {
-                return false;
-            }
-
-            let value_buf = self.value_data();
-            let other_value_buf = other.value_data();
-            let value_data = value_buf.data();
-            let other_value_data = other_value_buf.data();
-
-            if end_offset - start_offset > 0 {
-                let len = end_offset - start_offset;
-                if value_data[start_offset..(start_offset + len)]
-                    != other_value_data[other_start_offset..(other_start_offset + len)]
-                {
-                    return false;
-                }
-            }
-
-            j += 1;
-        }
-
-        true
-    }
-}
-
-impl ArrayEqual for FixedSizeBinaryArray {
-    fn equals(&self, other: &dyn Array) -> bool {
-        if !base_equal(&self.data(), &other.data()) {
-            return false;
-        }
-
-        let other = other
-            .as_any()
-            .downcast_ref::<FixedSizeBinaryArray>()
-            .unwrap();
-
-        if !value_offset_equal(self, other) {
-            return false;
-        }
-
-        // TODO: handle null & length == 0 case?
-
-        let value_buf = self.value_data();
-        let other_value_buf = other.value_data();
-        let value_data = value_buf.data();
-        let other_value_data = other_value_buf.data();
-
-        if self.null_count() == 0 {
-            // No offset in both - just do memcmp
-            if self.offset() == 0 && other.offset() == 0 {
-                let len = self.value_offset(self.len()) as usize;
-                return value_data[..len] == other_value_data[..len];
-            } else {
-                let start = self.value_offset(0) as usize;
-                let other_start = other.value_offset(0) as usize;
-                let len = (self.value_offset(self.len()) - self.value_offset(0)) as usize;
-                return value_data[start..(start + len)]
-                    == other_value_data[other_start..(other_start + len)];
-            }
-        } else {
-            for i in 0..self.len() {
-                if self.is_null(i) {
-                    continue;
-                }
-
-                let start = self.value_offset(i) as usize;
-                let other_start = other.value_offset(i) as usize;
-                let len = self.value_length() as usize;
-                if value_data[start..(start + len)]
-                    != other_value_data[other_start..(other_start + len)]
-                {
-                    return false;
-                }
-            }
-        }
-
-        true
-    }
-
-    fn range_equals(
-        &self,
-        other: &dyn Array,
-        start_idx: usize,
-        end_idx: usize,
-        other_start_idx: usize,
-    ) -> bool {
-        assert!(other_start_idx + (end_idx - start_idx) <= other.len());
-        let other = other
-            .as_any()
-            .downcast_ref::<FixedSizeBinaryArray>()
-            .unwrap();
-
-        let mut j = other_start_idx;
-        for i in start_idx..end_idx {
-            let is_null = self.is_null(i);
-            let other_is_null = other.is_null(j);
-
-            if is_null != other_is_null {
-                return false;
-            }
-
-            if is_null {
-                continue;
-            }
-
-            let start_offset = self.value_offset(i) as usize;
-            let end_offset = self.value_offset(i + 1) as usize;
-            let other_start_offset = other.value_offset(j) as usize;
-            let other_end_offset = other.value_offset(j + 1) as usize;
-
-            if end_offset - start_offset != other_end_offset - other_start_offset {
-                return false;
-            }
-
-            let value_buf = self.value_data();
-            let other_value_buf = other.value_data();
-            let value_data = value_buf.data();
-            let other_value_data = other_value_buf.data();
-
-            if end_offset - start_offset > 0 {
-                let len = end_offset - start_offset;
-                if value_data[start_offset..(start_offset + len)]
-                    != other_value_data[other_start_offset..(other_start_offset + len)]
-                {
-                    return false;
-                }
-            }
-
-            j += 1;
-        }
-
-        true
-    }
-}
-
-impl ArrayEqual for StructArray {
-    fn equals(&self, other: &dyn Array) -> bool {
-        if !base_equal(&self.data(), &other.data()) {
-            return false;
-        }
-
-        let other = other.as_any().downcast_ref::<StructArray>().unwrap();
-
-        for i in 0..self.len() {
-            let is_null = self.is_null(i);
-            let other_is_null = other.is_null(i);
-
-            if is_null != other_is_null {
-                return false;
-            }
-
-            if is_null {
-                continue;
-            }
-            for j in 0..self.num_columns() {
-                if !self.column(j).range_equals(&**other.column(j), i, i + 1, i) {
-                    return false;
-                }
-            }
-        }
-
-        true
-    }
-
-    fn range_equals(
-        &self,
-        other: &dyn Array,
-        start_idx: usize,
-        end_idx: usize,
-        other_start_idx: usize,
-    ) -> bool {
-        assert!(other_start_idx + (end_idx - start_idx) <= other.len());
-        let other = other.as_any().downcast_ref::<StructArray>().unwrap();
-
-        let mut j = other_start_idx;
-        for i in start_idx..end_idx {
-            let is_null = self.is_null(i);
-            let other_is_null = other.is_null(i);
-
-            if is_null != other_is_null {
-                return false;
-            }
-
-            if is_null {
-                continue;
-            }
-            for k in 0..self.num_columns() {
-                if !self.column(k).range_equals(&**other.column(k), i, i + 1, j) {
-                    return false;
-                }
-            }
-
-            j += 1;
-        }
-
-        true
-    }
-}
-
-impl ArrayEqual for UnionArray {
-    fn equals(&self, _other: &dyn Array) -> bool {
-        unimplemented!(
-            "Added to allow UnionArray to implement the Array trait: see ARROW-8576"
-        )
-    }
-
-    fn range_equals(
-        &self,
-        _other: &dyn Array,
-        _start_idx: usize,
-        _end_idx: usize,
-        _other_start_idx: usize,
-    ) -> bool {
-        unimplemented!(
-            "Added to allow UnionArray to implement the Array trait: see ARROW-8576"
-        )
-    }
-}
-
-impl ArrayEqual for NullArray {
-    fn equals(&self, other: &dyn Array) -> bool {
-        if other.data_type() != &DataType::Null {
-            return false;
-        }
-
-        if self.len() != other.len() {
-            return false;
-        }
-        if self.null_count() != other.null_count() {
-            return false;
-        }
-
-        true
-    }
-
-    fn range_equals(
-        &self,
-        _other: &dyn Array,
-        _start_idx: usize,
-        _end_idx: usize,
-        _other_start_idx: usize,
-    ) -> bool {
-        unimplemented!("Range comparison for null array not yet supported")
-    }
-}
-
-// Compare if the common basic fields between the two arrays are equal
-fn base_equal(this: &ArrayDataRef, other: &ArrayDataRef) -> bool {
-    if this.data_type() != other.data_type() {
-        return false;
-    }
-    if this.len != other.len {
-        return false;
-    }
-    if this.null_count != other.null_count {
-        return false;
-    }
-    if this.null_count > 0 {
-        let null_bitmap = this.null_bitmap().as_ref().unwrap();
-        let other_null_bitmap = other.null_bitmap().as_ref().unwrap();
-        let null_buf = null_bitmap.bits.data();
-        let other_null_buf = other_null_bitmap.bits.data();
-        for i in 0..this.len() {
-            if bit_util::get_bit(null_buf, i + this.offset())
-                != bit_util::get_bit(other_null_buf, i + other.offset())
-            {
-                return false;
-            }
-        }
-    }
-    true
-}
-
-// Compare if the value offsets are equal between the two list arrays
-fn value_offset_equal<T: Array + ListArrayOps>(this: &T, other: &T) -> bool {
-    // Check if offsets differ
-    if this.offset() == 0 && other.offset() == 0 {
-        let offset_data = &this.data_ref().buffers()[0];
-        let other_offset_data = &other.data_ref().buffers()[0];
-        return offset_data.data()[0..((this.len() + 1) * 4)]
-            == other_offset_data.data()[0..((other.len() + 1) * 4)];
-    }
-
-    // The expensive case
-    for i in 0..=this.len() {
-        if this.value_offset_at(i) - this.value_offset_at(0)
-            != other.value_offset_at(i) - other.value_offset_at(0)
-        {
-            return false;
-        }
-    }
-
-    true
-}
-
-// Compare if the value offsets are equal between the two list arrays
-fn large_value_offset_equal<T: Array + LargeListArrayOps>(this: &T, other: &T) -> bool {
-    // Check if offsets differ
-    if this.offset() == 0 && other.offset() == 0 {
-        let offset_data = &this.data_ref().buffers()[0];
-        let other_offset_data = &other.data_ref().buffers()[0];
-        return offset_data.data()[0..((this.len() + 1) * 4)]
-            == other_offset_data.data()[0..((other.len() + 1) * 4)];
-    }
-
-    // The expensive case
-    for i in 0..=this.len() {
-        if this.value_offset_at(i) - this.value_offset_at(0)
-            != other.value_offset_at(i) - other.value_offset_at(0)
-        {
-            return false;
-        }
-    }
-
-    true
-}
-
-/// Trait for comparing arrow array with json array
-pub trait JsonEqual {
-    /// Checks whether arrow array equals to json array.
-    fn equals_json(&self, json: &[&Value]) -> bool;
-
-    /// Checks whether arrow array equals to json array.
-    fn equals_json_values(&self, json: &[Value]) -> bool {
-        let refs = json.iter().collect::<Vec<&Value>>();
-
-        self.equals_json(&refs)
-    }
-}
-
-/// Implement array equals for numeric type
-impl<T: ArrowPrimitiveType> JsonEqual for PrimitiveArray<T> {
-    fn equals_json(&self, json: &[&Value]) -> bool {
-        if self.len() != json.len() {
-            return false;
-        }
-
-        (0..self.len()).all(|i| match json[i] {
-            Value::Null => self.is_null(i),
-            v => self.is_valid(i) && Some(v) == self.value(i).into_json_value().as_ref(),
-        })
-    }
-}
-
-impl<T: ArrowPrimitiveType> PartialEq<Value> for PrimitiveArray<T> {
-    fn eq(&self, json: &Value) -> bool {
-        match json {
-            Value::Array(array) => self.equals_json_values(&array),
-            _ => false,
-        }
-    }
-}
-
-impl<T: ArrowPrimitiveType> PartialEq<PrimitiveArray<T>> for Value {
-    fn eq(&self, arrow: &PrimitiveArray<T>) -> bool {
-        match self {
-            Value::Array(array) => arrow.equals_json_values(&array),
-            _ => false,
-        }
-    }
-}
-
-impl JsonEqual for ListArray {
-    fn equals_json(&self, json: &[&Value]) -> bool {
-        if self.len() != json.len() {
-            return false;
-        }
-
-        (0..self.len()).all(|i| match json[i] {
-            Value::Array(v) => self.is_valid(i) && self.value(i).equals_json_values(v),
-            Value::Null => self.is_null(i) || self.value_length(i) == 0,
-            _ => false,
-        })
-    }
-}
-
-impl JsonEqual for LargeListArray {
-    fn equals_json(&self, json: &[&Value]) -> bool {
-        if self.len() != json.len() {
-            return false;
-        }
-
-        (0..self.len()).all(|i| match json[i] {
-            Value::Array(v) => self.is_valid(i) && self.value(i).equals_json_values(v),
-            Value::Null => self.is_null(i) || self.value_length(i) == 0,
-            _ => false,
-        })
-    }
-}
-
-impl PartialEq<Value> for ListArray {
-    fn eq(&self, json: &Value) -> bool {
-        match json {
-            Value::Array(json_array) => self.equals_json_values(json_array),
-            _ => false,
-        }
-    }
-}
-
-impl PartialEq<Value> for LargeListArray {
-    fn eq(&self, json: &Value) -> bool {
-        match json {
-            Value::Array(json_array) => self.equals_json_values(json_array),
-            _ => false,
-        }
-    }
-}
-
-impl PartialEq<ListArray> for Value {
-    fn eq(&self, arrow: &ListArray) -> bool {
-        match self {
-            Value::Array(json_array) => arrow.equals_json_values(json_array),
-            _ => false,
-        }
-    }
-}
-
-impl PartialEq<LargeListArray> for Value {
-    fn eq(&self, arrow: &LargeListArray) -> bool {
-        match self {
-            Value::Array(json_array) => arrow.equals_json_values(json_array),
-            _ => false,
-        }
-    }
-}
-
-impl<T: ArrowPrimitiveType> JsonEqual for DictionaryArray<T> {
-    fn equals_json(&self, json: &[&Value]) -> bool {
-        self.keys().zip(json.iter()).all(|aj| match aj {
-            (None, Value::Null) => true,
-            (Some(a), Value::Number(j)) => {
-                a.to_usize().unwrap() as u64 == j.as_u64().unwrap()
-            }
-            _ => false,
-        })
-    }
-}
-
-impl<T: ArrowPrimitiveType> PartialEq<Value> for DictionaryArray<T> {
-    fn eq(&self, json: &Value) -> bool {
-        match json {
-            Value::Array(json_array) => self.equals_json_values(json_array),
-            _ => false,
-        }
-    }
-}
-
-impl<T: ArrowPrimitiveType> PartialEq<DictionaryArray<T>> for Value {
-    fn eq(&self, arrow: &DictionaryArray<T>) -> bool {
-        match self {
-            Value::Array(json_array) => arrow.equals_json_values(json_array),
-            _ => false,
-        }
-    }
-}
-
-impl JsonEqual for FixedSizeListArray {
-    fn equals_json(&self, json: &[&Value]) -> bool {
-        if self.len() != json.len() {
-            return false;
-        }
-
-        (0..self.len()).all(|i| match json[i] {
-            Value::Array(v) => self.is_valid(i) && self.value(i).equals_json_values(v),
-            Value::Null => self.is_null(i) || self.value_length() == 0,
-            _ => false,
-        })
-    }
-}
-
-impl PartialEq<Value> for FixedSizeListArray {
-    fn eq(&self, json: &Value) -> bool {
-        match json {
-            Value::Array(json_array) => self.equals_json_values(json_array),
-            _ => false,
-        }
-    }
-}
-
-impl PartialEq<FixedSizeListArray> for Value {
-    fn eq(&self, arrow: &FixedSizeListArray) -> bool {
-        match self {
-            Value::Array(json_array) => arrow.equals_json_values(json_array),
-            _ => false,
-        }
-    }
-}
-
-impl JsonEqual for StructArray {
-    fn equals_json(&self, json: &[&Value]) -> bool {
-        if self.len() != json.len() {
-            return false;
-        }
-
-        let all_object = json.iter().all(|v| match v {
-            Object(_) | JNull => true,
-            _ => false,
-        });
-
-        if !all_object {
-            return false;
-        }
-
-        for column_name in self.column_names() {
-            let json_values = json
-                .iter()
-                .map(|obj| obj.get(column_name).unwrap_or(&Value::Null))
-                .collect::<Vec<&Value>>();
-
-            if !self
-                .column_by_name(column_name)
-                .map(|arr| arr.equals_json(&json_values))
-                .unwrap_or(false)
-            {
-                return false;
-            }
-        }
-
-        true
-    }
-}
-
-impl PartialEq<Value> for StructArray {
-    fn eq(&self, json: &Value) -> bool {
-        match json {
-            Value::Array(json_array) => self.equals_json_values(&json_array),
-            _ => false,
-        }
-    }
-}
-
-impl PartialEq<StructArray> for Value {
-    fn eq(&self, arrow: &StructArray) -> bool {
-        match self {
-            Value::Array(json_array) => arrow.equals_json_values(&json_array),
-            _ => false,
-        }
-    }
-}
-
-impl JsonEqual for BinaryArray {
-    fn equals_json(&self, json: &[&Value]) -> bool {
-        if self.len() != json.len() {
-            return false;
-        }
-
-        (0..self.len()).all(|i| match json[i] {
-            JString(s) => {
-                // binary data is sometimes hex encoded, this checks if bytes are equal,
-                // and if not converting to hex is attempted
-                self.is_valid(i)
-                    && (s.as_str().as_bytes() == self.value(i)
-                        || Vec::from_hex(s.as_str()) == Ok(self.value(i).to_vec()))
-            }
-            JNull => self.is_null(i),
-            _ => false,
-        })
-    }
-}
-
-impl PartialEq<Value> for BinaryArray {
-    fn eq(&self, json: &Value) -> bool {
-        match json {
-            Value::Array(json_array) => self.equals_json_values(&json_array),
-            _ => false,
-        }
-    }
-}
-
-impl PartialEq<BinaryArray> for Value {
-    fn eq(&self, arrow: &BinaryArray) -> bool {
-        match self {
-            Value::Array(json_array) => arrow.equals_json_values(&json_array),
-            _ => false,
-        }
-    }
-}
-
-impl JsonEqual for LargeBinaryArray {
-    fn equals_json(&self, json: &[&Value]) -> bool {
-        if self.len() != json.len() {
-            return false;
-        }
-
-        (0..self.len()).all(|i| match json[i] {
-            JString(s) => {
-                // binary data is sometimes hex encoded, this checks if bytes are equal,
-                // and if not converting to hex is attempted
-                self.is_valid(i)
-                    && (s.as_str().as_bytes() == self.value(i)
-                        || Vec::from_hex(s.as_str()) == Ok(self.value(i).to_vec()))
-            }
-            JNull => self.is_null(i),
-            _ => false,
-        })
-    }
-}
-
-impl PartialEq<Value> for LargeBinaryArray {
-    fn eq(&self, json: &Value) -> bool {
-        match json {
-            Value::Array(json_array) => self.equals_json_values(&json_array),
-            _ => false,
-        }
-    }
-}
-
-impl PartialEq<LargeBinaryArray> for Value {
-    fn eq(&self, arrow: &LargeBinaryArray) -> bool {
-        match self {
-            Value::Array(json_array) => arrow.equals_json_values(&json_array),
-            _ => false,
-        }
-    }
-}
-
-impl JsonEqual for StringArray {
-    fn equals_json(&self, json: &[&Value]) -> bool {
-        if self.len() != json.len() {
-            return false;
-        }
-
-        (0..self.len()).all(|i| match json[i] {
-            JString(s) => self.is_valid(i) && s.as_str() == self.value(i),
-            JNull => self.is_null(i),
-            _ => false,
-        })
-    }
-}
-
-impl PartialEq<Value> for StringArray {
-    fn eq(&self, json: &Value) -> bool {
-        match json {
-            Value::Array(json_array) => self.equals_json_values(&json_array),
-            _ => false,
-        }
-    }
-}
-
-impl PartialEq<StringArray> for Value {
-    fn eq(&self, arrow: &StringArray) -> bool {
-        match self {
-            Value::Array(json_array) => arrow.equals_json_values(&json_array),
-            _ => false,
-        }
-    }
-}
-
-impl JsonEqual for LargeStringArray {
-    fn equals_json(&self, json: &[&Value]) -> bool {
-        if self.len() != json.len() {
-            return false;
-        }
-
-        (0..self.len()).all(|i| match json[i] {
-            JString(s) => self.is_valid(i) && s.as_str() == self.value(i),
-            JNull => self.is_null(i),
-            _ => false,
-        })
-    }
-}
-
-impl PartialEq<Value> for LargeStringArray {
-    fn eq(&self, json: &Value) -> bool {
-        match json {
-            Value::Array(json_array) => self.equals_json_values(&json_array),
-            _ => false,
-        }
-    }
-}
-
-impl PartialEq<LargeStringArray> for Value {
-    fn eq(&self, arrow: &LargeStringArray) -> bool {
-        match self {
-            Value::Array(json_array) => arrow.equals_json_values(&json_array),
-            _ => false,
-        }
-    }
-}
-
-impl JsonEqual for FixedSizeBinaryArray {
-    fn equals_json(&self, json: &[&Value]) -> bool {
-        if self.len() != json.len() {
-            return false;
-        }
-
-        (0..self.len()).all(|i| match json[i] {
-            JString(s) => {
-                // binary data is sometimes hex encoded, this checks if bytes are equal,
-                // and if not converting to hex is attempted
-                self.is_valid(i)
-                    && (s.as_str().as_bytes() == self.value(i)
-                        || Vec::from_hex(s.as_str()) == Ok(self.value(i).to_vec()))
-            }
-            JNull => self.is_null(i),
-            _ => false,
-        })
-    }
-}
-
-impl PartialEq<Value> for FixedSizeBinaryArray {
-    fn eq(&self, json: &Value) -> bool {
-        match json {
-            Value::Array(json_array) => self.equals_json_values(&json_array),
-            _ => false,
-        }
-    }
-}
-
-impl PartialEq<FixedSizeBinaryArray> for Value {
-    fn eq(&self, arrow: &FixedSizeBinaryArray) -> bool {
-        match self {
-            Value::Array(json_array) => arrow.equals_json_values(&json_array),
-            _ => false,
-        }
-    }
-}
-
-impl JsonEqual for UnionArray {
-    fn equals_json(&self, _json: &[&Value]) -> bool {
-        unimplemented!(
-            "Added to allow UnionArray to implement the Array trait: see ARROW-8547"
-        )
-    }
-}
-
-impl JsonEqual for NullArray {
-    fn equals_json(&self, json: &[&Value]) -> bool {
-        if self.len() != json.len() {
-            return false;
-        }
-
-        // all JSON values must be nulls
-        json.iter().all(|&v| v == &JNull)
-    }
-}
-
-impl PartialEq<NullArray> for Value {
-    fn eq(&self, arrow: &NullArray) -> bool {
-        match self {
-            Value::Array(json_array) => arrow.equals_json_values(&json_array),
-            _ => false,
-        }
-    }
-}
-
-impl PartialEq<Value> for NullArray {
-    fn eq(&self, json: &Value) -> bool {
-        match json {
-            Value::Array(json_array) => self.equals_json_values(&json_array),
-            _ => false,
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    use std::convert::TryFrom;
-
-    use crate::error::Result;
-
-    #[test]
-    fn test_primitive_equal() {
-        let a = Int32Array::from(vec![1, 2, 3]);
-        let b = Int32Array::from(vec![1, 2, 3]);
-        assert!(a.equals(&b));
-        assert!(b.equals(&a));
-
-        let b = Int32Array::from(vec![1, 2, 4]);
-        assert!(!a.equals(&b));
-        assert!(!b.equals(&a));
-
-        // Test the case where null_count > 0
-
-        let a = Int32Array::from(vec![Some(1), None, Some(2), Some(3)]);
-        let b = Int32Array::from(vec![Some(1), None, Some(2), Some(3)]);
-        assert!(a.equals(&b));
-        assert!(b.equals(&a));
-
-        let b = Int32Array::from(vec![Some(1), None, None, Some(3)]);
-        assert!(!a.equals(&b));
-        assert!(!b.equals(&a));
-
-        let b = Int32Array::from(vec![Some(1), None, Some(2), Some(4)]);
-        assert!(!a.equals(&b));
-        assert!(!b.equals(&a));
-
-        // Test the case where offset != 0
-
-        let a_slice = a.slice(1, 2);
-        let b_slice = b.slice(1, 2);
-        assert!(a_slice.equals(&*b_slice));
-        assert!(b_slice.equals(&*a_slice));
-    }
-
-    #[test]
-    fn test_boolean_equal() {
-        let a = BooleanArray::from(vec![false, false, true]);
-        let b = BooleanArray::from(vec![false, false, true]);
-        assert!(a.equals(&b));
-        assert!(b.equals(&a));
-
-        let b = BooleanArray::from(vec![false, false, false]);
-        assert!(!a.equals(&b));
-        assert!(!b.equals(&a));
-
-        // Test the case where null_count > 0
-
-        let a = BooleanArray::from(vec![Some(false), None, None, Some(true)]);
-        let b = BooleanArray::from(vec![Some(false), None, None, Some(true)]);
-        assert!(a.equals(&b));
-        assert!(b.equals(&a));
-
-        let b = BooleanArray::from(vec![None, None, None, Some(true)]);
-        assert!(!a.equals(&b));
-        assert!(!b.equals(&a));
-
-        let b = BooleanArray::from(vec![Some(true), None, None, Some(true)]);
-        assert!(!a.equals(&b));
-        assert!(!b.equals(&a));
-
-        // Test the case where offset != 0
-
-        let a = BooleanArray::from(vec![false, true, false, true, false, false, true]);
-        let b = BooleanArray::from(vec![false, false, false, true, false, true, true]);
-        assert!(!a.equals(&b));
-        assert!(!b.equals(&a));
-
-        let a_slice = a.slice(2, 3);
-        let b_slice = b.slice(2, 3);
-        assert!(a_slice.equals(&*b_slice));
-        assert!(b_slice.equals(&*a_slice));
-
-        let a_slice = a.slice(3, 4);
-        let b_slice = b.slice(3, 4);
-        assert!(!a_slice.equals(&*b_slice));
-        assert!(!b_slice.equals(&*a_slice));
-    }
-
-    #[test]
-    fn test_list_equal() {
-        let mut a_builder = ListBuilder::new(Int32Builder::new(10));
-        let mut b_builder = ListBuilder::new(Int32Builder::new(10));
-
-        let a = create_list_array(&mut a_builder, &[Some(&[1, 2, 3]), Some(&[4, 5, 6])])
-            .unwrap();
-        let b = create_list_array(&mut b_builder, &[Some(&[1, 2, 3]), Some(&[4, 5, 6])])
-            .unwrap();
-
-        assert!(a.equals(&b));
-        assert!(b.equals(&a));
-
-        let b = create_list_array(&mut a_builder, &[Some(&[1, 2, 3]), Some(&[4, 5, 7])])
-            .unwrap();
-        assert!(!a.equals(&b));
-        assert!(!b.equals(&a));
-
-        // Test the case where null_count > 0
-
-        let a = create_list_array(
-            &mut a_builder,
-            &[Some(&[1, 2]), None, None, Some(&[3, 4]), None, None],
-        )
-        .unwrap();
-        let b = create_list_array(
-            &mut a_builder,
-            &[Some(&[1, 2]), None, None, Some(&[3, 4]), None, None],
-        )
-        .unwrap();
-        assert!(a.equals(&b));
-        assert!(b.equals(&a));
-
-        let b = create_list_array(
-            &mut a_builder,
-            &[
-                Some(&[1, 2]),
-                None,
-                Some(&[5, 6]),
-                Some(&[3, 4]),
-                None,
-                None,
-            ],
-        )
-        .unwrap();
-        assert!(!a.equals(&b));
-        assert!(!b.equals(&a));
-
-        let b = create_list_array(
-            &mut a_builder,
-            &[Some(&[1, 2]), None, None, Some(&[3, 5]), None, None],
-        )
-        .unwrap();
-        assert!(!a.equals(&b));
-        assert!(!b.equals(&a));
-
-        // Test the case where offset != 0
-
-        let a_slice = a.slice(0, 3);
-        let b_slice = b.slice(0, 3);
-        assert!(a_slice.equals(&*b_slice));
-        assert!(b_slice.equals(&*a_slice));
-
-        let a_slice = a.slice(0, 5);
-        let b_slice = b.slice(0, 5);
-        assert!(!a_slice.equals(&*b_slice));
-        assert!(!b_slice.equals(&*a_slice));
-
-        let a_slice = a.slice(4, 1);
-        let b_slice = b.slice(4, 1);
-        assert!(a_slice.equals(&*b_slice));
-        assert!(b_slice.equals(&*a_slice));
-    }
-
-    #[test]
-    fn test_fixed_size_list_equal() {
-        let mut a_builder = FixedSizeListBuilder::new(Int32Builder::new(10), 3);
-        let mut b_builder = FixedSizeListBuilder::new(Int32Builder::new(10), 3);
-
-        let a = create_fixed_size_list_array(
-            &mut a_builder,
-            &[Some(&[1, 2, 3]), Some(&[4, 5, 6])],
-        )
-        .unwrap();
-        let b = create_fixed_size_list_array(
-            &mut b_builder,
-            &[Some(&[1, 2, 3]), Some(&[4, 5, 6])],
-        )
-        .unwrap();
-
-        assert!(a.equals(&b));
-        assert!(b.equals(&a));
-
-        let b = create_fixed_size_list_array(
-            &mut a_builder,
-            &[Some(&[1, 2, 3]), Some(&[4, 5, 7])],
-        )
-        .unwrap();
-        assert!(!a.equals(&b));
-        assert!(!b.equals(&a));
-
-        // Test the case where null_count > 0
-
-        let a = create_fixed_size_list_array(
-            &mut a_builder,
-            &[Some(&[1, 2, 3]), None, None, Some(&[4, 5, 6]), None, None],
-        )
-        .unwrap();
-        let b = create_fixed_size_list_array(
-            &mut a_builder,
-            &[Some(&[1, 2, 3]), None, None, Some(&[4, 5, 6]), None, None],
-        )
-        .unwrap();
-        assert!(a.equals(&b));
-        assert!(b.equals(&a));
-
-        let b = create_fixed_size_list_array(
-            &mut a_builder,
-            &[
-                Some(&[1, 2, 3]),
-                None,
-                Some(&[7, 8, 9]),
-                Some(&[4, 5, 6]),
-                None,
-                None,
-            ],
-        )
-        .unwrap();
-        assert!(!a.equals(&b));
-        assert!(!b.equals(&a));
-
-        let b = create_fixed_size_list_array(
-            &mut a_builder,
-            &[Some(&[1, 2, 3]), None, None, Some(&[3, 6, 9]), None, None],
-        )
-        .unwrap();
-        assert!(!a.equals(&b));
-        assert!(!b.equals(&a));
-
-        // Test the case where offset != 0
-
-        let a_slice = a.slice(0, 3);
-        let b_slice = b.slice(0, 3);
-        assert!(a_slice.equals(&*b_slice));
-        assert!(b_slice.equals(&*a_slice));
-
-        // let a_slice = a.slice(0, 5);
-        // let b_slice = b.slice(0, 5);
-        // assert!(!a_slice.equals(&*b_slice));
-        // assert!(!b_slice.equals(&*a_slice));
-
-        // let a_slice = a.slice(4, 1);
-        // let b_slice = b.slice(4, 1);
-        // assert!(a_slice.equals(&*b_slice));
-        // assert!(b_slice.equals(&*a_slice));
-    }
-
-    #[test]
-    fn test_string_equal() {
-        let a = StringArray::from(vec!["hello", "world"]);
-        let b = StringArray::from(vec!["hello", "world"]);
-        assert!(a.equals(&b));
-        assert!(b.equals(&a));
-
-        let b = StringArray::from(vec!["hello", "arrow"]);
-        assert!(!a.equals(&b));
-        assert!(!b.equals(&a));
-
-        // Test the case where null_count > 0
-
-        let a = StringArray::try_from(vec![
-            Some("hello"),
-            None,
-            None,
-            Some("world"),
-            None,
-            None,
-        ])
-        .unwrap();
-
-        let b = StringArray::try_from(vec![
-            Some("hello"),
-            None,
-            None,
-            Some("world"),
-            None,
-            None,
-        ])
-        .unwrap();
-        assert!(a.equals(&b));
-        assert!(b.equals(&a));
-
-        let b = StringArray::try_from(vec![
-            Some("hello"),
-            Some("foo"),
-            None,
-            Some("world"),
-            None,
-            None,
-        ])
-        .unwrap();
-        assert!(!a.equals(&b));
-        assert!(!b.equals(&a));
-
-        let b = StringArray::try_from(vec![
-            Some("hello"),
-            None,
-            None,
-            Some("arrow"),
-            None,
-            None,
-        ])
-        .unwrap();
-        assert!(!a.equals(&b));
-        assert!(!b.equals(&a));
-
-        // Test the case where offset != 0
-
-        let a_slice = a.slice(0, 3);
-        let b_slice = b.slice(0, 3);
-        assert!(a_slice.equals(&*b_slice));
-        assert!(b_slice.equals(&*a_slice));
-
-        let a_slice = a.slice(0, 5);
-        let b_slice = b.slice(0, 5);
-        assert!(!a_slice.equals(&*b_slice));
-        assert!(!b_slice.equals(&*a_slice));
-
-        let a_slice = a.slice(4, 1);
-        let b_slice = b.slice(4, 1);
-        assert!(a_slice.equals(&*b_slice));
-        assert!(b_slice.equals(&*a_slice));
-    }
-
-    #[test]
-    fn test_large_string_equal() {
-        let a = LargeStringArray::from(vec!["hello", "world"]);
-        let b = LargeStringArray::from(vec!["hello", "world"]);
-        assert!(a.equals(&b));
-        assert!(b.equals(&a));
-
-        let b = LargeStringArray::from(vec!["hello", "arrow"]);
-        assert!(!a.equals(&b));
-        assert!(!b.equals(&a));
-
-        // Test the case where null_count > 0
-
-        let a = LargeStringArray::try_from(vec![
-            Some("hello"),
-            None,
-            None,
-            Some("world"),
-            None,
-            None,
-        ])
-        .unwrap();
-
-        let b = LargeStringArray::try_from(vec![
-            Some("hello"),
-            None,
-            None,
-            Some("world"),
-            None,
-            None,
-        ])
-        .unwrap();
-        assert!(a.equals(&b));
-        assert!(b.equals(&a));
-
-        let b = LargeStringArray::try_from(vec![
-            Some("hello"),
-            Some("foo"),
-            None,
-            Some("world"),
-            None,
-            None,
-        ])
-        .unwrap();
-        assert!(!a.equals(&b));
-        assert!(!b.equals(&a));
-
-        let b = LargeStringArray::try_from(vec![
-            Some("hello"),
-            None,
-            None,
-            Some("arrow"),
-            None,
-            None,
-        ])
-        .unwrap();
-        assert!(!a.equals(&b));
-        assert!(!b.equals(&a));
-
-        // Test the case where offset != 0
-
-        let a_slice = a.slice(0, 3);
-        let b_slice = b.slice(0, 3);
-        assert!(a_slice.equals(&*b_slice));
-        assert!(b_slice.equals(&*a_slice));
-
-        let a_slice = a.slice(0, 5);
-        let b_slice = b.slice(0, 5);
-        assert!(!a_slice.equals(&*b_slice));
-        assert!(!b_slice.equals(&*a_slice));
-
-        let a_slice = a.slice(4, 1);
-        let b_slice = b.slice(4, 1);
-        assert!(a_slice.equals(&*b_slice));
-        assert!(b_slice.equals(&*a_slice));
-    }
-
-    #[test]
-    fn test_struct_equal() {
-        let string_builder = StringBuilder::new(5);
-        let int_builder = Int32Builder::new(5);
-
-        let mut fields = Vec::new();
-        let mut field_builders = Vec::new();
-        fields.push(Field::new("f1", DataType::Utf8, false));
-        field_builders.push(Box::new(string_builder) as Box<ArrayBuilder>);
-        fields.push(Field::new("f2", DataType::Int32, false));
-        field_builders.push(Box::new(int_builder) as Box<ArrayBuilder>);
-
-        let mut builder = StructBuilder::new(fields, field_builders);
-
-        let a = create_struct_array(
-            &mut builder,
-            &[Some("joe"), None, None, Some("mark"), Some("doe")],
-            &[Some(1), Some(2), None, Some(4), Some(5)],
-            &[true, true, false, true, true],
-        )
-        .unwrap();
-        let b = create_struct_array(
-            &mut builder,
-            &[Some("joe"), None, None, Some("mark"), Some("doe")],
-            &[Some(1), Some(2), None, Some(4), Some(5)],
-            &[true, true, false, true, true],
-        )
-        .unwrap();
-
-        assert!(a.equals(&b));
-        assert!(b.equals(&a));
-    }
-
-    #[test]
-    fn test_null_equal() {
-        let a = NullArray::new(12);
-        let b = NullArray::new(12);
-        assert!(a.equals(&b));
-        assert!(b.equals(&a));
-
-        let b = NullArray::new(10);
-        assert!(!a.equals(&b));
-        assert!(!b.equals(&a));
-
-        // Test the case where offset != 0
-
-        let a_slice = a.slice(2, 3);
-        let b_slice = b.slice(1, 3);
-        assert!(a_slice.equals(&*b_slice));
-        assert!(b_slice.equals(&*a_slice));
-
-        let a_slice = a.slice(5, 4);
-        let b_slice = b.slice(3, 3);
-        assert!(!a_slice.equals(&*b_slice));
-        assert!(!b_slice.equals(&*a_slice));
-    }
-
-    fn create_list_array<'a, U: AsRef<[i32]>, T: AsRef<[Option<U>]>>(
-        builder: &'a mut ListBuilder<Int32Builder>,
-        data: T,
-    ) -> Result<ListArray> {
-        for d in data.as_ref() {
-            if let Some(v) = d {
-                builder.values().append_slice(v.as_ref())?;
-                builder.append(true)?
-            } else {
-                builder.append(false)?
-            }
-        }
-        Ok(builder.finish())
-    }
-
-    /// Create a fixed size list of 2 value lengths
-    fn create_fixed_size_list_array<'a, U: AsRef<[i32]>, T: AsRef<[Option<U>]>>(
-        builder: &'a mut FixedSizeListBuilder<Int32Builder>,
-        data: T,
-    ) -> Result<FixedSizeListArray> {
-        for d in data.as_ref() {
-            if let Some(v) = d {
-                builder.values().append_slice(v.as_ref())?;
-                builder.append(true)?
-            } else {
-                for _ in 0..builder.value_length() {
-                    builder.values().append_null()?;
-                }
-                builder.append(false)?
-            }
-        }
-        Ok(builder.finish())
-    }
-
-    #[test]
-    fn test_primitive_json_equal() {
-        // Test equaled array
-        let arrow_array = Int32Array::from(vec![Some(1), None, Some(2), Some(3)]);
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                1, null, 2, 3
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.eq(&json_array));
-        assert!(json_array.eq(&arrow_array));
-
-        // Test unequaled array
-        let arrow_array = Int32Array::from(vec![Some(1), None, Some(2), Some(3)]);
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                1, 1, 2, 3
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-
-        // Test unequal length case
-        let arrow_array = Int32Array::from(vec![Some(1), None, Some(2), Some(3)]);
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                1, 1
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-
-        // Test not json array type case
-        let arrow_array = Int32Array::from(vec![Some(1), None, Some(2), Some(3)]);
-        let json_array: Value = serde_json::from_str(
-            r#"
-            {
-               "a": 1
-            }
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-    }
-
-    #[test]
-    fn test_list_json_equal() {
-        // Test equal case
-        let arrow_array = create_list_array(
-            &mut ListBuilder::new(Int32Builder::new(10)),
-            &[Some(&[1, 2, 3]), None, Some(&[4, 5, 6])],
-        )
-        .unwrap();
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                [1, 2, 3],
-                null,
-                [4, 5, 6]
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.eq(&json_array));
-        assert!(json_array.eq(&arrow_array));
-
-        // Test unequal case
-        let arrow_array = create_list_array(
-            &mut ListBuilder::new(Int32Builder::new(10)),
-            &[Some(&[1, 2, 3]), None, Some(&[4, 5, 6])],
-        )
-        .unwrap();
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                [1, 2, 3],
-                [7, 8],
-                [4, 5, 6]
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-
-        // Test incorrect type case
-        let arrow_array = create_list_array(
-            &mut ListBuilder::new(Int32Builder::new(10)),
-            &[Some(&[1, 2, 3]), None, Some(&[4, 5, 6])],
-        )
-        .unwrap();
-        let json_array: Value = serde_json::from_str(
-            r#"
-            {
-               "a": 1
-            }
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-    }
-
-    #[test]
-    fn test_fixed_size_list_json_equal() {
-        // Test equal case
-        let arrow_array = create_fixed_size_list_array(
-            &mut FixedSizeListBuilder::new(Int32Builder::new(10), 3),
-            &[Some(&[1, 2, 3]), None, Some(&[4, 5, 6])],
-        )
-        .unwrap();
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                [1, 2, 3],
-                null,
-                [4, 5, 6]
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.eq(&json_array));
-        assert!(json_array.eq(&arrow_array));
-
-        // Test unequal case
-        let arrow_array = create_fixed_size_list_array(
-            &mut FixedSizeListBuilder::new(Int32Builder::new(10), 3),
-            &[Some(&[1, 2, 3]), None, Some(&[4, 5, 6])],
-        )
-        .unwrap();
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                [1, 2, 3],
-                [7, 8, 9],
-                [4, 5, 6]
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-
-        // Test incorrect type case
-        let arrow_array = create_fixed_size_list_array(
-            &mut FixedSizeListBuilder::new(Int32Builder::new(10), 3),
-            &[Some(&[1, 2, 3]), None, Some(&[4, 5, 6])],
-        )
-        .unwrap();
-        let json_array: Value = serde_json::from_str(
-            r#"
-            {
-               "a": 1
-            }
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-    }
-
-    #[test]
-    fn test_string_json_equal() {
-        // Test the equal case
-        let arrow_array = StringArray::try_from(vec![
-            Some("hello"),
-            None,
-            None,
-            Some("world"),
-            None,
-            None,
-        ])
-        .unwrap();
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                "hello",
-                null,
-                null,
-                "world",
-                null,
-                null
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.eq(&json_array));
-        assert!(json_array.eq(&arrow_array));
-
-        // Test unequal case
-        let arrow_array = StringArray::try_from(vec![
-            Some("hello"),
-            None,
-            None,
-            Some("world"),
-            None,
-            None,
-        ])
-        .unwrap();
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                "hello",
-                null,
-                null,
-                "arrow",
-                null,
-                null
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-
-        // Test unequal length case
-        let arrow_array =
-            StringArray::try_from(vec![Some("hello"), None, None, Some("world"), None])
-                .unwrap();
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                "hello",
-                null,
-                null,
-                "arrow",
-                null,
-                null
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-
-        // Test incorrect type case
-        let arrow_array =
-            StringArray::try_from(vec![Some("hello"), None, None, Some("world"), None])
-                .unwrap();
-        let json_array: Value = serde_json::from_str(
-            r#"
-            {
-                "a": 1
-            }
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-
-        // Test incorrect value type case
-        let arrow_array =
-            StringArray::try_from(vec![Some("hello"), None, None, Some("world"), None])
-                .unwrap();
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                "hello",
-                null,
-                null,
-                1,
-                null,
-                null
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-    }
-
-    #[test]
-    fn test_binary_json_equal() {
-        // Test the equal case
-        let arrow_array = StringArray::try_from(vec![
-            Some("hello"),
-            None,
-            None,
-            Some("world"),
-            None,
-            None,
-        ])
-        .unwrap();
-        let arrow_array = BinaryArray::from(arrow_array.data());
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                "hello",
-                null,
-                null,
-                "world",
-                null,
-                null
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.eq(&json_array));
-        assert!(json_array.eq(&arrow_array));
-
-        // Test unequal case
-        let arrow_array = StringArray::try_from(vec![
-            Some("hello"),
-            None,
-            None,
-            Some("world"),
-            None,
-            None,
-        ])
-        .unwrap();
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                "hello",
-                null,
-                null,
-                "arrow",
-                null,
-                null
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-
-        // Test unequal length case
-        let arrow_array =
-            StringArray::try_from(vec![Some("hello"), None, None, Some("world"), None])
-                .unwrap();
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                "hello",
-                null,
-                null,
-                "arrow",
-                null,
-                null
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-
-        // Test incorrect type case
-        let arrow_array =
-            StringArray::try_from(vec![Some("hello"), None, None, Some("world"), None])
-                .unwrap();
-        let json_array: Value = serde_json::from_str(
-            r#"
-            {
-                "a": 1
-            }
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-
-        // Test incorrect value type case
-        let arrow_array =
-            StringArray::try_from(vec![Some("hello"), None, None, Some("world"), None])
-                .unwrap();
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                "hello",
-                null,
-                null,
-                1,
-                null,
-                null
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-    }
-
-    #[test]
-    fn test_fixed_size_binary_json_equal() {
-        // Test the equal case
-        let mut builder = FixedSizeBinaryBuilder::new(15, 5);
-        builder.append_value(b"hello").unwrap();
-        builder.append_null().unwrap();
-        builder.append_value(b"world").unwrap();
-        let arrow_array: FixedSizeBinaryArray = builder.finish();
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                "hello",
-                null,
-                "world"
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.eq(&json_array));
-        assert!(json_array.eq(&arrow_array));
-
-        // Test unequal case
-        builder.append_value(b"hello").unwrap();
-        builder.append_null().unwrap();
-        builder.append_value(b"world").unwrap();
-        let arrow_array: FixedSizeBinaryArray = builder.finish();
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                "hello",
-                null,
-                "arrow"
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-
-        // Test unequal length case
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                "hello",
-                null,
-                null,
-                "world"
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-
-        // Test incorrect type case
-        let json_array: Value = serde_json::from_str(
-            r#"
-            {
-                "a": 1
-            }
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-
-        // Test incorrect value type case
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                "hello",
-                null,
-                1
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-    }
-
-    #[test]
-    fn test_struct_json_equal() {
-        // Test equal case
-        let string_builder = StringBuilder::new(5);
-        let int_builder = Int32Builder::new(5);
-
-        let mut fields = Vec::new();
-        let mut field_builders = Vec::new();
-        fields.push(Field::new("f1", DataType::Utf8, false));
-        field_builders.push(Box::new(string_builder) as Box<ArrayBuilder>);
-        fields.push(Field::new("f2", DataType::Int32, false));
-        field_builders.push(Box::new(int_builder) as Box<ArrayBuilder>);
-
-        let mut builder = StructBuilder::new(fields, field_builders);
-
-        let arrow_array = create_struct_array(
-            &mut builder,
-            &[Some("joe"), None, None, Some("mark"), Some("doe")],
-            &[Some(1), Some(2), None, Some(4), Some(5)],
-            &[true, true, false, true, true],
-        )
-        .unwrap();
-
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-              {
-                "f1": "joe",
-                "f2": 1
-              },
-              {
-                "f2": 2
-              },
-              null,
-              {
-                "f1": "mark",
-                "f2": 4
-              },
-              {
-                "f1": "doe",
-                "f2": 5
-              }
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.eq(&json_array));
-        assert!(json_array.eq(&arrow_array));
-
-        // Test unequal length case
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-              {
-                "f1": "joe",
-                "f2": 1
-              },
-              {
-                "f2": 2
-              },
-              null,
-              {
-                "f1": "mark",
-                "f2": 4
-              }
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-
-        // Test incorrect type case
-        let json_array: Value = serde_json::from_str(
-            r#"
-              {
-                "f1": "joe",
-                "f2": 1
-              }
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-
-        // Test not all object case
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-              {
-                "f1": "joe",
-                "f2": 1
-              },
-              2,
-              null,
-              {
-                "f1": "mark",
-                "f2": 4
-              }
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-    }
-
-    fn create_struct_array<
-        'a,
-        T: AsRef<[Option<&'a str>]>,
-        U: AsRef<[Option<i32>]>,
-        V: AsRef<[bool]>,
-    >(
-        builder: &'a mut StructBuilder,
-        first: T,
-        second: U,
-        is_valid: V,
-    ) -> Result<StructArray> {
-        let string_builder = builder.field_builder::<StringBuilder>(0).unwrap();
-        for v in first.as_ref() {
-            if let Some(s) = v {
-                string_builder.append_value(s)?;
-            } else {
-                string_builder.append_null()?;
-            }
-        }
-
-        let int_builder = builder.field_builder::<Int32Builder>(1).unwrap();
-        for v in second.as_ref() {
-            if let Some(i) = v {
-                int_builder.append_value(*i)?;
-            } else {
-                int_builder.append_null()?;
-            }
-        }
-
-        for v in is_valid.as_ref() {
-            builder.append(*v)?
-        }
-
-        Ok(builder.finish())
-    }
-
-    #[test]
-    fn test_null_json_equal() {
-        // Test equaled array
-        let arrow_array = NullArray::new(4);
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                null, null, null, null
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.eq(&json_array));
-        assert!(json_array.eq(&arrow_array));
-
-        // Test unequaled array
-        let arrow_array = NullArray::new(2);
-        let json_array: Value = serde_json::from_str(
-            r#"
-            [
-                null, null, null
-            ]
-        "#,
-        )
-        .unwrap();
-        assert!(arrow_array.ne(&json_array));
-        assert!(json_array.ne(&arrow_array));
-    }
-}
diff --git a/rust/arrow/src/array/equal/boolean.rs b/rust/arrow/src/array/equal/boolean.rs
new file mode 100644
index 00000000000..4158080b81d
--- /dev/null
+++ b/rust/arrow/src/array/equal/boolean.rs
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::array::ArrayData;
+
+use super::utils::equal_bits;
+
+pub(super) fn boolean_equal(
+    lhs: &ArrayData,
+    rhs: &ArrayData,
+    lhs_start: usize,
+    rhs_start: usize,
+    len: usize,
+) -> bool {
+    let lhs_values = lhs.buffers()[0].data();
+    let rhs_values = rhs.buffers()[0].data();
+
+    // TODO: we can do this more efficiently if all values are not-null
+    (0..len).all(|i| {
+        let lhs_pos = lhs_start + i;
+        let rhs_pos = rhs_start + i;
+        let lhs_is_null = lhs.is_null(lhs_pos);
+        let rhs_is_null = rhs.is_null(rhs_pos);
+
+        lhs_is_null
+            || (lhs_is_null == rhs_is_null)
+                && equal_bits(
+                    lhs_values,
+                    rhs_values,
+                    lhs_pos + lhs.offset(),
+                    rhs_pos + rhs.offset(),
+                    1,
+                )
+    })
+}
diff --git a/rust/arrow/src/array/equal/dictionary.rs b/rust/arrow/src/array/equal/dictionary.rs
new file mode 100644
index 00000000000..a41b0a9b74e
--- /dev/null
+++ b/rust/arrow/src/array/equal/dictionary.rs
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::{array::ArrayData, datatypes::ArrowNativeType};
+
+use super::equal_range;
+
+pub(super) fn dictionary_equal<T: ArrowNativeType>(
+    lhs: &ArrayData,
+    rhs: &ArrayData,
+    lhs_start: usize,
+    rhs_start: usize,
+    len: usize,
+) -> bool {
+    let lhs_keys = lhs.buffer::<T>(0);
+    let rhs_keys = rhs.buffer::<T>(0);
+
+    let lhs_values = lhs.child_data()[0].as_ref();
+    let rhs_values = rhs.child_data()[0].as_ref();
+
+    if lhs.null_count() == 0 && rhs.null_count() == 0 {
+        (0..len).all(|i| {
+            let lhs_pos = lhs_start + i;
+            let rhs_pos = rhs_start + i;
+
+            equal_range(
+                lhs_values,
+                rhs_values,
+                lhs_keys[lhs_pos].to_usize().unwrap(),
+                rhs_keys[rhs_pos].to_usize().unwrap(),
+                1,
+            )
+        })
+    } else {
+        (0..len).all(|i| {
+            let lhs_pos = lhs_start + i;
+            let rhs_pos = rhs_start + i;
+
+            let lhs_is_null = lhs.is_null(lhs_pos);
+            let rhs_is_null = rhs.is_null(rhs_pos);
+
+            lhs_is_null
+                || (lhs_is_null == rhs_is_null)
+                    && equal_range(
+                        lhs_values,
+                        rhs_values,
+                        lhs_keys[lhs_pos].to_usize().unwrap(),
+                        rhs_keys[rhs_pos].to_usize().unwrap(),
+                        1,
+                    )
+        })
+    }
+}
diff --git a/rust/arrow/src/array/equal/fixed_binary.rs b/rust/arrow/src/array/equal/fixed_binary.rs
new file mode 100644
index 00000000000..e0fdf07ec85
--- /dev/null
+++ b/rust/arrow/src/array/equal/fixed_binary.rs
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::{array::ArrayData, datatypes::DataType};
+
+use super::utils::equal_len;
+
+pub(super) fn fixed_binary_equal(
+    lhs: &ArrayData,
+    rhs: &ArrayData,
+    lhs_start: usize,
+    rhs_start: usize,
+    len: usize,
+) -> bool {
+    let size = match lhs.data_type() {
+        DataType::FixedSizeBinary(i) => *i as usize,
+        _ => unreachable!(),
+    };
+
+    let lhs_values = lhs.buffer::<u8>(0);
+    let rhs_values = rhs.buffer::<u8>(0);
+
+    if lhs.null_count() == 0 && rhs.null_count() == 0 {
+        equal_len(
+            lhs_values,
+            rhs_values,
+            size * lhs_start,
+            size * rhs_start,
+            size * len,
+        )
+    } else {
+        // with nulls, we need to compare item by item whenever it is not null
+        (0..len).all(|i| {
+            let lhs_pos = lhs_start + i;
+            let rhs_pos = rhs_start + i;
+
+            let lhs_is_null = lhs.is_null(lhs_pos);
+            let rhs_is_null = rhs.is_null(rhs_pos);
+
+            lhs_is_null
+                || (lhs_is_null == rhs_is_null)
+                    && equal_len(
+                        lhs_values,
+                        rhs_values,
+                        lhs_pos * size,
+                        rhs_pos * size,
+                        size, // 1 * size since we are comparing a single entry
+                    )
+        })
+    }
+}
diff --git a/rust/arrow/src/array/equal/fixed_list.rs b/rust/arrow/src/array/equal/fixed_list.rs
new file mode 100644
index 00000000000..aeb0d1372c8
--- /dev/null
+++ b/rust/arrow/src/array/equal/fixed_list.rs
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::{array::ArrayData, datatypes::DataType};
+
+use super::equal_range;
+
+pub(super) fn fixed_list_equal(
+    lhs: &ArrayData,
+    rhs: &ArrayData,
+    lhs_start: usize,
+    rhs_start: usize,
+    len: usize,
+) -> bool {
+    let size = match lhs.data_type() {
+        DataType::FixedSizeList(_, i) => *i as usize,
+        _ => unreachable!(),
+    };
+
+    let lhs_values = lhs.child_data()[0].as_ref();
+    let rhs_values = rhs.child_data()[0].as_ref();
+
+    if lhs.null_count() == 0 && rhs.null_count() == 0 {
+        equal_range(
+            lhs_values,
+            rhs_values,
+            size * lhs_start,
+            size * rhs_start,
+            size * len,
+        )
+    } else {
+        // with nulls, we need to compare item by item whenever it is not null
+        (0..len).all(|i| {
+            let lhs_pos = lhs_start + i;
+            let rhs_pos = rhs_start + i;
+
+            let lhs_is_null = lhs.is_null(lhs_pos);
+            let rhs_is_null = rhs.is_null(rhs_pos);
+
+            lhs_is_null
+                || (lhs_is_null == rhs_is_null)
+                    && equal_range(
+                        lhs_values,
+                        rhs_values,
+                        lhs_pos * size,
+                        rhs_pos * size,
+                        size, // 1 * size since we are comparing a single entry
+                    )
+        })
+    }
+}
diff --git a/rust/arrow/src/array/equal/list.rs b/rust/arrow/src/array/equal/list.rs
new file mode 100644
index 00000000000..7e81a342443
--- /dev/null
+++ b/rust/arrow/src/array/equal/list.rs
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::{array::ArrayData, array::OffsetSizeTrait};
+
+use super::equal_range;
+
+fn lengths_equal<T: OffsetSizeTrait>(lhs: &[T], rhs: &[T]) -> bool {
+    // invariant from `base_equal`
+    debug_assert_eq!(lhs.len(), rhs.len());
+
+    if lhs.len() == 0 {
+        return true;
+    }
+
+    if lhs[0] == T::zero() && rhs[0] == T::zero() {
+        return lhs == rhs;
+    };
+
+    // The expensive case, e.g.
+    // [0, 2, 4, 6, 9] == [4, 6, 8, 10, 13]
+    lhs.windows(2)
+        .zip(rhs.windows(2))
+        .all(|(lhs_offsets, rhs_offsets)| {
+            // length of left == length of right
+            (lhs_offsets[1] - lhs_offsets[0]) == (rhs_offsets[1] - rhs_offsets[0])
+        })
+}
+
+#[inline]
+fn offset_value_equal<T: OffsetSizeTrait>(
+    lhs_values: &ArrayData,
+    rhs_values: &ArrayData,
+    lhs_offsets: &[T],
+    rhs_offsets: &[T],
+    lhs_pos: usize,
+    rhs_pos: usize,
+    len: usize,
+) -> bool {
+    let lhs_start = lhs_offsets[lhs_pos].to_usize().unwrap();
+    let rhs_start = rhs_offsets[rhs_pos].to_usize().unwrap();
+    let lhs_len = lhs_offsets[lhs_pos + len] - lhs_offsets[lhs_pos];
+    let rhs_len = rhs_offsets[rhs_pos + len] - rhs_offsets[rhs_pos];
+
+    lhs_len == rhs_len
+        && equal_range(
+            lhs_values,
+            rhs_values,
+            lhs_start,
+            rhs_start,
+            lhs_len.to_usize().unwrap(),
+        )
+}
+
+pub(super) fn list_equal<T: OffsetSizeTrait>(
+    lhs: &ArrayData,
+    rhs: &ArrayData,
+    lhs_start: usize,
+    rhs_start: usize,
+    len: usize,
+) -> bool {
+    let lhs_offsets = lhs.buffer::<T>(0);
+    let rhs_offsets = rhs.buffer::<T>(0);
+
+    let lhs_values = lhs.child_data()[0].as_ref();
+    let rhs_values = rhs.child_data()[0].as_ref();
+
+    if lhs.null_count() == 0 && rhs.null_count() == 0 {
+        lengths_equal(
+            &lhs_offsets[lhs_start..lhs_start + len],
+            &rhs_offsets[rhs_start..rhs_start + len],
+        ) && equal_range(
+            lhs_values,
+            rhs_values,
+            lhs_offsets[lhs_start].to_usize().unwrap(),
+            rhs_offsets[rhs_start].to_usize().unwrap(),
+            (lhs_offsets[len] - lhs_offsets[lhs_start])
+                .to_usize()
+                .unwrap(),
+        )
+    } else {
+        // with nulls, we need to compare item by item whenever it is not null
+        (0..len).all(|i| {
+            let lhs_pos = lhs_start + i;
+            let rhs_pos = rhs_start + i;
+
+            let lhs_is_null = lhs.is_null(lhs_pos);
+            let rhs_is_null = rhs.is_null(rhs_pos);
+
+            lhs_is_null
+                || (lhs_is_null == rhs_is_null)
+                    && offset_value_equal::<T>(
+                        lhs_values,
+                        rhs_values,
+                        lhs_offsets,
+                        rhs_offsets,
+                        lhs_pos,
+                        rhs_pos,
+                        1,
+                    )
+        })
+    }
+}
diff --git a/rust/arrow/src/array/equal/mod.rs b/rust/arrow/src/array/equal/mod.rs
new file mode 100644
index 00000000000..ab063754051
--- /dev/null
+++ b/rust/arrow/src/array/equal/mod.rs
@@ -0,0 +1,831 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Module containing functionality to compute array equality.
+//! This module uses [ArrayData] and does not
+//! depend on dynamic casting of `Array`.
+
+use super::{
+    Array, ArrayData, BinaryOffsetSizeTrait, FixedSizeBinaryArray, GenericBinaryArray,
+    GenericListArray, GenericStringArray, OffsetSizeTrait, PrimitiveArray,
+    StringOffsetSizeTrait, StructArray,
+};
+
+use crate::datatypes::{ArrowPrimitiveType, DataType, IntervalUnit};
+
+mod boolean;
+mod dictionary;
+mod fixed_binary;
+mod fixed_list;
+mod list;
+mod null;
+mod primitive;
+mod structure;
+mod utils;
+mod variable_size;
+
+// these methods assume the same type, len and null count.
+// For this reason, they are not exposed and are instead used
+// to build the generic functions below (`equal_range` and `equal`).
+use boolean::boolean_equal;
+use dictionary::dictionary_equal;
+use fixed_binary::fixed_binary_equal;
+use fixed_list::fixed_list_equal;
+use list::list_equal;
+use null::null_equal;
+use primitive::primitive_equal;
+use structure::struct_equal;
+use variable_size::variable_sized_equal;
+
+impl PartialEq for dyn Array {
+    fn eq(&self, other: &Self) -> bool {
+        equal(self.data().as_ref(), other.data().as_ref())
+    }
+}
+
+impl<T: Array> PartialEq<T> for dyn Array {
+    fn eq(&self, other: &T) -> bool {
+        equal(self.data().as_ref(), other.data().as_ref())
+    }
+}
+
+impl<T: ArrowPrimitiveType> PartialEq for PrimitiveArray<T> {
+    fn eq(&self, other: &PrimitiveArray<T>) -> bool {
+        equal(self.data().as_ref(), other.data().as_ref())
+    }
+}
+
+impl<OffsetSize: StringOffsetSizeTrait> PartialEq for GenericStringArray<OffsetSize> {
+    fn eq(&self, other: &Self) -> bool {
+        equal(self.data().as_ref(), other.data().as_ref())
+    }
+}
+
+impl<OffsetSize: BinaryOffsetSizeTrait> PartialEq for GenericBinaryArray<OffsetSize> {
+    fn eq(&self, other: &Self) -> bool {
+        equal(self.data().as_ref(), other.data().as_ref())
+    }
+}
+
+impl PartialEq for FixedSizeBinaryArray {
+    fn eq(&self, other: &Self) -> bool {
+        equal(self.data().as_ref(), other.data().as_ref())
+    }
+}
+
+impl<OffsetSize: OffsetSizeTrait> PartialEq for GenericListArray<OffsetSize> {
+    fn eq(&self, other: &Self) -> bool {
+        equal(self.data().as_ref(), other.data().as_ref())
+    }
+}
+
+impl PartialEq for StructArray {
+    fn eq(&self, other: &Self) -> bool {
+        equal(self.data().as_ref(), other.data().as_ref())
+    }
+}
+
+/// Compares the values of two [ArrayData] starting at `lhs_start` and `rhs_start` respectively
+/// for `len` slots.
+#[inline]
+fn equal_values(
+    lhs: &ArrayData,
+    rhs: &ArrayData,
+    lhs_start: usize,
+    rhs_start: usize,
+    len: usize,
+) -> bool {
+    match lhs.data_type() {
+        DataType::Null => null_equal(lhs, rhs, lhs_start, rhs_start, len),
+        DataType::Boolean => boolean_equal(lhs, rhs, lhs_start, rhs_start, len),
+        DataType::UInt8 => primitive_equal::<u8>(lhs, rhs, lhs_start, rhs_start, len),
+        DataType::UInt16 => primitive_equal::<u16>(lhs, rhs, lhs_start, rhs_start, len),
+        DataType::UInt32 => primitive_equal::<u32>(lhs, rhs, lhs_start, rhs_start, len),
+        DataType::UInt64 => primitive_equal::<u64>(lhs, rhs, lhs_start, rhs_start, len),
+        DataType::Int8 => primitive_equal::<i8>(lhs, rhs, lhs_start, rhs_start, len),
+        DataType::Int16 => primitive_equal::<i16>(lhs, rhs, lhs_start, rhs_start, len),
+        DataType::Int32 => primitive_equal::<i32>(lhs, rhs, lhs_start, rhs_start, len),
+        DataType::Int64 => primitive_equal::<i64>(lhs, rhs, lhs_start, rhs_start, len),
+        DataType::Float32 => primitive_equal::<f32>(lhs, rhs, lhs_start, rhs_start, len),
+        DataType::Float64 => primitive_equal::<f64>(lhs, rhs, lhs_start, rhs_start, len),
+        DataType::Date32(_)
+        | DataType::Time32(_)
+        | DataType::Interval(IntervalUnit::YearMonth) => {
+            primitive_equal::<i32>(lhs, rhs, lhs_start, rhs_start, len)
+        }
+        DataType::Date64(_)
+        | DataType::Interval(IntervalUnit::DayTime)
+        | DataType::Time64(_)
+        | DataType::Timestamp(_, _)
+        | DataType::Duration(_) => {
+            primitive_equal::<i64>(lhs, rhs, lhs_start, rhs_start, len)
+        }
+        DataType::Utf8 | DataType::Binary => {
+            variable_sized_equal::<i32>(lhs, rhs, lhs_start, rhs_start, len)
+        }
+        DataType::LargeUtf8 | DataType::LargeBinary => {
+            variable_sized_equal::<i64>(lhs, rhs, lhs_start, rhs_start, len)
+        }
+        DataType::FixedSizeBinary(_) => {
+            fixed_binary_equal(lhs, rhs, lhs_start, rhs_start, len)
+        }
+        DataType::List(_) => list_equal::<i32>(lhs, rhs, lhs_start, rhs_start, len),
+        DataType::LargeList(_) => list_equal::<i64>(lhs, rhs, lhs_start, rhs_start, len),
+        DataType::FixedSizeList(_, _) => {
+            fixed_list_equal(lhs, rhs, lhs_start, rhs_start, len)
+        }
+        DataType::Struct(_) => struct_equal(lhs, rhs, lhs_start, rhs_start, len),
+        DataType::Union(_) => unimplemented!("See ARROW-8576"),
+        DataType::Dictionary(data_type, _) => match data_type.as_ref() {
+            DataType::Int8 => dictionary_equal::<i8>(lhs, rhs, lhs_start, rhs_start, len),
+            DataType::Int16 => {
+                dictionary_equal::<i16>(lhs, rhs, lhs_start, rhs_start, len)
+            }
+            DataType::Int32 => {
+                dictionary_equal::<i32>(lhs, rhs, lhs_start, rhs_start, len)
+            }
+            DataType::Int64 => {
+                dictionary_equal::<i64>(lhs, rhs, lhs_start, rhs_start, len)
+            }
+            DataType::UInt8 => {
+                dictionary_equal::<u8>(lhs, rhs, lhs_start, rhs_start, len)
+            }
+            DataType::UInt16 => {
+                dictionary_equal::<u16>(lhs, rhs, lhs_start, rhs_start, len)
+            }
+            DataType::UInt32 => {
+                dictionary_equal::<u32>(lhs, rhs, lhs_start, rhs_start, len)
+            }
+            DataType::UInt64 => {
+                dictionary_equal::<u64>(lhs, rhs, lhs_start, rhs_start, len)
+            }
+            _ => unreachable!(),
+        },
+        DataType::Float16 => unreachable!(),
+    }
+}
+
+fn equal_range(
+    lhs: &ArrayData,
+    rhs: &ArrayData,
+    lhs_start: usize,
+    rhs_start: usize,
+    len: usize,
+) -> bool {
+    utils::base_equal(lhs, rhs)
+        && utils::equal_nulls(lhs, rhs, lhs_start, rhs_start, len)
+        && equal_values(lhs, rhs, lhs_start, rhs_start, len)
+}
+
+/// Logically compares two [ArrayData].
+/// Two arrays are logically equal if and only if:
+/// * their data types are equal
+/// * their lenghts are equal
+/// * their null counts are equal
+/// * their null bitmaps are equal
+/// * each of their items are equal
+/// two items are equal when their in-memory representation is physically equal (i.e. same bit content).
+/// The physical comparison depend on the data type.
+/// # Panics
+/// This function may panic whenever any of the [ArrayData] does not follow the Arrow specification.
+/// (e.g. wrong number of buffers, buffer `len` does not correspond to the declared `len`)
+pub fn equal(lhs: &ArrayData, rhs: &ArrayData) -> bool {
+    utils::base_equal(lhs, rhs)
+        && lhs.null_count() == rhs.null_count()
+        && utils::equal_nulls(lhs, rhs, 0, 0, lhs.len())
+        && equal_values(lhs, rhs, 0, 0, lhs.len())
+}
+
+#[cfg(test)]
+mod tests {
+    use std::convert::TryFrom;
+    use std::sync::Arc;
+
+    use crate::array::{
+        array::Array, ArrayDataRef, ArrayRef, BinaryOffsetSizeTrait, BooleanArray,
+        FixedSizeBinaryBuilder, FixedSizeListBuilder, GenericBinaryArray, Int32Builder,
+        ListBuilder, NullArray, PrimitiveBuilder, StringArray, StringDictionaryBuilder,
+        StringOffsetSizeTrait, StructArray,
+    };
+    use crate::array::{GenericStringArray, Int32Array};
+    use crate::datatypes::Int16Type;
+
+    use super::*;
+
+    #[test]
+    fn test_null_equal() {
+        let a = NullArray::new(12).data();
+        let b = NullArray::new(12).data();
+        test_equal(&a, &b, true);
+
+        let b = NullArray::new(10).data();
+        test_equal(&a, &b, false);
+
+        // Test the case where offset != 0
+
+        let a_slice = a.slice(2, 3);
+        let b_slice = b.slice(1, 3);
+        test_equal(&a_slice, &b_slice, true);
+
+        let a_slice = a.slice(5, 4);
+        let b_slice = b.slice(3, 3);
+        test_equal(&a_slice, &b_slice, false);
+    }
+
+    #[test]
+    fn test_boolean_equal() {
+        let a = BooleanArray::from(vec![false, false, true]).data();
+        let b = BooleanArray::from(vec![false, false, true]).data();
+        test_equal(a.as_ref(), b.as_ref(), true);
+
+        let b = BooleanArray::from(vec![false, false, false]).data();
+        test_equal(a.as_ref(), b.as_ref(), false);
+
+        // Test the case where null_count > 0
+
+        let a = BooleanArray::from(vec![Some(false), None, None, Some(true)]).data();
+        let b = BooleanArray::from(vec![Some(false), None, None, Some(true)]).data();
+        test_equal(a.as_ref(), b.as_ref(), true);
+
+        let b = BooleanArray::from(vec![None, None, None, Some(true)]).data();
+        test_equal(a.as_ref(), b.as_ref(), false);
+
+        let b = BooleanArray::from(vec![Some(true), None, None, Some(true)]).data();
+        test_equal(a.as_ref(), b.as_ref(), false);
+
+        // Test the case where offset != 0
+
+        let a =
+            BooleanArray::from(vec![false, true, false, true, false, false, true]).data();
+        let b =
+            BooleanArray::from(vec![false, false, false, true, false, true, true]).data();
+        assert_eq!(equal(a.as_ref(), b.as_ref()), false);
+        assert_eq!(equal(b.as_ref(), a.as_ref()), false);
+
+        let a_slice = a.slice(2, 3);
+        let b_slice = b.slice(2, 3);
+        assert_eq!(equal(&a_slice, &b_slice), true);
+        assert_eq!(equal(&b_slice, &a_slice), true);
+
+        let a_slice = a.slice(3, 4);
+        let b_slice = b.slice(3, 4);
+        assert_eq!(equal(&a_slice, &b_slice), false);
+        assert_eq!(equal(&b_slice, &a_slice), false);
+    }
+
+    #[test]
+    fn test_primitive() {
+        let cases = vec![
+            (
+                vec![Some(1), Some(2), Some(3)],
+                vec![Some(1), Some(2), Some(3)],
+                true,
+            ),
+            (
+                vec![Some(1), Some(2), Some(3)],
+                vec![Some(1), Some(2), Some(4)],
+                false,
+            ),
+            (
+                vec![Some(1), Some(2), None],
+                vec![Some(1), Some(2), None],
+                true,
+            ),
+            (
+                vec![Some(1), None, Some(3)],
+                vec![Some(1), Some(2), None],
+                false,
+            ),
+            (
+                vec![Some(1), None, None],
+                vec![Some(1), Some(2), None],
+                false,
+            ),
+        ];
+
+        for (lhs, rhs, expected) in cases {
+            let lhs = Int32Array::from(lhs).data();
+            let rhs = Int32Array::from(rhs).data();
+            test_equal(&lhs, &rhs, expected);
+        }
+    }
+
+    #[test]
+    fn test_primitive_slice() {
+        let cases = vec![
+            (
+                vec![Some(1), Some(2), Some(3)],
+                (0, 1),
+                vec![Some(1), Some(2), Some(3)],
+                (0, 1),
+                true,
+            ),
+            (
+                vec![Some(1), Some(2), Some(3)],
+                (1, 1),
+                vec![Some(1), Some(2), Some(3)],
+                (2, 1),
+                false,
+            ),
+            (
+                vec![Some(1), Some(2), None],
+                (1, 1),
+                vec![Some(1), None, Some(2)],
+                (2, 1),
+                true,
+            ),
+        ];
+
+        for (lhs, slice_lhs, rhs, slice_rhs, expected) in cases {
+            let lhs = Int32Array::from(lhs).data();
+            let lhs = lhs.slice(slice_lhs.0, slice_lhs.1);
+            let rhs = Int32Array::from(rhs).data();
+            let rhs = rhs.slice(slice_rhs.0, slice_rhs.1);
+
+            test_equal(&lhs, &rhs, expected);
+        }
+    }
+
+    fn test_equal(lhs: &ArrayData, rhs: &ArrayData, expected: bool) {
+        // equality is symetric
+        assert_eq!(equal(lhs, lhs), true, "\n{:?}\n{:?}", lhs, lhs);
+        assert_eq!(equal(rhs, rhs), true, "\n{:?}\n{:?}", rhs, rhs);
+
+        assert_eq!(equal(lhs, rhs), expected, "\n{:?}\n{:?}", lhs, rhs);
+        assert_eq!(equal(rhs, lhs), expected, "\n{:?}\n{:?}", rhs, lhs);
+    }
+
+    fn binary_cases() -> Vec<(Vec<Option<String>>, Vec<Option<String>>, bool)> {
+        let base = vec![
+            Some("hello".to_owned()),
+            None,
+            None,
+            Some("world".to_owned()),
+            None,
+            None,
+        ];
+        let not_base = vec![
+            Some("hello".to_owned()),
+            Some("foo".to_owned()),
+            None,
+            Some("world".to_owned()),
+            None,
+            None,
+        ];
+        vec![
+            (
+                vec![Some("hello".to_owned()), Some("world".to_owned())],
+                vec![Some("hello".to_owned()), Some("world".to_owned())],
+                true,
+            ),
+            (
+                vec![Some("hello".to_owned()), Some("world".to_owned())],
+                vec![Some("hello".to_owned()), Some("arrow".to_owned())],
+                false,
+            ),
+            (base.clone(), base.clone(), true),
+            (base.clone(), not_base.clone(), false),
+        ]
+    }
+
+    fn test_generic_string_equal<OffsetSize: StringOffsetSizeTrait>() {
+        let cases = binary_cases();
+
+        for (lhs, rhs, expected) in cases {
+            let lhs = lhs.iter().map(|x| x.as_deref()).collect();
+            let rhs = rhs.iter().map(|x| x.as_deref()).collect();
+            let lhs = GenericStringArray::<OffsetSize>::from_opt_vec(lhs).data();
+            let rhs = GenericStringArray::<OffsetSize>::from_opt_vec(rhs).data();
+            test_equal(lhs.as_ref(), rhs.as_ref(), expected);
+        }
+    }
+
+    #[test]
+    fn test_string_equal() {
+        test_generic_string_equal::<i32>()
+    }
+
+    #[test]
+    fn test_large_string_equal() {
+        test_generic_string_equal::<i64>()
+    }
+
+    fn test_generic_binary_equal<OffsetSize: BinaryOffsetSizeTrait>() {
+        let cases = binary_cases();
+
+        for (lhs, rhs, expected) in cases {
+            let lhs = lhs
+                .iter()
+                .map(|x| x.as_deref().map(|x| x.as_bytes()))
+                .collect();
+            let rhs = rhs
+                .iter()
+                .map(|x| x.as_deref().map(|x| x.as_bytes()))
+                .collect();
+            let lhs = GenericBinaryArray::<OffsetSize>::from_opt_vec(lhs).data();
+            let rhs = GenericBinaryArray::<OffsetSize>::from_opt_vec(rhs).data();
+            test_equal(lhs.as_ref(), rhs.as_ref(), expected);
+        }
+    }
+
+    #[test]
+    fn test_binary_equal() {
+        test_generic_binary_equal::<i32>()
+    }
+
+    #[test]
+    fn test_large_binary_equal() {
+        test_generic_binary_equal::<i64>()
+    }
+
+    #[test]
+    fn test_null() {
+        let a = NullArray::new(2).data();
+        let b = NullArray::new(2).data();
+        test_equal(a.as_ref(), b.as_ref(), true);
+
+        let b = NullArray::new(1).data();
+        test_equal(a.as_ref(), b.as_ref(), false);
+    }
+
+    fn create_list_array<U: AsRef<[i32]>, T: AsRef<[Option<U>]>>(
+        data: T,
+    ) -> ArrayDataRef {
+        let mut builder = ListBuilder::new(Int32Builder::new(10));
+        for d in data.as_ref() {
+            if let Some(v) = d {
+                builder.values().append_slice(v.as_ref()).unwrap();
+                builder.append(true).unwrap()
+            } else {
+                builder.append(false).unwrap()
+            }
+        }
+        builder.finish().data()
+    }
+
+    #[test]
+    fn test_list_equal() {
+        let a = create_list_array(&[Some(&[1, 2, 3]), Some(&[4, 5, 6])]);
+        let b = create_list_array(&[Some(&[1, 2, 3]), Some(&[4, 5, 6])]);
+        test_equal(a.as_ref(), b.as_ref(), true);
+
+        let b = create_list_array(&[Some(&[1, 2, 3]), Some(&[4, 5, 7])]);
+        test_equal(a.as_ref(), b.as_ref(), false);
+    }
+
+    // Test the case where null_count > 0
+    #[test]
+    fn test_list_null() {
+        let a =
+            create_list_array(&[Some(&[1, 2]), None, None, Some(&[3, 4]), None, None]);
+        let b =
+            create_list_array(&[Some(&[1, 2]), None, None, Some(&[3, 4]), None, None]);
+        test_equal(a.as_ref(), b.as_ref(), true);
+
+        let b = create_list_array(&[
+            Some(&[1, 2]),
+            None,
+            Some(&[5, 6]),
+            Some(&[3, 4]),
+            None,
+            None,
+        ]);
+        test_equal(a.as_ref(), b.as_ref(), false);
+
+        let b =
+            create_list_array(&[Some(&[1, 2]), None, None, Some(&[3, 5]), None, None]);
+        test_equal(a.as_ref(), b.as_ref(), false);
+    }
+
+    // Test the case where offset != 0
+    #[test]
+    fn test_list_offsets() {
+        let a =
+            create_list_array(&[Some(&[1, 2]), None, None, Some(&[3, 4]), None, None]);
+        let b =
+            create_list_array(&[Some(&[1, 2]), None, None, Some(&[3, 5]), None, None]);
+
+        let a_slice = a.slice(0, 3);
+        let b_slice = b.slice(0, 3);
+        test_equal(&a_slice, &b_slice, true);
+
+        let a_slice = a.slice(0, 5);
+        let b_slice = b.slice(0, 5);
+        test_equal(&a_slice, &b_slice, false);
+
+        let a_slice = a.slice(4, 1);
+        let b_slice = b.slice(4, 1);
+        test_equal(&a_slice, &b_slice, true);
+    }
+
+    fn create_fixed_size_binary_array<U: AsRef<[u8]>, T: AsRef<[Option<U>]>>(
+        data: T,
+    ) -> ArrayDataRef {
+        let mut builder = FixedSizeBinaryBuilder::new(15, 5);
+
+        for d in data.as_ref() {
+            if let Some(v) = d {
+                builder.append_value(v.as_ref()).unwrap();
+            } else {
+                builder.append_null().unwrap();
+            }
+        }
+        builder.finish().data()
+    }
+
+    #[test]
+    fn test_fixed_size_binary_equal() {
+        let a = create_fixed_size_binary_array(&[Some(b"hello"), Some(b"world")]);
+        let b = create_fixed_size_binary_array(&[Some(b"hello"), Some(b"world")]);
+        test_equal(a.as_ref(), b.as_ref(), true);
+
+        let b = create_fixed_size_binary_array(&[Some(b"hello"), Some(b"arrow")]);
+        test_equal(a.as_ref(), b.as_ref(), false);
+    }
+
+    // Test the case where null_count > 0
+    #[test]
+    fn test_fixed_size_binary_null() {
+        let a = create_fixed_size_binary_array(&[Some(b"hello"), None, Some(b"world")]);
+        let b = create_fixed_size_binary_array(&[Some(b"hello"), None, Some(b"world")]);
+        test_equal(a.as_ref(), b.as_ref(), true);
+
+        let b = create_fixed_size_binary_array(&[Some(b"hello"), Some(b"world"), None]);
+        test_equal(a.as_ref(), b.as_ref(), false);
+
+        let b = create_fixed_size_binary_array(&[Some(b"hello"), None, Some(b"arrow")]);
+        test_equal(a.as_ref(), b.as_ref(), false);
+    }
+
+    #[test]
+    fn test_fixed_size_binary_offsets() {
+        // Test the case where offset != 0
+        let a = create_fixed_size_binary_array(&[
+            Some(b"hello"),
+            None,
+            None,
+            Some(b"world"),
+            None,
+            None,
+        ]);
+        let b = create_fixed_size_binary_array(&[
+            Some(b"hello"),
+            None,
+            None,
+            Some(b"arrow"),
+            None,
+            None,
+        ]);
+
+        let a_slice = a.slice(0, 3);
+        let b_slice = b.slice(0, 3);
+        test_equal(&a_slice, &b_slice, true);
+
+        let a_slice = a.slice(0, 5);
+        let b_slice = b.slice(0, 5);
+        test_equal(&a_slice, &b_slice, false);
+
+        let a_slice = a.slice(4, 1);
+        let b_slice = b.slice(4, 1);
+        test_equal(&a_slice, &b_slice, true);
+    }
+
+    /// Create a fixed size list of 2 value lengths
+    fn create_fixed_size_list_array<U: AsRef<[i32]>, T: AsRef<[Option<U>]>>(
+        data: T,
+    ) -> ArrayDataRef {
+        let mut builder = FixedSizeListBuilder::new(Int32Builder::new(10), 3);
+
+        for d in data.as_ref() {
+            if let Some(v) = d {
+                builder.values().append_slice(v.as_ref()).unwrap();
+                builder.append(true).unwrap()
+            } else {
+                for _ in 0..builder.value_length() {
+                    builder.values().append_null().unwrap();
+                }
+                builder.append(false).unwrap()
+            }
+        }
+        builder.finish().data()
+    }
+
+    #[test]
+    fn test_fixed_size_list_equal() {
+        let a = create_fixed_size_list_array(&[Some(&[1, 2, 3]), Some(&[4, 5, 6])]);
+        let b = create_fixed_size_list_array(&[Some(&[1, 2, 3]), Some(&[4, 5, 6])]);
+        test_equal(a.as_ref(), b.as_ref(), true);
+
+        let b = create_fixed_size_list_array(&[Some(&[1, 2, 3]), Some(&[4, 5, 7])]);
+        test_equal(a.as_ref(), b.as_ref(), false);
+    }
+
+    // Test the case where null_count > 0
+    #[test]
+    fn test_fixed_list_null() {
+        let a = create_fixed_size_list_array(&[
+            Some(&[1, 2, 3]),
+            None,
+            None,
+            Some(&[4, 5, 6]),
+            None,
+            None,
+        ]);
+        let b = create_fixed_size_list_array(&[
+            Some(&[1, 2, 3]),
+            None,
+            None,
+            Some(&[4, 5, 6]),
+            None,
+            None,
+        ]);
+        test_equal(a.as_ref(), b.as_ref(), true);
+
+        let b = create_fixed_size_list_array(&[
+            Some(&[1, 2, 3]),
+            None,
+            Some(&[7, 8, 9]),
+            Some(&[4, 5, 6]),
+            None,
+            None,
+        ]);
+        test_equal(a.as_ref(), b.as_ref(), false);
+
+        let b = create_fixed_size_list_array(&[
+            Some(&[1, 2, 3]),
+            None,
+            None,
+            Some(&[3, 6, 9]),
+            None,
+            None,
+        ]);
+        test_equal(a.as_ref(), b.as_ref(), false);
+    }
+
+    #[test]
+    fn test_fixed_list_offsets() {
+        // Test the case where offset != 0
+        let a = create_fixed_size_list_array(&[
+            Some(&[1, 2, 3]),
+            None,
+            None,
+            Some(&[4, 5, 6]),
+            None,
+            None,
+        ]);
+        let b = create_fixed_size_list_array(&[
+            Some(&[1, 2, 3]),
+            None,
+            None,
+            Some(&[3, 6, 9]),
+            None,
+            None,
+        ]);
+
+        let a_slice = a.slice(0, 3);
+        let b_slice = b.slice(0, 3);
+        test_equal(&a_slice, &b_slice, true);
+
+        let a_slice = a.slice(0, 5);
+        let b_slice = b.slice(0, 5);
+        test_equal(&a_slice, &b_slice, false);
+
+        let a_slice = a.slice(4, 1);
+        let b_slice = b.slice(4, 1);
+        test_equal(&a_slice, &b_slice, true);
+    }
+
+    #[test]
+    fn test_struct_equal() {
+        let strings: ArrayRef = Arc::new(StringArray::from(vec![
+            Some("joe"),
+            None,
+            None,
+            Some("mark"),
+            Some("doe"),
+        ]));
+        let ints: ArrayRef = Arc::new(Int32Array::from(vec![
+            Some(1),
+            Some(2),
+            None,
+            Some(4),
+            Some(5),
+        ]));
+
+        let a =
+            StructArray::try_from(vec![("f1", strings.clone()), ("f2", ints.clone())])
+                .unwrap()
+                .data();
+
+        let b = StructArray::try_from(vec![("f1", strings), ("f2", ints)])
+            .unwrap()
+            .data();
+
+        test_equal(a.as_ref(), b.as_ref(), true);
+    }
+
+    fn create_dictionary_array(values: &[&str], keys: &[Option<&str>]) -> ArrayDataRef {
+        let values = StringArray::from(values.to_vec());
+        let mut builder = StringDictionaryBuilder::new_with_dictionary(
+            PrimitiveBuilder::<Int16Type>::new(3),
+            &values,
+        )
+        .unwrap();
+        for key in keys {
+            if let Some(v) = key {
+                builder.append(v).unwrap();
+            } else {
+                builder.append_null().unwrap()
+            }
+        }
+        builder.finish().data()
+    }
+
+    #[test]
+    fn test_dictionary_equal() {
+        // (a, b, c), (1, 2, 1, 3) => (a, b, a, c)
+        let a = create_dictionary_array(
+            &["a", "b", "c"],
+            &[Some("a"), Some("b"), Some("a"), Some("c")],
+        );
+        // different representation (values and keys are swapped), same result
+        let b = create_dictionary_array(
+            &["a", "c", "b"],
+            &[Some("a"), Some("b"), Some("a"), Some("c")],
+        );
+        test_equal(a.as_ref(), b.as_ref(), true);
+
+        // different len
+        let b =
+            create_dictionary_array(&["a", "c", "b"], &[Some("a"), Some("b"), Some("a")]);
+        test_equal(a.as_ref(), b.as_ref(), false);
+
+        // different key
+        let b = create_dictionary_array(
+            &["a", "c", "b"],
+            &[Some("a"), Some("b"), Some("a"), Some("a")],
+        );
+        test_equal(a.as_ref(), b.as_ref(), false);
+
+        // different values, same keys
+        let b = create_dictionary_array(
+            &["a", "b", "d"],
+            &[Some("a"), Some("b"), Some("a"), Some("d")],
+        );
+        test_equal(a.as_ref(), b.as_ref(), false);
+    }
+
+    #[test]
+    fn test_dictionary_equal_null() {
+        // (a, b, c), (1, 2, 1, 3) => (a, b, a, c)
+        let a = create_dictionary_array(
+            &["a", "b", "c"],
+            &[Some("a"), None, Some("a"), Some("c")],
+        );
+
+        // equal to self
+        test_equal(a.as_ref(), a.as_ref(), true);
+
+        // different representation (values and keys are swapped), same result
+        let b = create_dictionary_array(
+            &["a", "c", "b"],
+            &[Some("a"), None, Some("a"), Some("c")],
+        );
+        test_equal(a.as_ref(), b.as_ref(), true);
+
+        // different null position
+        let b = create_dictionary_array(
+            &["a", "c", "b"],
+            &[Some("a"), Some("b"), Some("a"), None],
+        );
+        test_equal(a.as_ref(), b.as_ref(), false);
+
+        // different key
+        let b = create_dictionary_array(
+            &["a", "c", "b"],
+            &[Some("a"), None, Some("a"), Some("a")],
+        );
+        test_equal(a.as_ref(), b.as_ref(), false);
+
+        // different values, same keys
+        let b = create_dictionary_array(
+            &["a", "b", "d"],
+            &[Some("a"), None, Some("a"), Some("d")],
+        );
+        test_equal(a.as_ref(), b.as_ref(), false);
+    }
+}
diff --git a/rust/arrow/src/array/equal/null.rs b/rust/arrow/src/array/equal/null.rs
new file mode 100644
index 00000000000..f287a382507
--- /dev/null
+++ b/rust/arrow/src/array/equal/null.rs
@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::array::ArrayData;
+
+#[inline]
+pub(super) fn null_equal(
+    _lhs: &ArrayData,
+    _rhs: &ArrayData,
+    _lhs_start: usize,
+    _rhs_start: usize,
+    _len: usize,
+) -> bool {
+    // a null buffer's range is always true, as every element is by definition equal (to null).
+    // We only need to compare data_types
+    true
+}
diff --git a/rust/arrow/src/array/equal/primitive.rs b/rust/arrow/src/array/equal/primitive.rs
new file mode 100644
index 00000000000..19602e46488
--- /dev/null
+++ b/rust/arrow/src/array/equal/primitive.rs
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::mem::size_of;
+
+use crate::array::ArrayData;
+
+use super::utils::equal_len;
+
+pub(super) fn primitive_equal<T>(
+    lhs: &ArrayData,
+    rhs: &ArrayData,
+    lhs_start: usize,
+    rhs_start: usize,
+    len: usize,
+) -> bool {
+    let byte_width = size_of::<T>();
+    let lhs_values = &lhs.buffers()[0].data()[lhs.offset() * byte_width..];
+    let rhs_values = &rhs.buffers()[0].data()[rhs.offset() * byte_width..];
+
+    if lhs.null_count() == 0 && rhs.null_count() == 0 {
+        // without nulls, we just need to compare slices
+        equal_len(
+            lhs_values,
+            rhs_values,
+            lhs_start * byte_width,
+            rhs_start * byte_width,
+            len * byte_width,
+        )
+    } else {
+        // with nulls, we need to compare item by item whenever it is not null
+        (0..len).all(|i| {
+            let lhs_pos = lhs_start + i;
+            let rhs_pos = rhs_start + i;
+            let lhs_is_null = lhs.is_null(lhs_pos);
+            let rhs_is_null = rhs.is_null(rhs_pos);
+
+            lhs_is_null
+                || (lhs_is_null == rhs_is_null)
+                    && equal_len(
+                        lhs_values,
+                        rhs_values,
+                        lhs_pos * byte_width,
+                        rhs_pos * byte_width,
+                        byte_width, // 1 * byte_width since we are comparing a single entry
+                    )
+        })
+    }
+}
diff --git a/rust/arrow/src/array/equal/structure.rs b/rust/arrow/src/array/equal/structure.rs
new file mode 100644
index 00000000000..1e8a1ff260b
--- /dev/null
+++ b/rust/arrow/src/array/equal/structure.rs
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::array::ArrayData;
+
+use super::equal_range;
+
+fn equal_values(
+    lhs: &ArrayData,
+    rhs: &ArrayData,
+    lhs_start: usize,
+    rhs_start: usize,
+    len: usize,
+) -> bool {
+    lhs.child_data()
+        .iter()
+        .zip(rhs.child_data())
+        .all(|(lhs_values, rhs_values)| {
+            equal_range(lhs_values, rhs_values, lhs_start, rhs_start, len)
+        })
+}
+
+pub(super) fn struct_equal(
+    lhs: &ArrayData,
+    rhs: &ArrayData,
+    lhs_start: usize,
+    rhs_start: usize,
+    len: usize,
+) -> bool {
+    if lhs.null_count() == 0 && rhs.null_count() == 0 {
+        equal_values(lhs, rhs, lhs_start, rhs_start, len)
+    } else {
+        // with nulls, we need to compare item by item whenever it is not null
+        (0..len).all(|i| {
+            let lhs_pos = lhs_start + i;
+            let rhs_pos = rhs_start + i;
+            let lhs_is_null = lhs.is_null(lhs_pos);
+            let rhs_is_null = rhs.is_null(rhs_pos);
+
+            lhs_is_null
+                || (lhs_is_null == rhs_is_null)
+                    && equal_values(lhs, rhs, lhs_pos, rhs_pos, 1)
+        })
+    }
+}
diff --git a/rust/arrow/src/array/equal/utils.rs b/rust/arrow/src/array/equal/utils.rs
new file mode 100644
index 00000000000..f9e8860a5bb
--- /dev/null
+++ b/rust/arrow/src/array/equal/utils.rs
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::{array::ArrayData, util::bit_util};
+
+// whether bits along the positions are equal
+// `lhs_start`, `rhs_start` and `len` are _measured in bits_.
+#[inline]
+pub(super) fn equal_bits(
+    lhs_values: &[u8],
+    rhs_values: &[u8],
+    lhs_start: usize,
+    rhs_start: usize,
+    len: usize,
+) -> bool {
+    (0..len).all(|i| {
+        bit_util::get_bit(lhs_values, lhs_start + i)
+            == bit_util::get_bit(rhs_values, rhs_start + i)
+    })
+}
+
+#[inline]
+pub(super) fn equal_nulls(
+    lhs: &ArrayData,
+    rhs: &ArrayData,
+    lhs_start: usize,
+    rhs_start: usize,
+    len: usize,
+) -> bool {
+    if lhs.null_count() > 0 || rhs.null_count() > 0 {
+        let lhs_null_bitmap = lhs.null_bitmap().as_ref().unwrap();
+        let rhs_null_bitmap = rhs.null_bitmap().as_ref().unwrap();
+        let lhs_values = lhs_null_bitmap.bits.data();
+        let rhs_values = rhs_null_bitmap.bits.data();
+        equal_bits(
+            lhs_values,
+            rhs_values,
+            lhs_start + lhs.offset(),
+            rhs_start + rhs.offset(),
+            len,
+        )
+    } else {
+        true
+    }
+}
+
+#[inline]
+pub(super) fn base_equal(lhs: &ArrayData, rhs: &ArrayData) -> bool {
+    lhs.data_type() == rhs.data_type() && lhs.len() == rhs.len()
+}
+
+// whether the two memory regions are equal
+#[inline]
+pub(super) fn equal_len(
+    lhs_values: &[u8],
+    rhs_values: &[u8],
+    lhs_start: usize,
+    rhs_start: usize,
+    len: usize,
+) -> bool {
+    lhs_values[lhs_start..(lhs_start + len)] == rhs_values[rhs_start..(rhs_start + len)]
+}
diff --git a/rust/arrow/src/array/equal/variable_size.rs b/rust/arrow/src/array/equal/variable_size.rs
new file mode 100644
index 00000000000..237b353d287
--- /dev/null
+++ b/rust/arrow/src/array/equal/variable_size.rs
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::array::{ArrayData, OffsetSizeTrait};
+
+use super::utils::equal_len;
+
+fn offset_value_equal<T: OffsetSizeTrait>(
+    lhs_values: &[u8],
+    rhs_values: &[u8],
+    lhs_offsets: &[T],
+    rhs_offsets: &[T],
+    lhs_pos: usize,
+    rhs_pos: usize,
+    len: usize,
+) -> bool {
+    let lhs_start = lhs_offsets[lhs_pos].to_usize().unwrap();
+    let rhs_start = rhs_offsets[rhs_pos].to_usize().unwrap();
+    let lhs_len = lhs_offsets[lhs_pos + len] - lhs_offsets[lhs_pos];
+    let rhs_len = rhs_offsets[rhs_pos + len] - rhs_offsets[rhs_pos];
+
+    lhs_len == rhs_len
+        && equal_len(
+            lhs_values,
+            rhs_values,
+            lhs_start,
+            rhs_start,
+            lhs_len.to_usize().unwrap(),
+        )
+}
+
+pub(super) fn variable_sized_equal<T: OffsetSizeTrait>(
+    lhs: &ArrayData,
+    rhs: &ArrayData,
+    lhs_start: usize,
+    rhs_start: usize,
+    len: usize,
+) -> bool {
+    let lhs_offsets = lhs.buffer::<T>(0);
+    let rhs_offsets = rhs.buffer::<T>(0);
+
+    // these are bytes, and thus the offset does not need to be multiplied
+    let lhs_values = &lhs.buffers()[1].data()[lhs.offset()..];
+    let rhs_values = &rhs.buffers()[1].data()[rhs.offset()..];
+
+    if lhs.null_count() == 0 && rhs.null_count() == 0 {
+        offset_value_equal(
+            lhs_values,
+            rhs_values,
+            lhs_offsets,
+            rhs_offsets,
+            lhs_start,
+            rhs_start,
+            len,
+        )
+    } else {
+        (0..len).all(|i| {
+            let lhs_pos = lhs_start + i;
+            let rhs_pos = rhs_start + i;
+
+            let lhs_is_null = lhs.is_null(lhs_pos);
+            let rhs_is_null = rhs.is_null(rhs_pos);
+
+            lhs_is_null
+                || (lhs_is_null == rhs_is_null)
+                    && offset_value_equal(
+                        lhs_values,
+                        rhs_values,
+                        lhs_offsets,
+                        rhs_offsets,
+                        lhs_pos,
+                        rhs_pos,
+                        1,
+                    )
+        })
+    }
+}
diff --git a/rust/arrow/src/array/equal_json.rs b/rust/arrow/src/array/equal_json.rs
new file mode 100644
index 00000000000..42c0964c68b
--- /dev/null
+++ b/rust/arrow/src/array/equal_json.rs
@@ -0,0 +1,982 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use super::*;
+use crate::datatypes::*;
+use array::Array;
+use hex::FromHex;
+use serde_json::value::Value::{Null as JNull, Object, String as JString};
+use serde_json::Value;
+
+/// Trait for comparing arrow array with json array
+pub trait JsonEqual {
+    /// Checks whether arrow array equals to json array.
+    fn equals_json(&self, json: &[&Value]) -> bool;
+
+    /// Checks whether arrow array equals to json array.
+    fn equals_json_values(&self, json: &[Value]) -> bool {
+        let refs = json.iter().collect::<Vec<&Value>>();
+
+        self.equals_json(&refs)
+    }
+}
+
+/// Implement array equals for numeric type
+impl<T: ArrowPrimitiveType> JsonEqual for PrimitiveArray<T> {
+    fn equals_json(&self, json: &[&Value]) -> bool {
+        if self.len() != json.len() {
+            return false;
+        }
+
+        (0..self.len()).all(|i| match json[i] {
+            Value::Null => self.is_null(i),
+            v => self.is_valid(i) && Some(v) == self.value(i).into_json_value().as_ref(),
+        })
+    }
+}
+
+impl<T: ArrowPrimitiveType> PartialEq<Value> for PrimitiveArray<T> {
+    fn eq(&self, json: &Value) -> bool {
+        match json {
+            Value::Array(array) => self.equals_json_values(&array),
+            _ => false,
+        }
+    }
+}
+
+impl<T: ArrowPrimitiveType> PartialEq<PrimitiveArray<T>> for Value {
+    fn eq(&self, arrow: &PrimitiveArray<T>) -> bool {
+        match self {
+            Value::Array(array) => arrow.equals_json_values(&array),
+            _ => false,
+        }
+    }
+}
+
+impl<OffsetSize: OffsetSizeTrait> JsonEqual for GenericListArray<OffsetSize> {
+    fn equals_json(&self, json: &[&Value]) -> bool {
+        if self.len() != json.len() {
+            return false;
+        }
+
+        (0..self.len()).all(|i| match json[i] {
+            Value::Array(v) => self.is_valid(i) && self.value(i).equals_json_values(v),
+            Value::Null => self.is_null(i) || self.value_length(i).is_zero(),
+            _ => false,
+        })
+    }
+}
+
+impl<OffsetSize: OffsetSizeTrait> PartialEq<Value> for GenericListArray<OffsetSize> {
+    fn eq(&self, json: &Value) -> bool {
+        match json {
+            Value::Array(json_array) => self.equals_json_values(json_array),
+            _ => false,
+        }
+    }
+}
+
+impl<OffsetSize: OffsetSizeTrait> PartialEq<GenericListArray<OffsetSize>> for Value {
+    fn eq(&self, arrow: &GenericListArray<OffsetSize>) -> bool {
+        match self {
+            Value::Array(json_array) => arrow.equals_json_values(json_array),
+            _ => false,
+        }
+    }
+}
+
+impl<T: ArrowPrimitiveType> JsonEqual for DictionaryArray<T> {
+    fn equals_json(&self, json: &[&Value]) -> bool {
+        // todo: this is wrong: we must test the values also
+        self.keys().equals_json(json)
+    }
+}
+
+impl<T: ArrowPrimitiveType> PartialEq<Value> for DictionaryArray<T> {
+    fn eq(&self, json: &Value) -> bool {
+        match json {
+            Value::Array(json_array) => self.equals_json_values(json_array),
+            _ => false,
+        }
+    }
+}
+
+impl<T: ArrowPrimitiveType> PartialEq<DictionaryArray<T>> for Value {
+    fn eq(&self, arrow: &DictionaryArray<T>) -> bool {
+        match self {
+            Value::Array(json_array) => arrow.equals_json_values(json_array),
+            _ => false,
+        }
+    }
+}
+
+impl JsonEqual for FixedSizeListArray {
+    fn equals_json(&self, json: &[&Value]) -> bool {
+        if self.len() != json.len() {
+            return false;
+        }
+
+        (0..self.len()).all(|i| match json[i] {
+            Value::Array(v) => self.is_valid(i) && self.value(i).equals_json_values(v),
+            Value::Null => self.is_null(i) || self.value_length() == 0,
+            _ => false,
+        })
+    }
+}
+
+impl PartialEq<Value> for FixedSizeListArray {
+    fn eq(&self, json: &Value) -> bool {
+        match json {
+            Value::Array(json_array) => self.equals_json_values(json_array),
+            _ => false,
+        }
+    }
+}
+
+impl PartialEq<FixedSizeListArray> for Value {
+    fn eq(&self, arrow: &FixedSizeListArray) -> bool {
+        match self {
+            Value::Array(json_array) => arrow.equals_json_values(json_array),
+            _ => false,
+        }
+    }
+}
+
+impl JsonEqual for StructArray {
+    fn equals_json(&self, json: &[&Value]) -> bool {
+        if self.len() != json.len() {
+            return false;
+        }
+
+        let all_object = json.iter().all(|v| match v {
+            Object(_) | JNull => true,
+            _ => false,
+        });
+
+        if !all_object {
+            return false;
+        }
+
+        for column_name in self.column_names() {
+            let json_values = json
+                .iter()
+                .map(|obj| obj.get(column_name).unwrap_or(&Value::Null))
+                .collect::<Vec<&Value>>();
+
+            if !self
+                .column_by_name(column_name)
+                .map(|arr| arr.equals_json(&json_values))
+                .unwrap_or(false)
+            {
+                return false;
+            }
+        }
+
+        true
+    }
+}
+
+impl PartialEq<Value> for StructArray {
+    fn eq(&self, json: &Value) -> bool {
+        match json {
+            Value::Array(json_array) => self.equals_json_values(&json_array),
+            _ => false,
+        }
+    }
+}
+
+impl PartialEq<StructArray> for Value {
+    fn eq(&self, arrow: &StructArray) -> bool {
+        match self {
+            Value::Array(json_array) => arrow.equals_json_values(&json_array),
+            _ => false,
+        }
+    }
+}
+
+impl<OffsetSize: BinaryOffsetSizeTrait> JsonEqual for GenericBinaryArray<OffsetSize> {
+    fn equals_json(&self, json: &[&Value]) -> bool {
+        if self.len() != json.len() {
+            return false;
+        }
+
+        (0..self.len()).all(|i| match json[i] {
+            JString(s) => {
+                // binary data is sometimes hex encoded, this checks if bytes are equal,
+                // and if not converting to hex is attempted
+                self.is_valid(i)
+                    && (s.as_str().as_bytes() == self.value(i)
+                        || Vec::from_hex(s.as_str()) == Ok(self.value(i).to_vec()))
+            }
+            JNull => self.is_null(i),
+            _ => false,
+        })
+    }
+}
+
+impl<OffsetSize: BinaryOffsetSizeTrait> PartialEq<Value>
+    for GenericBinaryArray<OffsetSize>
+{
+    fn eq(&self, json: &Value) -> bool {
+        match json {
+            Value::Array(json_array) => self.equals_json_values(&json_array),
+            _ => false,
+        }
+    }
+}
+
+impl<OffsetSize: BinaryOffsetSizeTrait> PartialEq<GenericBinaryArray<OffsetSize>>
+    for Value
+{
+    fn eq(&self, arrow: &GenericBinaryArray<OffsetSize>) -> bool {
+        match self {
+            Value::Array(json_array) => arrow.equals_json_values(&json_array),
+            _ => false,
+        }
+    }
+}
+
+impl<OffsetSize: StringOffsetSizeTrait> JsonEqual for GenericStringArray<OffsetSize> {
+    fn equals_json(&self, json: &[&Value]) -> bool {
+        if self.len() != json.len() {
+            return false;
+        }
+
+        (0..self.len()).all(|i| match json[i] {
+            JString(s) => self.is_valid(i) && s.as_str() == self.value(i),
+            JNull => self.is_null(i),
+            _ => false,
+        })
+    }
+}
+
+impl<OffsetSize: StringOffsetSizeTrait> PartialEq<Value>
+    for GenericStringArray<OffsetSize>
+{
+    fn eq(&self, json: &Value) -> bool {
+        match json {
+            Value::Array(json_array) => self.equals_json_values(&json_array),
+            _ => false,
+        }
+    }
+}
+
+impl<OffsetSize: StringOffsetSizeTrait> PartialEq<GenericStringArray<OffsetSize>>
+    for Value
+{
+    fn eq(&self, arrow: &GenericStringArray<OffsetSize>) -> bool {
+        match self {
+            Value::Array(json_array) => arrow.equals_json_values(&json_array),
+            _ => false,
+        }
+    }
+}
+
+impl JsonEqual for FixedSizeBinaryArray {
+    fn equals_json(&self, json: &[&Value]) -> bool {
+        if self.len() != json.len() {
+            return false;
+        }
+
+        (0..self.len()).all(|i| match json[i] {
+            JString(s) => {
+                // binary data is sometimes hex encoded, this checks if bytes are equal,
+                // and if not converting to hex is attempted
+                self.is_valid(i)
+                    && (s.as_str().as_bytes() == self.value(i)
+                        || Vec::from_hex(s.as_str()) == Ok(self.value(i).to_vec()))
+            }
+            JNull => self.is_null(i),
+            _ => false,
+        })
+    }
+}
+
+impl PartialEq<Value> for FixedSizeBinaryArray {
+    fn eq(&self, json: &Value) -> bool {
+        match json {
+            Value::Array(json_array) => self.equals_json_values(&json_array),
+            _ => false,
+        }
+    }
+}
+
+impl PartialEq<FixedSizeBinaryArray> for Value {
+    fn eq(&self, arrow: &FixedSizeBinaryArray) -> bool {
+        match self {
+            Value::Array(json_array) => arrow.equals_json_values(&json_array),
+            _ => false,
+        }
+    }
+}
+
+impl JsonEqual for UnionArray {
+    fn equals_json(&self, _json: &[&Value]) -> bool {
+        unimplemented!(
+            "Added to allow UnionArray to implement the Array trait: see ARROW-8547"
+        )
+    }
+}
+
+impl JsonEqual for NullArray {
+    fn equals_json(&self, json: &[&Value]) -> bool {
+        if self.len() != json.len() {
+            return false;
+        }
+
+        // all JSON values must be nulls
+        json.iter().all(|&v| v == &JNull)
+    }
+}
+
+impl PartialEq<NullArray> for Value {
+    fn eq(&self, arrow: &NullArray) -> bool {
+        match self {
+            Value::Array(json_array) => arrow.equals_json_values(&json_array),
+            _ => false,
+        }
+    }
+}
+
+impl PartialEq<Value> for NullArray {
+    fn eq(&self, json: &Value) -> bool {
+        match json {
+            Value::Array(json_array) => self.equals_json_values(&json_array),
+            _ => false,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::error::Result;
+    use std::{convert::TryFrom, sync::Arc};
+
+    fn create_list_array<'a, U: AsRef<[i32]>, T: AsRef<[Option<U>]>>(
+        builder: &'a mut ListBuilder<Int32Builder>,
+        data: T,
+    ) -> Result<ListArray> {
+        for d in data.as_ref() {
+            if let Some(v) = d {
+                builder.values().append_slice(v.as_ref())?;
+                builder.append(true)?
+            } else {
+                builder.append(false)?
+            }
+        }
+        Ok(builder.finish())
+    }
+
+    /// Create a fixed size list of 2 value lengths
+    fn create_fixed_size_list_array<'a, U: AsRef<[i32]>, T: AsRef<[Option<U>]>>(
+        builder: &'a mut FixedSizeListBuilder<Int32Builder>,
+        data: T,
+    ) -> Result<FixedSizeListArray> {
+        for d in data.as_ref() {
+            if let Some(v) = d {
+                builder.values().append_slice(v.as_ref())?;
+                builder.append(true)?
+            } else {
+                for _ in 0..builder.value_length() {
+                    builder.values().append_null()?;
+                }
+                builder.append(false)?
+            }
+        }
+        Ok(builder.finish())
+    }
+
+    #[test]
+    fn test_primitive_json_equal() {
+        // Test equaled array
+        let arrow_array = Int32Array::from(vec![Some(1), None, Some(2), Some(3)]);
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                1, null, 2, 3
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.eq(&json_array));
+        assert!(json_array.eq(&arrow_array));
+
+        // Test unequaled array
+        let arrow_array = Int32Array::from(vec![Some(1), None, Some(2), Some(3)]);
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                1, 1, 2, 3
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+
+        // Test unequal length case
+        let arrow_array = Int32Array::from(vec![Some(1), None, Some(2), Some(3)]);
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                1, 1
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+
+        // Test not json array type case
+        let arrow_array = Int32Array::from(vec![Some(1), None, Some(2), Some(3)]);
+        let json_array: Value = serde_json::from_str(
+            r#"
+            {
+               "a": 1
+            }
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+    }
+
+    #[test]
+    fn test_list_json_equal() {
+        // Test equal case
+        let arrow_array = create_list_array(
+            &mut ListBuilder::new(Int32Builder::new(10)),
+            &[Some(&[1, 2, 3]), None, Some(&[4, 5, 6])],
+        )
+        .unwrap();
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                [1, 2, 3],
+                null,
+                [4, 5, 6]
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.eq(&json_array));
+        assert!(json_array.eq(&arrow_array));
+
+        // Test unequal case
+        let arrow_array = create_list_array(
+            &mut ListBuilder::new(Int32Builder::new(10)),
+            &[Some(&[1, 2, 3]), None, Some(&[4, 5, 6])],
+        )
+        .unwrap();
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                [1, 2, 3],
+                [7, 8],
+                [4, 5, 6]
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+
+        // Test incorrect type case
+        let arrow_array = create_list_array(
+            &mut ListBuilder::new(Int32Builder::new(10)),
+            &[Some(&[1, 2, 3]), None, Some(&[4, 5, 6])],
+        )
+        .unwrap();
+        let json_array: Value = serde_json::from_str(
+            r#"
+            {
+               "a": 1
+            }
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+    }
+
+    #[test]
+    fn test_fixed_size_list_json_equal() {
+        // Test equal case
+        let arrow_array = create_fixed_size_list_array(
+            &mut FixedSizeListBuilder::new(Int32Builder::new(10), 3),
+            &[Some(&[1, 2, 3]), None, Some(&[4, 5, 6])],
+        )
+        .unwrap();
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                [1, 2, 3],
+                null,
+                [4, 5, 6]
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.eq(&json_array));
+        assert!(json_array.eq(&arrow_array));
+
+        // Test unequal case
+        let arrow_array = create_fixed_size_list_array(
+            &mut FixedSizeListBuilder::new(Int32Builder::new(10), 3),
+            &[Some(&[1, 2, 3]), None, Some(&[4, 5, 6])],
+        )
+        .unwrap();
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                [1, 2, 3],
+                [7, 8, 9],
+                [4, 5, 6]
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+
+        // Test incorrect type case
+        let arrow_array = create_fixed_size_list_array(
+            &mut FixedSizeListBuilder::new(Int32Builder::new(10), 3),
+            &[Some(&[1, 2, 3]), None, Some(&[4, 5, 6])],
+        )
+        .unwrap();
+        let json_array: Value = serde_json::from_str(
+            r#"
+            {
+               "a": 1
+            }
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+    }
+
+    #[test]
+    fn test_string_json_equal() {
+        // Test the equal case
+        let arrow_array =
+            StringArray::from(vec![Some("hello"), None, None, Some("world"), None, None]);
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                "hello",
+                null,
+                null,
+                "world",
+                null,
+                null
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.eq(&json_array));
+        assert!(json_array.eq(&arrow_array));
+
+        // Test unequal case
+        let arrow_array =
+            StringArray::from(vec![Some("hello"), None, None, Some("world"), None, None]);
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                "hello",
+                null,
+                null,
+                "arrow",
+                null,
+                null
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+
+        // Test unequal length case
+        let arrow_array =
+            StringArray::from(vec![Some("hello"), None, None, Some("world"), None]);
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                "hello",
+                null,
+                null,
+                "arrow",
+                null,
+                null
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+
+        // Test incorrect type case
+        let arrow_array =
+            StringArray::from(vec![Some("hello"), None, None, Some("world"), None]);
+        let json_array: Value = serde_json::from_str(
+            r#"
+            {
+                "a": 1
+            }
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+
+        // Test incorrect value type case
+        let arrow_array =
+            StringArray::from(vec![Some("hello"), None, None, Some("world"), None]);
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                "hello",
+                null,
+                null,
+                1,
+                null,
+                null
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+    }
+
+    #[test]
+    fn test_binary_json_equal() {
+        // Test the equal case
+        let mut builder = BinaryBuilder::new(6);
+        builder.append_value(b"hello").unwrap();
+        builder.append_null().unwrap();
+        builder.append_null().unwrap();
+        builder.append_value(b"world").unwrap();
+        builder.append_null().unwrap();
+        builder.append_null().unwrap();
+        let arrow_array = builder.finish();
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                "hello",
+                null,
+                null,
+                "world",
+                null,
+                null
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.eq(&json_array));
+        assert!(json_array.eq(&arrow_array));
+
+        // Test unequal case
+        let arrow_array =
+            StringArray::from(vec![Some("hello"), None, None, Some("world"), None, None]);
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                "hello",
+                null,
+                null,
+                "arrow",
+                null,
+                null
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+
+        // Test unequal length case
+        let arrow_array =
+            StringArray::from(vec![Some("hello"), None, None, Some("world"), None]);
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                "hello",
+                null,
+                null,
+                "arrow",
+                null,
+                null
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+
+        // Test incorrect type case
+        let arrow_array =
+            StringArray::from(vec![Some("hello"), None, None, Some("world"), None]);
+        let json_array: Value = serde_json::from_str(
+            r#"
+            {
+                "a": 1
+            }
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+
+        // Test incorrect value type case
+        let arrow_array =
+            StringArray::from(vec![Some("hello"), None, None, Some("world"), None]);
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                "hello",
+                null,
+                null,
+                1,
+                null,
+                null
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+    }
+
+    #[test]
+    fn test_fixed_size_binary_json_equal() {
+        // Test the equal case
+        let mut builder = FixedSizeBinaryBuilder::new(15, 5);
+        builder.append_value(b"hello").unwrap();
+        builder.append_null().unwrap();
+        builder.append_value(b"world").unwrap();
+        let arrow_array: FixedSizeBinaryArray = builder.finish();
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                "hello",
+                null,
+                "world"
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.eq(&json_array));
+        assert!(json_array.eq(&arrow_array));
+
+        // Test unequal case
+        builder.append_value(b"hello").unwrap();
+        builder.append_null().unwrap();
+        builder.append_value(b"world").unwrap();
+        let arrow_array: FixedSizeBinaryArray = builder.finish();
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                "hello",
+                null,
+                "arrow"
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+
+        // Test unequal length case
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                "hello",
+                null,
+                null,
+                "world"
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+
+        // Test incorrect type case
+        let json_array: Value = serde_json::from_str(
+            r#"
+            {
+                "a": 1
+            }
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+
+        // Test incorrect value type case
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                "hello",
+                null,
+                1
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+    }
+
+    #[test]
+    fn test_struct_json_equal() {
+        let strings: ArrayRef = Arc::new(StringArray::from(vec![
+            Some("joe"),
+            None,
+            None,
+            Some("mark"),
+            Some("doe"),
+        ]));
+        let ints: ArrayRef = Arc::new(Int32Array::from(vec![
+            Some(1),
+            Some(2),
+            None,
+            Some(4),
+            Some(5),
+        ]));
+
+        let arrow_array =
+            StructArray::try_from(vec![("f1", strings.clone()), ("f2", ints.clone())])
+                .unwrap();
+
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+              {
+                "f1": "joe",
+                "f2": 1
+              },
+              {
+                "f2": 2
+              },
+              null,
+              {
+                "f1": "mark",
+                "f2": 4
+              },
+              {
+                "f1": "doe",
+                "f2": 5
+              }
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.eq(&json_array));
+        assert!(json_array.eq(&arrow_array));
+
+        // Test unequal length case
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+              {
+                "f1": "joe",
+                "f2": 1
+              },
+              {
+                "f2": 2
+              },
+              null,
+              {
+                "f1": "mark",
+                "f2": 4
+              }
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+
+        // Test incorrect type case
+        let json_array: Value = serde_json::from_str(
+            r#"
+              {
+                "f1": "joe",
+                "f2": 1
+              }
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+
+        // Test not all object case
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+              {
+                "f1": "joe",
+                "f2": 1
+              },
+              2,
+              null,
+              {
+                "f1": "mark",
+                "f2": 4
+              }
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+    }
+
+    #[test]
+    fn test_null_json_equal() {
+        // Test equaled array
+        let arrow_array = NullArray::new(4);
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                null, null, null, null
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.eq(&json_array));
+        assert!(json_array.eq(&arrow_array));
+
+        // Test unequaled array
+        let arrow_array = NullArray::new(2);
+        let json_array: Value = serde_json::from_str(
+            r#"
+            [
+                null, null, null
+            ]
+        "#,
+        )
+        .unwrap();
+        assert!(arrow_array.ne(&json_array));
+        assert!(json_array.ne(&arrow_array));
+    }
+}
diff --git a/rust/arrow/src/array/iterator.rs b/rust/arrow/src/array/iterator.rs
new file mode 100644
index 00000000000..d93f86d9614
--- /dev/null
+++ b/rust/arrow/src/array/iterator.rs
@@ -0,0 +1,257 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::datatypes::ArrowPrimitiveType;
+
+use super::{
+    Array, BinaryOffsetSizeTrait, GenericBinaryArray, GenericStringArray, PrimitiveArray,
+    StringOffsetSizeTrait,
+};
+
+/// an iterator that returns Some(T) or None, that can be used on any non-boolean PrimitiveArray
+// Note: This implementation is based on std's [Vec]s' [IntoIter].
+#[derive(Debug)]
+pub struct PrimitiveIter<'a, T: ArrowPrimitiveType> {
+    array: &'a PrimitiveArray<T>,
+    current: usize,
+    current_end: usize,
+}
+
+impl<'a, T: ArrowPrimitiveType> PrimitiveIter<'a, T> {
+    /// create a new iterator
+    pub fn new(array: &'a PrimitiveArray<T>) -> Self {
+        PrimitiveIter::<T> {
+            array,
+            current: 0,
+            current_end: array.len(),
+        }
+    }
+}
+
+impl<'a, T: ArrowPrimitiveType> std::iter::Iterator for PrimitiveIter<'a, T> {
+    type Item = Option<T::Native>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.current == self.current_end {
+            None
+        } else if self.array.is_null(self.current) {
+            self.current += 1;
+            Some(None)
+        } else {
+            let old = self.current;
+            self.current += 1;
+            Some(Some(self.array.value(old)))
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        (self.array.len(), Some(self.array.len()))
+    }
+}
+
+impl<'a, T: ArrowPrimitiveType> std::iter::DoubleEndedIterator for PrimitiveIter<'a, T> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        if self.current_end == self.current {
+            None
+        } else {
+            self.current_end -= 1;
+            Some(if self.array.is_null(self.current_end) {
+                None
+            } else {
+                Some(self.array.value(self.current_end))
+            })
+        }
+    }
+}
+
+/// all arrays have known size.
+impl<'a, T: ArrowPrimitiveType> std::iter::ExactSizeIterator for PrimitiveIter<'a, T> {}
+
+/// an iterator that returns `Some(&str)` or `None`, for string arrays
+#[derive(Debug)]
+pub struct GenericStringIter<'a, T>
+where
+    T: StringOffsetSizeTrait,
+{
+    array: &'a GenericStringArray<T>,
+    i: usize,
+    len: usize,
+}
+
+impl<'a, T: StringOffsetSizeTrait> GenericStringIter<'a, T> {
+    /// create a new iterator
+    pub fn new(array: &'a GenericStringArray<T>) -> Self {
+        GenericStringIter::<T> {
+            array,
+            i: 0,
+            len: array.len(),
+        }
+    }
+}
+
+impl<'a, T: StringOffsetSizeTrait> std::iter::Iterator for GenericStringIter<'a, T> {
+    type Item = Option<&'a str>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let i = self.i;
+        if i >= self.len {
+            None
+        } else if self.array.is_null(i) {
+            self.i += 1;
+            Some(None)
+        } else {
+            self.i += 1;
+            Some(Some(self.array.value(i)))
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        (self.len, Some(self.len))
+    }
+}
+
+/// all arrays have known size.
+impl<'a, T: StringOffsetSizeTrait> std::iter::ExactSizeIterator
+    for GenericStringIter<'a, T>
+{
+}
+
+/// an iterator that returns `Some(&[u8])` or `None`, for binary arrays
+#[derive(Debug)]
+pub struct GenericBinaryIter<'a, T>
+where
+    T: BinaryOffsetSizeTrait,
+{
+    array: &'a GenericBinaryArray<T>,
+    i: usize,
+    len: usize,
+}
+
+impl<'a, T: BinaryOffsetSizeTrait> GenericBinaryIter<'a, T> {
+    /// create a new iterator
+    pub fn new(array: &'a GenericBinaryArray<T>) -> Self {
+        GenericBinaryIter::<T> {
+            array,
+            i: 0,
+            len: array.len(),
+        }
+    }
+}
+
+impl<'a, T: BinaryOffsetSizeTrait> std::iter::Iterator for GenericBinaryIter<'a, T> {
+    type Item = Option<&'a [u8]>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let i = self.i;
+        if i >= self.len {
+            None
+        } else if self.array.is_null(i) {
+            self.i += 1;
+            Some(None)
+        } else {
+            self.i += 1;
+            Some(Some(self.array.value(i)))
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        (self.len, Some(self.len))
+    }
+}
+
+/// all arrays have known size.
+impl<'a, T: BinaryOffsetSizeTrait> std::iter::ExactSizeIterator
+    for GenericBinaryIter<'a, T>
+{
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use crate::array::{ArrayRef, BinaryArray, Int32Array, StringArray};
+
+    #[test]
+    fn test_primitive_array_iter_round_trip() {
+        let array = Int32Array::from(vec![Some(0), None, Some(2), None, Some(4)]);
+        let array = Arc::new(array) as ArrayRef;
+
+        let array = array.as_any().downcast_ref::<Int32Array>().unwrap();
+
+        // to and from iter, with a +1
+        let result: Int32Array =
+            array.iter().map(|e| e.and_then(|e| Some(e + 1))).collect();
+
+        let expected = Int32Array::from(vec![Some(1), None, Some(3), None, Some(5)]);
+        assert_eq!(result, expected);
+    }
+
+    #[test]
+    fn test_double_ended() {
+        let array = Int32Array::from(vec![Some(0), None, Some(2), None, Some(4)]);
+        let mut a = array.iter();
+        assert_eq!(a.next(), Some(Some(0)));
+        assert_eq!(a.next(), Some(None));
+        assert_eq!(a.next_back(), Some(Some(4)));
+        assert_eq!(a.next_back(), Some(None));
+        assert_eq!(a.next_back(), Some(Some(2)));
+        // the two sides have met: None is returned by both
+        assert_eq!(a.next_back(), None);
+        assert_eq!(a.next(), None);
+    }
+
+    #[test]
+    fn test_string_array_iter_round_trip() {
+        let array =
+            StringArray::from(vec![Some("a"), None, Some("aaa"), None, Some("aaaaa")]);
+        let array = Arc::new(array) as ArrayRef;
+
+        let array = array.as_any().downcast_ref::<StringArray>().unwrap();
+
+        // to and from iter, with a +1
+        let result: StringArray = array
+            .iter()
+            .map(|e| {
+                e.and_then(|e| {
+                    let mut a = e.to_string();
+                    a.push_str("b");
+                    Some(a)
+                })
+            })
+            .collect();
+
+        let expected =
+            StringArray::from(vec![Some("ab"), None, Some("aaab"), None, Some("aaaaab")]);
+        assert_eq!(result, expected);
+    }
+
+    #[test]
+    fn test_binary_array_iter_round_trip() {
+        let array = BinaryArray::from(vec![
+            Some(b"a" as &[u8]),
+            None,
+            Some(b"aaa"),
+            None,
+            Some(b"aaaaa"),
+        ]);
+
+        // to and from iter
+        let result: BinaryArray = array.iter().collect();
+
+        assert_eq!(result, array);
+    }
+}
diff --git a/rust/arrow/src/array/mod.rs b/rust/arrow/src/array/mod.rs
index 3abc1423e4a..bbe454c9036 100644
--- a/rust/arrow/src/array/mod.rs
+++ b/rust/arrow/src/array/mod.rs
@@ -83,12 +83,21 @@
 
 #[allow(clippy::module_inception)]
 mod array;
+mod array_binary;
+mod array_dictionary;
+mod array_list;
+mod array_primitive;
+mod array_string;
+mod array_struct;
 mod builder;
 mod cast;
 mod data;
 mod equal;
+mod equal_json;
+mod iterator;
 mod null;
 mod ord;
+mod raw_pointer;
 mod union;
 
 use crate::datatypes::*;
@@ -101,21 +110,21 @@ pub use self::data::ArrayData;
 pub use self::data::ArrayDataBuilder;
 pub use self::data::ArrayDataRef;
 
-pub use self::array::BinaryArray;
-pub use self::array::DictionaryArray;
-pub use self::array::FixedSizeBinaryArray;
-pub use self::array::FixedSizeListArray;
-pub use self::array::LargeBinaryArray;
-pub use self::array::LargeListArray;
-pub use self::array::LargeStringArray;
-pub use self::array::ListArray;
-pub use self::array::PrimitiveArray;
-pub use self::array::StringArray;
-pub use self::array::StructArray;
+pub use self::array_binary::BinaryArray;
+pub use self::array_binary::FixedSizeBinaryArray;
+pub use self::array_binary::LargeBinaryArray;
+pub use self::array_dictionary::DictionaryArray;
+pub use self::array_list::FixedSizeListArray;
+pub use self::array_list::LargeListArray;
+pub use self::array_list::ListArray;
+pub use self::array_primitive::PrimitiveArray;
+pub use self::array_string::LargeStringArray;
+pub use self::array_string::StringArray;
+pub use self::array_struct::StructArray;
 pub use self::null::NullArray;
 pub use self::union::UnionArray;
 
-pub(crate) use self::array::make_array;
+pub use self::array::make_array;
 
 pub type BooleanArray = PrimitiveArray<BooleanType>;
 pub type Int8Array = PrimitiveArray<Int8Type>;
@@ -155,9 +164,12 @@ pub type DurationMillisecondArray = PrimitiveArray<DurationMillisecondType>;
 pub type DurationMicrosecondArray = PrimitiveArray<DurationMicrosecondType>;
 pub type DurationNanosecondArray = PrimitiveArray<DurationNanosecondType>;
 
-pub use self::array::LargeListArrayOps;
-pub use self::array::ListArrayOps;
-pub use self::array::PrimitiveArrayOps;
+pub use self::array_binary::BinaryOffsetSizeTrait;
+pub use self::array_binary::GenericBinaryArray;
+pub use self::array_list::GenericListArray;
+pub use self::array_list::OffsetSizeTrait;
+pub use self::array_string::GenericStringArray;
+pub use self::array_string::StringOffsetSizeTrait;
 
 // --------------------- Array Builder ---------------------
 
@@ -237,17 +249,21 @@ pub type DurationMillisecondBuilder = PrimitiveBuilder<DurationMillisecondType>;
 pub type DurationMicrosecondBuilder = PrimitiveBuilder<DurationMicrosecondType>;
 pub type DurationNanosecondBuilder = PrimitiveBuilder<DurationNanosecondType>;
 
+// --------------------- Array Iterator ---------------------
+
+pub use self::iterator::*;
+
 // --------------------- Array Equality ---------------------
 
-pub use self::equal::ArrayEqual;
-pub use self::equal::JsonEqual;
+pub use self::equal_json::JsonEqual;
 
-// --------------------- Sortable Array ---------------------
+// --------------------- Array's values comparison ---------------------
 
-pub use self::ord::{as_ordarray, OrdArray};
+pub use self::ord::{build_compare, DynComparator};
 
 // --------------------- Array downcast helper functions ---------------------
 
 pub use self::cast::{
-    as_boolean_array, as_null_array, as_primitive_array, as_string_array,
+    as_boolean_array, as_dictionary_array, as_null_array, as_primitive_array,
+    as_string_array,
 };
diff --git a/rust/arrow/src/array/null.rs b/rust/arrow/src/array/null.rs
index 867bd7cb879..08c7cf1f21e 100644
--- a/rust/arrow/src/array/null.rs
+++ b/rust/arrow/src/array/null.rs
@@ -82,7 +82,7 @@ impl Array for NullArray {
     /// Returns the total number of null values in this array.
     /// The null count of a `NullArray` always equals its length.
     fn null_count(&self) -> usize {
-        self.data().len()
+        self.data_ref().len()
     }
 
     /// Returns the total number of bytes of memory occupied by the buffers owned by this [NullArray].
@@ -113,7 +113,7 @@ impl From<ArrayDataRef> for NullArray {
 
 impl fmt::Debug for NullArray {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "NullArray")
+        write!(f, "NullArray({})", self.len())
     }
 }
 
@@ -146,4 +146,10 @@ mod tests {
         assert_eq!(array2.null_count(), 16);
         assert_eq!(array2.offset(), 8);
     }
+
+    #[test]
+    fn test_debug_null_array() {
+        let array = NullArray::new(1024 * 1024);
+        assert_eq!(format!("{:?}", array), "NullArray(1048576)");
+    }
 }
diff --git a/rust/arrow/src/array/ord.rs b/rust/arrow/src/array/ord.rs
index ab8a31f89a9..8ff5a3d7a74 100644
--- a/rust/arrow/src/array/ord.rs
+++ b/rust/arrow/src/array/ord.rs
@@ -15,118 +15,290 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//! Defines trait for array element comparison
+//! Contains functions and function factories to compare arrays.
 
 use std::cmp::Ordering;
 
 use crate::array::*;
+use crate::datatypes::TimeUnit;
 use crate::datatypes::*;
 use crate::error::{ArrowError, Result};
 
-use TimeUnit::*;
+use num::Float;
 
-/// Trait for Arrays that can be sorted
-///
-/// Example:
+/// Compare the values at two arbitrary indices in two arrays.
+pub type DynComparator<'a> = Box<dyn Fn(usize, usize) -> Ordering + 'a>;
+
+/// compares two floats, placing NaNs at last
+fn cmp_nans_last<T: Float>(a: &T, b: &T) -> Ordering {
+    match (a, b) {
+        (x, y) if x.is_nan() && y.is_nan() => Ordering::Equal,
+        (x, _) if x.is_nan() => Ordering::Greater,
+        (_, y) if y.is_nan() => Ordering::Less,
+        (_, _) => a.partial_cmp(b).unwrap(),
+    }
+}
+
+fn compare_primitives<'a, T: ArrowPrimitiveType>(
+    left: &'a Array,
+    right: &'a Array,
+) -> DynComparator<'a>
+where
+    T::Native: Ord,
+{
+    let left = left.as_any().downcast_ref::<PrimitiveArray<T>>().unwrap();
+    let right = right.as_any().downcast_ref::<PrimitiveArray<T>>().unwrap();
+    Box::new(move |i, j| left.value(i).cmp(&right.value(j)))
+}
+
+fn compare_float<'a, T: ArrowPrimitiveType>(
+    left: &'a Array,
+    right: &'a Array,
+) -> DynComparator<'a>
+where
+    T::Native: Float,
+{
+    let left = left.as_any().downcast_ref::<PrimitiveArray<T>>().unwrap();
+    let right = right.as_any().downcast_ref::<PrimitiveArray<T>>().unwrap();
+    Box::new(move |i, j| cmp_nans_last(&left.value(i), &right.value(j)))
+}
+
+fn compare_string<'a, T>(left: &'a Array, right: &'a Array) -> DynComparator<'a>
+where
+    T: StringOffsetSizeTrait,
+{
+    let left = left
+        .as_any()
+        .downcast_ref::<GenericStringArray<T>>()
+        .unwrap();
+    let right = right
+        .as_any()
+        .downcast_ref::<GenericStringArray<T>>()
+        .unwrap();
+    Box::new(move |i, j| left.value(i).cmp(&right.value(j)))
+}
+
+fn compare_dict_string<'a, T>(left: &'a Array, right: &'a Array) -> DynComparator<'a>
+where
+    T: ArrowDictionaryKeyType,
+{
+    let left = left.as_any().downcast_ref::<DictionaryArray<T>>().unwrap();
+    let right = right.as_any().downcast_ref::<DictionaryArray<T>>().unwrap();
+    let left_keys = left.keys_array();
+    let right_keys = right.keys_array();
+
+    let left_values = StringArray::from(left.values().data());
+    let right_values = StringArray::from(left.values().data());
+
+    Box::new(move |i: usize, j: usize| {
+        let key_left = left_keys.value(i).to_usize().unwrap();
+        let key_right = right_keys.value(j).to_usize().unwrap();
+        let left = left_values.value(key_left);
+        let right = right_values.value(key_right);
+        left.cmp(&right)
+    })
+}
+
+/// returns a comparison function that compares two values at two different positions
+/// between the two arrays.
+/// The arrays' types must be equal.
+/// # Example
 /// ```
-/// use std::cmp::Ordering;
-/// use arrow::array::*;
-/// use arrow::datatypes::*;
+/// use arrow::array::{build_compare, Int32Array};
+///
+/// # fn main() -> arrow::error::Result<()> {
+/// let array1 = Int32Array::from(vec![1, 2]);
+/// let array2 = Int32Array::from(vec![3, 4]);
 ///
-/// let arr: Box<dyn OrdArray> = Box::new(PrimitiveArray::<Int64Type>::from(vec![
-///     Some(-2),
-///     Some(89),
-///     Some(-64),
-///     Some(101),
-/// ]));
+/// let cmp = build_compare(&array1, &array2)?;
 ///
-/// assert_eq!(arr.cmp_value(1, 2), Ordering::Greater);
+/// // 1 (index 0 of array1) is smaller than 4 (index 1 of array2)
+/// assert_eq!(std::cmp::Ordering::Less, (cmp)(0, 1));
+/// # Ok(())
+/// # }
 /// ```
-pub trait OrdArray: Array {
-    /// Return ordering between array element at index i and j
-    fn cmp_value(&self, i: usize, j: usize) -> Ordering;
+// This is a factory of comparisons.
+// The lifetime 'a enforces that we cannot use the closure beyond any of the array's lifetime.
+pub fn build_compare<'a>(left: &'a Array, right: &'a Array) -> Result<DynComparator<'a>> {
+    use DataType::*;
+    use IntervalUnit::*;
+    use TimeUnit::*;
+    Ok(match (left.data_type(), right.data_type()) {
+        (a, b) if a != b => {
+            return Err(ArrowError::InvalidArgumentError(
+                "Can't compare arrays of different types".to_string(),
+            ));
+        }
+        (Boolean, Boolean) => compare_primitives::<BooleanType>(left, right),
+        (UInt8, UInt8) => compare_primitives::<UInt8Type>(left, right),
+        (UInt16, UInt16) => compare_primitives::<UInt16Type>(left, right),
+        (UInt32, UInt32) => compare_primitives::<UInt32Type>(left, right),
+        (UInt64, UInt64) => compare_primitives::<UInt64Type>(left, right),
+        (Int8, Int8) => compare_primitives::<Int8Type>(left, right),
+        (Int16, Int16) => compare_primitives::<Int16Type>(left, right),
+        (Int32, Int32) => compare_primitives::<Int32Type>(left, right),
+        (Int64, Int64) => compare_primitives::<Int64Type>(left, right),
+        (Float32, Float32) => compare_float::<Float32Type>(left, right),
+        (Float64, Float64) => compare_float::<Float64Type>(left, right),
+        (Date32(_), Date32(_)) => compare_primitives::<Date32Type>(left, right),
+        (Date64(_), Date64(_)) => compare_primitives::<Date64Type>(left, right),
+        (Time32(Second), Time32(Second)) => {
+            compare_primitives::<Time32SecondType>(left, right)
+        }
+        (Time32(Millisecond), Time32(Millisecond)) => {
+            compare_primitives::<Time32MillisecondType>(left, right)
+        }
+        (Time64(Microsecond), Time64(Microsecond)) => {
+            compare_primitives::<Time64MicrosecondType>(left, right)
+        }
+        (Time64(Nanosecond), Time64(Nanosecond)) => {
+            compare_primitives::<Time64NanosecondType>(left, right)
+        }
+        (Timestamp(Second, _), Timestamp(Second, _)) => {
+            compare_primitives::<TimestampSecondType>(left, right)
+        }
+        (Timestamp(Millisecond, _), Timestamp(Millisecond, _)) => {
+            compare_primitives::<TimestampMillisecondType>(left, right)
+        }
+        (Timestamp(Microsecond, _), Timestamp(Microsecond, _)) => {
+            compare_primitives::<TimestampMicrosecondType>(left, right)
+        }
+        (Timestamp(Nanosecond, _), Timestamp(Nanosecond, _)) => {
+            compare_primitives::<TimestampNanosecondType>(left, right)
+        }
+        (Interval(YearMonth), Interval(YearMonth)) => {
+            compare_primitives::<IntervalYearMonthType>(left, right)
+        }
+        (Interval(DayTime), Interval(DayTime)) => {
+            compare_primitives::<IntervalDayTimeType>(left, right)
+        }
+        (Duration(Second), Duration(Second)) => {
+            compare_primitives::<DurationSecondType>(left, right)
+        }
+        (Duration(Millisecond), Duration(Millisecond)) => {
+            compare_primitives::<DurationMillisecondType>(left, right)
+        }
+        (Duration(Microsecond), Duration(Microsecond)) => {
+            compare_primitives::<DurationMicrosecondType>(left, right)
+        }
+        (Duration(Nanosecond), Duration(Nanosecond)) => {
+            compare_primitives::<DurationNanosecondType>(left, right)
+        }
+        (Utf8, Utf8) => compare_string::<i32>(left, right),
+        (LargeUtf8, LargeUtf8) => compare_string::<i64>(left, right),
+        (
+            Dictionary(key_type_lhs, value_type_lhs),
+            Dictionary(key_type_rhs, value_type_rhs),
+        ) => {
+            if value_type_lhs.as_ref() != &DataType::Utf8
+                || value_type_rhs.as_ref() != &DataType::Utf8
+            {
+                return Err(ArrowError::InvalidArgumentError(
+                    "Arrow still does not support comparisons of non-string dictionary arrays"
+                        .to_string(),
+                ));
+            }
+            match (key_type_lhs.as_ref(), key_type_rhs.as_ref()) {
+                (a, b) if a != b => {
+                    return Err(ArrowError::InvalidArgumentError(
+                        "Can't compare arrays of different types".to_string(),
+                    ));
+                }
+                (UInt8, UInt8) => compare_dict_string::<UInt8Type>(left, right),
+                (UInt16, UInt16) => compare_dict_string::<UInt16Type>(left, right),
+                (UInt32, UInt32) => compare_dict_string::<UInt32Type>(left, right),
+                (UInt64, UInt64) => compare_dict_string::<UInt64Type>(left, right),
+                (Int8, Int8) => compare_dict_string::<Int8Type>(left, right),
+                (Int16, Int16) => compare_dict_string::<Int16Type>(left, right),
+                (Int32, Int32) => compare_dict_string::<Int32Type>(left, right),
+                (Int64, Int64) => compare_dict_string::<Int64Type>(left, right),
+                (lhs, _) => {
+                    return Err(ArrowError::InvalidArgumentError(format!(
+                        "Dictionaries do not support keys of type {:?}",
+                        lhs
+                    )))
+                }
+            }
+        }
+        (lhs, _) => {
+            return Err(ArrowError::InvalidArgumentError(format!(
+                "The data type type {:?} has no natural order",
+                lhs
+            )))
+        }
+    })
 }
 
-impl<T: ArrowPrimitiveType> OrdArray for PrimitiveArray<T>
-where
-    T::Native: std::cmp::Ord,
-{
-    fn cmp_value(&self, i: usize, j: usize) -> Ordering {
-        self.value(i).cmp(&self.value(j))
+#[cfg(test)]
+pub mod tests {
+    use super::*;
+    use crate::array::{Float64Array, Int32Array};
+    use crate::error::Result;
+    use std::cmp::Ordering;
+    use std::iter::FromIterator;
+
+    #[test]
+    fn test_i32() -> Result<()> {
+        let array = Int32Array::from(vec![1, 2]);
+
+        let cmp = build_compare(&array, &array)?;
+
+        assert_eq!(Ordering::Less, (cmp)(0, 1));
+        Ok(())
     }
-}
 
-impl OrdArray for StringArray {
-    fn cmp_value(&self, i: usize, j: usize) -> Ordering {
-        self.value(i).cmp(self.value(j))
+    #[test]
+    fn test_i32_i32() -> Result<()> {
+        let array1 = Int32Array::from(vec![1]);
+        let array2 = Int32Array::from(vec![2]);
+
+        let cmp = build_compare(&array1, &array2)?;
+
+        assert_eq!(Ordering::Less, (cmp)(0, 0));
+        Ok(())
     }
-}
 
-impl OrdArray for NullArray {
-    fn cmp_value(&self, _i: usize, _j: usize) -> Ordering {
-        Ordering::Equal
+    #[test]
+    fn test_f64() -> Result<()> {
+        let array = Float64Array::from(vec![1.0, 2.0]);
+
+        let cmp = build_compare(&array, &array)?;
+
+        assert_eq!(Ordering::Less, (cmp)(0, 1));
+        Ok(())
     }
-}
 
-/// Convert ArrayRef to OrdArray trait object
-pub fn as_ordarray(values: &ArrayRef) -> Result<&OrdArray> {
-    match values.data_type() {
-        DataType::Boolean => Ok(as_boolean_array(&values)),
-        DataType::Utf8 => Ok(as_string_array(&values)),
-        DataType::Null => Ok(as_null_array(&values)),
-        DataType::Int8 => Ok(as_primitive_array::<Int8Type>(&values)),
-        DataType::Int16 => Ok(as_primitive_array::<Int16Type>(&values)),
-        DataType::Int32 => Ok(as_primitive_array::<Int32Type>(&values)),
-        DataType::Int64 => Ok(as_primitive_array::<Int64Type>(&values)),
-        DataType::UInt8 => Ok(as_primitive_array::<UInt8Type>(&values)),
-        DataType::UInt16 => Ok(as_primitive_array::<UInt16Type>(&values)),
-        DataType::UInt32 => Ok(as_primitive_array::<UInt32Type>(&values)),
-        DataType::UInt64 => Ok(as_primitive_array::<UInt64Type>(&values)),
-        DataType::Date32(_) => Ok(as_primitive_array::<Date32Type>(&values)),
-        DataType::Date64(_) => Ok(as_primitive_array::<Date64Type>(&values)),
-        DataType::Time32(Second) => Ok(as_primitive_array::<Time32SecondType>(&values)),
-        DataType::Time32(Millisecond) => {
-            Ok(as_primitive_array::<Time32MillisecondType>(&values))
-        }
-        DataType::Time64(Microsecond) => {
-            Ok(as_primitive_array::<Time64MicrosecondType>(&values))
-        }
-        DataType::Time64(Nanosecond) => {
-            Ok(as_primitive_array::<Time64NanosecondType>(&values))
-        }
-        DataType::Timestamp(Second, _) => {
-            Ok(as_primitive_array::<TimestampSecondType>(&values))
-        }
-        DataType::Timestamp(Millisecond, _) => {
-            Ok(as_primitive_array::<TimestampMillisecondType>(&values))
-        }
-        DataType::Timestamp(Microsecond, _) => {
-            Ok(as_primitive_array::<TimestampMicrosecondType>(&values))
-        }
-        DataType::Timestamp(Nanosecond, _) => {
-            Ok(as_primitive_array::<TimestampNanosecondType>(&values))
-        }
-        DataType::Interval(IntervalUnit::YearMonth) => {
-            Ok(as_primitive_array::<IntervalYearMonthType>(&values))
-        }
-        DataType::Interval(IntervalUnit::DayTime) => {
-            Ok(as_primitive_array::<IntervalDayTimeType>(&values))
-        }
-        DataType::Duration(TimeUnit::Second) => {
-            Ok(as_primitive_array::<DurationSecondType>(&values))
-        }
-        DataType::Duration(TimeUnit::Millisecond) => {
-            Ok(as_primitive_array::<DurationMillisecondType>(&values))
-        }
-        DataType::Duration(TimeUnit::Microsecond) => {
-            Ok(as_primitive_array::<DurationMicrosecondType>(&values))
-        }
-        DataType::Duration(TimeUnit::Nanosecond) => {
-            Ok(as_primitive_array::<DurationNanosecondType>(&values))
-        }
-        t => Err(ArrowError::ComputeError(format!(
-            "Lexical Sort not supported for data type {:?}",
-            t
-        ))),
+    #[test]
+    fn test_f64_nan() -> Result<()> {
+        let array = Float64Array::from(vec![1.0, f64::NAN]);
+
+        let cmp = build_compare(&array, &array)?;
+
+        assert_eq!(Ordering::Less, (cmp)(0, 1));
+        Ok(())
+    }
+
+    #[test]
+    fn test_f64_zeros() -> Result<()> {
+        let array = Float64Array::from(vec![-0.0, 0.0]);
+
+        let cmp = build_compare(&array, &array)?;
+
+        assert_eq!(Ordering::Equal, (cmp)(0, 1));
+        assert_eq!(Ordering::Equal, (cmp)(1, 0));
+        Ok(())
+    }
+
+    #[test]
+    fn test_dict() -> Result<()> {
+        let data = vec!["a", "b", "c", "a", "a", "c", "c"];
+        let array = DictionaryArray::<Int16Type>::from_iter(data.into_iter());
+
+        let cmp = build_compare(&array, &array)?;
+
+        assert_eq!(Ordering::Less, (cmp)(0, 1));
+        assert_eq!(Ordering::Equal, (cmp)(3, 4));
+        assert_eq!(Ordering::Greater, (cmp)(2, 3));
+        Ok(())
     }
 }
diff --git a/rust/arrow/src/array/raw_pointer.rs b/rust/arrow/src/array/raw_pointer.rs
new file mode 100644
index 00000000000..8eeadfe9390
--- /dev/null
+++ b/rust/arrow/src/array/raw_pointer.rs
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::memory;
+
+pub(super) struct RawPtrBox<T> {
+    inner: *const T,
+}
+
+impl<T> RawPtrBox<T> {
+    pub(super) fn new(inner: *const T) -> Self {
+        Self { inner }
+    }
+
+    pub(super) fn get(&self) -> *const T {
+        self.inner
+    }
+}
+
+unsafe impl<T> Send for RawPtrBox<T> {}
+unsafe impl<T> Sync for RawPtrBox<T> {}
+
+pub(super) fn as_aligned_pointer<T>(p: *const u8) -> *const T {
+    assert!(
+        memory::is_aligned(p, std::mem::align_of::<T>()),
+        "memory is not aligned"
+    );
+    p as *const T
+}
diff --git a/rust/arrow/src/array/union.rs b/rust/arrow/src/array/union.rs
index 9bbf64e22a6..9fd13915aef 100644
--- a/rust/arrow/src/array/union.rs
+++ b/rust/arrow/src/array/union.rs
@@ -541,13 +541,11 @@ impl UnionBuilder {
         let mut field_data = match self.fields.remove(&type_name) {
             Some(data) => data,
             None => match self.value_offset_builder {
-                Some(_) => {
-                    FieldData::new(self.fields.len() as i8, T::get_data_type(), None)
-                }
+                Some(_) => FieldData::new(self.fields.len() as i8, T::DATA_TYPE, None),
                 None => {
                     let mut fd = FieldData::new(
                         self.fields.len() as i8,
-                        T::get_data_type(),
+                        T::DATA_TYPE,
                         Some(BooleanBufferBuilder::new(1)),
                     );
                     for _ in 0..self.len {
@@ -657,7 +655,7 @@ mod tests {
         // Check type ids
         assert_eq!(
             union.data().buffers()[0],
-            Buffer::from(&expected_type_ids.clone().to_byte_slice())
+            Buffer::from(&expected_type_ids.to_byte_slice())
         );
         for (i, id) in expected_type_ids.iter().enumerate() {
             assert_eq!(id, &union.type_id(i));
@@ -666,7 +664,7 @@ mod tests {
         // Check offsets
         assert_eq!(
             union.data().buffers()[1],
-            Buffer::from(expected_value_offsets.clone().to_byte_slice())
+            Buffer::from(expected_value_offsets.to_byte_slice())
         );
         for (i, id) in expected_value_offsets.iter().enumerate() {
             assert_eq!(&union.value_offset(i), id);
@@ -951,7 +949,7 @@ mod tests {
             .downcast_ref::<Float64Array>()
             .unwrap()
             .value(0);
-        assert_eq!(10.0, value);
+        assert!(10.0 - value < f64::EPSILON);
 
         let slot = array.value(4);
         let value = slot
@@ -983,7 +981,7 @@ mod tests {
 
         // Check type ids
         assert_eq!(
-            Buffer::from(&expected_type_ids.clone().to_byte_slice()),
+            Buffer::from(&expected_type_ids.to_byte_slice()),
             union.data().buffers()[0]
         );
         for (i, id) in expected_type_ids.iter().enumerate() {
@@ -1034,7 +1032,7 @@ mod tests {
 
         // Check type ids
         assert_eq!(
-            Buffer::from(&expected_type_ids.clone().to_byte_slice()),
+            Buffer::from(&expected_type_ids.to_byte_slice()),
             union.data().buffers()[0]
         );
         for (i, id) in expected_type_ids.iter().enumerate() {
@@ -1064,7 +1062,7 @@ mod tests {
                     let slot = slot.as_any().downcast_ref::<Float64Array>().unwrap();
                     assert_eq!(slot.len(), 1);
                     let value = slot.value(0);
-                    assert_eq!(value, 3_f64);
+                    assert!(value - 3_f64 < f64::EPSILON);
                 }
                 3 => {
                     let slot = slot.as_any().downcast_ref::<Int32Array>().unwrap();
@@ -1076,7 +1074,7 @@ mod tests {
                     let slot = slot.as_any().downcast_ref::<Float64Array>().unwrap();
                     assert_eq!(slot.len(), 1);
                     let value = slot.value(0);
-                    assert_eq!(5_f64, value);
+                    assert!(5_f64 - value < f64::EPSILON);
                 }
                 5 => {
                     let slot = slot.as_any().downcast_ref::<Int32Array>().unwrap();
@@ -1109,7 +1107,7 @@ mod tests {
 
         // Check type ids
         assert_eq!(
-            Buffer::from(&expected_type_ids.clone().to_byte_slice()),
+            Buffer::from(&expected_type_ids.to_byte_slice()),
             union.data().buffers()[0]
         );
         for (i, id) in expected_type_ids.iter().enumerate() {
@@ -1142,7 +1140,7 @@ mod tests {
                     assert_eq!(false, union.is_null(i));
                     assert_eq!(slot.len(), 1);
                     let value = slot.value(0);
-                    assert_eq!(value, 3_f64);
+                    assert!(value - 3_f64 < f64::EPSILON);
                 }
                 4 => {
                     let slot = slot.as_any().downcast_ref::<Int32Array>().unwrap();
@@ -1187,7 +1185,7 @@ mod tests {
                     assert_eq!(false, new_union.is_null(i));
                     assert_eq!(slot.len(), 1);
                     let value = slot.value(0);
-                    assert_eq!(value, 3_f64);
+                    assert!(value - 3_f64 < f64::EPSILON);
                 }
                 3 => assert!(new_union.is_null(i)),
                 4 => {
diff --git a/rust/arrow/src/bitmap.rs b/rust/arrow/src/bitmap.rs
index e9060e6869f..7df609fb88b 100644
--- a/rust/arrow/src/bitmap.rs
+++ b/rust/arrow/src/bitmap.rs
@@ -39,12 +39,8 @@ impl Bitmap {
         } else {
             num_bytes + 64 - r
         };
-        let mut v = Vec::with_capacity(len);
-        for _ in 0..len {
-            v.push(255); // 1 is not null
-        }
         Bitmap {
-            bits: Buffer::from(&v[..]),
+            bits: Buffer::from(&vec![0xFF; len]),
         }
     }
 
diff --git a/rust/arrow/src/buffer.rs b/rust/arrow/src/buffer.rs
index 33a0af9e4bd..5b51a2f5203 100644
--- a/rust/arrow/src/buffer.rs
+++ b/rust/arrow/src/buffer.rs
@@ -31,15 +31,18 @@ use std::ptr::NonNull;
 use std::slice::{from_raw_parts, from_raw_parts_mut};
 use std::sync::Arc;
 
-use crate::array::{BufferBuilderTrait, UInt8BufferBuilder};
 use crate::datatypes::ArrowNativeType;
 use crate::error::{ArrowError, Result};
 use crate::memory;
+use crate::util::bit_chunk_iterator::BitChunks;
 use crate::util::bit_util;
+use crate::util::bit_util::ceil;
+#[cfg(feature = "simd")]
+use std::borrow::BorrowMut;
 
 /// Buffer is a contiguous memory region of fixed size and is aligned at a 64-byte
 /// boundary. Buffer is immutable.
-#[derive(PartialEq, Debug)]
+#[derive(Clone, PartialEq, Debug)]
 pub struct Buffer {
     /// Reference-counted pointer to the internal byte buffer.
     data: Arc<BufferData>,
@@ -253,21 +256,27 @@ impl Buffer {
         )
     }
 
+    /// Returns a slice of this buffer starting at a certain bit offset.
+    /// If the offset is byte-aligned the returned buffer is a shallow clone,
+    /// otherwise a new buffer is allocated and filled with a copy of the bits in the range.
+    pub fn bit_slice(&self, offset: usize, len: usize) -> Self {
+        if offset % 8 == 0 && len % 8 == 0 {
+            return self.slice(offset / 8);
+        }
+
+        bitwise_unary_op_helper(&self, offset, len, |a| a)
+    }
+
+    pub fn bit_chunks(&self, offset: usize, len: usize) -> BitChunks {
+        BitChunks::new(&self, offset, len)
+    }
+
     /// Returns an empty buffer.
     pub fn empty() -> Self {
         unsafe { Self::from_raw_parts(BUFFER_INIT.as_ptr() as _, 0, 0) }
     }
 }
 
-impl Clone for Buffer {
-    fn clone(&self) -> Buffer {
-        Buffer {
-            data: self.data.clone(),
-            offset: self.offset,
-        }
-    }
-}
-
 /// Creating a `Buffer` instance by copying the memory from a `AsRef<[u8]>` into a newly
 /// allocated memory region.
 impl<T: AsRef<[u8]>> From<T> for Buffer {
@@ -279,32 +288,271 @@ impl<T: AsRef<[u8]>> From<T> for Buffer {
         let buffer = memory::allocate_aligned(capacity);
         unsafe {
             memory::memcpy(buffer, slice.as_ptr(), len);
-            Buffer::from_raw_parts(buffer, len, capacity)
+            Buffer::build_with_arguments(buffer, len, capacity, true)
         }
     }
 }
 
-///  Helper function for SIMD `BitAnd` and `BitOr` implementations
+/// Apply a bitwise operation `simd_op` / `scalar_op` to two inputs using simd instructions and return the result as a Buffer.
+/// The `simd_op` functions gets applied on chunks of 64 bytes (512 bits) at a time
+/// and the `scalar_op` gets applied to remaining bytes.
+/// Contrary to the non-simd version `bitwise_bin_op_helper`, the offset and length is specified in bytes
+/// and this version does not support operations starting at arbitrary bit offsets.
 #[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
-fn bitwise_bin_op_simd_helper<F>(left: &Buffer, right: &Buffer, op: F) -> Buffer
+fn bitwise_bin_op_simd_helper<F_SIMD, F_SCALAR>(
+    left: &Buffer,
+    left_offset: usize,
+    right: &Buffer,
+    right_offset: usize,
+    len: usize,
+    simd_op: F_SIMD,
+    scalar_op: F_SCALAR,
+) -> Buffer
 where
-    F: Fn(u8x64, u8x64) -> u8x64,
+    F_SIMD: Fn(u8x64, u8x64) -> u8x64,
+    F_SCALAR: Fn(u8, u8) -> u8,
 {
-    let mut result = MutableBuffer::new(left.len()).with_bitset(left.len(), false);
+    let mut result = MutableBuffer::new(len).with_bitset(len, false);
     let lanes = u8x64::lanes();
-    for i in (0..left.len()).step_by(lanes) {
-        let left_data = unsafe { from_raw_parts(left.raw_data().add(i), lanes) };
-        let right_data = unsafe { from_raw_parts(right.raw_data().add(i), lanes) };
-        let result_slice: &mut [u8] = unsafe {
-            from_raw_parts_mut((result.data_mut().as_mut_ptr() as *mut u8).add(i), lanes)
-        };
-        unsafe {
-            bit_util::bitwise_bin_op_simd(&left_data, &right_data, result_slice, &op)
-        };
-    }
+
+    let mut left_chunks = left.data()[left_offset..].chunks_exact(lanes);
+    let mut right_chunks = right.data()[right_offset..].chunks_exact(lanes);
+    let mut result_chunks = result.data_mut().chunks_exact_mut(lanes);
+
+    result_chunks
+        .borrow_mut()
+        .zip(left_chunks.borrow_mut().zip(right_chunks.borrow_mut()))
+        .for_each(|(res, (left, right))| {
+            unsafe { bit_util::bitwise_bin_op_simd(&left, &right, res, &simd_op) };
+        });
+
+    result_chunks
+        .into_remainder()
+        .iter_mut()
+        .zip(
+            left_chunks
+                .remainder()
+                .iter()
+                .zip(right_chunks.remainder().iter()),
+        )
+        .for_each(|(res, (left, right))| {
+            *res = scalar_op(*left, *right);
+        });
+
     result.freeze()
 }
 
+/// Apply a bitwise operation `simd_op` / `scalar_op` to one input using simd instructions and return the result as a Buffer.
+/// The `simd_op` functions gets applied on chunks of 64 bytes (512 bits) at a time
+/// and the `scalar_op` gets applied to remaining bytes.
+/// Contrary to the non-simd version `bitwise_unary_op_helper`, the offset and length is specified in bytes
+/// and this version does not support operations starting at arbitrary bit offsets.
+#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
+fn bitwise_unary_op_simd_helper<F_SIMD, F_SCALAR>(
+    left: &Buffer,
+    left_offset: usize,
+    len: usize,
+    simd_op: F_SIMD,
+    scalar_op: F_SCALAR,
+) -> Buffer
+where
+    F_SIMD: Fn(u8x64) -> u8x64,
+    F_SCALAR: Fn(u8) -> u8,
+{
+    let mut result = MutableBuffer::new(len).with_bitset(len, false);
+    let lanes = u8x64::lanes();
+
+    let mut left_chunks = left.data()[left_offset..].chunks_exact(lanes);
+    let mut result_chunks = result.data_mut().chunks_exact_mut(lanes);
+
+    result_chunks
+        .borrow_mut()
+        .zip(left_chunks.borrow_mut())
+        .for_each(|(res, left)| unsafe {
+            let data_simd = u8x64::from_slice_unaligned_unchecked(left);
+            let simd_result = simd_op(data_simd);
+            simd_result.write_to_slice_unaligned_unchecked(res);
+        });
+
+    result_chunks
+        .into_remainder()
+        .iter_mut()
+        .zip(left_chunks.remainder().iter())
+        .for_each(|(res, left)| {
+            *res = scalar_op(*left);
+        });
+
+    result.freeze()
+}
+
+/// Apply a bitwise operation `op` to two inputs and return the result as a Buffer.
+/// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits.
+fn bitwise_bin_op_helper<F>(
+    left: &Buffer,
+    left_offset_in_bits: usize,
+    right: &Buffer,
+    right_offset_in_bits: usize,
+    len_in_bits: usize,
+    op: F,
+) -> Buffer
+where
+    F: Fn(u64, u64) -> u64,
+{
+    // reserve capacity and set length so we can get a typed view of u64 chunks
+    let mut result =
+        MutableBuffer::new(ceil(len_in_bits, 8)).with_bitset(len_in_bits / 64 * 8, false);
+
+    let left_chunks = left.bit_chunks(left_offset_in_bits, len_in_bits);
+    let right_chunks = right.bit_chunks(right_offset_in_bits, len_in_bits);
+    let result_chunks = result.typed_data_mut::<u64>().iter_mut();
+
+    result_chunks
+        .zip(left_chunks.iter().zip(right_chunks.iter()))
+        .for_each(|(res, (left, right))| {
+            *res = op(left, right);
+        });
+
+    let remainder_bytes = ceil(left_chunks.remainder_len(), 8);
+    let rem = op(left_chunks.remainder_bits(), right_chunks.remainder_bits());
+    let rem = &rem.to_le_bytes()[0..remainder_bytes];
+    result
+        .write_all(rem)
+        .expect("not enough capacity in buffer");
+
+    result.freeze()
+}
+
+/// Apply a bitwise operation `op` to one input and return the result as a Buffer.
+/// The input is treated as a bitmap, meaning that offset and length are specified in number of bits.
+fn bitwise_unary_op_helper<F>(
+    left: &Buffer,
+    offset_in_bits: usize,
+    len_in_bits: usize,
+    op: F,
+) -> Buffer
+where
+    F: Fn(u64) -> u64,
+{
+    // reserve capacity and set length so we can get a typed view of u64 chunks
+    let mut result =
+        MutableBuffer::new(ceil(len_in_bits, 8)).with_bitset(len_in_bits / 64 * 8, false);
+
+    let left_chunks = left.bit_chunks(offset_in_bits, len_in_bits);
+    let result_chunks = result.typed_data_mut::<u64>().iter_mut();
+
+    result_chunks
+        .zip(left_chunks.iter())
+        .for_each(|(res, left)| {
+            *res = op(left);
+        });
+
+    let remainder_bytes = ceil(left_chunks.remainder_len(), 8);
+    let rem = op(left_chunks.remainder_bits());
+    let rem = &rem.to_le_bytes()[0..remainder_bytes];
+    result
+        .write_all(rem)
+        .expect("not enough capacity in buffer");
+
+    result.freeze()
+}
+
+pub(super) fn buffer_bin_and(
+    left: &Buffer,
+    left_offset_in_bits: usize,
+    right: &Buffer,
+    right_offset_in_bits: usize,
+    len_in_bits: usize,
+) -> Buffer {
+    // SIMD implementation if available and byte-aligned
+    #[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
+    if left_offset_in_bits % 8 == 0
+        && right_offset_in_bits % 8 == 0
+        && len_in_bits % 8 == 0
+    {
+        return bitwise_bin_op_simd_helper(
+            &left,
+            left_offset_in_bits / 8,
+            &right,
+            right_offset_in_bits / 8,
+            len_in_bits / 8,
+            |a, b| a & b,
+            |a, b| a & b,
+        );
+    }
+    // Default implementation
+    #[allow(unreachable_code)]
+    {
+        bitwise_bin_op_helper(
+            &left,
+            left_offset_in_bits,
+            right,
+            right_offset_in_bits,
+            len_in_bits,
+            |a, b| a & b,
+        )
+    }
+}
+
+pub(super) fn buffer_bin_or(
+    left: &Buffer,
+    left_offset_in_bits: usize,
+    right: &Buffer,
+    right_offset_in_bits: usize,
+    len_in_bits: usize,
+) -> Buffer {
+    // SIMD implementation if available and byte-aligned
+    #[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
+    if left_offset_in_bits % 8 == 0
+        && right_offset_in_bits % 8 == 0
+        && len_in_bits % 8 == 0
+    {
+        return bitwise_bin_op_simd_helper(
+            &left,
+            left_offset_in_bits / 8,
+            &right,
+            right_offset_in_bits / 8,
+            len_in_bits / 8,
+            |a, b| a | b,
+            |a, b| a | b,
+        );
+    }
+    // Default implementation
+    #[allow(unreachable_code)]
+    {
+        bitwise_bin_op_helper(
+            &left,
+            left_offset_in_bits,
+            right,
+            right_offset_in_bits,
+            len_in_bits,
+            |a, b| a | b,
+        )
+    }
+}
+
+pub(super) fn buffer_unary_not(
+    left: &Buffer,
+    offset_in_bits: usize,
+    len_in_bits: usize,
+) -> Buffer {
+    // SIMD implementation if available and byte-aligned
+    #[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
+    if offset_in_bits % 8 == 0 && len_in_bits % 8 == 0 {
+        return bitwise_unary_op_simd_helper(
+            &left,
+            offset_in_bits / 8,
+            len_in_bits / 8,
+            |a| !a,
+            |a| !a,
+        );
+    }
+    // Default implementation
+    #[allow(unreachable_code)]
+    {
+        bitwise_unary_op_helper(&left, offset_in_bits, len_in_bits, |a| !a)
+    }
+}
+
 impl<'a, 'b> BitAnd<&'b Buffer> for &'a Buffer {
     type Output = Result<Buffer>;
 
@@ -315,27 +563,8 @@ impl<'a, 'b> BitAnd<&'b Buffer> for &'a Buffer {
             ));
         }
 
-        // SIMD implementation if available
-        #[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
-        {
-            return Ok(bitwise_bin_op_simd_helper(&self, &rhs, |a, b| a & b));
-        }
-
-        // Default implementation
-        #[allow(unreachable_code)]
-        {
-            let mut builder = UInt8BufferBuilder::new(self.len());
-            for i in 0..self.len() {
-                unsafe {
-                    builder
-                        .append(
-                            self.data().get_unchecked(i) & rhs.data().get_unchecked(i),
-                        )
-                        .unwrap();
-                }
-            }
-            Ok(builder.finish())
-        }
+        let len_in_bits = self.len() * 8;
+        Ok(buffer_bin_and(&self, 0, &rhs, 0, len_in_bits))
     }
 }
 
@@ -349,27 +578,9 @@ impl<'a, 'b> BitOr<&'b Buffer> for &'a Buffer {
             ));
         }
 
-        // SIMD implementation if available
-        #[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
-        {
-            return Ok(bitwise_bin_op_simd_helper(&self, &rhs, |a, b| a | b));
-        }
+        let len_in_bits = self.len() * 8;
 
-        // Default implementation
-        #[allow(unreachable_code)]
-        {
-            let mut builder = UInt8BufferBuilder::new(self.len());
-            for i in 0..self.len() {
-                unsafe {
-                    builder
-                        .append(
-                            self.data().get_unchecked(i) | rhs.data().get_unchecked(i),
-                        )
-                        .unwrap();
-                }
-            }
-            Ok(builder.finish())
-        }
+        Ok(buffer_bin_or(&self, 0, &rhs, 0, len_in_bits))
     }
 }
 
@@ -377,38 +588,8 @@ impl Not for &Buffer {
     type Output = Buffer;
 
     fn not(self) -> Buffer {
-        // SIMD implementation if available
-        #[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
-        {
-            let mut result =
-                MutableBuffer::new(self.len()).with_bitset(self.len(), false);
-            let lanes = u8x64::lanes();
-            for i in (0..self.len()).step_by(lanes) {
-                unsafe {
-                    let data = from_raw_parts(self.raw_data().add(i), lanes);
-                    let data_simd = u8x64::from_slice_unaligned_unchecked(data);
-                    let simd_result = !data_simd;
-                    let result_slice: &mut [u8] = from_raw_parts_mut(
-                        (result.data_mut().as_mut_ptr() as *mut u8).add(i),
-                        lanes,
-                    );
-                    simd_result.write_to_slice_unaligned_unchecked(result_slice);
-                }
-            }
-            return result.freeze();
-        }
-
-        // Default implementation
-        #[allow(unreachable_code)]
-        {
-            let mut builder = UInt8BufferBuilder::new(self.len());
-            for i in 0..self.len() {
-                unsafe {
-                    builder.append(!self.data().get_unchecked(i)).unwrap();
-                }
-            }
-            builder.finish()
-        }
+        let len_in_bits = self.len() * 8;
+        buffer_unary_not(&self, 0, len_in_bits)
     }
 }
 
@@ -436,6 +617,12 @@ impl MutableBuffer {
         }
     }
 
+    /// creates a new [MutableBuffer] where every bit is initialized to `0`
+    pub fn new_null(len: usize) -> Self {
+        let num_bytes = bit_util::ceil(len, 8);
+        MutableBuffer::new(num_bytes).with_bitset(num_bytes, false)
+    }
+
     /// Set the bits in the range of `[0, end)` to 0 (if `val` is false), or 1 (if `val`
     /// is true). Also extend the length of this buffer to be `end`.
     ///
@@ -504,17 +691,20 @@ impl MutableBuffer {
     }
 
     /// Returns whether this buffer is empty or not.
-    pub fn is_empty(&self) -> bool {
+    #[inline]
+    pub const fn is_empty(&self) -> bool {
         self.len == 0
     }
 
     /// Returns the length (the number of bytes written) in this buffer.
-    pub fn len(&self) -> usize {
+    #[inline]
+    pub const fn len(&self) -> usize {
         self.len
     }
 
     /// Returns the total capacity in this buffer.
-    pub fn capacity(&self) -> usize {
+    #[inline]
+    pub const fn capacity(&self) -> usize {
         self.capacity
     }
 
@@ -546,7 +736,7 @@ impl MutableBuffer {
     /// Note that this should be used cautiously, and the returned pointer should not be
     /// stored anywhere, to avoid dangling pointers.
     #[inline]
-    pub fn raw_data(&self) -> *const u8 {
+    pub const fn raw_data(&self) -> *const u8 {
         self.data
     }
 
@@ -696,7 +886,7 @@ mod tests {
     #[test]
     fn test_copy() {
         let buf = Buffer::from(&[0, 1, 2, 3, 4]);
-        let buf2 = buf.clone();
+        let buf2 = buf;
         assert_eq!(5, buf2.len());
         assert_eq!(64, buf2.capacity());
         assert!(!buf2.raw_data().is_null());
@@ -797,19 +987,19 @@ mod tests {
     #[test]
     fn test_mutable_write() {
         let mut buf = MutableBuffer::new(100);
-        buf.write("hello".as_bytes()).expect("Ok");
+        buf.write_all(b"hello").expect("Ok");
         assert_eq!(5, buf.len());
-        assert_eq!("hello".as_bytes(), buf.data());
+        assert_eq!(b"hello", buf.data());
 
-        buf.write(" world".as_bytes()).expect("Ok");
+        buf.write_all(b" world").expect("Ok");
         assert_eq!(11, buf.len());
-        assert_eq!("hello world".as_bytes(), buf.data());
+        assert_eq!(b"hello world", buf.data());
 
         buf.clear();
         assert_eq!(0, buf.len());
-        buf.write("hello arrow".as_bytes()).expect("Ok");
+        buf.write_all(b"hello arrow").expect("Ok");
         assert_eq!(11, buf.len());
-        assert_eq!("hello arrow".as_bytes(), buf.data());
+        assert_eq!(b"hello arrow", buf.data());
     }
 
     #[test]
@@ -818,7 +1008,7 @@ mod tests {
         let mut buf = MutableBuffer::new(1);
         assert_eq!(64, buf.capacity());
         for _ in 0..10 {
-            buf.write(&[0, 0, 0, 0, 0, 0, 0, 0]).unwrap();
+            buf.write_all(&[0, 0, 0, 0, 0, 0, 0, 0]).unwrap();
         }
     }
 
@@ -867,16 +1057,16 @@ mod tests {
     #[test]
     fn test_mutable_freeze() {
         let mut buf = MutableBuffer::new(1);
-        buf.write("aaaa bbbb cccc dddd".as_bytes())
+        buf.write_all(b"aaaa bbbb cccc dddd")
             .expect("write should be OK");
         assert_eq!(19, buf.len());
         assert_eq!(64, buf.capacity());
-        assert_eq!("aaaa bbbb cccc dddd".as_bytes(), buf.data());
+        assert_eq!(b"aaaa bbbb cccc dddd", buf.data());
 
         let immutable_buf = buf.freeze();
         assert_eq!(19, immutable_buf.len());
         assert_eq!(64, immutable_buf.capacity());
-        assert_eq!("aaaa bbbb cccc dddd".as_bytes(), immutable_buf.data());
+        assert_eq!(b"aaaa bbbb cccc dddd", immutable_buf.data());
     }
 
     #[test]
@@ -884,11 +1074,11 @@ mod tests {
         let mut buf = MutableBuffer::new(1);
         let mut buf2 = MutableBuffer::new(1);
 
-        buf.write(&[0xaa])?;
-        buf2.write(&[0xaa, 0xbb])?;
+        buf.write_all(&[0xaa])?;
+        buf2.write_all(&[0xaa, 0xbb])?;
         assert!(buf != buf2);
 
-        buf.write(&[0xbb])?;
+        buf.write_all(&[0xbb])?;
         assert_eq!(buf, buf2);
 
         buf2.reserve(65)?;
@@ -905,7 +1095,7 @@ mod tests {
 
         let buffer_copy = thread::spawn(move || {
             // access buffer in another thread.
-            buffer.clone()
+            buffer
         })
         .join();
 
@@ -922,6 +1112,7 @@ mod tests {
     }
 
     #[test]
+    #[allow(clippy::float_cmp)]
     fn test_as_typed_data() {
         check_as_typed_data!(&[1i8, 3i8, 6i8], i8);
         check_as_typed_data!(&[1u8, 3u8, 6u8], u8);
diff --git a/rust/arrow/src/compute/kernels/aggregate.rs b/rust/arrow/src/compute/kernels/aggregate.rs
index 1af5ed666b3..444e2454a1c 100644
--- a/rust/arrow/src/compute/kernels/aggregate.rs
+++ b/rust/arrow/src/compute/kernels/aggregate.rs
@@ -19,15 +19,49 @@
 
 use std::ops::Add;
 
-use crate::array::{Array, PrimitiveArray};
+use crate::array::{Array, GenericStringArray, PrimitiveArray, StringOffsetSizeTrait};
 use crate::datatypes::ArrowNumericType;
 
+/// Helper macro to perform min/max of strings
+fn min_max_string<T: StringOffsetSizeTrait, F: Fn(&str, &str) -> bool>(
+    array: &GenericStringArray<T>,
+    cmp: F,
+) -> Option<&str> {
+    let null_count = array.null_count();
+
+    if null_count == array.len() {
+        return None;
+    }
+    let mut n = "";
+    let mut has_value = false;
+    let data = array.data();
+
+    if null_count == 0 {
+        for i in 0..data.len() {
+            let item = array.value(i);
+            if !has_value || cmp(&n, item) {
+                has_value = true;
+                n = item;
+            }
+        }
+    } else {
+        for i in 0..data.len() {
+            let item = array.value(i);
+            if data.is_valid(i) && (!has_value || cmp(&n, item)) {
+                has_value = true;
+                n = item;
+            }
+        }
+    }
+    Some(n)
+}
+
 /// Returns the minimum value in the array, according to the natural order.
 pub fn min<T>(array: &PrimitiveArray<T>) -> Option<T::Native>
 where
     T: ArrowNumericType,
 {
-    min_max_helper(array, |a, b| a < b)
+    min_max_helper(array, |a, b| a > b)
 }
 
 /// Returns the maximum value in the array, according to the natural order.
@@ -35,37 +69,63 @@ pub fn max<T>(array: &PrimitiveArray<T>) -> Option<T::Native>
 where
     T: ArrowNumericType,
 {
-    min_max_helper(array, |a, b| a > b)
+    min_max_helper(array, |a, b| a < b)
+}
+
+/// Returns the maximum value in the string array, according to the natural order.
+pub fn max_string<T: StringOffsetSizeTrait>(
+    array: &GenericStringArray<T>,
+) -> Option<&str> {
+    min_max_string(array, |a, b| a < b)
+}
+
+/// Returns the minimum value in the string array, according to the natural order.
+pub fn min_string<T: StringOffsetSizeTrait>(
+    array: &GenericStringArray<T>,
+) -> Option<&str> {
+    min_max_string(array, |a, b| a > b)
 }
 
 /// Helper function to perform min/max lambda function on values from a numeric array.
 fn min_max_helper<T, F>(array: &PrimitiveArray<T>, cmp: F) -> Option<T::Native>
 where
     T: ArrowNumericType,
-    F: Fn(T::Native, T::Native) -> bool,
+    F: Fn(&T::Native, &T::Native) -> bool,
 {
-    let mut n: Option<T::Native> = None;
+    let null_count = array.null_count();
+
+    if null_count == array.len() {
+        return None;
+    }
+
+    let mut n: T::Native = T::default_value();
+    let mut has_value = false;
     let data = array.data();
-    for i in 0..data.len() {
-        if data.is_null(i) {
-            continue;
+    let m = array.value_slice(0, data.len());
+
+    if null_count == 0 {
+        // optimized path for arrays without null values
+        for item in m {
+            if !has_value || cmp(&n, item) {
+                has_value = true;
+                n = *item
+            }
         }
-        let m = array.value(i);
-        match n {
-            None => n = Some(m),
-            Some(nn) => {
-                if cmp(m, nn) {
-                    n = Some(m)
-                }
+    } else {
+        for (i, item) in m.iter().enumerate() {
+            if data.is_valid(i) && (!has_value || cmp(&n, item)) {
+                has_value = true;
+                n = *item
             }
         }
     }
-    n
+    Some(n)
 }
 
 /// Returns the sum of values in the array.
 ///
 /// Returns `None` if the array is empty or only contains null values.
+#[cfg(not(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd")))]
 pub fn sum<T>(array: &PrimitiveArray<T>) -> Option<T::Native>
 where
     T: ArrowNumericType,
@@ -74,27 +134,129 @@ where
     let null_count = array.null_count();
 
     if null_count == array.len() {
-        None
-    } else if null_count == 0 {
-        // optimized path for arrays without null values
-        let mut n: T::Native = T::default_value();
-        let data = array.data();
-        let m = array.value_slice(0, data.len());
-        for item in m.iter().take(data.len()) {
-            n = n + *item;
+        return None;
+    }
+
+    let data: &[T::Native] = array.value_slice(0, array.len());
+
+    match array.data().null_buffer() {
+        None => {
+            let sum = data.iter().fold(T::default_value(), |accumulator, value| {
+                accumulator + *value
+            });
+
+            Some(sum)
         }
-        Some(n)
-    } else {
-        let mut n: T::Native = T::default_value();
-        let data = array.data();
-        let m = array.value_slice(0, data.len());
-        for (i, item) in m.iter().enumerate() {
-            if data.is_valid(i) {
-                n = n + *item;
-            }
+        Some(buffer) => {
+            let mut sum = T::default_value();
+            let data_chunks = data.chunks_exact(64);
+            let remainder = data_chunks.remainder();
+
+            let bit_chunks = buffer.bit_chunks(array.offset(), array.len());
+            &data_chunks
+                .zip(bit_chunks.iter())
+                .for_each(|(chunk, mask)| {
+                    chunk.iter().enumerate().for_each(|(i, value)| {
+                        if (mask & (1 << i)) != 0 {
+                            sum = sum + *value;
+                        }
+                    });
+                });
+
+            let remainder_bits = bit_chunks.remainder_bits();
+
+            remainder.iter().enumerate().for_each(|(i, value)| {
+                if remainder_bits & (1 << i) != 0 {
+                    sum = sum + *value;
+                }
+            });
+
+            Some(sum)
+        }
+    }
+}
+
+/// Returns the sum of values in the array.
+///
+/// Returns `None` if the array is empty or only contains null values.
+#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
+pub fn sum<T: ArrowNumericType>(array: &PrimitiveArray<T>) -> Option<T::Native>
+where
+    T::Native: Add<Output = T::Native>,
+{
+    let null_count = array.null_count();
+
+    if null_count == array.len() {
+        return None;
+    }
+
+    let data: &[T::Native] = array.value_slice(0, array.len());
+
+    let mut vector_sum = T::init(T::default_value());
+    let mut remainder_sum = T::default_value();
+
+    match array.data().null_buffer() {
+        None => {
+            let data_chunks = data.chunks_exact(64);
+            let remainder = data_chunks.remainder();
+
+            data_chunks.for_each(|chunk| {
+                chunk.chunks_exact(T::lanes()).for_each(|chunk| {
+                    let chunk = T::load(&chunk);
+                    vector_sum = vector_sum + chunk;
+                });
+            });
+
+            remainder.iter().for_each(|value| {
+                remainder_sum = remainder_sum + *value;
+            });
+        }
+        Some(buffer) => {
+            // process data in chunks of 64 elements since we also get 64 bits of validity information at a time
+            let data_chunks = data.chunks_exact(64);
+            let remainder = data_chunks.remainder();
+
+            let bit_chunks = buffer.bit_chunks(array.offset(), array.len());
+            let remainder_bits = bit_chunks.remainder_bits();
+
+            data_chunks.zip(bit_chunks).for_each(|(chunk, mut mask)| {
+                // split chunks further into slices corresponding to the vector length
+                // the compiler is able to unroll this inner loop and remove bounds checks
+                // since the outer chunk size (64) is always a multiple of the number of lanes
+                chunk.chunks_exact(T::lanes()).for_each(|chunk| {
+                    let zero = T::init(T::default_value());
+                    let vecmask = T::mask_from_u64(mask);
+                    let chunk = T::load(&chunk);
+                    let blended = T::mask_select(vecmask, chunk, zero);
+
+                    vector_sum = vector_sum + blended;
+
+                    mask = mask >> T::lanes();
+                });
+            });
+
+            remainder.iter().enumerate().for_each(|(i, value)| {
+                if remainder_bits & (1 << i) != 0 {
+                    remainder_sum = remainder_sum + *value;
+                }
+            });
         }
-        Some(n)
     }
+
+    // calculate horizontal sum of accumulator by writing to a temporary
+    // this is probably faster than extracting individual lanes
+    // the compiler is free to optimize this to something faster
+    let tmp = &mut [T::default_value(); 64];
+    T::write(vector_sum, &mut tmp[0..T::lanes()]);
+
+    let mut total_sum = T::default_value();
+    tmp[0..T::lanes()]
+        .iter()
+        .for_each(|lane| total_sum = total_sum + *lane);
+
+    total_sum = total_sum + remainder_sum;
+
+    Some(total_sum)
 }
 
 #[cfg(test)]
@@ -111,7 +273,7 @@ mod tests {
     #[test]
     fn test_primitive_array_float_sum() {
         let a = Float64Array::from(vec![1.1, 2.2, 3.3, 4.4, 5.5]);
-        assert_eq!(16.5, sum(&a).unwrap());
+        assert!(16.5 - sum(&a).unwrap() < f64::EPSILON);
     }
 
     #[test]
@@ -139,4 +301,32 @@ mod tests {
         assert_eq!(5, min(&a).unwrap());
         assert_eq!(9, max(&a).unwrap());
     }
+
+    #[test]
+    fn test_buffer_min_max_1() {
+        let a = Int32Array::from(vec![None, None, Some(5), Some(2)]);
+        assert_eq!(Some(2), min(&a));
+        assert_eq!(Some(5), max(&a));
+    }
+
+    #[test]
+    fn test_string_min_max_with_nulls() {
+        let a = StringArray::from(vec![Some("b"), None, None, Some("a"), Some("c")]);
+        assert_eq!("a", min_string(&a).unwrap());
+        assert_eq!("c", max_string(&a).unwrap());
+    }
+
+    #[test]
+    fn test_string_min_max_all_nulls() {
+        let a = StringArray::from(vec![None, None]);
+        assert_eq!(None, min_string(&a));
+        assert_eq!(None, max_string(&a));
+    }
+
+    #[test]
+    fn test_string_min_max_1() {
+        let a = StringArray::from(vec![None, None, Some("b"), Some("a")]);
+        assert_eq!(Some("a"), min_string(&a));
+        assert_eq!(Some("b"), max_string(&a));
+    }
 }
diff --git a/rust/arrow/src/compute/kernels/arithmetic.rs b/rust/arrow/src/compute/kernels/arithmetic.rs
index 94a66cd503a..fe1bda5e26f 100644
--- a/rust/arrow/src/compute/kernels/arithmetic.rs
+++ b/rust/arrow/src/compute/kernels/arithmetic.rs
@@ -31,23 +31,26 @@ use std::sync::Arc;
 
 use num::{One, Zero};
 
-use crate::array::*;
 #[cfg(feature = "simd")]
 use crate::bitmap::Bitmap;
 use crate::buffer::Buffer;
 #[cfg(feature = "simd")]
 use crate::buffer::MutableBuffer;
-use crate::compute::util::apply_bin_op_to_option_bitmap;
+use crate::compute::util::combine_option_bitmap;
 #[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
 use crate::compute::util::simd_load_set_invalid;
 use crate::datatypes;
 use crate::datatypes::ToByteSlice;
 use crate::error::{ArrowError, Result};
-use crate::util::bit_util;
+use crate::{array::*, util::bit_util};
 
 /// Helper function to perform math lambda function on values from two arrays. If either
 /// left or right value is null then the output value is also null, so `1 + null` is
 /// `null`.
+///
+/// # Errors
+///
+/// This function errors if the arrays have different lengths
 pub fn math_op<T, F>(
     left: &PrimitiveArray<T>,
     right: &PrimitiveArray<T>,
@@ -55,7 +58,7 @@ pub fn math_op<T, F>(
 ) -> Result<PrimitiveArray<T>>
 where
     T: datatypes::ArrowNumericType,
-    F: Fn(T::Native, T::Native) -> Result<T::Native>,
+    F: Fn(T::Native, T::Native) -> T::Native,
 {
     if left.len() != right.len() {
         return Err(ArrowError::ComputeError(
@@ -63,35 +66,84 @@ where
         ));
     }
 
-    let null_bit_buffer = apply_bin_op_to_option_bitmap(
-        left.data().null_bitmap(),
-        right.data().null_bitmap(),
-        |a, b| a & b,
-    )?;
+    let null_bit_buffer =
+        combine_option_bitmap(left.data_ref(), right.data_ref(), left.len())?;
+
+    let values = (0..left.len())
+        .map(|i| op(left.value(i), right.value(i)))
+        .collect::<Vec<T::Native>>();
+
+    let data = ArrayData::new(
+        T::DATA_TYPE,
+        left.len(),
+        None,
+        null_bit_buffer,
+        0,
+        vec![Buffer::from(values.to_byte_slice())],
+        vec![],
+    );
+    Ok(PrimitiveArray::<T>::from(Arc::new(data)))
+}
+
+/// Helper function to divide two arrays.
+///
+/// # Errors
+///
+/// This function errors if:
+/// * the arrays have different lengths
+/// * a division by zero is found
+fn math_divide<T>(
+    left: &PrimitiveArray<T>,
+    right: &PrimitiveArray<T>,
+) -> Result<PrimitiveArray<T>>
+where
+    T: datatypes::ArrowNumericType,
+    T::Native: Div<Output = T::Native> + Zero,
+{
+    if left.len() != right.len() {
+        return Err(ArrowError::ComputeError(
+            "Cannot perform math operation on arrays of different length".to_string(),
+        ));
+    }
+
+    let null_bit_buffer =
+        combine_option_bitmap(left.data_ref(), right.data_ref(), left.len())?;
 
     let mut values = Vec::with_capacity(left.len());
     if let Some(b) = &null_bit_buffer {
+        // some value is null
         for i in 0..left.len() {
-            unsafe {
+            values.push(unsafe {
                 if bit_util::get_bit_raw(b.raw_data(), i) {
-                    values.push(op(left.value(i), right.value(i))?);
+                    let right_value = right.value(i);
+                    if right_value.is_zero() {
+                        return Err(ArrowError::DivideByZero);
+                    } else {
+                        left.value(i) / right_value
+                    }
                 } else {
-                    values.push(T::default_value())
+                    T::default_value()
                 }
-            }
+            });
         }
     } else {
+        // no value is null
         for i in 0..left.len() {
-            values.push(op(left.value(i), right.value(i))?);
+            let right_value = right.value(i);
+            values.push(if right_value.is_zero() {
+                return Err(ArrowError::DivideByZero);
+            } else {
+                left.value(i) / right_value
+            });
         }
-    }
+    };
 
     let data = ArrayData::new(
-        T::get_data_type(),
+        T::DATA_TYPE,
         left.len(),
         None,
         null_bit_buffer,
-        left.offset(),
+        0,
         vec![Buffer::from(values.to_byte_slice())],
         vec![],
     );
@@ -119,11 +171,8 @@ where
         ));
     }
 
-    let null_bit_buffer = apply_bin_op_to_option_bitmap(
-        left.data().null_bitmap(),
-        right.data().null_bitmap(),
-        |a, b| a & b,
-    )?;
+    let null_bit_buffer =
+        combine_option_bitmap(left.data_ref(), right.data_ref(), left.len())?;
 
     let lanes = T::lanes();
     let buffer_size = left.len() * mem::size_of::<T::Native>();
@@ -144,11 +193,11 @@ where
     }
 
     let data = ArrayData::new(
-        T::get_data_type(),
+        T::DATA_TYPE,
         left.len(),
         None,
         null_bit_buffer,
-        left.offset(),
+        0,
         vec![result.freeze()],
         vec![],
     );
@@ -174,12 +223,9 @@ where
     }
 
     // Create the combined `Bitmap`
-    let null_bit_buffer = apply_bin_op_to_option_bitmap(
-        left.data().null_bitmap(),
-        right.data().null_bitmap(),
-        |a, b| a & b,
-    )?;
-    let bitmap = null_bit_buffer.map(Bitmap::from);
+    let null_bit_buffer =
+        combine_option_bitmap(left.data_ref(), right.data_ref(), left.len())?;
+    let bitmap = null_bit_buffer.clone().map(Bitmap::from);
 
     let lanes = T::lanes();
     let buffer_size = left.len() * mem::size_of::<T::Native>();
@@ -192,8 +238,6 @@ where
         if T::mask_any(is_zero) {
             return Err(ArrowError::DivideByZero);
         }
-        let right_no_invalid_zeros =
-            unsafe { simd_load_set_invalid(right, &bitmap, i, lanes, T::Native::one()) };
         let simd_left = T::load(left.value_slice(i, lanes));
         let simd_result = T::bin_op(simd_left, right_no_invalid_zeros, |a, b| a / b);
 
@@ -206,14 +250,12 @@ where
         T::write(simd_result, result_slice);
     }
 
-    let null_bit_buffer = bitmap.map(|b| b.bits);
-
     let data = ArrayData::new(
-        T::get_data_type(),
+        T::DATA_TYPE,
         left.len(),
         None,
         null_bit_buffer,
-        left.offset(),
+        0,
         vec![result.freeze()],
         vec![],
     );
@@ -238,7 +280,7 @@ where
     return simd_math_op(&left, &right, |a, b| a + b);
 
     #[allow(unreachable_code)]
-    math_op(left, right, |a, b| Ok(a + b))
+    math_op(left, right, |a, b| a + b)
 }
 
 /// Perform `left - right` operation on two arrays. If either left or right value is null
@@ -259,7 +301,7 @@ where
     return simd_math_op(&left, &right, |a, b| a - b);
 
     #[allow(unreachable_code)]
-    math_op(left, right, |a, b| Ok(a - b))
+    math_op(left, right, |a, b| a - b)
 }
 
 /// Perform `left * right` operation on two arrays. If either left or right value is null
@@ -280,7 +322,7 @@ where
     return simd_math_op(&left, &right, |a, b| a * b);
 
     #[allow(unreachable_code)]
-    math_op(left, right, |a, b| Ok(a * b))
+    math_op(left, right, |a, b| a * b)
 }
 
 /// Perform `left / right` operation on two arrays. If either left or right value is null
@@ -303,13 +345,7 @@ where
     return simd_divide(&left, &right);
 
     #[allow(unreachable_code)]
-    math_op(left, right, |a, b| {
-        if b.is_zero() {
-            Err(ArrowError::DivideByZero)
-        } else {
-            Ok(a / b)
-        }
-    })
+    math_divide(&left, &right)
 }
 
 #[cfg(test)]
@@ -329,6 +365,27 @@ mod tests {
         assert_eq!(17, c.value(4));
     }
 
+    #[test]
+    fn test_primitive_array_add_sliced() {
+        let a = Int32Array::from(vec![0, 0, 0, 5, 6, 7, 8, 9, 0]);
+        let b = Int32Array::from(vec![0, 0, 0, 6, 7, 8, 9, 8, 0]);
+        let a = a.slice(3, 5);
+        let b = b.slice(3, 5);
+        let a = a.as_any().downcast_ref::<Int32Array>().unwrap();
+        let b = b.as_any().downcast_ref::<Int32Array>().unwrap();
+
+        assert_eq!(5, a.value(0));
+        assert_eq!(6, b.value(0));
+
+        let c = add(&a, &b).unwrap();
+        assert_eq!(5, c.len());
+        assert_eq!(11, c.value(0));
+        assert_eq!(13, c.value(1));
+        assert_eq!(15, c.value(2));
+        assert_eq!(17, c.value(3));
+        assert_eq!(17, c.value(4));
+    }
+
     #[test]
     fn test_primitive_array_add_mismatched_length() {
         let a = Int32Array::from(vec![5, 6, 7, 8, 9]);
@@ -378,6 +435,24 @@ mod tests {
         assert_eq!(9, c.value(4));
     }
 
+    #[test]
+    fn test_primitive_array_divide_sliced() {
+        let a = Int32Array::from(vec![0, 0, 0, 15, 15, 8, 1, 9, 0]);
+        let b = Int32Array::from(vec![0, 0, 0, 5, 6, 8, 9, 1, 0]);
+        let a = a.slice(3, 5);
+        let b = b.slice(3, 5);
+        let a = a.as_any().downcast_ref::<Int32Array>().unwrap();
+        let b = b.as_any().downcast_ref::<Int32Array>().unwrap();
+
+        let c = divide(&a, &b).unwrap();
+        assert_eq!(5, c.len());
+        assert_eq!(3, c.value(0));
+        assert_eq!(2, c.value(1));
+        assert_eq!(1, c.value(2));
+        assert_eq!(0, c.value(3));
+        assert_eq!(9, c.value(4));
+    }
+
     #[test]
     fn test_primitive_array_divide_with_nulls() {
         let a = Int32Array::from(vec![Some(15), None, Some(8), Some(1), Some(9), None]);
@@ -391,6 +466,59 @@ mod tests {
         assert_eq!(true, c.is_null(5));
     }
 
+    #[test]
+    fn test_primitive_array_divide_with_nulls_sliced() {
+        let a = Int32Array::from(vec![
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            Some(15),
+            None,
+            Some(8),
+            Some(1),
+            Some(9),
+            None,
+            None,
+        ]);
+        let b = Int32Array::from(vec![
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            Some(5),
+            Some(6),
+            Some(8),
+            Some(9),
+            None,
+            None,
+            None,
+        ]);
+
+        let a = a.slice(8, 6);
+        let a = a.as_any().downcast_ref::<Int32Array>().unwrap();
+
+        let b = b.slice(8, 6);
+        let b = b.as_any().downcast_ref::<Int32Array>().unwrap();
+
+        let c = divide(&a, &b).unwrap();
+        assert_eq!(6, c.len());
+        assert_eq!(3, c.value(0));
+        assert_eq!(true, c.is_null(1));
+        assert_eq!(1, c.value(2));
+        assert_eq!(0, c.value(3));
+        assert_eq!(true, c.is_null(4));
+        assert_eq!(true, c.is_null(5));
+    }
+
     #[test]
     #[should_panic(expected = "DivideByZero")]
     fn test_primitive_array_divide_by_zero() {
@@ -404,9 +532,9 @@ mod tests {
         let a = Float64Array::from(vec![15.0, 15.0, 8.0]);
         let b = Float64Array::from(vec![5.0, 6.0, 8.0]);
         let c = divide(&a, &b).unwrap();
-        assert_eq!(3.0, c.value(0));
-        assert_eq!(2.5, c.value(1));
-        assert_eq!(1.0, c.value(2));
+        assert!(3.0 - c.value(0) < f64::EPSILON);
+        assert!(2.5 - c.value(1) < f64::EPSILON);
+        assert!(1.0 - c.value(2) < f64::EPSILON);
     }
 
     #[test]
diff --git a/rust/arrow/src/compute/kernels/boolean.rs b/rust/arrow/src/compute/kernels/boolean.rs
index b4981a90363..075ffb0d67b 100644
--- a/rust/arrow/src/compute/kernels/boolean.rs
+++ b/rust/arrow/src/compute/kernels/boolean.rs
@@ -24,11 +24,14 @@
 
 use std::sync::Arc;
 
-use crate::array::{Array, ArrayData, BooleanArray};
-use crate::buffer::Buffer;
-use crate::compute::util::apply_bin_op_to_option_bitmap;
+use crate::array::{Array, ArrayData, ArrayRef, BooleanArray};
+use crate::buffer::{
+    buffer_bin_and, buffer_bin_or, buffer_unary_not, Buffer, MutableBuffer,
+};
+use crate::compute::util::combine_option_bitmap;
 use crate::datatypes::DataType;
 use crate::error::{ArrowError, Result};
+use crate::util::bit_util::ceil;
 
 /// Helper function to implement binary kernels
 fn binary_boolean_kernel<F>(
@@ -37,29 +40,33 @@ fn binary_boolean_kernel<F>(
     op: F,
 ) -> Result<BooleanArray>
 where
-    F: Fn(&Buffer, &Buffer) -> Result<Buffer>,
+    F: Fn(&Buffer, usize, &Buffer, usize, usize) -> Buffer,
 {
-    if left.offset() != right.offset() {
+    if left.len() != right.len() {
         return Err(ArrowError::ComputeError(
-            "Cannot apply Bitwise binary op when arrays have different offsets."
-                .to_string(),
+            "Cannot perform bitwise operation on arrays of different length".to_string(),
         ));
     }
 
-    let left_data = left.data();
-    let right_data = right.data();
-    let null_bit_buffer = apply_bin_op_to_option_bitmap(
-        left_data.null_bitmap(),
-        right_data.null_bitmap(),
-        |a, b| a & b,
-    )?;
-    let values = op(&left_data.buffers()[0], &right_data.buffers()[0])?;
+    let len = left.len();
+
+    let left_data = left.data_ref();
+    let right_data = right.data_ref();
+    let null_bit_buffer = combine_option_bitmap(&left_data, &right_data, len)?;
+
+    let left_buffer = &left_data.buffers()[0];
+    let right_buffer = &right_data.buffers()[0];
+    let left_offset = left.offset();
+    let right_offset = right.offset();
+
+    let values = op(&left_buffer, left_offset, &right_buffer, right_offset, len);
+
     let data = ArrayData::new(
         DataType::Boolean,
-        left.len(),
+        len,
         None,
         null_bit_buffer,
-        left.offset(),
+        0,
         vec![values],
         vec![],
     );
@@ -69,37 +76,83 @@ where
 /// Performs `AND` operation on two arrays. If either left or right value is null then the
 /// result is also null.
 pub fn and(left: &BooleanArray, right: &BooleanArray) -> Result<BooleanArray> {
-    binary_boolean_kernel(&left, &right, |a, b| a & b)
+    binary_boolean_kernel(&left, &right, buffer_bin_and)
 }
 
 /// Performs `OR` operation on two arrays. If either left or right value is null then the
 /// result is also null.
 pub fn or(left: &BooleanArray, right: &BooleanArray) -> Result<BooleanArray> {
-    binary_boolean_kernel(&left, &right, |a, b| a | b)
+    binary_boolean_kernel(&left, &right, buffer_bin_or)
 }
 
 /// Performs unary `NOT` operation on an arrays. If value is null then the result is also
 /// null.
 pub fn not(left: &BooleanArray) -> Result<BooleanArray> {
-    let data = left.data();
-    let null_bit_buffer = data.null_bitmap().as_ref().map(|b| b.bits.clone());
+    let left_offset = left.offset();
+    let len = left.len();
+
+    let data = left.data_ref();
+    let null_bit_buffer = data
+        .null_bitmap()
+        .as_ref()
+        .map(|b| b.bits.slice(left_offset));
+
+    let values = buffer_unary_not(&data.buffers()[0], left_offset, len);
 
-    let values = !&data.buffers()[0];
     let data = ArrayData::new(
         DataType::Boolean,
-        left.len(),
+        len,
         None,
         null_bit_buffer,
-        left.offset(),
+        0,
         vec![values],
         vec![],
     );
     Ok(BooleanArray::from(Arc::new(data)))
 }
 
+pub fn is_null(input: &ArrayRef) -> Result<BooleanArray> {
+    let len = input.len();
+
+    let output = match input.data_ref().null_buffer() {
+        None => {
+            let len_bytes = ceil(len, 8);
+            MutableBuffer::new(len_bytes)
+                .with_bitset(len_bytes, false)
+                .freeze()
+        }
+        Some(buffer) => buffer_unary_not(buffer, input.offset(), len),
+    };
+
+    let data =
+        ArrayData::new(DataType::Boolean, len, None, None, 0, vec![output], vec![]);
+
+    Ok(BooleanArray::from(Arc::new(data)))
+}
+
+pub fn is_not_null(input: &ArrayRef) -> Result<BooleanArray> {
+    let len = input.len();
+
+    let output = match input.data_ref().null_buffer() {
+        None => {
+            let len_bytes = ceil(len, 8);
+            MutableBuffer::new(len_bytes)
+                .with_bitset(len_bytes, true)
+                .freeze()
+        }
+        Some(buffer) => buffer.bit_slice(input.offset(), len),
+    };
+
+    let data =
+        ArrayData::new(DataType::Boolean, len, None, None, 0, vec![output], vec![]);
+
+    Ok(BooleanArray::from(Arc::new(data)))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::array::Int32Array;
 
     #[test]
     fn test_bool_array_and() {
@@ -154,4 +207,254 @@ mod tests {
         assert_eq!(true, c.is_null(2));
         assert_eq!(false, c.is_null(3));
     }
+
+    #[test]
+    fn test_bool_array_and_sliced_same_offset() {
+        let a = BooleanArray::from(vec![
+            false, false, false, false, false, false, false, false, false, false, true,
+            true,
+        ]);
+        let b = BooleanArray::from(vec![
+            false, false, false, false, false, false, false, false, false, true, false,
+            true,
+        ]);
+
+        let a = a.slice(8, 4);
+        let a = a.as_any().downcast_ref::<BooleanArray>().unwrap();
+        let b = b.slice(8, 4);
+        let b = b.as_any().downcast_ref::<BooleanArray>().unwrap();
+
+        let c = and(&a, &b).unwrap();
+        assert_eq!(4, c.len());
+        assert_eq!(false, c.value(0));
+        assert_eq!(false, c.value(1));
+        assert_eq!(false, c.value(2));
+        assert_eq!(true, c.value(3));
+    }
+
+    #[test]
+    fn test_bool_array_and_sliced_same_offset_mod8() {
+        let a = BooleanArray::from(vec![
+            false, false, true, true, false, false, false, false, false, false, false,
+            false,
+        ]);
+        let b = BooleanArray::from(vec![
+            false, false, false, false, false, false, false, false, false, true, false,
+            true,
+        ]);
+
+        let a = a.slice(0, 4);
+        let a = a.as_any().downcast_ref::<BooleanArray>().unwrap();
+        let b = b.slice(8, 4);
+        let b = b.as_any().downcast_ref::<BooleanArray>().unwrap();
+
+        let c = and(&a, &b).unwrap();
+        assert_eq!(4, c.len());
+        assert_eq!(false, c.value(0));
+        assert_eq!(false, c.value(1));
+        assert_eq!(false, c.value(2));
+        assert_eq!(true, c.value(3));
+    }
+
+    #[test]
+    fn test_bool_array_and_sliced_offset1() {
+        let a = BooleanArray::from(vec![
+            false, false, false, false, false, false, false, false, false, false, true,
+            true,
+        ]);
+        let b = BooleanArray::from(vec![false, true, false, true]);
+
+        let a = a.slice(8, 4);
+        let a = a.as_any().downcast_ref::<BooleanArray>().unwrap();
+
+        let c = and(&a, &b).unwrap();
+        assert_eq!(4, c.len());
+        assert_eq!(false, c.value(0));
+        assert_eq!(false, c.value(1));
+        assert_eq!(false, c.value(2));
+        assert_eq!(true, c.value(3));
+    }
+
+    #[test]
+    fn test_bool_array_and_sliced_offset2() {
+        let a = BooleanArray::from(vec![false, false, true, true]);
+        let b = BooleanArray::from(vec![
+            false, false, false, false, false, false, false, false, false, true, false,
+            true,
+        ]);
+
+        let b = b.slice(8, 4);
+        let b = b.as_any().downcast_ref::<BooleanArray>().unwrap();
+
+        let c = and(&a, &b).unwrap();
+        assert_eq!(4, c.len());
+        assert_eq!(false, c.value(0));
+        assert_eq!(false, c.value(1));
+        assert_eq!(false, c.value(2));
+        assert_eq!(true, c.value(3));
+    }
+
+    #[test]
+    fn test_nonnull_array_is_null() {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
+
+        let res = is_null(&a).unwrap();
+
+        assert_eq!(4, res.len());
+        assert_eq!(0, res.null_count());
+        assert_eq!(&None, res.data_ref().null_bitmap());
+        assert_eq!(false, res.value(0));
+        assert_eq!(false, res.value(1));
+        assert_eq!(false, res.value(2));
+        assert_eq!(false, res.value(3));
+    }
+
+    #[test]
+    fn test_nonnull_array_with_offset_is_null() {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![
+            1, 2, 3, 4, 5, 6, 7, 8, 7, 6, 5, 4, 3, 2, 1,
+        ]));
+        let a = a.slice(8, 4);
+
+        let res = is_null(&a).unwrap();
+
+        assert_eq!(4, res.len());
+        assert_eq!(0, res.null_count());
+        assert_eq!(&None, res.data_ref().null_bitmap());
+        assert_eq!(false, res.value(0));
+        assert_eq!(false, res.value(1));
+        assert_eq!(false, res.value(2));
+        assert_eq!(false, res.value(3));
+    }
+
+    #[test]
+    fn test_nonnull_array_is_not_null() {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
+
+        let res = is_not_null(&a).unwrap();
+
+        assert_eq!(4, res.len());
+        assert_eq!(0, res.null_count());
+        assert_eq!(&None, res.data_ref().null_bitmap());
+        assert_eq!(true, res.value(0));
+        assert_eq!(true, res.value(1));
+        assert_eq!(true, res.value(2));
+        assert_eq!(true, res.value(3));
+    }
+
+    #[test]
+    fn test_nonnull_array_with_offset_is_not_null() {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![
+            1, 2, 3, 4, 5, 6, 7, 8, 7, 6, 5, 4, 3, 2, 1,
+        ]));
+        let a = a.slice(8, 4);
+
+        let res = is_not_null(&a).unwrap();
+
+        assert_eq!(4, res.len());
+        assert_eq!(0, res.null_count());
+        assert_eq!(&None, res.data_ref().null_bitmap());
+        assert_eq!(true, res.value(0));
+        assert_eq!(true, res.value(1));
+        assert_eq!(true, res.value(2));
+        assert_eq!(true, res.value(3));
+    }
+
+    #[test]
+    fn test_nullable_array_is_null() {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None, Some(3), None]));
+
+        let res = is_null(&a).unwrap();
+
+        assert_eq!(4, res.len());
+        assert_eq!(0, res.null_count());
+        assert_eq!(&None, res.data_ref().null_bitmap());
+        assert_eq!(false, res.value(0));
+        assert_eq!(true, res.value(1));
+        assert_eq!(false, res.value(2));
+        assert_eq!(true, res.value(3));
+    }
+
+    #[test]
+    fn test_nullable_array_with_offset_is_null() {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            // offset 8, previous None values are skipped by the slice
+            Some(1),
+            None,
+            Some(2),
+            None,
+            Some(3),
+            Some(4),
+            None,
+            None,
+        ]));
+        let a = a.slice(8, 4);
+
+        let res = is_null(&a).unwrap();
+
+        assert_eq!(4, res.len());
+        assert_eq!(0, res.null_count());
+        assert_eq!(&None, res.data_ref().null_bitmap());
+        assert_eq!(false, res.value(0));
+        assert_eq!(true, res.value(1));
+        assert_eq!(false, res.value(2));
+        assert_eq!(true, res.value(3));
+    }
+
+    #[test]
+    fn test_nullable_array_is_not_null() {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None, Some(3), None]));
+
+        let res = is_not_null(&a).unwrap();
+
+        assert_eq!(4, res.len());
+        assert_eq!(0, res.null_count());
+        assert_eq!(&None, res.data_ref().null_bitmap());
+        assert_eq!(true, res.value(0));
+        assert_eq!(false, res.value(1));
+        assert_eq!(true, res.value(2));
+        assert_eq!(false, res.value(3));
+    }
+
+    #[test]
+    fn test_nullable_array_with_offset_is_not_null() {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            // offset 8, previous None values are skipped by the slice
+            Some(1),
+            None,
+            Some(2),
+            None,
+            Some(3),
+            Some(4),
+            None,
+            None,
+        ]));
+        let a = a.slice(8, 4);
+
+        let res = is_not_null(&a).unwrap();
+
+        assert_eq!(4, res.len());
+        assert_eq!(0, res.null_count());
+        assert_eq!(&None, res.data_ref().null_bitmap());
+        assert_eq!(true, res.value(0));
+        assert_eq!(false, res.value(1));
+        assert_eq!(true, res.value(2));
+        assert_eq!(false, res.value(3));
+    }
 }
diff --git a/rust/arrow/src/compute/kernels/cast.rs b/rust/arrow/src/compute/kernels/cast.rs
index f443e5ab1f3..9d1b7fc6011 100644
--- a/rust/arrow/src/compute/kernels/cast.rs
+++ b/rust/arrow/src/compute/kernels/cast.rs
@@ -15,8 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//! Defines cast kernels for `ArrayRef`, allowing casting arrays between supported
-//! datatypes.
+//! Defines cast kernels for `ArrayRef`, to convert `Array`s between
+//! supported datatypes.
 //!
 //! Example:
 //!
@@ -38,13 +38,177 @@
 use std::str;
 use std::sync::Arc;
 
-use crate::array::*;
 use crate::buffer::Buffer;
 use crate::compute::kernels::arithmetic::{divide, multiply};
 use crate::datatypes::*;
 use crate::error::{ArrowError, Result};
+use crate::{array::*, compute::take};
 
-/// Cast array to provided data type
+/// Return true if a value of type `from_type` can be cast into a
+/// value of `to_type`. Note that such as cast may be lossy.
+///
+/// If this function returns true to stay consistent with the `cast` kernel below.
+pub fn can_cast_types(from_type: &DataType, to_type: &DataType) -> bool {
+    use self::DataType::*;
+    if from_type == to_type {
+        return true;
+    }
+
+    match (from_type, to_type) {
+        (Struct(_), _) => false,
+        (_, Struct(_)) => false,
+        (List(list_from), List(list_to)) => {
+            can_cast_types(list_from.data_type(), list_to.data_type())
+        }
+        (List(_), _) => false,
+        (_, List(list_to)) => can_cast_types(from_type, list_to.data_type()),
+        (Dictionary(_, from_value_type), Dictionary(_, to_value_type)) => {
+            can_cast_types(from_value_type, to_value_type)
+        }
+        (Dictionary(_, value_type), _) => can_cast_types(value_type, to_type),
+        (_, Dictionary(_, value_type)) => can_cast_types(from_type, value_type),
+
+        (_, Boolean) => DataType::is_numeric(from_type),
+        (Boolean, _) => DataType::is_numeric(to_type) || to_type == &Utf8,
+        (Utf8, _) => DataType::is_numeric(to_type),
+        (_, Utf8) => DataType::is_numeric(from_type) || from_type == &Binary,
+
+        // start numeric casts
+        (UInt8, UInt16) => true,
+        (UInt8, UInt32) => true,
+        (UInt8, UInt64) => true,
+        (UInt8, Int8) => true,
+        (UInt8, Int16) => true,
+        (UInt8, Int32) => true,
+        (UInt8, Int64) => true,
+        (UInt8, Float32) => true,
+        (UInt8, Float64) => true,
+
+        (UInt16, UInt8) => true,
+        (UInt16, UInt32) => true,
+        (UInt16, UInt64) => true,
+        (UInt16, Int8) => true,
+        (UInt16, Int16) => true,
+        (UInt16, Int32) => true,
+        (UInt16, Int64) => true,
+        (UInt16, Float32) => true,
+        (UInt16, Float64) => true,
+
+        (UInt32, UInt8) => true,
+        (UInt32, UInt16) => true,
+        (UInt32, UInt64) => true,
+        (UInt32, Int8) => true,
+        (UInt32, Int16) => true,
+        (UInt32, Int32) => true,
+        (UInt32, Int64) => true,
+        (UInt32, Float32) => true,
+        (UInt32, Float64) => true,
+
+        (UInt64, UInt8) => true,
+        (UInt64, UInt16) => true,
+        (UInt64, UInt32) => true,
+        (UInt64, Int8) => true,
+        (UInt64, Int16) => true,
+        (UInt64, Int32) => true,
+        (UInt64, Int64) => true,
+        (UInt64, Float32) => true,
+        (UInt64, Float64) => true,
+
+        (Int8, UInt8) => true,
+        (Int8, UInt16) => true,
+        (Int8, UInt32) => true,
+        (Int8, UInt64) => true,
+        (Int8, Int16) => true,
+        (Int8, Int32) => true,
+        (Int8, Int64) => true,
+        (Int8, Float32) => true,
+        (Int8, Float64) => true,
+
+        (Int16, UInt8) => true,
+        (Int16, UInt16) => true,
+        (Int16, UInt32) => true,
+        (Int16, UInt64) => true,
+        (Int16, Int8) => true,
+        (Int16, Int32) => true,
+        (Int16, Int64) => true,
+        (Int16, Float32) => true,
+        (Int16, Float64) => true,
+
+        (Int32, UInt8) => true,
+        (Int32, UInt16) => true,
+        (Int32, UInt32) => true,
+        (Int32, UInt64) => true,
+        (Int32, Int8) => true,
+        (Int32, Int16) => true,
+        (Int32, Int64) => true,
+        (Int32, Float32) => true,
+        (Int32, Float64) => true,
+
+        (Int64, UInt8) => true,
+        (Int64, UInt16) => true,
+        (Int64, UInt32) => true,
+        (Int64, UInt64) => true,
+        (Int64, Int8) => true,
+        (Int64, Int16) => true,
+        (Int64, Int32) => true,
+        (Int64, Float32) => true,
+        (Int64, Float64) => true,
+
+        (Float32, UInt8) => true,
+        (Float32, UInt16) => true,
+        (Float32, UInt32) => true,
+        (Float32, UInt64) => true,
+        (Float32, Int8) => true,
+        (Float32, Int16) => true,
+        (Float32, Int32) => true,
+        (Float32, Int64) => true,
+        (Float32, Float64) => true,
+
+        (Float64, UInt8) => true,
+        (Float64, UInt16) => true,
+        (Float64, UInt32) => true,
+        (Float64, UInt64) => true,
+        (Float64, Int8) => true,
+        (Float64, Int16) => true,
+        (Float64, Int32) => true,
+        (Float64, Int64) => true,
+        (Float64, Float32) => true,
+        // end numeric casts
+
+        // temporal casts
+        (Int32, Date32(_)) => true,
+        (Int32, Time32(_)) => true,
+        (Date32(_), Int32) => true,
+        (Time32(_), Int32) => true,
+        (Int64, Date64(_)) => true,
+        (Int64, Time64(_)) => true,
+        (Date64(_), Int64) => true,
+        (Time64(_), Int64) => true,
+        (Date32(DateUnit::Day), Date64(DateUnit::Millisecond)) => true,
+        (Date64(DateUnit::Millisecond), Date32(DateUnit::Day)) => true,
+        (Time32(TimeUnit::Second), Time32(TimeUnit::Millisecond)) => true,
+        (Time32(TimeUnit::Millisecond), Time32(TimeUnit::Second)) => true,
+        (Time32(_), Time64(_)) => true,
+        (Time64(TimeUnit::Microsecond), Time64(TimeUnit::Nanosecond)) => true,
+        (Time64(TimeUnit::Nanosecond), Time64(TimeUnit::Microsecond)) => true,
+        (Time64(_), Time32(to_unit)) => match to_unit {
+            TimeUnit::Second => true,
+            TimeUnit::Millisecond => true,
+            _ => false,
+        },
+        (Timestamp(_, _), Int64) => true,
+        (Int64, Timestamp(_, _)) => true,
+        (Timestamp(_, _), Timestamp(_, _)) => true,
+        (Timestamp(_, _), Date32(_)) => true,
+        (Timestamp(_, _), Date64(_)) => true,
+        // date64 to timestamp might not make sense,
+        (Null, Int32) => true,
+        (_, _) => false,
+    }
+}
+
+/// Cast `array` to the provided data type and return a new Array with
+/// type `to_type`, if possible.
 ///
 /// Behavior:
 /// * Boolean to Utf8: `true` => '1', `false` => `0`
@@ -81,9 +245,9 @@ pub fn cast(array: &ArrayRef, to_type: &DataType) -> Result<ArrayRef> {
         (List(_), List(ref to)) => {
             let data = array.data_ref();
             let underlying_array = make_array(data.child_data()[0].clone());
-            let cast_array = cast(&underlying_array, &to)?;
+            let cast_array = cast(&underlying_array, to.data_type())?;
             let array_data = ArrayData::new(
-                *to.clone(),
+                to.data_type().clone(),
                 array.len(),
                 Some(cast_array.null_count()),
                 cast_array
@@ -104,12 +268,12 @@ pub fn cast(array: &ArrayRef, to_type: &DataType) -> Result<ArrayRef> {
         )),
         (_, List(ref to)) => {
             // cast primitive to list's primitive
-            let cast_array = cast(array, &to)?;
+            let cast_array = cast(array, to.data_type())?;
             // create offsets, where if array.len() = 2, we have [0,1,2]
             let offsets: Vec<i32> = (0..=array.len() as i32).collect();
             let value_offsets = Buffer::from(offsets[..].to_byte_slice());
             let list_data = ArrayData::new(
-                *to.clone(),
+                to.data_type().clone(),
                 array.len(),
                 Some(cast_array.null_count()),
                 cast_array
@@ -125,6 +289,34 @@ pub fn cast(array: &ArrayRef, to_type: &DataType) -> Result<ArrayRef> {
 
             Ok(list_array)
         }
+        (Dictionary(index_type, _), _) => match **index_type {
+            DataType::Int8 => dictionary_cast::<Int8Type>(array, to_type),
+            DataType::Int16 => dictionary_cast::<Int16Type>(array, to_type),
+            DataType::Int32 => dictionary_cast::<Int32Type>(array, to_type),
+            DataType::Int64 => dictionary_cast::<Int64Type>(array, to_type),
+            DataType::UInt8 => dictionary_cast::<UInt8Type>(array, to_type),
+            DataType::UInt16 => dictionary_cast::<UInt16Type>(array, to_type),
+            DataType::UInt32 => dictionary_cast::<UInt32Type>(array, to_type),
+            DataType::UInt64 => dictionary_cast::<UInt64Type>(array, to_type),
+            _ => Err(ArrowError::ComputeError(format!(
+                "Casting from dictionary type {:?} to {:?} not supported",
+                from_type, to_type,
+            ))),
+        },
+        (_, Dictionary(index_type, value_type)) => match **index_type {
+            DataType::Int8 => cast_to_dictionary::<Int8Type>(array, value_type),
+            DataType::Int16 => cast_to_dictionary::<Int16Type>(array, value_type),
+            DataType::Int32 => cast_to_dictionary::<Int32Type>(array, value_type),
+            DataType::Int64 => cast_to_dictionary::<Int64Type>(array, value_type),
+            DataType::UInt8 => cast_to_dictionary::<UInt8Type>(array, value_type),
+            DataType::UInt16 => cast_to_dictionary::<UInt16Type>(array, value_type),
+            DataType::UInt32 => cast_to_dictionary::<UInt32Type>(array, value_type),
+            DataType::UInt64 => cast_to_dictionary::<UInt64Type>(array, value_type),
+            _ => Err(ArrowError::ComputeError(format!(
+                "Casting from type {:?} to dictionary type {:?} not supported",
+                from_type, to_type,
+            ))),
+        },
         (_, Boolean) => match from_type {
             UInt8 => cast_numeric_to_bool::<UInt8Type>(array),
             UInt16 => cast_numeric_to_bool::<UInt16Type>(array),
@@ -327,11 +519,24 @@ pub fn cast(array: &ArrayRef, to_type: &DataType) -> Result<ArrayRef> {
 
         // temporal casts
         (Int32, Date32(_)) => cast_array_data::<Date32Type>(array, to_type.clone()),
-        (Int32, Time32(_)) => cast_array_data::<Date32Type>(array, to_type.clone()),
+        (Int32, Time32(TimeUnit::Second)) => {
+            cast_array_data::<Time32SecondType>(array, to_type.clone())
+        }
+        (Int32, Time32(TimeUnit::Millisecond)) => {
+            cast_array_data::<Time32MillisecondType>(array, to_type.clone())
+        }
+        // No support for microsecond/nanosecond with i32
         (Date32(_), Int32) => cast_array_data::<Int32Type>(array, to_type.clone()),
         (Time32(_), Int32) => cast_array_data::<Int32Type>(array, to_type.clone()),
         (Int64, Date64(_)) => cast_array_data::<Date64Type>(array, to_type.clone()),
-        (Int64, Time64(_)) => cast_array_data::<Date64Type>(array, to_type.clone()),
+        // No support for second/milliseconds with i64
+        (Int64, Time64(TimeUnit::Microsecond)) => {
+            cast_array_data::<Time64MicrosecondType>(array, to_type.clone())
+        }
+        (Int64, Time64(TimeUnit::Nanosecond)) => {
+            cast_array_data::<Time64NanosecondType>(array, to_type.clone())
+        }
+
         (Date64(_), Int64) => cast_array_data::<Int64Type>(array, to_type.clone()),
         (Time64(_), Int64) => cast_array_data::<Int64Type>(array, to_type.clone()),
         (Date32(DateUnit::Day), Date64(DateUnit::Millisecond)) => {
@@ -378,7 +583,7 @@ pub fn cast(array: &ArrayRef, to_type: &DataType) -> Result<ArrayRef> {
         (Time32(from_unit), Time64(to_unit)) => {
             let time_array = Int32Array::from(array.data());
             // note: (numeric_cast + SIMD multiply) is faster than (cast & multiply)
-            let c: Int64Array = numeric_cast(&time_array)?;
+            let c: Int64Array = numeric_cast(&time_array);
             let from_size = time_unit_multiple(&from_unit);
             let to_size = time_unit_multiple(&to_unit);
             // from is only smaller than to if 64milli/64second don't exist
@@ -520,19 +725,36 @@ pub fn cast(array: &ArrayRef, to_type: &DataType) -> Result<ArrayRef> {
         (Timestamp(from_unit, _), Date64(_)) => {
             let from_size = time_unit_multiple(&from_unit);
             let to_size = MILLISECONDS;
-            if from_size != to_size {
-                let time_array = Date64Array::from(array.data());
-                Ok(Arc::new(divide(
-                    &time_array,
-                    &Date64Array::from(vec![from_size / to_size; array.len()]),
-                )?) as ArrayRef)
-            } else {
-                cast_array_data::<Date64Type>(array, to_type.clone())
+
+            // Scale time_array by (to_size / from_size) using a
+            // single integer operation, but need to avoid integer
+            // math rounding down to zero
+
+            match to_size.cmp(&from_size) {
+                std::cmp::Ordering::Less => {
+                    let time_array = Date64Array::from(array.data());
+                    Ok(Arc::new(divide(
+                        &time_array,
+                        &Date64Array::from(vec![from_size / to_size; array.len()]),
+                    )?) as ArrayRef)
+                }
+                std::cmp::Ordering::Equal => {
+                    cast_array_data::<Date64Type>(array, to_type.clone())
+                }
+                std::cmp::Ordering::Greater => {
+                    let time_array = Date64Array::from(array.data());
+                    Ok(Arc::new(multiply(
+                        &time_array,
+                        &Date64Array::from(vec![to_size / from_size; array.len()]),
+                    )?) as ArrayRef)
+                }
             }
         }
         // date64 to timestamp might not make sense,
 
-        // end temporal casts
+        // null to primitive/flat types
+        (Null, Int32) => Ok(Arc::new(Int32Array::from(vec![None; array.len()]))),
+
         (_, _) => Err(ArrowError::ComputeError(format!(
             "Casting from {:?} to {:?} not supported",
             from_type, to_type,
@@ -590,37 +812,27 @@ where
     FROM::Native: num::NumCast,
     TO::Native: num::NumCast,
 {
-    numeric_cast::<FROM, TO>(
+    Ok(Arc::new(numeric_cast::<FROM, TO>(
         from.as_any()
             .downcast_ref::<PrimitiveArray<FROM>>()
             .unwrap(),
-    )
-    .map(|to| Arc::new(to) as ArrayRef)
+    )))
 }
 
 /// Natural cast between numeric types
-fn numeric_cast<T, R>(from: &PrimitiveArray<T>) -> Result<PrimitiveArray<R>>
+fn numeric_cast<T, R>(from: &PrimitiveArray<T>) -> PrimitiveArray<R>
 where
     T: ArrowNumericType,
     R: ArrowNumericType,
     T::Native: num::NumCast,
     R::Native: num::NumCast,
 {
-    let mut b = PrimitiveBuilder::<R>::new(from.len());
-
-    for i in 0..from.len() {
-        if from.is_null(i) {
-            b.append_null()?;
-        } else {
-            // some casts return None, such as a negative value to u{8|16|32|64}
-            match num::cast::cast(from.value(i)) {
-                Some(v) => b.append_value(v)?,
-                None => b.append_null()?,
-            };
-        }
-    }
-
-    Ok(b.finish())
+    from.iter()
+        .map(|v| match v {
+            Some(v) => num::cast::cast::<T::Native, R::Native>(v),
+            None => None,
+        })
+        .collect()
 }
 
 /// Cast numeric types to Utf8
@@ -661,28 +873,27 @@ fn cast_string_to_numeric<TO>(from: &ArrayRef) -> Result<ArrayRef>
 where
     TO: ArrowNumericType,
 {
-    string_to_numeric_cast::<TO>(from.as_any().downcast_ref::<StringArray>().unwrap())
-        .map(|to| Arc::new(to) as ArrayRef)
+    Ok(Arc::new(string_to_numeric_cast::<TO>(
+        from.as_any().downcast_ref::<StringArray>().unwrap(),
+    )))
 }
 
-fn string_to_numeric_cast<T>(from: &StringArray) -> Result<PrimitiveArray<T>>
+fn string_to_numeric_cast<T>(from: &StringArray) -> PrimitiveArray<T>
 where
     T: ArrowNumericType,
 {
-    let mut b = PrimitiveBuilder::<T>::new(from.len());
-
-    for i in 0..from.len() {
-        if from.is_null(i) {
-            b.append_null()?;
-        } else {
-            match from.value(i).parse::<T::Native>() {
-                Ok(v) => b.append_value(v)?,
-                _ => b.append_null()?,
-            };
-        }
-    }
-
-    Ok(b.finish())
+    (0..from.len())
+        .map(|i| {
+            if from.is_null(i) {
+                None
+            } else {
+                match from.value(i).parse::<T::Native>() {
+                    Ok(v) => Some(v),
+                    Err(_) => None,
+                }
+            }
+        })
+        .collect()
 }
 
 /// Cast numeric types to Boolean
@@ -727,38 +938,227 @@ where
     TO: ArrowNumericType,
     TO::Native: num::cast::NumCast,
 {
-    bool_to_numeric_cast::<TO>(from.as_any().downcast_ref::<BooleanArray>().unwrap())
-        .map(|to| Arc::new(to) as ArrayRef)
+    Ok(Arc::new(bool_to_numeric_cast::<TO>(
+        from.as_any().downcast_ref::<BooleanArray>().unwrap(),
+    )))
 }
 
-fn bool_to_numeric_cast<T>(from: &BooleanArray) -> Result<PrimitiveArray<T>>
+fn bool_to_numeric_cast<T>(from: &BooleanArray) -> PrimitiveArray<T>
 where
     T: ArrowNumericType,
     T::Native: num::NumCast,
 {
-    let mut b = PrimitiveBuilder::<T>::new(from.len());
+    (0..from.len())
+        .map(|i| {
+            if from.is_null(i) {
+                None
+            } else if from.value(i) {
+                // a workaround to cast a primitive to T::Native, infallible
+                num::cast::cast(1)
+            } else {
+                Some(T::default_value())
+            }
+        })
+        .collect()
+}
 
-    for i in 0..from.len() {
-        if from.is_null(i) {
-            b.append_null()?;
-        } else if from.value(i) {
-            // a workaround to cast a primitive to T::Native, infallible
-            match num::cast::cast(1) {
-                Some(v) => b.append_value(v)?,
-                None => b.append_null()?,
+/// Attempts to cast an `ArrayDictionary` with index type K into
+/// `to_type` for supported types.
+///
+/// K is the key type
+fn dictionary_cast<K: ArrowDictionaryKeyType>(
+    array: &ArrayRef,
+    to_type: &DataType,
+) -> Result<ArrayRef> {
+    use DataType::*;
+
+    match to_type {
+        Dictionary(to_index_type, to_value_type) => {
+            let dict_array = array
+                .as_any()
+                .downcast_ref::<DictionaryArray<K>>()
+                .ok_or_else(|| {
+                    ArrowError::ComputeError(
+                        "Internal Error: Cannot cast dictionary to DictionaryArray of expected type".to_string(),
+                    )
+                })?;
+
+            let keys_array: ArrayRef = Arc::new(dict_array.keys_array());
+            let values_array: ArrayRef = dict_array.values();
+            let cast_keys = cast(&keys_array, to_index_type)?;
+            let cast_values = cast(&values_array, to_value_type)?;
+
+            // Failure to cast keys (because they don't fit in the
+            // target type) results in NULL values;
+            if cast_keys.null_count() > keys_array.null_count() {
+                return Err(ArrowError::ComputeError(format!(
+                    "Could not convert {} dictionary indexes from {:?} to {:?}",
+                    cast_keys.null_count() - keys_array.null_count(),
+                    keys_array.data_type(),
+                    to_index_type
+                )));
+            }
+
+            // keys are data, child_data is values (dictionary)
+            let data = Arc::new(ArrayData::new(
+                to_type.clone(),
+                cast_keys.len(),
+                Some(cast_keys.null_count()),
+                cast_keys
+                    .data()
+                    .null_bitmap()
+                    .clone()
+                    .map(|bitmap| bitmap.bits),
+                cast_keys.data().offset(),
+                cast_keys.data().buffers().to_vec(),
+                vec![cast_values.data()],
+            ));
+
+            // create the appropriate array type
+            let new_array: ArrayRef = match **to_index_type {
+                Int8 => Arc::new(DictionaryArray::<Int8Type>::from(data)),
+                Int16 => Arc::new(DictionaryArray::<Int16Type>::from(data)),
+                Int32 => Arc::new(DictionaryArray::<Int32Type>::from(data)),
+                Int64 => Arc::new(DictionaryArray::<Int64Type>::from(data)),
+                UInt8 => Arc::new(DictionaryArray::<UInt8Type>::from(data)),
+                UInt16 => Arc::new(DictionaryArray::<UInt16Type>::from(data)),
+                UInt32 => Arc::new(DictionaryArray::<UInt32Type>::from(data)),
+                UInt64 => Arc::new(DictionaryArray::<UInt64Type>::from(data)),
+                _ => {
+                    return Err(ArrowError::ComputeError(format!(
+                        "Unsupported type {:?} for dictionary index",
+                        to_index_type
+                    )))
+                }
             };
+
+            Ok(new_array)
+        }
+        _ => unpack_dictionary::<K>(array, to_type),
+    }
+}
+
+// Unpack a dictionary where the keys are of type <K> into a flattened array of type to_type
+fn unpack_dictionary<K>(array: &ArrayRef, to_type: &DataType) -> Result<ArrayRef>
+where
+    K: ArrowDictionaryKeyType,
+{
+    let dict_array = array
+        .as_any()
+        .downcast_ref::<DictionaryArray<K>>()
+        .ok_or_else(|| {
+            ArrowError::ComputeError(
+                "Internal Error: Cannot cast dictionary to DictionaryArray of expected type".to_string(),
+            )
+        })?;
+
+    // attempt to cast the dict values to the target type
+    // use the take kernel to expand out the dictionary
+    let cast_dict_values = cast(&dict_array.values(), to_type)?;
+
+    // Note take requires first casting the indices to u32
+    let keys_array: ArrayRef = Arc::new(dict_array.keys_array());
+    let indicies = cast(&keys_array, &DataType::UInt32)?;
+    let u32_indicies =
+        indicies
+            .as_any()
+            .downcast_ref::<UInt32Array>()
+            .ok_or_else(|| {
+                ArrowError::ComputeError(
+                    "Internal Error: Cannot cast dict indices to UInt32".to_string(),
+                )
+            })?;
+
+    take(&cast_dict_values, u32_indicies, None)
+}
+
+/// Attempts to encode an array into an `ArrayDictionary` with index
+/// type K and value (dictionary) type value_type
+///
+/// K is the key type
+fn cast_to_dictionary<K: ArrowDictionaryKeyType>(
+    array: &ArrayRef,
+    dict_value_type: &DataType,
+) -> Result<ArrayRef> {
+    use DataType::*;
+
+    match *dict_value_type {
+        Int8 => pack_numeric_to_dictionary::<K, Int8Type>(array, dict_value_type),
+        Int16 => pack_numeric_to_dictionary::<K, Int16Type>(array, dict_value_type),
+        Int32 => pack_numeric_to_dictionary::<K, Int32Type>(array, dict_value_type),
+        Int64 => pack_numeric_to_dictionary::<K, Int64Type>(array, dict_value_type),
+        UInt8 => pack_numeric_to_dictionary::<K, UInt8Type>(array, dict_value_type),
+        UInt16 => pack_numeric_to_dictionary::<K, UInt16Type>(array, dict_value_type),
+        UInt32 => pack_numeric_to_dictionary::<K, UInt32Type>(array, dict_value_type),
+        UInt64 => pack_numeric_to_dictionary::<K, UInt64Type>(array, dict_value_type),
+        Utf8 => pack_string_to_dictionary::<K>(array),
+        _ => Err(ArrowError::ComputeError(format!(
+            "Internal Error: Unsupported output type for dictionary packing: {:?}",
+            dict_value_type
+        ))),
+    }
+}
+
+// Packs the data from the primitive array of type <V> to a
+// DictionaryArray with keys of type K and values of value_type V
+fn pack_numeric_to_dictionary<K, V>(
+    array: &ArrayRef,
+    dict_value_type: &DataType,
+) -> Result<ArrayRef>
+where
+    K: ArrowDictionaryKeyType,
+    V: ArrowNumericType,
+{
+    // attempt to cast the source array values to the target value type (the dictionary values type)
+    let cast_values = cast(array, &dict_value_type)?;
+    let values = cast_values
+        .as_any()
+        .downcast_ref::<PrimitiveArray<V>>()
+        .unwrap();
+
+    let keys_builder = PrimitiveBuilder::<K>::new(values.len());
+    let values_builder = PrimitiveBuilder::<V>::new(values.len());
+    let mut b = PrimitiveDictionaryBuilder::new(keys_builder, values_builder);
+
+    // copy each element one at a time
+    for i in 0..values.len() {
+        if values.is_null(i) {
+            b.append_null()?;
         } else {
-            b.append_value(T::default_value())?;
+            b.append(values.value(i))?;
         }
     }
+    Ok(Arc::new(b.finish()))
+}
 
-    Ok(b.finish())
+// Packs the data as a StringDictionaryArray, if possible, with the
+// key types of K
+fn pack_string_to_dictionary<K>(array: &ArrayRef) -> Result<ArrayRef>
+where
+    K: ArrowDictionaryKeyType,
+{
+    let cast_values = cast(array, &DataType::Utf8)?;
+    let values = cast_values.as_any().downcast_ref::<StringArray>().unwrap();
+
+    let keys_builder = PrimitiveBuilder::<K>::new(values.len());
+    let values_builder = StringBuilder::new(values.len());
+    let mut b = StringDictionaryBuilder::new(keys_builder, values_builder);
+
+    // copy each element one at a time
+    for i in 0..values.len() {
+        if values.is_null(i) {
+            b.append_null()?;
+        } else {
+            b.append(values.value(i))?;
+        }
+    }
+    Ok(Arc::new(b.finish()))
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::buffer::Buffer;
+    use crate::{buffer::Buffer, util::display::array_value_to_string};
 
     #[test]
     fn test_cast_i32_to_f64() {
@@ -766,11 +1166,11 @@ mod tests {
         let array = Arc::new(a) as ArrayRef;
         let b = cast(&array, &DataType::Float64).unwrap();
         let c = b.as_any().downcast_ref::<Float64Array>().unwrap();
-        assert_eq!(5.0, c.value(0));
-        assert_eq!(6.0, c.value(1));
-        assert_eq!(7.0, c.value(2));
-        assert_eq!(8.0, c.value(3));
-        assert_eq!(9.0, c.value(4));
+        assert!(5.0 - c.value(0) < f64::EPSILON);
+        assert!(6.0 - c.value(1) < f64::EPSILON);
+        assert!(7.0 - c.value(2) < f64::EPSILON);
+        assert!(8.0 - c.value(3) < f64::EPSILON);
+        assert!(9.0 - c.value(4) < f64::EPSILON);
     }
 
     #[test]
@@ -821,7 +1221,11 @@ mod tests {
     fn test_cast_i32_to_list_i32() {
         let a = Int32Array::from(vec![5, 6, 7, 8, 9]);
         let array = Arc::new(a) as ArrayRef;
-        let b = cast(&array, &DataType::List(Box::new(DataType::Int32))).unwrap();
+        let b = cast(
+            &array,
+            &DataType::List(Box::new(Field::new("item", DataType::Int32, true))),
+        )
+        .unwrap();
         assert_eq!(5, b.len());
         let arr = b.as_any().downcast_ref::<ListArray>().unwrap();
         assert_eq!(0, arr.value_offset(0));
@@ -847,7 +1251,11 @@ mod tests {
     fn test_cast_i32_to_list_i32_nullable() {
         let a = Int32Array::from(vec![Some(5), None, Some(7), Some(8), Some(9)]);
         let array = Arc::new(a) as ArrayRef;
-        let b = cast(&array, &DataType::List(Box::new(DataType::Int32))).unwrap();
+        let b = cast(
+            &array,
+            &DataType::List(Box::new(Field::new("item", DataType::Int32, true))),
+        )
+        .unwrap();
         assert_eq!(5, b.len());
         assert_eq!(1, b.null_count());
         let arr = b.as_any().downcast_ref::<ListArray>().unwrap();
@@ -876,7 +1284,11 @@ mod tests {
         let a = Int32Array::from(vec![Some(5), None, Some(7), Some(8), None, Some(10)]);
         let array = Arc::new(a) as ArrayRef;
         let array = array.slice(2, 4);
-        let b = cast(&array, &DataType::List(Box::new(DataType::Float64))).unwrap();
+        let b = cast(
+            &array,
+            &DataType::List(Box::new(Field::new("item", DataType::Float64, true))),
+        )
+        .unwrap();
         assert_eq!(4, b.len());
         assert_eq!(1, b.null_count());
         let arr = b.as_any().downcast_ref::<ListArray>().unwrap();
@@ -891,10 +1303,10 @@ mod tests {
         let values = arr.values();
         let c = values.as_any().downcast_ref::<Float64Array>().unwrap();
         assert_eq!(1, c.null_count());
-        assert_eq!(7.0, c.value(0));
-        assert_eq!(8.0, c.value(1));
+        assert!(7.0 - c.value(0) < f64::EPSILON);
+        assert!(8.0 - c.value(1) < f64::EPSILON);
         assert_eq!(false, c.is_valid(2));
-        assert_eq!(10.0, c.value(3));
+        assert!(10.0 - c.value(3) < f64::EPSILON);
     }
 
     #[test]
@@ -927,8 +1339,8 @@ mod tests {
         let array = Arc::new(a) as ArrayRef;
         let b = cast(&array, &DataType::Float64).unwrap();
         let c = b.as_any().downcast_ref::<Float64Array>().unwrap();
-        assert_eq!(1.0, c.value(0));
-        assert_eq!(0.0, c.value(1));
+        assert!(1.0 - c.value(0) < f64::EPSILON);
+        assert!(0.0 - c.value(1) < f64::EPSILON);
         assert_eq!(false, c.is_valid(2));
     }
 
@@ -950,16 +1362,20 @@ mod tests {
         let value_offsets = Buffer::from(&[0, 3, 6, 8].to_byte_slice());
 
         // Construct a list array from the above two
-        let list_data_type = DataType::List(Box::new(DataType::Int32));
-        let list_data = ArrayData::builder(list_data_type.clone())
+        let list_data_type =
+            DataType::List(Box::new(Field::new("item", DataType::Int32, true)));
+        let list_data = ArrayData::builder(list_data_type)
             .len(3)
-            .add_buffer(value_offsets.clone())
-            .add_child_data(value_data.clone())
+            .add_buffer(value_offsets)
+            .add_child_data(value_data)
             .build();
         let list_array = Arc::new(ListArray::from(list_data)) as ArrayRef;
 
-        let cast_array =
-            cast(&list_array, &DataType::List(Box::new(DataType::UInt16))).unwrap();
+        let cast_array = cast(
+            &list_array,
+            &DataType::List(Box::new(Field::new("item", DataType::UInt16, true))),
+        )
+        .unwrap();
         // 3 negative values should get lost when casting to unsigned,
         // 1 value should overflow
         assert_eq!(4, cast_array.null_count());
@@ -1005,17 +1421,22 @@ mod tests {
         let value_offsets = Buffer::from(&[0, 3, 6, 9].to_byte_slice());
 
         // Construct a list array from the above two
-        let list_data_type = DataType::List(Box::new(DataType::Int32));
-        let list_data = ArrayData::builder(list_data_type.clone())
+        let list_data_type =
+            DataType::List(Box::new(Field::new("item", DataType::Int32, true)));
+        let list_data = ArrayData::builder(list_data_type)
             .len(3)
-            .add_buffer(value_offsets.clone())
-            .add_child_data(value_data.clone())
+            .add_buffer(value_offsets)
+            .add_child_data(value_data)
             .build();
         let list_array = Arc::new(ListArray::from(list_data)) as ArrayRef;
 
         cast(
             &list_array,
-            &DataType::List(Box::new(DataType::Timestamp(TimeUnit::Microsecond, None))),
+            &DataType::List(Box::new(Field::new(
+                "item",
+                DataType::Timestamp(TimeUnit::Microsecond, None),
+                true,
+            ))),
         )
         .unwrap();
     }
@@ -1131,7 +1552,7 @@ mod tests {
             std::u32::MAX as f64,
             std::u64::MAX as f64,
         ];
-        let f64_array: ArrayRef = Arc::new(Float64Array::from(f64_values.clone()));
+        let f64_array: ArrayRef = Arc::new(Float64Array::from(f64_values));
 
         let f64_expected = vec![
             "-9223372036854776000.0",
@@ -1275,7 +1696,7 @@ mod tests {
             std::u32::MAX as f32,
             std::u32::MAX as f32,
         ];
-        let f32_array: ArrayRef = Arc::new(Float32Array::from(f32_values.clone()));
+        let f32_array: ArrayRef = Arc::new(Float32Array::from(f32_values));
 
         let f64_expected = vec![
             "-2147483648.0",
@@ -1407,7 +1828,7 @@ mod tests {
             std::u32::MAX as u64,
             std::u64::MAX,
         ];
-        let u64_array: ArrayRef = Arc::new(UInt64Array::from(u64_values.clone()));
+        let u64_array: ArrayRef = Arc::new(UInt64Array::from(u64_values));
 
         let f64_expected = vec![
             "0.0",
@@ -1491,7 +1912,7 @@ mod tests {
             std::u16::MAX as u32,
             std::u32::MAX as u32,
         ];
-        let u32_array: ArrayRef = Arc::new(UInt32Array::from(u32_values.clone()));
+        let u32_array: ArrayRef = Arc::new(UInt32Array::from(u32_values));
 
         let f64_expected = vec!["0.0", "255.0", "65535.0", "4294967295.0"];
         assert_eq!(
@@ -1557,7 +1978,7 @@ mod tests {
     #[test]
     fn test_cast_from_uint16() {
         let u16_values: Vec<u16> = vec![0, std::u8::MAX as u16, std::u16::MAX as u16];
-        let u16_array: ArrayRef = Arc::new(UInt16Array::from(u16_values.clone()));
+        let u16_array: ArrayRef = Arc::new(UInt16Array::from(u16_values));
 
         let f64_expected = vec!["0.0", "255.0", "65535.0"];
         assert_eq!(
@@ -1623,7 +2044,7 @@ mod tests {
     #[test]
     fn test_cast_from_uint8() {
         let u8_values: Vec<u8> = vec![0, std::u8::MAX];
-        let u8_array: ArrayRef = Arc::new(UInt8Array::from(u8_values.clone()));
+        let u8_array: ArrayRef = Arc::new(UInt8Array::from(u8_values));
 
         let f64_expected = vec!["0.0", "255.0"];
         assert_eq!(
@@ -1699,7 +2120,7 @@ mod tests {
             std::i32::MAX as i64,
             std::i64::MAX,
         ];
-        let i64_array: ArrayRef = Arc::new(Int64Array::from(i64_values.clone()));
+        let i64_array: ArrayRef = Arc::new(Int64Array::from(i64_values));
 
         let f64_expected = vec![
             "-9223372036854776000.0",
@@ -1841,7 +2262,7 @@ mod tests {
             std::i16::MAX as i32,
             std::i32::MAX as i32,
         ];
-        let i32_array: ArrayRef = Arc::new(Int32Array::from(i32_values.clone()));
+        let i32_array: ArrayRef = Arc::new(Int32Array::from(i32_values));
 
         let f64_expected = vec![
             "-2147483648.0",
@@ -1919,7 +2340,7 @@ mod tests {
             std::i8::MAX as i16,
             std::i16::MAX,
         ];
-        let i16_array: ArrayRef = Arc::new(Int16Array::from(i16_values.clone()));
+        let i16_array: ArrayRef = Arc::new(Int16Array::from(i16_values));
 
         let f64_expected = vec!["-32768.0", "-128.0", "0.0", "127.0", "32767.0"];
         assert_eq!(
@@ -1985,7 +2406,7 @@ mod tests {
     #[test]
     fn test_cast_from_int8() {
         let i8_values: Vec<i8> = vec![std::i8::MIN, 0, std::i8::MAX];
-        let i8_array: ArrayRef = Arc::new(Int8Array::from(i8_values.clone()));
+        let i8_array: ArrayRef = Arc::new(Int8Array::from(i8_values));
 
         let f64_expected = vec!["-128.0", "0.0", "127.0"];
         assert_eq!(
@@ -2048,6 +2469,7 @@ mod tests {
         );
     }
 
+    /// Convert `array` into a vector of strings by casting to data type dt
     fn get_cast_values<T>(array: &ArrayRef, dt: &DataType) -> Vec<String>
     where
         T: ArrowNumericType,
@@ -2064,4 +2486,514 @@ mod tests {
         }
         v
     }
+
+    #[test]
+    fn test_cast_utf8_dict() {
+        // FROM a dictionary with of Utf8 values
+        use DataType::*;
+
+        let keys_builder = PrimitiveBuilder::<Int8Type>::new(10);
+        let values_builder = StringBuilder::new(10);
+        let mut builder = StringDictionaryBuilder::new(keys_builder, values_builder);
+        builder.append("one").unwrap();
+        builder.append_null().unwrap();
+        builder.append("three").unwrap();
+        let array: ArrayRef = Arc::new(builder.finish());
+
+        let expected = vec!["one", "null", "three"];
+
+        // Test casting TO StringArray
+        let cast_type = Utf8;
+        let cast_array = cast(&array, &cast_type).expect("cast to UTF-8 failed");
+        assert_eq!(cast_array.data_type(), &cast_type);
+        assert_eq!(array_to_strings(&cast_array), expected);
+
+        // Test casting TO Dictionary (with different index sizes)
+
+        let cast_type = Dictionary(Box::new(Int16), Box::new(Utf8));
+        let cast_array = cast(&array, &cast_type).expect("cast failed");
+        assert_eq!(cast_array.data_type(), &cast_type);
+        assert_eq!(array_to_strings(&cast_array), expected);
+
+        let cast_type = Dictionary(Box::new(Int32), Box::new(Utf8));
+        let cast_array = cast(&array, &cast_type).expect("cast failed");
+        assert_eq!(cast_array.data_type(), &cast_type);
+        assert_eq!(array_to_strings(&cast_array), expected);
+
+        let cast_type = Dictionary(Box::new(Int64), Box::new(Utf8));
+        let cast_array = cast(&array, &cast_type).expect("cast failed");
+        assert_eq!(cast_array.data_type(), &cast_type);
+        assert_eq!(array_to_strings(&cast_array), expected);
+
+        let cast_type = Dictionary(Box::new(UInt8), Box::new(Utf8));
+        let cast_array = cast(&array, &cast_type).expect("cast failed");
+        assert_eq!(cast_array.data_type(), &cast_type);
+        assert_eq!(array_to_strings(&cast_array), expected);
+
+        let cast_type = Dictionary(Box::new(UInt16), Box::new(Utf8));
+        let cast_array = cast(&array, &cast_type).expect("cast failed");
+        assert_eq!(cast_array.data_type(), &cast_type);
+        assert_eq!(array_to_strings(&cast_array), expected);
+
+        let cast_type = Dictionary(Box::new(UInt32), Box::new(Utf8));
+        let cast_array = cast(&array, &cast_type).expect("cast failed");
+        assert_eq!(cast_array.data_type(), &cast_type);
+        assert_eq!(array_to_strings(&cast_array), expected);
+
+        let cast_type = Dictionary(Box::new(UInt64), Box::new(Utf8));
+        let cast_array = cast(&array, &cast_type).expect("cast failed");
+        assert_eq!(cast_array.data_type(), &cast_type);
+        assert_eq!(array_to_strings(&cast_array), expected);
+    }
+
+    #[test]
+    fn test_cast_dict_to_dict_bad_index_value_primitive() {
+        use DataType::*;
+        // test converting from an array that has indexes of a type
+        // that are out of bounds for a particular other kind of
+        // index.
+
+        let keys_builder = PrimitiveBuilder::<Int32Type>::new(10);
+        let values_builder = PrimitiveBuilder::<Int64Type>::new(10);
+        let mut builder = PrimitiveDictionaryBuilder::new(keys_builder, values_builder);
+
+        // add 200 distinct values (which can be stored by a
+        // dictionary indexed by int32, but not a dictionary indexed
+        // with int8)
+        for i in 0..200 {
+            builder.append(i).unwrap();
+        }
+        let array: ArrayRef = Arc::new(builder.finish());
+
+        let cast_type = Dictionary(Box::new(Int8), Box::new(Utf8));
+        let res = cast(&array, &cast_type);
+        assert!(res.is_err());
+        let actual_error = format!("{:?}", res);
+        let expected_error = "Could not convert 72 dictionary indexes from Int32 to Int8";
+        assert!(
+            actual_error.contains(expected_error),
+            "did not find expected error '{}' in actual error '{}'",
+            actual_error,
+            expected_error
+        );
+    }
+
+    #[test]
+    fn test_cast_dict_to_dict_bad_index_value_utf8() {
+        use DataType::*;
+        // Same test as test_cast_dict_to_dict_bad_index_value but use
+        // string values (and encode the expected behavior here);
+
+        let keys_builder = PrimitiveBuilder::<Int32Type>::new(10);
+        let values_builder = StringBuilder::new(10);
+        let mut builder = StringDictionaryBuilder::new(keys_builder, values_builder);
+
+        // add 200 distinct values (which can be stored by a
+        // dictionary indexed by int32, but not a dictionary indexed
+        // with int8)
+        for i in 0..200 {
+            let val = format!("val{}", i);
+            builder.append(&val).unwrap();
+        }
+        let array: ArrayRef = Arc::new(builder.finish());
+
+        let cast_type = Dictionary(Box::new(Int8), Box::new(Utf8));
+        let res = cast(&array, &cast_type);
+        assert!(res.is_err());
+        let actual_error = format!("{:?}", res);
+        let expected_error = "Could not convert 72 dictionary indexes from Int32 to Int8";
+        assert!(
+            actual_error.contains(expected_error),
+            "did not find expected error '{}' in actual error '{}'",
+            actual_error,
+            expected_error
+        );
+    }
+
+    #[test]
+    fn test_cast_primitive_dict() {
+        // FROM a dictionary with of INT32 values
+        use DataType::*;
+
+        let keys_builder = PrimitiveBuilder::<Int8Type>::new(10);
+        let values_builder = PrimitiveBuilder::<Int32Type>::new(10);
+        let mut builder = PrimitiveDictionaryBuilder::new(keys_builder, values_builder);
+        builder.append(1).unwrap();
+        builder.append_null().unwrap();
+        builder.append(3).unwrap();
+        let array: ArrayRef = Arc::new(builder.finish());
+
+        let expected = vec!["1", "null", "3"];
+
+        // Test casting TO PrimitiveArray, different dictionary type
+        let cast_array = cast(&array, &Utf8).expect("cast to UTF-8 failed");
+        assert_eq!(array_to_strings(&cast_array), expected);
+        assert_eq!(cast_array.data_type(), &Utf8);
+
+        let cast_array = cast(&array, &Int64).expect("cast to int64 failed");
+        assert_eq!(array_to_strings(&cast_array), expected);
+        assert_eq!(cast_array.data_type(), &Int64);
+    }
+
+    #[test]
+    fn test_cast_primitive_array_to_dict() {
+        use DataType::*;
+
+        let mut builder = PrimitiveBuilder::<Int32Type>::new(10);
+        builder.append_value(1).unwrap();
+        builder.append_null().unwrap();
+        builder.append_value(3).unwrap();
+        let array: ArrayRef = Arc::new(builder.finish());
+
+        let expected = vec!["1", "null", "3"];
+
+        // Cast to a dictionary (same value type, Int32)
+        let cast_type = Dictionary(Box::new(UInt8), Box::new(Int32));
+        let cast_array = cast(&array, &cast_type).expect("cast failed");
+        assert_eq!(cast_array.data_type(), &cast_type);
+        assert_eq!(array_to_strings(&cast_array), expected);
+
+        // Cast to a dictionary (different value type, Int8)
+        let cast_type = Dictionary(Box::new(UInt8), Box::new(Int8));
+        let cast_array = cast(&array, &cast_type).expect("cast failed");
+        assert_eq!(cast_array.data_type(), &cast_type);
+        assert_eq!(array_to_strings(&cast_array), expected);
+    }
+
+    #[test]
+    fn test_cast_string_array_to_dict() {
+        use DataType::*;
+
+        let mut builder = StringBuilder::new(10);
+        builder.append_value("one").unwrap();
+        builder.append_null().unwrap();
+        builder.append_value("three").unwrap();
+        let array: ArrayRef = Arc::new(builder.finish());
+
+        let expected = vec!["one", "null", "three"];
+
+        // Cast to a dictionary (same value type, Utf8)
+        let cast_type = Dictionary(Box::new(UInt8), Box::new(Utf8));
+        let cast_array = cast(&array, &cast_type).expect("cast failed");
+        assert_eq!(cast_array.data_type(), &cast_type);
+        assert_eq!(array_to_strings(&cast_array), expected);
+    }
+
+    #[test]
+    fn test_cast_null_array_to_int32() {
+        let array = Arc::new(NullArray::new(6)) as ArrayRef;
+
+        let expected = Int32Array::from(vec![None; 6]);
+
+        // Cast to a dictionary (same value type, Utf8)
+        let cast_type = DataType::Int32;
+        let cast_array = cast(&array, &cast_type).expect("cast failed");
+        let cast_array = as_primitive_array::<Int32Type>(&cast_array);
+        assert_eq!(cast_array.data_type(), &cast_type);
+        assert_eq!(cast_array, &expected);
+    }
+
+    /// Print the `DictionaryArray` `array` as a vector of strings
+    fn array_to_strings(array: &ArrayRef) -> Vec<String> {
+        (0..array.len())
+            .map(|i| {
+                if array.is_null(i) {
+                    "null".to_string()
+                } else {
+                    array_value_to_string(array, i).expect("Convert array to String")
+                }
+            })
+            .collect()
+    }
+
+    #[test]
+    fn test_can_cast_types() {
+        // this function attempts to ensure that can_cast_types stays
+        // in sync with cast.  It simply tries all combinations of
+        // types and makes sure that if `can_cast_types` returns
+        // true, so does `cast`
+
+        let all_types = get_all_types();
+
+        for array in get_arrays_of_all_types() {
+            for to_type in &all_types {
+                println!("Test casting {:?} --> {:?}", array.data_type(), to_type);
+                let cast_result = cast(&array, &to_type);
+                let reported_cast_ability = can_cast_types(array.data_type(), to_type);
+
+                // check for mismatch
+                match (cast_result, reported_cast_ability) {
+                    (Ok(_), false) => {
+                        panic!("Was able to cast array from {:?} to {:?} but can_cast_types reported false",
+                               array.data_type(), to_type)
+                    },
+                    (Err(e), true) => {
+                        panic!("Was not able to cast array from {:?} to {:?} but can_cast_types reported true. \
+                                Error was {:?}",
+                               array.data_type(), to_type, e)
+                    },
+                    // otherwise it was a match
+                    _=> {},
+                };
+            }
+        }
+    }
+
+    /// Create instances of arrays with varying types for cast tests
+    fn get_arrays_of_all_types() -> Vec<ArrayRef> {
+        let tz_name = Arc::new(String::from("America/New_York"));
+        let binary_data: Vec<&[u8]> = vec![b"foo", b"bar"];
+        vec![
+            Arc::new(BinaryArray::from(binary_data.clone())),
+            Arc::new(LargeBinaryArray::from(binary_data.clone())),
+            make_dictionary_primitive::<Int8Type>(),
+            make_dictionary_primitive::<Int16Type>(),
+            make_dictionary_primitive::<Int32Type>(),
+            make_dictionary_primitive::<Int64Type>(),
+            make_dictionary_primitive::<UInt8Type>(),
+            make_dictionary_primitive::<UInt16Type>(),
+            make_dictionary_primitive::<UInt32Type>(),
+            make_dictionary_primitive::<UInt64Type>(),
+            make_dictionary_utf8::<Int8Type>(),
+            make_dictionary_utf8::<Int16Type>(),
+            make_dictionary_utf8::<Int32Type>(),
+            make_dictionary_utf8::<Int64Type>(),
+            make_dictionary_utf8::<UInt8Type>(),
+            make_dictionary_utf8::<UInt16Type>(),
+            make_dictionary_utf8::<UInt32Type>(),
+            make_dictionary_utf8::<UInt64Type>(),
+            Arc::new(make_list_array()),
+            Arc::new(make_large_list_array()),
+            Arc::new(make_fixed_size_list_array()),
+            Arc::new(make_fixed_size_binary_array()),
+            Arc::new(StructArray::from(vec![
+                (
+                    Field::new("a", DataType::Boolean, false),
+                    Arc::new(BooleanArray::from(vec![false, false, true, true]))
+                        as Arc<Array>,
+                ),
+                (
+                    Field::new("b", DataType::Int32, false),
+                    Arc::new(Int32Array::from(vec![42, 28, 19, 31])),
+                ),
+            ])),
+            //Arc::new(make_union_array()),
+            Arc::new(NullArray::new(10)),
+            Arc::new(StringArray::from(vec!["foo", "bar"])),
+            Arc::new(LargeStringArray::from(vec!["foo", "bar"])),
+            Arc::new(BooleanArray::from(vec![true, false])),
+            Arc::new(Int8Array::from(vec![1, 2])),
+            Arc::new(Int16Array::from(vec![1, 2])),
+            Arc::new(Int32Array::from(vec![1, 2])),
+            Arc::new(Int64Array::from(vec![1, 2])),
+            Arc::new(UInt8Array::from(vec![1, 2])),
+            Arc::new(UInt16Array::from(vec![1, 2])),
+            Arc::new(UInt32Array::from(vec![1, 2])),
+            Arc::new(UInt64Array::from(vec![1, 2])),
+            Arc::new(Float32Array::from(vec![1.0, 2.0])),
+            Arc::new(Float64Array::from(vec![1.0, 2.0])),
+            Arc::new(TimestampSecondArray::from_vec(vec![1000, 2000], None)),
+            Arc::new(TimestampMillisecondArray::from_vec(vec![1000, 2000], None)),
+            Arc::new(TimestampMicrosecondArray::from_vec(vec![1000, 2000], None)),
+            Arc::new(TimestampNanosecondArray::from_vec(vec![1000, 2000], None)),
+            Arc::new(TimestampSecondArray::from_vec(
+                vec![1000, 2000],
+                Some(tz_name.clone()),
+            )),
+            Arc::new(TimestampMillisecondArray::from_vec(
+                vec![1000, 2000],
+                Some(tz_name.clone()),
+            )),
+            Arc::new(TimestampMicrosecondArray::from_vec(
+                vec![1000, 2000],
+                Some(tz_name.clone()),
+            )),
+            Arc::new(TimestampNanosecondArray::from_vec(
+                vec![1000, 2000],
+                Some(tz_name),
+            )),
+            Arc::new(Date32Array::from(vec![1000, 2000])),
+            Arc::new(Date64Array::from(vec![1000, 2000])),
+            Arc::new(Time32SecondArray::from(vec![1000, 2000])),
+            Arc::new(Time32MillisecondArray::from(vec![1000, 2000])),
+            Arc::new(Time64MicrosecondArray::from(vec![1000, 2000])),
+            Arc::new(Time64NanosecondArray::from(vec![1000, 2000])),
+            Arc::new(IntervalYearMonthArray::from(vec![1000, 2000])),
+            Arc::new(IntervalDayTimeArray::from(vec![1000, 2000])),
+            Arc::new(DurationSecondArray::from(vec![1000, 2000])),
+            Arc::new(DurationMillisecondArray::from(vec![1000, 2000])),
+            Arc::new(DurationMicrosecondArray::from(vec![1000, 2000])),
+            Arc::new(DurationNanosecondArray::from(vec![1000, 2000])),
+        ]
+    }
+
+    fn make_list_array() -> ListArray {
+        // Construct a value array
+        let value_data = ArrayData::builder(DataType::Int32)
+            .len(8)
+            .add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
+            .build();
+
+        // Construct a buffer for value offsets, for the nested array:
+        //  [[0, 1, 2], [3, 4, 5], [6, 7]]
+        let value_offsets = Buffer::from(&[0, 3, 6, 8].to_byte_slice());
+
+        // Construct a list array from the above two
+        let list_data_type =
+            DataType::List(Box::new(Field::new("item", DataType::Int32, true)));
+        let list_data = ArrayData::builder(list_data_type.clone())
+            .len(3)
+            .add_buffer(value_offsets.clone())
+            .add_child_data(value_data.clone())
+            .build();
+        ListArray::from(list_data)
+    }
+
+    fn make_large_list_array() -> LargeListArray {
+        // Construct a value array
+        let value_data = ArrayData::builder(DataType::Int32)
+            .len(8)
+            .add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
+            .build();
+
+        // Construct a buffer for value offsets, for the nested array:
+        //  [[0, 1, 2], [3, 4, 5], [6, 7]]
+        let value_offsets = Buffer::from(&[0i64, 3, 6, 8].to_byte_slice());
+
+        // Construct a list array from the above two
+        let list_data_type =
+            DataType::LargeList(Box::new(Field::new("item", DataType::Int32, true)));
+        let list_data = ArrayData::builder(list_data_type.clone())
+            .len(3)
+            .add_buffer(value_offsets.clone())
+            .add_child_data(value_data.clone())
+            .build();
+        LargeListArray::from(list_data)
+    }
+
+    fn make_fixed_size_list_array() -> FixedSizeListArray {
+        // Construct a value array
+        let value_data = ArrayData::builder(DataType::Int32)
+            .len(10)
+            .add_buffer(Buffer::from(
+                &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9].to_byte_slice(),
+            ))
+            .build();
+
+        // Construct a fixed size list array from the above two
+        let list_data_type = DataType::FixedSizeList(
+            Box::new(Field::new("item", DataType::Int32, true)),
+            2,
+        );
+        let list_data = ArrayData::builder(list_data_type)
+            .len(5)
+            .add_child_data(value_data.clone())
+            .build();
+        FixedSizeListArray::from(list_data)
+    }
+
+    fn make_fixed_size_binary_array() -> FixedSizeBinaryArray {
+        let values: [u8; 15] = *b"hellotherearrow";
+
+        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
+            .len(3)
+            .add_buffer(Buffer::from(&values[..]))
+            .build();
+        FixedSizeBinaryArray::from(array_data)
+    }
+
+    fn make_union_array() -> UnionArray {
+        let mut builder = UnionBuilder::new_dense(7);
+        builder.append::<Int32Type>("a", 1).unwrap();
+        builder.append::<BooleanType>("b", false).unwrap();
+        builder.build().unwrap()
+    }
+
+    /// Creates a dictionary with primitive dictionary values, and keys of type K
+    fn make_dictionary_primitive<K: ArrowDictionaryKeyType>() -> ArrayRef {
+        let keys_builder = PrimitiveBuilder::<K>::new(2);
+        // Pick Int32 arbitrarily for dictionary values
+        let values_builder = PrimitiveBuilder::<Int32Type>::new(2);
+        let mut b = PrimitiveDictionaryBuilder::new(keys_builder, values_builder);
+        b.append(1).unwrap();
+        b.append(2).unwrap();
+        Arc::new(b.finish())
+    }
+
+    /// Creates a dictionary with utf8 values, and keys of type K
+    fn make_dictionary_utf8<K: ArrowDictionaryKeyType>() -> ArrayRef {
+        let keys_builder = PrimitiveBuilder::<K>::new(2);
+        // Pick Int32 arbitrarily for dictionary values
+        let values_builder = StringBuilder::new(2);
+        let mut b = StringDictionaryBuilder::new(keys_builder, values_builder);
+        b.append("foo").unwrap();
+        b.append("bar").unwrap();
+        Arc::new(b.finish())
+    }
+
+    // Get a selection of datatypes to try and cast to
+    fn get_all_types() -> Vec<DataType> {
+        use DataType::*;
+        let tz_name = Arc::new(String::from("America/New_York"));
+
+        vec![
+            Null,
+            Boolean,
+            Int8,
+            Int16,
+            Int32,
+            UInt64,
+            UInt8,
+            UInt16,
+            UInt32,
+            UInt64,
+            Float16,
+            Float32,
+            Float64,
+            Timestamp(TimeUnit::Second, None),
+            Timestamp(TimeUnit::Millisecond, None),
+            Timestamp(TimeUnit::Microsecond, None),
+            Timestamp(TimeUnit::Nanosecond, None),
+            Timestamp(TimeUnit::Second, Some(tz_name.clone())),
+            Timestamp(TimeUnit::Millisecond, Some(tz_name.clone())),
+            Timestamp(TimeUnit::Microsecond, Some(tz_name.clone())),
+            Timestamp(TimeUnit::Nanosecond, Some(tz_name.clone())),
+            Date32(DateUnit::Day),
+            Date64(DateUnit::Day),
+            Date32(DateUnit::Millisecond),
+            Date64(DateUnit::Millisecond),
+            Time32(TimeUnit::Second),
+            Time32(TimeUnit::Millisecond),
+            Time64(TimeUnit::Microsecond),
+            Time64(TimeUnit::Nanosecond),
+            Duration(TimeUnit::Second),
+            Duration(TimeUnit::Millisecond),
+            Duration(TimeUnit::Microsecond),
+            Duration(TimeUnit::Nanosecond),
+            Interval(IntervalUnit::YearMonth),
+            Interval(IntervalUnit::DayTime),
+            Binary,
+            FixedSizeBinary(10),
+            LargeBinary,
+            Utf8,
+            LargeUtf8,
+            List(Box::new(Field::new("item", DataType::Int8, true))),
+            List(Box::new(Field::new("item", DataType::Utf8, true))),
+            FixedSizeList(Box::new(Field::new("item", DataType::Int8, true)), 10),
+            FixedSizeList(Box::new(Field::new("item", DataType::Utf8, false)), 10),
+            LargeList(Box::new(Field::new("item", DataType::Int8, true))),
+            LargeList(Box::new(Field::new("item", DataType::Utf8, false))),
+            Struct(vec![
+                Field::new("f1", DataType::Int32, false),
+                Field::new("f2", DataType::Utf8, true),
+            ]),
+            Union(vec![
+                Field::new("f1", DataType::Int32, false),
+                Field::new("f2", DataType::Utf8, true),
+            ]),
+            Dictionary(Box::new(DataType::Int8), Box::new(DataType::Int32)),
+            Dictionary(Box::new(DataType::Int16), Box::new(DataType::Utf8)),
+            Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)),
+        ]
+    }
 }
diff --git a/rust/arrow/src/compute/kernels/comparison.rs b/rust/arrow/src/compute/kernels/comparison.rs
index a59e7b26a72..88bb49987af 100644
--- a/rust/arrow/src/compute/kernels/comparison.rs
+++ b/rust/arrow/src/compute/kernels/comparison.rs
@@ -27,9 +27,11 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use crate::array::*;
-use crate::compute::util::apply_bin_op_to_option_bitmap;
+use crate::buffer::{Buffer, MutableBuffer};
+use crate::compute::util::combine_option_bitmap;
 use crate::datatypes::{ArrowNumericType, BooleanType, DataType};
 use crate::error::{ArrowError, Result};
+use crate::util::bit_util;
 
 /// Helper function to perform boolean lambda function on values from two arrays, this
 /// version does not attempt to use SIMD.
@@ -42,11 +44,8 @@ macro_rules! compare_op {
             ));
         }
 
-        let null_bit_buffer = apply_bin_op_to_option_bitmap(
-            $left.data().null_bitmap(),
-            $right.data().null_bitmap(),
-            |a, b| a & b,
-        )?;
+        let null_bit_buffer =
+            combine_option_bitmap($left.data_ref(), $right.data_ref(), $left.len())?;
 
         let mut result = BooleanBufferBuilder::new($left.len());
         for i in 0..$left.len() {
@@ -120,11 +119,8 @@ pub fn like_utf8(left: &StringArray, right: &StringArray) -> Result<BooleanArray
         ));
     }
 
-    let null_bit_buffer = apply_bin_op_to_option_bitmap(
-        left.data().null_bitmap(),
-        right.data().null_bitmap(),
-        |a, b| a & b,
-    )?;
+    let null_bit_buffer =
+        combine_option_bitmap(left.data_ref(), right.data_ref(), left.len())?;
 
     let mut result = BooleanBufferBuilder::new(left.len());
     for i in 0..left.len() {
@@ -168,11 +164,8 @@ pub fn nlike_utf8(left: &StringArray, right: &StringArray) -> Result<BooleanArra
         ));
     }
 
-    let null_bit_buffer = apply_bin_op_to_option_bitmap(
-        left.data().null_bitmap(),
-        right.data().null_bitmap(),
-        |a, b| a & b,
-    )?;
+    let null_bit_buffer =
+        combine_option_bitmap(left.data_ref(), right.data_ref(), left.len())?;
 
     let mut result = BooleanBufferBuilder::new(left.len());
     for i in 0..left.len() {
@@ -267,7 +260,6 @@ where
     T: ArrowNumericType,
     F: Fn(T::Simd, T::Simd) -> T::SimdMask,
 {
-    use crate::buffer::MutableBuffer;
     use std::io::Write;
     use std::mem;
 
@@ -279,11 +271,7 @@ where
         ));
     }
 
-    let null_bit_buffer = apply_bin_op_to_option_bitmap(
-        left.data().null_bitmap(),
-        right.data().null_bitmap(),
-        |a, b| a & b,
-    )?;
+    let null_bit_buffer = combine_option_bitmap(left.data_ref(), right.data_ref(), len)?;
 
     let lanes = T::lanes();
     let mut result = MutableBuffer::new(left.len() * mem::size_of::<bool>());
@@ -333,7 +321,6 @@ where
     T: ArrowNumericType,
     F: Fn(T::Simd, T::Simd) -> T::SimdMask,
 {
-    use crate::buffer::MutableBuffer;
     use std::io::Write;
     use std::mem;
 
@@ -568,11 +555,134 @@ where
     compare_op_scalar!(left, right, |a, b| a >= b)
 }
 
+/// Checks if a `GenericListArray` contains a value in the `PrimitiveArray`
+pub fn contains<T, OffsetSize>(
+    left: &PrimitiveArray<T>,
+    right: &GenericListArray<OffsetSize>,
+) -> Result<BooleanArray>
+where
+    T: ArrowNumericType,
+    OffsetSize: OffsetSizeTrait,
+{
+    let left_len = left.len();
+    if left_len != right.len() {
+        return Err(ArrowError::ComputeError(
+            "Cannot perform comparison operation on arrays of different length"
+                .to_string(),
+        ));
+    }
+
+    let num_bytes = bit_util::ceil(left_len, 8);
+
+    let not_both_null_bit_buffer =
+        match combine_option_bitmap(left.data_ref(), right.data_ref(), left_len)? {
+            Some(buff) => buff,
+            None => new_all_set_buffer(num_bytes),
+        };
+    let not_both_null_bitmap = not_both_null_bit_buffer.data();
+
+    let mut bool_buf = MutableBuffer::new(num_bytes).with_bitset(num_bytes, false);
+    let bool_slice = bool_buf.data_mut();
+
+    // if both array slots are valid, check if list contains primitive
+    for i in 0..left_len {
+        if bit_util::get_bit(not_both_null_bitmap, i) {
+            let list = right.value(i);
+            let list = list.as_any().downcast_ref::<PrimitiveArray<T>>().unwrap();
+
+            for j in 0..list.len() {
+                if list.is_valid(j) && (left.value(i) == list.value(j)) {
+                    bit_util::set_bit(bool_slice, i);
+                    continue;
+                }
+            }
+        }
+    }
+
+    let data = ArrayData::new(
+        DataType::Boolean,
+        left.len(),
+        None,
+        None,
+        0,
+        vec![bool_buf.freeze()],
+        vec![],
+    );
+    Ok(PrimitiveArray::<BooleanType>::from(Arc::new(data)))
+}
+
+/// Checks if a `GenericListArray` contains a value in the `GenericStringArray`
+pub fn contains_utf8<OffsetSize>(
+    left: &GenericStringArray<OffsetSize>,
+    right: &ListArray,
+) -> Result<BooleanArray>
+where
+    OffsetSize: StringOffsetSizeTrait,
+{
+    let left_len = left.len();
+    if left_len != right.len() {
+        return Err(ArrowError::ComputeError(
+            "Cannot perform comparison operation on arrays of different length"
+                .to_string(),
+        ));
+    }
+
+    let num_bytes = bit_util::ceil(left_len, 8);
+
+    let not_both_null_bit_buffer =
+        match combine_option_bitmap(left.data_ref(), right.data_ref(), left_len)? {
+            Some(buff) => buff,
+            None => new_all_set_buffer(num_bytes),
+        };
+    let not_both_null_bitmap = not_both_null_bit_buffer.data();
+
+    let mut bool_buf = MutableBuffer::new(num_bytes).with_bitset(num_bytes, false);
+    let bool_slice = bool_buf.data_mut();
+
+    for i in 0..left_len {
+        // contains(null, null) = false
+        if bit_util::get_bit(not_both_null_bitmap, i) {
+            let list = right.value(i);
+            let list = list
+                .as_any()
+                .downcast_ref::<GenericStringArray<OffsetSize>>()
+                .unwrap();
+
+            for j in 0..list.len() {
+                if list.is_valid(j) && (left.value(i) == list.value(j)) {
+                    bit_util::set_bit(bool_slice, i);
+                    continue;
+                }
+            }
+        }
+    }
+
+    let data = ArrayData::new(
+        DataType::Boolean,
+        left.len(),
+        None,
+        None,
+        0,
+        vec![bool_buf.freeze()],
+        vec![],
+    );
+    Ok(PrimitiveArray::<BooleanType>::from(Arc::new(data)))
+}
+
+// create a buffer and fill it with valid bits
+#[inline]
+fn new_all_set_buffer(len: usize) -> Buffer {
+    let buffer = MutableBuffer::new(len);
+    let buffer = buffer.with_bitset(len, true);
+
+    buffer.freeze()
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::array::Int32Array;
-    use crate::datatypes::Int8Type;
+    use crate::datatypes::{Int8Type, ToByteSlice};
+    use crate::{array::Int32Array, datatypes::Field};
 
     #[test]
     fn test_primitive_array_eq() {
@@ -820,6 +930,112 @@ mod tests {
         );
     }
 
+    // Expected behaviour:
+    // contains(1, [1, 2, null]) = true
+    // contains(3, [1, 2, null]) = false
+    // contains(null, [1, 2, null]) = false
+    // contains(null, null) = false
+    #[test]
+    fn test_contains() {
+        let value_data = Int32Array::from(vec![
+            Some(0),
+            Some(1),
+            Some(2),
+            Some(3),
+            Some(4),
+            Some(5),
+            Some(6),
+            None,
+            Some(7),
+        ])
+        .data();
+        let value_offsets = Buffer::from(&[0i64, 3, 6, 6, 9].to_byte_slice());
+        let list_data_type =
+            DataType::LargeList(Box::new(Field::new("item", DataType::Int32, true)));
+        let list_data = ArrayData::builder(list_data_type)
+            .len(4)
+            .add_buffer(value_offsets)
+            .null_count(1)
+            .add_child_data(value_data)
+            .null_bit_buffer(Buffer::from([0b00001011]))
+            .build();
+
+        //  [[0, 1, 2], [3, 4, 5], null, [6, null, 7]]
+        let list_array = LargeListArray::from(list_data);
+
+        let nulls = Int32Array::from(vec![None, None, None, None]);
+        let nulls_result = contains(&nulls, &list_array).unwrap();
+        assert_eq!(
+            nulls_result
+                .as_any()
+                .downcast_ref::<BooleanArray>()
+                .unwrap(),
+            &BooleanArray::from(vec![false, false, false, false]),
+        );
+
+        let values = Int32Array::from(vec![Some(0), Some(0), Some(0), Some(0)]);
+        let values_result = contains(&values, &list_array).unwrap();
+        assert_eq!(
+            values_result
+                .as_any()
+                .downcast_ref::<BooleanArray>()
+                .unwrap(),
+            &BooleanArray::from(vec![true, false, false, false]),
+        );
+    }
+
+    // Expected behaviour:
+    // contains("ab", ["ab", "cd", null]) = true
+    // contains("ef", ["ab", "cd", null]) = false
+    // contains(null, ["ab", "cd", null]) = false
+    // contains(null, null) = false
+    #[test]
+    fn test_contains_utf8() {
+        let values_builder = StringBuilder::new(10);
+        let mut builder = ListBuilder::new(values_builder);
+
+        builder.values().append_value("Lorem").unwrap();
+        builder.values().append_value("ipsum").unwrap();
+        builder.values().append_null().unwrap();
+        builder.append(true).unwrap();
+        builder.values().append_value("sit").unwrap();
+        builder.values().append_value("amet").unwrap();
+        builder.values().append_value("Lorem").unwrap();
+        builder.append(true).unwrap();
+        builder.append(false).unwrap();
+        builder.values().append_value("ipsum").unwrap();
+        builder.append(true).unwrap();
+
+        //  [["Lorem", "ipsum", null], ["sit", "amet", "Lorem"], null, ["ipsum"]]
+        // value_offsets = [0, 3, 6, 6]
+        let list_array = builder.finish();
+
+        let nulls = StringArray::from(vec![None, None, None, None]);
+        let nulls_result = contains_utf8(&nulls, &list_array).unwrap();
+        assert_eq!(
+            nulls_result
+                .as_any()
+                .downcast_ref::<BooleanArray>()
+                .unwrap(),
+            &BooleanArray::from(vec![false, false, false, false]),
+        );
+
+        let values = StringArray::from(vec![
+            Some("Lorem"),
+            Some("Lorem"),
+            Some("Lorem"),
+            Some("Lorem"),
+        ]);
+        let values_result = contains_utf8(&values, &list_array).unwrap();
+        assert_eq!(
+            values_result
+                .as_any()
+                .downcast_ref::<BooleanArray>()
+                .unwrap(),
+            &BooleanArray::from(vec![true, true, false, false]),
+        );
+    }
+
     macro_rules! test_utf8 {
         ($test_name:ident, $left:expr, $right:expr, $op:expr, $expected:expr) => {
             #[test]
diff --git a/rust/arrow/src/compute/kernels/concat.rs b/rust/arrow/src/compute/kernels/concat.rs
index 48604dac81f..8c0965adbf8 100644
--- a/rust/arrow/src/compute/kernels/concat.rs
+++ b/rust/arrow/src/compute/kernels/concat.rs
@@ -53,7 +53,7 @@ pub fn concat(array_list: &[ArrayRef]) -> Result<ArrayRef> {
 
     match array_data_list[0].data_type() {
         DataType::Utf8 => {
-            let mut builder = StringArray::builder(0);
+            let mut builder = StringBuilder::new(0);
             builder.append_data(array_data_list)?;
             Ok(ArrayBuilder::finish(&mut builder))
         }
@@ -114,6 +114,9 @@ pub fn concat(array_list: &[ArrayRef]) -> Result<ArrayRef> {
         DataType::Duration(TimeUnit::Nanosecond) => {
             concat_primitive::<DurationNanosecondType>(array_data_list)
         }
+        DataType::List(nested_field) => {
+            concat_list(array_data_list, nested_field.data_type())
+        }
         t => Err(ArrowError::ComputeError(format!(
             "Concat not supported for data type {:?}",
             t
@@ -131,31 +134,62 @@ where
     Ok(ArrayBuilder::finish(&mut builder))
 }
 
+#[inline]
+fn concat_primitive_list<T>(array_data_list: &[ArrayDataRef]) -> Result<ArrayRef>
+where
+    T: ArrowNumericType,
+{
+    let mut builder = ListBuilder::new(PrimitiveArray::<T>::builder(0));
+    builder.append_data(array_data_list)?;
+    Ok(ArrayBuilder::finish(&mut builder))
+}
+
+#[inline]
+fn concat_list(
+    array_data_list: &[ArrayDataRef],
+    data_type: &DataType,
+) -> Result<ArrayRef> {
+    match data_type {
+        DataType::Int8 => concat_primitive_list::<Int8Type>(array_data_list),
+        DataType::Int16 => concat_primitive_list::<Int16Type>(array_data_list),
+        DataType::Int32 => concat_primitive_list::<Int32Type>(array_data_list),
+        DataType::Int64 => concat_primitive_list::<Int64Type>(array_data_list),
+        DataType::UInt8 => concat_primitive_list::<UInt8Type>(array_data_list),
+        DataType::UInt16 => concat_primitive_list::<UInt16Type>(array_data_list),
+        DataType::UInt32 => concat_primitive_list::<UInt32Type>(array_data_list),
+        DataType::UInt64 => concat_primitive_list::<UInt64Type>(array_data_list),
+        t => Err(ArrowError::ComputeError(format!(
+            "Concat not supported for list with data type {:?}",
+            t
+        ))),
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
-    use std::convert::TryFrom;
     use std::sync::Arc;
 
     #[test]
     fn test_concat_empty_vec() -> Result<()> {
-        let re = concat(&vec![]);
+        let re = concat(&[]);
         assert!(re.is_err());
         Ok(())
     }
 
     #[test]
     fn test_concat_incompatible_datatypes() -> Result<()> {
-        let re = concat(&vec![
+        let re = concat(&[
             Arc::new(PrimitiveArray::<Int64Type>::from(vec![
                 Some(-1),
                 Some(2),
                 None,
             ])) as ArrayRef,
-            Arc::new(
-                StringArray::try_from(vec![Some("hello"), Some("bar"), Some("world")])
-                    .expect("Unable to create string array"),
-            ) as ArrayRef,
+            Arc::new(StringArray::from(vec![
+                Some("hello"),
+                Some("bar"),
+                Some("world"),
+            ])) as ArrayRef,
         ]);
         assert!(re.is_err());
         Ok(())
@@ -163,46 +197,37 @@ mod tests {
 
     #[test]
     fn test_concat_string_arrays() -> Result<()> {
-        let arr = concat(&vec![
-            Arc::new(
-                StringArray::try_from(vec![Some("hello"), Some("world")])
-                    .expect("Unable to create string array"),
-            ) as ArrayRef,
+        let arr = concat(&[
+            Arc::new(StringArray::from(vec![Some("hello"), Some("world")])) as ArrayRef,
             Arc::new(StringArray::from(vec!["1", "2", "3", "4", "6"])).slice(1, 3),
-            Arc::new(
-                StringArray::try_from(vec![Some("foo"), Some("bar"), None, Some("baz")])
-                    .expect("Unable to create string array"),
-            ) as ArrayRef,
-        ])?;
-
-        let expected_output = Arc::new(
-            StringArray::try_from(vec![
-                Some("hello"),
-                Some("world"),
-                Some("2"),
-                Some("3"),
-                Some("4"),
+            Arc::new(StringArray::from(vec![
                 Some("foo"),
                 Some("bar"),
                 None,
                 Some("baz"),
-            ])
-            .expect("Unable to create string array"),
-        ) as ArrayRef;
+            ])) as ArrayRef,
+        ])?;
+
+        let expected_output = Arc::new(StringArray::from(vec![
+            Some("hello"),
+            Some("world"),
+            Some("2"),
+            Some("3"),
+            Some("4"),
+            Some("foo"),
+            Some("bar"),
+            None,
+            Some("baz"),
+        ])) as ArrayRef;
 
-        assert!(
-            arr.equals(&(*expected_output)),
-            "expect {:#?} to be: {:#?}",
-            arr,
-            &expected_output
-        );
+        assert_eq!(&arr, &expected_output);
 
         Ok(())
     }
 
     #[test]
     fn test_concat_primitive_arrays() -> Result<()> {
-        let arr = concat(&vec![
+        let arr = concat(&[
             Arc::new(PrimitiveArray::<Int64Type>::from(vec![
                 Some(-1),
                 Some(-1),
@@ -238,19 +263,14 @@ mod tests {
             Some(1024),
         ])) as ArrayRef;
 
-        assert!(
-            arr.equals(&(*expected_output)),
-            "expect {:#?} to be: {:#?}",
-            arr,
-            &expected_output
-        );
+        assert_eq!(&arr, &expected_output);
 
         Ok(())
     }
 
     #[test]
     fn test_concat_boolean_primitive_arrays() -> Result<()> {
-        let arr = concat(&vec![
+        let arr = concat(&[
             Arc::new(PrimitiveArray::<BooleanType>::from(vec![
                 Some(true),
                 Some(true),
@@ -280,12 +300,73 @@ mod tests {
             Some(false),
         ])) as ArrayRef;
 
-        assert!(
-            arr.equals(&(*expected_output)),
-            "expect {:#?} to be: {:#?}",
-            arr,
-            &expected_output
-        );
+        assert_eq!(&arr, &expected_output);
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_concat_primitive_list_arrays() -> Result<()> {
+        fn populate_list1(
+            b: &mut ListBuilder<PrimitiveBuilder<Int64Type>>,
+        ) -> Result<()> {
+            b.values().append_value(-1)?;
+            b.values().append_value(-1)?;
+            b.values().append_value(2)?;
+            b.values().append_null()?;
+            b.values().append_null()?;
+            b.append(true)?;
+            b.append(true)?;
+            b.append(false)?;
+            b.values().append_value(10)?;
+            b.append(true)?;
+            Ok(())
+        }
+
+        fn populate_list2(
+            b: &mut ListBuilder<PrimitiveBuilder<Int64Type>>,
+        ) -> Result<()> {
+            b.append(false)?;
+            b.values().append_value(100)?;
+            b.values().append_null()?;
+            b.values().append_value(101)?;
+            b.append(true)?;
+            b.values().append_value(102)?;
+            b.append(true)?;
+            Ok(())
+        }
+
+        fn populate_list3(
+            b: &mut ListBuilder<PrimitiveBuilder<Int64Type>>,
+        ) -> Result<()> {
+            b.values().append_value(1000)?;
+            b.values().append_value(1001)?;
+            b.append(true)?;
+            Ok(())
+        }
+
+        let mut builder_in1 = ListBuilder::new(PrimitiveArray::<Int64Type>::builder(0));
+        let mut builder_in2 = ListBuilder::new(PrimitiveArray::<Int64Type>::builder(0));
+        let mut builder_in3 = ListBuilder::new(PrimitiveArray::<Int64Type>::builder(0));
+        populate_list1(&mut builder_in1)?;
+        populate_list2(&mut builder_in2)?;
+        populate_list3(&mut builder_in3)?;
+
+        let mut builder_expected =
+            ListBuilder::new(PrimitiveArray::<Int64Type>::builder(0));
+        populate_list1(&mut builder_expected)?;
+        populate_list2(&mut builder_expected)?;
+        populate_list3(&mut builder_expected)?;
+
+        let array_result = concat(&[
+            Arc::new(builder_in1.finish()),
+            Arc::new(builder_in2.finish()),
+            Arc::new(builder_in3.finish()),
+        ])?;
+
+        let array_expected = Arc::new(builder_expected.finish()) as ArrayRef;
+
+        assert_eq!(&array_result, &array_expected);
 
         Ok(())
     }
diff --git a/rust/arrow/src/compute/kernels/filter.rs b/rust/arrow/src/compute/kernels/filter.rs
index 98d70f05ced..0d19443f689 100644
--- a/rust/arrow/src/compute/kernels/filter.rs
+++ b/rust/arrow/src/compute/kernels/filter.rs
@@ -18,7 +18,7 @@
 //! Defines miscellaneous array kernels.
 
 use crate::array::*;
-use crate::datatypes::{ArrowNumericType, DataType, TimeUnit};
+use crate::datatypes::*;
 use crate::error::{ArrowError, Result};
 use crate::record_batch::RecordBatch;
 use crate::{
@@ -161,7 +161,7 @@ fn filter_array_impl(
         // foreach u64 batch
         let filter_batch = *filter_batch;
         if filter_batch == 0 {
-            // if batch == 0: skip
+            // if batch == 0, all items are filtered out, so skip entire batch
             continue;
         } else if filter_batch == all_ones_batch {
             // if batch == all 1s: copy all 64 values in one go
@@ -230,6 +230,86 @@ macro_rules! filter_dictionary_array {
     }};
 }
 
+macro_rules! filter_primitive_item_list_array {
+    ($context:expr, $array:expr, $item_type:ident, $list_type:ident, $list_builder_type:ident) => {{
+        let input_array = $array.as_any().downcast_ref::<$list_type>().unwrap();
+        let values_builder = PrimitiveBuilder::<$item_type>::new($context.filtered_count);
+        let mut builder = $list_builder_type::new(values_builder);
+        for i in 0..$context.filter_u64.len() {
+            // foreach u64 batch
+            let filter_batch = $context.filter_u64[i];
+            if filter_batch == 0 {
+                // if batch == 0, all items are filtered out, so skip entire batch
+                continue;
+            }
+            for j in 0..64 {
+                // foreach bit in batch:
+                if (filter_batch & $context.filter_mask[j]) != 0 {
+                    let data_index = (i * 64) + j;
+                    if input_array.is_null(data_index) {
+                        builder.append(false)?;
+                    } else {
+                        let this_inner_list = input_array.value(data_index);
+                        let inner_list = this_inner_list
+                            .as_any()
+                            .downcast_ref::<PrimitiveArray<$item_type>>()
+                            .unwrap();
+                        for k in 0..inner_list.len() {
+                            if inner_list.is_null(k) {
+                                builder.values().append_null()?;
+                            } else {
+                                builder.values().append_value(inner_list.value(k))?;
+                            }
+                        }
+                        builder.append(true)?;
+                    }
+                }
+            }
+        }
+        Ok(Arc::new(builder.finish()))
+    }};
+}
+
+macro_rules! filter_non_primitive_item_list_array {
+    ($context:expr, $array:expr, $item_array_type:ident, $item_builder:ident, $list_type:ident, $list_builder_type:ident) => {{
+        let input_array = $array.as_any().downcast_ref::<$list_type>().unwrap();
+        let values_builder = $item_builder::new($context.filtered_count);
+        let mut builder = $list_builder_type::new(values_builder);
+        for i in 0..$context.filter_u64.len() {
+            // foreach u64 batch
+            let filter_batch = $context.filter_u64[i];
+            if filter_batch == 0 {
+                // if batch == 0, all items are filtered out, so skip entire batch
+                continue;
+            }
+            for j in 0..64 {
+                // foreach bit in batch:
+                if (filter_batch & $context.filter_mask[j]) != 0 {
+                    let data_index = (i * 64) + j;
+                    if input_array.is_null(data_index) {
+                        builder.append(false)?;
+                    } else {
+                        let this_inner_list = input_array.value(data_index);
+                        let inner_list = this_inner_list
+                            .as_any()
+                            .downcast_ref::<$item_array_type>()
+                            .unwrap();
+                        for k in 0..inner_list.len() {
+                            if inner_list.is_null(k) {
+                                builder.values().append_null()?;
+                            } else {
+                                builder.values().append_value(inner_list.value(k))?;
+                            }
+                        }
+                        builder.append(true)?;
+                    }
+                }
+            }
+        }
+        Ok(Arc::new(builder.finish()))
+    }};
+}
+
 impl FilterContext {
     /// Returns a new instance of FilterContext
     pub fn new(filter_array: &BooleanArray) -> Result<Self> {
@@ -240,12 +320,23 @@ impl FilterContext {
         }
         let filter_mask: Vec<u64> = (0..64).map(|x| 1u64 << x).collect();
         let filter_bytes = filter_array.data_ref().buffers()[0].data();
-        let filtered_count = bit_util::count_set_bits(filter_bytes);
+        let filtered_count =
+            bit_util::count_set_bits_offset(filter_bytes, 0, filter_array.len());
+
         // transmute filter_bytes to &[u64]
         let mut u64_buffer = MutableBuffer::new(filter_bytes.len());
-        u64_buffer
-            .write_bytes(filter_bytes, u64_buffer.capacity() - filter_bytes.len())?;
-        let filter_u64 = u64_buffer.typed_data_mut::<u64>().to_owned();
+        // add to the resulting len so is is a multiple of the size of u64
+        let pad_addional_len = (8 - filter_bytes.len() % 8) % 8;
+        u64_buffer.write_bytes(filter_bytes, pad_addional_len)?;
+        let mut filter_u64 = u64_buffer.typed_data_mut::<u64>().to_owned();
+
+        // mask of any bits outside of the given len
+        if filter_array.len() % 64 != 0 {
+            let last_idx = filter_u64.len() - 1;
+            let mask = u64::MAX >> (64 - filter_array.len() % 64);
+            filter_u64[last_idx] &= mask;
+        }
+
         Ok(FilterContext {
             filter_u64,
             filter_len: filter_array.len(),
@@ -274,7 +365,7 @@ impl FilterContext {
                     // foreach u64 batch
                     let filter_batch = self.filter_u64[i];
                     if filter_batch == 0 {
-                        // if batch == 0: skip
+                        // if batch == 0, all items are filtered out, so skip entire batch
                         continue;
                     }
                     for j in 0..64 {
@@ -331,19 +422,23 @@ impl FilterContext {
             }
             DataType::Binary => {
                 let input_array = array.as_any().downcast_ref::<BinaryArray>().unwrap();
-                let mut values: Vec<&[u8]> = Vec::with_capacity(self.filtered_count);
+                let mut values: Vec<Option<&[u8]>> = Vec::with_capacity(self.filtered_count);
                 for i in 0..self.filter_u64.len() {
                     // foreach u64 batch
                     let filter_batch = self.filter_u64[i];
                     if filter_batch == 0 {
-                        // if batch == 0: skip
+                        // if batch == 0, all items are filtered out, so skip entire batch
                         continue;
                     }
                     for j in 0..64 {
                         // foreach bit in batch:
                         if (filter_batch & self.filter_mask[j]) != 0 {
                             let data_index = (i * 64) + j;
-                            values.push(input_array.value(data_index));
+                            if input_array.is_null(data_index) {
+                                values.push(None)
+                            } else {
+                                values.push(Some(input_array.value(data_index)))
+                            }
                         }
                     }
                 }
@@ -351,19 +446,23 @@ impl FilterContext {
             }
             DataType::Utf8 => {
                 let input_array = array.as_any().downcast_ref::<StringArray>().unwrap();
-                let mut values: Vec<&str> = Vec::with_capacity(self.filtered_count);
+                let mut values: Vec<Option<&str>> = Vec::with_capacity(self.filtered_count);
                 for i in 0..self.filter_u64.len() {
                     // foreach u64 batch
                     let filter_batch = self.filter_u64[i];
                     if filter_batch == 0 {
-                        // if batch == 0: skip
+                        // if batch == 0, all items are filtered out, so skip entire batch
                         continue;
                     }
                     for j in 0..64 {
                         // foreach bit in batch:
                         if (filter_batch & self.filter_mask[j]) != 0 {
                             let data_index = (i * 64) + j;
-                            values.push(input_array.value(data_index));
+                            if input_array.is_null(data_index) {
+                                values.push(None)
+                            } else {
+                                values.push(Some(input_array.value(data_index)))
+                            }
                         }
                     }
                 }
@@ -389,6 +488,232 @@ impl FilterContext {
                     key_type, value_type
                 )))
             }
+            DataType::List(dt) => match dt.data_type() {
+                DataType::UInt8 => {
+                    filter_primitive_item_list_array!(self, array, UInt8Type, ListArray, ListBuilder)
+                }
+                DataType::UInt16 => {
+                    filter_primitive_item_list_array!(self, array, UInt16Type, ListArray, ListBuilder)
+                }
+                DataType::UInt32 => {
+                    filter_primitive_item_list_array!(self, array, UInt32Type, ListArray, ListBuilder)
+                }
+                DataType::UInt64 => {
+                    filter_primitive_item_list_array!(self, array, UInt64Type, ListArray, ListBuilder)
+                }
+                DataType::Int8 => filter_primitive_item_list_array!(self, array, Int8Type, ListArray, ListBuilder),
+                DataType::Int16 => {
+                    filter_primitive_item_list_array!(self, array, Int16Type, ListArray, ListBuilder)
+                }
+                DataType::Int32 => {
+                    filter_primitive_item_list_array!(self, array, Int32Type, ListArray, ListBuilder)
+                }
+                DataType::Int64 => {
+                    filter_primitive_item_list_array!(self, array, Int64Type, ListArray, ListBuilder)
+                }
+                DataType::Float32 => {
+                    filter_primitive_item_list_array!(self, array, Float32Type, ListArray, ListBuilder)
+                }
+                DataType::Float64 => {
+                    filter_primitive_item_list_array!(self, array, Float64Type, ListArray, ListBuilder)
+                }
+                DataType::Boolean => {
+                    filter_primitive_item_list_array!(self, array, BooleanType, ListArray, ListBuilder)
+                }
+                DataType::Date32(_) => {
+                    filter_primitive_item_list_array!(self, array, Date32Type, ListArray, ListBuilder)
+                }
+                DataType::Date64(_) => {
+                    filter_primitive_item_list_array!(self, array, Date64Type, ListArray, ListBuilder)
+                }
+                DataType::Time32(TimeUnit::Second) => {
+                    filter_primitive_item_list_array!(self, array, Time32SecondType, ListArray, ListBuilder)
+                }
+                DataType::Time32(TimeUnit::Millisecond) => {
+                    filter_primitive_item_list_array!(self, array, Time32MillisecondType, ListArray, ListBuilder)
+                }
+                DataType::Time64(TimeUnit::Microsecond) => {
+                    filter_primitive_item_list_array!(self, array, Time64MicrosecondType, ListArray, ListBuilder)
+                }
+                DataType::Time64(TimeUnit::Nanosecond) => {
+                    filter_primitive_item_list_array!(self, array, Time64NanosecondType, ListArray, ListBuilder)
+                }
+                DataType::Duration(TimeUnit::Second) => {
+                    filter_primitive_item_list_array!(self, array, DurationSecondType, ListArray, ListBuilder)
+                }
+                DataType::Duration(TimeUnit::Millisecond) => {
+                    filter_primitive_item_list_array!(self, array, DurationMillisecondType, ListArray, ListBuilder)
+                }
+                DataType::Duration(TimeUnit::Microsecond) => {
+                    filter_primitive_item_list_array!(self, array, DurationMicrosecondType, ListArray, ListBuilder)
+                }
+                DataType::Duration(TimeUnit::Nanosecond) => {
+                    filter_primitive_item_list_array!(self, array, DurationNanosecondType, ListArray, ListBuilder)
+                }
+                DataType::Timestamp(TimeUnit::Second, _) => {
+                    filter_primitive_item_list_array!(self, array, TimestampSecondType, ListArray, ListBuilder)
+                }
+                DataType::Timestamp(TimeUnit::Millisecond, _) => {
+                    filter_primitive_item_list_array!(self, array, TimestampMillisecondType, ListArray, ListBuilder)
+                }
+                DataType::Timestamp(TimeUnit::Microsecond, _) => {
+                    filter_primitive_item_list_array!(self, array, TimestampMicrosecondType, ListArray, ListBuilder)
+                }
+                DataType::Timestamp(TimeUnit::Nanosecond, _) => {
+                    filter_primitive_item_list_array!(self, array, TimestampNanosecondType, ListArray, ListBuilder)
+                }
+                DataType::Binary => filter_non_primitive_item_list_array!(
+                    self,
+                    array,
+                    BinaryArray,
+                    BinaryBuilder,
+                    ListArray,
+                    ListBuilder
+                ),
+                DataType::LargeBinary => filter_non_primitive_item_list_array!(
+                    self,
+                    array,
+                    LargeBinaryArray,
+                    LargeBinaryBuilder,
+                    ListArray,
+                    ListBuilder
+                ),
+                DataType::Utf8 => filter_non_primitive_item_list_array!(
+                    self,
+                    array,
+                    StringArray,
+                    StringBuilder,
+                    ListArray
+                    ,ListBuilder
+                ),
+                DataType::LargeUtf8 => filter_non_primitive_item_list_array!(
+                    self,
+                    array,
+                    LargeStringArray,
+                    LargeStringBuilder,
+                    ListArray,
+                    ListBuilder
+                ),
+                other => {
+                    Err(ArrowError::ComputeError(format!(
+                        "filter not supported for List({:?})",
+                        other
+                    )))
+                }
+            }
+            DataType::LargeList(dt) => match dt.data_type() {
+                DataType::UInt8 => {
+                    filter_primitive_item_list_array!(self, array, UInt8Type, LargeListArray, LargeListBuilder)
+                }
+                DataType::UInt16 => {
+                    filter_primitive_item_list_array!(self, array, UInt16Type, LargeListArray, LargeListBuilder)
+                }
+                DataType::UInt32 => {
+                    filter_primitive_item_list_array!(self, array, UInt32Type, LargeListArray, LargeListBuilder)
+                }
+                DataType::UInt64 => {
+                    filter_primitive_item_list_array!(self, array, UInt64Type, LargeListArray, LargeListBuilder)
+                }
+                DataType::Int8 => filter_primitive_item_list_array!(self, array, Int8Type, LargeListArray, LargeListBuilder),
+                DataType::Int16 => {
+                    filter_primitive_item_list_array!(self, array, Int16Type, LargeListArray, LargeListBuilder)
+                }
+                DataType::Int32 => {
+                    filter_primitive_item_list_array!(self, array, Int32Type, LargeListArray, LargeListBuilder)
+                }
+                DataType::Int64 => {
+                    filter_primitive_item_list_array!(self, array, Int64Type, LargeListArray, LargeListBuilder)
+                }
+                DataType::Float32 => {
+                    filter_primitive_item_list_array!(self, array, Float32Type, LargeListArray, LargeListBuilder)
+                }
+                DataType::Float64 => {
+                    filter_primitive_item_list_array!(self, array, Float64Type, LargeListArray, LargeListBuilder)
+                }
+                DataType::Boolean => {
+                    filter_primitive_item_list_array!(self, array, BooleanType, LargeListArray, LargeListBuilder)
+                }
+                DataType::Date32(_) => {
+                    filter_primitive_item_list_array!(self, array, Date32Type, LargeListArray, LargeListBuilder)
+                }
+                DataType::Date64(_) => {
+                    filter_primitive_item_list_array!(self, array, Date64Type, LargeListArray, LargeListBuilder)
+                }
+                DataType::Time32(TimeUnit::Second) => {
+                    filter_primitive_item_list_array!(self, array, Time32SecondType, LargeListArray, LargeListBuilder)
+                }
+                DataType::Time32(TimeUnit::Millisecond) => {
+                    filter_primitive_item_list_array!(self, array, Time32MillisecondType, LargeListArray, LargeListBuilder)
+                }
+                DataType::Time64(TimeUnit::Microsecond) => {
+                    filter_primitive_item_list_array!(self, array, Time64MicrosecondType, LargeListArray, LargeListBuilder)
+                }
+                DataType::Time64(TimeUnit::Nanosecond) => {
+                    filter_primitive_item_list_array!(self, array, Time64NanosecondType, LargeListArray, LargeListBuilder)
+                }
+                DataType::Duration(TimeUnit::Second) => {
+                    filter_primitive_item_list_array!(self, array, DurationSecondType, LargeListArray, LargeListBuilder)
+                }
+                DataType::Duration(TimeUnit::Millisecond) => {
+                    filter_primitive_item_list_array!(self, array, DurationMillisecondType, LargeListArray, LargeListBuilder)
+                }
+                DataType::Duration(TimeUnit::Microsecond) => {
+                    filter_primitive_item_list_array!(self, array, DurationMicrosecondType, LargeListArray, LargeListBuilder)
+                }
+                DataType::Duration(TimeUnit::Nanosecond) => {
+                    filter_primitive_item_list_array!(self, array, DurationNanosecondType, LargeListArray, LargeListBuilder)
+                }
+                DataType::Timestamp(TimeUnit::Second, _) => {
+                    filter_primitive_item_list_array!(self, array, TimestampSecondType, LargeListArray, LargeListBuilder)
+                }
+                DataType::Timestamp(TimeUnit::Millisecond, _) => {
+                    filter_primitive_item_list_array!(self, array, TimestampMillisecondType, LargeListArray, LargeListBuilder)
+                }
+                DataType::Timestamp(TimeUnit::Microsecond, _) => {
+                    filter_primitive_item_list_array!(self, array, TimestampMicrosecondType, LargeListArray, LargeListBuilder)
+                }
+                DataType::Timestamp(TimeUnit::Nanosecond, _) => {
+                    filter_primitive_item_list_array!(self, array, TimestampNanosecondType, LargeListArray, LargeListBuilder)
+                }
+                DataType::Binary => filter_non_primitive_item_list_array!(
+                    self,
+                    array,
+                    BinaryArray,
+                    BinaryBuilder,
+                    LargeListArray,
+                    LargeListBuilder
+                ),
+                DataType::LargeBinary => filter_non_primitive_item_list_array!(
+                    self,
+                    array,
+                    LargeBinaryArray,
+                    LargeBinaryBuilder,
+                    LargeListArray,
+                    LargeListBuilder
+                ),
+                DataType::Utf8 => filter_non_primitive_item_list_array!(
+                    self,
+                    array,
+                    StringArray,
+                    StringBuilder,
+                    LargeListArray,
+                    LargeListBuilder
+                ),
+                DataType::LargeUtf8 => filter_non_primitive_item_list_array!(
+                    self,
+                    array,
+                    LargeStringArray,
+                    LargeStringBuilder,
+                    LargeListArray,
+                    LargeListBuilder
+                ),
+                other => {
+                    Err(ArrowError::ComputeError(format!(
+                        "filter not supported for LargeList({:?})",
+                        other
+                    )))
+                }
+            }
             other => Err(ArrowError::ComputeError(format!(
                 "filter not supported for {:?}",
                 other
@@ -405,7 +730,7 @@ impl FilterContext {
     where
         T: ArrowNumericType,
     {
-        let array_type = T::get_data_type();
+        let array_type = T::DATA_TYPE;
         let value_size = mem::size_of::<T::Native>();
         let array_data_builder =
             filter_array_impl(self, data_array, array_type, value_size)?;
@@ -481,6 +806,8 @@ pub fn filter_record_batch(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::buffer::Buffer;
+    use crate::datatypes::ToByteSlice;
 
     macro_rules! def_temporal_test {
         ($test:ident, $array_type: ident, $data: expr) => {
@@ -597,9 +924,8 @@ mod tests {
     #[test]
     fn test_filter_array_low_density() {
         // this test exercises the all 0's branch of the filter algorithm
-        let mut data_values = (1..=65).into_iter().collect::<Vec<i32>>();
+        let mut data_values = (1..=65).collect::<Vec<i32>>();
         let mut filter_values = (1..=65)
-            .into_iter()
             .map(|i| match i % 65 {
                 0 => true,
                 _ => false,
@@ -620,9 +946,8 @@ mod tests {
     #[test]
     fn test_filter_array_high_density() {
         // this test exercises the all 1's branch of the filter algorithm
-        let mut data_values = (1..=65).into_iter().map(|x| Some(x)).collect::<Vec<_>>();
+        let mut data_values = (1..=65).map(Some).collect::<Vec<_>>();
         let mut filter_values = (1..=65)
-            .into_iter()
             .map(|i| match i % 65 {
                 0 => false,
                 _ => true,
@@ -658,7 +983,7 @@ mod tests {
     }
 
     #[test]
-    fn test_filter_array_with_null() {
+    fn test_filter_primative_array_with_null() {
         let a = Int32Array::from(vec![Some(5), None]);
         let b = BooleanArray::from(vec![false, true]);
         let c = filter(&a, &b).unwrap();
@@ -667,6 +992,31 @@ mod tests {
         assert_eq!(true, d.is_null(0));
     }
 
+    #[test]
+    fn test_filter_string_array_with_null() {
+        let a = StringArray::from(vec![Some("hello"), None, Some("world"), None]);
+        let b = BooleanArray::from(vec![true, false, false, true]);
+        let c = filter(&a, &b).unwrap();
+        let d = c.as_ref().as_any().downcast_ref::<StringArray>().unwrap();
+        assert_eq!(2, d.len());
+        assert_eq!("hello", d.value(0));
+        assert_eq!(false, d.is_null(0));
+        assert_eq!(true, d.is_null(1));
+    }
+
+    #[test]
+    fn test_filter_binary_array_with_null() {
+        let data: Vec<Option<&[u8]>> = vec![Some(b"hello"), None, Some(b"world"), None];
+        let a = BinaryArray::from(data);
+        let b = BooleanArray::from(vec![true, false, false, true]);
+        let c = filter(&a, &b).unwrap();
+        let d = c.as_ref().as_any().downcast_ref::<BinaryArray>().unwrap();
+        assert_eq!(2, d.len());
+        assert_eq!(b"hello", d.value(0));
+        assert_eq!(false, d.is_null(0));
+        assert_eq!(true, d.is_null(1));
+    }
+
     #[test]
     fn test_filter_array_slice_with_null() {
         let a_slice =
@@ -687,7 +1037,7 @@ mod tests {
     #[test]
     fn test_filter_dictionary_array() {
         let values = vec![Some("hello"), None, Some("world"), Some("!")];
-        let a: Int8DictionaryArray = values.iter().map(|&x| x).collect();
+        let a: Int8DictionaryArray = values.iter().copied().collect();
         let b = BooleanArray::from(vec![false, true, true, false]);
         let c = filter(&a, &b).unwrap();
         let d = c
@@ -702,9 +1052,78 @@ mod tests {
         // but keys are filtered
         assert_eq!(2, d.len());
         assert_eq!(true, d.is_null(0));
+        assert_eq!("world", values.value(d.keys().value(1) as usize));
+    }
+
+    #[test]
+    fn test_filter_string_array_with_negated_boolean_array() {
+        let a = StringArray::from(vec!["hello", " ", "world", "!"]);
+        let mut bb = BooleanBuilder::new(2);
+        bb.append_value(false).unwrap();
+        bb.append_value(true).unwrap();
+        bb.append_value(false).unwrap();
+        bb.append_value(true).unwrap();
+        let b = bb.finish();
+        let b = crate::compute::not(&b).unwrap();
+
+        let c = filter(&a, &b).unwrap();
+        let d = c.as_ref().as_any().downcast_ref::<StringArray>().unwrap();
+        assert_eq!(2, d.len());
+        assert_eq!("hello", d.value(0));
+        assert_eq!("world", d.value(1));
+    }
+
+    #[test]
+    fn test_filter_list_array() {
+        let value_data = ArrayData::builder(DataType::Int32)
+            .len(8)
+            .add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
+            .build();
+
+        let value_offsets = Buffer::from(&[0i64, 3, 6, 8, 8].to_byte_slice());
+
+        let list_data_type =
+            DataType::LargeList(Box::new(Field::new("item", DataType::Int32, false)));
+        let list_data = ArrayData::builder(list_data_type)
+            .len(4)
+            .add_buffer(value_offsets)
+            .add_child_data(value_data)
+            .null_bit_buffer(Buffer::from([0b00000111]))
+            .build();
+
+        //  a = [[0, 1, 2], [3, 4, 5], [6, 7], null]
+        let a = LargeListArray::from(list_data);
+        let b = BooleanArray::from(vec![false, true, false, true]);
+        let c = filter(&a, &b).unwrap();
+        let d = c
+            .as_ref()
+            .as_any()
+            .downcast_ref::<LargeListArray>()
+            .unwrap();
+
+        assert_eq!(DataType::Int32, d.value_type());
+
+        // result should be [[3, 4, 5], null]
+        assert_eq!(2, d.len());
+        assert_eq!(1, d.null_count());
+        assert_eq!(true, d.is_null(1));
+
+        assert_eq!(0, d.value_offset(0));
+        assert_eq!(3, d.value_length(0));
+        assert_eq!(3, d.value_offset(1));
+        assert_eq!(0, d.value_length(1));
+        assert_eq!(
+            Buffer::from(&[3, 4, 5].to_byte_slice()),
+            d.values().data().buffers()[0].clone()
+        );
         assert_eq!(
-            "world",
-            values.value(d.keys().nth(1).unwrap().unwrap() as usize)
+            Buffer::from(&[0i64, 3, 3].to_byte_slice()),
+            d.data().buffers()[0].clone()
         );
+        let inner_list = d.value(0);
+        let inner_list = inner_list.as_any().downcast_ref::<Int32Array>().unwrap();
+        assert_eq!(3, inner_list.len());
+        assert_eq!(0, inner_list.null_count());
+        assert_eq!(inner_list, &Int32Array::from(vec![3, 4, 5]));
     }
 }
diff --git a/rust/arrow/src/compute/kernels/length.rs b/rust/arrow/src/compute/kernels/length.rs
index b4ae26fa6b4..bf57f70fa35 100644
--- a/rust/arrow/src/compute/kernels/length.rs
+++ b/rust/arrow/src/compute/kernels/length.rs
@@ -17,52 +17,57 @@
 
 //! Defines kernel for length of a string array
 
-use crate::array::*;
+use crate::datatypes::ToByteSlice;
+use crate::{array::*, buffer::Buffer};
 use crate::{
     datatypes::DataType,
-    datatypes::UInt32Type,
     error::{ArrowError, Result},
 };
 use std::sync::Arc;
 
-/// Returns an array of UInt32 denoting the number of characters in each string in the array.
+fn length_string<OffsetSize>(array: &Array, data_type: DataType) -> Result<ArrayRef>
+where
+    OffsetSize: OffsetSizeTrait,
+{
+    // note: offsets are stored as u8, but they can be interpreted as OffsetSize
+    let offsets = array.data_ref().clone().buffers()[0].clone();
+    // this is a 30% improvement over iterating over u8s and building OffsetSize, which
+    // justifies the usage of `unsafe`.
+    let slice: &[OffsetSize] =
+        &unsafe { offsets.typed_data::<OffsetSize>() }[array.offset()..];
+
+    let lengths: Vec<OffsetSize> = slice
+        .windows(2)
+        .map(|offset| offset[1] - offset[0])
+        .collect();
+
+    let null_bit_buffer = array
+        .data_ref()
+        .null_bitmap()
+        .as_ref()
+        .map(|b| b.bits.clone());
+
+    let data = ArrayData::new(
+        data_type,
+        array.len(),
+        None,
+        null_bit_buffer,
+        0,
+        vec![Buffer::from(lengths.to_byte_slice())],
+        vec![],
+    );
+    Ok(make_array(Arc::new(data)))
+}
+
+/// Returns an array of Int32/Int64 denoting the number of characters in each string in the array.
 ///
-/// * this only accepts StringArray
+/// * this only accepts StringArray/Utf8 and LargeString/LargeUtf8
 /// * length of null is null.
 /// * length is in number of bytes
-pub fn length(array: &Array) -> Result<UInt32Array> {
+pub fn length(array: &Array) -> Result<ArrayRef> {
     match array.data_type() {
-        DataType::Utf8 => {
-            // note: offsets are stored as u8, but they can be interpreted as u32
-            let offsets = array.data_ref().clone().buffers()[0].clone();
-            // this is a 30% improvement over iterating over u8s and building u32, which
-            // justifies the usage of `unsafe`.
-            let slice: &[u32] = unsafe { offsets.typed_data::<u32>() };
-
-            let mut builder = UInt32BufferBuilder::new(array.len());
-            let lengths: Vec<u32> = slice
-                .windows(2)
-                .map(|offset| offset[1] - offset[0])
-                .collect();
-            builder.append_slice(lengths.as_slice())?;
-
-            let null_bit_buffer = array
-                .data_ref()
-                .null_bitmap()
-                .as_ref()
-                .map(|b| b.bits.clone());
-
-            let data = ArrayData::new(
-                DataType::UInt32,
-                array.len(),
-                None,
-                null_bit_buffer,
-                0,
-                vec![builder.finish()],
-                vec![],
-            );
-            Ok(PrimitiveArray::<UInt32Type>::from(Arc::new(data)))
-        }
+        DataType::Utf8 => length_string::<i32>(array, DataType::Int32),
+        DataType::LargeUtf8 => length_string::<i64>(array, DataType::Int64),
         _ => Err(ArrowError::ComputeError(format!(
             "length not supported for {:?}",
             array.data_type()
@@ -74,60 +79,12 @@ pub fn length(array: &Array) -> Result<UInt32Array> {
 mod tests {
     use super::*;
 
-    /// Tests a vector whose len is not a multiple of 4
-    #[test]
-    fn len_3() -> Result<()> {
-        let array = StringArray::from(vec!["hello", " ", "world"]);
-        let result = length(&array)?;
-        assert_eq!(3, result.len());
-        assert_eq!(
-            vec![5, 1, 5],
-            vec![result.value(0), result.value(1), result.value(2)]
-        );
-        Ok(())
-    }
-
-    /// Tests a vector whose len is multiple of 4
-    #[test]
-    fn len_4() -> Result<()> {
-        let array = StringArray::from(vec!["hello", " ", "world", "!"]);
-        let result = length(&array)?;
-        assert_eq!(4, result.len());
-        assert_eq!(
-            vec![5, 1, 5, 1],
-            vec![
-                result.value(0),
-                result.value(1),
-                result.value(2),
-                result.value(3)
-            ]
-        );
-        Ok(())
-    }
-
-    /// Tests a vector with a character with more than one code point.
-    #[test]
-    fn special() -> Result<()> {
-        let mut builder: StringBuilder = StringBuilder::new(1);
-        builder.append_value("💖")?;
-        let array = builder.finish();
-
-        let result = length(&array)?;
-
-        assert_eq!(1, result.len());
-
-        assert_eq!(4, result.value(0));
-        Ok(())
-    }
-
-    /// Tests a vector with more than 255 entries, to ensure that offsets are correctly computed beyond simple cases
-    #[test]
-    fn long_array() -> Result<()> {
+    fn cases() -> Vec<(Vec<&'static str>, usize, Vec<i32>)> {
         fn double_vec<T: Clone>(v: Vec<T>) -> Vec<T> {
             [&v[..], &v[..]].concat()
         }
 
-        // double ["hello", " ", "world", "!"] 10 times
+        // a large array
         let mut values = vec!["one", "on", "o", ""];
         let mut expected = vec![3, 2, 1, 0];
         for _ in 0..10 {
@@ -135,52 +92,120 @@ mod tests {
             expected = double_vec(expected);
         }
 
-        let a = StringArray::from(values);
+        vec![
+            (vec!["hello", " ", "world"], 3, vec![5, 1, 5]),
+            (vec!["hello", " ", "world", "!"], 4, vec![5, 1, 5, 1]),
+            (vec!["💖"], 1, vec![4]),
+            (values, 4096, expected),
+        ]
+    }
 
-        let result = length(&a)?;
+    #[test]
+    fn test_string() -> Result<()> {
+        cases()
+            .into_iter()
+            .map(|(input, len, expected)| {
+                let array = StringArray::from(input);
+                let result = length(&array)?;
+                assert_eq!(len, result.len());
+                let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
+                expected.iter().enumerate().for_each(|(i, value)| {
+                    assert_eq!(*value, result.value(i));
+                });
+                Ok(())
+            })
+            .collect::<Result<()>>()
+    }
 
-        assert_eq!(4096, result.len()); // 2^12
+    #[test]
+    fn test_large_string() -> Result<()> {
+        cases()
+            .into_iter()
+            .map(|(input, len, expected)| {
+                let array = LargeStringArray::from(input);
+                let result = length(&array)?;
+                assert_eq!(len, result.len());
+                let result = result.as_any().downcast_ref::<Int64Array>().unwrap();
+                expected.iter().enumerate().for_each(|(i, value)| {
+                    assert_eq!(*value as i64, result.value(i));
+                });
+                Ok(())
+            })
+            .collect::<Result<()>>()
+    }
 
-        let mut builder = UInt32Builder::new(expected.len());
-        for e in expected {
-            builder.append_value(e)?
-        }
-        assert_eq!(builder.finish(), result);
-        Ok(())
+    fn null_cases() -> Vec<(Vec<Option<&'static str>>, usize, Vec<Option<i32>>)> {
+        vec![(
+            vec![Some("one"), None, Some("three"), Some("four")],
+            4,
+            vec![Some(3), None, Some(5), Some(4)],
+        )]
     }
 
-    /// Tests handling of null values
     #[test]
-    fn null() -> Result<()> {
-        let mut builder: StringBuilder = StringBuilder::new(4);
-        builder.append_value("one")?;
-        builder.append_null()?;
-        builder.append_value("three")?;
-        builder.append_value("four")?;
-        let array = builder.finish();
-
-        let a = length(&array)?;
-        assert_eq!(a.len(), array.len());
-
-        let mut expected = UInt32Builder::new(4);
-        expected.append_value(3)?;
-        expected.append_null()?;
-        expected.append_value(5)?;
-        expected.append_value(4)?;
-        let expected = expected.finish();
-
-        assert_eq!(expected.data(), a.data());
-        Ok(())
+    fn null_string() -> Result<()> {
+        null_cases()
+            .into_iter()
+            .map(|(input, len, expected)| {
+                let array = StringArray::from(input);
+                let result = length(&array)?;
+                assert_eq!(len, result.len());
+                let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
+
+                let expected: Int32Array = expected.into();
+                assert_eq!(expected.data(), result.data());
+                Ok(())
+            })
+            .collect::<Result<()>>()
+    }
+
+    #[test]
+    fn null_large_string() -> Result<()> {
+        null_cases()
+            .into_iter()
+            .map(|(input, len, expected)| {
+                let array = LargeStringArray::from(input);
+                let result = length(&array)?;
+                assert_eq!(len, result.len());
+                let result = result.as_any().downcast_ref::<Int64Array>().unwrap();
+
+                // convert to i64
+                let expected: Int64Array = expected
+                    .iter()
+                    .map(|e| e.map(|e| e as i64))
+                    .collect::<Vec<_>>()
+                    .into();
+                assert_eq!(expected.data(), result.data());
+                Ok(())
+            })
+            .collect::<Result<()>>()
     }
 
     /// Tests that length is not valid for u64.
     #[test]
     fn wrong_type() -> Result<()> {
-        let mut builder = UInt64Builder::new(1);
-        builder.append_value(1)?;
-        let array = builder.finish();
+        let array: UInt64Array = vec![1u64].into();
 
         assert!(length(&array).is_err());
         Ok(())
     }
+
+    /// Tests with an offset
+    #[test]
+    fn offsets() -> Result<()> {
+        let a = StringArray::from(vec!["hello", " ", "world"]);
+        let b = make_array(
+            ArrayData::builder(DataType::Utf8)
+                .len(2)
+                .offset(1)
+                .buffers(a.data_ref().buffers().to_vec())
+                .build(),
+        );
+        let result = length(b.as_ref())?;
+
+        let expected = Int32Array::from(vec![1, 5]);
+        assert_eq!(expected.data(), result.data());
+
+        Ok(())
+    }
 }
diff --git a/rust/arrow/src/compute/kernels/limit.rs b/rust/arrow/src/compute/kernels/limit.rs
index e7c2467a9d7..65f66bce8e5 100644
--- a/rust/arrow/src/compute/kernels/limit.rs
+++ b/rust/arrow/src/compute/kernels/limit.rs
@@ -110,11 +110,12 @@ mod tests {
         bit_util::set_bit(&mut null_bits, 8);
 
         // Construct a list array from the above two
-        let list_data_type = DataType::List(Box::new(DataType::Int32));
-        let list_data = ArrayData::builder(list_data_type.clone())
+        let list_data_type =
+            DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
+        let list_data = ArrayData::builder(list_data_type)
             .len(9)
-            .add_buffer(value_offsets.clone())
-            .add_child_data(value_data.clone())
+            .add_buffer(value_offsets)
+            .add_child_data(value_data)
             .null_bit_buffer(Buffer::from(null_bits))
             .build();
         let list_array: ArrayRef = Arc::new(ListArray::from(list_data));
diff --git a/rust/arrow/src/compute/kernels/mod.rs b/rust/arrow/src/compute/kernels/mod.rs
index 0cd3d70e6db..5ac0f0b0c2e 100644
--- a/rust/arrow/src/compute/kernels/mod.rs
+++ b/rust/arrow/src/compute/kernels/mod.rs
@@ -27,5 +27,6 @@ pub mod filter;
 pub mod length;
 pub mod limit;
 pub mod sort;
+pub mod substring;
 pub mod take;
 pub mod temporal;
diff --git a/rust/arrow/src/compute/kernels/sort.rs b/rust/arrow/src/compute/kernels/sort.rs
index bb5b6375c41..da570ef3241 100644
--- a/rust/arrow/src/compute/kernels/sort.rs
+++ b/rust/arrow/src/compute/kernels/sort.rs
@@ -24,104 +24,178 @@ use crate::compute::take;
 use crate::datatypes::*;
 use crate::error::{ArrowError, Result};
 
+use crate::buffer::MutableBuffer;
+use num::ToPrimitive;
+use std::sync::Arc;
 use TimeUnit::*;
 
 /// Sort the `ArrayRef` using `SortOptions`.
 ///
-/// Performs a stable sort on values and indices, returning nulls after sorted valid values,
-/// while preserving the order of the nulls.
+/// Performs a stable sort on values and indices. Nulls are ordered according to the `nulls_first` flag in `options`.
+/// For floating point arrays any NaN values are considered to be greater than any other non-null value.
 ///
 /// Returns an `ArrowError::ComputeError(String)` if the array type is either unsupported by `sort_to_indices` or `take`.
 ///
-/// Null float values (e.g. f64::NAN) are sorted with non-null values, and are ordered higher than other values.
 pub fn sort(values: &ArrayRef, options: Option<SortOptions>) -> Result<ArrayRef> {
     let indices = sort_to_indices(values, options)?;
     take(values, &indices, None)
 }
 
-/// Sort elements from `ArrayRef` into an unsigned integer (`UInt32Array`) of indices
-pub fn sort_to_indices(
-    values: &ArrayRef,
-    options: Option<SortOptions>,
-) -> Result<UInt32Array> {
-    let options = options.unwrap_or_default();
-    let range = values.offset()..values.len();
-    // perform a custom range partition for floats, to account for NaN
-    let (v, n): (Vec<usize>, Vec<usize>) = if values.data_type() == &DataType::Float32 {
-        let array = values
+// partition indices into non-NaN and NaN
+fn partition_nan<T: ArrowPrimitiveType>(
+    array: &ArrayRef,
+    v: Vec<u32>,
+) -> (Vec<u32>, Vec<u32>) {
+    // partition by nan for float types
+    if T::DATA_TYPE == DataType::Float32 {
+        // T::Native has no `is_nan` and thus we need to downcast
+        let array = array
             .as_any()
             .downcast_ref::<Float32Array>()
             .expect("Unable to downcast array");
-        #[allow(clippy::cmp_nan)]
-        range.partition(|index| array.is_valid(*index) && array.value(*index) != f32::NAN)
-    } else if values.data_type() == &DataType::Float64 {
-        let array = values
+        let has_nan = v.iter().any(|index| array.value(*index as usize).is_nan());
+        if has_nan {
+            v.into_iter()
+                .partition(|index| !array.value(*index as usize).is_nan())
+        } else {
+            (v, vec![])
+        }
+    } else if T::DATA_TYPE == DataType::Float64 {
+        let array = array
             .as_any()
             .downcast_ref::<Float64Array>()
             .expect("Unable to downcast array");
-        #[allow(clippy::cmp_nan)]
-        range.partition(|index| array.is_valid(*index) && array.value(*index) != f64::NAN)
+        let has_nan = v.iter().any(|index| array.value(*index as usize).is_nan());
+        if has_nan {
+            v.into_iter()
+                .partition(|index| !array.value(*index as usize).is_nan())
+        } else {
+            (v, vec![])
+        }
     } else {
-        range.partition(|index| values.is_valid(*index))
-    };
-    let n = n.into_iter().map(|i| i as u32).collect();
+        unreachable!("Partition by nan is only applicable to float types")
+    }
+}
+
+// partition indices into valid and null indices
+fn partition_validity(array: &ArrayRef) -> (Vec<u32>, Vec<u32>) {
+    let indices = 0..(array.len().to_u32().unwrap());
+    indices.partition(|index| array.is_valid(*index as usize))
+}
+
+/// Sort elements from `ArrayRef` into an unsigned integer (`UInt32Array`) of indices.
+/// For floating point arrays any NaN values are considered to be greater than any other non-null value
+pub fn sort_to_indices(
+    values: &ArrayRef,
+    options: Option<SortOptions>,
+) -> Result<UInt32Array> {
+    let options = options.unwrap_or_default();
+
+    let (v, n) = partition_validity(values);
+
     match values.data_type() {
-        DataType::Boolean => sort_primitive::<BooleanType>(values, v, n, &options),
-        DataType::Int8 => sort_primitive::<Int8Type>(values, v, n, &options),
-        DataType::Int16 => sort_primitive::<Int16Type>(values, v, n, &options),
-        DataType::Int32 => sort_primitive::<Int32Type>(values, v, n, &options),
-        DataType::Int64 => sort_primitive::<Int64Type>(values, v, n, &options),
-        DataType::UInt8 => sort_primitive::<UInt8Type>(values, v, n, &options),
-        DataType::UInt16 => sort_primitive::<UInt16Type>(values, v, n, &options),
-        DataType::UInt32 => sort_primitive::<UInt32Type>(values, v, n, &options),
-        DataType::UInt64 => sort_primitive::<UInt64Type>(values, v, n, &options),
-        DataType::Float32 => sort_primitive::<Float32Type>(values, v, n, &options),
-        DataType::Float64 => sort_primitive::<Float64Type>(values, v, n, &options),
-        DataType::Date32(_) => sort_primitive::<Date32Type>(values, v, n, &options),
-        DataType::Date64(_) => sort_primitive::<Date64Type>(values, v, n, &options),
+        DataType::Boolean => {
+            sort_primitive::<BooleanType>(values, v, n, vec![], &options)
+        }
+        DataType::Int8 => sort_primitive::<Int8Type>(values, v, n, vec![], &options),
+        DataType::Int16 => sort_primitive::<Int16Type>(values, v, n, vec![], &options),
+        DataType::Int32 => sort_primitive::<Int32Type>(values, v, n, vec![], &options),
+        DataType::Int64 => sort_primitive::<Int64Type>(values, v, n, vec![], &options),
+        DataType::UInt8 => sort_primitive::<UInt8Type>(values, v, n, vec![], &options),
+        DataType::UInt16 => sort_primitive::<UInt16Type>(values, v, n, vec![], &options),
+        DataType::UInt32 => sort_primitive::<UInt32Type>(values, v, n, vec![], &options),
+        DataType::UInt64 => sort_primitive::<UInt64Type>(values, v, n, vec![], &options),
+        DataType::Float32 => {
+            let (v, nan) = partition_nan::<Float32Type>(values, v);
+            sort_primitive::<Float32Type>(values, v, n, nan, &options)
+        }
+        DataType::Float64 => {
+            let (v, nan) = partition_nan::<Float64Type>(values, v);
+            sort_primitive::<Float64Type>(values, v, n, nan, &options)
+        }
+        DataType::Date32(_) => {
+            sort_primitive::<Date32Type>(values, v, n, vec![], &options)
+        }
+        DataType::Date64(_) => {
+            sort_primitive::<Date64Type>(values, v, n, vec![], &options)
+        }
         DataType::Time32(Second) => {
-            sort_primitive::<Time32SecondType>(values, v, n, &options)
+            sort_primitive::<Time32SecondType>(values, v, n, vec![], &options)
         }
         DataType::Time32(Millisecond) => {
-            sort_primitive::<Time32MillisecondType>(values, v, n, &options)
+            sort_primitive::<Time32MillisecondType>(values, v, n, vec![], &options)
         }
         DataType::Time64(Microsecond) => {
-            sort_primitive::<Time64MicrosecondType>(values, v, n, &options)
+            sort_primitive::<Time64MicrosecondType>(values, v, n, vec![], &options)
         }
         DataType::Time64(Nanosecond) => {
-            sort_primitive::<Time64NanosecondType>(values, v, n, &options)
+            sort_primitive::<Time64NanosecondType>(values, v, n, vec![], &options)
         }
         DataType::Timestamp(Second, _) => {
-            sort_primitive::<TimestampSecondType>(values, v, n, &options)
+            sort_primitive::<TimestampSecondType>(values, v, n, vec![], &options)
         }
         DataType::Timestamp(Millisecond, _) => {
-            sort_primitive::<TimestampMillisecondType>(values, v, n, &options)
+            sort_primitive::<TimestampMillisecondType>(values, v, n, vec![], &options)
         }
         DataType::Timestamp(Microsecond, _) => {
-            sort_primitive::<TimestampMicrosecondType>(values, v, n, &options)
+            sort_primitive::<TimestampMicrosecondType>(values, v, n, vec![], &options)
         }
         DataType::Timestamp(Nanosecond, _) => {
-            sort_primitive::<TimestampNanosecondType>(values, v, n, &options)
+            sort_primitive::<TimestampNanosecondType>(values, v, n, vec![], &options)
         }
         DataType::Interval(IntervalUnit::YearMonth) => {
-            sort_primitive::<IntervalYearMonthType>(values, v, n, &options)
+            sort_primitive::<IntervalYearMonthType>(values, v, n, vec![], &options)
         }
         DataType::Interval(IntervalUnit::DayTime) => {
-            sort_primitive::<IntervalDayTimeType>(values, v, n, &options)
+            sort_primitive::<IntervalDayTimeType>(values, v, n, vec![], &options)
         }
         DataType::Duration(TimeUnit::Second) => {
-            sort_primitive::<DurationSecondType>(values, v, n, &options)
+            sort_primitive::<DurationSecondType>(values, v, n, vec![], &options)
         }
         DataType::Duration(TimeUnit::Millisecond) => {
-            sort_primitive::<DurationMillisecondType>(values, v, n, &options)
+            sort_primitive::<DurationMillisecondType>(values, v, n, vec![], &options)
         }
         DataType::Duration(TimeUnit::Microsecond) => {
-            sort_primitive::<DurationMicrosecondType>(values, v, n, &options)
+            sort_primitive::<DurationMicrosecondType>(values, v, n, vec![], &options)
         }
         DataType::Duration(TimeUnit::Nanosecond) => {
-            sort_primitive::<DurationNanosecondType>(values, v, n, &options)
+            sort_primitive::<DurationNanosecondType>(values, v, n, vec![], &options)
         }
         DataType::Utf8 => sort_string(values, v, n, &options),
+        DataType::Dictionary(key_type, value_type)
+            if *value_type.as_ref() == DataType::Utf8 =>
+        {
+            match key_type.as_ref() {
+                DataType::Int8 => {
+                    sort_string_dictionary::<Int8Type>(values, v, n, &options)
+                }
+                DataType::Int16 => {
+                    sort_string_dictionary::<Int16Type>(values, v, n, &options)
+                }
+                DataType::Int32 => {
+                    sort_string_dictionary::<Int32Type>(values, v, n, &options)
+                }
+                DataType::Int64 => {
+                    sort_string_dictionary::<Int64Type>(values, v, n, &options)
+                }
+                DataType::UInt8 => {
+                    sort_string_dictionary::<UInt8Type>(values, v, n, &options)
+                }
+                DataType::UInt16 => {
+                    sort_string_dictionary::<UInt16Type>(values, v, n, &options)
+                }
+                DataType::UInt32 => {
+                    sort_string_dictionary::<UInt32Type>(values, v, n, &options)
+                }
+                DataType::UInt64 => {
+                    sort_string_dictionary::<UInt64Type>(values, v, n, &options)
+                }
+                t => Err(ArrowError::ComputeError(format!(
+                    "Sort not supported for dictionary key type {:?}",
+                    t
+                ))),
+            }
+        }
         t => Err(ArrowError::ComputeError(format!(
             "Sort not supported for data type {:?}",
             t
@@ -148,11 +222,12 @@ impl Default for SortOptions {
     }
 }
 
-/// Sort primitive values, excluding floats
+/// Sort primitive values
 fn sort_primitive<T>(
     values: &ArrayRef,
-    value_indices: Vec<usize>,
+    value_indices: Vec<u32>,
     null_indices: Vec<u32>,
+    nan_indices: Vec<u32>,
     options: &SortOptions,
 ) -> Result<UInt32Array>
 where
@@ -160,46 +235,154 @@ where
     T::Native: std::cmp::PartialOrd,
 {
     let values = as_primitive_array::<T>(values);
+    let descending = options.descending;
+
     // create tuples that are used for sorting
     let mut valids = value_indices
         .into_iter()
-        .map(|index| (index as u32, values.value(index)))
+        .map(|index| (index, values.value(index as usize)))
         .collect::<Vec<(u32, T::Native)>>();
+
     let mut nulls = null_indices;
-    if !options.descending {
-        valids.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or_else(|| Ordering::Greater));
+    let mut nans = nan_indices;
+
+    let valids_len = valids.len();
+    let nulls_len = nulls.len();
+    let nans_len = nans.len();
+
+    if !descending {
+        valids.sort_by(|a, b| a.1.partial_cmp(&b.1).expect("unexpected NaN"));
     } else {
-        valids.sort_by(|a, b| {
-            a.1.partial_cmp(&b.1)
-                .unwrap_or_else(|| Ordering::Greater)
-                .reverse()
-        });
+        valids.sort_by(|a, b| a.1.partial_cmp(&b.1).expect("unexpected NaN").reverse());
+        // reverse to keep a stable ordering
+        nans.reverse();
         nulls.reverse();
     }
-    // collect the order of valid tuples
-    let mut valid_indices: Vec<u32> = valids.iter().map(|tuple| tuple.0).collect();
+
+    // collect results directly into a buffer instead of a vec to avoid another aligned allocation
+    let mut result = MutableBuffer::new(values.len() * std::mem::size_of::<u32>());
+    // sets len to capacity so we can access the whole buffer as a typed slice
+    result.resize(values.len() * std::mem::size_of::<u32>())?;
+    let result_slice: &mut [u32] = result.typed_data_mut();
+
+    debug_assert_eq!(result_slice.len(), nulls_len + nans_len + valids_len);
 
     if options.nulls_first {
-        nulls.append(&mut valid_indices);
-        return Ok(UInt32Array::from(nulls));
+        result_slice[0..nulls_len].copy_from_slice(&nulls);
+        insert_valid_and_nan_values(result_slice, nulls_len, valids, nans, descending);
+    } else {
+        // nulls last
+        insert_valid_and_nan_values(result_slice, 0, valids, nans, descending);
+        result_slice[valids_len + nans_len..].copy_from_slice(nulls.as_slice())
     }
-    // no need to sort nulls as they are in the correct order already
-    valid_indices.append(&mut nulls);
 
-    Ok(UInt32Array::from(valid_indices))
+    let result_data = Arc::new(ArrayData::new(
+        DataType::UInt32,
+        values.len(),
+        Some(0),
+        None,
+        0,
+        vec![result.freeze()],
+        vec![],
+    ));
+
+    Ok(UInt32Array::from(result_data))
+}
+
+// insert valid and nan values in the correct order depending on the descending flag
+fn insert_valid_and_nan_values<T: ArrowNativeType>(
+    result_slice: &mut [u32],
+    offset: usize,
+    valids: Vec<(u32, T)>,
+    nans: Vec<u32>,
+    descending: bool,
+) {
+    let valids_len = valids.len();
+    let nans_len = nans.len();
+
+    // helper to append the index part of the valid tuples
+    let append_valids = move |dst_slice: &mut [u32]| {
+        debug_assert_eq!(dst_slice.len(), valids_len);
+        dst_slice
+            .iter_mut()
+            .zip(valids.into_iter())
+            .for_each(|(dst, src)| *dst = src.0)
+    };
+
+    // NaNs are considered greater than all number which means
+    // for descending order they come before valid numbers
+    // for ascending order they come after valid numbers
+    if descending {
+        result_slice[offset..offset + nans_len].copy_from_slice(nans.as_slice());
+        append_valids(
+            &mut result_slice[offset + nans_len..offset + nans_len + valids_len],
+        );
+    } else {
+        append_valids(&mut result_slice[offset..offset + valids_len]);
+        result_slice[offset + valids_len..offset + valids_len + nans_len]
+            .copy_from_slice(nans.as_slice());
+    }
 }
 
 /// Sort strings
 fn sort_string(
     values: &ArrayRef,
-    value_indices: Vec<usize>,
+    value_indices: Vec<u32>,
     null_indices: Vec<u32>,
     options: &SortOptions,
 ) -> Result<UInt32Array> {
     let values = as_string_array(values);
+
+    sort_string_helper(
+        values,
+        value_indices,
+        null_indices,
+        options,
+        |array, idx| array.value(idx as usize),
+    )
+}
+
+/// Sort dictionary encoded strings
+fn sort_string_dictionary<T: ArrowDictionaryKeyType>(
+    values: &ArrayRef,
+    value_indices: Vec<u32>,
+    null_indices: Vec<u32>,
+    options: &SortOptions,
+) -> Result<UInt32Array> {
+    let values: &DictionaryArray<T> = as_dictionary_array::<T>(values);
+
+    let keys: &PrimitiveArray<T> = &values.keys_array();
+
+    let dict = values.values();
+    let dict: &StringArray = as_string_array(&dict);
+
+    sort_string_helper(
+        keys,
+        value_indices,
+        null_indices,
+        options,
+        |array: &PrimitiveArray<T>, idx| -> &str {
+            let key: T::Native = array.value(idx as usize);
+            dict.value(key.to_usize().unwrap())
+        },
+    )
+}
+
+/// shared implementation between dictionary encoded and plain string arrays
+#[inline]
+fn sort_string_helper<'a, A: Array, F>(
+    values: &'a A,
+    value_indices: Vec<u32>,
+    null_indices: Vec<u32>,
+    options: &SortOptions,
+    value_fn: F,
+) -> Result<UInt32Array>
+where
+    F: Fn(&'a A, u32) -> &str,
+{
     let mut valids = value_indices
         .into_iter()
-        .map(|index| (index as u32, values.value(index)))
+        .map(|index| (index, value_fn(&values, index)))
         .collect::<Vec<(u32, &str)>>();
     let mut nulls = null_indices;
     if !options.descending {
@@ -239,7 +422,7 @@ pub struct SortColumn {
 /// Example:
 ///
 /// ```
-/// use std::convert::TryFrom;
+/// use std::convert::From;
 /// use std::sync::Arc;
 /// use arrow::array::{ArrayRef, StringArray, PrimitiveArray, as_primitive_array};
 /// use arrow::compute::kernels::sort::{SortColumn, SortOptions, lexsort};
@@ -257,13 +440,13 @@ pub struct SortColumn {
 ///         options: None,
 ///     },
 ///     SortColumn {
-///         values: Arc::new(StringArray::try_from(vec![
+///         values: Arc::new(StringArray::from(vec![
 ///             Some("hello"),
 ///             Some("world"),
 ///             Some(","),
 ///             Some("foobar"),
 ///             Some("!"),
-///         ]).unwrap()) as ArrayRef,
+///         ])) as ArrayRef,
 ///         options: Some(SortOptions {
 ///             descending: true,
 ///             nulls_first: false,
@@ -285,47 +468,46 @@ pub fn lexsort(columns: &[SortColumn]) -> Result<Vec<ArrayRef>> {
 /// Sort elements lexicographically from a list of `ArrayRef` into an unsigned integer
 /// (`UInt32Array`) of indices.
 pub fn lexsort_to_indices(columns: &[SortColumn]) -> Result<UInt32Array> {
+    if columns.len() == 0 {
+        return Err(ArrowError::InvalidArgumentError(
+            "Sort requires at least one column".to_string(),
+        ));
+    }
     if columns.len() == 1 {
         // fallback to non-lexical sort
         let column = &columns[0];
         return sort_to_indices(&column.values, column.options);
     }
 
-    let mut row_count = None;
+    let row_count = columns[0].values.len();
+    if columns.iter().any(|item| item.values.len() != row_count) {
+        return Err(ArrowError::ComputeError(
+            "lexical sort columns have different row counts".to_string(),
+        ));
+    };
+
     // convert ArrayRefs to OrdArray trait objects and perform row count check
     let flat_columns = columns
         .iter()
-        .map(|column| -> Result<(&OrdArray, SortOptions)> {
-            // row count check
-            let curr_row_count = column.values.len() - column.values.offset();
-            match row_count {
-                None => {
-                    row_count = Some(curr_row_count);
-                }
-                Some(cnt) => {
-                    if curr_row_count != cnt {
-                        return Err(ArrowError::ComputeError(
-                            "lexical sort columns have different row counts".to_string(),
-                        ));
-                    }
-                }
-            }
-            // flatten and convert to OrdArray
+        .map(|column| -> Result<(&Array, DynComparator, SortOptions)> {
+            // flatten and convert build comparators
             Ok((
-                as_ordarray(&column.values)?,
+                column.values.as_ref(),
+                build_compare(column.values.as_ref(), column.values.as_ref())?,
                 column.options.unwrap_or_default(),
             ))
         })
-        .collect::<Result<Vec<(&OrdArray, SortOptions)>>>()?;
+        .collect::<Result<Vec<(&Array, DynComparator, SortOptions)>>>()?;
 
     let lex_comparator = |a_idx: &usize, b_idx: &usize| -> Ordering {
         for column in flat_columns.iter() {
             let values = &column.0;
-            let sort_option = column.1;
+            let comparator = &column.1;
+            let sort_option = column.2;
 
             match (values.is_valid(*a_idx), values.is_valid(*b_idx)) {
                 (true, true) => {
-                    match values.cmp_value(*a_idx, *b_idx) {
+                    match (comparator)(*a_idx, *b_idx) {
                         // equal, move on to next column
                         Ordering::Equal => continue,
                         order => {
@@ -359,7 +541,7 @@ pub fn lexsort_to_indices(columns: &[SortColumn]) -> Result<UInt32Array> {
         Ordering::Equal
     };
 
-    let mut value_indices = (0..row_count.unwrap()).collect::<Vec<usize>>();
+    let mut value_indices = (0..row_count).collect::<Vec<usize>>();
     value_indices.sort_by(lex_comparator);
 
     Ok(UInt32Array::from(
@@ -373,7 +555,9 @@ pub fn lexsort_to_indices(columns: &[SortColumn]) -> Result<UInt32Array> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use std::{convert::TryFrom, sync::Arc};
+    use std::convert::TryFrom;
+    use std::iter::FromIterator;
+    use std::sync::Arc;
 
     fn test_sort_to_indices_primitive_arrays<T>(
         data: Vec<Option<T::Native>>,
@@ -381,12 +565,12 @@ mod tests {
         expected_data: Vec<u32>,
     ) where
         T: ArrowPrimitiveType,
-        PrimitiveArray<T>: From<Vec<Option<T::Native>>> + ArrayEqual,
+        PrimitiveArray<T>: From<Vec<Option<T::Native>>>,
     {
         let output = PrimitiveArray::<T>::from(data);
         let expected = UInt32Array::from(expected_data);
         let output = sort_to_indices(&(Arc::new(output) as ArrayRef), options).unwrap();
-        assert!(output.equals(&expected))
+        assert_eq!(output, expected)
     }
 
     fn test_sort_primitive_arrays<T>(
@@ -395,13 +579,12 @@ mod tests {
         expected_data: Vec<Option<T::Native>>,
     ) where
         T: ArrowPrimitiveType,
-        PrimitiveArray<T>: From<Vec<Option<T::Native>>> + ArrayEqual,
+        PrimitiveArray<T>: From<Vec<Option<T::Native>>>,
     {
         let output = PrimitiveArray::<T>::from(data);
-        let expected = PrimitiveArray::<T>::from(expected_data);
+        let expected = Arc::new(PrimitiveArray::<T>::from(expected_data)) as ArrayRef;
         let output = sort(&(Arc::new(output) as ArrayRef), options).unwrap();
-        let output = output.as_any().downcast_ref::<PrimitiveArray<T>>().unwrap();
-        assert!(output.equals(&expected))
+        assert_eq!(&output, &expected)
     }
 
     fn test_sort_to_indices_string_arrays(
@@ -409,10 +592,10 @@ mod tests {
         options: Option<SortOptions>,
         expected_data: Vec<u32>,
     ) {
-        let output = StringArray::try_from(data).expect("Unable to create string array");
+        let output = StringArray::from(data);
         let expected = UInt32Array::from(expected_data);
         let output = sort_to_indices(&(Arc::new(output) as ArrayRef), options).unwrap();
-        assert!(output.equals(&expected))
+        assert_eq!(output, expected)
     }
 
     fn test_sort_string_arrays(
@@ -420,31 +603,61 @@ mod tests {
         options: Option<SortOptions>,
         expected_data: Vec<Option<&str>>,
     ) {
-        let output = StringArray::try_from(data).expect("Unable to create string array");
+        let output = StringArray::from(data);
+        let expected = Arc::new(StringArray::from(expected_data)) as ArrayRef;
+        let output = sort(&(Arc::new(output) as ArrayRef), options).unwrap();
+        assert_eq!(&output, &expected)
+    }
+
+    fn test_sort_string_dict_arrays<T: ArrowDictionaryKeyType>(
+        data: Vec<Option<&str>>,
+        options: Option<SortOptions>,
+        expected_data: Vec<Option<&str>>,
+    ) {
+        let array = DictionaryArray::<T>::from_iter(data.into_iter());
+        let array_values = array.values();
+        let dict = array_values
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .expect("Unable to get dictionary values");
+
+        let sorted = sort(&(Arc::new(array) as ArrayRef), options).unwrap();
+        let sorted = sorted
+            .as_any()
+            .downcast_ref::<DictionaryArray<T>>()
+            .unwrap();
+        let sorted_values = sorted.values();
+        let sorted_dict = sorted_values
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .expect("Unable to get dictionary values");
+        let sorted_keys = sorted.keys_array();
+
+        assert_eq!(sorted_dict, dict);
+
+        let sorted_strings = StringArray::try_from(
+            (0..sorted.len())
+                .map(|i| {
+                    if sorted.is_valid(i) {
+                        Some(sorted_dict.value(sorted_keys.value(i).to_usize().unwrap()))
+                    } else {
+                        None
+                    }
+                })
+                .collect::<Vec<Option<&str>>>(),
+        )
+        .expect("Unable to create string array from dictionary");
         let expected =
             StringArray::try_from(expected_data).expect("Unable to create string array");
-        let output = sort(&(Arc::new(output) as ArrayRef), options).unwrap();
-        let output = output.as_any().downcast_ref::<StringArray>().unwrap();
-        assert!(output.equals(&expected))
+
+        assert_eq!(sorted_strings, expected)
     }
 
     fn test_lex_sort_arrays(input: Vec<SortColumn>, expected_output: Vec<ArrayRef>) {
         let sorted = lexsort(&input).unwrap();
-        let sorted2cmp = sorted.iter().map(|arr| -> Box<&dyn ArrayEqual> {
-            match arr.data_type() {
-                DataType::Int64 => Box::new(as_primitive_array::<Int64Type>(&arr)),
-                DataType::UInt32 => Box::new(as_primitive_array::<UInt32Type>(&arr)),
-                DataType::Utf8 => Box::new(as_string_array(&arr)),
-                _ => panic!("unexpected array type"),
-            }
-        });
-        for (i, values) in sorted2cmp.enumerate() {
-            assert!(
-                values.equals(&(*expected_output[i])),
-                "expect {:#?} to be: {:#?}",
-                sorted,
-                expected_output
-            );
+
+        for (result, expected) in sorted.iter().zip(expected_output.iter()) {
+            assert_eq!(result, expected);
         }
     }
 
@@ -747,6 +960,14 @@ mod tests {
             }),
             vec![None, None, Some(f64::NAN), Some(2.0), Some(0.0), Some(-1.0)],
         );
+        test_sort_primitive_arrays::<Float64Type>(
+            vec![Some(f64::NAN), Some(f64::NAN), Some(f64::NAN), Some(1.0)],
+            Some(SortOptions {
+                descending: true,
+                nulls_first: true,
+            }),
+            vec![Some(f64::NAN), Some(f64::NAN), Some(f64::NAN), Some(1.0)],
+        );
 
         // int8 nulls first
         test_sort_primitive_arrays::<Int8Type>(
@@ -797,6 +1018,14 @@ mod tests {
             }),
             vec![None, None, Some(-1.0), Some(0.0), Some(2.0), Some(f64::NAN)],
         );
+        test_sort_primitive_arrays::<Float64Type>(
+            vec![Some(f64::NAN), Some(f64::NAN), Some(f64::NAN), Some(1.0)],
+            Some(SortOptions {
+                descending: false,
+                nulls_first: true,
+            }),
+            vec![Some(1.0), Some(f64::NAN), Some(f64::NAN), Some(f64::NAN)],
+        );
     }
 
     #[test]
@@ -955,6 +1184,98 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_sort_string_dicts() {
+        test_sort_string_dict_arrays::<Int8Type>(
+            vec![
+                None,
+                Some("bad"),
+                Some("sad"),
+                None,
+                Some("glad"),
+                Some("-ad"),
+            ],
+            None,
+            vec![
+                None,
+                None,
+                Some("-ad"),
+                Some("bad"),
+                Some("glad"),
+                Some("sad"),
+            ],
+        );
+
+        test_sort_string_dict_arrays::<Int16Type>(
+            vec![
+                None,
+                Some("bad"),
+                Some("sad"),
+                None,
+                Some("glad"),
+                Some("-ad"),
+            ],
+            Some(SortOptions {
+                descending: true,
+                nulls_first: false,
+            }),
+            vec![
+                Some("sad"),
+                Some("glad"),
+                Some("bad"),
+                Some("-ad"),
+                None,
+                None,
+            ],
+        );
+
+        test_sort_string_dict_arrays::<Int32Type>(
+            vec![
+                None,
+                Some("bad"),
+                Some("sad"),
+                None,
+                Some("glad"),
+                Some("-ad"),
+            ],
+            Some(SortOptions {
+                descending: false,
+                nulls_first: true,
+            }),
+            vec![
+                None,
+                None,
+                Some("-ad"),
+                Some("bad"),
+                Some("glad"),
+                Some("sad"),
+            ],
+        );
+
+        test_sort_string_dict_arrays::<Int16Type>(
+            vec![
+                None,
+                Some("bad"),
+                Some("sad"),
+                None,
+                Some("glad"),
+                Some("-ad"),
+            ],
+            Some(SortOptions {
+                descending: true,
+                nulls_first: true,
+            }),
+            vec![
+                None,
+                None,
+                Some("sad"),
+                Some("glad"),
+                Some("bad"),
+                Some("-ad"),
+            ],
+        );
+    }
+
     #[test]
     fn test_lex_sort_single_column() {
         let input = vec![SortColumn {
@@ -984,10 +1305,7 @@ mod tests {
                 options: None,
             },
             SortColumn {
-                values: Arc::new(
-                    StringArray::try_from(vec![Some("foo")])
-                        .expect("Unable to create string array"),
-                ) as ArrayRef,
+                values: Arc::new(StringArray::from(vec![Some("foo")])) as ArrayRef,
                 options: None,
             },
         ];
@@ -1065,15 +1383,12 @@ mod tests {
                 }),
             },
             SortColumn {
-                values: Arc::new(
-                    StringArray::try_from(vec![
-                        Some("foo"),
-                        Some("9"),
-                        Some("7"),
-                        Some("bar"),
-                    ])
-                    .expect("Unable to create string array"),
-                ) as ArrayRef,
+                values: Arc::new(StringArray::from(vec![
+                    Some("foo"),
+                    Some("9"),
+                    Some("7"),
+                    Some("bar"),
+                ])) as ArrayRef,
                 options: Some(SortOptions {
                     descending: true,
                     nulls_first: true,
@@ -1087,15 +1402,12 @@ mod tests {
                 Some(0),
                 Some(-1),
             ])) as ArrayRef,
-            Arc::new(
-                StringArray::try_from(vec![
-                    Some("9"),
-                    Some("foo"),
-                    Some("bar"),
-                    Some("7"),
-                ])
-                .expect("Unable to create string array"),
-            ) as ArrayRef,
+            Arc::new(StringArray::from(vec![
+                Some("9"),
+                Some("foo"),
+                Some("bar"),
+                Some("7"),
+            ])) as ArrayRef,
         ];
         test_lex_sort_arrays(input, expected);
 
@@ -1114,15 +1426,12 @@ mod tests {
                 }),
             },
             SortColumn {
-                values: Arc::new(
-                    StringArray::try_from(vec![
-                        Some("foo"),
-                        Some("world"),
-                        Some("hello"),
-                        None,
-                    ])
-                    .expect("Unable to create string array"),
-                ) as ArrayRef,
+                values: Arc::new(StringArray::from(vec![
+                    Some("foo"),
+                    Some("world"),
+                    Some("hello"),
+                    None,
+                ])) as ArrayRef,
                 options: Some(SortOptions {
                     descending: true,
                     nulls_first: true,
@@ -1136,15 +1445,12 @@ mod tests {
                 Some(2),
                 Some(-1),
             ])) as ArrayRef,
-            Arc::new(
-                StringArray::try_from(vec![
-                    None,
-                    Some("foo"),
-                    Some("hello"),
-                    Some("world"),
-                ])
-                .expect("Unable to create string array"),
-            ) as ArrayRef,
+            Arc::new(StringArray::from(vec![
+                None,
+                Some("foo"),
+                Some("hello"),
+                Some("world"),
+            ])) as ArrayRef,
         ];
         test_lex_sort_arrays(input, expected);
 
@@ -1163,15 +1469,12 @@ mod tests {
                 }),
             },
             SortColumn {
-                values: Arc::new(
-                    StringArray::try_from(vec![
-                        Some("foo"),
-                        Some("world"),
-                        Some("hello"),
-                        None,
-                    ])
-                    .expect("Unable to create string array"),
-                ) as ArrayRef,
+                values: Arc::new(StringArray::from(vec![
+                    Some("foo"),
+                    Some("world"),
+                    Some("hello"),
+                    None,
+                ])) as ArrayRef,
                 options: Some(SortOptions {
                     descending: true,
                     nulls_first: false,
@@ -1185,15 +1488,12 @@ mod tests {
                 None,
                 None,
             ])) as ArrayRef,
-            Arc::new(
-                StringArray::try_from(vec![
-                    Some("hello"),
-                    Some("world"),
-                    Some("foo"),
-                    None,
-                ])
-                .expect("Unable to create string array"),
-            ) as ArrayRef,
+            Arc::new(StringArray::from(vec![
+                Some("hello"),
+                Some("world"),
+                Some("foo"),
+                None,
+            ])) as ArrayRef,
         ];
         test_lex_sort_arrays(input, expected);
 
@@ -1213,16 +1513,13 @@ mod tests {
                 }),
             },
             SortColumn {
-                values: Arc::new(
-                    StringArray::try_from(vec![
-                        Some("foo"),
-                        Some("bar"),
-                        Some("world"),
-                        Some("hello"),
-                        None,
-                    ])
-                    .expect("Unable to create string array"),
-                ) as ArrayRef,
+                values: Arc::new(StringArray::from(vec![
+                    Some("foo"),
+                    Some("bar"),
+                    Some("world"),
+                    Some("hello"),
+                    None,
+                ])) as ArrayRef,
                 options: Some(SortOptions {
                     descending: true,
                     nulls_first: true,
@@ -1237,16 +1534,13 @@ mod tests {
                 None,
                 None,
             ])) as ArrayRef,
-            Arc::new(
-                StringArray::try_from(vec![
-                    Some("hello"),
-                    Some("bar"),
-                    Some("world"),
-                    None,
-                    Some("foo"),
-                ])
-                .expect("Unable to create string array"),
-            ) as ArrayRef,
+            Arc::new(StringArray::from(vec![
+                Some("hello"),
+                Some("bar"),
+                Some("world"),
+                None,
+                Some("foo"),
+            ])) as ArrayRef,
         ];
         test_lex_sort_arrays(input, expected);
     }
diff --git a/rust/arrow/src/compute/kernels/substring.rs b/rust/arrow/src/compute/kernels/substring.rs
new file mode 100644
index 00000000000..38c8040ef00
--- /dev/null
+++ b/rust/arrow/src/compute/kernels/substring.rs
@@ -0,0 +1,271 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Defines kernel to extract a substring of a \[Large\]StringArray
+
+use crate::{array::*, buffer::Buffer, datatypes::ToByteSlice};
+use crate::{
+    datatypes::DataType,
+    error::{ArrowError, Result},
+};
+use std::sync::Arc;
+
+fn generic_substring<OffsetSize: StringOffsetSizeTrait>(
+    array: &GenericStringArray<OffsetSize>,
+    start: OffsetSize,
+    length: &Option<OffsetSize>,
+) -> Result<ArrayRef> {
+    // compute current offsets
+    let offsets = array.data_ref().clone().buffers()[0].clone();
+    let offsets: &[OffsetSize] = unsafe { offsets.typed_data::<OffsetSize>() };
+
+    // compute null bitmap (copy)
+    let null_bit_buffer = array.data_ref().null_buffer().cloned();
+
+    // compute values
+    let values = &array.data_ref().buffers()[1];
+    let data = values.data();
+
+    let mut new_values = Vec::new(); // we have no way to estimate how much this will be.
+    let mut new_offsets: Vec<OffsetSize> = Vec::with_capacity(array.len() + 1);
+
+    let mut length_so_far = OffsetSize::zero();
+    new_offsets.push(length_so_far);
+    (0..array.len()).for_each(|i| {
+        // the length of this entry
+        let lenght_i: OffsetSize = offsets[i + 1] - offsets[i];
+        // compute where we should start slicing this entry
+        let start = offsets[i]
+            + if start >= OffsetSize::zero() {
+                start
+            } else {
+                lenght_i + start
+            };
+
+        let start = start.max(offsets[i]).min(offsets[i + 1]);
+        // compute the lenght of the slice
+        let length: OffsetSize = length
+            .unwrap_or(lenght_i)
+            // .max(0) is not needed as it is guaranteed
+            .min(offsets[i + 1] - start); // so we do not go beyond this entry
+
+        length_so_far = length_so_far + length;
+
+        new_offsets.push(length_so_far);
+
+        // we need usize for ranges
+        let start = start.to_usize().unwrap();
+        let length = length.to_usize().unwrap();
+
+        new_values.extend_from_slice(&data[start..start + length]);
+    });
+
+    let data = ArrayData::new(
+        <OffsetSize as StringOffsetSizeTrait>::DATA_TYPE,
+        array.len(),
+        None,
+        null_bit_buffer,
+        0,
+        vec![
+            Buffer::from(new_offsets.to_byte_slice()),
+            Buffer::from(&new_values[..]),
+        ],
+        vec![],
+    );
+    Ok(make_array(Arc::new(data)))
+}
+
+/// Returns an ArrayRef with a substring starting from `start` and with optional length `length` of each of the elements in `array`.
+/// `start` can be negative, in which case the start counts from the end of the string.
+/// this function errors when the passed array is not a \[Large\]String array.
+pub fn substring(array: &Array, start: i64, length: &Option<u64>) -> Result<ArrayRef> {
+    match array.data_type() {
+        DataType::LargeUtf8 => generic_substring(
+            array
+                .as_any()
+                .downcast_ref::<LargeStringArray>()
+                .expect("A large string is expected"),
+            start,
+            &length.map(|e| e as i64),
+        ),
+        DataType::Utf8 => generic_substring(
+            array
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .expect("A string is expected"),
+            start as i32,
+            &length.map(|e| e as i32),
+        ),
+        _ => Err(ArrowError::ComputeError(format!(
+            "substring does not support type {:?}",
+            array.data_type()
+        ))),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn with_nulls<T: 'static + Array + PartialEq + From<Vec<Option<&'static str>>>>(
+    ) -> Result<()> {
+        let cases = vec![
+            // identity
+            (
+                vec![Some("hello"), None, Some("word")],
+                0,
+                None,
+                vec![Some("hello"), None, Some("word")],
+            ),
+            // 0 length -> Nothing
+            (
+                vec![Some("hello"), None, Some("word")],
+                0,
+                Some(0),
+                vec![Some(""), None, Some("")],
+            ),
+            // high start -> Nothing
+            (
+                vec![Some("hello"), None, Some("word")],
+                1000,
+                Some(0),
+                vec![Some(""), None, Some("")],
+            ),
+            // high negative start -> identity
+            (
+                vec![Some("hello"), None, Some("word")],
+                -1000,
+                None,
+                vec![Some("hello"), None, Some("word")],
+            ),
+            // high length -> identity
+            (
+                vec![Some("hello"), None, Some("word")],
+                0,
+                Some(1000),
+                vec![Some("hello"), None, Some("word")],
+            ),
+        ];
+
+        cases
+            .into_iter()
+            .map(|(array, start, length, expected)| {
+                let array = T::from(array);
+                let result = substring(&array, start, &length)?;
+                assert_eq!(array.len(), result.len());
+
+                let result = result.as_any().downcast_ref::<T>().unwrap();
+                let expected = T::from(expected);
+                assert_eq!(&expected, result);
+                Ok(())
+            })
+            .collect::<Result<()>>()?;
+
+        Ok(())
+    }
+
+    #[test]
+    fn with_nulls_string() -> Result<()> {
+        with_nulls::<StringArray>()
+    }
+
+    #[test]
+    fn with_nulls_large_string() -> Result<()> {
+        with_nulls::<LargeStringArray>()
+    }
+
+    fn without_nulls<T: 'static + Array + PartialEq + From<Vec<Option<&'static str>>>>(
+    ) -> Result<()> {
+        let cases = vec![
+            // increase start
+            (
+                vec!["hello", "", "word"],
+                0,
+                None,
+                vec!["hello", "", "word"],
+            ),
+            (vec!["hello", "", "word"], 1, None, vec!["ello", "", "ord"]),
+            (vec!["hello", "", "word"], 2, None, vec!["llo", "", "rd"]),
+            (vec!["hello", "", "word"], 3, None, vec!["lo", "", "d"]),
+            (vec!["hello", "", "word"], 10, None, vec!["", "", ""]),
+            // increase start negatively
+            (vec!["hello", "", "word"], -1, None, vec!["o", "", "d"]),
+            (vec!["hello", "", "word"], -2, None, vec!["lo", "", "rd"]),
+            (vec!["hello", "", "word"], -3, None, vec!["llo", "", "ord"]),
+            (
+                vec!["hello", "", "word"],
+                -10,
+                None,
+                vec!["hello", "", "word"],
+            ),
+            // increase length
+            (vec!["hello", "", "word"], 1, Some(1), vec!["e", "", "o"]),
+            (vec!["hello", "", "word"], 1, Some(2), vec!["el", "", "or"]),
+            (
+                vec!["hello", "", "word"],
+                1,
+                Some(3),
+                vec!["ell", "", "ord"],
+            ),
+            (
+                vec!["hello", "", "word"],
+                1,
+                Some(4),
+                vec!["ello", "", "ord"],
+            ),
+            (vec!["hello", "", "word"], -3, Some(1), vec!["l", "", "o"]),
+            (vec!["hello", "", "word"], -3, Some(2), vec!["ll", "", "or"]),
+            (
+                vec!["hello", "", "word"],
+                -3,
+                Some(3),
+                vec!["llo", "", "ord"],
+            ),
+            (
+                vec!["hello", "", "word"],
+                -3,
+                Some(4),
+                vec!["llo", "", "ord"],
+            ),
+        ];
+
+        cases
+            .into_iter()
+            .map(|(array, start, length, expected)| {
+                let array = StringArray::from(array);
+                let result = substring(&array, start, &length)?;
+                assert_eq!(array.len(), result.len());
+                let result = result.as_any().downcast_ref::<StringArray>().unwrap();
+                let expected = StringArray::from(expected);
+                assert_eq!(&expected, result,);
+                Ok(())
+            })
+            .collect::<Result<()>>()?;
+
+        Ok(())
+    }
+
+    #[test]
+    fn without_nulls_string() -> Result<()> {
+        without_nulls::<StringArray>()
+    }
+
+    #[test]
+    fn without_nulls_large_string() -> Result<()> {
+        without_nulls::<LargeStringArray>()
+    }
+}
diff --git a/rust/arrow/src/compute/kernels/take.rs b/rust/arrow/src/compute/kernels/take.rs
index e88975cc6bf..9c9ca56fec3 100644
--- a/rust/arrow/src/compute/kernels/take.rs
+++ b/rust/arrow/src/compute/kernels/take.rs
@@ -17,18 +17,37 @@
 
 //! Defines take kernel for `ArrayRef`
 
-use std::sync::Arc;
+use std::{ops::AddAssign, sync::Arc};
 
-use crate::array::*;
 use crate::buffer::{Buffer, MutableBuffer};
 use crate::compute::util::take_value_indices_from_list;
 use crate::datatypes::*;
 use crate::error::{ArrowError, Result};
 use crate::util::bit_util;
+use crate::{array::*, buffer::buffer_bin_and};
 
+use num::{ToPrimitive, Zero};
 use TimeUnit::*;
 
-/// Take elements from `ArrayRef` by supplying an array of indices.
+/// Take elements from `ArrayRef` by copying the data from `values` at
+/// each index in `indices` into a new `ArrayRef`
+///
+/// For example:
+/// ```
+/// use std::sync::Arc;
+/// use arrow::array::{Array, StringArray, UInt32Array};
+/// use arrow::compute::take;
+///
+/// let values = StringArray::from(vec!["zero", "one", "two"]);
+/// let values: Arc<dyn Array> = Arc::new(values);
+///
+/// // Take items at index 2, and 1:
+/// let indices = UInt32Array::from(vec![2, 1]);
+/// let taken = take(&values, &indices, None).unwrap();
+/// let taken = taken.as_any().downcast_ref::<StringArray>().unwrap();
+///
+/// assert_eq!(*taken, StringArray::from(vec!["two", "one"]));
+/// ```
 ///
 /// Supports:
 ///  * null indices, returning a null value for the index
@@ -38,12 +57,26 @@ pub fn take(
     indices: &UInt32Array,
     options: Option<TakeOptions>,
 ) -> Result<ArrayRef> {
+    take_impl::<UInt32Type>(values, indices, options)
+}
+
+fn take_impl<IndexType>(
+    values: &ArrayRef,
+    indices: &PrimitiveArray<IndexType>,
+    options: Option<TakeOptions>,
+) -> Result<ArrayRef>
+where
+    IndexType: ArrowNumericType,
+    IndexType::Native: ToPrimitive,
+{
     let options = options.unwrap_or_default();
     if options.check_bounds {
         let len = values.len();
         for i in 0..indices.len() {
             if indices.is_valid(i) {
-                let ix = indices.value(i) as usize;
+                let ix = ToPrimitive::to_usize(&indices.value(i)).ok_or_else(|| {
+                    ArrowError::ComputeError("Cast to usize failed".to_string())
+                })?;
                 if ix >= len {
                     return Err(ArrowError::ComputeError(
                     format!("Array index out of bounds, cannot get item at index {} from {} entries", ix, len))
@@ -53,74 +86,89 @@ pub fn take(
         }
     }
     match values.data_type() {
-        DataType::Boolean => take_primitive::<BooleanType>(values, indices),
-        DataType::Int8 => take_primitive::<Int8Type>(values, indices),
-        DataType::Int16 => take_primitive::<Int16Type>(values, indices),
-        DataType::Int32 => take_primitive::<Int32Type>(values, indices),
-        DataType::Int64 => take_primitive::<Int64Type>(values, indices),
-        DataType::UInt8 => take_primitive::<UInt8Type>(values, indices),
-        DataType::UInt16 => take_primitive::<UInt16Type>(values, indices),
-        DataType::UInt32 => take_primitive::<UInt32Type>(values, indices),
-        DataType::UInt64 => take_primitive::<UInt64Type>(values, indices),
-        DataType::Float32 => take_primitive::<Float32Type>(values, indices),
-        DataType::Float64 => take_primitive::<Float64Type>(values, indices),
-        DataType::Date32(_) => take_primitive::<Date32Type>(values, indices),
-        DataType::Date64(_) => take_primitive::<Date64Type>(values, indices),
-        DataType::Time32(Second) => take_primitive::<Time32SecondType>(values, indices),
+        DataType::Boolean => take_boolean(values, indices),
+        DataType::Int8 => take_primitive::<Int8Type, _>(values, indices),
+        DataType::Int16 => take_primitive::<Int16Type, _>(values, indices),
+        DataType::Int32 => take_primitive::<Int32Type, _>(values, indices),
+        DataType::Int64 => take_primitive::<Int64Type, _>(values, indices),
+        DataType::UInt8 => take_primitive::<UInt8Type, _>(values, indices),
+        DataType::UInt16 => take_primitive::<UInt16Type, _>(values, indices),
+        DataType::UInt32 => take_primitive::<UInt32Type, _>(values, indices),
+        DataType::UInt64 => take_primitive::<UInt64Type, _>(values, indices),
+        DataType::Float32 => take_primitive::<Float32Type, _>(values, indices),
+        DataType::Float64 => take_primitive::<Float64Type, _>(values, indices),
+        DataType::Date32(_) => take_primitive::<Date32Type, _>(values, indices),
+        DataType::Date64(_) => take_primitive::<Date64Type, _>(values, indices),
+        DataType::Time32(Second) => {
+            take_primitive::<Time32SecondType, _>(values, indices)
+        }
         DataType::Time32(Millisecond) => {
-            take_primitive::<Time32MillisecondType>(values, indices)
+            take_primitive::<Time32MillisecondType, _>(values, indices)
         }
         DataType::Time64(Microsecond) => {
-            take_primitive::<Time64MicrosecondType>(values, indices)
+            take_primitive::<Time64MicrosecondType, _>(values, indices)
         }
         DataType::Time64(Nanosecond) => {
-            take_primitive::<Time64NanosecondType>(values, indices)
+            take_primitive::<Time64NanosecondType, _>(values, indices)
         }
         DataType::Timestamp(Second, _) => {
-            take_primitive::<TimestampSecondType>(values, indices)
+            take_primitive::<TimestampSecondType, _>(values, indices)
         }
         DataType::Timestamp(Millisecond, _) => {
-            take_primitive::<TimestampMillisecondType>(values, indices)
+            take_primitive::<TimestampMillisecondType, _>(values, indices)
         }
         DataType::Timestamp(Microsecond, _) => {
-            take_primitive::<TimestampMicrosecondType>(values, indices)
+            take_primitive::<TimestampMicrosecondType, _>(values, indices)
         }
         DataType::Timestamp(Nanosecond, _) => {
-            take_primitive::<TimestampNanosecondType>(values, indices)
+            take_primitive::<TimestampNanosecondType, _>(values, indices)
         }
         DataType::Interval(IntervalUnit::YearMonth) => {
-            take_primitive::<IntervalYearMonthType>(values, indices)
+            take_primitive::<IntervalYearMonthType, _>(values, indices)
         }
         DataType::Interval(IntervalUnit::DayTime) => {
-            take_primitive::<IntervalDayTimeType>(values, indices)
+            take_primitive::<IntervalDayTimeType, _>(values, indices)
         }
         DataType::Duration(TimeUnit::Second) => {
-            take_primitive::<DurationSecondType>(values, indices)
+            take_primitive::<DurationSecondType, _>(values, indices)
         }
         DataType::Duration(TimeUnit::Millisecond) => {
-            take_primitive::<DurationMillisecondType>(values, indices)
+            take_primitive::<DurationMillisecondType, _>(values, indices)
         }
         DataType::Duration(TimeUnit::Microsecond) => {
-            take_primitive::<DurationMicrosecondType>(values, indices)
+            take_primitive::<DurationMicrosecondType, _>(values, indices)
         }
         DataType::Duration(TimeUnit::Nanosecond) => {
-            take_primitive::<DurationNanosecondType>(values, indices)
+            take_primitive::<DurationNanosecondType, _>(values, indices)
         }
-        DataType::Utf8 => take_string(values, indices),
-        DataType::List(_) => take_list(values, indices),
+        DataType::Utf8 => take_string::<i32, _>(values, indices),
+        DataType::LargeUtf8 => take_string::<i64, _>(values, indices),
+        DataType::List(_) => take_list::<_, Int32Type>(values, indices),
+        DataType::LargeList(_) => take_list::<_, Int64Type>(values, indices),
         DataType::Struct(fields) => {
             let struct_: &StructArray =
                 values.as_any().downcast_ref::<StructArray>().unwrap();
             let arrays: Result<Vec<ArrayRef>> = struct_
                 .columns()
                 .iter()
-                .map(|a| take(a, indices, Some(options.clone())))
+                .map(|a| take_impl(a, indices, Some(options.clone())))
                 .collect();
             let arrays = arrays?;
             let pairs: Vec<(Field, ArrayRef)> =
                 fields.clone().into_iter().zip(arrays).collect();
             Ok(Arc::new(StructArray::from(pairs)) as ArrayRef)
         }
+        DataType::Dictionary(key_type, _) => match key_type.as_ref() {
+            DataType::Int8 => take_dict::<Int8Type, _>(values, indices),
+            DataType::Int16 => take_dict::<Int16Type, _>(values, indices),
+            DataType::Int32 => take_dict::<Int32Type, _>(values, indices),
+            DataType::Int64 => take_dict::<Int64Type, _>(values, indices),
+            DataType::UInt8 => take_dict::<UInt8Type, _>(values, indices),
+            DataType::UInt16 => take_dict::<UInt16Type, _>(values, indices),
+            DataType::UInt32 => take_dict::<UInt32Type, _>(values, indices),
+            DataType::UInt64 => take_dict::<UInt64Type, _>(values, indices),
+            t => unimplemented!("Take not supported for dictionary key type {:?}", t),
+        },
         t => unimplemented!("Take not supported for data type {:?}", t),
     }
 }
@@ -142,7 +190,7 @@ impl Default for TakeOptions {
     }
 }
 
-/// `take` implementation for primitive arrays
+/// `take` implementation for all primitive arrays except boolean
 ///
 /// This checks if an `indices` slot is populated, and gets the value from `values`
 ///  as the populated index.
@@ -151,46 +199,165 @@ impl Default for TakeOptions {
 ///     values:  [1, 2, 3, null, 5]
 ///     indices: [0, null, 4, 3]
 /// The result is: [1 (slot 0), null (null slot), 5 (slot 4), null (slot 3)]
-fn take_primitive<T>(values: &ArrayRef, indices: &UInt32Array) -> Result<ArrayRef>
+fn take_primitive<T, I>(
+    values: &ArrayRef,
+    indices: &PrimitiveArray<I>,
+) -> Result<ArrayRef>
 where
     T: ArrowPrimitiveType,
+    I: ArrowNumericType,
+    I::Native: ToPrimitive,
 {
-    let mut builder = PrimitiveBuilder::<T>::new(indices.len());
-    let a = values.as_any().downcast_ref::<PrimitiveArray<T>>().unwrap();
-    for i in 0..indices.len() {
-        if indices.is_null(i) {
-            // populate with null if index is null
-            builder.append_null()?;
-        } else {
-            // get index value to use in looking up the value from `values`
-            let ix = indices.value(i) as usize;
-            if a.is_valid(ix) {
-                builder.append_value(a.value(ix))?;
-            } else {
-                builder.append_null()?;
-            }
+    let data_len = indices.len();
+
+    let array = values.as_any().downcast_ref::<PrimitiveArray<T>>().unwrap();
+
+    let num_bytes = bit_util::ceil(data_len, 8);
+    let mut null_buf = MutableBuffer::new(num_bytes).with_bitset(num_bytes, true);
+
+    let null_slice = null_buf.data_mut();
+
+    // This iteration is implemented with a while loop, rather than a
+    // map()/collect(), since the while loop performs better in the benchmarks.
+    let mut new_values: Vec<T::Native> = Vec::with_capacity(data_len);
+    let mut i = 0;
+    while i < data_len {
+        let index = ToPrimitive::to_usize(&indices.value(i)).ok_or_else(|| {
+            ArrowError::ComputeError("Cast to usize failed".to_string())
+        })?;
+
+        if array.is_null(index) {
+            bit_util::unset_bit(null_slice, i);
         }
+
+        new_values.push(array.value(index));
+
+        i += 1;
     }
-    Ok(Arc::new(builder.finish()) as ArrayRef)
+
+    let nulls = match indices.data_ref().null_buffer() {
+        Some(buffer) => buffer_bin_and(buffer, 0, &null_buf.freeze(), 0, indices.len()),
+        None => null_buf.freeze(),
+    };
+
+    let data = ArrayData::new(
+        T::DATA_TYPE,
+        indices.len(),
+        None,
+        Some(nulls),
+        0,
+        vec![Buffer::from(new_values.to_byte_slice())],
+        vec![],
+    );
+    Ok(Arc::new(PrimitiveArray::<T>::from(Arc::new(data))))
+}
+
+/// `take` implementation for boolean arrays
+fn take_boolean<IndexType>(
+    values: &ArrayRef,
+    indices: &PrimitiveArray<IndexType>,
+) -> Result<ArrayRef>
+where
+    IndexType: ArrowNumericType,
+    IndexType::Native: ToPrimitive,
+{
+    let data_len = indices.len();
+
+    let array = values.as_any().downcast_ref::<BooleanArray>().unwrap();
+
+    let num_byte = bit_util::ceil(data_len, 8);
+    let mut null_buf = MutableBuffer::new(num_byte).with_bitset(num_byte, true);
+    let mut val_buf = MutableBuffer::new(num_byte).with_bitset(num_byte, false);
+
+    let null_slice = null_buf.data_mut();
+    let val_slice = val_buf.data_mut();
+
+    (0..data_len).try_for_each::<_, Result<()>>(|i| {
+        let index = ToPrimitive::to_usize(&indices.value(i)).ok_or_else(|| {
+            ArrowError::ComputeError("Cast to usize failed".to_string())
+        })?;
+
+        if array.is_null(index) {
+            bit_util::unset_bit(null_slice, i);
+        } else if array.value(index) {
+            bit_util::set_bit(val_slice, i);
+        }
+
+        Ok(())
+    })?;
+
+    let nulls = match indices.data_ref().null_buffer() {
+        Some(buffer) => buffer_bin_and(buffer, 0, &null_buf.freeze(), 0, indices.len()),
+        None => null_buf.freeze(),
+    };
+
+    let data = ArrayData::new(
+        DataType::Boolean,
+        indices.len(),
+        None,
+        Some(nulls),
+        0,
+        vec![val_buf.freeze()],
+        vec![],
+    );
+    Ok(Arc::new(BooleanArray::from(Arc::new(data))))
 }
 
 /// `take` implementation for string arrays
-fn take_string(values: &ArrayRef, indices: &UInt32Array) -> Result<ArrayRef> {
-    let mut builder = StringBuilder::new(indices.len());
-    let a = values.as_any().downcast_ref::<StringArray>().unwrap();
-    for i in 0..indices.len() {
-        if indices.is_null(i) {
-            builder.append(false)?;
+fn take_string<OffsetSize, IndexType>(
+    values: &ArrayRef,
+    indices: &PrimitiveArray<IndexType>,
+) -> Result<ArrayRef>
+where
+    OffsetSize: Zero + AddAssign + StringOffsetSizeTrait,
+    IndexType: ArrowNumericType,
+    IndexType::Native: ToPrimitive,
+{
+    let data_len = indices.len();
+
+    let array = values
+        .as_any()
+        .downcast_ref::<GenericStringArray<OffsetSize>>()
+        .unwrap();
+
+    let num_bytes = bit_util::ceil(data_len, 8);
+    let mut null_buf = MutableBuffer::new(num_bytes).with_bitset(num_bytes, true);
+    let null_slice = null_buf.data_mut();
+
+    let mut offsets = Vec::with_capacity(data_len + 1);
+    let mut values = Vec::with_capacity(data_len);
+    let mut length_so_far = OffsetSize::zero();
+
+    offsets.push(length_so_far);
+    for i in 0..data_len {
+        let index = ToPrimitive::to_usize(&indices.value(i)).ok_or_else(|| {
+            ArrowError::ComputeError("Cast to usize failed".to_string())
+        })?;
+
+        if array.is_valid(index) && indices.is_valid(i) {
+            let s = array.value(index);
+
+            length_so_far += OffsetSize::from_usize(s.len()).unwrap();
+            values.extend_from_slice(s.as_bytes());
         } else {
-            let ix = indices.value(i) as usize;
-            if a.is_null(ix) {
-                builder.append(false)?;
-            } else {
-                builder.append_value(a.value(ix))?;
-            }
+            // set null bit
+            bit_util::unset_bit(null_slice, i);
         }
+        offsets.push(length_so_far);
     }
-    Ok(Arc::new(builder.finish()) as ArrayRef)
+
+    let nulls = match indices.data_ref().null_buffer() {
+        Some(buffer) => buffer_bin_and(buffer, 0, &null_buf.freeze(), 0, data_len),
+        None => null_buf.freeze(),
+    };
+
+    let data = ArrayData::builder(<OffsetSize as StringOffsetSizeTrait>::DATA_TYPE)
+        .len(data_len)
+        .null_bit_buffer(nulls)
+        .add_buffer(Buffer::from(offsets.to_byte_slice()))
+        .add_buffer(Buffer::from(&values[..]))
+        .build();
+    Ok(Arc::new(GenericStringArray::<OffsetSize>::from(data)))
 }
 
 /// `take` implementation for list arrays
@@ -198,29 +365,43 @@ fn take_string(values: &ArrayRef, indices: &UInt32Array) -> Result<ArrayRef> {
 /// Calculates the index and indexed offset for the inner array,
 /// applying `take` on the inner array, then reconstructing a list array
 /// with the indexed offsets
-fn take_list(values: &ArrayRef, indices: &UInt32Array) -> Result<ArrayRef> {
+fn take_list<IndexType, OffsetType>(
+    values: &ArrayRef,
+    indices: &PrimitiveArray<IndexType>,
+) -> Result<ArrayRef>
+where
+    IndexType: ArrowNumericType,
+    IndexType::Native: ToPrimitive,
+    OffsetType: ArrowNumericType,
+    OffsetType::Native: ToPrimitive + OffsetSizeTrait,
+    PrimitiveArray<OffsetType>: From<Vec<Option<OffsetType::Native>>>,
+{
     // TODO: Some optimizations can be done here such as if it is
     // taking the whole list or a contiguous sublist
-    let list: &ListArray = values.as_any().downcast_ref::<ListArray>().unwrap();
-    let (list_indices, offsets) = take_value_indices_from_list(values, indices);
-    let taken = take(&list.values(), &list_indices, None)?;
+    let list = values
+        .as_any()
+        .downcast_ref::<GenericListArray<OffsetType::Native>>()
+        .unwrap();
+
+    let (list_indices, offsets) =
+        take_value_indices_from_list::<IndexType, OffsetType>(values, indices)?;
+
+    let taken = take_impl::<OffsetType>(&list.values(), &list_indices, None)?;
     // determine null count and null buffer, which are a function of `values` and `indices`
     let mut null_count = 0;
     let num_bytes = bit_util::ceil(indices.len(), 8);
-    let mut null_buf = MutableBuffer::new(num_bytes).with_bitset(num_bytes, false);
+    let mut null_buf = MutableBuffer::new(num_bytes).with_bitset(num_bytes, true);
     {
         let null_slice = null_buf.data_mut();
-        offsets[..]
-            .windows(2)
-            .enumerate()
-            .for_each(|(i, window): (usize, &[i32])| {
-                if window[0] != window[1] {
-                    // offsets are unequal, slot is not null
-                    bit_util::set_bit(null_slice, i);
-                } else {
+        offsets[..].windows(2).enumerate().for_each(
+            |(i, window): (usize, &[OffsetType::Native])| {
+                if window[0] == window[1] {
+                    // offsets are equal, slot is null
+                    bit_util::unset_bit(null_slice, i);
                     null_count += 1;
                 }
-            });
+            },
+        );
     }
     let value_offsets = Buffer::from(offsets[..].to_byte_slice());
     // create a new list with taken data and computed null information
@@ -232,28 +413,77 @@ fn take_list(values: &ArrayRef, indices: &UInt32Array) -> Result<ArrayRef> {
         .add_child_data(taken.data())
         .add_buffer(value_offsets)
         .build();
-    let list_array = Arc::new(ListArray::from(list_data)) as ArrayRef;
+    let list_array =
+        Arc::new(GenericListArray::<OffsetType::Native>::from(list_data)) as ArrayRef;
     Ok(list_array)
 }
 
+/// `take` implementation for dictionary arrays
+///
+/// applies `take` to the keys of the dictionary array and returns a new dictionary array
+/// with the same dictionary values and reordered keys
+fn take_dict<T, I>(values: &ArrayRef, indices: &PrimitiveArray<I>) -> Result<ArrayRef>
+where
+    T: ArrowPrimitiveType,
+    I: ArrowNumericType,
+    I::Native: ToPrimitive,
+{
+    let dict = values
+        .as_any()
+        .downcast_ref::<DictionaryArray<T>>()
+        .unwrap();
+    let keys: ArrayRef = Arc::new(dict.keys_array());
+    let new_keys = take_primitive::<T, I>(&keys, indices)?;
+    let new_keys_data = new_keys.data_ref();
+
+    let data = Arc::new(ArrayData::new(
+        dict.data_type().clone(),
+        new_keys.len(),
+        Some(new_keys_data.null_count()),
+        new_keys_data.null_buffer().cloned(),
+        0,
+        new_keys_data.buffers().to_vec(),
+        dict.data().child_data().to_vec(),
+    ));
+
+    Ok(Arc::new(DictionaryArray::<T>::from(data)))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
-    fn test_take_primitive_arrays<'a, T>(
+    fn test_take_primitive_arrays<T>(
         data: Vec<Option<T::Native>>,
         index: &UInt32Array,
         options: Option<TakeOptions>,
         expected_data: Vec<Option<T::Native>>,
     ) where
         T: ArrowPrimitiveType,
-        PrimitiveArray<T>: From<Vec<Option<T::Native>>> + ArrayEqual,
+        PrimitiveArray<T>: From<Vec<Option<T::Native>>>,
     {
         let output = PrimitiveArray::<T>::from(data);
-        let expected = PrimitiveArray::<T>::from(expected_data);
+        let expected = Arc::new(PrimitiveArray::<T>::from(expected_data)) as ArrayRef;
         let output = take(&(Arc::new(output) as ArrayRef), index, options).unwrap();
+        assert_eq!(&output, &expected)
+    }
+
+    fn test_take_impl_primitive_arrays<T, I>(
+        data: Vec<Option<T::Native>>,
+        index: &PrimitiveArray<I>,
+        options: Option<TakeOptions>,
+        expected_data: Vec<Option<T::Native>>,
+    ) where
+        T: ArrowPrimitiveType,
+        PrimitiveArray<T>: From<Vec<Option<T::Native>>>,
+        I: ArrowNumericType,
+        I::Native: ToPrimitive,
+    {
+        let output = PrimitiveArray::<T>::from(data);
+        let expected = PrimitiveArray::<T>::from(expected_data);
+        let output = take_impl(&(Arc::new(output) as ArrayRef), index, options).unwrap();
         let output = output.as_any().downcast_ref::<PrimitiveArray<T>>().unwrap();
-        assert!(output.equals(&expected))
+        assert_eq!(output, &expected)
     }
 
     // create a simple struct for testing purposes
@@ -277,6 +507,38 @@ mod tests {
     fn test_take_primitive() {
         let index = UInt32Array::from(vec![Some(3), None, Some(1), Some(3), Some(2)]);
 
+        // int8
+        test_take_primitive_arrays::<Int8Type>(
+            vec![Some(0), None, Some(2), Some(3), None],
+            &index,
+            None,
+            vec![Some(3), None, None, Some(3), Some(2)],
+        );
+
+        // int16
+        test_take_primitive_arrays::<Int16Type>(
+            vec![Some(0), None, Some(2), Some(3), None],
+            &index,
+            None,
+            vec![Some(3), None, None, Some(3), Some(2)],
+        );
+
+        // int32
+        test_take_primitive_arrays::<Int32Type>(
+            vec![Some(0), None, Some(2), Some(3), None],
+            &index,
+            None,
+            vec![Some(3), None, None, Some(3), Some(2)],
+        );
+
+        // int64
+        test_take_primitive_arrays::<Int64Type>(
+            vec![Some(0), None, Some(2), Some(3), None],
+            &index,
+            None,
+            vec![Some(3), None, None, Some(3), Some(2)],
+        );
+
         // uint8
         test_take_primitive_arrays::<UInt8Type>(
             vec![Some(0), None, Some(2), Some(3), None],
@@ -372,7 +634,85 @@ mod tests {
             None,
             vec![Some(-3.1), None, None, Some(-3.1), Some(2.21)],
         );
+    }
+
+    #[test]
+    fn test_take_impl_primitive_with_int64_indices() {
+        let index = Int64Array::from(vec![Some(3), None, Some(1), Some(3), Some(2)]);
 
+        // int16
+        test_take_impl_primitive_arrays::<Int16Type, Int64Type>(
+            vec![Some(0), None, Some(2), Some(3), None],
+            &index,
+            None,
+            vec![Some(3), None, None, Some(3), Some(2)],
+        );
+
+        // int64
+        test_take_impl_primitive_arrays::<Int64Type, Int64Type>(
+            vec![Some(0), None, Some(2), Some(-15), None],
+            &index,
+            None,
+            vec![Some(-15), None, None, Some(-15), Some(2)],
+        );
+
+        // uint64
+        test_take_impl_primitive_arrays::<UInt64Type, Int64Type>(
+            vec![Some(0), None, Some(2), Some(3), None],
+            &index,
+            None,
+            vec![Some(3), None, None, Some(3), Some(2)],
+        );
+
+        // duration_millisecond
+        test_take_impl_primitive_arrays::<DurationMillisecondType, Int64Type>(
+            vec![Some(0), None, Some(2), Some(-15), None],
+            &index,
+            None,
+            vec![Some(-15), None, None, Some(-15), Some(2)],
+        );
+
+        // float32
+        test_take_impl_primitive_arrays::<Float32Type, Int64Type>(
+            vec![Some(0.0), None, Some(2.21), Some(-3.1), None],
+            &index,
+            None,
+            vec![Some(-3.1), None, None, Some(-3.1), Some(2.21)],
+        );
+    }
+
+    #[test]
+    fn test_take_impl_primitive_with_uint8_indices() {
+        let index = UInt8Array::from(vec![Some(3), None, Some(1), Some(3), Some(2)]);
+
+        // int16
+        test_take_impl_primitive_arrays::<Int16Type, UInt8Type>(
+            vec![Some(0), None, Some(2), Some(3), None],
+            &index,
+            None,
+            vec![Some(3), None, None, Some(3), Some(2)],
+        );
+
+        // duration_millisecond
+        test_take_impl_primitive_arrays::<DurationMillisecondType, UInt8Type>(
+            vec![Some(0), None, Some(2), Some(-15), None],
+            &index,
+            None,
+            vec![Some(-15), None, None, Some(-15), Some(2)],
+        );
+
+        // float32
+        test_take_impl_primitive_arrays::<Float32Type, UInt8Type>(
+            vec![Some(0.0), None, Some(2.21), Some(-3.1), None],
+            &index,
+            None,
+            vec![Some(-3.1), None, None, Some(-3.1), Some(2.21)],
+        );
+    }
+
+    #[test]
+    fn test_take_primitive_bool() {
+        let index = UInt32Array::from(vec![Some(3), None, Some(1), Some(3), Some(2)]);
         // boolean
         test_take_primitive_arrays::<BooleanType>(
             vec![Some(false), None, Some(true), Some(false), None],
@@ -382,207 +722,304 @@ mod tests {
         );
     }
 
+    fn _test_take_string<'a, K: 'static>()
+    where
+        K: Array + PartialEq + From<Vec<Option<&'a str>>>,
+    {
+        let index = UInt32Array::from(vec![Some(3), None, Some(1), Some(3), Some(4)]);
+
+        let array = K::from(vec![
+            Some("one"),
+            None,
+            Some("three"),
+            Some("four"),
+            Some("five"),
+        ]);
+        let array = Arc::new(array) as ArrayRef;
+
+        let actual = take(&array, &index, None).unwrap();
+        assert_eq!(actual.len(), index.len());
+
+        let actual = actual.as_any().downcast_ref::<K>().unwrap();
+
+        let expected =
+            K::from(vec![Some("four"), None, None, Some("four"), Some("five")]);
+
+        assert_eq!(actual, &expected);
+    }
+
     #[test]
     fn test_take_string() {
-        let index = UInt32Array::from(vec![Some(3), None, Some(1), Some(3), Some(4)]);
-        let mut builder: StringBuilder = StringBuilder::new(6);
-        builder.append_value("one").unwrap();
-        builder.append_null().unwrap();
-        builder.append_value("three").unwrap();
-        builder.append_value("four").unwrap();
-        builder.append_value("five").unwrap();
-        let array = Arc::new(builder.finish()) as ArrayRef;
-        let a = take(&array, &index, None).unwrap();
-        assert_eq!(a.len(), index.len());
-        builder.append_value("four").unwrap();
-        builder.append_null().unwrap();
-        builder.append_null().unwrap();
-        builder.append_value("four").unwrap();
-        builder.append_value("five").unwrap();
-        let b = builder.finish();
-        assert_eq!(a.data(), b.data());
+        _test_take_string::<StringArray>()
     }
 
     #[test]
-    fn test_take_list() {
-        // Construct a value array, [[0,0,0], [-1,-2,-1], [2,3]]
-        let value_data = Int32Array::from(vec![0, 0, 0, -1, -2, -1, 2, 3]).data();
-        // Construct offsets
-        let value_offsets = Buffer::from(&[0, 3, 6, 8].to_byte_slice());
-        // Construct a list array from the above two
-        let list_data_type = DataType::List(Box::new(DataType::Int32));
-        let list_data = ArrayData::builder(list_data_type.clone())
-            .len(3)
-            .add_buffer(value_offsets)
-            .add_child_data(value_data)
-            .build();
-        let list_array = Arc::new(ListArray::from(list_data)) as ArrayRef;
+    fn test_take_large_string() {
+        _test_take_string::<LargeStringArray>()
+    }
 
-        // index returns: [[2,3], null, [-1,-2,-1], [2,3], [0,0,0]]
-        let index = UInt32Array::from(vec![Some(2), None, Some(1), Some(2), Some(0)]);
+    macro_rules! test_take_list {
+        ($offset_type:ty, $list_data_type:ident, $list_array_type:ident) => {{
+            // Construct a value array, [[0,0,0], [-1,-2,-1], [2,3]]
+            let value_data = Int32Array::from(vec![0, 0, 0, -1, -2, -1, 2, 3]).data();
+            // Construct offsets
+            let value_offsets: [$offset_type; 4] = [0, 3, 6, 8];
+            let value_offsets = Buffer::from(&value_offsets.to_byte_slice());
+            // Construct a list array from the above two
+            let list_data_type = DataType::$list_data_type(Box::new(Field::new(
+                "item",
+                DataType::Int32,
+                false,
+            )));
+            let list_data = ArrayData::builder(list_data_type.clone())
+                .len(3)
+                .add_buffer(value_offsets)
+                .add_child_data(value_data)
+                .build();
+            let list_array = Arc::new($list_array_type::from(list_data)) as ArrayRef;
 
-        let a = take(&list_array, &index, None).unwrap();
-        let a: &ListArray = a.as_any().downcast_ref::<ListArray>().unwrap();
+            // index returns: [[2,3], null, [-1,-2,-1], [2,3], [0,0,0]]
+            let index = UInt32Array::from(vec![Some(2), None, Some(1), Some(2), Some(0)]);
 
-        // construct a value array with expected results:
-        // [[2,3], null, [-1,-2,-1], [2,3], [0,0,0]]
-        let expected_data = Int32Array::from(vec![
-            Some(2),
-            Some(3),
-            Some(-1),
-            Some(-2),
-            Some(-1),
-            Some(2),
-            Some(3),
-            Some(0),
-            Some(0),
-            Some(0),
-        ])
-        .data();
-        // construct offsets
-        let expected_offsets = Buffer::from(&[0, 2, 2, 5, 7, 10].to_byte_slice());
-        // construct list array from the two
-        let expected_list_data = ArrayData::builder(list_data_type.clone())
-            .len(5)
-            .null_count(1)
-            // null buffer remains the same as only the indices have nulls
-            .null_bit_buffer(index.data().null_bitmap().as_ref().unwrap().bits.clone())
-            .add_buffer(expected_offsets)
-            .add_child_data(expected_data)
-            .build();
-        let expected_list_array = ListArray::from(expected_list_data);
+            let a = take(&list_array, &index, None).unwrap();
+            let a: &$list_array_type =
+                a.as_any().downcast_ref::<$list_array_type>().unwrap();
+
+            // construct a value array with expected results:
+            // [[2,3], null, [-1,-2,-1], [2,3], [0,0,0]]
+            let expected_data = Int32Array::from(vec![
+                Some(2),
+                Some(3),
+                Some(-1),
+                Some(-2),
+                Some(-1),
+                Some(2),
+                Some(3),
+                Some(0),
+                Some(0),
+                Some(0),
+            ])
+            .data();
+            // construct offsets
+            let expected_offsets: [$offset_type; 6] = [0, 2, 2, 5, 7, 10];
+            let expected_offsets = Buffer::from(&expected_offsets.to_byte_slice());
+            // construct list array from the two
+            let expected_list_data = ArrayData::builder(list_data_type)
+                .len(5)
+                .null_count(1)
+                // null buffer remains the same as only the indices have nulls
+                .null_bit_buffer(
+                    index.data().null_bitmap().as_ref().unwrap().bits.clone(),
+                )
+                .add_buffer(expected_offsets)
+                .add_child_data(expected_data)
+                .build();
+            let expected_list_array = $list_array_type::from(expected_list_data);
+
+            assert_eq!(a, &expected_list_array);
+        }};
+    }
+
+    macro_rules! test_take_list_with_value_nulls {
+        ($offset_type:ty, $list_data_type:ident, $list_array_type:ident) => {{
+            // Construct a value array, [[0,null,0], [-1,-2,3], [null], [5,null]]
+            let value_data = Int32Array::from(vec![
+                Some(0),
+                None,
+                Some(0),
+                Some(-1),
+                Some(-2),
+                Some(3),
+                None,
+                Some(5),
+                None,
+            ])
+            .data();
+            // Construct offsets
+            let value_offsets: [$offset_type; 5] = [0, 3, 6, 7, 9];
+            let value_offsets = Buffer::from(&value_offsets.to_byte_slice());
+            // Construct a list array from the above two
+            let list_data_type = DataType::$list_data_type(Box::new(Field::new(
+                "item",
+                DataType::Int32,
+                false,
+            )));
+            let list_data = ArrayData::builder(list_data_type.clone())
+                .len(4)
+                .add_buffer(value_offsets)
+                .null_count(0)
+                .null_bit_buffer(Buffer::from([0b10111101, 0b00000000]))
+                .add_child_data(value_data)
+                .build();
+            let list_array = Arc::new($list_array_type::from(list_data)) as ArrayRef;
+
+            // index returns: [[null], null, [-1,-2,3], [2,null], [0,null,0]]
+            let index = UInt32Array::from(vec![Some(2), None, Some(1), Some(3), Some(0)]);
+
+            let a = take(&list_array, &index, None).unwrap();
+            let a: &$list_array_type =
+                a.as_any().downcast_ref::<$list_array_type>().unwrap();
+
+            // construct a value array with expected results:
+            // [[null], null, [-1,-2,3], [5,null], [0,null,0]]
+            let expected_data = Int32Array::from(vec![
+                None,
+                Some(-1),
+                Some(-2),
+                Some(3),
+                Some(5),
+                None,
+                Some(0),
+                None,
+                Some(0),
+            ])
+            .data();
+            // construct offsets
+            let expected_offsets: [$offset_type; 6] = [0, 1, 1, 4, 6, 9];
+            let expected_offsets = Buffer::from(&expected_offsets.to_byte_slice());
+            // construct list array from the two
+            let expected_list_data = ArrayData::builder(list_data_type)
+                .len(5)
+                .null_count(1)
+                // null buffer remains the same as only the indices have nulls
+                .null_bit_buffer(
+                    index.data().null_bitmap().as_ref().unwrap().bits.clone(),
+                )
+                .add_buffer(expected_offsets)
+                .add_child_data(expected_data)
+                .build();
+            let expected_list_array = $list_array_type::from(expected_list_data);
+
+            assert_eq!(a, &expected_list_array);
+        }};
+    }
+
+    macro_rules! test_take_list_with_nulls {
+        ($offset_type:ty, $list_data_type:ident, $list_array_type:ident) => {{
+            // Construct a value array, [[0,null,0], [-1,-2,3], null, [5,null]]
+            let value_data = Int32Array::from(vec![
+                Some(0),
+                None,
+                Some(0),
+                Some(-1),
+                Some(-2),
+                Some(3),
+                Some(5),
+                None,
+            ])
+            .data();
+            // Construct offsets
+            let value_offsets: [$offset_type; 5] = [0, 3, 6, 6, 8];
+            let value_offsets = Buffer::from(&value_offsets.to_byte_slice());
+            // Construct a list array from the above two
+            let list_data_type = DataType::$list_data_type(Box::new(Field::new(
+                "item",
+                DataType::Int32,
+                false,
+            )));
+            let list_data = ArrayData::builder(list_data_type.clone())
+                .len(4)
+                .add_buffer(value_offsets)
+                .null_count(1)
+                .null_bit_buffer(Buffer::from([0b01111101]))
+                .add_child_data(value_data)
+                .build();
+            let list_array = Arc::new($list_array_type::from(list_data)) as ArrayRef;
+
+            // index returns: [null, null, [-1,-2,3], [5,null], [0,null,0]]
+            let index = UInt32Array::from(vec![Some(2), None, Some(1), Some(3), Some(0)]);
+
+            let a = take(&list_array, &index, None).unwrap();
+            let a: &$list_array_type =
+                a.as_any().downcast_ref::<$list_array_type>().unwrap();
+
+            // construct a value array with expected results:
+            // [null, null, [-1,-2,3], [5,null], [0,null,0]]
+            let expected_data = Int32Array::from(vec![
+                Some(-1),
+                Some(-2),
+                Some(3),
+                Some(5),
+                None,
+                Some(0),
+                None,
+                Some(0),
+            ])
+            .data();
+            // construct offsets
+            let expected_offsets: [$offset_type; 6] = [0, 0, 0, 3, 5, 8];
+            let expected_offsets = Buffer::from(&expected_offsets.to_byte_slice());
+            // construct list array from the two
+            let mut null_bits: [u8; 1] = [0; 1];
+            bit_util::set_bit(&mut null_bits, 2);
+            bit_util::set_bit(&mut null_bits, 3);
+            bit_util::set_bit(&mut null_bits, 4);
+            let expected_list_data = ArrayData::builder(list_data_type)
+                .len(5)
+                .null_count(2)
+                // null buffer must be recalculated as both values and indices have nulls
+                .null_bit_buffer(Buffer::from(null_bits))
+                .add_buffer(expected_offsets)
+                .add_child_data(expected_data)
+                .build();
+            let expected_list_array = $list_array_type::from(expected_list_data);
 
-        assert!(a.equals(&expected_list_array));
+            assert_eq!(a, &expected_list_array);
+        }};
     }
 
     #[test]
-    fn test_take_list_with_value_nulls() {
-        // Construct a value array, [[0,null,0], [-1,-2,3], [null], [5,null]]
-        let value_data = Int32Array::from(vec![
-            Some(0),
-            None,
-            Some(0),
-            Some(-1),
-            Some(-2),
-            Some(3),
-            None,
-            Some(5),
-            None,
-        ])
-        .data();
-        // Construct offsets
-        let value_offsets = Buffer::from(&[0, 3, 6, 7, 9].to_byte_slice());
-        // Construct a list array from the above two
-        let list_data_type = DataType::List(Box::new(DataType::Int32));
-        let list_data = ArrayData::builder(list_data_type.clone())
-            .len(4)
-            .add_buffer(value_offsets)
-            .null_count(0)
-            .null_bit_buffer(Buffer::from([0b10111101, 0b00000000]))
-            .add_child_data(value_data)
-            .build();
-        let list_array = Arc::new(ListArray::from(list_data)) as ArrayRef;
+    fn test_take_list() {
+        test_take_list!(i32, List, ListArray);
+    }
 
-        // index returns: [[null], null, [-1,-2,3], [2,null], [0,null,0]]
-        let index = UInt32Array::from(vec![Some(2), None, Some(1), Some(3), Some(0)]);
+    #[test]
+    fn test_take_large_list() {
+        test_take_list!(i64, LargeList, LargeListArray);
+    }
 
-        let a = take(&list_array, &index, None).unwrap();
-        let a: &ListArray = a.as_any().downcast_ref::<ListArray>().unwrap();
+    #[test]
+    fn test_take_list_with_value_nulls() {
+        test_take_list_with_value_nulls!(i32, List, ListArray);
+    }
 
-        // construct a value array with expected results:
-        // [[null], null, [-1,-2,3], [5,null], [0,null,0]]
-        let expected_data = Int32Array::from(vec![
-            None,
-            Some(-1),
-            Some(-2),
-            Some(3),
-            Some(5),
-            None,
-            Some(0),
-            None,
-            Some(0),
-        ])
-        .data();
-        // construct offsets
-        let expected_offsets = Buffer::from(&[0, 1, 1, 4, 6, 9].to_byte_slice());
-        // construct list array from the two
-        let expected_list_data = ArrayData::builder(list_data_type.clone())
-            .len(5)
-            .null_count(1)
-            // null buffer remains the same as only the indices have nulls
-            .null_bit_buffer(index.data().null_bitmap().as_ref().unwrap().bits.clone())
-            .add_buffer(expected_offsets)
-            .add_child_data(expected_data)
-            .build();
-        let expected_list_array = ListArray::from(expected_list_data);
+    #[test]
+    fn test_take_large_list_with_value_nulls() {
+        test_take_list_with_value_nulls!(i64, LargeList, LargeListArray);
+    }
 
-        assert!(a.equals(&expected_list_array));
+    #[test]
+    fn test_test_take_list_with_nulls() {
+        test_take_list_with_nulls!(i32, List, ListArray);
     }
 
     #[test]
-    fn test_take_list_with_list_nulls() {
-        // Construct a value array, [[0,null,0], [-1,-2,3], null, [5,null]]
-        let value_data = Int32Array::from(vec![
-            Some(0),
-            None,
-            Some(0),
-            Some(-1),
-            Some(-2),
-            Some(3),
-            Some(5),
-            None,
-        ])
-        .data();
+    fn test_test_take_large_list_with_nulls() {
+        test_take_list_with_nulls!(i64, LargeList, LargeListArray);
+    }
+
+    #[test]
+    #[should_panic(expected = "index out of bounds: the len is 4 but the index is 1000")]
+    fn test_take_list_out_of_bounds() {
+        // Construct a value array, [[0,0,0], [-1,-2,-1], [2,3]]
+        let value_data = Int32Array::from(vec![0, 0, 0, -1, -2, -1, 2, 3]).data();
         // Construct offsets
-        let value_offsets = Buffer::from(&[0, 3, 6, 6, 8].to_byte_slice());
+        let value_offsets = Buffer::from(&[0, 3, 6, 8].to_byte_slice());
         // Construct a list array from the above two
-        let list_data_type = DataType::List(Box::new(DataType::Int32));
+        let list_data_type =
+            DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
         let list_data = ArrayData::builder(list_data_type.clone())
-            .len(4)
+            .len(3)
             .add_buffer(value_offsets)
-            .null_count(1)
-            .null_bit_buffer(Buffer::from([0b01111101]))
             .add_child_data(value_data)
             .build();
         let list_array = Arc::new(ListArray::from(list_data)) as ArrayRef;
 
-        // index returns: [null, null, [-1,-2,3], [5,null], [0,null,0]]
-        let index = UInt32Array::from(vec![Some(2), None, Some(1), Some(3), Some(0)]);
-
-        let a = take(&list_array, &index, None).unwrap();
-        let a: &ListArray = a.as_any().downcast_ref::<ListArray>().unwrap();
-
-        // construct a value array with expected results:
-        // [null, null, [-1,-2,3], [5,null], [0,null,0]]
-        let expected_data = Int32Array::from(vec![
-            Some(-1),
-            Some(-2),
-            Some(3),
-            Some(5),
-            None,
-            Some(0),
-            None,
-            Some(0),
-        ])
-        .data();
-        // construct offsets
-        let expected_offsets = Buffer::from(&[0, 0, 0, 3, 5, 8].to_byte_slice());
-        // construct list array from the two
-        let mut null_bits: [u8; 1] = [0; 1];
-        bit_util::set_bit(&mut null_bits, 2);
-        bit_util::set_bit(&mut null_bits, 3);
-        bit_util::set_bit(&mut null_bits, 4);
-        let expected_list_data = ArrayData::builder(list_data_type.clone())
-            .len(5)
-            .null_count(2)
-            // null buffer must be recalculated as both values and indices have nulls
-            .null_bit_buffer(Buffer::from(null_bits))
-            .add_buffer(expected_offsets)
-            .add_child_data(expected_data)
-            .build();
-        let expected_list_array = ListArray::from(expected_list_data);
+        let index = UInt32Array::from(vec![1000]);
 
-        assert!(a.equals(&expected_list_array));
+        // A panic is expected here since we have not supplied the check_bounds
+        // option.
+        take(&list_array, &index, None).unwrap();
     }
 
     #[test]
@@ -608,7 +1045,8 @@ mod tests {
             .add_child_data(expected_int_data)
             .build();
         let struct_array = StructArray::from(struct_array_data);
-        assert!(a.equals(&struct_array));
+
+        assert_eq!(a, &struct_array);
     }
 
     #[test]
@@ -638,7 +1076,7 @@ mod tests {
             .add_child_data(expected_int_data)
             .build();
         let struct_array = StructArray::from(struct_array_data);
-        assert!(a.equals(&struct_array));
+        assert_eq!(a, &struct_array);
     }
 
     #[test]
@@ -657,4 +1095,60 @@ mod tests {
             vec![None],
         );
     }
+
+    #[test]
+    fn test_take_dict() {
+        let keys_builder = Int16Builder::new(8);
+        let values_builder = StringBuilder::new(4);
+
+        let mut dict_builder = StringDictionaryBuilder::new(keys_builder, values_builder);
+
+        dict_builder.append("foo").unwrap();
+        dict_builder.append("bar").unwrap();
+        dict_builder.append("").unwrap();
+        dict_builder.append_null().unwrap();
+        dict_builder.append("foo").unwrap();
+        dict_builder.append("bar").unwrap();
+        dict_builder.append("bar").unwrap();
+        dict_builder.append("foo").unwrap();
+
+        let array = dict_builder.finish();
+        let dict_values = array.values().clone();
+        let dict_values = dict_values.as_any().downcast_ref::<StringArray>().unwrap();
+        let array: Arc<dyn Array> = Arc::new(array);
+
+        let indices = UInt32Array::from(vec![
+            Some(0), // first "foo"
+            Some(7), // last "foo"
+            None,    // null index should return null
+            Some(5), // second "bar"
+            Some(6), // another "bar"
+            Some(2), // empty string
+            Some(3), // input is null at this index
+        ]);
+
+        let result = take(&array, &indices, None).unwrap();
+        let result = result
+            .as_any()
+            .downcast_ref::<DictionaryArray<Int16Type>>()
+            .unwrap();
+
+        let result_values: StringArray = result.values().data().into();
+
+        // dictionary values should stay the same
+        let expected_values = StringArray::from(vec!["foo", "bar", ""]);
+        assert_eq!(&expected_values, dict_values);
+        assert_eq!(&expected_values, &result_values);
+
+        let expected_keys = Int16Array::from(vec![
+            Some(0),
+            Some(0),
+            None,
+            Some(1),
+            Some(1),
+            Some(2),
+            None,
+        ]);
+        assert_eq!(result.keys(), &expected_keys);
+    }
 }
diff --git a/rust/arrow/src/compute/util.rs b/rust/arrow/src/compute/util.rs
index 05e7490f206..3b578011640 100644
--- a/rust/arrow/src/compute/util.rs
+++ b/rust/arrow/src/compute/util.rs
@@ -18,35 +18,78 @@
 //! Common utilities for computation kernels.
 
 use crate::array::*;
-use crate::bitmap::Bitmap;
-use crate::buffer::Buffer;
 #[cfg(feature = "simd")]
+use crate::bitmap::Bitmap;
+use crate::buffer::{buffer_bin_and, buffer_bin_or, Buffer};
 use crate::datatypes::*;
-use crate::error::Result;
-#[cfg(feature = "simd")]
-use num::One;
+use crate::error::{ArrowError, Result};
+use num::{One, ToPrimitive, Zero};
 #[cfg(feature = "simd")]
 use std::cmp::min;
+use std::ops::Add;
 
-/// Applies a given binary operation, `op`, to two references to `Option<Bitmap>`'s.
+/// Combines the null bitmaps of two arrays using a bitwise `and` operation.
 ///
 /// This function is useful when implementing operations on higher level arrays.
-pub(super) fn apply_bin_op_to_option_bitmap<F>(
-    left: &Option<Bitmap>,
-    right: &Option<Bitmap>,
-    op: F,
-) -> Result<Option<Buffer>>
-where
-    F: Fn(&Buffer, &Buffer) -> Result<Buffer>,
-{
-    match *left {
-        None => match *right {
+pub(super) fn combine_option_bitmap(
+    left_data: &ArrayDataRef,
+    right_data: &ArrayDataRef,
+    len_in_bits: usize,
+) -> Result<Option<Buffer>> {
+    let left_offset_in_bits = left_data.offset();
+    let right_offset_in_bits = right_data.offset();
+
+    let left = left_data.null_buffer();
+    let right = right_data.null_buffer();
+
+    match left {
+        None => match right {
+            None => Ok(None),
+            Some(r) => Ok(Some(r.bit_slice(right_offset_in_bits, len_in_bits))),
+        },
+        Some(l) => match right {
+            None => Ok(Some(l.bit_slice(left_offset_in_bits, len_in_bits))),
+
+            Some(r) => Ok(Some(buffer_bin_and(
+                &l,
+                left_offset_in_bits,
+                &r,
+                right_offset_in_bits,
+                len_in_bits,
+            ))),
+        },
+    }
+}
+
+/// Compares the null bitmaps of two arrays using a bitwise `or` operation.
+///
+/// This function is useful when implementing operations on higher level arrays.
+pub(super) fn compare_option_bitmap(
+    left_data: &ArrayDataRef,
+    right_data: &ArrayDataRef,
+    len_in_bits: usize,
+) -> Result<Option<Buffer>> {
+    let left_offset_in_bits = left_data.offset();
+    let right_offset_in_bits = right_data.offset();
+
+    let left = left_data.null_buffer();
+    let right = right_data.null_buffer();
+
+    match left {
+        None => match right {
             None => Ok(None),
-            Some(ref r) => Ok(Some(r.bits.clone())),
+            Some(r) => Ok(Some(r.bit_slice(right_offset_in_bits, len_in_bits))),
         },
-        Some(ref l) => match *right {
-            None => Ok(Some(l.bits.clone())),
-            Some(ref r) => Ok(Some(op(&l.bits, &r.bits)?)),
+        Some(l) => match right {
+            None => Ok(Some(l.bit_slice(left_offset_in_bits, len_in_bits))),
+
+            Some(r) => Ok(Some(buffer_bin_or(
+                &l,
+                left_offset_in_bits,
+                &r,
+                right_offset_in_bits,
+                len_in_bits,
+            ))),
         },
     }
 }
@@ -56,41 +99,55 @@ where
 /// Where a list array has indices `[0,2,5,10]`, taking indices of `[2,0]` returns
 /// an array of the indices `[5..10, 0..2]` and offsets `[0,5,7]` (5 elements and 2
 /// elements)
-pub(super) fn take_value_indices_from_list(
+pub(super) fn take_value_indices_from_list<IndexType, OffsetType>(
     values: &ArrayRef,
-    indices: &UInt32Array,
-) -> (UInt32Array, Vec<i32>) {
+    indices: &PrimitiveArray<IndexType>,
+) -> Result<(PrimitiveArray<OffsetType>, Vec<OffsetType::Native>)>
+where
+    IndexType: ArrowNumericType,
+    IndexType::Native: ToPrimitive,
+    OffsetType: ArrowNumericType,
+    OffsetType::Native: OffsetSizeTrait + Add + Zero + One,
+    PrimitiveArray<OffsetType>: From<Vec<Option<OffsetType::Native>>>,
+{
     // TODO: benchmark this function, there might be a faster unsafe alternative
     // get list array's offsets
-    let list: &ListArray = values.as_any().downcast_ref::<ListArray>().unwrap();
-    let offsets: Vec<u32> = (0..=list.len())
-        .map(|i| list.value_offset(i) as u32)
-        .collect();
+    let list = values
+        .as_any()
+        .downcast_ref::<GenericListArray<OffsetType::Native>>()
+        .unwrap();
+    let offsets: Vec<OffsetType::Native> =
+        (0..=list.len()).map(|i| list.value_offset(i)).collect();
+
     let mut new_offsets = Vec::with_capacity(indices.len());
     let mut values = Vec::new();
-    let mut current_offset = 0;
+    let mut current_offset = OffsetType::Native::zero();
     // add first offset
-    new_offsets.push(0);
+    new_offsets.push(OffsetType::Native::zero());
     // compute the value indices, and set offsets accordingly
     for i in 0..indices.len() {
         if indices.is_valid(i) {
-            let ix = indices.value(i) as usize;
+            let ix = ToPrimitive::to_usize(&indices.value(i)).ok_or_else(|| {
+                ArrowError::ComputeError("Cast to usize failed".to_string())
+            })?;
             let start = offsets[ix];
             let end = offsets[ix + 1];
-            current_offset += (end - start) as i32;
+            current_offset = current_offset + (end - start);
             new_offsets.push(current_offset);
+
+            let mut curr = start;
+
             // if start == end, this slot is empty
-            if start != end {
-                // type annotation needed to guide compiler a bit
-                let mut offsets: Vec<Option<u32>> =
-                    (start..end).map(Some).collect::<Vec<Option<u32>>>();
-                values.append(&mut offsets);
+            while curr < end {
+                values.push(Some(curr));
+                curr = curr + OffsetType::Native::one();
             }
         } else {
             new_offsets.push(current_offset);
         }
     }
-    (UInt32Array::from(values), new_offsets)
+
+    Ok((PrimitiveArray::<OffsetType>::from(values), new_offsets))
 }
 
 /// Creates a new SIMD mask, i.e. `packed_simd::m32x16` or similar. that indicates if the
@@ -166,66 +223,131 @@ mod tests {
     use crate::array::ArrayData;
     use crate::datatypes::{DataType, ToByteSlice};
 
+    fn make_data_with_null_bit_buffer(
+        len: usize,
+        offset: usize,
+        null_bit_buffer: Option<Buffer>,
+    ) -> Arc<ArrayData> {
+        // empty vec for buffers and children is not really correct, but for these tests we only care about the null bitmap
+        Arc::new(ArrayData::new(
+            DataType::UInt8,
+            len,
+            None,
+            null_bit_buffer,
+            offset,
+            vec![],
+            vec![],
+        ))
+    }
+
+    #[test]
+    fn test_combine_option_bitmap() {
+        let none_bitmap = make_data_with_null_bit_buffer(8, 0, None);
+        let some_bitmap =
+            make_data_with_null_bit_buffer(8, 0, Some(Buffer::from([0b01001010])));
+        let inverse_bitmap =
+            make_data_with_null_bit_buffer(8, 0, Some(Buffer::from([0b10110101])));
+        assert_eq!(
+            None,
+            combine_option_bitmap(&none_bitmap, &none_bitmap, 8).unwrap()
+        );
+        assert_eq!(
+            Some(Buffer::from([0b01001010])),
+            combine_option_bitmap(&some_bitmap, &none_bitmap, 8).unwrap()
+        );
+        assert_eq!(
+            Some(Buffer::from([0b01001010])),
+            combine_option_bitmap(&none_bitmap, &some_bitmap, 8,).unwrap()
+        );
+        assert_eq!(
+            Some(Buffer::from([0b01001010])),
+            combine_option_bitmap(&some_bitmap, &some_bitmap, 8,).unwrap()
+        );
+        assert_eq!(
+            Some(Buffer::from([0b0])),
+            combine_option_bitmap(&some_bitmap, &inverse_bitmap, 8,).unwrap()
+        );
+    }
+
     #[test]
-    fn test_apply_bin_op_to_option_bitmap() {
+    fn test_compare_option_bitmap() {
+        let none_bitmap = make_data_with_null_bit_buffer(8, 0, None);
+        let some_bitmap =
+            make_data_with_null_bit_buffer(8, 0, Some(Buffer::from([0b01001010])));
+        let inverse_bitmap =
+            make_data_with_null_bit_buffer(8, 0, Some(Buffer::from([0b10110101])));
         assert_eq!(
             None,
-            apply_bin_op_to_option_bitmap(&None, &None, |a, b| a & b).unwrap()
+            compare_option_bitmap(&none_bitmap, &none_bitmap, 8).unwrap()
         );
         assert_eq!(
-            Some(Buffer::from([0b01101010])),
-            apply_bin_op_to_option_bitmap(
-                &Some(Bitmap::from(Buffer::from([0b01101010]))),
-                &None,
-                |a, b| a & b,
-            )
-            .unwrap()
+            Some(Buffer::from([0b01001010])),
+            compare_option_bitmap(&some_bitmap, &none_bitmap, 8).unwrap()
         );
         assert_eq!(
-            Some(Buffer::from([0b01001110])),
-            apply_bin_op_to_option_bitmap(
-                &None,
-                &Some(Bitmap::from(Buffer::from([0b01001110]))),
-                |a, b| a & b,
-            )
-            .unwrap()
+            Some(Buffer::from([0b01001010])),
+            compare_option_bitmap(&none_bitmap, &some_bitmap, 8,).unwrap()
         );
         assert_eq!(
             Some(Buffer::from([0b01001010])),
-            apply_bin_op_to_option_bitmap(
-                &Some(Bitmap::from(Buffer::from([0b01101010]))),
-                &Some(Bitmap::from(Buffer::from([0b01001110]))),
-                |a, b| a & b,
-            )
-            .unwrap()
+            compare_option_bitmap(&some_bitmap, &some_bitmap, 8,).unwrap()
+        );
+        assert_eq!(
+            Some(Buffer::from([0b11111111])),
+            compare_option_bitmap(&some_bitmap, &inverse_bitmap, 8,).unwrap()
         );
     }
 
+    fn build_list<P, S>(
+        list_data_type: DataType,
+        values: PrimitiveArray<P>,
+        offsets: Vec<S>,
+    ) -> ArrayRef
+    where
+        P: ArrowPrimitiveType,
+        S: OffsetSizeTrait,
+    {
+        let value_data = values.data();
+        let value_offsets = Buffer::from(&offsets[..].to_byte_slice());
+        let list_data = ArrayData::builder(list_data_type)
+            .len(offsets.len() - 1)
+            .add_buffer(value_offsets)
+            .add_child_data(value_data)
+            .build();
+        let array = Arc::new(GenericListArray::<S>::from(list_data)) as ArrayRef;
+        array
+    }
+
     #[test]
     fn test_take_value_index_from_list() {
-        let value_data = Int32Array::from((0..10).collect::<Vec<i32>>()).data();
-        let value_offsets = Buffer::from(&[0, 2, 5, 10].to_byte_slice());
-        let list_data_type = DataType::List(Box::new(DataType::Int32));
-        let list_data = ArrayData::builder(list_data_type.clone())
-            .len(3)
-            .add_buffer(value_offsets.clone())
-            .add_child_data(value_data.clone())
-            .build();
-        let array = Arc::new(ListArray::from(list_data)) as ArrayRef;
-        let index = UInt32Array::from(vec![2, 0]);
-        let (indexed, offsets) = take_value_indices_from_list(&array, &index);
-        assert_eq!(vec![0, 5, 7], offsets);
-        let data = UInt32Array::from(vec![
-            Some(5),
-            Some(6),
-            Some(7),
-            Some(8),
-            Some(9),
-            Some(0),
-            Some(1),
-        ])
-        .data();
-        assert_eq!(data, indexed.data());
+        let list = build_list(
+            DataType::List(Box::new(Field::new("item", DataType::Int32, true))),
+            Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]),
+            vec![0i32, 2i32, 5i32, 10i32],
+        );
+        let indices = UInt32Array::from(vec![2, 0]);
+
+        let (indexed, offsets) =
+            take_value_indices_from_list::<_, Int32Type>(&list, &indices).unwrap();
+
+        assert_eq!(indexed, Int32Array::from(vec![5, 6, 7, 8, 9, 0, 1]));
+        assert_eq!(offsets, vec![0, 5, 7]);
+    }
+
+    #[test]
+    fn test_take_value_index_from_large_list() {
+        let list = build_list(
+            DataType::LargeList(Box::new(Field::new("item", DataType::Int32, false))),
+            Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]),
+            vec![0i64, 2i64, 5i64, 10i64],
+        );
+        let indices = UInt32Array::from(vec![2, 0]);
+
+        let (indexed, offsets) =
+            take_value_indices_from_list::<_, Int64Type>(&list, &indices).unwrap();
+
+        assert_eq!(indexed, Int64Array::from(vec![5, 6, 7, 8, 9, 0, 1]));
+        assert_eq!(offsets, vec![0, 5, 7]);
     }
 
     #[test]
diff --git a/rust/arrow/src/csv/reader.rs b/rust/arrow/src/csv/reader.rs
index 9822788e064..4f926f9a98f 100644
--- a/rust/arrow/src/csv/reader.rs
+++ b/rust/arrow/src/csv/reader.rs
@@ -36,26 +36,29 @@
 //!
 //! let file = File::open("test/data/uk_cities.csv").unwrap();
 //!
-//! let mut csv = csv::Reader::new(file, Arc::new(schema), false, None, 1024, None);
+//! let mut csv = csv::Reader::new(file, Arc::new(schema), false, None, 1024, None, None);
 //! let batch = csv.next().unwrap().unwrap();
 //! ```
 
 use lazy_static::lazy_static;
 use regex::{Regex, RegexBuilder};
-use std::collections::HashSet;
-use std::fmt;
 use std::fs::File;
 use std::io::{BufReader, Read, Seek, SeekFrom};
 use std::sync::Arc;
+use std::{collections::HashSet, iter::Skip};
+use std::{fmt, iter::Take};
 
 use csv as csv_crate;
 
-use crate::array::{ArrayRef, PrimitiveBuilder, StringBuilder};
 use crate::datatypes::*;
 use crate::error::{ArrowError, Result};
 use crate::record_batch::RecordBatch;
+use crate::{
+    array::{ArrayRef, PrimitiveArray, StringBuilder},
+    util::buffered_iterator::Buffered,
+};
 
-use self::csv_crate::{StringRecord, StringRecordsIntoIter};
+use self::csv_crate::{Error, StringRecord, StringRecordsIntoIter};
 
 lazy_static! {
     static ref DECIMAL_RE: Regex = Regex::new(r"^-?(\d+\.\d+)$").unwrap();
@@ -216,6 +219,9 @@ pub fn infer_schema_from_files(
     Schema::try_merge(&schemas)
 }
 
+// optional bounds of the reader, of the form (min line, max line).
+type Bounds = Option<(usize, usize)>;
+
 /// CSV file reader
 pub struct Reader<R: Read> {
     /// Explicit schema for the CSV file
@@ -223,10 +229,9 @@ pub struct Reader<R: Read> {
     /// Optional projection for which columns to load (zero-based column indices)
     projection: Option<Vec<usize>>,
     /// File reader
-    record_iter: StringRecordsIntoIter<BufReader<R>>,
-    /// Batch size (number of records to load each time)
-    batch_size: usize,
-    /// Current line number, used in error reporting
+    record_iter:
+        Buffered<Skip<Take<StringRecordsIntoIter<BufReader<R>>>>, StringRecord, Error>,
+    /// Current line number
     line_number: usize,
 }
 
@@ -238,7 +243,6 @@ where
         f.debug_struct("Reader")
             .field("schema", &self.schema)
             .field("projection", &self.projection)
-            .field("batch_size", &self.batch_size)
             .field("line_number", &self.line_number)
             .finish()
     }
@@ -256,6 +260,7 @@ impl<R: Read> Reader<R> {
         has_header: bool,
         delimiter: Option<u8>,
         batch_size: usize,
+        bounds: Bounds,
         projection: Option<Vec<usize>>,
     ) -> Self {
         Self::from_buf_reader(
@@ -264,6 +269,7 @@ impl<R: Read> Reader<R> {
             has_header,
             delimiter,
             batch_size,
+            bounds,
             projection,
         )
     }
@@ -293,6 +299,7 @@ impl<R: Read> Reader<R> {
         has_header: bool,
         delimiter: Option<u8>,
         batch_size: usize,
+        bounds: Bounds,
         projection: Option<Vec<usize>>,
     ) -> Self {
         let mut reader_builder = csv_crate::ReaderBuilder::new();
@@ -304,148 +311,176 @@ impl<R: Read> Reader<R> {
 
         let csv_reader = reader_builder.from_reader(buf_reader);
         let record_iter = csv_reader.into_records();
+
+        let (start, end) = match bounds {
+            None => (0, usize::MAX),
+            Some((start, end)) => (start, end),
+        };
+        // Create an iterator that:
+        // * skips the first `start` items
+        // * runs up to `end` items
+        // * buffers `batch_size` items
+        // note that this skips by iteration. This is because in general it is not possible
+        // to seek in CSV. However, skiping still saves the burden of creating arrow arrays,
+        // which is a slow operation that scales with the number of columns
+        let record_iter = Buffered::new(record_iter.take(end).skip(start), batch_size);
+
         Self {
             schema,
             projection,
             record_iter,
-            batch_size,
-            line_number: if has_header { 1 } else { 0 },
+            line_number: if has_header { start + 1 } else { start + 0 },
         }
     }
+}
 
-    /// Read the next batch of rows
-    #[allow(clippy::should_implement_trait)]
-    pub fn next(&mut self) -> Result<Option<RecordBatch>> {
-        // read a batch of rows into memory
-        let mut rows: Vec<StringRecord> = Vec::with_capacity(self.batch_size);
-        for i in 0..self.batch_size {
-            match self.record_iter.next() {
-                Some(Ok(r)) => {
-                    rows.push(r);
-                }
-                Some(Err(e)) => {
-                    return Err(ArrowError::ParseError(format!(
-                        "Error parsing line {}: {:?}",
-                        self.line_number + i,
-                        e
-                    )));
-                }
-                None => break,
+impl<R: Read> Iterator for Reader<R> {
+    type Item = Result<RecordBatch>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let rows = match self.record_iter.next() {
+            Some(Ok(r)) => r,
+            Some(Err(e)) => {
+                return Some(Err(ArrowError::ParseError(format!(
+                    "Error parsing line {}: {:?}",
+                    self.line_number + self.record_iter.n(),
+                    e
+                ))));
             }
-        }
+            None => return None,
+        };
 
         // return early if no data was loaded
         if rows.is_empty() {
-            return Ok(None);
+            return None;
         }
 
-        let projection: Vec<usize> = match self.projection {
-            Some(ref v) => v.clone(),
-            None => self
-                .schema
-                .fields()
-                .iter()
-                .enumerate()
-                .map(|(i, _)| i)
-                .collect(),
-        };
+        // parse the batches into a RecordBatch
+        let result = parse(
+            &rows,
+            &self.schema.fields(),
+            &self.projection,
+            self.line_number,
+        );
 
-        let rows = &rows[..];
-        let arrays: Result<Vec<ArrayRef>> = projection
-            .iter()
-            .map(|i| {
-                let i = *i;
-                let field = self.schema.field(i);
-                match field.data_type() {
-                    &DataType::Boolean => {
-                        self.build_primitive_array::<BooleanType>(rows, i)
-                    }
-                    &DataType::Int8 => self.build_primitive_array::<Int8Type>(rows, i),
-                    &DataType::Int16 => self.build_primitive_array::<Int16Type>(rows, i),
-                    &DataType::Int32 => self.build_primitive_array::<Int32Type>(rows, i),
-                    &DataType::Int64 => self.build_primitive_array::<Int64Type>(rows, i),
-                    &DataType::UInt8 => self.build_primitive_array::<UInt8Type>(rows, i),
-                    &DataType::UInt16 => {
-                        self.build_primitive_array::<UInt16Type>(rows, i)
-                    }
-                    &DataType::UInt32 => {
-                        self.build_primitive_array::<UInt32Type>(rows, i)
-                    }
-                    &DataType::UInt64 => {
-                        self.build_primitive_array::<UInt64Type>(rows, i)
-                    }
-                    &DataType::Float32 => {
-                        self.build_primitive_array::<Float32Type>(rows, i)
-                    }
-                    &DataType::Float64 => {
-                        self.build_primitive_array::<Float64Type>(rows, i)
-                    }
-                    &DataType::Utf8 => {
-                        let mut builder = StringBuilder::new(rows.len());
-                        for row in rows.iter() {
-                            match row.get(i) {
-                                Some(s) => builder.append_value(s).unwrap(),
-                                _ => builder.append(false).unwrap(),
-                            }
+        self.line_number += rows.len();
+
+        Some(result)
+    }
+}
+
+/// parses a slice of [csv_crate::StringRecord] into a [array::record_batch::RecordBatch].
+fn parse(
+    rows: &[StringRecord],
+    fields: &Vec<Field>,
+    projection: &Option<Vec<usize>>,
+    line_number: usize,
+) -> Result<RecordBatch> {
+    let projection: Vec<usize> = match projection {
+        Some(ref v) => v.clone(),
+        None => fields.iter().enumerate().map(|(i, _)| i).collect(),
+    };
+
+    let arrays: Result<Vec<ArrayRef>> = projection
+        .iter()
+        .map(|i| {
+            let i = *i;
+            let field = &fields[i];
+            match field.data_type() {
+                &DataType::Boolean => {
+                    build_primitive_array::<BooleanType>(line_number, rows, i)
+                }
+                &DataType::Int8 => {
+                    build_primitive_array::<Int8Type>(line_number, rows, i)
+                }
+                &DataType::Int16 => {
+                    build_primitive_array::<Int16Type>(line_number, rows, i)
+                }
+                &DataType::Int32 => {
+                    build_primitive_array::<Int32Type>(line_number, rows, i)
+                }
+                &DataType::Int64 => {
+                    build_primitive_array::<Int64Type>(line_number, rows, i)
+                }
+                &DataType::UInt8 => {
+                    build_primitive_array::<UInt8Type>(line_number, rows, i)
+                }
+                &DataType::UInt16 => {
+                    build_primitive_array::<UInt16Type>(line_number, rows, i)
+                }
+                &DataType::UInt32 => {
+                    build_primitive_array::<UInt32Type>(line_number, rows, i)
+                }
+                &DataType::UInt64 => {
+                    build_primitive_array::<UInt64Type>(line_number, rows, i)
+                }
+                &DataType::Float32 => {
+                    build_primitive_array::<Float32Type>(line_number, rows, i)
+                }
+                &DataType::Float64 => {
+                    build_primitive_array::<Float64Type>(line_number, rows, i)
+                }
+                &DataType::Utf8 => {
+                    let mut builder = StringBuilder::new(rows.len());
+                    for row in rows.iter() {
+                        match row.get(i) {
+                            Some(s) => builder.append_value(s).unwrap(),
+                            _ => builder.append(false).unwrap(),
                         }
-                        Ok(Arc::new(builder.finish()) as ArrayRef)
                     }
-                    other => Err(ArrowError::ParseError(format!(
-                        "Unsupported data type {:?}",
-                        other
-                    ))),
+                    Ok(Arc::new(builder.finish()) as ArrayRef)
                 }
-            })
-            .collect();
-
-        self.line_number += rows.len();
-
-        let schema_fields = self.schema.fields();
+                other => Err(ArrowError::ParseError(format!(
+                    "Unsupported data type {:?}",
+                    other
+                ))),
+            }
+        })
+        .collect();
 
-        let projected_fields: Vec<Field> = projection
-            .iter()
-            .map(|i| schema_fields[*i].clone())
-            .collect();
+    let projected_fields: Vec<Field> =
+        projection.iter().map(|i| fields[*i].clone()).collect();
 
-        let projected_schema = Arc::new(Schema::new(projected_fields));
+    let projected_schema = Arc::new(Schema::new(projected_fields));
 
-        arrays.and_then(|arr| RecordBatch::try_new(projected_schema, arr).map(Some))
-    }
+    arrays.and_then(|arr| RecordBatch::try_new(projected_schema, arr))
+}
 
-    fn build_primitive_array<T: ArrowPrimitiveType>(
-        &self,
-        rows: &[StringRecord],
-        col_idx: usize,
-    ) -> Result<ArrayRef> {
-        let mut builder = PrimitiveBuilder::<T>::new(rows.len());
-        let is_boolean_type =
-            *self.schema.field(col_idx).data_type() == DataType::Boolean;
-        for (row_index, row) in rows.iter().enumerate() {
+// parses a specific column (col_idx) into an Arrow Array.
+fn build_primitive_array<T: ArrowPrimitiveType>(
+    line_number: usize,
+    rows: &[StringRecord],
+    col_idx: usize,
+) -> Result<ArrayRef> {
+    rows.iter()
+        .enumerate()
+        .map(|(row_index, row)| {
             match row.get(col_idx) {
-                Some(s) if !s.is_empty() => {
-                    let t = if is_boolean_type {
+                Some(s) => {
+                    if s.is_empty() {
+                        return Ok(None);
+                    }
+                    let parsed = if T::DATA_TYPE == DataType::Boolean {
                         s.to_lowercase().parse::<T::Native>()
                     } else {
                         s.parse::<T::Native>()
                     };
-                    match t {
-                        Ok(v) => builder.append_value(v)?,
-                        Err(_) => {
+                    match parsed {
+                        Ok(e) => Ok(Some(e)),
+                        Err(_) => Err(ArrowError::ParseError(format!(
                             // TODO: we should surface the underlying error here.
-                            return Err(ArrowError::ParseError(format!(
-                                "Error while parsing value {} for column {} at line {}",
-                                s,
-                                col_idx,
-                                self.line_number + row_index
-                            )));
-                        }
+                            "Error while parsing value {} for column {} at line {}",
+                            s,
+                            col_idx,
+                            line_number + row_index
+                        ))),
                     }
                 }
-                _ => builder.append_null()?,
+                None => Ok(None),
             }
-        }
-        Ok(Arc::new(builder.finish()))
-    }
+        })
+        .collect::<Result<PrimitiveArray<T>>>()
+        .map(|e| Arc::new(e) as ArrayRef)
 }
 
 /// CSV file reader builder
@@ -471,6 +506,8 @@ pub struct ReaderBuilder {
     ///
     /// The default batch size when using the `ReaderBuilder` is 1024 records
     batch_size: usize,
+    /// The bounds over which to scan the reader. `None` starts from 0 and runs until EOF.
+    bounds: Bounds,
     /// Optional projection for which columns to load (zero-based column indices)
     projection: Option<Vec<usize>>,
 }
@@ -483,6 +520,7 @@ impl Default for ReaderBuilder {
             delimiter: None,
             max_records: None,
             batch_size: 1024,
+            bounds: None,
             projection: None,
         }
     }
@@ -572,18 +610,15 @@ impl ReaderBuilder {
                 Arc::new(inferred_schema)
             }
         };
-        let csv_reader = csv_crate::ReaderBuilder::new()
-            .delimiter(delimiter)
-            .has_headers(self.has_header)
-            .from_reader(buf_reader);
-        let record_iter = csv_reader.into_records();
-        Ok(Reader {
+        Ok(Reader::from_buf_reader(
+            buf_reader,
             schema,
-            projection: self.projection.clone(),
-            record_iter,
-            batch_size: self.batch_size,
-            line_number: if self.has_header { 1 } else { 0 },
-        })
+            self.has_header,
+            self.delimiter,
+            self.batch_size,
+            None,
+            self.projection.clone(),
+        ))
     }
 }
 
@@ -608,8 +643,15 @@ mod tests {
 
         let file = File::open("test/data/uk_cities.csv").unwrap();
 
-        let mut csv =
-            Reader::new(file, Arc::new(schema.clone()), false, None, 1024, None);
+        let mut csv = Reader::new(
+            file,
+            Arc::new(schema.clone()),
+            false,
+            None,
+            1024,
+            None,
+            None,
+        );
         assert_eq!(Arc::new(schema), csv.schema());
         let batch = csv.next().unwrap().unwrap();
         assert_eq!(37, batch.num_rows());
@@ -621,7 +663,7 @@ mod tests {
             .as_any()
             .downcast_ref::<Float64Array>()
             .unwrap();
-        assert_eq!(57.653484, lat.value(0));
+        assert!(57.653484 - lat.value(0) < f64::EPSILON);
 
         // access data from a string array (ListArray<u8>)
         let city = batch
@@ -654,6 +696,7 @@ mod tests {
             None,
             1024,
             None,
+            None,
         );
         let batch = csv.next().unwrap().unwrap();
         assert_eq!(74, batch.num_rows());
@@ -683,7 +726,7 @@ mod tests {
             .as_any()
             .downcast_ref::<Float64Array>()
             .unwrap();
-        assert_eq!(57.653484, lat.value(0));
+        assert!(57.653484 - lat.value(0) < f64::EPSILON);
 
         // access data from a string array (ListArray<u8>)
         let city = batch
@@ -721,7 +764,7 @@ mod tests {
             .as_any()
             .downcast_ref::<Float64Array>()
             .unwrap();
-        assert_eq!(57.653484, lat.value(0));
+        assert!(57.653484 - lat.value(0) < f64::EPSILON);
 
         // access data from a string array (ListArray<u8>)
         let city = batch
@@ -743,13 +786,20 @@ mod tests {
 
         let file = File::open("test/data/uk_cities.csv").unwrap();
 
-        let mut csv =
-            Reader::new(file, Arc::new(schema), false, None, 1024, Some(vec![0, 1]));
+        let mut csv = Reader::new(
+            file,
+            Arc::new(schema),
+            false,
+            None,
+            1024,
+            None,
+            Some(vec![0, 1]),
+        );
         let projected_schema = Arc::new(Schema::new(vec![
             Field::new("city", DataType::Utf8, false),
             Field::new("lat", DataType::Float64, false),
         ]));
-        assert_eq!(projected_schema.clone(), csv.schema());
+        assert_eq!(projected_schema, csv.schema());
         let batch = csv.next().unwrap().unwrap();
         assert_eq!(projected_schema, batch.schema());
         assert_eq!(37, batch.num_rows());
@@ -766,7 +816,7 @@ mod tests {
 
         let file = File::open("test/data/null_test.csv").unwrap();
 
-        let mut csv = Reader::new(file, Arc::new(schema), true, None, 1024, None);
+        let mut csv = Reader::new(file, Arc::new(schema), true, None, 1024, None, None);
         let batch = csv.next().unwrap().unwrap();
 
         assert_eq!(false, batch.column(1).is_null(0));
@@ -832,11 +882,14 @@ mod tests {
 
         let mut csv = builder.build(file).unwrap();
         match csv.next() {
-            Err(e) => assert_eq!(
-                "ParseError(\"Error while parsing value 4.x4 for column 1 at line 4\")",
-                format!("{:?}", e)
-            ),
-            Ok(_) => panic!("should have failed"),
+            Some(e) => match e {
+                Err(e) => assert_eq!(
+                    "ParseError(\"Error while parsing value 4.x4 for column 1 at line 4\")",
+                    format!("{:?}", e)
+                ),
+                Ok(_) => panic!("should have failed"),
+            }
+            None => panic!("should have failed"),
         }
     }
 
@@ -867,7 +920,7 @@ mod tests {
         writeln!(csv4, "10,\"foo\",")?;
 
         let schema = infer_schema_from_files(
-            &vec![
+            &[
                 csv3.path().to_str().unwrap().to_string(),
                 csv1.path().to_str().unwrap().to_string(),
                 csv2.path().to_str().unwrap().to_string(),
@@ -891,4 +944,51 @@ mod tests {
 
         Ok(())
     }
+
+    #[test]
+    fn test_bounded() -> Result<()> {
+        let schema = Schema::new(vec![Field::new("int", DataType::UInt32, false)]);
+        let data = vec![
+            vec!["0"],
+            vec!["1"],
+            vec!["2"],
+            vec!["3"],
+            vec!["4"],
+            vec!["5"],
+            vec!["6"],
+        ];
+
+        let data = data
+            .iter()
+            .map(|x| x.join(","))
+            .collect::<Vec<_>>()
+            .join("\n");
+        let data = data.as_bytes();
+
+        let reader = std::io::Cursor::new(data);
+
+        let mut csv = Reader::new(
+            reader,
+            Arc::new(schema),
+            false,
+            None,
+            2,
+            // starting at row 2 and up to row 6.
+            Some((2, 6)),
+            Some(vec![0]),
+        );
+
+        let batch = csv.next().unwrap().unwrap();
+        let a = batch.column(0);
+        let a = a.as_any().downcast_ref::<UInt32Array>().unwrap();
+        assert_eq!(a, &UInt32Array::from(vec![2, 3]));
+
+        let batch = csv.next().unwrap().unwrap();
+        let a = batch.column(0);
+        let a = a.as_any().downcast_ref::<UInt32Array>().unwrap();
+        assert_eq!(a, &UInt32Array::from(vec![4, 5]));
+
+        assert!(csv.next().is_none());
+        Ok(())
+    }
 }
diff --git a/rust/arrow/src/datatypes.rs b/rust/arrow/src/datatypes.rs
index 45ac221e8a3..099355e7596 100644
--- a/rust/arrow/src/datatypes.rs
+++ b/rust/arrow/src/datatypes.rs
@@ -23,6 +23,7 @@
 //!  * [`DataType`](crate::datatypes::DataType) to describe the type of a field.
 
 use std::collections::HashMap;
+use std::default::Default;
 use std::fmt;
 use std::mem::size_of;
 #[cfg(feature = "simd")]
@@ -39,6 +40,7 @@ use serde_json::{
 };
 
 use crate::error::{ArrowError, Result};
+use crate::util::bit_util;
 
 /// The set of datatypes that are supported by this implementation of Apache Arrow.
 ///
@@ -123,17 +125,25 @@ pub enum DataType {
     /// A variable-length string in Unicode with UFT-8 encoding and 64-bit offsets.
     LargeUtf8,
     /// A list of some logical data type with variable length.
-    List(Box<DataType>),
+    List(Box<Field>),
     /// A list of some logical data type with fixed length.
-    FixedSizeList(Box<DataType>, i32),
+    FixedSizeList(Box<Field>, i32),
     /// A list of some logical data type with variable length and 64-bit offsets.
-    LargeList(Box<DataType>),
+    LargeList(Box<Field>),
     /// A nested datatype that contains a number of sub-fields.
     Struct(Vec<Field>),
     /// A nested datatype that can represent slots of differing types.
     Union(Vec<Field>),
-    /// A dictionary array where each element is a single value indexed by an integer key.
-    /// This is mostly used to represent strings or a limited set of primitive types as integers.
+    /// A dictionary encoded array (`key_type`, `value_type`), where
+    /// each array element is an index of `key_type` into an
+    /// associated dictionary of `value_type`.
+    ///
+    /// Dictionary arrays are used to store columns of `value_type`
+    /// that contain many repeated values using less memory, but with
+    /// a higher CPU overhead for some operations.
+    ///
+    /// This type mostly used to represent low cardinality string
+    /// arrays or a limited set of primitive types as integers.
     Dictionary(Box<DataType>, Box<DataType>),
 }
 
@@ -184,7 +194,7 @@ pub struct Field {
 }
 
 pub trait ArrowNativeType:
-    fmt::Debug + Send + Sync + Copy + PartialOrd + FromStr + 'static
+    fmt::Debug + Send + Sync + Copy + PartialOrd + FromStr + Default + 'static
 {
     fn into_json_value(self) -> Option<Value>;
 
@@ -204,16 +214,29 @@ pub trait ArrowPrimitiveType: 'static {
     /// Corresponding Rust native type for the primitive type.
     type Native: ArrowNativeType;
 
-    /// Returns the corresponding Arrow data type of this primitive type.
-    fn get_data_type() -> DataType;
+    /// the corresponding Arrow data type of this primitive type.
+    const DATA_TYPE: DataType;
 
     /// Returns the bit width of this primitive type.
-    fn get_bit_width() -> usize;
+    fn get_bit_width() -> usize {
+        size_of::<Self::Native>() * 8
+    }
 
     /// Returns a default value of this primitive type.
     ///
     /// This is useful for aggregate array ops like `sum()`, `mean()`.
-    fn default_value() -> Self::Native;
+    fn default_value() -> Self::Native {
+        Default::default()
+    }
+
+    /// Returns a value offset from the given pointer by the given index. The default
+    /// implementation (used for all non-boolean types) is simply equivalent to pointer-arithmetic.
+    /// # Safety
+    /// Just like array-access in C: the raw_ptr must be the start of a valid array, and the index
+    /// must be less than the size of the array.
+    unsafe fn index(raw_ptr: *const Self::Native, i: usize) -> Self::Native {
+        *(raw_ptr.add(i))
+    }
 }
 
 impl ArrowNativeType for bool {
@@ -346,145 +369,116 @@ impl ArrowNativeType for f64 {
     }
 }
 
+// BooleanType is special: its bit-width is not the size of the primitive type, and its `index`
+// operation assumes bit-packing.
+#[derive(Debug)]
+pub struct BooleanType {}
+
+impl ArrowPrimitiveType for BooleanType {
+    type Native = bool;
+    const DATA_TYPE: DataType = DataType::Boolean;
+
+    fn get_bit_width() -> usize {
+        1
+    }
+
+    /// # Safety
+    /// The pointer must be part of a bit-packed boolean array, and the index must be less than the
+    /// size of the array.
+    unsafe fn index(raw_ptr: *const Self::Native, i: usize) -> Self::Native {
+        bit_util::get_bit_raw(raw_ptr as *const u8, i)
+    }
+}
+
 macro_rules! make_type {
-    ($name:ident, $native_ty:ty, $data_ty:expr, $bit_width:expr, $default_val:expr) => {
+    ($name:ident, $native_ty:ty, $data_ty:expr) => {
         #[derive(Debug)]
         pub struct $name {}
 
         impl ArrowPrimitiveType for $name {
             type Native = $native_ty;
-
-            fn get_data_type() -> DataType {
-                $data_ty
-            }
-
-            fn get_bit_width() -> usize {
-                $bit_width
-            }
-
-            fn default_value() -> Self::Native {
-                $default_val
-            }
+            const DATA_TYPE: DataType = $data_ty;
         }
     };
 }
 
-make_type!(BooleanType, bool, DataType::Boolean, 1, false);
-make_type!(Int8Type, i8, DataType::Int8, 8, 0i8);
-make_type!(Int16Type, i16, DataType::Int16, 16, 0i16);
-make_type!(Int32Type, i32, DataType::Int32, 32, 0i32);
-make_type!(Int64Type, i64, DataType::Int64, 64, 0i64);
-make_type!(UInt8Type, u8, DataType::UInt8, 8, 0u8);
-make_type!(UInt16Type, u16, DataType::UInt16, 16, 0u16);
-make_type!(UInt32Type, u32, DataType::UInt32, 32, 0u32);
-make_type!(UInt64Type, u64, DataType::UInt64, 64, 0u64);
-make_type!(Float32Type, f32, DataType::Float32, 32, 0.0f32);
-make_type!(Float64Type, f64, DataType::Float64, 64, 0.0f64);
+make_type!(Int8Type, i8, DataType::Int8);
+make_type!(Int16Type, i16, DataType::Int16);
+make_type!(Int32Type, i32, DataType::Int32);
+make_type!(Int64Type, i64, DataType::Int64);
+make_type!(UInt8Type, u8, DataType::UInt8);
+make_type!(UInt16Type, u16, DataType::UInt16);
+make_type!(UInt32Type, u32, DataType::UInt32);
+make_type!(UInt64Type, u64, DataType::UInt64);
+make_type!(Float32Type, f32, DataType::Float32);
+make_type!(Float64Type, f64, DataType::Float64);
 make_type!(
     TimestampSecondType,
     i64,
-    DataType::Timestamp(TimeUnit::Second, None),
-    64,
-    0i64
+    DataType::Timestamp(TimeUnit::Second, None)
 );
 make_type!(
     TimestampMillisecondType,
     i64,
-    DataType::Timestamp(TimeUnit::Millisecond, None),
-    64,
-    0i64
+    DataType::Timestamp(TimeUnit::Millisecond, None)
 );
 make_type!(
     TimestampMicrosecondType,
     i64,
-    DataType::Timestamp(TimeUnit::Microsecond, None),
-    64,
-    0i64
+    DataType::Timestamp(TimeUnit::Microsecond, None)
 );
 make_type!(
     TimestampNanosecondType,
     i64,
-    DataType::Timestamp(TimeUnit::Nanosecond, None),
-    64,
-    0i64
-);
-make_type!(Date32Type, i32, DataType::Date32(DateUnit::Day), 32, 0i32);
-make_type!(
-    Date64Type,
-    i64,
-    DataType::Date64(DateUnit::Millisecond),
-    64,
-    0i64
-);
-make_type!(
-    Time32SecondType,
-    i32,
-    DataType::Time32(TimeUnit::Second),
-    32,
-    0i32
+    DataType::Timestamp(TimeUnit::Nanosecond, None)
 );
+make_type!(Date32Type, i32, DataType::Date32(DateUnit::Day));
+make_type!(Date64Type, i64, DataType::Date64(DateUnit::Millisecond));
+make_type!(Time32SecondType, i32, DataType::Time32(TimeUnit::Second));
 make_type!(
     Time32MillisecondType,
     i32,
-    DataType::Time32(TimeUnit::Millisecond),
-    32,
-    0i32
+    DataType::Time32(TimeUnit::Millisecond)
 );
 make_type!(
     Time64MicrosecondType,
     i64,
-    DataType::Time64(TimeUnit::Microsecond),
-    64,
-    0i64
+    DataType::Time64(TimeUnit::Microsecond)
 );
 make_type!(
     Time64NanosecondType,
     i64,
-    DataType::Time64(TimeUnit::Nanosecond),
-    64,
-    0i64
+    DataType::Time64(TimeUnit::Nanosecond)
 );
 make_type!(
     IntervalYearMonthType,
     i32,
-    DataType::Interval(IntervalUnit::YearMonth),
-    32,
-    0i32
+    DataType::Interval(IntervalUnit::YearMonth)
 );
 make_type!(
     IntervalDayTimeType,
     i64,
-    DataType::Interval(IntervalUnit::DayTime),
-    64,
-    0i64
+    DataType::Interval(IntervalUnit::DayTime)
 );
 make_type!(
     DurationSecondType,
     i64,
-    DataType::Duration(TimeUnit::Second),
-    64,
-    0i64
+    DataType::Duration(TimeUnit::Second)
 );
 make_type!(
     DurationMillisecondType,
     i64,
-    DataType::Duration(TimeUnit::Millisecond),
-    64,
-    0i64
+    DataType::Duration(TimeUnit::Millisecond)
 );
 make_type!(
     DurationMicrosecondType,
     i64,
-    DataType::Duration(TimeUnit::Microsecond),
-    64,
-    0i64
+    DataType::Duration(TimeUnit::Microsecond)
 );
 make_type!(
     DurationNanosecondType,
     i64,
-    DataType::Duration(TimeUnit::Nanosecond),
-    64,
-    0i64
+    DataType::Duration(TimeUnit::Nanosecond)
 );
 
 /// A subtype of primitive type that represents legal dictionary keys.
@@ -537,6 +531,9 @@ where
     /// Creates a new SIMD mask for this SIMD type filling it with `value`
     fn mask_init(value: bool) -> Self::SimdMask;
 
+    /// Creates a new SIMD mask for this SIMD type from the lower-most bits of the given `mask`
+    fn mask_from_u64(mask: u64) -> Self::SimdMask;
+
     /// Gets the value of a single lane in a SIMD mask
     fn mask_get(mask: &Self::SimdMask, idx: usize) -> bool;
 
@@ -597,22 +594,109 @@ macro_rules! make_numeric_type {
 
             type SimdMask = $simd_mask_ty;
 
+            #[inline]
             fn lanes() -> usize {
                 Self::Simd::lanes()
             }
 
+            #[inline]
             fn init(value: Self::Native) -> Self::Simd {
                 Self::Simd::splat(value)
             }
 
+            #[inline]
             fn load(slice: &[Self::Native]) -> Self::Simd {
                 unsafe { Self::Simd::from_slice_unaligned_unchecked(slice) }
             }
 
+            #[inline]
             fn mask_init(value: bool) -> Self::SimdMask {
                 Self::SimdMask::splat(value)
             }
 
+            #[inline]
+            fn mask_from_u64(mask: u64) -> Self::SimdMask {
+                match Self::lanes() {
+                    8 => {
+                        let vecidx = i64x8::new(128, 64, 32, 16, 8, 4, 2, 1);
+
+                        let vecmask = i64x8::splat((mask & 0xFF) as i64);
+                        let vecmask = (vecidx & vecmask).eq(vecidx);
+
+                        unsafe { std::mem::transmute(vecmask) }
+                    }
+                    16 => {
+                        let vecidx = i32x16::new(
+                            32768, 16384, 8192, 4096, 2048, 1024, 512, 256, 128, 64, 32,
+                            16, 8, 4, 2, 1,
+                        );
+
+                        let vecmask = i32x16::splat((mask & 0xFFFF) as i32);
+                        let vecmask = (vecidx & vecmask).eq(vecidx);
+
+                        unsafe { std::mem::transmute(vecmask) }
+                    }
+                    32 => {
+                        let tmp = &mut [0_i16; 32];
+
+                        let vecidx = i32x16::new(
+                            32768, 16384, 8192, 4096, 2048, 1024, 512, 256, 128, 64, 32,
+                            16, 8, 4, 2, 1,
+                        );
+
+                        let vecmask = i32x16::splat((mask & 0xFFFF) as i32);
+                        let vecmask = (vecidx & vecmask).eq(vecidx);
+
+                        i16x16::from_cast(vecmask)
+                            .write_to_slice_unaligned(&mut tmp[0..16]);
+
+                        let vecmask = i32x16::splat(((mask >> 16) & 0xFFFF) as i32);
+                        let vecmask = (vecidx & vecmask).eq(vecidx);
+
+                        i16x16::from_cast(vecmask)
+                            .write_to_slice_unaligned(&mut tmp[16..32]);
+
+                        unsafe { std::mem::transmute(i16x32::from_slice_unaligned(tmp)) }
+                    }
+                    64 => {
+                        let tmp = &mut [0_i8; 64];
+
+                        let vecidx = i32x16::new(
+                            32768, 16384, 8192, 4096, 2048, 1024, 512, 256, 128, 64, 32,
+                            16, 8, 4, 2, 1,
+                        );
+
+                        let vecmask = i32x16::splat((mask & 0xFFFF) as i32);
+                        let vecmask = (vecidx & vecmask).eq(vecidx);
+
+                        i8x16::from_cast(vecmask)
+                            .write_to_slice_unaligned(&mut tmp[0..16]);
+
+                        let vecmask = i32x16::splat(((mask >> 16) & 0xFFFF) as i32);
+                        let vecmask = (vecidx & vecmask).eq(vecidx);
+
+                        i8x16::from_cast(vecmask)
+                            .write_to_slice_unaligned(&mut tmp[16..32]);
+
+                        let vecmask = i32x16::splat(((mask >> 32) & 0xFFFF) as i32);
+                        let vecmask = (vecidx & vecmask).eq(vecidx);
+
+                        i8x16::from_cast(vecmask)
+                            .write_to_slice_unaligned(&mut tmp[32..48]);
+
+                        let vecmask = i32x16::splat(((mask >> 48) & 0xFFFF) as i32);
+                        let vecmask = (vecidx & vecmask).eq(vecidx);
+
+                        i8x16::from_cast(vecmask)
+                            .write_to_slice_unaligned(&mut tmp[48..64]);
+
+                        unsafe { std::mem::transmute(i8x64::from_slice_unaligned(tmp)) }
+                    }
+                    _ => panic!("Invalid number of vector lanes"),
+                }
+            }
+
+            #[inline]
             fn mask_get(mask: &Self::SimdMask, idx: usize) -> bool {
                 unsafe { mask.extract_unchecked(idx) }
             }
@@ -624,11 +708,13 @@ macro_rules! make_numeric_type {
                 action(mask.bitmask().to_byte_slice());
             }
 
+            #[inline]
             fn mask_set(mask: Self::SimdMask, idx: usize, value: bool) -> Self::SimdMask {
                 unsafe { mask.replace_unchecked(idx, value) }
             }
 
             /// Selects elements of `a` and `b` using `mask`
+            #[inline]
             fn mask_select(
                 mask: Self::SimdMask,
                 a: Self::Simd,
@@ -637,10 +723,12 @@ macro_rules! make_numeric_type {
                 mask.select(a, b)
             }
 
+            #[inline]
             fn mask_any(mask: Self::SimdMask) -> bool {
                 mask.any()
             }
 
+            #[inline]
             fn bin_op<F: Fn(Self::Simd, Self::Simd) -> Self::Simd>(
                 left: Self::Simd,
                 right: Self::Simd,
@@ -649,30 +737,37 @@ macro_rules! make_numeric_type {
                 op(left, right)
             }
 
+            #[inline]
             fn eq(left: Self::Simd, right: Self::Simd) -> Self::SimdMask {
                 left.eq(right)
             }
 
+            #[inline]
             fn ne(left: Self::Simd, right: Self::Simd) -> Self::SimdMask {
                 left.ne(right)
             }
 
+            #[inline]
             fn lt(left: Self::Simd, right: Self::Simd) -> Self::SimdMask {
                 left.lt(right)
             }
 
+            #[inline]
             fn le(left: Self::Simd, right: Self::Simd) -> Self::SimdMask {
                 left.le(right)
             }
 
+            #[inline]
             fn gt(left: Self::Simd, right: Self::Simd) -> Self::SimdMask {
                 left.gt(right)
             }
 
+            #[inline]
             fn ge(left: Self::Simd, right: Self::Simd) -> Self::SimdMask {
                 left.ge(right)
             }
 
+            #[inline]
             fn write(simd_result: Self::Simd, slice: &mut [Self::Native]) {
                 unsafe { simd_result.write_to_slice_unaligned_unchecked(slice) };
             }
@@ -779,6 +874,7 @@ impl<T: ArrowNativeType> ToByteSlice for T {
 impl DataType {
     /// Parse a data type from a JSON representation
     fn from(json: &Value) -> Result<DataType> {
+        let default_field = Field::new("", DataType::Boolean, true);
         match *json {
             Value::Object(ref map) => match map.get("name") {
                 Some(s) if s == "null" => Ok(DataType::Null),
@@ -912,17 +1008,17 @@ impl DataType {
                 },
                 Some(s) if s == "list" => {
                     // return a list with any type as its child isn't defined in the map
-                    Ok(DataType::List(Box::new(DataType::Boolean)))
+                    Ok(DataType::List(Box::new(default_field)))
                 }
                 Some(s) if s == "largelist" => {
                     // return a largelist with any type as its child isn't defined in the map
-                    Ok(DataType::LargeList(Box::new(DataType::Boolean)))
+                    Ok(DataType::LargeList(Box::new(default_field)))
                 }
                 Some(s) if s == "fixedsizelist" => {
                     // return a list with any type as its child isn't defined in the map
                     if let Some(Value::Number(size)) = map.get("listSize") {
                         Ok(DataType::FixedSizeList(
-                            Box::new(DataType::Boolean),
+                            Box::new(default_field),
                             size.as_i64().unwrap() as i32,
                         ))
                     } else {
@@ -1028,6 +1124,16 @@ impl DataType {
             DataType::Dictionary(_, _) => json!({ "name": "dictionary"}),
         }
     }
+
+    /// Returns true if this type is numeric: (UInt*, Unit*, or Float*)
+    pub fn is_numeric(t: &DataType) -> bool {
+        use DataType::*;
+        match t {
+            UInt8 | UInt16 | UInt32 | UInt64 | Int8 | Int16 | Int32 | Int64 | Float32
+            | Float64 => true,
+            _ => false,
+        }
+    }
 }
 
 impl Field {
@@ -1060,20 +1166,35 @@ impl Field {
     }
 
     /// Returns an immutable reference to the `Field`'s name
-    pub fn name(&self) -> &String {
+    #[inline]
+    pub const fn name(&self) -> &String {
         &self.name
     }
 
     /// Returns an immutable reference to the `Field`'s  data-type
-    pub fn data_type(&self) -> &DataType {
+    #[inline]
+    pub const fn data_type(&self) -> &DataType {
         &self.data_type
     }
 
     /// Indicates whether this `Field` supports null values
-    pub fn is_nullable(&self) -> bool {
+    #[inline]
+    pub const fn is_nullable(&self) -> bool {
         self.nullable
     }
 
+    /// Returns the dictionary ID
+    #[inline]
+    pub const fn dict_id(&self) -> i64 {
+        self.dict_id
+    }
+
+    /// Indicates whether this `Field`'s dictionary is ordered
+    #[inline]
+    pub const fn dict_is_ordered(&self) -> bool {
+        self.dict_is_ordered
+    }
+
     /// Parse a `Field` definition from a JSON representation
     pub fn from(json: &Value) -> Result<Self> {
         match *json {
@@ -1115,14 +1236,14 @@ impl Field {
                             }
                             match data_type {
                                     DataType::List(_) => DataType::List(Box::new(
-                                        Self::from(&values[0])?.data_type,
+                                        Self::from(&values[0])?,
                                     )),
                                     DataType::LargeList(_) => DataType::LargeList(Box::new(
-                                        Self::from(&values[0])?.data_type,
+                                        Self::from(&values[0])?,
                                     )),
                                     DataType::FixedSizeList(_, int) => {
                                         DataType::FixedSizeList(
-                                            Box::new(Self::from(&values[0])?.data_type),
+                                            Box::new(Self::from(&values[0])?),
                                             int,
                                         )
                                     }
@@ -1214,18 +1335,9 @@ impl Field {
     pub fn to_json(&self) -> Value {
         let children: Vec<Value> = match self.data_type() {
             DataType::Struct(fields) => fields.iter().map(|f| f.to_json()).collect(),
-            DataType::List(dtype) => {
-                let item = Field::new("item", *dtype.clone(), self.nullable);
-                vec![item.to_json()]
-            }
-            DataType::LargeList(dtype) => {
-                let item = Field::new("item", *dtype.clone(), self.nullable);
-                vec![item.to_json()]
-            }
-            DataType::FixedSizeList(dtype, _) => {
-                let item = Field::new("item", *dtype.clone(), self.nullable);
-                vec![item.to_json()]
-            }
+            DataType::List(field) => vec![field.to_json()],
+            DataType::LargeList(field) => vec![field.to_json()],
+            DataType::FixedSizeList(field, _) => vec![field.to_json()],
             _ => vec![],
         };
         match self.data_type() {
@@ -1423,7 +1535,8 @@ impl Schema {
     ///
     /// let schema = Schema::new_with_metadata(vec![field_a, field_b], metadata);
     /// ```
-    pub fn new_with_metadata(
+    #[inline]
+    pub const fn new_with_metadata(
         fields: Vec<Field>,
         metadata: HashMap<String, String>,
     ) -> Self {
@@ -1499,7 +1612,8 @@ impl Schema {
     }
 
     /// Returns an immutable reference of the vector of `Field` instances
-    pub fn fields(&self) -> &Vec<Field> {
+    #[inline]
+    pub const fn fields(&self) -> &Vec<Field> {
         &self.fields
     }
 
@@ -1530,7 +1644,8 @@ impl Schema {
     }
 
     /// Returns an immutable reference to the Map of custom metadata key-value pairs.
-    pub fn metadata(&self) -> &HashMap<String, String> {
+    #[inline]
+    pub const fn metadata(&self) -> &HashMap<String, String> {
         &self.metadata
     }
 
@@ -1621,7 +1736,6 @@ pub type SchemaRef = Arc<Schema>;
 #[cfg(test)]
 mod tests {
     use super::*;
-    use serde_json;
     use serde_json::Number;
     use serde_json::Value::{Bool, Number as VNumber};
     use std::f32::NAN;
@@ -1848,17 +1962,30 @@ mod tests {
                 ),
                 Field::new("c19", DataType::Interval(IntervalUnit::DayTime), false),
                 Field::new("c20", DataType::Interval(IntervalUnit::YearMonth), false),
-                Field::new("c21", DataType::List(Box::new(DataType::Boolean)), false),
+                Field::new(
+                    "c21",
+                    DataType::List(Box::new(Field::new("item", DataType::Boolean, true))),
+                    false,
+                ),
                 Field::new(
                     "c22",
-                    DataType::FixedSizeList(Box::new(DataType::Boolean), 5),
+                    DataType::FixedSizeList(
+                        Box::new(Field::new("bools", DataType::Boolean, false)),
+                        5,
+                    ),
                     false,
                 ),
                 Field::new(
                     "c23",
-                    DataType::List(Box::new(DataType::List(Box::new(DataType::Struct(
-                        vec![],
-                    ))))),
+                    DataType::List(Box::new(Field::new(
+                        "inner_list",
+                        DataType::List(Box::new(Field::new(
+                            "struct",
+                            DataType::Struct(vec![]),
+                            true,
+                        ))),
+                        false,
+                    ))),
                     true,
                 ),
                 Field::new(
@@ -1889,9 +2016,15 @@ mod tests {
                 Field::new("c33", DataType::LargeUtf8, true),
                 Field::new(
                     "c34",
-                    DataType::LargeList(Box::new(DataType::LargeList(Box::new(
-                        DataType::Struct(vec![]),
-                    )))),
+                    DataType::LargeList(Box::new(Field::new(
+                        "inner_large_list",
+                        DataType::LargeList(Box::new(Field::new(
+                            "struct",
+                            DataType::Struct(vec![]),
+                            false,
+                        ))),
+                        true,
+                    ))),
                     true,
                 ),
             ],
@@ -2097,7 +2230,7 @@ mod tests {
                         "children": [
                             {
                                 "name": "item",
-                                "nullable": false,
+                                "nullable": true,
                                 "type": {
                                     "name": "bool"
                                 },
@@ -2114,7 +2247,7 @@ mod tests {
                         },
                         "children": [
                             {
-                                "name": "item",
+                                "name": "bools",
                                 "nullable": false,
                                 "type": {
                                     "name": "bool"
@@ -2131,14 +2264,14 @@ mod tests {
                         },
                         "children": [
                             {
-                                "name": "item",
-                                "nullable": true,
+                                "name": "inner_list",
+                                "nullable": false,
                                 "type": {
                                     "name": "list"
                                 },
                                 "children": [
                                     {
-                                        "name": "item",
+                                        "name": "struct",
                                         "nullable": true,
                                         "type": {
                                             "name": "struct"
@@ -2271,15 +2404,15 @@ mod tests {
                         },
                         "children": [
                             {
-                                "name": "item",
+                                "name": "inner_large_list",
                                 "nullable": true,
                                 "type": {
                                     "name": "largelist"
                                 },
                                 "children": [
                                     {
-                                        "name": "item",
-                                        "nullable": true,
+                                        "name": "struct",
+                                        "nullable": false,
                                         "type": {
                                             "name": "struct"
                                         },
@@ -2461,7 +2594,7 @@ mod tests {
 
     #[test]
     fn test_schema_merge() -> Result<()> {
-        let merged = Schema::try_merge(&vec![
+        let merged = Schema::try_merge(&[
             Schema::new(vec![
                 Field::new("first_name", DataType::Utf8, false),
                 Field::new("last_name", DataType::Utf8, false),
@@ -2520,7 +2653,7 @@ mod tests {
 
         // support merge union fields
         assert_eq!(
-            Schema::try_merge(&vec![
+            Schema::try_merge(&[
                 Schema::new(vec![Field::new(
                     "c1",
                     DataType::Union(vec![
@@ -2550,17 +2683,17 @@ mod tests {
         );
 
         // incompatible field should throw error
-        assert!(Schema::try_merge(&vec![
+        assert!(Schema::try_merge(&[
             Schema::new(vec![
                 Field::new("first_name", DataType::Utf8, false),
                 Field::new("last_name", DataType::Utf8, false),
             ]),
-            Schema::new(vec![Field::new("last_name", DataType::Int64, false),]),
+            Schema::new(vec![Field::new("last_name", DataType::Int64, false),])
         ])
         .is_err());
 
         // incompatible metadata should throw error
-        assert!(Schema::try_merge(&vec![
+        assert!(Schema::try_merge(&[
             Schema::new_with_metadata(
                 vec![Field::new("first_name", DataType::Utf8, false)],
                 [("foo".to_string(), "bar".to_string()),]
@@ -2574,10 +2707,93 @@ mod tests {
                     .iter()
                     .cloned()
                     .collect::<HashMap<String, String>>()
-            ),
+            )
         ])
         .is_err());
 
         Ok(())
     }
 }
+
+#[cfg(all(
+    test,
+    any(target_arch = "x86", target_arch = "x86_64"),
+    feature = "simd"
+))]
+mod arrow_numeric_type_tests {
+    use crate::datatypes::{
+        ArrowNumericType, Float32Type, Float64Type, Int32Type, Int64Type, Int8Type,
+        UInt16Type,
+    };
+    use packed_simd::*;
+    use FromCast;
+
+    #[test]
+    fn test_mask_f64() {
+        let mask = Float64Type::mask_from_u64(0b10101010);
+
+        let expected =
+            m64x8::from_cast(i64x8::from_slice_unaligned(&[-1, 0, -1, 0, -1, 0, -1, 0]));
+
+        assert_eq!(expected, mask);
+    }
+
+    #[test]
+    fn test_mask_u64() {
+        let mask = Int64Type::mask_from_u64(0b01010101);
+
+        let expected =
+            m64x8::from_cast(i64x8::from_slice_unaligned(&[0, -1, 0, -1, 0, -1, 0, -1]));
+
+        assert_eq!(expected, mask);
+    }
+
+    #[test]
+    fn test_mask_f32() {
+        let mask = Float32Type::mask_from_u64(0b10101010_10101010);
+
+        let expected = m32x16::from_cast(i32x16::from_slice_unaligned(&[
+            -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0,
+        ]));
+
+        assert_eq!(expected, mask);
+    }
+
+    #[test]
+    fn test_mask_i32() {
+        let mask = Int32Type::mask_from_u64(0b01010101_01010101);
+
+        let expected = m32x16::from_cast(i32x16::from_slice_unaligned(&[
+            0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1,
+        ]));
+
+        assert_eq!(expected, mask);
+    }
+
+    #[test]
+    fn test_mask_u16() {
+        let mask = UInt16Type::mask_from_u64(0b01010101_01010101_10101010_10101010);
+
+        let expected = m16x32::from_cast(i16x32::from_slice_unaligned(&[
+            -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, 0, -1, 0, -1, 0, -1,
+            0, -1, 0, -1, 0, -1, 0, -1, 0, -1,
+        ]));
+
+        assert_eq!(expected, mask);
+    }
+
+    #[test]
+    fn test_mask_i8() {
+        let mask = Int8Type::mask_from_u64(
+            0b01010101_01010101_10101010_10101010_01010101_01010101_10101010_10101010,
+        );
+
+        let expected = m8x64::from_cast(i8x64::from_slice_unaligned(&[
+            -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, 0, -1, 0, -1, 0, -1,
+            0, -1, 0, -1, 0, -1, 0, -1, 0, -1, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0,
+            -1, 0, -1, 0, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1,
+        ]));
+
+        assert_eq!(expected, mask);
+    }
+}
diff --git a/rust/arrow/src/ipc/convert.rs b/rust/arrow/src/ipc/convert.rs
index a5bad4b99b5..feb4133d383 100644
--- a/rust/arrow/src/ipc/convert.rs
+++ b/rust/arrow/src/ipc/convert.rs
@@ -34,18 +34,8 @@ pub fn schema_to_fb(schema: &Schema) -> FlatBufferBuilder {
 
     let mut fields = vec![];
     for field in schema.fields() {
-        let fb_field_name = fbb.create_string(field.name().as_str());
-        let field_type = get_fb_field_type(field.data_type(), &mut fbb);
-        let mut field_builder = ipc::FieldBuilder::new(&mut fbb);
-        field_builder.add_name(fb_field_name);
-        field_builder.add_type_type(field_type.type_type);
-        field_builder.add_nullable(field.is_nullable());
-        match field_type.children {
-            None => {}
-            Some(children) => field_builder.add_children(children),
-        };
-        field_builder.add_type_(field_type.type_);
-        fields.push(field_builder.finish());
+        let fb_field = build_field(&mut fbb, field);
+        fields.push(fb_field);
     }
 
     let mut custom_metadata = vec![];
@@ -80,18 +70,8 @@ pub fn schema_to_fb_offset<'a: 'b, 'b>(
 ) -> WIPOffset<ipc::Schema<'b>> {
     let mut fields = vec![];
     for field in schema.fields() {
-        let fb_field_name = fbb.create_string(field.name().as_str());
-        let field_type = get_fb_field_type(field.data_type(), fbb);
-        let mut field_builder = ipc::FieldBuilder::new(fbb);
-        field_builder.add_name(fb_field_name);
-        field_builder.add_type_type(field_type.type_type);
-        field_builder.add_nullable(field.is_nullable());
-        match field_type.children {
-            None => {}
-            Some(children) => field_builder.add_children(children),
-        };
-        field_builder.add_type_(field_type.type_);
-        fields.push(field_builder.finish());
+        let fb_field = build_field(fbb, field);
+        fields.push(fb_field);
     }
 
     let mut custom_metadata = vec![];
@@ -165,7 +145,7 @@ pub fn fb_to_schema(fb: ipc::Schema) -> Schema {
 /// Deserialize an IPC message into a schema
 pub fn schema_from_bytes(bytes: &[u8]) -> Option<Schema> {
     let ipc = ipc::get_root_as_message(bytes);
-    ipc.header_as_schema().map(|schema| fb_to_schema(schema))
+    ipc.header_as_schema().map(fb_to_schema)
 }
 
 /// Get the Arrow data type from the flatbuffer Field table
@@ -288,30 +268,22 @@ pub(crate) fn get_data_type(field: ipc::Field, may_be_dictionary: bool) -> DataT
             if children.len() != 1 {
                 panic!("expect a list to have one child")
             }
-            let child_field = children.get(0);
-            // returning int16 for now, to test, not sure how to get data type
-            DataType::List(Box::new(get_data_type(child_field, false)))
+            DataType::List(Box::new(children.get(0).into()))
         }
         ipc::Type::LargeList => {
             let children = field.children().unwrap();
             if children.len() != 1 {
                 panic!("expect a large list to have one child")
             }
-            let child_field = children.get(0);
-            // returning int16 for now, to test, not sure how to get data type
-            DataType::LargeList(Box::new(get_data_type(child_field, false)))
+            DataType::LargeList(Box::new(children.get(0).into()))
         }
         ipc::Type::FixedSizeList => {
             let children = field.children().unwrap();
             if children.len() != 1 {
                 panic!("expect a list to have one child")
             }
-            let child_field = children.get(0);
             let fsl = field.type_as_fixed_size_list().unwrap();
-            DataType::FixedSizeList(
-                Box::new(get_data_type(child_field, false)),
-                fsl.listSize(),
-            )
+            DataType::FixedSizeList(Box::new(children.get(0).into()), fsl.listSize())
         }
         ipc::Type::Struct_ => {
             let mut fields = vec![];
@@ -333,6 +305,40 @@ pub(crate) struct FBFieldType<'b> {
     pub(crate) children: Option<WIPOffset<Vector<'b, ForwardsUOffset<ipc::Field<'b>>>>>,
 }
 
+/// Create an IPC Field from an Arrow Field
+pub(crate) fn build_field<'a: 'b, 'b>(
+    fbb: &mut FlatBufferBuilder<'a>,
+    field: &Field,
+) -> WIPOffset<ipc::Field<'b>> {
+    let fb_field_name = fbb.create_string(field.name().as_str());
+    let field_type = get_fb_field_type(field.data_type(), fbb);
+
+    let fb_dictionary = if let Dictionary(index_type, _) = field.data_type() {
+        Some(get_fb_dictionary(
+            index_type,
+            field.dict_id(),
+            field.dict_is_ordered(),
+            fbb,
+        ))
+    } else {
+        None
+    };
+
+    let mut field_builder = ipc::FieldBuilder::new(fbb);
+    field_builder.add_name(fb_field_name);
+    if let Some(dictionary) = fb_dictionary {
+        field_builder.add_dictionary(dictionary)
+    }
+    field_builder.add_type_type(field_type.type_type);
+    field_builder.add_nullable(field.is_nullable());
+    match field_type.children {
+        None => {}
+        Some(children) => field_builder.add_children(children),
+    };
+    field_builder.add_type_(field_type.type_);
+    field_builder.finish()
+}
+
 /// Get the IPC type of a data type
 pub(crate) fn get_fb_field_type<'a: 'b, 'b>(
     data_type: &DataType,
@@ -345,7 +351,7 @@ pub(crate) fn get_fb_field_type<'a: 'b, 'b>(
         Null => FBFieldType {
             type_type: ipc::Type::Null,
             type_: ipc::NullBuilder::new(fbb).finish().as_union_value(),
-            children: None,
+            children: Some(fbb.create_vector(&empty_fields[..])),
         },
         Boolean => FBFieldType {
             type_type: ipc::Type::Bool,
@@ -523,19 +529,7 @@ pub(crate) fn get_fb_field_type<'a: 'b, 'b>(
             }
         }
         List(ref list_type) => {
-            let inner_types = get_fb_field_type(list_type, fbb);
-            let child = ipc::Field::create(
-                fbb,
-                &ipc::FieldArgs {
-                    name: None,
-                    nullable: false,
-                    type_type: inner_types.type_type,
-                    type_: Some(inner_types.type_),
-                    children: inner_types.children,
-                    dictionary: None,
-                    custom_metadata: None,
-                },
-            );
+            let child = build_field(fbb, list_type);
             FBFieldType {
                 type_type: ipc::Type::List,
                 type_: ipc::ListBuilder::new(fbb).finish().as_union_value(),
@@ -543,19 +537,7 @@ pub(crate) fn get_fb_field_type<'a: 'b, 'b>(
             }
         }
         LargeList(ref list_type) => {
-            let inner_types = get_fb_field_type(list_type, fbb);
-            let child = ipc::Field::create(
-                fbb,
-                &ipc::FieldArgs {
-                    name: None,
-                    nullable: false,
-                    type_type: inner_types.type_type,
-                    type_: Some(inner_types.type_),
-                    dictionary: None,
-                    children: inner_types.children,
-                    custom_metadata: None,
-                },
-            );
+            let child = build_field(fbb, list_type);
             FBFieldType {
                 type_type: ipc::Type::LargeList,
                 type_: ipc::LargeListBuilder::new(fbb).finish().as_union_value(),
@@ -563,19 +545,7 @@ pub(crate) fn get_fb_field_type<'a: 'b, 'b>(
             }
         }
         FixedSizeList(ref list_type, len) => {
-            let inner_types = get_fb_field_type(list_type, fbb);
-            let child = ipc::Field::create(
-                fbb,
-                &ipc::FieldArgs {
-                    name: None,
-                    nullable: false,
-                    type_type: inner_types.type_type,
-                    type_: Some(inner_types.type_),
-                    dictionary: None,
-                    children: inner_types.children,
-                    custom_metadata: None,
-                },
-            );
+            let child = build_field(fbb, list_type);
             let mut builder = ipc::FixedSizeListBuilder::new(fbb);
             builder.add_listSize(*len as i32);
             FBFieldType {
@@ -609,10 +579,51 @@ pub(crate) fn get_fb_field_type<'a: 'b, 'b>(
                 children: Some(fbb.create_vector(&children[..])),
             }
         }
+        Dictionary(_, value_type) => {
+            // In this library, the dictionary "type" is a logical construct. Here we
+            // pass through to the value type, as we've already captured the index
+            // type in the DictionaryEncoding metadata in the parent field
+            get_fb_field_type(value_type, fbb)
+        }
         t => unimplemented!("Type {:?} not supported", t),
     }
 }
 
+/// Create an IPC dictionary encoding
+pub(crate) fn get_fb_dictionary<'a: 'b, 'b>(
+    index_type: &DataType,
+    dict_id: i64,
+    dict_is_ordered: bool,
+    fbb: &mut FlatBufferBuilder<'a>,
+) -> WIPOffset<ipc::DictionaryEncoding<'b>> {
+    // We assume that the dictionary index type (as an integer) has already been
+    // validated elsewhere, and can safely assume we are dealing with integers
+    let mut index_builder = ipc::IntBuilder::new(fbb);
+
+    match *index_type {
+        Int8 | Int16 | Int32 | Int64 => index_builder.add_is_signed(true),
+        UInt8 | UInt16 | UInt32 | UInt64 => index_builder.add_is_signed(false),
+        _ => {}
+    }
+
+    match *index_type {
+        Int8 | UInt8 => index_builder.add_bitWidth(8),
+        Int16 | UInt16 => index_builder.add_bitWidth(16),
+        Int32 | UInt32 => index_builder.add_bitWidth(32),
+        Int64 | UInt64 => index_builder.add_bitWidth(64),
+        _ => {}
+    }
+
+    let index_builder = index_builder.finish();
+
+    let mut builder = ipc::DictionaryEncodingBuilder::new(fbb);
+    builder.add_id(dict_id);
+    builder.add_indexType(index_builder);
+    builder.add_isOrdered(dict_is_ordered);
+
+    builder.finish()
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -680,14 +691,22 @@ mod tests {
                 ),
                 Field::new("utf8", DataType::Utf8, false),
                 Field::new("binary", DataType::Binary, false),
-                Field::new("list[u8]", DataType::List(Box::new(DataType::UInt8)), true),
+                Field::new(
+                    "list[u8]",
+                    DataType::List(Box::new(Field::new("item", DataType::UInt8, false))),
+                    true,
+                ),
                 Field::new(
                     "list[struct<float32, int32, bool>]",
-                    DataType::List(Box::new(DataType::Struct(vec![
-                        Field::new("float32", DataType::UInt8, false),
-                        Field::new("int32", DataType::Int32, true),
-                        Field::new("bool", DataType::Boolean, true),
-                    ]))),
+                    DataType::List(Box::new(Field::new(
+                        "struct",
+                        DataType::Struct(vec![
+                            Field::new("float32", DataType::UInt8, false),
+                            Field::new("int32", DataType::Int32, true),
+                            Field::new("bool", DataType::Boolean, true),
+                        ]),
+                        true,
+                    ))),
                     false,
                 ),
                 Field::new(
@@ -696,24 +715,52 @@ mod tests {
                         Field::new("int64", DataType::Int64, true),
                         Field::new(
                             "list[struct<date32, list[struct<>]>]",
-                            DataType::List(Box::new(DataType::Struct(vec![
-                                Field::new(
-                                    "date32",
-                                    DataType::Date32(DateUnit::Day),
-                                    true,
-                                ),
-                                Field::new(
-                                    "list[struct<>]",
-                                    DataType::List(Box::new(DataType::Struct(vec![]))),
-                                    false,
-                                ),
-                            ]))),
+                            DataType::List(Box::new(Field::new(
+                                "struct",
+                                DataType::Struct(vec![
+                                    Field::new(
+                                        "date32",
+                                        DataType::Date32(DateUnit::Day),
+                                        true,
+                                    ),
+                                    Field::new(
+                                        "list[struct<>]",
+                                        DataType::List(Box::new(Field::new(
+                                            "struct",
+                                            DataType::Struct(vec![]),
+                                            false,
+                                        ))),
+                                        false,
+                                    ),
+                                ]),
+                                false,
+                            ))),
                             false,
                         ),
                     ]),
                     false,
                 ),
                 Field::new("struct<>", DataType::Struct(vec![]), true),
+                Field::new_dict(
+                    "dictionary<int32, utf8>",
+                    DataType::Dictionary(
+                        Box::new(DataType::Int32),
+                        Box::new(DataType::Utf8),
+                    ),
+                    true,
+                    123,
+                    true,
+                ),
+                Field::new_dict(
+                    "dictionary<uint8, uint32>",
+                    DataType::Dictionary(
+                        Box::new(DataType::UInt8),
+                        Box::new(DataType::UInt32),
+                    ),
+                    true,
+                    123,
+                    true,
+                ),
             ],
             md,
         );
diff --git a/rust/arrow/src/ipc/gen/File.rs b/rust/arrow/src/ipc/gen/File.rs
index a805100baf7..c268f127fe4 100644
--- a/rust/arrow/src/ipc/gen/File.rs
+++ b/rust/arrow/src/ipc/gen/File.rs
@@ -88,7 +88,7 @@ impl Block {
         self.metaDataLength_.from_little_endian()
     }
     /// Length of the data (this is aligned so there can be a gap between this and
-    /// the metatdata).
+    /// the metadata).
     pub fn bodyLength<'a>(&'a self) -> i64 {
         self.bodyLength_.from_little_endian()
     }
diff --git a/rust/arrow/src/ipc/gen/Message.rs b/rust/arrow/src/ipc/gen/Message.rs
index 0907ea84fb9..6765348302a 100644
--- a/rust/arrow/src/ipc/gen/Message.rs
+++ b/rust/arrow/src/ipc/gen/Message.rs
@@ -25,6 +25,123 @@ use flatbuffers::EndianScalar;
 use std::{cmp::Ordering, mem};
 // automatically generated by the FlatBuffers compiler, do not modify
 
+#[allow(non_camel_case_types)]
+#[repr(i8)]
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
+pub enum CompressionType {
+    LZ4_FRAME = 0,
+    ZSTD = 1,
+}
+
+const ENUM_MIN_COMPRESSION_TYPE: i8 = 0;
+const ENUM_MAX_COMPRESSION_TYPE: i8 = 1;
+
+impl<'a> flatbuffers::Follow<'a> for CompressionType {
+    type Inner = Self;
+    #[inline]
+    fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+        flatbuffers::read_scalar_at::<Self>(buf, loc)
+    }
+}
+
+impl flatbuffers::EndianScalar for CompressionType {
+    #[inline]
+    fn to_little_endian(self) -> Self {
+        let n = i8::to_le(self as i8);
+        let p = &n as *const i8 as *const CompressionType;
+        unsafe { *p }
+    }
+    #[inline]
+    fn from_little_endian(self) -> Self {
+        let n = i8::from_le(self as i8);
+        let p = &n as *const i8 as *const CompressionType;
+        unsafe { *p }
+    }
+}
+
+impl flatbuffers::Push for CompressionType {
+    type Output = CompressionType;
+    #[inline]
+    fn push(&self, dst: &mut [u8], _rest: &[u8]) {
+        flatbuffers::emplace_scalar::<CompressionType>(dst, *self);
+    }
+}
+
+#[allow(non_camel_case_types)]
+const ENUM_VALUES_COMPRESSION_TYPE: [CompressionType; 2] =
+    [CompressionType::LZ4_FRAME, CompressionType::ZSTD];
+
+#[allow(non_camel_case_types)]
+const ENUM_NAMES_COMPRESSION_TYPE: [&'static str; 2] = ["LZ4_FRAME", "ZSTD"];
+
+pub fn enum_name_compression_type(e: CompressionType) -> &'static str {
+    let index = e as i8;
+    ENUM_NAMES_COMPRESSION_TYPE[index as usize]
+}
+
+/// Provided for forward compatibility in case we need to support different
+/// strategies for compressing the IPC message body (like whole-body
+/// compression rather than buffer-level) in the future
+#[allow(non_camel_case_types)]
+#[repr(i8)]
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
+pub enum BodyCompressionMethod {
+    /// Each constituent buffer is first compressed with the indicated
+    /// compressor, and then written with the uncompressed length in the first 8
+    /// bytes as a 64-bit little-endian signed integer followed by the compressed
+    /// buffer bytes (and then padding as required by the protocol). The
+    /// uncompressed length may be set to -1 to indicate that the data that
+    /// follows is not compressed, which can be useful for cases where
+    /// compression does not yield appreciable savings.
+    BUFFER = 0,
+}
+
+const ENUM_MIN_BODY_COMPRESSION_METHOD: i8 = 0;
+const ENUM_MAX_BODY_COMPRESSION_METHOD: i8 = 0;
+
+impl<'a> flatbuffers::Follow<'a> for BodyCompressionMethod {
+    type Inner = Self;
+    #[inline]
+    fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+        flatbuffers::read_scalar_at::<Self>(buf, loc)
+    }
+}
+
+impl flatbuffers::EndianScalar for BodyCompressionMethod {
+    #[inline]
+    fn to_little_endian(self) -> Self {
+        let n = i8::to_le(self as i8);
+        let p = &n as *const i8 as *const BodyCompressionMethod;
+        unsafe { *p }
+    }
+    #[inline]
+    fn from_little_endian(self) -> Self {
+        let n = i8::from_le(self as i8);
+        let p = &n as *const i8 as *const BodyCompressionMethod;
+        unsafe { *p }
+    }
+}
+
+impl flatbuffers::Push for BodyCompressionMethod {
+    type Output = BodyCompressionMethod;
+    #[inline]
+    fn push(&self, dst: &mut [u8], _rest: &[u8]) {
+        flatbuffers::emplace_scalar::<BodyCompressionMethod>(dst, *self);
+    }
+}
+
+#[allow(non_camel_case_types)]
+const ENUM_VALUES_BODY_COMPRESSION_METHOD: [BodyCompressionMethod; 1] =
+    [BodyCompressionMethod::BUFFER];
+
+#[allow(non_camel_case_types)]
+const ENUM_NAMES_BODY_COMPRESSION_METHOD: [&'static str; 1] = ["BUFFER"];
+
+pub fn enum_name_body_compression_method(e: BodyCompressionMethod) -> &'static str {
+    let index = e as i8;
+    ENUM_NAMES_BODY_COMPRESSION_METHOD[index as usize]
+}
+
 /// ----------------------------------------------------------------------
 /// The root Message type
 /// This union enables us to easily send different message types without
@@ -111,7 +228,7 @@ pub struct MessageHeaderUnionTableOffset {}
 /// Metadata about a field at some level of a nested type tree (but not
 /// its children).
 ///
-/// For example, a List<Int16> with values [[1, 2, 3], null, [4], [5, 6], null]
+/// For example, a List<Int16> with values `[[1, 2, 3], null, [4], [5, 6], null]`
 /// would have {length: 5, null_count: 2} for its List node, and {length: 6,
 /// null_count: 0} for its Int16 node, as separate FieldNode structs
 // struct FieldNode, aligned to 8
@@ -184,6 +301,118 @@ impl FieldNode {
     }
 }
 
+pub enum BodyCompressionOffset {}
+#[derive(Copy, Clone, Debug, PartialEq)]
+
+/// Optional compression for the memory buffers constituting IPC message
+/// bodies. Intended for use with RecordBatch but could be used for other
+/// message types
+pub struct BodyCompression<'a> {
+    pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for BodyCompression<'a> {
+    type Inner = BodyCompression<'a>;
+    #[inline]
+    fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+        Self {
+            _tab: flatbuffers::Table { buf: buf, loc: loc },
+        }
+    }
+}
+
+impl<'a> BodyCompression<'a> {
+    #[inline]
+    pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+        BodyCompression { _tab: table }
+    }
+    #[allow(unused_mut)]
+    pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>(
+        _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>,
+        args: &'args BodyCompressionArgs,
+    ) -> flatbuffers::WIPOffset<BodyCompression<'bldr>> {
+        let mut builder = BodyCompressionBuilder::new(_fbb);
+        builder.add_method(args.method);
+        builder.add_codec(args.codec);
+        builder.finish()
+    }
+
+    pub const VT_CODEC: flatbuffers::VOffsetT = 4;
+    pub const VT_METHOD: flatbuffers::VOffsetT = 6;
+
+    /// Compressor library
+    #[inline]
+    pub fn codec(&self) -> CompressionType {
+        self._tab
+            .get::<CompressionType>(
+                BodyCompression::VT_CODEC,
+                Some(CompressionType::LZ4_FRAME),
+            )
+            .unwrap()
+    }
+    /// Indicates the way the record batch body was compressed
+    #[inline]
+    pub fn method(&self) -> BodyCompressionMethod {
+        self._tab
+            .get::<BodyCompressionMethod>(
+                BodyCompression::VT_METHOD,
+                Some(BodyCompressionMethod::BUFFER),
+            )
+            .unwrap()
+    }
+}
+
+pub struct BodyCompressionArgs {
+    pub codec: CompressionType,
+    pub method: BodyCompressionMethod,
+}
+impl<'a> Default for BodyCompressionArgs {
+    #[inline]
+    fn default() -> Self {
+        BodyCompressionArgs {
+            codec: CompressionType::LZ4_FRAME,
+            method: BodyCompressionMethod::BUFFER,
+        }
+    }
+}
+pub struct BodyCompressionBuilder<'a: 'b, 'b> {
+    fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>,
+    start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+}
+impl<'a: 'b, 'b> BodyCompressionBuilder<'a, 'b> {
+    #[inline]
+    pub fn add_codec(&mut self, codec: CompressionType) {
+        self.fbb_.push_slot::<CompressionType>(
+            BodyCompression::VT_CODEC,
+            codec,
+            CompressionType::LZ4_FRAME,
+        );
+    }
+    #[inline]
+    pub fn add_method(&mut self, method: BodyCompressionMethod) {
+        self.fbb_.push_slot::<BodyCompressionMethod>(
+            BodyCompression::VT_METHOD,
+            method,
+            BodyCompressionMethod::BUFFER,
+        );
+    }
+    #[inline]
+    pub fn new(
+        _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>,
+    ) -> BodyCompressionBuilder<'a, 'b> {
+        let start = _fbb.start_table();
+        BodyCompressionBuilder {
+            fbb_: _fbb,
+            start_: start,
+        }
+    }
+    #[inline]
+    pub fn finish(self) -> flatbuffers::WIPOffset<BodyCompression<'a>> {
+        let o = self.fbb_.end_table(self.start_);
+        flatbuffers::WIPOffset::new(o.value())
+    }
+}
+
 pub enum RecordBatchOffset {}
 #[derive(Copy, Clone, Debug, PartialEq)]
 
@@ -216,6 +445,9 @@ impl<'a> RecordBatch<'a> {
     ) -> flatbuffers::WIPOffset<RecordBatch<'bldr>> {
         let mut builder = RecordBatchBuilder::new(_fbb);
         builder.add_length(args.length);
+        if let Some(x) = args.compression {
+            builder.add_compression(x);
+        }
         if let Some(x) = args.buffers {
             builder.add_buffers(x);
         }
@@ -228,6 +460,7 @@ impl<'a> RecordBatch<'a> {
     pub const VT_LENGTH: flatbuffers::VOffsetT = 4;
     pub const VT_NODES: flatbuffers::VOffsetT = 6;
     pub const VT_BUFFERS: flatbuffers::VOffsetT = 8;
+    pub const VT_COMPRESSION: flatbuffers::VOffsetT = 10;
 
     /// number of records / rows. The arrays in the batch should all have this
     /// length
@@ -262,12 +495,22 @@ impl<'a> RecordBatch<'a> {
             )
             .map(|v| v.safe_slice())
     }
+    /// Optional compression of the message body
+    #[inline]
+    pub fn compression(&self) -> Option<BodyCompression<'a>> {
+        self._tab
+            .get::<flatbuffers::ForwardsUOffset<BodyCompression<'a>>>(
+                RecordBatch::VT_COMPRESSION,
+                None,
+            )
+    }
 }
 
 pub struct RecordBatchArgs<'a> {
     pub length: i64,
     pub nodes: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, FieldNode>>>,
     pub buffers: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, Buffer>>>,
+    pub compression: Option<flatbuffers::WIPOffset<BodyCompression<'a>>>,
 }
 impl<'a> Default for RecordBatchArgs<'a> {
     #[inline]
@@ -276,6 +519,7 @@ impl<'a> Default for RecordBatchArgs<'a> {
             length: 0,
             nodes: None,
             buffers: None,
+            compression: None,
         }
     }
 }
@@ -308,6 +552,17 @@ impl<'a: 'b, 'b> RecordBatchBuilder<'a, 'b> {
         );
     }
     #[inline]
+    pub fn add_compression(
+        &mut self,
+        compression: flatbuffers::WIPOffset<BodyCompression<'b>>,
+    ) {
+        self.fbb_
+            .push_slot_always::<flatbuffers::WIPOffset<BodyCompression>>(
+                RecordBatch::VT_COMPRESSION,
+                compression,
+            );
+    }
+    #[inline]
     pub fn new(
         _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>,
     ) -> RecordBatchBuilder<'a, 'b> {
diff --git a/rust/arrow/src/ipc/gen/Schema.rs b/rust/arrow/src/ipc/gen/Schema.rs
index 24136adef6a..91ad7177545 100644
--- a/rust/arrow/src/ipc/gen/Schema.rs
+++ b/rust/arrow/src/ipc/gen/Schema.rs
@@ -26,18 +26,24 @@ use std::{cmp::Ordering, mem};
 #[repr(i16)]
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
 pub enum MetadataVersion {
-    /// 0.1.0
+    /// 0.1.0 (October 2016).
     V1 = 0,
-    /// 0.2.0
+    /// 0.2.0 (February 2017). Non-backwards compatible with V1.
     V2 = 1,
-    /// 0.3.0 -> 0.7.1
+    /// 0.3.0 -> 0.7.1 (May - December 2017). Non-backwards compatible with V2.
     V3 = 2,
-    /// >= 0.8.0
+    /// >= 0.8.0 (December 2017). Non-backwards compatible with V3.
     V4 = 3,
+    /// >= 1.0.0 (July 2020. Backwards compatible with V4 (V5 readers can read V4
+    /// metadata and IPC messages). Implementations are recommended to provide a
+    /// V4 compatibility mode with V5 format changes disabled.
+    ///
+    /// TODO: Add list of non-forward compatible changes.
+    V5 = 4,
 }
 
 const ENUM_MIN_METADATA_VERSION: i16 = 0;
-const ENUM_MAX_METADATA_VERSION: i16 = 3;
+const ENUM_MAX_METADATA_VERSION: i16 = 4;
 
 impl<'a> flatbuffers::Follow<'a> for MetadataVersion {
     type Inner = Self;
@@ -71,21 +77,104 @@ impl flatbuffers::Push for MetadataVersion {
 }
 
 #[allow(non_camel_case_types)]
-const ENUM_VALUES_METADATA_VERSION: [MetadataVersion; 4] = [
+const ENUM_VALUES_METADATA_VERSION: [MetadataVersion; 5] = [
     MetadataVersion::V1,
     MetadataVersion::V2,
     MetadataVersion::V3,
     MetadataVersion::V4,
+    MetadataVersion::V5,
 ];
 
 #[allow(non_camel_case_types)]
-const ENUM_NAMES_METADATA_VERSION: [&'static str; 4] = ["V1", "V2", "V3", "V4"];
+const ENUM_NAMES_METADATA_VERSION: [&'static str; 5] = ["V1", "V2", "V3", "V4", "V5"];
 
 pub fn enum_name_metadata_version(e: MetadataVersion) -> &'static str {
     let index = e as i16;
     ENUM_NAMES_METADATA_VERSION[index as usize]
 }
 
+/// Represents Arrow Features that might not have full support
+/// within implementations. This is intended to be used in
+/// two scenarios:
+///  1.  A mechanism for readers of Arrow Streams
+///      and files to understand that the stream or file makes
+///      use of a feature that isn't supported or unknown to
+///      the implementation (and therefore can meet the Arrow
+///      forward compatibility guarantees).
+///  2.  A means of negotiating between a client and server
+///      what features a stream is allowed to use. The enums
+///      values here are intented to represent higher level
+///      features, additional details maybe negotiated
+///      with key-value pairs specific to the protocol.
+///
+/// Enums added to this list should be assigned power-of-two values
+/// to facilitate exchanging and comparing bitmaps for supported
+/// features.
+#[allow(non_camel_case_types)]
+#[repr(i64)]
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
+pub enum Feature {
+    /// Needed to make flatbuffers happy.
+    UNUSED = 0,
+    /// The stream makes use of multiple full dictionaries with the
+    /// same ID and assumes clients implement dictionary replacement
+    /// correctly.
+    DICTIONARY_REPLACEMENT = 1,
+    /// The stream makes use of compressed bodies as described
+    /// in Message.fbs.
+    COMPRESSED_BODY = 2,
+}
+
+const ENUM_MIN_FEATURE: i64 = 0;
+const ENUM_MAX_FEATURE: i64 = 2;
+
+impl<'a> flatbuffers::Follow<'a> for Feature {
+    type Inner = Self;
+    #[inline]
+    fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+        flatbuffers::read_scalar_at::<Self>(buf, loc)
+    }
+}
+
+impl flatbuffers::EndianScalar for Feature {
+    #[inline]
+    fn to_little_endian(self) -> Self {
+        let n = i64::to_le(self as i64);
+        let p = &n as *const i64 as *const Feature;
+        unsafe { *p }
+    }
+    #[inline]
+    fn from_little_endian(self) -> Self {
+        let n = i64::from_le(self as i64);
+        let p = &n as *const i64 as *const Feature;
+        unsafe { *p }
+    }
+}
+
+impl flatbuffers::Push for Feature {
+    type Output = Feature;
+    #[inline]
+    fn push(&self, dst: &mut [u8], _rest: &[u8]) {
+        flatbuffers::emplace_scalar::<Feature>(dst, *self);
+    }
+}
+
+#[allow(non_camel_case_types)]
+const ENUM_VALUES_FEATURE: [Feature; 3] = [
+    Feature::UNUSED,
+    Feature::DICTIONARY_REPLACEMENT,
+    Feature::COMPRESSED_BODY,
+];
+
+#[allow(non_camel_case_types)]
+const ENUM_NAMES_FEATURE: [&'static str; 3] =
+    ["UNUSED", "DICTIONARY_REPLACEMENT", "COMPRESSED_BODY"];
+
+pub fn enum_name_feature(e: Feature) -> &'static str {
+    let index = e as i64;
+    ENUM_NAMES_FEATURE[index as usize]
+}
+
 #[allow(non_camel_case_types)]
 #[repr(i16)]
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
@@ -1008,9 +1097,10 @@ pub enum MapOffset {}
 /// not enforced.
 ///
 /// Map
-///   - child[0] entries: Struct
-///     - child[0] key: K
-///     - child[1] value: V
+///
+///   - `child[0] entries: Struct`
+///     - `child[0] key: K`
+///     - `child[1] value: V`
 ///
 /// Neither the "entries" field nor the "key" field may be nullable.
 ///
@@ -1097,7 +1187,7 @@ pub enum UnionOffset {}
 /// A union is a complex type with children in Field
 /// By default ids in the type vector refer to the offsets in the children
 /// optionally typeIds provides an indirection between the child offset and the type id
-/// for each child typeIds[offset] is the id used in the type vector
+/// for each child `typeIds[offset]` is the id used in the type vector
 pub struct Union<'a> {
     pub _tab: flatbuffers::Table<'a>,
 }
@@ -1757,6 +1847,10 @@ impl<'a: 'b, 'b> BoolBuilder<'a, 'b> {
 pub enum DecimalOffset {}
 #[derive(Copy, Clone, Debug, PartialEq)]
 
+/// Exact decimal value represented as an integer value in two's
+/// complement. Currently only 128-bit (16-byte) integers are used but this may
+/// be expanded in the future. The representation uses the endianness indicated
+/// in the Schema.
 pub struct Decimal<'a> {
     pub _tab: flatbuffers::Table<'a>,
 }
@@ -1782,6 +1876,7 @@ impl<'a> Decimal<'a> {
         args: &'args DecimalArgs,
     ) -> flatbuffers::WIPOffset<Decimal<'bldr>> {
         let mut builder = DecimalBuilder::new(_fbb);
+        builder.add_bitWidth(args.bitWidth);
         builder.add_scale(args.scale);
         builder.add_precision(args.precision);
         builder.finish()
@@ -1789,6 +1884,7 @@ impl<'a> Decimal<'a> {
 
     pub const VT_PRECISION: flatbuffers::VOffsetT = 4;
     pub const VT_SCALE: flatbuffers::VOffsetT = 6;
+    pub const VT_BITWIDTH: flatbuffers::VOffsetT = 8;
 
     /// Total number of decimal digits
     #[inline]
@@ -1802,11 +1898,22 @@ impl<'a> Decimal<'a> {
     pub fn scale(&self) -> i32 {
         self._tab.get::<i32>(Decimal::VT_SCALE, Some(0)).unwrap()
     }
+    /// Number of bits per value. The only accepted width right now is 128 but
+    /// this field exists for forward compatibility so that other bit widths may
+    /// be supported in future format versions. We use bitWidth for consistency
+    /// with Int::bitWidth.
+    #[inline]
+    pub fn bitWidth(&self) -> i32 {
+        self._tab
+            .get::<i32>(Decimal::VT_BITWIDTH, Some(128))
+            .unwrap()
+    }
 }
 
 pub struct DecimalArgs {
     pub precision: i32,
     pub scale: i32,
+    pub bitWidth: i32,
 }
 impl<'a> Default for DecimalArgs {
     #[inline]
@@ -1814,6 +1921,7 @@ impl<'a> Default for DecimalArgs {
         DecimalArgs {
             precision: 0,
             scale: 0,
+            bitWidth: 128,
         }
     }
 }
@@ -1832,6 +1940,11 @@ impl<'a: 'b, 'b> DecimalBuilder<'a, 'b> {
         self.fbb_.push_slot::<i32>(Decimal::VT_SCALE, scale, 0);
     }
     #[inline]
+    pub fn add_bitWidth(&mut self, bitWidth: i32) {
+        self.fbb_
+            .push_slot::<i32>(Decimal::VT_BITWIDTH, bitWidth, 128);
+    }
+    #[inline]
     pub fn new(
         _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>,
     ) -> DecimalBuilder<'a, 'b> {
@@ -2463,8 +2576,11 @@ impl<'a> DictionaryEncoding<'a> {
             .get::<i64>(DictionaryEncoding::VT_ID, Some(0))
             .unwrap()
     }
-    /// The dictionary indices are constrained to be positive integers. If this
-    /// field is null, the indices must be signed int32
+    /// The dictionary indices are constrained to be non-negative integers. If
+    /// this field is null, the indices must be signed int32. To maximize
+    /// cross-language compatibility and performance, implementations are
+    /// recommended to prefer signed integer types over unsigned integer types
+    /// and to avoid uint64 indices unless they are required by an application.
     #[inline]
     pub fn indexType(&self) -> Option<Int<'a>> {
         self._tab.get::<flatbuffers::ForwardsUOffset<Int<'a>>>(
@@ -3019,6 +3135,9 @@ impl<'a> Schema<'a> {
         args: &'args SchemaArgs<'args>,
     ) -> flatbuffers::WIPOffset<Schema<'bldr>> {
         let mut builder = SchemaBuilder::new(_fbb);
+        if let Some(x) = args.features {
+            builder.add_features(x);
+        }
         if let Some(x) = args.custom_metadata {
             builder.add_custom_metadata(x);
         }
@@ -3032,6 +3151,7 @@ impl<'a> Schema<'a> {
     pub const VT_ENDIANNESS: flatbuffers::VOffsetT = 4;
     pub const VT_FIELDS: flatbuffers::VOffsetT = 6;
     pub const VT_CUSTOM_METADATA: flatbuffers::VOffsetT = 8;
+    pub const VT_FEATURES: flatbuffers::VOffsetT = 10;
 
     /// endianness of the buffer
     /// it is Little Endian by default
@@ -3058,6 +3178,15 @@ impl<'a> Schema<'a> {
             flatbuffers::Vector<flatbuffers::ForwardsUOffset<KeyValue<'a>>>,
         >>(Schema::VT_CUSTOM_METADATA, None)
     }
+    /// Features used in the stream/file.
+    #[inline]
+    pub fn features(&self) -> Option<flatbuffers::Vector<'a, Feature>> {
+        self._tab
+            .get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, Feature>>>(
+                Schema::VT_FEATURES,
+                None,
+            )
+    }
 }
 
 pub struct SchemaArgs<'a> {
@@ -3072,6 +3201,7 @@ pub struct SchemaArgs<'a> {
             flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset<KeyValue<'a>>>,
         >,
     >,
+    pub features: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, Feature>>>,
 }
 impl<'a> Default for SchemaArgs<'a> {
     #[inline]
@@ -3080,6 +3210,7 @@ impl<'a> Default for SchemaArgs<'a> {
             endianness: Endianness::Little,
             fields: None,
             custom_metadata: None,
+            features: None,
         }
     }
 }
@@ -3119,6 +3250,14 @@ impl<'a: 'b, 'b> SchemaBuilder<'a, 'b> {
         );
     }
     #[inline]
+    pub fn add_features(
+        &mut self,
+        features: flatbuffers::WIPOffset<flatbuffers::Vector<'b, Feature>>,
+    ) {
+        self.fbb_
+            .push_slot_always::<flatbuffers::WIPOffset<_>>(Schema::VT_FEATURES, features);
+    }
+    #[inline]
     pub fn new(
         _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>,
     ) -> SchemaBuilder<'a, 'b> {
diff --git a/rust/arrow/src/ipc/gen/SparseTensor.rs b/rust/arrow/src/ipc/gen/SparseTensor.rs
index c7168973ebf..a6d037ae476 100644
--- a/rust/arrow/src/ipc/gen/SparseTensor.rs
+++ b/rust/arrow/src/ipc/gen/SparseTensor.rs
@@ -89,10 +89,11 @@ pub enum SparseTensorIndex {
     NONE = 0,
     SparseTensorIndexCOO = 1,
     SparseMatrixIndexCSX = 2,
+    SparseTensorIndexCSF = 3,
 }
 
 const ENUM_MIN_SPARSE_TENSOR_INDEX: u8 = 0;
-const ENUM_MAX_SPARSE_TENSOR_INDEX: u8 = 2;
+const ENUM_MAX_SPARSE_TENSOR_INDEX: u8 = 3;
 
 impl<'a> flatbuffers::Follow<'a> for SparseTensorIndex {
     type Inner = Self;
@@ -126,15 +127,20 @@ impl flatbuffers::Push for SparseTensorIndex {
 }
 
 #[allow(non_camel_case_types)]
-const ENUM_VALUES_SPARSE_TENSOR_INDEX: [SparseTensorIndex; 3] = [
+const ENUM_VALUES_SPARSE_TENSOR_INDEX: [SparseTensorIndex; 4] = [
     SparseTensorIndex::NONE,
     SparseTensorIndex::SparseTensorIndexCOO,
     SparseTensorIndex::SparseMatrixIndexCSX,
+    SparseTensorIndex::SparseTensorIndexCSF,
 ];
 
 #[allow(non_camel_case_types)]
-const ENUM_NAMES_SPARSE_TENSOR_INDEX: [&'static str; 3] =
-    ["NONE", "SparseTensorIndexCOO", "SparseMatrixIndexCSX"];
+const ENUM_NAMES_SPARSE_TENSOR_INDEX: [&'static str; 4] = [
+    "NONE",
+    "SparseTensorIndexCOO",
+    "SparseMatrixIndexCSX",
+    "SparseTensorIndexCSF",
+];
 
 pub fn enum_name_sparse_tensor_index(e: SparseTensorIndex) -> &'static str {
     let index = e as u8;
@@ -218,13 +224,16 @@ impl<'a> SparseTensorIndexCOO<'a> {
 
     /// The type of values in indicesBuffer
     #[inline]
-    pub fn indicesType(&self) -> Option<Int<'a>> {
-        self._tab.get::<flatbuffers::ForwardsUOffset<Int<'a>>>(
-            SparseTensorIndexCOO::VT_INDICESTYPE,
-            None,
-        )
+    pub fn indicesType(&self) -> Int<'a> {
+        self._tab
+            .get::<flatbuffers::ForwardsUOffset<Int<'a>>>(
+                SparseTensorIndexCOO::VT_INDICESTYPE,
+                None,
+            )
+            .unwrap()
     }
     /// Non-negative byte offsets to advance one value cell along each dimension
+    /// If omitted, default to row-major order (C-like).
     #[inline]
     pub fn indicesStrides(&self) -> Option<flatbuffers::Vector<'a, i64>> {
         self._tab
@@ -235,9 +244,10 @@ impl<'a> SparseTensorIndexCOO<'a> {
     }
     /// The location and size of the indices matrix's data
     #[inline]
-    pub fn indicesBuffer(&self) -> Option<&'a Buffer> {
+    pub fn indicesBuffer(&self) -> &'a Buffer {
         self._tab
             .get::<Buffer>(SparseTensorIndexCOO::VT_INDICESBUFFER, None)
+            .unwrap()
     }
 }
 
@@ -250,9 +260,9 @@ impl<'a> Default for SparseTensorIndexCOOArgs<'a> {
     #[inline]
     fn default() -> Self {
         SparseTensorIndexCOOArgs {
-            indicesType: None,
+            indicesType: None, // required field
             indicesStrides: None,
-            indicesBuffer: None,
+            indicesBuffer: None, // required field
         }
     }
 }
@@ -298,6 +308,10 @@ impl<'a: 'b, 'b> SparseTensorIndexCOOBuilder<'a, 'b> {
     #[inline]
     pub fn finish(self) -> flatbuffers::WIPOffset<SparseTensorIndexCOO<'a>> {
         let o = self.fbb_.end_table(self.start_);
+        self.fbb_
+            .required(o, SparseTensorIndexCOO::VT_INDICESTYPE, "indices_type");
+        self.fbb_
+            .required(o, SparseTensorIndexCOO::VT_INDICESBUFFER, "indices_buffer");
         flatbuffers::WIPOffset::new(o.value())
     }
 }
@@ -365,15 +379,17 @@ impl<'a> SparseMatrixIndexCSX<'a> {
     }
     /// The type of values in indptrBuffer
     #[inline]
-    pub fn indptrType(&self) -> Option<Int<'a>> {
-        self._tab.get::<flatbuffers::ForwardsUOffset<Int<'a>>>(
-            SparseMatrixIndexCSX::VT_INDPTRTYPE,
-            None,
-        )
+    pub fn indptrType(&self) -> Int<'a> {
+        self._tab
+            .get::<flatbuffers::ForwardsUOffset<Int<'a>>>(
+                SparseMatrixIndexCSX::VT_INDPTRTYPE,
+                None,
+            )
+            .unwrap()
     }
     /// indptrBuffer stores the location and size of indptr array that
     /// represents the range of the rows.
-    /// The i-th row spans from indptr[i] to indptr[i+1] in the data.
+    /// The i-th row spans from `indptr[i]` to `indptr[i+1]` in the data.
     /// The length of this array is 1 + (the number of rows), and the type
     /// of index value is long.
     ///
@@ -394,17 +410,20 @@ impl<'a> SparseMatrixIndexCSX<'a> {
     ///
     ///   indptr(X) = [0, 2, 3, 5, 5, 8, 10].
     #[inline]
-    pub fn indptrBuffer(&self) -> Option<&'a Buffer> {
+    pub fn indptrBuffer(&self) -> &'a Buffer {
         self._tab
             .get::<Buffer>(SparseMatrixIndexCSX::VT_INDPTRBUFFER, None)
+            .unwrap()
     }
     /// The type of values in indicesBuffer
     #[inline]
-    pub fn indicesType(&self) -> Option<Int<'a>> {
-        self._tab.get::<flatbuffers::ForwardsUOffset<Int<'a>>>(
-            SparseMatrixIndexCSX::VT_INDICESTYPE,
-            None,
-        )
+    pub fn indicesType(&self) -> Int<'a> {
+        self._tab
+            .get::<flatbuffers::ForwardsUOffset<Int<'a>>>(
+                SparseMatrixIndexCSX::VT_INDICESTYPE,
+                None,
+            )
+            .unwrap()
     }
     /// indicesBuffer stores the location and size of the array that
     /// contains the column indices of the corresponding non-zero values.
@@ -416,9 +435,10 @@ impl<'a> SparseMatrixIndexCSX<'a> {
     ///
     /// Note that the indices are sorted in lexicographical order for each row.
     #[inline]
-    pub fn indicesBuffer(&self) -> Option<&'a Buffer> {
+    pub fn indicesBuffer(&self) -> &'a Buffer {
         self._tab
             .get::<Buffer>(SparseMatrixIndexCSX::VT_INDICESBUFFER, None)
+            .unwrap()
     }
 }
 
@@ -434,10 +454,10 @@ impl<'a> Default for SparseMatrixIndexCSXArgs<'a> {
     fn default() -> Self {
         SparseMatrixIndexCSXArgs {
             compressedAxis: SparseMatrixCompressedAxis::Row,
-            indptrType: None,
-            indptrBuffer: None,
-            indicesType: None,
-            indicesBuffer: None,
+            indptrType: None,    // required field
+            indptrBuffer: None,  // required field
+            indicesType: None,   // required field
+            indicesBuffer: None, // required field
         }
     }
 }
@@ -495,6 +515,276 @@ impl<'a: 'b, 'b> SparseMatrixIndexCSXBuilder<'a, 'b> {
     #[inline]
     pub fn finish(self) -> flatbuffers::WIPOffset<SparseMatrixIndexCSX<'a>> {
         let o = self.fbb_.end_table(self.start_);
+        self.fbb_
+            .required(o, SparseMatrixIndexCSX::VT_INDPTRTYPE, "indptr_type");
+        self.fbb_
+            .required(o, SparseMatrixIndexCSX::VT_INDPTRBUFFER, "indptr_buffer");
+        self.fbb_
+            .required(o, SparseMatrixIndexCSX::VT_INDICESTYPE, "indices_type");
+        self.fbb_
+            .required(o, SparseMatrixIndexCSX::VT_INDICESBUFFER, "indices_buffer");
+        flatbuffers::WIPOffset::new(o.value())
+    }
+}
+
+pub enum SparseTensorIndexCSFOffset {}
+#[derive(Copy, Clone, Debug, PartialEq)]
+
+/// Compressed Sparse Fiber (CSF) sparse tensor index.
+pub struct SparseTensorIndexCSF<'a> {
+    pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for SparseTensorIndexCSF<'a> {
+    type Inner = SparseTensorIndexCSF<'a>;
+    #[inline]
+    fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+        Self {
+            _tab: flatbuffers::Table { buf: buf, loc: loc },
+        }
+    }
+}
+
+impl<'a> SparseTensorIndexCSF<'a> {
+    #[inline]
+    pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+        SparseTensorIndexCSF { _tab: table }
+    }
+    #[allow(unused_mut)]
+    pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>(
+        _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>,
+        args: &'args SparseTensorIndexCSFArgs<'args>,
+    ) -> flatbuffers::WIPOffset<SparseTensorIndexCSF<'bldr>> {
+        let mut builder = SparseTensorIndexCSFBuilder::new(_fbb);
+        if let Some(x) = args.axisOrder {
+            builder.add_axisOrder(x);
+        }
+        if let Some(x) = args.indicesBuffers {
+            builder.add_indicesBuffers(x);
+        }
+        if let Some(x) = args.indicesType {
+            builder.add_indicesType(x);
+        }
+        if let Some(x) = args.indptrBuffers {
+            builder.add_indptrBuffers(x);
+        }
+        if let Some(x) = args.indptrType {
+            builder.add_indptrType(x);
+        }
+        builder.finish()
+    }
+
+    pub const VT_INDPTRTYPE: flatbuffers::VOffsetT = 4;
+    pub const VT_INDPTRBUFFERS: flatbuffers::VOffsetT = 6;
+    pub const VT_INDICESTYPE: flatbuffers::VOffsetT = 8;
+    pub const VT_INDICESBUFFERS: flatbuffers::VOffsetT = 10;
+    pub const VT_AXISORDER: flatbuffers::VOffsetT = 12;
+
+    /// CSF is a generalization of compressed sparse row (CSR) index.
+    /// See [smith2017knl](http://shaden.io/pub-files/smith2017knl.pdf)
+    ///
+    /// CSF index recursively compresses each dimension of a tensor into a set
+    /// of prefix trees. Each path from a root to leaf forms one tensor
+    /// non-zero index. CSF is implemented with two arrays of buffers and one
+    /// arrays of integers.
+    ///
+    /// For example, let X be a 2x3x4x5 tensor and let it have the following
+    /// 8 non-zero values:
+    ///
+    ///   X[0, 0, 0, 1] := 1
+    ///   X[0, 0, 0, 2] := 2
+    ///   X[0, 1, 0, 0] := 3
+    ///   X[0, 1, 0, 2] := 4
+    ///   X[0, 1, 1, 0] := 5
+    ///   X[1, 1, 1, 0] := 6
+    ///   X[1, 1, 1, 1] := 7
+    ///   X[1, 1, 1, 2] := 8
+    ///
+    /// As a prefix tree this would be represented as:
+    ///
+    /// //        0          1
+    /// //       / \         |
+    /// //      0   1        1
+    /// //     /   / \       |
+    /// //    0   0   1      1
+    /// //   /|  /|   |    /| |
+    /// //  1 2 0 2   0   0 1 2
+    /// The type of values in indptrBuffers
+    #[inline]
+    pub fn indptrType(&self) -> Int<'a> {
+        self._tab
+            .get::<flatbuffers::ForwardsUOffset<Int<'a>>>(
+                SparseTensorIndexCSF::VT_INDPTRTYPE,
+                None,
+            )
+            .unwrap()
+    }
+    /// indptrBuffers stores the sparsity structure.
+    /// Each two consecutive dimensions in a tensor correspond to a buffer in
+    /// indptrBuffers. A pair of consecutive values at `indptrBuffers[dim][i]`
+    /// and `indptrBuffers[dim][i + 1]` signify a range of nodes in
+    /// `indicesBuffers[dim + 1]` who are children of `indicesBuffers[dim][i]` node.
+    ///
+    /// For example, the indptrBuffers for the above X is:
+    ///
+    ///   indptrBuffer(X) = [
+    ///                       [0, 2, 3],
+    ///                       [0, 1, 3, 4],
+    ///                       [0, 2, 4, 5, 8]
+    ///                     ].
+    ///
+    #[inline]
+    pub fn indptrBuffers(&self) -> &'a [Buffer] {
+        self._tab
+            .get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<Buffer>>>(
+                SparseTensorIndexCSF::VT_INDPTRBUFFERS,
+                None,
+            )
+            .map(|v| v.safe_slice())
+            .unwrap()
+    }
+    /// The type of values in indicesBuffers
+    #[inline]
+    pub fn indicesType(&self) -> Int<'a> {
+        self._tab
+            .get::<flatbuffers::ForwardsUOffset<Int<'a>>>(
+                SparseTensorIndexCSF::VT_INDICESTYPE,
+                None,
+            )
+            .unwrap()
+    }
+    /// indicesBuffers stores values of nodes.
+    /// Each tensor dimension corresponds to a buffer in indicesBuffers.
+    /// For example, the indicesBuffers for the above X is:
+    ///
+    ///   indicesBuffer(X) = [
+    ///                        [0, 1],
+    ///                        [0, 1, 1],
+    ///                        [0, 0, 1, 1],
+    ///                        [1, 2, 0, 2, 0, 0, 1, 2]
+    ///                      ].
+    ///
+    #[inline]
+    pub fn indicesBuffers(&self) -> &'a [Buffer] {
+        self._tab
+            .get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<Buffer>>>(
+                SparseTensorIndexCSF::VT_INDICESBUFFERS,
+                None,
+            )
+            .map(|v| v.safe_slice())
+            .unwrap()
+    }
+    /// axisOrder stores the sequence in which dimensions were traversed to
+    /// produce the prefix tree.
+    /// For example, the axisOrder for the above X is:
+    ///
+    ///   axisOrder(X) = [0, 1, 2, 3].
+    ///
+    #[inline]
+    pub fn axisOrder(&self) -> flatbuffers::Vector<'a, i32> {
+        self._tab
+            .get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, i32>>>(
+                SparseTensorIndexCSF::VT_AXISORDER,
+                None,
+            )
+            .unwrap()
+    }
+}
+
+pub struct SparseTensorIndexCSFArgs<'a> {
+    pub indptrType: Option<flatbuffers::WIPOffset<Int<'a>>>,
+    pub indptrBuffers: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, Buffer>>>,
+    pub indicesType: Option<flatbuffers::WIPOffset<Int<'a>>>,
+    pub indicesBuffers: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, Buffer>>>,
+    pub axisOrder: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, i32>>>,
+}
+impl<'a> Default for SparseTensorIndexCSFArgs<'a> {
+    #[inline]
+    fn default() -> Self {
+        SparseTensorIndexCSFArgs {
+            indptrType: None,     // required field
+            indptrBuffers: None,  // required field
+            indicesType: None,    // required field
+            indicesBuffers: None, // required field
+            axisOrder: None,      // required field
+        }
+    }
+}
+pub struct SparseTensorIndexCSFBuilder<'a: 'b, 'b> {
+    fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>,
+    start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+}
+impl<'a: 'b, 'b> SparseTensorIndexCSFBuilder<'a, 'b> {
+    #[inline]
+    pub fn add_indptrType(&mut self, indptrType: flatbuffers::WIPOffset<Int<'b>>) {
+        self.fbb_.push_slot_always::<flatbuffers::WIPOffset<Int>>(
+            SparseTensorIndexCSF::VT_INDPTRTYPE,
+            indptrType,
+        );
+    }
+    #[inline]
+    pub fn add_indptrBuffers(
+        &mut self,
+        indptrBuffers: flatbuffers::WIPOffset<flatbuffers::Vector<'b, Buffer>>,
+    ) {
+        self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(
+            SparseTensorIndexCSF::VT_INDPTRBUFFERS,
+            indptrBuffers,
+        );
+    }
+    #[inline]
+    pub fn add_indicesType(&mut self, indicesType: flatbuffers::WIPOffset<Int<'b>>) {
+        self.fbb_.push_slot_always::<flatbuffers::WIPOffset<Int>>(
+            SparseTensorIndexCSF::VT_INDICESTYPE,
+            indicesType,
+        );
+    }
+    #[inline]
+    pub fn add_indicesBuffers(
+        &mut self,
+        indicesBuffers: flatbuffers::WIPOffset<flatbuffers::Vector<'b, Buffer>>,
+    ) {
+        self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(
+            SparseTensorIndexCSF::VT_INDICESBUFFERS,
+            indicesBuffers,
+        );
+    }
+    #[inline]
+    pub fn add_axisOrder(
+        &mut self,
+        axisOrder: flatbuffers::WIPOffset<flatbuffers::Vector<'b, i32>>,
+    ) {
+        self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(
+            SparseTensorIndexCSF::VT_AXISORDER,
+            axisOrder,
+        );
+    }
+    #[inline]
+    pub fn new(
+        _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>,
+    ) -> SparseTensorIndexCSFBuilder<'a, 'b> {
+        let start = _fbb.start_table();
+        SparseTensorIndexCSFBuilder {
+            fbb_: _fbb,
+            start_: start,
+        }
+    }
+    #[inline]
+    pub fn finish(self) -> flatbuffers::WIPOffset<SparseTensorIndexCSF<'a>> {
+        let o = self.fbb_.end_table(self.start_);
+        self.fbb_
+            .required(o, SparseTensorIndexCSF::VT_INDPTRTYPE, "indptr_type");
+        self.fbb_
+            .required(o, SparseTensorIndexCSF::VT_INDPTRBUFFERS, "indptr_buffers");
+        self.fbb_
+            .required(o, SparseTensorIndexCSF::VT_INDICESTYPE, "indices_type");
+        self.fbb_.required(
+            o,
+            SparseTensorIndexCSF::VT_INDICESBUFFERS,
+            "indices_buffers",
+        );
+        self.fbb_
+            .required(o, SparseTensorIndexCSF::VT_AXISORDER, "axis_order");
         flatbuffers::WIPOffset::new(o.value())
     }
 }
@@ -563,22 +853,24 @@ impl<'a> SparseTensor<'a> {
     /// Currently only fixed-width value types are supported,
     /// no strings or nested types.
     #[inline]
-    pub fn type_(&self) -> Option<flatbuffers::Table<'a>> {
+    pub fn type_(&self) -> flatbuffers::Table<'a> {
         self._tab
             .get::<flatbuffers::ForwardsUOffset<flatbuffers::Table<'a>>>(
                 SparseTensor::VT_TYPE_,
                 None,
             )
+            .unwrap()
     }
     /// The dimensions of the tensor, optionally named.
     #[inline]
     pub fn shape(
         &self,
-    ) -> Option<flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset<TensorDim<'a>>>>
-    {
-        self._tab.get::<flatbuffers::ForwardsUOffset<
-            flatbuffers::Vector<flatbuffers::ForwardsUOffset<TensorDim<'a>>>,
-        >>(SparseTensor::VT_SHAPE, None)
+    ) -> flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset<TensorDim<'a>>> {
+        self._tab
+            .get::<flatbuffers::ForwardsUOffset<
+                flatbuffers::Vector<flatbuffers::ForwardsUOffset<TensorDim<'a>>>,
+            >>(SparseTensor::VT_SHAPE, None)
+            .unwrap()
     }
     /// The number of non-zero values in a sparse tensor.
     #[inline]
@@ -598,23 +890,26 @@ impl<'a> SparseTensor<'a> {
     }
     /// Sparse tensor index
     #[inline]
-    pub fn sparseIndex(&self) -> Option<flatbuffers::Table<'a>> {
+    pub fn sparseIndex(&self) -> flatbuffers::Table<'a> {
         self._tab
             .get::<flatbuffers::ForwardsUOffset<flatbuffers::Table<'a>>>(
                 SparseTensor::VT_SPARSEINDEX,
                 None,
             )
+            .unwrap()
     }
     /// The location and size of the tensor's data
     #[inline]
-    pub fn data(&self) -> Option<&'a Buffer> {
-        self._tab.get::<Buffer>(SparseTensor::VT_DATA, None)
+    pub fn data(&self) -> &'a Buffer {
+        self._tab
+            .get::<Buffer>(SparseTensor::VT_DATA, None)
+            .unwrap()
     }
     #[inline]
     #[allow(non_snake_case)]
     pub fn type_as_null(&self) -> Option<Null<'a>> {
         if self.type_type() == Type::Null {
-            self.type_().map(|u| Null::init_from_table(u))
+            Some(Null::init_from_table(self.type_()))
         } else {
             None
         }
@@ -624,7 +919,7 @@ impl<'a> SparseTensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_int(&self) -> Option<Int<'a>> {
         if self.type_type() == Type::Int {
-            self.type_().map(|u| Int::init_from_table(u))
+            Some(Int::init_from_table(self.type_()))
         } else {
             None
         }
@@ -634,7 +929,7 @@ impl<'a> SparseTensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_floating_point(&self) -> Option<FloatingPoint<'a>> {
         if self.type_type() == Type::FloatingPoint {
-            self.type_().map(|u| FloatingPoint::init_from_table(u))
+            Some(FloatingPoint::init_from_table(self.type_()))
         } else {
             None
         }
@@ -644,7 +939,7 @@ impl<'a> SparseTensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_binary(&self) -> Option<Binary<'a>> {
         if self.type_type() == Type::Binary {
-            self.type_().map(|u| Binary::init_from_table(u))
+            Some(Binary::init_from_table(self.type_()))
         } else {
             None
         }
@@ -654,7 +949,7 @@ impl<'a> SparseTensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_utf_8(&self) -> Option<Utf8<'a>> {
         if self.type_type() == Type::Utf8 {
-            self.type_().map(|u| Utf8::init_from_table(u))
+            Some(Utf8::init_from_table(self.type_()))
         } else {
             None
         }
@@ -664,7 +959,7 @@ impl<'a> SparseTensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_bool(&self) -> Option<Bool<'a>> {
         if self.type_type() == Type::Bool {
-            self.type_().map(|u| Bool::init_from_table(u))
+            Some(Bool::init_from_table(self.type_()))
         } else {
             None
         }
@@ -674,7 +969,7 @@ impl<'a> SparseTensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_decimal(&self) -> Option<Decimal<'a>> {
         if self.type_type() == Type::Decimal {
-            self.type_().map(|u| Decimal::init_from_table(u))
+            Some(Decimal::init_from_table(self.type_()))
         } else {
             None
         }
@@ -684,7 +979,7 @@ impl<'a> SparseTensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_date(&self) -> Option<Date<'a>> {
         if self.type_type() == Type::Date {
-            self.type_().map(|u| Date::init_from_table(u))
+            Some(Date::init_from_table(self.type_()))
         } else {
             None
         }
@@ -694,7 +989,7 @@ impl<'a> SparseTensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_time(&self) -> Option<Time<'a>> {
         if self.type_type() == Type::Time {
-            self.type_().map(|u| Time::init_from_table(u))
+            Some(Time::init_from_table(self.type_()))
         } else {
             None
         }
@@ -704,7 +999,7 @@ impl<'a> SparseTensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_timestamp(&self) -> Option<Timestamp<'a>> {
         if self.type_type() == Type::Timestamp {
-            self.type_().map(|u| Timestamp::init_from_table(u))
+            Some(Timestamp::init_from_table(self.type_()))
         } else {
             None
         }
@@ -714,7 +1009,7 @@ impl<'a> SparseTensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_interval(&self) -> Option<Interval<'a>> {
         if self.type_type() == Type::Interval {
-            self.type_().map(|u| Interval::init_from_table(u))
+            Some(Interval::init_from_table(self.type_()))
         } else {
             None
         }
@@ -724,7 +1019,7 @@ impl<'a> SparseTensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_list(&self) -> Option<List<'a>> {
         if self.type_type() == Type::List {
-            self.type_().map(|u| List::init_from_table(u))
+            Some(List::init_from_table(self.type_()))
         } else {
             None
         }
@@ -734,7 +1029,7 @@ impl<'a> SparseTensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_struct_(&self) -> Option<Struct_<'a>> {
         if self.type_type() == Type::Struct_ {
-            self.type_().map(|u| Struct_::init_from_table(u))
+            Some(Struct_::init_from_table(self.type_()))
         } else {
             None
         }
@@ -744,7 +1039,7 @@ impl<'a> SparseTensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_union(&self) -> Option<Union<'a>> {
         if self.type_type() == Type::Union {
-            self.type_().map(|u| Union::init_from_table(u))
+            Some(Union::init_from_table(self.type_()))
         } else {
             None
         }
@@ -754,7 +1049,7 @@ impl<'a> SparseTensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_fixed_size_binary(&self) -> Option<FixedSizeBinary<'a>> {
         if self.type_type() == Type::FixedSizeBinary {
-            self.type_().map(|u| FixedSizeBinary::init_from_table(u))
+            Some(FixedSizeBinary::init_from_table(self.type_()))
         } else {
             None
         }
@@ -764,7 +1059,7 @@ impl<'a> SparseTensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_fixed_size_list(&self) -> Option<FixedSizeList<'a>> {
         if self.type_type() == Type::FixedSizeList {
-            self.type_().map(|u| FixedSizeList::init_from_table(u))
+            Some(FixedSizeList::init_from_table(self.type_()))
         } else {
             None
         }
@@ -774,7 +1069,7 @@ impl<'a> SparseTensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_map(&self) -> Option<Map<'a>> {
         if self.type_type() == Type::Map {
-            self.type_().map(|u| Map::init_from_table(u))
+            Some(Map::init_from_table(self.type_()))
         } else {
             None
         }
@@ -784,7 +1079,7 @@ impl<'a> SparseTensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_duration(&self) -> Option<Duration<'a>> {
         if self.type_type() == Type::Duration {
-            self.type_().map(|u| Duration::init_from_table(u))
+            Some(Duration::init_from_table(self.type_()))
         } else {
             None
         }
@@ -794,7 +1089,7 @@ impl<'a> SparseTensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_large_binary(&self) -> Option<LargeBinary<'a>> {
         if self.type_type() == Type::LargeBinary {
-            self.type_().map(|u| LargeBinary::init_from_table(u))
+            Some(LargeBinary::init_from_table(self.type_()))
         } else {
             None
         }
@@ -804,7 +1099,7 @@ impl<'a> SparseTensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_large_utf_8(&self) -> Option<LargeUtf8<'a>> {
         if self.type_type() == Type::LargeUtf8 {
-            self.type_().map(|u| LargeUtf8::init_from_table(u))
+            Some(LargeUtf8::init_from_table(self.type_()))
         } else {
             None
         }
@@ -814,7 +1109,7 @@ impl<'a> SparseTensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_large_list(&self) -> Option<LargeList<'a>> {
         if self.type_type() == Type::LargeList {
-            self.type_().map(|u| LargeList::init_from_table(u))
+            Some(LargeList::init_from_table(self.type_()))
         } else {
             None
         }
@@ -826,8 +1121,7 @@ impl<'a> SparseTensor<'a> {
         &self,
     ) -> Option<SparseTensorIndexCOO<'a>> {
         if self.sparseIndex_type() == SparseTensorIndex::SparseTensorIndexCOO {
-            self.sparseIndex()
-                .map(|u| SparseTensorIndexCOO::init_from_table(u))
+            Some(SparseTensorIndexCOO::init_from_table(self.sparseIndex()))
         } else {
             None
         }
@@ -839,8 +1133,19 @@ impl<'a> SparseTensor<'a> {
         &self,
     ) -> Option<SparseMatrixIndexCSX<'a>> {
         if self.sparseIndex_type() == SparseTensorIndex::SparseMatrixIndexCSX {
-            self.sparseIndex()
-                .map(|u| SparseMatrixIndexCSX::init_from_table(u))
+            Some(SparseMatrixIndexCSX::init_from_table(self.sparseIndex()))
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    #[allow(non_snake_case)]
+    pub fn sparseIndex_as_sparse_tensor_index_csf(
+        &self,
+    ) -> Option<SparseTensorIndexCSF<'a>> {
+        if self.sparseIndex_type() == SparseTensorIndex::SparseTensorIndexCSF {
+            Some(SparseTensorIndexCSF::init_from_table(self.sparseIndex()))
         } else {
             None
         }
@@ -865,12 +1170,12 @@ impl<'a> Default for SparseTensorArgs<'a> {
     fn default() -> Self {
         SparseTensorArgs {
             type_type: Type::NONE,
-            type_: None,
-            shape: None,
+            type_: None, // required field
+            shape: None, // required field
             non_zero_length: 0,
             sparseIndex_type: SparseTensorIndex::NONE,
-            sparseIndex: None,
-            data: None,
+            sparseIndex: None, // required field
+            data: None,        // required field
         }
     }
 }
@@ -943,6 +1248,11 @@ impl<'a: 'b, 'b> SparseTensorBuilder<'a, 'b> {
     #[inline]
     pub fn finish(self) -> flatbuffers::WIPOffset<SparseTensor<'a>> {
         let o = self.fbb_.end_table(self.start_);
+        self.fbb_.required(o, SparseTensor::VT_TYPE_, "type_");
+        self.fbb_.required(o, SparseTensor::VT_SHAPE, "shape");
+        self.fbb_
+            .required(o, SparseTensor::VT_SPARSEINDEX, "sparse_index");
+        self.fbb_.required(o, SparseTensor::VT_DATA, "data");
         flatbuffers::WIPOffset::new(o.value())
     }
 }
diff --git a/rust/arrow/src/ipc/gen/Tensor.rs b/rust/arrow/src/ipc/gen/Tensor.rs
index 9cc2885e338..8532cee4770 100644
--- a/rust/arrow/src/ipc/gen/Tensor.rs
+++ b/rust/arrow/src/ipc/gen/Tensor.rs
@@ -180,24 +180,27 @@ impl<'a> Tensor<'a> {
     /// The type of data contained in a value cell. Currently only fixed-width
     /// value types are supported, no strings or nested types
     #[inline]
-    pub fn type_(&self) -> Option<flatbuffers::Table<'a>> {
+    pub fn type_(&self) -> flatbuffers::Table<'a> {
         self._tab
             .get::<flatbuffers::ForwardsUOffset<flatbuffers::Table<'a>>>(
                 Tensor::VT_TYPE_,
                 None,
             )
+            .unwrap()
     }
     /// The dimensions of the tensor, optionally named
     #[inline]
     pub fn shape(
         &self,
-    ) -> Option<flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset<TensorDim<'a>>>>
-    {
-        self._tab.get::<flatbuffers::ForwardsUOffset<
-            flatbuffers::Vector<flatbuffers::ForwardsUOffset<TensorDim<'a>>>,
-        >>(Tensor::VT_SHAPE, None)
+    ) -> flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset<TensorDim<'a>>> {
+        self._tab
+            .get::<flatbuffers::ForwardsUOffset<
+                flatbuffers::Vector<flatbuffers::ForwardsUOffset<TensorDim<'a>>>,
+            >>(Tensor::VT_SHAPE, None)
+            .unwrap()
     }
     /// Non-negative byte offsets to advance one value cell along each dimension
+    /// If omitted, default to row-major order (C-like).
     #[inline]
     pub fn strides(&self) -> Option<flatbuffers::Vector<'a, i64>> {
         self._tab
@@ -208,14 +211,14 @@ impl<'a> Tensor<'a> {
     }
     /// The location and size of the tensor's data
     #[inline]
-    pub fn data(&self) -> Option<&'a Buffer> {
-        self._tab.get::<Buffer>(Tensor::VT_DATA, None)
+    pub fn data(&self) -> &'a Buffer {
+        self._tab.get::<Buffer>(Tensor::VT_DATA, None).unwrap()
     }
     #[inline]
     #[allow(non_snake_case)]
     pub fn type_as_null(&self) -> Option<Null<'a>> {
         if self.type_type() == Type::Null {
-            self.type_().map(|u| Null::init_from_table(u))
+            Some(Null::init_from_table(self.type_()))
         } else {
             None
         }
@@ -225,7 +228,7 @@ impl<'a> Tensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_int(&self) -> Option<Int<'a>> {
         if self.type_type() == Type::Int {
-            self.type_().map(|u| Int::init_from_table(u))
+            Some(Int::init_from_table(self.type_()))
         } else {
             None
         }
@@ -235,7 +238,7 @@ impl<'a> Tensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_floating_point(&self) -> Option<FloatingPoint<'a>> {
         if self.type_type() == Type::FloatingPoint {
-            self.type_().map(|u| FloatingPoint::init_from_table(u))
+            Some(FloatingPoint::init_from_table(self.type_()))
         } else {
             None
         }
@@ -245,7 +248,7 @@ impl<'a> Tensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_binary(&self) -> Option<Binary<'a>> {
         if self.type_type() == Type::Binary {
-            self.type_().map(|u| Binary::init_from_table(u))
+            Some(Binary::init_from_table(self.type_()))
         } else {
             None
         }
@@ -255,7 +258,7 @@ impl<'a> Tensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_utf_8(&self) -> Option<Utf8<'a>> {
         if self.type_type() == Type::Utf8 {
-            self.type_().map(|u| Utf8::init_from_table(u))
+            Some(Utf8::init_from_table(self.type_()))
         } else {
             None
         }
@@ -265,7 +268,7 @@ impl<'a> Tensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_bool(&self) -> Option<Bool<'a>> {
         if self.type_type() == Type::Bool {
-            self.type_().map(|u| Bool::init_from_table(u))
+            Some(Bool::init_from_table(self.type_()))
         } else {
             None
         }
@@ -275,7 +278,7 @@ impl<'a> Tensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_decimal(&self) -> Option<Decimal<'a>> {
         if self.type_type() == Type::Decimal {
-            self.type_().map(|u| Decimal::init_from_table(u))
+            Some(Decimal::init_from_table(self.type_()))
         } else {
             None
         }
@@ -285,7 +288,7 @@ impl<'a> Tensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_date(&self) -> Option<Date<'a>> {
         if self.type_type() == Type::Date {
-            self.type_().map(|u| Date::init_from_table(u))
+            Some(Date::init_from_table(self.type_()))
         } else {
             None
         }
@@ -295,7 +298,7 @@ impl<'a> Tensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_time(&self) -> Option<Time<'a>> {
         if self.type_type() == Type::Time {
-            self.type_().map(|u| Time::init_from_table(u))
+            Some(Time::init_from_table(self.type_()))
         } else {
             None
         }
@@ -305,7 +308,7 @@ impl<'a> Tensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_timestamp(&self) -> Option<Timestamp<'a>> {
         if self.type_type() == Type::Timestamp {
-            self.type_().map(|u| Timestamp::init_from_table(u))
+            Some(Timestamp::init_from_table(self.type_()))
         } else {
             None
         }
@@ -315,7 +318,7 @@ impl<'a> Tensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_interval(&self) -> Option<Interval<'a>> {
         if self.type_type() == Type::Interval {
-            self.type_().map(|u| Interval::init_from_table(u))
+            Some(Interval::init_from_table(self.type_()))
         } else {
             None
         }
@@ -325,7 +328,7 @@ impl<'a> Tensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_list(&self) -> Option<List<'a>> {
         if self.type_type() == Type::List {
-            self.type_().map(|u| List::init_from_table(u))
+            Some(List::init_from_table(self.type_()))
         } else {
             None
         }
@@ -335,7 +338,7 @@ impl<'a> Tensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_struct_(&self) -> Option<Struct_<'a>> {
         if self.type_type() == Type::Struct_ {
-            self.type_().map(|u| Struct_::init_from_table(u))
+            Some(Struct_::init_from_table(self.type_()))
         } else {
             None
         }
@@ -345,7 +348,7 @@ impl<'a> Tensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_union(&self) -> Option<Union<'a>> {
         if self.type_type() == Type::Union {
-            self.type_().map(|u| Union::init_from_table(u))
+            Some(Union::init_from_table(self.type_()))
         } else {
             None
         }
@@ -355,7 +358,7 @@ impl<'a> Tensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_fixed_size_binary(&self) -> Option<FixedSizeBinary<'a>> {
         if self.type_type() == Type::FixedSizeBinary {
-            self.type_().map(|u| FixedSizeBinary::init_from_table(u))
+            Some(FixedSizeBinary::init_from_table(self.type_()))
         } else {
             None
         }
@@ -365,7 +368,7 @@ impl<'a> Tensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_fixed_size_list(&self) -> Option<FixedSizeList<'a>> {
         if self.type_type() == Type::FixedSizeList {
-            self.type_().map(|u| FixedSizeList::init_from_table(u))
+            Some(FixedSizeList::init_from_table(self.type_()))
         } else {
             None
         }
@@ -375,7 +378,7 @@ impl<'a> Tensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_map(&self) -> Option<Map<'a>> {
         if self.type_type() == Type::Map {
-            self.type_().map(|u| Map::init_from_table(u))
+            Some(Map::init_from_table(self.type_()))
         } else {
             None
         }
@@ -385,7 +388,7 @@ impl<'a> Tensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_duration(&self) -> Option<Duration<'a>> {
         if self.type_type() == Type::Duration {
-            self.type_().map(|u| Duration::init_from_table(u))
+            Some(Duration::init_from_table(self.type_()))
         } else {
             None
         }
@@ -395,7 +398,7 @@ impl<'a> Tensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_large_binary(&self) -> Option<LargeBinary<'a>> {
         if self.type_type() == Type::LargeBinary {
-            self.type_().map(|u| LargeBinary::init_from_table(u))
+            Some(LargeBinary::init_from_table(self.type_()))
         } else {
             None
         }
@@ -405,7 +408,7 @@ impl<'a> Tensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_large_utf_8(&self) -> Option<LargeUtf8<'a>> {
         if self.type_type() == Type::LargeUtf8 {
-            self.type_().map(|u| LargeUtf8::init_from_table(u))
+            Some(LargeUtf8::init_from_table(self.type_()))
         } else {
             None
         }
@@ -415,7 +418,7 @@ impl<'a> Tensor<'a> {
     #[allow(non_snake_case)]
     pub fn type_as_large_list(&self) -> Option<LargeList<'a>> {
         if self.type_type() == Type::LargeList {
-            self.type_().map(|u| LargeList::init_from_table(u))
+            Some(LargeList::init_from_table(self.type_()))
         } else {
             None
         }
@@ -438,10 +441,10 @@ impl<'a> Default for TensorArgs<'a> {
     fn default() -> Self {
         TensorArgs {
             type_type: Type::NONE,
-            type_: None,
-            shape: None,
+            type_: None, // required field
+            shape: None, // required field
             strides: None,
-            data: None,
+            data: None, // required field
         }
     }
 }
@@ -498,6 +501,9 @@ impl<'a: 'b, 'b> TensorBuilder<'a, 'b> {
     #[inline]
     pub fn finish(self) -> flatbuffers::WIPOffset<Tensor<'a>> {
         let o = self.fbb_.end_table(self.start_);
+        self.fbb_.required(o, Tensor::VT_TYPE_, "type_");
+        self.fbb_.required(o, Tensor::VT_SHAPE, "shape");
+        self.fbb_.required(o, Tensor::VT_DATA, "data");
         flatbuffers::WIPOffset::new(o.value())
     }
 }
diff --git a/rust/arrow/src/ipc/gen/mod.rs b/rust/arrow/src/ipc/gen/mod.rs
index 39b05db7c56..ceeb6b2c5c7 100644
--- a/rust/arrow/src/ipc/gen/mod.rs
+++ b/rust/arrow/src/ipc/gen/mod.rs
@@ -19,8 +19,13 @@
 
 #![allow(non_snake_case)]
 
+#[allow(clippy::all)]
 pub mod File;
+#[allow(clippy::all)]
 pub mod Message;
+#[allow(clippy::all)]
 pub mod Schema;
+#[allow(clippy::all)]
 pub mod SparseTensor;
+#[allow(clippy::all)]
 pub mod Tensor;
diff --git a/rust/arrow/src/ipc/mod.rs b/rust/arrow/src/ipc/mod.rs
index 410e85c8afd..cba8fb269a4 100644
--- a/rust/arrow/src/ipc/mod.rs
+++ b/rust/arrow/src/ipc/mod.rs
@@ -22,6 +22,11 @@ pub mod convert;
 pub mod reader;
 pub mod writer;
 
+#[allow(clippy::redundant_closure)]
+#[allow(clippy::needless_lifetimes)]
+#[allow(clippy::extra_unused_lifetimes)]
+#[allow(clippy::redundant_static_lifetimes)]
+#[allow(clippy::redundant_field_names)]
 pub mod gen;
 
 pub use self::gen::File::*;
@@ -31,3 +36,4 @@ pub use self::gen::SparseTensor::*;
 pub use self::gen::Tensor::*;
 
 static ARROW_MAGIC: [u8; 6] = [b'A', b'R', b'R', b'O', b'W', b'1'];
+static CONTINUATION_MARKER: [u8; 4] = [0xff; 4];
diff --git a/rust/arrow/src/ipc/reader.rs b/rust/arrow/src/ipc/reader.rs
index 370e480393f..05cf00b822b 100644
--- a/rust/arrow/src/ipc/reader.rs
+++ b/rust/arrow/src/ipc/reader.rs
@@ -31,9 +31,9 @@ use crate::datatypes::{DataType, Field, IntervalUnit, Schema, SchemaRef};
 use crate::error::{ArrowError, Result};
 use crate::ipc;
 use crate::record_batch::{RecordBatch, RecordBatchReader};
-use DataType::*;
 
-const CONTINUATION_MARKER: u32 = 0xffff_ffff;
+use ipc::CONTINUATION_MARKER;
+use DataType::*;
 
 /// Read a buffer based on offset and length
 fn read_buffer(buf: &ipc::Buffer, a_data: &[u8]) -> Buffer {
@@ -89,7 +89,7 @@ fn create_array(
             buffer_index += 2;
             array
         }
-        List(ref list_data_type) | LargeList(ref list_data_type) => {
+        List(ref list_field) | LargeList(ref list_field) => {
             let list_node = &nodes[node_index];
             let list_buffers: Vec<Buffer> = buffers[buffer_index..buffer_index + 2]
                 .iter()
@@ -99,7 +99,7 @@ fn create_array(
             buffer_index += 2;
             let triple = create_array(
                 nodes,
-                list_data_type,
+                list_field.data_type(),
                 data,
                 buffers,
                 dictionaries,
@@ -111,7 +111,7 @@ fn create_array(
 
             create_list_array(list_node, data_type, &list_buffers[..], triple.0)
         }
-        FixedSizeList(ref list_data_type, _) => {
+        FixedSizeList(ref list_field, _) => {
             let list_node = &nodes[node_index];
             let list_buffers: Vec<Buffer> = buffers[buffer_index..=buffer_index]
                 .iter()
@@ -121,7 +121,7 @@ fn create_array(
             buffer_index += 1;
             let triple = create_array(
                 nodes,
-                list_data_type,
+                list_field.data_type(),
                 data,
                 buffers,
                 dictionaries,
@@ -414,7 +414,7 @@ pub fn read_record_batch(
     batch: ipc::RecordBatch,
     schema: SchemaRef,
     dictionaries: &[Option<ArrayRef>],
-) -> Result<Option<RecordBatch>> {
+) -> Result<RecordBatch> {
     let buffers = batch.buffers().ok_or_else(|| {
         ArrowError::IoError("Unable to get buffers from IPC RecordBatch".to_string())
     })?;
@@ -442,7 +442,70 @@ pub fn read_record_batch(
         arrays.push(triple.0);
     }
 
-    RecordBatch::try_new(schema, arrays).map(|batch| Some(batch))
+    RecordBatch::try_new(schema, arrays)
+}
+
+/// Read the dictionary from the buffer and provided metadata,
+/// updating the `dictionaries_by_field` with the resulting dictionary
+fn read_dictionary(
+    buf: &[u8],
+    batch: ipc::DictionaryBatch,
+    ipc_schema: &ipc::Schema,
+    schema: &Schema,
+    dictionaries_by_field: &mut [Option<ArrayRef>],
+) -> Result<()> {
+    if batch.isDelta() {
+        return Err(ArrowError::IoError(
+            "delta dictionary batches not supported".to_string(),
+        ));
+    }
+
+    let id = batch.id();
+
+    // As the dictionary batch does not contain the type of the
+    // values array, we need to retrieve this from the schema.
+    let first_field = find_dictionary_field(ipc_schema, id).ok_or_else(|| {
+        ArrowError::InvalidArgumentError("dictionary id not found in schema".to_string())
+    })?;
+
+    // Get an array representing this dictionary's values.
+    let dictionary_values: ArrayRef = match schema.field(first_field).data_type() {
+        DataType::Dictionary(_, ref value_type) => {
+            // Make a fake schema for the dictionary batch.
+            let schema = Schema {
+                fields: vec![Field::new("", value_type.as_ref().clone(), false)],
+                metadata: HashMap::new(),
+            };
+            // Read a single column
+            let record_batch = read_record_batch(
+                &buf,
+                batch.data().unwrap(),
+                Arc::new(schema),
+                &dictionaries_by_field,
+            )?;
+            Some(record_batch.column(0).clone())
+        }
+        _ => None,
+    }
+    .ok_or_else(|| {
+        ArrowError::InvalidArgumentError("dictionary id not found in schema".to_string())
+    })?;
+
+    // for all fields with this dictionary id, update the dictionaries vector
+    // in the reader. Note that a dictionary batch may be shared between many fields.
+    // We don't currently record the isOrdered field. This could be general
+    // attributes of arrays.
+    let fields = ipc_schema.fields().unwrap();
+    for (i, field) in fields.iter().enumerate() {
+        if let Some(dictionary) = field.dictionary() {
+            if dictionary.id() == id {
+                // Add (possibly multiple) array refs to the dictionaries array.
+                dictionaries_by_field[i] = Some(dictionary_values.clone());
+            }
+        }
+    }
+
+    Ok(())
 }
 
 // Linear search for the first dictionary field with a dictionary id.
@@ -482,6 +545,9 @@ pub struct FileReader<R: Read + Seek> {
     ///
     /// Dictionaries may be appended to in the streaming format.
     dictionaries_by_field: Vec<Option<ArrayRef>>,
+
+    /// Metadata version
+    metadata_version: ipc::MetadataVersion,
 }
 
 impl<R: Read + Seek> FileReader<R> {
@@ -506,12 +572,11 @@ impl<R: Read + Seek> FileReader<R> {
                 "Arrow file does not contain correct footer".to_string(),
             ));
         }
-
-        // what does the footer contain?
+        // read footer length
         let mut footer_size: [u8; 4] = [0; 4];
         reader.seek(SeekFrom::End(-10))?;
         reader.read_exact(&mut footer_size)?;
-        let footer_len = u32::from_le_bytes(footer_size);
+        let footer_len = i32::from_le_bytes(footer_size);
 
         // read footer
         let mut footer_data = vec![0; footer_len as usize];
@@ -534,6 +599,7 @@ impl<R: Read + Seek> FileReader<R> {
         let mut dictionaries_by_field = vec![None; schema.fields().len()];
         for block in footer.dictionaries().unwrap() {
             // read length from end of offset
+            // TODO: ARROW-9848: dictionary metadata has not been tested
             let meta_len = block.metaDataLength() - 4;
 
             let mut block_data = vec![0; meta_len as usize];
@@ -553,60 +619,19 @@ impl<R: Read + Seek> FileReader<R> {
                     ))?;
                     reader.read_exact(&mut buf)?;
 
-                    if batch.isDelta() {
-                        panic!("delta dictionary batches not supported");
-                    }
-
-                    let id = batch.id();
-
-                    // As the dictionary batch does not contain the type of the
-                    // values array, we need to retieve this from the schema.
-                    let first_field = find_dictionary_field(&ipc_schema, id)
-                        .expect("dictionary id not found in shchema");
-
-                    // Get an array representing this dictionary's values.
-                    let dictionary_values: ArrayRef =
-                        match schema.field(first_field).data_type() {
-                            DataType::Dictionary(_, ref value_type) => {
-                                // Make a fake schema for the dictionary batch.
-                                let schema = Schema {
-                                    fields: vec![Field::new(
-                                        "",
-                                        value_type.as_ref().clone(),
-                                        false,
-                                    )],
-                                    metadata: HashMap::new(),
-                                };
-                                // Read a single column
-                                let record_batch = read_record_batch(
-                                    &buf,
-                                    batch.data().unwrap(),
-                                    Arc::new(schema),
-                                    &dictionaries_by_field,
-                                )?
-                                .unwrap();
-                                Some(record_batch.column(0).clone())
-                            }
-                            _ => None,
-                        }
-                        .expect("dictionary id not found in schema");
-
-                    // for all fields with this dictionary id, update the dictionaries vector
-                    // in the reader. Note that a dictionary batch may be shared between many fields.
-                    // We don't currently record the isOrdered field. This could be general
-                    // attributes of arrays.
-                    let fields = ipc_schema.fields().unwrap();
-                    for (i, field) in fields.iter().enumerate() {
-                        if let Some(dictionary) = field.dictionary() {
-                            if dictionary.id() == id {
-                                // Add (possibly multiple) array refs to the dictionaries array.
-                                dictionaries_by_field[i] =
-                                    Some(dictionary_values.clone());
-                            }
-                        }
-                    }
+                    read_dictionary(
+                        &buf,
+                        batch,
+                        &ipc_schema,
+                        &schema,
+                        &mut dictionaries_by_field,
+                    )?;
+                }
+                _ => {
+                    return Err(ArrowError::IoError(
+                        "Expecting DictionaryBatch in dictionary blocks.".to_string(),
+                    ))
                 }
-                _ => panic!("Expecting DictionaryBatch in dictionary blocks."),
             };
         }
 
@@ -617,6 +642,7 @@ impl<R: Read + Seek> FileReader<R> {
             current_block: 0,
             total_blocks,
             dictionaries_by_field,
+            metadata_version: footer.version(),
         })
     }
 
@@ -644,81 +670,110 @@ impl<R: Read + Seek> FileReader<R> {
             Ok(())
         }
     }
-}
-
-impl<R: Read + Seek> RecordBatchReader for FileReader<R> {
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-
-    fn next_batch(&mut self) -> Result<Option<RecordBatch>> {
-        // get current block
-        if self.current_block < self.total_blocks {
-            let block = self.blocks[self.current_block];
-            self.current_block += 1;
 
-            // read length from end of offset
-            let meta_len = block.metaDataLength() - 4;
+    fn maybe_next(&mut self) -> Result<Option<RecordBatch>> {
+        let block = self.blocks[self.current_block];
+        self.current_block += 1;
+
+        // read length
+        self.reader.seek(SeekFrom::Start(block.offset() as u64))?;
+        let mut meta_buf = [0; 4];
+        self.reader.read_exact(&mut meta_buf)?;
+        if meta_buf == CONTINUATION_MARKER {
+            // continuation marker encountered, read message next
+            self.reader.read_exact(&mut meta_buf)?;
+        }
+        let meta_len = i32::from_le_bytes(meta_buf);
 
-            let mut block_data = vec![0; meta_len as usize];
-            self.reader
-                .seek(SeekFrom::Start(block.offset() as u64 + 4))?;
-            self.reader.read_exact(&mut block_data)?;
+        let mut block_data = vec![0; meta_len as usize];
+        self.reader.read_exact(&mut block_data)?;
 
-            let message = ipc::get_root_as_message(&block_data[..]);
+        let message = ipc::get_root_as_message(&block_data[..]);
 
-            match message.header_type() {
-                ipc::MessageHeader::Schema => Err(ArrowError::IoError(
-                    "Not expecting a schema when messages are read".to_string(),
-                )),
-                ipc::MessageHeader::RecordBatch => {
-                    let batch = message.header_as_record_batch().ok_or_else(|| {
-                        ArrowError::IoError(
-                            "Unable to read IPC message as record batch".to_string(),
-                        )
-                    })?;
-                    // read the block that makes up the record batch into a buffer
-                    let mut buf = vec![0; block.bodyLength() as usize];
-                    self.reader.seek(SeekFrom::Start(
-                        block.offset() as u64 + block.metaDataLength() as u64,
-                    ))?;
-                    self.reader.read_exact(&mut buf)?;
+        // some old test data's footer metadata is not set, so we account for that
+        if self.metadata_version != ipc::MetadataVersion::V1
+            && message.version() != self.metadata_version
+        {
+            return Err(ArrowError::IoError(
+                "Could not read IPC message as metadata versions mismatch".to_string(),
+            ));
+        }
 
-                    read_record_batch(
-                        &buf,
-                        batch,
-                        self.schema(),
-                        &self.dictionaries_by_field,
+        match message.header_type() {
+            ipc::MessageHeader::Schema => Err(ArrowError::IoError(
+                "Not expecting a schema when messages are read".to_string(),
+            )),
+            ipc::MessageHeader::RecordBatch => {
+                let batch = message.header_as_record_batch().ok_or_else(|| {
+                    ArrowError::IoError(
+                        "Unable to read IPC message as record batch".to_string(),
                     )
-                }
-                ipc::MessageHeader::NONE => {
-                    Ok(None)
-                }
-                t => Err(ArrowError::IoError(format!(
-                    "Reading types other than record batches not yet supported, unable to read {:?}", t
-                ))),
+                })?;
+                // read the block that makes up the record batch into a buffer
+                let mut buf = vec![0; block.bodyLength() as usize];
+                self.reader.seek(SeekFrom::Start(
+                    block.offset() as u64 + block.metaDataLength() as u64,
+                ))?;
+                self.reader.read_exact(&mut buf)?;
+
+                read_record_batch(
+                    &buf,
+                    batch,
+                    self.schema(),
+                    &self.dictionaries_by_field,
+                ).map(Some)
+            }
+            ipc::MessageHeader::NONE => {
+                Ok(None)
             }
+            t => Err(ArrowError::IoError(format!(
+                "Reading types other than record batches not yet supported, unable to read {:?}", t
+            ))),
+        }
+    }
+}
+
+impl<R: Read + Seek> Iterator for FileReader<R> {
+    type Item = Result<RecordBatch>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        // get current block
+        if self.current_block < self.total_blocks {
+            self.maybe_next().transpose()
         } else {
-            Ok(None)
+            None
         }
     }
 }
 
+impl<R: Read + Seek> RecordBatchReader for FileReader<R> {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+}
+
 /// Arrow Stream reader
 pub struct StreamReader<R: Read> {
     /// Buffered stream reader
     reader: BufReader<R>,
+
     /// The schema that is read from the stream's first message
     schema: SchemaRef,
-    /// An indicator of whether the strewam is complete.
+
+    /// The bytes of the IPC schema that is read from the stream's first message
     ///
-    /// This value is set to `true` the first time the reader's `next()` returns `None`.
-    finished: bool,
+    /// This is kept in order to interpret dictionary data
+    ipc_schema: Vec<u8>,
 
     /// Optional dictionaries for each schema field.
     ///
     /// Dictionaries may be appended to in the streaming format.
     dictionaries_by_field: Vec<Option<ArrayRef>>,
+
+    /// An indicator of whether the stream is complete.
+    ///
+    /// This value is set to `true` the first time the reader's `next()` returns `None`.
+    finished: bool,
 }
 
 impl<R: Read> StreamReader<R> {
@@ -733,23 +788,18 @@ impl<R: Read> StreamReader<R> {
         let mut meta_size: [u8; 4] = [0; 4];
         reader.read_exact(&mut meta_size)?;
         let meta_len = {
-            let meta_len = u32::from_le_bytes(meta_size);
-
             // If a continuation marker is encountered, skip over it and read
             // the size from the next four bytes.
-            if meta_len == CONTINUATION_MARKER {
+            if meta_size == CONTINUATION_MARKER {
                 reader.read_exact(&mut meta_size)?;
-                u32::from_le_bytes(meta_size)
-            } else {
-                meta_len
             }
+            i32::from_le_bytes(meta_size)
         };
 
         let mut meta_buffer = vec![0; meta_len as usize];
         reader.read_exact(&mut meta_buffer)?;
 
-        let vecs = &meta_buffer.to_vec();
-        let message = ipc::get_root_as_message(vecs);
+        let message = ipc::get_root_as_message(meta_buffer.as_slice());
         // message header is a Schema, so read it
         let ipc_schema: ipc::Schema = message.header_as_schema().ok_or_else(|| {
             ArrowError::IoError("Unable to read IPC message as schema".to_string())
@@ -762,6 +812,7 @@ impl<R: Read> StreamReader<R> {
         Ok(Self {
             reader,
             schema: Arc::new(schema),
+            ipc_schema: meta_buffer,
             finished: false,
             dictionaries_by_field,
         })
@@ -776,14 +827,8 @@ impl<R: Read> StreamReader<R> {
     pub fn is_finished(&self) -> bool {
         self.finished
     }
-}
 
-impl<R: Read> RecordBatchReader for StreamReader<R> {
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-
-    fn next_batch(&mut self) -> Result<Option<RecordBatch>> {
+    fn maybe_next(&mut self) -> Result<Option<RecordBatch>> {
         if self.finished {
             return Ok(None);
         }
@@ -806,16 +851,12 @@ impl<R: Read> RecordBatchReader for StreamReader<R> {
         }
 
         let meta_len = {
-            let meta_len = u32::from_le_bytes(meta_size);
-
             // If a continuation marker is encountered, skip over it and read
             // the size from the next four bytes.
-            if meta_len == CONTINUATION_MARKER {
+            if meta_size == CONTINUATION_MARKER {
                 self.reader.read_exact(&mut meta_size)?;
-                u32::from_le_bytes(meta_size)
-            } else {
-                meta_len
             }
+            i32::from_le_bytes(meta_size)
         };
 
         if meta_len == 0 {
@@ -844,7 +885,31 @@ impl<R: Read> RecordBatchReader for StreamReader<R> {
                 let mut buf = vec![0; message.bodyLength() as usize];
                 self.reader.read_exact(&mut buf)?;
 
-                read_record_batch(&buf, batch, self.schema(), &self.dictionaries_by_field)
+                read_record_batch(&buf, batch, self.schema(), &self.dictionaries_by_field).map(Some)
+            }
+            ipc::MessageHeader::DictionaryBatch => {
+                let batch = message.header_as_dictionary_batch().ok_or_else(|| {
+                    ArrowError::IoError(
+                        "Unable to read IPC message as dictionary batch".to_string(),
+                    )
+                })?;
+                // read the block that makes up the dictionary batch into a buffer
+                let mut buf = vec![0; message.bodyLength() as usize];
+                self.reader.read_exact(&mut buf)?;
+
+                let ipc_schema = ipc::get_root_as_message(&self.ipc_schema).header_as_schema()
+                .ok_or_else(|| {
+                    ArrowError::IoError(
+                        "Unable to read schema from stored message header".to_string(),
+                    )
+                })?;
+
+                read_dictionary(
+                    &buf, batch, &ipc_schema, &self.schema, &mut self.dictionaries_by_field
+                )?;
+
+                // read the next message until we encounter a RecordBatch
+                self.maybe_next()
             }
             ipc::MessageHeader::NONE => {
                 Ok(None)
@@ -856,6 +921,20 @@ impl<R: Read> RecordBatchReader for StreamReader<R> {
     }
 }
 
+impl<R: Read> Iterator for StreamReader<R> {
+    type Item = Result<RecordBatch>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.maybe_next().transpose()
+    }
+}
+
+impl<R: Read> RecordBatchReader for StreamReader<R> {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -901,7 +980,7 @@ mod tests {
         let paths = vec![
             "generated_interval",
             "generated_datetime",
-            // "generated_dictionary",
+            "generated_dictionary",
             "generated_nested",
             "generated_primitive_no_batches",
             "generated_primitive_zerolength",
@@ -920,7 +999,7 @@ mod tests {
             let arrow_json = read_gzip_json(path);
             assert!(arrow_json.equals_reader(&mut reader));
             // the next batch must be empty
-            assert!(reader.next_batch().unwrap().is_none());
+            assert!(reader.next().is_none());
             // the stream must indicate that it's finished
             assert!(reader.is_finished());
         });
@@ -950,8 +1029,10 @@ mod tests {
 
         // read stream back
         let file = File::open("target/debug/testdata/float.stream").unwrap();
-        let mut reader = StreamReader::try_new(file).unwrap();
-        while let Some(batch) = reader.next_batch().unwrap() {
+        let reader = StreamReader::try_new(file).unwrap();
+
+        reader.for_each(|batch| {
+            let batch = batch.unwrap();
             assert!(
                 batch
                     .column(0)
@@ -970,7 +1051,7 @@ mod tests {
                     .value(0)
                     != 0.0
             );
-        }
+        })
     }
 
     /// Read gzipped JSON file
diff --git a/rust/arrow/src/ipc/writer.rs b/rust/arrow/src/ipc/writer.rs
index b2376fe2bd1..7678a5cd200 100644
--- a/rust/arrow/src/ipc/writer.rs
+++ b/rust/arrow/src/ipc/writer.rs
@@ -32,9 +32,76 @@ use crate::ipc;
 use crate::record_batch::RecordBatch;
 use crate::util::bit_util;
 
+use ipc::CONTINUATION_MARKER;
+
+/// IPC write options used to control the behaviour of the writer
+#[derive(Debug)]
+pub struct IpcWriteOptions {
+    /// Write padding after memory buffers to this multiple of bytes.
+    /// Generally 8 or 64, defaults to 8
+    alignment: usize,
+    /// The legacy format is for releases before 0.15.0, and uses metadata V4
+    write_legacy_ipc_format: bool,
+    /// The metadata version to write. The Rust IPC writer supports V4+
+    metadata_version: ipc::MetadataVersion,
+}
+
+impl IpcWriteOptions {
+    /// Try create IpcWriteOptions, checking for incompatible settings
+    pub fn try_new(
+        alignment: usize,
+        write_legacy_ipc_format: bool,
+        metadata_version: ipc::MetadataVersion,
+    ) -> Result<Self> {
+        if alignment == 0 || alignment % 8 != 0 {
+            return Err(ArrowError::InvalidArgumentError(
+                "Alignment should be greater than 0 and be a multiple of 8".to_string(),
+            ));
+        }
+        match metadata_version {
+            ipc::MetadataVersion::V1
+            | ipc::MetadataVersion::V2
+            | ipc::MetadataVersion::V3 => Err(ArrowError::InvalidArgumentError(
+                "Writing IPC metadata version 3 and lower not supported".to_string(),
+            )),
+            ipc::MetadataVersion::V4 => Ok(Self {
+                alignment,
+                write_legacy_ipc_format,
+                metadata_version,
+            }),
+            ipc::MetadataVersion::V5 => {
+                if write_legacy_ipc_format {
+                    Err(ArrowError::InvalidArgumentError(
+                        "Legacy IPC format only supported on metadata version 4"
+                            .to_string(),
+                    ))
+                } else {
+                    Ok(Self {
+                        alignment,
+                        write_legacy_ipc_format,
+                        metadata_version,
+                    })
+                }
+            }
+        }
+    }
+}
+
+impl Default for IpcWriteOptions {
+    fn default() -> Self {
+        Self {
+            alignment: 8,
+            write_legacy_ipc_format: true,
+            metadata_version: ipc::MetadataVersion::V4,
+        }
+    }
+}
+
 pub struct FileWriter<W: Write> {
     /// The object to write to
     writer: BufWriter<W>,
+    /// IPC write options
+    write_options: IpcWriteOptions,
     /// A reference to the schema, used in validating record batches
     schema: Schema,
     /// The number of bytes between each block of bytes, as an offset for random access
@@ -50,17 +117,29 @@ pub struct FileWriter<W: Write> {
 impl<W: Write> FileWriter<W> {
     /// Try create a new writer, with the schema written as part of the header
     pub fn try_new(writer: W, schema: &Schema) -> Result<Self> {
+        let write_options = IpcWriteOptions::default();
+        Self::try_new_with_options(writer, schema, write_options)
+    }
+
+    /// Try create a new writer with IpcWriteOptions
+    pub fn try_new_with_options(
+        writer: W,
+        schema: &Schema,
+        write_options: IpcWriteOptions,
+    ) -> Result<Self> {
         let mut writer = BufWriter::new(writer);
         // write magic to header
         writer.write_all(&super::ARROW_MAGIC[..])?;
         // create an 8-byte boundary after the header
         writer.write_all(&[0, 0])?;
         // write the schema, set the written bytes to the schema + header
-        let written = write_schema(&mut writer, schema)? + 8;
+        let message = Message::Schema(schema, &write_options);
+        let (meta, data) = write_message(&mut writer, &message, &write_options)?;
         Ok(Self {
             writer,
+            write_options,
             schema: schema.clone(),
-            block_offsets: written,
+            block_offsets: meta + data + 8,
             dictionary_blocks: vec![],
             record_blocks: vec![],
             finished: false,
@@ -74,19 +153,25 @@ impl<W: Write> FileWriter<W> {
                 "Cannot write record batch to file writer as it is closed".to_string(),
             ));
         }
-        let (meta, data) = write_record_batch(&mut self.writer, batch, false)?;
+        let message = Message::RecordBatch(batch, &self.write_options);
+        let (meta, data) =
+            write_message(&mut self.writer, &message, &self.write_options)?;
         // add a record block for the footer
-        self.record_blocks.push(ipc::Block::new(
+        let block = ipc::Block::new(
             self.block_offsets as i64,
-            (meta as i32) + 4,
+            meta as i32, // TODO: is this still applicable?
             data as i64,
-        ));
+        );
+        self.record_blocks.push(block);
         self.block_offsets += meta + data;
         Ok(())
     }
 
     /// Write footer and closing tag, then mark the writer as done
     pub fn finish(&mut self) -> Result<()> {
+        // write EOS
+        write_continuation(&mut self.writer, &self.write_options, 0)?;
+
         let mut fbb = FlatBufferBuilder::new();
         let dictionaries = fbb.create_vector(&self.dictionary_blocks);
         let record_batches = fbb.create_vector(&self.record_blocks);
@@ -130,14 +215,17 @@ impl<W: Write> FileWriter<W> {
         };
         let root = {
             let mut footer_builder = ipc::FooterBuilder::new(&mut fbb);
-            footer_builder.add_version(ipc::MetadataVersion::V4);
+            footer_builder.add_version(self.write_options.metadata_version);
             footer_builder.add_schema(schema);
             footer_builder.add_dictionaries(dictionaries);
             footer_builder.add_recordBatches(record_batches);
             footer_builder.finish()
         };
         fbb.finish(root, None);
-        write_padded_data(&mut self.writer, fbb.finished_data(), WriteDataType::Footer)?;
+        let footer_data = fbb.finished_data();
+        self.writer.write_all(footer_data)?;
+        self.writer
+            .write_all(&(footer_data.len() as i32).to_le_bytes())?;
         self.writer.write_all(&super::ARROW_MAGIC)?;
         self.writer.flush()?;
         self.finished = true;
@@ -158,6 +246,8 @@ impl<W: Write> Drop for FileWriter<W> {
 pub struct StreamWriter<W: Write> {
     /// The object to write to
     writer: BufWriter<W>,
+    /// IPC write options
+    write_options: IpcWriteOptions,
     /// A reference to the schema, used in validating record batches
     schema: Schema,
     /// Whether the writer footer has been written, and the writer is finished
@@ -167,11 +257,22 @@ pub struct StreamWriter<W: Write> {
 impl<W: Write> StreamWriter<W> {
     /// Try create a new writer, with the schema written as part of the header
     pub fn try_new(writer: W, schema: &Schema) -> Result<Self> {
+        let write_options = IpcWriteOptions::default();
+        Self::try_new_with_options(writer, schema, write_options)
+    }
+
+    pub fn try_new_with_options(
+        writer: W,
+        schema: &Schema,
+        write_options: IpcWriteOptions,
+    ) -> Result<Self> {
         let mut writer = BufWriter::new(writer);
         // write the schema, set the written bytes to the schema
-        write_schema(&mut writer, schema)?;
+        let message = Message::Schema(schema, &write_options);
+        write_message(&mut writer, &message, &write_options)?;
         Ok(Self {
             writer,
+            write_options,
             schema: schema.clone(),
             finished: false,
         })
@@ -184,15 +285,14 @@ impl<W: Write> StreamWriter<W> {
                 "Cannot write record batch to stream writer as it is closed".to_string(),
             ));
         }
-        write_record_batch(&mut self.writer, batch, true)?;
+        let message = Message::RecordBatch(batch, &self.write_options);
+        write_message(&mut self.writer, &message, &self.write_options)?;
         Ok(())
     }
 
     /// Write continuation bytes, and mark the stream as done
     pub fn finish(&mut self) -> Result<()> {
-        self.writer.write_all(&[255u8, 255, 255, 255])?;
-        self.writer.write_all(&[0u8, 0, 0, 0])?;
-        self.writer.flush()?;
+        write_continuation(&mut self.writer, &self.write_options, 0)?;
 
         self.finished = true;
 
@@ -209,7 +309,15 @@ impl<W: Write> Drop for StreamWriter<W> {
     }
 }
 
-pub fn schema_to_bytes(schema: &Schema) -> Vec<u8> {
+/// Stores the encoded data, which is an ipc::Message, and optional Arrow data
+pub struct EncodedData {
+    /// An encoded ipc::Message
+    pub ipc_message: Vec<u8>,
+    /// Arrow buffers to be written, should be an empty vec for schema messages
+    pub arrow_data: Vec<u8>,
+}
+
+pub fn schema_to_bytes(schema: &Schema, write_options: &IpcWriteOptions) -> EncodedData {
     let mut fbb = FlatBufferBuilder::new();
     let schema = {
         let fb = ipc::convert::schema_to_fb_offset(&mut fbb, schema);
@@ -217,7 +325,7 @@ pub fn schema_to_bytes(schema: &Schema) -> Vec<u8> {
     };
 
     let mut message = ipc::MessageBuilder::new(&mut fbb);
-    message.add_version(ipc::MetadataVersion::V4);
+    message.add_version(write_options.metadata_version);
     message.add_header_type(ipc::MessageHeader::Schema);
     message.add_bodyLength(0);
     message.add_header(schema);
@@ -226,51 +334,101 @@ pub fn schema_to_bytes(schema: &Schema) -> Vec<u8> {
     fbb.finish(data, None);
 
     let data = fbb.finished_data();
-    data.to_vec()
+    EncodedData {
+        ipc_message: data.to_vec(),
+        arrow_data: vec![],
+    }
 }
 
-/// Convert the schema to its IPC representation, and write it to the `writer`
-fn write_schema<R: Write>(writer: &mut BufWriter<R>, schema: &Schema) -> Result<usize> {
-    let data = schema_to_bytes(schema);
-    write_padded_data(writer, &data[..], WriteDataType::Header)
+enum Message<'a> {
+    Schema(&'a Schema, &'a IpcWriteOptions),
+    RecordBatch(&'a RecordBatch, &'a IpcWriteOptions),
+    DictionaryBatch(&'a IpcWriteOptions),
 }
 
-/// The message type being written. This determines whether to write the data length or not.
-/// Data length is written before the header, after the footer, and never for the body.
-#[derive(PartialEq)]
-enum WriteDataType {
-    Header,
-    Body,
-    Footer,
+impl<'a> Message<'a> {
+    /// Encode message to a ipc::Message and return data as bytes
+    fn encode(&'a self) -> EncodedData {
+        match self {
+            Message::Schema(schema, options) => schema_to_bytes(*schema, *options),
+            Message::RecordBatch(batch, options) => {
+                record_batch_to_bytes(*batch, *options)
+            }
+            Message::DictionaryBatch(_) => {
+                unimplemented!("Writing dictionary batches not implemented")
+            }
+        }
+    }
 }
 
-/// Write a slice of data to the writer, ensuring that it is padded to 8 bytes
-fn write_padded_data<R: Write>(
-    writer: &mut BufWriter<R>,
-    data: &[u8],
-    data_type: WriteDataType,
-) -> Result<usize> {
+/// Write a message's IPC data and buffers, returning metadata and buffer data lengths written
+fn write_message<W: Write>(
+    mut writer: &mut BufWriter<W>,
+    message: &Message,
+    write_options: &IpcWriteOptions,
+) -> Result<(usize, usize)> {
+    let encoded = message.encode();
+    let arrow_data_len = encoded.arrow_data.len();
+    if arrow_data_len % 8 != 0 {
+        return Err(ArrowError::MemoryError(
+            "Arrow data not aligned".to_string(),
+        ));
+    }
+
+    let a = write_options.alignment - 1;
+    let buffer = encoded.ipc_message;
+    let flatbuf_size = buffer.len();
+    let prefix_size = if write_options.write_legacy_ipc_format {
+        4
+    } else {
+        8
+    };
+    let aligned_size = (flatbuf_size + prefix_size + a) & !a;
+    let padding_bytes = aligned_size - flatbuf_size - prefix_size;
+
+    write_continuation(
+        &mut writer,
+        &write_options,
+        (aligned_size - prefix_size) as i32,
+    )?;
+
+    // write the flatbuf
+    if flatbuf_size > 0 {
+        writer.write_all(&buffer)?;
+    }
+    // write padding
+    writer.write_all(&vec![0; padding_bytes])?;
+
+    // write arrow data
+    let body_len = if arrow_data_len > 0 {
+        write_body_buffers(&mut writer, &encoded.arrow_data)?
+    } else {
+        0
+    };
+
+    Ok((aligned_size, body_len))
+}
+
+fn write_body_buffers<W: Write>(writer: &mut BufWriter<W>, data: &[u8]) -> Result<usize> {
     let len = data.len() as u32;
     let pad_len = pad_to_8(len) as u32;
     let total_len = len + pad_len;
-    // write data length
-    if data_type == WriteDataType::Header {
-        writer.write_all(&total_len.to_le_bytes()[..])?;
-    }
-    // write flatbuffer data
+
+    // write body buffer
     writer.write_all(data)?;
     if pad_len > 0 {
         writer.write_all(&vec![0u8; pad_len as usize][..])?;
     }
-    if data_type == WriteDataType::Footer {
-        writer.write_all(&total_len.to_le_bytes()[..])?;
-    }
+
     writer.flush()?;
     Ok(total_len as usize)
 }
 
 /// Write a `RecordBatch` into a tuple of bytes, one for the header (ipc::Message) and the other for the batch's data
-pub fn record_batch_to_bytes(batch: &RecordBatch) -> (Vec<u8>, Vec<u8>) {
+pub fn record_batch_to_bytes(
+    batch: &RecordBatch,
+    write_options: &IpcWriteOptions,
+) -> EncodedData {
     let mut fbb = FlatBufferBuilder::new();
 
     let mut nodes: Vec<ipc::FieldNode> = vec![];
@@ -304,7 +462,7 @@ pub fn record_batch_to_bytes(batch: &RecordBatch) -> (Vec<u8>, Vec<u8>) {
     };
     // create an ipc::Message
     let mut message = ipc::MessageBuilder::new(&mut fbb);
-    message.add_version(ipc::MetadataVersion::V4);
+    message.add_version(write_options.metadata_version);
     message.add_header_type(ipc::MessageHeader::RecordBatch);
     message.add_bodyLength(arrow_data.len() as i64);
     message.add_header(root);
@@ -312,26 +470,46 @@ pub fn record_batch_to_bytes(batch: &RecordBatch) -> (Vec<u8>, Vec<u8>) {
     fbb.finish(root, None);
     let finished_data = fbb.finished_data();
 
-    (finished_data.to_vec(), arrow_data)
+    EncodedData {
+        ipc_message: finished_data.to_vec(),
+        arrow_data,
+    }
 }
 
 /// Write a record batch to the writer, writing the message size before the message
 /// if the record batch is being written to a stream
-fn write_record_batch<R: Write>(
-    writer: &mut BufWriter<R>,
-    batch: &RecordBatch,
-    is_stream: bool,
-) -> Result<(usize, usize)> {
-    let (meta_data, arrow_data) = record_batch_to_bytes(batch);
-    // write the length of data if writing to stream
-    if is_stream {
-        let total_len: u32 = meta_data.len() as u32;
-        writer.write_all(&total_len.to_le_bytes()[..])?;
-    }
-    let meta_written = write_padded_data(writer, &meta_data[..], WriteDataType::Body)?;
-    let arrow_data_written =
-        write_padded_data(writer, &arrow_data[..], WriteDataType::Body)?;
-    Ok((meta_written, arrow_data_written))
+fn write_continuation<W: Write>(
+    writer: &mut BufWriter<W>,
+    write_options: &IpcWriteOptions,
+    total_len: i32,
+) -> Result<usize> {
+    let mut written = 8;
+
+    // the version of the writer determines whether continuation markers should be added
+    match write_options.metadata_version {
+        ipc::MetadataVersion::V1
+        | ipc::MetadataVersion::V2
+        | ipc::MetadataVersion::V3 => {
+            unreachable!("Options with the metadata version cannot be created")
+        }
+        ipc::MetadataVersion::V4 => {
+            if !write_options.write_legacy_ipc_format {
+                // v0.15.0 format
+                writer.write_all(&CONTINUATION_MARKER)?;
+                written = 4;
+            }
+            writer.write_all(&total_len.to_le_bytes()[..])?;
+        }
+        ipc::MetadataVersion::V5 => {
+            // write continuation marker and message length
+            writer.write_all(&CONTINUATION_MARKER)?;
+            writer.write_all(&total_len.to_le_bytes()[..])?;
+        }
+    };
+
+    writer.flush()?;
+
+    Ok(written)
 }
 
 /// Write array data to a vector of bytes
@@ -383,7 +561,7 @@ fn write_array_data(
     offset
 }
 
-/// Write a buffer to a vector of bytes, and add its ipc Buffer to a vector
+/// Write a buffer to a vector of bytes, and add its ipc::Buffer to a vector
 fn write_buffer(
     buffer: &Buffer,
     buffers: &mut Vec<ipc::Buffer>,
@@ -401,11 +579,9 @@ fn write_buffer(
 }
 
 /// Calculate an 8-byte boundary and return the number of bytes needed to pad to 8 bytes
-fn pad_to_8<'a>(len: u32) -> usize {
-    match len % 8 {
-        0 => 0 as usize,
-        v => 8 - v as usize,
-    }
+#[inline]
+fn pad_to_8(len: u32) -> usize {
+    (((len + 7) & !7) - len) as usize
 }
 
 #[cfg(test)]
@@ -417,7 +593,6 @@ mod tests {
     use crate::array::*;
     use crate::datatypes::Field;
     use crate::ipc::reader::*;
-    use crate::record_batch::RecordBatchReader;
     use crate::util::integration_util::*;
     use std::env;
     use std::fs::File;
@@ -457,7 +632,7 @@ mod tests {
                 File::open(format!("target/debug/testdata/{}.arrow_file", "arrow"))
                     .unwrap();
             let mut reader = FileReader::try_new(file).unwrap();
-            while let Ok(Some(read_batch)) = reader.next_batch() {
+            while let Some(Ok(read_batch)) = reader.next() {
                 read_batch
                     .columns()
                     .iter()
@@ -503,9 +678,10 @@ mod tests {
 
         {
             let file = File::open("target/debug/testdata/nulls.arrow_file").unwrap();
-            let mut reader = FileReader::try_new(file).unwrap();
-            while let Ok(Some(read_batch)) = reader.next_batch() {
-                read_batch
+            let reader = FileReader::try_new(file).unwrap();
+            reader.for_each(|maybe_batch| {
+                maybe_batch
+                    .unwrap()
                     .columns()
                     .iter()
                     .zip(batch.columns())
@@ -514,7 +690,7 @@ mod tests {
                         assert_eq!(a.len(), b.len());
                         assert_eq!(a.null_count(), b.null_count());
                     });
-            }
+            });
         }
     }
 
@@ -545,7 +721,7 @@ mod tests {
                     File::create(format!("target/debug/testdata/{}.arrow_file", path))
                         .unwrap();
                 let mut writer = FileWriter::try_new(file, &reader.schema()).unwrap();
-                while let Ok(Some(batch)) = reader.next_batch() {
+                while let Some(Ok(batch)) = reader.next() {
                     writer.write(&batch).unwrap();
                 }
                 writer.finish().unwrap();
@@ -580,16 +756,16 @@ mod tests {
             ))
             .unwrap();
 
-            let mut reader = StreamReader::try_new(file).unwrap();
+            let reader = StreamReader::try_new(file).unwrap();
 
             // read and rewrite the stream to a temp location
             {
                 let file = File::create(format!("target/debug/testdata/{}.stream", path))
                     .unwrap();
                 let mut writer = StreamWriter::try_new(file, &reader.schema()).unwrap();
-                while let Ok(Some(batch)) = reader.next_batch() {
-                    writer.write(&batch).unwrap();
-                }
+                reader.for_each(|batch| {
+                    writer.write(&batch.unwrap()).unwrap();
+                });
                 writer.finish().unwrap();
             }
 
diff --git a/rust/arrow/src/json/reader.rs b/rust/arrow/src/json/reader.rs
index 5bcd1c87b94..7a414db8795 100644
--- a/rust/arrow/src/json/reader.rs
+++ b/rust/arrow/src/json/reader.rs
@@ -44,6 +44,7 @@
 
 use indexmap::map::IndexMap as HashMap;
 use indexmap::set::IndexSet as HashSet;
+use std::cell::RefCell;
 use std::io::{BufRead, BufReader, Read, Seek, SeekFrom};
 use std::sync::Arc;
 
@@ -64,55 +65,86 @@ fn coerce_data_type(dt: Vec<&DataType>) -> Result<DataType> {
         1 => Ok(dt[0].clone()),
         2 => {
             // there can be a case where a list and scalar both exist
-            if dt.contains(&&DataType::List(Box::new(DataType::Float64)))
-                || dt.contains(&&DataType::List(Box::new(DataType::Int64)))
-                || dt.contains(&&DataType::List(Box::new(DataType::Boolean)))
-                || dt.contains(&&DataType::List(Box::new(DataType::Utf8)))
-            {
+            if dt.contains(&&DataType::List(Box::new(Field::new(
+                "item",
+                DataType::Float64,
+                true,
+            )))) || dt.contains(&&DataType::List(Box::new(Field::new(
+                "item",
+                DataType::Int64,
+                true,
+            )))) || dt.contains(&&DataType::List(Box::new(Field::new(
+                "item",
+                DataType::Boolean,
+                true,
+            )))) || dt.contains(&&DataType::List(Box::new(Field::new(
+                "item",
+                DataType::Utf8,
+                true,
+            )))) {
                 // we have a list and scalars, so we should get the values and coerce them
                 let mut dt = dt;
                 // sorting guarantees that the list will be the second value
                 dt.sort();
                 match (dt[0], dt[1]) {
-                    (t1, DataType::List(e)) if **e == DataType::Float64 => {
+                    (t1, DataType::List(e)) if e.data_type() == &DataType::Float64 => {
                         if t1 == &DataType::Float64 {
-                            Ok(DataType::List(Box::new(DataType::Float64)))
+                            Ok(DataType::List(Box::new(Field::new(
+                                "item",
+                                DataType::Float64,
+                                true,
+                            ))))
                         } else {
-                            Ok(DataType::List(Box::new(coerce_data_type(vec![
-                                t1,
-                                &DataType::Float64,
-                            ])?)))
+                            Ok(DataType::List(Box::new(Field::new(
+                                "item",
+                                coerce_data_type(vec![t1, &DataType::Float64])?,
+                                true,
+                            ))))
                         }
                     }
-                    (t1, DataType::List(e)) if **e == DataType::Int64 => {
+                    (t1, DataType::List(e)) if e.data_type() == &DataType::Int64 => {
                         if t1 == &DataType::Int64 {
-                            Ok(DataType::List(Box::new(DataType::Int64)))
+                            Ok(DataType::List(Box::new(Field::new(
+                                "item",
+                                DataType::Int64,
+                                true,
+                            ))))
                         } else {
-                            Ok(DataType::List(Box::new(coerce_data_type(vec![
-                                t1,
-                                &DataType::Int64,
-                            ])?)))
+                            Ok(DataType::List(Box::new(Field::new(
+                                "item",
+                                coerce_data_type(vec![t1, &DataType::Int64])?,
+                                true,
+                            ))))
                         }
                     }
-                    (t1, DataType::List(e)) if **e == DataType::Boolean => {
+                    (t1, DataType::List(e)) if e.data_type() == &DataType::Boolean => {
                         if t1 == &DataType::Boolean {
-                            Ok(DataType::List(Box::new(DataType::Boolean)))
+                            Ok(DataType::List(Box::new(Field::new(
+                                "item",
+                                DataType::Boolean,
+                                true,
+                            ))))
                         } else {
-                            Ok(DataType::List(Box::new(coerce_data_type(vec![
-                                t1,
-                                &DataType::Boolean,
-                            ])?)))
+                            Ok(DataType::List(Box::new(Field::new(
+                                "item",
+                                coerce_data_type(vec![t1, &DataType::Boolean])?,
+                                true,
+                            ))))
                         }
                     }
-                    (t1, DataType::List(e)) if **e == DataType::Utf8 => {
+                    (t1, DataType::List(e)) if e.data_type() == &DataType::Utf8 => {
                         if t1 == &DataType::Utf8 {
-                            Ok(DataType::List(Box::new(DataType::Utf8)))
+                            Ok(DataType::List(Box::new(Field::new(
+                                "item",
+                                DataType::Utf8,
+                                true,
+                            ))))
                         } else {
-                            dbg!(&t1);
-                            Ok(DataType::List(Box::new(coerce_data_type(vec![
-                                t1,
-                                &DataType::Utf8,
-                            ])?)))
+                            Ok(DataType::List(Box::new(Field::new(
+                                "item",
+                                coerce_data_type(vec![t1, &DataType::Utf8])?,
+                                true,
+                            ))))
                         }
                     }
                     (t1, t2) => Err(ArrowError::JsonError(format!(
@@ -129,7 +161,11 @@ fn coerce_data_type(dt: Vec<&DataType>) -> Result<DataType> {
         _ => {
             // TODO(nevi_me) It's possible to have [float, int, list(float)], which should
             // return list(float). Will hash this out later
-            Ok(DataType::List(Box::new(DataType::Utf8)))
+            Ok(DataType::List(Box::new(Field::new(
+                "item",
+                DataType::Utf8,
+                true,
+            ))))
         }
     }
 }
@@ -268,11 +304,15 @@ pub fn infer_json_schema<R: Read>(
 
                                             if values.contains_key(k) {
                                                 let x = values.get_mut(k).unwrap();
-                                                x.insert(DataType::List(Box::new(dt)));
+                                                x.insert(DataType::List(Box::new(
+                                                    Field::new("item", dt, true),
+                                                )));
                                             } else {
                                                 // create hashset and add value type
                                                 let mut hs = HashSet::new();
-                                                hs.insert(DataType::List(Box::new(dt)));
+                                                hs.insert(DataType::List(Box::new(
+                                                    Field::new("item", dt, true),
+                                                )));
                                                 values.insert(k.to_string(), hs);
                                             }
                                         }
@@ -448,131 +488,185 @@ impl<R: Read> Reader<R> {
 
         let rows = &rows[..];
         let projection = self.projection.clone().unwrap_or_else(Vec::new);
-        let arrays: Result<Vec<ArrayRef>> = self
-            .schema
-            .clone()
-            .fields()
-            .iter()
-            .filter(|field| {
-                if projection.is_empty() {
-                    return true;
-                }
-                projection.contains(field.name())
-            })
-            .map(|field| {
-                match field.data_type().clone() {
-                    DataType::Null => unimplemented!(),
-                    DataType::Boolean => self.build_boolean_array(rows, field.name()),
-                    DataType::Float64 => {
-                        self.build_primitive_array::<Float64Type>(rows, field.name())
-                    }
-                    DataType::Float32 => {
-                        self.build_primitive_array::<Float32Type>(rows, field.name())
-                    }
-                    DataType::Int64 => self.build_primitive_array::<Int64Type>(rows, field.name()),
-                    DataType::Int32 => self.build_primitive_array::<Int32Type>(rows, field.name()),
-                    DataType::Int16 => self.build_primitive_array::<Int16Type>(rows, field.name()),
-                    DataType::Int8 => self.build_primitive_array::<Int8Type>(rows, field.name()),
-                    DataType::UInt64 => {
-                        self.build_primitive_array::<UInt64Type>(rows, field.name())
-                    }
-                    DataType::UInt32 => {
-                        self.build_primitive_array::<UInt32Type>(rows, field.name())
-                    }
-                    DataType::UInt16 => {
-                        self.build_primitive_array::<UInt16Type>(rows, field.name())
-                    }
-                    DataType::UInt8 => self.build_primitive_array::<UInt8Type>(rows, field.name()),
-                    DataType::Utf8 => {
-                        let mut builder = StringBuilder::new(rows.len());
-                        for row in rows {
-                            if let Some(value) = row.get(field.name()) {
-                                if let Some(str_v) = value.as_str() {
-                                    builder.append_value(str_v)?
-                                } else {
-                                    builder.append(false)?
-                                }
-                            } else {
-                                builder.append(false)?
-                            }
-                        }
-                        Ok(Arc::new(builder.finish()) as ArrayRef)
+        let arrays: Result<Vec<ArrayRef>> =
+            self.schema
+                .clone()
+                .fields()
+                .iter()
+                .filter(|field| {
+                    if projection.is_empty() {
+                        return true;
                     }
-                    DataType::List(ref t) => match **t {
-                        DataType::Int8 => self.build_list_array::<Int8Type>(rows, field.name()),
-                        DataType::Int16 => self.build_list_array::<Int16Type>(rows, field.name()),
-                        DataType::Int32 => self.build_list_array::<Int32Type>(rows, field.name()),
-                        DataType::Int64 => self.build_list_array::<Int64Type>(rows, field.name()),
-                        DataType::UInt8 => self.build_list_array::<UInt8Type>(rows, field.name()),
-                        DataType::UInt16 => self.build_list_array::<UInt16Type>(rows, field.name()),
-                        DataType::UInt32 => self.build_list_array::<UInt32Type>(rows, field.name()),
-                        DataType::UInt64 => self.build_list_array::<UInt64Type>(rows, field.name()),
-                        DataType::Float32 => self.build_list_array::<Float32Type>(rows, field.name()),
-                        DataType::Float64 => self.build_list_array::<Float64Type>(rows, field.name()),
+                    projection.contains(field.name())
+                })
+                .map(|field| {
+                    match field.data_type().clone() {
                         DataType::Null => unimplemented!(),
-                        DataType::Boolean => self.build_boolean_list_array(rows, field.name()),
+                        DataType::Boolean => self.build_boolean_array(rows, field.name()),
+                        DataType::Float64 => {
+                            self.build_primitive_array::<Float64Type>(rows, field.name())
+                        }
+                        DataType::Float32 => {
+                            self.build_primitive_array::<Float32Type>(rows, field.name())
+                        }
+                        DataType::Int64 => {
+                            self.build_primitive_array::<Int64Type>(rows, field.name())
+                        }
+                        DataType::Int32 => {
+                            self.build_primitive_array::<Int32Type>(rows, field.name())
+                        }
+                        DataType::Int16 => {
+                            self.build_primitive_array::<Int16Type>(rows, field.name())
+                        }
+                        DataType::Int8 => {
+                            self.build_primitive_array::<Int8Type>(rows, field.name())
+                        }
+                        DataType::UInt64 => {
+                            self.build_primitive_array::<UInt64Type>(rows, field.name())
+                        }
+                        DataType::UInt32 => {
+                            self.build_primitive_array::<UInt32Type>(rows, field.name())
+                        }
+                        DataType::UInt16 => {
+                            self.build_primitive_array::<UInt16Type>(rows, field.name())
+                        }
+                        DataType::UInt8 => {
+                            self.build_primitive_array::<UInt8Type>(rows, field.name())
+                        }
+                        DataType::Timestamp(unit, _) => match unit {
+                            TimeUnit::Second => self
+                                .build_primitive_array::<TimestampSecondType>(
+                                    rows,
+                                    field.name(),
+                                ),
+                            TimeUnit::Microsecond => self
+                                .build_primitive_array::<TimestampMicrosecondType>(
+                                    rows,
+                                    field.name(),
+                                ),
+                            TimeUnit::Millisecond => self
+                                .build_primitive_array::<TimestampMillisecondType>(
+                                    rows,
+                                    field.name(),
+                                ),
+                            TimeUnit::Nanosecond => self
+                                .build_primitive_array::<TimestampNanosecondType>(
+                                    rows,
+                                    field.name(),
+                                ),
+                        },
+                        DataType::Date64(_) => {
+                            self.build_primitive_array::<Date64Type>(rows, field.name())
+                        }
+                        DataType::Date32(_) => {
+                            self.build_primitive_array::<Date32Type>(rows, field.name())
+                        }
+                        DataType::Time64(unit) => match unit {
+                            TimeUnit::Microsecond => self
+                                .build_primitive_array::<Time64MicrosecondType>(
+                                    rows,
+                                    field.name(),
+                                ),
+                            TimeUnit::Nanosecond => self
+                                .build_primitive_array::<Time64NanosecondType>(
+                                    rows,
+                                    field.name(),
+                                ),
+                            _ => unimplemented!(),
+                        },
+                        DataType::Time32(unit) => match unit {
+                            TimeUnit::Second => self
+                                .build_primitive_array::<Time32SecondType>(
+                                    rows,
+                                    field.name(),
+                                ),
+                            TimeUnit::Millisecond => self
+                                .build_primitive_array::<Time32MillisecondType>(
+                                    rows,
+                                    field.name(),
+                                ),
+                            _ => unimplemented!(),
+                        },
                         DataType::Utf8 => {
-                            let values_builder = StringBuilder::new(rows.len() * 5);
-                            let mut builder = ListBuilder::new(values_builder);
+                            let mut builder = StringBuilder::new(rows.len());
                             for row in rows {
                                 if let Some(value) = row.get(field.name()) {
-                                    // value can be an array or a scalar
-                                    let vals: Vec<Option<String>> = if let Value::String(v) = value {
-                                        vec![Some(v.to_string())]
-                                    } else if let Value::Array(n) = value {
-                                        n.iter().map(|v: &Value| {
-                                            if v.is_string() {
-                                                Some(v.as_str().unwrap().to_string())
-                                            } else if v.is_array() || v.is_object() {
-                                                // implicitly drop nested values
-                                                // TODO support deep-nesting
-                                                None
-                                            } else {
-                                                Some(v.to_string())
-                                            }
-                                        }).collect()
-                                    } else if let Value::Null = value {
-                                        vec![None]
-                                    } else if !value.is_object() {
-                                        vec![Some(value.to_string())]
+                                    if let Some(str_v) = value.as_str() {
+                                        builder.append_value(str_v)?
                                     } else {
-                                        return Err(ArrowError::JsonError("Only scalars are currently supported in JSON arrays".to_string()));
-                                    };
-                                    for val in vals {
-                                        if let Some(v) = val {
-                                            builder.values().append_value(&v)?
-                                        } else {
-                                            builder.values().append_null()?
-                                        };
+                                        builder.append(false)?
                                     }
+                                } else {
+                                    builder.append(false)?
                                 }
-                                builder.append(true)?
                             }
                             Ok(Arc::new(builder.finish()) as ArrayRef)
                         }
-                        _ => Err(ArrowError::JsonError("Data type is currently not supported in a list".to_string())),
-                    },
-                    DataType::Dictionary(ref key_typ, ref value_type) => {
-                        if let DataType::Utf8 = **value_type {
-                            match **key_typ {
-                                DataType::Int8 => self.build_dictionary_array::<Int8Type>(rows, field.name()),
-                                DataType::Int16 => self.build_dictionary_array::<Int16Type>(rows, field.name()),
-                                DataType::Int32 => self.build_dictionary_array::<Int32Type>(rows, field.name()),
-                                DataType::Int64 => self.build_dictionary_array::<Int64Type>(rows, field.name()),
-                                DataType::UInt8 => self.build_dictionary_array::<UInt8Type>(rows, field.name()),
-                                DataType::UInt16 => self.build_dictionary_array::<UInt16Type>(rows, field.name()),
-                                DataType::UInt32 => self.build_dictionary_array::<UInt32Type>(rows, field.name()),
-                                DataType::UInt64 => self.build_dictionary_array::<UInt64Type>(rows, field.name()),
-                                _ => Err(ArrowError::JsonError("unsupported dictionary key type".to_string()))
+                        DataType::List(ref t) => {
+                            match t.data_type() {
+                                DataType::Int8 => {
+                                    self.build_list_array::<Int8Type>(rows, field.name())
+                                }
+                                DataType::Int16 => {
+                                    self.build_list_array::<Int16Type>(rows, field.name())
+                                }
+                                DataType::Int32 => {
+                                    self.build_list_array::<Int32Type>(rows, field.name())
+                                }
+                                DataType::Int64 => {
+                                    self.build_list_array::<Int64Type>(rows, field.name())
+                                }
+                                DataType::UInt8 => {
+                                    self.build_list_array::<UInt8Type>(rows, field.name())
+                                }
+                                DataType::UInt16 => self
+                                    .build_list_array::<UInt16Type>(rows, field.name()),
+                                DataType::UInt32 => self
+                                    .build_list_array::<UInt32Type>(rows, field.name()),
+                                DataType::UInt64 => self
+                                    .build_list_array::<UInt64Type>(rows, field.name()),
+                                DataType::Float32 => self
+                                    .build_list_array::<Float32Type>(rows, field.name()),
+                                DataType::Float64 => self
+                                    .build_list_array::<Float64Type>(rows, field.name()),
+                                DataType::Null => unimplemented!(),
+                                DataType::Boolean => {
+                                    self.build_boolean_list_array(rows, field.name())
+                                }
+                                ref dtype @ DataType::Utf8 => {
+                                    // UInt64Type passed down below is a fake type for dictionary builder.
+                                    // It is there to make compiler happy.
+                                    self.list_array_string_array_builder::<UInt64Type>(
+                                        &dtype,
+                                        field.name(),
+                                        rows,
+                                    )
+                                }
+                                DataType::Dictionary(ref key_ty, _) => self
+                                    .build_wrapped_list_array(rows, field.name(), key_ty),
+                                ref e => Err(ArrowError::JsonError(format!(
+                            "Data type is currently not supported in a list : {:?}",
+                            e
+                        ))),
                             }
-                        } else {
-                            Err(ArrowError::JsonError("dictionary types other than UTF-8 not yet supported".to_string()))
                         }
+                        DataType::Dictionary(ref key_ty, ref val_ty) => self
+                            .build_string_dictionary_array(
+                                rows,
+                                field.name(),
+                                key_ty,
+                                val_ty,
+                            ),
+                        DataType::Struct(_) => Err(ArrowError::JsonError(
+                            "struct types are not yet supported".to_string(),
+                        )),
+                        _ => Err(ArrowError::JsonError(format!(
+                            "{:?} type is not supported",
+                            field.data_type()
+                        ))),
                     }
-                    _ => Err(ArrowError::JsonError("struct types are not yet supported".to_string())),
-                }
-            })
-            .collect();
+                })
+                .collect();
 
         let projected_fields: Vec<Field> = if projection.is_empty() {
             self.schema.fields().to_vec()
@@ -590,6 +684,240 @@ impl<R: Read> Reader<R> {
         arrays.and_then(|arr| RecordBatch::try_new(projected_schema, arr).map(Some))
     }
 
+    fn build_wrapped_list_array(
+        &self,
+        rows: &[Value],
+        col_name: &str,
+        key_type: &DataType,
+    ) -> Result<ArrayRef> {
+        match *key_type {
+            DataType::Int8 => {
+                let dtype = DataType::Dictionary(
+                    Box::new(DataType::Int8),
+                    Box::new(DataType::Utf8),
+                );
+                self.list_array_string_array_builder::<Int8Type>(&dtype, col_name, rows)
+            }
+            DataType::Int16 => {
+                let dtype = DataType::Dictionary(
+                    Box::new(DataType::Int16),
+                    Box::new(DataType::Utf8),
+                );
+                self.list_array_string_array_builder::<Int16Type>(&dtype, col_name, rows)
+            }
+            DataType::Int32 => {
+                let dtype = DataType::Dictionary(
+                    Box::new(DataType::Int32),
+                    Box::new(DataType::Utf8),
+                );
+                self.list_array_string_array_builder::<Int32Type>(&dtype, col_name, rows)
+            }
+            DataType::Int64 => {
+                let dtype = DataType::Dictionary(
+                    Box::new(DataType::Int64),
+                    Box::new(DataType::Utf8),
+                );
+                self.list_array_string_array_builder::<Int64Type>(&dtype, col_name, rows)
+            }
+            DataType::UInt8 => {
+                let dtype = DataType::Dictionary(
+                    Box::new(DataType::UInt8),
+                    Box::new(DataType::Utf8),
+                );
+                self.list_array_string_array_builder::<UInt8Type>(&dtype, col_name, rows)
+            }
+            DataType::UInt16 => {
+                let dtype = DataType::Dictionary(
+                    Box::new(DataType::UInt16),
+                    Box::new(DataType::Utf8),
+                );
+                self.list_array_string_array_builder::<UInt16Type>(&dtype, col_name, rows)
+            }
+            DataType::UInt32 => {
+                let dtype = DataType::Dictionary(
+                    Box::new(DataType::UInt32),
+                    Box::new(DataType::Utf8),
+                );
+                self.list_array_string_array_builder::<UInt32Type>(&dtype, col_name, rows)
+            }
+            DataType::UInt64 => {
+                let dtype = DataType::Dictionary(
+                    Box::new(DataType::UInt64),
+                    Box::new(DataType::Utf8),
+                );
+                self.list_array_string_array_builder::<UInt64Type>(&dtype, col_name, rows)
+            }
+            ref e => Err(ArrowError::JsonError(format!(
+                "Data type is currently not supported for dictionaries in list : {:?}",
+                e
+            ))),
+        }
+    }
+
+    #[inline(always)]
+    fn list_array_string_array_builder<DICT_TY>(
+        &self,
+        data_type: &DataType,
+        col_name: &str,
+        rows: &[Value],
+    ) -> Result<ArrayRef>
+    where
+        DICT_TY: ArrowPrimitiveType + ArrowDictionaryKeyType,
+    {
+        let mut builder: RefCell<Box<dyn ArrayBuilder>> = match data_type {
+            DataType::Utf8 => {
+                let values_builder = StringBuilder::new(rows.len() * 5);
+                RefCell::new(Box::new(ListBuilder::new(values_builder)))
+            }
+            DataType::Dictionary(_, _) => {
+                let values_builder =
+                    self.build_string_dictionary_builder::<DICT_TY>(rows.len() * 5)?;
+                RefCell::new(Box::new(ListBuilder::new(values_builder)))
+            }
+            e => {
+                return Err(ArrowError::JsonError(format!(
+                    "Nested list data builder type is not supported: {:?}",
+                    e
+                )))
+            }
+        };
+
+        for row in rows {
+            if let Some(value) = row.get(col_name) {
+                // value can be an array or a scalar
+                let vals: Vec<Option<String>> = if let Value::String(v) = value {
+                    vec![Some(v.to_string())]
+                } else if let Value::Array(n) = value {
+                    n.iter()
+                        .map(|v: &Value| {
+                            if v.is_string() {
+                                Some(v.as_str().unwrap().to_string())
+                            } else if v.is_array() || v.is_object() || v.is_null() {
+                                // implicitly drop nested values
+                                // TODO support deep-nesting
+                                None
+                            } else {
+                                Some(v.to_string())
+                            }
+                        })
+                        .collect()
+                } else if let Value::Null = value {
+                    vec![None]
+                } else if !value.is_object() {
+                    vec![Some(value.to_string())]
+                } else {
+                    return Err(ArrowError::JsonError(
+                        "Only scalars are currently supported in JSON arrays".to_string(),
+                    ));
+                };
+
+                // TODO: ARROW-10335: APIs of dictionary arrays and others are different. Unify
+                // them.
+                match data_type {
+                    DataType::Utf8 => {
+                        let builder = &mut builder.borrow_mut();
+                        let builder = builder
+                            .as_any_mut()
+                            .downcast_mut::<ListBuilder<StringBuilder>>()
+                            .ok_or(ArrowError::JsonError(
+                                "Cast failed for ListBuilder<StringBuilder> during nested data parsing".to_string(),
+                            ))?;
+                        for val in vals {
+                            if let Some(v) = val {
+                                builder.values().append_value(&v)?
+                            } else {
+                                builder.values().append_null()?
+                            };
+                        }
+
+                        // Append to the list
+                        builder.append(true)?;
+                    }
+                    DataType::Dictionary(_, _) => {
+                        let builder = &mut builder.borrow_mut();
+                        let builder = builder.as_any_mut().downcast_mut::<ListBuilder<StringDictionaryBuilder<DICT_TY>>>().ok_or(ArrowError::JsonError(
+                            "Cast failed for ListBuilder<StringDictionaryBuilder> during nested data parsing".to_string(),
+                        ))?;
+                        for val in vals {
+                            if let Some(v) = val {
+                                let _ = builder.values().append(&v)?;
+                            } else {
+                                builder.values().append_null()?
+                            };
+                        }
+
+                        // Append to the list
+                        builder.append(true)?;
+                    }
+                    e => {
+                        return Err(ArrowError::JsonError(format!(
+                            "Nested list data builder type is not supported: {:?}",
+                            e
+                        )))
+                    }
+                }
+            }
+        }
+
+        Ok(builder.get_mut().finish() as ArrayRef)
+    }
+
+    #[inline(always)]
+    fn build_string_dictionary_builder<T>(
+        &self,
+        row_len: usize,
+    ) -> Result<StringDictionaryBuilder<T>>
+    where
+        T: ArrowPrimitiveType + ArrowDictionaryKeyType,
+    {
+        let key_builder = PrimitiveBuilder::<T>::new(row_len);
+        let values_builder = StringBuilder::new(row_len * 5);
+        Ok(StringDictionaryBuilder::new(key_builder, values_builder))
+    }
+
+    #[inline(always)]
+    fn build_string_dictionary_array(
+        &self,
+        rows: &[Value],
+        col_name: &str,
+        key_type: &DataType,
+        value_type: &DataType,
+    ) -> Result<ArrayRef> {
+        if let DataType::Utf8 = *value_type {
+            match *key_type {
+                DataType::Int8 => self.build_dictionary_array::<Int8Type>(rows, col_name),
+                DataType::Int16 => {
+                    self.build_dictionary_array::<Int16Type>(rows, col_name)
+                }
+                DataType::Int32 => {
+                    self.build_dictionary_array::<Int32Type>(rows, col_name)
+                }
+                DataType::Int64 => {
+                    self.build_dictionary_array::<Int64Type>(rows, col_name)
+                }
+                DataType::UInt8 => {
+                    self.build_dictionary_array::<UInt8Type>(rows, col_name)
+                }
+                DataType::UInt16 => {
+                    self.build_dictionary_array::<UInt16Type>(rows, col_name)
+                }
+                DataType::UInt32 => {
+                    self.build_dictionary_array::<UInt32Type>(rows, col_name)
+                }
+                DataType::UInt64 => {
+                    self.build_dictionary_array::<UInt64Type>(rows, col_name)
+                }
+                _ => Err(ArrowError::JsonError(
+                    "unsupported dictionary key type".to_string(),
+                )),
+            }
+        } else {
+            Err(ArrowError::JsonError(
+                "dictionary types other than UTF-8 not yet supported".to_string(),
+            ))
+        }
+    }
+
     fn build_boolean_array(&self, rows: &[Value], col_name: &str) -> Result<ArrayRef> {
         let mut builder = BooleanBuilder::new(rows.len());
         for row in rows {
@@ -649,22 +977,15 @@ impl<R: Read> Reader<R> {
         T: ArrowNumericType,
         T::Native: num::NumCast,
     {
-        let mut builder = PrimitiveBuilder::<T>::new(rows.len());
-        for row in rows {
-            if let Some(value) = row.get(&col_name) {
-                // check that value is of expected datatype
-                match value.as_f64() {
-                    Some(v) => match num::cast::cast(v) {
-                        Some(v) => builder.append_value(v)?,
-                        None => builder.append_null()?,
-                    },
-                    None => builder.append_null()?,
-                }
-            } else {
-                builder.append_null()?;
-            }
-        }
-        Ok(Arc::new(builder.finish()))
+        Ok(Arc::new(
+            rows.iter()
+                .map(|row| {
+                    row.get(&col_name)
+                        .and_then(|value| value.as_f64())
+                        .and_then(num::cast::cast)
+                })
+                .collect::<PrimitiveArray<T>>(),
+        ))
     }
 
     fn build_list_array<T: ArrowPrimitiveType>(
@@ -707,18 +1028,18 @@ impl<R: Read> Reader<R> {
         Ok(Arc::new(builder.finish()))
     }
 
-    fn build_dictionary_array<T: ArrowPrimitiveType>(
+    #[inline(always)]
+    fn build_dictionary_array<T>(
         &self,
         rows: &[Value],
         col_name: &str,
     ) -> Result<ArrayRef>
     where
         T::Native: num::NumCast,
-        T: ArrowDictionaryKeyType,
+        T: ArrowPrimitiveType + ArrowDictionaryKeyType,
     {
-        let key_builder = PrimitiveBuilder::<T>::new(rows.len());
-        let value_builder = StringBuilder::new(100);
-        let mut builder = StringDictionaryBuilder::new(key_builder, value_builder);
+        let mut builder: StringDictionaryBuilder<T> =
+            self.build_string_dictionary_builder(rows.len())?;
         for row in rows {
             if let Some(value) = row.get(&col_name) {
                 if let Some(str_v) = value.as_str() {
@@ -840,7 +1161,7 @@ impl ReaderBuilder {
 
 #[cfg(test)]
 mod tests {
-    use crate::datatypes::DataType::Dictionary;
+    use crate::datatypes::DataType::{Dictionary, List};
 
     use super::*;
     use flate2::read::GzDecoder;
@@ -886,8 +1207,8 @@ mod tests {
             .as_any()
             .downcast_ref::<Float64Array>()
             .unwrap();
-        assert_eq!(2.0, bb.value(0));
-        assert_eq!(-3.5, bb.value(1));
+        assert!(2.0 - bb.value(0) < f64::EPSILON);
+        assert!(-3.5 - bb.value(1) < f64::EPSILON);
         let cc = batch
             .column(c.0)
             .as_any()
@@ -1009,8 +1330,8 @@ mod tests {
             .as_any()
             .downcast_ref::<Float32Array>()
             .unwrap();
-        assert_eq!(2.0, bb.value(0));
-        assert_eq!(-3.5, bb.value(1));
+        assert!(2.0 - bb.value(0) < f32::EPSILON);
+        assert!(-3.5 - bb.value(1) < f32::EPSILON);
     }
 
     #[test]
@@ -1035,7 +1356,7 @@ mod tests {
             Field::new("a", DataType::Int32, false),
             Field::new("c", DataType::Boolean, false),
         ]));
-        assert_eq!(reader_schema.clone(), expected_schema);
+        assert_eq!(reader_schema, expected_schema);
 
         let batch = reader.next().unwrap().unwrap();
 
@@ -1071,12 +1392,12 @@ mod tests {
         assert_eq!(&DataType::Int64, a.1.data_type());
         let b = schema.column_with_name("b").unwrap();
         assert_eq!(
-            &DataType::List(Box::new(DataType::Float64)),
+            &DataType::List(Box::new(Field::new("item", DataType::Float64, true))),
             b.1.data_type()
         );
         let c = schema.column_with_name("c").unwrap();
         assert_eq!(
-            &DataType::List(Box::new(DataType::Boolean)),
+            &DataType::List(Box::new(Field::new("item", DataType::Boolean, true))),
             c.1.data_type()
         );
         let d = schema.column_with_name("d").unwrap();
@@ -1097,8 +1418,8 @@ mod tests {
         let bb = bb.values();
         let bb = bb.as_any().downcast_ref::<Float64Array>().unwrap();
         assert_eq!(9, bb.len());
-        assert_eq!(2.0, bb.value(0));
-        assert_eq!(-6.1, bb.value(5));
+        assert!(2.0 - bb.value(0) < f64::EPSILON);
+        assert!(-6.1 - bb.value(5) < f64::EPSILON);
         assert_eq!(false, bb.is_valid(7));
 
         let cc = batch
@@ -1129,21 +1450,37 @@ mod tests {
         use crate::datatypes::DataType::*;
 
         assert_eq!(
-            List(Box::new(Float64)),
-            coerce_data_type(vec![&Float64, &List(Box::new(Float64))]).unwrap()
+            List(Box::new(Field::new("item", Float64, true))),
+            coerce_data_type(vec![
+                &Float64,
+                &List(Box::new(Field::new("item", Float64, true)))
+            ])
+            .unwrap()
         );
         assert_eq!(
-            List(Box::new(Float64)),
-            coerce_data_type(vec![&Float64, &List(Box::new(Int64))]).unwrap()
+            List(Box::new(Field::new("item", Float64, true))),
+            coerce_data_type(vec![
+                &Float64,
+                &List(Box::new(Field::new("item", Int64, true)))
+            ])
+            .unwrap()
         );
         assert_eq!(
-            List(Box::new(Int64)),
-            coerce_data_type(vec![&Int64, &List(Box::new(Int64))]).unwrap()
+            List(Box::new(Field::new("item", Int64, true))),
+            coerce_data_type(vec![
+                &Int64,
+                &List(Box::new(Field::new("item", Int64, true)))
+            ])
+            .unwrap()
         );
         // boolean and number are incompatible, return utf8
         assert_eq!(
-            List(Box::new(Utf8)),
-            coerce_data_type(vec![&Boolean, &List(Box::new(Float64))]).unwrap()
+            List(Box::new(Field::new("item", Utf8, true))),
+            coerce_data_type(vec![
+                &Boolean,
+                &List(Box::new(Field::new("item", Float64, true)))
+            ])
+            .unwrap()
         );
     }
 
@@ -1174,16 +1511,19 @@ mod tests {
             assert_eq!(&DataType::Int64, a.1.data_type());
             let b = schema.column_with_name("b").unwrap();
             assert_eq!(
-                &DataType::List(Box::new(DataType::Float64)),
+                &DataType::List(Box::new(Field::new("item", DataType::Float64, true))),
                 b.1.data_type()
             );
             let c = schema.column_with_name("c").unwrap();
             assert_eq!(
-                &DataType::List(Box::new(DataType::Boolean)),
+                &DataType::List(Box::new(Field::new("item", DataType::Boolean, true))),
                 c.1.data_type()
             );
             let d = schema.column_with_name("d").unwrap();
-            assert_eq!(&DataType::List(Box::new(DataType::Utf8)), d.1.data_type());
+            assert_eq!(
+                &DataType::List(Box::new(Field::new("item", DataType::Utf8, true))),
+                d.1.data_type()
+            );
 
             let bb = batch
                 .column(b.0)
@@ -1193,7 +1533,7 @@ mod tests {
             let bb = bb.values();
             let bb = bb.as_any().downcast_ref::<Float64Array>().unwrap();
             assert_eq!(10, bb.len());
-            assert_eq!(4.0, bb.value(9));
+            assert!(4.0 - bb.value(9) < f64::EPSILON);
 
             let cc = batch
                 .column(c.0)
@@ -1263,10 +1603,9 @@ mod tests {
         assert_eq!(true, dd.is_valid(2));
         assert_eq!(false, dd.is_valid(11));
 
-        let keys: Vec<_> = dd.keys().collect();
         assert_eq!(
-            keys,
-            vec![
+            dd.keys(),
+            &Int16Array::from(vec![
                 None,
                 Some(0),
                 Some(1),
@@ -1279,7 +1618,7 @@ mod tests {
                 Some(0),
                 Some(0),
                 None
-            ]
+            ])
         );
     }
 
@@ -1370,6 +1709,127 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_list_of_string_dictionary_from_json() {
+        let schema = Schema::new(vec![Field::new(
+            "events",
+            List(Box::new(Field::new(
+                "item",
+                Dictionary(Box::new(DataType::UInt64), Box::new(DataType::Utf8)),
+                true,
+            ))),
+            true,
+        )]);
+        let builder = ReaderBuilder::new()
+            .with_schema(Arc::new(schema))
+            .with_batch_size(64);
+        let mut reader: Reader<File> = builder
+            .build::<File>(File::open("test/data/list_string_dict_nested.json").unwrap())
+            .unwrap();
+        let batch = reader.next().unwrap().unwrap();
+
+        assert_eq!(1, batch.num_columns());
+        assert_eq!(3, batch.num_rows());
+
+        let schema = reader.schema();
+        let batch_schema = batch.schema();
+        assert_eq!(schema, batch_schema);
+
+        let events = schema.column_with_name("events").unwrap();
+        assert_eq!(
+            &List(Box::new(Field::new(
+                "item",
+                Dictionary(Box::new(DataType::UInt64), Box::new(DataType::Utf8)),
+                true
+            ))),
+            events.1.data_type()
+        );
+
+        let evs_list = batch
+            .column(events.0)
+            .as_any()
+            .downcast_ref::<ListArray>()
+            .unwrap();
+        let evs_list = evs_list.values();
+        let evs_list = evs_list
+            .as_any()
+            .downcast_ref::<DictionaryArray<UInt64Type>>()
+            .unwrap();
+        assert_eq!(6, evs_list.len());
+        assert_eq!(true, evs_list.is_valid(1));
+        assert_eq!(DataType::Utf8, evs_list.value_type());
+
+        // dict from the events list
+        let dict_el = evs_list.values();
+        let dict_el = dict_el.as_any().downcast_ref::<StringArray>().unwrap();
+        assert_eq!(3, dict_el.len());
+        assert_eq!("Elect Leader", dict_el.value(0));
+        assert_eq!("Do Ballot", dict_el.value(1));
+        assert_eq!("Send Data", dict_el.value(2));
+    }
+
+    #[test]
+    fn test_list_of_string_dictionary_from_json_with_nulls() {
+        let schema = Schema::new(vec![Field::new(
+            "events",
+            List(Box::new(Field::new(
+                "item",
+                Dictionary(Box::new(DataType::UInt64), Box::new(DataType::Utf8)),
+                true,
+            ))),
+            true,
+        )]);
+        let builder = ReaderBuilder::new()
+            .with_schema(Arc::new(schema))
+            .with_batch_size(64);
+        let mut reader: Reader<File> = builder
+            .build::<File>(
+                File::open("test/data/list_string_dict_nested_nulls.json").unwrap(),
+            )
+            .unwrap();
+        let batch = reader.next().unwrap().unwrap();
+
+        assert_eq!(1, batch.num_columns());
+        assert_eq!(3, batch.num_rows());
+
+        let schema = reader.schema();
+        let batch_schema = batch.schema();
+        assert_eq!(schema, batch_schema);
+
+        let events = schema.column_with_name("events").unwrap();
+        assert_eq!(
+            &List(Box::new(Field::new(
+                "item",
+                Dictionary(Box::new(DataType::UInt64), Box::new(DataType::Utf8)),
+                true
+            ))),
+            events.1.data_type()
+        );
+
+        let evs_list = batch
+            .column(events.0)
+            .as_any()
+            .downcast_ref::<ListArray>()
+            .unwrap();
+        let evs_list = evs_list.values();
+        let evs_list = evs_list
+            .as_any()
+            .downcast_ref::<DictionaryArray<UInt64Type>>()
+            .unwrap();
+        assert_eq!(8, evs_list.len());
+        assert_eq!(true, evs_list.is_valid(1));
+        assert_eq!(DataType::Utf8, evs_list.value_type());
+
+        // dict from the events list
+        let dict_el = evs_list.values();
+        let dict_el = dict_el.as_any().downcast_ref::<StringArray>().unwrap();
+        assert_eq!(2, evs_list.null_count());
+        assert_eq!(3, dict_el.len());
+        assert_eq!("Elect Leader", dict_el.value(0));
+        assert_eq!("Do Ballot", dict_el.value(1));
+        assert_eq!("Send Data", dict_el.value(2));
+    }
+
     #[test]
     fn test_dictionary_from_json_uint8() {
         let schema = Schema::new(vec![Field::new(
@@ -1456,6 +1916,7 @@ mod tests {
             d.1.data_type()
         );
     }
+
     #[test]
     fn test_with_multiple_batches() {
         let builder = ReaderBuilder::new()
@@ -1477,9 +1938,21 @@ mod tests {
     fn test_json_infer_schema() {
         let schema = Schema::new(vec![
             Field::new("a", DataType::Int64, true),
-            Field::new("b", DataType::List(Box::new(DataType::Float64)), true),
-            Field::new("c", DataType::List(Box::new(DataType::Boolean)), true),
-            Field::new("d", DataType::List(Box::new(DataType::Utf8)), true),
+            Field::new(
+                "b",
+                DataType::List(Box::new(Field::new("item", DataType::Float64, true))),
+                true,
+            ),
+            Field::new(
+                "c",
+                DataType::List(Box::new(Field::new("item", DataType::Boolean, true))),
+                true,
+            ),
+            Field::new(
+                "d",
+                DataType::List(Box::new(Field::new("item", DataType::Utf8, true))),
+                true,
+            ),
         ]);
 
         let mut reader =
@@ -1494,4 +1967,166 @@ mod tests {
 
         assert_eq!(inferred_schema, Arc::new(schema));
     }
+
+    #[test]
+    fn test_timestamp_from_json_seconds() {
+        let schema = Schema::new(vec![Field::new(
+            "a",
+            DataType::Timestamp(TimeUnit::Second, None),
+            true,
+        )]);
+
+        let builder = ReaderBuilder::new()
+            .with_schema(Arc::new(schema))
+            .with_batch_size(64);
+        let mut reader: Reader<File> = builder
+            .build::<File>(File::open("test/data/basic_nulls.json").unwrap())
+            .unwrap();
+        let batch = reader.next().unwrap().unwrap();
+
+        assert_eq!(1, batch.num_columns());
+        assert_eq!(12, batch.num_rows());
+
+        let schema = reader.schema();
+        let batch_schema = batch.schema();
+        assert_eq!(schema, batch_schema);
+
+        let a = schema.column_with_name("a").unwrap();
+        assert_eq!(
+            &DataType::Timestamp(TimeUnit::Second, None),
+            a.1.data_type()
+        );
+
+        let aa = batch
+            .column(a.0)
+            .as_any()
+            .downcast_ref::<TimestampSecondArray>()
+            .unwrap();
+        assert_eq!(true, aa.is_valid(0));
+        assert_eq!(false, aa.is_valid(1));
+        assert_eq!(false, aa.is_valid(2));
+        assert_eq!(1, aa.value(0));
+        assert_eq!(1, aa.value(3));
+        assert_eq!(5, aa.value(7));
+    }
+
+    #[test]
+    fn test_timestamp_from_json_milliseconds() {
+        let schema = Schema::new(vec![Field::new(
+            "a",
+            DataType::Timestamp(TimeUnit::Millisecond, None),
+            true,
+        )]);
+
+        let builder = ReaderBuilder::new()
+            .with_schema(Arc::new(schema))
+            .with_batch_size(64);
+        let mut reader: Reader<File> = builder
+            .build::<File>(File::open("test/data/basic_nulls.json").unwrap())
+            .unwrap();
+        let batch = reader.next().unwrap().unwrap();
+
+        assert_eq!(1, batch.num_columns());
+        assert_eq!(12, batch.num_rows());
+
+        let schema = reader.schema();
+        let batch_schema = batch.schema();
+        assert_eq!(schema, batch_schema);
+
+        let a = schema.column_with_name("a").unwrap();
+        assert_eq!(
+            &DataType::Timestamp(TimeUnit::Millisecond, None),
+            a.1.data_type()
+        );
+
+        let aa = batch
+            .column(a.0)
+            .as_any()
+            .downcast_ref::<TimestampMillisecondArray>()
+            .unwrap();
+        assert_eq!(true, aa.is_valid(0));
+        assert_eq!(false, aa.is_valid(1));
+        assert_eq!(false, aa.is_valid(2));
+        assert_eq!(1, aa.value(0));
+        assert_eq!(1, aa.value(3));
+        assert_eq!(5, aa.value(7));
+    }
+
+    #[test]
+    fn test_date_from_json_milliseconds() {
+        let schema = Schema::new(vec![Field::new(
+            "a",
+            DataType::Date64(DateUnit::Millisecond),
+            true,
+        )]);
+
+        let builder = ReaderBuilder::new()
+            .with_schema(Arc::new(schema))
+            .with_batch_size(64);
+        let mut reader: Reader<File> = builder
+            .build::<File>(File::open("test/data/basic_nulls.json").unwrap())
+            .unwrap();
+        let batch = reader.next().unwrap().unwrap();
+
+        assert_eq!(1, batch.num_columns());
+        assert_eq!(12, batch.num_rows());
+
+        let schema = reader.schema();
+        let batch_schema = batch.schema();
+        assert_eq!(schema, batch_schema);
+
+        let a = schema.column_with_name("a").unwrap();
+        assert_eq!(&DataType::Date64(DateUnit::Millisecond), a.1.data_type());
+
+        let aa = batch
+            .column(a.0)
+            .as_any()
+            .downcast_ref::<Date64Array>()
+            .unwrap();
+        assert_eq!(true, aa.is_valid(0));
+        assert_eq!(false, aa.is_valid(1));
+        assert_eq!(false, aa.is_valid(2));
+        assert_eq!(1, aa.value(0));
+        assert_eq!(1, aa.value(3));
+        assert_eq!(5, aa.value(7));
+    }
+
+    #[test]
+    fn test_time_from_json_nanoseconds() {
+        let schema = Schema::new(vec![Field::new(
+            "a",
+            DataType::Time64(TimeUnit::Nanosecond),
+            true,
+        )]);
+
+        let builder = ReaderBuilder::new()
+            .with_schema(Arc::new(schema))
+            .with_batch_size(64);
+        let mut reader: Reader<File> = builder
+            .build::<File>(File::open("test/data/basic_nulls.json").unwrap())
+            .unwrap();
+        let batch = reader.next().unwrap().unwrap();
+
+        assert_eq!(1, batch.num_columns());
+        assert_eq!(12, batch.num_rows());
+
+        let schema = reader.schema();
+        let batch_schema = batch.schema();
+        assert_eq!(schema, batch_schema);
+
+        let a = schema.column_with_name("a").unwrap();
+        assert_eq!(&DataType::Time64(TimeUnit::Nanosecond), a.1.data_type());
+
+        let aa = batch
+            .column(a.0)
+            .as_any()
+            .downcast_ref::<Time64NanosecondArray>()
+            .unwrap();
+        assert_eq!(true, aa.is_valid(0));
+        assert_eq!(false, aa.is_valid(1));
+        assert_eq!(false, aa.is_valid(2));
+        assert_eq!(1, aa.value(0));
+        assert_eq!(1, aa.value(3));
+        assert_eq!(5, aa.value(7));
+    }
 }
diff --git a/rust/arrow/src/lib.rs b/rust/arrow/src/lib.rs
index 13a1321af4b..09c04be4b2c 100644
--- a/rust/arrow/src/lib.rs
+++ b/rust/arrow/src/lib.rs
@@ -18,14 +18,117 @@
 //! A native Rust implementation of [Apache Arrow](https://arrow.apache.org), a cross-language
 //! development platform for in-memory data.
 //!
-//! Currently the project is developed and tested against nightly Rust. To learn more
-//! about the status of Arrow in Rust, see `README.md`.
+//! ### DataType
+//!
+//! Every [`Array`](array::Array) in this crate has an associated [`DataType`](datatypes::DataType),
+//! that specifies how its data is layed in memory and represented.
+//! Thus, a central enum of this crate is [`DataType`](datatypes::DataType), that contains the set of valid
+//! DataTypes in the specification. For example, [`DataType::Utf8`](datatypes::DataType::Utf8).
+//!
+//! ## Array
+//!
+//! The central trait of this package is the dynamically-typed [`Array`](array::Array) that
+//! represents a fixed-sized, immutable, Send + Sync Array of nullable elements. An example of such an array is [`UInt32Array`](array::UInt32Array).
+//! One way to think about an arrow [`Array`](array::Array) is a `Arc<[Option<T>; len]>` where T can be anything ranging from an integer to a string, or even
+//! another [`Array`](array::Array).
+//!
+//! [`Arrays`](array::Array) have [`len()`](array::Array::len), [`data_type()`](array::Array::data_type), and the nullability of each of its elements,
+//! can be obtained via [`is_null(index)`](array::Array::is_null). To downcast an [`Array`](array::Array) to a specific implementation, you can use
+//!
+//! ```rust
+//! use arrow::array::{Array, UInt32Array};
+//! let array = UInt32Array::from(vec![Some(1), None, Some(3)]);
+//! assert_eq!(array.len(), 3);
+//! assert_eq!(array.value(0), 1);
+//! assert_eq!(array.is_null(1), true);
+//! ```
+//!
+//! To make the array dynamically typed, we wrap it in an [`Arc`](std::sync::Arc):
+//!
+//! ```rust
+//! # use std::sync::Arc;
+//! use arrow::datatypes::DataType;
+//! use arrow::array::{UInt32Array, ArrayRef};
+//! # let array = UInt32Array::from(vec![Some(1), None, Some(3)]);
+//! let array: ArrayRef = Arc::new(array);
+//! assert_eq!(array.len(), 3);
+//! // array.value() is not available in the dynamically-typed version
+//! assert_eq!(array.is_null(1), true);
+//! assert_eq!(array.data_type(), &DataType::UInt32);
+//! ```
+//!
+//! to downcast, use `as_any()`:
+//!
+//! ```rust
+//! # use std::sync::Arc;
+//! # use arrow::array::{UInt32Array, ArrayRef};
+//! # let array = UInt32Array::from(vec![Some(1), None, Some(3)]);
+//! # let array: ArrayRef = Arc::new(array);
+//! let array = array.as_any().downcast_ref::<UInt32Array>().unwrap();
+//! assert_eq!(array.value(0), 1);
+//! ```
+//!
+//! ## Memory and Buffers
+//!
+//! Data in [`Array`](array::Array) is stored in [`ArrayData`](array::data::ArrayData), that in turn
+//! is a collection of other [`ArrayData`](array::data::ArrayData) and [`Buffers`](buffer::Buffer).
+//! [`Buffers`](buffer::Buffer) is the central struct that array implementations use keep allocated memory and pointers.
+//! The [`MutableBuffer`](buffer::MutableBuffer) is the mutable counter-part of[`Buffer`](buffer::Buffer).
+//! These are the lowest abstractions of this crate, and are used throughout the crate to
+//! efficiently allocate, write, read and deallocate memory.
+//!
+//! ## Field, Schema and RecordBatch
+//!
+//! [`Field`](datatypes::Field) is a struct that contains an array's metadata (datatype and whether its values
+//! can be null), and a name. [`Schema`](datatypes::Schema) is a vector of fields with optional metadata.
+//! Together, they form the basis of a schematic representation of a group of [`Arrays`](array::Array).
+//!
+//! In fact, [`RecordBatch`](record_batch::RecordBatch) is a struct with a [`Schema`](datatypes::Schema) and a vector of
+//! [`Array`](array::Array)s, all with the same `len`. A record batch is the highest order struct that this crate currently offers
+//! and is broadly used to represent a table where each column in an `Array`.
+//!
+//! ## Compute
+//!
+//! This crate offers many operations (called kernels) to operate on `Array`s, that you can find at [compute::kernels].
+//! It has both vertial and horizontal operations, and some of them have an SIMD implementation.
+//!
+//! ## Status
+//!
+//! This crate has most of the implementation of the arrow specification. Specifically, it supports the following types:
+//!
+//! * All arrow primitive types, such as [`Int32Array`](array::UInt8Array), [`BooleanArray`](array::BooleanArray) and [`Float64Array`](array::Float64Array).
+//! * All arrow variable length types, such as [`StringArray`](array::StringArray) and [`BinaryArray`](array::BinaryArray)
+//! * All composite types such as [`StructArray`](array::StructArray) and [`ListArray`](array::ListArray)
+//! * Dictionary types  [`DictionaryArray`](array::DictionaryArray)
+
+//!
+//! This crate also implements many common vertical operations:
+//! * all mathematical binary operators, such as [`subtract`](compute::kernels::arithmetic::subtract)
+//! * all boolean binary operators such as [`equality`](compute::kernels::comparison::eq)
+//! * [`cast`](compute::kernels::cast::cast)
+//! * [`filter`](compute::kernels::filter::filter)
+//! * [`take`](compute::kernels::take::take) and [`limit`](compute::kernels::limit::limit)
+//! * [`sort`](compute::kernels::sort::sort)
+//! * some string operators such as [`substring`](compute::kernels::substring::substring) and [`length`](compute::kernels::length::length)
+//!
+//! as well as some horizontal operations, such as
+//!
+//! * [`min`](compute::kernels::aggregate::min) and [`max`](compute::kernels::aggregate::max)
+//! * [`sum`](compute::kernels::aggregate::sum)
+//!
+//! Finally, this crate implements some readers and writers to different formats:
+//!
+//! * json: [reader](json::reader::Reader)
+//! * csv: [reader](csv::reader::Reader) and [writer](csv::writer::Writer)
+//! * ipc: [reader](ipc::reader::StreamReader) and [writer](ipc::writer::FileWriter)
+//!
+//! The parquet implementation is on a [separate crate](https://crates.io/crates/parquet)
 
-#![feature(specialization)]
 #![allow(dead_code)]
 #![allow(non_camel_case_types)]
 #![allow(bare_trait_objects)]
 #![warn(missing_debug_implementations)]
+#![deny(clippy::redundant_clone)]
 
 pub mod array;
 pub mod bitmap;
@@ -34,11 +137,6 @@ pub mod compute;
 pub mod csv;
 pub mod datatypes;
 pub mod error;
-#[allow(clippy::redundant_closure)]
-#[allow(clippy::needless_lifetimes)]
-#[allow(clippy::extra_unused_lifetimes)]
-#[allow(clippy::redundant_static_lifetimes)]
-#[allow(clippy::redundant_field_names)]
 pub mod ipc;
 pub mod json;
 pub mod memory;
diff --git a/rust/arrow/src/memory.rs b/rust/arrow/src/memory.rs
index b5218f7a00a..802498abd22 100644
--- a/rust/arrow/src/memory.rs
+++ b/rust/arrow/src/memory.rs
@@ -201,9 +201,9 @@ pub unsafe fn reallocate(ptr: *mut u8, old_size: usize, new_size: usize) -> *mut
 ///
 /// Behavior is undefined if any of the following conditions are violated:
 ///
-/// * `src` must be [valid] for reads of `len * size_of::<T>()` bytes.
+/// * `src` must be valid for reads of `len * size_of::<T>()` bytes.
 ///
-/// * `dst` must be [valid] for writes of `len * size_of::<T>()` bytes.
+/// * `dst` must be valid for writes of `len * size_of::<T>()` bytes.
 ///
 /// * Both `src` and `dst` must be properly aligned.
 ///
diff --git a/rust/arrow/src/record_batch.rs b/rust/arrow/src/record_batch.rs
index 76b154ddb25..b4aa97dd2a2 100644
--- a/rust/arrow/src/record_batch.rs
+++ b/rust/arrow/src/record_batch.rs
@@ -33,7 +33,7 @@ use crate::error::{ArrowError, Result};
 /// datatypes.
 ///
 /// Record batches are a convenient unit of work for various
-/// serialization and computation functions, possibly incremental.  
+/// serialization and computation functions, possibly incremental.
 /// See also [CSV reader](crate::csv::Reader) and
 /// [JSON reader](crate::json::Reader).
 #[derive(Clone, Debug)]
@@ -99,6 +99,7 @@ impl RecordBatch {
                     "all columns in a record batch must have the same length".to_string(),
                 ));
             }
+            // list types can have different names, but we only need the data types to be the same
             if column.data_type() != schema.field(i).data_type() {
                 return Err(ArrowError::InvalidArgumentError(format!(
                     "column types must match schema types, expected {:?} but found {:?} at column index {}",
@@ -217,7 +218,7 @@ impl Into<StructArray> for RecordBatch {
 }
 
 /// Trait for types that can read `RecordBatch`'s.
-pub trait RecordBatchReader {
+pub trait RecordBatchReader: Iterator<Item = Result<RecordBatch>> {
     /// Returns the schema of this `RecordBatchReader`.
     ///
     /// Implementation of this trait should guarantee that all `RecordBatch`'s returned by this
@@ -225,7 +226,13 @@ pub trait RecordBatchReader {
     fn schema(&self) -> SchemaRef;
 
     /// Reads the next `RecordBatch`.
-    fn next_batch(&mut self) -> Result<Option<RecordBatch>>;
+    #[deprecated(
+        since = "2.0.0",
+        note = "This method is deprecated in favour of `next` from the trait Iterator."
+    )]
+    fn next_batch(&mut self) -> Result<Option<RecordBatch>> {
+        self.next().transpose()
+    }
 }
 
 #[cfg(test)]
@@ -255,7 +262,7 @@ mod tests {
             .add_buffer(Buffer::from(offset_data.to_byte_slice()))
             .add_buffer(Buffer::from(v.to_byte_slice()))
             .build();
-        let b = BinaryArray::from(array_data);
+        let b = StringArray::from(array_data);
 
         let record_batch =
             RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)])
diff --git a/rust/arrow/src/tensor.rs b/rust/arrow/src/tensor.rs
index 57388b88cfd..92c5cfee7b1 100644
--- a/rust/arrow/src/tensor.rs
+++ b/rust/arrow/src/tensor.rs
@@ -24,13 +24,22 @@ use std::mem;
 use crate::buffer::Buffer;
 use crate::datatypes::*;
 
+use crate::error::{ArrowError, Result};
+
 /// Computes the strides required assuming a row major memory layout
-fn compute_row_major_strides<T: ArrowPrimitiveType>(shape: &[usize]) -> Vec<usize> {
+fn compute_row_major_strides<T: ArrowPrimitiveType>(
+    shape: &[usize],
+) -> Result<Vec<usize>> {
     let mut remaining_bytes = mem::size_of::<T::Native>();
+
     for i in shape {
-        remaining_bytes = remaining_bytes
-            .checked_mul(*i)
-            .expect("Overflow occurred when computing row major strides.");
+        if let Some(val) = remaining_bytes.checked_mul(*i) {
+            remaining_bytes = val;
+        } else {
+            return Err(ArrowError::ComputeError(
+                "overflow occurred when computing row major strides.".to_string(),
+            ));
+        }
     }
 
     let mut strides = Vec::<usize>::new();
@@ -38,20 +47,30 @@ fn compute_row_major_strides<T: ArrowPrimitiveType>(shape: &[usize]) -> Vec<usiz
         remaining_bytes /= *i;
         strides.push(remaining_bytes);
     }
-    strides
+
+    Ok(strides)
 }
 
 /// Computes the strides required assuming a column major memory layout
-fn compute_column_major_strides<T: ArrowPrimitiveType>(shape: &[usize]) -> Vec<usize> {
+fn compute_column_major_strides<T: ArrowPrimitiveType>(
+    shape: &[usize],
+) -> Result<Vec<usize>> {
     let mut remaining_bytes = mem::size_of::<T::Native>();
     let mut strides = Vec::<usize>::new();
+
     for i in shape {
         strides.push(remaining_bytes);
-        remaining_bytes = remaining_bytes
-            .checked_mul(*i)
-            .expect("Overflow occurred when computing column major strides.");
+
+        if let Some(val) = remaining_bytes.checked_mul(*i) {
+            remaining_bytes = val;
+        } else {
+            return Err(ArrowError::ComputeError(
+                "overflow occurred when computing column major strides.".to_string(),
+            ));
+        }
     }
-    strides
+
+    Ok(strides)
 }
 
 /// Tensor of primitive types
@@ -79,49 +98,95 @@ pub type Float64Tensor<'a> = Tensor<'a, Float64Type>;
 
 impl<'a, T: ArrowPrimitiveType> Tensor<'a, T> {
     /// Creates a new `Tensor`
-    pub fn new(
+    pub fn try_new(
         buffer: Buffer,
         shape: Option<Vec<usize>>,
         strides: Option<Vec<usize>>,
         names: Option<Vec<&'a str>>,
-    ) -> Self {
-        match &shape {
+    ) -> Result<Self> {
+        match shape {
             None => {
-                assert_eq!(
-                    buffer.len(),
-                    mem::size_of::<T::Native>(),
-                    "underlying buffer should only contain a single tensor element"
-                );
-                assert_eq!(None, strides);
-                assert_eq!(None, names);
+                if buffer.len() != mem::size_of::<T::Native>() {
+                    return Err(ArrowError::InvalidArgumentError(
+                        "underlying buffer should only contain a single tensor element"
+                            .to_string(),
+                    ));
+                }
+
+                if strides != None {
+                    return Err(ArrowError::InvalidArgumentError(
+                        "expected None strides for tensor with no shape".to_string(),
+                    ));
+                }
+
+                if names != None {
+                    return Err(ArrowError::InvalidArgumentError(
+                        "expected None names for tensor with no shape".to_string(),
+                    ));
+                }
             }
+
             Some(ref s) => {
-                strides
-                    .iter()
-                    .map(|i| {
-                        assert_eq!(s.len(), i.len(), "shape and stride dimensions differ")
-                    })
-                    .next();
-                names
-                    .iter()
-                    .map(|i| {
-                        assert_eq!(
-                            s.len(),
-                            i.len(),
+                if let Some(ref st) = strides {
+                    if st.len() != s.len() {
+                        return Err(ArrowError::InvalidArgumentError(
+                            "shape and stride dimensions differ".to_string(),
+                        ));
+                    }
+                }
+
+                if let Some(ref n) = names {
+                    if n.len() != s.len() {
+                        return Err(ArrowError::InvalidArgumentError(
                             "number of dimensions and number of dimension names differ"
-                        )
-                    })
-                    .next();
+                                .to_string(),
+                        ));
+                    }
+                }
+
+                let total_elements: usize = s.iter().product();
+                if total_elements != (buffer.len() / mem::size_of::<T::Native>()) {
+                    return Err(ArrowError::InvalidArgumentError(
+                        "number of elements in buffer does not match dimensions"
+                            .to_string(),
+                    ));
+                }
+            }
+        };
+
+        // Checking that the tensor strides used for construction are correct
+        // otherwise a row major stride is calculated and used as value for the tensor
+        let tensor_strides = {
+            if let Some(st) = strides {
+                if let Some(ref s) = shape {
+                    if compute_row_major_strides::<T>(s)? == st
+                        || compute_column_major_strides::<T>(s)? == st
+                    {
+                        Some(st)
+                    } else {
+                        return Err(ArrowError::InvalidArgumentError(
+                            "the input stride does not match the selected shape"
+                                .to_string(),
+                        ));
+                    }
+                } else {
+                    Some(st)
+                }
+            } else if let Some(ref s) = shape {
+                Some(compute_row_major_strides::<T>(s)?)
+            } else {
+                None
             }
         };
-        Self {
-            data_type: T::get_data_type(),
+
+        Ok(Self {
+            data_type: T::DATA_TYPE,
             buffer,
             shape,
-            strides,
+            strides: tensor_strides,
             names,
             _marker: PhantomData,
-        }
+        })
     }
 
     /// Creates a new Tensor using row major memory layout
@@ -129,11 +194,16 @@ impl<'a, T: ArrowPrimitiveType> Tensor<'a, T> {
         buffer: Buffer,
         shape: Option<Vec<usize>>,
         names: Option<Vec<&'a str>>,
-    ) -> Self {
-        let strides = shape
-            .as_ref()
-            .map(|ref s| compute_row_major_strides::<T>(&s));
-        Self::new(buffer, shape, strides, names)
+    ) -> Result<Self> {
+        if let Some(ref s) = shape {
+            let strides = Some(compute_row_major_strides::<T>(&s)?);
+
+            Self::try_new(buffer, shape, strides, names)
+        } else {
+            Err(ArrowError::InvalidArgumentError(
+                "shape required to create row major tensor".to_string(),
+            ))
+        }
     }
 
     /// Creates a new Tensor using column major memory layout
@@ -141,11 +211,16 @@ impl<'a, T: ArrowPrimitiveType> Tensor<'a, T> {
         buffer: Buffer,
         shape: Option<Vec<usize>>,
         names: Option<Vec<&'a str>>,
-    ) -> Self {
-        let strides = shape
-            .as_ref()
-            .map(|ref s| compute_column_major_strides::<T>(&s));
-        Self::new(buffer, shape, strides, names)
+    ) -> Result<Self> {
+        if let Some(ref s) = shape {
+            let strides = Some(compute_column_major_strides::<T>(&s)?);
+
+            Self::try_new(buffer, shape, strides, names)
+        } else {
+            Err(ArrowError::InvalidArgumentError(
+                "shape required to create column major tensor".to_string(),
+            ))
+        }
     }
 
     /// The data type of the `Tensor`
@@ -195,23 +270,25 @@ impl<'a, T: ArrowPrimitiveType> Tensor<'a, T> {
     }
 
     /// Indicates if the data is laid out contiguously in memory
-    pub fn is_contiguous(&self) -> bool {
-        self.is_row_major() || self.is_column_major()
+    pub fn is_contiguous(&self) -> Result<bool> {
+        Ok(self.is_row_major()? || self.is_column_major()?)
     }
 
     /// Indicates if the memory layout row major
-    pub fn is_row_major(&self) -> bool {
+    pub fn is_row_major(&self) -> Result<bool> {
         match self.shape {
-            None => false,
-            Some(ref s) => Some(compute_row_major_strides::<T>(s)) == self.strides,
+            None => Ok(false),
+            Some(ref s) => Ok(Some(compute_row_major_strides::<T>(s)?) == self.strides),
         }
     }
 
     /// Indicates if the memory layout column major
-    pub fn is_column_major(&self) -> bool {
+    pub fn is_column_major(&self) -> Result<bool> {
         match self.shape {
-            None => false,
-            Some(ref s) => Some(compute_column_major_strides::<T>(s)) == self.strides,
+            None => Ok(false),
+            Some(ref s) => {
+                Ok(Some(compute_column_major_strides::<T>(s)?) == self.strides)
+            }
         }
     }
 }
@@ -226,56 +303,56 @@ mod tests {
     #[test]
     fn test_compute_row_major_strides() {
         assert_eq!(
-            vec![48, 8],
-            compute_row_major_strides::<Int64Type>(&vec![4_usize, 6])
+            vec![48_usize, 8],
+            compute_row_major_strides::<Int64Type>(&[4_usize, 6]).unwrap()
         );
         assert_eq!(
-            vec![24, 4],
-            compute_row_major_strides::<Int32Type>(&vec![4_usize, 6])
+            vec![24_usize, 4],
+            compute_row_major_strides::<Int32Type>(&[4_usize, 6]).unwrap()
         );
         assert_eq!(
-            vec![6, 1],
-            compute_row_major_strides::<Int8Type>(&vec![4_usize, 6])
+            vec![6_usize, 1],
+            compute_row_major_strides::<Int8Type>(&[4_usize, 6]).unwrap()
         );
     }
 
     #[test]
     fn test_compute_column_major_strides() {
         assert_eq!(
-            vec![8, 32],
-            compute_column_major_strides::<Int64Type>(&vec![4_usize, 6])
+            vec![8_usize, 32],
+            compute_column_major_strides::<Int64Type>(&[4_usize, 6]).unwrap()
         );
         assert_eq!(
-            vec![4, 16],
-            compute_column_major_strides::<Int32Type>(&vec![4_usize, 6])
+            vec![4_usize, 16],
+            compute_column_major_strides::<Int32Type>(&[4_usize, 6]).unwrap()
         );
         assert_eq!(
-            vec![1, 4],
-            compute_column_major_strides::<Int8Type>(&vec![4_usize, 6])
+            vec![1_usize, 4],
+            compute_column_major_strides::<Int8Type>(&[4_usize, 6]).unwrap()
         );
     }
 
     #[test]
     fn test_zero_dim() {
         let buf = Buffer::from(&[1]);
-        let tensor = UInt8Tensor::new(buf, None, None, None);
+        let tensor = UInt8Tensor::try_new(buf, None, None, None).unwrap();
         assert_eq!(0, tensor.size());
         assert_eq!(None, tensor.shape());
         assert_eq!(None, tensor.names());
         assert_eq!(0, tensor.ndim());
-        assert_eq!(false, tensor.is_row_major());
-        assert_eq!(false, tensor.is_column_major());
-        assert_eq!(false, tensor.is_contiguous());
+        assert_eq!(false, tensor.is_row_major().unwrap());
+        assert_eq!(false, tensor.is_column_major().unwrap());
+        assert_eq!(false, tensor.is_contiguous().unwrap());
 
         let buf = Buffer::from(&[1, 2, 2, 2]);
-        let tensor = Int32Tensor::new(buf, None, None, None);
+        let tensor = Int32Tensor::try_new(buf, None, None, None).unwrap();
         assert_eq!(0, tensor.size());
         assert_eq!(None, tensor.shape());
         assert_eq!(None, tensor.names());
         assert_eq!(0, tensor.ndim());
-        assert_eq!(false, tensor.is_row_major());
-        assert_eq!(false, tensor.is_column_major());
-        assert_eq!(false, tensor.is_contiguous());
+        assert_eq!(false, tensor.is_row_major().unwrap());
+        assert_eq!(false, tensor.is_column_major().unwrap());
+        assert_eq!(false, tensor.is_contiguous().unwrap());
     }
 
     #[test]
@@ -285,10 +362,10 @@ mod tests {
             builder.append(i).unwrap();
         }
         let buf = builder.finish();
-        let tensor = Int32Tensor::new(buf, Some(vec![2, 8]), None, None);
+        let tensor = Int32Tensor::try_new(buf, Some(vec![2, 8]), None, None).unwrap();
         assert_eq!(16, tensor.size());
         assert_eq!(Some(vec![2_usize, 8]).as_ref(), tensor.shape());
-        assert_eq!(None, tensor.strides());
+        assert_eq!(Some(vec![32_usize, 4]).as_ref(), tensor.strides());
         assert_eq!(2, tensor.ndim());
         assert_eq!(None, tensor.names());
     }
@@ -300,15 +377,15 @@ mod tests {
             builder.append(i).unwrap();
         }
         let buf = builder.finish();
-        let tensor = Int32Tensor::new_row_major(buf, Some(vec![2, 8]), None);
+        let tensor = Int32Tensor::new_row_major(buf, Some(vec![2, 8]), None).unwrap();
         assert_eq!(16, tensor.size());
         assert_eq!(Some(vec![2_usize, 8]).as_ref(), tensor.shape());
         assert_eq!(Some(vec![32_usize, 4]).as_ref(), tensor.strides());
         assert_eq!(None, tensor.names());
         assert_eq!(2, tensor.ndim());
-        assert_eq!(true, tensor.is_row_major());
-        assert_eq!(false, tensor.is_column_major());
-        assert_eq!(true, tensor.is_contiguous());
+        assert_eq!(true, tensor.is_row_major().unwrap());
+        assert_eq!(false, tensor.is_column_major().unwrap());
+        assert_eq!(true, tensor.is_contiguous().unwrap());
     }
 
     #[test]
@@ -318,15 +395,15 @@ mod tests {
             builder.append(i).unwrap();
         }
         let buf = builder.finish();
-        let tensor = Int32Tensor::new_column_major(buf, Some(vec![2, 8]), None);
+        let tensor = Int32Tensor::new_column_major(buf, Some(vec![2, 8]), None).unwrap();
         assert_eq!(16, tensor.size());
         assert_eq!(Some(vec![2_usize, 8]).as_ref(), tensor.shape());
         assert_eq!(Some(vec![4_usize, 8]).as_ref(), tensor.strides());
         assert_eq!(None, tensor.names());
         assert_eq!(2, tensor.ndim());
-        assert_eq!(false, tensor.is_row_major());
-        assert_eq!(true, tensor.is_column_major());
-        assert_eq!(true, tensor.is_contiguous());
+        assert_eq!(false, tensor.is_row_major().unwrap());
+        assert_eq!(true, tensor.is_column_major().unwrap());
+        assert_eq!(true, tensor.is_contiguous().unwrap());
     }
 
     #[test]
@@ -337,44 +414,82 @@ mod tests {
         }
         let buf = builder.finish();
         let names = vec!["Dim 1", "Dim 2"];
-        let tensor = Int64Tensor::new_column_major(buf, Some(vec![2, 4]), Some(names));
+        let tensor =
+            Int64Tensor::new_column_major(buf, Some(vec![2, 4]), Some(names)).unwrap();
         assert_eq!(8, tensor.size());
         assert_eq!(Some(vec![2_usize, 4]).as_ref(), tensor.shape());
         assert_eq!(Some(vec![8_usize, 16]).as_ref(), tensor.strides());
         assert_eq!("Dim 1", tensor.dim_name(0).unwrap());
         assert_eq!("Dim 2", tensor.dim_name(1).unwrap());
         assert_eq!(2, tensor.ndim());
-        assert_eq!(false, tensor.is_row_major());
-        assert_eq!(true, tensor.is_column_major());
-        assert_eq!(true, tensor.is_contiguous());
+        assert_eq!(false, tensor.is_row_major().unwrap());
+        assert_eq!(true, tensor.is_column_major().unwrap());
+        assert_eq!(true, tensor.is_contiguous().unwrap());
     }
 
     #[test]
-    #[should_panic(expected = "shape and stride dimensions differ")]
     fn test_inconsistent_strides() {
         let mut builder = Int32BufferBuilder::new(16);
         for i in 0..16 {
             builder.append(i).unwrap();
         }
         let buf = builder.finish();
-        Int32Tensor::new(buf, Some(vec![2, 8]), Some(vec![2, 8, 1]), None);
+
+        let result =
+            Int32Tensor::try_new(buf, Some(vec![2, 8]), Some(vec![2, 8, 1]), None);
+
+        if result.is_ok() {
+            panic!("shape and stride dimensions are different")
+        }
     }
 
     #[test]
-    #[should_panic(
-        expected = "number of dimensions and number of dimension names differ"
-    )]
     fn test_inconsistent_names() {
         let mut builder = Int32BufferBuilder::new(16);
         for i in 0..16 {
             builder.append(i).unwrap();
         }
         let buf = builder.finish();
-        Int32Tensor::new(
+
+        let result = Int32Tensor::try_new(
             buf,
             Some(vec![2, 8]),
             Some(vec![4, 8]),
             Some(vec!["1", "2", "3"]),
         );
+
+        if result.is_ok() {
+            panic!("dimensions and names have different shape")
+        }
+    }
+
+    #[test]
+    fn test_incorrect_shape() {
+        let mut builder = Int32BufferBuilder::new(16);
+        for i in 0..16 {
+            builder.append(i).unwrap();
+        }
+        let buf = builder.finish();
+
+        let result = Int32Tensor::try_new(buf, Some(vec![2, 6]), None, None);
+
+        if result.is_ok() {
+            panic!("number of elements does not match for the shape")
+        }
+    }
+
+    #[test]
+    fn test_incorrect_stride() {
+        let mut builder = Int32BufferBuilder::new(16);
+        for i in 0..16 {
+            builder.append(i).unwrap();
+        }
+        let buf = builder.finish();
+
+        let result = Int32Tensor::try_new(buf, Some(vec![2, 8]), Some(vec![30, 4]), None);
+
+        if result.is_ok() {
+            panic!("the input stride does not match the selected shape")
+        }
     }
 }
diff --git a/rust/arrow/src/util/bit_chunk_iterator.rs b/rust/arrow/src/util/bit_chunk_iterator.rs
new file mode 100644
index 00000000000..df6caf9a9cd
--- /dev/null
+++ b/rust/arrow/src/util/bit_chunk_iterator.rs
@@ -0,0 +1,223 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+use crate::buffer::Buffer;
+use crate::util::bit_util::ceil;
+use std::fmt::Debug;
+
+#[derive(Debug)]
+pub struct BitChunks<'a> {
+    buffer: &'a Buffer,
+    raw_data: *const u8,
+    offset: usize,
+    chunk_len: usize,
+    remainder_len: usize,
+}
+
+impl<'a> BitChunks<'a> {
+    pub fn new(buffer: &'a Buffer, offset: usize, len: usize) -> Self {
+        assert!(ceil(offset + len, 8) <= buffer.len() * 8);
+
+        let byte_offset = offset / 8;
+        let offset = offset % 8;
+
+        let raw_data = unsafe { buffer.raw_data().add(byte_offset) };
+
+        let chunk_bits = 64;
+
+        let chunk_len = len / chunk_bits;
+        let remainder_len = len & (chunk_bits - 1);
+
+        BitChunks::<'a> {
+            buffer: &buffer,
+            raw_data,
+            offset,
+            chunk_len,
+            remainder_len,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct BitChunkIterator<'a> {
+    buffer: &'a Buffer,
+    raw_data: *const u8,
+    offset: usize,
+    chunk_len: usize,
+    index: usize,
+}
+
+impl<'a> BitChunks<'a> {
+    #[inline]
+    pub const fn remainder_len(&self) -> usize {
+        self.remainder_len
+    }
+
+    #[inline]
+    pub fn remainder_bits(&self) -> u64 {
+        let bit_len = self.remainder_len;
+        if bit_len == 0 {
+            0
+        } else {
+            let byte_len = ceil(bit_len, 8);
+
+            let mut bits = 0;
+            for i in 0..byte_len {
+                let byte = unsafe {
+                    std::ptr::read(
+                        self.raw_data
+                            .add(self.chunk_len * std::mem::size_of::<u64>() + i),
+                    )
+                };
+                bits |= (byte as u64) << (i * 8);
+            }
+
+            let offset = self.offset as u64;
+
+            (bits >> offset) & ((1 << bit_len) - 1)
+        }
+    }
+
+    #[inline]
+    pub const fn iter(&self) -> BitChunkIterator<'a> {
+        BitChunkIterator::<'a> {
+            buffer: self.buffer,
+            raw_data: self.raw_data,
+            offset: self.offset,
+            chunk_len: self.chunk_len,
+            index: 0,
+        }
+    }
+}
+
+impl<'a> IntoIterator for BitChunks<'a> {
+    type Item = u64;
+    type IntoIter = BitChunkIterator<'a>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.iter()
+    }
+}
+
+impl Iterator for BitChunkIterator<'_> {
+    type Item = u64;
+
+    #[inline]
+    fn next(&mut self) -> Option<u64> {
+        if self.index >= self.chunk_len {
+            return None;
+        }
+
+        // cast to *const u64 should be fine since we are using read_unaligned
+        #[allow(clippy::cast_ptr_alignment)]
+        let current = unsafe {
+            std::ptr::read_unaligned((self.raw_data as *const u64).add(self.index))
+        };
+
+        let combined = if self.offset == 0 {
+            current
+        } else {
+            // cast to *const u64 should be fine since we are using read_unaligned
+            #[allow(clippy::cast_ptr_alignment)]
+            let next = unsafe {
+                std::ptr::read_unaligned(
+                    (self.raw_data as *const u64).add(self.index + 1),
+                )
+            };
+            current >> self.offset
+                | (next & ((1 << self.offset) - 1)) << (64 - self.offset)
+        };
+
+        self.index += 1;
+
+        Some(combined)
+    }
+
+    #[inline]
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        (
+            self.chunk_len - self.index,
+            Some(self.chunk_len - self.index),
+        )
+    }
+}
+
+impl ExactSizeIterator for BitChunkIterator<'_> {
+    #[inline]
+    fn len(&self) -> usize {
+        self.chunk_len - self.index
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::buffer::Buffer;
+
+    #[test]
+    fn test_iter_aligned() {
+        let input: &[u8] = &[0, 1, 2, 3, 4, 5, 6, 7];
+        let buffer: Buffer = Buffer::from(input);
+
+        let bitchunks = buffer.bit_chunks(0, 64);
+        let result = bitchunks.into_iter().collect::<Vec<_>>();
+
+        assert_eq!(vec![0x0706050403020100], result);
+    }
+
+    #[test]
+    fn test_iter_unaligned() {
+        let input: &[u8] = &[
+            0b00000000, 0b00000001, 0b00000010, 0b00000100, 0b00001000, 0b00010000,
+            0b00100000, 0b01000000, 0b11111111,
+        ];
+        let buffer: Buffer = Buffer::from(input);
+
+        let bitchunks = buffer.bit_chunks(4, 64);
+
+        assert_eq!(0, bitchunks.remainder_len());
+        assert_eq!(0, bitchunks.remainder_bits());
+
+        let result = bitchunks.into_iter().collect::<Vec<_>>();
+
+        //assert_eq!(vec![0b00010000, 0b00100000, 0b01000000, 0b10000000, 0b00000000, 0b00000001, 0b00000010, 0b11110100], result);
+        assert_eq!(
+            vec![0b1111010000000010000000010000000010000000010000000010000000010000],
+            result
+        );
+    }
+
+    #[test]
+    fn test_iter_unaligned_remainder_1_byte() {
+        let input: &[u8] = &[
+            0b00000000, 0b00000001, 0b00000010, 0b00000100, 0b00001000, 0b00010000,
+            0b00100000, 0b01000000, 0b11111111,
+        ];
+        let buffer: Buffer = Buffer::from(input);
+
+        let bitchunks = buffer.bit_chunks(4, 66);
+
+        assert_eq!(2, bitchunks.remainder_len());
+        assert_eq!(0b00000011, bitchunks.remainder_bits());
+
+        let result = bitchunks.into_iter().collect::<Vec<_>>();
+
+        //assert_eq!(vec![0b00010000, 0b00100000, 0b01000000, 0b10000000, 0b00000000, 0b00000001, 0b00000010, 0b11110100], result);
+        assert_eq!(
+            vec![0b1111010000000010000000010000000010000000010000000010000000010000],
+            result
+        );
+    }
+}
diff --git a/rust/arrow/src/util/bit_util.rs b/rust/arrow/src/util/bit_util.rs
index d8ffa6f19c5..2c21d482a38 100644
--- a/rust/arrow/src/util/bit_util.rs
+++ b/rust/arrow/src/util/bit_util.rs
@@ -20,9 +20,9 @@
 #[cfg(feature = "simd")]
 use packed_simd::u8x64;
 
-static BIT_MASK: [u8; 8] = [1, 2, 4, 8, 16, 32, 64, 128];
+const BIT_MASK: [u8; 8] = [1, 2, 4, 8, 16, 32, 64, 128];
 
-static POPCOUNT_TABLE: [u8; 256] = [
+const POPCOUNT_TABLE: [u8; 256] = [
     0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4,
     3, 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5,
     3, 4, 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4,
@@ -43,7 +43,7 @@ pub fn round_upto_multiple_of_64(num: usize) -> usize {
 
 /// Returns the nearest multiple of `factor` that is `>=` than `num`. Here `factor` must
 /// be a power of 2.
-fn round_upto_power_of_2(num: usize, factor: usize) -> usize {
+pub fn round_upto_power_of_2(num: usize, factor: usize) -> usize {
     debug_assert!(factor > 0 && (factor & (factor - 1)) == 0);
     (num + (factor - 1)) & !(factor - 1)
 }
@@ -409,6 +409,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(all(any(target_arch = "x86", target_arch = "x86_64")))]
     fn test_ceil() {
         assert_eq!(ceil(0, 1), 0);
         assert_eq!(ceil(1, 1), 1);
diff --git a/rust/arrow/src/util/buffered_iterator.rs b/rust/arrow/src/util/buffered_iterator.rs
new file mode 100644
index 00000000000..059b82424a8
--- /dev/null
+++ b/rust/arrow/src/util/buffered_iterator.rs
@@ -0,0 +1,138 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! [Buffered] is an iterator useful to build an [arrow::array::Array] and other
+//! containers that benefit from batching or chunking.
+
+use std::marker::PhantomData;
+
+/// An iterator that buffers results in a vector so that the iterator returns a vector of `size` items.
+/// The items must be a [std::result::Result] and if an error is returned, tha error is returned
+/// and the iterator continues.
+/// An invariant of this iterator is that every returned vector's size is at most the specified size.
+#[derive(Debug)]
+pub struct Buffered<I, T, R>
+where
+    T: Clone,
+    I: Iterator<Item = Result<T, R>>,
+{
+    iter: I,
+    size: usize,
+    buffer: Vec<T>,
+    phantom: PhantomData<R>,
+}
+
+impl<I, T, R> Buffered<I, T, R>
+where
+    T: Clone,
+    I: Iterator<Item = Result<T, R>>,
+{
+    pub fn new(iter: I, size: usize) -> Self {
+        Buffered {
+            iter,
+            size,
+            buffer: Vec::with_capacity(size),
+            phantom: PhantomData,
+        }
+    }
+
+    /// returns the number of items buffered so far.
+    /// Useful to extract the exact item where an error occurred
+    #[inline]
+    pub fn n(&self) -> usize {
+        return self.buffer.len();
+    }
+}
+
+impl<I, T, R> Iterator for Buffered<I, T, R>
+where
+    T: Clone,
+    I: Iterator<Item = Result<T, R>>,
+{
+    type Item = Result<Vec<T>, R>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        for _ in 0..(self.size - self.n()) {
+            match self.iter.next() {
+                Some(Ok(item)) => self.buffer.push(item),
+                Some(Err(error)) => return Some(Err(error)),
+                None => break,
+            }
+        }
+        if self.buffer.is_empty() {
+            None
+        } else {
+            let result = self.buffer.clone();
+            self.buffer.clear();
+            Some(Ok(result))
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[derive(Debug, PartialEq)]
+    struct AError {}
+
+    impl std::fmt::Display for AError {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            write!(f, "Bla")
+        }
+    }
+    impl std::error::Error for AError {}
+
+    #[test]
+    fn test_basic() {
+        let a: Vec<Result<i32, AError>> = vec![Ok(1), Ok(2), Ok(3)];
+        let iter = a.into_iter();
+        let mut iter = Buffered::new(iter, 2);
+
+        assert_eq!(iter.next(), Some(Ok(vec![1, 2])));
+        assert_eq!(iter.next(), Some(Ok(vec![3])));
+        assert_eq!(iter.next(), None);
+    }
+
+    #[test]
+    fn test_error_first() {
+        let a: Vec<Result<i32, AError>> =
+            vec![Ok(1), Ok(2), Err(AError {}), Ok(4), Ok(5)];
+        let iter = a.into_iter();
+        let mut iter = Buffered::new(iter, 2);
+
+        assert_eq!(iter.next(), Some(Ok(vec![1, 2])));
+        assert_eq!(iter.next(), Some(Err(AError {})));
+        // 4 is here: it was not skipped on the previous
+        assert_eq!(iter.n(), 0);
+        assert_eq!(iter.next(), Some(Ok(vec![4, 5])));
+        assert_eq!(iter.next(), None);
+    }
+
+    #[test]
+    fn test_error_last() {
+        let a: Vec<Result<i32, AError>> = vec![Ok(1), Err(AError {}), Ok(3), Ok(4)];
+        let iter = a.into_iter();
+        let mut iter = Buffered::new(iter, 2);
+
+        assert_eq!(iter.next(), Some(Err(AError {})));
+        assert_eq!(iter.n(), 1);
+        assert_eq!(iter.next(), Some(Ok(vec![1, 3])));
+        assert_eq!(iter.next(), Some(Ok(vec![4])));
+        assert_eq!(iter.next(), None);
+    }
+}
diff --git a/rust/arrow/src/util/display.rs b/rust/arrow/src/util/display.rs
new file mode 100644
index 00000000000..1a873f1b083
--- /dev/null
+++ b/rust/arrow/src/util/display.rs
@@ -0,0 +1,152 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Functions for printing array values, as strings, for debugging
+//! purposes. See the `pretty` crate for additional functions for
+//! record batch pretty printing.
+
+use crate::array;
+use crate::array::Array;
+use crate::datatypes::{
+    ArrowNativeType, ArrowPrimitiveType, DataType, Int16Type, Int32Type, Int64Type,
+    Int8Type, TimeUnit, UInt16Type, UInt32Type, UInt64Type, UInt8Type,
+};
+
+use array::DictionaryArray;
+
+use crate::error::{ArrowError, Result};
+
+macro_rules! make_string {
+    ($array_type:ty, $column: ident, $row: ident) => {{
+        let array = $column.as_any().downcast_ref::<$array_type>().unwrap();
+
+        let s = if array.is_null($row) {
+            "".to_string()
+        } else {
+            array.value($row).to_string()
+        };
+
+        Ok(s)
+    }};
+}
+
+macro_rules! make_string_from_list {
+    ($column: ident, $row: ident) => {{
+        let list = $column
+            .as_any()
+            .downcast_ref::<array::ListArray>()
+            .ok_or(ArrowError::InvalidArgumentError(format!(
+                "Repl error: could not convert list column to list array."
+            )))?
+            .value($row);
+        let string_values = (0..list.len())
+            .map(|i| array_value_to_string(&list.clone(), i))
+            .collect::<Result<Vec<String>>>()?;
+        Ok(format!("[{}]", string_values.join(", ")))
+    }};
+}
+
+/// Get the value at the given row in an array as a String.
+///
+/// Note this function is quite inefficient and is unlikely to be
+/// suitable for converting large arrays or record batches.
+pub fn array_value_to_string(column: &array::ArrayRef, row: usize) -> Result<String> {
+    match column.data_type() {
+        DataType::Utf8 => make_string!(array::StringArray, column, row),
+        DataType::Boolean => make_string!(array::BooleanArray, column, row),
+        DataType::Int8 => make_string!(array::Int8Array, column, row),
+        DataType::Int16 => make_string!(array::Int16Array, column, row),
+        DataType::Int32 => make_string!(array::Int32Array, column, row),
+        DataType::Int64 => make_string!(array::Int64Array, column, row),
+        DataType::UInt8 => make_string!(array::UInt8Array, column, row),
+        DataType::UInt16 => make_string!(array::UInt16Array, column, row),
+        DataType::UInt32 => make_string!(array::UInt32Array, column, row),
+        DataType::UInt64 => make_string!(array::UInt64Array, column, row),
+        DataType::Float16 => make_string!(array::Float32Array, column, row),
+        DataType::Float32 => make_string!(array::Float32Array, column, row),
+        DataType::Float64 => make_string!(array::Float64Array, column, row),
+        DataType::Timestamp(unit, _) if *unit == TimeUnit::Second => {
+            make_string!(array::TimestampSecondArray, column, row)
+        }
+        DataType::Timestamp(unit, _) if *unit == TimeUnit::Millisecond => {
+            make_string!(array::TimestampMillisecondArray, column, row)
+        }
+        DataType::Timestamp(unit, _) if *unit == TimeUnit::Microsecond => {
+            make_string!(array::TimestampMicrosecondArray, column, row)
+        }
+        DataType::Timestamp(unit, _) if *unit == TimeUnit::Nanosecond => {
+            make_string!(array::TimestampNanosecondArray, column, row)
+        }
+        DataType::Date32(_) => make_string!(array::Date32Array, column, row),
+        DataType::Date64(_) => make_string!(array::Date64Array, column, row),
+        DataType::Time32(unit) if *unit == TimeUnit::Second => {
+            make_string!(array::Time32SecondArray, column, row)
+        }
+        DataType::Time32(unit) if *unit == TimeUnit::Millisecond => {
+            make_string!(array::Time32MillisecondArray, column, row)
+        }
+        DataType::Time32(unit) if *unit == TimeUnit::Microsecond => {
+            make_string!(array::Time64MicrosecondArray, column, row)
+        }
+        DataType::Time64(unit) if *unit == TimeUnit::Nanosecond => {
+            make_string!(array::Time64NanosecondArray, column, row)
+        }
+        DataType::List(_) => make_string_from_list!(column, row),
+        DataType::Dictionary(index_type, _value_type) => match **index_type {
+            DataType::Int8 => dict_array_value_to_string::<Int8Type>(column, row),
+            DataType::Int16 => dict_array_value_to_string::<Int16Type>(column, row),
+            DataType::Int32 => dict_array_value_to_string::<Int32Type>(column, row),
+            DataType::Int64 => dict_array_value_to_string::<Int64Type>(column, row),
+            DataType::UInt8 => dict_array_value_to_string::<UInt8Type>(column, row),
+            DataType::UInt16 => dict_array_value_to_string::<UInt16Type>(column, row),
+            DataType::UInt32 => dict_array_value_to_string::<UInt32Type>(column, row),
+            DataType::UInt64 => dict_array_value_to_string::<UInt64Type>(column, row),
+            _ => Err(ArrowError::InvalidArgumentError(format!(
+                "Pretty printing not supported for {:?} due to index type",
+                column.data_type()
+            ))),
+        },
+        _ => Err(ArrowError::InvalidArgumentError(format!(
+            "Pretty printing not implemented for {:?} type",
+            column.data_type()
+        ))),
+    }
+}
+
+/// Converts the value of the dictionary array at `row` to a String
+fn dict_array_value_to_string<K: ArrowPrimitiveType>(
+    colum: &array::ArrayRef,
+    row: usize,
+) -> Result<String> {
+    let dict_array = colum.as_any().downcast_ref::<DictionaryArray<K>>().unwrap();
+
+    let keys_array = dict_array.keys_array();
+
+    if keys_array.is_null(row) {
+        return Ok(String::from(""));
+    }
+
+    let dict_index = keys_array.value(row).to_usize().ok_or_else(|| {
+        ArrowError::InvalidArgumentError(format!(
+            "Can not convert value {:?} at index {:?} to usize for string conversion.",
+            keys_array.value(row),
+            row
+        ))
+    })?;
+
+    array_value_to_string(&dict_array.values(), dict_index)
+}
diff --git a/rust/arrow/src/util/integration_util.rs b/rust/arrow/src/util/integration_util.rs
index 3705aeeda14..22d271618b6 100644
--- a/rust/arrow/src/util/integration_util.rs
+++ b/rust/arrow/src/util/integration_util.rs
@@ -78,9 +78,9 @@ impl ArrowJson {
             return false;
         }
         self.batches.iter().all(|col| {
-            let batch = reader.next_batch();
+            let batch = reader.next();
             match batch {
-                Ok(Some(batch)) => col.equals_batch(&batch),
+                Some(Ok(batch)) => col.equals_batch(&batch),
                 _ => false,
             }
         })
@@ -375,9 +375,9 @@ impl ArrowJsonBatch {
 /// Convert an Arrow JSON column/array into a vector of `Value`
 fn json_from_col(col: &ArrowJsonColumn, data_type: &DataType) -> Vec<Value> {
     match data_type {
-        DataType::List(dt) => json_from_list_col(col, &**dt),
-        DataType::FixedSizeList(dt, list_size) => {
-            json_from_fixed_size_list_col(col, &**dt, *list_size as usize)
+        DataType::List(field) => json_from_list_col(col, field.data_type()),
+        DataType::FixedSizeList(field, list_size) => {
+            json_from_fixed_size_list_col(col, field.data_type(), *list_size as usize)
         }
         DataType::Struct(fields) => json_from_struct_col(col, fields),
         DataType::Int64
@@ -474,7 +474,7 @@ fn json_from_list_col(col: &ArrowJsonColumn, data_type: &DataType) -> Vec<Value>
         })
         .collect();
     let inner = match data_type {
-        DataType::List(ref dt) => json_from_col(child, &**dt),
+        DataType::List(ref field) => json_from_col(child, field.data_type()),
         DataType::Struct(fields) => json_from_struct_col(col, fields),
         _ => merge_json_array(
             child.validity.as_ref().unwrap().as_slice(),
@@ -511,8 +511,8 @@ fn json_from_fixed_size_list_col(
     // get the inner array
     let child = &col.children.clone().expect("list type must have children")[0];
     let inner = match data_type {
-        DataType::List(ref dt) => json_from_col(child, &**dt),
-        DataType::FixedSizeList(ref dt, _) => json_from_col(child, &**dt),
+        DataType::List(ref field) => json_from_col(child, field.data_type()),
+        DataType::FixedSizeList(ref field, _) => json_from_col(child, field.data_type()),
         DataType::Struct(fields) => json_from_struct_col(col, fields),
         _ => merge_json_array(
             child.validity.as_ref().unwrap().as_slice(),
@@ -540,7 +540,6 @@ fn json_from_fixed_size_list_col(
 mod tests {
     use super::*;
 
-    use std::convert::TryFrom;
     use std::fs::File;
     use std::io::Read;
     use std::sync::Arc;
@@ -578,13 +577,13 @@ mod tests {
                     "nullable": true,
                     "children": [
                         {
-                            "name": "item",
+                            "name": "custom_item",
                             "type": {
                                 "name": "int",
                                 "isSigned": true,
                                 "bitWidth": 32
                             },
-                            "nullable": true,
+                            "nullable": false,
                             "children": []
                         }
                     ]
@@ -596,7 +595,15 @@ mod tests {
             Field::new("c1", DataType::Int32, true),
             Field::new("c2", DataType::Float64, true),
             Field::new("c3", DataType::Utf8, true),
-            Field::new("c4", DataType::List(Box::new(DataType::Int32)), true),
+            Field::new(
+                "c4",
+                DataType::List(Box::new(Field::new(
+                    "custom_item",
+                    DataType::Int32,
+                    false,
+                ))),
+                true,
+            ),
         ]);
         assert!(json_schema.equals_schema(&schema));
     }
@@ -662,7 +669,11 @@ mod tests {
                 true,
             ),
             Field::new("utf8s", DataType::Utf8, true),
-            Field::new("lists", DataType::List(Box::new(DataType::Int32)), true),
+            Field::new(
+                "lists",
+                DataType::List(Box::new(Field::new("item", DataType::Int32, true))),
+                true,
+            ),
             Field::new(
                 "structs",
                 DataType::Struct(vec![
@@ -732,11 +743,12 @@ mod tests {
             vec![None, None, Some(-6473623571954960143)],
             nanos_tz,
         );
-        let utf8s = StringArray::try_from(vec![Some("aa"), None, Some("bbb")]).unwrap();
+        let utf8s = StringArray::from(vec![Some("aa"), None, Some("bbb")]);
 
         let value_data = Int32Array::from(vec![None, Some(2), None, None]);
         let value_offsets = Buffer::from(&[0, 3, 4, 4].to_byte_slice());
-        let list_data_type = DataType::List(Box::new(DataType::Int32));
+        let list_data_type =
+            DataType::List(Box::new(Field::new("item", DataType::Int32, true)));
         let list_data = ArrayData::builder(list_data_type)
             .len(3)
             .add_buffer(value_offsets)
@@ -745,8 +757,7 @@ mod tests {
         let lists = ListArray::from(list_data);
 
         let structs_int32s = Int32Array::from(vec![None, Some(-2), None]);
-        let structs_utf8s =
-            StringArray::try_from(vec![None, None, Some("aaaaaa")]).unwrap();
+        let structs_utf8s = StringArray::from(vec![None, None, Some("aaaaaa")]);
         let structs = StructArray::from(vec![
             (
                 Field::new("int32s", DataType::Int32, true),
diff --git a/rust/arrow/src/util/mod.rs b/rust/arrow/src/util/mod.rs
index f3bcc23a1d4..053d1329631 100644
--- a/rust/arrow/src/util/mod.rs
+++ b/rust/arrow/src/util/mod.rs
@@ -15,7 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
+pub mod bit_chunk_iterator;
 pub mod bit_util;
+pub mod buffered_iterator;
+pub mod display;
 pub mod integration_util;
 #[cfg(feature = "prettyprint")]
 pub mod pretty;
diff --git a/rust/arrow/src/util/pretty.rs b/rust/arrow/src/util/pretty.rs
index 4d3c64408a7..7eacba3c1b5 100644
--- a/rust/arrow/src/util/pretty.rs
+++ b/rust/arrow/src/util/pretty.rs
@@ -15,16 +15,17 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//! Utilities for printing record batches
+//! Utilities for printing record batches. Note this module is not
+//! available unless `feature = "prettyprint"` is enabled.
 
-use crate::array;
-use crate::datatypes::{DataType, TimeUnit};
 use crate::record_batch::RecordBatch;
 
 use prettytable::format;
 use prettytable::{Cell, Row, Table};
 
-use crate::error::{ArrowError, Result};
+use crate::error::Result;
+
+use super::display::array_value_to_string;
 
 ///! Create a visual representation of record batches
 pub fn pretty_format_batches(results: &[RecordBatch]) -> Result<String> {
@@ -59,7 +60,7 @@ fn create_table(results: &[RecordBatch]) -> Result<Table> {
             let mut cells = Vec::new();
             for col in 0..batch.num_columns() {
                 let column = batch.column(col);
-                cells.push(Cell::new(&array_value_to_string(column.clone(), row)?));
+                cells.push(Cell::new(&array_value_to_string(&column, row)?));
             }
             table.add_row(Row::new(cells));
         }
@@ -68,109 +69,94 @@ fn create_table(results: &[RecordBatch]) -> Result<Table> {
     Ok(table)
 }
 
-macro_rules! make_string {
-    ($array_type:ty, $column: ident, $row: ident) => {{
-        Ok($column
-            .as_any()
-            .downcast_ref::<$array_type>()
-            .unwrap()
-            .value($row)
-            .to_string())
-    }};
-}
-
-/// Get the value at the given row in an array as a string
-fn array_value_to_string(column: array::ArrayRef, row: usize) -> Result<String> {
-    match column.data_type() {
-        DataType::Utf8 => Ok(column
-            .as_any()
-            .downcast_ref::<array::StringArray>()
-            .unwrap()
-            .value(row)
-            .to_string()),
-        DataType::Boolean => make_string!(array::BooleanArray, column, row),
-        DataType::Int16 => make_string!(array::Int16Array, column, row),
-        DataType::Int32 => make_string!(array::Int32Array, column, row),
-        DataType::Int64 => make_string!(array::Int64Array, column, row),
-        DataType::UInt8 => make_string!(array::UInt8Array, column, row),
-        DataType::UInt16 => make_string!(array::UInt16Array, column, row),
-        DataType::UInt32 => make_string!(array::UInt32Array, column, row),
-        DataType::UInt64 => make_string!(array::UInt64Array, column, row),
-        DataType::Float16 => make_string!(array::Float32Array, column, row),
-        DataType::Float32 => make_string!(array::Float32Array, column, row),
-        DataType::Float64 => make_string!(array::Float64Array, column, row),
-        DataType::Timestamp(unit, _) if *unit == TimeUnit::Second => {
-            make_string!(array::TimestampSecondArray, column, row)
-        }
-        DataType::Timestamp(unit, _) if *unit == TimeUnit::Millisecond => {
-            make_string!(array::TimestampMillisecondArray, column, row)
-        }
-        DataType::Timestamp(unit, _) if *unit == TimeUnit::Microsecond => {
-            make_string!(array::TimestampMicrosecondArray, column, row)
-        }
-        DataType::Timestamp(unit, _) if *unit == TimeUnit::Nanosecond => {
-            make_string!(array::TimestampNanosecondArray, column, row)
-        }
-        DataType::Date32(_) => make_string!(array::Date32Array, column, row),
-        DataType::Date64(_) => make_string!(array::Date64Array, column, row),
-        DataType::Time32(unit) if *unit == TimeUnit::Second => {
-            make_string!(array::Time32SecondArray, column, row)
-        }
-        DataType::Time32(unit) if *unit == TimeUnit::Millisecond => {
-            make_string!(array::Time32MillisecondArray, column, row)
-        }
-        DataType::Time32(unit) if *unit == TimeUnit::Microsecond => {
-            make_string!(array::Time64MicrosecondArray, column, row)
-        }
-        DataType::Time64(unit) if *unit == TimeUnit::Nanosecond => {
-            make_string!(array::Time64NanosecondArray, column, row)
-        }
-        _ => Err(ArrowError::InvalidArgumentError(format!(
-            "Unsupported {:?} type for repl.",
-            column.data_type()
-        ))),
-    }
-}
-
 #[cfg(test)]
 mod tests {
+    use crate::array::{self, PrimitiveBuilder, StringBuilder, StringDictionaryBuilder};
+
     use super::*;
-    use crate::datatypes::{Field, Schema};
+    use crate::datatypes::{DataType, Field, Int32Type, Schema};
     use std::sync::Arc;
 
     #[test]
     fn test_pretty_format_batches() -> Result<()> {
         // define a schema.
         let schema = Arc::new(Schema::new(vec![
-            Field::new("a", DataType::Utf8, false),
-            Field::new("b", DataType::Int32, false),
+            Field::new("a", DataType::Utf8, true),
+            Field::new("b", DataType::Int32, true),
         ]));
 
         // define data.
         let batch = RecordBatch::try_new(
-            schema.clone(),
+            schema,
             vec![
-                Arc::new(array::StringArray::from(vec!["a", "b", "c", "d"])),
-                Arc::new(array::Int32Array::from(vec![1, 10, 10, 100])),
+                Arc::new(array::StringArray::from(vec![
+                    Some("a"),
+                    Some("b"),
+                    None,
+                    Some("d"),
+                ])),
+                Arc::new(array::Int32Array::from(vec![
+                    Some(1),
+                    None,
+                    Some(10),
+                    Some(100),
+                ])),
             ],
         )?;
 
-        let table = pretty_format_batches(&vec![batch])?;
+        let table = pretty_format_batches(&[batch])?;
 
         let expected = vec![
             "+---+-----+",
             "| a | b   |",
             "+---+-----+",
             "| a | 1   |",
-            "| b | 10  |",
-            "| c | 10  |",
+            "| b |     |",
+            "|   | 10  |",
             "| d | 100 |",
             "+---+-----+",
         ];
 
         let actual: Vec<&str> = table.lines().collect();
 
-        assert_eq!(expected, actual);
+        assert_eq!(expected, actual, "Actual result:\n{}", table);
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_pretty_format_dictionary() -> Result<()> {
+        // define a schema.
+        let field_type =
+            DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));
+        let schema = Arc::new(Schema::new(vec![Field::new("d1", field_type, true)]));
+
+        let keys_builder = PrimitiveBuilder::<Int32Type>::new(10);
+        let values_builder = StringBuilder::new(10);
+        let mut builder = StringDictionaryBuilder::new(keys_builder, values_builder);
+
+        builder.append("one")?;
+        builder.append_null()?;
+        builder.append("three")?;
+        let array = Arc::new(builder.finish());
+
+        let batch = RecordBatch::try_new(schema.clone(), vec![array])?;
+
+        let table = pretty_format_batches(&[batch])?;
+
+        let expected = vec![
+            "+-------+",
+            "| d1    |",
+            "+-------+",
+            "| one   |",
+            "|       |",
+            "| three |",
+            "+-------+",
+        ];
+
+        let actual: Vec<&str> = table.lines().collect();
+
+        assert_eq!(expected, actual, "Actual result:\n{}", table);
 
         Ok(())
     }
diff --git a/rust/arrow/test/data/integration.json b/rust/arrow/test/data/integration.json
index d54277981d5..193636ff136 100644
--- a/rust/arrow/test/data/integration.json
+++ b/rust/arrow/test/data/integration.json
@@ -251,19 +251,19 @@
       },
       {
         "name": "lists",
+        "nullable": true,
         "type": {
           "name": "list"
         },
-        "nullable": true,
         "children": [
           {
             "name": "item",
+            "nullable": true,
             "type": {
               "name": "int",
-              "isSigned": true,
-              "bitWidth": 32
+              "bitWidth": 32,
+              "isSigned": true
             },
-            "nullable": true,
             "children": []
           }
         ]
diff --git a/rust/arrow/test/data/list_string_dict_nested.json b/rust/arrow/test/data/list_string_dict_nested.json
new file mode 100644
index 00000000000..d215b318bae
--- /dev/null
+++ b/rust/arrow/test/data/list_string_dict_nested.json
@@ -0,0 +1,3 @@
+{"machine": "a", "events": ["Elect Leader", "Do Ballot"]}
+{"machine": "b", "events": ["Do Ballot", "Send Data", "Elect Leader"]}
+{"machine": "c", "events": ["Send Data"]}
diff --git a/rust/arrow/test/data/list_string_dict_nested_nulls.json b/rust/arrow/test/data/list_string_dict_nested_nulls.json
new file mode 100644
index 00000000000..9300b14ce27
--- /dev/null
+++ b/rust/arrow/test/data/list_string_dict_nested_nulls.json
@@ -0,0 +1,3 @@
+{"machine": "a", "events": [null, "Elect Leader", "Do Ballot"]}
+{"machine": "b", "events": ["Do Ballot", null, "Send Data", "Elect Leader"]}
+{"machine": "c", "events": ["Send Data"]}
diff --git a/rust/benchmarks/Cargo.toml b/rust/benchmarks/Cargo.toml
index ead493b96bf..1762969ad93 100644
--- a/rust/benchmarks/Cargo.toml
+++ b/rust/benchmarks/Cargo.toml
@@ -18,7 +18,7 @@
 [package]
 name = "arrow-benchmarks"
 description = "Apache Arrow Benchmarks"
-version = "2.0.0-SNAPSHOT"
+version = "3.0.0-SNAPSHOT"
 edition = "2018"
 authors = ["Apache Arrow <dev@arrow.apache.org>"]
 homepage = "https://github.com/apache/arrow"
@@ -28,5 +28,5 @@ license = "Apache-2.0"
 [dependencies]
 arrow = { path = "../arrow" }
 datafusion = { path = "../datafusion" }
-
 structopt = { version = "0.3", default-features = false }
+tokio = { version = "0.2", features = ["macros", "rt-core", "rt-threaded"] }
diff --git a/rust/benchmarks/README.md b/rust/benchmarks/README.md
index d38f58672c5..89bb4ecc5a3 100644
--- a/rust/benchmarks/README.md
+++ b/rust/benchmarks/README.md
@@ -19,12 +19,81 @@
 
 # Apache Arrow Rust Benchmarks
 
-This crate contains benchmarks based on the [New York Taxi and Limousine Commission](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page) data set.
+This crate contains benchmarks based on popular public data sets and open source benchmark suites, making it easy to
+run real-world benchmarks to help with performance and scalability testing and for comparing performance with other Arrow
+implementations as well as other query engines.
 
-Currently, only DataFusion benchmarks exist, but the plan is to add benchmarks for the arrow, flight, and parquet crates as well.
+Currently, only DataFusion benchmarks exist, but the plan is to add benchmarks for the arrow, flight, and parquet
+crates as well.
+
+## Benchmark derived from TPC-H
+
+These benchmarks are derived from the [TPC-H][1] benchmark.
+
+Data for this benchmark can be generated using the [tpch-dbgen][2] command-line tool. Run the following commands to
+clone the repository and build the source code.
+
+```bash
+git clone git@github.com:databricks/tpch-dbgen.git
+cd tpch-dbgen
+make
+```
+
+Data can now be generated with the following command. Note that `-s 1` means use Scale Factor 1 or ~1 GB of
+data. This value can be increased to generate larger data sets.
+
+```bash
+./dbgen -vf -s 1
+```
+
+The benchmark can then be run (assuming the data created from `dbgen` is in `/mnt/tpch-dbgen`) with a command such as:
 
 ```bash
-cargo run --release -- --iterations 3 --path /mnt/nyctaxi/csv --format csv --batch-size 4096
+cargo run --release --bin tpch -- --iterations 3 --path /mnt/tpch-dbgen --format tbl --query 1 --batch-size 4096
+```
+
+The benchmark program also supports CSV and Parquet input file formats.
+
+This crate does not currently provide a method for converting the generated tbl format to CSV or Parquet so it is 
+necessary to use other tools to perform this conversion.
+
+One option is to use the following Docker image to perform the conversion from `tbl` files to CSV or Parquet.
+
+```bash
+docker run -it ballistacompute/spark-benchmarks:0.4.0-SNAPSHOT 
+  -h, --help   Show help message
+
+Subcommand: convert-tpch
+  -i, --input  <arg>
+      --input-format  <arg>
+  -o, --output  <arg>
+      --output-format  <arg>
+  -p, --partitions  <arg>
+  -h, --help                   Show help message
+```
+
+Note that it is necessary to mount volumes into the Docker container as appropriate so that the file conversion process
+can access files on the host system.
+
+Here is a full example that assumes that data is stored in the `/mnt` path on the host system.
+
+```bash
+docker run -v /mnt:/mnt -it ballistacompute/spark-benchmarks:0.4.0-SNAPSHOT \
+  convert-tpch \
+  --input /mnt/tpch/csv \
+  --input-format tbl \
+  --output /mnt/tpch/parquet \
+  --output-format parquet \
+  --partitions 64
+```
+
+
+## NYC Taxi Benchmark
+
+These benchmarks are based on the [New York Taxi and Limousine Commission][3] data set.
+
+```bash
+cargo run --release --bin nyctaxi -- --iterations 3 --path /mnt/nyctaxi/csv --format csv --batch-size 4096
 ```
 
 Example output:
@@ -35,4 +104,8 @@ Executing 'fare_amt_by_passenger'
 Query 'fare_amt_by_passenger' iteration 0 took 7138 ms
 Query 'fare_amt_by_passenger' iteration 1 took 7599 ms
 Query 'fare_amt_by_passenger' iteration 2 took 7969 ms
-```
\ No newline at end of file
+```
+
+[1]: http://www.tpc.org/tpch/
+[2]: https://github.com/databricks/tpch-dbgen
+[3]: https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page
\ No newline at end of file
diff --git a/rust/benchmarks/src/main.rs b/rust/benchmarks/src/bin/nyctaxi.rs
similarity index 83%
rename from rust/benchmarks/src/main.rs
rename to rust/benchmarks/src/bin/nyctaxi.rs
index eec58ca483c..02a790bdcaf 100644
--- a/rust/benchmarks/src/main.rs
+++ b/rust/benchmarks/src/bin/nyctaxi.rs
@@ -25,9 +25,9 @@ use std::time::Instant;
 use arrow::datatypes::{DataType, Field, Schema};
 use arrow::util::pretty;
 use datafusion::error::Result;
-use datafusion::execution::context::ExecutionContext;
+use datafusion::execution::context::{ExecutionConfig, ExecutionContext};
 
-use datafusion::execution::physical_plan::csv::CsvReadOptions;
+use datafusion::physical_plan::csv::CsvReadOptions;
 use structopt::StructOpt;
 
 #[derive(Debug, StructOpt)]
@@ -41,6 +41,10 @@ struct Opt {
     #[structopt(short = "i", long = "iterations", default_value = "3")]
     iterations: usize,
 
+    /// Number of threads for query execution
+    #[structopt(short = "c", long = "concurrency", default_value = "2")]
+    concurrency: usize,
+
     /// Batch size when reading CSV or Parquet files
     #[structopt(short = "s", long = "batch-size", default_value = "4096")]
     batch_size: usize,
@@ -54,11 +58,15 @@ struct Opt {
     file_format: String,
 }
 
-fn main() -> Result<()> {
+#[tokio::main]
+async fn main() -> Result<()> {
     let opt = Opt::from_args();
     println!("Running benchmarks with the following options: {:?}", opt);
 
-    let mut ctx = ExecutionContext::new();
+    let config = ExecutionConfig::new()
+        .with_concurrency(opt.concurrency)
+        .with_batch_size(opt.batch_size);
+    let mut ctx = ExecutionContext::with_config(config);
 
     let path = opt.path.to_str().unwrap();
 
@@ -75,13 +83,12 @@ fn main() -> Result<()> {
         }
     }
 
-    datafusion_sql_benchmarks(&mut ctx, opt.iterations, opt.batch_size, opt.debug)
+    datafusion_sql_benchmarks(&mut ctx, opt.iterations, opt.debug).await
 }
 
-fn datafusion_sql_benchmarks(
+async fn datafusion_sql_benchmarks(
     ctx: &mut ExecutionContext,
     iterations: usize,
-    batch_size: usize,
     debug: bool,
 ) -> Result<()> {
     let mut queries = HashMap::new();
@@ -90,7 +97,7 @@ fn datafusion_sql_benchmarks(
         println!("Executing '{}'", name);
         for i in 0..iterations {
             let start = Instant::now();
-            execute_sql(ctx, sql, batch_size, debug)?;
+            execute_sql(ctx, sql, debug).await?;
             println!(
                 "Query '{}' iteration {} took {} ms",
                 name,
@@ -102,19 +109,14 @@ fn datafusion_sql_benchmarks(
     Ok(())
 }
 
-fn execute_sql(
-    ctx: &mut ExecutionContext,
-    sql: &str,
-    batch_size: usize,
-    debug: bool,
-) -> Result<()> {
+async fn execute_sql(ctx: &mut ExecutionContext, sql: &str, debug: bool) -> Result<()> {
     let plan = ctx.create_logical_plan(sql)?;
     let plan = ctx.optimize(&plan)?;
     if debug {
         println!("Optimized logical plan:\n{:?}", plan);
     }
-    let physical_plan = ctx.create_physical_plan(&plan, batch_size)?;
-    let result = ctx.collect(physical_plan.as_ref())?;
+    let physical_plan = ctx.create_physical_plan(&plan)?;
+    let result = ctx.collect(physical_plan).await?;
     if debug {
         pretty::print_batches(&result)?;
     }
diff --git a/rust/benchmarks/src/bin/tpch.rs b/rust/benchmarks/src/bin/tpch.rs
new file mode 100644
index 00000000000..b76ae13b380
--- /dev/null
+++ b/rust/benchmarks/src/bin/tpch.rs
@@ -0,0 +1,202 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Benchmark derived from TPC-H. This is not an official TPC-H benchmark.
+
+use std::path::PathBuf;
+use std::process;
+use std::time::Instant;
+
+use arrow::datatypes::{DataType, Field, Schema};
+use arrow::util::pretty;
+use datafusion::error::Result;
+use datafusion::execution::context::{ExecutionConfig, ExecutionContext};
+
+use datafusion::datasource::parquet::ParquetTable;
+use datafusion::datasource::{CsvFile, MemTable, TableProvider};
+use datafusion::physical_plan::csv::CsvReadOptions;
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+#[structopt(name = "Benchmarks", about = "Apache Arrow Rust Benchmarks.")]
+struct TpchOpt {
+    /// Query number
+    #[structopt(short, long)]
+    query: usize,
+
+    /// Activate debug mode to see query results
+    #[structopt(short, long)]
+    debug: bool,
+
+    /// Number of iterations of each test run
+    #[structopt(short = "i", long = "iterations", default_value = "3")]
+    iterations: usize,
+
+    /// Number of threads to use for parallel execution
+    #[structopt(short = "c", long = "concurrency", default_value = "2")]
+    concurrency: usize,
+
+    /// Batch size when reading CSV or Parquet files
+    #[structopt(short = "s", long = "batch-size", default_value = "4096")]
+    batch_size: usize,
+
+    /// Path to data files
+    #[structopt(parse(from_os_str), required = true, short = "p", long = "path")]
+    path: PathBuf,
+
+    /// File format: `csv` or `parquet`
+    #[structopt(short = "f", long = "format", default_value = "csv")]
+    file_format: String,
+
+    /// Load the data into a MemTable before executing the query
+    #[structopt(short = "m", long = "mem-table")]
+    mem_table: bool,
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let opt = TpchOpt::from_args();
+    println!("Running benchmarks with the following options: {:?}", opt);
+
+    let config = ExecutionConfig::new()
+        .with_concurrency(opt.concurrency)
+        .with_batch_size(opt.batch_size);
+    let mut ctx = ExecutionContext::with_config(config);
+
+    let path = opt.path.to_str().unwrap();
+
+    let tableprovider: Box<dyn TableProvider + Send + Sync> =
+        match opt.file_format.as_str() {
+            // dbgen creates .tbl ('|' delimited) files
+            "tbl" => {
+                let path = format!("{}/lineitem.tbl", path);
+                let schema = lineitem_schema();
+                let options = CsvReadOptions::new()
+                    .schema(&schema)
+                    .delimiter(b'|')
+                    .file_extension(".tbl");
+
+                Box::new(CsvFile::try_new(&path, options)?)
+            }
+            "csv" => {
+                let path = format!("{}/lineitem", path);
+                let schema = lineitem_schema();
+                let options = CsvReadOptions::new().schema(&schema).has_header(true);
+
+                Box::new(CsvFile::try_new(&path, options)?)
+            }
+            "parquet" => {
+                let path = format!("{}/lineitem", path);
+                Box::new(ParquetTable::try_new(&path)?)
+            }
+            other => {
+                println!("Invalid file format '{}'", other);
+                process::exit(-1);
+            }
+        };
+
+    if opt.mem_table {
+        println!("Loading data into memory");
+        let start = Instant::now();
+
+        let memtable = MemTable::load(tableprovider.as_ref(), opt.batch_size).await?;
+        println!(
+            "Loaded data into memory in {} ms",
+            start.elapsed().as_millis()
+        );
+
+        ctx.register_table("lineitem", Box::new(memtable));
+    } else {
+        ctx.register_table("lineitem", tableprovider);
+    }
+
+    let sql = match opt.query {
+        1 => {
+            "select
+                l_returnflag,
+                l_linestatus,
+                sum(l_quantity),
+                sum(l_extendedprice),
+                sum(l_extendedprice * (1 - l_discount)),
+                sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)),
+                avg(l_quantity),
+                avg(l_extendedprice),
+                avg(l_discount),
+                count(*)
+            from
+                lineitem
+            where
+                l_shipdate <= '1998-12-01'
+            group by
+                l_returnflag,
+                l_linestatus
+            order by
+                l_returnflag,
+                l_linestatus"
+        }
+
+        _ => unimplemented!("unsupported query"),
+    };
+
+    for i in 0..opt.iterations {
+        let start = Instant::now();
+        execute_sql(&mut ctx, sql, opt.debug).await?;
+        println!(
+            "Query {} iteration {} took {} ms",
+            opt.query,
+            i,
+            start.elapsed().as_millis()
+        );
+    }
+
+    Ok(())
+}
+
+async fn execute_sql(ctx: &mut ExecutionContext, sql: &str, debug: bool) -> Result<()> {
+    let plan = ctx.create_logical_plan(sql)?;
+    let plan = ctx.optimize(&plan)?;
+    if debug {
+        println!("Optimized logical plan:\n{:?}", plan);
+    }
+    let physical_plan = ctx.create_physical_plan(&plan)?;
+    let result = ctx.collect(physical_plan).await?;
+    if debug {
+        pretty::print_batches(&result)?;
+    }
+    Ok(())
+}
+
+fn lineitem_schema() -> Schema {
+    Schema::new(vec![
+        Field::new("l_orderkey", DataType::UInt32, true),
+        Field::new("l_partkey", DataType::UInt32, true),
+        Field::new("l_suppkey", DataType::UInt32, true),
+        Field::new("l_linenumber", DataType::UInt32, true),
+        Field::new("l_quantity", DataType::Float64, true),
+        Field::new("l_extendedprice", DataType::Float64, true),
+        Field::new("l_discount", DataType::Float64, true),
+        Field::new("l_tax", DataType::Float64, true),
+        Field::new("l_returnflag", DataType::Utf8, true),
+        Field::new("l_linestatus", DataType::Utf8, true),
+        Field::new("l_shipdate", DataType::Utf8, true),
+        Field::new("l_commitdate", DataType::Utf8, true),
+        Field::new("l_receiptdate", DataType::Utf8, true),
+        Field::new("l_shipinstruct", DataType::Utf8, true),
+        Field::new("l_shipmode", DataType::Utf8, true),
+        Field::new("l_comment", DataType::Utf8, true),
+    ])
+}
diff --git a/rust/datafusion/Cargo.toml b/rust/datafusion/Cargo.toml
index 3d12f93e649..380e04cf6b8 100644
--- a/rust/datafusion/Cargo.toml
+++ b/rust/datafusion/Cargo.toml
@@ -18,7 +18,7 @@
 [package]
 name = "datafusion"
 description = "DataFusion is an in-memory query engine that uses Apache Arrow as the memory model"
-version = "2.0.0-SNAPSHOT"
+version = "3.0.0-SNAPSHOT"
 homepage = "https://github.com/apache/arrow"
 repository = "https://github.com/apache/arrow"
 authors = ["Apache Arrow <dev@arrow.apache.org>"]
@@ -45,23 +45,35 @@ cli = ["rustyline"]
 
 [dependencies]
 fnv = "1.0"
-arrow = { path = "../arrow", version = "2.0.0-SNAPSHOT", features = ["prettyprint"] }
-parquet = { path = "../parquet", version = "2.0.0-SNAPSHOT", features = ["arrow"] }
+arrow = { path = "../arrow", version = "3.0.0-SNAPSHOT", features = ["prettyprint"] }
+parquet = { path = "../parquet", version = "3.0.0-SNAPSHOT", features = ["arrow"] }
 sqlparser = "0.6.1"
 clap = "2.33"
 rustyline = {version = "6.0", optional = true}
 crossbeam = "0.7"
 paste = "0.1"
+num_cpus = "1.13.0"
+chrono = "0.4"
+async-trait = "0.1.41"
+futures = "0.3"
+tokio = { version = "0.2", features = ["macros", "rt-core", "rt-threaded"] }
 
 [dev-dependencies]
+rand = "0.7"
 criterion = "0.3"
-tempdir = "0.3"
-futures = "0.3"
+tempfile = "3"
 prost = "0.6"
-tokio = { version = "0.2", features = ["macros"] }
-arrow-flight = { path = "../arrow-flight", version = "2.0.0-SNAPSHOT" }
-tonic = "0.2"
+arrow-flight = { path = "../arrow-flight", version = "3.0.0-SNAPSHOT" }
+tonic = "0.3"
 
 [[bench]]
 name = "aggregate_query_sql"
 harness = false
+
+[[bench]]
+name = "sort_limit_query_sql"
+harness = false
+
+[[bench]]
+name = "math_query_sql"
+harness = false
diff --git a/rust/datafusion/README.md b/rust/datafusion/README.md
index 46160aea819..307443b02b4 100644
--- a/rust/datafusion/README.md
+++ b/rust/datafusion/README.md
@@ -27,7 +27,7 @@ DataFusion can be used as a library by adding the following to your `Cargo.toml`
 
 ```toml
 [dependencies]
-datafusion = "2.0.0-SNAPSHOT"
+datafusion = "3.0.0-SNAPSHOT"
 ```
 
 ## Using DataFusion as a binary
@@ -42,20 +42,28 @@ DataFusion includes a simple command-line interactive SQL utility. See the [CLI
 - [x] SQL Query Planner
 - [x] Query Optimizer
 - [x] Projection push down
-- [ ] Predicate push down
+- [x] Predicate push down
 - [x] Type coercion
 - [x] Parallel query execution
 
 ## SQL Support
 
 - [x] Projection
-- [x] Selection
+- [x] Filter (WHERE)
 - [x] Limit
 - [x] Aggregate
-- [x] UDFs
+- [x] UDFs (user-defined functions)
+- [x] UDAFs (user-defined aggregate functions)
 - [x] Common math functions
-- [ ] Common string functions
-- [ ] Common date/time functions
+- String functions
+  - [x] Length
+  - [x] Concatenate
+- Common date/time functions
+  - [ ] Basic date functions
+  - [ ] Basic time functions
+  - [x] Basic timestamp functions
+- nested functions
+  - [x] Array of columns
 - [x] Sorting
 - [ ] Nested types
 - [ ] Lists
@@ -79,4 +87,87 @@ This library currently supports the following SQL constructs:
 * most mathematical unary and binary expressions such as `+`, `/`, `sqrt`, `tan`, `>=`.
 * `WHERE` to filter
 * `GROUP BY` together with one of the following aggregations: `MIN`, `MAX`, `COUNT`, `SUM`, `AVG`
-* `ORDER BY` together with an expression and optional `DESC`
+* `ORDER BY` together with an expression and optional `ASC` or `DESC` and also optional `NULLS FIRST` or `NULLS LAST`
+
+## Supported Data Types
+
+DataFusion uses Arrow, and thus the Arrow type system, for query
+execution. The SQL types from
+[sqlparser-rs](https://github.com/ballista-compute/sqlparser-rs/blob/main/src/ast/data_type.rs#L57)
+are mapped to Arrow types according to the following table
+
+
+| SQL Data Type   | Arrow DataType                   |
+| --------------- | -------------------------------- |
+| `CHAR`          | `Utf8`                           |
+| `VARCHAR`       | `Utf8`                           |
+| `UUID`          | *Not yet supported*              |
+| `CLOB`          | *Not yet supported*              |
+| `BINARY`        | *Not yet supported*              |
+| `VARBINARY`     | *Not yet supported*              |
+| `DECIMAL`       | `Float64`                        |
+| `FLOAT`         | `Float32`                        |
+| `SMALLINT`      | `Int16`                          |
+| `INT`           | `Int32`                          |
+| `BIGINT`        | `Int64`                          |
+| `REAL`          | `Float64`                        |
+| `DOUBLE`        | `Float64`                        |
+| `BOOLEAN`       | `Boolean`                        |
+| `DATE`          | `Date64(DateUnit::Day)`          |
+| `TIME`          | `Time64(TimeUnit::Millisecond)`  |
+| `TIMESTAMP`     | `Date64(DateUnit::Millisecond)`  |
+| `INTERVAL`      | *Not yet supported*              |
+| `REGCLASS`      | *Not yet supported*              |
+| `TEXT`          | *Not yet supported*              |
+| `BYTEA`         | *Not yet supported*              |
+| `CUSTOM`        | *Not yet supported*              |
+| `ARRAY`         | *Not yet supported*              |
+
+# Developer's guide
+
+This section describes how you can get started at developing DataFusion.
+
+### Bootstrap environment
+
+DataFusion is written in Rust and it uses a standard rust toolkit:
+
+* `cargo build`
+* `cargo fmt` to format the code
+* `cargo test` to test
+* etc.
+
+## How to add a new scalar function
+
+Below is a checklist of what you need to do to add a new scalar function to DataFusion:
+
+* Add the actual implementation of the function:
+  * [here](src/physical_plan/string_expressions.rs) for string functions
+  * [here](src/physical_plan/math_expressions.rs) for math functions
+  * [here](src/physical_plan/datetime_expressions.rs) for datetime functions
+  * create a new module [here](src/physical_plan) for other functions
+* In [src/physical_plan/functions](src/physical_plan/functions.rs), add:
+  * a new variant to `BuiltinScalarFunction`
+  * a new entry to `FromStr` with the name of the function as called by SQL
+  * a new line in `return_type` with the expected return type of the function, given an incoming type
+  * a new line in `signature` with the signature of the function (number and types of its arguments)
+  * a new line in `create_physical_expr` mapping the built-in to the implementation
+  * tests to the function.
+* In [tests/sql.rs](tests/sql.rs), add a new test where the function is called through SQL against well known data and returns the expected result.
+
+## How to add a new aggregate function
+
+Below is a checklist of what you need to do to add a new aggregate function to DataFusion:
+
+* Add the actual implementation of an `Accumulator` and `AggregateExpr`:
+  * [here](src/physical_plan/string_expressions.rs) for string functions
+  * [here](src/physical_plan/math_expressions.rs) for math functions
+  * [here](src/physical_plan/datetime_expressions.rs) for datetime functions
+  * create a new module [here](src/physical_plan) for other functions
+* In [src/physical_plan/aggregates](src/physical_plan/aggregates.rs), add:
+  * a new variant to `BuiltinAggregateFunction`
+  * a new entry to `FromStr` with the name of the function as called by SQL
+  * a new line in `return_type` with the expected return type of the function, given an incoming type
+  * a new line in `signature` with the signature of the function (number and types of its arguments)
+  * a new line in `create_aggregate_expr` mapping the built-in to the implementation
+  * tests to the function.
+* In [tests/sql.rs](tests/sql.rs), add a new test where the function is called through SQL against well known data and returns the expected result.
diff --git a/rust/datafusion/benches/aggregate_query_sql.rs b/rust/datafusion/benches/aggregate_query_sql.rs
index ab374a0c641..54741043fd2 100644
--- a/rust/datafusion/benches/aggregate_query_sql.rs
+++ b/rust/datafusion/benches/aggregate_query_sql.rs
@@ -19,91 +19,200 @@
 extern crate criterion;
 use criterion::Criterion;
 
-use std::env;
-use std::sync::Arc;
+use rand::seq::SliceRandom;
+use rand::Rng;
+use std::sync::{Arc, Mutex};
+use tokio::runtime::Runtime;
 
 extern crate arrow;
 extern crate datafusion;
 
-use arrow::datatypes::{DataType, Field, Schema};
-
-use datafusion::datasource::{CsvFile, CsvReadOptions, MemTable};
+use arrow::{
+    array::Float32Array,
+    array::Float64Array,
+    array::StringArray,
+    array::UInt64Array,
+    datatypes::{DataType, Field, Schema},
+    record_batch::RecordBatch,
+};
+
+use datafusion::datasource::MemTable;
+use datafusion::error::Result;
 use datafusion::execution::context::ExecutionContext;
 
-fn aggregate_query(ctx: &mut ExecutionContext, sql: &str) {
+fn query(ctx: Arc<Mutex<ExecutionContext>>, sql: &str) {
+    let mut rt = Runtime::new().unwrap();
+
     // execute the query
-    let results = ctx.sql(&sql, 1024 * 1024).unwrap();
+    let df = ctx.lock().unwrap().sql(&sql).unwrap();
+    rt.block_on(df.collect()).unwrap();
+}
 
-    // display the relation
-    for _batch in results {}
+fn create_data(size: usize, null_density: f64) -> Vec<Option<f64>> {
+    // use random numbers to avoid spurious compiler optimizations wrt to branching
+    let mut rng = rand::thread_rng();
+
+    (0..size)
+        .map(|_| {
+            if rng.gen::<f64>() > null_density {
+                None
+            } else {
+                Some(rng.gen::<f64>())
+            }
+        })
+        .collect()
 }
 
-fn create_context() -> ExecutionContext {
-    // define schema for data source (csv file)
+fn create_integer_data(size: usize, value_density: f64) -> Vec<Option<u64>> {
+    // use random numbers to avoid spurious compiler optimizations wrt to branching
+    let mut rng = rand::thread_rng();
+
+    (0..size)
+        .map(|_| {
+            if rng.gen::<f64>() > value_density {
+                None
+            } else {
+                Some(rng.gen::<u64>())
+            }
+        })
+        .collect()
+}
+
+fn create_context(
+    partitions_len: usize,
+    array_len: usize,
+    batch_size: usize,
+) -> Result<Arc<Mutex<ExecutionContext>>> {
+    // define a schema.
     let schema = Arc::new(Schema::new(vec![
-        Field::new("c1", DataType::Utf8, false),
-        Field::new("c2", DataType::UInt32, false),
-        Field::new("c3", DataType::Int8, false),
-        Field::new("c4", DataType::Int16, false),
-        Field::new("c5", DataType::Int32, false),
-        Field::new("c6", DataType::Int64, false),
-        Field::new("c7", DataType::UInt8, false),
-        Field::new("c8", DataType::UInt16, false),
-        Field::new("c9", DataType::UInt32, false),
-        Field::new("c10", DataType::UInt64, false),
-        Field::new("c11", DataType::Float32, false),
-        Field::new("c12", DataType::Float64, false),
-        Field::new("c13", DataType::Utf8, false),
+        Field::new("utf8", DataType::Utf8, false),
+        Field::new("f32", DataType::Float32, false),
+        Field::new("f64", DataType::Float64, false),
+        // This field will contain integers randomly selected from a large
+        // range of values, i.e. [0, u64::MAX], such that there are none (or
+        // very few) repeated values.
+        Field::new("u64_wide", DataType::UInt64, false),
+        // This field will contain integers randomly selected from a narrow
+        // range of values such that there are a few distinct values, but they
+        // are repeated often.
+        Field::new("u64_narrow", DataType::UInt64, false),
     ]));
 
-    let testdata = env::var("ARROW_TEST_DATA").expect("ARROW_TEST_DATA not defined");
+    // define data.
+    let partitions = (0..partitions_len)
+        .map(|_| {
+            (0..array_len / batch_size / partitions_len)
+                .map(|i| {
+                    // the 4 here is the number of different keys.
+                    // a higher number increase sparseness
+                    let vs = vec![0, 1, 2, 3];
+                    let keys: Vec<String> = (0..batch_size)
+                        .map(
+                            // use random numbers to avoid spurious compiler optimizations wrt to branching
+                            |_| format!("hi{:?}", vs.choose(&mut rand::thread_rng())),
+                        )
+                        .collect();
+                    let keys: Vec<&str> = keys.iter().map(|e| &**e).collect();
+
+                    let values = create_data(batch_size, 0.5);
+
+                    // Integer values between [0, u64::MAX].
+                    let integer_values_wide = create_integer_data(batch_size, 9.0);
+
+                    // Integer values between [0, 9].
+                    let integer_values_narrow_choices = (0..10).collect::<Vec<u64>>();
+                    let integer_values_narrow = (0..batch_size)
+                        .map(|_| {
+                            *integer_values_narrow_choices
+                                .choose(&mut rand::thread_rng())
+                                .unwrap()
+                        })
+                        .collect::<Vec<u64>>();
+
+                    RecordBatch::try_new(
+                        schema.clone(),
+                        vec![
+                            Arc::new(StringArray::from(keys)),
+                            Arc::new(Float32Array::from(vec![i as f32; batch_size])),
+                            Arc::new(Float64Array::from(values)),
+                            Arc::new(UInt64Array::from(integer_values_wide)),
+                            Arc::new(UInt64Array::from(integer_values_narrow)),
+                        ],
+                    )
+                    .unwrap()
+                })
+                .collect::<Vec<_>>()
+        })
+        .collect::<Vec<_>>();
 
-    // create CSV data source
-    let csv = CsvFile::try_new(
-        &format!("{}/csv/aggregate_test_100.csv", testdata),
-        CsvReadOptions::new().schema(&schema),
-    )
-    .unwrap();
+    let mut ctx = ExecutionContext::new();
 
-    let mem_table = MemTable::load(&csv).unwrap();
+    // declare a table in memory. In spark API, this corresponds to createDataFrame(...).
+    let provider = MemTable::new(schema, partitions)?;
+    ctx.register_table("t", Box::new(provider));
 
-    // create local execution context
-    let mut ctx = ExecutionContext::new();
-    ctx.register_table("aggregate_test_100", Box::new(mem_table));
-    ctx
+    Ok(Arc::new(Mutex::new(ctx)))
 }
 
 fn criterion_benchmark(c: &mut Criterion) {
-    c.bench_function("aggregate_query_no_group_by", |b| {
-        let mut ctx = create_context();
+    let partitions_len = 8;
+    let array_len = 32768 * 2; // 2^16
+    let batch_size = 2048; // 2^11
+    let ctx = create_context(partitions_len, array_len, batch_size).unwrap();
+
+    c.bench_function("aggregate_query_no_group_by 15 12", |b| {
         b.iter(|| {
-            aggregate_query(
-                &mut ctx,
-                "SELECT MIN(c12), MAX(c12) \
-                 FROM aggregate_test_100",
+            query(
+                ctx.clone(),
+                "SELECT MIN(f64), AVG(f64), COUNT(f64) \
+                 FROM t",
             )
         })
     });
 
-    c.bench_function("aggregate_query_group_by", |b| {
-        let mut ctx = create_context();
+    c.bench_function(
+        "aggregate_query_no_group_by_count_distinct_wide 15 12",
+        |b| {
+            b.iter(|| {
+                query(
+                    ctx.clone(),
+                    "SELECT COUNT(DISTINCT u64_wide) \
+                 FROM t",
+                )
+            })
+        },
+    );
+
+    c.bench_function(
+        "aggregate_query_no_group_by_count_distinct_narrow 15 12",
+        |b| {
+            b.iter(|| {
+                query(
+                    ctx.clone(),
+                    "SELECT COUNT(DISTINCT u64_narrow) \
+                 FROM t",
+                )
+            })
+        },
+    );
+
+    c.bench_function("aggregate_query_group_by 15 12", |b| {
         b.iter(|| {
-            aggregate_query(
-                &mut ctx,
-                "SELECT c1, MIN(c12), MAX(c12) \
-                 FROM aggregate_test_100 GROUP BY c1",
+            query(
+                ctx.clone(),
+                "SELECT utf8, MIN(f64), AVG(f64), COUNT(f64) \
+                 FROM t GROUP BY utf8",
             )
         })
     });
 
-    c.bench_function("aggregate_query_group_by_with_filter", |b| {
-        let mut ctx = create_context();
+    c.bench_function("aggregate_query_group_by_with_filter 15 12", |b| {
         b.iter(|| {
-            aggregate_query(
-                &mut ctx,
-                "SELECT c1, MIN(c12), MAX(c12) \
-                 FROM aggregate_test_100 \
-                 WHERE c11 > 0.1 AND c11 < 0.9 GROUP BY c1",
+            query(
+                ctx.clone(),
+                "SELECT utf8, MIN(f64), AVG(f64), COUNT(f64) \
+                 FROM t \
+                 WHERE f32 > 10 AND f32 < 20 GROUP BY utf8",
             )
         })
     });
diff --git a/rust/datafusion/benches/math_query_sql.rs b/rust/datafusion/benches/math_query_sql.rs
new file mode 100644
index 00000000000..65f613b6cdd
--- /dev/null
+++ b/rust/datafusion/benches/math_query_sql.rs
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[macro_use]
+extern crate criterion;
+use criterion::Criterion;
+
+use std::sync::{Arc, Mutex};
+
+use tokio::runtime::Runtime;
+
+extern crate arrow;
+extern crate datafusion;
+
+use arrow::{
+    array::{Float32Array, Float64Array},
+    datatypes::{DataType, Field, Schema},
+    record_batch::RecordBatch,
+};
+use datafusion::error::Result;
+
+use datafusion::datasource::MemTable;
+use datafusion::execution::context::ExecutionContext;
+
+fn query(ctx: Arc<Mutex<ExecutionContext>>, sql: &str) {
+    let mut rt = Runtime::new().unwrap();
+
+    // execute the query
+    let df = ctx.lock().unwrap().sql(&sql).unwrap();
+    rt.block_on(df.collect()).unwrap();
+}
+
+fn create_context(
+    array_len: usize,
+    batch_size: usize,
+) -> Result<Arc<Mutex<ExecutionContext>>> {
+    // define a schema.
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("f32", DataType::Float32, false),
+        Field::new("f64", DataType::Float64, false),
+    ]));
+
+    // define data.
+    let batches = (0..array_len / batch_size)
+        .map(|i| {
+            RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Float32Array::from(vec![i as f32; batch_size])),
+                    Arc::new(Float64Array::from(vec![i as f64; batch_size])),
+                ],
+            )
+            .unwrap()
+        })
+        .collect::<Vec<_>>();
+
+    let mut ctx = ExecutionContext::new();
+
+    // declare a table in memory. In spark API, this corresponds to createDataFrame(...).
+    let provider = MemTable::new(schema, vec![batches])?;
+    ctx.register_table("t", Box::new(provider));
+
+    Ok(Arc::new(Mutex::new(ctx)))
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    let array_len = 1048576; // 2^20
+    let batch_size = 512; // 2^9
+    let ctx = create_context(array_len, batch_size).unwrap();
+    c.bench_function("sqrt_20_9", |b| {
+        b.iter(|| query(ctx.clone(), "SELECT sqrt(f32) FROM t"))
+    });
+
+    let array_len = 1048576; // 2^20
+    let batch_size = 4096; // 2^12
+    let ctx = create_context(array_len, batch_size).unwrap();
+    c.bench_function("sqrt_20_12", |b| {
+        b.iter(|| query(ctx.clone(), "SELECT sqrt(f32) FROM t"))
+    });
+
+    let array_len = 4194304; // 2^22
+    let batch_size = 4096; // 2^12
+    let ctx = create_context(array_len, batch_size).unwrap();
+    c.bench_function("sqrt_22_12", |b| {
+        b.iter(|| query(ctx.clone(), "SELECT sqrt(f32) FROM t"))
+    });
+
+    let array_len = 4194304; // 2^22
+    let batch_size = 16384; // 2^14
+    let ctx = create_context(array_len, batch_size).unwrap();
+    c.bench_function("sqrt_22_14", |b| {
+        b.iter(|| query(ctx.clone(), "SELECT sqrt(f32) FROM t"))
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/rust/datafusion/benches/sort_limit_query_sql.rs b/rust/datafusion/benches/sort_limit_query_sql.rs
new file mode 100644
index 00000000000..02440046b99
--- /dev/null
+++ b/rust/datafusion/benches/sort_limit_query_sql.rs
@@ -0,0 +1,143 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[macro_use]
+extern crate criterion;
+use criterion::Criterion;
+
+use std::env;
+use std::sync::{Arc, Mutex};
+
+extern crate arrow;
+extern crate datafusion;
+
+use arrow::datatypes::{DataType, Field, Schema};
+
+use datafusion::datasource::{CsvFile, CsvReadOptions, MemTable};
+use datafusion::execution::context::ExecutionContext;
+
+use tokio::runtime::Runtime;
+
+fn query(ctx: Arc<Mutex<ExecutionContext>>, sql: &str) {
+    let mut rt = Runtime::new().unwrap();
+
+    // execute the query
+    let df = ctx.lock().unwrap().sql(&sql).unwrap();
+    rt.block_on(df.collect()).unwrap();
+}
+
+fn create_context() -> Arc<Mutex<ExecutionContext>> {
+    // define schema for data source (csv file)
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("c1", DataType::Utf8, false),
+        Field::new("c2", DataType::UInt32, false),
+        Field::new("c3", DataType::Int8, false),
+        Field::new("c4", DataType::Int16, false),
+        Field::new("c5", DataType::Int32, false),
+        Field::new("c6", DataType::Int64, false),
+        Field::new("c7", DataType::UInt8, false),
+        Field::new("c8", DataType::UInt16, false),
+        Field::new("c9", DataType::UInt32, false),
+        Field::new("c10", DataType::UInt64, false),
+        Field::new("c11", DataType::Float32, false),
+        Field::new("c12", DataType::Float64, false),
+        Field::new("c13", DataType::Utf8, false),
+    ]));
+
+    let testdata = env::var("ARROW_TEST_DATA").expect("ARROW_TEST_DATA not defined");
+
+    // create CSV data source
+    let csv = CsvFile::try_new(
+        &format!("{}/csv/aggregate_test_100.csv", testdata),
+        CsvReadOptions::new().schema(&schema),
+    )
+    .unwrap();
+
+    let mut rt = Runtime::new().unwrap();
+
+    let ctx_holder: Arc<Mutex<Vec<Arc<Mutex<ExecutionContext>>>>> =
+        Arc::new(Mutex::new(vec![]));
+    rt.block_on(async {
+        let mem_table = MemTable::load(&csv, 16 * 1024).await.unwrap();
+
+        // create local execution context
+        let mut ctx = ExecutionContext::new();
+        ctx.state.config.concurrency = 1;
+        ctx.register_table("aggregate_test_100", Box::new(mem_table));
+        ctx_holder.lock().unwrap().push(Arc::new(Mutex::new(ctx)))
+    });
+
+    let ctx = ctx_holder.lock().unwrap().get(0).unwrap().clone();
+    ctx
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    c.bench_function("sort_and_limit_by_int", |b| {
+        let ctx = create_context();
+        b.iter(|| {
+            query(
+                ctx.clone(),
+                "SELECT c1, c13, c6, c10 \
+                 FROM aggregate_test_100 \
+                 ORDER BY 3
+                 LIMIT 10",
+            )
+        })
+    });
+
+    c.bench_function("sort_and_limit_by_float", |b| {
+        let ctx = create_context();
+        b.iter(|| {
+            query(
+                ctx.clone(),
+                "SELECT c1, c13, c12 \
+                 FROM aggregate_test_100 \
+                 ORDER BY 2
+                 LIMIT 10",
+            )
+        })
+    });
+
+    c.bench_function("sort_and_limit_lex_by_int", |b| {
+        let ctx = create_context();
+        b.iter(|| {
+            query(
+                ctx.clone(),
+                "SELECT c1, c13, c6, c10 \
+                 FROM aggregate_test_100 \
+                 ORDER BY 3 DESC, 4 DESC
+                 LIMIT 10",
+            )
+        })
+    });
+
+    c.bench_function("sort_and_limit_lex_by_string", |b| {
+        let ctx = create_context();
+        b.iter(|| {
+            query(
+                ctx.clone(),
+                "SELECT c1, c13, c6, c10 \
+                 FROM aggregate_test_100 \
+                 ORDER BY 1, 2
+                 LIMIT 10",
+            )
+        })
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/rust/datafusion/examples/csv_sql.rs b/rust/datafusion/examples/csv_sql.rs
index 7300bc862fd..78f2416907f 100644
--- a/rust/datafusion/examples/csv_sql.rs
+++ b/rust/datafusion/examples/csv_sql.rs
@@ -17,13 +17,13 @@
 
 use arrow::util::pretty;
 
-use datafusion::datasource::csv::CsvReadOptions;
 use datafusion::error::Result;
-use datafusion::execution::context::ExecutionContext;
+use datafusion::prelude::*;
 
 /// This example demonstrates executing a simple query against an Arrow data source (CSV) and
 /// fetching results
-fn main() -> Result<()> {
+#[tokio::main]
+async fn main() -> Result<()> {
     // create local execution context
     let mut ctx = ExecutionContext::new();
 
@@ -36,15 +36,14 @@ fn main() -> Result<()> {
         CsvReadOptions::new(),
     )?;
 
-    let sql = "SELECT c1, MIN(c12), MAX(c12) FROM aggregate_test_100 WHERE c11 > 0.1 AND c11 < 0.9 GROUP BY c1";
-
-    // create the query plan
-    let plan = ctx.create_logical_plan(sql)?;
-    let plan = ctx.optimize(&plan)?;
-    let plan = ctx.create_physical_plan(&plan, 1024 * 1024)?;
-
     // execute the query
-    let results = ctx.collect(plan.as_ref())?;
+    let df = ctx.sql(
+        "SELECT c1, MIN(c12), MAX(c12) \
+        FROM aggregate_test_100 \
+        WHERE c11 > 0.1 AND c11 < 0.9 \
+        GROUP BY c1",
+    )?;
+    let results = df.collect().await?;
 
     // print the results
     pretty::print_batches(&results)?;
diff --git a/rust/datafusion/examples/dataframe.rs b/rust/datafusion/examples/dataframe.rs
new file mode 100644
index 00000000000..41442026094
--- /dev/null
+++ b/rust/datafusion/examples/dataframe.rs
@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::util::pretty;
+
+use datafusion::error::Result;
+use datafusion::prelude::*;
+
+/// This example demonstrates executing a simple query against an Arrow data source (Parquet) and
+/// fetching results, using the DataFrame trait
+#[tokio::main]
+async fn main() -> Result<()> {
+    // create local execution context
+    let mut ctx = ExecutionContext::new();
+
+    let testdata =
+        std::env::var("PARQUET_TEST_DATA").expect("PARQUET_TEST_DATA not defined");
+
+    let filename = &format!("{}/alltypes_plain.parquet", testdata);
+
+    // define the query using the DataFrame trait
+    let df = ctx
+        .read_parquet(filename)?
+        .select_columns(vec!["id", "bool_col", "timestamp_col"])?
+        .filter(col("id").gt(lit(1)))?;
+
+    // execute the query
+    let results = df.collect().await?;
+
+    // print the results
+    pretty::print_batches(&results)?;
+
+    Ok(())
+}
diff --git a/rust/datafusion/examples/memory_table_api.rs b/rust/datafusion/examples/dataframe_in_memory.rs
similarity index 84%
rename from rust/datafusion/examples/memory_table_api.rs
rename to rust/datafusion/examples/dataframe_in_memory.rs
index 937d80eeb3e..5f5a3d272ec 100644
--- a/rust/datafusion/examples/memory_table_api.rs
+++ b/rust/datafusion/examples/dataframe_in_memory.rs
@@ -25,11 +25,11 @@ use arrow::util::pretty;
 
 use datafusion::datasource::MemTable;
 use datafusion::error::Result;
-use datafusion::execution::context::ExecutionContext;
-use datafusion::logicalplan::lit;
+use datafusion::prelude::*;
 
-/// This example demonstrates basic uses of the Table API on an in-memory table
-fn main() -> Result<()> {
+/// This example demonstrates how to use the DataFrame API against in-memory data.
+#[tokio::main]
+async fn main() -> Result<()> {
     // define a schema.
     let schema = Arc::new(Schema::new(vec![
         Field::new("a", DataType::Utf8, false),
@@ -51,15 +51,15 @@ fn main() -> Result<()> {
     // declare a table in memory. In spark API, this corresponds to createDataFrame(...).
     let provider = MemTable::new(schema, vec![vec![batch]])?;
     ctx.register_table("t", Box::new(provider));
-    let t = ctx.table("t")?;
+    let df = ctx.table("t")?;
 
     // construct an expression corresponding to "SELECT a, b FROM t WHERE b = 10" in SQL
-    let filter = t.col("b")?.eq(&lit(10));
+    let filter = col("b").eq(lit(10));
 
-    let t = t.select_columns(vec!["a", "b"])?.filter(filter)?;
+    let df = df.select_columns(vec!["a", "b"])?.filter(filter)?;
 
     // execute
-    let results = t.collect(&mut ctx, 10)?;
+    let results = df.collect().await?;
 
     // print the results
     pretty::print_batches(&results)?;
diff --git a/rust/datafusion/examples/flight_client.rs b/rust/datafusion/examples/flight_client.rs
index f33714f553c..be3a4bdade7 100644
--- a/rust/datafusion/examples/flight_client.rs
+++ b/rust/datafusion/examples/flight_client.rs
@@ -23,9 +23,12 @@ use arrow::util::pretty;
 
 use arrow_flight::flight_descriptor;
 use arrow_flight::flight_service_client::FlightServiceClient;
-use arrow_flight::utils::flight_data_to_batch;
+use arrow_flight::utils::flight_data_to_arrow_batch;
 use arrow_flight::{FlightDescriptor, Ticket};
 
+/// This example shows how to wrap DataFusion with `FlightService` to support looking up schema information for
+/// Parquet files and executing SQL queries against them on a remote server.
+/// This example is run along-side the example `flight_server`.
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let testdata =
@@ -62,7 +65,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let mut results = vec![];
     while let Some(flight_data) = stream.message().await? {
         // the unwrap is infallible and thus safe
-        let record_batch = flight_data_to_batch(&flight_data, schema.clone())?.unwrap();
+        let record_batch =
+            flight_data_to_arrow_batch(&flight_data, schema.clone()).unwrap()?;
         results.push(record_batch);
     }
 
diff --git a/rust/datafusion/examples/flight_server.rs b/rust/datafusion/examples/flight_server.rs
index a3eff83884d..a601b7cafdd 100644
--- a/rust/datafusion/examples/flight_server.rs
+++ b/rust/datafusion/examples/flight_server.rs
@@ -23,7 +23,7 @@ use tonic::{Request, Response, Status, Streaming};
 
 use datafusion::datasource::parquet::ParquetTable;
 use datafusion::datasource::TableProvider;
-use datafusion::execution::context::ExecutionContext;
+use datafusion::prelude::*;
 
 use arrow_flight::{
     flight_service_server::FlightService, flight_service_server::FlightServiceServer,
@@ -34,10 +34,6 @@ use arrow_flight::{
 #[derive(Clone)]
 pub struct FlightServiceImpl {}
 
-/**
- * Example Flight Server wrapping DataFusion that supports looking up schema information for
- * Parquet files and executing SQL queries against Parquet files.
- */
 #[tonic::async_trait]
 impl FlightService for FlightServiceImpl {
     type HandshakeStream = Pin<
@@ -99,11 +95,14 @@ impl FlightService for FlightServiceImpl {
                 let plan = ctx
                     .create_logical_plan(&sql)
                     .and_then(|plan| ctx.optimize(&plan))
-                    .and_then(|plan| ctx.create_physical_plan(&plan, 1024 * 1024))
+                    .and_then(|plan| ctx.create_physical_plan(&plan))
                     .map_err(|e| to_tonic_err(&e))?;
 
                 // execute the query
-                let results = ctx.collect(plan.as_ref()).map_err(|e| to_tonic_err(&e))?;
+                let results = ctx
+                    .collect(plan.clone())
+                    .await
+                    .map_err(|e| to_tonic_err(&e))?;
                 if results.is_empty() {
                     return Err(Status::internal("There were no results from ticket"));
                 }
@@ -179,10 +178,13 @@ impl FlightService for FlightServiceImpl {
     }
 }
 
-fn to_tonic_err(e: &datafusion::error::ExecutionError) -> Status {
+fn to_tonic_err(e: &datafusion::error::DataFusionError) -> Status {
     Status::internal(format!("{:?}", e))
 }
 
+/// This example shows how to wrap DataFusion with `FlightService` to support looking up schema information for
+/// Parquet files and executing SQL queries against them on a remote server.
+/// This example is run along-side the example `flight_client`.
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let addr = "0.0.0.0:50051".parse()?;
diff --git a/rust/datafusion/examples/parquet_sql.rs b/rust/datafusion/examples/parquet_sql.rs
index 6e635dcc733..04293058620 100644
--- a/rust/datafusion/examples/parquet_sql.rs
+++ b/rust/datafusion/examples/parquet_sql.rs
@@ -16,12 +16,14 @@
 // under the License.
 
 use arrow::util::pretty;
+
 use datafusion::error::Result;
-use datafusion::execution::context::ExecutionContext;
+use datafusion::prelude::*;
 
 /// This example demonstrates executing a simple query against an Arrow data source (Parquet) and
 /// fetching results
-fn main() -> Result<()> {
+#[tokio::main]
+async fn main() -> Result<()> {
     // create local execution context
     let mut ctx = ExecutionContext::new();
 
@@ -34,16 +36,13 @@ fn main() -> Result<()> {
         &format!("{}/alltypes_plain.parquet", testdata),
     )?;
 
-    // simple selection
-    let sql = "SELECT int_col, double_col, CAST(date_string_col as VARCHAR) FROM alltypes_plain WHERE id > 1 AND tinyint_col < double_col";
-
-    // create the query plan
-    let plan = ctx.create_logical_plan(&sql)?;
-    let plan = ctx.optimize(&plan)?;
-    let plan = ctx.create_physical_plan(&plan, 1024 * 1024)?;
-
     // execute the query
-    let results = ctx.collect(plan.as_ref())?;
+    let df = ctx.sql(
+        "SELECT int_col, double_col, CAST(date_string_col as VARCHAR) \
+        FROM alltypes_plain \
+        WHERE id > 1 AND tinyint_col < double_col",
+    )?;
+    let results = df.collect().await?;
 
     // print the results
     pretty::print_batches(&results)?;
diff --git a/rust/datafusion/examples/simple_udaf.rs b/rust/datafusion/examples/simple_udaf.rs
new file mode 100644
index 00000000000..c3cd49fec73
--- /dev/null
+++ b/rust/datafusion/examples/simple_udaf.rs
@@ -0,0 +1,170 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// In this example we will declare a single-type, single return type UDAF that computes the geometric mean.
+/// The geometric mean is described here: https://en.wikipedia.org/wiki/Geometric_mean
+use arrow::{
+    array::Float32Array, array::Float64Array, datatypes::DataType,
+    record_batch::RecordBatch,
+};
+
+use datafusion::{error::Result, logical_plan::create_udaf, physical_plan::Accumulator};
+use datafusion::{prelude::*, scalar::ScalarValue};
+use std::sync::Arc;
+
+// create local execution context with an in-memory table
+fn create_context() -> Result<ExecutionContext> {
+    use arrow::datatypes::{Field, Schema};
+    use datafusion::datasource::MemTable;
+    // define a schema.
+    let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Float32, false)]));
+
+    // define data in two partitions
+    let batch1 = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(Float32Array::from(vec![2.0, 4.0, 8.0]))],
+    )?;
+    let batch2 = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(Float32Array::from(vec![64.0]))],
+    )?;
+
+    // declare a new context. In spark API, this corresponds to a new spark SQLsession
+    let mut ctx = ExecutionContext::new();
+
+    // declare a table in memory. In spark API, this corresponds to createDataFrame(...).
+    let provider = MemTable::new(schema, vec![vec![batch1], vec![batch2]])?;
+    ctx.register_table("t", Box::new(provider));
+    Ok(ctx)
+}
+
+/// A UDAF has state across multiple rows, and thus we require a `struct` with that state.
+#[derive(Debug)]
+struct GeometricMean {
+    n: u32,
+    prod: f64,
+}
+
+impl GeometricMean {
+    // how the struct is initialized
+    pub fn new() -> Self {
+        GeometricMean { n: 0, prod: 1.0 }
+    }
+}
+
+// UDAFs are built using the trait `Accumulator`, that offers DataFusion the necessary functions
+// to use them.
+impl Accumulator for GeometricMean {
+    // this function serializes our state to `ScalarValue`, which DataFusion uses
+    // to pass this state between execution stages.
+    // Note that this can be arbitrary data.
+    fn state(&self) -> Result<Vec<ScalarValue>> {
+        Ok(vec![
+            ScalarValue::from(self.prod),
+            ScalarValue::from(self.n),
+        ])
+    }
+
+    // this function receives one entry per argument of this accumulator.
+    // DataFusion calls this function on every row, and expects this function to update the accumulator's state.
+    fn update(&mut self, values: &Vec<ScalarValue>) -> Result<()> {
+        // this is a one-argument UDAF, and thus we use `0`.
+        let value = &values[0];
+        match value {
+            // here we map `ScalarValue` to our internal state. `Float64` indicates that this function
+            // only accepts Float64 as its argument (DataFusion does try to coerce arguments to this type)
+            //
+            // Note that `.map` here ensures that we ignore Nulls.
+            ScalarValue::Float64(e) => e.map(|value| {
+                self.prod *= value;
+                self.n += 1;
+            }),
+            _ => unreachable!(""),
+        };
+        Ok(())
+    }
+
+    // this function receives states from other accumulators (Vec<ScalarValue>)
+    // and updates the accumulator.
+    fn merge(&mut self, states: &Vec<ScalarValue>) -> Result<()> {
+        let prod = &states[0];
+        let n = &states[1];
+        match (prod, n) {
+            (ScalarValue::Float64(Some(prod)), ScalarValue::UInt32(Some(n))) => {
+                self.prod *= prod;
+                self.n += n;
+            }
+            _ => unreachable!(""),
+        };
+        Ok(())
+    }
+
+    // DataFusion expects this function to return the final value of this aggregator.
+    // in this case, this is the formula of the geometric mean
+    fn evaluate(&self) -> Result<ScalarValue> {
+        let value = self.prod.powf(1.0 / self.n as f64);
+        Ok(ScalarValue::from(value))
+    }
+
+    // Optimization hint: this trait also supports `update_batch` and `merge_batch`,
+    // that can be used to perform these operations on arrays instead of single values.
+    // By default, these methods call `update` and `merge` row by row
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut ctx = create_context()?;
+
+    // here is where we define the UDAF. We also declare its signature:
+    let geometric_mean = create_udaf(
+        // the name; used to represent it in plan descriptions and in the registry, to use in SQL.
+        "geo_mean",
+        // the input type; DataFusion guarantees that the first entry of `values` in `update` has this type.
+        DataType::Float64,
+        // the return type; DataFusion expects this to match the type returned by `evaluate`.
+        Arc::new(DataType::Float64),
+        // This is the accumulator factory; DataFusion uses it to create new accumulators.
+        Arc::new(|| Ok(Box::new(GeometricMean::new()))),
+        // This is the description of the state. `state()` must match the types here.
+        Arc::new(vec![DataType::Float64, DataType::UInt32]),
+    );
+
+    // get a DataFrame from the context
+    // this table has 1 column `a` f32 with values {2,4,8,64}, whose geometric mean is 8.0.
+    let df = ctx.table("t")?;
+
+    // perform the aggregation
+    let df = df.aggregate(vec![], vec![geometric_mean.call(vec![col("a")])])?;
+
+    // note that "a" is f32, not f64. DataFusion coerces it to match the UDAF's signature.
+
+    // execute the query
+    let results = df.collect().await?;
+
+    // downcast the array to the expected type
+    let result = results[0]
+        .column(0)
+        .as_any()
+        .downcast_ref::<Float64Array>()
+        .unwrap();
+
+    // verify that the calculation is correct
+    assert_eq!(result.value(0), 8.0);
+    println!("The geometric mean of [2,4,8,64] is {}", result.value(0));
+
+    Ok(())
+}
diff --git a/rust/datafusion/examples/simple_udf.rs b/rust/datafusion/examples/simple_udf.rs
new file mode 100644
index 00000000000..c0d4a04bd15
--- /dev/null
+++ b/rust/datafusion/examples/simple_udf.rs
@@ -0,0 +1,147 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::{
+    array::{Array, ArrayRef, Float32Array, Float64Array, Float64Builder},
+    datatypes::DataType,
+    record_batch::RecordBatch,
+    util::pretty,
+};
+
+use datafusion::error::Result;
+use datafusion::{physical_plan::functions::ScalarFunctionImplementation, prelude::*};
+use std::sync::Arc;
+
+// create local execution context with an in-memory table
+fn create_context() -> Result<ExecutionContext> {
+    use arrow::datatypes::{Field, Schema};
+    use datafusion::datasource::MemTable;
+    // define a schema.
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("a", DataType::Float32, false),
+        Field::new("b", DataType::Float64, false),
+    ]));
+
+    // define data.
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(Float32Array::from(vec![2.1, 3.1, 4.1, 5.1])),
+            Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0, 4.0])),
+        ],
+    )?;
+
+    // declare a new context. In spark API, this corresponds to a new spark SQLsession
+    let mut ctx = ExecutionContext::new();
+
+    // declare a table in memory. In spark API, this corresponds to createDataFrame(...).
+    let provider = MemTable::new(schema, vec![vec![batch]])?;
+    ctx.register_table("t", Box::new(provider));
+    Ok(ctx)
+}
+
+/// In this example we will declare a single-type, single return type UDF that exponentiates f64, a^b
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut ctx = create_context()?;
+
+    // First, declare the actual implementation of the calculation
+    let pow: ScalarFunctionImplementation = Arc::new(|args: &[ArrayRef]| {
+        // in DataFusion, all `args` and output are dynamically-typed arrays, which means that we need to:
+        // 1. cast the values to the type we want
+        // 2. perform the computation for every element in the array (using a loop or SIMD)
+        // 3. construct the resulting array
+
+        // this is guaranteed by DataFusion based on the function's signature.
+        assert_eq!(args.len(), 2);
+
+        // 1. cast both arguments to f64. These casts MUST be aligned with the signature or this function panics!
+        let base = &args[0]
+            .as_any()
+            .downcast_ref::<Float64Array>()
+            .expect("cast failed");
+        let exponent = &args[1]
+            .as_any()
+            .downcast_ref::<Float64Array>()
+            .expect("cast failed");
+
+        // this is guaranteed by DataFusion. We place it just to make it obvious.
+        assert_eq!(exponent.len(), base.len());
+
+        // 2. Arrow's builder is used to construct an Arrow array.
+        let mut builder = Float64Builder::new(base.len());
+        for index in 0..base.len() {
+            // in arrow, any value can be null.
+            // Here we decide to make our UDF to return null when either base or exponent is null.
+            if base.is_null(index) || exponent.is_null(index) {
+                builder.append_null()?;
+            } else {
+                // 3. computation. Since we do not have any SIMD `pow` operation at our hands,
+                // we loop over each entry. Array's values are obtained via `.value(index)`.
+                let value = base.value(index).powf(exponent.value(index));
+                builder.append_value(value)?;
+            }
+        }
+        Ok(Arc::new(builder.finish()))
+    });
+
+    // Next:
+    // * give it a name so that it shows nicely when the plan is printed
+    // * declare what input it expects
+    // * declare its return type
+    let pow = create_udf(
+        "pow",
+        // expects two f64
+        vec![DataType::Float64, DataType::Float64],
+        // returns f64
+        Arc::new(DataType::Float64),
+        pow,
+    );
+
+    // at this point, we can use it or register it, depending on the use-case:
+    // * if the UDF is expected to be used throughout the program in different contexts,
+    //   we can register it, and call it later:
+    ctx.register_udf(pow.clone()); // clone is only required in this example because we show both usages
+
+    // * if the UDF is expected to be used directly in the scope, `.call` it directly:
+    let expr = pow.call(vec![col("a"), col("b")]);
+
+    // get a DataFrame from the context
+    let df = ctx.table("t")?;
+
+    // if we do not have `pow` in the scope and we registered it, we can get it from the registry
+    let pow = df.registry().udf("pow")?;
+    // equivalent to expr
+    let expr1 = pow.call(vec![col("a"), col("b")]);
+
+    // equivalent to `'SELECT pow(a, b), pow(a, b) AS pow1 FROM t'`
+    let df = df.select(vec![
+        expr,
+        // alias so that they have different column names
+        expr1.alias("pow1"),
+    ])?;
+
+    // note that "b" is f32, not f64. DataFusion coerces the types to match the UDF's signature.
+
+    // execute the query
+    let results = df.collect().await?;
+
+    // print the results
+    pretty::print_batches(&results)?;
+
+    Ok(())
+}
diff --git a/rust/datafusion/src/bin/repl.rs b/rust/datafusion/src/bin/repl.rs
index 4a8464b600e..df27f984319 100644
--- a/rust/datafusion/src/bin/repl.rs
+++ b/rust/datafusion/src/bin/repl.rs
@@ -20,13 +20,14 @@
 use arrow::util::pretty;
 use clap::{crate_version, App, Arg};
 use datafusion::error::Result;
-use datafusion::execution::context::ExecutionContext;
+use datafusion::execution::context::{ExecutionConfig, ExecutionContext};
 use rustyline::Editor;
 use std::env;
 use std::path::Path;
 use std::time::Instant;
 
-pub fn main() {
+#[tokio::main]
+pub async fn main() {
     let matches = App::new("DataFusion")
         .version(crate_version!())
         .about(
@@ -60,7 +61,8 @@ pub fn main() {
         .map(|size| size.parse::<usize>().unwrap())
         .unwrap_or(1_048_576);
 
-    let mut ctx = ExecutionContext::new();
+    let mut ctx =
+        ExecutionContext::with_config(ExecutionConfig::new().with_batch_size(batch_size));
 
     let mut rl = Editor::<()>::new();
     rl.load_history(".history").ok();
@@ -75,7 +77,7 @@ pub fn main() {
             Ok(ref line) if line.trim_end().ends_with(';') => {
                 query.push_str(line.trim_end());
                 rl.add_history_entry(query.clone());
-                match exec_and_print(&mut ctx, query, batch_size) {
+                match exec_and_print(&mut ctx, query).await {
                     Ok(_) => {}
                     Err(err) => println!("{:?}", err),
                 }
@@ -99,14 +101,11 @@ fn is_exit_command(line: &str) -> bool {
     line == "quit" || line == "exit"
 }
 
-fn exec_and_print(
-    ctx: &mut ExecutionContext,
-    sql: String,
-    batch_size: usize,
-) -> Result<()> {
+async fn exec_and_print(ctx: &mut ExecutionContext, sql: String) -> Result<()> {
     let now = Instant::now();
 
-    let results = ctx.sql(&sql, batch_size)?;
+    let df = ctx.sql(&sql)?;
+    let results = df.collect().await?;
 
     if results.is_empty() {
         println!(
diff --git a/rust/datafusion/src/dataframe.rs b/rust/datafusion/src/dataframe.rs
new file mode 100644
index 00000000000..d2ad7e6917a
--- /dev/null
+++ b/rust/datafusion/src/dataframe.rs
@@ -0,0 +1,211 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! DataFrame API for building and executing query plans.
+
+use crate::arrow::record_batch::RecordBatch;
+use crate::error::Result;
+use crate::logical_plan::{Expr, FunctionRegistry, LogicalPlan};
+use arrow::datatypes::Schema;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+
+/// DataFrame represents a logical set of rows with the same named columns.
+/// Similar to a [Pandas DataFrame](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html) or
+/// [Spark DataFrame](https://spark.apache.org/docs/latest/sql-programming-guide.html)
+///
+/// DataFrames are typically created by the `read_csv` and `read_parquet` methods on the
+/// [ExecutionContext](../execution/context/struct.ExecutionContext.html) and can then be modified
+/// by calling the transformation methods, such as `filter`, `select`, `aggregate`, and `limit`
+/// to build up a query definition.
+///
+/// The query can be executed by calling the `collect` method.
+///
+/// ```
+/// # use datafusion::prelude::*;
+/// # use datafusion::error::Result;
+/// # fn main() -> Result<()> {
+/// let mut ctx = ExecutionContext::new();
+/// let df = ctx.read_csv("tests/example.csv", CsvReadOptions::new())?;
+/// let df = df.filter(col("a").lt_eq(col("b")))?
+///            .aggregate(vec![col("a")], vec![min(col("b"))])?
+///            .limit(100)?;
+/// let results = df.collect();
+/// # Ok(())
+/// # }
+/// ```
+#[async_trait]
+pub trait DataFrame {
+    /// Filter the DataFrame by column. Returns a new DataFrame only containing the
+    /// specified columns.
+    ///
+    /// ```
+    /// # use datafusion::prelude::*;
+    /// # use datafusion::error::Result;
+    /// # fn main() -> Result<()> {
+    /// let mut ctx = ExecutionContext::new();
+    /// let df = ctx.read_csv("tests/example.csv", CsvReadOptions::new())?;
+    /// let df = df.select_columns(vec!["a", "b"])?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    fn select_columns(&self, columns: Vec<&str>) -> Result<Arc<dyn DataFrame>>;
+
+    /// Create a projection based on arbitrary expressions.
+    ///
+    /// ```
+    /// # use datafusion::prelude::*;
+    /// # use datafusion::error::Result;
+    /// # fn main() -> Result<()> {
+    /// let mut ctx = ExecutionContext::new();
+    /// let df = ctx.read_csv("tests/example.csv", CsvReadOptions::new())?;
+    /// let df = df.select(vec![col("a") * col("b"), col("c")])?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    fn select(&self, expr: Vec<Expr>) -> Result<Arc<dyn DataFrame>>;
+
+    /// Filter a DataFrame to only include rows that match the specified filter expression.
+    ///
+    /// ```
+    /// # use datafusion::prelude::*;
+    /// # use datafusion::error::Result;
+    /// # fn main() -> Result<()> {
+    /// let mut ctx = ExecutionContext::new();
+    /// let df = ctx.read_csv("tests/example.csv", CsvReadOptions::new())?;
+    /// let df = df.filter(col("a").lt_eq(col("b")))?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    fn filter(&self, expr: Expr) -> Result<Arc<dyn DataFrame>>;
+
+    /// Perform an aggregate query with optional grouping expressions.
+    ///
+    /// ```
+    /// # use datafusion::prelude::*;
+    /// # use datafusion::error::Result;
+    /// # fn main() -> Result<()> {
+    /// let mut ctx = ExecutionContext::new();
+    /// let df = ctx.read_csv("tests/example.csv", CsvReadOptions::new())?;
+    ///
+    /// // The following use is the equivalent of "SELECT MIN(b) GROUP BY a"
+    /// let _ = df.aggregate(vec![col("a")], vec![min(col("b"))])?;
+    ///
+    /// // The following use is the equivalent of "SELECT MIN(b)"
+    /// let _ = df.aggregate(vec![], vec![min(col("b"))])?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    fn aggregate(
+        &self,
+        group_expr: Vec<Expr>,
+        aggr_expr: Vec<Expr>,
+    ) -> Result<Arc<dyn DataFrame>>;
+
+    /// Limit the number of rows returned from this DataFrame.
+    ///
+    /// ```
+    /// # use datafusion::prelude::*;
+    /// # use datafusion::error::Result;
+    /// # fn main() -> Result<()> {
+    /// let mut ctx = ExecutionContext::new();
+    /// let df = ctx.read_csv("tests/example.csv", CsvReadOptions::new())?;
+    /// let df = df.limit(100)?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    fn limit(&self, n: usize) -> Result<Arc<dyn DataFrame>>;
+
+    /// Sort the DataFrame by the specified sorting expressions. Any expression can be turned into
+    /// a sort expression by calling its [sort](../logical_plan/enum.Expr.html#method.sort) method.
+    ///
+    /// ```
+    /// # use datafusion::prelude::*;
+    /// # use datafusion::error::Result;
+    /// # fn main() -> Result<()> {
+    /// let mut ctx = ExecutionContext::new();
+    /// let df = ctx.read_csv("tests/example.csv", CsvReadOptions::new())?;
+    /// let df = df.sort(vec![col("a").sort(true, true), col("b").sort(false, false)])?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    fn sort(&self, expr: Vec<Expr>) -> Result<Arc<dyn DataFrame>>;
+
+    /// Executes this DataFrame and collects all results into a vector of RecordBatch.
+    ///
+    /// ```
+    /// # use datafusion::prelude::*;
+    /// # use datafusion::error::Result;
+    /// # #[tokio::main]
+    /// # async fn main() -> Result<()> {
+    /// let mut ctx = ExecutionContext::new();
+    /// let df = ctx.read_csv("tests/example.csv", CsvReadOptions::new())?;
+    /// let batches = df.collect().await?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    async fn collect(&self) -> Result<Vec<RecordBatch>>;
+
+    /// Returns the schema describing the output of this DataFrame in terms of columns returned,
+    /// where each column has a name, data type, and nullability attribute.
+
+    /// ```
+    /// # use datafusion::prelude::*;
+    /// # use datafusion::error::Result;
+    /// # fn main() -> Result<()> {
+    /// let mut ctx = ExecutionContext::new();
+    /// let df = ctx.read_csv("tests/example.csv", CsvReadOptions::new())?;
+    /// let schema = df.schema();
+    /// # Ok(())
+    /// # }
+    /// ```
+    fn schema(&self) -> &Schema;
+
+    /// Return the logical plan represented by this DataFrame.
+    fn to_logical_plan(&self) -> LogicalPlan;
+
+    /// Return a DataFrame with the explanation of its plan so far.
+    ///
+    /// ```
+    /// # use datafusion::prelude::*;
+    /// # use datafusion::error::Result;
+    /// # #[tokio::main]
+    /// # async fn main() -> Result<()> {
+    /// let mut ctx = ExecutionContext::new();
+    /// let df = ctx.read_csv("tests/example.csv", CsvReadOptions::new())?;
+    /// let batches = df.limit(100)?.explain(false)?.collect().await?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    fn explain(&self, verbose: bool) -> Result<Arc<dyn DataFrame>>;
+
+    /// Return a `FunctionRegistry` used to plan udf's calls
+    ///
+    /// ```
+    /// # use datafusion::prelude::*;
+    /// # use datafusion::error::Result;
+    /// # fn main() -> Result<()> {
+    /// let mut ctx = ExecutionContext::new();
+    /// let df = ctx.read_csv("tests/example.csv", CsvReadOptions::new())?;
+    /// let f = df.registry();
+    /// // use f.udf("name", vec![...]) to use the udf
+    /// # Ok(())
+    /// # }
+    /// ```
+    fn registry(&self) -> &dyn FunctionRegistry;
+}
diff --git a/rust/datafusion/src/datasource/csv.rs b/rust/datafusion/src/datasource/csv.rs
index 225ebfbf77b..b0317c16d33 100644
--- a/rust/datafusion/src/datasource/csv.rs
+++ b/rust/datafusion/src/datasource/csv.rs
@@ -33,24 +33,20 @@
 //! let schema = csvdata.schema();
 //! ```
 
-use std::fs::File;
-
-use arrow::csv;
-use arrow::datatypes::{Field, Schema, SchemaRef};
-use arrow::error::Result as ArrowResult;
-use arrow::record_batch::{RecordBatch, RecordBatchReader};
+use arrow::datatypes::SchemaRef;
 use std::string::String;
 use std::sync::Arc;
 
-use crate::datasource::{ScanResult, TableProvider};
-use crate::error::Result;
-use crate::execution::physical_plan::csv::CsvExec;
-pub use crate::execution::physical_plan::csv::CsvReadOptions;
-use crate::execution::physical_plan::ExecutionPlan;
+use crate::datasource::TableProvider;
+use crate::error::{DataFusionError, Result};
+use crate::physical_plan::csv::CsvExec;
+pub use crate::physical_plan::csv::CsvReadOptions;
+use crate::physical_plan::{common, ExecutionPlan};
 
 /// Represents a CSV file with a provided schema
 pub struct CsvFile {
-    filename: String,
+    /// Path to a single CSV file or a directory containing one of more CSV files
+    path: String,
     schema: SchemaRef,
     has_header: bool,
     delimiter: u8,
@@ -59,14 +55,21 @@ pub struct CsvFile {
 
 impl CsvFile {
     /// Attempt to initialize a new `CsvFile` from a file path
-    pub fn try_new(filename: &str, options: CsvReadOptions) -> Result<Self> {
+    pub fn try_new(path: &str, options: CsvReadOptions) -> Result<Self> {
         let schema = Arc::new(match options.schema {
             Some(s) => s.clone(),
-            None => CsvExec::try_infer_schema(filename, &options)?,
+            None => {
+                let mut filenames: Vec<String> = vec![];
+                common::build_file_list(path, &mut filenames, options.file_extension)?;
+                if filenames.is_empty() {
+                    return Err(DataFusionError::Plan("No files found".to_string()));
+                }
+                CsvExec::try_infer_schema(&filenames, &options)?
+            }
         });
 
         Ok(Self {
-            filename: String::from(filename),
+            path: String::from(path),
             schema,
             has_header: options.has_header,
             delimiter: options.delimiter,
@@ -84,9 +87,9 @@ impl TableProvider for CsvFile {
         &self,
         projection: &Option<Vec<usize>>,
         batch_size: usize,
-    ) -> Result<Vec<ScanResult>> {
-        let exec = CsvExec::try_new(
-            &self.filename,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        Ok(Arc::new(CsvExec::try_new(
+            &self.path,
             CsvReadOptions::new()
                 .schema(&self.schema)
                 .has_header(self.has_header)
@@ -94,66 +97,6 @@ impl TableProvider for CsvFile {
                 .file_extension(self.file_extension.as_str()),
             projection.clone(),
             batch_size,
-        )?;
-        let partitions = exec.partitions()?;
-        let iterators = partitions
-            .iter()
-            .map(|p| p.execute())
-            .collect::<Result<Vec<_>>>()?;
-        Ok(iterators)
-    }
-}
-
-/// Iterator over CSV batches
-// TODO: usage example (rather than documenting `new()`)
-pub struct CsvBatchIterator {
-    schema: SchemaRef,
-    reader: csv::Reader<File>,
-}
-
-impl CsvBatchIterator {
-    #[allow(missing_docs)]
-    pub fn try_new(
-        filename: &str,
-        schema: SchemaRef,
-        has_header: bool,
-        delimiter: Option<u8>,
-        projection: &Option<Vec<usize>>,
-        batch_size: usize,
-    ) -> Result<Self> {
-        let file = File::open(filename)?;
-        let reader = csv::Reader::new(
-            file,
-            schema.clone(),
-            has_header,
-            delimiter,
-            batch_size,
-            projection.clone(),
-        );
-
-        let projected_schema = match projection {
-            Some(p) => {
-                let projected_fields: Vec<Field> =
-                    p.iter().map(|i| schema.fields()[*i].clone()).collect();
-
-                Arc::new(Schema::new(projected_fields))
-            }
-            None => schema,
-        };
-
-        Ok(Self {
-            schema: projected_schema,
-            reader,
-        })
-    }
-}
-
-impl RecordBatchReader for CsvBatchIterator {
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-
-    fn next_batch(&mut self) -> ArrowResult<Option<RecordBatch>> {
-        self.reader.next()
+        )?))
     }
 }
diff --git a/rust/datafusion/src/datasource/datasource.rs b/rust/datafusion/src/datasource/datasource.rs
index 9d83023a1f0..3b1cf0c778b 100644
--- a/rust/datafusion/src/datasource/datasource.rs
+++ b/rust/datafusion/src/datasource/datasource.rs
@@ -17,27 +17,21 @@
 
 //! Data source traits
 
-use std::sync::{Arc, Mutex};
-
-use arrow::datatypes::SchemaRef;
-use arrow::record_batch::RecordBatchReader;
+use std::sync::Arc;
 
+use crate::arrow::datatypes::SchemaRef;
 use crate::error::Result;
-
-/// Returned by implementors of `Table#scan`, this `SendableRecordBatchReader` is wrapped with
-/// an `Arc` and `Mutex` so that it can be shared across threads as it is used.
-pub type ScanResult = Arc<Mutex<dyn RecordBatchReader + Send + Sync>>;
+use crate::physical_plan::ExecutionPlan;
 
 /// Source table
 pub trait TableProvider {
     /// Get a reference to the schema for this table
     fn schema(&self) -> SchemaRef;
 
-    /// Perform a scan of a table and return a sequence of iterators over the data (one
-    /// iterator per partition)
+    /// Create an ExecutionPlan that will scan the table.
     fn scan(
         &self,
         projection: &Option<Vec<usize>>,
         batch_size: usize,
-    ) -> Result<Vec<ScanResult>>;
+    ) -> Result<Arc<dyn ExecutionPlan>>;
 }
diff --git a/rust/datafusion/src/datasource/memory.rs b/rust/datafusion/src/datasource/memory.rs
index 5f2a5b7501e..12db5b5e9e8 100644
--- a/rust/datafusion/src/datasource/memory.rs
+++ b/rust/datafusion/src/datasource/memory.rs
@@ -24,10 +24,11 @@ use std::sync::Arc;
 use arrow::datatypes::{Field, Schema, SchemaRef};
 use arrow::record_batch::RecordBatch;
 
-use crate::datasource::{ScanResult, TableProvider};
-use crate::error::{ExecutionError, Result};
-use crate::execution::physical_plan::memory::MemoryExec;
-use crate::execution::physical_plan::ExecutionPlan;
+use crate::datasource::TableProvider;
+use crate::error::{DataFusionError, Result};
+use crate::physical_plan::common;
+use crate::physical_plan::memory::MemoryExec;
+use crate::physical_plan::ExecutionPlan;
 
 /// In-memory table
 pub struct MemTable {
@@ -48,24 +49,35 @@ impl MemTable {
                 batches: partitions,
             })
         } else {
-            Err(ExecutionError::General(
+            Err(DataFusionError::Plan(
                 "Mismatch between schema and batches".to_string(),
             ))
         }
     }
 
     /// Create a mem table by reading from another data source
-    pub fn load(t: &dyn TableProvider) -> Result<Self> {
+    pub async fn load(t: &dyn TableProvider, batch_size: usize) -> Result<Self> {
         let schema = t.schema();
-        let partitions = t.scan(&None, 1024 * 1024)?;
-
-        let mut data: Vec<Vec<RecordBatch>> = Vec::with_capacity(partitions.len());
-        for it in &partitions {
-            let mut partition = vec![];
-            while let Ok(Some(batch)) = it.lock().unwrap().next_batch() {
-                partition.push(batch);
-            }
-            data.push(partition);
+        let exec = t.scan(&None, batch_size)?;
+        let partition_count = exec.output_partitioning().partition_count();
+
+        let tasks = (0..partition_count)
+            .map(|part_i| {
+                let exec = exec.clone();
+                tokio::spawn(async move {
+                    let stream = exec.execute(part_i).await?;
+                    common::collect(stream).await
+                })
+            })
+            // this collect *is needed* so that the join below can
+            // switch between tasks
+            .collect::<Vec<_>>();
+
+        let mut data: Vec<Vec<RecordBatch>> =
+            Vec::with_capacity(exec.output_partitioning().partition_count());
+        for task in tasks {
+            let result = task.await.expect("MemTable::load could not join task")?;
+            data.push(result);
         }
 
         MemTable::new(schema.clone(), data)
@@ -81,7 +93,7 @@ impl TableProvider for MemTable {
         &self,
         projection: &Option<Vec<usize>>,
         _batch_size: usize,
-    ) -> Result<Vec<ScanResult>> {
+    ) -> Result<Arc<dyn ExecutionPlan>> {
         let columns: Vec<usize> = match projection {
             Some(p) => p.clone(),
             None => {
@@ -100,7 +112,7 @@ impl TableProvider for MemTable {
                 if *i < self.schema.fields().len() {
                     Ok(self.schema.field(*i).clone())
                 } else {
-                    Err(ExecutionError::General(
+                    Err(DataFusionError::Internal(
                         "Projection index out of range".to_string(),
                     ))
                 }
@@ -109,17 +121,11 @@ impl TableProvider for MemTable {
 
         let projected_schema = Arc::new(Schema::new(projected_columns?));
 
-        let exec = MemoryExec::try_new(
+        Ok(Arc::new(MemoryExec::try_new(
             &self.batches.clone(),
             projected_schema,
             projection.clone(),
-        )?;
-        let partitions = exec.partitions()?;
-        let iterators = partitions
-            .iter()
-            .map(|p| p.execute())
-            .collect::<Result<Vec<_>>>()?;
-        Ok(iterators)
+        )?))
     }
 }
 
@@ -128,9 +134,10 @@ mod tests {
     use super::*;
     use arrow::array::Int32Array;
     use arrow::datatypes::{DataType, Field, Schema};
+    use futures::StreamExt;
 
-    #[test]
-    fn test_with_projection() {
+    #[tokio::test]
+    async fn test_with_projection() -> Result<()> {
         let schema = Arc::new(Schema::new(vec![
             Field::new("a", DataType::Int32, false),
             Field::new("b", DataType::Int32, false),
@@ -144,22 +151,24 @@ mod tests {
                 Arc::new(Int32Array::from(vec![4, 5, 6])),
                 Arc::new(Int32Array::from(vec![7, 8, 9])),
             ],
-        )
-        .unwrap();
+        )?;
 
-        let provider = MemTable::new(schema, vec![vec![batch]]).unwrap();
+        let provider = MemTable::new(schema, vec![vec![batch]])?;
 
         // scan with projection
-        let partitions = provider.scan(&Some(vec![2, 1]), 1024).unwrap();
-        let batch2 = partitions[0].lock().unwrap().next_batch().unwrap().unwrap();
+        let exec = provider.scan(&Some(vec![2, 1]), 1024)?;
+        let mut it = exec.execute(0).await?;
+        let batch2 = it.next().await.unwrap()?;
         assert_eq!(2, batch2.schema().fields().len());
         assert_eq!("c", batch2.schema().field(0).name());
         assert_eq!("b", batch2.schema().field(1).name());
         assert_eq!(2, batch2.num_columns());
+
+        Ok(())
     }
 
-    #[test]
-    fn test_without_projection() {
+    #[tokio::test]
+    async fn test_without_projection() -> Result<()> {
         let schema = Arc::new(Schema::new(vec![
             Field::new("a", DataType::Int32, false),
             Field::new("b", DataType::Int32, false),
@@ -173,19 +182,21 @@ mod tests {
                 Arc::new(Int32Array::from(vec![4, 5, 6])),
                 Arc::new(Int32Array::from(vec![7, 8, 9])),
             ],
-        )
-        .unwrap();
+        )?;
 
-        let provider = MemTable::new(schema, vec![vec![batch]]).unwrap();
+        let provider = MemTable::new(schema, vec![vec![batch]])?;
 
-        let partitions = provider.scan(&None, 1024).unwrap();
-        let batch1 = partitions[0].lock().unwrap().next_batch().unwrap().unwrap();
+        let exec = provider.scan(&None, 1024)?;
+        let mut it = exec.execute(0).await?;
+        let batch1 = it.next().await.unwrap()?;
         assert_eq!(3, batch1.schema().fields().len());
         assert_eq!(3, batch1.num_columns());
+
+        Ok(())
     }
 
     #[test]
-    fn test_invalid_projection() {
+    fn test_invalid_projection() -> Result<()> {
         let schema = Arc::new(Schema::new(vec![
             Field::new("a", DataType::Int32, false),
             Field::new("b", DataType::Int32, false),
@@ -199,23 +210,24 @@ mod tests {
                 Arc::new(Int32Array::from(vec![4, 5, 6])),
                 Arc::new(Int32Array::from(vec![7, 8, 9])),
             ],
-        )
-        .unwrap();
+        )?;
 
-        let provider = MemTable::new(schema, vec![vec![batch]]).unwrap();
+        let provider = MemTable::new(schema, vec![vec![batch]])?;
 
         let projection: Vec<usize> = vec![0, 4];
 
         match provider.scan(&Some(projection), 1024) {
-            Err(ExecutionError::General(e)) => {
+            Err(DataFusionError::Internal(e)) => {
                 assert_eq!("\"Projection index out of range\"", format!("{:?}", e))
             }
             _ => assert!(false, "Scan should failed on invalid projection"),
         };
+
+        Ok(())
     }
 
     #[test]
-    fn test_schema_validation() {
+    fn test_schema_validation() -> Result<()> {
         let schema1 = Arc::new(Schema::new(vec![
             Field::new("a", DataType::Int32, false),
             Field::new("b", DataType::Int32, false),
@@ -235,11 +247,10 @@ mod tests {
                 Arc::new(Int32Array::from(vec![4, 5, 6])),
                 Arc::new(Int32Array::from(vec![7, 8, 9])),
             ],
-        )
-        .unwrap();
+        )?;
 
         match MemTable::new(schema2, vec![vec![batch]]) {
-            Err(ExecutionError::General(e)) => assert_eq!(
+            Err(DataFusionError::Plan(e)) => assert_eq!(
                 "\"Mismatch between schema and batches\"",
                 format!("{:?}", e)
             ),
@@ -248,5 +259,7 @@ mod tests {
                 "MemTable::new should have failed due to schema mismatch"
             ),
         }
+
+        Ok(())
     }
 }
diff --git a/rust/datafusion/src/datasource/mod.rs b/rust/datafusion/src/datasource/mod.rs
index 7ecc4a18d6d..5c3b9401be3 100644
--- a/rust/datafusion/src/datasource/mod.rs
+++ b/rust/datafusion/src/datasource/mod.rs
@@ -22,6 +22,6 @@ pub mod datasource;
 pub mod memory;
 pub mod parquet;
 
-pub use self::csv::{CsvBatchIterator, CsvFile, CsvReadOptions};
-pub use self::datasource::{ScanResult, TableProvider};
+pub use self::csv::{CsvFile, CsvReadOptions};
+pub use self::datasource::TableProvider;
 pub use self::memory::MemTable;
diff --git a/rust/datafusion/src/datasource/parquet.rs b/rust/datafusion/src/datasource/parquet.rs
index ff323270529..eb4e0beef34 100644
--- a/rust/datafusion/src/datasource/parquet.rs
+++ b/rust/datafusion/src/datasource/parquet.rs
@@ -18,13 +18,14 @@
 //! Parquet data source
 
 use std::string::String;
+use std::sync::Arc;
 
 use arrow::datatypes::*;
 
-use crate::datasource::{ScanResult, TableProvider};
+use crate::datasource::TableProvider;
 use crate::error::Result;
-use crate::execution::physical_plan::parquet::ParquetExec;
-use crate::execution::physical_plan::ExecutionPlan;
+use crate::physical_plan::parquet::ParquetExec;
+use crate::physical_plan::ExecutionPlan;
 
 /// Table-based representation of a `ParquetFile`.
 pub struct ParquetTable {
@@ -56,17 +57,12 @@ impl TableProvider for ParquetTable {
         &self,
         projection: &Option<Vec<usize>>,
         batch_size: usize,
-    ) -> Result<Vec<ScanResult>> {
-        let parquet_exec =
-            ParquetExec::try_new(&self.path, projection.clone(), batch_size)?;
-
-        let partitions = parquet_exec.partitions()?;
-
-        let iterators = partitions
-            .iter()
-            .map(|p| p.execute())
-            .collect::<Result<Vec<_>>>()?;
-        Ok(iterators)
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        Ok(Arc::new(ParquetExec::try_new(
+            &self.path,
+            projection.clone(),
+            batch_size,
+        )?))
     }
 }
 
@@ -77,30 +73,35 @@ mod tests {
         BinaryArray, BooleanArray, Float32Array, Float64Array, Int32Array,
         TimestampNanosecondArray,
     };
+    use arrow::record_batch::RecordBatch;
+    use futures::StreamExt;
     use std::env;
 
-    #[test]
-    fn read_small_batches() {
-        let table = load_table("alltypes_plain.parquet");
-
+    #[tokio::test]
+    async fn read_small_batches() -> Result<()> {
+        let table = load_table("alltypes_plain.parquet")?;
         let projection = None;
-        let scan = table.scan(&projection, 2).unwrap();
-        let mut it = scan[0].lock().unwrap();
-
-        let mut count = 0;
-        while let Some(batch) = it.next_batch().unwrap() {
-            assert_eq!(11, batch.num_columns());
-            assert_eq!(2, batch.num_rows());
-            count += 1;
-        }
+        let exec = table.scan(&projection, 2)?;
+        let stream = exec.execute(0).await?;
+
+        let count = stream
+            .map(|batch| {
+                let batch = batch.unwrap();
+                assert_eq!(11, batch.num_columns());
+                assert_eq!(2, batch.num_rows());
+            })
+            .fold(0, |acc, _| async move { acc + 1i32 })
+            .await;
 
         // we should have seen 4 batches of 2 rows
         assert_eq!(4, count);
+
+        Ok(())
     }
 
-    #[test]
-    fn read_alltypes_plain_parquet() {
-        let table = load_table("alltypes_plain.parquet");
+    #[tokio::test]
+    async fn read_alltypes_plain_parquet() -> Result<()> {
+        let table = load_table("alltypes_plain.parquet")?;
 
         let x: Vec<String> = table
             .schema()
@@ -125,22 +126,19 @@ mod tests {
         );
 
         let projection = None;
-        let scan = table.scan(&projection, 1024).unwrap();
-        let mut it = scan[0].lock().unwrap();
-        let batch = it.next_batch().unwrap().unwrap();
+        let batch = get_first_batch(table, &projection).await?;
 
         assert_eq!(11, batch.num_columns());
         assert_eq!(8, batch.num_rows());
-    }
 
-    #[test]
-    fn read_bool_alltypes_plain_parquet() {
-        let table = load_table("alltypes_plain.parquet");
+        Ok(())
+    }
 
+    #[tokio::test]
+    async fn read_bool_alltypes_plain_parquet() -> Result<()> {
+        let table = load_table("alltypes_plain.parquet")?;
         let projection = Some(vec![1]);
-        let scan = table.scan(&projection, 1024).unwrap();
-        let mut it = scan[0].lock().unwrap();
-        let batch = it.next_batch().unwrap().unwrap();
+        let batch = get_first_batch(table, &projection).await?;
 
         assert_eq!(1, batch.num_columns());
         assert_eq!(8, batch.num_rows());
@@ -159,16 +157,15 @@ mod tests {
             "[true, false, true, false, true, false, true, false]",
             format!("{:?}", values)
         );
-    }
 
-    #[test]
-    fn read_i32_alltypes_plain_parquet() {
-        let table = load_table("alltypes_plain.parquet");
+        Ok(())
+    }
 
+    #[tokio::test]
+    async fn read_i32_alltypes_plain_parquet() -> Result<()> {
+        let table = load_table("alltypes_plain.parquet")?;
         let projection = Some(vec![0]);
-        let scan = table.scan(&projection, 1024).unwrap();
-        let mut it = scan[0].lock().unwrap();
-        let batch = it.next_batch().unwrap().unwrap();
+        let batch = get_first_batch(table, &projection).await?;
 
         assert_eq!(1, batch.num_columns());
         assert_eq!(8, batch.num_rows());
@@ -184,16 +181,15 @@ mod tests {
         }
 
         assert_eq!("[4, 5, 6, 7, 2, 3, 0, 1]", format!("{:?}", values));
-    }
 
-    #[test]
-    fn read_i96_alltypes_plain_parquet() {
-        let table = load_table("alltypes_plain.parquet");
+        Ok(())
+    }
 
+    #[tokio::test]
+    async fn read_i96_alltypes_plain_parquet() -> Result<()> {
+        let table = load_table("alltypes_plain.parquet")?;
         let projection = Some(vec![10]);
-        let scan = table.scan(&projection, 1024).unwrap();
-        let mut it = scan[0].lock().unwrap();
-        let batch = it.next_batch().unwrap().unwrap();
+        let batch = get_first_batch(table, &projection).await?;
 
         assert_eq!(1, batch.num_columns());
         assert_eq!(8, batch.num_rows());
@@ -209,16 +205,16 @@ mod tests {
         }
 
         assert_eq!("[1235865600000000000, 1235865660000000000, 1238544000000000000, 1238544060000000000, 1233446400000000000, 1233446460000000000, 1230768000000000000, 1230768060000000000]", format!("{:?}", values));
-    }
 
-    #[test]
-    fn read_f32_alltypes_plain_parquet() {
-        let table = load_table("alltypes_plain.parquet");
+        Ok(())
+    }
 
+    #[tokio::test]
+    async fn read_f32_alltypes_plain_parquet() -> Result<()> {
+        let table = load_table("alltypes_plain.parquet")?;
         let projection = Some(vec![6]);
-        let scan = table.scan(&projection, 1024).unwrap();
-        let mut it = scan[0].lock().unwrap();
-        let batch = it.next_batch().unwrap().unwrap();
+        let batch = get_first_batch(table, &projection).await?;
+
         assert_eq!(1, batch.num_columns());
         assert_eq!(8, batch.num_rows());
 
@@ -236,16 +232,15 @@ mod tests {
             "[0.0, 1.1, 0.0, 1.1, 0.0, 1.1, 0.0, 1.1]",
             format!("{:?}", values)
         );
-    }
 
-    #[test]
-    fn read_f64_alltypes_plain_parquet() {
-        let table = load_table("alltypes_plain.parquet");
+        Ok(())
+    }
 
+    #[tokio::test]
+    async fn read_f64_alltypes_plain_parquet() -> Result<()> {
+        let table = load_table("alltypes_plain.parquet")?;
         let projection = Some(vec![7]);
-        let scan = table.scan(&projection, 1024).unwrap();
-        let mut it = scan[0].lock().unwrap();
-        let batch = it.next_batch().unwrap().unwrap();
+        let batch = get_first_batch(table, &projection).await?;
 
         assert_eq!(1, batch.num_columns());
         assert_eq!(8, batch.num_rows());
@@ -264,16 +259,15 @@ mod tests {
             "[0.0, 10.1, 0.0, 10.1, 0.0, 10.1, 0.0, 10.1]",
             format!("{:?}", values)
         );
-    }
 
-    #[test]
-    fn read_binary_alltypes_plain_parquet() {
-        let table = load_table("alltypes_plain.parquet");
+        Ok(())
+    }
 
+    #[tokio::test]
+    async fn read_binary_alltypes_plain_parquet() -> Result<()> {
+        let table = load_table("alltypes_plain.parquet")?;
         let projection = Some(vec![9]);
-        let scan = table.scan(&projection, 1024).unwrap();
-        let mut it = scan[0].lock().unwrap();
-        let batch = it.next_batch().unwrap().unwrap();
+        let batch = get_first_batch(table, &projection).await?;
 
         assert_eq!(1, batch.num_columns());
         assert_eq!(8, batch.num_rows());
@@ -292,13 +286,27 @@ mod tests {
             "[\"0\", \"1\", \"0\", \"1\", \"0\", \"1\", \"0\", \"1\"]",
             format!("{:?}", values)
         );
+
+        Ok(())
     }
 
-    fn load_table(name: &str) -> Box<dyn TableProvider> {
+    fn load_table(name: &str) -> Result<Box<dyn TableProvider>> {
         let testdata =
             env::var("PARQUET_TEST_DATA").expect("PARQUET_TEST_DATA not defined");
         let filename = format!("{}/{}", testdata, name);
-        let table = ParquetTable::try_new(&filename).unwrap();
-        Box::new(table)
+        let table = ParquetTable::try_new(&filename)?;
+        Ok(Box::new(table))
+    }
+
+    async fn get_first_batch(
+        table: Box<dyn TableProvider>,
+        projection: &Option<Vec<usize>>,
+    ) -> Result<RecordBatch> {
+        let exec = table.scan(projection, 1024)?;
+        let mut it = exec.execute(0).await?;
+        it.next()
+            .await
+            .expect("should have received at least one batch")
+            .map_err(|e| e.into())
     }
 }
diff --git a/rust/datafusion/src/error.rs b/rust/datafusion/src/error.rs
index 6f903025dee..b4c8dcc026b 100644
--- a/rust/datafusion/src/error.rs
+++ b/rust/datafusion/src/error.rs
@@ -19,109 +19,102 @@
 
 use std::error;
 use std::fmt::{Display, Formatter};
-use std::io::Error;
+use std::io;
 use std::result;
 
 use arrow::error::ArrowError;
 use parquet::errors::ParquetError;
 use sqlparser::parser::ParserError;
 
-/// Result type for operations that could result in an `ExecutionError`
-pub type Result<T> = result::Result<T, ExecutionError>;
+/// Result type for operations that could result in an [DataFusionError]
+pub type Result<T> = result::Result<T, DataFusionError>;
 
 /// DataFusion error
 #[derive(Debug)]
 #[allow(missing_docs)]
-pub enum ExecutionError {
-    /// Wraps an error from the Arrow crate
+pub enum DataFusionError {
+    /// Error returned by arrow.
     ArrowError(ArrowError),
     /// Wraps an error from the Parquet crate
     ParquetError(ParquetError),
-    /// I/O error
-    IoError(Error),
-    /// SQL parser error
-    ParserError(ParserError),
-    /// General error
-    General(String),
-    /// Invalid column error
-    InvalidColumn(String),
-    /// Missing functionality
+    /// Error associated to I/O operations and associated traits.
+    IoError(io::Error),
+    /// Error returned when SQL is syntatically incorrect.
+    SQL(ParserError),
+    /// Error returned on a branch that we know it is possible
+    /// but to which we still have no implementation for.
+    /// Often, these errors are tracked in our issue tracker.
     NotImplemented(String),
-    /// Internal error
-    InternalError(String),
-    /// Query engine execution error
-    ExecutionError(String),
+    /// Error returned as a consequence of an error in DataFusion.
+    /// This error should not happen in normal usage of DataFusion.
+    // DataFusions has internal invariants that we are unable to ask the compiler to check for us.
+    // This error is raised when one of those invariants is not verified during execution.
+    Internal(String),
+    /// This error happens whenever a plan is not valid. Examples include
+    /// impossible casts, schema inference not possible and non-unique column names.
+    Plan(String),
+    /// Error returned during execution of the query.
+    /// Examples include files not found, errors in parsing certain types.
+    Execution(String),
 }
 
-impl ExecutionError {
-    /// Wraps this `ExecutionError` in arrow's `ExternalError` variant.
+impl DataFusionError {
+    /// Wraps this [DataFusionError] as an [Arrow::error::ArrowError].
     pub fn into_arrow_external_error(self) -> ArrowError {
         ArrowError::from_external_error(Box::new(self))
     }
 }
 
-impl From<Error> for ExecutionError {
-    fn from(e: Error) -> Self {
-        ExecutionError::IoError(e)
+impl From<io::Error> for DataFusionError {
+    fn from(e: io::Error) -> Self {
+        DataFusionError::IoError(e)
     }
 }
 
-impl From<String> for ExecutionError {
-    fn from(e: String) -> Self {
-        ExecutionError::General(e)
-    }
-}
-
-impl From<&'static str> for ExecutionError {
-    fn from(e: &'static str) -> Self {
-        ExecutionError::General(e.to_string())
-    }
-}
-
-impl From<ArrowError> for ExecutionError {
+impl From<ArrowError> for DataFusionError {
     fn from(e: ArrowError) -> Self {
-        ExecutionError::ArrowError(e)
+        DataFusionError::ArrowError(e)
     }
 }
 
-impl From<ParquetError> for ExecutionError {
+impl From<ParquetError> for DataFusionError {
     fn from(e: ParquetError) -> Self {
-        ExecutionError::ParquetError(e)
+        DataFusionError::ParquetError(e)
     }
 }
 
-impl From<ParserError> for ExecutionError {
+impl From<ParserError> for DataFusionError {
     fn from(e: ParserError) -> Self {
-        ExecutionError::ParserError(e)
+        DataFusionError::SQL(e)
     }
 }
 
-impl Display for ExecutionError {
+impl Display for DataFusionError {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match *self {
-            ExecutionError::ArrowError(ref desc) => write!(f, "Arrow error: {}", desc),
-            ExecutionError::ParquetError(ref desc) => {
+            DataFusionError::ArrowError(ref desc) => write!(f, "Arrow error: {}", desc),
+            DataFusionError::ParquetError(ref desc) => {
                 write!(f, "Parquet error: {}", desc)
             }
-            ExecutionError::IoError(ref desc) => write!(f, "IO error: {}", desc),
-            ExecutionError::ParserError(ref desc) => {
-                write!(f, "Parser error: {:?}", desc)
+            DataFusionError::IoError(ref desc) => write!(f, "IO error: {}", desc),
+            DataFusionError::SQL(ref desc) => {
+                write!(f, "SQL error: {:?}", desc)
             }
-            ExecutionError::General(ref desc) => write!(f, "General error: {}", desc),
-            ExecutionError::InvalidColumn(ref desc) => {
-                write!(f, "Invalid column error: {}", desc)
+            DataFusionError::NotImplemented(ref desc) => {
+                write!(f, "This feature is not implemented: {}", desc)
             }
-            ExecutionError::NotImplemented(ref desc) => {
-                write!(f, "NotImplemented: {}", desc)
+            DataFusionError::Internal(ref desc) => {
+                write!(f, "Internal error: {}. This was likely caused by a bug in DataFusion's \
+                    code and we would welcome that you file an bug report in our issue tracker", desc)
             }
-            ExecutionError::InternalError(ref desc) => {
-                write!(f, "Internal error: {}", desc)
+            DataFusionError::Plan(ref desc) => {
+                write!(f, "Error during planning: {}", desc)
             }
-            ExecutionError::ExecutionError(ref desc) => {
+            DataFusionError::Execution(ref desc) => {
                 write!(f, "Execution error: {}", desc)
             }
         }
     }
 }
 
-impl error::Error for ExecutionError {}
+impl error::Error for DataFusionError {}
diff --git a/rust/datafusion/src/execution/context.rs b/rust/datafusion/src/execution/context.rs
index dcbfe7bd257..af10a18fb24 100644
--- a/rust/datafusion/src/execution/context.rs
+++ b/rust/datafusion/src/execution/context.rs
@@ -22,7 +22,8 @@ use std::fs;
 use std::path::Path;
 use std::string::String;
 use std::sync::Arc;
-use std::thread::{self, JoinHandle};
+
+use futures::{StreamExt, TryStreamExt};
 
 use arrow::csv;
 use arrow::datatypes::*;
@@ -31,78 +32,100 @@ use arrow::record_batch::RecordBatch;
 use crate::datasource::csv::CsvFile;
 use crate::datasource::parquet::ParquetTable;
 use crate::datasource::TableProvider;
-use crate::error::{ExecutionError, Result};
-use crate::execution::physical_plan::common;
-use crate::execution::physical_plan::csv::{CsvExec, CsvReadOptions};
-use crate::execution::physical_plan::datasource::DatasourceExec;
-use crate::execution::physical_plan::expressions::{
-    Avg, BinaryExpr, CastExpr, Column, Count, Literal, Max, Min, PhysicalSortExpr, Sum,
-};
-use crate::execution::physical_plan::hash_aggregate::HashAggregateExec;
-use crate::execution::physical_plan::limit::LimitExec;
-use crate::execution::physical_plan::math_expressions::register_math_functions;
-use crate::execution::physical_plan::memory::MemoryExec;
-use crate::execution::physical_plan::merge::MergeExec;
-use crate::execution::physical_plan::parquet::ParquetExec;
-use crate::execution::physical_plan::projection::ProjectionExec;
-use crate::execution::physical_plan::selection::SelectionExec;
-use crate::execution::physical_plan::sort::{SortExec, SortOptions};
-use crate::execution::physical_plan::udf::{ScalarFunction, ScalarFunctionExpr};
-use crate::execution::physical_plan::{AggregateExpr, ExecutionPlan, PhysicalExpr};
-use crate::execution::table_impl::TableImpl;
-use crate::logicalplan::{
-    Expr, FunctionMeta, FunctionType, LogicalPlan, LogicalPlanBuilder,
+use crate::error::{DataFusionError, Result};
+use crate::execution::dataframe_impl::DataFrameImpl;
+use crate::logical_plan::{
+    FunctionRegistry, LogicalPlan, LogicalPlanBuilder, TableSource,
 };
+use crate::optimizer::filter_push_down::FilterPushDown;
 use crate::optimizer::optimizer::OptimizerRule;
 use crate::optimizer::projection_push_down::ProjectionPushDown;
-use crate::optimizer::type_coercion::TypeCoercionRule;
+use crate::physical_plan::common;
+use crate::physical_plan::csv::CsvReadOptions;
+use crate::physical_plan::merge::MergeExec;
+use crate::physical_plan::planner::DefaultPhysicalPlanner;
+use crate::physical_plan::udf::ScalarUDF;
+use crate::physical_plan::ExecutionPlan;
+use crate::physical_plan::PhysicalPlanner;
 use crate::sql::{
     parser::{DFParser, FileType},
     planner::{SchemaProvider, SqlToRel},
 };
-use crate::table::Table;
-
-/// Execution context for registering data sources and executing queries
+use crate::variable::{VarProvider, VarType};
+use crate::{dataframe::DataFrame, physical_plan::udaf::AggregateUDF};
+
+/// ExecutionContext is the main interface for executing queries with DataFusion. The context
+/// provides the following functionality:
+///
+/// * Create DataFrame from a CSV or Parquet data source.
+/// * Register a CSV or Parquet data source as a table that can be referenced from a SQL query.
+/// * Register a custom data source that can be referenced from a SQL query.
+/// * Execution a SQL query
+///
+/// The following example demonstrates how to use the context to execute a query against a CSV
+/// data source using the DataFrame API:
+///
+/// ```
+/// use datafusion::prelude::*;
+/// # use datafusion::error::Result;
+/// # fn main() -> Result<()> {
+/// let mut ctx = ExecutionContext::new();
+/// let df = ctx.read_csv("tests/example.csv", CsvReadOptions::new())?;
+/// let df = df.filter(col("a").lt_eq(col("b")))?
+///            .aggregate(vec![col("a")], vec![min(col("b"))])?
+///            .limit(100)?;
+/// let results = df.collect();
+/// # Ok(())
+/// # }
+/// ```
+///
+/// The following example demonstrates how to execute the same query using SQL:
+///
+/// ```
+/// use datafusion::prelude::*;
+///
+/// # use datafusion::error::Result;
+/// # fn main() -> Result<()> {
+/// let mut ctx = ExecutionContext::new();
+/// ctx.register_csv("example", "tests/example.csv", CsvReadOptions::new())?;
+/// let results = ctx.sql("SELECT a, MIN(b) FROM example GROUP BY a LIMIT 100")?;
+/// # Ok(())
+/// # }
+/// ```
 pub struct ExecutionContext {
-    datasources: HashMap<String, Box<dyn TableProvider + Send + Sync>>,
-    scalar_functions: HashMap<String, Box<ScalarFunction>>,
-}
-
-fn tuple_err<T, R>(value: (Result<T>, Result<R>)) -> Result<(T, R)> {
-    match value {
-        (Ok(e), Ok(e1)) => Ok((e, e1)),
-        (Err(e), Ok(_)) => Err(e),
-        (Ok(_), Err(e1)) => Err(e1),
-        (Err(e), Err(_)) => Err(e),
-    }
+    /// Internal state for the context
+    pub state: ExecutionContextState,
 }
 
 impl ExecutionContext {
-    /// Create a new execution context for in-memory queries
+    /// Create a new execution context using a default configuration.
     pub fn new() -> Self {
-        let mut ctx = Self {
-            datasources: HashMap::new(),
-            scalar_functions: HashMap::new(),
+        Self::with_config(ExecutionConfig::new())
+    }
+
+    /// Create a new execution context using the provided configuration
+    pub fn with_config(config: ExecutionConfig) -> Self {
+        let ctx = Self {
+            state: ExecutionContextState {
+                datasources: HashMap::new(),
+                scalar_functions: HashMap::new(),
+                var_provider: HashMap::new(),
+                aggregate_functions: HashMap::new(),
+                config,
+            },
         };
-        register_math_functions(&mut ctx);
         ctx
     }
 
-    /// Execute a SQL query and produce a Relation (a schema-aware iterator over a series
-    /// of RecordBatch instances)
-    pub fn sql(&mut self, sql: &str, batch_size: usize) -> Result<Vec<RecordBatch>> {
-        let plan = self.create_logical_plan(sql)?;
-
-        return self.collect_plan(&plan, batch_size);
+    /// Get the configuration of this execution context
+    pub fn config(&self) -> &ExecutionConfig {
+        &self.state.config
     }
 
-    /// Executes a logical plan and produce a Relation (a schema-aware iterator over a series
+    /// Execute a SQL query and produce a Relation (a schema-aware iterator over a series
     /// of RecordBatch instances)
-    pub fn collect_plan(
-        &mut self,
-        plan: &LogicalPlan,
-        batch_size: usize,
-    ) -> Result<Vec<RecordBatch>> {
+    pub fn sql(&mut self, sql: &str) -> Result<Arc<dyn DataFrame>> {
+        let plan = self.create_logical_plan(sql)?;
         match plan {
             LogicalPlan::CreateExternalTable {
                 ref schema,
@@ -119,57 +142,124 @@ impl ExecutionContext {
                             .schema(&schema)
                             .has_header(*has_header),
                     )?;
-                    Ok(vec![])
+                    let plan = LogicalPlanBuilder::empty().build()?;
+                    Ok(Arc::new(DataFrameImpl::new(self.state.clone(), &plan)))
                 }
                 FileType::Parquet => {
                     self.register_parquet(name, location)?;
-                    Ok(vec![])
+                    let plan = LogicalPlanBuilder::empty().build()?;
+                    Ok(Arc::new(DataFrameImpl::new(self.state.clone(), &plan)))
                 }
-                _ => Err(ExecutionError::ExecutionError(format!(
+                _ => Err(DataFusionError::NotImplemented(format!(
                     "Unsupported file type {:?}.",
                     file_type
                 ))),
             },
 
-            plan => {
-                let plan = self.optimize(&plan)?;
-                let plan = self.create_physical_plan(&plan, batch_size)?;
-                Ok(self.collect(plan.as_ref())?)
-            }
+            plan => Ok(Arc::new(DataFrameImpl::new(self.state.clone(), &plan))),
         }
     }
 
-    /// Creates a logical plan
-    pub fn create_logical_plan(&mut self, sql: &str) -> Result<LogicalPlan> {
+    /// Creates a logical plan. This function is intended for internal use and should not be
+    /// called directly.
+    pub fn create_logical_plan(&self, sql: &str) -> Result<LogicalPlan> {
         let statements = DFParser::parse_sql(sql)?;
 
         if statements.len() != 1 {
-            return Err(ExecutionError::NotImplemented(format!(
+            return Err(DataFusionError::NotImplemented(format!(
                 "The context currently only supports a single SQL statement",
             )));
         }
 
-        let schema_provider = ExecutionContextSchemaProvider {
-            datasources: &self.datasources,
-            scalar_functions: &self.scalar_functions,
-        };
-
         // create a query planner
-        let query_planner = SqlToRel::new(schema_provider);
+        let query_planner = SqlToRel::new(&self.state);
         Ok(query_planner.statement_to_plan(&statements[0])?)
     }
 
+    /// Register variable
+    pub fn register_variable(
+        &mut self,
+        variable_type: VarType,
+        provider: Arc<dyn VarProvider + Send + Sync>,
+    ) {
+        self.state.var_provider.insert(variable_type, provider);
+    }
+
     /// Register a scalar UDF
-    pub fn register_udf(&mut self, f: ScalarFunction) {
-        self.scalar_functions.insert(f.name.clone(), Box::new(f));
+    pub fn register_udf(&mut self, f: ScalarUDF) {
+        self.state
+            .scalar_functions
+            .insert(f.name.clone(), Arc::new(f));
+    }
+
+    /// Register a aggregate UDF
+    pub fn register_udaf(&mut self, f: AggregateUDF) {
+        self.state
+            .aggregate_functions
+            .insert(f.name.clone(), Arc::new(f));
+    }
+
+    /// Creates a DataFrame for reading a CSV data source.
+    pub fn read_csv(
+        &mut self,
+        filename: &str,
+        options: CsvReadOptions,
+    ) -> Result<Arc<dyn DataFrame>> {
+        let csv = CsvFile::try_new(filename, options)?;
+
+        let table_scan = LogicalPlan::CsvScan {
+            path: filename.to_string(),
+            schema: csv.schema().clone(),
+            has_header: options.has_header,
+            delimiter: Some(options.delimiter),
+            projection: None,
+            projected_schema: csv.schema().clone(),
+        };
+
+        Ok(Arc::new(DataFrameImpl::new(
+            self.state.clone(),
+            &LogicalPlanBuilder::from(&table_scan).build()?,
+        )))
+    }
+
+    /// Creates a DataFrame for reading a Parquet data source.
+    pub fn read_parquet(&mut self, filename: &str) -> Result<Arc<dyn DataFrame>> {
+        let parquet = ParquetTable::try_new(filename)?;
+
+        let table_scan = LogicalPlan::ParquetScan {
+            path: filename.to_string(),
+            schema: parquet.schema().clone(),
+            projection: None,
+            projected_schema: parquet.schema().clone(),
+        };
+
+        Ok(Arc::new(DataFrameImpl::new(
+            self.state.clone(),
+            &LogicalPlanBuilder::from(&table_scan).build()?,
+        )))
     }
 
-    /// Get a reference to the registered scalar functions
-    pub fn scalar_functions(&self) -> &HashMap<String, Box<ScalarFunction>> {
-        &self.scalar_functions
+    /// Creates a DataFrame for reading a custom TableProvider
+    pub fn read_table(
+        &mut self,
+        provider: Arc<dyn TableProvider + Send + Sync>,
+    ) -> Result<Arc<dyn DataFrame>> {
+        let schema = provider.schema().clone();
+        let table_scan = LogicalPlan::TableScan {
+            schema_name: "".to_string(),
+            source: TableSource::FromProvider(provider),
+            table_schema: schema.clone(),
+            projected_schema: schema,
+            projection: None,
+        };
+        Ok(Arc::new(DataFrameImpl::new(
+            self.state.clone(),
+            &LogicalPlanBuilder::from(&table_scan).build()?,
+        )))
     }
 
-    /// Register a CSV file as a table so that it can be queried from SQL
+    /// Register a CSV data source so that it can be referenced from SQL statements
+    /// executed against this context.
     pub fn register_csv(
         &mut self,
         name: &str,
@@ -180,39 +270,46 @@ impl ExecutionContext {
         Ok(())
     }
 
-    /// Register a Parquet file as a table so that it can be queried from SQL
+    /// Register a Parquet data source so that it can be referenced from SQL statements
+    /// executed against this context.
     pub fn register_parquet(&mut self, name: &str, filename: &str) -> Result<()> {
         let table = ParquetTable::try_new(&filename)?;
         self.register_table(name, Box::new(table));
         Ok(())
     }
 
-    /// Register a table so that it can be queried from SQL
+    /// Register a table using a custom TableProvider so that it can be referenced from SQL
+    /// statements executed against this context.
     pub fn register_table(
         &mut self,
         name: &str,
         provider: Box<dyn TableProvider + Send + Sync>,
     ) {
-        self.datasources.insert(name.to_string(), provider);
+        self.state
+            .datasources
+            .insert(name.to_string(), provider.into());
     }
 
-    /// Get a table by name
-    pub fn table(&mut self, table_name: &str) -> Result<Arc<dyn Table>> {
-        match self.datasources.get(table_name) {
+    /// Retrieves a DataFrame representing a table previously registered by calling the
+    /// register_table function. An Err result will be returned if no table has been
+    /// registered with the provided name.
+    pub fn table(&mut self, table_name: &str) -> Result<Arc<dyn DataFrame>> {
+        match self.state.datasources.get(table_name) {
             Some(provider) => {
-                let schema = provider.schema().as_ref().clone();
+                let schema = provider.schema().clone();
                 let table_scan = LogicalPlan::TableScan {
                     schema_name: "".to_string(),
-                    table_name: table_name.to_string(),
-                    table_schema: Box::new(schema.to_owned()),
-                    projected_schema: Box::new(schema),
+                    source: TableSource::FromContext(table_name.to_string()),
+                    table_schema: schema.clone(),
+                    projected_schema: schema,
                     projection: None,
                 };
-                Ok(Arc::new(TableImpl::new(
+                Ok(Arc::new(DataFrameImpl::new(
+                    self.state.clone(),
                     &LogicalPlanBuilder::from(&table_scan).build()?,
                 )))
             }
-            _ => Err(ExecutionError::General(format!(
+            _ => Err(DataFusionError::Plan(format!(
                 "No table named '{}'",
                 table_name
             ))),
@@ -221,422 +318,228 @@ impl ExecutionContext {
 
     /// The set of available tables. Use `table` to get a specific table.
     pub fn tables(&self) -> HashSet<String> {
-        self.datasources.keys().cloned().collect()
+        self.state.datasources.keys().cloned().collect()
     }
 
     /// Optimize the logical plan by applying optimizer rules
     pub fn optimize(&self, plan: &LogicalPlan) -> Result<LogicalPlan> {
-        let rules: Vec<Box<dyn OptimizerRule>> = vec![
-            Box::new(ProjectionPushDown::new()),
-            Box::new(TypeCoercionRule::new(&self.scalar_functions)),
-        ];
-        let mut plan = plan.clone();
+        // Apply standard rewrites and optimizations
+        let mut plan = ProjectionPushDown::new().optimize(&plan)?;
+        plan = FilterPushDown::new().optimize(&plan)?;
 
-        for mut rule in rules {
-            plan = rule.optimize(&plan)?;
-        }
-        Ok(plan)
+        self.state.config.query_planner.rewrite_logical_plan(plan)
     }
 
     /// Create a physical plan from a logical plan
     pub fn create_physical_plan(
         &self,
         logical_plan: &LogicalPlan,
-        batch_size: usize,
     ) -> Result<Arc<dyn ExecutionPlan>> {
-        match logical_plan {
-            LogicalPlan::TableScan {
-                table_name,
-                projection,
-                ..
-            } => match self.datasources.get(table_name) {
-                Some(provider) => {
-                    let partitions = provider.scan(projection, batch_size)?;
-                    if partitions.is_empty() {
-                        Err(ExecutionError::General(
-                            "Table provider returned no partitions".to_string(),
-                        ))
-                    } else {
-                        let partition = partitions[0].lock().unwrap();
-                        let schema = partition.schema();
-                        let exec =
-                            DatasourceExec::new(schema.clone(), partitions.clone());
-                        Ok(Arc::new(exec))
-                    }
-                }
-                _ => Err(ExecutionError::General(format!(
-                    "No table named {}",
-                    table_name
-                ))),
-            },
-            LogicalPlan::InMemoryScan {
-                data,
-                projection,
-                projected_schema,
-                ..
-            } => Ok(Arc::new(MemoryExec::try_new(
-                data,
-                Arc::new(projected_schema.as_ref().to_owned()),
-                projection.to_owned(),
-            )?)),
-            LogicalPlan::CsvScan {
-                path,
-                schema,
-                has_header,
-                delimiter,
-                projection,
-                ..
-            } => Ok(Arc::new(CsvExec::try_new(
-                path,
-                CsvReadOptions::new()
-                    .schema(schema.as_ref())
-                    .delimiter_option(*delimiter)
-                    .has_header(*has_header),
-                projection.to_owned(),
-                batch_size,
-            )?)),
-            LogicalPlan::ParquetScan {
-                path, projection, ..
-            } => Ok(Arc::new(ParquetExec::try_new(
-                path,
-                projection.to_owned(),
-                batch_size,
-            )?)),
-            LogicalPlan::Projection { input, expr, .. } => {
-                let input = self.create_physical_plan(input, batch_size)?;
-                let input_schema = input.as_ref().schema().clone();
-                let runtime_expr = expr
-                    .iter()
-                    .map(|e| {
-                        tuple_err((
-                            self.create_physical_expr(e, &input_schema),
-                            e.name(&input_schema),
-                        ))
-                    })
-                    .collect::<Result<Vec<_>>>()?;
-                Ok(Arc::new(ProjectionExec::try_new(runtime_expr, input)?))
-            }
-            LogicalPlan::Aggregate {
-                input,
-                group_expr,
-                aggr_expr,
-                ..
-            } => {
-                // Initially need to perform the aggregate and then merge the partitions
-                let input = self.create_physical_plan(input, batch_size)?;
-                let input_schema = input.as_ref().schema().clone();
-
-                let groups = group_expr
-                    .iter()
-                    .map(|e| {
-                        tuple_err((
-                            self.create_physical_expr(e, &input_schema),
-                            e.name(&input_schema),
-                        ))
-                    })
-                    .collect::<Result<Vec<_>>>()?;
-                let aggregates = aggr_expr
-                    .iter()
-                    .map(|e| {
-                        tuple_err((
-                            self.create_aggregate_expr(e, &input_schema),
-                            e.name(&input_schema),
-                        ))
-                    })
-                    .collect::<Result<Vec<_>>>()?;
-
-                let initial_aggr = HashAggregateExec::try_new(
-                    groups.clone(),
-                    aggregates.clone(),
-                    input,
-                )?;
-
-                let schema = initial_aggr.schema();
-                let partitions = initial_aggr.partitions()?;
-
-                if partitions.len() == 1 {
-                    return Ok(Arc::new(initial_aggr));
-                }
-
-                let merge = Arc::new(MergeExec::new(schema.clone(), partitions));
-
-                // construct the expressions for the final aggregation
-                let (final_group, final_aggr) = initial_aggr.make_final_expr(
-                    groups.iter().map(|x| x.1.clone()).collect(),
-                    aggregates.iter().map(|x| x.1.clone()).collect(),
-                );
-
-                // construct a second aggregation, keeping the final column name equal to the first aggregation
-                // and the expressions corresponding to the respective aggregate
-                Ok(Arc::new(HashAggregateExec::try_new(
-                    final_group
-                        .iter()
-                        .enumerate()
-                        .map(|(i, expr)| (expr.clone(), groups[i].1.clone()))
-                        .collect(),
-                    final_aggr
-                        .iter()
-                        .enumerate()
-                        .map(|(i, expr)| (expr.clone(), aggregates[i].1.clone()))
-                        .collect(),
-                    merge,
-                )?))
-            }
-            LogicalPlan::Selection { input, expr, .. } => {
-                let input = self.create_physical_plan(input, batch_size)?;
-                let input_schema = input.as_ref().schema().clone();
-                let runtime_expr = self.create_physical_expr(expr, &input_schema)?;
-                Ok(Arc::new(SelectionExec::try_new(runtime_expr, input)?))
-            }
-            LogicalPlan::Sort { expr, input, .. } => {
-                let input = self.create_physical_plan(input, batch_size)?;
-                let input_schema = input.as_ref().schema().clone();
-
-                let sort_expr = expr
-                    .iter()
-                    .map(|e| match e {
-                        Expr::Sort {
-                            expr,
-                            asc,
-                            nulls_first,
-                        } => self.create_physical_sort_expr(
-                            expr,
-                            &input_schema,
-                            SortOptions {
-                                descending: !*asc,
-                                nulls_first: *nulls_first,
-                            },
-                        ),
-                        _ => Err(ExecutionError::ExecutionError(
-                            "Sort only accepts sort expressions".to_string(),
-                        )),
-                    })
-                    .collect::<Result<Vec<_>>>()?;
-
-                Ok(Arc::new(SortExec::try_new(sort_expr, input)?))
-            }
-            LogicalPlan::Limit { input, n, .. } => {
-                let input = self.create_physical_plan(input, batch_size)?;
-                let input_schema = input.as_ref().schema().clone();
-
-                Ok(Arc::new(LimitExec::new(
-                    input_schema.clone(),
-                    input.partitions()?,
-                    *n,
-                )))
-            }
-            _ => Err(ExecutionError::General(
-                "Unsupported logical plan variant".to_string(),
-            )),
-        }
-    }
-
-    /// Create a physical expression from a logical expression
-    pub fn create_physical_expr(
-        &self,
-        e: &Expr,
-        input_schema: &Schema,
-    ) -> Result<Arc<dyn PhysicalExpr>> {
-        match e {
-            Expr::Alias(expr, ..) => Ok(self.create_physical_expr(expr, input_schema)?),
-            Expr::Column(name) => {
-                // check that name exists
-                input_schema.field_with_name(&name)?;
-                Ok(Arc::new(Column::new(name)))
-            }
-            Expr::Literal(value) => Ok(Arc::new(Literal::new(value.clone()))),
-            Expr::BinaryExpr { left, op, right } => Ok(Arc::new(BinaryExpr::new(
-                self.create_physical_expr(left, input_schema)?,
-                op.clone(),
-                self.create_physical_expr(right, input_schema)?,
-            ))),
-            Expr::Cast { expr, data_type } => Ok(Arc::new(CastExpr::try_new(
-                self.create_physical_expr(expr, input_schema)?,
-                input_schema,
-                data_type.clone(),
-            )?)),
-            Expr::ScalarFunction {
-                name,
-                args,
-                return_type,
-            } => match &self.scalar_functions.get(name) {
-                Some(f) => {
-                    let mut physical_args = vec![];
-                    for e in args {
-                        physical_args.push(self.create_physical_expr(e, input_schema)?);
-                    }
-                    Ok(Arc::new(ScalarFunctionExpr::new(
-                        name,
-                        Box::new(f.fun.clone()),
-                        physical_args,
-                        return_type,
-                    )))
-                }
-                _ => Err(ExecutionError::General(format!(
-                    "Invalid scalar function '{:?}'",
-                    name
-                ))),
-            },
-            other => Err(ExecutionError::NotImplemented(format!(
-                "Physical plan does not support logical expression {:?}",
-                other
-            ))),
-        }
-    }
-
-    /// Create an aggregate expression from a logical expression
-    pub fn create_aggregate_expr(
-        &self,
-        e: &Expr,
-        input_schema: &Schema,
-    ) -> Result<Arc<dyn AggregateExpr>> {
-        match e {
-            Expr::AggregateFunction { name, args, .. } => {
-                match name.to_lowercase().as_ref() {
-                    "sum" => Ok(Arc::new(Sum::new(
-                        self.create_physical_expr(&args[0], input_schema)?,
-                    ))),
-                    "avg" => Ok(Arc::new(Avg::new(
-                        self.create_physical_expr(&args[0], input_schema)?,
-                    ))),
-                    "max" => Ok(Arc::new(Max::new(
-                        self.create_physical_expr(&args[0], input_schema)?,
-                    ))),
-                    "min" => Ok(Arc::new(Min::new(
-                        self.create_physical_expr(&args[0], input_schema)?,
-                    ))),
-                    "count" => Ok(Arc::new(Count::new(
-                        self.create_physical_expr(&args[0], input_schema)?,
-                    ))),
-                    other => Err(ExecutionError::NotImplemented(format!(
-                        "Unsupported aggregate function '{}'",
-                        other
-                    ))),
-                }
-            }
-            other => Err(ExecutionError::General(format!(
-                "Invalid aggregate expression '{:?}'",
-                other
-            ))),
-        }
-    }
-
-    /// Create an aggregate expression from a logical expression
-    pub fn create_physical_sort_expr(
-        &self,
-        e: &Expr,
-        input_schema: &Schema,
-        options: SortOptions,
-    ) -> Result<PhysicalSortExpr> {
-        Ok(PhysicalSortExpr {
-            expr: self.create_physical_expr(e, input_schema)?,
-            options: options,
-        })
+        self.state
+            .config
+            .query_planner
+            .create_physical_plan(logical_plan, &self.state)
     }
 
     /// Execute a physical plan and collect the results in memory
-    pub fn collect(&self, plan: &dyn ExecutionPlan) -> Result<Vec<RecordBatch>> {
-        let partitions = plan.partitions()?;
-
-        match partitions.len() {
+    pub async fn collect(
+        &self,
+        plan: Arc<dyn ExecutionPlan>,
+    ) -> Result<Vec<RecordBatch>> {
+        match plan.output_partitioning().partition_count() {
             0 => Ok(vec![]),
             1 => {
-                let it = partitions[0].execute()?;
-                common::collect(it)
+                let it = plan.execute(0).await?;
+                common::collect(it).await
             }
             _ => {
                 // merge into a single partition
-                let plan = MergeExec::new(plan.schema().clone(), partitions);
-                let partitions = plan.partitions()?;
-                if partitions.len() == 1 {
-                    common::collect(partitions[0].execute()?)
-                } else {
-                    Err(ExecutionError::InternalError(format!(
-                        "MergeExec returned {} partitions",
-                        partitions.len()
-                    )))
-                }
+                let plan = MergeExec::new(plan.clone());
+                // MergeExec must produce a single partition
+                assert_eq!(1, plan.output_partitioning().partition_count());
+                common::collect(plan.execute(0).await?).await
             }
         }
     }
 
     /// Execute a query and write the results to a partitioned CSV file
-    pub fn write_csv(&self, plan: &dyn ExecutionPlan, path: &str) -> Result<()> {
+    pub async fn write_csv(
+        &self,
+        plan: Arc<dyn ExecutionPlan>,
+        path: String,
+    ) -> Result<()> {
         // create directory to contain the CSV files (one per partition)
-        let path = path.to_string();
+        let path = path.to_owned();
         fs::create_dir(&path)?;
 
-        let threads: Vec<JoinHandle<Result<()>>> = plan
-            .partitions()?
-            .iter()
-            .enumerate()
-            .map(|(i, p)| {
-                let p = p.clone();
-                let path = path.clone();
-                thread::spawn(move || {
-                    let filename = format!("part-{}.csv", i);
-                    let path = Path::new(&path).join(&filename);
-                    let file = fs::File::create(path)?;
-                    let mut writer = csv::Writer::new(file);
-                    let reader = p.execute()?;
-                    let mut reader = reader.lock().unwrap();
-                    loop {
-                        match reader.next_batch() {
-                            Ok(Some(batch)) => {
-                                writer.write(&batch)?;
-                            }
-                            Ok(None) => break,
-                            Err(e) => return Err(ExecutionError::from(e)),
-                        }
-                    }
-                    Ok(())
-                })
-            })
-            .collect();
-
-        // combine the results from each thread
-        for thread in threads {
-            let join = thread.join().expect("Failed to join thread");
-            join?;
+        for i in 0..plan.output_partitioning().partition_count() {
+            let path = path.clone();
+            let plan = plan.clone();
+            let filename = format!("part-{}.csv", i);
+            let path = Path::new(&path).join(&filename);
+            let file = fs::File::create(path)?;
+            let mut writer = csv::Writer::new(file);
+            let stream = plan.execute(i).await?;
+
+            stream
+                .map(|batch| writer.write(&batch?))
+                .try_collect()
+                .await
+                .map_err(|e| DataFusionError::from(e))?;
         }
-
         Ok(())
     }
+
+    /// get the registry, that allows to construct logical expressions of UDFs
+    pub fn registry(&self) -> &dyn FunctionRegistry {
+        &self.state
+    }
 }
 
-/// Get schema and scalar functions for execution context
-pub struct ExecutionContextSchemaProvider<'a> {
-    datasources: &'a HashMap<String, Box<dyn TableProvider + Send + Sync>>,
-    scalar_functions: &'a HashMap<String, Box<ScalarFunction>>,
+impl From<ExecutionContextState> for ExecutionContext {
+    fn from(state: ExecutionContextState) -> Self {
+        ExecutionContext { state }
+    }
 }
 
-impl<'a> ExecutionContextSchemaProvider<'a> {
-    /// Create a new ExecutionContextSchemaProvider based on data sources and scalar functions
-    pub fn new(
-        datasources: &'a HashMap<String, Box<dyn TableProvider + Send + Sync>>,
-        scalar_functions: &'a HashMap<String, Box<ScalarFunction>>,
-    ) -> Self {
-        ExecutionContextSchemaProvider {
-            datasources,
-            scalar_functions,
+/// A planner used to add extensions to DataFusion logical and physical plans.
+pub trait QueryPlanner {
+    /// Given a `LogicalPlan`, create a new, modified `LogicalPlan`
+    /// plan. This method is run after built in `OptimizerRule`s. By
+    /// default returns the `plan` unmodified.
+    fn rewrite_logical_plan(&self, plan: LogicalPlan) -> Result<LogicalPlan> {
+        Ok(plan)
+    }
+
+    /// Given a `LogicalPlan`, create an `ExecutionPlan` suitable for execution
+    fn create_physical_plan(
+        &self,
+        logical_plan: &LogicalPlan,
+        ctx_state: &ExecutionContextState,
+    ) -> Result<Arc<dyn ExecutionPlan>>;
+}
+
+/// The query planner used if no user defined planner is provided
+struct DefaultQueryPlanner {}
+
+impl QueryPlanner for DefaultQueryPlanner {
+    /// Given a `LogicalPlan`, create an `ExecutionPlan` suitable for execution
+    fn create_physical_plan(
+        &self,
+        logical_plan: &LogicalPlan,
+        ctx_state: &ExecutionContextState,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        let planner = DefaultPhysicalPlanner::default();
+        planner.create_physical_plan(logical_plan, ctx_state)
+    }
+}
+
+/// Configuration options for execution context
+#[derive(Clone)]
+pub struct ExecutionConfig {
+    /// Number of concurrent threads for query execution.
+    pub concurrency: usize,
+    /// Default batch size when reading data sources
+    pub batch_size: usize,
+    /// Responsible for planning `LogicalPlan`s, and `ExecutionPlan`
+    query_planner: Arc<dyn QueryPlanner + Send + Sync>,
+}
+
+impl ExecutionConfig {
+    /// Create an execution config with default setting
+    pub fn new() -> Self {
+        Self {
+            concurrency: num_cpus::get(),
+            batch_size: 4096,
+            query_planner: Arc::new(DefaultQueryPlanner {}),
         }
     }
+
+    /// Customize max_concurrency
+    pub fn with_concurrency(mut self, n: usize) -> Self {
+        // concurrency must be greater than zero
+        assert!(n > 0);
+        self.concurrency = n;
+        self
+    }
+
+    /// Customize batch size
+    pub fn with_batch_size(mut self, n: usize) -> Self {
+        // batch size must be greater than zero
+        assert!(n > 0);
+        self.batch_size = n;
+        self
+    }
+
+    /// Replace the default query planner
+    pub fn with_query_planner(
+        mut self,
+        query_planner: Arc<dyn QueryPlanner + Send + Sync>,
+    ) -> Self {
+        self.query_planner = query_planner;
+        self
+    }
+}
+
+/// Execution context for registering data sources and executing queries
+#[derive(Clone)]
+pub struct ExecutionContextState {
+    /// Data sources that are registered with the context
+    pub datasources: HashMap<String, Arc<dyn TableProvider + Send + Sync>>,
+    /// Scalar functions that are registered with the context
+    pub scalar_functions: HashMap<String, Arc<ScalarUDF>>,
+    /// Variable provider that are registered with the context
+    pub var_provider: HashMap<VarType, Arc<dyn VarProvider + Send + Sync>>,
+    /// Aggregate functions registered in the context
+    pub aggregate_functions: HashMap<String, Arc<AggregateUDF>>,
+    /// Context configuration
+    pub config: ExecutionConfig,
 }
 
-impl SchemaProvider for ExecutionContextSchemaProvider<'_> {
+impl SchemaProvider for ExecutionContextState {
     fn get_table_meta(&self, name: &str) -> Option<SchemaRef> {
         self.datasources.get(name).map(|ds| ds.schema().clone())
     }
 
-    fn get_function_meta(&self, name: &str) -> Option<Arc<FunctionMeta>> {
-        self.scalar_functions.get(name).map(|f| {
-            Arc::new(FunctionMeta::new(
-                name.to_owned(),
-                f.args.clone(),
-                f.return_type.clone(),
-                FunctionType::Scalar,
+    fn get_function_meta(&self, name: &str) -> Option<Arc<ScalarUDF>> {
+        self.scalar_functions
+            .get(name)
+            .and_then(|func| Some(func.clone()))
+    }
+
+    fn get_aggregate_meta(&self, name: &str) -> Option<Arc<AggregateUDF>> {
+        self.aggregate_functions
+            .get(name)
+            .and_then(|func| Some(func.clone()))
+    }
+}
+
+impl FunctionRegistry for ExecutionContextState {
+    fn udfs(&self) -> HashSet<String> {
+        self.scalar_functions.keys().cloned().collect()
+    }
+
+    fn udf(&self, name: &str) -> Result<&ScalarUDF> {
+        let result = self.scalar_functions.get(name);
+        if result.is_none() {
+            Err(DataFusionError::Plan(
+                format!("There is no UDF named \"{}\" in the registry", name).to_string(),
             ))
-        })
+        } else {
+            Ok(result.unwrap())
+        }
+    }
+
+    fn udaf(&self, name: &str) -> Result<&AggregateUDF> {
+        let result = self.aggregate_functions.get(name);
+        if result.is_none() {
+            Err(DataFusionError::Plan(
+                format!("There is no UDAF named \"{}\" in the registry", name)
+                    .to_string(),
+            ))
+        } else {
+            Ok(result.unwrap())
+        }
     }
 }
 
@@ -644,21 +547,26 @@ impl SchemaProvider for ExecutionContextSchemaProvider<'_> {
 mod tests {
 
     use super::*;
-    use crate::datasource::MemTable;
-    use crate::execution::physical_plan::udf::ScalarUdf;
-    use crate::logicalplan::{aggregate_expr, col, scalar_function};
+    use crate::logical_plan::{col, create_udf, sum};
+    use crate::physical_plan::functions::ScalarFunctionImplementation;
     use crate::test;
-    use arrow::array::{ArrayRef, Int32Array};
+    use crate::variable::VarType;
+    use crate::{
+        datasource::MemTable, logical_plan::create_udaf,
+        physical_plan::expressions::AvgAccumulator,
+    };
+    use arrow::array::{ArrayRef, Float64Array, Int32Array, StringArray};
     use arrow::compute::add;
     use std::fs::File;
-    use std::io::prelude::*;
-    use tempdir::TempDir;
+    use std::thread::{self, JoinHandle};
+    use std::{io::prelude::*, sync::Mutex};
+    use tempfile::TempDir;
     use test::*;
 
-    #[test]
-    fn parallel_projection() -> Result<()> {
+    #[tokio::test]
+    async fn parallel_projection() -> Result<()> {
         let partition_count = 4;
-        let results = execute("SELECT c1, c2 FROM test", partition_count)?;
+        let results = execute("SELECT c1, c2 FROM test", partition_count).await?;
 
         // there should be one batch per partition
         assert_eq!(results.len(), partition_count);
@@ -674,19 +582,57 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn parallel_selection() -> Result<()> {
-        let tmp_dir = TempDir::new("parallel_selection")?;
+    #[tokio::test]
+    async fn create_variable_expr() -> Result<()> {
+        let tmp_dir = TempDir::new()?;
         let partition_count = 4;
         let mut ctx = create_ctx(&tmp_dir, partition_count)?;
 
+        let variable_provider = test::variable::SystemVar::new();
+        ctx.register_variable(VarType::System, Arc::new(variable_provider));
+        let variable_provider = test::variable::UserDefinedVar::new();
+        ctx.register_variable(VarType::UserDefined, Arc::new(variable_provider));
+
+        let provider = test::create_table_dual();
+        ctx.register_table("dual", provider);
+
+        let results = collect(&mut ctx, "SELECT @@version, @name FROM dual").await?;
+
+        let batch = &results[0];
+        assert_eq!(2, batch.num_columns());
+        assert_eq!(1, batch.num_rows());
+        assert_eq!(field_names(batch), vec!["@@version", "@name"]);
+
+        let version = batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .expect("failed to cast version");
+        assert_eq!(version.value(0), "system-var-@@version");
+
+        let name = batch
+            .column(1)
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .expect("failed to cast name");
+        assert_eq!(name.value(0), "user-defined-var-@name");
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn parallel_query_with_filter() -> Result<()> {
+        let tmp_dir = TempDir::new()?;
+        let partition_count = 4;
+        let ctx = create_ctx(&tmp_dir, partition_count)?;
+
         let logical_plan =
             ctx.create_logical_plan("SELECT c1, c2 FROM test WHERE c1 > 0 AND c1 < 3")?;
         let logical_plan = ctx.optimize(&logical_plan)?;
 
-        let physical_plan = ctx.create_physical_plan(&logical_plan, 1024)?;
+        let physical_plan = ctx.create_physical_plan(&logical_plan)?;
 
-        let results = ctx.collect(physical_plan.as_ref())?;
+        let results = ctx.collect(physical_plan).await?;
 
         // there should be one batch per partition
         assert_eq!(results.len(), partition_count);
@@ -697,9 +643,9 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn projection_on_table_scan() -> Result<()> {
-        let tmp_dir = TempDir::new("projection_on_table_scan")?;
+    #[tokio::test]
+    async fn projection_on_table_scan() -> Result<()> {
+        let tmp_dir = TempDir::new()?;
         let partition_count = 4;
         let mut ctx = create_ctx(&tmp_dir, partition_count)?;
 
@@ -728,12 +674,12 @@ mod tests {
         \n  TableScan: test projection=Some([1])";
         assert_eq!(format!("{:?}", optimized_plan), expected);
 
-        let physical_plan = ctx.create_physical_plan(&optimized_plan, 1024)?;
+        let physical_plan = ctx.create_physical_plan(&optimized_plan)?;
 
         assert_eq!(1, physical_plan.schema().fields().len());
         assert_eq!("c2", physical_plan.schema().field(0).name().as_str());
 
-        let batches = ctx.collect(physical_plan.as_ref())?;
+        let batches = ctx.collect(physical_plan).await?;
         assert_eq!(4, batches.len());
         assert_eq!(1, batches[0].num_columns());
         assert_eq!(10, batches[0].num_rows());
@@ -743,10 +689,10 @@ mod tests {
 
     #[test]
     fn preserve_nullability_on_projection() -> Result<()> {
-        let tmp_dir = TempDir::new("execute")?;
+        let tmp_dir = TempDir::new()?;
         let ctx = create_ctx(&tmp_dir, 1)?;
 
-        let schema = ctx.datasources.get("test").unwrap().schema();
+        let schema = ctx.state.datasources.get("test").unwrap().schema();
         assert_eq!(schema.field_with_name("c1")?.is_nullable(), false);
 
         let plan = LogicalPlanBuilder::scan("default", "test", schema.as_ref(), None)?
@@ -754,7 +700,7 @@ mod tests {
             .build()?;
 
         let plan = ctx.optimize(&plan)?;
-        let physical_plan = ctx.create_physical_plan(&Arc::new(plan), 1024)?;
+        let physical_plan = ctx.create_physical_plan(&Arc::new(plan))?;
         assert_eq!(
             physical_plan.schema().field_with_name("c1")?.is_nullable(),
             false
@@ -762,25 +708,26 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn projection_on_memory_scan() -> Result<()> {
+    #[tokio::test]
+    async fn projection_on_memory_scan() -> Result<()> {
         let schema = Schema::new(vec![
             Field::new("a", DataType::Int32, false),
             Field::new("b", DataType::Int32, false),
             Field::new("c", DataType::Int32, false),
         ]);
+        let schema = SchemaRef::new(schema);
         let plan = LogicalPlanBuilder::from(&LogicalPlan::InMemoryScan {
             data: vec![vec![RecordBatch::try_new(
-                Arc::new(schema.clone()),
+                schema.clone(),
                 vec![
                     Arc::new(Int32Array::from(vec![1, 10, 10, 100])),
                     Arc::new(Int32Array::from(vec![2, 12, 12, 120])),
                     Arc::new(Int32Array::from(vec![3, 12, 12, 120])),
                 ],
             )?]],
-            schema: Box::new(schema.clone()),
+            schema: schema.clone(),
             projection: None,
-            projected_schema: Box::new(schema.clone()),
+            projected_schema: schema.clone(),
         })
         .project(vec![col("b")])?
         .build()?;
@@ -807,12 +754,12 @@ mod tests {
         \n  InMemoryScan: projection=Some([1])";
         assert_eq!(format!("{:?}", optimized_plan), expected);
 
-        let physical_plan = ctx.create_physical_plan(&optimized_plan, 1024)?;
+        let physical_plan = ctx.create_physical_plan(&optimized_plan)?;
 
         assert_eq!(1, physical_plan.schema().fields().len());
         assert_eq!("b", physical_plan.schema().field(0).name().as_str());
 
-        let batches = ctx.collect(physical_plan.as_ref())?;
+        let batches = ctx.collect(physical_plan).await?;
         assert_eq!(1, batches.len());
         assert_eq!(1, batches[0].num_columns());
         assert_eq!(4, batches[0].num_rows());
@@ -820,9 +767,10 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn sort() -> Result<()> {
-        let results = execute("SELECT c1, c2 FROM test ORDER BY c1 DESC, c2 ASC", 4)?;
+    #[tokio::test]
+    async fn sort() -> Result<()> {
+        let results =
+            execute("SELECT c1, c2 FROM test ORDER BY c1 DESC, c2 ASC", 4).await?;
         assert_eq!(results.len(), 1);
 
         let batch = &results[0];
@@ -837,9 +785,9 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn aggregate() -> Result<()> {
-        let results = execute("SELECT SUM(c1), SUM(c2) FROM test", 4)?;
+    #[tokio::test]
+    async fn aggregate() -> Result<()> {
+        let results = execute("SELECT SUM(c1), SUM(c2) FROM test", 4).await?;
         assert_eq!(results.len(), 1);
 
         let batch = &results[0];
@@ -854,9 +802,9 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn aggregate_avg() -> Result<()> {
-        let results = execute("SELECT AVG(c1), AVG(c2) FROM test", 4)?;
+    #[tokio::test]
+    async fn aggregate_avg() -> Result<()> {
+        let results = execute("SELECT AVG(c1), AVG(c2) FROM test", 4).await?;
         assert_eq!(results.len(), 1);
 
         let batch = &results[0];
@@ -871,9 +819,9 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn aggregate_max() -> Result<()> {
-        let results = execute("SELECT MAX(c1), MAX(c2) FROM test", 4)?;
+    #[tokio::test]
+    async fn aggregate_max() -> Result<()> {
+        let results = execute("SELECT MAX(c1), MAX(c2) FROM test", 4).await?;
         assert_eq!(results.len(), 1);
 
         let batch = &results[0];
@@ -888,9 +836,9 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn aggregate_min() -> Result<()> {
-        let results = execute("SELECT MIN(c1), MIN(c2) FROM test", 4)?;
+    #[tokio::test]
+    async fn aggregate_min() -> Result<()> {
+        let results = execute("SELECT MIN(c1), MIN(c2) FROM test", 4).await?;
         assert_eq!(results.len(), 1);
 
         let batch = &results[0];
@@ -905,9 +853,9 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn aggregate_grouped() -> Result<()> {
-        let results = execute("SELECT c1, SUM(c2) FROM test GROUP BY c1", 4)?;
+    #[tokio::test]
+    async fn aggregate_grouped() -> Result<()> {
+        let results = execute("SELECT c1, SUM(c2) FROM test GROUP BY c1", 4).await?;
         assert_eq!(results.len(), 1);
 
         let batch = &results[0];
@@ -922,9 +870,9 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn aggregate_grouped_avg() -> Result<()> {
-        let results = execute("SELECT c1, AVG(c2) FROM test GROUP BY c1", 4)?;
+    #[tokio::test]
+    async fn aggregate_grouped_avg() -> Result<()> {
+        let results = execute("SELECT c1, AVG(c2) FROM test GROUP BY c1", 4).await?;
         assert_eq!(results.len(), 1);
 
         let batch = &results[0];
@@ -939,9 +887,27 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn aggregate_grouped_max() -> Result<()> {
-        let results = execute("SELECT c1, MAX(c2) FROM test GROUP BY c1", 4)?;
+    #[tokio::test]
+    async fn aggregate_grouped_empty() -> Result<()> {
+        let results =
+            execute("SELECT c1, AVG(c2) FROM test WHERE c1 = 123 GROUP BY c1", 4).await?;
+        assert_eq!(results.len(), 1);
+
+        let batch = &results[0];
+
+        assert_eq!(field_names(batch), vec!["c1", "AVG(c2)"]);
+
+        let expected: Vec<&str> = vec![];
+        let mut rows = test::format_batch(&batch);
+        rows.sort();
+        assert_eq!(rows, expected);
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn aggregate_grouped_max() -> Result<()> {
+        let results = execute("SELECT c1, MAX(c2) FROM test GROUP BY c1", 4).await?;
         assert_eq!(results.len(), 1);
 
         let batch = &results[0];
@@ -956,9 +922,9 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn aggregate_grouped_min() -> Result<()> {
-        let results = execute("SELECT c1, MIN(c2) FROM test GROUP BY c1", 4)?;
+    #[tokio::test]
+    async fn aggregate_grouped_min() -> Result<()> {
+        let results = execute("SELECT c1, MIN(c2) FROM test GROUP BY c1", 4).await?;
         assert_eq!(results.len(), 1);
 
         let batch = &results[0];
@@ -973,9 +939,9 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn count_basic() -> Result<()> {
-        let results = execute("SELECT COUNT(c1), COUNT(c2) FROM test", 1)?;
+    #[tokio::test]
+    async fn count_basic() -> Result<()> {
+        let results = execute("SELECT COUNT(c1), COUNT(c2) FROM test", 1).await?;
         assert_eq!(results.len(), 1);
 
         let batch = &results[0];
@@ -989,9 +955,9 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn count_partitioned() -> Result<()> {
-        let results = execute("SELECT COUNT(c1), COUNT(c2) FROM test", 4)?;
+    #[tokio::test]
+    async fn count_partitioned() -> Result<()> {
+        let results = execute("SELECT COUNT(c1), COUNT(c2) FROM test", 4).await?;
         assert_eq!(results.len(), 1);
 
         let batch = &results[0];
@@ -1005,9 +971,9 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn count_aggregated() -> Result<()> {
-        let results = execute("SELECT c1, COUNT(c2) FROM test GROUP BY c1", 4)?;
+    #[tokio::test]
+    async fn count_aggregated() -> Result<()> {
+        let results = execute("SELECT c1, COUNT(c2) FROM test GROUP BY c1", 4).await?;
         assert_eq!(results.len(), 1);
 
         let batch = &results[0];
@@ -1021,9 +987,136 @@ mod tests {
         Ok(())
     }
 
+    async fn run_count_distinct_integers_aggregated_scenario(
+        partitions: Vec<Vec<(&str, u64)>>,
+    ) -> Result<Vec<RecordBatch>> {
+        let tmp_dir = TempDir::new()?;
+        let mut ctx = ExecutionContext::new();
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("c_group", DataType::Utf8, false),
+            Field::new("c_int8", DataType::Int8, false),
+            Field::new("c_int16", DataType::Int16, false),
+            Field::new("c_int32", DataType::Int32, false),
+            Field::new("c_int64", DataType::Int64, false),
+            Field::new("c_uint8", DataType::UInt8, false),
+            Field::new("c_uint16", DataType::UInt16, false),
+            Field::new("c_uint32", DataType::UInt32, false),
+            Field::new("c_uint64", DataType::UInt64, false),
+        ]));
+
+        for (i, partition) in partitions.iter().enumerate() {
+            let filename = format!("partition-{}.csv", i);
+            let file_path = tmp_dir.path().join(&filename);
+            let mut file = File::create(file_path)?;
+            for row in partition {
+                let row_str = format!(
+                    "{},{}\n",
+                    row.0,
+                    // Populate values for each of the integer fields in the
+                    // schema.
+                    (0..8)
+                        .map(|_| { row.1.to_string() })
+                        .collect::<Vec<_>>()
+                        .join(","),
+                );
+                file.write_all(row_str.as_bytes())?;
+            }
+        }
+        ctx.register_csv(
+            "test",
+            tmp_dir.path().to_str().unwrap(),
+            CsvReadOptions::new().schema(&schema).has_header(false),
+        )?;
+
+        let results = collect(
+            &mut ctx,
+            "
+              SELECT
+                c_group,
+                COUNT(c_uint64),
+                COUNT(DISTINCT c_int8),
+                COUNT(DISTINCT c_int16),
+                COUNT(DISTINCT c_int32),
+                COUNT(DISTINCT c_int64),
+                COUNT(DISTINCT c_uint8),
+                COUNT(DISTINCT c_uint16),
+                COUNT(DISTINCT c_uint32),
+                COUNT(DISTINCT c_uint64)
+              FROM test
+              GROUP BY c_group
+            ",
+        )
+        .await?;
+
+        Ok(results)
+    }
+
+    #[tokio::test]
+    async fn count_distinct_integers_aggregated_single_partition() -> Result<()> {
+        let partitions = vec![
+            // The first member of each tuple will be the value for the
+            // `c_group` column, and the second member will be the value for
+            // each of the int/uint fields.
+            vec![
+                ("a", 1),
+                ("a", 1),
+                ("a", 2),
+                ("b", 9),
+                ("c", 9),
+                ("c", 10),
+                ("c", 9),
+            ],
+        ];
+
+        let results = run_count_distinct_integers_aggregated_scenario(partitions).await?;
+        assert_eq!(results.len(), 1);
+
+        let batch = &results[0];
+        assert_eq!(batch.num_rows(), 3);
+        assert_eq!(batch.num_columns(), 10);
+        assert_eq!(
+            test::format_batch(&batch),
+            vec![
+                "a,3,2,2,2,2,2,2,2,2",
+                "c,3,2,2,2,2,2,2,2,2",
+                "b,1,1,1,1,1,1,1,1,1",
+            ],
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn count_distinct_integers_aggregated_multiple_partitions() -> Result<()> {
+        let partitions = vec![
+            // The first member of each tuple will be the value for the
+            // `c_group` column, and the second member will be the value for
+            // each of the int/uint fields.
+            vec![("a", 1), ("a", 1), ("a", 2), ("b", 9), ("c", 9)],
+            vec![("a", 1), ("a", 3), ("b", 8), ("b", 9), ("b", 10), ("b", 11)],
+        ];
+
+        let results = run_count_distinct_integers_aggregated_scenario(partitions).await?;
+        assert_eq!(results.len(), 1);
+
+        let batch = &results[0];
+        assert_eq!(batch.num_rows(), 3);
+        assert_eq!(batch.num_columns(), 10);
+        assert_eq!(
+            test::format_batch(&batch),
+            vec![
+                "a,5,3,3,3,3,3,3,3,3",
+                "c,1,1,1,1,1,1,1,1,1",
+                "b,5,4,4,4,4,4,4,4,4",
+            ],
+        );
+
+        Ok(())
+    }
+
     #[test]
     fn aggregate_with_alias() -> Result<()> {
-        let tmp_dir = TempDir::new("execute")?;
+        let tmp_dir = TempDir::new()?;
         let ctx = create_ctx(&tmp_dir, 1)?;
 
         let schema = Arc::new(Schema::new(vec![
@@ -1032,16 +1125,13 @@ mod tests {
         ]));
 
         let plan = LogicalPlanBuilder::scan("default", "test", schema.as_ref(), None)?
-            .aggregate(
-                vec![col("c1")],
-                vec![aggregate_expr("SUM", col("c2"), DataType::UInt32)],
-            )?
+            .aggregate(vec![col("c1")], vec![sum(col("c2"))])?
             .project(vec![col("c1"), col("SUM(c2)").alias("total_salary")])?
             .build()?;
 
         let plan = ctx.optimize(&plan)?;
 
-        let physical_plan = ctx.create_physical_plan(&Arc::new(plan), 1024)?;
+        let physical_plan = ctx.create_physical_plan(&Arc::new(plan))?;
         assert_eq!("c1", physical_plan.schema().field(0).name().as_str());
         assert_eq!(
             "total_salary",
@@ -1050,15 +1140,15 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn write_csv_results() -> Result<()> {
+    #[tokio::test]
+    async fn write_csv_results() -> Result<()> {
         // create partitioned input file and context
-        let tmp_dir = TempDir::new("write_csv_results_temp")?;
+        let tmp_dir = TempDir::new()?;
         let mut ctx = create_ctx(&tmp_dir, 4)?;
 
         // execute a simple query and write the results to CSV
         let out_dir = tmp_dir.as_ref().to_str().unwrap().to_string() + "/out";
-        write_csv(&mut ctx, "SELECT c1, c2 FROM test", &out_dir)?;
+        write_csv(&mut ctx, "SELECT c1, c2 FROM test", &out_dir).await?;
 
         // create a new context and verify that the results were saved to a partitioned csv file
         let mut ctx = ExecutionContext::new();
@@ -1076,11 +1166,11 @@ mod tests {
         ctx.register_csv("part3", &format!("{}/part-3.csv", out_dir), csv_read_option)?;
         ctx.register_csv("allparts", &out_dir, csv_read_option)?;
 
-        let part0 = collect(&mut ctx, "SELECT c1, c2 FROM part0")?;
-        let part1 = collect(&mut ctx, "SELECT c1, c2 FROM part1")?;
-        let part2 = collect(&mut ctx, "SELECT c1, c2 FROM part2")?;
-        let part3 = collect(&mut ctx, "SELECT c1, c2 FROM part3")?;
-        let allparts = collect(&mut ctx, "SELECT c1, c2 FROM allparts")?;
+        let part0 = collect(&mut ctx, "SELECT c1, c2 FROM part0").await?;
+        let part1 = collect(&mut ctx, "SELECT c1, c2 FROM part1").await?;
+        let part2 = collect(&mut ctx, "SELECT c1, c2 FROM part2").await?;
+        let part3 = collect(&mut ctx, "SELECT c1, c2 FROM part3").await?;
+        let allparts = collect(&mut ctx, "SELECT c1, c2 FROM allparts").await?;
 
         let part0_count: usize = part0.iter().map(|batch| batch.num_rows()).sum();
         let part1_count: usize = part1.iter().map(|batch| batch.num_rows()).sum();
@@ -1097,9 +1187,9 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn query_csv_with_custom_partition_extension() -> Result<()> {
-        let tmp_dir = TempDir::new("query_csv_with_custom_partition_extension")?;
+    #[tokio::test]
+    async fn query_csv_with_custom_partition_extension() -> Result<()> {
+        let tmp_dir = TempDir::new()?;
 
         // The main stipulation of this test: use a file extension that isn't .csv.
         let file_extension = ".tst";
@@ -1113,7 +1203,8 @@ mod tests {
                 .schema(&schema)
                 .file_extension(file_extension),
         )?;
-        let results = collect(&mut ctx, "SELECT SUM(c1), SUM(c2), COUNT(*) FROM test")?;
+        let results =
+            collect(&mut ctx, "SELECT SUM(c1), SUM(c2), COUNT(*) FROM test").await?;
 
         assert_eq!(results.len(), 1);
         assert_eq!(results[0].num_rows(), 1);
@@ -1123,7 +1214,34 @@ mod tests {
     }
 
     #[test]
-    fn scalar_udf() -> Result<()> {
+    fn send_context_to_threads() -> Result<()> {
+        // ensure ExecutionContexts can be used in a multi-threaded
+        // environment. Usecase is for concurrent planing.
+        let tmp_dir = TempDir::new()?;
+        let partition_count = 4;
+        let ctx = Arc::new(Mutex::new(create_ctx(&tmp_dir, partition_count)?));
+
+        let threads: Vec<JoinHandle<Result<_>>> = (0..2)
+            .map(|_| ctx.clone())
+            .map(|ctx_clone| {
+                thread::spawn(move || {
+                    let ctx = ctx_clone.lock().expect("Locked context");
+                    // Ensure we can create logical plan code on a separate thread.
+                    ctx.create_logical_plan(
+                        "SELECT c1, c2 FROM test WHERE c1 > 0 AND c1 < 3",
+                    )
+                })
+            })
+            .collect();
+
+        for thread in threads {
+            thread.join().expect("Failed to join thread")?;
+        }
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn scalar_udf() -> Result<()> {
         let schema = Schema::new(vec![
             Field::new("a", DataType::Int32, false),
             Field::new("b", DataType::Int32, false),
@@ -1142,7 +1260,7 @@ mod tests {
         let provider = MemTable::new(Arc::new(schema), vec![vec![batch]])?;
         ctx.register_table("t", Box::new(provider));
 
-        let myfunc: ScalarUdf = Arc::new(|args: &[ArrayRef]| {
+        let myfunc: ScalarFunctionImplementation = Arc::new(|args: &[ArrayRef]| {
             let l = &args[0]
                 .as_any()
                 .downcast_ref::<Int32Array>()
@@ -1154,17 +1272,15 @@ mod tests {
             Ok(Arc::new(add(l, r)?))
         });
 
-        let my_add = ScalarFunction::new(
+        ctx.register_udf(create_udf(
             "my_add",
-            vec![
-                Field::new("a", DataType::Int32, true),
-                Field::new("b", DataType::Int32, true),
-            ],
-            DataType::Int32,
+            vec![DataType::Int32, DataType::Int32],
+            Arc::new(DataType::Int32),
             myfunc,
-        );
+        ));
 
-        ctx.register_udf(my_add);
+        // from here on, we may be in a different scope. We would still like to be able
+        // to call UDFs.
 
         let t = ctx.table("t")?;
 
@@ -1172,7 +1288,7 @@ mod tests {
             .project(vec![
                 col("a"),
                 col("b"),
-                scalar_function("my_add", vec![col("a"), col("b")], DataType::Int32),
+                ctx.registry().udf("my_add")?.call(vec![col("a"), col("b")]),
             ])?
             .build()?;
 
@@ -1182,8 +1298,8 @@ mod tests {
         );
 
         let plan = ctx.optimize(&plan)?;
-        let plan = ctx.create_physical_plan(&plan, 1024)?;
-        let result = ctx.collect(plan.as_ref())?;
+        let plan = ctx.create_physical_plan(&plan)?;
+        let result = ctx.collect(plan).await?;
 
         let batch = &result[0];
         assert_eq!(3, batch.num_columns());
@@ -1216,12 +1332,132 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test]
+    async fn simple_avg() -> Result<()> {
+        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+
+        let batch1 = RecordBatch::try_new(
+            Arc::new(schema.clone()),
+            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
+        )?;
+        let batch2 = RecordBatch::try_new(
+            Arc::new(schema.clone()),
+            vec![Arc::new(Int32Array::from(vec![4, 5]))],
+        )?;
+
+        let mut ctx = ExecutionContext::new();
+
+        let provider = MemTable::new(Arc::new(schema), vec![vec![batch1], vec![batch2]])?;
+        ctx.register_table("t", Box::new(provider));
+
+        let result = collect(&mut ctx, "SELECT AVG(a) FROM t").await?;
+
+        let batch = &result[0];
+        assert_eq!(1, batch.num_columns());
+        assert_eq!(1, batch.num_rows());
+
+        let values = batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<Float64Array>()
+            .expect("failed to cast version");
+        assert_eq!(values.len(), 1);
+        // avg(1,2,3,4,5) = 3.0
+        assert_eq!(values.value(0), 3.0_f64);
+        Ok(())
+    }
+
+    /// tests the creation, registration and usage of a UDAF
+    #[tokio::test]
+    async fn simple_udaf() -> Result<()> {
+        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+
+        let batch1 = RecordBatch::try_new(
+            Arc::new(schema.clone()),
+            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
+        )?;
+        let batch2 = RecordBatch::try_new(
+            Arc::new(schema.clone()),
+            vec![Arc::new(Int32Array::from(vec![4, 5]))],
+        )?;
+
+        let mut ctx = ExecutionContext::new();
+
+        let provider = MemTable::new(Arc::new(schema), vec![vec![batch1], vec![batch2]])?;
+        ctx.register_table("t", Box::new(provider));
+
+        // define a udaf, using a DataFusion's accumulator
+        let my_avg = create_udaf(
+            "MY_AVG",
+            DataType::Float64,
+            Arc::new(DataType::Float64),
+            Arc::new(|| Ok(Box::new(AvgAccumulator::try_new(&DataType::Float64)?))),
+            Arc::new(vec![DataType::UInt64, DataType::Float64]),
+        );
+
+        ctx.register_udaf(my_avg);
+
+        let result = collect(&mut ctx, "SELECT MY_AVG(a) FROM t").await?;
+
+        let batch = &result[0];
+        assert_eq!(1, batch.num_columns());
+        assert_eq!(1, batch.num_rows());
+
+        let values = batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<Float64Array>()
+            .expect("failed to cast version");
+        assert_eq!(values.len(), 1);
+        // avg(1,2,3,4,5) = 3.0
+        assert_eq!(values.value(0), 3.0_f64);
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn custom_query_planner() -> Result<()> {
+        let mut ctx = ExecutionContext::with_config(
+            ExecutionConfig::new().with_query_planner(Arc::new(MyQueryPlanner {})),
+        );
+
+        let df = ctx.sql("SELECT 1")?;
+        df.collect().await.expect_err("query not supported");
+        Ok(())
+    }
+
+    struct MyPhysicalPlanner {}
+
+    impl PhysicalPlanner for MyPhysicalPlanner {
+        fn create_physical_plan(
+            &self,
+            _logical_plan: &LogicalPlan,
+            _ctx_state: &ExecutionContextState,
+        ) -> Result<Arc<dyn ExecutionPlan>> {
+            Err(DataFusionError::NotImplemented(
+                "query not supported".to_string(),
+            ))
+        }
+    }
+
+    struct MyQueryPlanner {}
+
+    impl QueryPlanner for MyQueryPlanner {
+        fn create_physical_plan(
+            &self,
+            logical_plan: &LogicalPlan,
+            ctx_state: &ExecutionContextState,
+        ) -> Result<Arc<dyn ExecutionPlan>> {
+            let physical_planner = MyPhysicalPlanner {};
+            physical_planner.create_physical_plan(logical_plan, ctx_state)
+        }
+    }
+
     /// Execute SQL and return results
-    fn collect(ctx: &mut ExecutionContext, sql: &str) -> Result<Vec<RecordBatch>> {
+    async fn collect(ctx: &mut ExecutionContext, sql: &str) -> Result<Vec<RecordBatch>> {
         let logical_plan = ctx.create_logical_plan(sql)?;
         let logical_plan = ctx.optimize(&logical_plan)?;
-        let physical_plan = ctx.create_physical_plan(&logical_plan, 1024)?;
-        ctx.collect(physical_plan.as_ref())
+        let physical_plan = ctx.create_physical_plan(&logical_plan)?;
+        ctx.collect(physical_plan).await
     }
 
     fn field_names(result: &RecordBatch) -> Vec<String> {
@@ -1234,18 +1470,22 @@ mod tests {
     }
 
     /// Execute SQL and return results
-    fn execute(sql: &str, partition_count: usize) -> Result<Vec<RecordBatch>> {
-        let tmp_dir = TempDir::new("execute")?;
+    async fn execute(sql: &str, partition_count: usize) -> Result<Vec<RecordBatch>> {
+        let tmp_dir = TempDir::new()?;
         let mut ctx = create_ctx(&tmp_dir, partition_count)?;
-        collect(&mut ctx, sql)
+        collect(&mut ctx, sql).await
     }
 
     /// Execute SQL and write results to partitioned csv files
-    fn write_csv(ctx: &mut ExecutionContext, sql: &str, out_dir: &str) -> Result<()> {
+    async fn write_csv(
+        ctx: &mut ExecutionContext,
+        sql: &str,
+        out_dir: &str,
+    ) -> Result<()> {
         let logical_plan = ctx.create_logical_plan(sql)?;
         let logical_plan = ctx.optimize(&logical_plan)?;
-        let physical_plan = ctx.create_physical_plan(&logical_plan, 1024)?;
-        ctx.write_csv(physical_plan.as_ref(), out_dir)
+        let physical_plan = ctx.create_physical_plan(&logical_plan)?;
+        ctx.write_csv(physical_plan, out_dir.to_string()).await
     }
 
     /// Generate CSV partitions within the supplied directory
diff --git a/rust/datafusion/src/execution/table_impl.rs b/rust/datafusion/src/execution/dataframe_impl.rs
similarity index 53%
rename from rust/datafusion/src/execution/table_impl.rs
rename to rust/datafusion/src/execution/dataframe_impl.rs
index 7494ba51bd9..a93e5745a03 100644
--- a/rust/datafusion/src/execution/table_impl.rs
+++ b/rust/datafusion/src/execution/dataframe_impl.rs
@@ -15,33 +15,39 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//! Implementation of Table API
+//! Implementation of DataFrame API
 
 use std::sync::Arc;
 
-use crate::arrow::datatypes::DataType;
 use crate::arrow::record_batch::RecordBatch;
-use crate::error::{ExecutionError, Result};
-use crate::execution::context::ExecutionContext;
-use crate::logicalplan::{col, Expr, LogicalPlan, LogicalPlanBuilder};
-use crate::table::*;
+use crate::dataframe::*;
+use crate::error::Result;
+use crate::execution::context::{ExecutionContext, ExecutionContextState};
+use crate::logical_plan::{col, Expr, FunctionRegistry, LogicalPlan, LogicalPlanBuilder};
 use arrow::datatypes::Schema;
 
-/// Implementation of Table API
-pub struct TableImpl {
+use async_trait::async_trait;
+
+/// Implementation of DataFrame API
+pub struct DataFrameImpl {
+    ctx_state: ExecutionContextState,
     plan: LogicalPlan,
 }
 
-impl TableImpl {
+impl DataFrameImpl {
     /// Create a new Table based on an existing logical plan
-    pub fn new(plan: &LogicalPlan) -> Self {
-        Self { plan: plan.clone() }
+    pub fn new(ctx_state: ExecutionContextState, plan: &LogicalPlan) -> Self {
+        Self {
+            ctx_state,
+            plan: plan.clone(),
+        }
     }
 }
 
-impl Table for TableImpl {
+#[async_trait]
+impl DataFrame for DataFrameImpl {
     /// Apply a projection based on a list of column names
-    fn select_columns(&self, columns: Vec<&str>) -> Result<Arc<dyn Table>> {
+    fn select_columns(&self, columns: Vec<&str>) -> Result<Arc<dyn DataFrame>> {
         let exprs = columns
             .iter()
             .map(|name| {
@@ -57,17 +63,19 @@ impl Table for TableImpl {
     }
 
     /// Create a projection based on arbitrary expressions
-    fn select(&self, expr_list: Vec<Expr>) -> Result<Arc<dyn Table>> {
+    fn select(&self, expr_list: Vec<Expr>) -> Result<Arc<dyn DataFrame>> {
         let plan = LogicalPlanBuilder::from(&self.plan)
             .project(expr_list)?
             .build()?;
-        Ok(Arc::new(TableImpl::new(&plan)))
+        Ok(Arc::new(DataFrameImpl::new(self.ctx_state.clone(), &plan)))
     }
 
-    /// Create a selection based on a filter expression
-    fn filter(&self, expr: Expr) -> Result<Arc<dyn Table>> {
-        let plan = LogicalPlanBuilder::from(&self.plan).filter(expr)?.build()?;
-        Ok(Arc::new(TableImpl::new(&plan)))
+    /// Create a filter based on a predicate expression
+    fn filter(&self, predicate: Expr) -> Result<Arc<dyn DataFrame>> {
+        let plan = LogicalPlanBuilder::from(&self.plan)
+            .filter(predicate)?
+            .build()?;
+        Ok(Arc::new(DataFrameImpl::new(self.ctx_state.clone(), &plan)))
     }
 
     /// Perform an aggregate query
@@ -75,48 +83,23 @@ impl Table for TableImpl {
         &self,
         group_expr: Vec<Expr>,
         aggr_expr: Vec<Expr>,
-    ) -> Result<Arc<dyn Table>> {
+    ) -> Result<Arc<dyn DataFrame>> {
         let plan = LogicalPlanBuilder::from(&self.plan)
             .aggregate(group_expr, aggr_expr)?
             .build()?;
-        Ok(Arc::new(TableImpl::new(&plan)))
+        Ok(Arc::new(DataFrameImpl::new(self.ctx_state.clone(), &plan)))
     }
 
     /// Limit the number of rows
-    fn limit(&self, n: usize) -> Result<Arc<dyn Table>> {
+    fn limit(&self, n: usize) -> Result<Arc<dyn DataFrame>> {
         let plan = LogicalPlanBuilder::from(&self.plan).limit(n)?.build()?;
-        Ok(Arc::new(TableImpl::new(&plan)))
-    }
-
-    /// Return an expression representing a column within this table
-    fn col(&self, name: &str) -> Result<Expr> {
-        self.plan.schema().index_of(name)?; // check that the column exists
-        Ok(col(name))
-    }
-
-    /// Create an expression to represent the min() aggregate function
-    fn min(&self, expr: &Expr) -> Result<Expr> {
-        self.aggregate_expr("MIN", expr)
-    }
-
-    /// Create an expression to represent the max() aggregate function
-    fn max(&self, expr: &Expr) -> Result<Expr> {
-        self.aggregate_expr("MAX", expr)
-    }
-
-    /// Create an expression to represent the sum() aggregate function
-    fn sum(&self, expr: &Expr) -> Result<Expr> {
-        self.aggregate_expr("SUM", expr)
-    }
-
-    /// Create an expression to represent the avg() aggregate function
-    fn avg(&self, expr: &Expr) -> Result<Expr> {
-        self.aggregate_expr("AVG", expr)
+        Ok(Arc::new(DataFrameImpl::new(self.ctx_state.clone(), &plan)))
     }
 
-    /// Create an expression to represent the count() aggregate function
-    fn count(&self, expr: &Expr) -> Result<Expr> {
-        self.aggregate_expr("COUNT", expr)
+    /// Sort by specified sorting expressions
+    fn sort(&self, expr: Vec<Expr>) -> Result<Arc<dyn DataFrame>> {
+        let plan = LogicalPlanBuilder::from(&self.plan).sort(expr)?.build()?;
+        Ok(Arc::new(DataFrameImpl::new(self.ctx_state.clone(), &plan)))
     }
 
     /// Convert to logical plan
@@ -124,45 +107,29 @@ impl Table for TableImpl {
         self.plan.clone()
     }
 
-    fn collect(
-        &self,
-        ctx: &mut ExecutionContext,
-        batch_size: usize,
-    ) -> Result<Vec<RecordBatch>> {
-        ctx.collect_plan(&self.plan.clone(), batch_size)
+    // Convert the logical plan represented by this DataFrame into a physical plan and
+    // execute it
+    async fn collect(&self) -> Result<Vec<RecordBatch>> {
+        let ctx = ExecutionContext::from(self.ctx_state.clone());
+        let plan = ctx.optimize(&self.plan)?;
+        let plan = ctx.create_physical_plan(&plan)?;
+        Ok(ctx.collect(plan).await?)
     }
 
     /// Returns the schema from the logical plan
     fn schema(&self) -> &Schema {
-        self.plan.schema().as_ref()
+        self.plan.schema()
     }
-}
 
-impl TableImpl {
-    /// Determine the data type for a given expression
-    fn get_data_type(&self, expr: &Expr) -> Result<DataType> {
-        match expr {
-            Expr::Column(name) => Ok(self
-                .plan
-                .schema()
-                .field_with_name(name)?
-                .data_type()
-                .clone()),
-            _ => Err(ExecutionError::General(format!(
-                "Could not determine data type for expr {:?}",
-                expr
-            ))),
-        }
+    fn explain(&self, verbose: bool) -> Result<Arc<dyn DataFrame>> {
+        let plan = LogicalPlanBuilder::from(&self.plan)
+            .explain(verbose)?
+            .build()?;
+        Ok(Arc::new(DataFrameImpl::new(self.ctx_state.clone(), &plan)))
     }
 
-    /// Create an expression to represent a named aggregate function
-    fn aggregate_expr(&self, name: &str, expr: &Expr) -> Result<Expr> {
-        let return_type = self.get_data_type(expr)?;
-        Ok(Expr::AggregateFunction {
-            name: name.to_string(),
-            args: vec![expr.clone()],
-            return_type,
-        })
+    fn registry(&self) -> &dyn FunctionRegistry {
+        &self.ctx_state
     }
 }
 
@@ -171,7 +138,9 @@ mod tests {
     use super::*;
     use crate::datasource::csv::CsvReadOptions;
     use crate::execution::context::ExecutionContext;
-    use crate::test;
+    use crate::logical_plan::*;
+    use crate::{physical_plan::functions::ScalarFunctionImplementation, test};
+    use arrow::{array::ArrayRef, datatypes::DataType};
 
     #[test]
     fn select_columns() -> Result<()> {
@@ -193,7 +162,7 @@ mod tests {
     fn select_expr() -> Result<()> {
         // build plan using Table API
         let t = test_table()?;
-        let t2 = t.select(vec![t.col("c1")?, t.col("c2")?, t.col("c11")?])?;
+        let t2 = t.select(vec![col("c1"), col("c2"), col("c11")])?;
         let plan = t2.to_logical_plan();
 
         // build query using SQL
@@ -207,21 +176,20 @@ mod tests {
 
     #[test]
     fn aggregate() -> Result<()> {
-        // build plan using Table API
-        let t = test_table()?;
-        let group_expr = vec![t.col("c1")?];
-        let c12 = t.col("c12")?;
+        // build plan using DataFrame API
+        let df = test_table()?;
+        let group_expr = vec![col("c1")];
         let aggr_expr = vec![
-            t.min(&c12)?,
-            t.max(&c12)?,
-            t.avg(&c12)?,
-            t.sum(&c12)?,
-            t.count(&c12)?,
+            min(col("c12")),
+            max(col("c12")),
+            avg(col("c12")),
+            sum(col("c12")),
+            count(col("c12")),
         ];
 
-        let t2 = t.aggregate(group_expr.clone(), aggr_expr.clone())?;
+        let df = df.aggregate(group_expr.clone(), aggr_expr.clone())?;
 
-        let plan = t2.to_logical_plan();
+        let plan = df.to_logical_plan();
 
         // build same plan using SQL API
         let sql = "SELECT c1, MIN(c12), MAX(c12), AVG(c12), SUM(c12), COUNT(c12) \
@@ -252,6 +220,61 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn explain() -> Result<()> {
+        // build query using Table API
+        let df = test_table()?;
+        let df = df
+            .select_columns(vec!["c1", "c2", "c11"])?
+            .limit(10)?
+            .explain(false)?;
+        let plan = df.to_logical_plan();
+
+        // build query using SQL
+        let sql_plan =
+            create_plan("EXPLAIN SELECT c1, c2, c11 FROM aggregate_test_100 LIMIT 10")?;
+
+        // the two plans should be identical
+        assert_same_plan(&plan, &sql_plan);
+
+        Ok(())
+    }
+
+    #[test]
+    fn registry() -> Result<()> {
+        let mut ctx = ExecutionContext::new();
+        register_aggregate_csv(&mut ctx)?;
+
+        // declare the udf
+        let my_fn: ScalarFunctionImplementation =
+            Arc::new(|_: &[ArrayRef]| unimplemented!("my_fn is not implemented"));
+
+        // create and register the udf
+        ctx.register_udf(create_udf(
+            "my_fn",
+            vec![DataType::Float64],
+            Arc::new(DataType::Float64),
+            my_fn,
+        ));
+
+        // build query with a UDF using DataFrame API
+        let df = ctx.table("aggregate_test_100")?;
+
+        let f = df.registry();
+
+        let df = df.select(vec![f.udf("my_fn")?.call(vec![col("c12")])])?;
+        let plan = df.to_logical_plan();
+
+        // build query using SQL
+        let sql_plan =
+            ctx.create_logical_plan("SELECT my_fn(c12) FROM aggregate_test_100")?;
+
+        // the two plans should be identical
+        assert_same_plan(&plan, &sql_plan);
+
+        Ok(())
+    }
+
     /// Compare the formatted string representation of two plans for equality
     fn assert_same_plan(plan1: &LogicalPlan, plan2: &LogicalPlan) {
         assert_eq!(format!("{:?}", plan1), format!("{:?}", plan2));
@@ -264,7 +287,7 @@ mod tests {
         ctx.create_logical_plan(sql)
     }
 
-    fn test_table() -> Result<Arc<dyn Table + 'static>> {
+    fn test_table() -> Result<Arc<dyn DataFrame + 'static>> {
         let mut ctx = ExecutionContext::new();
         register_aggregate_csv(&mut ctx)?;
         ctx.table("aggregate_test_100")
diff --git a/rust/datafusion/src/execution/mod.rs b/rust/datafusion/src/execution/mod.rs
index 5a0e7525bff..ff44dd43f83 100644
--- a/rust/datafusion/src/execution/mod.rs
+++ b/rust/datafusion/src/execution/mod.rs
@@ -18,5 +18,4 @@
 //! DataFusion query execution
 
 pub mod context;
-pub mod physical_plan;
-pub mod table_impl;
+pub mod dataframe_impl;
diff --git a/rust/datafusion/src/execution/physical_plan/common.rs b/rust/datafusion/src/execution/physical_plan/common.rs
deleted file mode 100644
index f418012b03d..00000000000
--- a/rust/datafusion/src/execution/physical_plan/common.rs
+++ /dev/null
@@ -1,196 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Defines common code used in execution plans
-
-use std::fs;
-use std::fs::metadata;
-use std::sync::{Arc, Mutex};
-
-use crate::error::{ExecutionError, Result};
-
-use crate::logicalplan::ScalarValue;
-use arrow::array::{self, ArrayRef};
-use arrow::datatypes::{DataType, SchemaRef};
-use arrow::error::Result as ArrowResult;
-use arrow::record_batch::{RecordBatch, RecordBatchReader};
-
-/// Iterator over a vector of record batches
-pub struct RecordBatchIterator {
-    schema: SchemaRef,
-    batches: Vec<Arc<RecordBatch>>,
-    index: usize,
-}
-
-impl RecordBatchIterator {
-    /// Create a new RecordBatchIterator
-    pub fn new(schema: SchemaRef, batches: Vec<Arc<RecordBatch>>) -> Self {
-        RecordBatchIterator {
-            schema,
-            index: 0,
-            batches,
-        }
-    }
-}
-
-impl RecordBatchReader for RecordBatchIterator {
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-
-    fn next_batch(&mut self) -> ArrowResult<Option<RecordBatch>> {
-        if self.index < self.batches.len() {
-            self.index += 1;
-            Ok(Some(self.batches[self.index - 1].as_ref().clone()))
-        } else {
-            Ok(None)
-        }
-    }
-}
-
-/// Create a vector of record batches from an iterator
-pub fn collect(
-    it: Arc<Mutex<dyn RecordBatchReader + Send + Sync>>,
-) -> Result<Vec<RecordBatch>> {
-    let mut reader = it.lock().unwrap();
-    let mut results: Vec<RecordBatch> = vec![];
-    loop {
-        match reader.next_batch() {
-            Ok(Some(batch)) => {
-                results.push(batch);
-            }
-            Ok(None) => {
-                // end of result set
-                return Ok(results);
-            }
-            Err(e) => return Err(ExecutionError::from(e)),
-        }
-    }
-}
-
-/// Recursively build a list of files in a directory with a given extension
-pub fn build_file_list(dir: &str, filenames: &mut Vec<String>, ext: &str) -> Result<()> {
-    let metadata = metadata(dir)?;
-    if metadata.is_file() {
-        if dir.ends_with(ext) {
-            filenames.push(dir.to_string());
-        }
-    } else {
-        for entry in fs::read_dir(dir)? {
-            let entry = entry?;
-            let path = entry.path();
-            if let Some(path_name) = path.to_str() {
-                if path.is_dir() {
-                    build_file_list(path_name, filenames, ext)?;
-                } else {
-                    if path_name.ends_with(ext) {
-                        filenames.push(path_name.to_string());
-                    }
-                }
-            } else {
-                return Err(ExecutionError::General("Invalid path".to_string()));
-            }
-        }
-    }
-    Ok(())
-}
-
-/// Get a value from an array as a ScalarValue
-pub fn get_scalar_value(array: &ArrayRef, row: usize) -> Result<Option<ScalarValue>> {
-    if array.is_null(row) {
-        return Ok(None);
-    }
-    let value: Option<ScalarValue> = match array.data_type() {
-        DataType::UInt8 => {
-            let array = array
-                .as_any()
-                .downcast_ref::<array::UInt8Array>()
-                .expect("Failed to cast array");
-            Some(ScalarValue::UInt8(array.value(row)))
-        }
-        DataType::UInt16 => {
-            let array = array
-                .as_any()
-                .downcast_ref::<array::UInt16Array>()
-                .expect("Failed to cast array");
-            Some(ScalarValue::UInt16(array.value(row)))
-        }
-        DataType::UInt32 => {
-            let array = array
-                .as_any()
-                .downcast_ref::<array::UInt32Array>()
-                .expect("Failed to cast array");
-            Some(ScalarValue::UInt32(array.value(row)))
-        }
-        DataType::UInt64 => {
-            let array = array
-                .as_any()
-                .downcast_ref::<array::UInt64Array>()
-                .expect("Failed to cast array");
-            Some(ScalarValue::UInt64(array.value(row)))
-        }
-        DataType::Int8 => {
-            let array = array
-                .as_any()
-                .downcast_ref::<array::Int8Array>()
-                .expect("Failed to cast array");
-            Some(ScalarValue::Int8(array.value(row)))
-        }
-        DataType::Int16 => {
-            let array = array
-                .as_any()
-                .downcast_ref::<array::Int16Array>()
-                .expect("Failed to cast array");
-            Some(ScalarValue::Int16(array.value(row)))
-        }
-        DataType::Int32 => {
-            let array = array
-                .as_any()
-                .downcast_ref::<array::Int32Array>()
-                .expect("Failed to cast array");
-            Some(ScalarValue::Int32(array.value(row)))
-        }
-        DataType::Int64 => {
-            let array = array
-                .as_any()
-                .downcast_ref::<array::Int64Array>()
-                .expect("Failed to cast array");
-            Some(ScalarValue::Int64(array.value(row)))
-        }
-        DataType::Float32 => {
-            let array = array
-                .as_any()
-                .downcast_ref::<array::Float32Array>()
-                .unwrap();
-            Some(ScalarValue::Float32(array.value(row)))
-        }
-        DataType::Float64 => {
-            let array = array
-                .as_any()
-                .downcast_ref::<array::Float64Array>()
-                .unwrap();
-            Some(ScalarValue::Float64(array.value(row)))
-        }
-        other => {
-            return Err(ExecutionError::ExecutionError(format!(
-                "Unsupported data type {:?} for result of aggregate expression",
-                other
-            )));
-        }
-    };
-    Ok(value)
-}
diff --git a/rust/datafusion/src/execution/physical_plan/datasource.rs b/rust/datafusion/src/execution/physical_plan/datasource.rs
deleted file mode 100644
index 73de47e4f62..00000000000
--- a/rust/datafusion/src/execution/physical_plan/datasource.rs
+++ /dev/null
@@ -1,92 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! ExecutionPlan implementation for DataFusion data sources
-
-use std::{
-    fmt::{self, Debug, Formatter},
-    sync::{Arc, Mutex},
-};
-
-use crate::error::Result;
-use crate::execution::physical_plan::{ExecutionPlan, Partition};
-use arrow::datatypes::SchemaRef;
-use arrow::record_batch::RecordBatchReader;
-
-/// Datasource execution plan
-pub struct DatasourceExec {
-    schema: SchemaRef,
-    partitions: Vec<Arc<Mutex<dyn RecordBatchReader + Send + Sync>>>,
-}
-
-impl Debug for DatasourceExec {
-    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
-        f.debug_struct("DataSourceExec")
-            .field("schema", &self.schema)
-            .field("partitions.len", &self.partitions.len())
-            .finish()
-    }
-}
-
-impl DatasourceExec {
-    /// Create a new data source execution plan
-    pub fn new(
-        schema: SchemaRef,
-        partitions: Vec<Arc<Mutex<dyn RecordBatchReader + Send + Sync>>>,
-    ) -> Self {
-        Self { schema, partitions }
-    }
-}
-
-impl ExecutionPlan for DatasourceExec {
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-
-    fn partitions(&self) -> Result<Vec<Arc<dyn Partition>>> {
-        Ok(self
-            .partitions
-            .iter()
-            .map(|it| {
-                Arc::new(DatasourcePartition::new(it.clone())) as Arc<dyn Partition>
-            })
-            .collect::<Vec<_>>())
-    }
-}
-
-/// Wrapper to convert a `SendableRecordBatchReader` into a `Partition`.
-pub struct DatasourcePartition {
-    batch_iter: Arc<Mutex<dyn RecordBatchReader + Send + Sync>>,
-}
-
-impl Debug for DatasourcePartition {
-    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
-        f.debug_struct("DatasourcePartition").finish()
-    }
-}
-
-impl DatasourcePartition {
-    fn new(batch_iter: Arc<Mutex<dyn RecordBatchReader + Send + Sync>>) -> Self {
-        Self { batch_iter }
-    }
-}
-
-impl Partition for DatasourcePartition {
-    fn execute(&self) -> Result<Arc<Mutex<dyn RecordBatchReader + Send + Sync>>> {
-        Ok(self.batch_iter.clone())
-    }
-}
diff --git a/rust/datafusion/src/execution/physical_plan/expressions.rs b/rust/datafusion/src/execution/physical_plan/expressions.rs
deleted file mode 100644
index dff913b9162..00000000000
--- a/rust/datafusion/src/execution/physical_plan/expressions.rs
+++ /dev/null
@@ -1,2018 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Defines physical expressions that can evaluated at runtime during query execution
-
-use std::cell::RefCell;
-use std::fmt;
-use std::rc::Rc;
-use std::sync::Arc;
-
-use crate::error::{ExecutionError, Result};
-use crate::execution::physical_plan::common::get_scalar_value;
-use crate::execution::physical_plan::{Accumulator, AggregateExpr, PhysicalExpr};
-use crate::logicalplan::{Operator, ScalarValue};
-use arrow::array::{
-    ArrayRef, BooleanArray, Float32Array, Float64Array, Int16Array, Int32Array,
-    Int64Array, Int8Array, StringArray, TimestampNanosecondArray, UInt16Array,
-    UInt32Array, UInt64Array, UInt8Array,
-};
-use arrow::array::{
-    Float32Builder, Float64Builder, Int16Builder, Int32Builder, Int64Builder,
-    Int8Builder, StringBuilder, UInt16Builder, UInt32Builder, UInt64Builder,
-    UInt8Builder,
-};
-use arrow::compute;
-use arrow::compute::kernels::arithmetic::{add, divide, multiply, subtract};
-use arrow::compute::kernels::boolean::{and, or};
-use arrow::compute::kernels::cast::cast;
-use arrow::compute::kernels::comparison::{eq, gt, gt_eq, lt, lt_eq, neq};
-use arrow::compute::kernels::comparison::{
-    eq_utf8, gt_eq_utf8, gt_utf8, like_utf8, lt_eq_utf8, lt_utf8, neq_utf8, nlike_utf8,
-};
-use arrow::compute::kernels::sort::{SortColumn, SortOptions};
-use arrow::datatypes::{DataType, Schema, TimeUnit};
-use arrow::record_batch::RecordBatch;
-
-/// Represents the column at a given index in a RecordBatch
-#[derive(Debug)]
-pub struct Column {
-    name: String,
-}
-
-impl Column {
-    /// Create a new column expression
-    pub fn new(name: &str) -> Self {
-        Self {
-            name: name.to_owned(),
-        }
-    }
-}
-
-impl fmt::Display for Column {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", self.name)
-    }
-}
-
-impl PhysicalExpr for Column {
-    /// Get the data type of this expression, given the schema of the input
-    fn data_type(&self, input_schema: &Schema) -> Result<DataType> {
-        Ok(input_schema
-            .field_with_name(&self.name)?
-            .data_type()
-            .clone())
-    }
-
-    /// Decide whehter this expression is nullable, given the schema of the input
-    fn nullable(&self, input_schema: &Schema) -> Result<bool> {
-        Ok(input_schema.field_with_name(&self.name)?.is_nullable())
-    }
-
-    /// Evaluate the expression
-    fn evaluate(&self, batch: &RecordBatch) -> Result<ArrayRef> {
-        Ok(batch.column(batch.schema().index_of(&self.name)?).clone())
-    }
-}
-
-/// Create a column expression
-pub fn col(name: &str) -> Arc<dyn PhysicalExpr> {
-    Arc::new(Column::new(name))
-}
-
-/// SUM aggregate expression
-#[derive(Debug)]
-pub struct Sum {
-    expr: Arc<dyn PhysicalExpr>,
-}
-
-impl Sum {
-    /// Create a new SUM aggregate function
-    pub fn new(expr: Arc<dyn PhysicalExpr>) -> Self {
-        Self { expr }
-    }
-}
-
-impl AggregateExpr for Sum {
-    fn data_type(&self, input_schema: &Schema) -> Result<DataType> {
-        match self.expr.data_type(input_schema)? {
-            DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::Int64 => {
-                Ok(DataType::Int64)
-            }
-            DataType::UInt8 | DataType::UInt16 | DataType::UInt32 | DataType::UInt64 => {
-                Ok(DataType::UInt64)
-            }
-            DataType::Float32 => Ok(DataType::Float32),
-            DataType::Float64 => Ok(DataType::Float64),
-            other => Err(ExecutionError::General(format!(
-                "SUM does not support {:?}",
-                other
-            ))),
-        }
-    }
-
-    fn evaluate_input(&self, batch: &RecordBatch) -> Result<ArrayRef> {
-        self.expr.evaluate(batch)
-    }
-
-    fn create_accumulator(&self) -> Rc<RefCell<dyn Accumulator>> {
-        Rc::new(RefCell::new(SumAccumulator { sum: None }))
-    }
-
-    fn create_reducer(&self, column_name: &str) -> Arc<dyn AggregateExpr> {
-        Arc::new(Sum::new(Arc::new(Column::new(column_name))))
-    }
-}
-
-macro_rules! sum_accumulate {
-    ($SELF:ident, $VALUE:expr, $ARRAY_TYPE:ident, $SCALAR_VARIANT:ident, $TY:ty) => {{
-        $SELF.sum = match $SELF.sum {
-            Some(ScalarValue::$SCALAR_VARIANT(n)) => {
-                Some(ScalarValue::$SCALAR_VARIANT(n + $VALUE as $TY))
-            }
-            Some(_) => {
-                return Err(ExecutionError::InternalError(
-                    "Unexpected ScalarValue variant".to_string(),
-                ))
-            }
-            None => Some(ScalarValue::$SCALAR_VARIANT($VALUE as $TY)),
-        };
-    }};
-}
-
-#[derive(Debug)]
-struct SumAccumulator {
-    sum: Option<ScalarValue>,
-}
-
-impl Accumulator for SumAccumulator {
-    fn accumulate_scalar(&mut self, value: Option<ScalarValue>) -> Result<()> {
-        if let Some(value) = value {
-            match value {
-                ScalarValue::Int8(value) => {
-                    sum_accumulate!(self, value, Int8Array, Int64, i64);
-                }
-                ScalarValue::Int16(value) => {
-                    sum_accumulate!(self, value, Int16Array, Int64, i64);
-                }
-                ScalarValue::Int32(value) => {
-                    sum_accumulate!(self, value, Int32Array, Int64, i64);
-                }
-                ScalarValue::Int64(value) => {
-                    sum_accumulate!(self, value, Int64Array, Int64, i64);
-                }
-                ScalarValue::UInt8(value) => {
-                    sum_accumulate!(self, value, UInt8Array, UInt64, u64);
-                }
-                ScalarValue::UInt16(value) => {
-                    sum_accumulate!(self, value, UInt16Array, UInt64, u64);
-                }
-                ScalarValue::UInt32(value) => {
-                    sum_accumulate!(self, value, UInt32Array, UInt64, u64);
-                }
-                ScalarValue::UInt64(value) => {
-                    sum_accumulate!(self, value, UInt64Array, UInt64, u64);
-                }
-                ScalarValue::Float32(value) => {
-                    sum_accumulate!(self, value, Float32Array, Float32, f32);
-                }
-                ScalarValue::Float64(value) => {
-                    sum_accumulate!(self, value, Float64Array, Float64, f64);
-                }
-                other => {
-                    return Err(ExecutionError::General(format!(
-                        "SUM does not support {:?}",
-                        other
-                    )))
-                }
-            }
-        }
-        Ok(())
-    }
-
-    fn accumulate_batch(&mut self, array: &ArrayRef) -> Result<()> {
-        let sum = match array.data_type() {
-            DataType::UInt8 => {
-                match compute::sum(array.as_any().downcast_ref::<UInt8Array>().unwrap()) {
-                    Some(n) => Ok(Some(ScalarValue::UInt8(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::UInt16 => {
-                match compute::sum(array.as_any().downcast_ref::<UInt16Array>().unwrap())
-                {
-                    Some(n) => Ok(Some(ScalarValue::UInt16(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::UInt32 => {
-                match compute::sum(array.as_any().downcast_ref::<UInt32Array>().unwrap())
-                {
-                    Some(n) => Ok(Some(ScalarValue::UInt32(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::UInt64 => {
-                match compute::sum(array.as_any().downcast_ref::<UInt64Array>().unwrap())
-                {
-                    Some(n) => Ok(Some(ScalarValue::UInt64(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::Int8 => {
-                match compute::sum(array.as_any().downcast_ref::<Int8Array>().unwrap()) {
-                    Some(n) => Ok(Some(ScalarValue::Int8(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::Int16 => {
-                match compute::sum(array.as_any().downcast_ref::<Int16Array>().unwrap()) {
-                    Some(n) => Ok(Some(ScalarValue::Int16(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::Int32 => {
-                match compute::sum(array.as_any().downcast_ref::<Int32Array>().unwrap()) {
-                    Some(n) => Ok(Some(ScalarValue::Int32(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::Int64 => {
-                match compute::sum(array.as_any().downcast_ref::<Int64Array>().unwrap()) {
-                    Some(n) => Ok(Some(ScalarValue::Int64(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::Float32 => {
-                match compute::sum(array.as_any().downcast_ref::<Float32Array>().unwrap())
-                {
-                    Some(n) => Ok(Some(ScalarValue::Float32(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::Float64 => {
-                match compute::sum(array.as_any().downcast_ref::<Float64Array>().unwrap())
-                {
-                    Some(n) => Ok(Some(ScalarValue::Float64(n))),
-                    None => Ok(None),
-                }
-            }
-            _ => Err(ExecutionError::ExecutionError(
-                "Unsupported data type for SUM".to_string(),
-            )),
-        }?;
-        self.accumulate_scalar(sum)
-    }
-
-    fn get_value(&self) -> Result<Option<ScalarValue>> {
-        Ok(self.sum.clone())
-    }
-}
-
-/// Create a sum expression
-pub fn sum(expr: Arc<dyn PhysicalExpr>) -> Arc<dyn AggregateExpr> {
-    Arc::new(Sum::new(expr))
-}
-
-/// AVG aggregate expression
-#[derive(Debug)]
-pub struct Avg {
-    expr: Arc<dyn PhysicalExpr>,
-}
-
-impl Avg {
-    /// Create a new AVG aggregate function
-    pub fn new(expr: Arc<dyn PhysicalExpr>) -> Self {
-        Self { expr }
-    }
-}
-
-impl AggregateExpr for Avg {
-    fn data_type(&self, input_schema: &Schema) -> Result<DataType> {
-        match self.expr.data_type(input_schema)? {
-            DataType::Int8
-            | DataType::Int16
-            | DataType::Int32
-            | DataType::Int64
-            | DataType::UInt8
-            | DataType::UInt16
-            | DataType::UInt32
-            | DataType::UInt64
-            | DataType::Float32
-            | DataType::Float64 => Ok(DataType::Float64),
-            other => Err(ExecutionError::General(format!(
-                "AVG does not support {:?}",
-                other
-            ))),
-        }
-    }
-
-    fn evaluate_input(&self, batch: &RecordBatch) -> Result<ArrayRef> {
-        self.expr.evaluate(batch)
-    }
-
-    fn create_accumulator(&self) -> Rc<RefCell<dyn Accumulator>> {
-        Rc::new(RefCell::new(AvgAccumulator {
-            sum: None,
-            count: None,
-        }))
-    }
-
-    fn create_reducer(&self, column_name: &str) -> Arc<dyn AggregateExpr> {
-        Arc::new(Avg::new(Arc::new(Column::new(column_name))))
-    }
-}
-
-macro_rules! avg_accumulate {
-    ($SELF:ident, $VALUE:expr, $ARRAY_TYPE:ident) => {{
-        match ($SELF.sum, $SELF.count) {
-            (Some(sum), Some(count)) => {
-                $SELF.sum = Some(sum + $VALUE as f64);
-                $SELF.count = Some(count + 1);
-            }
-            _ => {
-                $SELF.sum = Some($VALUE as f64);
-                $SELF.count = Some(1);
-            }
-        };
-    }};
-}
-#[derive(Debug)]
-struct AvgAccumulator {
-    sum: Option<f64>,
-    count: Option<i64>,
-}
-
-impl Accumulator for AvgAccumulator {
-    fn accumulate_scalar(&mut self, value: Option<ScalarValue>) -> Result<()> {
-        if let Some(value) = value {
-            match value {
-                ScalarValue::Int8(value) => avg_accumulate!(self, value, Int8Array),
-                ScalarValue::Int16(value) => avg_accumulate!(self, value, Int16Array),
-                ScalarValue::Int32(value) => avg_accumulate!(self, value, Int32Array),
-                ScalarValue::Int64(value) => avg_accumulate!(self, value, Int64Array),
-                ScalarValue::UInt8(value) => avg_accumulate!(self, value, UInt8Array),
-                ScalarValue::UInt16(value) => avg_accumulate!(self, value, UInt16Array),
-                ScalarValue::UInt32(value) => avg_accumulate!(self, value, UInt32Array),
-                ScalarValue::UInt64(value) => avg_accumulate!(self, value, UInt64Array),
-                ScalarValue::Float32(value) => avg_accumulate!(self, value, Float32Array),
-                ScalarValue::Float64(value) => avg_accumulate!(self, value, Float64Array),
-                other => {
-                    return Err(ExecutionError::General(format!(
-                        "AVG does not support {:?}",
-                        other
-                    )))
-                }
-            }
-        }
-        Ok(())
-    }
-
-    fn accumulate_batch(&mut self, array: &ArrayRef) -> Result<()> {
-        for row in 0..array.len() {
-            self.accumulate_scalar(get_scalar_value(array, row)?)?;
-        }
-        Ok(())
-    }
-
-    fn get_value(&self) -> Result<Option<ScalarValue>> {
-        match (self.sum, self.count) {
-            (Some(sum), Some(count)) => {
-                Ok(Some(ScalarValue::Float64(sum / count as f64)))
-            }
-            _ => Ok(None),
-        }
-    }
-}
-
-/// Create a avg expression
-pub fn avg(expr: Arc<dyn PhysicalExpr>) -> Arc<dyn AggregateExpr> {
-    Arc::new(Avg::new(expr))
-}
-
-/// MAX aggregate expression
-#[derive(Debug)]
-pub struct Max {
-    expr: Arc<dyn PhysicalExpr>,
-}
-
-impl Max {
-    /// Create a new MAX aggregate function
-    pub fn new(expr: Arc<dyn PhysicalExpr>) -> Self {
-        Self { expr }
-    }
-}
-
-impl AggregateExpr for Max {
-    fn data_type(&self, input_schema: &Schema) -> Result<DataType> {
-        self.expr.data_type(input_schema)
-    }
-
-    fn evaluate_input(&self, batch: &RecordBatch) -> Result<ArrayRef> {
-        self.expr.evaluate(batch)
-    }
-
-    fn create_accumulator(&self) -> Rc<RefCell<dyn Accumulator>> {
-        Rc::new(RefCell::new(MaxAccumulator { max: None }))
-    }
-
-    fn create_reducer(&self, column_name: &str) -> Arc<dyn AggregateExpr> {
-        Arc::new(Max::new(Arc::new(Column::new(column_name))))
-    }
-}
-
-macro_rules! max_accumulate {
-    ($SELF:ident, $VALUE:expr, $ARRAY_TYPE:ident, $SCALAR_VARIANT:ident) => {{
-        $SELF.max = match $SELF.max {
-            Some(ScalarValue::$SCALAR_VARIANT(n)) => {
-                if n > ($VALUE) {
-                    Some(ScalarValue::$SCALAR_VARIANT(n))
-                } else {
-                    Some(ScalarValue::$SCALAR_VARIANT($VALUE))
-                }
-            }
-            Some(_) => {
-                return Err(ExecutionError::InternalError(
-                    "Unexpected ScalarValue variant".to_string(),
-                ))
-            }
-            None => Some(ScalarValue::$SCALAR_VARIANT($VALUE)),
-        };
-    }};
-}
-#[derive(Debug)]
-struct MaxAccumulator {
-    max: Option<ScalarValue>,
-}
-
-impl Accumulator for MaxAccumulator {
-    fn accumulate_scalar(&mut self, value: Option<ScalarValue>) -> Result<()> {
-        if let Some(value) = value {
-            match value {
-                ScalarValue::Int8(value) => {
-                    max_accumulate!(self, value, Int8Array, Int8);
-                }
-                ScalarValue::Int16(value) => {
-                    max_accumulate!(self, value, Int16Array, Int16)
-                }
-                ScalarValue::Int32(value) => {
-                    max_accumulate!(self, value, Int32Array, Int32)
-                }
-                ScalarValue::Int64(value) => {
-                    max_accumulate!(self, value, Int64Array, Int64)
-                }
-                ScalarValue::UInt8(value) => {
-                    max_accumulate!(self, value, UInt8Array, UInt8)
-                }
-                ScalarValue::UInt16(value) => {
-                    max_accumulate!(self, value, UInt16Array, UInt16)
-                }
-                ScalarValue::UInt32(value) => {
-                    max_accumulate!(self, value, UInt32Array, UInt32)
-                }
-                ScalarValue::UInt64(value) => {
-                    max_accumulate!(self, value, UInt64Array, UInt64)
-                }
-                ScalarValue::Float32(value) => {
-                    max_accumulate!(self, value, Float32Array, Float32)
-                }
-                ScalarValue::Float64(value) => {
-                    max_accumulate!(self, value, Float64Array, Float64)
-                }
-                other => {
-                    return Err(ExecutionError::General(format!(
-                        "MAX does not support {:?}",
-                        other
-                    )))
-                }
-            }
-        }
-        Ok(())
-    }
-
-    fn accumulate_batch(&mut self, array: &ArrayRef) -> Result<()> {
-        let max = match array.data_type() {
-            DataType::UInt8 => {
-                match compute::max(array.as_any().downcast_ref::<UInt8Array>().unwrap()) {
-                    Some(n) => Ok(Some(ScalarValue::UInt8(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::UInt16 => {
-                match compute::max(array.as_any().downcast_ref::<UInt16Array>().unwrap())
-                {
-                    Some(n) => Ok(Some(ScalarValue::UInt16(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::UInt32 => {
-                match compute::max(array.as_any().downcast_ref::<UInt32Array>().unwrap())
-                {
-                    Some(n) => Ok(Some(ScalarValue::UInt32(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::UInt64 => {
-                match compute::max(array.as_any().downcast_ref::<UInt64Array>().unwrap())
-                {
-                    Some(n) => Ok(Some(ScalarValue::UInt64(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::Int8 => {
-                match compute::max(array.as_any().downcast_ref::<Int8Array>().unwrap()) {
-                    Some(n) => Ok(Some(ScalarValue::Int8(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::Int16 => {
-                match compute::max(array.as_any().downcast_ref::<Int16Array>().unwrap()) {
-                    Some(n) => Ok(Some(ScalarValue::Int16(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::Int32 => {
-                match compute::max(array.as_any().downcast_ref::<Int32Array>().unwrap()) {
-                    Some(n) => Ok(Some(ScalarValue::Int32(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::Int64 => {
-                match compute::max(array.as_any().downcast_ref::<Int64Array>().unwrap()) {
-                    Some(n) => Ok(Some(ScalarValue::Int64(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::Float32 => {
-                match compute::max(array.as_any().downcast_ref::<Float32Array>().unwrap())
-                {
-                    Some(n) => Ok(Some(ScalarValue::Float32(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::Float64 => {
-                match compute::max(array.as_any().downcast_ref::<Float64Array>().unwrap())
-                {
-                    Some(n) => Ok(Some(ScalarValue::Float64(n))),
-                    None => Ok(None),
-                }
-            }
-            _ => Err(ExecutionError::ExecutionError(
-                "Unsupported data type for MAX".to_string(),
-            )),
-        }?;
-        self.accumulate_scalar(max)
-    }
-
-    fn get_value(&self) -> Result<Option<ScalarValue>> {
-        Ok(self.max.clone())
-    }
-}
-
-/// Create a max expression
-pub fn max(expr: Arc<dyn PhysicalExpr>) -> Arc<dyn AggregateExpr> {
-    Arc::new(Max::new(expr))
-}
-
-/// MIN aggregate expression
-#[derive(Debug)]
-pub struct Min {
-    expr: Arc<dyn PhysicalExpr>,
-}
-
-impl Min {
-    /// Create a new MIN aggregate function
-    pub fn new(expr: Arc<dyn PhysicalExpr>) -> Self {
-        Self { expr }
-    }
-}
-
-impl AggregateExpr for Min {
-    fn data_type(&self, input_schema: &Schema) -> Result<DataType> {
-        self.expr.data_type(input_schema)
-    }
-
-    fn evaluate_input(&self, batch: &RecordBatch) -> Result<ArrayRef> {
-        self.expr.evaluate(batch)
-    }
-
-    fn create_accumulator(&self) -> Rc<RefCell<dyn Accumulator>> {
-        Rc::new(RefCell::new(MinAccumulator { min: None }))
-    }
-
-    fn create_reducer(&self, column_name: &str) -> Arc<dyn AggregateExpr> {
-        Arc::new(Min::new(Arc::new(Column::new(column_name))))
-    }
-}
-
-macro_rules! min_accumulate {
-    ($SELF:ident, $VALUE:expr, $ARRAY_TYPE:ident, $SCALAR_VARIANT:ident) => {{
-        $SELF.min = match $SELF.min {
-            Some(ScalarValue::$SCALAR_VARIANT(n)) => {
-                if n < ($VALUE) {
-                    Some(ScalarValue::$SCALAR_VARIANT(n))
-                } else {
-                    Some(ScalarValue::$SCALAR_VARIANT($VALUE))
-                }
-            }
-            Some(_) => {
-                return Err(ExecutionError::InternalError(
-                    "Unexpected ScalarValue variant".to_string(),
-                ))
-            }
-            None => Some(ScalarValue::$SCALAR_VARIANT($VALUE)),
-        };
-    }};
-}
-#[derive(Debug)]
-struct MinAccumulator {
-    min: Option<ScalarValue>,
-}
-
-impl Accumulator for MinAccumulator {
-    fn accumulate_scalar(&mut self, value: Option<ScalarValue>) -> Result<()> {
-        if let Some(value) = value {
-            match value {
-                ScalarValue::Int8(value) => {
-                    min_accumulate!(self, value, Int8Array, Int8);
-                }
-                ScalarValue::Int16(value) => {
-                    min_accumulate!(self, value, Int16Array, Int16)
-                }
-                ScalarValue::Int32(value) => {
-                    min_accumulate!(self, value, Int32Array, Int32)
-                }
-                ScalarValue::Int64(value) => {
-                    min_accumulate!(self, value, Int64Array, Int64)
-                }
-                ScalarValue::UInt8(value) => {
-                    min_accumulate!(self, value, UInt8Array, UInt8)
-                }
-                ScalarValue::UInt16(value) => {
-                    min_accumulate!(self, value, UInt16Array, UInt16)
-                }
-                ScalarValue::UInt32(value) => {
-                    min_accumulate!(self, value, UInt32Array, UInt32)
-                }
-                ScalarValue::UInt64(value) => {
-                    min_accumulate!(self, value, UInt64Array, UInt64)
-                }
-                ScalarValue::Float32(value) => {
-                    min_accumulate!(self, value, Float32Array, Float32)
-                }
-                ScalarValue::Float64(value) => {
-                    min_accumulate!(self, value, Float64Array, Float64)
-                }
-                other => {
-                    return Err(ExecutionError::General(format!(
-                        "MIN does not support {:?}",
-                        other
-                    )))
-                }
-            }
-        }
-        Ok(())
-    }
-
-    fn accumulate_batch(&mut self, array: &ArrayRef) -> Result<()> {
-        let min = match array.data_type() {
-            DataType::UInt8 => {
-                match compute::min(array.as_any().downcast_ref::<UInt8Array>().unwrap()) {
-                    Some(n) => Ok(Some(ScalarValue::UInt8(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::UInt16 => {
-                match compute::min(array.as_any().downcast_ref::<UInt16Array>().unwrap())
-                {
-                    Some(n) => Ok(Some(ScalarValue::UInt16(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::UInt32 => {
-                match compute::min(array.as_any().downcast_ref::<UInt32Array>().unwrap())
-                {
-                    Some(n) => Ok(Some(ScalarValue::UInt32(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::UInt64 => {
-                match compute::min(array.as_any().downcast_ref::<UInt64Array>().unwrap())
-                {
-                    Some(n) => Ok(Some(ScalarValue::UInt64(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::Int8 => {
-                match compute::min(array.as_any().downcast_ref::<Int8Array>().unwrap()) {
-                    Some(n) => Ok(Some(ScalarValue::Int8(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::Int16 => {
-                match compute::min(array.as_any().downcast_ref::<Int16Array>().unwrap()) {
-                    Some(n) => Ok(Some(ScalarValue::Int16(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::Int32 => {
-                match compute::min(array.as_any().downcast_ref::<Int32Array>().unwrap()) {
-                    Some(n) => Ok(Some(ScalarValue::Int32(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::Int64 => {
-                match compute::min(array.as_any().downcast_ref::<Int64Array>().unwrap()) {
-                    Some(n) => Ok(Some(ScalarValue::Int64(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::Float32 => {
-                match compute::min(array.as_any().downcast_ref::<Float32Array>().unwrap())
-                {
-                    Some(n) => Ok(Some(ScalarValue::Float32(n))),
-                    None => Ok(None),
-                }
-            }
-            DataType::Float64 => {
-                match compute::min(array.as_any().downcast_ref::<Float64Array>().unwrap())
-                {
-                    Some(n) => Ok(Some(ScalarValue::Float64(n))),
-                    None => Ok(None),
-                }
-            }
-            _ => Err(ExecutionError::ExecutionError(
-                "Unsupported data type for MIN".to_string(),
-            )),
-        }?;
-        self.accumulate_scalar(min)
-    }
-
-    fn get_value(&self) -> Result<Option<ScalarValue>> {
-        Ok(self.min.clone())
-    }
-}
-
-/// Create a min expression
-pub fn min(expr: Arc<dyn PhysicalExpr>) -> Arc<dyn AggregateExpr> {
-    Arc::new(Min::new(expr))
-}
-
-/// COUNT aggregate expression
-/// Returns the amount of non-null values of the given expression.
-#[derive(Debug)]
-pub struct Count {
-    expr: Arc<dyn PhysicalExpr>,
-}
-
-impl Count {
-    /// Create a new COUNT aggregate function.
-    pub fn new(expr: Arc<dyn PhysicalExpr>) -> Self {
-        Self { expr: expr }
-    }
-}
-
-impl AggregateExpr for Count {
-    fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
-        Ok(DataType::UInt64)
-    }
-
-    fn evaluate_input(&self, batch: &RecordBatch) -> Result<ArrayRef> {
-        self.expr.evaluate(batch)
-    }
-
-    fn create_accumulator(&self) -> Rc<RefCell<dyn Accumulator>> {
-        Rc::new(RefCell::new(CountAccumulator { count: 0 }))
-    }
-
-    fn create_reducer(&self, column_name: &str) -> Arc<dyn AggregateExpr> {
-        Arc::new(Sum::new(Arc::new(Column::new(column_name))))
-    }
-}
-
-#[derive(Debug)]
-struct CountAccumulator {
-    count: u64,
-}
-
-impl Accumulator for CountAccumulator {
-    fn accumulate_scalar(&mut self, value: Option<ScalarValue>) -> Result<()> {
-        if value.is_some() {
-            self.count += 1;
-        }
-        Ok(())
-    }
-
-    fn accumulate_batch(&mut self, array: &ArrayRef) -> Result<()> {
-        self.count += array.len() as u64 - array.null_count() as u64;
-        Ok(())
-    }
-
-    fn get_value(&self) -> Result<Option<ScalarValue>> {
-        Ok(Some(ScalarValue::UInt64(self.count)))
-    }
-}
-
-/// Create a count expression
-pub fn count(expr: Arc<dyn PhysicalExpr>) -> Arc<dyn AggregateExpr> {
-    Arc::new(Count::new(expr))
-}
-
-/// Invoke a compute kernel on a pair of binary data arrays
-macro_rules! compute_utf8_op {
-    ($LEFT:expr, $RIGHT:expr, $OP:ident, $DT:ident) => {{
-        let ll = $LEFT
-            .as_any()
-            .downcast_ref::<$DT>()
-            .expect("compute_op failed to downcast array");
-        let rr = $RIGHT
-            .as_any()
-            .downcast_ref::<$DT>()
-            .expect("compute_op failed to downcast array");
-        Ok(Arc::new(paste::expr! {[<$OP _utf8>]}(&ll, &rr)?))
-    }};
-}
-
-/// Invoke a compute kernel on a pair of arrays
-macro_rules! compute_op {
-    ($LEFT:expr, $RIGHT:expr, $OP:ident, $DT:ident) => {{
-        let ll = $LEFT
-            .as_any()
-            .downcast_ref::<$DT>()
-            .expect("compute_op failed to downcast array");
-        let rr = $RIGHT
-            .as_any()
-            .downcast_ref::<$DT>()
-            .expect("compute_op failed to downcast array");
-        Ok(Arc::new($OP(&ll, &rr)?))
-    }};
-}
-
-macro_rules! binary_string_array_op {
-    ($LEFT:expr, $RIGHT:expr, $OP:ident) => {{
-        match $LEFT.data_type() {
-            DataType::Utf8 => compute_utf8_op!($LEFT, $RIGHT, $OP, StringArray),
-            other => Err(ExecutionError::General(format!(
-                "Unsupported data type {:?}",
-                other
-            ))),
-        }
-    }};
-}
-
-/// Invoke a compute kernel on a pair of arrays
-/// The binary_primitive_array_op macro only evaluates for primitive types
-/// like integers and floats.
-macro_rules! binary_primitive_array_op {
-    ($LEFT:expr, $RIGHT:expr, $OP:ident) => {{
-        match $LEFT.data_type() {
-            DataType::Int8 => compute_op!($LEFT, $RIGHT, $OP, Int8Array),
-            DataType::Int16 => compute_op!($LEFT, $RIGHT, $OP, Int16Array),
-            DataType::Int32 => compute_op!($LEFT, $RIGHT, $OP, Int32Array),
-            DataType::Int64 => compute_op!($LEFT, $RIGHT, $OP, Int64Array),
-            DataType::UInt8 => compute_op!($LEFT, $RIGHT, $OP, UInt8Array),
-            DataType::UInt16 => compute_op!($LEFT, $RIGHT, $OP, UInt16Array),
-            DataType::UInt32 => compute_op!($LEFT, $RIGHT, $OP, UInt32Array),
-            DataType::UInt64 => compute_op!($LEFT, $RIGHT, $OP, UInt64Array),
-            DataType::Float32 => compute_op!($LEFT, $RIGHT, $OP, Float32Array),
-            DataType::Float64 => compute_op!($LEFT, $RIGHT, $OP, Float64Array),
-            other => Err(ExecutionError::General(format!(
-                "Unsupported data type {:?}",
-                other
-            ))),
-        }
-    }};
-}
-
-/// The binary_array_op macro includes types that extend beyond the primitive,
-/// such as Utf8 strings.
-macro_rules! binary_array_op {
-    ($LEFT:expr, $RIGHT:expr, $OP:ident) => {{
-        match $LEFT.data_type() {
-            DataType::Int8 => compute_op!($LEFT, $RIGHT, $OP, Int8Array),
-            DataType::Int16 => compute_op!($LEFT, $RIGHT, $OP, Int16Array),
-            DataType::Int32 => compute_op!($LEFT, $RIGHT, $OP, Int32Array),
-            DataType::Int64 => compute_op!($LEFT, $RIGHT, $OP, Int64Array),
-            DataType::UInt8 => compute_op!($LEFT, $RIGHT, $OP, UInt8Array),
-            DataType::UInt16 => compute_op!($LEFT, $RIGHT, $OP, UInt16Array),
-            DataType::UInt32 => compute_op!($LEFT, $RIGHT, $OP, UInt32Array),
-            DataType::UInt64 => compute_op!($LEFT, $RIGHT, $OP, UInt64Array),
-            DataType::Float32 => compute_op!($LEFT, $RIGHT, $OP, Float32Array),
-            DataType::Float64 => compute_op!($LEFT, $RIGHT, $OP, Float64Array),
-            DataType::Utf8 => compute_utf8_op!($LEFT, $RIGHT, $OP, StringArray),
-            DataType::Timestamp(TimeUnit::Nanosecond, None) => {
-                compute_op!($LEFT, $RIGHT, $OP, TimestampNanosecondArray)
-            }
-            other => Err(ExecutionError::General(format!(
-                "Unsupported data type {:?}",
-                other
-            ))),
-        }
-    }};
-}
-
-/// Invoke a boolean kernel on a pair of arrays
-macro_rules! boolean_op {
-    ($LEFT:expr, $RIGHT:expr, $OP:ident) => {{
-        let ll = $LEFT
-            .as_any()
-            .downcast_ref::<BooleanArray>()
-            .expect("boolean_op failed to downcast array");
-        let rr = $RIGHT
-            .as_any()
-            .downcast_ref::<BooleanArray>()
-            .expect("boolean_op failed to downcast array");
-        Ok(Arc::new($OP(&ll, &rr)?))
-    }};
-}
-/// Binary expression
-#[derive(Debug)]
-pub struct BinaryExpr {
-    left: Arc<dyn PhysicalExpr>,
-    op: Operator,
-    right: Arc<dyn PhysicalExpr>,
-}
-
-impl BinaryExpr {
-    /// Create new binary expression
-    pub fn new(
-        left: Arc<dyn PhysicalExpr>,
-        op: Operator,
-        right: Arc<dyn PhysicalExpr>,
-    ) -> Self {
-        Self { left, op, right }
-    }
-}
-
-impl fmt::Display for BinaryExpr {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{} {} {}", self.left, self.op, self.right)
-    }
-}
-
-impl PhysicalExpr for BinaryExpr {
-    fn data_type(&self, input_schema: &Schema) -> Result<DataType> {
-        self.left.data_type(input_schema)
-    }
-
-    fn nullable(&self, _input_schema: &Schema) -> Result<bool> {
-        // binary operator should always return a boolean value
-        Ok(false)
-    }
-
-    fn evaluate(&self, batch: &RecordBatch) -> Result<ArrayRef> {
-        let left = self.left.evaluate(batch)?;
-        let right = self.right.evaluate(batch)?;
-        if left.data_type() != right.data_type() {
-            return Err(ExecutionError::General(format!(
-                "Cannot evaluate binary expression {:?} with types {:?} and {:?}",
-                self.op,
-                left.data_type(),
-                right.data_type()
-            )));
-        }
-        match &self.op {
-            Operator::Like => binary_string_array_op!(left, right, like),
-            Operator::NotLike => binary_string_array_op!(left, right, nlike),
-            Operator::Lt => binary_array_op!(left, right, lt),
-            Operator::LtEq => binary_array_op!(left, right, lt_eq),
-            Operator::Gt => binary_array_op!(left, right, gt),
-            Operator::GtEq => binary_array_op!(left, right, gt_eq),
-            Operator::Eq => binary_array_op!(left, right, eq),
-            Operator::NotEq => binary_array_op!(left, right, neq),
-            Operator::Plus => binary_primitive_array_op!(left, right, add),
-            Operator::Minus => binary_primitive_array_op!(left, right, subtract),
-            Operator::Multiply => binary_primitive_array_op!(left, right, multiply),
-            Operator::Divide => binary_primitive_array_op!(left, right, divide),
-            Operator::And => {
-                if left.data_type() == &DataType::Boolean {
-                    boolean_op!(left, right, and)
-                } else {
-                    return Err(ExecutionError::General(format!(
-                        "Cannot evaluate binary expression {:?} with types {:?} and {:?}",
-                        self.op,
-                        left.data_type(),
-                        right.data_type()
-                    )));
-                }
-            }
-            Operator::Or => {
-                if left.data_type() == &DataType::Boolean {
-                    boolean_op!(left, right, or)
-                } else {
-                    return Err(ExecutionError::General(format!(
-                        "Cannot evaluate binary expression {:?} with types {:?} and {:?}",
-                        self.op,
-                        left.data_type(),
-                        right.data_type()
-                    )));
-                }
-            }
-            _ => Err(ExecutionError::General("Unsupported operator".to_string())),
-        }
-    }
-}
-
-/// Create a binary expression
-pub fn binary(
-    l: Arc<dyn PhysicalExpr>,
-    op: Operator,
-    r: Arc<dyn PhysicalExpr>,
-) -> Arc<dyn PhysicalExpr> {
-    Arc::new(BinaryExpr::new(l, op, r))
-}
-
-/// Not expression
-#[derive(Debug)]
-pub struct NotExpr {
-    arg: Arc<dyn PhysicalExpr>,
-}
-
-impl NotExpr {
-    /// Create new not expression
-    pub fn new(arg: Arc<dyn PhysicalExpr>) -> Self {
-        Self { arg }
-    }
-}
-
-impl fmt::Display for NotExpr {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "NOT {}", self.arg)
-    }
-}
-impl PhysicalExpr for NotExpr {
-    fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
-        return Ok(DataType::Boolean);
-    }
-
-    fn nullable(&self, _input_schema: &Schema) -> Result<bool> {
-        // !Null == true
-        Ok(false)
-    }
-
-    fn evaluate(&self, batch: &RecordBatch) -> Result<ArrayRef> {
-        let arg = self.arg.evaluate(batch)?;
-        if arg.data_type() != &DataType::Boolean {
-            return Err(ExecutionError::General(format!(
-                "Cannot evaluate \"not\" expression with type {:?}",
-                arg.data_type(),
-            )));
-        }
-        let arg = arg
-            .as_any()
-            .downcast_ref::<BooleanArray>()
-            .expect("boolean_op failed to downcast array");
-        return Ok(Arc::new(arrow::compute::kernels::boolean::not(arg)?));
-    }
-}
-
-/// Create a unary expression
-pub fn not(arg: Arc<dyn PhysicalExpr>) -> Arc<dyn PhysicalExpr> {
-    Arc::new(NotExpr::new(arg))
-}
-
-/// CAST expression casts an expression to a specific data type
-#[derive(Debug)]
-pub struct CastExpr {
-    /// The expression to cast
-    expr: Arc<dyn PhysicalExpr>,
-    /// The data type to cast to
-    cast_type: DataType,
-}
-
-/// Determine if a DataType is numeric or not
-fn is_numeric(dt: &DataType) -> bool {
-    match dt {
-        DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::Int64 => true,
-        DataType::UInt8 | DataType::UInt16 | DataType::UInt32 | DataType::UInt64 => true,
-        DataType::Float16 | DataType::Float32 | DataType::Float64 => true,
-        _ => false,
-    }
-}
-
-impl CastExpr {
-    /// Create a CAST expression
-    pub fn try_new(
-        expr: Arc<dyn PhysicalExpr>,
-        input_schema: &Schema,
-        cast_type: DataType,
-    ) -> Result<Self> {
-        let expr_type = expr.data_type(input_schema)?;
-        // numbers can be cast to numbers and strings
-        if is_numeric(&expr_type)
-            && (is_numeric(&cast_type) || cast_type == DataType::Utf8)
-        {
-            Ok(Self { expr, cast_type })
-        } else if expr_type == DataType::Binary && cast_type == DataType::Utf8 {
-            Ok(Self { expr, cast_type })
-        } else if is_numeric(&expr_type)
-            && cast_type == DataType::Timestamp(TimeUnit::Nanosecond, None)
-        {
-            Ok(Self { expr, cast_type })
-        } else {
-            Err(ExecutionError::General(format!(
-                "Invalid CAST from {:?} to {:?}",
-                expr_type, cast_type
-            )))
-        }
-    }
-}
-
-impl fmt::Display for CastExpr {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "CAST({} AS {:?})", self.expr, self.cast_type)
-    }
-}
-
-impl PhysicalExpr for CastExpr {
-    fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
-        Ok(self.cast_type.clone())
-    }
-
-    fn nullable(&self, input_schema: &Schema) -> Result<bool> {
-        self.expr.nullable(input_schema)
-    }
-
-    fn evaluate(&self, batch: &RecordBatch) -> Result<ArrayRef> {
-        let value = self.expr.evaluate(batch)?;
-        Ok(cast(&value, &self.cast_type)?)
-    }
-}
-
-/// Represents a non-null literal value
-#[derive(Debug)]
-pub struct Literal {
-    value: ScalarValue,
-}
-
-impl Literal {
-    /// Create a literal value expression
-    pub fn new(value: ScalarValue) -> Self {
-        Self { value }
-    }
-}
-
-/// Build array containing the same literal value repeated. This is necessary because the Arrow
-/// memory model does not have the concept of a scalar value currently.
-macro_rules! build_literal_array {
-    ($BATCH:ident, $BUILDER:ident, $VALUE:expr) => {{
-        let mut builder = $BUILDER::new($BATCH.num_rows());
-        for _ in 0..$BATCH.num_rows() {
-            builder.append_value($VALUE)?;
-        }
-        Ok(Arc::new(builder.finish()))
-    }};
-}
-
-impl fmt::Display for Literal {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", self.value)
-    }
-}
-
-impl PhysicalExpr for Literal {
-    fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
-        self.value.get_datatype()
-    }
-
-    fn nullable(&self, _input_schema: &Schema) -> Result<bool> {
-        match &self.value {
-            ScalarValue::Null => Ok(true),
-            _ => Ok(false),
-        }
-    }
-
-    fn evaluate(&self, batch: &RecordBatch) -> Result<ArrayRef> {
-        match &self.value {
-            ScalarValue::Int8(value) => build_literal_array!(batch, Int8Builder, *value),
-            ScalarValue::Int16(value) => {
-                build_literal_array!(batch, Int16Builder, *value)
-            }
-            ScalarValue::Int32(value) => {
-                build_literal_array!(batch, Int32Builder, *value)
-            }
-            ScalarValue::Int64(value) => {
-                build_literal_array!(batch, Int64Builder, *value)
-            }
-            ScalarValue::UInt8(value) => {
-                build_literal_array!(batch, UInt8Builder, *value)
-            }
-            ScalarValue::UInt16(value) => {
-                build_literal_array!(batch, UInt16Builder, *value)
-            }
-            ScalarValue::UInt32(value) => {
-                build_literal_array!(batch, UInt32Builder, *value)
-            }
-            ScalarValue::UInt64(value) => {
-                build_literal_array!(batch, UInt64Builder, *value)
-            }
-            ScalarValue::Float32(value) => {
-                build_literal_array!(batch, Float32Builder, *value)
-            }
-            ScalarValue::Float64(value) => {
-                build_literal_array!(batch, Float64Builder, *value)
-            }
-            ScalarValue::Utf8(value) => build_literal_array!(batch, StringBuilder, value),
-            other => Err(ExecutionError::General(format!(
-                "Unsupported literal type {:?}",
-                other
-            ))),
-        }
-    }
-}
-
-/// Create a literal expression
-pub fn lit(value: ScalarValue) -> Arc<dyn PhysicalExpr> {
-    Arc::new(Literal::new(value))
-}
-
-/// Represents Sort operation for a column in a RecordBatch
-#[derive(Clone, Debug)]
-pub struct PhysicalSortExpr {
-    /// Physical expression representing the column to sort
-    pub expr: Arc<dyn PhysicalExpr>,
-    /// Option to specify how the given column should be sorted
-    pub options: SortOptions,
-}
-
-impl PhysicalSortExpr {
-    /// evaluate the sort expression into SortColumn that can be passed into arrow sort kernel
-    pub fn evaluate_to_sort_column(&self, batch: &RecordBatch) -> Result<SortColumn> {
-        Ok(SortColumn {
-            values: self.expr.evaluate(batch)?,
-            options: Some(self.options),
-        })
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::error::Result;
-    use crate::execution::physical_plan::common::get_scalar_value;
-    use arrow::array::{PrimitiveArray, StringArray, Time64NanosecondArray};
-    use arrow::datatypes::*;
-
-    #[test]
-    fn binary_comparison() -> Result<()> {
-        let schema = Schema::new(vec![
-            Field::new("a", DataType::Int32, false),
-            Field::new("b", DataType::Int32, false),
-        ]);
-        let a = Int32Array::from(vec![1, 2, 3, 4, 5]);
-        let b = Int32Array::from(vec![1, 2, 4, 8, 16]);
-        let batch = RecordBatch::try_new(
-            Arc::new(schema.clone()),
-            vec![Arc::new(a), Arc::new(b)],
-        )?;
-
-        // expression: "a < b"
-        let lt = binary(col("a"), Operator::Lt, col("b"));
-        let result = lt.evaluate(&batch)?;
-        assert_eq!(result.len(), 5);
-
-        let expected = vec![false, false, true, true, true];
-        let result = result
-            .as_any()
-            .downcast_ref::<BooleanArray>()
-            .expect("failed to downcast to BooleanArray");
-        for i in 0..5 {
-            assert_eq!(result.value(i), expected[i]);
-        }
-
-        Ok(())
-    }
-
-    #[test]
-    fn binary_nested() -> Result<()> {
-        let schema = Schema::new(vec![
-            Field::new("a", DataType::Int32, false),
-            Field::new("b", DataType::Int32, false),
-        ]);
-        let a = Int32Array::from(vec![2, 4, 6, 8, 10]);
-        let b = Int32Array::from(vec![2, 5, 4, 8, 8]);
-        let batch = RecordBatch::try_new(
-            Arc::new(schema.clone()),
-            vec![Arc::new(a), Arc::new(b)],
-        )?;
-
-        // expression: "a < b OR a == b"
-        let expr = binary(
-            binary(col("a"), Operator::Lt, col("b")),
-            Operator::Or,
-            binary(col("a"), Operator::Eq, col("b")),
-        );
-        assert_eq!("a < b OR a = b", format!("{}", expr));
-
-        let result = expr.evaluate(&batch)?;
-        assert_eq!(result.len(), 5);
-
-        let expected = vec![true, true, false, true, false];
-        let result = result
-            .as_any()
-            .downcast_ref::<BooleanArray>()
-            .expect("failed to downcast to BooleanArray");
-        for i in 0..5 {
-            print!("{}", i);
-            assert_eq!(result.value(i), expected[i]);
-        }
-
-        Ok(())
-    }
-
-    #[test]
-    fn literal_i32() -> Result<()> {
-        // create an arbitrary record bacth
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-        let a = Int32Array::from(vec![Some(1), None, Some(3), Some(4), Some(5)]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        // create and evaluate a literal expression
-        let literal_expr = lit(ScalarValue::Int32(42));
-        assert_eq!("42", format!("{}", literal_expr));
-
-        let literal_array = literal_expr.evaluate(&batch)?;
-        let literal_array = literal_array.as_any().downcast_ref::<Int32Array>().unwrap();
-
-        // note that the contents of the literal array are unrelated to the batch contents except for the length of the array
-        assert_eq!(literal_array.len(), 5); // 5 rows in the batch
-        for i in 0..literal_array.len() {
-            assert_eq!(literal_array.value(i), 42);
-        }
-
-        Ok(())
-    }
-
-    #[test]
-    fn cast_i32_to_u32() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-        let a = Int32Array::from(vec![1, 2, 3, 4, 5]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        let cast = CastExpr::try_new(col("a"), &schema, DataType::UInt32)?;
-        assert_eq!("CAST(a AS UInt32)", format!("{}", cast));
-        let result = cast.evaluate(&batch)?;
-        assert_eq!(result.len(), 5);
-
-        let result = result
-            .as_any()
-            .downcast_ref::<UInt32Array>()
-            .expect("failed to downcast to UInt32Array");
-        assert_eq!(result.value(0), 1_u32);
-
-        Ok(())
-    }
-
-    #[test]
-    fn cast_i32_to_utf8() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-        let a = Int32Array::from(vec![1, 2, 3, 4, 5]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        let cast = CastExpr::try_new(col("a"), &schema, DataType::Utf8)?;
-        let result = cast.evaluate(&batch)?;
-        assert_eq!(result.len(), 5);
-
-        let result = result
-            .as_any()
-            .downcast_ref::<StringArray>()
-            .expect("failed to downcast to StringArray");
-        assert_eq!(result.value(0), "1");
-
-        Ok(())
-    }
-
-    #[test]
-    fn cast_i64_to_timestamp_nanoseconds() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int64, false)]);
-        let a = Int64Array::from(vec![1, 2, 3, 4, 5]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        let cast = CastExpr::try_new(
-            col("a"),
-            &schema,
-            DataType::Timestamp(TimeUnit::Nanosecond, None),
-        )?;
-        let result = cast.evaluate(&batch)?;
-        assert_eq!(result.len(), 5);
-        let expected_result = Time64NanosecondArray::from(vec![1, 2, 3, 4]);
-        let result = result
-            .as_any()
-            .downcast_ref::<TimestampNanosecondArray>()
-            .expect("failed to downcast to TimestampNanosecondArray");
-        assert_eq!(result.value(0), expected_result.value(0));
-
-        Ok(())
-    }
-
-    #[test]
-    fn invalid_cast() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Utf8, false)]);
-        let result = CastExpr::try_new(col("a"), &schema, DataType::Int32);
-        result.expect_err("Invalid CAST from Utf8 to Int32");
-        Ok(())
-    }
-
-    #[test]
-    fn sum_contract() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-
-        let sum = sum(col("a"));
-        assert_eq!(DataType::Int64, sum.data_type(&schema)?);
-
-        // after the aggr expression is applied, the schema changes to:
-        let schema = Schema::new(vec![
-            schema.field(0).clone(),
-            Field::new("SUM(a)", sum.data_type(&schema)?, false),
-        ]);
-
-        let combiner = sum.create_reducer("SUM(a)");
-        assert_eq!(DataType::Int64, combiner.data_type(&schema)?);
-
-        Ok(())
-    }
-
-    #[test]
-    fn max_contract() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-
-        let max = max(col("a"));
-        assert_eq!(DataType::Int32, max.data_type(&schema)?);
-
-        // after the aggr expression is applied, the schema changes to:
-        let schema = Schema::new(vec![
-            schema.field(0).clone(),
-            Field::new("Max(a)", max.data_type(&schema)?, false),
-        ]);
-
-        let combiner = max.create_reducer("Max(a)");
-        assert_eq!(DataType::Int32, combiner.data_type(&schema)?);
-
-        Ok(())
-    }
-
-    #[test]
-    fn min_contract() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-
-        let min = min(col("a"));
-        assert_eq!(DataType::Int32, min.data_type(&schema)?);
-
-        // after the aggr expression is applied, the schema changes to:
-        let schema = Schema::new(vec![
-            schema.field(0).clone(),
-            Field::new("MIN(a)", min.data_type(&schema)?, false),
-        ]);
-        let combiner = min.create_reducer("MIN(a)");
-        assert_eq!(DataType::Int32, combiner.data_type(&schema)?);
-
-        Ok(())
-    }
-    #[test]
-    fn avg_contract() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-
-        let avg = avg(col("a"));
-        assert_eq!(DataType::Float64, avg.data_type(&schema)?);
-
-        // after the aggr expression is applied, the schema changes to:
-        let schema = Schema::new(vec![
-            schema.field(0).clone(),
-            Field::new("SUM(a)", avg.data_type(&schema)?, false),
-        ]);
-
-        let combiner = avg.create_reducer("SUM(a)");
-        assert_eq!(DataType::Float64, combiner.data_type(&schema)?);
-
-        Ok(())
-    }
-
-    #[test]
-    fn sum_i32() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-
-        let a = Int32Array::from(vec![1, 2, 3, 4, 5]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_sum(&batch)?, Some(ScalarValue::Int64(15)));
-
-        Ok(())
-    }
-
-    #[test]
-    fn avg_i32() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-
-        let a = Int32Array::from(vec![1, 2, 3, 4, 5]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_avg(&batch)?, Some(ScalarValue::Float64(3_f64)));
-
-        Ok(())
-    }
-
-    #[test]
-    fn max_i32() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-
-        let a = Int32Array::from(vec![1, 2, 3, 4, 5]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_max(&batch)?, Some(ScalarValue::Int32(5)));
-
-        Ok(())
-    }
-
-    #[test]
-    fn min_i32() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-
-        let a = Int32Array::from(vec![1, 2, 3, 4, 5]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_min(&batch)?, Some(ScalarValue::Int32(1)));
-
-        Ok(())
-    }
-
-    #[test]
-    fn sum_i32_with_nulls() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-
-        let a = Int32Array::from(vec![Some(1), None, Some(3), Some(4), Some(5)]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_sum(&batch)?, Some(ScalarValue::Int64(13)));
-
-        Ok(())
-    }
-
-    #[test]
-    fn avg_i32_with_nulls() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-
-        let a = Int32Array::from(vec![Some(1), None, Some(3), Some(4), Some(5)]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_avg(&batch)?, Some(ScalarValue::Float64(3.25)));
-
-        Ok(())
-    }
-
-    #[test]
-    fn max_i32_with_nulls() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-
-        let a = Int32Array::from(vec![Some(1), None, Some(3), Some(4), Some(5)]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_max(&batch)?, Some(ScalarValue::Int32(5)));
-
-        Ok(())
-    }
-
-    #[test]
-    fn min_i32_with_nulls() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-
-        let a = Int32Array::from(vec![Some(1), None, Some(3), Some(4), Some(5)]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_min(&batch)?, Some(ScalarValue::Int32(1)));
-
-        Ok(())
-    }
-
-    #[test]
-    fn sum_i32_all_nulls() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-
-        let a = Int32Array::from(vec![None, None]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_sum(&batch)?, None);
-
-        Ok(())
-    }
-
-    #[test]
-    fn max_i32_all_nulls() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-
-        let a = Int32Array::from(vec![None, None]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_max(&batch)?, None);
-
-        Ok(())
-    }
-
-    #[test]
-    fn min_i32_all_nulls() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-
-        let a = Int32Array::from(vec![None, None]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_min(&batch)?, None);
-
-        Ok(())
-    }
-
-    #[test]
-    fn avg_i32_all_nulls() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-
-        let a = Int32Array::from(vec![None, None]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_avg(&batch)?, None);
-
-        Ok(())
-    }
-
-    #[test]
-    fn sum_u32() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::UInt32, false)]);
-
-        let a = UInt32Array::from(vec![1_u32, 2_u32, 3_u32, 4_u32, 5_u32]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_sum(&batch)?, Some(ScalarValue::UInt64(15_u64)));
-
-        Ok(())
-    }
-
-    #[test]
-    fn avg_u32() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::UInt32, false)]);
-
-        let a = UInt32Array::from(vec![1_u32, 2_u32, 3_u32, 4_u32, 5_u32]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_avg(&batch)?, Some(ScalarValue::Float64(3_f64)));
-
-        Ok(())
-    }
-
-    #[test]
-    fn max_u32() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::UInt32, false)]);
-
-        let a = UInt32Array::from(vec![1_u32, 2_u32, 3_u32, 4_u32, 5_u32]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_max(&batch)?, Some(ScalarValue::UInt32(5_u32)));
-
-        Ok(())
-    }
-
-    #[test]
-    fn min_u32() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::UInt32, false)]);
-
-        let a = UInt32Array::from(vec![1_u32, 2_u32, 3_u32, 4_u32, 5_u32]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_min(&batch)?, Some(ScalarValue::UInt32(1_u32)));
-
-        Ok(())
-    }
-
-    #[test]
-    fn sum_f32() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Float32, false)]);
-
-        let a = Float32Array::from(vec![1_f32, 2_f32, 3_f32, 4_f32, 5_f32]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_sum(&batch)?, Some(ScalarValue::Float32(15_f32)));
-
-        Ok(())
-    }
-
-    #[test]
-    fn avg_f32() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Float32, false)]);
-
-        let a = Float32Array::from(vec![1_f32, 2_f32, 3_f32, 4_f32, 5_f32]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_avg(&batch)?, Some(ScalarValue::Float64(3_f64)));
-
-        Ok(())
-    }
-
-    #[test]
-    fn max_f32() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Float32, false)]);
-
-        let a = Float32Array::from(vec![1_f32, 2_f32, 3_f32, 4_f32, 5_f32]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_max(&batch)?, Some(ScalarValue::Float32(5_f32)));
-
-        Ok(())
-    }
-
-    #[test]
-    fn min_f32() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Float32, false)]);
-
-        let a = Float32Array::from(vec![1_f32, 2_f32, 3_f32, 4_f32, 5_f32]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_min(&batch)?, Some(ScalarValue::Float32(1_f32)));
-
-        Ok(())
-    }
-
-    #[test]
-    fn sum_f64() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Float64, false)]);
-
-        let a = Float64Array::from(vec![1_f64, 2_f64, 3_f64, 4_f64, 5_f64]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_sum(&batch)?, Some(ScalarValue::Float64(15_f64)));
-
-        Ok(())
-    }
-
-    #[test]
-    fn avg_f64() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Float64, false)]);
-
-        let a = Float64Array::from(vec![1_f64, 2_f64, 3_f64, 4_f64, 5_f64]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_avg(&batch)?, Some(ScalarValue::Float64(3_f64)));
-
-        Ok(())
-    }
-
-    #[test]
-    fn max_f64() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Float64, false)]);
-
-        let a = Float64Array::from(vec![1_f64, 2_f64, 3_f64, 4_f64, 5_f64]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_max(&batch)?, Some(ScalarValue::Float64(5_f64)));
-
-        Ok(())
-    }
-
-    #[test]
-    fn min_f64() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Float64, false)]);
-
-        let a = Float64Array::from(vec![1_f64, 2_f64, 3_f64, 4_f64, 5_f64]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        assert_eq!(do_min(&batch)?, Some(ScalarValue::Float64(1_f64)));
-
-        Ok(())
-    }
-
-    #[test]
-    fn count_elements() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-        let a = Int32Array::from(vec![1, 2, 3, 4, 5]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-        assert_eq!(do_count(&batch)?, Some(ScalarValue::UInt64(5)));
-        Ok(())
-    }
-
-    #[test]
-    fn count_with_nulls() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-        let a = Int32Array::from(vec![Some(1), Some(2), None, None, Some(3), None]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-        assert_eq!(do_count(&batch)?, Some(ScalarValue::UInt64(3)));
-        Ok(())
-    }
-
-    #[test]
-    fn count_all_nulls() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Boolean, false)]);
-        let a = BooleanArray::from(vec![None, None, None, None, None, None, None, None]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-        assert_eq!(do_count(&batch)?, Some(ScalarValue::UInt64(0)));
-        Ok(())
-    }
-
-    #[test]
-    fn count_empty() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Boolean, false)]);
-        let a = BooleanArray::from(Vec::<bool>::new());
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-        assert_eq!(do_count(&batch)?, Some(ScalarValue::UInt64(0)));
-        Ok(())
-    }
-
-    fn do_sum(batch: &RecordBatch) -> Result<Option<ScalarValue>> {
-        let sum = sum(col("a"));
-        let accum = sum.create_accumulator();
-        let input = sum.evaluate_input(batch)?;
-        let mut accum = accum.borrow_mut();
-        for i in 0..batch.num_rows() {
-            accum.accumulate_scalar(get_scalar_value(&input, i)?)?;
-        }
-        accum.get_value()
-    }
-
-    fn do_max(batch: &RecordBatch) -> Result<Option<ScalarValue>> {
-        let max = max(col("a"));
-        let accum = max.create_accumulator();
-        let input = max.evaluate_input(batch)?;
-        let mut accum = accum.borrow_mut();
-        for i in 0..batch.num_rows() {
-            accum.accumulate_scalar(get_scalar_value(&input, i)?)?;
-        }
-        accum.get_value()
-    }
-
-    fn do_min(batch: &RecordBatch) -> Result<Option<ScalarValue>> {
-        let min = min(col("a"));
-        let accum = min.create_accumulator();
-        let input = min.evaluate_input(batch)?;
-        let mut accum = accum.borrow_mut();
-        for i in 0..batch.num_rows() {
-            accum.accumulate_scalar(get_scalar_value(&input, i)?)?;
-        }
-        accum.get_value()
-    }
-
-    fn do_count(batch: &RecordBatch) -> Result<Option<ScalarValue>> {
-        let count = count(col("a"));
-        let accum = count.create_accumulator();
-        let input = count.evaluate_input(batch)?;
-        let mut accum = accum.borrow_mut();
-        for i in 0..batch.num_rows() {
-            accum.accumulate_scalar(get_scalar_value(&input, i)?)?;
-        }
-        accum.get_value()
-    }
-
-    fn do_avg(batch: &RecordBatch) -> Result<Option<ScalarValue>> {
-        let avg = avg(col("a"));
-        let accum = avg.create_accumulator();
-        let input = avg.evaluate_input(batch)?;
-        let mut accum = accum.borrow_mut();
-        for i in 0..batch.num_rows() {
-            accum.accumulate_scalar(get_scalar_value(&input, i)?)?;
-        }
-        accum.get_value()
-    }
-
-    #[test]
-    fn plus_op() -> Result<()> {
-        let schema = Schema::new(vec![
-            Field::new("a", DataType::Int32, false),
-            Field::new("b", DataType::Int32, false),
-        ]);
-        let a = Int32Array::from(vec![1, 2, 3, 4, 5]);
-        let b = Int32Array::from(vec![1, 2, 4, 8, 16]);
-
-        apply_arithmetic::<Int32Type>(
-            Arc::new(schema),
-            vec![Arc::new(a), Arc::new(b)],
-            Operator::Plus,
-            Int32Array::from(vec![2, 4, 7, 12, 21]),
-        )?;
-
-        Ok(())
-    }
-
-    #[test]
-    fn minus_op() -> Result<()> {
-        let schema = Arc::new(Schema::new(vec![
-            Field::new("a", DataType::Int32, false),
-            Field::new("b", DataType::Int32, false),
-        ]));
-        let a = Arc::new(Int32Array::from(vec![1, 2, 4, 8, 16]));
-        let b = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]));
-
-        apply_arithmetic::<Int32Type>(
-            schema.clone(),
-            vec![a.clone(), b.clone()],
-            Operator::Minus,
-            Int32Array::from(vec![0, 0, 1, 4, 11]),
-        )?;
-
-        // should handle have negative values in result (for signed)
-        apply_arithmetic::<Int32Type>(
-            schema.clone(),
-            vec![b.clone(), a.clone()],
-            Operator::Minus,
-            Int32Array::from(vec![0, 0, -1, -4, -11]),
-        )?;
-
-        Ok(())
-    }
-
-    #[test]
-    fn multiply_op() -> Result<()> {
-        let schema = Arc::new(Schema::new(vec![
-            Field::new("a", DataType::Int32, false),
-            Field::new("b", DataType::Int32, false),
-        ]));
-        let a = Arc::new(Int32Array::from(vec![4, 8, 16, 32, 64]));
-        let b = Arc::new(Int32Array::from(vec![2, 4, 8, 16, 32]));
-
-        apply_arithmetic::<Int32Type>(
-            schema,
-            vec![a, b],
-            Operator::Multiply,
-            Int32Array::from(vec![8, 32, 128, 512, 2048]),
-        )?;
-
-        Ok(())
-    }
-
-    #[test]
-    fn divide_op() -> Result<()> {
-        let schema = Arc::new(Schema::new(vec![
-            Field::new("a", DataType::Int32, false),
-            Field::new("b", DataType::Int32, false),
-        ]));
-        let a = Arc::new(Int32Array::from(vec![8, 32, 128, 512, 2048]));
-        let b = Arc::new(Int32Array::from(vec![2, 4, 8, 16, 32]));
-
-        apply_arithmetic::<Int32Type>(
-            schema,
-            vec![a, b],
-            Operator::Divide,
-            Int32Array::from(vec![4, 8, 16, 32, 64]),
-        )?;
-
-        Ok(())
-    }
-
-    fn apply_arithmetic<T: ArrowNumericType>(
-        schema: SchemaRef,
-        data: Vec<ArrayRef>,
-        op: Operator,
-        expected: PrimitiveArray<T>,
-    ) -> Result<()> {
-        let arithmetic_op = binary(col("a"), op, col("b"));
-        let batch = RecordBatch::try_new(schema, data)?;
-        let result = arithmetic_op.evaluate(&batch)?;
-
-        assert_array_eq::<T>(expected, result);
-
-        Ok(())
-    }
-
-    fn assert_array_eq<T: ArrowNumericType>(
-        expected: PrimitiveArray<T>,
-        actual: ArrayRef,
-    ) {
-        let actual = actual
-            .as_any()
-            .downcast_ref::<PrimitiveArray<T>>()
-            .expect("Actual array should unwrap to type of expected array");
-
-        for i in 0..expected.len() {
-            assert_eq!(expected.value(i), actual.value(i));
-        }
-    }
-
-    #[test]
-    fn neg_op() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("a", DataType::Boolean, true)]);
-        let a = BooleanArray::from(vec![true, false]);
-        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
-
-        // expression: "!a"
-        let lt = not(col("a"));
-        let result = lt.evaluate(&batch)?;
-        assert_eq!(result.len(), 2);
-
-        let expected = vec![false, true];
-        let result = result
-            .as_any()
-            .downcast_ref::<BooleanArray>()
-            .expect("failed to downcast to BooleanArray");
-        for i in 0..2 {
-            assert_eq!(result.value(i), expected[i]);
-        }
-
-        Ok(())
-    }
-}
diff --git a/rust/datafusion/src/execution/physical_plan/hash_aggregate.rs b/rust/datafusion/src/execution/physical_plan/hash_aggregate.rs
deleted file mode 100644
index 0b20c8df571..00000000000
--- a/rust/datafusion/src/execution/physical_plan/hash_aggregate.rs
+++ /dev/null
@@ -1,787 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Defines the execution plan for the hash aggregate operation
-
-use std::cell::RefCell;
-use std::rc::Rc;
-use std::sync::{Arc, Mutex};
-
-use crate::error::{ExecutionError, Result};
-use crate::execution::physical_plan::{
-    Accumulator, AggregateExpr, ExecutionPlan, Partition, PhysicalExpr,
-};
-
-use arrow::array::{
-    ArrayBuilder, ArrayRef, Float32Array, Float64Array, Int16Array, Int32Array,
-    Int64Array, Int8Array, StringArray, UInt16Array, UInt32Array, UInt64Array,
-    UInt8Array,
-};
-use arrow::array::{
-    Float32Builder, Float64Builder, Int16Builder, Int32Builder, Int64Builder,
-    Int8Builder, StringBuilder, UInt16Builder, UInt32Builder, UInt64Builder,
-    UInt8Builder,
-};
-use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
-use arrow::error::Result as ArrowResult;
-use arrow::record_batch::{RecordBatch, RecordBatchReader};
-
-use crate::execution::physical_plan::expressions::col;
-use crate::logicalplan::ScalarValue;
-use fnv::FnvHashMap;
-
-/// Hash aggregate execution plan
-#[derive(Debug)]
-pub struct HashAggregateExec {
-    group_expr: Vec<Arc<dyn PhysicalExpr>>,
-    aggr_expr: Vec<Arc<dyn AggregateExpr>>,
-    input: Arc<dyn ExecutionPlan>,
-    schema: SchemaRef,
-}
-
-impl HashAggregateExec {
-    /// Create a new hash aggregate execution plan
-    pub fn try_new(
-        group_expr: Vec<(Arc<dyn PhysicalExpr>, String)>,
-        aggr_expr: Vec<(Arc<dyn AggregateExpr>, String)>,
-        input: Arc<dyn ExecutionPlan>,
-    ) -> Result<Self> {
-        let input_schema = input.schema();
-
-        let mut fields = Vec::with_capacity(group_expr.len() + aggr_expr.len());
-        for (expr, name) in &group_expr {
-            fields.push(Field::new(name, expr.data_type(&input_schema)?, true))
-        }
-        for (expr, name) in &aggr_expr {
-            fields.push(Field::new(&name, expr.data_type(&input_schema)?, true))
-        }
-        let schema = Arc::new(Schema::new(fields));
-
-        Ok(HashAggregateExec {
-            group_expr: group_expr.iter().map(|x| x.0.clone()).collect(),
-            aggr_expr: aggr_expr.iter().map(|x| x.0.clone()).collect(),
-            input,
-            schema,
-        })
-    }
-
-    /// Create the final group and aggregate expressions from the initial group and aggregate
-    /// expressions
-    pub fn make_final_expr(
-        &self,
-        group_names: Vec<String>,
-        agg_names: Vec<String>,
-    ) -> (Vec<Arc<dyn PhysicalExpr>>, Vec<Arc<dyn AggregateExpr>>) {
-        let final_group: Vec<Arc<dyn PhysicalExpr>> = (0..self.group_expr.len())
-            .map(|i| col(&group_names[i]) as Arc<dyn PhysicalExpr>)
-            .collect();
-
-        let final_aggr: Vec<Arc<dyn AggregateExpr>> = (0..self.aggr_expr.len())
-            .map(|i| self.aggr_expr[i].create_reducer(&agg_names[i]))
-            .collect();
-
-        (final_group, final_aggr)
-    }
-}
-
-impl ExecutionPlan for HashAggregateExec {
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-
-    fn partitions(&self) -> Result<Vec<Arc<dyn Partition>>> {
-        Ok(self
-            .input
-            .partitions()?
-            .iter()
-            .map(|p| {
-                let aggregate: Arc<dyn Partition> =
-                    Arc::new(HashAggregatePartition::new(
-                        self.group_expr.clone(),
-                        self.aggr_expr.clone(),
-                        p.clone() as Arc<dyn Partition>,
-                        self.schema.clone(),
-                    ));
-
-                aggregate
-            })
-            .collect::<Vec<Arc<dyn Partition>>>())
-    }
-}
-
-#[derive(Debug)]
-struct HashAggregatePartition {
-    group_expr: Vec<Arc<dyn PhysicalExpr>>,
-    aggr_expr: Vec<Arc<dyn AggregateExpr>>,
-    input: Arc<dyn Partition>,
-    schema: SchemaRef,
-}
-
-impl HashAggregatePartition {
-    /// Create a new HashAggregatePartition
-    pub fn new(
-        group_expr: Vec<Arc<dyn PhysicalExpr>>,
-        aggr_expr: Vec<Arc<dyn AggregateExpr>>,
-        input: Arc<dyn Partition>,
-        schema: SchemaRef,
-    ) -> Self {
-        HashAggregatePartition {
-            group_expr,
-            aggr_expr,
-            input,
-            schema,
-        }
-    }
-}
-
-impl Partition for HashAggregatePartition {
-    fn execute(&self) -> Result<Arc<Mutex<dyn RecordBatchReader + Send + Sync>>> {
-        if self.group_expr.is_empty() {
-            Ok(Arc::new(Mutex::new(HashAggregateIterator::new(
-                self.schema.clone(),
-                self.aggr_expr.clone(),
-                self.input.execute()?,
-            ))))
-        } else {
-            Ok(Arc::new(Mutex::new(GroupedHashAggregateIterator::new(
-                self.schema.clone(),
-                self.group_expr.clone(),
-                self.aggr_expr.clone(),
-                self.input.execute()?,
-            ))))
-        }
-    }
-}
-
-/// Create array from single accumulator value
-macro_rules! accum_val {
-    ($BUILDER:ident, $SCALAR_TY:ident, $VALUE:expr) => {{
-        let mut builder = $BUILDER::new(1);
-        match $VALUE {
-            Some(ScalarValue::$SCALAR_TY(n)) => {
-                builder.append_value(n)?;
-                Ok(Arc::new(builder.finish()) as ArrayRef)
-            }
-            None => {
-                builder.append_null()?;
-                Ok(Arc::new(builder.finish()) as ArrayRef)
-            }
-            _ => Err(ExecutionError::ExecutionError(
-                "unexpected type when creating aggregate array from no-group aggregate"
-                    .to_string(),
-            )),
-        }
-    }};
-}
-
-struct GroupedHashAggregateIterator {
-    schema: SchemaRef,
-    group_expr: Vec<Arc<dyn PhysicalExpr>>,
-    aggr_expr: Vec<Arc<dyn AggregateExpr>>,
-    input: Arc<Mutex<dyn RecordBatchReader + Send + Sync>>,
-    finished: bool,
-}
-
-impl GroupedHashAggregateIterator {
-    /// Create a new HashAggregateIterator
-    pub fn new(
-        schema: SchemaRef,
-        group_expr: Vec<Arc<dyn PhysicalExpr>>,
-        aggr_expr: Vec<Arc<dyn AggregateExpr>>,
-        input: Arc<Mutex<dyn RecordBatchReader + Send + Sync>>,
-    ) -> Self {
-        GroupedHashAggregateIterator {
-            schema,
-            group_expr,
-            aggr_expr,
-            input,
-            finished: false,
-        }
-    }
-}
-
-type AccumulatorSet = Vec<Rc<RefCell<dyn Accumulator>>>;
-
-macro_rules! update_accum {
-    ($ARRAY:ident, $ARRAY_TY:ident, $SCALAR_TY:expr, $COL:expr, $ACCUM:expr) => {{
-        let primitive_array = $ARRAY.as_any().downcast_ref::<$ARRAY_TY>().unwrap();
-
-        for row in 0..$ARRAY.len() {
-            if $ARRAY.is_valid(row) {
-                let value = Some($SCALAR_TY(primitive_array.value(row)));
-                let mut accum = $ACCUM[row][$COL].borrow_mut();
-                accum
-                    .accumulate_scalar(value)
-                    .map_err(ExecutionError::into_arrow_external_error)?;
-            }
-        }
-    }};
-}
-
-impl RecordBatchReader for GroupedHashAggregateIterator {
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-
-    fn next_batch(&mut self) -> ArrowResult<Option<RecordBatch>> {
-        if self.finished {
-            return Ok(None);
-        }
-
-        self.finished = true;
-
-        // create map to store accumulators for each unique grouping key
-        let mut map: FnvHashMap<Vec<GroupByScalar>, Rc<AccumulatorSet>> =
-            FnvHashMap::default();
-
-        // iterate over all input batches and update the accumulators
-        let mut input = self.input.lock().unwrap();
-
-        // iterate over input and perform aggregation
-        while let Some(batch) = input.next_batch()? {
-            // evaluate the grouping expressions for this batch
-            let group_values = self
-                .group_expr
-                .iter()
-                .map(|expr| {
-                    expr.evaluate(&batch)
-                        .map_err(ExecutionError::into_arrow_external_error)
-                })
-                .collect::<ArrowResult<Vec<_>>>()?;
-
-            // evaluate the inputs to the aggregate expressions for this batch
-            let aggr_input_values = self
-                .aggr_expr
-                .iter()
-                .map(|expr| {
-                    expr.evaluate_input(&batch)
-                        .map_err(ExecutionError::into_arrow_external_error)
-                })
-                .collect::<ArrowResult<Vec<_>>>()?;
-
-            // create vector large enough to hold the grouping key
-            let mut key = Vec::with_capacity(group_values.len());
-            for _ in 0..group_values.len() {
-                key.push(GroupByScalar::UInt32(0));
-            }
-
-            // iterate over each row in the batch and create the accumulators for each grouping key
-            let mut accums: Vec<Rc<AccumulatorSet>> =
-                Vec::with_capacity(batch.num_rows());
-
-            for row in 0..batch.num_rows() {
-                // create grouping key for this row
-                create_key(&group_values, row, &mut key)
-                    .map_err(ExecutionError::into_arrow_external_error)?;
-
-                if let Some(accumulator_set) = map.get(&key) {
-                    accums.push(accumulator_set.clone());
-                } else {
-                    let accumulator_set: AccumulatorSet = self
-                        .aggr_expr
-                        .iter()
-                        .map(|expr| expr.create_accumulator())
-                        .collect();
-
-                    let accumulator_set = Rc::new(accumulator_set);
-
-                    map.insert(key.clone(), accumulator_set.clone());
-                    accums.push(accumulator_set);
-                }
-            }
-
-            // iterate over each non-grouping column in the batch and update the accumulator
-            // for each row
-            for col in 0..aggr_input_values.len() {
-                let array = &aggr_input_values[col];
-
-                match array.data_type() {
-                    DataType::Int8 => {
-                        update_accum!(array, Int8Array, ScalarValue::Int8, col, accums)
-                    }
-                    DataType::Int16 => {
-                        update_accum!(array, Int16Array, ScalarValue::Int16, col, accums)
-                    }
-                    DataType::Int32 => {
-                        update_accum!(array, Int32Array, ScalarValue::Int32, col, accums)
-                    }
-                    DataType::Int64 => {
-                        update_accum!(array, Int64Array, ScalarValue::Int64, col, accums)
-                    }
-                    DataType::UInt8 => {
-                        update_accum!(array, UInt8Array, ScalarValue::UInt8, col, accums)
-                    }
-                    DataType::UInt16 => update_accum!(
-                        array,
-                        UInt16Array,
-                        ScalarValue::UInt16,
-                        col,
-                        accums
-                    ),
-                    DataType::UInt32 => update_accum!(
-                        array,
-                        UInt32Array,
-                        ScalarValue::UInt32,
-                        col,
-                        accums
-                    ),
-                    DataType::UInt64 => update_accum!(
-                        array,
-                        UInt64Array,
-                        ScalarValue::UInt64,
-                        col,
-                        accums
-                    ),
-                    DataType::Float32 => update_accum!(
-                        array,
-                        Float32Array,
-                        ScalarValue::Float32,
-                        col,
-                        accums
-                    ),
-                    DataType::Float64 => update_accum!(
-                        array,
-                        Float64Array,
-                        ScalarValue::Float64,
-                        col,
-                        accums
-                    ),
-                    other => {
-                        return Err(ExecutionError::ExecutionError(format!(
-                            "Unsupported data type {:?} for result of aggregate expression",
-                            other
-                        )).into_arrow_external_error());
-                    }
-                };
-            }
-        }
-
-        let batch = create_batch_from_map(
-            &map,
-            self.group_expr.len(),
-            self.aggr_expr.len(),
-            &self.schema,
-        )
-        .map_err(ExecutionError::into_arrow_external_error)?;
-
-        Ok(Some(batch))
-    }
-}
-
-struct HashAggregateIterator {
-    schema: SchemaRef,
-    aggr_expr: Vec<Arc<dyn AggregateExpr>>,
-    input: Arc<Mutex<dyn RecordBatchReader + Send + Sync>>,
-    finished: bool,
-}
-
-impl HashAggregateIterator {
-    /// Create a new HashAggregateIterator
-    pub fn new(
-        schema: SchemaRef,
-        aggr_expr: Vec<Arc<dyn AggregateExpr>>,
-        input: Arc<Mutex<dyn RecordBatchReader + Send + Sync>>,
-    ) -> Self {
-        HashAggregateIterator {
-            schema,
-            aggr_expr,
-            input,
-            finished: false,
-        }
-    }
-}
-
-impl RecordBatchReader for HashAggregateIterator {
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-
-    fn next_batch(&mut self) -> ArrowResult<Option<RecordBatch>> {
-        if self.finished {
-            return Ok(None);
-        }
-
-        self.finished = true;
-
-        let accumulators: Vec<Rc<RefCell<dyn Accumulator>>> = self
-            .aggr_expr
-            .iter()
-            .map(|expr| expr.create_accumulator())
-            .collect();
-
-        // iterate over all input batches and update the accumulators
-        let mut input = self.input.lock().unwrap();
-
-        // iterate over input and perform aggregation
-        while let Some(batch) = input.next_batch()? {
-            // evaluate the inputs to the aggregate expressions for this batch
-            let aggr_input_values = self
-                .aggr_expr
-                .iter()
-                .map(|expr| {
-                    expr.evaluate_input(&batch)
-                        .map_err(ExecutionError::into_arrow_external_error)
-                })
-                .collect::<ArrowResult<Vec<_>>>()?;
-
-            // iterate over each row in the batch
-            let _ = accumulators
-                .iter()
-                .zip(aggr_input_values.iter())
-                .map(|(accum, input)| {
-                    accum
-                        .borrow_mut()
-                        .accumulate_batch(input)
-                        .map_err(ExecutionError::into_arrow_external_error)
-                })
-                .collect::<ArrowResult<Vec<_>>>()?;
-        }
-
-        let input_schema = input.schema();
-
-        // build the result arrays
-        let mut result_arrays: Vec<ArrayRef> = Vec::with_capacity(self.aggr_expr.len());
-
-        // aggregate values
-        for i in 0..self.aggr_expr.len() {
-            let aggr_data_type = self.aggr_expr[i]
-                .data_type(&input_schema)
-                .map_err(ExecutionError::into_arrow_external_error)?;
-            let value = accumulators[i]
-                .borrow_mut()
-                .get_value()
-                .map_err(ExecutionError::into_arrow_external_error)?;
-            let array = match aggr_data_type {
-                DataType::UInt8 => accum_val!(UInt8Builder, UInt8, value),
-                DataType::UInt16 => accum_val!(UInt16Builder, UInt16, value),
-                DataType::UInt32 => accum_val!(UInt32Builder, UInt32, value),
-                DataType::UInt64 => accum_val!(UInt64Builder, UInt64, value),
-                DataType::Int8 => accum_val!(Int8Builder, Int8, value),
-                DataType::Int16 => accum_val!(Int16Builder, Int16, value),
-                DataType::Int32 => accum_val!(Int32Builder, Int32, value),
-                DataType::Int64 => accum_val!(Int64Builder, Int64, value),
-                DataType::Float32 => accum_val!(Float32Builder, Float32, value),
-                DataType::Float64 => accum_val!(Float64Builder, Float64, value),
-                _ => Err(ExecutionError::ExecutionError(
-                    "Unsupported aggregate expr".to_string(),
-                )),
-            };
-            result_arrays.push(array.map_err(ExecutionError::into_arrow_external_error)?);
-        }
-
-        let batch = RecordBatch::try_new(self.schema.clone(), result_arrays)?;
-        Ok(Some(batch))
-    }
-}
-
-/// Append a grouping expression value to a builder
-macro_rules! group_val {
-    ($BUILDER:expr, $BUILDER_TY:ident, $VALUE:expr) => {{
-        let builder = $BUILDER
-            .downcast_mut::<$BUILDER_TY>()
-            .expect("failed to downcast group value builder to expected type");
-        builder.append_value($VALUE)?;
-    }};
-}
-
-/// Append an aggregate expression value to a builder
-macro_rules! aggr_val {
-    ($BUILDER:expr, $BUILDER_TY:ident, $VALUE:expr, $SCALAR_TY:ident) => {{
-        let builder = $BUILDER
-            .downcast_mut::<$BUILDER_TY>()
-            .expect("failed to downcast aggregate value builder to expected type");
-        match $VALUE {
-            Some(ScalarValue::$SCALAR_TY(n)) => builder.append_value(n)?,
-            None => builder.append_null()?,
-            Some(other) => {
-                return Err(ExecutionError::General(format!(
-                    "Unexpected data type {:?} for aggregate value",
-                    other
-                )))
-            }
-        }
-    }};
-}
-
-/// Create a RecordBatch representing the accumulated results in a map
-fn create_batch_from_map(
-    map: &FnvHashMap<Vec<GroupByScalar>, Rc<AccumulatorSet>>,
-    num_group_expr: usize,
-    num_aggr_expr: usize,
-    output_schema: &Schema,
-) -> Result<RecordBatch> {
-    // create builders based on the output schema data types
-    let output_types: Vec<&DataType> = output_schema
-        .fields()
-        .iter()
-        .map(|f| f.data_type())
-        .collect();
-    let mut builders: Vec<Box<dyn ArrayBuilder>> = vec![];
-    for data_type in &output_types {
-        let builder: Box<dyn ArrayBuilder> = match data_type {
-            DataType::Int8 => Box::new(Int8Builder::new(map.len())),
-            DataType::Int16 => Box::new(Int16Builder::new(map.len())),
-            DataType::Int32 => Box::new(Int32Builder::new(map.len())),
-            DataType::Int64 => Box::new(Int64Builder::new(map.len())),
-            DataType::UInt8 => Box::new(UInt8Builder::new(map.len())),
-            DataType::UInt16 => Box::new(UInt16Builder::new(map.len())),
-            DataType::UInt32 => Box::new(UInt32Builder::new(map.len())),
-            DataType::UInt64 => Box::new(UInt64Builder::new(map.len())),
-            DataType::Float32 => Box::new(Float32Builder::new(map.len())),
-            DataType::Float64 => Box::new(Float64Builder::new(map.len())),
-            DataType::Utf8 => Box::new(StringBuilder::new(map.len())),
-            _ => {
-                return Err(ExecutionError::ExecutionError(
-                    "Unsupported data type in final aggregate result".to_string(),
-                ))
-            }
-        };
-        builders.push(builder);
-    }
-
-    // iterate over the map
-    for (k, v) in map.iter() {
-        // add group values to builders
-        for i in 0..num_group_expr {
-            let builder = builders[i].as_any_mut();
-            match &k[i] {
-                GroupByScalar::Int8(n) => group_val!(builder, Int8Builder, *n),
-                GroupByScalar::Int16(n) => group_val!(builder, Int16Builder, *n),
-                GroupByScalar::Int32(n) => group_val!(builder, Int32Builder, *n),
-                GroupByScalar::Int64(n) => group_val!(builder, Int64Builder, *n),
-                GroupByScalar::UInt8(n) => group_val!(builder, UInt8Builder, *n),
-                GroupByScalar::UInt16(n) => group_val!(builder, UInt16Builder, *n),
-                GroupByScalar::UInt32(n) => group_val!(builder, UInt32Builder, *n),
-                GroupByScalar::UInt64(n) => group_val!(builder, UInt64Builder, *n),
-                GroupByScalar::Utf8(str) => group_val!(builder, StringBuilder, str),
-            }
-        }
-
-        // add aggregate values to builders
-        for i in 0..num_aggr_expr {
-            let value = v[i].borrow().get_value()?;
-            let index = num_group_expr + i;
-            let builder = builders[index].as_any_mut();
-            match output_types[index] {
-                DataType::Int8 => aggr_val!(builder, Int8Builder, value, Int8),
-                DataType::Int16 => aggr_val!(builder, Int16Builder, value, Int16),
-                DataType::Int32 => aggr_val!(builder, Int32Builder, value, Int32),
-                DataType::Int64 => aggr_val!(builder, Int64Builder, value, Int64),
-                DataType::UInt8 => aggr_val!(builder, UInt8Builder, value, UInt8),
-                DataType::UInt16 => aggr_val!(builder, UInt16Builder, value, UInt16),
-                DataType::UInt32 => aggr_val!(builder, UInt32Builder, value, UInt32),
-                DataType::UInt64 => aggr_val!(builder, UInt64Builder, value, UInt64),
-                DataType::Float32 => aggr_val!(builder, Float32Builder, value, Float32),
-                DataType::Float64 => aggr_val!(builder, Float64Builder, value, Float64),
-                // The aggr_val! macro doesn't work for ScalarValue::Utf8 because it contains
-                // String and the builder wants &str. In all other cases the scalar and builder
-                // types are the same.
-                DataType::Utf8 => {
-                    let builder = builder
-                        .downcast_mut::<StringBuilder>()
-                        .expect("failed to downcast builder to expected type");
-                    match value {
-                        Some(ScalarValue::Utf8(str)) => builder.append_value(&str)?,
-                        None => builder.append_null()?,
-                        Some(_) => {
-                            return Err(ExecutionError::ExecutionError(
-                                "Invalid value for accumulator".to_string(),
-                            ))
-                        }
-                    }
-                }
-                _ => {
-                    return Err(ExecutionError::ExecutionError(
-                        "Unsupported aggregate data type".to_string(),
-                    ))
-                }
-            };
-        }
-    }
-
-    let arrays: Vec<ArrayRef> = builders
-        .iter_mut()
-        .map(|builder| builder.finish())
-        .collect();
-
-    let batch = RecordBatch::try_new(Arc::new(output_schema.to_owned()), arrays)?;
-
-    Ok(batch)
-}
-
-/// Enumeration of types that can be used in a GROUP BY expression (all primitives except
-/// for floating point numerics)
-#[derive(Debug, PartialEq, Eq, Hash, Clone)]
-enum GroupByScalar {
-    UInt8(u8),
-    UInt16(u16),
-    UInt32(u32),
-    UInt64(u64),
-    Int8(i8),
-    Int16(i16),
-    Int32(i32),
-    Int64(i64),
-    Utf8(String),
-}
-
-/// Create a Vec<GroupByScalar> that can be used as a map key
-fn create_key(
-    group_by_keys: &[ArrayRef],
-    row: usize,
-    vec: &mut Vec<GroupByScalar>,
-) -> Result<()> {
-    for i in 0..group_by_keys.len() {
-        let col = &group_by_keys[i];
-        match col.data_type() {
-            DataType::UInt8 => {
-                let array = col.as_any().downcast_ref::<UInt8Array>().unwrap();
-                vec[i] = GroupByScalar::UInt8(array.value(row))
-            }
-            DataType::UInt16 => {
-                let array = col.as_any().downcast_ref::<UInt16Array>().unwrap();
-                vec[i] = GroupByScalar::UInt16(array.value(row))
-            }
-            DataType::UInt32 => {
-                let array = col.as_any().downcast_ref::<UInt32Array>().unwrap();
-                vec[i] = GroupByScalar::UInt32(array.value(row))
-            }
-            DataType::UInt64 => {
-                let array = col.as_any().downcast_ref::<UInt64Array>().unwrap();
-                vec[i] = GroupByScalar::UInt64(array.value(row))
-            }
-            DataType::Int8 => {
-                let array = col.as_any().downcast_ref::<Int8Array>().unwrap();
-                vec[i] = GroupByScalar::Int8(array.value(row))
-            }
-            DataType::Int16 => {
-                let array = col.as_any().downcast_ref::<Int16Array>().unwrap();
-                vec[i] = GroupByScalar::Int16(array.value(row))
-            }
-            DataType::Int32 => {
-                let array = col.as_any().downcast_ref::<Int32Array>().unwrap();
-                vec[i] = GroupByScalar::Int32(array.value(row))
-            }
-            DataType::Int64 => {
-                let array = col.as_any().downcast_ref::<Int64Array>().unwrap();
-                vec[i] = GroupByScalar::Int64(array.value(row))
-            }
-            DataType::Utf8 => {
-                let array = col.as_any().downcast_ref::<StringArray>().unwrap();
-                vec[i] = GroupByScalar::Utf8(String::from(array.value(row)))
-            }
-            _ => {
-                return Err(ExecutionError::ExecutionError(
-                    "Unsupported GROUP BY data type".to_string(),
-                ))
-            }
-        }
-    }
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-
-    use super::*;
-    use crate::execution::physical_plan::csv::{CsvExec, CsvReadOptions};
-    use crate::execution::physical_plan::expressions::{col, sum};
-    use crate::execution::physical_plan::merge::MergeExec;
-    use crate::test;
-
-    #[test]
-    fn aggregate() -> Result<()> {
-        let schema = test::aggr_test_schema();
-
-        let partitions = 4;
-        let path = test::create_partitioned_csv("aggregate_test_100.csv", partitions)?;
-
-        let csv =
-            CsvExec::try_new(&path, CsvReadOptions::new().schema(&schema), None, 1024)?;
-
-        let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =
-            vec![(col("c2"), "c2".to_string())];
-
-        let aggregates: Vec<(Arc<dyn AggregateExpr>, String)> =
-            vec![(sum(col("c4")), "SUM(c4)".to_string())];
-
-        let partition_aggregate = HashAggregateExec::try_new(
-            groups.clone(),
-            aggregates.clone(),
-            Arc::new(csv),
-        )?;
-
-        let schema = partition_aggregate.schema();
-        let partitions = partition_aggregate.partitions()?;
-
-        // construct the expressions for the final aggregation
-        let (final_group, final_aggr) = partition_aggregate.make_final_expr(
-            groups.iter().map(|x| x.1.clone()).collect(),
-            aggregates.iter().map(|x| x.1.clone()).collect(),
-        );
-
-        let merge = Arc::new(MergeExec::new(schema.clone(), partitions));
-
-        let merged_aggregate = HashAggregateExec::try_new(
-            final_group
-                .iter()
-                .enumerate()
-                .map(|(i, expr)| (expr.clone(), groups[i].1.clone()))
-                .collect(),
-            final_aggr
-                .iter()
-                .enumerate()
-                .map(|(i, expr)| (expr.clone(), aggregates[i].1.clone()))
-                .collect(),
-            merge,
-        )?;
-
-        let result = test::execute(&merged_aggregate)?;
-        assert_eq!(result.len(), 1);
-
-        let batch = &result[0];
-        assert_eq!(batch.num_columns(), 2);
-        assert_eq!(batch.num_rows(), 5);
-
-        let a = batch
-            .column(0)
-            .as_any()
-            .downcast_ref::<UInt32Array>()
-            .unwrap();
-        let b = batch
-            .column(1)
-            .as_any()
-            .downcast_ref::<Int64Array>()
-            .unwrap();
-
-        let mut group_values = vec![];
-        for i in 0..a.len() {
-            group_values.push(a.value(i))
-        }
-
-        let mut aggr_values = vec![];
-        for i in 1..=5 {
-            // find index of row with this value for the grouping column
-            let index = group_values.iter().position(|&r| r == i).unwrap();
-            aggr_values.push(b.value(index));
-        }
-
-        let expected: Vec<i64> = vec![88722, 90999, 80899, -120910, 92287];
-        assert_eq!(aggr_values, expected);
-
-        Ok(())
-    }
-}
diff --git a/rust/datafusion/src/execution/physical_plan/limit.rs b/rust/datafusion/src/execution/physical_plan/limit.rs
deleted file mode 100644
index 49d612b4d0d..00000000000
--- a/rust/datafusion/src/execution/physical_plan/limit.rs
+++ /dev/null
@@ -1,208 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Defines the LIMIT plan
-
-use crate::error::{ExecutionError, Result};
-use crate::execution::physical_plan::common::RecordBatchIterator;
-use crate::execution::physical_plan::ExecutionPlan;
-use crate::execution::physical_plan::Partition;
-use arrow::array::ArrayRef;
-use arrow::compute::limit;
-use arrow::datatypes::SchemaRef;
-use arrow::record_batch::{RecordBatch, RecordBatchReader};
-use std::sync::{Arc, Mutex};
-use std::thread;
-use std::thread::JoinHandle;
-
-/// Limit execution plan
-#[derive(Debug)]
-pub struct LimitExec {
-    /// Input schema
-    schema: SchemaRef,
-    /// Input partitions
-    partitions: Vec<Arc<dyn Partition>>,
-    /// Maximum number of rows to return
-    limit: usize,
-}
-
-impl LimitExec {
-    /// Create a new MergeExec
-    pub fn new(
-        schema: SchemaRef,
-        partitions: Vec<Arc<dyn Partition>>,
-        limit: usize,
-    ) -> Self {
-        LimitExec {
-            schema,
-            partitions,
-            limit,
-        }
-    }
-}
-
-impl ExecutionPlan for LimitExec {
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-
-    fn partitions(&self) -> Result<Vec<Arc<dyn Partition>>> {
-        Ok(vec![Arc::new(LimitPartition {
-            schema: self.schema.clone(),
-            partitions: self.partitions.clone(),
-            limit: self.limit,
-        })])
-    }
-}
-
-#[derive(Debug)]
-struct LimitPartition {
-    /// Input schema
-    schema: SchemaRef,
-    /// Input partitions
-    partitions: Vec<Arc<dyn Partition>>,
-    /// Maximum number of rows to return
-    limit: usize,
-}
-
-impl Partition for LimitPartition {
-    fn execute(&self) -> Result<Arc<Mutex<dyn RecordBatchReader + Send + Sync>>> {
-        // collect up to "limit" rows on each partition
-        let threads: Vec<JoinHandle<Result<Vec<RecordBatch>>>> = self
-            .partitions
-            .iter()
-            .map(|p| {
-                let p = p.clone();
-                let limit = self.limit;
-                thread::spawn(move || {
-                    let it = p.execute()?;
-                    collect_with_limit(it, limit)
-                })
-            })
-            .collect();
-
-        // combine the results from each thread, up to the limit
-        let mut combined_results: Vec<Arc<RecordBatch>> = vec![];
-        let mut count = 0;
-        for thread in threads {
-            let join = thread.join().expect("Failed to join thread");
-            let result = join?;
-            for batch in result {
-                let capacity = self.limit - count;
-                if batch.num_rows() <= capacity {
-                    count += batch.num_rows();
-                    combined_results.push(Arc::new(batch.clone()))
-                } else {
-                    let batch = truncate_batch(&batch, capacity)?;
-                    count += batch.num_rows();
-                    combined_results.push(Arc::new(batch.clone()))
-                }
-                if count == self.limit {
-                    break;
-                }
-            }
-        }
-
-        Ok(Arc::new(Mutex::new(RecordBatchIterator::new(
-            self.schema.clone(),
-            combined_results,
-        ))))
-    }
-}
-
-/// Truncate a RecordBatch to maximum of n rows
-pub fn truncate_batch(batch: &RecordBatch, n: usize) -> Result<RecordBatch> {
-    let limited_columns: Result<Vec<ArrayRef>> = (0..batch.num_columns())
-        .map(|i| limit(batch.column(i), n).map_err(|error| ExecutionError::from(error)))
-        .collect();
-
-    Ok(RecordBatch::try_new(
-        batch.schema().clone(),
-        limited_columns?,
-    )?)
-}
-
-/// Create a vector of record batches from an iterator
-fn collect_with_limit(
-    reader: Arc<Mutex<dyn RecordBatchReader + Send + Sync>>,
-    limit: usize,
-) -> Result<Vec<RecordBatch>> {
-    let mut count = 0;
-    let mut reader = reader.lock().unwrap();
-    let mut results: Vec<RecordBatch> = vec![];
-    loop {
-        match reader.next_batch() {
-            Ok(Some(batch)) => {
-                let capacity = limit - count;
-                if batch.num_rows() <= capacity {
-                    count += batch.num_rows();
-                    results.push(batch);
-                } else {
-                    let batch = truncate_batch(&batch, capacity)?;
-                    count += batch.num_rows();
-                    results.push(batch);
-                }
-                if count == limit {
-                    return Ok(results);
-                }
-            }
-            Ok(None) => {
-                // end of result set
-                return Ok(results);
-            }
-            Err(e) => return Err(ExecutionError::from(e)),
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-
-    use super::*;
-    use crate::execution::physical_plan::common;
-    use crate::execution::physical_plan::csv::{CsvExec, CsvReadOptions};
-    use crate::test;
-
-    #[test]
-    fn limit() -> Result<()> {
-        let schema = test::aggr_test_schema();
-
-        let num_partitions = 4;
-        let path =
-            test::create_partitioned_csv("aggregate_test_100.csv", num_partitions)?;
-
-        let csv =
-            CsvExec::try_new(&path, CsvReadOptions::new().schema(&schema), None, 1024)?;
-
-        // input should have 4 partitions
-        let input = csv.partitions()?;
-        assert_eq!(input.len(), num_partitions);
-
-        let limit = LimitExec::new(schema.clone(), input, 7);
-        let partitions = limit.partitions()?;
-
-        // the result should contain 4 batches (one per input partition)
-        let iter = partitions[0].execute()?;
-        let batches = common::collect(iter)?;
-
-        // there should be a total of 100 rows
-        let row_count: usize = batches.iter().map(|batch| batch.num_rows()).sum();
-        assert_eq!(row_count, 7);
-
-        Ok(())
-    }
-}
diff --git a/rust/datafusion/src/execution/physical_plan/math_expressions.rs b/rust/datafusion/src/execution/physical_plan/math_expressions.rs
deleted file mode 100644
index aa578b697da..00000000000
--- a/rust/datafusion/src/execution/physical_plan/math_expressions.rs
+++ /dev/null
@@ -1,114 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Math expressions
-
-use crate::error::ExecutionError;
-use crate::execution::context::ExecutionContext;
-use crate::execution::physical_plan::udf::ScalarFunction;
-
-use arrow::array::{Array, ArrayRef, Float64Array, Float64Builder};
-use arrow::datatypes::{DataType, Field};
-
-use std::sync::Arc;
-
-macro_rules! math_unary_function {
-    ($NAME:expr, $FUNC:ident) => {
-        ScalarFunction::new(
-            $NAME,
-            vec![Field::new("n", DataType::Float64, true)],
-            DataType::Float64,
-            Arc::new(|args: &[ArrayRef]| {
-                let n = &args[0].as_any().downcast_ref::<Float64Array>();
-                match n {
-                    Some(array) => {
-                        let mut builder = Float64Builder::new(array.len());
-                        for i in 0..array.len() {
-                            if array.is_null(i) {
-                                builder.append_null()?;
-                            } else {
-                                builder.append_value(array.value(i).$FUNC())?;
-                            }
-                        }
-                        Ok(Arc::new(builder.finish()))
-                    }
-                    _ => Err(ExecutionError::General(format!(
-                        "Invalid data type for {}",
-                        $NAME
-                    ))),
-                }
-            }),
-        )
-    };
-}
-
-/// Register math scalar functions with the context
-pub fn register_math_functions(ctx: &mut ExecutionContext) {
-    ctx.register_udf(math_unary_function!("sqrt", sqrt));
-    ctx.register_udf(math_unary_function!("sin", sin));
-    ctx.register_udf(math_unary_function!("cos", cos));
-    ctx.register_udf(math_unary_function!("tan", tan));
-    ctx.register_udf(math_unary_function!("asin", asin));
-    ctx.register_udf(math_unary_function!("acos", acos));
-    ctx.register_udf(math_unary_function!("atan", atan));
-    ctx.register_udf(math_unary_function!("floor", floor));
-    ctx.register_udf(math_unary_function!("ceil", ceil));
-    ctx.register_udf(math_unary_function!("round", round));
-    ctx.register_udf(math_unary_function!("trunc", trunc));
-    ctx.register_udf(math_unary_function!("abs", abs));
-    ctx.register_udf(math_unary_function!("signum", signum));
-    ctx.register_udf(math_unary_function!("exp", exp));
-    ctx.register_udf(math_unary_function!("log", ln));
-    ctx.register_udf(math_unary_function!("log2", log2));
-    ctx.register_udf(math_unary_function!("log10", log10));
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::error::Result;
-    use crate::logicalplan::{col, sqrt, LogicalPlanBuilder};
-    use arrow::datatypes::Schema;
-
-    #[test]
-    fn cast_i8_input() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("c0", DataType::Int8, true)]);
-        let plan = LogicalPlanBuilder::scan("", "", &schema, None)?
-            .project(vec![sqrt(col("c0"))])?
-            .build()?;
-        let ctx = ExecutionContext::new();
-        let plan = ctx.optimize(&plan)?;
-        let expected = "Projection: sqrt(CAST(#c0 AS Float64))\
-        \n  TableScan:  projection=Some([0])";
-        assert_eq!(format!("{:?}", plan), expected);
-        Ok(())
-    }
-
-    #[test]
-    fn no_cast_f64_input() -> Result<()> {
-        let schema = Schema::new(vec![Field::new("c0", DataType::Float64, true)]);
-        let plan = LogicalPlanBuilder::scan("", "", &schema, None)?
-            .project(vec![sqrt(col("c0"))])?
-            .build()?;
-        let ctx = ExecutionContext::new();
-        let plan = ctx.optimize(&plan)?;
-        let expected = "Projection: sqrt(#c0)\
-        \n  TableScan:  projection=Some([0])";
-        assert_eq!(format!("{:?}", plan), expected);
-        Ok(())
-    }
-}
diff --git a/rust/datafusion/src/execution/physical_plan/merge.rs b/rust/datafusion/src/execution/physical_plan/merge.rs
deleted file mode 100644
index dfb4f578c82..00000000000
--- a/rust/datafusion/src/execution/physical_plan/merge.rs
+++ /dev/null
@@ -1,143 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Defines the merge plan for executing partitions in parallel and then merging the results
-//! into a single partition
-
-use crate::error::{ExecutionError, Result};
-use crate::execution::physical_plan::common::RecordBatchIterator;
-use crate::execution::physical_plan::Partition;
-use crate::execution::physical_plan::{common, ExecutionPlan};
-use arrow::datatypes::SchemaRef;
-use arrow::record_batch::{RecordBatch, RecordBatchReader};
-use std::sync::{Arc, Mutex};
-use std::thread;
-use std::thread::JoinHandle;
-
-/// Merge execution plan executes partitions in parallel and combines them into a single
-/// partition. No guarantees are made about the order of the resulting partition.
-#[derive(Debug)]
-pub struct MergeExec {
-    /// Input schema
-    schema: SchemaRef,
-    /// Input partitions
-    partitions: Vec<Arc<dyn Partition>>,
-}
-
-impl MergeExec {
-    /// Create a new MergeExec
-    pub fn new(schema: SchemaRef, partitions: Vec<Arc<dyn Partition>>) -> Self {
-        MergeExec { schema, partitions }
-    }
-}
-
-impl ExecutionPlan for MergeExec {
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-
-    fn partitions(&self) -> Result<Vec<Arc<dyn Partition>>> {
-        Ok(vec![Arc::new(MergePartition {
-            schema: self.schema.clone(),
-            partitions: self.partitions.clone(),
-        })])
-    }
-}
-
-#[derive(Debug)]
-struct MergePartition {
-    /// Input schema
-    schema: SchemaRef,
-    /// Input partitions
-    partitions: Vec<Arc<dyn Partition>>,
-}
-
-impl Partition for MergePartition {
-    fn execute(&self) -> Result<Arc<Mutex<dyn RecordBatchReader + Send + Sync>>> {
-        let threads: Vec<JoinHandle<Result<Vec<RecordBatch>>>> = self
-            .partitions
-            .iter()
-            .map(|p| {
-                let p = p.clone();
-                thread::spawn(move || {
-                    let it = p.execute()?;
-                    common::collect(it)
-                })
-            })
-            .collect();
-
-        // combine the results from each thread
-        let mut combined_results: Vec<Arc<RecordBatch>> = vec![];
-        for thread in threads {
-            match thread.join() {
-                Ok(join) => {
-                    join?
-                        .iter()
-                        .for_each(|batch| combined_results.push(Arc::new(batch.clone())));
-                }
-                Err(e) => return Err(ExecutionError::General(format!("{:?}", e))),
-            }
-        }
-
-        Ok(Arc::new(Mutex::new(RecordBatchIterator::new(
-            self.schema.clone(),
-            combined_results,
-        ))))
-    }
-}
-
-#[cfg(test)]
-mod tests {
-
-    use super::*;
-    use crate::execution::physical_plan::common;
-    use crate::execution::physical_plan::csv::{CsvExec, CsvReadOptions};
-    use crate::test;
-
-    #[test]
-    fn merge() -> Result<()> {
-        let schema = test::aggr_test_schema();
-
-        let num_partitions = 4;
-        let path =
-            test::create_partitioned_csv("aggregate_test_100.csv", num_partitions)?;
-
-        let csv =
-            CsvExec::try_new(&path, CsvReadOptions::new().schema(&schema), None, 1024)?;
-
-        // input should have 4 partitions
-        let input = csv.partitions()?;
-        assert_eq!(input.len(), num_partitions);
-
-        let merge = MergeExec::new(schema.clone(), input);
-
-        // output of MergeExec should have a single partition
-        let merged = merge.partitions()?;
-        assert_eq!(merged.len(), 1);
-
-        // the result should contain 4 batches (one per input partition)
-        let iter = merged[0].execute()?;
-        let batches = common::collect(iter)?;
-        assert_eq!(batches.len(), num_partitions);
-
-        // there should be a total of 100 rows
-        let row_count: usize = batches.iter().map(|batch| batch.num_rows()).sum();
-        assert_eq!(row_count, 100);
-
-        Ok(())
-    }
-}
diff --git a/rust/datafusion/src/execution/physical_plan/mod.rs b/rust/datafusion/src/execution/physical_plan/mod.rs
deleted file mode 100644
index 7f5a9fb2328..00000000000
--- a/rust/datafusion/src/execution/physical_plan/mod.rs
+++ /dev/null
@@ -1,93 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Traits for physical query plan, supporting parallel execution for partitioned relations.
-
-use std::cell::RefCell;
-use std::fmt::{Debug, Display};
-use std::rc::Rc;
-use std::sync::{Arc, Mutex};
-
-use crate::error::Result;
-use crate::logicalplan::ScalarValue;
-use arrow::array::ArrayRef;
-use arrow::datatypes::{DataType, Schema, SchemaRef};
-use arrow::record_batch::{RecordBatch, RecordBatchReader};
-
-/// Partition-aware execution plan for a relation
-pub trait ExecutionPlan: Debug {
-    /// Get the schema for this execution plan
-    fn schema(&self) -> SchemaRef;
-    /// Get the partitions for this execution plan. Each partition can be executed in parallel.
-    fn partitions(&self) -> Result<Vec<Arc<dyn Partition>>>;
-}
-
-/// Represents a partition of an execution plan that can be executed on a thread
-pub trait Partition: Send + Sync + Debug {
-    /// Execute this partition and return an iterator over RecordBatch
-    fn execute(&self) -> Result<Arc<Mutex<dyn RecordBatchReader + Send + Sync>>>;
-}
-
-/// Expression that can be evaluated against a RecordBatch
-/// A Physical expression knows its type, nullability and how to evaluate itself.
-pub trait PhysicalExpr: Send + Sync + Display + Debug {
-    /// Get the data type of this expression, given the schema of the input
-    fn data_type(&self, input_schema: &Schema) -> Result<DataType>;
-    /// Decide whehter this expression is nullable, given the schema of the input
-    fn nullable(&self, input_schema: &Schema) -> Result<bool>;
-    /// Evaluate an expression against a RecordBatch
-    fn evaluate(&self, batch: &RecordBatch) -> Result<ArrayRef>;
-}
-
-/// Aggregate expression that can be evaluated against a RecordBatch
-pub trait AggregateExpr: Send + Sync + Debug {
-    /// Get the data type of this expression, given the schema of the input
-    fn data_type(&self, input_schema: &Schema) -> Result<DataType>;
-    /// Evaluate the expression being aggregated
-    fn evaluate_input(&self, batch: &RecordBatch) -> Result<ArrayRef>;
-    /// Create an accumulator for this aggregate expression
-    fn create_accumulator(&self) -> Rc<RefCell<dyn Accumulator>>;
-    /// Create an aggregate expression for combining the results of accumulators from partitions.
-    /// For example, to combine the results of a parallel SUM we just need to do another SUM, but
-    /// to combine the results of parallel COUNT we would also use SUM.
-    fn create_reducer(&self, column_name: &str) -> Arc<dyn AggregateExpr>;
-}
-
-/// Aggregate accumulator
-pub trait Accumulator: Debug {
-    /// Update the accumulator based on a row in a batch
-    fn accumulate_scalar(&mut self, value: Option<ScalarValue>) -> Result<()>;
-    /// Update the accumulator based on an array in a batch
-    fn accumulate_batch(&mut self, array: &ArrayRef) -> Result<()>;
-    /// Get the final value for the accumulator
-    fn get_value(&self) -> Result<Option<ScalarValue>>;
-}
-
-pub mod common;
-pub mod csv;
-pub mod datasource;
-pub mod expressions;
-pub mod hash_aggregate;
-pub mod limit;
-pub mod math_expressions;
-pub mod memory;
-pub mod merge;
-pub mod parquet;
-pub mod projection;
-pub mod selection;
-pub mod sort;
-pub mod udf;
diff --git a/rust/datafusion/src/execution/physical_plan/projection.rs b/rust/datafusion/src/execution/physical_plan/projection.rs
deleted file mode 100644
index a5ad0ef3e03..00000000000
--- a/rust/datafusion/src/execution/physical_plan/projection.rs
+++ /dev/null
@@ -1,185 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Defines the projection execution plan. A projection determines which columns or expressions
-//! are returned from a query. The SQL statement `SELECT a, b, a+b FROM t1` is an example
-//! of a projection on table `t1` where the expressions `a`, `b`, and `a+b` are the
-//! projection expressions.
-
-use std::sync::{Arc, Mutex};
-
-use crate::error::{ExecutionError, Result};
-use crate::execution::physical_plan::{ExecutionPlan, Partition, PhysicalExpr};
-use arrow::datatypes::{Field, Schema, SchemaRef};
-use arrow::error::Result as ArrowResult;
-use arrow::record_batch::{RecordBatch, RecordBatchReader};
-
-/// Execution plan for a projection
-#[derive(Debug)]
-pub struct ProjectionExec {
-    /// The projection expressions
-    expr: Vec<Arc<dyn PhysicalExpr>>,
-    /// The schema once the projection has been applied to the input
-    schema: SchemaRef,
-    /// The input plan
-    input: Arc<dyn ExecutionPlan>,
-}
-
-impl ProjectionExec {
-    /// Create a projection on an input
-    pub fn try_new(
-        expr: Vec<(Arc<dyn PhysicalExpr>, String)>,
-        input: Arc<dyn ExecutionPlan>,
-    ) -> Result<Self> {
-        let input_schema = input.schema();
-
-        let fields: Result<Vec<_>> = expr
-            .iter()
-            .map(|(e, name)| {
-                Ok(Field::new(
-                    name,
-                    e.data_type(&input_schema)?,
-                    e.nullable(&input_schema)?,
-                ))
-            })
-            .collect();
-
-        let schema = Arc::new(Schema::new(fields?));
-
-        Ok(Self {
-            expr: expr.iter().map(|x| x.0.clone()).collect(),
-            schema,
-            input: input.clone(),
-        })
-    }
-}
-
-impl ExecutionPlan for ProjectionExec {
-    /// Get the schema for this execution plan
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-
-    /// Get the partitions for this execution plan
-    fn partitions(&self) -> Result<Vec<Arc<dyn Partition>>> {
-        let partitions: Vec<Arc<dyn Partition>> = self
-            .input
-            .partitions()?
-            .iter()
-            .map(|p| {
-                let projection: Arc<dyn Partition> = Arc::new(ProjectionPartition {
-                    schema: self.schema.clone(),
-                    expr: self.expr.clone(),
-                    input: p.clone() as Arc<dyn Partition>,
-                });
-
-                projection
-            })
-            .collect();
-
-        Ok(partitions)
-    }
-}
-
-/// Represents a single partition of a projection execution plan
-#[derive(Debug)]
-struct ProjectionPartition {
-    schema: SchemaRef,
-    expr: Vec<Arc<dyn PhysicalExpr>>,
-    input: Arc<dyn Partition>,
-}
-
-impl Partition for ProjectionPartition {
-    /// Execute the projection
-    fn execute(&self) -> Result<Arc<Mutex<dyn RecordBatchReader + Send + Sync>>> {
-        Ok(Arc::new(Mutex::new(ProjectionIterator {
-            schema: self.schema.clone(),
-            expr: self.expr.clone(),
-            input: self.input.execute()?,
-        })))
-    }
-}
-
-/// Projection iterator
-struct ProjectionIterator {
-    schema: SchemaRef,
-    expr: Vec<Arc<dyn PhysicalExpr>>,
-    input: Arc<Mutex<dyn RecordBatchReader + Send + Sync>>,
-}
-
-impl RecordBatchReader for ProjectionIterator {
-    /// Get the schema
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-
-    /// Get the next batch
-    fn next_batch(&mut self) -> ArrowResult<Option<RecordBatch>> {
-        let mut input = self.input.lock().unwrap();
-        match input.next_batch()? {
-            Some(batch) => {
-                let arrays: Result<Vec<_>> =
-                    self.expr.iter().map(|expr| expr.evaluate(&batch)).collect();
-                Ok(Some(RecordBatch::try_new(
-                    self.schema.clone(),
-                    arrays.map_err(ExecutionError::into_arrow_external_error)?,
-                )?))
-            }
-            None => Ok(None),
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-
-    use super::*;
-    use crate::execution::physical_plan::csv::{CsvExec, CsvReadOptions};
-    use crate::execution::physical_plan::expressions::col;
-    use crate::test;
-
-    #[test]
-    fn project_first_column() -> Result<()> {
-        let schema = test::aggr_test_schema();
-
-        let partitions = 4;
-        let path = test::create_partitioned_csv("aggregate_test_100.csv", partitions)?;
-
-        let csv =
-            CsvExec::try_new(&path, CsvReadOptions::new().schema(&schema), None, 1024)?;
-
-        // pick column c1 and name it column c1 in the output schema
-        let projection =
-            ProjectionExec::try_new(vec![(col("c1"), "c1".to_string())], Arc::new(csv))?;
-
-        let mut partition_count = 0;
-        let mut row_count = 0;
-        for partition in projection.partitions()? {
-            partition_count += 1;
-            let iterator = partition.execute()?;
-            let mut iterator = iterator.lock().unwrap();
-            while let Some(batch) = iterator.next_batch()? {
-                assert_eq!(1, batch.num_columns());
-                row_count += batch.num_rows();
-            }
-        }
-        assert_eq!(partitions, partition_count);
-        assert_eq!(100, row_count);
-
-        Ok(())
-    }
-}
diff --git a/rust/datafusion/src/execution/physical_plan/selection.rs b/rust/datafusion/src/execution/physical_plan/selection.rs
deleted file mode 100644
index 549b38f4e48..00000000000
--- a/rust/datafusion/src/execution/physical_plan/selection.rs
+++ /dev/null
@@ -1,189 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Defines the selection execution plan. A selection filters rows based on a predicate
-
-use std::sync::{Arc, Mutex};
-
-use crate::error::{ExecutionError, Result};
-use crate::execution::physical_plan::{ExecutionPlan, Partition, PhysicalExpr};
-use arrow::array::BooleanArray;
-use arrow::compute::filter;
-use arrow::datatypes::SchemaRef;
-use arrow::error::Result as ArrowResult;
-use arrow::record_batch::{RecordBatch, RecordBatchReader};
-
-/// Execution plan for a Selection
-#[derive(Debug)]
-pub struct SelectionExec {
-    /// The selection predicate expression
-    expr: Arc<dyn PhysicalExpr>,
-    /// The input plan
-    input: Arc<dyn ExecutionPlan>,
-}
-
-impl SelectionExec {
-    /// Create a selection on an input
-    pub fn try_new(
-        expr: Arc<dyn PhysicalExpr>,
-        input: Arc<dyn ExecutionPlan>,
-    ) -> Result<Self> {
-        Ok(Self {
-            expr: expr.clone(),
-            input: input.clone(),
-        })
-    }
-}
-
-impl ExecutionPlan for SelectionExec {
-    /// Get the schema for this execution plan
-    fn schema(&self) -> SchemaRef {
-        // The selection operator does not make any changes to the schema of its input
-        self.input.schema()
-    }
-
-    /// Get the partitions for this execution plan
-    fn partitions(&self) -> Result<Vec<Arc<dyn Partition>>> {
-        let partitions: Vec<Arc<dyn Partition>> = self
-            .input
-            .partitions()?
-            .iter()
-            .map(|p| {
-                let expr = self.expr.clone();
-                let partition: Arc<dyn Partition> = Arc::new(SelectionPartition {
-                    schema: self.input.schema(),
-                    expr,
-                    input: p.clone() as Arc<dyn Partition>,
-                });
-
-                partition
-            })
-            .collect();
-
-        Ok(partitions)
-    }
-}
-
-/// Represents a single partition of a Selection execution plan
-#[derive(Debug)]
-struct SelectionPartition {
-    schema: SchemaRef,
-    expr: Arc<dyn PhysicalExpr>,
-    input: Arc<dyn Partition>,
-}
-
-impl Partition for SelectionPartition {
-    /// Execute the Selection
-    fn execute(&self) -> Result<Arc<Mutex<dyn RecordBatchReader + Send + Sync>>> {
-        Ok(Arc::new(Mutex::new(SelectionIterator {
-            schema: self.schema.clone(),
-            expr: self.expr.clone(),
-            input: self.input.execute()?,
-        })))
-    }
-}
-
-/// Selection iterator
-struct SelectionIterator {
-    schema: SchemaRef,
-    expr: Arc<dyn PhysicalExpr>,
-    input: Arc<Mutex<dyn RecordBatchReader + Send + Sync>>,
-}
-
-impl RecordBatchReader for SelectionIterator {
-    /// Get the schema
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-
-    /// Get the next batch
-    fn next_batch(&mut self) -> ArrowResult<Option<RecordBatch>> {
-        let mut input = self.input.lock().unwrap();
-        match input.next_batch()? {
-            Some(batch) => {
-                // evaluate the selection predicate to get a boolean array
-                let predicate_result = self
-                    .expr
-                    .evaluate(&batch)
-                    .map_err(ExecutionError::into_arrow_external_error)?;
-
-                if let Some(f) = predicate_result.as_any().downcast_ref::<BooleanArray>()
-                {
-                    // filter each array
-                    let mut filtered_arrays = vec![];
-                    for i in 0..batch.num_columns() {
-                        let array = batch.column(i);
-                        let filtered_array = filter(array.as_ref(), f)?;
-                        filtered_arrays.push(filtered_array);
-                    }
-                    Ok(Some(RecordBatch::try_new(
-                        batch.schema().clone(),
-                        filtered_arrays,
-                    )?))
-                } else {
-                    Err(ExecutionError::InternalError(
-                        "Predicate evaluated to non-boolean value".to_string(),
-                    )
-                    .into_arrow_external_error())
-                }
-            }
-            None => Ok(None),
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-
-    use super::*;
-    use crate::execution::physical_plan::csv::{CsvExec, CsvReadOptions};
-    use crate::execution::physical_plan::expressions::*;
-    use crate::execution::physical_plan::ExecutionPlan;
-    use crate::logicalplan::{Operator, ScalarValue};
-    use crate::test;
-    use std::iter::Iterator;
-
-    #[test]
-    fn simple_predicate() -> Result<()> {
-        let schema = test::aggr_test_schema();
-
-        let partitions = 4;
-        let path = test::create_partitioned_csv("aggregate_test_100.csv", partitions)?;
-
-        let csv =
-            CsvExec::try_new(&path, CsvReadOptions::new().schema(&schema), None, 1024)?;
-
-        let predicate: Arc<dyn PhysicalExpr> = binary(
-            binary(col("c2"), Operator::Gt, lit(ScalarValue::UInt32(1))),
-            Operator::And,
-            binary(col("c2"), Operator::Lt, lit(ScalarValue::UInt32(4))),
-        );
-
-        let selection: Arc<dyn ExecutionPlan> =
-            Arc::new(SelectionExec::try_new(predicate, Arc::new(csv))?);
-
-        let results = test::execute(selection.as_ref())?;
-
-        results
-            .iter()
-            .for_each(|batch| assert_eq!(13, batch.num_columns()));
-        let row_count: usize = results.iter().map(|batch| batch.num_rows()).sum();
-        assert_eq!(41, row_count);
-
-        Ok(())
-    }
-}
diff --git a/rust/datafusion/src/execution/physical_plan/sort.rs b/rust/datafusion/src/execution/physical_plan/sort.rs
deleted file mode 100644
index e0fcd0b8564..00000000000
--- a/rust/datafusion/src/execution/physical_plan/sort.rs
+++ /dev/null
@@ -1,213 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Defines the SORT plan
-
-use std::sync::{Arc, Mutex};
-use std::thread;
-use std::thread::JoinHandle;
-
-use arrow::array::ArrayRef;
-pub use arrow::compute::SortOptions;
-use arrow::compute::{concat, lexsort_to_indices, take, SortColumn, TakeOptions};
-use arrow::datatypes::SchemaRef;
-use arrow::record_batch::{RecordBatch, RecordBatchReader};
-
-use crate::error::Result;
-use crate::execution::physical_plan::common::RecordBatchIterator;
-use crate::execution::physical_plan::expressions::PhysicalSortExpr;
-use crate::execution::physical_plan::{common, ExecutionPlan, Partition};
-
-/// Sort execution plan
-#[derive(Debug)]
-pub struct SortExec {
-    /// Input schema
-    input: Arc<dyn ExecutionPlan>,
-    expr: Vec<PhysicalSortExpr>,
-}
-
-impl SortExec {
-    /// Create a new sort execution plan
-    pub fn try_new(
-        expr: Vec<PhysicalSortExpr>,
-        input: Arc<dyn ExecutionPlan>,
-    ) -> Result<Self> {
-        Ok(Self { expr, input })
-    }
-}
-
-impl ExecutionPlan for SortExec {
-    fn schema(&self) -> SchemaRef {
-        self.input.schema().clone()
-    }
-
-    fn partitions(&self) -> Result<Vec<Arc<dyn Partition>>> {
-        Ok(vec![
-            (Arc::new(SortPartition {
-                input: self.input.partitions()?,
-                expr: self.expr.clone(),
-                schema: self.schema(),
-            })),
-        ])
-    }
-}
-
-/// Represents a single partition of a Sort execution plan
-#[derive(Debug)]
-struct SortPartition {
-    schema: SchemaRef,
-    expr: Vec<PhysicalSortExpr>,
-    input: Vec<Arc<dyn Partition>>,
-}
-
-impl Partition for SortPartition {
-    /// Execute the sort
-    fn execute(&self) -> Result<Arc<Mutex<dyn RecordBatchReader + Send + Sync>>> {
-        let threads: Vec<JoinHandle<Result<Vec<RecordBatch>>>> = self
-            .input
-            .iter()
-            .map(|p| {
-                let p = p.clone();
-                thread::spawn(move || {
-                    let it = p.execute()?;
-                    common::collect(it)
-                })
-            })
-            .collect();
-
-        // generate record batches from input in parallel
-        let mut all_batches: Vec<Arc<RecordBatch>> = vec![];
-        for thread in threads {
-            let join = thread.join().expect("Failed to join thread");
-            let result = join?;
-            result
-                .iter()
-                .for_each(|batch| all_batches.push(Arc::new(batch.clone())));
-        }
-
-        // combine all record batches into one for each column
-        let combined_batch = RecordBatch::try_new(
-            self.schema.clone(),
-            self.schema
-                .fields()
-                .iter()
-                .enumerate()
-                .map(|(i, _)| -> Result<ArrayRef> {
-                    Ok(concat(
-                        &all_batches
-                            .iter()
-                            .map(|batch| batch.columns()[i].clone())
-                            .collect::<Vec<ArrayRef>>(),
-                    )?)
-                })
-                .collect::<Result<Vec<ArrayRef>>>()?,
-        )?;
-
-        // sort combined record batch
-        let indices = lexsort_to_indices(
-            &self
-                .expr
-                .iter()
-                .map(|e| e.evaluate_to_sort_column(&combined_batch))
-                .collect::<Result<Vec<SortColumn>>>()?,
-        )?;
-
-        // reorder all rows based on sorted indices
-        let sorted_batch = RecordBatch::try_new(
-            self.schema.clone(),
-            combined_batch
-                .columns()
-                .iter()
-                .map(|column| -> Result<ArrayRef> {
-                    Ok(take(
-                        column,
-                        &indices,
-                        // disable bound check overhead since indices are already generated from
-                        // the same record batch
-                        Some(TakeOptions {
-                            check_bounds: false,
-                        }),
-                    )?)
-                })
-                .collect::<Result<Vec<ArrayRef>>>()?,
-        )?;
-
-        Ok(Arc::new(Mutex::new(RecordBatchIterator::new(
-            self.schema.clone(),
-            vec![Arc::new(sorted_batch)],
-        ))))
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::execution::physical_plan::csv::{CsvExec, CsvReadOptions};
-    use crate::execution::physical_plan::expressions::col;
-    use crate::test;
-    use arrow::array::*;
-    use arrow::datatypes::*;
-
-    #[test]
-    fn test_sort() -> Result<()> {
-        let schema = test::aggr_test_schema();
-        let partitions = 4;
-        let path = test::create_partitioned_csv("aggregate_test_100.csv", partitions)?;
-        let csv =
-            CsvExec::try_new(&path, CsvReadOptions::new().schema(&schema), None, 1024)?;
-
-        let sort_exec = SortExec::try_new(
-            vec![
-                // c1 string column
-                PhysicalSortExpr {
-                    expr: col("c1"),
-                    options: SortOptions::default(),
-                },
-                // c2 uin32 column
-                PhysicalSortExpr {
-                    expr: col("c2"),
-                    options: SortOptions::default(),
-                },
-                // c7 uin8 column
-                PhysicalSortExpr {
-                    expr: col("c7"),
-                    options: SortOptions::default(),
-                },
-            ],
-            Arc::new(csv),
-        )?;
-
-        let result: Vec<RecordBatch> = test::execute(&sort_exec)?;
-        assert_eq!(result.len(), 1);
-
-        let columns = result[0].columns();
-
-        let c1 = as_string_array(&columns[0]);
-        assert_eq!(c1.value(0), "a");
-        assert_eq!(c1.value(c1.len() - 1), "e");
-
-        let c2 = as_primitive_array::<UInt32Type>(&columns[1]);
-        assert_eq!(c2.value(0), 1);
-        assert_eq!(c2.value(c2.len() - 1), 5,);
-
-        let c7 = as_primitive_array::<UInt8Type>(&columns[6]);
-        assert_eq!(c7.value(0), 15);
-        assert_eq!(c7.value(c7.len() - 1), 254,);
-
-        Ok(())
-    }
-}
diff --git a/rust/datafusion/src/execution/physical_plan/udf.rs b/rust/datafusion/src/execution/physical_plan/udf.rs
deleted file mode 100644
index 944b5c9bef3..00000000000
--- a/rust/datafusion/src/execution/physical_plan/udf.rs
+++ /dev/null
@@ -1,148 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! UDF support
-
-use std::fmt;
-
-use arrow::array::ArrayRef;
-use arrow::datatypes::{DataType, Field, Schema};
-
-use crate::error::Result;
-use crate::execution::physical_plan::PhysicalExpr;
-
-use arrow::record_batch::RecordBatch;
-use fmt::{Debug, Formatter};
-use std::sync::Arc;
-
-/// Scalar UDF
-pub type ScalarUdf = Arc<dyn Fn(&[ArrayRef]) -> Result<ArrayRef> + Send + Sync>;
-
-/// Scalar UDF Expression
-#[derive(Clone)]
-pub struct ScalarFunction {
-    /// Function name
-    pub name: String,
-    /// Function argument meta-data
-    pub args: Vec<Field>,
-    /// Return type
-    pub return_type: DataType,
-    /// UDF implementation
-    pub fun: ScalarUdf,
-}
-
-impl Debug for ScalarFunction {
-    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
-        f.debug_struct("ScalarFunction")
-            .field("name", &self.name)
-            .field("args", &self.args)
-            .field("return_type", &self.return_type)
-            .field("fun", &"<FUNC>")
-            .finish()
-    }
-}
-
-impl ScalarFunction {
-    /// Create a new ScalarFunction
-    pub fn new(
-        name: &str,
-        args: Vec<Field>,
-        return_type: DataType,
-        fun: ScalarUdf,
-    ) -> Self {
-        Self {
-            name: name.to_owned(),
-            args,
-            return_type,
-            fun,
-        }
-    }
-}
-
-/// Scalar UDF Physical Expression
-pub struct ScalarFunctionExpr {
-    fun: Box<ScalarUdf>,
-    name: String,
-    args: Vec<Arc<dyn PhysicalExpr>>,
-    return_type: DataType,
-}
-
-impl Debug for ScalarFunctionExpr {
-    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
-        f.debug_struct("ScalarFunctionExpr")
-            .field("fun", &"<FUNC>")
-            .field("name", &self.name)
-            .field("args", &self.args)
-            .field("return_type", &self.return_type)
-            .finish()
-    }
-}
-
-impl ScalarFunctionExpr {
-    /// Create a new Scalar function
-    pub fn new(
-        name: &str,
-        fun: Box<ScalarUdf>,
-        args: Vec<Arc<dyn PhysicalExpr>>,
-        return_type: &DataType,
-    ) -> Self {
-        Self {
-            fun,
-            name: name.to_owned(),
-            args,
-            return_type: return_type.clone(),
-        }
-    }
-}
-
-impl fmt::Display for ScalarFunctionExpr {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "{}({})",
-            self.name,
-            self.args
-                .iter()
-                .map(|e| format!("{}", e))
-                .collect::<Vec<String>>()
-                .join(", ")
-        )
-    }
-}
-
-impl PhysicalExpr for ScalarFunctionExpr {
-    fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
-        Ok(self.return_type.clone())
-    }
-
-    fn nullable(&self, _input_schema: &Schema) -> Result<bool> {
-        Ok(true)
-    }
-
-    fn evaluate(&self, batch: &RecordBatch) -> Result<ArrayRef> {
-        // evaluate the arguments
-        let inputs = self
-            .args
-            .iter()
-            .map(|e| e.evaluate(batch))
-            .collect::<Result<Vec<_>>>()?;
-
-        // evaluate the function
-        let fun = self.fun.as_ref();
-        (fun)(&inputs)
-    }
-}
diff --git a/rust/datafusion/src/lib.rs b/rust/datafusion/src/lib.rs
index fb4e5af303f..45a73eb74b8 100644
--- a/rust/datafusion/src/lib.rs
+++ b/rust/datafusion/src/lib.rs
@@ -15,27 +15,152 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#![warn(missing_docs)]
+
 //! DataFusion is an extensible query execution framework that uses
-//! Apache Arrow as the memory model.
+//! [Apache Arrow](https://arrow.apache.org) as its in-memory format.
 //!
-//! DataFusion supports both SQL and a Table/DataFrame-style API for building logical query plans
-//! and also provides a query optimizer and execution engine capable of parallel execution
+//! DataFusion supports both an SQL and a DataFrame API for building logical query plans
+//! as well as a query optimizer and execution engine capable of parallel execution
 //! against partitioned data sources (CSV and Parquet) using threads.
 //!
-//! DataFusion currently supports simple projection, selection, and aggregate queries.
-
-#![warn(missing_docs)]
+//! Below is an example of how to execute a query against a CSV using [`DataFrames`](dataframe::DataFrame):
+//!
+//! ```rust
+//! # use datafusion::prelude::*;
+//! # use datafusion::error::Result;
+//! # use arrow::record_batch::RecordBatch;
+//!
+//! # #[tokio::main]
+//! # async fn main() -> Result<()> {
+//! let mut ctx = ExecutionContext::new();
+//!
+//! // create the dataframe
+//! let df = ctx.read_csv("tests/example.csv", CsvReadOptions::new())?;
+//!
+//! // create a plan
+//! let df = df.filter(col("a").lt_eq(col("b")))?
+//!            .aggregate(vec![col("a")], vec![min(col("b"))])?
+//!            .limit(100)?;
+//!
+//! // execute the plan
+//! let results: Vec<RecordBatch> = df.collect().await?;
+//! # Ok(())
+//! # }
+//! ```
+//!
+//! and how to execute a query against a CSV using SQL:
+//!
+//! ```
+//! # use datafusion::prelude::*;
+//! # use datafusion::error::Result;
+//! # use arrow::record_batch::RecordBatch;
+//!
+//! # #[tokio::main]
+//! # async fn main() -> Result<()> {
+//! let mut ctx = ExecutionContext::new();
+//!
+//! ctx.register_csv("example", "tests/example.csv", CsvReadOptions::new())?;
+//!
+//! // create a plan
+//! let df = ctx.sql("SELECT a, MIN(b) FROM example GROUP BY a LIMIT 100")?;
+//!
+//! // execute the plan
+//! let results: Vec<RecordBatch> = df.collect().await?;
+//! # Ok(())
+//! # }
+//! ```
+//!
+//! ## Parse, Plan, Optimize, Execute
+//!
+//! DataFusion is a fully fledged query engine capable of performing complex operations.
+//! Specifically, when DataFusion receives an SQL query, there are different steps
+//! that it passes through until a result is obtained. Broadly, they are:
+//!
+//! 1. The string is parsed to an Abstract syntax tree (AST) using [sqlparser](https://docs.rs/sqlparser/0.6.1/sqlparser/).
+//! 2. The planner [`SqlToRel`](sql::planner::SqlToRel) converts logical expressions on the AST to logical expressions [`Expr`s](logical_plan::Expr).
+//! 3. The planner [`SqlToRel`](sql::planner::SqlToRel) converts logical nodes on the AST to a [`LogicalPlan`](logical_plan::LogicalPlan).
+//! 4. [`OptimizerRules`](optimizer::optimizer::OptimizerRule) are applied to the [`LogicalPlan`](logical_plan::LogicalPlan) to optimize it.
+//! 5. The [`LogicalPlan`](logical_plan::LogicalPlan) is converted to an [`ExecutionPlan`](physical_plan::ExecutionPlan) by a [`PhysicalPlanner`](physical_plan::PhysicalPlanner)
+//! 6. The [`ExecutionPlan`](physical_plan::ExecutionPlan) is executed against data through the [`ExecutionContext`](execution::context::ExecutionContext)
+//!
+//! With a [`DataFrame`](dataframe::DataFrame) API, steps 1-3 are not used as the DataFrame builds the [`LogicalPlan`](logical_plan::LogicalPlan) directly.
+//!
+//! Phases 1-5 are typically cheap when compared to phase 6, and thus DataFusion puts a
+//! lot of effort to ensure that phase 6 runs efficiently and without errors.
+//!
+//! DataFusion's planning is divided in two main parts: logical planning and physical planning.
+//!
+//! ### Logical plan
+//!
+//! Logical planning yields [`logical plans`](logical_plan::LogicalPlan) and [`logical expressions`](logical_plan::Expr).
+//! These are [`Schema`](arrow::datatypes::Schema)-aware traits that represent statements whose result is independent of how it should physically be executed.
+//!
+//! A [`LogicalPlan`](logical_plan::LogicalPlan) is a Direct Asyclic graph of other [`LogicalPlan`s](logical_plan::LogicalPlan) and each node contains logical expressions ([`Expr`s](logical_plan::Expr)).
+//! All of these are located in [`logical_plan`](logical_plan).
+//!
+//! ### Physical plan
+//!
+//! A Physical plan ([`ExecutionPlan`](physical_plan::ExecutionPlan)) is a plan that can be executed against data.
+//! Contrarily to a logical plan, the physical plan has concrete information about how the calculation
+//! should be performed (e.g. what Rust functions are used) and how data should be loaded into memory.
+//!
+//! [`ExecutionPlan`](physical_plan::ExecutionPlan) uses the Arrow format as its in-memory representation of data, through the [arrow] crate.
+//! We recommend going through [its documentation](arrow) for details on how the data is physically represented.
+//!
+//! A [`ExecutionPlan`](physical_plan::ExecutionPlan) is composed by nodes (implement the trait [`ExecutionPlan`](physical_plan::ExecutionPlan)),
+//! and each node is composed by physical expressions ([`PhysicalExpr`](physical_plan::PhysicalExpr))
+//! or aggreagate expressions ([`AggregateExpr`](physical_plan::AggregateExpr)).
+//! All of these are located in the module [`physical_plan`](physical_plan).
+//!
+//! Broadly speaking,
+//!
+//! * an [`ExecutionPlan`](physical_plan::ExecutionPlan) receives a partition number and asyncronosly returns
+//!   an iterator over [`RecordBatch`](arrow::record_batch::RecordBatch)
+//!   (a node-specific struct that implements [`RecordBatchReader`](arrow::record_batch::RecordBatchReader))
+//! * a [`PhysicalExpr`](physical_plan::PhysicalExpr) receives a [`RecordBatch`](arrow::record_batch::RecordBatch)
+//!   and returns an [`Array`](arrow::array::Array)
+//! * an [`AggregateExpr`](physical_plan::AggregateExpr) receives [`RecordBatch`es](arrow::record_batch::RecordBatch)
+//!   and returns a [`RecordBatch`](arrow::record_batch::RecordBatch) of a single row(*)
+//!
+//! (*) Technically, it aggregates the results on each partition and then merges the results into a single partition.
+//!
+//! The following physical nodes are currently implemented:
+//!
+//! * Projection: [`ProjectionExec`](physical_plan::projection::ProjectionExec)
+//! * Filter: [`FilterExec`](physical_plan::filter::FilterExec)
+//! * Hash and Grouped aggregations: [`HashAggregateExec`](physical_plan::hash_aggregate::HashAggregateExec)
+//! * Sort: [`SortExec`](physical_plan::sort::SortExec)
+//! * Merge (partitions): [`MergeExec`](physical_plan::merge::MergeExec)
+//! * Limit: [`LocalLimitExec`](physical_plan::limit::LocalLimitExec) and [`GlobalLimitExec`](physical_plan::limit::GlobalLimitExec)
+//! * Scan a CSV: [`CsvExec`](physical_plan::csv::CsvExec)
+//! * Scan a Parquet: [`ParquetExec`](physical_plan::parquet::ParquetExec)
+//! * Scan from memory: [`MemoryExec`](physical_plan::memory::MemoryExec)
+//! * Explain the plan: [`ExplainExec`](physical_plan::explain::ExplainExec)
+//!
+//! ## Customize
+//!
+//! DataFusion allows users to
+//! * extend the planner to use user-defined logical and physical nodes ([`QueryPlanner`](execution::context::QueryPlanner))
+//! * declare and use user-defined scalar functions ([`ScalarUDF`](physical_plan::udf::ScalarUDF))
+//! * declare and use user-defined aggregate functions ([`AggregateUDF`](physical_plan::udaf::AggregateUDF))
+//!
+//! you can find examples of each of them in examples section.
 
 extern crate arrow;
 extern crate sqlparser;
 
+pub mod dataframe;
 pub mod datasource;
 pub mod error;
 pub mod execution;
-pub mod logicalplan;
+pub mod logical_plan;
 pub mod optimizer;
+pub mod physical_plan;
+pub mod prelude;
+pub mod scalar;
 pub mod sql;
-pub mod table;
+pub mod variable;
 
 #[cfg(test)]
 pub mod test;
diff --git a/rust/datafusion/src/logical_plan/mod.rs b/rust/datafusion/src/logical_plan/mod.rs
new file mode 100644
index 00000000000..19ee56d76ba
--- /dev/null
+++ b/rust/datafusion/src/logical_plan/mod.rs
@@ -0,0 +1,1532 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! This module provides a logical query plan enum that can describe queries. Logical query
+//! plans can be created from a SQL statement or built programmatically via the Table API.
+//!
+//! Logical query plans can then be optimized and executed directly, or translated into
+//! physical query plans and executed.
+
+use fmt::Debug;
+use std::{any::Any, collections::HashMap, collections::HashSet, fmt, sync::Arc};
+
+use aggregates::{AccumulatorFunctionImplementation, StateTypeFunction};
+use arrow::{
+    compute::can_cast_types,
+    datatypes::{DataType, Field, Schema, SchemaRef},
+};
+
+use crate::datasource::parquet::ParquetTable;
+use crate::datasource::TableProvider;
+use crate::error::{DataFusionError, Result};
+use crate::{
+    datasource::csv::{CsvFile, CsvReadOptions},
+    physical_plan::udaf::AggregateUDF,
+    scalar::ScalarValue,
+};
+use crate::{
+    physical_plan::{
+        aggregates, expressions::binary_operator_data_type, functions, udf::ScalarUDF,
+    },
+    sql::parser::FileType,
+};
+use arrow::record_batch::RecordBatch;
+use functions::{ReturnTypeFunction, ScalarFunctionImplementation, Signature};
+
+mod operators;
+pub use operators::Operator;
+
+fn create_function_name(
+    fun: &String,
+    distinct: bool,
+    args: &[Expr],
+    input_schema: &Schema,
+) -> Result<String> {
+    let names: Vec<String> = args
+        .iter()
+        .map(|e| create_name(e, input_schema))
+        .collect::<Result<_>>()?;
+    let distinct_str = match distinct {
+        true => "DISTINCT ",
+        false => "",
+    };
+    Ok(format!("{}({}{})", fun, distinct_str, names.join(",")))
+}
+
+/// Returns a readable name of an expression based on the input schema.
+/// This function recursively transverses the expression for names such as "CAST(a > 2)".
+fn create_name(e: &Expr, input_schema: &Schema) -> Result<String> {
+    match e {
+        Expr::Alias(_, name) => Ok(name.clone()),
+        Expr::Column(name) => Ok(name.clone()),
+        Expr::ScalarVariable(variable_names) => Ok(variable_names.join(".")),
+        Expr::Literal(value) => Ok(format!("{:?}", value)),
+        Expr::BinaryExpr { left, op, right } => {
+            let left = create_name(left, input_schema)?;
+            let right = create_name(right, input_schema)?;
+            Ok(format!("{} {:?} {}", left, op, right))
+        }
+        Expr::Cast { expr, data_type } => {
+            let expr = create_name(expr, input_schema)?;
+            Ok(format!("CAST({} AS {:?})", expr, data_type))
+        }
+        Expr::Not(expr) => {
+            let expr = create_name(expr, input_schema)?;
+            Ok(format!("NOT {}", expr))
+        }
+        Expr::IsNull(expr) => {
+            let expr = create_name(expr, input_schema)?;
+            Ok(format!("{} IS NULL", expr))
+        }
+        Expr::IsNotNull(expr) => {
+            let expr = create_name(expr, input_schema)?;
+            Ok(format!("{} IS NOT NULL", expr))
+        }
+        Expr::ScalarFunction { fun, args, .. } => {
+            create_function_name(&fun.to_string(), false, args, input_schema)
+        }
+        Expr::ScalarUDF { fun, args, .. } => {
+            create_function_name(&fun.name, false, args, input_schema)
+        }
+        Expr::AggregateFunction {
+            fun,
+            distinct,
+            args,
+            ..
+        } => create_function_name(&fun.to_string(), *distinct, args, input_schema),
+        Expr::AggregateUDF { fun, args } => {
+            let mut names = Vec::with_capacity(args.len());
+            for e in args {
+                names.push(create_name(e, input_schema)?);
+            }
+            Ok(format!("{}({})", fun.name, names.join(",")))
+        }
+        other => Err(DataFusionError::NotImplemented(format!(
+            "Physical plan does not support logical expression {:?}",
+            other
+        ))),
+    }
+}
+
+/// Create field meta-data from an expression, for use in a result set schema
+pub fn exprlist_to_fields(expr: &[Expr], input_schema: &Schema) -> Result<Vec<Field>> {
+    expr.iter().map(|e| e.to_field(input_schema)).collect()
+}
+
+/// `Expr` is a logical expression. A logical expression is something like `1 + 1`, or `CAST(c1 AS int)`.
+/// Logical expressions know how to compute its [arrow::datatypes::DataType] and nullability.
+/// `Expr` is a central struct of DataFusion's query API.
+///
+/// # Examples
+///
+/// ```
+/// # use datafusion::logical_plan::Expr;
+/// # use datafusion::error::Result;
+/// # fn main() -> Result<()> {
+/// let expr = Expr::Column("c1".to_string()) + Expr::Column("c2".to_string());
+/// println!("{:?}", expr);
+/// # Ok(())
+/// # }
+/// ```
+#[derive(Clone)]
+pub enum Expr {
+    /// An expression with a specific name.
+    Alias(Box<Expr>, String),
+    /// A named reference to a field in a schema.
+    Column(String),
+    /// A named reference to a variable in a registry.
+    ScalarVariable(Vec<String>),
+    /// A constant value.
+    Literal(ScalarValue),
+    /// A binary expression such as "age > 21"
+    BinaryExpr {
+        /// Left-hand side of the expression
+        left: Box<Expr>,
+        /// The comparison operator
+        op: Operator,
+        /// Right-hand side of the expression
+        right: Box<Expr>,
+    },
+    /// Parenthesized expression. E.g. `(foo > bar)` or `(1)`
+    Nested(Box<Expr>),
+    /// Negation of an expression. The expression's type must be a boolean to make sense.
+    Not(Box<Expr>),
+    /// Whether an expression is not Null. This expression is never null.
+    IsNotNull(Box<Expr>),
+    /// Whether an expression is Null. This expression is never null.
+    IsNull(Box<Expr>),
+    /// Casts the expression to a given type. This expression is guaranteed to have a fixed type.
+    Cast {
+        /// The expression being cast
+        expr: Box<Expr>,
+        /// The `DataType` the expression will yield
+        data_type: DataType,
+    },
+    /// A sort expression, that can be used to sort values.
+    Sort {
+        /// The expression to sort on
+        expr: Box<Expr>,
+        /// The direction of the sort
+        asc: bool,
+        /// Whether to put Nulls before all other data values
+        nulls_first: bool,
+    },
+    /// Represents the call of a built-in scalar function with a set of arguments.
+    ScalarFunction {
+        /// The function
+        fun: functions::BuiltinScalarFunction,
+        /// List of expressions to feed to the functions as arguments
+        args: Vec<Expr>,
+    },
+    /// Represents the call of a user-defined scalar function with arguments.
+    ScalarUDF {
+        /// The function
+        fun: Arc<ScalarUDF>,
+        /// List of expressions to feed to the functions as arguments
+        args: Vec<Expr>,
+    },
+    /// Represents the call of an aggregate built-in function with arguments.
+    AggregateFunction {
+        /// Name of the function
+        fun: aggregates::AggregateFunction,
+        /// List of expressions to feed to the functions as arguments
+        args: Vec<Expr>,
+        /// Whether this is a DISTINCT aggregation or not
+        distinct: bool,
+    },
+    /// aggregate function
+    AggregateUDF {
+        /// The function
+        fun: Arc<AggregateUDF>,
+        /// List of expressions to feed to the functions as arguments
+        args: Vec<Expr>,
+    },
+    /// Represents a reference to all fields in a schema.
+    Wildcard,
+}
+
+impl Expr {
+    /// Returns the [arrow::datatypes::DataType] of the expression based on [arrow::datatypes::Schema].
+    ///
+    /// # Errors
+    ///
+    /// This function errors when it is not possible to compute its [arrow::datatypes::DataType].
+    /// This happens when e.g. the expression refers to a column that does not exist in the schema, or when
+    /// the expression is incorrectly typed (e.g. `[utf8] + [bool]`).
+    pub fn get_type(&self, schema: &Schema) -> Result<DataType> {
+        match self {
+            Expr::Alias(expr, _) => expr.get_type(schema),
+            Expr::Column(name) => Ok(schema.field_with_name(name)?.data_type().clone()),
+            Expr::ScalarVariable(_) => Ok(DataType::Utf8),
+            Expr::Literal(l) => Ok(l.get_datatype()),
+            Expr::Cast { data_type, .. } => Ok(data_type.clone()),
+            Expr::ScalarUDF { fun, args } => {
+                let data_types = args
+                    .iter()
+                    .map(|e| e.get_type(schema))
+                    .collect::<Result<Vec<_>>>()?;
+                Ok((fun.return_type)(&data_types)?.as_ref().clone())
+            }
+            Expr::ScalarFunction { fun, args } => {
+                let data_types = args
+                    .iter()
+                    .map(|e| e.get_type(schema))
+                    .collect::<Result<Vec<_>>>()?;
+                functions::return_type(fun, &data_types)
+            }
+            Expr::AggregateFunction { fun, args, .. } => {
+                let data_types = args
+                    .iter()
+                    .map(|e| e.get_type(schema))
+                    .collect::<Result<Vec<_>>>()?;
+                aggregates::return_type(fun, &data_types)
+            }
+            Expr::AggregateUDF { fun, args, .. } => {
+                let data_types = args
+                    .iter()
+                    .map(|e| e.get_type(schema))
+                    .collect::<Result<Vec<_>>>()?;
+                Ok((fun.return_type)(&data_types)?.as_ref().clone())
+            }
+            Expr::Not(_) => Ok(DataType::Boolean),
+            Expr::IsNull(_) => Ok(DataType::Boolean),
+            Expr::IsNotNull(_) => Ok(DataType::Boolean),
+            Expr::BinaryExpr {
+                ref left,
+                ref right,
+                ref op,
+            } => binary_operator_data_type(
+                &left.get_type(schema)?,
+                op,
+                &right.get_type(schema)?,
+            ),
+            Expr::Sort { ref expr, .. } => expr.get_type(schema),
+            Expr::Wildcard => Err(DataFusionError::Internal(
+                "Wildcard expressions are not valid in a logical query plan".to_owned(),
+            )),
+            Expr::Nested(e) => e.get_type(schema),
+        }
+    }
+
+    /// Returns the nullability of the expression based on [arrow::datatypes::Schema].
+    ///
+    /// # Errors
+    ///
+    /// This function errors when it is not possible to compute its nullability.
+    /// This happens when the expression refers to a column that does not exist in the schema.
+    pub fn nullable(&self, input_schema: &Schema) -> Result<bool> {
+        match self {
+            Expr::Alias(expr, _) => expr.nullable(input_schema),
+            Expr::Column(name) => Ok(input_schema.field_with_name(name)?.is_nullable()),
+            Expr::Literal(value) => Ok(value.is_null()),
+            Expr::ScalarVariable(_) => Ok(true),
+            Expr::Cast { expr, .. } => expr.nullable(input_schema),
+            Expr::ScalarFunction { .. } => Ok(true),
+            Expr::ScalarUDF { .. } => Ok(true),
+            Expr::AggregateFunction { .. } => Ok(true),
+            Expr::AggregateUDF { .. } => Ok(true),
+            Expr::Not(expr) => expr.nullable(input_schema),
+            Expr::IsNull(_) => Ok(false),
+            Expr::IsNotNull(_) => Ok(false),
+            Expr::BinaryExpr {
+                ref left,
+                ref right,
+                ..
+            } => Ok(left.nullable(input_schema)? || right.nullable(input_schema)?),
+            Expr::Sort { ref expr, .. } => expr.nullable(input_schema),
+            Expr::Nested(e) => e.nullable(input_schema),
+            Expr::Wildcard => Err(DataFusionError::Internal(
+                "Wildcard expressions are not valid in a logical query plan".to_owned(),
+            )),
+        }
+    }
+
+    /// Returns the name of this expression based on [arrow::datatypes::Schema].
+    ///
+    /// This represents how a column with this expression is named when no alias is chosen
+    pub fn name(&self, input_schema: &Schema) -> Result<String> {
+        create_name(self, input_schema)
+    }
+
+    /// Returns a [arrow::datatypes::Field] compatible with this expression.
+    pub fn to_field(&self, input_schema: &Schema) -> Result<Field> {
+        Ok(Field::new(
+            &self.name(input_schema)?,
+            self.get_type(input_schema)?,
+            self.nullable(input_schema)?,
+        ))
+    }
+
+    /// Wraps this expression in a cast to a target [arrow::datatypes::DataType].
+    ///
+    /// # Errors
+    ///
+    /// This function errors when it is impossible to cast the
+    /// expression to the target [arrow::datatypes::DataType].
+    pub fn cast_to(&self, cast_to_type: &DataType, schema: &Schema) -> Result<Expr> {
+        let this_type = self.get_type(schema)?;
+        if this_type == *cast_to_type {
+            Ok(self.clone())
+        } else if can_cast_types(&this_type, cast_to_type) {
+            Ok(Expr::Cast {
+                expr: Box::new(self.clone()),
+                data_type: cast_to_type.clone(),
+            })
+        } else {
+            Err(DataFusionError::Plan(format!(
+                "Cannot automatically convert {:?} to {:?}",
+                this_type, cast_to_type
+            )))
+        }
+    }
+
+    /// Equal
+    pub fn eq(&self, other: Expr) -> Expr {
+        binary_expr(self.clone(), Operator::Eq, other.clone())
+    }
+
+    /// Not equal
+    pub fn not_eq(&self, other: Expr) -> Expr {
+        binary_expr(self.clone(), Operator::NotEq, other.clone())
+    }
+
+    /// Greater than
+    pub fn gt(&self, other: Expr) -> Expr {
+        binary_expr(self.clone(), Operator::Gt, other.clone())
+    }
+
+    /// Greater than or equal to
+    pub fn gt_eq(&self, other: Expr) -> Expr {
+        binary_expr(self.clone(), Operator::GtEq, other.clone())
+    }
+
+    /// Less than
+    pub fn lt(&self, other: Expr) -> Expr {
+        binary_expr(self.clone(), Operator::Lt, other.clone())
+    }
+
+    /// Less than or equal to
+    pub fn lt_eq(&self, other: Expr) -> Expr {
+        binary_expr(self.clone(), Operator::LtEq, other.clone())
+    }
+
+    /// And
+    pub fn and(&self, other: Expr) -> Expr {
+        binary_expr(self.clone(), Operator::And, other)
+    }
+
+    /// Or
+    pub fn or(&self, other: Expr) -> Expr {
+        binary_expr(self.clone(), Operator::Or, other)
+    }
+
+    /// Not
+    pub fn not(&self) -> Expr {
+        Expr::Not(Box::new(self.clone()))
+    }
+
+    /// Calculate the modulus of two expressions
+    pub fn modulus(&self, other: Expr) -> Expr {
+        binary_expr(self.clone(), Operator::Modulus, other.clone())
+    }
+
+    /// like (string) another expression
+    pub fn like(&self, other: Expr) -> Expr {
+        binary_expr(self.clone(), Operator::Like, other.clone())
+    }
+
+    /// not like another expression
+    pub fn not_like(&self, other: Expr) -> Expr {
+        binary_expr(self.clone(), Operator::NotLike, other.clone())
+    }
+
+    /// Alias
+    pub fn alias(&self, name: &str) -> Expr {
+        Expr::Alias(Box::new(self.clone()), name.to_owned())
+    }
+
+    /// Create a sort expression from an existing expression.
+    ///
+    /// ```
+    /// # use datafusion::logical_plan::col;
+    /// let sort_expr = col("foo").sort(true, true); // SORT ASC NULLS_FIRST
+    /// ```
+    pub fn sort(&self, asc: bool, nulls_first: bool) -> Expr {
+        Expr::Sort {
+            expr: Box::new(self.clone()),
+            asc,
+            nulls_first,
+        }
+    }
+}
+
+fn binary_expr(l: Expr, op: Operator, r: Expr) -> Expr {
+    Expr::BinaryExpr {
+        left: Box::new(l),
+        op,
+        right: Box::new(r),
+    }
+}
+
+/// return a new expression with a logical AND
+pub fn and(left: &Expr, right: &Expr) -> Expr {
+    Expr::BinaryExpr {
+        left: Box::new(left.clone()),
+        op: Operator::And,
+        right: Box::new(right.clone()),
+    }
+}
+
+/// Create a column expression based on a column name
+pub fn col(name: &str) -> Expr {
+    Expr::Column(name.to_owned())
+}
+
+/// Create an expression to represent the min() aggregate function
+pub fn min(expr: Expr) -> Expr {
+    Expr::AggregateFunction {
+        fun: aggregates::AggregateFunction::Min,
+        distinct: false,
+        args: vec![expr],
+    }
+}
+
+/// Create an expression to represent the max() aggregate function
+pub fn max(expr: Expr) -> Expr {
+    Expr::AggregateFunction {
+        fun: aggregates::AggregateFunction::Max,
+        distinct: false,
+        args: vec![expr],
+    }
+}
+
+/// Create an expression to represent the sum() aggregate function
+pub fn sum(expr: Expr) -> Expr {
+    Expr::AggregateFunction {
+        fun: aggregates::AggregateFunction::Sum,
+        distinct: false,
+        args: vec![expr],
+    }
+}
+
+/// Create an expression to represent the avg() aggregate function
+pub fn avg(expr: Expr) -> Expr {
+    Expr::AggregateFunction {
+        fun: aggregates::AggregateFunction::Avg,
+        distinct: false,
+        args: vec![expr],
+    }
+}
+
+/// Create an expression to represent the count() aggregate function
+pub fn count(expr: Expr) -> Expr {
+    Expr::AggregateFunction {
+        fun: aggregates::AggregateFunction::Count,
+        distinct: false,
+        args: vec![expr],
+    }
+}
+
+/// Whether it can be represented as a literal expression
+pub trait Literal {
+    /// convert the value to a Literal expression
+    fn lit(&self) -> Expr;
+}
+
+impl Literal for &str {
+    fn lit(&self) -> Expr {
+        Expr::Literal(ScalarValue::Utf8(Some((*self).to_owned())))
+    }
+}
+
+impl Literal for String {
+    fn lit(&self) -> Expr {
+        Expr::Literal(ScalarValue::Utf8(Some((*self).to_owned())))
+    }
+}
+
+macro_rules! make_literal {
+    ($TYPE:ty, $SCALAR:ident) => {
+        #[allow(missing_docs)]
+        impl Literal for $TYPE {
+            fn lit(&self) -> Expr {
+                Expr::Literal(ScalarValue::$SCALAR(Some(self.clone())))
+            }
+        }
+    };
+}
+
+make_literal!(bool, Boolean);
+make_literal!(f32, Float32);
+make_literal!(f64, Float64);
+make_literal!(i8, Int8);
+make_literal!(i16, Int16);
+make_literal!(i32, Int32);
+make_literal!(i64, Int64);
+make_literal!(u8, UInt8);
+make_literal!(u16, UInt16);
+make_literal!(u32, UInt32);
+make_literal!(u64, UInt64);
+
+/// Create a literal expression
+pub fn lit<T: Literal>(n: T) -> Expr {
+    n.lit()
+}
+
+/// Create an convenience function representing a unary scalar function
+macro_rules! unary_math_expr {
+    ($ENUM:ident, $FUNC:ident) => {
+        #[allow(missing_docs)]
+        pub fn $FUNC(e: Expr) -> Expr {
+            Expr::ScalarFunction {
+                fun: functions::BuiltinScalarFunction::$ENUM,
+                args: vec![e],
+            }
+        }
+    };
+}
+
+// generate methods for creating the supported unary math expressions
+unary_math_expr!(Sqrt, sqrt);
+unary_math_expr!(Sin, sin);
+unary_math_expr!(Cos, cos);
+unary_math_expr!(Tan, tan);
+unary_math_expr!(Asin, asin);
+unary_math_expr!(Acos, acos);
+unary_math_expr!(Atan, atan);
+unary_math_expr!(Floor, floor);
+unary_math_expr!(Ceil, ceil);
+unary_math_expr!(Round, round);
+unary_math_expr!(Trunc, trunc);
+unary_math_expr!(Abs, abs);
+unary_math_expr!(Signum, signum);
+unary_math_expr!(Exp, exp);
+unary_math_expr!(Log, ln);
+unary_math_expr!(Log2, log2);
+unary_math_expr!(Log10, log10);
+
+/// returns the length of a string in bytes
+pub fn length(e: Expr) -> Expr {
+    Expr::ScalarFunction {
+        fun: functions::BuiltinScalarFunction::Length,
+        args: vec![e],
+    }
+}
+
+/// returns the concatenation of string expressions
+pub fn concat(args: Vec<Expr>) -> Expr {
+    Expr::ScalarFunction {
+        fun: functions::BuiltinScalarFunction::Concat,
+        args,
+    }
+}
+
+/// returns an array of fixed size with each argument on it.
+pub fn array(args: Vec<Expr>) -> Expr {
+    Expr::ScalarFunction {
+        fun: functions::BuiltinScalarFunction::Array,
+        args,
+    }
+}
+
+/// Creates a new UDF with a specific signature and specific return type.
+/// This is a helper function to create a new UDF.
+/// The function `create_udf` returns a subset of all possible `ScalarFunction`:
+/// * the UDF has a fixed return type
+/// * the UDF has a fixed signature (e.g. [f64, f64])
+pub fn create_udf(
+    name: &str,
+    input_types: Vec<DataType>,
+    return_type: Arc<DataType>,
+    fun: ScalarFunctionImplementation,
+) -> ScalarUDF {
+    let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(return_type.clone()));
+    ScalarUDF::new(name, &Signature::Exact(input_types), &return_type, &fun)
+}
+
+/// Creates a new UDAF with a specific signature, state type and return type.
+/// The signature and state type must match the `Acumulator's implementation`.
+pub fn create_udaf(
+    name: &str,
+    input_type: DataType,
+    return_type: Arc<DataType>,
+    accumulator: AccumulatorFunctionImplementation,
+    state_type: Arc<Vec<DataType>>,
+) -> AggregateUDF {
+    let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(return_type.clone()));
+    let state_type: StateTypeFunction = Arc::new(move |_| Ok(state_type.clone()));
+    AggregateUDF::new(
+        name,
+        &Signature::Exact(vec![input_type]),
+        &return_type,
+        &accumulator,
+        &state_type,
+    )
+}
+
+fn fmt_function(
+    f: &mut fmt::Formatter,
+    fun: &String,
+    distinct: bool,
+    args: &Vec<Expr>,
+) -> fmt::Result {
+    let args: Vec<String> = args.iter().map(|arg| format!("{:?}", arg)).collect();
+    let distinct_str = match distinct {
+        true => "DISTINCT ",
+        false => "",
+    };
+    write!(f, "{}({}{})", fun, distinct_str, args.join(", "))
+}
+
+impl fmt::Debug for Expr {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Expr::Alias(expr, alias) => write!(f, "{:?} AS {}", expr, alias),
+            Expr::Column(name) => write!(f, "#{}", name),
+            Expr::ScalarVariable(var_names) => write!(f, "{}", var_names.join(".")),
+            Expr::Literal(v) => write!(f, "{:?}", v),
+            Expr::Cast { expr, data_type } => {
+                write!(f, "CAST({:?} AS {:?})", expr, data_type)
+            }
+            Expr::Not(expr) => write!(f, "NOT {:?}", expr),
+            Expr::IsNull(expr) => write!(f, "{:?} IS NULL", expr),
+            Expr::IsNotNull(expr) => write!(f, "{:?} IS NOT NULL", expr),
+            Expr::BinaryExpr { left, op, right } => {
+                write!(f, "{:?} {:?} {:?}", left, op, right)
+            }
+            Expr::Sort {
+                expr,
+                asc,
+                nulls_first,
+            } => {
+                if *asc {
+                    write!(f, "{:?} ASC", expr)?;
+                } else {
+                    write!(f, "{:?} DESC", expr)?;
+                }
+                if *nulls_first {
+                    write!(f, " NULLS FIRST")
+                } else {
+                    write!(f, " NULLS LAST")
+                }
+            }
+            Expr::ScalarFunction { fun, args, .. } => {
+                fmt_function(f, &fun.to_string(), false, args)
+            }
+            Expr::ScalarUDF { fun, ref args, .. } => {
+                fmt_function(f, &fun.name, false, args)
+            }
+            Expr::AggregateFunction {
+                fun,
+                distinct,
+                ref args,
+                ..
+            } => fmt_function(f, &fun.to_string(), *distinct, args),
+            Expr::AggregateUDF { fun, ref args, .. } => {
+                fmt_function(f, &fun.name, false, args)
+            }
+            Expr::Wildcard => write!(f, "*"),
+            Expr::Nested(expr) => write!(f, "({:?})", expr),
+        }
+    }
+}
+
+/// This defines the interface for `LogicalPlan` nodes that can be
+/// used to extend DataFusion with custom relational operators.
+///
+/// See the example in
+/// [user_defined_plan.rs](../../tests/user_defined_plan.rs) for an
+/// example of how to use this extension API
+pub trait UserDefinedLogicalNode: Debug {
+    /// Return a reference to self as Any, to support dynamic downcasting
+    fn as_any(&self) -> &dyn Any;
+
+    /// Return the logical plan's inputs
+    fn inputs(&self) -> Vec<&LogicalPlan>;
+
+    /// Return the output schema of this logical plan node
+    fn schema(&self) -> &SchemaRef;
+
+    /// returns all expressions in the current logical plan node. This
+    /// should not include expressions of any inputs (aka
+    /// non-recursively) These expressions are used for optimizer
+    /// passes and rewrites.
+    fn expressions(&self) -> Vec<Expr>;
+
+    /// A list of output columns (e.g. the names of columns in
+    /// self.schema()) for which predicates can not be pushed below
+    /// this node without changing the output.
+    ///
+    /// By default, this returns all columns and thus prevents any
+    /// predicates from being pushed below this node.
+    fn prevent_predicate_push_down_columns(&self) -> HashSet<String> {
+        // default (safe) is all columns in the schema.
+        self.schema()
+            .fields()
+            .iter()
+            .map(|f| f.name().clone())
+            .collect()
+    }
+
+    /// Write a single line, human readable string to `f` for use in explain plan
+    ///
+    /// For example: `TopK: k=10`
+    fn fmt_for_explain(&self, f: &mut fmt::Formatter) -> fmt::Result;
+
+    /// Create a new `ExtensionPlanNode` with the specified children
+    /// and expressions. This function is used during optimization
+    /// when the plan is being rewritten and a new instance of the
+    /// `ExtensionPlanNode` must be created.
+    ///
+    /// Note that exprs and inputs are in the same order as the result
+    /// of self.inputs and self.exprs.
+    ///
+    /// So, `self.from_template(exprs, ..).expressions() == exprs
+    fn from_template(
+        &self,
+        exprs: &Vec<Expr>,
+        inputs: &Vec<LogicalPlan>,
+    ) -> Arc<dyn UserDefinedLogicalNode + Send + Sync>;
+}
+
+/// Describes the source of the table, either registered on the context or by reference
+#[derive(Clone)]
+pub enum TableSource {
+    /// The source provider is registered in the context with the corresponding name
+    FromContext(String),
+    /// The source provider is passed directly by reference
+    FromProvider(Arc<dyn TableProvider + Send + Sync>),
+}
+
+/// A LogicalPlan represents the different types of relational
+/// operators (such as Projection, Filter, etc) and can be created by
+/// the SQL query planner and the DataFrame API.
+///
+/// A LogicalPlan represents transforming an input relation (table) to
+/// an output relation (table) with a (potentially) different
+/// schema. A plan represents a dataflow tree where data flows
+/// from leaves up to the root to produce the query result.
+#[derive(Clone)]
+pub enum LogicalPlan {
+    /// Evaluates an arbitrary list of expressions (essentially a
+    /// SELECT with an expression list) on its input.
+    Projection {
+        /// The list of expressions
+        expr: Vec<Expr>,
+        /// The incoming logical plan
+        input: Arc<LogicalPlan>,
+        /// The schema description of the output
+        schema: SchemaRef,
+    },
+    /// Filters rows from its input that do not match an
+    /// expression (essentially a WHERE clause with a predicate
+    /// expression).
+    ///
+    /// Semantically, `<predicate>` is evaluated for each row of the input;
+    /// If the value of `<predicate>` is true, the input row is passed to
+    /// the output. If the value of `<predicate>` is false, the row is
+    /// discarded.
+    Filter {
+        /// The predicate expression, which must have Boolean type.
+        predicate: Expr,
+        /// The incoming logical plan
+        input: Arc<LogicalPlan>,
+    },
+    /// Aggregates its input based on a set of grouping and aggregate
+    /// expressions (e.g. SUM).
+    Aggregate {
+        /// The incoming logical plan
+        input: Arc<LogicalPlan>,
+        /// Grouping expressions
+        group_expr: Vec<Expr>,
+        /// Aggregate expressions
+        aggr_expr: Vec<Expr>,
+        /// The schema description of the aggregate output
+        schema: SchemaRef,
+    },
+    /// Sorts its input according to a list of sort expressions.
+    Sort {
+        /// The sort expressions
+        expr: Vec<Expr>,
+        /// The incoming logical plan
+        input: Arc<LogicalPlan>,
+    },
+    /// Produces rows from a table provider by reference or from the context
+    TableScan {
+        /// The name of the schema
+        schema_name: String,
+        /// The source of the table
+        source: TableSource,
+        /// The schema of the source data
+        table_schema: SchemaRef,
+        /// Optional column indices to use as a projection
+        projection: Option<Vec<usize>>,
+        /// The schema description of the output
+        projected_schema: SchemaRef,
+    },
+    /// Produces rows that come from a `Vec` of in memory `RecordBatch`es
+    InMemoryScan {
+        /// Record batch partitions
+        data: Vec<Vec<RecordBatch>>,
+        /// The schema of the record batches
+        schema: SchemaRef,
+        /// Optional column indices to use as a projection
+        projection: Option<Vec<usize>>,
+        /// The schema description of the output
+        projected_schema: SchemaRef,
+    },
+    /// Produces rows by scanning Parquet file(s)
+    ParquetScan {
+        /// The path to the files
+        path: String,
+        /// The schema of the Parquet file(s)
+        schema: SchemaRef,
+        /// Optional column indices to use as a projection
+        projection: Option<Vec<usize>>,
+        /// The schema description of the output
+        projected_schema: SchemaRef,
+    },
+    /// Produces rows by scanning a CSV file(s)
+    CsvScan {
+        /// The path to the files
+        path: String,
+        /// The underlying table schema
+        schema: SchemaRef,
+        /// Whether the CSV file(s) have a header containing column names
+        has_header: bool,
+        /// An optional column delimiter. Defaults to `b','`
+        delimiter: Option<u8>,
+        /// Optional column indices to use as a projection
+        projection: Option<Vec<usize>>,
+        /// The schema description of the output
+        projected_schema: SchemaRef,
+    },
+    /// Produces no rows: An empty relation with an empty schema
+    EmptyRelation {
+        /// The schema description of the output
+        schema: SchemaRef,
+    },
+    /// Produces the first `n` tuples from its input and discards the rest.
+    Limit {
+        /// The limit
+        n: usize,
+        /// The logical plan
+        input: Arc<LogicalPlan>,
+    },
+    /// Creates an external table.
+    CreateExternalTable {
+        /// The table schema
+        schema: SchemaRef,
+        /// The table name
+        name: String,
+        /// The physical location
+        location: String,
+        /// The file type of physical file
+        file_type: FileType,
+        /// Whether the CSV file contains a header
+        has_header: bool,
+    },
+    /// Produces a relation with string representations of
+    /// various parts of the plan
+    Explain {
+        /// Should extra (detailed, intermediate plans) be included?
+        verbose: bool,
+        /// The logical plan that is being EXPLAIN'd
+        plan: Arc<LogicalPlan>,
+        /// Represent the various stages plans have gone through
+        stringified_plans: Vec<StringifiedPlan>,
+        /// The output schema of the explain (2 columns of text)
+        schema: SchemaRef,
+    },
+    /// Extension operator defined outside of DataFusion
+    Extension {
+        /// The runtime extension operator
+        node: Arc<dyn UserDefinedLogicalNode + Send + Sync>,
+    },
+}
+
+impl LogicalPlan {
+    /// Get a reference to the logical plan's schema
+    pub fn schema(&self) -> &SchemaRef {
+        match self {
+            LogicalPlan::EmptyRelation { schema } => &schema,
+            LogicalPlan::InMemoryScan {
+                projected_schema, ..
+            } => &projected_schema,
+            LogicalPlan::CsvScan {
+                projected_schema, ..
+            } => &projected_schema,
+            LogicalPlan::ParquetScan {
+                projected_schema, ..
+            } => &projected_schema,
+            LogicalPlan::TableScan {
+                projected_schema, ..
+            } => &projected_schema,
+            LogicalPlan::Projection { schema, .. } => &schema,
+            LogicalPlan::Filter { input, .. } => input.schema(),
+            LogicalPlan::Aggregate { schema, .. } => &schema,
+            LogicalPlan::Sort { input, .. } => input.schema(),
+            LogicalPlan::Limit { input, .. } => input.schema(),
+            LogicalPlan::CreateExternalTable { schema, .. } => &schema,
+            LogicalPlan::Explain { schema, .. } => &schema,
+            LogicalPlan::Extension { node } => &node.schema(),
+        }
+    }
+
+    /// Returns the (fixed) output schema for explain plans
+    pub fn explain_schema() -> SchemaRef {
+        SchemaRef::new(Schema::new(vec![
+            Field::new("plan_type", DataType::Utf8, false),
+            Field::new("plan", DataType::Utf8, false),
+        ]))
+    }
+}
+
+impl LogicalPlan {
+    fn fmt_with_indent(&self, f: &mut fmt::Formatter, indent: usize) -> fmt::Result {
+        if indent > 0 {
+            writeln!(f)?;
+            for _ in 0..indent {
+                write!(f, "  ")?;
+            }
+        }
+        match *self {
+            LogicalPlan::EmptyRelation { .. } => write!(f, "EmptyRelation"),
+            LogicalPlan::TableScan {
+                ref source,
+                ref projection,
+                ..
+            } => match source {
+                TableSource::FromContext(table_name) => {
+                    write!(f, "TableScan: {} projection={:?}", table_name, projection)
+                }
+                TableSource::FromProvider(_) => {
+                    write!(f, "TableScan: projection={:?}", projection)
+                }
+            },
+            LogicalPlan::InMemoryScan { ref projection, .. } => {
+                write!(f, "InMemoryScan: projection={:?}", projection)
+            }
+            LogicalPlan::CsvScan {
+                ref path,
+                ref projection,
+                ..
+            } => write!(f, "CsvScan: {} projection={:?}", path, projection),
+            LogicalPlan::ParquetScan {
+                ref path,
+                ref projection,
+                ..
+            } => write!(f, "ParquetScan: {} projection={:?}", path, projection),
+            LogicalPlan::Projection {
+                ref expr,
+                ref input,
+                ..
+            } => {
+                write!(f, "Projection: ")?;
+                for i in 0..expr.len() {
+                    if i > 0 {
+                        write!(f, ", ")?;
+                    }
+                    write!(f, "{:?}", expr[i])?;
+                }
+                input.fmt_with_indent(f, indent + 1)
+            }
+            LogicalPlan::Filter {
+                predicate: ref expr,
+                ref input,
+                ..
+            } => {
+                write!(f, "Filter: {:?}", expr)?;
+                input.fmt_with_indent(f, indent + 1)
+            }
+            LogicalPlan::Aggregate {
+                ref input,
+                ref group_expr,
+                ref aggr_expr,
+                ..
+            } => {
+                write!(
+                    f,
+                    "Aggregate: groupBy=[{:?}], aggr=[{:?}]",
+                    group_expr, aggr_expr
+                )?;
+                input.fmt_with_indent(f, indent + 1)
+            }
+            LogicalPlan::Sort {
+                ref input,
+                ref expr,
+                ..
+            } => {
+                write!(f, "Sort: ")?;
+                for i in 0..expr.len() {
+                    if i > 0 {
+                        write!(f, ", ")?;
+                    }
+                    write!(f, "{:?}", expr[i])?;
+                }
+                input.fmt_with_indent(f, indent + 1)
+            }
+            LogicalPlan::Limit {
+                ref input, ref n, ..
+            } => {
+                write!(f, "Limit: {}", n)?;
+                input.fmt_with_indent(f, indent + 1)
+            }
+            LogicalPlan::CreateExternalTable { ref name, .. } => {
+                write!(f, "CreateExternalTable: {:?}", name)
+            }
+            LogicalPlan::Explain { ref plan, .. } => {
+                write!(f, "Explain")?;
+                plan.fmt_with_indent(f, indent + 1)
+            }
+            LogicalPlan::Extension { ref node } => {
+                node.fmt_for_explain(f)?;
+                node.inputs()
+                    .iter()
+                    .map(|input| input.fmt_with_indent(f, indent + 1))
+                    .collect()
+            }
+        }
+    }
+}
+
+impl fmt::Debug for LogicalPlan {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        self.fmt_with_indent(f, 0)
+    }
+}
+
+/// A registry knows how to build logical expressions out of user-defined function' names
+pub trait FunctionRegistry {
+    /// Set of all available udfs.
+    fn udfs(&self) -> HashSet<String>;
+
+    /// Returns a reference to the udf named `name`.
+    fn udf(&self, name: &str) -> Result<&ScalarUDF>;
+
+    /// Returns a reference to the udaf named `name`.
+    fn udaf(&self, name: &str) -> Result<&AggregateUDF>;
+}
+
+/// Builder for logical plans
+pub struct LogicalPlanBuilder {
+    plan: LogicalPlan,
+}
+
+impl LogicalPlanBuilder {
+    /// Create a builder from an existing plan
+    pub fn from(plan: &LogicalPlan) -> Self {
+        Self { plan: plan.clone() }
+    }
+
+    /// Create an empty relation
+    pub fn empty() -> Self {
+        Self::from(&LogicalPlan::EmptyRelation {
+            schema: SchemaRef::new(Schema::empty()),
+        })
+    }
+
+    /// Scan a CSV data source
+    pub fn scan_csv(
+        path: &str,
+        options: CsvReadOptions,
+        projection: Option<Vec<usize>>,
+    ) -> Result<Self> {
+        let has_header = options.has_header;
+        let delimiter = options.delimiter;
+        let schema: Schema = match options.schema {
+            Some(s) => s.to_owned(),
+            None => CsvFile::try_new(path, options)?
+                .schema()
+                .as_ref()
+                .to_owned(),
+        };
+
+        let projected_schema = SchemaRef::new(
+            projection
+                .clone()
+                .map(|p| {
+                    Schema::new(p.iter().map(|i| schema.field(*i).clone()).collect())
+                })
+                .or(Some(schema.clone()))
+                .unwrap(),
+        );
+
+        Ok(Self::from(&LogicalPlan::CsvScan {
+            path: path.to_owned(),
+            schema: SchemaRef::new(schema),
+            has_header,
+            delimiter: Some(delimiter),
+            projection,
+            projected_schema,
+        }))
+    }
+
+    /// Scan a Parquet data source
+    pub fn scan_parquet(path: &str, projection: Option<Vec<usize>>) -> Result<Self> {
+        let p = ParquetTable::try_new(path)?;
+        let schema = p.schema().clone();
+
+        let projected_schema = projection
+            .clone()
+            .map(|p| Schema::new(p.iter().map(|i| schema.field(*i).clone()).collect()));
+        let projected_schema =
+            projected_schema.map_or(schema.clone(), |s| SchemaRef::new(s));
+
+        Ok(Self::from(&LogicalPlan::ParquetScan {
+            path: path.to_owned(),
+            schema,
+            projection,
+            projected_schema,
+        }))
+    }
+
+    /// Scan a data source
+    pub fn scan(
+        schema_name: &str,
+        table_name: &str,
+        table_schema: &Schema,
+        projection: Option<Vec<usize>>,
+    ) -> Result<Self> {
+        let table_schema = SchemaRef::new(table_schema.clone());
+        let projected_schema = projection.clone().map(|p| {
+            Schema::new(p.iter().map(|i| table_schema.field(*i).clone()).collect())
+        });
+        let projected_schema =
+            projected_schema.map_or(table_schema.clone(), |s| SchemaRef::new(s));
+
+        Ok(Self::from(&LogicalPlan::TableScan {
+            schema_name: schema_name.to_owned(),
+            source: TableSource::FromContext(table_name.to_owned()),
+            table_schema,
+            projected_schema,
+            projection,
+        }))
+    }
+
+    /// Apply a projection.
+    ///
+    /// # Errors
+    /// This function errors under any of the following conditions:
+    /// * Two or more expressions have the same name
+    /// * An invalid expression is used (e.g. a `sort` expression)
+    pub fn project(&self, expr: Vec<Expr>) -> Result<Self> {
+        let input_schema = self.plan.schema();
+        let mut projected_expr = vec![];
+        (0..expr.len()).for_each(|i| match &expr[i] {
+            Expr::Wildcard => {
+                (0..input_schema.fields().len())
+                    .for_each(|i| projected_expr.push(col(input_schema.field(i).name())));
+            }
+            _ => projected_expr.push(expr[i].clone()),
+        });
+
+        validate_unique_names("Projections", &projected_expr, input_schema)?;
+
+        let schema = Schema::new(exprlist_to_fields(&projected_expr, input_schema)?);
+
+        Ok(Self::from(&LogicalPlan::Projection {
+            expr: projected_expr,
+            input: Arc::new(self.plan.clone()),
+            schema: SchemaRef::new(schema),
+        }))
+    }
+
+    /// Apply a filter
+    pub fn filter(&self, expr: Expr) -> Result<Self> {
+        Ok(Self::from(&LogicalPlan::Filter {
+            predicate: expr,
+            input: Arc::new(self.plan.clone()),
+        }))
+    }
+
+    /// Apply a limit
+    pub fn limit(&self, n: usize) -> Result<Self> {
+        Ok(Self::from(&LogicalPlan::Limit {
+            n,
+            input: Arc::new(self.plan.clone()),
+        }))
+    }
+
+    /// Apply a sort
+    pub fn sort(&self, expr: Vec<Expr>) -> Result<Self> {
+        Ok(Self::from(&LogicalPlan::Sort {
+            expr,
+            input: Arc::new(self.plan.clone()),
+        }))
+    }
+
+    /// Apply an aggregate
+    pub fn aggregate(&self, group_expr: Vec<Expr>, aggr_expr: Vec<Expr>) -> Result<Self> {
+        let mut all_expr: Vec<Expr> = group_expr.clone();
+        aggr_expr.iter().for_each(|x| all_expr.push(x.clone()));
+
+        validate_unique_names("Aggregations", &all_expr, self.plan.schema())?;
+
+        let aggr_schema = Schema::new(exprlist_to_fields(&all_expr, self.plan.schema())?);
+
+        Ok(Self::from(&LogicalPlan::Aggregate {
+            input: Arc::new(self.plan.clone()),
+            group_expr,
+            aggr_expr,
+            schema: SchemaRef::new(aggr_schema),
+        }))
+    }
+
+    /// Create an expression to represent the explanation of the plan
+    pub fn explain(&self, verbose: bool) -> Result<Self> {
+        let stringified_plans = vec![StringifiedPlan::new(
+            PlanType::LogicalPlan,
+            format!("{:#?}", self.plan.clone()),
+        )];
+
+        let schema = LogicalPlan::explain_schema();
+
+        Ok(Self::from(&LogicalPlan::Explain {
+            verbose,
+            plan: Arc::new(self.plan.clone()),
+            stringified_plans,
+            schema,
+        }))
+    }
+
+    /// Build the plan
+    pub fn build(&self) -> Result<LogicalPlan> {
+        Ok(self.plan.clone())
+    }
+}
+
+/// Errors if one or more expressions have equal names.
+fn validate_unique_names(
+    node_name: &str,
+    expressions: &[Expr],
+    input_schema: &Schema,
+) -> Result<()> {
+    let mut unique_names = HashMap::new();
+    expressions.iter().enumerate().map(|(position, expr)| {
+        let name = expr.name(input_schema)?;
+        match unique_names.get(&name) {
+            None => {
+                unique_names.insert(name, (position, expr));
+                Ok(())
+            },
+            Some((existing_position, existing_expr)) => {
+                Err(DataFusionError::Plan(
+                    format!("{} require unique expression names \
+                             but the expression \"{:?}\" at position {} and \"{:?}\" \
+                             at position {} have the same name. Consider aliasing (\"AS\") one of them.",
+                             node_name, existing_expr, existing_position, expr, position,
+                            )
+                ))
+            }
+        }
+    }).collect::<Result<()>>()
+}
+
+/// Represents which type of plan
+#[derive(Debug, Clone, PartialEq)]
+pub enum PlanType {
+    /// The initial LogicalPlan provided to DataFusion
+    LogicalPlan,
+    /// The LogicalPlan which results from applying an optimizer pass
+    OptimizedLogicalPlan {
+        /// The name of the optimizer which produced this plan
+        optimizer_name: String,
+    },
+    /// The physical plan, prepared for execution
+    PhysicalPlan,
+}
+
+impl From<&PlanType> for String {
+    fn from(t: &PlanType) -> Self {
+        match t {
+            PlanType::LogicalPlan => "logical_plan".into(),
+            PlanType::OptimizedLogicalPlan { optimizer_name } => {
+                format!("logical_plan after {}", optimizer_name)
+            }
+            PlanType::PhysicalPlan => "physical_plan".into(),
+        }
+    }
+}
+
+/// Represents some sort of execution plan, in String form
+#[derive(Debug, Clone, PartialEq)]
+pub struct StringifiedPlan {
+    /// An identifier of what type of plan this string represents
+    pub plan_type: PlanType,
+    /// The string representation of the plan
+    pub plan: Arc<String>,
+}
+
+impl StringifiedPlan {
+    /// Create a new Stringified plan of `plan_type` with string
+    /// representation `plan`
+    pub fn new(plan_type: PlanType, plan: impl Into<String>) -> Self {
+        StringifiedPlan {
+            plan_type,
+            plan: Arc::new(plan.into()),
+        }
+    }
+
+    /// returns true if this plan should be displayed. Generally
+    /// `verbose_mode = true` will display all available plans
+    pub fn should_display(&self, verbose_mode: bool) -> bool {
+        self.plan_type == PlanType::LogicalPlan || verbose_mode
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn plan_builder_simple() -> Result<()> {
+        let plan = LogicalPlanBuilder::scan(
+            "default",
+            "employee.csv",
+            &employee_schema(),
+            Some(vec![0, 3]),
+        )?
+        .filter(col("state").eq(lit("CO")))?
+        .project(vec![col("id")])?
+        .build()?;
+
+        let expected = "Projection: #id\
+        \n  Filter: #state Eq Utf8(\"CO\")\
+        \n    TableScan: employee.csv projection=Some([0, 3])";
+
+        assert_eq!(expected, format!("{:?}", plan));
+
+        Ok(())
+    }
+
+    #[test]
+    fn plan_builder_csv() -> Result<()> {
+        let plan = LogicalPlanBuilder::scan_csv(
+            "employee.csv",
+            CsvReadOptions::new().schema(&employee_schema()),
+            Some(vec![0, 3]),
+        )?
+        .filter(col("state").eq(lit("CO")))?
+        .project(vec![col("id")])?
+        .build()?;
+
+        let expected = "Projection: #id\
+        \n  Filter: #state Eq Utf8(\"CO\")\
+        \n    CsvScan: employee.csv projection=Some([0, 3])";
+
+        assert_eq!(expected, format!("{:?}", plan));
+
+        Ok(())
+    }
+
+    #[test]
+    fn plan_builder_aggregate() -> Result<()> {
+        let plan = LogicalPlanBuilder::scan(
+            "default",
+            "employee.csv",
+            &employee_schema(),
+            Some(vec![3, 4]),
+        )?
+        .aggregate(
+            vec![col("state")],
+            vec![sum(col("salary")).alias("total_salary")],
+        )?
+        .project(vec![col("state"), col("total_salary")])?
+        .build()?;
+
+        let expected = "Projection: #state, #total_salary\
+        \n  Aggregate: groupBy=[[#state]], aggr=[[SUM(#salary) AS total_salary]]\
+        \n    TableScan: employee.csv projection=Some([3, 4])";
+
+        assert_eq!(expected, format!("{:?}", plan));
+
+        Ok(())
+    }
+
+    #[test]
+    fn plan_builder_sort() -> Result<()> {
+        let plan = LogicalPlanBuilder::scan(
+            "default",
+            "employee.csv",
+            &employee_schema(),
+            Some(vec![3, 4]),
+        )?
+        .sort(vec![
+            Expr::Sort {
+                expr: Box::new(col("state")),
+                asc: true,
+                nulls_first: true,
+            },
+            Expr::Sort {
+                expr: Box::new(col("total_salary")),
+                asc: false,
+                nulls_first: false,
+            },
+        ])?
+        .build()?;
+
+        let expected = "Sort: #state ASC NULLS FIRST, #total_salary DESC NULLS LAST\
+        \n  TableScan: employee.csv projection=Some([3, 4])";
+
+        assert_eq!(expected, format!("{:?}", plan));
+
+        Ok(())
+    }
+
+    #[test]
+    fn projection_non_unique_names() -> Result<()> {
+        let plan = LogicalPlanBuilder::scan(
+            "default",
+            "employee.csv",
+            &employee_schema(),
+            Some(vec![0, 3]),
+        )?
+        // two columns with the same name => error
+        .project(vec![col("id"), col("first_name").alias("id")]);
+
+        match plan {
+            Err(DataFusionError::Plan(e)) => {
+                assert_eq!(e, "Projections require unique expression names \
+                    but the expression \"#id\" at position 0 and \"#first_name AS id\" at \
+                    position 1 have the same name. Consider aliasing (\"AS\") one of them.");
+                Ok(())
+            }
+            _ => Err(DataFusionError::Plan(
+                "Plan should have returned an DataFusionError::Plan".to_string(),
+            )),
+        }
+    }
+
+    #[test]
+    fn aggregate_non_unique_names() -> Result<()> {
+        let plan = LogicalPlanBuilder::scan(
+            "default",
+            "employee.csv",
+            &employee_schema(),
+            Some(vec![0, 3]),
+        )?
+        // two columns with the same name => error
+        .aggregate(vec![col("state")], vec![sum(col("salary")).alias("state")]);
+
+        match plan {
+            Err(DataFusionError::Plan(e)) => {
+                assert_eq!(e, "Aggregations require unique expression names \
+                    but the expression \"#state\" at position 0 and \"SUM(#salary) AS state\" at \
+                    position 1 have the same name. Consider aliasing (\"AS\") one of them.");
+                Ok(())
+            }
+            _ => Err(DataFusionError::Plan(
+                "Plan should have returned an DataFusionError::Plan".to_string(),
+            )),
+        }
+    }
+
+    fn employee_schema() -> Schema {
+        Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("first_name", DataType::Utf8, false),
+            Field::new("last_name", DataType::Utf8, false),
+            Field::new("state", DataType::Utf8, false),
+            Field::new("salary", DataType::Int32, false),
+        ])
+    }
+
+    #[test]
+    fn stringified_plan() -> Result<()> {
+        let stringified_plan =
+            StringifiedPlan::new(PlanType::LogicalPlan, "...the plan...");
+        assert!(stringified_plan.should_display(true));
+        assert!(stringified_plan.should_display(false)); // display in non verbose mode too
+
+        let stringified_plan =
+            StringifiedPlan::new(PlanType::PhysicalPlan, "...the plan...");
+        assert!(stringified_plan.should_display(true));
+        assert!(!stringified_plan.should_display(false));
+
+        let stringified_plan = StringifiedPlan::new(
+            PlanType::OptimizedLogicalPlan {
+                optimizer_name: "random opt pass".into(),
+            },
+            "...the plan...",
+        );
+        assert!(stringified_plan.should_display(true));
+        assert!(!stringified_plan.should_display(false));
+
+        Ok(())
+    }
+}
diff --git a/rust/datafusion/src/logical_plan/operators.rs b/rust/datafusion/src/logical_plan/operators.rs
new file mode 100644
index 00000000000..f002d013140
--- /dev/null
+++ b/rust/datafusion/src/logical_plan/operators.rs
@@ -0,0 +1,138 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::{fmt, ops};
+
+use super::{binary_expr, Expr};
+
+/// Operators applied to expressions
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Operator {
+    /// Expressions are equal
+    Eq,
+    /// Expressions are not equal
+    NotEq,
+    /// Left side is smaller than right side
+    Lt,
+    /// Left side is smaller or equal to right side
+    LtEq,
+    /// Left side is greater than right side
+    Gt,
+    /// Left side is greater or equal to right side
+    GtEq,
+    /// Addition
+    Plus,
+    /// Subtraction
+    Minus,
+    /// Multiplication operator, like `*`
+    Multiply,
+    /// Division operator, like `/`
+    Divide,
+    /// Remainder operator, like `%`
+    Modulus,
+    /// Logical AND, like `&&`
+    And,
+    /// Logical OR, like `||`
+    Or,
+    /// Matches a wildcard pattern
+    Like,
+    /// Does not match a wildcard pattern
+    NotLike,
+}
+
+impl fmt::Display for Operator {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let display = match &self {
+            Operator::Eq => "=",
+            Operator::NotEq => "!=",
+            Operator::Lt => "<",
+            Operator::LtEq => "<=",
+            Operator::Gt => ">",
+            Operator::GtEq => ">=",
+            Operator::Plus => "+",
+            Operator::Minus => "-",
+            Operator::Multiply => "*",
+            Operator::Divide => "/",
+            Operator::Modulus => "%",
+            Operator::And => "AND",
+            Operator::Or => "OR",
+            Operator::Like => "LIKE",
+            Operator::NotLike => "NOT LIKE",
+        };
+        write!(f, "{}", display)
+    }
+}
+
+impl ops::Add for Expr {
+    type Output = Self;
+
+    fn add(self, rhs: Self) -> Self {
+        binary_expr(self.clone(), Operator::Plus, rhs.clone())
+    }
+}
+
+impl ops::Sub for Expr {
+    type Output = Self;
+
+    fn sub(self, rhs: Self) -> Self {
+        binary_expr(self.clone(), Operator::Minus, rhs.clone())
+    }
+}
+
+impl ops::Mul for Expr {
+    type Output = Self;
+
+    fn mul(self, rhs: Self) -> Self {
+        binary_expr(self.clone(), Operator::Multiply, rhs.clone())
+    }
+}
+
+impl ops::Div for Expr {
+    type Output = Self;
+
+    fn div(self, rhs: Self) -> Self {
+        binary_expr(self.clone(), Operator::Divide, rhs.clone())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::error::Result;
+    use crate::prelude::lit;
+
+    #[test]
+    fn test_operators() -> Result<()> {
+        assert_eq!(
+            format!("{:?}", lit(1u32) + lit(2u32)),
+            "UInt32(1) Plus UInt32(2)"
+        );
+        assert_eq!(
+            format!("{:?}", lit(1u32) - lit(2u32)),
+            "UInt32(1) Minus UInt32(2)"
+        );
+        assert_eq!(
+            format!("{:?}", lit(1u32) * lit(2u32)),
+            "UInt32(1) Multiply UInt32(2)"
+        );
+        assert_eq!(
+            format!("{:?}", lit(1u32) / lit(2u32)),
+            "UInt32(1) Divide UInt32(2)"
+        );
+
+        Ok(())
+    }
+}
diff --git a/rust/datafusion/src/logicalplan.rs b/rust/datafusion/src/logicalplan.rs
deleted file mode 100644
index d13e51b4934..00000000000
--- a/rust/datafusion/src/logicalplan.rs
+++ /dev/null
@@ -1,1240 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! This module provides a logical query plan enum that can describe queries. Logical query
-//! plans can be created from a SQL statement or built programmatically via the Table API.
-//!
-//! Logical query plans can then be optimized and executed directly, or translated into
-//! physical query plans and executed.
-
-use std::fmt;
-
-use arrow::datatypes::{DataType, Field, Schema};
-
-use crate::datasource::csv::{CsvFile, CsvReadOptions};
-use crate::datasource::parquet::ParquetTable;
-use crate::datasource::TableProvider;
-use crate::error::{ExecutionError, Result};
-use crate::optimizer::utils;
-use crate::sql::parser::FileType;
-use arrow::record_batch::RecordBatch;
-
-/// Enumeration of supported function types (Scalar and Aggregate)
-#[derive(Debug, Clone)]
-pub enum FunctionType {
-    /// Simple function returning a value per DataFrame
-    Scalar,
-    /// Aggregate functions produce a value by sampling multiple DataFrames
-    Aggregate,
-}
-
-/// Logical representation of a UDF (user-defined function)
-#[derive(Debug, Clone)]
-pub struct FunctionMeta {
-    /// Function name
-    name: String,
-    /// Function arguments
-    args: Vec<Field>,
-    /// Function return type
-    return_type: DataType,
-    /// Function type (Scalar or Aggregate)
-    function_type: FunctionType,
-}
-
-impl FunctionMeta {
-    #[allow(missing_docs)]
-    pub fn new(
-        name: String,
-        args: Vec<Field>,
-        return_type: DataType,
-        function_type: FunctionType,
-    ) -> Self {
-        FunctionMeta {
-            name,
-            args,
-            return_type,
-            function_type,
-        }
-    }
-    /// Getter for the function name
-    pub fn name(&self) -> &String {
-        &self.name
-    }
-    /// Getter for the arg list
-    pub fn args(&self) -> &Vec<Field> {
-        &self.args
-    }
-    /// Getter for the `DataType` the function returns
-    pub fn return_type(&self) -> &DataType {
-        &self.return_type
-    }
-    /// Getter for the `FunctionType`
-    pub fn function_type(&self) -> &FunctionType {
-        &self.function_type
-    }
-}
-
-/// Operators applied to expressions
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum Operator {
-    /// Expressions are equal
-    Eq,
-    /// Expressions are not equal
-    NotEq,
-    /// Left side is smaller than right side
-    Lt,
-    /// Left side is smaller or equal to right side
-    LtEq,
-    /// Left side is greater than right side
-    Gt,
-    /// Left side is greater or equal to right side
-    GtEq,
-    /// Addition
-    Plus,
-    /// Subtraction
-    Minus,
-    /// Multiplication operator, like `*`
-    Multiply,
-    /// Division operator, like `/`
-    Divide,
-    /// Remainder operator, like `%`
-    Modulus,
-    /// Logical AND, like `&&`
-    And,
-    /// Logical OR, like `||`
-    Or,
-    /// Logical NOT, like `!`
-    Not,
-    /// Matches a wildcard pattern
-    Like,
-    /// Does not match a wildcard pattern
-    NotLike,
-}
-
-impl fmt::Display for Operator {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let display = match &self {
-            Operator::Eq => "=",
-            Operator::NotEq => "!=",
-            Operator::Lt => "<",
-            Operator::LtEq => "<=",
-            Operator::Gt => ">",
-            Operator::GtEq => ">=",
-            Operator::Plus => "+",
-            Operator::Minus => "-",
-            Operator::Multiply => "*",
-            Operator::Divide => "/",
-            Operator::Modulus => "%",
-            Operator::And => "AND",
-            Operator::Or => "OR",
-            Operator::Not => "NOT",
-            Operator::Like => "LIKE",
-            Operator::NotLike => "NOT LIKE",
-        };
-        write!(f, "{}", display)
-    }
-}
-
-/// ScalarValue enumeration
-#[derive(Debug, Clone, PartialEq)]
-pub enum ScalarValue {
-    /// null value
-    Null,
-    /// true or false value
-    Boolean(bool),
-    /// 32bit float
-    Float32(f32),
-    /// 64bit float
-    Float64(f64),
-    /// signed 8bit int
-    Int8(i8),
-    /// signed 16bit int
-    Int16(i16),
-    /// signed 32bit int
-    Int32(i32),
-    /// signed 64bit int
-    Int64(i64),
-    /// unsigned 8bit int
-    UInt8(u8),
-    /// unsigned 16bit int
-    UInt16(u16),
-    /// unsigned 32bit int
-    UInt32(u32),
-    /// unsigned 64bit int
-    UInt64(u64),
-    /// utf-8 encoded string
-    Utf8(String),
-    /// List of scalars packed as a struct
-    Struct(Vec<ScalarValue>),
-}
-
-impl ScalarValue {
-    /// Getter for the `DataType` of the value
-    pub fn get_datatype(&self) -> Result<DataType> {
-        match *self {
-            ScalarValue::Boolean(_) => Ok(DataType::Boolean),
-            ScalarValue::UInt8(_) => Ok(DataType::UInt8),
-            ScalarValue::UInt16(_) => Ok(DataType::UInt16),
-            ScalarValue::UInt32(_) => Ok(DataType::UInt32),
-            ScalarValue::UInt64(_) => Ok(DataType::UInt64),
-            ScalarValue::Int8(_) => Ok(DataType::Int8),
-            ScalarValue::Int16(_) => Ok(DataType::Int16),
-            ScalarValue::Int32(_) => Ok(DataType::Int32),
-            ScalarValue::Int64(_) => Ok(DataType::Int64),
-            ScalarValue::Float32(_) => Ok(DataType::Float32),
-            ScalarValue::Float64(_) => Ok(DataType::Float64),
-            ScalarValue::Utf8(_) => Ok(DataType::Utf8),
-            _ => Err(ExecutionError::General(format!(
-                "Cannot treat {:?} as scalar value",
-                self
-            ))),
-        }
-    }
-}
-
-impl fmt::Display for ScalarValue {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            ScalarValue::Boolean(value) => write!(f, "{}", value),
-            ScalarValue::UInt8(value) => write!(f, "{}", value),
-            ScalarValue::UInt16(value) => write!(f, "{}", value),
-            ScalarValue::UInt32(value) => write!(f, "{}", value),
-            ScalarValue::UInt64(value) => write!(f, "{}", value),
-            ScalarValue::Int8(value) => write!(f, "{}", value),
-            ScalarValue::Int16(value) => write!(f, "{}", value),
-            ScalarValue::Int32(value) => write!(f, "{}", value),
-            ScalarValue::Int64(value) => write!(f, "{}", value),
-            ScalarValue::Float32(value) => write!(f, "{}", value),
-            ScalarValue::Float64(value) => write!(f, "{}", value),
-            ScalarValue::Utf8(value) => write!(f, "{}", value),
-            ScalarValue::Null => write!(f, "NULL"),
-            ScalarValue::Struct(_) => write!(f, "STRUCT"),
-        }
-    }
-}
-
-/// Returns a readable name of an expression based on the input schema.
-/// This function recursively transverses the expression for names such as "CAST(a > 2)".
-fn create_name(e: &Expr, input_schema: &Schema) -> Result<String> {
-    match e {
-        Expr::Alias(_, name) => Ok(name.clone()),
-        Expr::Column(name) => Ok(name.clone()),
-        Expr::Literal(value) => Ok(format!("{:?}", value)),
-        Expr::BinaryExpr { left, op, right } => {
-            let left = create_name(left, input_schema)?;
-            let right = create_name(right, input_schema)?;
-            Ok(format!("{} {:?} {}", left, op, right))
-        }
-        Expr::Cast { expr, data_type } => {
-            let expr = create_name(expr, input_schema)?;
-            Ok(format!("CAST({} as {:?})", expr, data_type))
-        }
-        Expr::ScalarFunction { name, args, .. } => {
-            let mut names = Vec::with_capacity(args.len());
-            for e in args {
-                names.push(create_name(e, input_schema)?);
-            }
-            Ok(format!("{}({})", name, names.join(",")))
-        }
-        Expr::AggregateFunction { name, args, .. } => {
-            let mut names = Vec::with_capacity(args.len());
-            for e in args {
-                names.push(create_name(e, input_schema)?);
-            }
-            Ok(format!("{}({})", name, names.join(",")))
-        }
-        other => Err(ExecutionError::NotImplemented(format!(
-            "Physical plan does not support logical expression {:?}",
-            other
-        ))),
-    }
-}
-
-/// Returns the datatype of the expression given the input schema
-// note: the physical plan derived from an expression must match the datatype on this function.
-pub fn expr_to_field(e: &Expr, input_schema: &Schema) -> Result<Field> {
-    let data_type = match e {
-        Expr::Alias(expr, ..) => expr.get_type(input_schema),
-        Expr::Column(name) => Ok(input_schema.field_with_name(name)?.data_type().clone()),
-        Expr::Literal(ref lit) => lit.get_datatype(),
-        Expr::ScalarFunction {
-            ref return_type, ..
-        } => Ok(return_type.clone()),
-        Expr::AggregateFunction {
-            ref return_type, ..
-        } => Ok(return_type.clone()),
-        Expr::Cast { ref data_type, .. } => Ok(data_type.clone()),
-        Expr::BinaryExpr {
-            ref left,
-            ref right,
-            ..
-        } => {
-            let left_type = left.get_type(input_schema)?;
-            let right_type = right.get_type(input_schema)?;
-            Ok(utils::get_supertype(&left_type, &right_type).unwrap())
-        }
-        _ => Err(ExecutionError::NotImplemented(format!(
-            "Cannot determine schema type for expression {:?}",
-            e
-        ))),
-    };
-
-    match data_type {
-        Ok(d) => Ok(Field::new(&e.name(input_schema)?, d, true)),
-        Err(e) => Err(e),
-    }
-}
-
-/// Create field meta-data from an expression, for use in a result set schema
-pub fn exprlist_to_fields(expr: &[Expr], input_schema: &Schema) -> Result<Vec<Field>> {
-    expr.iter()
-        .map(|e| expr_to_field(e, input_schema))
-        .collect()
-}
-
-/// Relation expression
-#[derive(Clone, PartialEq)]
-pub enum Expr {
-    /// An aliased expression
-    Alias(Box<Expr>, String),
-    /// column of a table scan
-    Column(String),
-    /// literal value
-    Literal(ScalarValue),
-    /// binary expression e.g. "age > 21"
-    BinaryExpr {
-        /// Left-hand side of the expression
-        left: Box<Expr>,
-        /// The comparison operator
-        op: Operator,
-        /// Right-hand side of the expression
-        right: Box<Expr>,
-    },
-    /// Nested expression e.g. `(foo > bar)` or `(1)`
-    Nested(Box<Expr>),
-    /// unary NOT
-    Not(Box<Expr>),
-    /// unary IS NOT NULL
-    IsNotNull(Box<Expr>),
-    /// unary IS NULL
-    IsNull(Box<Expr>),
-    /// cast a value to a different type
-    Cast {
-        /// The expression being cast
-        expr: Box<Expr>,
-        /// The `DataType` the expression will yield
-        data_type: DataType,
-    },
-    /// sort expression
-    Sort {
-        /// The expression to sort on
-        expr: Box<Expr>,
-        /// The direction of the sort
-        asc: bool,
-        /// Whether to put Nulls before all other data values
-        nulls_first: bool,
-    },
-    /// scalar function
-    ScalarFunction {
-        /// Name of the function
-        name: String,
-        /// List of expressions to feed to the functions as arguments
-        args: Vec<Expr>,
-        /// The `DataType` the expression will yield
-        return_type: DataType,
-    },
-    /// aggregate function
-    AggregateFunction {
-        /// Name of the function
-        name: String,
-        /// List of expressions to feed to the functions as arguments
-        args: Vec<Expr>,
-        /// The `DataType` the expression will yield
-        return_type: DataType,
-    },
-    /// Wildcard
-    Wildcard,
-}
-
-impl Expr {
-    /// Find the `DataType` for the expression
-    pub fn get_type(&self, schema: &Schema) -> Result<DataType> {
-        match self {
-            Expr::Alias(expr, _) => expr.get_type(schema),
-            Expr::Column(name) => Ok(schema.field_with_name(name)?.data_type().clone()),
-            Expr::Literal(l) => l.get_datatype(),
-            Expr::Cast { data_type, .. } => Ok(data_type.clone()),
-            Expr::ScalarFunction { return_type, .. } => Ok(return_type.clone()),
-            Expr::AggregateFunction { return_type, .. } => Ok(return_type.clone()),
-            Expr::Not(_) => Ok(DataType::Boolean),
-            Expr::IsNull(_) => Ok(DataType::Boolean),
-            Expr::IsNotNull(_) => Ok(DataType::Boolean),
-            Expr::BinaryExpr {
-                ref left,
-                ref right,
-                ref op,
-            } => match op {
-                Operator::Not => Ok(DataType::Boolean),
-                Operator::Like | Operator::NotLike => Ok(DataType::Boolean),
-                Operator::Eq | Operator::NotEq => Ok(DataType::Boolean),
-                Operator::Lt | Operator::LtEq => Ok(DataType::Boolean),
-                Operator::Gt | Operator::GtEq => Ok(DataType::Boolean),
-                Operator::And | Operator::Or => Ok(DataType::Boolean),
-                _ => {
-                    let left_type = left.get_type(schema)?;
-                    let right_type = right.get_type(schema)?;
-                    utils::get_supertype(&left_type, &right_type)
-                }
-            },
-            Expr::Sort { ref expr, .. } => expr.get_type(schema),
-            Expr::Wildcard => Err(ExecutionError::General(
-                "Wildcard expressions are not valid in a logical query plan".to_owned(),
-            )),
-            Expr::Nested(e) => e.get_type(schema),
-        }
-    }
-
-    /// Return the name of this expression
-    ///
-    /// This represents how a column with this expression is named when no alias is chosen
-    pub fn name(&self, input_schema: &Schema) -> Result<String> {
-        create_name(self, input_schema)
-    }
-
-    /// Perform a type cast on the expression value.
-    ///
-    /// Will `Err` if the type cast cannot be performed.
-    pub fn cast_to(&self, cast_to_type: &DataType, schema: &Schema) -> Result<Expr> {
-        let this_type = self.get_type(schema)?;
-        if this_type == *cast_to_type {
-            Ok(self.clone())
-        } else if can_coerce_from(cast_to_type, &this_type) {
-            Ok(Expr::Cast {
-                expr: Box::new(self.clone()),
-                data_type: cast_to_type.clone(),
-            })
-        } else {
-            Err(ExecutionError::General(format!(
-                "Cannot automatically convert {:?} to {:?}",
-                this_type, cast_to_type
-            )))
-        }
-    }
-
-    /// Equal
-    pub fn eq(&self, other: &Expr) -> Expr {
-        Expr::BinaryExpr {
-            left: Box::new(self.clone()),
-            op: Operator::Eq,
-            right: Box::new(other.clone()),
-        }
-    }
-
-    /// Not equal
-    pub fn not_eq(&self, other: &Expr) -> Expr {
-        Expr::BinaryExpr {
-            left: Box::new(self.clone()),
-            op: Operator::NotEq,
-            right: Box::new(other.clone()),
-        }
-    }
-
-    /// Greater than
-    pub fn gt(&self, other: &Expr) -> Expr {
-        Expr::BinaryExpr {
-            left: Box::new(self.clone()),
-            op: Operator::Gt,
-            right: Box::new(other.clone()),
-        }
-    }
-
-    /// Greater than or equal to
-    pub fn gt_eq(&self, other: &Expr) -> Expr {
-        Expr::BinaryExpr {
-            left: Box::new(self.clone()),
-            op: Operator::GtEq,
-            right: Box::new(other.clone()),
-        }
-    }
-
-    /// Less than
-    pub fn lt(&self, other: &Expr) -> Expr {
-        Expr::BinaryExpr {
-            left: Box::new(self.clone()),
-            op: Operator::Lt,
-            right: Box::new(other.clone()),
-        }
-    }
-
-    /// Less than or equal to
-    pub fn lt_eq(&self, other: &Expr) -> Expr {
-        Expr::BinaryExpr {
-            left: Box::new(self.clone()),
-            op: Operator::LtEq,
-            right: Box::new(other.clone()),
-        }
-    }
-
-    /// Not
-    pub fn not(&self) -> Expr {
-        Expr::Not(Box::new(self.clone()))
-    }
-
-    /// Alias
-    pub fn alias(&self, name: &str) -> Expr {
-        Expr::Alias(Box::new(self.clone()), name.to_owned())
-    }
-}
-
-/// Create a column expression based on a column name
-pub fn col(name: &str) -> Expr {
-    Expr::Column(name.to_owned())
-}
-
-/// Whether it can be represented as a literal expression
-pub trait Literal {
-    /// convert the value to a Literal expression
-    fn lit(&self) -> Expr;
-}
-
-impl Literal for &str {
-    fn lit(&self) -> Expr {
-        Expr::Literal(ScalarValue::Utf8((*self).to_owned()))
-    }
-}
-
-impl Literal for String {
-    fn lit(&self) -> Expr {
-        Expr::Literal(ScalarValue::Utf8((*self).to_owned()))
-    }
-}
-
-macro_rules! make_literal {
-    ($TYPE:ty, $SCALAR:ident) => {
-        #[allow(missing_docs)]
-        impl Literal for $TYPE {
-            fn lit(&self) -> Expr {
-                Expr::Literal(ScalarValue::$SCALAR(self.clone()))
-            }
-        }
-    };
-}
-
-make_literal!(bool, Boolean);
-make_literal!(f32, Float32);
-make_literal!(f64, Float64);
-make_literal!(i8, Int8);
-make_literal!(i16, Int16);
-make_literal!(i32, Int32);
-make_literal!(i64, Int64);
-make_literal!(u8, UInt8);
-make_literal!(u16, UInt16);
-make_literal!(u32, UInt32);
-make_literal!(u64, UInt64);
-
-/// Create a literal expression
-pub fn lit<T: Literal>(n: T) -> Expr {
-    n.lit()
-}
-
-/// Create an convenience function representing a unary scalar function
-macro_rules! unary_math_expr {
-    ($NAME:expr, $FUNC:ident) => {
-        #[allow(missing_docs)]
-        pub fn $FUNC(e: Expr) -> Expr {
-            scalar_function($NAME, vec![e], DataType::Float64)
-        }
-    };
-}
-
-// generate methods for creating the supported unary math expressions
-unary_math_expr!("sqrt", sqrt);
-unary_math_expr!("sin", sin);
-unary_math_expr!("cos", cos);
-unary_math_expr!("tan", tan);
-unary_math_expr!("asin", asin);
-unary_math_expr!("acos", acos);
-unary_math_expr!("atan", atan);
-unary_math_expr!("floor", floor);
-unary_math_expr!("ceil", ceil);
-unary_math_expr!("round", round);
-unary_math_expr!("trunc", trunc);
-unary_math_expr!("abs", abs);
-unary_math_expr!("signum", signum);
-unary_math_expr!("exp", exp);
-unary_math_expr!("log", ln);
-unary_math_expr!("log2", log2);
-unary_math_expr!("log10", log10);
-
-/// Create an aggregate expression
-pub fn aggregate_expr(name: &str, expr: Expr, return_type: DataType) -> Expr {
-    Expr::AggregateFunction {
-        name: name.to_owned(),
-        args: vec![expr],
-        return_type,
-    }
-}
-
-/// Create an aggregate expression
-pub fn scalar_function(name: &str, expr: Vec<Expr>, return_type: DataType) -> Expr {
-    Expr::ScalarFunction {
-        name: name.to_owned(),
-        args: expr,
-        return_type,
-    }
-}
-
-impl fmt::Debug for Expr {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            Expr::Alias(expr, alias) => write!(f, "{:?} AS {}", expr, alias),
-            Expr::Column(name) => write!(f, "#{}", name),
-            Expr::Literal(v) => write!(f, "{:?}", v),
-            Expr::Cast { expr, data_type } => {
-                write!(f, "CAST({:?} AS {:?})", expr, data_type)
-            }
-            Expr::Not(expr) => write!(f, "NOT {:?}", expr),
-            Expr::IsNull(expr) => write!(f, "{:?} IS NULL", expr),
-            Expr::IsNotNull(expr) => write!(f, "{:?} IS NOT NULL", expr),
-            Expr::BinaryExpr { left, op, right } => {
-                write!(f, "{:?} {:?} {:?}", left, op, right)
-            }
-            Expr::Sort {
-                expr,
-                asc,
-                nulls_first,
-            } => {
-                if *asc {
-                    write!(f, "{:?} ASC", expr)?;
-                } else {
-                    write!(f, "{:?} DESC", expr)?;
-                }
-                if *nulls_first {
-                    write!(f, " NULLS FIRST")
-                } else {
-                    write!(f, " NULLS LAST")
-                }
-            }
-            Expr::ScalarFunction { name, ref args, .. } => {
-                write!(f, "{}(", name)?;
-                for i in 0..args.len() {
-                    if i > 0 {
-                        write!(f, ", ")?;
-                    }
-                    write!(f, "{:?}", args[i])?;
-                }
-
-                write!(f, ")")
-            }
-            Expr::AggregateFunction { name, ref args, .. } => {
-                write!(f, "{}(", name)?;
-                for i in 0..args.len() {
-                    if i > 0 {
-                        write!(f, ", ")?;
-                    }
-                    write!(f, "{:?}", args[i])?;
-                }
-
-                write!(f, ")")
-            }
-            Expr::Wildcard => write!(f, "*"),
-            Expr::Nested(expr) => write!(f, "({:?})", expr),
-        }
-    }
-}
-
-/// The LogicalPlan represents different types of relations (such as Projection,
-/// Selection, etc) and can be created by the SQL query planner and the DataFrame API.
-#[derive(Clone)]
-pub enum LogicalPlan {
-    /// Evaluates an arbitrary list of expressions (essentially a
-    /// SELECT with an expression list) on its input.
-    Projection {
-        /// The list of expressions
-        expr: Vec<Expr>,
-        /// The incoming logical plan
-        input: Box<LogicalPlan>,
-        /// The schema description of the output
-        schema: Box<Schema>,
-    },
-    /// Filters rows from its input that do not match an
-    /// expression (essentially a WHERE clause with a predicate
-    /// expression).
-    ///
-    /// Semantically, `<expr>` is evaluated for each row of the input;
-    /// If the value of `<expr>` is true, the input row is passed to
-    /// the output. If the value of `<expr>` is false, the row is
-    /// discarded.
-    Selection {
-        /// The expression. Must have Boolean type.
-        expr: Expr,
-        /// The incoming logical plan
-        input: Box<LogicalPlan>,
-    },
-    /// Aggregates its input based on a set of grouping and aggregate
-    /// expressions (e.g. SUM).
-    Aggregate {
-        /// The incoming logical plan
-        input: Box<LogicalPlan>,
-        /// Grouping expressions
-        group_expr: Vec<Expr>,
-        /// Aggregate expressions
-        aggr_expr: Vec<Expr>,
-        /// The schema description of the aggregate output
-        schema: Box<Schema>,
-    },
-    /// Sorts its input according to a list of sort expressions.
-    Sort {
-        /// The sort expressions
-        expr: Vec<Expr>,
-        /// The incoming logical plan
-        input: Box<LogicalPlan>,
-        /// The schema description of the otuput
-        schema: Box<Schema>,
-    },
-    /// Produces rows from a table that has been registered on a
-    /// context
-    TableScan {
-        /// The name of the schema
-        schema_name: String,
-        /// The name of the table
-        table_name: String,
-        /// The schema of the CSV file(s)
-        table_schema: Box<Schema>,
-        /// Optional column indices to use as a projection
-        projection: Option<Vec<usize>>,
-        /// The schema description of the output
-        projected_schema: Box<Schema>,
-    },
-    /// Produces rows that come from a `Vec` of in memory `RecordBatch`es
-    InMemoryScan {
-        /// Record batch partitions
-        data: Vec<Vec<RecordBatch>>,
-        /// The schema of the record batches
-        schema: Box<Schema>,
-        /// Optional column indices to use as a projection
-        projection: Option<Vec<usize>>,
-        /// The schema description of the output
-        projected_schema: Box<Schema>,
-    },
-    /// Produces rows by scanning Parquet file(s)
-    ParquetScan {
-        /// The path to the files
-        path: String,
-        /// The schema of the Parquet file(s)
-        schema: Box<Schema>,
-        /// Optional column indices to use as a projection
-        projection: Option<Vec<usize>>,
-        /// The schema description of the output
-        projected_schema: Box<Schema>,
-    },
-    /// Produces rows by scanning a CSV file(s)
-    CsvScan {
-        /// The path to the files
-        path: String,
-        /// The underlying table schema
-        schema: Box<Schema>,
-        /// Whether the CSV file(s) have a header containing column names
-        has_header: bool,
-        /// An optional column delimiter. Defaults to `b','`
-        delimiter: Option<u8>,
-        /// Optional column indices to use as a projection
-        projection: Option<Vec<usize>>,
-        /// The schema description of the output
-        projected_schema: Box<Schema>,
-    },
-    /// Produces no rows: An empty relation with an empty schema
-    EmptyRelation {
-        /// The schema description of the output
-        schema: Box<Schema>,
-    },
-    /// Produces the first `n` tuples from its input and discards the rest.
-    Limit {
-        /// The limit
-        n: usize,
-        /// The logical plan
-        input: Box<LogicalPlan>,
-        /// The schema description of the output
-        schema: Box<Schema>,
-    },
-    /// Creates an external table.
-    CreateExternalTable {
-        /// The table schema
-        schema: Box<Schema>,
-        /// The table name
-        name: String,
-        /// The physical location
-        location: String,
-        /// The file type of physical file
-        file_type: FileType,
-        /// Whether the CSV file contains a header
-        has_header: bool,
-    },
-}
-
-impl LogicalPlan {
-    /// Get a reference to the logical plan's schema
-    pub fn schema(&self) -> &Box<Schema> {
-        match self {
-            LogicalPlan::EmptyRelation { schema } => &schema,
-            LogicalPlan::InMemoryScan {
-                projected_schema, ..
-            } => &projected_schema,
-            LogicalPlan::CsvScan {
-                projected_schema, ..
-            } => &projected_schema,
-            LogicalPlan::ParquetScan {
-                projected_schema, ..
-            } => &projected_schema,
-            LogicalPlan::TableScan {
-                projected_schema, ..
-            } => &projected_schema,
-            LogicalPlan::Projection { schema, .. } => &schema,
-            LogicalPlan::Selection { input, .. } => input.schema(),
-            LogicalPlan::Aggregate { schema, .. } => &schema,
-            LogicalPlan::Sort { schema, .. } => &schema,
-            LogicalPlan::Limit { schema, .. } => &schema,
-            LogicalPlan::CreateExternalTable { schema, .. } => &schema,
-        }
-    }
-}
-
-impl LogicalPlan {
-    fn fmt_with_indent(&self, f: &mut fmt::Formatter, indent: usize) -> fmt::Result {
-        if indent > 0 {
-            writeln!(f)?;
-            for _ in 0..indent {
-                write!(f, "  ")?;
-            }
-        }
-        match *self {
-            LogicalPlan::EmptyRelation { .. } => write!(f, "EmptyRelation"),
-            LogicalPlan::TableScan {
-                ref table_name,
-                ref projection,
-                ..
-            } => write!(f, "TableScan: {} projection={:?}", table_name, projection),
-            LogicalPlan::InMemoryScan { ref projection, .. } => {
-                write!(f, "InMemoryScan: projection={:?}", projection)
-            }
-            LogicalPlan::CsvScan {
-                ref path,
-                ref projection,
-                ..
-            } => write!(f, "CsvScan: {} projection={:?}", path, projection),
-            LogicalPlan::ParquetScan {
-                ref path,
-                ref projection,
-                ..
-            } => write!(f, "ParquetScan: {} projection={:?}", path, projection),
-            LogicalPlan::Projection {
-                ref expr,
-                ref input,
-                ..
-            } => {
-                write!(f, "Projection: ")?;
-                for i in 0..expr.len() {
-                    if i > 0 {
-                        write!(f, ", ")?;
-                    }
-                    write!(f, "{:?}", expr[i])?;
-                }
-                input.fmt_with_indent(f, indent + 1)
-            }
-            LogicalPlan::Selection {
-                ref expr,
-                ref input,
-                ..
-            } => {
-                write!(f, "Selection: {:?}", expr)?;
-                input.fmt_with_indent(f, indent + 1)
-            }
-            LogicalPlan::Aggregate {
-                ref input,
-                ref group_expr,
-                ref aggr_expr,
-                ..
-            } => {
-                write!(
-                    f,
-                    "Aggregate: groupBy=[{:?}], aggr=[{:?}]",
-                    group_expr, aggr_expr
-                )?;
-                input.fmt_with_indent(f, indent + 1)
-            }
-            LogicalPlan::Sort {
-                ref input,
-                ref expr,
-                ..
-            } => {
-                write!(f, "Sort: ")?;
-                for i in 0..expr.len() {
-                    if i > 0 {
-                        write!(f, ", ")?;
-                    }
-                    write!(f, "{:?}", expr[i])?;
-                }
-                input.fmt_with_indent(f, indent + 1)
-            }
-            LogicalPlan::Limit {
-                ref input, ref n, ..
-            } => {
-                write!(f, "Limit: {}", n)?;
-                input.fmt_with_indent(f, indent + 1)
-            }
-            LogicalPlan::CreateExternalTable { ref name, .. } => {
-                write!(f, "CreateExternalTable: {:?}", name)
-            }
-        }
-    }
-}
-
-impl fmt::Debug for LogicalPlan {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        self.fmt_with_indent(f, 0)
-    }
-}
-
-/// Verify a given type cast can be performed
-pub fn can_coerce_from(type_into: &DataType, type_from: &DataType) -> bool {
-    use self::DataType::*;
-    match type_into {
-        Int8 => match type_from {
-            Int8 => true,
-            _ => false,
-        },
-        Int16 => match type_from {
-            Int8 | Int16 | UInt8 => true,
-            _ => false,
-        },
-        Int32 => match type_from {
-            Int8 | Int16 | Int32 | UInt8 | UInt16 => true,
-            _ => false,
-        },
-        Int64 => match type_from {
-            Int8 | Int16 | Int32 | Int64 | UInt8 | UInt16 | UInt32 => true,
-            _ => false,
-        },
-        UInt8 => match type_from {
-            UInt8 => true,
-            _ => false,
-        },
-        UInt16 => match type_from {
-            UInt8 | UInt16 => true,
-            _ => false,
-        },
-        UInt32 => match type_from {
-            UInt8 | UInt16 | UInt32 => true,
-            _ => false,
-        },
-        UInt64 => match type_from {
-            UInt8 | UInt16 | UInt32 | UInt64 => true,
-            _ => false,
-        },
-        Float32 => match type_from {
-            Int8 | Int16 | Int32 | Int64 => true,
-            UInt8 | UInt16 | UInt32 | UInt64 => true,
-            Float32 => true,
-            _ => false,
-        },
-        Float64 => match type_from {
-            Int8 | Int16 | Int32 | Int64 => true,
-            UInt8 | UInt16 | UInt32 | UInt64 => true,
-            Float32 | Float64 => true,
-            _ => false,
-        },
-        Utf8 => true,
-        _ => false,
-    }
-}
-
-/// Builder for logical plans
-pub struct LogicalPlanBuilder {
-    plan: LogicalPlan,
-}
-
-impl LogicalPlanBuilder {
-    /// Create a builder from an existing plan
-    pub fn from(plan: &LogicalPlan) -> Self {
-        Self { plan: plan.clone() }
-    }
-
-    /// Create an empty relation
-    pub fn empty() -> Self {
-        Self::from(&LogicalPlan::EmptyRelation {
-            schema: Box::new(Schema::empty()),
-        })
-    }
-
-    /// Scan a CSV data source
-    pub fn scan_csv(
-        path: &str,
-        options: CsvReadOptions,
-        projection: Option<Vec<usize>>,
-    ) -> Result<Self> {
-        let has_header = options.has_header;
-        let delimiter = options.delimiter;
-        let schema: Schema = match options.schema {
-            Some(s) => s.to_owned(),
-            None => CsvFile::try_new(path, options)?
-                .schema()
-                .as_ref()
-                .to_owned(),
-        };
-
-        let projected_schema = Box::new(
-            projection
-                .clone()
-                .map(|p| {
-                    Schema::new(p.iter().map(|i| schema.field(*i).clone()).collect())
-                })
-                .or(Some(schema.clone()))
-                .unwrap(),
-        );
-
-        Ok(Self::from(&LogicalPlan::CsvScan {
-            path: path.to_owned(),
-            schema: Box::new(schema),
-            has_header: has_header,
-            delimiter: Some(delimiter),
-            projection,
-            projected_schema,
-        }))
-    }
-
-    /// Scan a Parquet data source
-    pub fn scan_parquet(path: &str, projection: Option<Vec<usize>>) -> Result<Self> {
-        let p = ParquetTable::try_new(path)?;
-        let schema = p.schema().as_ref().to_owned();
-        let projected_schema = projection
-            .clone()
-            .map(|p| Schema::new(p.iter().map(|i| schema.field(*i).clone()).collect()));
-        Ok(Self::from(&LogicalPlan::ParquetScan {
-            path: path.to_owned(),
-            schema: Box::new(schema.clone()),
-            projection,
-            projected_schema: Box::new(
-                projected_schema.or(Some(schema.clone())).unwrap(),
-            ),
-        }))
-    }
-
-    /// Scan a data source
-    pub fn scan(
-        schema_name: &str,
-        table_name: &str,
-        table_schema: &Schema,
-        projection: Option<Vec<usize>>,
-    ) -> Result<Self> {
-        let projected_schema = projection.clone().map(|p| {
-            Schema::new(p.iter().map(|i| table_schema.field(*i).clone()).collect())
-        });
-        Ok(Self::from(&LogicalPlan::TableScan {
-            schema_name: schema_name.to_owned(),
-            table_name: table_name.to_owned(),
-            table_schema: Box::new(table_schema.clone()),
-            projected_schema: Box::new(
-                projected_schema.or(Some(table_schema.clone())).unwrap(),
-            ),
-            projection,
-        }))
-    }
-
-    /// Apply a projection
-    pub fn project(&self, expr: Vec<Expr>) -> Result<Self> {
-        let input_schema = self.plan.schema();
-        let projected_expr = if expr.contains(&Expr::Wildcard) {
-            let mut expr_vec = vec![];
-            (0..expr.len()).for_each(|i| match &expr[i] {
-                Expr::Wildcard => {
-                    (0..input_schema.fields().len())
-                        .for_each(|i| expr_vec.push(col(input_schema.field(i).name())));
-                }
-                _ => expr_vec.push(expr[i].clone()),
-            });
-            expr_vec
-        } else {
-            expr.clone()
-        };
-
-        let schema =
-            Schema::new(exprlist_to_fields(&projected_expr, input_schema.as_ref())?);
-
-        Ok(Self::from(&LogicalPlan::Projection {
-            expr: projected_expr,
-            input: Box::new(self.plan.clone()),
-            schema: Box::new(schema),
-        }))
-    }
-
-    /// Apply a filter
-    pub fn filter(&self, expr: Expr) -> Result<Self> {
-        Ok(Self::from(&LogicalPlan::Selection {
-            expr,
-            input: Box::new(self.plan.clone()),
-        }))
-    }
-
-    /// Apply a limit
-    pub fn limit(&self, n: usize) -> Result<Self> {
-        Ok(Self::from(&LogicalPlan::Limit {
-            n,
-            input: Box::new(self.plan.clone()),
-            schema: self.plan.schema().clone(),
-        }))
-    }
-
-    /// Apply a sort
-    pub fn sort(&self, expr: Vec<Expr>) -> Result<Self> {
-        Ok(Self::from(&LogicalPlan::Sort {
-            expr,
-            input: Box::new(self.plan.clone()),
-            schema: self.plan.schema().clone(),
-        }))
-    }
-
-    /// Apply an aggregate
-    pub fn aggregate(&self, group_expr: Vec<Expr>, aggr_expr: Vec<Expr>) -> Result<Self> {
-        let mut all_expr: Vec<Expr> = group_expr.clone();
-        aggr_expr.iter().for_each(|x| all_expr.push(x.clone()));
-
-        let aggr_schema = Schema::new(exprlist_to_fields(&all_expr, self.plan.schema())?);
-
-        Ok(Self::from(&LogicalPlan::Aggregate {
-            input: Box::new(self.plan.clone()),
-            group_expr,
-            aggr_expr,
-            schema: Box::new(aggr_schema),
-        }))
-    }
-
-    /// Build the plan
-    pub fn build(&self) -> Result<LogicalPlan> {
-        Ok(self.plan.clone())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn plan_builder_simple() -> Result<()> {
-        let plan = LogicalPlanBuilder::scan(
-            "default",
-            "employee.csv",
-            &employee_schema(),
-            Some(vec![0, 3]),
-        )?
-        .filter(col("state").eq(&lit("CO")))?
-        .project(vec![col("id")])?
-        .build()?;
-
-        let expected = "Projection: #id\
-        \n  Selection: #state Eq Utf8(\"CO\")\
-        \n    TableScan: employee.csv projection=Some([0, 3])";
-
-        assert_eq!(expected, format!("{:?}", plan));
-
-        Ok(())
-    }
-
-    #[test]
-    fn plan_builder_csv() -> Result<()> {
-        let plan = LogicalPlanBuilder::scan_csv(
-            "employee.csv",
-            CsvReadOptions::new().schema(&employee_schema()),
-            Some(vec![0, 3]),
-        )?
-        .filter(col("state").eq(&lit("CO")))?
-        .project(vec![col("id")])?
-        .build()?;
-
-        let expected = "Projection: #id\
-        \n  Selection: #state Eq Utf8(\"CO\")\
-        \n    CsvScan: employee.csv projection=Some([0, 3])";
-
-        assert_eq!(expected, format!("{:?}", plan));
-
-        Ok(())
-    }
-
-    #[test]
-    fn plan_builder_aggregate() -> Result<()> {
-        let plan = LogicalPlanBuilder::scan(
-            "default",
-            "employee.csv",
-            &employee_schema(),
-            Some(vec![3, 4]),
-        )?
-        .aggregate(
-            vec![col("state")],
-            vec![aggregate_expr("SUM", col("salary"), DataType::Int32)
-                .alias("total_salary")],
-        )?
-        .project(vec![col("state"), col("total_salary")])?
-        .build()?;
-
-        let expected = "Projection: #state, #total_salary\
-        \n  Aggregate: groupBy=[[#state]], aggr=[[SUM(#salary) AS total_salary]]\
-        \n    TableScan: employee.csv projection=Some([3, 4])";
-
-        assert_eq!(expected, format!("{:?}", plan));
-
-        Ok(())
-    }
-
-    #[test]
-    fn plan_builder_sort() -> Result<()> {
-        let plan = LogicalPlanBuilder::scan(
-            "default",
-            "employee.csv",
-            &employee_schema(),
-            Some(vec![3, 4]),
-        )?
-        .sort(vec![
-            Expr::Sort {
-                expr: Box::new(col("state")),
-                asc: true,
-                nulls_first: true,
-            },
-            Expr::Sort {
-                expr: Box::new(col("total_salary")),
-                asc: false,
-                nulls_first: false,
-            },
-        ])?
-        .build()?;
-
-        let expected = "Sort: #state ASC NULLS FIRST, #total_salary DESC NULLS LAST\
-        \n  TableScan: employee.csv projection=Some([3, 4])";
-
-        assert_eq!(expected, format!("{:?}", plan));
-
-        Ok(())
-    }
-
-    fn employee_schema() -> Schema {
-        Schema::new(vec![
-            Field::new("id", DataType::Int32, false),
-            Field::new("first_name", DataType::Utf8, false),
-            Field::new("last_name", DataType::Utf8, false),
-            Field::new("state", DataType::Utf8, false),
-            Field::new("salary", DataType::Int32, false),
-        ])
-    }
-}
diff --git a/rust/datafusion/src/optimizer/filter_push_down.rs b/rust/datafusion/src/optimizer/filter_push_down.rs
new file mode 100644
index 00000000000..28a99eedfe1
--- /dev/null
+++ b/rust/datafusion/src/optimizer/filter_push_down.rs
@@ -0,0 +1,622 @@
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Filter Push Down optimizer rule ensures that filters are applied as early as possible in the plan
+
+use crate::error::Result;
+use crate::logical_plan::Expr;
+use crate::logical_plan::{and, LogicalPlan};
+use crate::optimizer::optimizer::OptimizerRule;
+use crate::optimizer::utils;
+use std::{
+    collections::{BTreeMap, HashMap, HashSet},
+    sync::Arc,
+};
+
+/// Filter Push Down optimizer rule pushes filter clauses down the plan
+///
+/// This optimization looks for the maximum depth of each column in the plan where a filter can be applied and
+/// re-writes the plan with filters on those locations.
+/// It performs two passes on the plan:
+/// 1. identify filters, which columns they use, and projections along the path
+/// 2. move filters down, re-writing the expressions using the projections
+/*
+A filter-commutative operation is an operation whose result of filter(op(data)) = op(filter(data)).
+An example of a filter-commutative operation is a projection; a counter-example is `limit`.
+
+The filter-commutative property is column-specific. An aggregate grouped by A on SUM(B)
+can commute with a filter that depends on A only, but does not commute with a filter that depends
+on SUM(B).
+
+A location in this module is identified by a number, depth, which is 0 for the last operation
+and highest for the first operation (typically a scan).
+
+This optimizer commutes filters with filter-commutative operations to push the filters
+to the maximum possible depth, consequently re-writing the filter expressions by every
+projection that changes the filter's expression.
+
+    Filter: #b Gt Int64(10)
+        Projection: #a AS b
+
+is optimized to
+
+    Projection: #a AS b
+        Filter: #a Gt Int64(10)  <--- changed from #b to #a
+
+To perform such optimization, we first analyze the plan to identify three items:
+
+1. Where are the filters located in the plan
+2. Where are non-commutable operations' columns located in the plan (break_points)
+3. Where are projections located in the plan
+
+With this information, we re-write the plan by:
+
+1. Computing the maximum possible depth of each column between breakpoints
+2. Computing the maximum possible depth of each filter expression based on the columns it depends on
+3. re-write the filter expression for every projection that it commutes with from its original depth to its max possible depth
+4. recursively re-write the plan by deleting old filter expressions and adding new filter expressions on their max possible depth.
+*/
+pub struct FilterPushDown {}
+
+impl OptimizerRule for FilterPushDown {
+    fn name(&self) -> &str {
+        return "filter_push_down";
+    }
+
+    fn optimize(&mut self, plan: &LogicalPlan) -> Result<LogicalPlan> {
+        let result = analyze_plan(plan, 0)?;
+        let break_points = result.break_points.clone();
+
+        // get max depth over all breakpoints
+        let max_depth = break_points.keys().max();
+        if max_depth.is_none() {
+            // it is unlikely that the plan is correct without break points as all scans
+            // adds breakpoints. We just return the plan and let others handle the error
+            return Ok(plan.clone());
+        }
+        let max_depth = *max_depth.unwrap(); // unwrap is safe by previous if
+
+        // construct optimized position of each of the new filters
+        // E.g. when we have a filter (c1 + c2 > 2), c1's max depth is 10 and c2 is 11, we
+        // can push the filter to depth 10
+        let mut new_filtersnew_filters: BTreeMap<usize, Expr> = BTreeMap::new();
+        for (filter_depth, expr) in result.filters {
+            // get all columns on the filter expression
+            let mut filter_columns: HashSet<String> = HashSet::new();
+            utils::expr_to_column_names(&expr, &mut filter_columns)?;
+
+            // identify the depths that are filter-commutable with this filter
+            let mut new_depth = filter_depth;
+            for depth in filter_depth..max_depth {
+                if let Some(break_columns) = break_points.get(&depth) {
+                    if filter_columns
+                        .intersection(break_columns)
+                        .peekable()
+                        .peek()
+                        .is_none()
+                    {
+                        new_depth += 1
+                    } else {
+                        // non-commutable: can't advance any further
+                        break;
+                    }
+                } else {
+                    new_depth += 1
+                }
+            }
+
+            // re-write the new filters based on all projections that it crossed.
+            // E.g. in `Filter: #b\n  Projection: #a > 1 as b`, we can swap them, but the filter must be "#a > 1"
+            let mut new_expression = expr.clone();
+            for depth_i in filter_depth..new_depth {
+                if let Some(projection) = result.projections.get(&depth_i) {
+                    new_expression = rewrite(&new_expression, projection)?;
+                }
+            }
+
+            // AND filter expressions that would be placed on the same depth
+            if let Some(existing_expression) = new_filtersnew_filters.get(&new_depth) {
+                new_expression = and(existing_expression, &new_expression)
+            }
+            new_filtersnew_filters.insert(new_depth, new_expression);
+        }
+
+        optimize_plan(plan, &new_filtersnew_filters, 0)
+    }
+}
+
+/// The result of a plan analysis suitable to perform a filter push down optimization
+// BTreeMap are ordered, which ensures stability in ordered operations.
+// Also, most inserts here are at the end
+struct AnalysisResult {
+    /// maps the depths of non filter-commutative nodes to their columns
+    /// depths not in here indicate that the node is commutative
+    pub break_points: BTreeMap<usize, HashSet<String>>,
+    /// maps the depths of filter nodes to expressions
+    pub filters: BTreeMap<usize, Expr>,
+    /// maps the depths of projection nodes to their expressions
+    pub projections: BTreeMap<usize, HashMap<String, Expr>>,
+}
+
+/// Recursively transverses the logical plan looking for depths that break filter pushdown
+fn analyze_plan(plan: &LogicalPlan, depth: usize) -> Result<AnalysisResult> {
+    match plan {
+        LogicalPlan::Filter { input, predicate } => {
+            let mut result = analyze_plan(&input, depth + 1)?;
+            result.filters.insert(depth, predicate.clone());
+            Ok(result)
+        }
+        LogicalPlan::Projection {
+            input,
+            expr,
+            schema,
+        } => {
+            let mut result = analyze_plan(&input, depth + 1)?;
+
+            // collect projection.
+            let mut projection = HashMap::new();
+            schema.fields().iter().enumerate().for_each(|(i, field)| {
+                // strip alias, as they should not be part of filters
+                let expr = match &expr[i] {
+                    Expr::Alias(expr, _) => expr.as_ref().clone(),
+                    expr => expr.clone(),
+                };
+
+                projection.insert(field.name().clone(), expr);
+            });
+            result.projections.insert(depth, projection);
+            Ok(result)
+        }
+        LogicalPlan::Aggregate {
+            input, aggr_expr, ..
+        } => {
+            let mut result = analyze_plan(&input, depth + 1)?;
+
+            // construct set of columns that `aggr_expr` depends on
+            let mut agg_columns = HashSet::new();
+            utils::exprlist_to_column_names(aggr_expr, &mut agg_columns)?;
+
+            // collect all columns that break at this depth:
+            // * columns whose aggregation expression depends on
+            // * the aggregation columns themselves
+            let mut columns = agg_columns.iter().cloned().collect::<HashSet<_>>();
+            let agg_columns = aggr_expr
+                .iter()
+                .map(|x| x.name(input.schema()))
+                .collect::<Result<HashSet<_>>>()?;
+            columns.extend(agg_columns);
+            result.break_points.insert(depth, columns);
+
+            Ok(result)
+        }
+        LogicalPlan::Sort { input, .. } => analyze_plan(&input, depth + 1),
+        LogicalPlan::Limit { input, .. } => {
+            let mut result = analyze_plan(&input, depth + 1)?;
+
+            // collect all columns that break at this depth
+            let columns = input
+                .schema()
+                .fields()
+                .iter()
+                .map(|f| f.name().clone())
+                .collect::<HashSet<_>>();
+            result.break_points.insert(depth, columns);
+            Ok(result)
+        }
+        // all other plans add breaks to all their columns to indicate that filters can't proceed further.
+        _ => {
+            let columns = plan
+                .schema()
+                .fields()
+                .iter()
+                .map(|f| f.name().clone())
+                .collect::<HashSet<_>>();
+            let mut break_points = BTreeMap::new();
+
+            break_points.insert(depth, columns);
+            Ok(AnalysisResult {
+                break_points,
+                filters: BTreeMap::new(),
+                projections: BTreeMap::new(),
+            })
+        }
+    }
+}
+
+impl FilterPushDown {
+    #[allow(missing_docs)]
+    pub fn new() -> Self {
+        Self {}
+    }
+}
+
+/// Returns a re-written logical plan where all old filters are removed and the new ones are added.
+fn optimize_plan(
+    plan: &LogicalPlan,
+    new_filters: &BTreeMap<usize, Expr>,
+    depth: usize,
+) -> Result<LogicalPlan> {
+    // optimize the plan recursively:
+    let new_plan = match plan {
+        LogicalPlan::Filter { input, .. } => {
+            // ignore old filters
+            Ok(optimize_plan(&input, new_filters, depth + 1)?)
+        }
+        _ => {
+            // all other nodes are copied, optimizing recursively.
+            let expr = utils::expressions(plan);
+
+            let inputs = utils::inputs(plan);
+            let new_inputs = inputs
+                .iter()
+                .map(|plan| optimize_plan(plan, new_filters, depth + 1))
+                .collect::<Result<Vec<_>>>()?;
+
+            utils::from_plan(plan, &expr, &new_inputs)
+        }
+    }?;
+
+    // if a new filter is to be applied, apply it
+    if let Some(expr) = new_filters.get(&depth) {
+        return Ok(LogicalPlan::Filter {
+            predicate: expr.clone(),
+            input: Arc::new(new_plan),
+        });
+    } else {
+        Ok(new_plan)
+    }
+}
+
+/// replaces columns by its name on the projection.
+fn rewrite(expr: &Expr, projection: &HashMap<String, Expr>) -> Result<Expr> {
+    let expressions = utils::expr_sub_expressions(&expr)?;
+
+    let expressions = expressions
+        .iter()
+        .map(|e| rewrite(e, &projection))
+        .collect::<Result<Vec<_>>>()?;
+
+    match expr {
+        Expr::Column(name) => {
+            if let Some(expr) = projection.get(name) {
+                return Ok(expr.clone());
+            }
+        }
+        _ => {}
+    }
+
+    utils::rewrite_expression(&expr, &expressions)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::logical_plan::col;
+    use crate::logical_plan::{lit, sum, Expr, LogicalPlanBuilder, Operator};
+    use crate::test::*;
+
+    fn assert_optimized_plan_eq(plan: &LogicalPlan, expected: &str) {
+        let mut rule = FilterPushDown::new();
+        let optimized_plan = rule.optimize(plan).expect("failed to optimize plan");
+        let formatted_plan = format!("{:?}", optimized_plan);
+        assert_eq!(formatted_plan, expected);
+    }
+
+    #[test]
+    fn filter_before_projection() -> Result<()> {
+        let table_scan = test_table_scan()?;
+        let plan = LogicalPlanBuilder::from(&table_scan)
+            .project(vec![col("a"), col("b")])?
+            .filter(col("a").eq(lit(1i64)))?
+            .build()?;
+        // filter is before projection
+        let expected = "\
+            Projection: #a, #b\
+            \n  Filter: #a Eq Int64(1)\
+            \n    TableScan: test projection=None";
+        assert_optimized_plan_eq(&plan, expected);
+        Ok(())
+    }
+
+    #[test]
+    fn filter_after_limit() -> Result<()> {
+        let table_scan = test_table_scan()?;
+        let plan = LogicalPlanBuilder::from(&table_scan)
+            .project(vec![col("a"), col("b")])?
+            .limit(10)?
+            .filter(col("a").eq(lit(1i64)))?
+            .build()?;
+        // filter is before single projection
+        let expected = "\
+            Filter: #a Eq Int64(1)\
+            \n  Limit: 10\
+            \n    Projection: #a, #b\
+            \n      TableScan: test projection=None";
+        assert_optimized_plan_eq(&plan, expected);
+        Ok(())
+    }
+
+    #[test]
+    fn filter_jump_2_plans() -> Result<()> {
+        let table_scan = test_table_scan()?;
+        let plan = LogicalPlanBuilder::from(&table_scan)
+            .project(vec![col("a"), col("b"), col("c")])?
+            .project(vec![col("c"), col("b")])?
+            .filter(col("a").eq(lit(1i64)))?
+            .build()?;
+        // filter is before double projection
+        let expected = "\
+            Projection: #c, #b\
+            \n  Projection: #a, #b, #c\
+            \n    Filter: #a Eq Int64(1)\
+            \n      TableScan: test projection=None";
+        assert_optimized_plan_eq(&plan, expected);
+        Ok(())
+    }
+
+    #[test]
+    fn filter_move_agg() -> Result<()> {
+        let table_scan = test_table_scan()?;
+        let plan = LogicalPlanBuilder::from(&table_scan)
+            .aggregate(vec![col("a")], vec![sum(col("b")).alias("total_salary")])?
+            .filter(col("a").gt(lit(10i64)))?
+            .build()?;
+        // filter of key aggregation is commutative
+        let expected = "\
+            Aggregate: groupBy=[[#a]], aggr=[[SUM(#b) AS total_salary]]\
+            \n  Filter: #a Gt Int64(10)\
+            \n    TableScan: test projection=None";
+        assert_optimized_plan_eq(&plan, expected);
+        Ok(())
+    }
+
+    #[test]
+    fn filter_keep_agg() -> Result<()> {
+        let table_scan = test_table_scan()?;
+        let plan = LogicalPlanBuilder::from(&table_scan)
+            .aggregate(vec![col("a")], vec![sum(col("b")).alias("b")])?
+            .filter(col("b").gt(lit(10i64)))?
+            .build()?;
+        // filter of aggregate is after aggregation since they are non-commutative
+        let expected = "\
+            Filter: #b Gt Int64(10)\
+            \n  Aggregate: groupBy=[[#a]], aggr=[[SUM(#b) AS b]]\
+            \n    TableScan: test projection=None";
+        assert_optimized_plan_eq(&plan, expected);
+        Ok(())
+    }
+
+    /// verifies that a filter is pushed to before a projection, the filter expression is correctly re-written
+    #[test]
+    fn alias() -> Result<()> {
+        let table_scan = test_table_scan()?;
+        let plan = LogicalPlanBuilder::from(&table_scan)
+            .project(vec![col("a").alias("b"), col("c")])?
+            .filter(col("b").eq(lit(1i64)))?
+            .build()?;
+        // filter is before projection
+        let expected = "\
+            Projection: #a AS b, #c\
+            \n  Filter: #a Eq Int64(1)\
+            \n    TableScan: test projection=None";
+        assert_optimized_plan_eq(&plan, expected);
+        Ok(())
+    }
+
+    fn add(left: Expr, right: Expr) -> Expr {
+        Expr::BinaryExpr {
+            left: Box::new(left),
+            op: Operator::Plus,
+            right: Box::new(right),
+        }
+    }
+
+    fn multiply(left: Expr, right: Expr) -> Expr {
+        Expr::BinaryExpr {
+            left: Box::new(left),
+            op: Operator::Multiply,
+            right: Box::new(right),
+        }
+    }
+
+    /// verifies that a filter is pushed to before a projection with a complex expression, the filter expression is correctly re-written
+    #[test]
+    fn complex_expression() -> Result<()> {
+        let table_scan = test_table_scan()?;
+        let plan = LogicalPlanBuilder::from(&table_scan)
+            .project(vec![
+                add(multiply(col("a"), lit(2)), col("c")).alias("b"),
+                col("c"),
+            ])?
+            .filter(col("b").eq(lit(1i64)))?
+            .build()?;
+
+        // not part of the test, just good to know:
+        assert_eq!(
+            format!("{:?}", plan),
+            "\
+            Filter: #b Eq Int64(1)\
+            \n  Projection: #a Multiply Int32(2) Plus #c AS b, #c\
+            \n    TableScan: test projection=None"
+        );
+
+        // filter is before projection
+        let expected = "\
+            Projection: #a Multiply Int32(2) Plus #c AS b, #c\
+            \n  Filter: #a Multiply Int32(2) Plus #c Eq Int64(1)\
+            \n    TableScan: test projection=None";
+        assert_optimized_plan_eq(&plan, expected);
+        Ok(())
+    }
+
+    /// verifies that when a filter is pushed to after 2 projections, the filter expression is correctly re-written
+    #[test]
+    fn complex_plan() -> Result<()> {
+        let table_scan = test_table_scan()?;
+        let plan = LogicalPlanBuilder::from(&table_scan)
+            .project(vec![
+                add(multiply(col("a"), lit(2)), col("c")).alias("b"),
+                col("c"),
+            ])?
+            // second projection where we rename columns, just to make it difficult
+            .project(vec![multiply(col("b"), lit(3)).alias("a"), col("c")])?
+            .filter(col("a").eq(lit(1i64)))?
+            .build()?;
+
+        // not part of the test, just good to know:
+        assert_eq!(
+            format!("{:?}", plan),
+            "\
+            Filter: #a Eq Int64(1)\
+            \n  Projection: #b Multiply Int32(3) AS a, #c\
+            \n    Projection: #a Multiply Int32(2) Plus #c AS b, #c\
+            \n      TableScan: test projection=None"
+        );
+
+        // filter is before the projections
+        let expected = "\
+        Projection: #b Multiply Int32(3) AS a, #c\
+        \n  Projection: #a Multiply Int32(2) Plus #c AS b, #c\
+        \n    Filter: #a Multiply Int32(2) Plus #c Multiply Int32(3) Eq Int64(1)\
+        \n      TableScan: test projection=None";
+        assert_optimized_plan_eq(&plan, expected);
+        Ok(())
+    }
+
+    /// verifies that when two filters apply after an aggregation that only allows one to be pushed, one is pushed
+    /// and the other not.
+    #[test]
+    fn multi_filter() -> Result<()> {
+        // the aggregation allows one filter to pass (b), and the other one to not pass (SUM(c))
+        let table_scan = test_table_scan()?;
+        let plan = LogicalPlanBuilder::from(&table_scan)
+            .project(vec![col("a").alias("b"), col("c")])?
+            .aggregate(vec![col("b")], vec![sum(col("c"))])?
+            .filter(col("b").gt(lit(10i64)))?
+            .filter(col("SUM(c)").gt(lit(10i64)))?
+            .build()?;
+
+        // not part of the test, just good to know:
+        assert_eq!(
+            format!("{:?}", plan),
+            "\
+            Filter: #SUM(c) Gt Int64(10)\
+            \n  Filter: #b Gt Int64(10)\
+            \n    Aggregate: groupBy=[[#b]], aggr=[[SUM(#c)]]\
+            \n      Projection: #a AS b, #c\
+            \n        TableScan: test projection=None"
+        );
+
+        // filter is before the projections
+        let expected = "\
+        Filter: #SUM(c) Gt Int64(10)\
+        \n  Aggregate: groupBy=[[#b]], aggr=[[SUM(#c)]]\
+        \n    Projection: #a AS b, #c\
+        \n      Filter: #a Gt Int64(10)\
+        \n        TableScan: test projection=None";
+        assert_optimized_plan_eq(&plan, expected);
+
+        Ok(())
+    }
+
+    /// verifies that when two limits are in place, we jump neither
+    #[test]
+    fn double_limit() -> Result<()> {
+        let table_scan = test_table_scan()?;
+        let plan = LogicalPlanBuilder::from(&table_scan)
+            .project(vec![col("a"), col("b")])?
+            .limit(20)?
+            .limit(10)?
+            .project(vec![col("a"), col("b")])?
+            .filter(col("a").eq(lit(1i64)))?
+            .build()?;
+        // filter does not just any of the limits
+        let expected = "\
+            Projection: #a, #b\
+            \n  Filter: #a Eq Int64(1)\
+            \n    Limit: 10\
+            \n      Limit: 20\
+            \n        Projection: #a, #b\
+            \n          TableScan: test projection=None";
+        assert_optimized_plan_eq(&plan, expected);
+        Ok(())
+    }
+
+    /// verifies that filters with the same columns are correctly placed
+    #[test]
+    fn filter_2_breaks_limits() -> Result<()> {
+        let table_scan = test_table_scan()?;
+        let plan = LogicalPlanBuilder::from(&table_scan)
+            .project(vec![col("a")])?
+            .filter(col("a").lt_eq(lit(1i64)))?
+            .limit(1)?
+            .project(vec![col("a")])?
+            .filter(col("a").gt_eq(lit(1i64)))?
+            .build()?;
+        // Should be able to move both filters below the projections
+
+        // not part of the test
+        assert_eq!(
+            format!("{:?}", plan),
+            "Filter: #a GtEq Int64(1)\
+             \n  Projection: #a\
+             \n    Limit: 1\
+             \n      Filter: #a LtEq Int64(1)\
+             \n        Projection: #a\
+             \n          TableScan: test projection=None"
+        );
+
+        let expected = "\
+        Projection: #a\
+        \n  Filter: #a GtEq Int64(1)\
+        \n    Limit: 1\
+        \n      Projection: #a\
+        \n        Filter: #a LtEq Int64(1)\
+        \n          TableScan: test projection=None";
+
+        assert_optimized_plan_eq(&plan, expected);
+        Ok(())
+    }
+
+    /// verifies that filters to be placed on the same depth are ANDed
+    #[test]
+    fn two_filters_on_same_depth() -> Result<()> {
+        let table_scan = test_table_scan()?;
+        let plan = LogicalPlanBuilder::from(&table_scan)
+            .limit(1)?
+            .filter(col("a").lt_eq(lit(1i64)))?
+            .filter(col("a").gt_eq(lit(1i64)))?
+            .project(vec![col("a")])?
+            .build()?;
+
+        // not part of the test
+        assert_eq!(
+            format!("{:?}", plan),
+            "Projection: #a\
+            \n  Filter: #a GtEq Int64(1)\
+            \n    Filter: #a LtEq Int64(1)\
+            \n      Limit: 1\
+            \n        TableScan: test projection=None"
+        );
+
+        let expected = "\
+        Projection: #a\
+        \n  Filter: #a GtEq Int64(1) And #a LtEq Int64(1)\
+        \n    Limit: 1\
+        \n      TableScan: test projection=None";
+
+        assert_optimized_plan_eq(&plan, expected);
+        Ok(())
+    }
+}
diff --git a/rust/datafusion/src/optimizer/mod.rs b/rust/datafusion/src/optimizer/mod.rs
index e60c7db824b..dffae5328b7 100644
--- a/rust/datafusion/src/optimizer/mod.rs
+++ b/rust/datafusion/src/optimizer/mod.rs
@@ -18,7 +18,7 @@
 //! This module contains a query optimizer that operates against a logical plan and applies
 //! some simple rules to a logical plan, such as "Projection Push Down" and "Type Coercion".
 
+pub mod filter_push_down;
 pub mod optimizer;
 pub mod projection_push_down;
-pub mod type_coercion;
 pub mod utils;
diff --git a/rust/datafusion/src/optimizer/optimizer.rs b/rust/datafusion/src/optimizer/optimizer.rs
index e041ce3f5e5..30015197bca 100644
--- a/rust/datafusion/src/optimizer/optimizer.rs
+++ b/rust/datafusion/src/optimizer/optimizer.rs
@@ -17,12 +17,29 @@
 
 //! Query optimizer traits
 
+use super::utils;
 use crate::error::Result;
-use crate::logicalplan::LogicalPlan;
+use crate::logical_plan::LogicalPlan;
 
 /// An optimizer rules performs a transformation on a logical plan to produce an optimized
 /// logical plan.
 pub trait OptimizerRule {
     /// Perform optimizations on the plan
     fn optimize(&mut self, plan: &LogicalPlan) -> Result<LogicalPlan>;
+    /// Produce a human readable name for this optimizer rule
+    fn name(&self) -> &str;
+
+    /// Convenience rule for writing optimizers: recursively invoke
+    /// optimize on plan's children and then return a node of the same
+    /// type. Useful for optimizer rules which want to leave the type
+    /// of plan unchanged but still apply to the children.
+    fn optimize_children(&mut self, plan: &LogicalPlan) -> Result<LogicalPlan> {
+        let new_exprs = utils::expressions(&plan);
+        let new_inputs = utils::inputs(&plan)
+            .into_iter()
+            .map(|plan| self.optimize(plan))
+            .collect::<Result<Vec<_>>>()?;
+
+        utils::from_plan(plan, &new_exprs, &new_inputs)
+    }
 }
diff --git a/rust/datafusion/src/optimizer/projection_push_down.rs b/rust/datafusion/src/optimizer/projection_push_down.rs
index e99a996d7c9..f4f190cd193 100644
--- a/rust/datafusion/src/optimizer/projection_push_down.rs
+++ b/rust/datafusion/src/optimizer/projection_push_down.rs
@@ -18,23 +18,33 @@
 //! Projection Push Down optimizer rule ensures that only referenced columns are
 //! loaded into memory
 
-use crate::error::{ExecutionError, Result};
-use crate::logicalplan::LogicalPlan;
+use crate::error::{DataFusionError, Result};
+use crate::logical_plan::LogicalPlan;
 use crate::optimizer::optimizer::OptimizerRule;
 use crate::optimizer::utils;
-use arrow::datatypes::{Field, Schema};
+use arrow::datatypes::{Field, Schema, SchemaRef};
 use arrow::error::Result as ArrowResult;
-use std::collections::HashSet;
+use std::{collections::HashSet, sync::Arc};
+use utils::optimize_explain;
 
-/// Projection Push Down optimizer rule ensures that only referenced columns are
-/// loaded into memory
+/// Optimizer that removes unused projections and aggregations from plans
+/// This reduces both scans and
 pub struct ProjectionPushDown {}
 
 impl OptimizerRule for ProjectionPushDown {
     fn optimize(&mut self, plan: &LogicalPlan) -> Result<LogicalPlan> {
-        // set of all columns refered from a scan.
-        let mut accum: HashSet<String> = HashSet::new();
-        self.optimize_plan(plan, &mut accum, false)
+        // set of all columns refered by the plan (and thus considered required by the root)
+        let required_columns = plan
+            .schema()
+            .fields()
+            .iter()
+            .map(|f| f.name().clone())
+            .collect::<HashSet<String>>();
+        return optimize_plan(self, plan, &required_columns, false);
+    }
+
+    fn name(&self) -> &str {
+        return "projection_push_down";
     }
 }
 
@@ -43,161 +53,16 @@ impl ProjectionPushDown {
     pub fn new() -> Self {
         Self {}
     }
-
-    fn optimize_plan(
-        &self,
-        plan: &LogicalPlan,
-        accum: &mut HashSet<String>,
-        has_projection: bool,
-    ) -> Result<LogicalPlan> {
-        match plan {
-            LogicalPlan::Projection {
-                expr,
-                input,
-                schema,
-            } => {
-                // collect all columns referenced by projection expressions
-                utils::exprlist_to_column_names(&expr, accum)?;
-
-                Ok(LogicalPlan::Projection {
-                    expr: expr.clone(),
-                    input: Box::new(self.optimize_plan(&input, accum, true)?),
-                    schema: schema.clone(),
-                })
-            }
-            LogicalPlan::Selection { expr, input } => {
-                // collect all columns referenced by filter expression
-                utils::expr_to_column_names(expr, accum)?;
-
-                Ok(LogicalPlan::Selection {
-                    expr: expr.clone(),
-                    input: Box::new(self.optimize_plan(&input, accum, has_projection)?),
-                })
-            }
-            LogicalPlan::Aggregate {
-                input,
-                group_expr,
-                aggr_expr,
-                schema,
-            } => {
-                // collect all columns referenced by grouping and aggregate expressions
-                utils::exprlist_to_column_names(&group_expr, accum)?;
-                utils::exprlist_to_column_names(&aggr_expr, accum)?;
-
-                Ok(LogicalPlan::Aggregate {
-                    input: Box::new(self.optimize_plan(&input, accum, has_projection)?),
-                    group_expr: group_expr.clone(),
-                    aggr_expr: aggr_expr.clone(),
-                    schema: schema.clone(),
-                })
-            }
-            LogicalPlan::Sort {
-                expr,
-                input,
-                schema,
-            } => {
-                // collect all columns referenced by sort expressions
-                utils::exprlist_to_column_names(&expr, accum)?;
-
-                Ok(LogicalPlan::Sort {
-                    expr: expr.clone(),
-                    input: Box::new(self.optimize_plan(&input, accum, has_projection)?),
-                    schema: schema.clone(),
-                })
-            }
-            LogicalPlan::Limit { n, input, schema } => Ok(LogicalPlan::Limit {
-                n: n.clone(),
-                input: Box::new(self.optimize_plan(&input, accum, has_projection)?),
-                schema: schema.clone(),
-            }),
-            LogicalPlan::EmptyRelation { .. } => Ok(plan.clone()),
-            LogicalPlan::TableScan {
-                schema_name,
-                table_name,
-                table_schema,
-                projection,
-                ..
-            } => {
-                let (projection, projected_schema) = get_projected_schema(
-                    &table_schema,
-                    projection,
-                    accum,
-                    has_projection,
-                )?;
-
-                // return the table scan with projection
-                Ok(LogicalPlan::TableScan {
-                    schema_name: schema_name.to_string(),
-                    table_name: table_name.to_string(),
-                    table_schema: table_schema.clone(),
-                    projection: Some(projection),
-                    projected_schema: Box::new(projected_schema),
-                })
-            }
-            LogicalPlan::InMemoryScan {
-                data,
-                schema,
-                projection,
-                ..
-            } => {
-                let (projection, projected_schema) =
-                    get_projected_schema(&schema, projection, accum, has_projection)?;
-                Ok(LogicalPlan::InMemoryScan {
-                    data: data.clone(),
-                    schema: schema.clone(),
-                    projection: Some(projection),
-                    projected_schema: Box::new(projected_schema),
-                })
-            }
-            LogicalPlan::CsvScan {
-                path,
-                has_header,
-                delimiter,
-                schema,
-                projection,
-                ..
-            } => {
-                let (projection, projected_schema) =
-                    get_projected_schema(&schema, projection, accum, has_projection)?;
-
-                Ok(LogicalPlan::CsvScan {
-                    path: path.to_owned(),
-                    has_header: *has_header,
-                    schema: schema.clone(),
-                    delimiter: *delimiter,
-                    projection: Some(projection),
-                    projected_schema: Box::new(projected_schema),
-                })
-            }
-            LogicalPlan::ParquetScan {
-                path,
-                schema,
-                projection,
-                ..
-            } => {
-                let (projection, projected_schema) =
-                    get_projected_schema(&schema, projection, accum, has_projection)?;
-
-                Ok(LogicalPlan::ParquetScan {
-                    path: path.to_owned(),
-                    schema: schema.clone(),
-                    projection: Some(projection),
-                    projected_schema: Box::new(projected_schema),
-                })
-            }
-            LogicalPlan::CreateExternalTable { .. } => Ok(plan.clone()),
-        }
-    }
 }
 
 fn get_projected_schema(
-    table_schema: &Schema,
+    schema: &Schema,
     projection: &Option<Vec<usize>>,
-    accum: &HashSet<String>,
+    required_columns: &HashSet<String>,
     has_projection: bool,
-) -> Result<(Vec<usize>, Schema)> {
+) -> Result<(Vec<usize>, SchemaRef)> {
     if projection.is_some() {
-        return Err(ExecutionError::General(
+        return Err(DataFusionError::Internal(
             "Cannot run projection push-down rule more than once".to_string(),
         ));
     }
@@ -207,9 +72,9 @@ fn get_projected_schema(
     //
     // we discard non-existing columns because some column names are not part of the schema,
     // e.g. when the column derives from an aggregation
-    let mut projection: Vec<usize> = accum
+    let mut projection: Vec<usize> = required_columns
         .iter()
-        .map(|name| table_schema.index_of(name))
+        .map(|name| schema.index_of(name))
         .filter_map(ArrowResult::ok)
         .collect();
 
@@ -220,7 +85,7 @@ fn get_projected_schema(
             projection.push(0);
         } else {
             // for table scan without projection, we default to return all columns
-            projection = table_schema
+            projection = schema
                 .fields()
                 .iter()
                 .enumerate()
@@ -235,18 +100,245 @@ fn get_projected_schema(
     // create the projected schema
     let mut projected_fields: Vec<Field> = Vec::with_capacity(projection.len());
     for i in &projection {
-        projected_fields.push(table_schema.fields()[*i].clone());
+        projected_fields.push(schema.fields()[*i].clone());
     }
 
-    Ok((projection, Schema::new(projected_fields)))
+    Ok((projection, SchemaRef::new(Schema::new(projected_fields))))
+}
+
+/// Recursively transverses the logical plan removing expressions and that are not needed.
+fn optimize_plan(
+    optimizer: &mut ProjectionPushDown,
+    plan: &LogicalPlan,
+    required_columns: &HashSet<String>, // set of columns required up to this step
+    has_projection: bool,
+) -> Result<LogicalPlan> {
+    match plan {
+        LogicalPlan::Projection {
+            input,
+            expr,
+            schema,
+        } => {
+            // projection:
+            // * remove any expression that is not required
+            // * construct the new set of required columns
+
+            let mut new_expr = Vec::new();
+            let mut new_fields = Vec::new();
+            let mut new_required_columns = HashSet::new();
+
+            // Gather all columns needed for expressions in this Projection
+            schema
+                .fields()
+                .iter()
+                .enumerate()
+                .map(|(i, field)| {
+                    if required_columns.contains(field.name()) {
+                        new_expr.push(expr[i].clone());
+                        new_fields.push(field.clone());
+
+                        // gather the new set of required columns
+                        utils::expr_to_column_names(&expr[i], &mut new_required_columns)
+                    } else {
+                        Ok(())
+                    }
+                })
+                .collect::<Result<()>>()?;
+
+            let new_input =
+                optimize_plan(optimizer, &input, &new_required_columns, true)?;
+            if new_fields.len() == 0 {
+                // no need for an expression at all
+                Ok(new_input)
+            } else {
+                Ok(LogicalPlan::Projection {
+                    expr: new_expr,
+                    input: Arc::new(new_input),
+                    schema: SchemaRef::new(Schema::new(new_fields)),
+                })
+            }
+        }
+        LogicalPlan::Aggregate {
+            schema,
+            input,
+            group_expr,
+            aggr_expr,
+            ..
+        } => {
+            // aggregate:
+            // * remove any aggregate expression that is not required
+            // * construct the new set of required columns
+
+            let mut new_required_columns = HashSet::new();
+            utils::exprlist_to_column_names(group_expr, &mut new_required_columns)?;
+
+            // Gather all columns needed for expressions in this Aggregate
+            let mut new_aggr_expr = Vec::new();
+            aggr_expr
+                .iter()
+                .map(|expr| {
+                    let name = &expr.name(&schema)?;
+
+                    if required_columns.contains(name) {
+                        new_aggr_expr.push(expr.clone());
+                        new_required_columns.insert(name.clone());
+
+                        // add to the new set of required columns
+                        utils::expr_to_column_names(expr, &mut new_required_columns)
+                    } else {
+                        Ok(())
+                    }
+                })
+                .collect::<Result<()>>()?;
+
+            let new_schema = Schema::new(
+                schema
+                    .fields()
+                    .iter()
+                    .filter(|x| new_required_columns.contains(x.name()))
+                    .cloned()
+                    .collect(),
+            );
+
+            Ok(LogicalPlan::Aggregate {
+                group_expr: group_expr.clone(),
+                aggr_expr: new_aggr_expr,
+                input: Arc::new(optimize_plan(
+                    optimizer,
+                    &input,
+                    &new_required_columns,
+                    true,
+                )?),
+                schema: SchemaRef::new(new_schema),
+            })
+        }
+        // scans:
+        // * remove un-used columns from the scan projection
+        LogicalPlan::TableScan {
+            schema_name,
+            source,
+            table_schema,
+            projection,
+            ..
+        } => {
+            let (projection, projected_schema) = get_projected_schema(
+                &table_schema,
+                projection,
+                required_columns,
+                has_projection,
+            )?;
+
+            // return the table scan with projection
+            Ok(LogicalPlan::TableScan {
+                schema_name: schema_name.to_string(),
+                source: source.clone(),
+                table_schema: table_schema.clone(),
+                projection: Some(projection),
+                projected_schema: projected_schema,
+            })
+        }
+        LogicalPlan::InMemoryScan {
+            data,
+            schema,
+            projection,
+            ..
+        } => {
+            let (projection, projected_schema) = get_projected_schema(
+                &schema,
+                projection,
+                required_columns,
+                has_projection,
+            )?;
+            Ok(LogicalPlan::InMemoryScan {
+                data: data.clone(),
+                schema: schema.clone(),
+                projection: Some(projection),
+                projected_schema: projected_schema,
+            })
+        }
+        LogicalPlan::CsvScan {
+            path,
+            has_header,
+            delimiter,
+            schema,
+            projection,
+            ..
+        } => {
+            let (projection, projected_schema) = get_projected_schema(
+                &schema,
+                projection,
+                required_columns,
+                has_projection,
+            )?;
+
+            Ok(LogicalPlan::CsvScan {
+                path: path.to_owned(),
+                has_header: *has_header,
+                schema: schema.clone(),
+                delimiter: *delimiter,
+                projection: Some(projection),
+                projected_schema: projected_schema,
+            })
+        }
+        LogicalPlan::ParquetScan {
+            path,
+            schema,
+            projection,
+            ..
+        } => {
+            let (projection, projected_schema) = get_projected_schema(
+                &schema,
+                projection,
+                required_columns,
+                has_projection,
+            )?;
+
+            Ok(LogicalPlan::ParquetScan {
+                path: path.to_owned(),
+                schema: schema.clone(),
+                projection: Some(projection),
+                projected_schema: projected_schema,
+            })
+        }
+        LogicalPlan::Explain {
+            verbose,
+            plan,
+            stringified_plans,
+            schema,
+        } => optimize_explain(optimizer, *verbose, &*plan, stringified_plans, &*schema),
+        // all other nodes: Add any additional columns used by
+        // expressions in this node to the list of required columns
+        LogicalPlan::Limit { .. }
+        | LogicalPlan::Filter { .. }
+        | LogicalPlan::EmptyRelation { .. }
+        | LogicalPlan::Sort { .. }
+        | LogicalPlan::CreateExternalTable { .. }
+        | LogicalPlan::Extension { .. } => {
+            let expr = utils::expressions(plan);
+            // collect all required columns by this plan
+            let mut new_required_columns = required_columns.clone();
+            utils::exprlist_to_column_names(&expr, &mut new_required_columns)?;
+
+            // apply the optimization to all inputs of the plan
+            let inputs = utils::inputs(plan);
+            let new_inputs = inputs
+                .iter()
+                .map(|plan| {
+                    optimize_plan(optimizer, plan, &new_required_columns, has_projection)
+                })
+                .collect::<Result<Vec<_>>>()?;
+
+            utils::from_plan(plan, &expr, &new_inputs)
+        }
+    }
 }
 
 #[cfg(test)]
 mod tests {
 
     use super::*;
-    use crate::logicalplan::{col, lit};
-    use crate::logicalplan::{Expr, LogicalPlanBuilder};
+    use crate::logical_plan::{col, lit};
+    use crate::logical_plan::{max, min, Expr, LogicalPlanBuilder};
     use crate::test::*;
     use arrow::datatypes::DataType;
 
@@ -283,7 +375,7 @@ mod tests {
     }
 
     #[test]
-    fn aggregate_no_group_by_with_selection() -> Result<()> {
+    fn aggregate_no_group_by_with_filter() -> Result<()> {
         let table_scan = test_table_scan()?;
 
         let plan = LogicalPlanBuilder::from(&table_scan)
@@ -292,7 +384,7 @@ mod tests {
             .build()?;
 
         let expected = "Aggregate: groupBy=[[]], aggr=[[MAX(#b)]]\
-        \n  Selection: #c\
+        \n  Filter: #c\
         \n    TableScan: test projection=Some([1, 2])";
 
         assert_optimized_plan_eq(&plan, expected);
@@ -383,6 +475,84 @@ mod tests {
         Ok(())
     }
 
+    /// tests that it removes unused columns in projections
+    #[test]
+    fn table_unused_column() -> Result<()> {
+        let table_scan = test_table_scan()?;
+        assert_eq!(3, table_scan.schema().fields().len());
+        assert_fields_eq(&table_scan, vec!["a", "b", "c"]);
+
+        // we never use "b" in the first projection => remove it
+        let plan = LogicalPlanBuilder::from(&table_scan)
+            .project(vec![col("c"), col("a"), col("b")])?
+            .filter(col("c").gt(lit(1)))?
+            .aggregate(vec![col("c")], vec![max(col("a"))])?
+            .build()?;
+
+        assert_fields_eq(&plan, vec!["c", "MAX(a)"]);
+
+        let expected = "\
+        Aggregate: groupBy=[[#c]], aggr=[[MAX(#a)]]\
+        \n  Filter: #c Gt Int32(1)\
+        \n    Projection: #c, #a\
+        \n      TableScan: test projection=Some([0, 2])";
+
+        assert_optimized_plan_eq(&plan, expected);
+
+        Ok(())
+    }
+
+    /// tests that it removes un-needed projections
+    #[test]
+    fn table_unused_projection() -> Result<()> {
+        let table_scan = test_table_scan()?;
+        assert_eq!(3, table_scan.schema().fields().len());
+        assert_fields_eq(&table_scan, vec!["a", "b", "c"]);
+
+        // there is no need for the first projection
+        let plan = LogicalPlanBuilder::from(&table_scan)
+            .project(vec![col("b")])?
+            .project(vec![lit(1).alias("a")])?
+            .build()?;
+
+        assert_fields_eq(&plan, vec!["a"]);
+
+        let expected = "\
+        Projection: Int32(1) AS a\
+        \n  TableScan: test projection=Some([0])";
+
+        assert_optimized_plan_eq(&plan, expected);
+
+        Ok(())
+    }
+
+    /// tests that it removes an aggregate is never used downstream
+    #[test]
+    fn table_unused_aggregate() -> Result<()> {
+        let table_scan = test_table_scan()?;
+        assert_eq!(3, table_scan.schema().fields().len());
+        assert_fields_eq(&table_scan, vec!["a", "b", "c"]);
+
+        // we never use "min(b)" => remove it
+        let plan = LogicalPlanBuilder::from(&table_scan)
+            .aggregate(vec![col("a"), col("c")], vec![max(col("b")), min(col("b"))])?
+            .filter(col("c").gt(lit(1)))?
+            .project(vec![col("c"), col("a"), col("MAX(b)")])?
+            .build()?;
+
+        assert_fields_eq(&plan, vec!["c", "a", "MAX(b)"]);
+
+        let expected = "\
+        Projection: #c, #a, #MAX(b)\
+        \n  Filter: #c Gt Int32(1)\
+        \n    Aggregate: groupBy=[[#a, #c]], aggr=[[MAX(#b)]]\
+        \n      TableScan: test projection=Some([0, 1, 2])";
+
+        assert_optimized_plan_eq(&plan, expected);
+
+        Ok(())
+    }
+
     fn assert_optimized_plan_eq(plan: &LogicalPlan, expected: &str) {
         let optimized_plan = optimize(plan).expect("failed to optimize plan");
         let formatted_plan = format!("{:?}", optimized_plan);
diff --git a/rust/datafusion/src/optimizer/type_coercion.rs b/rust/datafusion/src/optimizer/type_coercion.rs
deleted file mode 100644
index 78fcf52f1c6..00000000000
--- a/rust/datafusion/src/optimizer/type_coercion.rs
+++ /dev/null
@@ -1,286 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! The type_coercion optimizer rule ensures that all operators are operating on
-//! compatible types by adding explicit cast operations to expressions. For example,
-//! the operation `c_float + c_int` would be rewritten as `c_float + CAST(c_int AS
-//! float)`. This keeps the runtime query execution code much simpler.
-
-use std::collections::HashMap;
-
-use arrow::datatypes::Schema;
-
-use crate::error::{ExecutionError, Result};
-use crate::execution::physical_plan::udf::ScalarFunction;
-use crate::logicalplan::LogicalPlan;
-use crate::logicalplan::{Expr, LogicalPlanBuilder};
-use crate::optimizer::optimizer::OptimizerRule;
-use crate::optimizer::utils;
-
-/// Implementation of type coercion optimizer rule
-pub struct TypeCoercionRule<'a> {
-    scalar_functions: &'a HashMap<String, Box<ScalarFunction>>,
-}
-
-impl<'a> TypeCoercionRule<'a> {
-    /// Create a new type coercion optimizer rule using meta-data about registered
-    /// scalar functions
-    pub fn new(scalar_functions: &'a HashMap<String, Box<ScalarFunction>>) -> Self {
-        Self { scalar_functions }
-    }
-
-    /// Rewrite an expression list to include explicit CAST operations when required
-    fn rewrite_expr_list(&self, expr: &[Expr], schema: &Schema) -> Result<Vec<Expr>> {
-        Ok(expr
-            .iter()
-            .map(|e| self.rewrite_expr(e, schema))
-            .collect::<Result<Vec<_>>>()?)
-    }
-
-    /// Rewrite an expression to include explicit CAST operations when required
-    fn rewrite_expr(&self, expr: &Expr, schema: &Schema) -> Result<Expr> {
-        match expr {
-            Expr::BinaryExpr { left, op, right } => {
-                let left = self.rewrite_expr(left, schema)?;
-                let right = self.rewrite_expr(right, schema)?;
-                let left_type = left.get_type(schema)?;
-                let right_type = right.get_type(schema)?;
-                if left_type == right_type {
-                    Ok(Expr::BinaryExpr {
-                        left: Box::new(left),
-                        op: op.clone(),
-                        right: Box::new(right),
-                    })
-                } else {
-                    let super_type = utils::get_supertype(&left_type, &right_type)?;
-                    Ok(Expr::BinaryExpr {
-                        left: Box::new(left.cast_to(&super_type, schema)?),
-                        op: op.clone(),
-                        right: Box::new(right.cast_to(&super_type, schema)?),
-                    })
-                }
-            }
-            Expr::IsNull(e) => Ok(Expr::IsNull(Box::new(self.rewrite_expr(e, schema)?))),
-            Expr::IsNotNull(e) => {
-                Ok(Expr::IsNotNull(Box::new(self.rewrite_expr(e, schema)?)))
-            }
-            Expr::ScalarFunction {
-                name,
-                args,
-                return_type,
-            } => {
-                // cast the inputs of scalar functions to the appropriate type where possible
-                match self.scalar_functions.get(name) {
-                    Some(func_meta) => {
-                        let mut func_args = Vec::with_capacity(args.len());
-                        for i in 0..args.len() {
-                            let field = &func_meta.args[i];
-                            let expr = self.rewrite_expr(&args[i], schema)?;
-                            let actual_type = expr.get_type(schema)?;
-                            let required_type = field.data_type();
-                            if &actual_type == required_type {
-                                func_args.push(expr)
-                            } else {
-                                let super_type =
-                                    utils::get_supertype(&actual_type, required_type)?;
-                                func_args.push(expr.cast_to(&super_type, schema)?);
-                            }
-                        }
-
-                        Ok(Expr::ScalarFunction {
-                            name: name.clone(),
-                            args: func_args,
-                            return_type: return_type.clone(),
-                        })
-                    }
-                    _ => Err(ExecutionError::General(format!(
-                        "Invalid scalar function {}",
-                        name
-                    ))),
-                }
-            }
-            Expr::AggregateFunction {
-                name,
-                args,
-                return_type,
-            } => Ok(Expr::AggregateFunction {
-                name: name.clone(),
-                args: args
-                    .iter()
-                    .map(|a| self.rewrite_expr(a, schema))
-                    .collect::<Result<Vec<_>>>()?,
-                return_type: return_type.clone(),
-            }),
-            Expr::Cast { .. } => Ok(expr.clone()),
-            Expr::Column(_) => Ok(expr.clone()),
-            Expr::Alias(expr, alias) => Ok(Expr::Alias(
-                Box::new(self.rewrite_expr(expr, schema)?),
-                alias.to_owned(),
-            )),
-            Expr::Literal(_) => Ok(expr.clone()),
-            Expr::Not(_) => Ok(expr.clone()),
-            Expr::Sort { .. } => Ok(expr.clone()),
-            Expr::Wildcard { .. } => Err(ExecutionError::General(
-                "Wildcard expressions are not valid in a logical query plan".to_owned(),
-            )),
-            Expr::Nested(e) => self.rewrite_expr(e, schema),
-        }
-    }
-}
-
-impl<'a> OptimizerRule for TypeCoercionRule<'a> {
-    fn optimize(&mut self, plan: &LogicalPlan) -> Result<LogicalPlan> {
-        match plan {
-            LogicalPlan::Projection { expr, input, .. } => {
-                LogicalPlanBuilder::from(&self.optimize(input)?)
-                    .project(self.rewrite_expr_list(expr, input.schema())?)?
-                    .build()
-            }
-            LogicalPlan::Selection { expr, input, .. } => {
-                LogicalPlanBuilder::from(&self.optimize(input)?)
-                    .filter(self.rewrite_expr(expr, input.schema())?)?
-                    .build()
-            }
-            LogicalPlan::Aggregate {
-                input,
-                group_expr,
-                aggr_expr,
-                ..
-            } => LogicalPlanBuilder::from(&self.optimize(input)?)
-                .aggregate(
-                    self.rewrite_expr_list(group_expr, input.schema())?,
-                    self.rewrite_expr_list(aggr_expr, input.schema())?,
-                )?
-                .build(),
-            LogicalPlan::TableScan { .. } => Ok(plan.clone()),
-            LogicalPlan::InMemoryScan { .. } => Ok(plan.clone()),
-            LogicalPlan::ParquetScan { .. } => Ok(plan.clone()),
-            LogicalPlan::CsvScan { .. } => Ok(plan.clone()),
-            LogicalPlan::EmptyRelation { .. } => Ok(plan.clone()),
-            LogicalPlan::Limit { .. } => Ok(plan.clone()),
-            LogicalPlan::Sort { .. } => Ok(plan.clone()),
-            LogicalPlan::CreateExternalTable { .. } => Ok(plan.clone()),
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::execution::context::ExecutionContext;
-    use crate::execution::physical_plan::csv::CsvReadOptions;
-    use crate::logicalplan::{col, Operator};
-    use crate::test::arrow_testdata_path;
-    use arrow::datatypes::{DataType, Field, Schema};
-
-    #[test]
-    fn test_with_csv_plan() -> Result<()> {
-        let testdata = arrow_testdata_path();
-        let path = format!("{}/csv/aggregate_test_100.csv", testdata);
-
-        let options = CsvReadOptions::new().schema_infer_max_records(100);
-        let plan = LogicalPlanBuilder::scan_csv(&path, options, None)?
-            .filter(col("c7").lt(&col("c12")))?
-            .build()?;
-
-        let scalar_functions = HashMap::new();
-        let mut rule = TypeCoercionRule::new(&scalar_functions);
-        let plan = rule.optimize(&plan)?;
-
-        assert!(
-            format!("{:?}", plan).starts_with("Selection: CAST(#c7 AS Float64) Lt #c12")
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn test_add_i32_i64() {
-        binary_cast_test(
-            DataType::Int32,
-            DataType::Int64,
-            "CAST(#c0 AS Int64) Plus #c1",
-        );
-        binary_cast_test(
-            DataType::Int64,
-            DataType::Int32,
-            "#c0 Plus CAST(#c1 AS Int64)",
-        );
-    }
-
-    #[test]
-    fn test_add_f32_f64() {
-        binary_cast_test(
-            DataType::Float32,
-            DataType::Float64,
-            "CAST(#c0 AS Float64) Plus #c1",
-        );
-        binary_cast_test(
-            DataType::Float64,
-            DataType::Float32,
-            "#c0 Plus CAST(#c1 AS Float64)",
-        );
-    }
-
-    #[test]
-    fn test_add_i32_f32() {
-        binary_cast_test(
-            DataType::Int32,
-            DataType::Float32,
-            "CAST(#c0 AS Float32) Plus #c1",
-        );
-        binary_cast_test(
-            DataType::Float32,
-            DataType::Int32,
-            "#c0 Plus CAST(#c1 AS Float32)",
-        );
-    }
-
-    #[test]
-    fn test_add_u32_i64() {
-        binary_cast_test(
-            DataType::UInt32,
-            DataType::Int64,
-            "CAST(#c0 AS Int64) Plus #c1",
-        );
-        binary_cast_test(
-            DataType::Int64,
-            DataType::UInt32,
-            "#c0 Plus CAST(#c1 AS Int64)",
-        );
-    }
-
-    fn binary_cast_test(left_type: DataType, right_type: DataType, expected: &str) {
-        let schema = Schema::new(vec![
-            Field::new("c0", left_type, true),
-            Field::new("c1", right_type, true),
-        ]);
-
-        let expr = Expr::BinaryExpr {
-            left: Box::new(col("c0")),
-            op: Operator::Plus,
-            right: Box::new(col("c1")),
-        };
-
-        let ctx = ExecutionContext::new();
-        let rule = TypeCoercionRule::new(ctx.scalar_functions());
-
-        let expr2 = rule.rewrite_expr(&expr, &schema).unwrap();
-
-        assert_eq!(expected, format!("{:?}", expr2));
-    }
-}
diff --git a/rust/datafusion/src/optimizer/utils.rs b/rust/datafusion/src/optimizer/utils.rs
index 6c9d409d344..3b83fcbd939 100644
--- a/rust/datafusion/src/optimizer/utils.rs
+++ b/rust/datafusion/src/optimizer/utils.rs
@@ -17,12 +17,13 @@
 
 //! Collection of utility functions that are leveraged by the query optimizer rules
 
-use std::collections::HashSet;
+use std::{collections::HashSet, sync::Arc};
 
-use arrow::datatypes::DataType;
+use arrow::datatypes::{Schema, SchemaRef};
 
-use crate::error::{ExecutionError, Result};
-use crate::logicalplan::Expr;
+use super::optimizer::OptimizerRule;
+use crate::error::{DataFusionError, Result};
+use crate::logical_plan::{Expr, LogicalPlan, PlanType, StringifiedPlan};
 
 /// Recursively walk a list of expression trees, collecting the unique set of column
 /// names referenced in the expression
@@ -45,6 +46,10 @@ pub fn expr_to_column_names(expr: &Expr, accum: &mut HashSet<String>) -> Result<
             accum.insert(name.clone());
             Ok(())
         }
+        Expr::ScalarVariable(var_names) => {
+            accum.insert(var_names.join("."));
+            Ok(())
+        }
         Expr::Literal(_) => {
             // not needed
             Ok(())
@@ -60,133 +65,220 @@ pub fn expr_to_column_names(expr: &Expr, accum: &mut HashSet<String>) -> Result<
         Expr::Cast { expr, .. } => expr_to_column_names(expr, accum),
         Expr::Sort { expr, .. } => expr_to_column_names(expr, accum),
         Expr::AggregateFunction { args, .. } => exprlist_to_column_names(args, accum),
+        Expr::AggregateUDF { args, .. } => exprlist_to_column_names(args, accum),
         Expr::ScalarFunction { args, .. } => exprlist_to_column_names(args, accum),
-        Expr::Wildcard => Err(ExecutionError::General(
+        Expr::ScalarUDF { args, .. } => exprlist_to_column_names(args, accum),
+        Expr::Wildcard => Err(DataFusionError::Internal(
             "Wildcard expressions are not valid in a logical query plan".to_owned(),
         )),
         Expr::Nested(e) => expr_to_column_names(e, accum),
     }
 }
 
-/// Given two datatypes, determine the supertype that both types can safely be cast to
-pub fn get_supertype(l: &DataType, r: &DataType) -> Result<DataType> {
-    match _get_supertype(l, r) {
-        Some(dt) => Ok(dt),
-        None => _get_supertype(r, l).ok_or_else(|| {
-            ExecutionError::InternalError(format!(
-                "Failed to determine supertype of {:?} and {:?}",
-                l, r
-            ))
+/// Create a `LogicalPlan::Explain` node by running `optimizer` on the
+/// input plan and capturing the resulting plan string
+pub fn optimize_explain(
+    optimizer: &mut impl OptimizerRule,
+    verbose: bool,
+    plan: &LogicalPlan,
+    stringified_plans: &Vec<StringifiedPlan>,
+    schema: &Schema,
+) -> Result<LogicalPlan> {
+    // These are the fields of LogicalPlan::Explain It might be nice
+    // to transform that enum Variant into its own struct and avoid
+    // passing the fields individually
+    let plan = Arc::new(optimizer.optimize(plan)?);
+    let mut stringified_plans = stringified_plans.clone();
+    let optimizer_name = optimizer.name().into();
+    stringified_plans.push(StringifiedPlan::new(
+        PlanType::OptimizedLogicalPlan { optimizer_name },
+        format!("{:#?}", plan),
+    ));
+    let schema = SchemaRef::new(schema.clone());
+
+    Ok(LogicalPlan::Explain {
+        verbose,
+        plan,
+        stringified_plans,
+        schema,
+    })
+}
+
+/// returns all expressions (non-recursively) in the current logical plan node.
+pub fn expressions(plan: &LogicalPlan) -> Vec<Expr> {
+    match plan {
+        LogicalPlan::Projection { expr, .. } => expr.clone(),
+        LogicalPlan::Filter { predicate, .. } => vec![predicate.clone()],
+        LogicalPlan::Aggregate {
+            group_expr,
+            aggr_expr,
+            ..
+        } => {
+            let mut result = group_expr.clone();
+            result.extend(aggr_expr.clone());
+            result
+        }
+        LogicalPlan::Sort { expr, .. } => expr.clone(),
+        LogicalPlan::Extension { node } => node.expressions(),
+        // plans without expressions
+        LogicalPlan::TableScan { .. }
+        | LogicalPlan::InMemoryScan { .. }
+        | LogicalPlan::ParquetScan { .. }
+        | LogicalPlan::CsvScan { .. }
+        | LogicalPlan::EmptyRelation { .. }
+        | LogicalPlan::Limit { .. }
+        | LogicalPlan::CreateExternalTable { .. }
+        | LogicalPlan::Explain { .. } => vec![],
+    }
+}
+
+/// returns all inputs in the logical plan
+pub fn inputs(plan: &LogicalPlan) -> Vec<&LogicalPlan> {
+    match plan {
+        LogicalPlan::Projection { input, .. } => vec![input],
+        LogicalPlan::Filter { input, .. } => vec![input],
+        LogicalPlan::Aggregate { input, .. } => vec![input],
+        LogicalPlan::Sort { input, .. } => vec![input],
+        LogicalPlan::Limit { input, .. } => vec![input],
+        LogicalPlan::Extension { node } => node.inputs(),
+        // plans without inputs
+        LogicalPlan::TableScan { .. }
+        | LogicalPlan::InMemoryScan { .. }
+        | LogicalPlan::ParquetScan { .. }
+        | LogicalPlan::CsvScan { .. }
+        | LogicalPlan::EmptyRelation { .. }
+        | LogicalPlan::CreateExternalTable { .. }
+        | LogicalPlan::Explain { .. } => vec![],
+    }
+}
+
+/// Returns a new logical plan based on the original one with inputs and expressions replaced
+pub fn from_plan(
+    plan: &LogicalPlan,
+    expr: &Vec<Expr>,
+    inputs: &Vec<LogicalPlan>,
+) -> Result<LogicalPlan> {
+    match plan {
+        LogicalPlan::Projection { schema, .. } => Ok(LogicalPlan::Projection {
+            expr: expr.clone(),
+            input: Arc::new(inputs[0].clone()),
+            schema: schema.clone(),
+        }),
+        LogicalPlan::Filter { .. } => Ok(LogicalPlan::Filter {
+            predicate: expr[0].clone(),
+            input: Arc::new(inputs[0].clone()),
         }),
+        LogicalPlan::Aggregate {
+            group_expr, schema, ..
+        } => Ok(LogicalPlan::Aggregate {
+            group_expr: expr[0..group_expr.len()].to_vec(),
+            aggr_expr: expr[group_expr.len()..].to_vec(),
+            input: Arc::new(inputs[0].clone()),
+            schema: schema.clone(),
+        }),
+        LogicalPlan::Sort { .. } => Ok(LogicalPlan::Sort {
+            expr: expr.clone(),
+            input: Arc::new(inputs[0].clone()),
+        }),
+        LogicalPlan::Limit { n, .. } => Ok(LogicalPlan::Limit {
+            n: *n,
+            input: Arc::new(inputs[0].clone()),
+        }),
+        LogicalPlan::Extension { node } => Ok(LogicalPlan::Extension {
+            node: node.from_template(expr, inputs),
+        }),
+        LogicalPlan::EmptyRelation { .. }
+        | LogicalPlan::TableScan { .. }
+        | LogicalPlan::InMemoryScan { .. }
+        | LogicalPlan::ParquetScan { .. }
+        | LogicalPlan::CsvScan { .. }
+        | LogicalPlan::CreateExternalTable { .. }
+        | LogicalPlan::Explain { .. } => Ok(plan.clone()),
     }
 }
 
-/// Given two datatypes, determine the supertype that both types can safely be cast to
-fn _get_supertype(l: &DataType, r: &DataType) -> Option<DataType> {
-    use arrow::datatypes::DataType::*;
-    match (l, r) {
-        (UInt8, Int8) => Some(Int8),
-        (UInt8, Int16) => Some(Int16),
-        (UInt8, Int32) => Some(Int32),
-        (UInt8, Int64) => Some(Int64),
-
-        (UInt16, Int16) => Some(Int16),
-        (UInt16, Int32) => Some(Int32),
-        (UInt16, Int64) => Some(Int64),
-
-        (UInt32, Int32) => Some(Int32),
-        (UInt32, Int64) => Some(Int64),
-
-        (UInt64, Int64) => Some(Int64),
-
-        (Int8, UInt8) => Some(Int8),
-
-        (Int16, UInt8) => Some(Int16),
-        (Int16, UInt16) => Some(Int16),
-
-        (Int32, UInt8) => Some(Int32),
-        (Int32, UInt16) => Some(Int32),
-        (Int32, UInt32) => Some(Int32),
-
-        (Int64, UInt8) => Some(Int64),
-        (Int64, UInt16) => Some(Int64),
-        (Int64, UInt32) => Some(Int64),
-        (Int64, UInt64) => Some(Int64),
-
-        (UInt8, UInt8) => Some(UInt8),
-        (UInt8, UInt16) => Some(UInt16),
-        (UInt8, UInt32) => Some(UInt32),
-        (UInt8, UInt64) => Some(UInt64),
-        (UInt8, Float32) => Some(Float32),
-        (UInt8, Float64) => Some(Float64),
-
-        (UInt16, UInt8) => Some(UInt16),
-        (UInt16, UInt16) => Some(UInt16),
-        (UInt16, UInt32) => Some(UInt32),
-        (UInt16, UInt64) => Some(UInt64),
-        (UInt16, Float32) => Some(Float32),
-        (UInt16, Float64) => Some(Float64),
-
-        (UInt32, UInt8) => Some(UInt32),
-        (UInt32, UInt16) => Some(UInt32),
-        (UInt32, UInt32) => Some(UInt32),
-        (UInt32, UInt64) => Some(UInt64),
-        (UInt32, Float32) => Some(Float32),
-        (UInt32, Float64) => Some(Float64),
-
-        (UInt64, UInt8) => Some(UInt64),
-        (UInt64, UInt16) => Some(UInt64),
-        (UInt64, UInt32) => Some(UInt64),
-        (UInt64, UInt64) => Some(UInt64),
-        (UInt64, Float32) => Some(Float32),
-        (UInt64, Float64) => Some(Float64),
-
-        (Int8, Int8) => Some(Int8),
-        (Int8, Int16) => Some(Int16),
-        (Int8, Int32) => Some(Int32),
-        (Int8, Int64) => Some(Int64),
-        (Int8, Float32) => Some(Float32),
-        (Int8, Float64) => Some(Float64),
-
-        (Int16, Int8) => Some(Int16),
-        (Int16, Int16) => Some(Int16),
-        (Int16, Int32) => Some(Int32),
-        (Int16, Int64) => Some(Int64),
-        (Int16, Float32) => Some(Float32),
-        (Int16, Float64) => Some(Float64),
-
-        (Int32, Int8) => Some(Int32),
-        (Int32, Int16) => Some(Int32),
-        (Int32, Int32) => Some(Int32),
-        (Int32, Int64) => Some(Int64),
-        (Int32, Float32) => Some(Float32),
-        (Int32, Float64) => Some(Float64),
-
-        (Int64, Int8) => Some(Int64),
-        (Int64, Int16) => Some(Int64),
-        (Int64, Int32) => Some(Int64),
-        (Int64, Int64) => Some(Int64),
-        (Int64, Float32) => Some(Float32),
-        (Int64, Float64) => Some(Float64),
-
-        (Float32, Float32) => Some(Float32),
-        (Float32, Float64) => Some(Float64),
-        (Float64, Float32) => Some(Float64),
-        (Float64, Float64) => Some(Float64),
-
-        (Utf8, _) => Some(Utf8),
-        (_, Utf8) => Some(Utf8),
-
-        (Boolean, Boolean) => Some(Boolean),
-
-        _ => None,
+/// Returns all direct children `Expression`s of `expr`.
+/// E.g. if the expression is "(a + 1) + 1", it returns ["a + 1", "1"] (as Expr objects)
+pub fn expr_sub_expressions(expr: &Expr) -> Result<Vec<&Expr>> {
+    match expr {
+        Expr::BinaryExpr { left, right, .. } => Ok(vec![left, right]),
+        Expr::IsNull(e) => Ok(vec![e]),
+        Expr::IsNotNull(e) => Ok(vec![e]),
+        Expr::ScalarFunction { args, .. } => Ok(args.iter().collect()),
+        Expr::ScalarUDF { args, .. } => Ok(args.iter().collect()),
+        Expr::AggregateFunction { args, .. } => Ok(args.iter().collect()),
+        Expr::AggregateUDF { args, .. } => Ok(args.iter().collect()),
+        Expr::Cast { expr, .. } => Ok(vec![expr]),
+        Expr::Column(_) => Ok(vec![]),
+        Expr::Alias(expr, ..) => Ok(vec![expr]),
+        Expr::Literal(_) => Ok(vec![]),
+        Expr::ScalarVariable(_) => Ok(vec![]),
+        Expr::Not(expr) => Ok(vec![expr]),
+        Expr::Sort { expr, .. } => Ok(vec![expr]),
+        Expr::Wildcard { .. } => Err(DataFusionError::Internal(
+            "Wildcard expressions are not valid in a logical query plan".to_owned(),
+        )),
+        Expr::Nested(expr) => Ok(vec![expr]),
+    }
+}
+
+/// returns a new expression where the expressions in expr are replaced by the ones in `expr`.
+/// This is used in conjunction with ``expr_expressions`` to re-write expressions.
+pub fn rewrite_expression(expr: &Expr, expressions: &Vec<Expr>) -> Result<Expr> {
+    match expr {
+        Expr::BinaryExpr { op, .. } => Ok(Expr::BinaryExpr {
+            left: Box::new(expressions[0].clone()),
+            op: op.clone(),
+            right: Box::new(expressions[1].clone()),
+        }),
+        Expr::IsNull(_) => Ok(Expr::IsNull(Box::new(expressions[0].clone()))),
+        Expr::IsNotNull(_) => Ok(Expr::IsNotNull(Box::new(expressions[0].clone()))),
+        Expr::ScalarFunction { fun, .. } => Ok(Expr::ScalarFunction {
+            fun: fun.clone(),
+            args: expressions.clone(),
+        }),
+        Expr::ScalarUDF { fun, .. } => Ok(Expr::ScalarUDF {
+            fun: fun.clone(),
+            args: expressions.clone(),
+        }),
+        Expr::AggregateFunction { fun, distinct, .. } => Ok(Expr::AggregateFunction {
+            fun: fun.clone(),
+            args: expressions.clone(),
+            distinct: *distinct,
+        }),
+        Expr::AggregateUDF { fun, .. } => Ok(Expr::AggregateUDF {
+            fun: fun.clone(),
+            args: expressions.clone(),
+        }),
+        Expr::Cast { data_type, .. } => Ok(Expr::Cast {
+            expr: Box::new(expressions[0].clone()),
+            data_type: data_type.clone(),
+        }),
+        Expr::Alias(_, alias) => {
+            Ok(Expr::Alias(Box::new(expressions[0].clone()), alias.clone()))
+        }
+        Expr::Not(_) => Ok(Expr::Not(Box::new(expressions[0].clone()))),
+        Expr::Column(_) => Ok(expr.clone()),
+        Expr::Literal(_) => Ok(expr.clone()),
+        Expr::ScalarVariable(_) => Ok(expr.clone()),
+        Expr::Sort {
+            asc, nulls_first, ..
+        } => Ok(Expr::Sort {
+            expr: Box::new(expressions[0].clone()),
+            asc: asc.clone(),
+            nulls_first: nulls_first.clone(),
+        }),
+        Expr::Wildcard { .. } => Err(DataFusionError::Internal(
+            "Wildcard expressions are not valid in a logical query plan".to_owned(),
+        )),
+        Expr::Nested(_) => Ok(Expr::Nested(Box::new(expressions[0].clone()))),
     }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::logicalplan::col;
+    use crate::logical_plan::{col, LogicalPlanBuilder};
     use arrow::datatypes::DataType;
     use std::collections::HashSet;
 
@@ -211,4 +303,60 @@ mod tests {
         assert!(accum.contains("a"));
         Ok(())
     }
+
+    struct TestOptimizer {}
+
+    impl OptimizerRule for TestOptimizer {
+        fn optimize(&mut self, plan: &LogicalPlan) -> Result<LogicalPlan> {
+            Ok(plan.clone())
+        }
+
+        fn name(&self) -> &str {
+            return "test_optimizer";
+        }
+    }
+
+    #[test]
+    fn test_optimize_explain() -> Result<()> {
+        let mut optimizer = TestOptimizer {};
+
+        let empty_plan = LogicalPlanBuilder::empty().build()?;
+        let schema = LogicalPlan::explain_schema();
+
+        let optimized_explain = optimize_explain(
+            &mut optimizer,
+            true,
+            &empty_plan,
+            &vec![StringifiedPlan::new(PlanType::LogicalPlan, "...")],
+            &*schema,
+        )?;
+
+        match &optimized_explain {
+            LogicalPlan::Explain {
+                verbose,
+                stringified_plans,
+                ..
+            } => {
+                assert_eq!(*verbose, true);
+
+                let expected_stringified_plans = vec![
+                    StringifiedPlan::new(PlanType::LogicalPlan, "..."),
+                    StringifiedPlan::new(
+                        PlanType::OptimizedLogicalPlan {
+                            optimizer_name: "test_optimizer".into(),
+                        },
+                        "EmptyRelation",
+                    ),
+                ];
+                assert_eq!(*stringified_plans, expected_stringified_plans);
+            }
+            _ => assert!(
+                false,
+                "Expected explain plan but got {:?}",
+                optimized_explain
+            ),
+        }
+
+        Ok(())
+    }
 }
diff --git a/rust/datafusion/src/physical_plan/aggregates.rs b/rust/datafusion/src/physical_plan/aggregates.rs
new file mode 100644
index 00000000000..eba8c7552a4
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/aggregates.rs
@@ -0,0 +1,250 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Declaration of built-in (aggregate) functions.
+//! This module contains built-in aggregates' enumeration and metadata.
+//!
+//! Generally, an aggregate has:
+//! * a signature
+//! * a return type, that is a function of the incoming argument's types
+//! * the computation, that must accept each valid signature
+//!
+//! * Signature: see `Signature`
+//! * Return type: a function `(arg_types) -> return_type`. E.g. for min, ([f32]) -> f32, ([f64]) -> f64.
+
+use super::{
+    functions::Signature,
+    type_coercion::{coerce, data_types},
+    Accumulator, AggregateExpr, PhysicalExpr,
+};
+use crate::error::{DataFusionError, Result};
+use crate::physical_plan::distinct_expressions;
+use crate::physical_plan::expressions;
+use arrow::datatypes::{DataType, Schema};
+use expressions::{avg_return_type, sum_return_type};
+use std::{fmt, str::FromStr, sync::Arc};
+
+/// the implementation of an aggregate function
+pub type AccumulatorFunctionImplementation =
+    Arc<dyn Fn() -> Result<Box<dyn Accumulator>> + Send + Sync>;
+
+/// This signature corresponds to which types an aggregator serializes
+/// its state, given its return datatype.
+pub type StateTypeFunction =
+    Arc<dyn Fn(&DataType) -> Result<Arc<Vec<DataType>>> + Send + Sync>;
+
+/// Enum of all built-in scalar functions
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum AggregateFunction {
+    /// count
+    Count,
+    /// sum
+    Sum,
+    /// min
+    Min,
+    /// max
+    Max,
+    /// avg
+    Avg,
+}
+
+impl fmt::Display for AggregateFunction {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        // uppercase of the debug.
+        write!(f, "{}", format!("{:?}", self).to_uppercase())
+    }
+}
+
+impl FromStr for AggregateFunction {
+    type Err = DataFusionError;
+    fn from_str(name: &str) -> Result<AggregateFunction> {
+        Ok(match &*name.to_uppercase() {
+            "MIN" => AggregateFunction::Min,
+            "MAX" => AggregateFunction::Max,
+            "COUNT" => AggregateFunction::Count,
+            "AVG" => AggregateFunction::Avg,
+            "SUM" => AggregateFunction::Sum,
+            _ => {
+                return Err(DataFusionError::Plan(format!(
+                    "There is no built-in function named {}",
+                    name
+                )))
+            }
+        })
+    }
+}
+
+/// Returns the datatype of the scalar function
+pub fn return_type(
+    fun: &AggregateFunction,
+    arg_types: &Vec<DataType>,
+) -> Result<DataType> {
+    // Note that this function *must* return the same type that the respective physical expression returns
+    // or the execution panics.
+
+    // verify that this is a valid set of data types for this function
+    data_types(arg_types, &signature(fun))?;
+
+    match fun {
+        AggregateFunction::Count => Ok(DataType::UInt64),
+        AggregateFunction::Max | AggregateFunction::Min => Ok(arg_types[0].clone()),
+        AggregateFunction::Sum => sum_return_type(&arg_types[0]),
+        AggregateFunction::Avg => avg_return_type(&arg_types[0]),
+    }
+}
+
+/// Create a physical (function) expression.
+/// This function errors when `args`' can't be coerced to a valid argument type of the function.
+pub fn create_aggregate_expr(
+    fun: &AggregateFunction,
+    distinct: bool,
+    args: &Vec<Arc<dyn PhysicalExpr>>,
+    input_schema: &Schema,
+    name: String,
+) -> Result<Arc<dyn AggregateExpr>> {
+    // coerce
+    let arg = coerce(args, input_schema, &signature(fun))?[0].clone();
+
+    let arg_types = args
+        .iter()
+        .map(|e| e.data_type(input_schema))
+        .collect::<Result<Vec<_>>>()?;
+
+    let return_type = return_type(&fun, &arg_types)?;
+
+    Ok(match (fun, distinct) {
+        (AggregateFunction::Count, false) => {
+            Arc::new(expressions::Count::new(arg, name, return_type))
+        }
+        (AggregateFunction::Count, true) => {
+            Arc::new(distinct_expressions::DistinctCount::new(
+                arg_types,
+                args.clone(),
+                name,
+                return_type,
+            ))
+        }
+        (AggregateFunction::Sum, false) => {
+            Arc::new(expressions::Sum::new(arg, name, return_type))
+        }
+        (AggregateFunction::Sum, true) => {
+            return Err(DataFusionError::NotImplemented(
+                "SUM(DISTINCT) aggregations are not available".to_string(),
+            ));
+        }
+        (AggregateFunction::Min, _) => {
+            Arc::new(expressions::Min::new(arg, name, return_type))
+        }
+        (AggregateFunction::Max, _) => {
+            Arc::new(expressions::Max::new(arg, name, return_type))
+        }
+        (AggregateFunction::Avg, false) => {
+            Arc::new(expressions::Avg::new(arg, name, return_type))
+        }
+        (AggregateFunction::Avg, true) => {
+            return Err(DataFusionError::NotImplemented(
+                "AVG(DISTINCT) aggregations are not available".to_string(),
+            ));
+        }
+    })
+}
+
+static NUMERICS: &'static [DataType] = &[
+    DataType::Int8,
+    DataType::Int16,
+    DataType::Int32,
+    DataType::Int64,
+    DataType::UInt8,
+    DataType::UInt16,
+    DataType::UInt32,
+    DataType::UInt64,
+    DataType::Float32,
+    DataType::Float64,
+];
+
+/// the signatures supported by the function `fun`.
+fn signature(fun: &AggregateFunction) -> Signature {
+    // note: the physical expression must accept the type returned by this function or the execution panics.
+    match fun {
+        AggregateFunction::Count => Signature::Any(1),
+        AggregateFunction::Min | AggregateFunction::Max => {
+            let mut valid = vec![DataType::Utf8, DataType::LargeUtf8];
+            valid.extend_from_slice(NUMERICS);
+            Signature::Uniform(1, valid)
+        }
+        AggregateFunction::Avg | AggregateFunction::Sum => {
+            Signature::Uniform(1, NUMERICS.to_vec())
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::error::Result;
+
+    #[test]
+    fn test_min_max() -> Result<()> {
+        let observed = return_type(&AggregateFunction::Min, &vec![DataType::Utf8])?;
+        assert_eq!(DataType::Utf8, observed);
+
+        let observed = return_type(&AggregateFunction::Max, &vec![DataType::Int32])?;
+        assert_eq!(DataType::Int32, observed);
+        Ok(())
+    }
+
+    #[test]
+    fn test_sum_no_utf8() -> Result<()> {
+        let observed = return_type(&AggregateFunction::Sum, &vec![DataType::Utf8]);
+        assert!(observed.is_err());
+        Ok(())
+    }
+
+    #[test]
+    fn test_sum_upcasts() -> Result<()> {
+        let observed = return_type(&AggregateFunction::Sum, &vec![DataType::UInt32])?;
+        assert_eq!(DataType::UInt64, observed);
+        Ok(())
+    }
+
+    #[test]
+    fn test_count_return_type() -> Result<()> {
+        let observed = return_type(&AggregateFunction::Count, &vec![DataType::Utf8])?;
+        assert_eq!(DataType::UInt64, observed);
+
+        let observed = return_type(&AggregateFunction::Count, &vec![DataType::Int8])?;
+        assert_eq!(DataType::UInt64, observed);
+        Ok(())
+    }
+
+    #[test]
+    fn test_avg_return_type() -> Result<()> {
+        let observed = return_type(&AggregateFunction::Avg, &vec![DataType::Float32])?;
+        assert_eq!(DataType::Float64, observed);
+
+        let observed = return_type(&AggregateFunction::Avg, &vec![DataType::Float64])?;
+        assert_eq!(DataType::Float64, observed);
+        Ok(())
+    }
+
+    #[test]
+    fn test_avg_no_utf8() -> Result<()> {
+        let observed = return_type(&AggregateFunction::Avg, &vec![DataType::Utf8]);
+        assert!(observed.is_err());
+        Ok(())
+    }
+}
diff --git a/rust/datafusion/src/physical_plan/array_expressions.rs b/rust/datafusion/src/physical_plan/array_expressions.rs
new file mode 100644
index 00000000000..528cbb3da8d
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/array_expressions.rs
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Array expressions
+
+use crate::error::{DataFusionError, Result};
+use arrow::array::*;
+use arrow::datatypes::DataType;
+use std::sync::Arc;
+
+macro_rules! downcast_vec {
+    ($ARGS:expr, $ARRAY_TYPE:ident) => {{
+        $ARGS
+            .iter()
+            .map(|e| match e.as_any().downcast_ref::<$ARRAY_TYPE>() {
+                Some(array) => Ok(array),
+                _ => Err(DataFusionError::Internal("failed to downcast".to_string())),
+            })
+    }};
+}
+
+macro_rules! array {
+    ($ARGS:expr, $ARRAY_TYPE:ident, $BUILDER_TYPE:ident) => {{
+        // downcast all arguments to their common format
+        let args =
+            downcast_vec!($ARGS, $ARRAY_TYPE).collect::<Result<Vec<&$ARRAY_TYPE>>>()?;
+
+        let mut builder = FixedSizeListBuilder::<$BUILDER_TYPE>::new(
+            <$BUILDER_TYPE>::new(args[0].len()),
+            args.len() as i32,
+        );
+        // for each entry in the array
+        for index in 0..args[0].len() {
+            for arg in &args {
+                if arg.is_null(index) {
+                    builder.values().append_null()?;
+                } else {
+                    builder.values().append_value(arg.value(index))?;
+                }
+            }
+            builder.append(true)?;
+        }
+        Ok(Arc::new(builder.finish()))
+    }};
+}
+
+/// put values in an array.
+pub fn array(args: &[ArrayRef]) -> Result<ArrayRef> {
+    // do not accept 0 arguments.
+    if args.len() == 0 {
+        return Err(DataFusionError::Internal(
+            "array requires at least one argument".to_string(),
+        ));
+    }
+
+    match args[0].data_type() {
+        DataType::Utf8 => array!(args, StringArray, StringBuilder),
+        DataType::LargeUtf8 => array!(args, LargeStringArray, LargeStringBuilder),
+        DataType::Boolean => array!(args, BooleanArray, BooleanBuilder),
+        DataType::Float32 => array!(args, Float32Array, Float32Builder),
+        DataType::Float64 => array!(args, Float64Array, Float64Builder),
+        DataType::Int8 => array!(args, Int8Array, Int8Builder),
+        DataType::Int16 => array!(args, Int16Array, Int16Builder),
+        DataType::Int32 => array!(args, Int32Array, Int32Builder),
+        DataType::Int64 => array!(args, Int64Array, Int64Builder),
+        DataType::UInt8 => array!(args, UInt8Array, UInt8Builder),
+        DataType::UInt16 => array!(args, UInt16Array, UInt16Builder),
+        DataType::UInt32 => array!(args, UInt32Array, UInt32Builder),
+        DataType::UInt64 => array!(args, UInt64Array, UInt64Builder),
+        data_type => Err(DataFusionError::NotImplemented(format!(
+            "Array is not implemented for type '{:?}'.",
+            data_type
+        ))),
+    }
+}
+
+/// Currently supported types by the array function.
+/// The order of these types correspond to the order on which coercion applies
+/// This should thus be from least informative to most informative
+pub static SUPPORTED_ARRAY_TYPES: &'static [DataType] = &[
+    DataType::Boolean,
+    DataType::UInt8,
+    DataType::UInt16,
+    DataType::UInt32,
+    DataType::UInt64,
+    DataType::Int8,
+    DataType::Int16,
+    DataType::Int32,
+    DataType::Int64,
+    DataType::Float32,
+    DataType::Float64,
+    DataType::Utf8,
+    DataType::LargeUtf8,
+];
diff --git a/rust/datafusion/src/physical_plan/common.rs b/rust/datafusion/src/physical_plan/common.rs
new file mode 100644
index 00000000000..7c4f2179024
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/common.rs
@@ -0,0 +1,170 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Defines common code used in execution plans
+
+use std::fs;
+use std::fs::metadata;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+
+use super::{RecordBatchStream, SendableRecordBatchStream};
+use crate::error::{DataFusionError, Result};
+
+use array::{
+    BooleanArray, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array,
+    Int8Array, LargeStringArray, StringArray, UInt16Array, UInt32Array, UInt64Array,
+    UInt8Array,
+};
+use arrow::datatypes::{DataType, SchemaRef};
+use arrow::error::Result as ArrowResult;
+use arrow::record_batch::RecordBatch;
+use arrow::{
+    array::{self, ArrayRef},
+    datatypes::Schema,
+};
+use futures::{Stream, TryStreamExt};
+
+/// Stream of record batches
+pub struct SizedRecordBatchStream {
+    schema: SchemaRef,
+    batches: Vec<Arc<RecordBatch>>,
+    index: usize,
+}
+
+impl SizedRecordBatchStream {
+    /// Create a new RecordBatchIterator
+    pub fn new(schema: SchemaRef, batches: Vec<Arc<RecordBatch>>) -> Self {
+        SizedRecordBatchStream {
+            schema,
+            index: 0,
+            batches,
+        }
+    }
+}
+
+impl Stream for SizedRecordBatchStream {
+    type Item = ArrowResult<RecordBatch>;
+
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        _: &mut Context<'_>,
+    ) -> Poll<Option<Self::Item>> {
+        Poll::Ready(if self.index < self.batches.len() {
+            self.index += 1;
+            Some(Ok(self.batches[self.index - 1].as_ref().clone()))
+        } else {
+            None
+        })
+    }
+}
+
+impl RecordBatchStream for SizedRecordBatchStream {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+}
+
+/// Create a vector of record batches from a stream
+pub async fn collect(stream: SendableRecordBatchStream) -> Result<Vec<RecordBatch>> {
+    stream
+        .try_collect::<Vec<_>>()
+        .await
+        .map_err(|e| DataFusionError::from(e))
+}
+
+/// Recursively build a list of files in a directory with a given extension
+pub fn build_file_list(dir: &str, filenames: &mut Vec<String>, ext: &str) -> Result<()> {
+    let metadata = metadata(dir)?;
+    if metadata.is_file() {
+        if dir.ends_with(ext) {
+            filenames.push(dir.to_string());
+        }
+    } else {
+        for entry in fs::read_dir(dir)? {
+            let entry = entry?;
+            let path = entry.path();
+            if let Some(path_name) = path.to_str() {
+                if path.is_dir() {
+                    build_file_list(path_name, filenames, ext)?;
+                } else {
+                    if path_name.ends_with(ext) {
+                        filenames.push(path_name.to_string());
+                    }
+                }
+            } else {
+                return Err(DataFusionError::Plan("Invalid path".to_string()));
+            }
+        }
+    }
+    Ok(())
+}
+
+/// creates an empty record batch.
+pub fn create_batch_empty(schema: &Schema) -> ArrowResult<RecordBatch> {
+    let columns = schema
+        .fields()
+        .iter()
+        .map(|f| match f.data_type() {
+            DataType::Float32 => {
+                Ok(Arc::new(Float32Array::from(vec![] as Vec<f32>)) as ArrayRef)
+            }
+            DataType::Float64 => {
+                Ok(Arc::new(Float64Array::from(vec![] as Vec<f64>)) as ArrayRef)
+            }
+            DataType::Int64 => {
+                Ok(Arc::new(Int64Array::from(vec![] as Vec<i64>)) as ArrayRef)
+            }
+            DataType::Int32 => {
+                Ok(Arc::new(Int32Array::from(vec![] as Vec<i32>)) as ArrayRef)
+            }
+            DataType::Int16 => {
+                Ok(Arc::new(Int16Array::from(vec![] as Vec<i16>)) as ArrayRef)
+            }
+            DataType::Int8 => {
+                Ok(Arc::new(Int8Array::from(vec![] as Vec<i8>)) as ArrayRef)
+            }
+            DataType::UInt64 => {
+                Ok(Arc::new(UInt64Array::from(vec![] as Vec<u64>)) as ArrayRef)
+            }
+            DataType::UInt32 => {
+                Ok(Arc::new(UInt32Array::from(vec![] as Vec<u32>)) as ArrayRef)
+            }
+            DataType::UInt16 => {
+                Ok(Arc::new(UInt16Array::from(vec![] as Vec<u16>)) as ArrayRef)
+            }
+            DataType::UInt8 => {
+                Ok(Arc::new(UInt8Array::from(vec![] as Vec<u8>)) as ArrayRef)
+            }
+            DataType::Utf8 => {
+                Ok(Arc::new(StringArray::from(vec![] as Vec<&str>)) as ArrayRef)
+            }
+            DataType::LargeUtf8 => {
+                Ok(Arc::new(LargeStringArray::from(vec![] as Vec<&str>)) as ArrayRef)
+            }
+            DataType::Boolean => {
+                Ok(Arc::new(BooleanArray::from(vec![] as Vec<bool>)) as ArrayRef)
+            }
+            _ => Err(DataFusionError::NotImplemented(format!(
+                "Cannot convert datatype {:?} to array",
+                f.data_type()
+            ))),
+        })
+        .collect::<Result<_>>()
+        .map_err(DataFusionError::into_arrow_external_error)?;
+    RecordBatch::try_new(Arc::new(schema.to_owned()), columns)
+}
diff --git a/rust/datafusion/src/execution/physical_plan/csv.rs b/rust/datafusion/src/physical_plan/csv.rs
similarity index 70%
rename from rust/datafusion/src/execution/physical_plan/csv.rs
rename to rust/datafusion/src/physical_plan/csv.rs
index c53b21a75ae..f28523fc468 100644
--- a/rust/datafusion/src/execution/physical_plan/csv.rs
+++ b/rust/datafusion/src/physical_plan/csv.rs
@@ -17,16 +17,23 @@
 
 //! Execution plan for reading CSV files
 
+use std::any::Any;
 use std::fs::File;
-use std::sync::{Arc, Mutex};
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
 
-use crate::error::{ExecutionError, Result};
-use crate::execution::physical_plan::common;
-use crate::execution::physical_plan::{ExecutionPlan, Partition};
+use crate::error::{DataFusionError, Result};
+use crate::physical_plan::ExecutionPlan;
+use crate::physical_plan::{common, Partitioning};
 use arrow::csv;
 use arrow::datatypes::{Schema, SchemaRef};
 use arrow::error::Result as ArrowResult;
-use arrow::record_batch::{RecordBatch, RecordBatchReader};
+use arrow::record_batch::RecordBatch;
+use futures::Stream;
+
+use super::{RecordBatchStream, SendableRecordBatchStream};
+use async_trait::async_trait;
 
 /// CSV file read option
 #[derive(Copy, Clone)]
@@ -103,10 +110,12 @@ impl<'a> CsvReadOptions<'a> {
 }
 
 /// Execution plan for scanning a CSV file
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct CsvExec {
     /// Path to directory containing partitioned CSV files with the same schema
     path: String,
+    /// The individual files under path
+    filenames: Vec<String>,
     /// Schema representing the CSV file
     schema: SchemaRef,
     /// Does the CSV file have a header?
@@ -131,9 +140,17 @@ impl CsvExec {
         projection: Option<Vec<usize>>,
         batch_size: usize,
     ) -> Result<Self> {
+        let file_extension = String::from(options.file_extension);
+
+        let mut filenames: Vec<String> = vec![];
+        common::build_file_list(path, &mut filenames, file_extension.as_str())?;
+        if filenames.is_empty() {
+            return Err(DataFusionError::Execution("No files found".to_string()));
+        }
+
         let schema = match options.schema {
             Some(s) => s.clone(),
-            None => CsvExec::try_infer_schema(path, &options)?,
+            None => CsvExec::try_infer_schema(&filenames, &options)?,
         };
 
         let projected_schema = match &projection {
@@ -143,10 +160,11 @@ impl CsvExec {
 
         Ok(Self {
             path: path.to_string(),
+            filenames,
             schema: Arc::new(schema),
             has_header: options.has_header,
             delimiter: Some(options.delimiter),
-            file_extension: String::from(options.file_extension),
+            file_extension,
             projection,
             projected_schema: Arc::new(projected_schema),
             batch_size,
@@ -154,16 +172,12 @@ impl CsvExec {
     }
 
     /// Infer schema for given CSV dataset
-    pub fn try_infer_schema(path: &str, options: &CsvReadOptions) -> Result<Schema> {
-        let mut filenames: Vec<String> = vec![];
-        common::build_file_list(path, &mut filenames, options.file_extension)?;
-
-        if filenames.is_empty() {
-            return Err(ExecutionError::General("No files found".to_string()));
-        }
-
+    pub fn try_infer_schema(
+        filenames: &[String],
+        options: &CsvReadOptions,
+    ) -> Result<Schema> {
         Ok(csv::infer_schema_from_files(
-            &filenames,
+            filenames,
             options.delimiter,
             Some(options.schema_infer_max_records),
             options.has_header,
@@ -171,95 +185,61 @@ impl CsvExec {
     }
 }
 
+#[async_trait]
 impl ExecutionPlan for CsvExec {
+    /// Return a reference to Any that can be used for downcasting
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
     /// Get the schema for this execution plan
     fn schema(&self) -> SchemaRef {
         self.projected_schema.clone()
     }
 
-    /// Get the partitions for this execution plan. Each partition can be executed in parallel.
-    fn partitions(&self) -> Result<Vec<Arc<dyn Partition>>> {
-        let mut filenames: Vec<String> = vec![];
-        common::build_file_list(
-            &self.path,
-            &mut filenames,
-            self.file_extension.as_str(),
-        )?;
-        let partitions = filenames
-            .iter()
-            .map(|filename| {
-                Arc::new(CsvPartition::new(
-                    &filename,
-                    self.schema.clone(),
-                    self.has_header,
-                    self.delimiter,
-                    self.projection.clone(),
-                    self.batch_size,
-                )) as Arc<dyn Partition>
-            })
-            .collect();
-        Ok(partitions)
+    /// Get the output partitioning of this plan
+    fn output_partitioning(&self) -> Partitioning {
+        Partitioning::UnknownPartitioning(self.filenames.len())
     }
-}
 
-/// CSV Partition
-#[derive(Debug)]
-struct CsvPartition {
-    /// Path to the CSV File
-    path: String,
-    /// Schema representing the CSV file
-    schema: SchemaRef,
-    /// Does the CSV file have a header?
-    has_header: bool,
-    /// An optional column delimiter. Defaults to `b','`
-    delimiter: Option<u8>,
-    /// Optional projection for which columns to load
-    projection: Option<Vec<usize>>,
-    /// Batch size
-    batch_size: usize,
-}
+    fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
+        // this is a leaf node and has no children
+        vec![]
+    }
 
-impl CsvPartition {
-    fn new(
-        path: &str,
-        schema: SchemaRef,
-        has_header: bool,
-        delimiter: Option<u8>,
-        projection: Option<Vec<usize>>,
-        batch_size: usize,
-    ) -> Self {
-        Self {
-            path: path.to_string(),
-            schema,
-            has_header,
-            delimiter,
-            projection,
-            batch_size,
+    fn with_new_children(
+        &self,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        if children.is_empty() {
+            Ok(Arc::new(self.clone()))
+        } else {
+            Err(DataFusionError::Internal(format!(
+                "Children cannot be replaced in {:?}",
+                self
+            )))
         }
     }
-}
 
-impl Partition for CsvPartition {
-    /// Execute this partition and return an iterator over RecordBatch
-    fn execute(&self) -> Result<Arc<Mutex<dyn RecordBatchReader + Send + Sync>>> {
-        Ok(Arc::new(Mutex::new(CsvIterator::try_new(
-            &self.path,
+    async fn execute(&self, partition: usize) -> Result<SendableRecordBatchStream> {
+        Ok(Box::pin(CsvStream::try_new(
+            &self.filenames[partition],
             self.schema.clone(),
             self.has_header,
             self.delimiter,
             &self.projection,
             self.batch_size,
-        )?)))
+        )?))
     }
 }
 
 /// Iterator over batches
-struct CsvIterator {
+struct CsvStream {
     /// Arrow CSV reader
     reader: csv::Reader<File>,
 }
 
-impl CsvIterator {
+impl CsvStream {
     /// Create an iterator for a CSV file
     pub fn try_new(
         filename: &str,
@@ -276,6 +256,7 @@ impl CsvIterator {
             has_header,
             delimiter,
             batch_size,
+            None,
             projection.clone(),
         );
 
@@ -283,25 +264,32 @@ impl CsvIterator {
     }
 }
 
-impl RecordBatchReader for CsvIterator {
+impl Stream for CsvStream {
+    type Item = ArrowResult<RecordBatch>;
+
+    fn poll_next(
+        mut self: Pin<&mut Self>,
+        _: &mut Context<'_>,
+    ) -> Poll<Option<Self::Item>> {
+        Poll::Ready(self.reader.next())
+    }
+}
+
+impl RecordBatchStream for CsvStream {
     /// Get the schema
     fn schema(&self) -> SchemaRef {
         self.reader.schema()
     }
-
-    /// Get the next RecordBatch
-    fn next_batch(&mut self) -> ArrowResult<Option<RecordBatch>> {
-        Ok(self.reader.next()?)
-    }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::test::{aggr_test_schema, arrow_testdata_path};
+    use futures::StreamExt;
 
-    #[test]
-    fn csv_exec_with_projection() -> Result<()> {
+    #[tokio::test]
+    async fn csv_exec_with_projection() -> Result<()> {
         let schema = aggr_test_schema();
         let testdata = arrow_testdata_path();
         let filename = "aggregate_test_100.csv";
@@ -315,10 +303,8 @@ mod tests {
         assert_eq!(13, csv.schema.fields().len());
         assert_eq!(3, csv.projected_schema.fields().len());
         assert_eq!(3, csv.schema().fields().len());
-        let partitions = csv.partitions()?;
-        let results = partitions[0].execute()?;
-        let mut it = results.lock().unwrap();
-        let batch = it.next_batch()?.unwrap();
+        let mut stream = csv.execute(0).await?;
+        let batch = stream.next().await.unwrap()?;
         assert_eq!(3, batch.num_columns());
         let batch_schema = batch.schema();
         assert_eq!(3, batch_schema.fields().len());
@@ -328,8 +314,8 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn csv_exec_without_projection() -> Result<()> {
+    #[tokio::test]
+    async fn csv_exec_without_projection() -> Result<()> {
         let schema = aggr_test_schema();
         let testdata = arrow_testdata_path();
         let filename = "aggregate_test_100.csv";
@@ -339,10 +325,8 @@ mod tests {
         assert_eq!(13, csv.schema.fields().len());
         assert_eq!(13, csv.projected_schema.fields().len());
         assert_eq!(13, csv.schema().fields().len());
-        let partitions = csv.partitions()?;
-        let results = partitions[0].execute()?;
-        let mut it = results.lock().unwrap();
-        let batch = it.next_batch()?.unwrap();
+        let mut it = csv.execute(0).await?;
+        let batch = it.next().await.unwrap()?;
         assert_eq!(13, batch.num_columns());
         let batch_schema = batch.schema();
         assert_eq!(13, batch_schema.fields().len());
diff --git a/rust/datafusion/src/physical_plan/datetime_expressions.rs b/rust/datafusion/src/physical_plan/datetime_expressions.rs
new file mode 100644
index 00000000000..02d950ac7be
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/datetime_expressions.rs
@@ -0,0 +1,406 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! DateTime expressions
+
+use std::sync::Arc;
+
+use crate::error::{DataFusionError, Result};
+use arrow::{
+    array::{Array, ArrayData, ArrayRef, StringArray, TimestampNanosecondArray},
+    buffer::Buffer,
+    datatypes::{DataType, TimeUnit, ToByteSlice},
+};
+use chrono::{prelude::*, LocalResult};
+
+#[inline]
+/// Accepts a string in RFC3339 / ISO8601 standard format and some
+/// variants and converts it to a nanosecond precision timestamp.
+///
+/// Implements the `to_timestamp` function to convert a string to a
+/// timestamp, following the model of spark SQL’s to_`timestamp`.
+///
+/// In addition to RFC3339 / ISO8601 standard timestamps, it also
+/// accepts strings that use a space ` ` to separate the date and time
+/// as well as strings that have no explicit timezone offset.
+///
+/// Examples of accepted inputs:
+/// * `1997-01-31T09:26:56.123Z`        # RCF3339
+/// * `1997-01-31T09:26:56.123-05:00`   # RCF3339
+/// * `1997-01-31 09:26:56.123-05:00`   # close to RCF3339 but with a space rather than T
+/// * `1997-01-31T09:26:56.123`         # close to RCF3339 but no timezone offset specified
+/// * `1997-01-31 09:26:56.123`         # close to RCF3339 but uses a space and no timezone offset
+/// * `1997-01-31 09:26:56`             # close to RCF3339, no fractional seconds
+//
+/// Internally, this function uses the `chrono` library for the
+/// datetime parsing
+///
+/// We hope to extend this function in the future with a second
+/// parameter to specifying the format string.
+///
+/// ## Timestamp Precision
+///
+/// DataFusion uses the maximum precision timestamps supported by
+/// Arrow (nanoseconds stored as a 64-bit integer) timestamps. This
+/// means the range of dates that timestamps can represent is ~1677 AD
+/// to 2262 AM
+///
+///
+/// ## Timezone / Offset Handling
+///
+/// By using the Arrow format, DataFusion inherits Arrow’s handling of
+/// timestamp values. Specifically, the stored numerical values of
+/// timestamps are stored compared to offset UTC.
+///
+/// This function intertprets strings without an explicit time zone as
+/// timestamps with offsets of the local time on the machine that ran
+/// the datafusion query
+///
+/// For example, `1997-01-31 09:26:56.123Z` is interpreted as UTC, as
+/// it has an explicit timezone specifier (“Z” for Zulu/UTC)
+///
+/// `1997-01-31T09:26:56.123` is interpreted as a local timestamp in
+/// the timezone of the machine that ran DataFusion. For example, if
+/// the system timezone is set to Americas/New_York (UTC-5) the
+/// timestamp will be interpreted as though it were
+/// `1997-01-31T09:26:56.123-05:00`
+fn string_to_timestamp_nanos(s: &str) -> Result<i64> {
+    // Fast path:  RFC3339 timestamp (with a T)
+    // Example: 2020-09-08T13:42:29.190855Z
+    if let Ok(ts) = DateTime::parse_from_rfc3339(s) {
+        return Ok(ts.timestamp_nanos());
+    }
+
+    // Implement quasi-RFC3339 support by trying to parse the
+    // timestamp with various other format specifiers to to support
+    // separating the date and time with a space ' ' rather than 'T' to be
+    // (more) compatible with Apache Spark SQL
+
+    // timezone offset, using ' ' as a separator
+    // Example: 2020-09-08 13:42:29.190855-05:00
+    if let Ok(ts) = DateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.f%:z") {
+        return Ok(ts.timestamp_nanos());
+    }
+
+    // with an explicit Z, using ' ' as a separator
+    // Example: 2020-09-08 13:42:29Z
+    if let Ok(ts) = Utc.datetime_from_str(s, "%Y-%m-%d %H:%M:%S%.fZ") {
+        return Ok(ts.timestamp_nanos());
+    }
+
+    // Support timestamps without an explicit timezone offset, again
+    // to be compatible with what Apache Spark SQL does.
+
+    // without a timezone specifier as a local time, using T as a separator
+    // Example: 2020-09-08T13:42:29.190855
+    if let Ok(ts) = NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%f") {
+        return naive_datetime_to_timestamp(s, ts);
+    }
+
+    // without a timezone specifier as a local time, using T as a
+    // separator, no fractional seconds
+    // Example: 2020-09-08T13:42:29
+    if let Ok(ts) = NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S") {
+        return naive_datetime_to_timestamp(s, ts);
+    }
+
+    // without a timezone specifier as a local time, using ' ' as a separator
+    // Example: 2020-09-08 13:42:29.190855
+    if let Ok(ts) = NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S.%f") {
+        return naive_datetime_to_timestamp(s, ts);
+    }
+
+    // without a timezone specifier as a local time, using ' ' as a
+    // separator, no fractional seconds
+    // Example: 2020-09-08 13:42:29
+    if let Ok(ts) = NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S") {
+        return naive_datetime_to_timestamp(s, ts);
+    }
+
+    // Note we don't pass along the error message from the underlying
+    // chrono parsing because we tried several different format
+    // strings and we don't know which the user was trying to
+    // match. Ths any of the specific error messages is likely to be
+    // be more confusing than helpful
+    Err(DataFusionError::Execution(format!(
+        "Error parsing '{}' as timestamp",
+        s
+    )))
+}
+
+/// Converts the naive datetime (which has no specific timezone) to a
+/// nanosecond epoch timestamp relative to UTC.
+fn naive_datetime_to_timestamp(s: &str, datetime: NaiveDateTime) -> Result<i64> {
+    let l = Local {};
+
+    match l.from_local_datetime(&datetime) {
+        LocalResult::None => Err(DataFusionError::Execution(format!(
+            "Error parsing '{}' as timestamp: local time representation is invalid",
+            s
+        ))),
+        LocalResult::Single(local_datetime) => {
+            Ok(local_datetime.with_timezone(&Utc).timestamp_nanos())
+        }
+        // Ambiguous times can happen if the timestamp is exactly when
+        // a daylight savings time transition occurs, for example, and
+        // so the datetime could validly be said to be in two
+        // potential offsets. However, since we are about to convert
+        // to UTC anyways, we can pick one arbitrarily
+        LocalResult::Ambiguous(local_datetime, _) => {
+            Ok(local_datetime.with_timezone(&Utc).timestamp_nanos())
+        }
+    }
+}
+
+/// convert an array of strings into `Timestamp(Nanosecond, None)`
+pub fn to_timestamp(args: &[ArrayRef]) -> Result<TimestampNanosecondArray> {
+    let num_rows = args[0].len();
+    let string_args =
+        &args[0]
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .ok_or_else(|| {
+                DataFusionError::Internal(format!(
+                    "could not cast to_timestamp input to StringArray"
+                ))
+            })?;
+
+    let result = (0..num_rows)
+        .map(|i| {
+            if string_args.is_null(i) {
+                // NB: Since we use the same null bitset as the input,
+                // the output for this value will be ignored, but we
+                // need some value in the array we are building.
+                Ok(0)
+            } else {
+                string_to_timestamp_nanos(string_args.value(i))
+            }
+        })
+        .collect::<Result<Vec<_>>>()?;
+
+    let data = ArrayData::new(
+        DataType::Timestamp(TimeUnit::Nanosecond, None),
+        num_rows,
+        Some(string_args.null_count()),
+        string_args.data().null_buffer().cloned(),
+        0,
+        vec![Buffer::from(result.to_byte_slice())],
+        vec![],
+    );
+
+    Ok(TimestampNanosecondArray::from(Arc::new(data)))
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow::array::{Int64Array, StringBuilder};
+
+    use super::*;
+
+    #[test]
+    fn string_to_timestamp_timezone() -> Result<()> {
+        // Explicit timezone
+        assert_eq!(
+            1599572549190855000,
+            parse_timestamp("2020-09-08T13:42:29.190855+00:00")?
+        );
+        assert_eq!(
+            1599572549190855000,
+            parse_timestamp("2020-09-08T13:42:29.190855Z")?
+        );
+        assert_eq!(
+            1599572549000000000,
+            parse_timestamp("2020-09-08T13:42:29Z")?
+        ); // no fractional part
+        assert_eq!(
+            1599590549190855000,
+            parse_timestamp("2020-09-08T13:42:29.190855-05:00")?
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn string_to_timestamp_timezone_space() -> Result<()> {
+        // Ensure space rather than T between time and date is accepted
+        assert_eq!(
+            1599572549190855000,
+            parse_timestamp("2020-09-08 13:42:29.190855+00:00")?
+        );
+        assert_eq!(
+            1599572549190855000,
+            parse_timestamp("2020-09-08 13:42:29.190855Z")?
+        );
+        assert_eq!(
+            1599572549000000000,
+            parse_timestamp("2020-09-08 13:42:29Z")?
+        ); // no fractional part
+        assert_eq!(
+            1599590549190855000,
+            parse_timestamp("2020-09-08 13:42:29.190855-05:00")?
+        );
+        Ok(())
+    }
+
+    /// Interprets a naive_datetime (with no explicit timzone offset)
+    /// using the local timezone and returns the timestamp in UTC (0
+    /// offset)
+    fn naive_datetime_to_timestamp(naive_datetime: &NaiveDateTime) -> i64 {
+        // Note: Use chrono APIs that are different than
+        // naive_datetime_to_timestamp to compute the utc offset to
+        // try and double check the logic
+        let utc_offset_secs = match Local.offset_from_local_datetime(&naive_datetime) {
+            LocalResult::Single(local_offset) => {
+                local_offset.fix().local_minus_utc() as i64
+            }
+            _ => panic!("Unexpected failure converting to local datetime"),
+        };
+        let utc_offset_nanos = utc_offset_secs * 1_000_000_000;
+        naive_datetime.timestamp_nanos() - utc_offset_nanos
+    }
+
+    #[test]
+    fn string_to_timestamp_no_timezone() -> Result<()> {
+        // This test is designed to succeed in regardless of the local
+        // timezone the test machine is running. Thus it is still
+        // somewhat suceptable to bugs in the use of chrono
+        let naive_datetime = NaiveDateTime::new(
+            NaiveDate::from_ymd(2020, 09, 08),
+            NaiveTime::from_hms_nano(13, 42, 29, 190855),
+        );
+
+        // Ensure both T and ' ' variants work
+        assert_eq!(
+            naive_datetime_to_timestamp(&naive_datetime),
+            parse_timestamp("2020-09-08T13:42:29.190855")?
+        );
+
+        assert_eq!(
+            naive_datetime_to_timestamp(&naive_datetime),
+            parse_timestamp("2020-09-08 13:42:29.190855")?
+        );
+
+        // Also ensure that parsing timestamps with no fractional
+        // second part works as well
+        let naive_datetime_whole_secs = NaiveDateTime::new(
+            NaiveDate::from_ymd(2020, 09, 08),
+            NaiveTime::from_hms(13, 42, 29),
+        );
+
+        // Ensure both T and ' ' variants work
+        assert_eq!(
+            naive_datetime_to_timestamp(&naive_datetime_whole_secs),
+            parse_timestamp("2020-09-08T13:42:29")?
+        );
+
+        assert_eq!(
+            naive_datetime_to_timestamp(&naive_datetime_whole_secs),
+            parse_timestamp("2020-09-08 13:42:29")?
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn string_to_timestamp_invalid() -> Result<()> {
+        // Test parsing invalid formats
+
+        // It would be nice to make these messages better
+        expect_timestamp_parse_error("", "Error parsing '' as timestamp");
+        expect_timestamp_parse_error("SS", "Error parsing 'SS' as timestamp");
+        expect_timestamp_parse_error(
+            "Wed, 18 Feb 2015 23:16:09 GMT",
+            "Error parsing 'Wed, 18 Feb 2015 23:16:09 GMT' as timestamp",
+        );
+
+        Ok(())
+    }
+
+    // Parse a timestamp to timestamp int with a useful human readable error message
+    fn parse_timestamp(s: &str) -> Result<i64> {
+        let result = string_to_timestamp_nanos(s);
+        if let Err(e) = &result {
+            eprintln!("Error parsing timestamp '{}': {:?}", s, e);
+        }
+        result
+    }
+
+    fn expect_timestamp_parse_error(s: &str, expected_err: &str) {
+        match string_to_timestamp_nanos(s) {
+            Ok(v) => assert!(
+                false,
+                "Expected error '{}' while parsing '{}', but parsed {} instead",
+                expected_err, s, v
+            ),
+            Err(e) => {
+                assert!(e.to_string().contains(expected_err),
+                        "Can not find expected error '{}' while parsing '{}'. Actual error '{}'",
+                        expected_err, s, e);
+            }
+        }
+    }
+
+    #[test]
+    fn to_timestamp_arrays_and_nulls() -> Result<()> {
+        // ensure that arrow array implementation is wired up and handles nulls correctly
+
+        let mut string_builder = StringBuilder::new(2);
+        let mut ts_builder = TimestampNanosecondArray::builder(2);
+
+        string_builder.append_value("2020-09-08T13:42:29.190855Z")?;
+        ts_builder.append_value(1599572549190855000)?;
+
+        string_builder.append_null()?;
+        ts_builder.append_null()?;
+
+        let string_array = Arc::new(string_builder.finish());
+        let parsed_timestamps = to_timestamp(&[string_array])
+            .expect("that to_timestamp parsed values without error");
+
+        let expected_timestamps = ts_builder.finish();
+
+        assert_eq!(parsed_timestamps.len(), 2);
+        assert_eq!(expected_timestamps, parsed_timestamps);
+        Ok(())
+    }
+
+    #[test]
+    fn to_timestamp_invalid_input_type() -> Result<()> {
+        // pass the wrong type of input array to to_timestamp and test
+        // that we get an error.
+
+        let mut builder = Int64Array::builder(1);
+        builder.append_value(1)?;
+        let int64array = Arc::new(builder.finish());
+
+        let expected_err =
+            "Internal error: could not cast to_timestamp input to StringArray";
+        match to_timestamp(&[int64array]) {
+            Ok(_) => panic!("Expected error but got success"),
+            Err(e) => {
+                assert!(
+                    e.to_string().contains(expected_err),
+                    "Can not find expected error '{}'. Actual error '{}'",
+                    expected_err,
+                    e
+                );
+            }
+        }
+        Ok(())
+    }
+}
diff --git a/rust/datafusion/src/physical_plan/distinct_expressions.rs b/rust/datafusion/src/physical_plan/distinct_expressions.rs
new file mode 100644
index 00000000000..ed90cb36bc4
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/distinct_expressions.rs
@@ -0,0 +1,557 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Implementations for DISTINCT expressions, e.g. `COUNT(DISTINCT c)`
+
+use std::convert::TryFrom;
+use std::fmt::Debug;
+use std::hash::Hash;
+use std::sync::Arc;
+
+use arrow::datatypes::{DataType, Field};
+
+use fnv::FnvHashSet;
+
+use crate::error::{DataFusionError, Result};
+use crate::physical_plan::group_scalar::GroupByScalar;
+use crate::physical_plan::{Accumulator, AggregateExpr, PhysicalExpr};
+use crate::scalar::ScalarValue;
+
+#[derive(Debug, PartialEq, Eq, Hash, Clone)]
+struct DistinctScalarValues(Vec<GroupByScalar>);
+
+fn format_state_name(name: &str, state_name: &str) -> String {
+    format!("{}[{}]", name, state_name)
+}
+
+/// Expression for a COUNT(DISTINCT) aggregation.
+#[derive(Debug)]
+pub struct DistinctCount {
+    /// Column name
+    name: String,
+    /// The DataType for the final count
+    data_type: DataType,
+    /// The DataType for each input argument
+    input_data_types: Vec<DataType>,
+    /// The input arguments
+    exprs: Vec<Arc<dyn PhysicalExpr>>,
+}
+
+impl DistinctCount {
+    /// Create a new COUNT(DISTINCT) aggregate function.
+    pub fn new(
+        input_data_types: Vec<DataType>,
+        exprs: Vec<Arc<dyn PhysicalExpr>>,
+        name: String,
+        data_type: DataType,
+    ) -> Self {
+        Self {
+            input_data_types,
+            exprs,
+            name,
+            data_type,
+        }
+    }
+}
+
+impl AggregateExpr for DistinctCount {
+    fn field(&self) -> Result<Field> {
+        Ok(Field::new(&self.name, self.data_type.clone(), true))
+    }
+
+    fn state_fields(&self) -> Result<Vec<Field>> {
+        Ok(self
+            .input_data_types
+            .iter()
+            .map(|data_type| {
+                Field::new(
+                    &format_state_name(&self.name, "count distinct"),
+                    DataType::List(Box::new(Field::new("item", data_type.clone(), true))),
+                    false,
+                )
+            })
+            .collect::<Vec<_>>())
+    }
+
+    fn expressions(&self) -> Vec<Arc<dyn PhysicalExpr>> {
+        self.exprs.clone()
+    }
+
+    fn create_accumulator(&self) -> Result<Box<dyn Accumulator>> {
+        Ok(Box::new(DistinctCountAccumulator {
+            values: FnvHashSet::default(),
+            data_types: self.input_data_types.clone(),
+            count_data_type: self.data_type.clone(),
+        }))
+    }
+}
+
+#[derive(Debug)]
+struct DistinctCountAccumulator {
+    values: FnvHashSet<DistinctScalarValues>,
+    data_types: Vec<DataType>,
+    count_data_type: DataType,
+}
+
+impl Accumulator for DistinctCountAccumulator {
+    fn update(&mut self, values: &Vec<ScalarValue>) -> Result<()> {
+        // If a row has a NULL, it is not included in the final count.
+        if !values.iter().any(|v| v.is_null()) {
+            self.values.insert(DistinctScalarValues(
+                values
+                    .iter()
+                    .map(GroupByScalar::try_from)
+                    .collect::<Result<Vec<_>>>()?,
+            ));
+        }
+
+        Ok(())
+    }
+
+    fn merge(&mut self, states: &Vec<ScalarValue>) -> Result<()> {
+        if states.len() == 0 {
+            return Ok(());
+        }
+
+        let col_values = states
+            .iter()
+            .map(|state| match state {
+                ScalarValue::List(Some(values), _) => Ok(values),
+                _ => Err(DataFusionError::Internal(
+                    "Unexpected accumulator state".to_string(),
+                )),
+            })
+            .collect::<Result<Vec<_>>>()?;
+
+        (0..col_values[0].len())
+            .map(|row_index| {
+                let row_values = col_values
+                    .iter()
+                    .map(|col| col[row_index].clone())
+                    .collect::<Vec<_>>();
+                self.update(&row_values)
+            })
+            .collect::<Result<_>>()
+    }
+
+    fn state(&self) -> Result<Vec<ScalarValue>> {
+        let mut cols_out = self
+            .data_types
+            .iter()
+            .map(|data_type| ScalarValue::List(Some(Vec::new()), data_type.clone()))
+            .collect::<Vec<_>>();
+
+        let mut cols_vec = cols_out
+            .iter_mut()
+            .map(|c| match c {
+                ScalarValue::List(Some(ref mut v), _) => v,
+                _ => unreachable!(),
+            })
+            .collect::<Vec<_>>();
+
+        self.values.iter().for_each(|distinct_values| {
+            distinct_values.0.iter().enumerate().for_each(
+                |(col_index, distinct_value)| {
+                    cols_vec[col_index].push(ScalarValue::from(distinct_value));
+                },
+            )
+        });
+
+        Ok(cols_out)
+    }
+
+    fn evaluate(&self) -> Result<ScalarValue> {
+        match &self.count_data_type {
+            DataType::UInt64 => Ok(ScalarValue::UInt64(Some(self.values.len() as u64))),
+            t => {
+                return Err(DataFusionError::Internal(format!(
+                    "Invalid data type {:?} for count distinct aggregation",
+                    t
+                )))
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use arrow::array::ArrayRef;
+    use arrow::array::{
+        Int16Array, Int32Array, Int64Array, Int8Array, ListArray, UInt16Array,
+        UInt32Array, UInt64Array, UInt8Array,
+    };
+    use arrow::array::{Int32Builder, ListBuilder, UInt64Builder};
+    use arrow::datatypes::DataType;
+
+    macro_rules! build_list {
+        ($LISTS:expr, $BUILDER_TYPE:ident) => {{
+            let mut builder = ListBuilder::new($BUILDER_TYPE::new(0));
+            for list in $LISTS.iter() {
+                match list {
+                    Some(values) => {
+                        for value in values.iter() {
+                            match value {
+                                Some(v) => builder.values().append_value((*v).into())?,
+                                None => builder.values().append_null()?,
+                            }
+                        }
+
+                        builder.append(true)?;
+                    }
+                    None => {
+                        builder.append(false)?;
+                    }
+                }
+            }
+
+            let array = Arc::new(builder.finish()) as ArrayRef;
+
+            Ok(array) as Result<ArrayRef>
+        }};
+    }
+
+    macro_rules! state_to_vec {
+        ($LIST:expr, $DATA_TYPE:ident, $PRIM_TY:ty) => {{
+            match $LIST {
+                ScalarValue::List(_, data_type) => match data_type {
+                    DataType::$DATA_TYPE => (),
+                    _ => panic!("Unexpected DataType for list"),
+                },
+                _ => panic!("Expected a ScalarValue::List"),
+            }
+
+            match $LIST {
+                ScalarValue::List(None, _) => None,
+                ScalarValue::List(Some(scalar_values), _) => {
+                    let vec = scalar_values
+                        .iter()
+                        .map(|scalar_value| match scalar_value {
+                            ScalarValue::$DATA_TYPE(value) => *value,
+                            _ => panic!("Unexpected ScalarValue variant"),
+                        })
+                        .collect::<Vec<Option<$PRIM_TY>>>();
+
+                    Some(vec)
+                }
+                _ => unreachable!(),
+            }
+        }};
+    }
+
+    fn collect_states<T: Ord + Clone, S: Ord + Clone>(
+        state1: &Vec<Option<T>>,
+        state2: &Vec<Option<S>>,
+    ) -> Vec<(Option<T>, Option<S>)> {
+        let mut states = state1
+            .iter()
+            .zip(state2.iter())
+            .map(|(a, b)| (a.clone(), b.clone()))
+            .collect::<Vec<(Option<T>, Option<S>)>>();
+        states.sort();
+        states
+    }
+
+    fn run_update_batch(
+        arrays: &Vec<ArrayRef>,
+    ) -> Result<(Vec<ScalarValue>, ScalarValue)> {
+        let agg = DistinctCount::new(
+            arrays
+                .iter()
+                .map(|a| a.data_type().clone())
+                .collect::<Vec<_>>(),
+            vec![],
+            String::from("__col_name__"),
+            DataType::UInt64,
+        );
+
+        let mut accum = agg.create_accumulator()?;
+        accum.update_batch(arrays)?;
+
+        Ok((accum.state()?, accum.evaluate()?))
+    }
+
+    fn run_update(
+        data_types: &Vec<DataType>,
+        rows: &Vec<Vec<ScalarValue>>,
+    ) -> Result<(Vec<ScalarValue>, ScalarValue)> {
+        let agg = DistinctCount::new(
+            data_types.clone(),
+            vec![],
+            String::from("__col_name__"),
+            DataType::UInt64,
+        );
+
+        let mut accum = agg.create_accumulator()?;
+
+        for row in rows.iter() {
+            accum.update(row)?
+        }
+
+        Ok((accum.state()?, accum.evaluate()?))
+    }
+
+    fn run_merge_batch(
+        arrays: &Vec<ArrayRef>,
+    ) -> Result<(Vec<ScalarValue>, ScalarValue)> {
+        let agg = DistinctCount::new(
+            arrays
+                .iter()
+                .map(|a| a.as_any().downcast_ref::<ListArray>().unwrap())
+                .map(|a| a.values().data_type().clone())
+                .collect::<Vec<_>>(),
+            vec![],
+            String::from("__col_name__"),
+            DataType::UInt64,
+        );
+
+        let mut accum = agg.create_accumulator()?;
+        accum.merge_batch(arrays)?;
+
+        Ok((accum.state()?, accum.evaluate()?))
+    }
+
+    macro_rules! test_count_distinct_update_batch_numeric {
+        ($ARRAY_TYPE:ident, $DATA_TYPE:ident, $PRIM_TYPE:ty) => {{
+            let values: Vec<Option<$PRIM_TYPE>> = vec![
+                Some(1),
+                Some(1),
+                None,
+                Some(3),
+                Some(2),
+                None,
+                Some(2),
+                Some(3),
+                Some(1),
+            ];
+
+            let arrays = vec![Arc::new($ARRAY_TYPE::from(values)) as ArrayRef];
+
+            let (states, result) = run_update_batch(&arrays)?;
+
+            let mut state_vec =
+                state_to_vec!(&states[0], $DATA_TYPE, $PRIM_TYPE).unwrap();
+            state_vec.sort();
+
+            assert_eq!(states.len(), 1);
+            assert_eq!(state_vec, vec![Some(1), Some(2), Some(3)]);
+            assert_eq!(result, ScalarValue::UInt64(Some(3)));
+
+            Ok(())
+        }};
+    }
+
+    #[test]
+    fn count_distinct_update_batch_i8() -> Result<()> {
+        test_count_distinct_update_batch_numeric!(Int8Array, Int8, i8)
+    }
+
+    #[test]
+    fn count_distinct_update_batch_i16() -> Result<()> {
+        test_count_distinct_update_batch_numeric!(Int16Array, Int16, i16)
+    }
+
+    #[test]
+    fn count_distinct_update_batch_i32() -> Result<()> {
+        test_count_distinct_update_batch_numeric!(Int32Array, Int32, i32)
+    }
+
+    #[test]
+    fn count_distinct_update_batch_i64() -> Result<()> {
+        test_count_distinct_update_batch_numeric!(Int64Array, Int64, i64)
+    }
+
+    #[test]
+    fn count_distinct_update_batch_u8() -> Result<()> {
+        test_count_distinct_update_batch_numeric!(UInt8Array, UInt8, u8)
+    }
+
+    #[test]
+    fn count_distinct_update_batch_u16() -> Result<()> {
+        test_count_distinct_update_batch_numeric!(UInt16Array, UInt16, u16)
+    }
+
+    #[test]
+    fn count_distinct_update_batch_u32() -> Result<()> {
+        test_count_distinct_update_batch_numeric!(UInt32Array, UInt32, u32)
+    }
+
+    #[test]
+    fn count_distinct_update_batch_u64() -> Result<()> {
+        test_count_distinct_update_batch_numeric!(UInt64Array, UInt64, u64)
+    }
+
+    #[test]
+    fn count_distinct_update_batch_all_nulls() -> Result<()> {
+        let arrays = vec![Arc::new(Int32Array::from(
+            vec![None, None, None, None] as Vec<Option<i32>>
+        )) as ArrayRef];
+
+        let (states, result) = run_update_batch(&arrays)?;
+
+        assert_eq!(states.len(), 1);
+        assert_eq!(state_to_vec!(&states[0], Int32, i32), Some(vec![]));
+        assert_eq!(result, ScalarValue::UInt64(Some(0)));
+
+        Ok(())
+    }
+
+    #[test]
+    fn count_distinct_update_batch_empty() -> Result<()> {
+        let arrays =
+            vec![Arc::new(Int32Array::from(vec![] as Vec<Option<i32>>)) as ArrayRef];
+
+        let (states, result) = run_update_batch(&arrays)?;
+
+        assert_eq!(states.len(), 1);
+        assert_eq!(state_to_vec!(&states[0], Int32, i32), Some(vec![]));
+        assert_eq!(result, ScalarValue::UInt64(Some(0)));
+
+        Ok(())
+    }
+
+    #[test]
+    fn count_distinct_update_batch_multiple_columns() -> Result<()> {
+        let array_int8: ArrayRef = Arc::new(Int8Array::from(vec![1, 1, 2]));
+        let array_int16: ArrayRef = Arc::new(Int16Array::from(vec![3, 3, 4]));
+        let arrays = vec![array_int8, array_int16];
+
+        let (states, result) = run_update_batch(&arrays)?;
+
+        let state_vec1 = state_to_vec!(&states[0], Int8, i8).unwrap();
+        let state_vec2 = state_to_vec!(&states[1], Int16, i16).unwrap();
+        let state_pairs = collect_states::<i8, i16>(&state_vec1, &state_vec2);
+
+        assert_eq!(states.len(), 2);
+        assert_eq!(
+            state_pairs,
+            vec![(Some(1_i8), Some(3_i16)), (Some(2_i8), Some(4_i16))]
+        );
+
+        assert_eq!(result, ScalarValue::UInt64(Some(2)));
+
+        Ok(())
+    }
+
+    #[test]
+    fn count_distinct_update() -> Result<()> {
+        let (states, result) = run_update(
+            &vec![DataType::Int32, DataType::UInt64],
+            &vec![
+                vec![ScalarValue::Int32(Some(-1)), ScalarValue::UInt64(Some(5))],
+                vec![ScalarValue::Int32(Some(5)), ScalarValue::UInt64(Some(1))],
+                vec![ScalarValue::Int32(Some(-1)), ScalarValue::UInt64(Some(5))],
+                vec![ScalarValue::Int32(Some(5)), ScalarValue::UInt64(Some(1))],
+                vec![ScalarValue::Int32(Some(-1)), ScalarValue::UInt64(Some(6))],
+                vec![ScalarValue::Int32(Some(-1)), ScalarValue::UInt64(Some(7))],
+                vec![ScalarValue::Int32(Some(2)), ScalarValue::UInt64(Some(7))],
+            ],
+        )?;
+
+        let state_vec1 = state_to_vec!(&states[0], Int32, i32).unwrap();
+        let state_vec2 = state_to_vec!(&states[1], UInt64, u64).unwrap();
+        let state_pairs = collect_states::<i32, u64>(&state_vec1, &state_vec2);
+
+        assert_eq!(states.len(), 2);
+        assert_eq!(
+            state_pairs,
+            vec![
+                (Some(-1_i32), Some(5_u64)),
+                (Some(-1_i32), Some(6_u64)),
+                (Some(-1_i32), Some(7_u64)),
+                (Some(2_i32), Some(7_u64)),
+                (Some(5_i32), Some(1_u64)),
+            ]
+        );
+        assert_eq!(result, ScalarValue::UInt64(Some(5)));
+
+        Ok(())
+    }
+
+    #[test]
+    fn count_distinct_update_with_nulls() -> Result<()> {
+        let (states, result) = run_update(
+            &vec![DataType::Int32, DataType::UInt64],
+            &vec![
+                // None of these updates contains a None, so these are accumulated.
+                vec![ScalarValue::Int32(Some(-1)), ScalarValue::UInt64(Some(5))],
+                vec![ScalarValue::Int32(Some(-1)), ScalarValue::UInt64(Some(5))],
+                vec![ScalarValue::Int32(Some(-2)), ScalarValue::UInt64(Some(5))],
+                // Each of these updates contains at least one None, so these
+                // won't be accumulated.
+                vec![ScalarValue::Int32(Some(-1)), ScalarValue::UInt64(None)],
+                vec![ScalarValue::Int32(None), ScalarValue::UInt64(Some(5))],
+                vec![ScalarValue::Int32(None), ScalarValue::UInt64(None)],
+            ],
+        )?;
+
+        let state_vec1 = state_to_vec!(&states[0], Int32, i32).unwrap();
+        let state_vec2 = state_to_vec!(&states[1], UInt64, u64).unwrap();
+        let state_pairs = collect_states::<i32, u64>(&state_vec1, &state_vec2);
+
+        assert_eq!(states.len(), 2);
+        assert_eq!(
+            state_pairs,
+            vec![(Some(-2_i32), Some(5_u64)), (Some(-1_i32), Some(5_u64))]
+        );
+
+        assert_eq!(result, ScalarValue::UInt64(Some(2)));
+
+        Ok(())
+    }
+
+    #[test]
+    fn count_distinct_merge_batch() -> Result<()> {
+        let state_in1 = build_list!(
+            vec![
+                Some(vec![Some(-1_i32), Some(-1_i32), Some(-2_i32), Some(-2_i32)]),
+                Some(vec![Some(-2_i32), Some(-3_i32)]),
+            ],
+            Int32Builder
+        )?;
+
+        let state_in2 = build_list!(
+            vec![
+                Some(vec![Some(5_u64), Some(6_u64), Some(5_u64), Some(7_u64)]),
+                Some(vec![Some(5_u64), Some(7_u64)]),
+            ],
+            UInt64Builder
+        )?;
+
+        let (states, result) = run_merge_batch(&vec![state_in1, state_in2])?;
+
+        let state_out_vec1 = state_to_vec!(&states[0], Int32, i32).unwrap();
+        let state_out_vec2 = state_to_vec!(&states[1], UInt64, u64).unwrap();
+        let state_pairs = collect_states::<i32, u64>(&state_out_vec1, &state_out_vec2);
+
+        assert_eq!(
+            state_pairs,
+            vec![
+                (Some(-3_i32), Some(7_u64)),
+                (Some(-2_i32), Some(5_u64)),
+                (Some(-2_i32), Some(7_u64)),
+                (Some(-1_i32), Some(5_u64)),
+                (Some(-1_i32), Some(6_u64)),
+            ]
+        );
+
+        assert_eq!(result, ScalarValue::UInt64(Some(5)));
+
+        Ok(())
+    }
+}
diff --git a/rust/datafusion/src/physical_plan/empty.rs b/rust/datafusion/src/physical_plan/empty.rs
new file mode 100644
index 00000000000..ecdbeba9eea
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/empty.rs
@@ -0,0 +1,146 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! EmptyRelation execution plan
+
+use std::any::Any;
+use std::sync::Arc;
+
+use crate::error::{DataFusionError, Result};
+use crate::physical_plan::memory::MemoryStream;
+use crate::physical_plan::{Distribution, ExecutionPlan, Partitioning};
+use arrow::datatypes::SchemaRef;
+
+use super::SendableRecordBatchStream;
+
+use async_trait::async_trait;
+
+/// Execution plan for empty relation (produces no rows)
+#[derive(Debug)]
+pub struct EmptyExec {
+    schema: SchemaRef,
+}
+
+impl EmptyExec {
+    /// Create a new EmptyExec
+    pub fn new(schema: SchemaRef) -> Self {
+        EmptyExec { schema }
+    }
+}
+
+#[async_trait]
+impl ExecutionPlan for EmptyExec {
+    /// Return a reference to Any that can be used for downcasting
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+
+    fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
+        vec![]
+    }
+
+    fn required_child_distribution(&self) -> Distribution {
+        Distribution::UnspecifiedDistribution
+    }
+
+    /// Get the output partitioning of this plan
+    fn output_partitioning(&self) -> Partitioning {
+        Partitioning::UnknownPartitioning(1)
+    }
+
+    fn with_new_children(
+        &self,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        match children.len() {
+            0 => Ok(Arc::new(EmptyExec::new(self.schema.clone()))),
+            _ => Err(DataFusionError::Internal(
+                "EmptyExec wrong number of children".to_string(),
+            )),
+        }
+    }
+
+    async fn execute(&self, partition: usize) -> Result<SendableRecordBatchStream> {
+        // GlobalLimitExec has a single output partition
+        if 0 != partition {
+            return Err(DataFusionError::Internal(format!(
+                "EmptyExec invalid partition {} (expected 0)",
+                partition
+            )));
+        }
+
+        let data = vec![];
+        Ok(Box::pin(MemoryStream::try_new(
+            data,
+            self.schema.clone(),
+            None,
+        )?))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::physical_plan::common;
+    use crate::test;
+
+    #[tokio::test]
+    async fn empty() -> Result<()> {
+        let schema = test::aggr_test_schema();
+
+        let empty = EmptyExec::new(schema.clone());
+        assert_eq!(empty.schema(), schema);
+
+        // we should have no results
+        let iter = empty.execute(0).await?;
+        let batches = common::collect(iter).await?;
+        assert!(batches.is_empty());
+
+        Ok(())
+    }
+
+    #[test]
+    fn with_new_children() -> Result<()> {
+        let schema = test::aggr_test_schema();
+        let empty = EmptyExec::new(schema.clone());
+
+        let empty2 = empty.with_new_children(vec![])?;
+        assert_eq!(empty.schema(), empty2.schema());
+
+        let too_many_kids = vec![empty2];
+        assert!(
+            empty.with_new_children(too_many_kids).is_err(),
+            "expected error when providing list of kids"
+        );
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn invalid_execute() -> Result<()> {
+        let schema = test::aggr_test_schema();
+        let empty = EmptyExec::new(schema.clone());
+
+        // ask for the wrong partition
+        assert!(empty.execute(1).await.is_err());
+        assert!(empty.execute(20).await.is_err());
+        Ok(())
+    }
+}
diff --git a/rust/datafusion/src/physical_plan/explain.rs b/rust/datafusion/src/physical_plan/explain.rs
new file mode 100644
index 00000000000..40cd7065ba2
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/explain.rs
@@ -0,0 +1,121 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Defines the EXPLAIN operator
+
+use std::any::Any;
+use std::sync::Arc;
+
+use crate::error::{DataFusionError, Result};
+use crate::{
+    logical_plan::StringifiedPlan,
+    physical_plan::{common::SizedRecordBatchStream, ExecutionPlan},
+};
+use arrow::{array::StringBuilder, datatypes::SchemaRef, record_batch::RecordBatch};
+
+use crate::physical_plan::Partitioning;
+
+use super::SendableRecordBatchStream;
+use async_trait::async_trait;
+
+/// Explain execution plan operator. This operator contains the string
+/// values of the various plans it has when it is created, and passes
+/// them to its output.
+#[derive(Debug, Clone)]
+pub struct ExplainExec {
+    /// The schema that this exec plan node outputs
+    schema: SchemaRef,
+
+    /// The strings to be printed
+    stringified_plans: Vec<StringifiedPlan>,
+}
+
+impl ExplainExec {
+    /// Create a new ExplainExec
+    pub fn new(schema: SchemaRef, stringified_plans: Vec<StringifiedPlan>) -> Self {
+        ExplainExec {
+            schema,
+            stringified_plans,
+        }
+    }
+}
+
+#[async_trait]
+impl ExecutionPlan for ExplainExec {
+    /// Return a reference to Any that can be used for downcasting
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+
+    fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
+        // this is a leaf node and has no children
+        vec![]
+    }
+
+    /// Get the output partitioning of this plan
+    fn output_partitioning(&self) -> Partitioning {
+        Partitioning::UnknownPartitioning(1)
+    }
+
+    fn with_new_children(
+        &self,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        if children.is_empty() {
+            Ok(Arc::new(self.clone()))
+        } else {
+            Err(DataFusionError::Internal(format!(
+                "Children cannot be replaced in {:?}",
+                self
+            )))
+        }
+    }
+
+    async fn execute(&self, partition: usize) -> Result<SendableRecordBatchStream> {
+        if 0 != partition {
+            return Err(DataFusionError::Internal(format!(
+                "ExplainExec invalid partition {}",
+                partition
+            )));
+        }
+
+        let mut type_builder = StringBuilder::new(self.stringified_plans.len());
+        let mut plan_builder = StringBuilder::new(self.stringified_plans.len());
+
+        for p in &self.stringified_plans {
+            type_builder.append_value(&String::from(&p.plan_type))?;
+            plan_builder.append_value(&p.plan)?;
+        }
+
+        let record_batch = RecordBatch::try_new(
+            self.schema.clone(),
+            vec![
+                Arc::new(type_builder.finish()),
+                Arc::new(plan_builder.finish()),
+            ],
+        )?;
+
+        Ok(Box::pin(SizedRecordBatchStream::new(
+            self.schema.clone(),
+            vec![Arc::new(record_batch)],
+        )))
+    }
+}
diff --git a/rust/datafusion/src/physical_plan/expressions.rs b/rust/datafusion/src/physical_plan/expressions.rs
new file mode 100644
index 00000000000..ce3d038f726
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/expressions.rs
@@ -0,0 +1,2810 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Defines physical expressions that can evaluated at runtime during query execution
+
+use std::convert::TryFrom;
+use std::fmt;
+use std::sync::Arc;
+
+use crate::error::{DataFusionError, Result};
+use crate::logical_plan::Operator;
+use crate::physical_plan::{Accumulator, AggregateExpr, PhysicalExpr};
+use crate::scalar::ScalarValue;
+use arrow::array::{
+    Float32Builder, Float64Builder, Int16Builder, Int32Builder, Int64Builder,
+    Int8Builder, LargeStringArray, StringBuilder, UInt16Builder, UInt32Builder,
+    UInt64Builder, UInt8Builder,
+};
+use arrow::compute;
+use arrow::compute::kernels;
+use arrow::compute::kernels::arithmetic::{add, divide, multiply, subtract};
+use arrow::compute::kernels::boolean::{and, or};
+use arrow::compute::kernels::comparison::{eq, gt, gt_eq, lt, lt_eq, neq};
+use arrow::compute::kernels::comparison::{
+    eq_utf8, gt_eq_utf8, gt_utf8, like_utf8, lt_eq_utf8, lt_utf8, neq_utf8, nlike_utf8,
+};
+use arrow::compute::kernels::sort::{SortColumn, SortOptions};
+use arrow::datatypes::{DataType, Schema, TimeUnit};
+use arrow::record_batch::RecordBatch;
+use arrow::{
+    array::{
+        ArrayRef, BooleanArray, Float32Array, Float64Array, Int16Array, Int32Array,
+        Int64Array, Int8Array, StringArray, TimestampNanosecondArray, UInt16Array,
+        UInt32Array, UInt64Array, UInt8Array,
+    },
+    datatypes::Field,
+};
+use compute::can_cast_types;
+
+/// returns the name of the state
+pub fn format_state_name(name: &str, state_name: &str) -> String {
+    format!("{}[{}]", name, state_name)
+}
+
+/// Represents the column at a given index in a RecordBatch
+#[derive(Debug)]
+pub struct Column {
+    name: String,
+}
+
+impl Column {
+    /// Create a new column expression
+    pub fn new(name: &str) -> Self {
+        Self {
+            name: name.to_owned(),
+        }
+    }
+}
+
+impl fmt::Display for Column {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.name)
+    }
+}
+
+impl PhysicalExpr for Column {
+    /// Get the data type of this expression, given the schema of the input
+    fn data_type(&self, input_schema: &Schema) -> Result<DataType> {
+        Ok(input_schema
+            .field_with_name(&self.name)?
+            .data_type()
+            .clone())
+    }
+
+    /// Decide whehter this expression is nullable, given the schema of the input
+    fn nullable(&self, input_schema: &Schema) -> Result<bool> {
+        Ok(input_schema.field_with_name(&self.name)?.is_nullable())
+    }
+
+    /// Evaluate the expression
+    fn evaluate(&self, batch: &RecordBatch) -> Result<ArrayRef> {
+        Ok(batch.column(batch.schema().index_of(&self.name)?).clone())
+    }
+}
+
+/// Create a column expression
+pub fn col(name: &str) -> Arc<dyn PhysicalExpr> {
+    Arc::new(Column::new(name))
+}
+
+/// SUM aggregate expression
+#[derive(Debug)]
+pub struct Sum {
+    name: String,
+    data_type: DataType,
+    expr: Arc<dyn PhysicalExpr>,
+    nullable: bool,
+}
+
+/// function return type of a sum
+pub fn sum_return_type(arg_type: &DataType) -> Result<DataType> {
+    match arg_type {
+        DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::Int64 => {
+            Ok(DataType::Int64)
+        }
+        DataType::UInt8 | DataType::UInt16 | DataType::UInt32 | DataType::UInt64 => {
+            Ok(DataType::UInt64)
+        }
+        DataType::Float32 => Ok(DataType::Float32),
+        DataType::Float64 => Ok(DataType::Float64),
+        other => Err(DataFusionError::Plan(format!(
+            "SUM does not support type \"{:?}\"",
+            other
+        ))),
+    }
+}
+
+impl Sum {
+    /// Create a new SUM aggregate function
+    pub fn new(expr: Arc<dyn PhysicalExpr>, name: String, data_type: DataType) -> Self {
+        Self {
+            name,
+            expr,
+            data_type,
+            nullable: true,
+        }
+    }
+}
+
+impl AggregateExpr for Sum {
+    fn field(&self) -> Result<Field> {
+        Ok(Field::new(
+            &self.name,
+            self.data_type.clone(),
+            self.nullable,
+        ))
+    }
+
+    fn state_fields(&self) -> Result<Vec<Field>> {
+        Ok(vec![Field::new(
+            &format_state_name(&self.name, "sum"),
+            self.data_type.clone(),
+            self.nullable,
+        )])
+    }
+
+    fn expressions(&self) -> Vec<Arc<dyn PhysicalExpr>> {
+        vec![self.expr.clone()]
+    }
+
+    fn create_accumulator(&self) -> Result<Box<dyn Accumulator>> {
+        Ok(Box::new(SumAccumulator::try_new(&self.data_type)?))
+    }
+}
+
+#[derive(Debug)]
+struct SumAccumulator {
+    sum: ScalarValue,
+}
+
+impl SumAccumulator {
+    /// new sum accumulator
+    pub fn try_new(data_type: &DataType) -> Result<Self> {
+        Ok(Self {
+            sum: ScalarValue::try_from(data_type)?,
+        })
+    }
+}
+
+// returns the new value after sum with the new values, taking nullability into account
+macro_rules! typed_sum_delta_batch {
+    ($VALUES:expr, $ARRAYTYPE:ident, $SCALAR:ident) => {{
+        let array = $VALUES.as_any().downcast_ref::<$ARRAYTYPE>().unwrap();
+        let delta = compute::sum(array);
+        ScalarValue::$SCALAR(delta)
+    }};
+}
+
+// sums the array and returns a ScalarValue of its corresponding type.
+fn sum_batch(values: &ArrayRef) -> Result<ScalarValue> {
+    Ok(match values.data_type() {
+        DataType::Float64 => typed_sum_delta_batch!(values, Float64Array, Float64),
+        DataType::Float32 => typed_sum_delta_batch!(values, Float32Array, Float32),
+        DataType::Int64 => typed_sum_delta_batch!(values, Int64Array, Int64),
+        DataType::Int32 => typed_sum_delta_batch!(values, Int32Array, Int32),
+        DataType::Int16 => typed_sum_delta_batch!(values, Int16Array, Int16),
+        DataType::Int8 => typed_sum_delta_batch!(values, Int8Array, Int8),
+        DataType::UInt64 => typed_sum_delta_batch!(values, UInt64Array, UInt64),
+        DataType::UInt32 => typed_sum_delta_batch!(values, UInt32Array, UInt32),
+        DataType::UInt16 => typed_sum_delta_batch!(values, UInt16Array, UInt16),
+        DataType::UInt8 => typed_sum_delta_batch!(values, UInt8Array, UInt8),
+        e => {
+            return Err(DataFusionError::Internal(format!(
+                "Sum is not expected to receive the type {:?}",
+                e
+            )))
+        }
+    })
+}
+
+// returns the sum of two scalar values, including coercion into $TYPE.
+macro_rules! typed_sum {
+    ($OLD_VALUE:expr, $DELTA:expr, $SCALAR:ident, $TYPE:ident) => {{
+        ScalarValue::$SCALAR(match ($OLD_VALUE, $DELTA) {
+            (None, None) => None,
+            (Some(a), None) => Some(a.clone()),
+            (None, Some(b)) => Some(b.clone() as $TYPE),
+            (Some(a), Some(b)) => Some(a + (*b as $TYPE)),
+        })
+    }};
+}
+
+fn sum(lhs: &ScalarValue, rhs: &ScalarValue) -> Result<ScalarValue> {
+    Ok(match (lhs, rhs) {
+        // float64 coerces everything to f64
+        (ScalarValue::Float64(lhs), ScalarValue::Float64(rhs)) => {
+            typed_sum!(lhs, rhs, Float64, f64)
+        }
+        (ScalarValue::Float64(lhs), ScalarValue::Float32(rhs)) => {
+            typed_sum!(lhs, rhs, Float64, f64)
+        }
+        (ScalarValue::Float64(lhs), ScalarValue::Int64(rhs)) => {
+            typed_sum!(lhs, rhs, Float64, f64)
+        }
+        (ScalarValue::Float64(lhs), ScalarValue::Int32(rhs)) => {
+            typed_sum!(lhs, rhs, Float64, f64)
+        }
+        (ScalarValue::Float64(lhs), ScalarValue::Int16(rhs)) => {
+            typed_sum!(lhs, rhs, Float64, f64)
+        }
+        (ScalarValue::Float64(lhs), ScalarValue::Int8(rhs)) => {
+            typed_sum!(lhs, rhs, Float64, f64)
+        }
+        (ScalarValue::Float64(lhs), ScalarValue::UInt64(rhs)) => {
+            typed_sum!(lhs, rhs, Float64, f64)
+        }
+        (ScalarValue::Float64(lhs), ScalarValue::UInt32(rhs)) => {
+            typed_sum!(lhs, rhs, Float64, f64)
+        }
+        (ScalarValue::Float64(lhs), ScalarValue::UInt16(rhs)) => {
+            typed_sum!(lhs, rhs, Float64, f64)
+        }
+        (ScalarValue::Float64(lhs), ScalarValue::UInt8(rhs)) => {
+            typed_sum!(lhs, rhs, Float64, f64)
+        }
+        // float32 has no cast
+        (ScalarValue::Float32(lhs), ScalarValue::Float32(rhs)) => {
+            typed_sum!(lhs, rhs, Float32, f32)
+        }
+        // u64 coerces u* to u64
+        (ScalarValue::UInt64(lhs), ScalarValue::UInt64(rhs)) => {
+            typed_sum!(lhs, rhs, UInt64, u64)
+        }
+        (ScalarValue::UInt64(lhs), ScalarValue::UInt32(rhs)) => {
+            typed_sum!(lhs, rhs, UInt64, u64)
+        }
+        (ScalarValue::UInt64(lhs), ScalarValue::UInt16(rhs)) => {
+            typed_sum!(lhs, rhs, UInt64, u64)
+        }
+        (ScalarValue::UInt64(lhs), ScalarValue::UInt8(rhs)) => {
+            typed_sum!(lhs, rhs, UInt64, u64)
+        }
+        // i64 coerces i* to u64
+        (ScalarValue::Int64(lhs), ScalarValue::Int64(rhs)) => {
+            typed_sum!(lhs, rhs, Int64, i64)
+        }
+        (ScalarValue::Int64(lhs), ScalarValue::Int32(rhs)) => {
+            typed_sum!(lhs, rhs, Int64, i64)
+        }
+        (ScalarValue::Int64(lhs), ScalarValue::Int16(rhs)) => {
+            typed_sum!(lhs, rhs, Int64, i64)
+        }
+        (ScalarValue::Int64(lhs), ScalarValue::Int8(rhs)) => {
+            typed_sum!(lhs, rhs, Int64, i64)
+        }
+        e => {
+            return Err(DataFusionError::Internal(format!(
+                "Sum is not expected to receive a scalar {:?}",
+                e
+            )))
+        }
+    })
+}
+
+impl Accumulator for SumAccumulator {
+    fn update_batch(&mut self, values: &Vec<ArrayRef>) -> Result<()> {
+        let values = &values[0];
+        self.sum = sum(&self.sum, &sum_batch(values)?)?;
+        Ok(())
+    }
+
+    fn update(&mut self, values: &Vec<ScalarValue>) -> Result<()> {
+        // sum(v1, v2, v3) = v1 + v2 + v3
+        self.sum = sum(&self.sum, &values[0])?;
+        Ok(())
+    }
+
+    fn merge(&mut self, states: &Vec<ScalarValue>) -> Result<()> {
+        // sum(sum1, sum2) = sum1 + sum2
+        self.update(states)
+    }
+
+    fn merge_batch(&mut self, states: &Vec<ArrayRef>) -> Result<()> {
+        // sum(sum1, sum2, sum3, ...) = sum1 + sum2 + sum3 + ...
+        self.update_batch(states)
+    }
+
+    fn state(&self) -> Result<Vec<ScalarValue>> {
+        Ok(vec![self.sum.clone()])
+    }
+
+    fn evaluate(&self) -> Result<ScalarValue> {
+        Ok(self.sum.clone())
+    }
+}
+
+/// AVG aggregate expression
+#[derive(Debug)]
+pub struct Avg {
+    name: String,
+    data_type: DataType,
+    nullable: bool,
+    expr: Arc<dyn PhysicalExpr>,
+}
+
+/// function return type of an average
+pub fn avg_return_type(arg_type: &DataType) -> Result<DataType> {
+    match arg_type {
+        DataType::Int8
+        | DataType::Int16
+        | DataType::Int32
+        | DataType::Int64
+        | DataType::UInt8
+        | DataType::UInt16
+        | DataType::UInt32
+        | DataType::UInt64
+        | DataType::Float32
+        | DataType::Float64 => Ok(DataType::Float64),
+        other => Err(DataFusionError::Plan(format!(
+            "AVG does not support {:?}",
+            other
+        ))),
+    }
+}
+
+impl Avg {
+    /// Create a new AVG aggregate function
+    pub fn new(expr: Arc<dyn PhysicalExpr>, name: String, data_type: DataType) -> Self {
+        Self {
+            name,
+            expr,
+            data_type,
+            nullable: true,
+        }
+    }
+}
+
+impl AggregateExpr for Avg {
+    fn field(&self) -> Result<Field> {
+        Ok(Field::new(&self.name, DataType::Float64, true))
+    }
+
+    fn state_fields(&self) -> Result<Vec<Field>> {
+        Ok(vec![
+            Field::new(
+                &format_state_name(&self.name, "count"),
+                DataType::UInt64,
+                true,
+            ),
+            Field::new(
+                &format_state_name(&self.name, "sum"),
+                DataType::Float64,
+                true,
+            ),
+        ])
+    }
+
+    fn create_accumulator(&self) -> Result<Box<dyn Accumulator>> {
+        Ok(Box::new(AvgAccumulator::try_new(
+            // avg is f64
+            &DataType::Float64,
+        )?))
+    }
+
+    fn expressions(&self) -> Vec<Arc<dyn PhysicalExpr>> {
+        vec![self.expr.clone()]
+    }
+}
+
+/// An accumulator to compute the average
+#[derive(Debug)]
+pub(crate) struct AvgAccumulator {
+    // sum is used for null
+    sum: ScalarValue,
+    count: u64,
+}
+
+impl AvgAccumulator {
+    pub fn try_new(datatype: &DataType) -> Result<Self> {
+        Ok(Self {
+            sum: ScalarValue::try_from(datatype)?,
+            count: 0,
+        })
+    }
+}
+
+impl Accumulator for AvgAccumulator {
+    fn state(&self) -> Result<Vec<ScalarValue>> {
+        Ok(vec![ScalarValue::from(self.count), self.sum.clone()])
+    }
+
+    fn update(&mut self, values: &Vec<ScalarValue>) -> Result<()> {
+        let values = &values[0];
+
+        self.count += (!values.is_null()) as u64;
+        self.sum = sum(&self.sum, values)?;
+
+        Ok(())
+    }
+
+    fn update_batch(&mut self, values: &Vec<ArrayRef>) -> Result<()> {
+        let values = &values[0];
+
+        self.count += (values.len() - values.data().null_count()) as u64;
+        self.sum = sum(&self.sum, &sum_batch(values)?)?;
+        Ok(())
+    }
+
+    fn merge(&mut self, states: &Vec<ScalarValue>) -> Result<()> {
+        let count = &states[0];
+        // counts are summed
+        if let ScalarValue::UInt64(Some(c)) = count {
+            self.count += c
+        } else {
+            unreachable!()
+        };
+
+        // sums are summed
+        self.sum = sum(&self.sum, &states[1])?;
+        Ok(())
+    }
+
+    fn merge_batch(&mut self, states: &Vec<ArrayRef>) -> Result<()> {
+        let counts = states[0].as_any().downcast_ref::<UInt64Array>().unwrap();
+        // counts are summed
+        self.count += compute::sum(counts).unwrap_or(0);
+
+        // sums are summed
+        self.sum = sum(&self.sum, &sum_batch(&states[1])?)?;
+        Ok(())
+    }
+
+    fn evaluate(&self) -> Result<ScalarValue> {
+        match self.sum {
+            ScalarValue::Float64(e) => Ok(ScalarValue::Float64(match e {
+                Some(f) => Some(f / self.count as f64),
+                None => None,
+            })),
+            _ => Err(DataFusionError::Internal(
+                "Sum should be f64 on average".to_string(),
+            )),
+        }
+    }
+}
+
+/// MAX aggregate expression
+#[derive(Debug)]
+pub struct Max {
+    name: String,
+    data_type: DataType,
+    nullable: bool,
+    expr: Arc<dyn PhysicalExpr>,
+}
+
+impl Max {
+    /// Create a new MAX aggregate function
+    pub fn new(expr: Arc<dyn PhysicalExpr>, name: String, data_type: DataType) -> Self {
+        Self {
+            name,
+            expr,
+            data_type,
+            nullable: true,
+        }
+    }
+}
+
+impl AggregateExpr for Max {
+    fn field(&self) -> Result<Field> {
+        Ok(Field::new(
+            &self.name,
+            self.data_type.clone(),
+            self.nullable,
+        ))
+    }
+
+    fn state_fields(&self) -> Result<Vec<Field>> {
+        Ok(vec![Field::new(
+            &format_state_name(&self.name, "max"),
+            self.data_type.clone(),
+            true,
+        )])
+    }
+
+    fn expressions(&self) -> Vec<Arc<dyn PhysicalExpr>> {
+        vec![self.expr.clone()]
+    }
+
+    fn create_accumulator(&self) -> Result<Box<dyn Accumulator>> {
+        Ok(Box::new(MaxAccumulator::try_new(&self.data_type)?))
+    }
+}
+
+// Statically-typed version of min/max(array) -> ScalarValue for string types.
+macro_rules! typed_min_max_batch_string {
+    ($VALUES:expr, $ARRAYTYPE:ident, $SCALAR:ident, $OP:ident) => {{
+        let array = $VALUES.as_any().downcast_ref::<$ARRAYTYPE>().unwrap();
+        let value = compute::$OP(array);
+        let value = value.and_then(|e| Some(e.to_string()));
+        ScalarValue::$SCALAR(value)
+    }};
+}
+
+// Statically-typed version of min/max(array) -> ScalarValue for non-string types.
+macro_rules! typed_min_max_batch {
+    ($VALUES:expr, $ARRAYTYPE:ident, $SCALAR:ident, $OP:ident) => {{
+        let array = $VALUES.as_any().downcast_ref::<$ARRAYTYPE>().unwrap();
+        let value = compute::$OP(array);
+        ScalarValue::$SCALAR(value)
+    }};
+}
+
+// Statically-typed version of min/max(array) -> ScalarValue  for non-string types.
+// this is a macro to support both operations (min and max).
+macro_rules! min_max_batch {
+    ($VALUES:expr, $OP:ident) => {{
+        match $VALUES.data_type() {
+            // all types that have a natural order
+            DataType::Float64 => {
+                typed_min_max_batch!($VALUES, Float64Array, Float64, $OP)
+            }
+            DataType::Float32 => {
+                typed_min_max_batch!($VALUES, Float32Array, Float32, $OP)
+            }
+            DataType::Int64 => typed_min_max_batch!($VALUES, Int64Array, Int64, $OP),
+            DataType::Int32 => typed_min_max_batch!($VALUES, Int32Array, Int32, $OP),
+            DataType::Int16 => typed_min_max_batch!($VALUES, Int16Array, Int16, $OP),
+            DataType::Int8 => typed_min_max_batch!($VALUES, Int8Array, Int8, $OP),
+            DataType::UInt64 => typed_min_max_batch!($VALUES, UInt64Array, UInt64, $OP),
+            DataType::UInt32 => typed_min_max_batch!($VALUES, UInt32Array, UInt32, $OP),
+            DataType::UInt16 => typed_min_max_batch!($VALUES, UInt16Array, UInt16, $OP),
+            DataType::UInt8 => typed_min_max_batch!($VALUES, UInt8Array, UInt8, $OP),
+            other => {
+                // This should have been handled before
+                return Err(DataFusionError::Internal(format!(
+                    "Min/Max accumulator not implemented for type {:?}",
+                    other
+                )));
+            }
+        }
+    }};
+}
+
+/// dynamically-typed min(array) -> ScalarValue
+fn min_batch(values: &ArrayRef) -> Result<ScalarValue> {
+    Ok(match values.data_type() {
+        DataType::Utf8 => {
+            typed_min_max_batch_string!(values, StringArray, Utf8, min_string)
+        }
+        DataType::LargeUtf8 => {
+            typed_min_max_batch_string!(values, LargeStringArray, LargeUtf8, min_string)
+        }
+        _ => min_max_batch!(values, min),
+    })
+}
+
+/// dynamically-typed max(array) -> ScalarValue
+fn max_batch(values: &ArrayRef) -> Result<ScalarValue> {
+    Ok(match values.data_type() {
+        DataType::Utf8 => {
+            typed_min_max_batch_string!(values, StringArray, Utf8, max_string)
+        }
+        DataType::LargeUtf8 => {
+            typed_min_max_batch_string!(values, LargeStringArray, LargeUtf8, max_string)
+        }
+        _ => min_max_batch!(values, max),
+    })
+}
+
+// min/max of two non-string scalar values.
+macro_rules! typed_min_max {
+    ($VALUE:expr, $DELTA:expr, $SCALAR:ident, $OP:ident) => {{
+        ScalarValue::$SCALAR(match ($VALUE, $DELTA) {
+            (None, None) => None,
+            (Some(a), None) => Some(a.clone()),
+            (None, Some(b)) => Some(b.clone()),
+            (Some(a), Some(b)) => Some((*a).$OP(*b)),
+        })
+    }};
+}
+
+// min/max of two scalar string values.
+macro_rules! typed_min_max_string {
+    ($VALUE:expr, $DELTA:expr, $SCALAR:ident, $OP:ident) => {{
+        ScalarValue::$SCALAR(match ($VALUE, $DELTA) {
+            (None, None) => None,
+            (Some(a), None) => Some(a.clone()),
+            (None, Some(b)) => Some(b.clone()),
+            (Some(a), Some(b)) => Some((a).$OP(b).clone()),
+        })
+    }};
+}
+
+// min/max of two scalar values of the same type
+macro_rules! min_max {
+    ($VALUE:expr, $DELTA:expr, $OP:ident) => {{
+        Ok(match ($VALUE, $DELTA) {
+            (ScalarValue::Float64(lhs), ScalarValue::Float64(rhs)) => {
+                typed_min_max!(lhs, rhs, Float64, $OP)
+            }
+            (ScalarValue::Float32(lhs), ScalarValue::Float32(rhs)) => {
+                typed_min_max!(lhs, rhs, Float32, $OP)
+            }
+            (ScalarValue::UInt64(lhs), ScalarValue::UInt64(rhs)) => {
+                typed_min_max!(lhs, rhs, UInt64, $OP)
+            }
+            (ScalarValue::UInt32(lhs), ScalarValue::UInt32(rhs)) => {
+                typed_min_max!(lhs, rhs, UInt32, $OP)
+            }
+            (ScalarValue::UInt16(lhs), ScalarValue::UInt16(rhs)) => {
+                typed_min_max!(lhs, rhs, UInt16, $OP)
+            }
+            (ScalarValue::UInt8(lhs), ScalarValue::UInt8(rhs)) => {
+                typed_min_max!(lhs, rhs, UInt8, $OP)
+            }
+            (ScalarValue::Int64(lhs), ScalarValue::Int64(rhs)) => {
+                typed_min_max!(lhs, rhs, Int64, $OP)
+            }
+            (ScalarValue::Int32(lhs), ScalarValue::Int32(rhs)) => {
+                typed_min_max!(lhs, rhs, Int32, $OP)
+            }
+            (ScalarValue::Int16(lhs), ScalarValue::Int16(rhs)) => {
+                typed_min_max!(lhs, rhs, Int16, $OP)
+            }
+            (ScalarValue::Int8(lhs), ScalarValue::Int8(rhs)) => {
+                typed_min_max!(lhs, rhs, Int8, $OP)
+            }
+            (ScalarValue::Utf8(lhs), ScalarValue::Utf8(rhs)) => {
+                typed_min_max_string!(lhs, rhs, Utf8, $OP)
+            }
+            (ScalarValue::LargeUtf8(lhs), ScalarValue::LargeUtf8(rhs)) => {
+                typed_min_max_string!(lhs, rhs, LargeUtf8, $OP)
+            }
+            e => {
+                return Err(DataFusionError::Internal(format!(
+                    "MIN/MAX is not expected to receive a scalar {:?}",
+                    e
+                )))
+            }
+        })
+    }};
+}
+
+/// the minimum of two scalar values
+fn min(lhs: &ScalarValue, rhs: &ScalarValue) -> Result<ScalarValue> {
+    min_max!(lhs, rhs, min)
+}
+
+/// the maximum of two scalar values
+fn max(lhs: &ScalarValue, rhs: &ScalarValue) -> Result<ScalarValue> {
+    min_max!(lhs, rhs, max)
+}
+
+#[derive(Debug)]
+struct MaxAccumulator {
+    max: ScalarValue,
+}
+
+impl MaxAccumulator {
+    /// new max accumulator
+    pub fn try_new(datatype: &DataType) -> Result<Self> {
+        Ok(Self {
+            max: ScalarValue::try_from(datatype)?,
+        })
+    }
+}
+
+impl Accumulator for MaxAccumulator {
+    fn update_batch(&mut self, values: &Vec<ArrayRef>) -> Result<()> {
+        let values = &values[0];
+        let delta = &max_batch(values)?;
+        self.max = max(&self.max, delta)?;
+        Ok(())
+    }
+
+    fn update(&mut self, values: &Vec<ScalarValue>) -> Result<()> {
+        let value = &values[0];
+        self.max = max(&self.max, value)?;
+        Ok(())
+    }
+
+    fn merge(&mut self, states: &Vec<ScalarValue>) -> Result<()> {
+        self.update(states)
+    }
+
+    fn merge_batch(&mut self, states: &Vec<ArrayRef>) -> Result<()> {
+        self.update_batch(states)
+    }
+
+    fn state(&self) -> Result<Vec<ScalarValue>> {
+        Ok(vec![self.max.clone()])
+    }
+
+    fn evaluate(&self) -> Result<ScalarValue> {
+        Ok(self.max.clone())
+    }
+}
+
+/// MIN aggregate expression
+#[derive(Debug)]
+pub struct Min {
+    name: String,
+    data_type: DataType,
+    nullable: bool,
+    expr: Arc<dyn PhysicalExpr>,
+}
+
+impl Min {
+    /// Create a new MIN aggregate function
+    pub fn new(expr: Arc<dyn PhysicalExpr>, name: String, data_type: DataType) -> Self {
+        Self {
+            name,
+            expr,
+            data_type,
+            nullable: true,
+        }
+    }
+}
+
+impl AggregateExpr for Min {
+    fn field(&self) -> Result<Field> {
+        Ok(Field::new(
+            &self.name,
+            self.data_type.clone(),
+            self.nullable,
+        ))
+    }
+
+    fn state_fields(&self) -> Result<Vec<Field>> {
+        Ok(vec![Field::new(
+            &format_state_name(&self.name, "min"),
+            self.data_type.clone(),
+            true,
+        )])
+    }
+
+    fn expressions(&self) -> Vec<Arc<dyn PhysicalExpr>> {
+        vec![self.expr.clone()]
+    }
+
+    fn create_accumulator(&self) -> Result<Box<dyn Accumulator>> {
+        Ok(Box::new(MinAccumulator::try_new(&self.data_type)?))
+    }
+}
+
+#[derive(Debug)]
+struct MinAccumulator {
+    min: ScalarValue,
+}
+
+impl MinAccumulator {
+    /// new min accumulator
+    pub fn try_new(datatype: &DataType) -> Result<Self> {
+        Ok(Self {
+            min: ScalarValue::try_from(datatype)?,
+        })
+    }
+}
+
+impl Accumulator for MinAccumulator {
+    fn state(&self) -> Result<Vec<ScalarValue>> {
+        Ok(vec![self.min.clone()])
+    }
+
+    fn update_batch(&mut self, values: &Vec<ArrayRef>) -> Result<()> {
+        let values = &values[0];
+        let delta = &min_batch(values)?;
+        self.min = min(&self.min, delta)?;
+        Ok(())
+    }
+
+    fn update(&mut self, values: &Vec<ScalarValue>) -> Result<()> {
+        let value = &values[0];
+        self.min = min(&self.min, value)?;
+        Ok(())
+    }
+
+    fn merge(&mut self, states: &Vec<ScalarValue>) -> Result<()> {
+        self.update(states)
+    }
+
+    fn merge_batch(&mut self, states: &Vec<ArrayRef>) -> Result<()> {
+        self.update_batch(states)
+    }
+
+    fn evaluate(&self) -> Result<ScalarValue> {
+        Ok(self.min.clone())
+    }
+}
+
+/// COUNT aggregate expression
+/// Returns the amount of non-null values of the given expression.
+#[derive(Debug)]
+pub struct Count {
+    name: String,
+    data_type: DataType,
+    nullable: bool,
+    expr: Arc<dyn PhysicalExpr>,
+}
+
+impl Count {
+    /// Create a new COUNT aggregate function.
+    pub fn new(expr: Arc<dyn PhysicalExpr>, name: String, data_type: DataType) -> Self {
+        Self {
+            name,
+            expr,
+            data_type,
+            nullable: true,
+        }
+    }
+}
+
+impl AggregateExpr for Count {
+    fn field(&self) -> Result<Field> {
+        Ok(Field::new(
+            &self.name,
+            self.data_type.clone(),
+            self.nullable,
+        ))
+    }
+
+    fn state_fields(&self) -> Result<Vec<Field>> {
+        Ok(vec![Field::new(
+            &format_state_name(&self.name, "count"),
+            self.data_type.clone(),
+            true,
+        )])
+    }
+
+    fn expressions(&self) -> Vec<Arc<dyn PhysicalExpr>> {
+        vec![self.expr.clone()]
+    }
+
+    fn create_accumulator(&self) -> Result<Box<dyn Accumulator>> {
+        Ok(Box::new(CountAccumulator::new()))
+    }
+}
+
+#[derive(Debug)]
+struct CountAccumulator {
+    count: ScalarValue,
+}
+
+impl CountAccumulator {
+    /// new count accumulator
+    pub fn new() -> Self {
+        Self {
+            count: ScalarValue::from(0u64),
+        }
+    }
+
+    fn update_from_option(&mut self, delta: &Option<u64>) -> Result<()> {
+        self.count = ScalarValue::UInt64(match (&self.count, delta) {
+            (ScalarValue::UInt64(None), None) => None,
+            (ScalarValue::UInt64(None), Some(rhs)) => Some(rhs.clone()),
+            (ScalarValue::UInt64(Some(lhs)), None) => Some(lhs.clone()),
+            (ScalarValue::UInt64(Some(lhs)), Some(rhs)) => Some(lhs + rhs),
+            _ => {
+                return Err(DataFusionError::Internal(
+                    "Code should not be reached reach".to_string(),
+                ))
+            }
+        });
+        Ok(())
+    }
+}
+
+impl Accumulator for CountAccumulator {
+    fn update_batch(&mut self, values: &Vec<ArrayRef>) -> Result<()> {
+        let array = &values[0];
+        let delta = if array.len() == array.data().null_count() {
+            None
+        } else {
+            Some((array.len() - array.data().null_count()) as u64)
+        };
+        self.update_from_option(&delta)
+    }
+
+    fn update(&mut self, values: &Vec<ScalarValue>) -> Result<()> {
+        let value = &values[0];
+        self.count = match (&self.count, value.is_null()) {
+            (ScalarValue::UInt64(None), false) => ScalarValue::from(1u64),
+            (ScalarValue::UInt64(Some(count)), false) => ScalarValue::from(count + 1),
+            // value is null => no change in count
+            (e, true) => e.clone(),
+            (_, false) => {
+                return Err(DataFusionError::Internal(
+                    "Count is always of type u64".to_string(),
+                ))
+            }
+        };
+        Ok(())
+    }
+
+    fn merge(&mut self, states: &Vec<ScalarValue>) -> Result<()> {
+        let count = &states[0];
+        if let ScalarValue::UInt64(delta) = count {
+            self.update_from_option(delta)
+        } else {
+            unreachable!()
+        }
+    }
+
+    fn merge_batch(&mut self, states: &Vec<ArrayRef>) -> Result<()> {
+        let counts = states[0].as_any().downcast_ref::<UInt64Array>().unwrap();
+        let delta = &compute::sum(counts);
+        self.update_from_option(delta)
+    }
+
+    fn state(&self) -> Result<Vec<ScalarValue>> {
+        Ok(vec![self.count.clone()])
+    }
+
+    fn evaluate(&self) -> Result<ScalarValue> {
+        Ok(self.count.clone())
+    }
+}
+
+/// Invoke a compute kernel on a pair of binary data arrays
+macro_rules! compute_utf8_op {
+    ($LEFT:expr, $RIGHT:expr, $OP:ident, $DT:ident) => {{
+        let ll = $LEFT
+            .as_any()
+            .downcast_ref::<$DT>()
+            .expect("compute_op failed to downcast array");
+        let rr = $RIGHT
+            .as_any()
+            .downcast_ref::<$DT>()
+            .expect("compute_op failed to downcast array");
+        Ok(Arc::new(paste::expr! {[<$OP _utf8>]}(&ll, &rr)?))
+    }};
+}
+
+/// Invoke a compute kernel on a pair of arrays
+macro_rules! compute_op {
+    ($LEFT:expr, $RIGHT:expr, $OP:ident, $DT:ident) => {{
+        let ll = $LEFT
+            .as_any()
+            .downcast_ref::<$DT>()
+            .expect("compute_op failed to downcast array");
+        let rr = $RIGHT
+            .as_any()
+            .downcast_ref::<$DT>()
+            .expect("compute_op failed to downcast array");
+        Ok(Arc::new($OP(&ll, &rr)?))
+    }};
+}
+
+macro_rules! binary_string_array_op {
+    ($LEFT:expr, $RIGHT:expr, $OP:ident) => {{
+        match $LEFT.data_type() {
+            DataType::Utf8 => compute_utf8_op!($LEFT, $RIGHT, $OP, StringArray),
+            other => Err(DataFusionError::Internal(format!(
+                "Unsupported data type {:?}",
+                other
+            ))),
+        }
+    }};
+}
+
+/// Invoke a compute kernel on a pair of arrays
+/// The binary_primitive_array_op macro only evaluates for primitive types
+/// like integers and floats.
+macro_rules! binary_primitive_array_op {
+    ($LEFT:expr, $RIGHT:expr, $OP:ident) => {{
+        match $LEFT.data_type() {
+            DataType::Int8 => compute_op!($LEFT, $RIGHT, $OP, Int8Array),
+            DataType::Int16 => compute_op!($LEFT, $RIGHT, $OP, Int16Array),
+            DataType::Int32 => compute_op!($LEFT, $RIGHT, $OP, Int32Array),
+            DataType::Int64 => compute_op!($LEFT, $RIGHT, $OP, Int64Array),
+            DataType::UInt8 => compute_op!($LEFT, $RIGHT, $OP, UInt8Array),
+            DataType::UInt16 => compute_op!($LEFT, $RIGHT, $OP, UInt16Array),
+            DataType::UInt32 => compute_op!($LEFT, $RIGHT, $OP, UInt32Array),
+            DataType::UInt64 => compute_op!($LEFT, $RIGHT, $OP, UInt64Array),
+            DataType::Float32 => compute_op!($LEFT, $RIGHT, $OP, Float32Array),
+            DataType::Float64 => compute_op!($LEFT, $RIGHT, $OP, Float64Array),
+            other => Err(DataFusionError::Internal(format!(
+                "Unsupported data type {:?}",
+                other
+            ))),
+        }
+    }};
+}
+
+/// The binary_array_op macro includes types that extend beyond the primitive,
+/// such as Utf8 strings.
+macro_rules! binary_array_op {
+    ($LEFT:expr, $RIGHT:expr, $OP:ident) => {{
+        match $LEFT.data_type() {
+            DataType::Int8 => compute_op!($LEFT, $RIGHT, $OP, Int8Array),
+            DataType::Int16 => compute_op!($LEFT, $RIGHT, $OP, Int16Array),
+            DataType::Int32 => compute_op!($LEFT, $RIGHT, $OP, Int32Array),
+            DataType::Int64 => compute_op!($LEFT, $RIGHT, $OP, Int64Array),
+            DataType::UInt8 => compute_op!($LEFT, $RIGHT, $OP, UInt8Array),
+            DataType::UInt16 => compute_op!($LEFT, $RIGHT, $OP, UInt16Array),
+            DataType::UInt32 => compute_op!($LEFT, $RIGHT, $OP, UInt32Array),
+            DataType::UInt64 => compute_op!($LEFT, $RIGHT, $OP, UInt64Array),
+            DataType::Float32 => compute_op!($LEFT, $RIGHT, $OP, Float32Array),
+            DataType::Float64 => compute_op!($LEFT, $RIGHT, $OP, Float64Array),
+            DataType::Utf8 => compute_utf8_op!($LEFT, $RIGHT, $OP, StringArray),
+            DataType::Timestamp(TimeUnit::Nanosecond, None) => {
+                compute_op!($LEFT, $RIGHT, $OP, TimestampNanosecondArray)
+            }
+            other => Err(DataFusionError::Internal(format!(
+                "Unsupported data type {:?}",
+                other
+            ))),
+        }
+    }};
+}
+
+/// Invoke a boolean kernel on a pair of arrays
+macro_rules! boolean_op {
+    ($LEFT:expr, $RIGHT:expr, $OP:ident) => {{
+        let ll = $LEFT
+            .as_any()
+            .downcast_ref::<BooleanArray>()
+            .expect("boolean_op failed to downcast array");
+        let rr = $RIGHT
+            .as_any()
+            .downcast_ref::<BooleanArray>()
+            .expect("boolean_op failed to downcast array");
+        Ok(Arc::new($OP(&ll, &rr)?))
+    }};
+}
+/// Binary expression
+#[derive(Debug)]
+pub struct BinaryExpr {
+    left: Arc<dyn PhysicalExpr>,
+    op: Operator,
+    right: Arc<dyn PhysicalExpr>,
+}
+
+impl BinaryExpr {
+    /// Create new binary expression
+    pub fn new(
+        left: Arc<dyn PhysicalExpr>,
+        op: Operator,
+        right: Arc<dyn PhysicalExpr>,
+    ) -> Self {
+        Self { left, op, right }
+    }
+}
+
+impl fmt::Display for BinaryExpr {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{} {} {}", self.left, self.op, self.right)
+    }
+}
+
+/// Coercion rules for dictionary values (aka the type of the  dictionary itself)
+fn dictionary_value_coercion(
+    lhs_type: &DataType,
+    rhs_type: &DataType,
+) -> Option<DataType> {
+    numerical_coercion(lhs_type, rhs_type).or_else(|| string_coercion(lhs_type, rhs_type))
+}
+
+/// Coercion rules for Dictionaries: the type that both lhs and rhs
+/// can be casted to for the purpose of a computation.
+///
+/// It would likely be preferable to cast primitive values to
+/// dictionaries, and thus avoid unpacking dictionary as well as doing
+/// faster comparisons. However, the arrow compute kernels (e.g. eq)
+/// don't have DictionaryArray support yet, so fall back to unpacking
+/// the dictionaries
+fn dictionary_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {
+    match (lhs_type, rhs_type) {
+        (
+            DataType::Dictionary(_lhs_index_type, lhs_value_type),
+            DataType::Dictionary(_rhs_index_type, rhs_value_type),
+        ) => dictionary_value_coercion(lhs_value_type, rhs_value_type),
+        (DataType::Dictionary(_index_type, value_type), _) => {
+            dictionary_value_coercion(value_type, rhs_type)
+        }
+        (_, DataType::Dictionary(_index_type, value_type)) => {
+            dictionary_value_coercion(lhs_type, value_type)
+        }
+        _ => None,
+    }
+}
+
+/// Coercion rules for Strings: the type that both lhs and rhs can be
+/// casted to for the purpose of a string computation
+fn string_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {
+    use arrow::datatypes::DataType::*;
+    match (lhs_type, rhs_type) {
+        (Utf8, Utf8) => Some(Utf8),
+        (LargeUtf8, Utf8) => Some(LargeUtf8),
+        (Utf8, LargeUtf8) => Some(LargeUtf8),
+        (LargeUtf8, LargeUtf8) => Some(LargeUtf8),
+        _ => None,
+    }
+}
+
+/// Coercion rule for numerical types: The type that both lhs and rhs
+/// can be casted to for numerical calculation, while maintaining
+/// maximum precision
+pub fn numerical_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {
+    use arrow::datatypes::DataType::*;
+
+    // error on any non-numeric type
+    if !is_numeric(lhs_type) || !is_numeric(rhs_type) {
+        return None;
+    };
+
+    // same type => all good
+    if lhs_type == rhs_type {
+        return Some(lhs_type.clone());
+    }
+
+    // these are ordered from most informative to least informative so
+    // that the coercion removes the least amount of information
+    match (lhs_type, rhs_type) {
+        (Float64, _) => Some(Float64),
+        (_, Float64) => Some(Float64),
+
+        (_, Float32) => Some(Float32),
+        (Float32, _) => Some(Float32),
+
+        (Int64, _) => Some(Int64),
+        (_, Int64) => Some(Int64),
+
+        (Int32, _) => Some(Int32),
+        (_, Int32) => Some(Int32),
+
+        (Int16, _) => Some(Int16),
+        (_, Int16) => Some(Int16),
+
+        (Int8, _) => Some(Int8),
+        (_, Int8) => Some(Int8),
+
+        (UInt64, _) => Some(UInt64),
+        (_, UInt64) => Some(UInt64),
+
+        (UInt32, _) => Some(UInt32),
+        (_, UInt32) => Some(UInt32),
+
+        (UInt16, _) => Some(UInt16),
+        (_, UInt16) => Some(UInt16),
+
+        (UInt8, _) => Some(UInt8),
+        (_, UInt8) => Some(UInt8),
+
+        _ => None,
+    }
+}
+
+// coercion rules for equality operations. This is a superset of all numerical coercion rules.
+fn eq_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {
+    if lhs_type == rhs_type {
+        // same type => equality is possible
+        return Some(lhs_type.clone());
+    }
+    numerical_coercion(lhs_type, rhs_type)
+        .or_else(|| dictionary_coercion(lhs_type, rhs_type))
+}
+
+// coercion rules that assume an ordered set, such as "less than".
+// These are the union of all numerical coercion rules and all string coercion rules
+fn order_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {
+    if lhs_type == rhs_type {
+        // same type => all good
+        return Some(lhs_type.clone());
+    }
+
+    numerical_coercion(lhs_type, rhs_type)
+        .or_else(|| string_coercion(lhs_type, rhs_type))
+        .or_else(|| dictionary_coercion(lhs_type, rhs_type))
+}
+
+/// Coercion rules for all binary operators. Returns the output type
+/// of applying `op` to an argument of `lhs_type` and `rhs_type`.
+fn common_binary_type(
+    lhs_type: &DataType,
+    op: &Operator,
+    rhs_type: &DataType,
+) -> Result<DataType> {
+    // This result MUST be compatible with `binary_coerce`
+    let result = match op {
+        Operator::And | Operator::Or => match (lhs_type, rhs_type) {
+            // logical binary boolean operators can only be evaluated in bools
+            (DataType::Boolean, DataType::Boolean) => Some(DataType::Boolean),
+            _ => None,
+        },
+        // logical equality operators have their own rules, and always return a boolean
+        Operator::Eq | Operator::NotEq => eq_coercion(lhs_type, rhs_type),
+        // "like" operators operate on strings and always return a boolean
+        Operator::Like | Operator::NotLike => string_coercion(lhs_type, rhs_type),
+        // order-comparison operators have their own rules
+        Operator::Lt | Operator::Gt | Operator::GtEq | Operator::LtEq => {
+            order_coercion(lhs_type, rhs_type)
+        }
+        // for math expressions, the final value of the coercion is also the return type
+        // because coercion favours higher information types
+        Operator::Plus | Operator::Minus | Operator::Divide | Operator::Multiply => {
+            numerical_coercion(lhs_type, rhs_type)
+        }
+        Operator::Modulus => {
+            return Err(DataFusionError::NotImplemented(
+                "Modulus operator is still not supported".to_string(),
+            ))
+        }
+    };
+
+    // re-write the error message of failed coercions to include the operator's information
+    match result {
+        None => Err(DataFusionError::Plan(
+            format!(
+                "'{:?} {} {:?}' can't be evaluated because there isn't a common type to coerce the types to",
+                lhs_type, op, rhs_type
+            )
+            .to_string(),
+        )),
+        Some(t) => Ok(t)
+    }
+}
+
+/// Returns the return type of a binary operator or an error when the binary operator cannot
+/// perform the computation between the argument's types, even after type coercion.
+///
+/// This function makes some assumptions about the underlying available computations.
+pub fn binary_operator_data_type(
+    lhs_type: &DataType,
+    op: &Operator,
+    rhs_type: &DataType,
+) -> Result<DataType> {
+    // validate that it is possible to perform the operation on incoming types.
+    // (or the return datatype cannot be infered)
+    let common_type = common_binary_type(lhs_type, op, rhs_type)?;
+
+    match op {
+        // operators that return a boolean
+        Operator::Eq
+        | Operator::NotEq
+        | Operator::And
+        | Operator::Or
+        | Operator::Like
+        | Operator::NotLike
+        | Operator::Lt
+        | Operator::Gt
+        | Operator::GtEq
+        | Operator::LtEq => Ok(DataType::Boolean),
+        // math operations return the same value as the common coerced type
+        Operator::Plus | Operator::Minus | Operator::Divide | Operator::Multiply => {
+            Ok(common_type)
+        }
+        Operator::Modulus => Err(DataFusionError::NotImplemented(
+            "Modulus operator is still not supported".to_string(),
+        )),
+    }
+}
+
+/// return two physical expressions that are optionally coerced to a
+/// common type that the binary operator supports.
+fn binary_cast(
+    lhs: Arc<dyn PhysicalExpr>,
+    op: &Operator,
+    rhs: Arc<dyn PhysicalExpr>,
+    input_schema: &Schema,
+) -> Result<(Arc<dyn PhysicalExpr>, Arc<dyn PhysicalExpr>)> {
+    let lhs_type = &lhs.data_type(input_schema)?;
+    let rhs_type = &rhs.data_type(input_schema)?;
+
+    let cast_type = common_binary_type(lhs_type, op, rhs_type)?;
+
+    Ok((
+        cast(lhs, input_schema, cast_type.clone())?,
+        cast(rhs, input_schema, cast_type)?,
+    ))
+}
+
+impl PhysicalExpr for BinaryExpr {
+    fn data_type(&self, input_schema: &Schema) -> Result<DataType> {
+        binary_operator_data_type(
+            &self.left.data_type(input_schema)?,
+            &self.op,
+            &self.right.data_type(input_schema)?,
+        )
+    }
+
+    fn nullable(&self, input_schema: &Schema) -> Result<bool> {
+        Ok(self.left.nullable(input_schema)? || self.right.nullable(input_schema)?)
+    }
+
+    fn evaluate(&self, batch: &RecordBatch) -> Result<ArrayRef> {
+        let left = self.left.evaluate(batch)?;
+        let right = self.right.evaluate(batch)?;
+        if left.data_type() != right.data_type() {
+            // this should have been captured during planning
+            return Err(DataFusionError::Internal(format!(
+                "Cannot evaluate binary expression {:?} with types {:?} and {:?}",
+                self.op,
+                left.data_type(),
+                right.data_type()
+            )));
+        }
+        match &self.op {
+            Operator::Like => binary_string_array_op!(left, right, like),
+            Operator::NotLike => binary_string_array_op!(left, right, nlike),
+            Operator::Lt => binary_array_op!(left, right, lt),
+            Operator::LtEq => binary_array_op!(left, right, lt_eq),
+            Operator::Gt => binary_array_op!(left, right, gt),
+            Operator::GtEq => binary_array_op!(left, right, gt_eq),
+            Operator::Eq => binary_array_op!(left, right, eq),
+            Operator::NotEq => binary_array_op!(left, right, neq),
+            Operator::Plus => binary_primitive_array_op!(left, right, add),
+            Operator::Minus => binary_primitive_array_op!(left, right, subtract),
+            Operator::Multiply => binary_primitive_array_op!(left, right, multiply),
+            Operator::Divide => binary_primitive_array_op!(left, right, divide),
+            Operator::And => {
+                if left.data_type() == &DataType::Boolean {
+                    boolean_op!(left, right, and)
+                } else {
+                    return Err(DataFusionError::Internal(format!(
+                        "Cannot evaluate binary expression {:?} with types {:?} and {:?}",
+                        self.op,
+                        left.data_type(),
+                        right.data_type()
+                    )));
+                }
+            }
+            Operator::Or => {
+                if left.data_type() == &DataType::Boolean {
+                    boolean_op!(left, right, or)
+                } else {
+                    return Err(DataFusionError::Internal(format!(
+                        "Cannot evaluate binary expression {:?} with types {:?} and {:?}",
+                        self.op,
+                        left.data_type(),
+                        right.data_type()
+                    )));
+                }
+            }
+            Operator::Modulus => Err(DataFusionError::NotImplemented(
+                "Modulus operator is still not supported".to_string(),
+            )),
+        }
+    }
+}
+
+/// Create a binary expression whose arguments are correctly coerced.
+/// This function errors if it is not possible to coerce the arguments
+/// to computational types supported by the operator.
+pub fn binary(
+    lhs: Arc<dyn PhysicalExpr>,
+    op: Operator,
+    rhs: Arc<dyn PhysicalExpr>,
+    input_schema: &Schema,
+) -> Result<Arc<dyn PhysicalExpr>> {
+    let (l, r) = binary_cast(lhs, &op, rhs, input_schema)?;
+    Ok(Arc::new(BinaryExpr::new(l, op, r)))
+}
+
+/// Not expression
+#[derive(Debug)]
+pub struct NotExpr {
+    arg: Arc<dyn PhysicalExpr>,
+}
+
+impl NotExpr {
+    /// Create new not expression
+    pub fn new(arg: Arc<dyn PhysicalExpr>) -> Self {
+        Self { arg }
+    }
+}
+
+impl fmt::Display for NotExpr {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "NOT {}", self.arg)
+    }
+}
+impl PhysicalExpr for NotExpr {
+    fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
+        return Ok(DataType::Boolean);
+    }
+
+    fn nullable(&self, input_schema: &Schema) -> Result<bool> {
+        self.arg.nullable(input_schema)
+    }
+
+    fn evaluate(&self, batch: &RecordBatch) -> Result<ArrayRef> {
+        let arg = self.arg.evaluate(batch)?;
+        let arg = arg
+            .as_any()
+            .downcast_ref::<BooleanArray>()
+            .expect("boolean_op failed to downcast array");
+        return Ok(Arc::new(arrow::compute::kernels::boolean::not(arg)?));
+    }
+}
+
+/// Creates a unary expression NOT
+///
+/// # Errors
+///
+/// This function errors when the argument's type is not boolean
+pub fn not(
+    arg: Arc<dyn PhysicalExpr>,
+    input_schema: &Schema,
+) -> Result<Arc<dyn PhysicalExpr>> {
+    let data_type = arg.data_type(input_schema)?;
+    if data_type != DataType::Boolean {
+        Err(DataFusionError::Internal(
+            format!(
+                "NOT '{:?}' can't be evaluated because the expression's type is {:?}, not boolean",
+                arg, data_type,
+            )
+            .to_string(),
+        ))
+    } else {
+        Ok(Arc::new(NotExpr::new(arg)))
+    }
+}
+
+/// IS NULL expression
+#[derive(Debug)]
+pub struct IsNullExpr {
+    arg: Arc<dyn PhysicalExpr>,
+}
+
+impl IsNullExpr {
+    /// Create new not expression
+    pub fn new(arg: Arc<dyn PhysicalExpr>) -> Self {
+        Self { arg }
+    }
+}
+
+impl fmt::Display for IsNullExpr {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{} IS NULL", self.arg)
+    }
+}
+impl PhysicalExpr for IsNullExpr {
+    fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
+        return Ok(DataType::Boolean);
+    }
+
+    fn nullable(&self, _input_schema: &Schema) -> Result<bool> {
+        Ok(false)
+    }
+
+    fn evaluate(&self, batch: &RecordBatch) -> Result<ArrayRef> {
+        let arg = self.arg.evaluate(batch)?;
+        return Ok(Arc::new(arrow::compute::is_null(&arg)?));
+    }
+}
+
+/// Create an IS NULL expression
+pub fn is_null(arg: Arc<dyn PhysicalExpr>) -> Result<Arc<dyn PhysicalExpr>> {
+    Ok(Arc::new(IsNullExpr::new(arg)))
+}
+
+/// IS NULL expression
+#[derive(Debug)]
+pub struct IsNotNullExpr {
+    arg: Arc<dyn PhysicalExpr>,
+}
+
+impl IsNotNullExpr {
+    /// Create new not expression
+    pub fn new(arg: Arc<dyn PhysicalExpr>) -> Self {
+        Self { arg }
+    }
+}
+
+impl fmt::Display for IsNotNullExpr {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{} IS NOT NULL", self.arg)
+    }
+}
+impl PhysicalExpr for IsNotNullExpr {
+    fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
+        return Ok(DataType::Boolean);
+    }
+
+    fn nullable(&self, _input_schema: &Schema) -> Result<bool> {
+        Ok(false)
+    }
+
+    fn evaluate(&self, batch: &RecordBatch) -> Result<ArrayRef> {
+        let arg = self.arg.evaluate(batch)?;
+        return Ok(Arc::new(arrow::compute::is_not_null(&arg)?));
+    }
+}
+
+/// Create an IS NOT NULL expression
+pub fn is_not_null(arg: Arc<dyn PhysicalExpr>) -> Result<Arc<dyn PhysicalExpr>> {
+    Ok(Arc::new(IsNotNullExpr::new(arg)))
+}
+
+/// CAST expression casts an expression to a specific data type
+#[derive(Debug)]
+pub struct CastExpr {
+    /// The expression to cast
+    expr: Arc<dyn PhysicalExpr>,
+    /// The data type to cast to
+    cast_type: DataType,
+}
+
+/// Determine if a DataType is numeric or not
+pub fn is_numeric(dt: &DataType) -> bool {
+    match dt {
+        DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::Int64 => true,
+        DataType::UInt8 | DataType::UInt16 | DataType::UInt32 | DataType::UInt64 => true,
+        DataType::Float16 | DataType::Float32 | DataType::Float64 => true,
+        _ => false,
+    }
+}
+
+impl fmt::Display for CastExpr {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "CAST({} AS {:?})", self.expr, self.cast_type)
+    }
+}
+
+impl PhysicalExpr for CastExpr {
+    fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
+        Ok(self.cast_type.clone())
+    }
+
+    fn nullable(&self, input_schema: &Schema) -> Result<bool> {
+        self.expr.nullable(input_schema)
+    }
+
+    fn evaluate(&self, batch: &RecordBatch) -> Result<ArrayRef> {
+        let value = self.expr.evaluate(batch)?;
+        Ok(kernels::cast::cast(&value, &self.cast_type)?)
+    }
+}
+
+/// Return a PhysicalExpression representing `expr` casted to
+/// `cast_type`, if any casting is needed.
+///
+/// Note that such casts may lose type information
+pub fn cast(
+    expr: Arc<dyn PhysicalExpr>,
+    input_schema: &Schema,
+    cast_type: DataType,
+) -> Result<Arc<dyn PhysicalExpr>> {
+    let expr_type = expr.data_type(input_schema)?;
+    if expr_type == cast_type {
+        Ok(expr.clone())
+    } else if can_cast_types(&expr_type, &cast_type) {
+        Ok(Arc::new(CastExpr { expr, cast_type }))
+    } else {
+        Err(DataFusionError::Internal(format!(
+            "Unsupported CAST from {:?} to {:?}",
+            expr_type, cast_type
+        )))
+    }
+}
+
+/// Represents a non-null literal value
+#[derive(Debug)]
+pub struct Literal {
+    value: ScalarValue,
+}
+
+impl Literal {
+    /// Create a literal value expression
+    pub fn new(value: ScalarValue) -> Self {
+        Self { value }
+    }
+}
+
+/// Build array containing the same literal value repeated. This is necessary because the Arrow
+/// memory model does not have the concept of a scalar value currently.
+macro_rules! build_literal_array {
+    ($BATCH:ident, $BUILDER:ident, $VALUE:expr) => {{
+        let mut builder = $BUILDER::new($BATCH.num_rows());
+        if $VALUE.is_none() {
+            for _ in 0..$BATCH.num_rows() {
+                builder.append_null()?;
+            }
+        } else {
+            for _ in 0..$BATCH.num_rows() {
+                builder.append_value($VALUE.unwrap())?;
+            }
+        }
+        Ok(Arc::new(builder.finish()))
+    }};
+}
+
+impl fmt::Display for Literal {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.value)
+    }
+}
+
+impl PhysicalExpr for Literal {
+    fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
+        Ok(self.value.get_datatype())
+    }
+
+    fn nullable(&self, _input_schema: &Schema) -> Result<bool> {
+        Ok(self.value.is_null())
+    }
+
+    fn evaluate(&self, batch: &RecordBatch) -> Result<ArrayRef> {
+        match &self.value {
+            ScalarValue::Int8(value) => build_literal_array!(batch, Int8Builder, *value),
+            ScalarValue::Int16(value) => {
+                build_literal_array!(batch, Int16Builder, *value)
+            }
+            ScalarValue::Int32(value) => {
+                build_literal_array!(batch, Int32Builder, *value)
+            }
+            ScalarValue::Int64(value) => {
+                build_literal_array!(batch, Int64Builder, *value)
+            }
+            ScalarValue::UInt8(value) => {
+                build_literal_array!(batch, UInt8Builder, *value)
+            }
+            ScalarValue::UInt16(value) => {
+                build_literal_array!(batch, UInt16Builder, *value)
+            }
+            ScalarValue::UInt32(value) => {
+                build_literal_array!(batch, UInt32Builder, *value)
+            }
+            ScalarValue::UInt64(value) => {
+                build_literal_array!(batch, UInt64Builder, *value)
+            }
+            ScalarValue::Float32(value) => {
+                build_literal_array!(batch, Float32Builder, *value)
+            }
+            ScalarValue::Float64(value) => {
+                build_literal_array!(batch, Float64Builder, *value)
+            }
+            ScalarValue::Utf8(value) => build_literal_array!(
+                batch,
+                StringBuilder,
+                value.as_ref().and_then(|e| Some(&*e))
+            ),
+            other => Err(DataFusionError::Internal(format!(
+                "Unsupported literal type {:?}",
+                other
+            ))),
+        }
+    }
+}
+
+/// Create a literal expression
+pub fn lit(value: ScalarValue) -> Arc<dyn PhysicalExpr> {
+    Arc::new(Literal::new(value))
+}
+
+/// Represents Sort operation for a column in a RecordBatch
+#[derive(Clone, Debug)]
+pub struct PhysicalSortExpr {
+    /// Physical expression representing the column to sort
+    pub expr: Arc<dyn PhysicalExpr>,
+    /// Option to specify how the given column should be sorted
+    pub options: SortOptions,
+}
+
+impl PhysicalSortExpr {
+    /// evaluate the sort expression into SortColumn that can be passed into arrow sort kernel
+    pub fn evaluate_to_sort_column(&self, batch: &RecordBatch) -> Result<SortColumn> {
+        Ok(SortColumn {
+            values: self.expr.evaluate(batch)?,
+            options: Some(self.options),
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::error::Result;
+    use arrow::datatypes::*;
+    use arrow::{
+        array::{
+            LargeStringArray, PrimitiveArray, PrimitiveBuilder, StringArray,
+            StringDictionaryBuilder, Time64NanosecondArray,
+        },
+        util::display::array_value_to_string,
+    };
+
+    // Create a binary expression without coercion. Used here when we do not want to coerce the expressions
+    // to valid types. Usage can result in an execution (after plan) error.
+    fn binary_simple(
+        l: Arc<dyn PhysicalExpr>,
+        op: Operator,
+        r: Arc<dyn PhysicalExpr>,
+    ) -> Arc<dyn PhysicalExpr> {
+        Arc::new(BinaryExpr::new(l, op, r))
+    }
+
+    #[test]
+    fn binary_comparison() -> Result<()> {
+        let schema = Schema::new(vec![
+            Field::new("a", DataType::Int32, false),
+            Field::new("b", DataType::Int32, false),
+        ]);
+        let a = Int32Array::from(vec![1, 2, 3, 4, 5]);
+        let b = Int32Array::from(vec![1, 2, 4, 8, 16]);
+        let batch = RecordBatch::try_new(
+            Arc::new(schema.clone()),
+            vec![Arc::new(a), Arc::new(b)],
+        )?;
+
+        // expression: "a < b"
+        let lt = binary_simple(col("a"), Operator::Lt, col("b"));
+        let result = lt.evaluate(&batch)?;
+        assert_eq!(result.len(), 5);
+
+        let expected = vec![false, false, true, true, true];
+        let result = result
+            .as_any()
+            .downcast_ref::<BooleanArray>()
+            .expect("failed to downcast to BooleanArray");
+        for i in 0..5 {
+            assert_eq!(result.value(i), expected[i]);
+        }
+
+        Ok(())
+    }
+
+    #[test]
+    fn binary_nested() -> Result<()> {
+        let schema = Schema::new(vec![
+            Field::new("a", DataType::Int32, false),
+            Field::new("b", DataType::Int32, false),
+        ]);
+        let a = Int32Array::from(vec![2, 4, 6, 8, 10]);
+        let b = Int32Array::from(vec![2, 5, 4, 8, 8]);
+        let batch = RecordBatch::try_new(
+            Arc::new(schema.clone()),
+            vec![Arc::new(a), Arc::new(b)],
+        )?;
+
+        // expression: "a < b OR a == b"
+        let expr = binary_simple(
+            binary_simple(col("a"), Operator::Lt, col("b")),
+            Operator::Or,
+            binary_simple(col("a"), Operator::Eq, col("b")),
+        );
+        assert_eq!("a < b OR a = b", format!("{}", expr));
+
+        let result = expr.evaluate(&batch)?;
+        assert_eq!(result.len(), 5);
+
+        let expected = vec![true, true, false, true, false];
+        let result = result
+            .as_any()
+            .downcast_ref::<BooleanArray>()
+            .expect("failed to downcast to BooleanArray");
+        for i in 0..5 {
+            assert_eq!(result.value(i), expected[i]);
+        }
+
+        Ok(())
+    }
+
+    #[test]
+    fn literal_i32() -> Result<()> {
+        // create an arbitrary record bacth
+        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+        let a = Int32Array::from(vec![Some(1), None, Some(3), Some(4), Some(5)]);
+        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
+
+        // create and evaluate a literal expression
+        let literal_expr = lit(ScalarValue::from(42i32));
+        assert_eq!("42", format!("{}", literal_expr));
+
+        let literal_array = literal_expr.evaluate(&batch)?;
+        let literal_array = literal_array.as_any().downcast_ref::<Int32Array>().unwrap();
+
+        // note that the contents of the literal array are unrelated to the batch contents except for the length of the array
+        assert_eq!(literal_array.len(), 5); // 5 rows in the batch
+        for i in 0..literal_array.len() {
+            assert_eq!(literal_array.value(i), 42);
+        }
+
+        Ok(())
+    }
+
+    // runs an end-to-end test of physical type coercion:
+    // 1. construct a record batch with two columns of type A and B
+    //  (*_ARRAY is the Rust Arrow array type, and *_TYPE is the DataType of the elements)
+    // 2. construct a physical expression of A OP B
+    // 3. evaluate the expression
+    // 4. verify that the resulting expression is of type C
+    // 5. verify that the results of evaluation are $VEC
+    macro_rules! test_coercion {
+        ($A_ARRAY:ident, $A_TYPE:expr, $A_VEC:expr, $B_ARRAY:ident, $B_TYPE:expr, $B_VEC:expr, $OP:expr, $C_ARRAY:ident, $C_TYPE:expr, $VEC:expr) => {{
+            let schema = Schema::new(vec![
+                Field::new("a", $A_TYPE, false),
+                Field::new("b", $B_TYPE, false),
+            ]);
+            let a = $A_ARRAY::from($A_VEC);
+            let b = $B_ARRAY::from($B_VEC);
+            let batch = RecordBatch::try_new(
+                Arc::new(schema.clone()),
+                vec![Arc::new(a), Arc::new(b)],
+            )?;
+
+            // verify that we can construct the expression
+            let expression = binary(col("a"), $OP, col("b"), &schema)?;
+
+            // verify that the expression's type is correct
+            assert_eq!(expression.data_type(&schema)?, $C_TYPE);
+
+            // compute
+            let result = expression.evaluate(&batch)?;
+
+            // verify that the array's data_type is correct
+            assert_eq!(*result.data_type(), $C_TYPE);
+
+            // verify that the data itself is downcastable
+            let result = result
+                .as_any()
+                .downcast_ref::<$C_ARRAY>()
+                .expect("failed to downcast");
+            // verify that the result itself is correct
+            for (i, x) in $VEC.iter().enumerate() {
+                assert_eq!(result.value(i), *x);
+            }
+        }};
+    }
+
+    #[test]
+    fn test_type_coersion() -> Result<()> {
+        test_coercion!(
+            Int32Array,
+            DataType::Int32,
+            vec![1i32, 2i32],
+            UInt32Array,
+            DataType::UInt32,
+            vec![1u32, 2u32],
+            Operator::Plus,
+            Int32Array,
+            DataType::Int32,
+            vec![2i32, 4i32]
+        );
+        test_coercion!(
+            Int32Array,
+            DataType::Int32,
+            vec![1i32],
+            UInt16Array,
+            DataType::UInt16,
+            vec![1u16],
+            Operator::Plus,
+            Int32Array,
+            DataType::Int32,
+            vec![2i32]
+        );
+        test_coercion!(
+            Float32Array,
+            DataType::Float32,
+            vec![1f32],
+            UInt16Array,
+            DataType::UInt16,
+            vec![1u16],
+            Operator::Plus,
+            Float32Array,
+            DataType::Float32,
+            vec![2f32]
+        );
+        test_coercion!(
+            Float32Array,
+            DataType::Float32,
+            vec![2f32],
+            UInt16Array,
+            DataType::UInt16,
+            vec![1u16],
+            Operator::Multiply,
+            Float32Array,
+            DataType::Float32,
+            vec![2f32]
+        );
+        test_coercion!(
+            StringArray,
+            DataType::Utf8,
+            vec!["hello world", "world"],
+            StringArray,
+            DataType::Utf8,
+            vec!["%hello%", "%hello%"],
+            Operator::Like,
+            BooleanArray,
+            DataType::Boolean,
+            vec![true, false]
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn test_dictionary_type_coersion() -> Result<()> {
+        use DataType::*;
+
+        // TODO: In the future, this would ideally return Dictionary types and avoid unpacking
+        let lhs_type = Dictionary(Box::new(Int8), Box::new(Int32));
+        let rhs_type = Dictionary(Box::new(Int8), Box::new(Int16));
+        assert_eq!(dictionary_coercion(&lhs_type, &rhs_type), Some(Int32));
+
+        let lhs_type = Dictionary(Box::new(Int8), Box::new(Utf8));
+        let rhs_type = Dictionary(Box::new(Int8), Box::new(Int16));
+        assert_eq!(dictionary_coercion(&lhs_type, &rhs_type), None);
+
+        let lhs_type = Dictionary(Box::new(Int8), Box::new(Utf8));
+        let rhs_type = Utf8;
+        assert_eq!(dictionary_coercion(&lhs_type, &rhs_type), Some(Utf8));
+
+        let lhs_type = Utf8;
+        let rhs_type = Dictionary(Box::new(Int8), Box::new(Utf8));
+        assert_eq!(dictionary_coercion(&lhs_type, &rhs_type), Some(Utf8));
+
+        Ok(())
+    }
+
+    // Note it would be nice to use the same test_coercion macro as
+    // above, but sadly the type of the values of the dictionary are
+    // not encoded in the rust type of the DictionaryArray. Thus there
+    // is no way at the time of this writing to create a dictionary
+    // array using the `From` trait
+    #[test]
+    fn test_dictionary_type_to_array_coersion() -> Result<()> {
+        // Test string  a string dictionary
+        let dict_type =
+            DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));
+        let string_type = DataType::Utf8;
+
+        // build dictionary
+        let keys_builder = PrimitiveBuilder::<Int32Type>::new(10);
+        let values_builder = StringBuilder::new(10);
+        let mut dict_builder = StringDictionaryBuilder::new(keys_builder, values_builder);
+
+        dict_builder.append("one")?;
+        dict_builder.append_null()?;
+        dict_builder.append("three")?;
+        dict_builder.append("four")?;
+        let dict_array = dict_builder.finish();
+
+        let str_array =
+            StringArray::from(vec![Some("not one"), Some("two"), None, Some("four")]);
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("dict", dict_type.clone(), true),
+            Field::new("str", string_type.clone(), true),
+        ]));
+
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(dict_array), Arc::new(str_array)],
+        )?;
+
+        let expected = "false\n\n\ntrue";
+
+        // Test 1: dict = str
+
+        // verify that we can construct the expression
+        let expression = binary(col("dict"), Operator::Eq, col("str"), &schema)?;
+        assert_eq!(expression.data_type(&schema)?, DataType::Boolean);
+
+        // evaluate and verify the result type matched
+        let result = expression.evaluate(&batch)?;
+        assert_eq!(result.data_type(), &DataType::Boolean);
+
+        // verify that the result itself is correct
+        assert_eq!(expected, array_to_string(&result)?);
+
+        // Test 2: now test the other direction
+        // str = dict
+
+        // verify that we can construct the expression
+        let expression = binary(col("str"), Operator::Eq, col("dict"), &schema)?;
+        assert_eq!(expression.data_type(&schema)?, DataType::Boolean);
+
+        // evaluate and verify the result type matched
+        let result = expression.evaluate(&batch)?;
+        assert_eq!(result.data_type(), &DataType::Boolean);
+
+        // verify that the result itself is correct
+        assert_eq!(expected, array_to_string(&result)?);
+
+        Ok(())
+    }
+
+    // Convert the array to a newline delimited string of pretty printed values
+    fn array_to_string(array: &ArrayRef) -> Result<String> {
+        let s = (0..array.len())
+            .map(|i| array_value_to_string(array, i))
+            .collect::<std::result::Result<Vec<_>, arrow::error::ArrowError>>()?
+            .join("\n");
+        Ok(s)
+    }
+
+    #[test]
+    fn test_coersion_error() -> Result<()> {
+        let expr =
+            common_binary_type(&DataType::Float32, &Operator::Plus, &DataType::Utf8);
+
+        if let Err(DataFusionError::Plan(e)) = expr {
+            assert_eq!(e, "'Float32 + Utf8' can't be evaluated because there isn't a common type to coerce the types to");
+            Ok(())
+        } else {
+            Err(DataFusionError::Internal(
+                "Coercion should have returned an DataFusionError::Internal".to_string(),
+            ))
+        }
+    }
+
+    // runs an end-to-end test of physical type cast
+    // 1. construct a record batch with a column "a" of type A
+    // 2. construct a physical expression of CAST(a AS B)
+    // 3. evaluate the expression
+    // 4. verify that the resulting expression is of type B
+    // 5. verify that the resulting values are downcastable and correct
+    macro_rules! generic_test_cast {
+        ($A_ARRAY:ident, $A_TYPE:expr, $A_VEC:expr, $TYPEARRAY:ident, $TYPE:expr, $VEC:expr) => {{
+            let schema = Schema::new(vec![Field::new("a", $A_TYPE, false)]);
+            let a = $A_ARRAY::from($A_VEC);
+            let batch =
+                RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
+
+            // verify that we can construct the expression
+            let expression = cast(col("a"), &schema, $TYPE)?;
+
+            // verify that its display is correct
+            assert_eq!(format!("CAST(a AS {:?})", $TYPE), format!("{}", expression));
+
+            // verify that the expression's type is correct
+            assert_eq!(expression.data_type(&schema)?, $TYPE);
+
+            // compute
+            let result = expression.evaluate(&batch)?;
+
+            // verify that the array's data_type is correct
+            assert_eq!(*result.data_type(), $TYPE);
+
+            // verify that the len is correct
+            assert_eq!(result.len(), $A_VEC.len());
+
+            // verify that the data itself is downcastable
+            let result = result
+                .as_any()
+                .downcast_ref::<$TYPEARRAY>()
+                .expect("failed to downcast");
+
+            // verify that the result itself is correct
+            for (i, x) in $VEC.iter().enumerate() {
+                assert_eq!(result.value(i), *x);
+            }
+        }};
+    }
+
+    #[test]
+    fn test_cast_i32_u32() -> Result<()> {
+        generic_test_cast!(
+            Int32Array,
+            DataType::Int32,
+            vec![1, 2, 3, 4, 5],
+            UInt32Array,
+            DataType::UInt32,
+            vec![1_u32, 2_u32, 3_u32, 4_u32, 5_u32]
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn test_cast_i32_utf8() -> Result<()> {
+        generic_test_cast!(
+            Int32Array,
+            DataType::Int32,
+            vec![1, 2, 3, 4, 5],
+            StringArray,
+            DataType::Utf8,
+            vec!["1", "2", "3", "4", "5"]
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn test_cast_i64_t64() -> Result<()> {
+        let original = vec![1, 2, 3, 4, 5];
+        let expected: Vec<i64> = original
+            .iter()
+            .map(|i| Time64NanosecondArray::from(vec![*i]).value(0))
+            .collect();
+        generic_test_cast!(
+            Int64Array,
+            DataType::Int64,
+            original.clone(),
+            TimestampNanosecondArray,
+            DataType::Timestamp(TimeUnit::Nanosecond, None),
+            expected
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn invalid_cast() -> Result<()> {
+        // Ensure a useful error happens at plan time if invalid casts are used
+        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+        let result = cast(col("a"), &schema, DataType::LargeBinary);
+        result.expect_err("expected Invalid CAST");
+        Ok(())
+    }
+
+    /// macro to perform an aggregation and verify the result.
+    macro_rules! generic_test_op {
+        ($ARRAY:expr, $DATATYPE:expr, $OP:ident, $EXPECTED:expr, $EXPECTED_DATATYPE:expr) => {{
+            let schema = Schema::new(vec![Field::new("a", $DATATYPE, false)]);
+
+            let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![$ARRAY])?;
+
+            let agg =
+                Arc::new(<$OP>::new(col("a"), "bla".to_string(), $EXPECTED_DATATYPE));
+            let actual = aggregate(&batch, agg)?;
+            let expected = ScalarValue::from($EXPECTED);
+
+            assert_eq!(expected, actual);
+
+            Ok(())
+        }};
+    }
+
+    #[test]
+    fn sum_i32() -> Result<()> {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]));
+        generic_test_op!(
+            a,
+            DataType::Int32,
+            Sum,
+            ScalarValue::from(15i64),
+            DataType::Int64
+        )
+    }
+
+    #[test]
+    fn avg_i32() -> Result<()> {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]));
+        generic_test_op!(
+            a,
+            DataType::Int32,
+            Avg,
+            ScalarValue::from(3_f64),
+            DataType::Float64
+        )
+    }
+
+    #[test]
+    fn max_i32() -> Result<()> {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]));
+        generic_test_op!(
+            a,
+            DataType::Int32,
+            Max,
+            ScalarValue::from(5i32),
+            DataType::Int32
+        )
+    }
+
+    #[test]
+    fn min_i32() -> Result<()> {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]));
+        generic_test_op!(
+            a,
+            DataType::Int32,
+            Min,
+            ScalarValue::from(1i32),
+            DataType::Int32
+        )
+    }
+
+    #[test]
+    fn max_utf8() -> Result<()> {
+        let a: ArrayRef = Arc::new(StringArray::from(vec!["d", "a", "c", "b"]));
+        generic_test_op!(
+            a,
+            DataType::Utf8,
+            Max,
+            ScalarValue::Utf8(Some("d".to_string())),
+            DataType::Utf8
+        )
+    }
+
+    #[test]
+    fn max_large_utf8() -> Result<()> {
+        let a: ArrayRef = Arc::new(LargeStringArray::from(vec!["d", "a", "c", "b"]));
+        generic_test_op!(
+            a,
+            DataType::LargeUtf8,
+            Max,
+            ScalarValue::LargeUtf8(Some("d".to_string())),
+            DataType::LargeUtf8
+        )
+    }
+
+    #[test]
+    fn min_utf8() -> Result<()> {
+        let a: ArrayRef = Arc::new(StringArray::from(vec!["d", "a", "c", "b"]));
+        generic_test_op!(
+            a,
+            DataType::Utf8,
+            Min,
+            ScalarValue::Utf8(Some("a".to_string())),
+            DataType::Utf8
+        )
+    }
+
+    #[test]
+    fn min_large_utf8() -> Result<()> {
+        let a: ArrayRef = Arc::new(LargeStringArray::from(vec!["d", "a", "c", "b"]));
+        generic_test_op!(
+            a,
+            DataType::LargeUtf8,
+            Min,
+            ScalarValue::LargeUtf8(Some("a".to_string())),
+            DataType::LargeUtf8
+        )
+    }
+
+    #[test]
+    fn sum_i32_with_nulls() -> Result<()> {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![
+            Some(1),
+            None,
+            Some(3),
+            Some(4),
+            Some(5),
+        ]));
+        generic_test_op!(
+            a,
+            DataType::Int32,
+            Sum,
+            ScalarValue::from(13i64),
+            DataType::Int64
+        )
+    }
+
+    #[test]
+    fn avg_i32_with_nulls() -> Result<()> {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![
+            Some(1),
+            None,
+            Some(3),
+            Some(4),
+            Some(5),
+        ]));
+        generic_test_op!(
+            a,
+            DataType::Int32,
+            Avg,
+            ScalarValue::from(3.25f64),
+            DataType::Float64
+        )
+    }
+
+    #[test]
+    fn max_i32_with_nulls() -> Result<()> {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![
+            Some(1),
+            None,
+            Some(3),
+            Some(4),
+            Some(5),
+        ]));
+        generic_test_op!(
+            a,
+            DataType::Int32,
+            Max,
+            ScalarValue::from(5i32),
+            DataType::Int32
+        )
+    }
+
+    #[test]
+    fn min_i32_with_nulls() -> Result<()> {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![
+            Some(1),
+            None,
+            Some(3),
+            Some(4),
+            Some(5),
+        ]));
+        generic_test_op!(
+            a,
+            DataType::Int32,
+            Min,
+            ScalarValue::from(1i32),
+            DataType::Int32
+        )
+    }
+
+    #[test]
+    fn sum_i32_all_nulls() -> Result<()> {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![None, None]));
+        generic_test_op!(
+            a,
+            DataType::Int32,
+            Sum,
+            ScalarValue::Int64(None),
+            DataType::Int64
+        )
+    }
+
+    #[test]
+    fn max_i32_all_nulls() -> Result<()> {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![None, None]));
+        generic_test_op!(
+            a,
+            DataType::Int32,
+            Max,
+            ScalarValue::Int32(None),
+            DataType::Int32
+        )
+    }
+
+    #[test]
+    fn min_i32_all_nulls() -> Result<()> {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![None, None]));
+        generic_test_op!(
+            a,
+            DataType::Int32,
+            Min,
+            ScalarValue::Int32(None),
+            DataType::Int32
+        )
+    }
+
+    #[test]
+    fn avg_i32_all_nulls() -> Result<()> {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![None, None]));
+        generic_test_op!(
+            a,
+            DataType::Int32,
+            Avg,
+            ScalarValue::Float64(None),
+            DataType::Float64
+        )
+    }
+
+    #[test]
+    fn sum_u32() -> Result<()> {
+        let a: ArrayRef =
+            Arc::new(UInt32Array::from(vec![1_u32, 2_u32, 3_u32, 4_u32, 5_u32]));
+        generic_test_op!(
+            a,
+            DataType::UInt32,
+            Sum,
+            ScalarValue::from(15u64),
+            DataType::UInt64
+        )
+    }
+
+    #[test]
+    fn avg_u32() -> Result<()> {
+        let a: ArrayRef =
+            Arc::new(UInt32Array::from(vec![1_u32, 2_u32, 3_u32, 4_u32, 5_u32]));
+        generic_test_op!(
+            a,
+            DataType::UInt32,
+            Avg,
+            ScalarValue::from(3.0f64),
+            DataType::Float64
+        )
+    }
+
+    #[test]
+    fn max_u32() -> Result<()> {
+        let a: ArrayRef =
+            Arc::new(UInt32Array::from(vec![1_u32, 2_u32, 3_u32, 4_u32, 5_u32]));
+        generic_test_op!(
+            a,
+            DataType::UInt32,
+            Max,
+            ScalarValue::from(5_u32),
+            DataType::UInt32
+        )
+    }
+
+    #[test]
+    fn min_u32() -> Result<()> {
+        let a: ArrayRef =
+            Arc::new(UInt32Array::from(vec![1_u32, 2_u32, 3_u32, 4_u32, 5_u32]));
+        generic_test_op!(
+            a,
+            DataType::UInt32,
+            Min,
+            ScalarValue::from(1u32),
+            DataType::UInt32
+        )
+    }
+
+    #[test]
+    fn sum_f32() -> Result<()> {
+        let a: ArrayRef =
+            Arc::new(Float32Array::from(vec![1_f32, 2_f32, 3_f32, 4_f32, 5_f32]));
+        generic_test_op!(
+            a,
+            DataType::Float32,
+            Sum,
+            ScalarValue::from(15_f32),
+            DataType::Float32
+        )
+    }
+
+    #[test]
+    fn avg_f32() -> Result<()> {
+        let a: ArrayRef =
+            Arc::new(Float32Array::from(vec![1_f32, 2_f32, 3_f32, 4_f32, 5_f32]));
+        generic_test_op!(
+            a,
+            DataType::Float32,
+            Avg,
+            ScalarValue::from(3_f64),
+            DataType::Float64
+        )
+    }
+
+    #[test]
+    fn max_f32() -> Result<()> {
+        let a: ArrayRef =
+            Arc::new(Float32Array::from(vec![1_f32, 2_f32, 3_f32, 4_f32, 5_f32]));
+        generic_test_op!(
+            a,
+            DataType::Float32,
+            Max,
+            ScalarValue::from(5_f32),
+            DataType::Float32
+        )
+    }
+
+    #[test]
+    fn min_f32() -> Result<()> {
+        let a: ArrayRef =
+            Arc::new(Float32Array::from(vec![1_f32, 2_f32, 3_f32, 4_f32, 5_f32]));
+        generic_test_op!(
+            a,
+            DataType::Float32,
+            Min,
+            ScalarValue::from(1_f32),
+            DataType::Float32
+        )
+    }
+
+    #[test]
+    fn sum_f64() -> Result<()> {
+        let a: ArrayRef =
+            Arc::new(Float64Array::from(vec![1_f64, 2_f64, 3_f64, 4_f64, 5_f64]));
+        generic_test_op!(
+            a,
+            DataType::Float64,
+            Sum,
+            ScalarValue::from(15_f64),
+            DataType::Float64
+        )
+    }
+
+    #[test]
+    fn avg_f64() -> Result<()> {
+        let a: ArrayRef =
+            Arc::new(Float64Array::from(vec![1_f64, 2_f64, 3_f64, 4_f64, 5_f64]));
+        generic_test_op!(
+            a,
+            DataType::Float64,
+            Avg,
+            ScalarValue::from(3_f64),
+            DataType::Float64
+        )
+    }
+
+    #[test]
+    fn max_f64() -> Result<()> {
+        let a: ArrayRef =
+            Arc::new(Float64Array::from(vec![1_f64, 2_f64, 3_f64, 4_f64, 5_f64]));
+        generic_test_op!(
+            a,
+            DataType::Float64,
+            Max,
+            ScalarValue::from(5_f64),
+            DataType::Float64
+        )
+    }
+
+    #[test]
+    fn min_f64() -> Result<()> {
+        let a: ArrayRef =
+            Arc::new(Float64Array::from(vec![1_f64, 2_f64, 3_f64, 4_f64, 5_f64]));
+        generic_test_op!(
+            a,
+            DataType::Float64,
+            Min,
+            ScalarValue::from(1_f64),
+            DataType::Float64
+        )
+    }
+
+    #[test]
+    fn count_elements() -> Result<()> {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]));
+        generic_test_op!(
+            a,
+            DataType::Int32,
+            Count,
+            ScalarValue::from(5u64),
+            DataType::UInt64
+        )
+    }
+
+    #[test]
+    fn count_with_nulls() -> Result<()> {
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![
+            Some(1),
+            Some(2),
+            None,
+            None,
+            Some(3),
+            None,
+        ]));
+        generic_test_op!(
+            a,
+            DataType::Int32,
+            Count,
+            ScalarValue::from(3u64),
+            DataType::UInt64
+        )
+    }
+
+    #[test]
+    fn count_all_nulls() -> Result<()> {
+        let a: ArrayRef = Arc::new(BooleanArray::from(vec![
+            None, None, None, None, None, None, None, None,
+        ]));
+        generic_test_op!(
+            a,
+            DataType::Boolean,
+            Count,
+            ScalarValue::from(0u64),
+            DataType::UInt64
+        )
+    }
+
+    #[test]
+    fn count_empty() -> Result<()> {
+        let a: Vec<bool> = vec![];
+        let a: ArrayRef = Arc::new(BooleanArray::from(a));
+        generic_test_op!(
+            a,
+            DataType::Boolean,
+            Count,
+            ScalarValue::from(0u64),
+            DataType::UInt64
+        )
+    }
+
+    #[test]
+    fn count_utf8() -> Result<()> {
+        let a: ArrayRef =
+            Arc::new(StringArray::from(vec!["a", "bb", "ccc", "dddd", "ad"]));
+        generic_test_op!(
+            a,
+            DataType::Utf8,
+            Count,
+            ScalarValue::from(5u64),
+            DataType::UInt64
+        )
+    }
+
+    #[test]
+    fn count_large_utf8() -> Result<()> {
+        let a: ArrayRef =
+            Arc::new(LargeStringArray::from(vec!["a", "bb", "ccc", "dddd", "ad"]));
+        generic_test_op!(
+            a,
+            DataType::LargeUtf8,
+            Count,
+            ScalarValue::from(5u64),
+            DataType::UInt64
+        )
+    }
+
+    fn aggregate(
+        batch: &RecordBatch,
+        agg: Arc<dyn AggregateExpr>,
+    ) -> Result<ScalarValue> {
+        let mut accum = agg.create_accumulator()?;
+        let expr = agg.expressions();
+        let values = expr
+            .iter()
+            .map(|e| e.evaluate(batch))
+            .collect::<Result<Vec<_>>>()?;
+        accum.update_batch(&values)?;
+        accum.evaluate()
+    }
+
+    #[test]
+    fn plus_op() -> Result<()> {
+        let schema = Schema::new(vec![
+            Field::new("a", DataType::Int32, false),
+            Field::new("b", DataType::Int32, false),
+        ]);
+        let a = Int32Array::from(vec![1, 2, 3, 4, 5]);
+        let b = Int32Array::from(vec![1, 2, 4, 8, 16]);
+
+        apply_arithmetic::<Int32Type>(
+            Arc::new(schema),
+            vec![Arc::new(a), Arc::new(b)],
+            Operator::Plus,
+            Int32Array::from(vec![2, 4, 7, 12, 21]),
+        )?;
+
+        Ok(())
+    }
+
+    #[test]
+    fn minus_op() -> Result<()> {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("a", DataType::Int32, false),
+            Field::new("b", DataType::Int32, false),
+        ]));
+        let a = Arc::new(Int32Array::from(vec![1, 2, 4, 8, 16]));
+        let b = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]));
+
+        apply_arithmetic::<Int32Type>(
+            schema.clone(),
+            vec![a.clone(), b.clone()],
+            Operator::Minus,
+            Int32Array::from(vec![0, 0, 1, 4, 11]),
+        )?;
+
+        // should handle have negative values in result (for signed)
+        apply_arithmetic::<Int32Type>(
+            schema.clone(),
+            vec![b.clone(), a.clone()],
+            Operator::Minus,
+            Int32Array::from(vec![0, 0, -1, -4, -11]),
+        )?;
+
+        Ok(())
+    }
+
+    #[test]
+    fn multiply_op() -> Result<()> {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("a", DataType::Int32, false),
+            Field::new("b", DataType::Int32, false),
+        ]));
+        let a = Arc::new(Int32Array::from(vec![4, 8, 16, 32, 64]));
+        let b = Arc::new(Int32Array::from(vec![2, 4, 8, 16, 32]));
+
+        apply_arithmetic::<Int32Type>(
+            schema,
+            vec![a, b],
+            Operator::Multiply,
+            Int32Array::from(vec![8, 32, 128, 512, 2048]),
+        )?;
+
+        Ok(())
+    }
+
+    #[test]
+    fn divide_op() -> Result<()> {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("a", DataType::Int32, false),
+            Field::new("b", DataType::Int32, false),
+        ]));
+        let a = Arc::new(Int32Array::from(vec![8, 32, 128, 512, 2048]));
+        let b = Arc::new(Int32Array::from(vec![2, 4, 8, 16, 32]));
+
+        apply_arithmetic::<Int32Type>(
+            schema,
+            vec![a, b],
+            Operator::Divide,
+            Int32Array::from(vec![4, 8, 16, 32, 64]),
+        )?;
+
+        Ok(())
+    }
+
+    fn apply_arithmetic<T: ArrowNumericType>(
+        schema: SchemaRef,
+        data: Vec<ArrayRef>,
+        op: Operator,
+        expected: PrimitiveArray<T>,
+    ) -> Result<()> {
+        let arithmetic_op = binary_simple(col("a"), op, col("b"));
+        let batch = RecordBatch::try_new(schema, data)?;
+        let result = arithmetic_op.evaluate(&batch)?;
+
+        assert_array_eq::<T>(expected, result);
+
+        Ok(())
+    }
+
+    fn assert_array_eq<T: ArrowNumericType>(
+        expected: PrimitiveArray<T>,
+        actual: ArrayRef,
+    ) {
+        let actual = actual
+            .as_any()
+            .downcast_ref::<PrimitiveArray<T>>()
+            .expect("Actual array should unwrap to type of expected array");
+
+        for i in 0..expected.len() {
+            assert_eq!(expected.value(i), actual.value(i));
+        }
+    }
+
+    #[test]
+    fn neg_op() -> Result<()> {
+        let schema = Schema::new(vec![Field::new("a", DataType::Boolean, true)]);
+
+        let expr = not(col("a"), &schema)?;
+        assert_eq!(expr.data_type(&schema)?, DataType::Boolean);
+        assert_eq!(expr.nullable(&schema)?, true);
+
+        let input = BooleanArray::from(vec![Some(true), None, Some(false)]);
+        let expected = &BooleanArray::from(vec![Some(false), None, Some(true)]);
+
+        let batch =
+            RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(input)])?;
+
+        let result = expr.evaluate(&batch)?;
+        let result = result
+            .as_any()
+            .downcast_ref::<BooleanArray>()
+            .expect("failed to downcast to BooleanArray");
+        assert_eq!(result, expected);
+
+        Ok(())
+    }
+
+    /// verify that expression errors when the input expression is not a boolean.
+    #[test]
+    fn neg_op_not_null() -> Result<()> {
+        let schema = Schema::new(vec![Field::new("a", DataType::Utf8, true)]);
+
+        let expr = not(col("a"), &schema);
+        assert!(expr.is_err());
+
+        Ok(())
+    }
+
+    #[test]
+    fn is_null_op() -> Result<()> {
+        let schema = Schema::new(vec![Field::new("a", DataType::Utf8, true)]);
+        let a = StringArray::from(vec![Some("foo"), None]);
+        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
+
+        // expression: "a is null"
+        let expr = is_null(col("a")).unwrap();
+        let result = expr.evaluate(&batch)?;
+        let result = result
+            .as_any()
+            .downcast_ref::<BooleanArray>()
+            .expect("failed to downcast to BooleanArray");
+
+        let expected = &BooleanArray::from(vec![false, true]);
+
+        assert_eq!(expected, result);
+
+        Ok(())
+    }
+
+    #[test]
+    fn is_not_null_op() -> Result<()> {
+        let schema = Schema::new(vec![Field::new("a", DataType::Utf8, true)]);
+        let a = StringArray::from(vec![Some("foo"), None]);
+        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
+
+        // expression: "a is not null"
+        let expr = is_not_null(col("a")).unwrap();
+        let result = expr.evaluate(&batch)?;
+        let result = result
+            .as_any()
+            .downcast_ref::<BooleanArray>()
+            .expect("failed to downcast to BooleanArray");
+
+        let expected = &BooleanArray::from(vec![true, false]);
+
+        assert_eq!(expected, result);
+
+        Ok(())
+    }
+}
diff --git a/rust/datafusion/src/physical_plan/filter.rs b/rust/datafusion/src/physical_plan/filter.rs
new file mode 100644
index 00000000000..4a61d7d9dac
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/filter.rs
@@ -0,0 +1,232 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! FilterExec evaluates a boolean predicate against all input batches to determine which rows to
+//! include in its output batches.
+
+use std::any::Any;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+
+use super::{RecordBatchStream, SendableRecordBatchStream};
+use crate::error::{DataFusionError, Result};
+use crate::physical_plan::{ExecutionPlan, Partitioning, PhysicalExpr};
+use arrow::array::BooleanArray;
+use arrow::compute::filter;
+use arrow::datatypes::{DataType, SchemaRef};
+use arrow::error::Result as ArrowResult;
+use arrow::record_batch::RecordBatch;
+
+use async_trait::async_trait;
+
+use futures::stream::{Stream, StreamExt};
+
+/// FilterExec evaluates a boolean predicate against all input batches to determine which rows to
+/// include in its output batches.
+#[derive(Debug)]
+pub struct FilterExec {
+    /// The expression to filter on. This expression must evaluate to a boolean value.
+    predicate: Arc<dyn PhysicalExpr>,
+    /// The input plan
+    input: Arc<dyn ExecutionPlan>,
+}
+
+impl FilterExec {
+    /// Create a FilterExec on an input
+    pub fn try_new(
+        predicate: Arc<dyn PhysicalExpr>,
+        input: Arc<dyn ExecutionPlan>,
+    ) -> Result<Self> {
+        match predicate.data_type(input.schema().as_ref())? {
+            DataType::Boolean => Ok(Self {
+                predicate: predicate.clone(),
+                input: input.clone(),
+            }),
+            other => Err(DataFusionError::Plan(format!(
+                "Filter predicate must return boolean values, not {:?}",
+                other
+            ))),
+        }
+    }
+}
+
+#[async_trait]
+impl ExecutionPlan for FilterExec {
+    /// Return a reference to Any that can be used for downcasting
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    /// Get the schema for this execution plan
+    fn schema(&self) -> SchemaRef {
+        // The filter operator does not make any changes to the schema of its input
+        self.input.schema()
+    }
+
+    fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
+        vec![self.input.clone()]
+    }
+
+    /// Get the output partitioning of this plan
+    fn output_partitioning(&self) -> Partitioning {
+        self.input.output_partitioning()
+    }
+
+    fn with_new_children(
+        &self,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        match children.len() {
+            1 => Ok(Arc::new(FilterExec::try_new(
+                self.predicate.clone(),
+                children[0].clone(),
+            )?)),
+            _ => Err(DataFusionError::Internal(
+                "FilterExec wrong number of children".to_string(),
+            )),
+        }
+    }
+
+    async fn execute(&self, partition: usize) -> Result<SendableRecordBatchStream> {
+        Ok(Box::pin(FilterExecStream {
+            schema: self.input.schema().clone(),
+            predicate: self.predicate.clone(),
+            input: self.input.execute(partition).await?,
+        }))
+    }
+}
+
+/// The FilterExec streams wraps the input iterator and applies the predicate expression to
+/// determine which rows to include in its output batches
+struct FilterExecStream {
+    /// Output schema, which is the same as the input schema for this operator
+    schema: SchemaRef,
+    /// The expression to filter on. This expression must evaluate to a boolean value.
+    predicate: Arc<dyn PhysicalExpr>,
+    /// The input partition to filter.
+    input: SendableRecordBatchStream,
+}
+
+fn batch_filter(
+    batch: &RecordBatch,
+    predicate: &Arc<dyn PhysicalExpr>,
+) -> ArrowResult<RecordBatch> {
+    predicate
+        .evaluate(&batch)
+        .map_err(DataFusionError::into_arrow_external_error)
+        .and_then(|array| {
+            array
+                .as_any()
+                .downcast_ref::<BooleanArray>()
+                .ok_or(
+                    DataFusionError::Internal(
+                        "Filter predicate evaluated to non-boolean value".to_string(),
+                    )
+                    .into_arrow_external_error(),
+                )
+                // apply predicate to each column
+                .and_then(|predicate| {
+                    batch
+                        .columns()
+                        .iter()
+                        .map(|column| filter(column.as_ref(), predicate))
+                        .collect::<ArrowResult<Vec<_>>>()
+                })
+        })
+        // build RecordBatch
+        .and_then(|columns| RecordBatch::try_new(batch.schema().clone(), columns))
+}
+
+impl Stream for FilterExecStream {
+    type Item = ArrowResult<RecordBatch>;
+
+    fn poll_next(
+        mut self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<Option<Self::Item>> {
+        self.input.poll_next_unpin(cx).map(|x| match x {
+            Some(Ok(batch)) => Some(batch_filter(&batch, &self.predicate)),
+            other => other,
+        })
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        // same number of record batches
+        self.input.size_hint()
+    }
+}
+
+impl RecordBatchStream for FilterExecStream {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::*;
+    use crate::logical_plan::Operator;
+    use crate::physical_plan::csv::{CsvExec, CsvReadOptions};
+    use crate::physical_plan::expressions::*;
+    use crate::physical_plan::ExecutionPlan;
+    use crate::scalar::ScalarValue;
+    use crate::test;
+    use std::iter::Iterator;
+
+    #[tokio::test]
+    async fn simple_predicate() -> Result<()> {
+        let schema = test::aggr_test_schema();
+
+        let partitions = 4;
+        let path = test::create_partitioned_csv("aggregate_test_100.csv", partitions)?;
+
+        let csv =
+            CsvExec::try_new(&path, CsvReadOptions::new().schema(&schema), None, 1024)?;
+
+        let predicate: Arc<dyn PhysicalExpr> = binary(
+            binary(
+                col("c2"),
+                Operator::Gt,
+                lit(ScalarValue::from(1u32)),
+                &schema,
+            )?,
+            Operator::And,
+            binary(
+                col("c2"),
+                Operator::Lt,
+                lit(ScalarValue::from(4u32)),
+                &schema,
+            )?,
+            &schema,
+        )?;
+
+        let filter: Arc<dyn ExecutionPlan> =
+            Arc::new(FilterExec::try_new(predicate, Arc::new(csv))?);
+
+        let results = test::execute(filter).await?;
+
+        results
+            .iter()
+            .for_each(|batch| assert_eq!(13, batch.num_columns()));
+        let row_count: usize = results.iter().map(|batch| batch.num_rows()).sum();
+        assert_eq!(41, row_count);
+
+        Ok(())
+    }
+}
diff --git a/rust/datafusion/src/physical_plan/functions.rs b/rust/datafusion/src/physical_plan/functions.rs
new file mode 100644
index 00000000000..d0a121139d8
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/functions.rs
@@ -0,0 +1,518 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Declaration of built-in (scalar) functions.
+//! This module contains built-in functions' enumeration and metadata.
+//!
+//! Generally, a function has:
+//! * a signature
+//! * a return type, that is a function of the incoming argument's types
+//! * the computation, that must accept each valid signature
+//!
+//! * Signature: see `Signature`
+//! * Return type: a function `(arg_types) -> return_type`. E.g. for sqrt, ([f32]) -> f32, ([f64]) -> f64.
+//!
+//! This module also has a set of coercion rules to improve user experience: if an argument i32 is passed
+//! to a function that supports f64, it is coerced to f64.
+
+use super::{
+    type_coercion::{coerce, data_types},
+    PhysicalExpr,
+};
+use crate::error::{DataFusionError, Result};
+use crate::physical_plan::array_expressions;
+use crate::physical_plan::datetime_expressions;
+use crate::physical_plan::math_expressions;
+use crate::physical_plan::string_expressions;
+use arrow::{
+    array::ArrayRef,
+    compute::kernels::length::length,
+    datatypes::TimeUnit,
+    datatypes::{DataType, Field, Schema},
+    record_batch::RecordBatch,
+};
+use fmt::{Debug, Formatter};
+use std::{fmt, str::FromStr, sync::Arc};
+
+/// A function's signature, which defines the function's supported argument types.
+#[derive(Debug, Clone)]
+pub enum Signature {
+    /// arbitrary number of arguments of an common type out of a list of valid types
+    // A function such as `concat` is `Variadic(vec![DataType::Utf8, DataType::LargeUtf8])`
+    Variadic(Vec<DataType>),
+    /// arbitrary number of arguments of an arbitrary but equal type
+    // A function such as `array` is `VariadicEqual`
+    // The first argument decides the type used for coercion
+    VariadicEqual,
+    /// fixed number of arguments of an arbitrary but equal type out of a list of valid types
+    // A function of one argument of f64 is `Uniform(1, vec![DataType::Float64])`
+    // A function of two arguments of f64 or f32 is `Uniform(1, vec![DataType::Float32, DataType::Float64])`
+    Uniform(usize, Vec<DataType>),
+    /// exact number of arguments of an exact type
+    Exact(Vec<DataType>),
+    /// fixed number of arguments of arbitrary types
+    Any(usize),
+}
+
+/// Scalar function
+pub type ScalarFunctionImplementation =
+    Arc<dyn Fn(&[ArrayRef]) -> Result<ArrayRef> + Send + Sync>;
+
+/// A function's return type
+pub type ReturnTypeFunction =
+    Arc<dyn Fn(&[DataType]) -> Result<Arc<DataType>> + Send + Sync>;
+
+/// Enum of all built-in scalar functions
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum BuiltinScalarFunction {
+    /// sqrt
+    Sqrt,
+    /// sin
+    Sin,
+    /// cos
+    Cos,
+    /// tan
+    Tan,
+    /// asin
+    Asin,
+    /// acos
+    Acos,
+    /// atan
+    Atan,
+    /// exp
+    Exp,
+    /// log, also known as ln
+    Log,
+    /// log2
+    Log2,
+    /// log10
+    Log10,
+    /// floor
+    Floor,
+    /// ceil
+    Ceil,
+    /// round
+    Round,
+    /// trunc
+    Trunc,
+    /// abs
+    Abs,
+    /// signum
+    Signum,
+    /// length
+    Length,
+    /// concat
+    Concat,
+    /// to_timestamp
+    ToTimestamp,
+    /// construct an array from columns
+    Array,
+}
+
+impl fmt::Display for BuiltinScalarFunction {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        // lowercase of the debug.
+        write!(f, "{}", format!("{:?}", self).to_lowercase())
+    }
+}
+
+impl FromStr for BuiltinScalarFunction {
+    type Err = DataFusionError;
+    fn from_str(name: &str) -> Result<BuiltinScalarFunction> {
+        Ok(match name {
+            "sqrt" => BuiltinScalarFunction::Sqrt,
+            "sin" => BuiltinScalarFunction::Sin,
+            "cos" => BuiltinScalarFunction::Cos,
+            "tan" => BuiltinScalarFunction::Tan,
+            "asin" => BuiltinScalarFunction::Asin,
+            "acos" => BuiltinScalarFunction::Acos,
+            "atan" => BuiltinScalarFunction::Atan,
+            "exp" => BuiltinScalarFunction::Exp,
+            "log" => BuiltinScalarFunction::Log,
+            "log2" => BuiltinScalarFunction::Log2,
+            "log10" => BuiltinScalarFunction::Log10,
+            "floor" => BuiltinScalarFunction::Floor,
+            "ceil" => BuiltinScalarFunction::Ceil,
+            "round" => BuiltinScalarFunction::Round,
+            "truc" => BuiltinScalarFunction::Trunc,
+            "abs" => BuiltinScalarFunction::Abs,
+            "signum" => BuiltinScalarFunction::Signum,
+            "length" => BuiltinScalarFunction::Length,
+            "concat" => BuiltinScalarFunction::Concat,
+            "to_timestamp" => BuiltinScalarFunction::ToTimestamp,
+            "array" => BuiltinScalarFunction::Array,
+            _ => {
+                return Err(DataFusionError::Plan(format!(
+                    "There is no built-in function named {}",
+                    name
+                )))
+            }
+        })
+    }
+}
+
+/// Returns the datatype of the scalar function
+pub fn return_type(
+    fun: &BuiltinScalarFunction,
+    arg_types: &Vec<DataType>,
+) -> Result<DataType> {
+    // Note that this function *must* return the same type that the respective physical expression returns
+    // or the execution panics.
+
+    // verify that this is a valid set of data types for this function
+    data_types(&arg_types, &signature(fun))?;
+
+    if arg_types.len() == 0 {
+        // functions currently cannot be evaluated without arguments, as they can't
+        // know the number of rows to return.
+        return Err(DataFusionError::Plan(
+            format!("Function '{}' requires at least one argument", fun).to_string(),
+        ));
+    }
+
+    // the return type of the built in function. Eventually there
+    // will be built-in functions whose return type depends on the
+    // incoming type.
+    match fun {
+        BuiltinScalarFunction::Length => Ok(match arg_types[0] {
+            DataType::LargeUtf8 => DataType::Int64,
+            DataType::Utf8 => DataType::Int32,
+            _ => {
+                // this error is internal as `data_types` should have captured this.
+                return Err(DataFusionError::Internal(
+                    "The length function can only accept strings.".to_string(),
+                ));
+            }
+        }),
+        BuiltinScalarFunction::Concat => Ok(DataType::Utf8),
+        BuiltinScalarFunction::ToTimestamp => {
+            Ok(DataType::Timestamp(TimeUnit::Nanosecond, None))
+        }
+        BuiltinScalarFunction::Array => Ok(DataType::FixedSizeList(
+            Box::new(Field::new("item", arg_types[0].clone(), true)),
+            arg_types.len() as i32,
+        )),
+        _ => Ok(DataType::Float64),
+    }
+}
+
+/// Create a physical (function) expression.
+/// This function errors when `args`' can't be coerced to a valid argument type of the function.
+pub fn create_physical_expr(
+    fun: &BuiltinScalarFunction,
+    args: &Vec<Arc<dyn PhysicalExpr>>,
+    input_schema: &Schema,
+) -> Result<Arc<dyn PhysicalExpr>> {
+    let fun_expr: ScalarFunctionImplementation = Arc::new(match fun {
+        BuiltinScalarFunction::Sqrt => math_expressions::sqrt,
+        BuiltinScalarFunction::Sin => math_expressions::sin,
+        BuiltinScalarFunction::Cos => math_expressions::cos,
+        BuiltinScalarFunction::Tan => math_expressions::tan,
+        BuiltinScalarFunction::Asin => math_expressions::asin,
+        BuiltinScalarFunction::Acos => math_expressions::acos,
+        BuiltinScalarFunction::Atan => math_expressions::atan,
+        BuiltinScalarFunction::Exp => math_expressions::exp,
+        BuiltinScalarFunction::Log => math_expressions::ln,
+        BuiltinScalarFunction::Log2 => math_expressions::log2,
+        BuiltinScalarFunction::Log10 => math_expressions::log10,
+        BuiltinScalarFunction::Floor => math_expressions::floor,
+        BuiltinScalarFunction::Ceil => math_expressions::ceil,
+        BuiltinScalarFunction::Round => math_expressions::round,
+        BuiltinScalarFunction::Trunc => math_expressions::trunc,
+        BuiltinScalarFunction::Abs => math_expressions::abs,
+        BuiltinScalarFunction::Signum => math_expressions::signum,
+        BuiltinScalarFunction::Length => |args| Ok(length(args[0].as_ref())?),
+        BuiltinScalarFunction::Concat => {
+            |args| Ok(Arc::new(string_expressions::concatenate(args)?))
+        }
+        BuiltinScalarFunction::ToTimestamp => {
+            |args| Ok(Arc::new(datetime_expressions::to_timestamp(args)?))
+        }
+        BuiltinScalarFunction::Array => |args| Ok(array_expressions::array(args)?),
+    });
+    // coerce
+    let args = coerce(args, input_schema, &signature(fun))?;
+
+    let arg_types = args
+        .iter()
+        .map(|e| e.data_type(input_schema))
+        .collect::<Result<Vec<_>>>()?;
+
+    Ok(Arc::new(ScalarFunctionExpr::new(
+        &format!("{}", fun),
+        fun_expr,
+        args,
+        &return_type(&fun, &arg_types)?,
+    )))
+}
+
+/// the signatures supported by the function `fun`.
+fn signature(fun: &BuiltinScalarFunction) -> Signature {
+    // note: the physical expression must accept the type returned by this function or the execution panics.
+
+    // for now, the list is small, as we do not have many built-in functions.
+    match fun {
+        BuiltinScalarFunction::Length => {
+            Signature::Uniform(1, vec![DataType::Utf8, DataType::LargeUtf8])
+        }
+        BuiltinScalarFunction::Concat => Signature::Variadic(vec![DataType::Utf8]),
+        BuiltinScalarFunction::ToTimestamp => Signature::Uniform(1, vec![DataType::Utf8]),
+        BuiltinScalarFunction::Array => {
+            Signature::Variadic(array_expressions::SUPPORTED_ARRAY_TYPES.to_vec())
+        }
+        // math expressions expect 1 argument of type f64 or f32
+        // priority is given to f64 because e.g. `sqrt(1i32)` is in IR (real numbers) and thus we
+        // return the best approximation for it (in f64).
+        // We accept f32 because in this case it is clear that the best approximation
+        // will be as good as the number of digits in the number
+        _ => Signature::Uniform(1, vec![DataType::Float64, DataType::Float32]),
+    }
+}
+
+/// Physical expression of a scalar function
+pub struct ScalarFunctionExpr {
+    fun: ScalarFunctionImplementation,
+    name: String,
+    args: Vec<Arc<dyn PhysicalExpr>>,
+    return_type: DataType,
+}
+
+impl Debug for ScalarFunctionExpr {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        f.debug_struct("ScalarFunctionExpr")
+            .field("fun", &"<FUNC>")
+            .field("name", &self.name)
+            .field("args", &self.args)
+            .field("return_type", &self.return_type)
+            .finish()
+    }
+}
+
+impl ScalarFunctionExpr {
+    /// Create a new Scalar function
+    pub fn new(
+        name: &str,
+        fun: ScalarFunctionImplementation,
+        args: Vec<Arc<dyn PhysicalExpr>>,
+        return_type: &DataType,
+    ) -> Self {
+        Self {
+            fun,
+            name: name.to_owned(),
+            args,
+            return_type: return_type.clone(),
+        }
+    }
+}
+
+impl fmt::Display for ScalarFunctionExpr {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "{}({})",
+            self.name,
+            self.args
+                .iter()
+                .map(|e| format!("{}", e))
+                .collect::<Vec<String>>()
+                .join(", ")
+        )
+    }
+}
+
+impl PhysicalExpr for ScalarFunctionExpr {
+    fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
+        Ok(self.return_type.clone())
+    }
+
+    fn nullable(&self, _input_schema: &Schema) -> Result<bool> {
+        Ok(true)
+    }
+
+    fn evaluate(&self, batch: &RecordBatch) -> Result<ArrayRef> {
+        // evaluate the arguments
+        let inputs = self
+            .args
+            .iter()
+            .map(|e| e.evaluate(batch))
+            .collect::<Result<Vec<_>>>()?;
+
+        // evaluate the function
+        let fun = self.fun.as_ref();
+        (fun)(&inputs)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{error::Result, physical_plan::expressions::lit, scalar::ScalarValue};
+    use arrow::{
+        array::{ArrayRef, FixedSizeListArray, Float64Array, Int32Array, StringArray},
+        datatypes::Field,
+        record_batch::RecordBatch,
+    };
+
+    fn generic_test_math(value: ScalarValue, expected: &str) -> Result<()> {
+        // any type works here: we evaluate against a literal of `value`
+        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+        let columns: Vec<ArrayRef> = vec![Arc::new(Int32Array::from(vec![1]))];
+
+        let arg = lit(value);
+
+        let expr =
+            create_physical_expr(&BuiltinScalarFunction::Exp, &vec![arg], &schema)?;
+
+        // type is correct
+        assert_eq!(expr.data_type(&schema)?, DataType::Float64);
+
+        // evaluate works
+        let result =
+            expr.evaluate(&RecordBatch::try_new(Arc::new(schema.clone()), columns)?)?;
+
+        // downcast works
+        let result = result.as_any().downcast_ref::<Float64Array>().unwrap();
+
+        // value is correct
+        assert_eq!(format!("{}", result.value(0)), expected);
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_math_function() -> Result<()> {
+        // 2.71828182845904523536... : https://oeis.org/A001113
+        let exp_f64 = "2.718281828459045";
+        let exp_f32 = "2.7182817459106445";
+        generic_test_math(ScalarValue::from(1i32), exp_f64)?;
+        generic_test_math(ScalarValue::from(1u32), exp_f64)?;
+        generic_test_math(ScalarValue::from(1u64), exp_f64)?;
+        generic_test_math(ScalarValue::from(1f64), exp_f64)?;
+        generic_test_math(ScalarValue::from(1f32), exp_f32)?;
+        Ok(())
+    }
+
+    fn test_concat(value: ScalarValue, expected: &str) -> Result<()> {
+        // any type works here: we evaluate against a literal of `value`
+        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+        let columns: Vec<ArrayRef> = vec![Arc::new(Int32Array::from(vec![1]))];
+
+        // concat(value, value)
+        let expr = create_physical_expr(
+            &BuiltinScalarFunction::Concat,
+            &vec![lit(value.clone()), lit(value)],
+            &schema,
+        )?;
+
+        // type is correct
+        assert_eq!(expr.data_type(&schema)?, DataType::Utf8);
+
+        // evaluate works
+        let result =
+            expr.evaluate(&RecordBatch::try_new(Arc::new(schema.clone()), columns)?)?;
+
+        // downcast works
+        let result = result.as_any().downcast_ref::<StringArray>().unwrap();
+
+        // value is correct
+        assert_eq!(format!("{}", result.value(0)), expected);
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_concat_utf8() -> Result<()> {
+        test_concat(ScalarValue::Utf8(Some("aa".to_string())), "aaaa")
+    }
+
+    #[test]
+    fn test_concat_error() -> Result<()> {
+        let result = return_type(&BuiltinScalarFunction::Concat, &vec![]);
+        if let Ok(_) = result {
+            Err(DataFusionError::Plan(
+                "Function 'concat' cannot accept zero arguments".to_string(),
+            ))
+        } else {
+            Ok(())
+        }
+    }
+
+    fn generic_test_array(
+        value1: ScalarValue,
+        value2: ScalarValue,
+        expected_type: DataType,
+        expected: &str,
+    ) -> Result<()> {
+        // any type works here: we evaluate against a literal of `value`
+        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+        let columns: Vec<ArrayRef> = vec![Arc::new(Int32Array::from(vec![1]))];
+
+        let expr = create_physical_expr(
+            &BuiltinScalarFunction::Array,
+            &vec![lit(value1.clone()), lit(value2.clone())],
+            &schema,
+        )?;
+
+        // type is correct
+        assert_eq!(
+            expr.data_type(&schema)?,
+            // type equals to a common coercion
+            DataType::FixedSizeList(Box::new(Field::new("item", expected_type, true)), 2)
+        );
+
+        // evaluate works
+        let result =
+            expr.evaluate(&RecordBatch::try_new(Arc::new(schema.clone()), columns)?)?;
+
+        // downcast works
+        let result = result
+            .as_any()
+            .downcast_ref::<FixedSizeListArray>()
+            .unwrap();
+
+        // value is correct
+        assert_eq!(format!("{:?}", result.value(0)), expected);
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_array() -> Result<()> {
+        generic_test_array(
+            ScalarValue::Utf8(Some("aa".to_string())),
+            ScalarValue::Utf8(Some("aa".to_string())),
+            DataType::Utf8,
+            "StringArray\n[\n  \"aa\",\n  \"aa\",\n]",
+        )?;
+
+        // different types, to validate that casting happens
+        generic_test_array(
+            ScalarValue::from(1u32),
+            ScalarValue::from(1u64),
+            DataType::UInt64,
+            "PrimitiveArray<UInt64>\n[\n  1,\n  1,\n]",
+        )?;
+
+        // different types (another order), to validate that casting happens
+        generic_test_array(
+            ScalarValue::from(1u64),
+            ScalarValue::from(1u32),
+            DataType::UInt64,
+            "PrimitiveArray<UInt64>\n[\n  1,\n  1,\n]",
+        )
+    }
+}
diff --git a/rust/datafusion/src/physical_plan/group_scalar.rs b/rust/datafusion/src/physical_plan/group_scalar.rs
new file mode 100644
index 00000000000..bb1e204c7f5
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/group_scalar.rs
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Defines scalars used to construct groups, ex. in GROUP BY clauses.
+
+use std::convert::{From, TryFrom};
+
+use crate::error::{DataFusionError, Result};
+use crate::scalar::ScalarValue;
+
+/// Enumeration of types that can be used in a GROUP BY expression (all primitives except
+/// for floating point numerics)
+#[derive(Debug, PartialEq, Eq, Hash, Clone)]
+pub(crate) enum GroupByScalar {
+    UInt8(u8),
+    UInt16(u16),
+    UInt32(u32),
+    UInt64(u64),
+    Int8(i8),
+    Int16(i16),
+    Int32(i32),
+    Int64(i64),
+    Utf8(String),
+}
+
+impl TryFrom<&ScalarValue> for GroupByScalar {
+    type Error = DataFusionError;
+
+    fn try_from(scalar_value: &ScalarValue) -> Result<Self> {
+        Ok(match scalar_value {
+            ScalarValue::Int8(Some(v)) => GroupByScalar::Int8(*v),
+            ScalarValue::Int16(Some(v)) => GroupByScalar::Int16(*v),
+            ScalarValue::Int32(Some(v)) => GroupByScalar::Int32(*v),
+            ScalarValue::Int64(Some(v)) => GroupByScalar::Int64(*v),
+            ScalarValue::UInt8(Some(v)) => GroupByScalar::UInt8(*v),
+            ScalarValue::UInt16(Some(v)) => GroupByScalar::UInt16(*v),
+            ScalarValue::UInt32(Some(v)) => GroupByScalar::UInt32(*v),
+            ScalarValue::UInt64(Some(v)) => GroupByScalar::UInt64(*v),
+            ScalarValue::Utf8(Some(v)) => GroupByScalar::Utf8(v.clone()),
+            ScalarValue::Int8(None)
+            | ScalarValue::Int16(None)
+            | ScalarValue::Int32(None)
+            | ScalarValue::Int64(None)
+            | ScalarValue::UInt8(None)
+            | ScalarValue::UInt16(None)
+            | ScalarValue::UInt32(None)
+            | ScalarValue::UInt64(None)
+            | ScalarValue::Utf8(None) => {
+                return Err(DataFusionError::Internal(format!(
+                    "Cannot convert a ScalarValue holding NULL ({:?})",
+                    scalar_value
+                )));
+            }
+            v => {
+                return Err(DataFusionError::Internal(format!(
+                    "Cannot convert a ScalarValue with associated DataType {:?}",
+                    v.get_datatype()
+                )))
+            }
+        })
+    }
+}
+
+impl From<&GroupByScalar> for ScalarValue {
+    fn from(group_by_scalar: &GroupByScalar) -> Self {
+        match group_by_scalar {
+            GroupByScalar::Int8(v) => ScalarValue::Int8(Some(*v)),
+            GroupByScalar::Int16(v) => ScalarValue::Int16(Some(*v)),
+            GroupByScalar::Int32(v) => ScalarValue::Int32(Some(*v)),
+            GroupByScalar::Int64(v) => ScalarValue::Int64(Some(*v)),
+            GroupByScalar::UInt8(v) => ScalarValue::UInt8(Some(*v)),
+            GroupByScalar::UInt16(v) => ScalarValue::UInt16(Some(*v)),
+            GroupByScalar::UInt32(v) => ScalarValue::UInt32(Some(*v)),
+            GroupByScalar::UInt64(v) => ScalarValue::UInt64(Some(*v)),
+            GroupByScalar::Utf8(v) => ScalarValue::Utf8(Some(v.clone())),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::error::{DataFusionError, Result};
+
+    #[test]
+    fn from_scalar_holding_none() -> Result<()> {
+        let scalar_value = ScalarValue::Int8(None);
+        let result = GroupByScalar::try_from(&scalar_value);
+
+        match result {
+            Err(DataFusionError::Internal(error_message)) => assert_eq!(
+                error_message,
+                String::from("Cannot convert a ScalarValue holding NULL (Int8(NULL))")
+            ),
+            _ => panic!("Unexpected result"),
+        }
+
+        Ok(())
+    }
+
+    #[test]
+    fn from_scalar_unsupported() -> Result<()> {
+        // Use any ScalarValue type not supported by GroupByScalar.
+        let scalar_value = ScalarValue::Float32(Some(1.1));
+        let result = GroupByScalar::try_from(&scalar_value);
+
+        match result {
+            Err(DataFusionError::Internal(error_message)) => assert_eq!(
+                error_message,
+                String::from(
+                    "Cannot convert a ScalarValue with associated DataType Float32"
+                )
+            ),
+            _ => panic!("Unexpected result"),
+        }
+
+        Ok(())
+    }
+}
diff --git a/rust/datafusion/src/physical_plan/hash_aggregate.rs b/rust/datafusion/src/physical_plan/hash_aggregate.rs
new file mode 100644
index 00000000000..8d7beadb740
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/hash_aggregate.rs
@@ -0,0 +1,889 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Defines the execution plan for the hash aggregate operation
+
+use std::any::Any;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+
+use futures::stream::{Stream, StreamExt, TryStreamExt};
+use futures::FutureExt;
+
+use crate::error::{DataFusionError, Result};
+use crate::physical_plan::{Accumulator, AggregateExpr};
+use crate::physical_plan::{Distribution, ExecutionPlan, Partitioning, PhysicalExpr};
+
+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use arrow::error::Result as ArrowResult;
+use arrow::record_batch::RecordBatch;
+use arrow::{
+    array::{
+        ArrayRef, Int16Array, Int32Array, Int64Array, Int8Array, StringArray,
+        UInt16Array, UInt32Array, UInt64Array, UInt8Array,
+    },
+    compute,
+};
+
+use fnv::FnvHashMap;
+
+use super::{
+    common, expressions::Column, group_scalar::GroupByScalar, RecordBatchStream,
+    SendableRecordBatchStream,
+};
+
+use async_trait::async_trait;
+
+/// Hash aggregate modes
+#[derive(Debug, Copy, Clone)]
+pub enum AggregateMode {
+    /// Partial aggregate that can be applied in parallel across input partitions
+    Partial,
+    /// Final aggregate that produces a single partition of output
+    Final,
+}
+
+/// Hash aggregate execution plan
+#[derive(Debug)]
+pub struct HashAggregateExec {
+    mode: AggregateMode,
+    group_expr: Vec<(Arc<dyn PhysicalExpr>, String)>,
+    aggr_expr: Vec<Arc<dyn AggregateExpr>>,
+    input: Arc<dyn ExecutionPlan>,
+    schema: SchemaRef,
+}
+
+fn create_schema(
+    input_schema: &Schema,
+    group_expr: &Vec<(Arc<dyn PhysicalExpr>, String)>,
+    aggr_expr: &Vec<Arc<dyn AggregateExpr>>,
+    mode: AggregateMode,
+) -> Result<Schema> {
+    let mut fields = Vec::with_capacity(group_expr.len() + aggr_expr.len());
+    for (expr, name) in group_expr {
+        fields.push(Field::new(
+            name,
+            expr.data_type(&input_schema)?,
+            expr.nullable(&input_schema)?,
+        ))
+    }
+
+    match mode {
+        AggregateMode::Partial => {
+            // in partial mode, the fields of the accumulator's state
+            for expr in aggr_expr {
+                fields.extend(expr.state_fields()?.iter().cloned())
+            }
+        }
+        AggregateMode::Final => {
+            // in final mode, the field with the final result of the accumulator
+            for expr in aggr_expr {
+                fields.push(expr.field()?)
+            }
+        }
+    }
+
+    Ok(Schema::new(fields))
+}
+
+impl HashAggregateExec {
+    /// Create a new hash aggregate execution plan
+    pub fn try_new(
+        mode: AggregateMode,
+        group_expr: Vec<(Arc<dyn PhysicalExpr>, String)>,
+        aggr_expr: Vec<Arc<dyn AggregateExpr>>,
+        input: Arc<dyn ExecutionPlan>,
+    ) -> Result<Self> {
+        let schema = create_schema(&input.schema(), &group_expr, &aggr_expr, mode)?;
+
+        let schema = Arc::new(schema);
+
+        Ok(HashAggregateExec {
+            mode,
+            group_expr,
+            aggr_expr,
+            input,
+            schema,
+        })
+    }
+}
+
+#[async_trait]
+impl ExecutionPlan for HashAggregateExec {
+    /// Return a reference to Any that can be used for downcasting
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+
+    fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
+        vec![self.input.clone()]
+    }
+
+    fn required_child_distribution(&self) -> Distribution {
+        match &self.mode {
+            AggregateMode::Partial => Distribution::UnspecifiedDistribution,
+            AggregateMode::Final => Distribution::SinglePartition,
+        }
+    }
+
+    /// Get the output partitioning of this plan
+    fn output_partitioning(&self) -> Partitioning {
+        self.input.output_partitioning()
+    }
+
+    async fn execute(&self, partition: usize) -> Result<SendableRecordBatchStream> {
+        let input = self.input.execute(partition).await?;
+        let group_expr = self.group_expr.iter().map(|x| x.0.clone()).collect();
+
+        if self.group_expr.is_empty() {
+            Ok(Box::pin(HashAggregateStream::new(
+                self.mode,
+                self.schema.clone(),
+                self.aggr_expr.clone(),
+                input,
+            )))
+        } else {
+            Ok(Box::pin(GroupedHashAggregateStream::new(
+                self.mode.clone(),
+                self.schema.clone(),
+                group_expr,
+                self.aggr_expr.clone(),
+                input,
+            )))
+        }
+    }
+
+    fn with_new_children(
+        &self,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        match children.len() {
+            1 => Ok(Arc::new(HashAggregateExec::try_new(
+                self.mode,
+                self.group_expr.clone(),
+                self.aggr_expr.clone(),
+                children[0].clone(),
+            )?)),
+            _ => Err(DataFusionError::Internal(
+                "HashAggregateExec wrong number of children".to_string(),
+            )),
+        }
+    }
+}
+
+/*
+The architecture is the following:
+
+1. An accumulator has state that is updated on each batch.
+2. At the end of the aggregation (e.g. end of batches in a partition), the accumulator converts its state to a RecordBatch of a single row
+3. The RecordBatches of all accumulators are merged (`concatenate` in `rust/arrow`) together to a single RecordBatch.
+4. The state's RecordBatch is `merge`d to a new state
+5. The state is mapped to the final value
+
+Why:
+
+* Accumulators' state can be statically typed, but it is more efficient to transmit data from the accumulators via `Array`
+* The `merge` operation must have access to the state of the aggregators because it uses it to correctly merge
+* It uses Arrow's native dynamically typed object, `Array`.
+* Arrow shines in batch operations and both `merge` and `concatenate` of uniform types are very performant.
+
+Example: average
+
+* the state is `n: u32` and `sum: f64`
+* For every batch, we update them accordingly.
+* At the end of the accumulation (of a partition), we convert `n` and `sum` to a RecordBatch of 1 row and two columns: `[n, sum]`
+* The RecordBatch is (sent back / transmitted over network)
+* Once all N record batches arrive, `merge` is performed, which builds a RecordBatch with N rows and 2 columns.
+* Finally, `get_value` returns an array with one entry computed from the state
+*/
+struct GroupedHashAggregateStream {
+    mode: AggregateMode,
+    schema: SchemaRef,
+    group_expr: Vec<Arc<dyn PhysicalExpr>>,
+    aggr_expr: Vec<Arc<dyn AggregateExpr>>,
+    input: SendableRecordBatchStream,
+    finished: bool,
+}
+
+fn group_aggregate_batch(
+    mode: &AggregateMode,
+    group_expr: &Vec<Arc<dyn PhysicalExpr>>,
+    aggr_expr: &Vec<Arc<dyn AggregateExpr>>,
+    batch: RecordBatch,
+    mut accumulators: Accumulators,
+    aggregate_expressions: &Vec<Vec<Arc<dyn PhysicalExpr>>>,
+) -> Result<Accumulators> {
+    // evaluate the grouping expressions
+    let group_values = evaluate(group_expr, &batch)?;
+
+    // evaluate the aggregation expressions.
+    // We could evaluate them after the `take`, but since we need to evaluate all
+    // of them anyways, it is more performant to do it while they are together.
+    let aggr_input_values = evaluate_many(aggregate_expressions, &batch)?;
+
+    // create vector large enough to hold the grouping key
+    // this is an optimization to avoid allocating `key` on every row.
+    // it will be overwritten on every iteration of the loop below
+    let mut key = Vec::with_capacity(group_values.len());
+    for _ in 0..group_values.len() {
+        key.push(GroupByScalar::UInt32(0));
+    }
+
+    // 1.1 construct the key from the group values
+    // 1.2 construct the mapping key if it does not exist
+    // 1.3 add the row' index to `indices`
+    for row in 0..batch.num_rows() {
+        // 1.1
+        create_key(&group_values, row, &mut key)
+            .map_err(DataFusionError::into_arrow_external_error)?;
+
+        match accumulators.get_mut(&key) {
+            // 1.2
+            None => {
+                let accumulator_set = create_accumulators(aggr_expr)
+                    .map_err(DataFusionError::into_arrow_external_error)?;
+
+                accumulators
+                    .insert(key.clone(), (accumulator_set, Box::new(vec![row as u32])));
+            }
+            // 1.3
+            Some((_, v)) => v.push(row as u32),
+        }
+    }
+
+    // 2.1 for each key
+    // 2.2 for each aggregation
+    // 2.3 `take` from each of its arrays the keys' values
+    // 2.4 update / merge the accumulator with the values
+    // 2.5 clear indices
+    accumulators
+        .iter_mut()
+        // 2.1
+        .map(|(_, (accumulator_set, indices))| {
+            // 2.2
+            accumulator_set
+                .into_iter()
+                .zip(&aggr_input_values)
+                .map(|(accumulator, aggr_array)| {
+                    (
+                        accumulator,
+                        aggr_array
+                            .iter()
+                            .map(|array| {
+                                // 2.3
+                                compute::take(
+                                    array,
+                                    &UInt32Array::from(*indices.clone()),
+                                    None, // None: no index check
+                                )
+                                .unwrap()
+                            })
+                            .collect::<Vec<ArrayRef>>(),
+                    )
+                })
+                // 2.4
+                .map(|(accumulator, values)| match mode {
+                    AggregateMode::Partial => accumulator.update_batch(&values),
+                    AggregateMode::Final => {
+                        // note: the aggregation here is over states, not values, thus the merge
+                        accumulator.merge_batch(&values)
+                    }
+                })
+                .collect::<Result<()>>()
+                // 2.5
+                .and(Ok(indices.clear()))
+        })
+        .collect::<Result<()>>()?;
+    Ok(accumulators)
+}
+
+impl GroupedHashAggregateStream {
+    /// Create a new HashAggregateStream
+    pub fn new(
+        mode: AggregateMode,
+        schema: SchemaRef,
+        group_expr: Vec<Arc<dyn PhysicalExpr>>,
+        aggr_expr: Vec<Arc<dyn AggregateExpr>>,
+        input: SendableRecordBatchStream,
+    ) -> Self {
+        GroupedHashAggregateStream {
+            mode,
+            schema,
+            group_expr,
+            aggr_expr,
+            input,
+            finished: false,
+        }
+    }
+}
+
+type AccumulatorSet = Vec<Box<dyn Accumulator>>;
+type Accumulators = FnvHashMap<Vec<GroupByScalar>, (AccumulatorSet, Box<Vec<u32>>)>;
+
+impl Stream for GroupedHashAggregateStream {
+    type Item = ArrowResult<RecordBatch>;
+
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<Option<Self::Item>> {
+        if self.finished {
+            return Poll::Ready(None);
+        }
+
+        // return single batch
+        self.finished = true;
+
+        let mode = self.mode.clone();
+        let group_expr = self.group_expr.clone();
+        let aggr_expr = self.aggr_expr.clone();
+        let schema = self.schema.clone();
+
+        // the expressions to evaluate the batch, one vec of expressions per aggregation
+        let aggregate_expressions = match aggregate_expressions(&aggr_expr, &mode) {
+            Ok(e) => e,
+            Err(e) => {
+                return Poll::Ready(Some(Err(
+                    DataFusionError::into_arrow_external_error(e),
+                )))
+            }
+        };
+
+        // mapping key -> (set of accumulators, indices of the key in the batch)
+        // * the indexes are updated at each row
+        // * the accumulators are updated at the end of each batch
+        // * the indexes are `clear`ed at the end of each batch
+        //let mut accumulators: Accumulators = FnvHashMap::default();
+
+        // iterate over all input batches and update the accumulators
+        let future = self.input.as_mut().try_fold(
+            Accumulators::default(),
+            |accumulators, batch| async {
+                group_aggregate_batch(
+                    &mode,
+                    &group_expr,
+                    &aggr_expr,
+                    batch,
+                    accumulators,
+                    &aggregate_expressions,
+                )
+                .map_err(DataFusionError::into_arrow_external_error)
+            },
+        );
+
+        let future = future.map(|maybe_accumulators| {
+            maybe_accumulators.map(|accumulators| {
+                create_batch_from_map(&mode, &accumulators, group_expr.len(), &schema)
+            })?
+        });
+
+        // send the stream to the heap, so that it outlives this function.
+        let mut combined = Box::pin(future.into_stream());
+
+        combined.poll_next_unpin(cx)
+    }
+}
+
+impl RecordBatchStream for GroupedHashAggregateStream {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+}
+
+/// Evaluates expressions against a record batch.
+fn evaluate(
+    expr: &Vec<Arc<dyn PhysicalExpr>>,
+    batch: &RecordBatch,
+) -> Result<Vec<ArrayRef>> {
+    expr.iter()
+        .map(|expr| expr.evaluate(&batch))
+        .collect::<Result<Vec<_>>>()
+}
+
+/// Evaluates expressions against a record batch.
+fn evaluate_many(
+    expr: &Vec<Vec<Arc<dyn PhysicalExpr>>>,
+    batch: &RecordBatch,
+) -> Result<Vec<Vec<ArrayRef>>> {
+    expr.iter()
+        .map(|expr| evaluate(expr, batch))
+        .collect::<Result<Vec<_>>>()
+}
+
+/// uses `state_fields` to build a vec of expressions required to merge the AggregateExpr' accumulator's state.
+fn merge_expressions(
+    expr: &Arc<dyn AggregateExpr>,
+) -> Result<Vec<Arc<dyn PhysicalExpr>>> {
+    Ok(expr
+        .state_fields()?
+        .iter()
+        .map(|f| Arc::new(Column::new(f.name())) as Arc<dyn PhysicalExpr>)
+        .collect::<Vec<_>>())
+}
+
+/// returns physical expressions to evaluate against a batch
+/// The expressions are different depending on `mode`:
+/// * Partial: AggregateExpr::expressions
+/// * Final: columns of `AggregateExpr::state_fields()`
+/// The return value is to be understood as:
+/// * index 0 is the aggregation
+/// * index 1 is the expression i of the aggregation
+fn aggregate_expressions(
+    aggr_expr: &[Arc<dyn AggregateExpr>],
+    mode: &AggregateMode,
+) -> Result<Vec<Vec<Arc<dyn PhysicalExpr>>>> {
+    match mode {
+        AggregateMode::Partial => {
+            Ok(aggr_expr.iter().map(|agg| agg.expressions()).collect())
+        }
+        // in this mode, we build the merge expressions of the aggregation
+        AggregateMode::Final => Ok(aggr_expr
+            .iter()
+            .map(|agg| merge_expressions(agg))
+            .collect::<Result<Vec<_>>>()?),
+    }
+}
+
+struct HashAggregateStream {
+    mode: AggregateMode,
+    schema: SchemaRef,
+    aggr_expr: Vec<Arc<dyn AggregateExpr>>,
+    input: SendableRecordBatchStream,
+    finished: bool,
+}
+
+impl HashAggregateStream {
+    /// Create a new HashAggregateStream
+    pub fn new(
+        mode: AggregateMode,
+        schema: SchemaRef,
+        aggr_expr: Vec<Arc<dyn AggregateExpr>>,
+        input: SendableRecordBatchStream,
+    ) -> Self {
+        HashAggregateStream {
+            mode,
+            schema,
+            aggr_expr,
+            input,
+            finished: false,
+        }
+    }
+}
+
+fn aggregate_batch(
+    mode: &AggregateMode,
+    batch: &RecordBatch,
+    accumulators: AccumulatorSet,
+    expressions: &Vec<Vec<Arc<dyn PhysicalExpr>>>,
+) -> Result<AccumulatorSet> {
+    // 1.1 iterate accumulators and respective expressions together
+    // 1.2 evaluate expressions
+    // 1.3 update / merge accumulators with the expressions' values
+
+    // 1.1
+    accumulators
+        .into_iter()
+        .zip(expressions)
+        .map(|(mut accum, expr)| {
+            // 1.2
+            let values = &expr
+                .iter()
+                .map(|e| e.evaluate(batch))
+                .collect::<Result<Vec<_>>>()?;
+
+            // 1.3
+            match mode {
+                AggregateMode::Partial => {
+                    accum.update_batch(values)?;
+                }
+                AggregateMode::Final => {
+                    accum.merge_batch(values)?;
+                }
+            }
+            Ok(accum)
+        })
+        .collect::<Result<Vec<_>>>()
+}
+
+impl Stream for HashAggregateStream {
+    type Item = ArrowResult<RecordBatch>;
+
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<Option<Self::Item>> {
+        if self.finished {
+            return Poll::Ready(None);
+        }
+
+        // return single batch
+        self.finished = true;
+
+        let accumulators = match create_accumulators(&self.aggr_expr) {
+            Ok(e) => e,
+            Err(e) => {
+                return Poll::Ready(Some(Err(
+                    DataFusionError::into_arrow_external_error(e),
+                )))
+            }
+        };
+
+        let expressions = match aggregate_expressions(&self.aggr_expr, &self.mode) {
+            Ok(e) => e,
+            Err(e) => {
+                return Poll::Ready(Some(Err(
+                    DataFusionError::into_arrow_external_error(e),
+                )))
+            }
+        };
+        let expressions = Arc::new(expressions);
+
+        let mode = self.mode;
+        let schema = self.schema();
+
+        // 1 for each batch, update / merge accumulators with the expressions' values
+        // future is ready when all batches are computed
+        let future = self
+            .input
+            .as_mut()
+            .try_fold(
+                // pass the expressions on every fold to handle closures' mutability
+                (accumulators, expressions),
+                |(acc, expr), batch| async move {
+                    aggregate_batch(&mode, &batch, acc, &expr)
+                        .map_err(DataFusionError::into_arrow_external_error)
+                        .map(|agg| (agg, expr))
+                },
+            )
+            // pick the accumulators (disregard the expressions)
+            .map(|e| e.map(|e| e.0));
+
+        let future = future.map(|maybe_accumulators| {
+            maybe_accumulators.map(|accumulators| {
+                // 2. convert values to a record batch
+                finalize_aggregation(&accumulators, &mode)
+                    .map_err(DataFusionError::into_arrow_external_error)
+                    .and_then(|columns| RecordBatch::try_new(schema.clone(), columns))
+            })?
+        });
+
+        Box::pin(future.into_stream()).poll_next_unpin(cx)
+    }
+}
+
+impl RecordBatchStream for HashAggregateStream {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+}
+
+/// Given Vec<Vec<ArrayRef>>, concatenates the inners `Vec<ArrayRef>` into `ArrayRef`, returning `Vec<ArrayRef>`
+/// This assumes that `arrays` is not empty.
+fn concatenate(arrays: Vec<Vec<ArrayRef>>) -> ArrowResult<Vec<ArrayRef>> {
+    (0..arrays[0].len())
+        .map(|column| {
+            let array_list = arrays.iter().map(|a| a[column].clone()).collect::<Vec<_>>();
+            compute::concat(&array_list)
+        })
+        .collect::<ArrowResult<Vec<_>>>()
+}
+
+/// Create a RecordBatch with all group keys and accumulator' states or values.
+fn create_batch_from_map(
+    mode: &AggregateMode,
+    accumulators: &Accumulators,
+    num_group_expr: usize,
+    output_schema: &Schema,
+) -> ArrowResult<RecordBatch> {
+    // 1. for each key
+    // 2. create single-row ArrayRef with all group expressions
+    // 3. create single-row ArrayRef with all aggregate states or values
+    // 4. collect all in a vector per key of vec<ArrayRef>, vec[i][j]
+    // 5. concatenate the arrays over the second index [j] into a single vec<ArrayRef>.
+    let arrays = accumulators
+        .iter()
+        .map(|(k, (accumulator_set, _))| {
+            // 2.
+            let mut groups = (0..num_group_expr)
+                .map(|i| match &k[i] {
+                    GroupByScalar::Int8(n) => {
+                        Arc::new(Int8Array::from(vec![*n])) as ArrayRef
+                    }
+                    GroupByScalar::Int16(n) => Arc::new(Int16Array::from(vec![*n])),
+                    GroupByScalar::Int32(n) => Arc::new(Int32Array::from(vec![*n])),
+                    GroupByScalar::Int64(n) => Arc::new(Int64Array::from(vec![*n])),
+                    GroupByScalar::UInt8(n) => Arc::new(UInt8Array::from(vec![*n])),
+                    GroupByScalar::UInt16(n) => Arc::new(UInt16Array::from(vec![*n])),
+                    GroupByScalar::UInt32(n) => Arc::new(UInt32Array::from(vec![*n])),
+                    GroupByScalar::UInt64(n) => Arc::new(UInt64Array::from(vec![*n])),
+                    GroupByScalar::Utf8(str) => Arc::new(StringArray::from(vec![&**str])),
+                })
+                .collect::<Vec<ArrayRef>>();
+
+            // 3.
+            groups.extend(
+                finalize_aggregation(accumulator_set, mode)
+                    .map_err(DataFusionError::into_arrow_external_error)?,
+            );
+
+            Ok(groups)
+        })
+        // 4.
+        .collect::<ArrowResult<Vec<Vec<ArrayRef>>>>()?;
+
+    let batch = if arrays.len() != 0 {
+        // 5.
+        let columns = concatenate(arrays)?;
+        RecordBatch::try_new(Arc::new(output_schema.to_owned()), columns)?
+    } else {
+        common::create_batch_empty(output_schema)?
+    };
+    Ok(batch)
+}
+
+fn create_accumulators(
+    aggr_expr: &Vec<Arc<dyn AggregateExpr>>,
+) -> Result<AccumulatorSet> {
+    aggr_expr
+        .iter()
+        .map(|expr| expr.create_accumulator())
+        .collect::<Result<Vec<_>>>()
+}
+
+/// returns a vector of ArrayRefs, where each entry corresponds to either the
+/// final value (mode = Final) or states (mode = Partial)
+fn finalize_aggregation(
+    accumulators: &AccumulatorSet,
+    mode: &AggregateMode,
+) -> Result<Vec<ArrayRef>> {
+    match mode {
+        AggregateMode::Partial => {
+            // build the vector of states
+            let a = accumulators
+                .iter()
+                .map(|accumulator| accumulator.state())
+                .map(|value| {
+                    value.and_then(|e| {
+                        Ok(e.iter().map(|v| v.to_array()).collect::<Vec<ArrayRef>>())
+                    })
+                })
+                .collect::<Result<Vec<_>>>()?;
+            Ok(a.iter().flatten().cloned().collect::<Vec<_>>())
+        }
+        AggregateMode::Final => {
+            // merge the state to the final value
+            accumulators
+                .iter()
+                .map(|accumulator| accumulator.evaluate().and_then(|v| Ok(v.to_array())))
+                .collect::<Result<Vec<ArrayRef>>>()
+        }
+    }
+}
+
+/// Create a Vec<GroupByScalar> that can be used as a map key
+fn create_key(
+    group_by_keys: &[ArrayRef],
+    row: usize,
+    vec: &mut Vec<GroupByScalar>,
+) -> Result<()> {
+    for i in 0..group_by_keys.len() {
+        let col = &group_by_keys[i];
+        match col.data_type() {
+            DataType::UInt8 => {
+                let array = col.as_any().downcast_ref::<UInt8Array>().unwrap();
+                vec[i] = GroupByScalar::UInt8(array.value(row))
+            }
+            DataType::UInt16 => {
+                let array = col.as_any().downcast_ref::<UInt16Array>().unwrap();
+                vec[i] = GroupByScalar::UInt16(array.value(row))
+            }
+            DataType::UInt32 => {
+                let array = col.as_any().downcast_ref::<UInt32Array>().unwrap();
+                vec[i] = GroupByScalar::UInt32(array.value(row))
+            }
+            DataType::UInt64 => {
+                let array = col.as_any().downcast_ref::<UInt64Array>().unwrap();
+                vec[i] = GroupByScalar::UInt64(array.value(row))
+            }
+            DataType::Int8 => {
+                let array = col.as_any().downcast_ref::<Int8Array>().unwrap();
+                vec[i] = GroupByScalar::Int8(array.value(row))
+            }
+            DataType::Int16 => {
+                let array = col.as_any().downcast_ref::<Int16Array>().unwrap();
+                vec[i] = GroupByScalar::Int16(array.value(row))
+            }
+            DataType::Int32 => {
+                let array = col.as_any().downcast_ref::<Int32Array>().unwrap();
+                vec[i] = GroupByScalar::Int32(array.value(row))
+            }
+            DataType::Int64 => {
+                let array = col.as_any().downcast_ref::<Int64Array>().unwrap();
+                vec[i] = GroupByScalar::Int64(array.value(row))
+            }
+            DataType::Utf8 => {
+                let array = col.as_any().downcast_ref::<StringArray>().unwrap();
+                vec[i] = GroupByScalar::Utf8(String::from(array.value(row)))
+            }
+            _ => {
+                // This is internal because we should have caught this before.
+                return Err(DataFusionError::Internal(
+                    "Unsupported GROUP BY data type".to_string(),
+                ));
+            }
+        }
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+
+    use arrow::array::Float64Array;
+
+    use super::*;
+    use crate::physical_plan::expressions::{col, Avg};
+    use crate::physical_plan::merge::MergeExec;
+    use crate::physical_plan::{common, memory::MemoryExec};
+
+    fn some_data() -> ArrowResult<(Arc<Schema>, Vec<RecordBatch>)> {
+        // define a schema.
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("a", DataType::UInt32, false),
+            Field::new("b", DataType::Float64, false),
+        ]));
+
+        // define data.
+        Ok((
+            schema.clone(),
+            vec![
+                RecordBatch::try_new(
+                    schema.clone(),
+                    vec![
+                        Arc::new(UInt32Array::from(vec![2, 3, 4, 4])),
+                        Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0, 4.0])),
+                    ],
+                )?,
+                RecordBatch::try_new(
+                    schema.clone(),
+                    vec![
+                        Arc::new(UInt32Array::from(vec![2, 3, 3, 4])),
+                        Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0, 4.0])),
+                    ],
+                )?,
+            ],
+        ))
+    }
+
+    #[tokio::test]
+    async fn aggregate() -> Result<()> {
+        let (schema, batches) = some_data().unwrap();
+
+        let input: Arc<dyn ExecutionPlan> = Arc::new(
+            MemoryExec::try_new(&vec![batches.clone(), batches], schema, None).unwrap(),
+        );
+
+        let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =
+            vec![(col("a"), "a".to_string())];
+
+        let aggregates: Vec<Arc<dyn AggregateExpr>> = vec![Arc::new(Avg::new(
+            col("b"),
+            "AVG(b)".to_string(),
+            DataType::Float64,
+        ))];
+
+        let partial_aggregate = Arc::new(HashAggregateExec::try_new(
+            AggregateMode::Partial,
+            groups.clone(),
+            aggregates.clone(),
+            input,
+        )?);
+
+        let result = common::collect(partial_aggregate.execute(0).await?).await?;
+
+        let keys = result[0]
+            .column(0)
+            .as_any()
+            .downcast_ref::<UInt32Array>()
+            .unwrap();
+        assert_eq!(*keys, UInt32Array::from(vec![2, 3, 4]));
+
+        let ns = result[0]
+            .column(1)
+            .as_any()
+            .downcast_ref::<UInt64Array>()
+            .unwrap();
+        assert_eq!(*ns, UInt64Array::from(vec![2, 3, 3]));
+
+        let sums = result[0]
+            .column(2)
+            .as_any()
+            .downcast_ref::<Float64Array>()
+            .unwrap();
+        assert_eq!(*sums, Float64Array::from(vec![2.0, 7.0, 11.0]));
+
+        let merge = Arc::new(MergeExec::new(partial_aggregate));
+
+        let final_group: Vec<Arc<dyn PhysicalExpr>> =
+            (0..groups.len()).map(|i| col(&groups[i].1)).collect();
+
+        let merged_aggregate = Arc::new(HashAggregateExec::try_new(
+            AggregateMode::Final,
+            final_group
+                .iter()
+                .enumerate()
+                .map(|(i, expr)| (expr.clone(), groups[i].1.clone()))
+                .collect(),
+            aggregates,
+            merge,
+        )?);
+
+        let result = common::collect(merged_aggregate.execute(0).await?).await?;
+        assert_eq!(result.len(), 1);
+
+        let batch = &result[0];
+        assert_eq!(batch.num_columns(), 2);
+        assert_eq!(batch.num_rows(), 3);
+
+        let a = batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<UInt32Array>()
+            .unwrap();
+        let b = batch
+            .column(1)
+            .as_any()
+            .downcast_ref::<Float64Array>()
+            .unwrap();
+
+        assert_eq!(*a, UInt32Array::from(vec![2, 3, 4]));
+        assert_eq!(
+            *b,
+            Float64Array::from(vec![
+                1.0,
+                (2.0 + 3.0 + 2.0) / 3.0,
+                (3.0 + 4.0 + 4.0) / 3.0
+            ])
+        );
+
+        Ok(())
+    }
+}
diff --git a/rust/datafusion/src/physical_plan/limit.rs b/rust/datafusion/src/physical_plan/limit.rs
new file mode 100644
index 00000000000..b685ca43b6a
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/limit.rs
@@ -0,0 +1,259 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Defines the LIMIT plan
+
+use std::any::Any;
+use std::sync::Arc;
+
+use crate::error::{DataFusionError, Result};
+use crate::physical_plan::memory::MemoryStream;
+use crate::physical_plan::{Distribution, ExecutionPlan, Partitioning};
+use arrow::array::ArrayRef;
+use arrow::compute::limit;
+use arrow::datatypes::SchemaRef;
+use arrow::record_batch::RecordBatch;
+use futures::StreamExt;
+
+use super::SendableRecordBatchStream;
+
+use async_trait::async_trait;
+
+/// Limit execution plan
+#[derive(Debug)]
+pub struct GlobalLimitExec {
+    /// Input execution plan
+    input: Arc<dyn ExecutionPlan>,
+    /// Maximum number of rows to return
+    limit: usize,
+    /// Number of threads to run parallel LocalLimitExec on
+    concurrency: usize,
+}
+
+impl GlobalLimitExec {
+    /// Create a new MergeExec
+    pub fn new(input: Arc<dyn ExecutionPlan>, limit: usize, concurrency: usize) -> Self {
+        GlobalLimitExec {
+            input,
+            limit,
+            concurrency,
+        }
+    }
+}
+
+#[async_trait]
+impl ExecutionPlan for GlobalLimitExec {
+    /// Return a reference to Any that can be used for downcasting
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.input.schema()
+    }
+
+    fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
+        vec![self.input.clone()]
+    }
+
+    fn required_child_distribution(&self) -> Distribution {
+        Distribution::SinglePartition
+    }
+
+    /// Get the output partitioning of this plan
+    fn output_partitioning(&self) -> Partitioning {
+        Partitioning::UnknownPartitioning(1)
+    }
+
+    fn with_new_children(
+        &self,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        match children.len() {
+            1 => Ok(Arc::new(GlobalLimitExec::new(
+                children[0].clone(),
+                self.limit,
+                self.concurrency,
+            ))),
+            _ => Err(DataFusionError::Internal(
+                "GlobalLimitExec wrong number of children".to_string(),
+            )),
+        }
+    }
+
+    async fn execute(&self, partition: usize) -> Result<SendableRecordBatchStream> {
+        // GlobalLimitExec has a single output partition
+        if 0 != partition {
+            return Err(DataFusionError::Internal(format!(
+                "GlobalLimitExec invalid partition {}",
+                partition
+            )));
+        }
+
+        // GlobalLimitExec requires a single input partition
+        if 1 != self.input.output_partitioning().partition_count() {
+            return Err(DataFusionError::Internal(
+                "GlobalLimitExec requires a single input partition".to_owned(),
+            ));
+        }
+
+        let mut it = self.input.execute(0).await?;
+        Ok(Box::pin(MemoryStream::try_new(
+            collect_with_limit(&mut it, self.limit).await?,
+            self.input.schema(),
+            None,
+        )?))
+    }
+}
+
+/// LocalLimitExec applies a limit to a single partition
+#[derive(Debug)]
+pub struct LocalLimitExec {
+    input: Arc<dyn ExecutionPlan>,
+    limit: usize,
+}
+
+impl LocalLimitExec {
+    /// Create a new LocalLimitExec partition
+    pub fn new(input: Arc<dyn ExecutionPlan>, limit: usize) -> Self {
+        Self { input, limit }
+    }
+}
+
+#[async_trait]
+impl ExecutionPlan for LocalLimitExec {
+    /// Return a reference to Any that can be used for downcasting
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.input.schema()
+    }
+
+    fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
+        vec![self.input.clone()]
+    }
+
+    fn output_partitioning(&self) -> Partitioning {
+        self.input.output_partitioning()
+    }
+
+    fn with_new_children(
+        &self,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        match children.len() {
+            1 => Ok(Arc::new(LocalLimitExec::new(
+                children[0].clone(),
+                self.limit,
+            ))),
+            _ => Err(DataFusionError::Internal(
+                "LocalLimitExec wrong number of children".to_string(),
+            )),
+        }
+    }
+
+    async fn execute(&self, _: usize) -> Result<SendableRecordBatchStream> {
+        let mut it = self.input.execute(0).await?;
+        Ok(Box::pin(MemoryStream::try_new(
+            collect_with_limit(&mut it, self.limit).await?,
+            self.input.schema(),
+            None,
+        )?))
+    }
+}
+
+/// Truncate a RecordBatch to maximum of n rows
+pub fn truncate_batch(batch: &RecordBatch, n: usize) -> Result<RecordBatch> {
+    let limited_columns: Result<Vec<ArrayRef>> = (0..batch.num_columns())
+        .map(|i| limit(batch.column(i), n).map_err(|error| DataFusionError::from(error)))
+        .collect();
+
+    Ok(RecordBatch::try_new(
+        batch.schema().clone(),
+        limited_columns?,
+    )?)
+}
+
+/// Create a vector of record batches from an iterator
+async fn collect_with_limit(
+    reader: &mut SendableRecordBatchStream,
+    limit: usize,
+) -> Result<Vec<RecordBatch>> {
+    let mut count = 0;
+    let mut results: Vec<RecordBatch> = vec![];
+    loop {
+        match reader.as_mut().next().await {
+            Some(Ok(batch)) => {
+                let capacity = limit - count;
+                if batch.num_rows() <= capacity {
+                    count += batch.num_rows();
+                    results.push(batch);
+                } else {
+                    let batch = truncate_batch(&batch, capacity)?;
+                    count += batch.num_rows();
+                    results.push(batch);
+                }
+                if count == limit {
+                    return Ok(results);
+                }
+            }
+            None => {
+                return Ok(results);
+            }
+            Some(Err(e)) => return Err(DataFusionError::from(e)),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::*;
+    use crate::physical_plan::common;
+    use crate::physical_plan::csv::{CsvExec, CsvReadOptions};
+    use crate::physical_plan::merge::MergeExec;
+    use crate::test;
+
+    #[tokio::test]
+    async fn limit() -> Result<()> {
+        let schema = test::aggr_test_schema();
+
+        let num_partitions = 4;
+        let path =
+            test::create_partitioned_csv("aggregate_test_100.csv", num_partitions)?;
+
+        let csv =
+            CsvExec::try_new(&path, CsvReadOptions::new().schema(&schema), None, 1024)?;
+
+        // input should have 4 partitions
+        assert_eq!(csv.output_partitioning().partition_count(), num_partitions);
+
+        let limit = GlobalLimitExec::new(Arc::new(MergeExec::new(Arc::new(csv))), 7, 2);
+
+        // the result should contain 4 batches (one per input partition)
+        let iter = limit.execute(0).await?;
+        let batches = common::collect(iter).await?;
+
+        // there should be a total of 100 rows
+        let row_count: usize = batches.iter().map(|batch| batch.num_rows()).sum();
+        assert_eq!(row_count, 7);
+
+        Ok(())
+    }
+}
diff --git a/rust/datafusion/src/physical_plan/math_expressions.rs b/rust/datafusion/src/physical_plan/math_expressions.rs
new file mode 100644
index 00000000000..9ad0e2540df
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/math_expressions.rs
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Math expressions
+
+use std::sync::Arc;
+
+use arrow::array::{make_array, Array, ArrayData, ArrayRef, Float32Array, Float64Array};
+use arrow::buffer::Buffer;
+use arrow::datatypes::{DataType, ToByteSlice};
+
+use crate::error::{DataFusionError, Result};
+
+macro_rules! compute_op {
+    ($ARRAY:expr, $FUNC:ident, $TYPE:ident) => {{
+        let len = $ARRAY.len();
+        let result = (0..len)
+            .map(|i| $ARRAY.value(i).$FUNC() as f64)
+            .collect::<Vec<f64>>();
+        let data = ArrayData::new(
+            DataType::Float64,
+            len,
+            Some($ARRAY.null_count()),
+            $ARRAY.data().null_buffer().cloned(),
+            0,
+            vec![Buffer::from(result.to_byte_slice())],
+            vec![],
+        );
+        Ok(make_array(Arc::new(data)))
+    }};
+}
+
+macro_rules! downcast_compute_op {
+    ($ARRAY:expr, $NAME:expr, $FUNC:ident, $TYPE:ident) => {{
+        let n = $ARRAY.as_any().downcast_ref::<$TYPE>();
+        match n {
+            Some(array) => compute_op!(array, $FUNC, $TYPE),
+            _ => Err(DataFusionError::Internal(format!(
+                "Invalid data type for {}",
+                $NAME
+            ))),
+        }
+    }};
+}
+
+macro_rules! unary_primitive_array_op {
+    ($ARRAY:expr, $NAME:expr, $FUNC:ident) => {{
+        match ($ARRAY).data_type() {
+            DataType::Float32 => downcast_compute_op!($ARRAY, $NAME, $FUNC, Float32Array),
+            DataType::Float64 => downcast_compute_op!($ARRAY, $NAME, $FUNC, Float64Array),
+            other => Err(DataFusionError::Internal(format!(
+                "Unsupported data type {:?} for function {}",
+                other, $NAME,
+            ))),
+        }
+    }};
+}
+
+macro_rules! math_unary_function {
+    ($NAME:expr, $FUNC:ident) => {
+        /// mathematical function that accepts f32 or f64 and returns f64
+        pub fn $FUNC(args: &[ArrayRef]) -> Result<ArrayRef> {
+            unary_primitive_array_op!(args[0], $NAME, $FUNC)
+        }
+    };
+}
+
+math_unary_function!("sqrt", sqrt);
+math_unary_function!("sin", sin);
+math_unary_function!("cos", cos);
+math_unary_function!("tan", tan);
+math_unary_function!("asin", asin);
+math_unary_function!("acos", acos);
+math_unary_function!("atan", atan);
+math_unary_function!("floor", floor);
+math_unary_function!("ceil", ceil);
+math_unary_function!("round", round);
+math_unary_function!("trunc", trunc);
+math_unary_function!("abs", abs);
+math_unary_function!("signum", signum);
+math_unary_function!("exp", exp);
+math_unary_function!("log", ln);
+math_unary_function!("log2", log2);
+math_unary_function!("log10", log10);
diff --git a/rust/datafusion/src/execution/physical_plan/memory.rs b/rust/datafusion/src/physical_plan/memory.rs
similarity index 60%
rename from rust/datafusion/src/execution/physical_plan/memory.rs
rename to rust/datafusion/src/physical_plan/memory.rs
index 40c19eb6394..e7778f9d564 100644
--- a/rust/datafusion/src/execution/physical_plan/memory.rs
+++ b/rust/datafusion/src/physical_plan/memory.rs
@@ -17,13 +17,18 @@
 
 //! Execution plan for reading in-memory batches of data
 
-use std::sync::{Arc, Mutex};
+use std::any::Any;
+use std::sync::Arc;
+use std::task::{Context, Poll};
 
-use crate::error::Result;
-use crate::execution::physical_plan::{ExecutionPlan, Partition};
+use super::{ExecutionPlan, Partitioning, RecordBatchStream, SendableRecordBatchStream};
+use crate::error::{DataFusionError, Result};
 use arrow::datatypes::SchemaRef;
 use arrow::error::Result as ArrowResult;
-use arrow::record_batch::{RecordBatch, RecordBatchReader};
+use arrow::record_batch::RecordBatch;
+
+use async_trait::async_trait;
+use futures::Stream;
 
 /// Execution plan for reading in-memory batches of data
 #[derive(Debug)]
@@ -36,26 +41,44 @@ pub struct MemoryExec {
     projection: Option<Vec<usize>>,
 }
 
+#[async_trait]
 impl ExecutionPlan for MemoryExec {
+    /// Return a reference to Any that can be used for downcasting
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
     /// Get the schema for this execution plan
     fn schema(&self) -> SchemaRef {
         self.schema.clone()
     }
 
-    /// Get the partitions for this execution plan. Each partition can be executed in parallel.
-    fn partitions(&self) -> Result<Vec<Arc<dyn Partition>>> {
-        let partitions = self
-            .partitions
-            .iter()
-            .map(|vec| {
-                Arc::new(MemoryPartition::new(
-                    vec.clone(),
-                    self.schema.clone(),
-                    self.projection.clone(),
-                )) as Arc<dyn Partition>
-            })
-            .collect();
-        Ok(partitions)
+    fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
+        // this is a leaf node and has no children
+        vec![]
+    }
+
+    /// Get the output partitioning of this plan
+    fn output_partitioning(&self) -> Partitioning {
+        Partitioning::UnknownPartitioning(self.partitions.len())
+    }
+
+    fn with_new_children(
+        &self,
+        _: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        Err(DataFusionError::Internal(format!(
+            "Children cannot be replaced in {:?}",
+            self
+        )))
+    }
+
+    async fn execute(&self, partition: usize) -> Result<SendableRecordBatchStream> {
+        Ok(Box::pin(MemoryStream::try_new(
+            self.partitions[partition].clone(),
+            self.schema.clone(),
+            self.projection.clone(),
+        )?))
     }
 }
 
@@ -74,45 +97,8 @@ impl MemoryExec {
     }
 }
 
-/// Memory partition
-#[derive(Debug)]
-struct MemoryPartition {
-    /// Vector of record batches
-    data: Vec<RecordBatch>,
-    /// Schema representing the data
-    schema: SchemaRef,
-    /// Optional projection
-    projection: Option<Vec<usize>>,
-}
-
-impl MemoryPartition {
-    /// Create a new in-memory partition
-    fn new(
-        data: Vec<RecordBatch>,
-        schema: SchemaRef,
-        projection: Option<Vec<usize>>,
-    ) -> Self {
-        Self {
-            data,
-            schema,
-            projection,
-        }
-    }
-}
-
-impl Partition for MemoryPartition {
-    /// Execute this partition and return an iterator over RecordBatch
-    fn execute(&self) -> Result<Arc<Mutex<dyn RecordBatchReader + Send + Sync>>> {
-        Ok(Arc::new(Mutex::new(MemoryIterator::try_new(
-            self.data.clone(),
-            self.schema.clone(),
-            self.projection.clone(),
-        )?)))
-    }
-}
-
 /// Iterator over batches
-struct MemoryIterator {
+pub(crate) struct MemoryStream {
     /// Vector of record batches
     data: Vec<RecordBatch>,
     /// Schema representing the data
@@ -123,7 +109,7 @@ struct MemoryIterator {
     index: usize,
 }
 
-impl MemoryIterator {
+impl MemoryStream {
     /// Create an iterator for a vector of record batches
     pub fn try_new(
         data: Vec<RecordBatch>,
@@ -139,27 +125,37 @@ impl MemoryIterator {
     }
 }
 
-impl RecordBatchReader for MemoryIterator {
-    /// Get the schema
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
+impl Stream for MemoryStream {
+    type Item = ArrowResult<RecordBatch>;
 
-    /// Get the next RecordBatch
-    fn next_batch(&mut self) -> ArrowResult<Option<RecordBatch>> {
-        if self.index < self.data.len() {
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        _: &mut Context<'_>,
+    ) -> Poll<Option<Self::Item>> {
+        Poll::Ready(if self.index < self.data.len() {
             self.index += 1;
             let batch = &self.data[self.index - 1];
             // apply projection
             match &self.projection {
-                Some(columns) => Ok(Some(RecordBatch::try_new(
+                Some(columns) => Some(RecordBatch::try_new(
                     self.schema.clone(),
                     columns.iter().map(|i| batch.column(*i).clone()).collect(),
-                )?)),
-                None => Ok(Some(batch.clone())),
+                )),
+                None => Some(Ok(batch.clone())),
             }
         } else {
-            Ok(None)
-        }
+            None
+        })
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        (self.data.len(), Some(self.data.len()))
+    }
+}
+
+impl RecordBatchStream for MemoryStream {
+    /// Get the schema
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
     }
 }
diff --git a/rust/datafusion/src/physical_plan/merge.rs b/rust/datafusion/src/physical_plan/merge.rs
new file mode 100644
index 00000000000..f50bfee92ba
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/merge.rs
@@ -0,0 +1,175 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Defines the merge plan for executing partitions in parallel and then merging the results
+//! into a single partition
+
+use std::any::Any;
+use std::iter::Iterator;
+use std::sync::Arc;
+
+use futures::future;
+
+use super::common;
+use crate::error::{DataFusionError, Result};
+use crate::physical_plan::ExecutionPlan;
+use crate::physical_plan::Partitioning;
+
+use arrow::record_batch::RecordBatch;
+use arrow::{datatypes::SchemaRef, error::ArrowError};
+
+use super::SendableRecordBatchStream;
+
+use async_trait::async_trait;
+use tokio;
+
+/// Merge execution plan executes partitions in parallel and combines them into a single
+/// partition. No guarantees are made about the order of the resulting partition.
+#[derive(Debug)]
+pub struct MergeExec {
+    /// Input execution plan
+    input: Arc<dyn ExecutionPlan>,
+}
+
+impl MergeExec {
+    /// Create a new MergeExec
+    pub fn new(input: Arc<dyn ExecutionPlan>) -> Self {
+        MergeExec { input }
+    }
+}
+
+#[async_trait]
+impl ExecutionPlan for MergeExec {
+    /// Return a reference to Any that can be used for downcasting
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.input.schema()
+    }
+
+    fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
+        vec![self.input.clone()]
+    }
+
+    /// Get the output partitioning of this plan
+    fn output_partitioning(&self) -> Partitioning {
+        Partitioning::UnknownPartitioning(1)
+    }
+
+    fn with_new_children(
+        &self,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        match children.len() {
+            1 => Ok(Arc::new(MergeExec::new(children[0].clone()))),
+            _ => Err(DataFusionError::Internal(
+                "MergeExec wrong number of children".to_string(),
+            )),
+        }
+    }
+
+    async fn execute(&self, partition: usize) -> Result<SendableRecordBatchStream> {
+        // MergeExec produces a single partition
+        if 0 != partition {
+            return Err(DataFusionError::Internal(format!(
+                "MergeExec invalid partition {}",
+                partition
+            )));
+        }
+
+        let input_partitions = self.input.output_partitioning().partition_count();
+        match input_partitions {
+            0 => Err(DataFusionError::Internal(
+                "MergeExec requires at least one input partition".to_owned(),
+            )),
+            1 => {
+                // bypass any threading if there is a single partition
+                self.input.execute(0).await
+            }
+            _ => {
+                let tasks = (0..input_partitions).map(|part_i| {
+                    let input = self.input.clone();
+                    tokio::spawn(async move {
+                        let stream = input.execute(part_i).await?;
+                        common::collect(stream).await
+                    })
+                });
+
+                let results = future::try_join_all(tasks)
+                    .await
+                    .map_err(|e| ArrowError::from_external_error(Box::new(e)))?;
+
+                let combined_results = results
+                    .into_iter()
+                    .try_fold(Vec::<RecordBatch>::new(), |mut acc, maybe_batches| {
+                        acc.append(&mut maybe_batches?);
+                        Result::Ok(acc)
+                    })?
+                    .into_iter()
+                    .map(|x| Arc::new(x))
+                    .collect::<Vec<_>>();
+
+                Ok(Box::pin(common::SizedRecordBatchStream::new(
+                    self.input.schema(),
+                    combined_results,
+                )))
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::*;
+    use crate::physical_plan::common;
+    use crate::physical_plan::csv::{CsvExec, CsvReadOptions};
+    use crate::test;
+
+    #[tokio::test]
+    async fn merge() -> Result<()> {
+        let schema = test::aggr_test_schema();
+
+        let num_partitions = 4;
+        let path =
+            test::create_partitioned_csv("aggregate_test_100.csv", num_partitions)?;
+
+        let csv =
+            CsvExec::try_new(&path, CsvReadOptions::new().schema(&schema), None, 1024)?;
+
+        // input should have 4 partitions
+        assert_eq!(csv.output_partitioning().partition_count(), num_partitions);
+
+        let merge = MergeExec::new(Arc::new(csv));
+
+        // output of MergeExec should have a single partition
+        assert_eq!(merge.output_partitioning().partition_count(), 1);
+
+        // the result should contain 4 batches (one per input partition)
+        let iter = merge.execute(0).await?;
+        let batches = common::collect(iter).await?;
+        assert_eq!(batches.len(), num_partitions);
+
+        // there should be a total of 100 rows
+        let row_count: usize = batches.iter().map(|batch| batch.num_rows()).sum();
+        assert_eq!(row_count, 100);
+
+        Ok(())
+    }
+}
diff --git a/rust/datafusion/src/physical_plan/mod.rs b/rust/datafusion/src/physical_plan/mod.rs
new file mode 100644
index 00000000000..a2bc3bef7f4
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/mod.rs
@@ -0,0 +1,225 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Traits for physical query plan, supporting parallel execution for partitioned relations.
+
+use std::fmt::{Debug, Display};
+use std::sync::Arc;
+use std::{any::Any, pin::Pin};
+
+use crate::execution::context::ExecutionContextState;
+use crate::logical_plan::LogicalPlan;
+use crate::{error::Result, scalar::ScalarValue};
+use arrow::datatypes::{DataType, Schema, SchemaRef};
+use arrow::error::Result as ArrowResult;
+use arrow::record_batch::RecordBatch;
+use arrow::{array::ArrayRef, datatypes::Field};
+
+use async_trait::async_trait;
+use futures::stream::Stream;
+
+/// Trait for types that stream [arrow::record_batch::RecordBatch]
+pub trait RecordBatchStream: Stream<Item = ArrowResult<RecordBatch>> {
+    /// Returns the schema of this `RecordBatchStream`.
+    ///
+    /// Implementation of this trait should guarantee that all `RecordBatch`'s returned by this
+    /// stream should have the same schema as returned from this method.
+    fn schema(&self) -> SchemaRef;
+}
+
+/// Trait for a stream of record batches.
+pub type SendableRecordBatchStream = Pin<Box<dyn RecordBatchStream + Send>>;
+
+/// Physical query planner that converts a `LogicalPlan` to an
+/// `ExecutionPlan` suitable for execution.
+pub trait PhysicalPlanner {
+    /// Create a physical plan from a logical plan
+    fn create_physical_plan(
+        &self,
+        logical_plan: &LogicalPlan,
+        ctx_state: &ExecutionContextState,
+    ) -> Result<Arc<dyn ExecutionPlan>>;
+}
+
+/// Partition-aware execution plan for a relation
+#[async_trait]
+pub trait ExecutionPlan: Debug + Send + Sync {
+    /// Returns the execution plan as [`Any`](std::any::Any) so that it can be
+    /// downcast to a specific implementation.
+    fn as_any(&self) -> &dyn Any;
+    /// Get the schema for this execution plan
+    fn schema(&self) -> SchemaRef;
+    /// Specifies the output partitioning scheme of this plan
+    fn output_partitioning(&self) -> Partitioning;
+    /// Specifies the data distribution requirements of all the children for this operator
+    fn required_child_distribution(&self) -> Distribution {
+        Distribution::UnspecifiedDistribution
+    }
+    /// Get a list of child execution plans that provide the input for this plan. The returned list
+    /// will be empty for leaf nodes, will contain a single value for unary nodes, or two
+    /// values for binary nodes (such as joins).
+    fn children(&self) -> Vec<Arc<dyn ExecutionPlan>>;
+    /// Returns a new plan where all children were replaced by new plans.
+    /// The size of `children` must be equal to the size of `ExecutionPlan::children()`.
+    fn with_new_children(
+        &self,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>>;
+
+    /// creates an iterator
+    async fn execute(&self, partition: usize) -> Result<SendableRecordBatchStream>;
+}
+
+/// Partitioning schemes supported by operators.
+#[derive(Debug, Clone)]
+pub enum Partitioning {
+    /// Unknown partitioning scheme
+    UnknownPartitioning(usize),
+}
+
+impl Partitioning {
+    /// Returns the number of partitions in this partitioning scheme
+    pub fn partition_count(&self) -> usize {
+        use Partitioning::*;
+        match self {
+            UnknownPartitioning(n) => *n,
+        }
+    }
+}
+
+/// Distribution schemes
+#[derive(Debug, Clone)]
+pub enum Distribution {
+    /// Unspecified distribution
+    UnspecifiedDistribution,
+    /// A single partition is required
+    SinglePartition,
+}
+
+/// Expression that can be evaluated against a RecordBatch
+/// A Physical expression knows its type, nullability and how to evaluate itself.
+pub trait PhysicalExpr: Send + Sync + Display + Debug {
+    /// Get the data type of this expression, given the schema of the input
+    fn data_type(&self, input_schema: &Schema) -> Result<DataType>;
+    /// Determine whether this expression is nullable, given the schema of the input
+    fn nullable(&self, input_schema: &Schema) -> Result<bool>;
+    /// Evaluate an expression against a RecordBatch
+    fn evaluate(&self, batch: &RecordBatch) -> Result<ArrayRef>;
+}
+
+/// An aggregate expression that:
+/// * knows its resulting field
+/// * knows how to create its accumulator
+/// * knows its accumulator's state's field
+/// * knows the expressions from whose its accumulator will receive values
+pub trait AggregateExpr: Send + Sync + Debug {
+    /// the field of the final result of this aggregation.
+    fn field(&self) -> Result<Field>;
+
+    /// the accumulator used to accumulate values from the expressions.
+    /// the accumulator expects the same number of arguments as `expressions` and must
+    /// return states with the same description as `state_fields`
+    fn create_accumulator(&self) -> Result<Box<dyn Accumulator>>;
+
+    /// the fields that encapsulate the Accumulator's state
+    /// the number of fields here equals the number of states that the accumulator contains
+    fn state_fields(&self) -> Result<Vec<Field>>;
+
+    /// expressions that are passed to the Accumulator.
+    /// Single-column aggregations such as `sum` return a single value, others (e.g. `cov`) return many.
+    fn expressions(&self) -> Vec<Arc<dyn PhysicalExpr>>;
+}
+
+/// An accumulator represents a stateful object that lives throughout the evaluation of multiple rows and
+/// generically accumulates values. An accumulator knows how to:
+/// * update its state from inputs via `update`
+/// * convert its internal state to a vector of scalar values
+/// * update its state from multiple accumulators' states via `merge`
+/// * compute the final value from its internal state via `evaluate`
+pub trait Accumulator: Send + Sync + Debug {
+    /// Returns the state of the accumulator at the end of the accumulation.
+    // in the case of an average on which we track `sum` and `n`, this function should return a vector
+    // of two values, sum and n.
+    fn state(&self) -> Result<Vec<ScalarValue>>;
+
+    /// updates the accumulator's state from a vector of scalars.
+    fn update(&mut self, values: &Vec<ScalarValue>) -> Result<()>;
+
+    /// updates the accumulator's state from a vector of arrays.
+    fn update_batch(&mut self, values: &Vec<ArrayRef>) -> Result<()> {
+        if values.len() == 0 {
+            return Ok(());
+        };
+        (0..values[0].len())
+            .map(|index| {
+                let v = values
+                    .iter()
+                    .map(|array| ScalarValue::try_from_array(array, index))
+                    .collect::<Result<Vec<_>>>()?;
+                self.update(&v)
+            })
+            .collect::<Result<_>>()
+    }
+
+    /// updates the accumulator's state from a vector of scalars.
+    fn merge(&mut self, states: &Vec<ScalarValue>) -> Result<()>;
+
+    /// updates the accumulator's state from a vector of states.
+    fn merge_batch(&mut self, states: &Vec<ArrayRef>) -> Result<()> {
+        if states.len() == 0 {
+            return Ok(());
+        };
+        (0..states[0].len())
+            .map(|index| {
+                let v = states
+                    .iter()
+                    .map(|array| ScalarValue::try_from_array(array, index))
+                    .collect::<Result<Vec<_>>>()?;
+                self.merge(&v)
+            })
+            .collect::<Result<_>>()
+    }
+
+    /// returns its value based on its current state.
+    fn evaluate(&self) -> Result<ScalarValue>;
+}
+
+pub mod aggregates;
+pub mod array_expressions;
+pub mod common;
+pub mod csv;
+pub mod datetime_expressions;
+pub mod distinct_expressions;
+pub mod empty;
+pub mod explain;
+pub mod expressions;
+pub mod filter;
+pub mod functions;
+pub mod group_scalar;
+pub mod hash_aggregate;
+pub mod limit;
+pub mod math_expressions;
+pub mod memory;
+pub mod merge;
+pub mod parquet;
+pub mod planner;
+pub mod projection;
+pub mod sort;
+pub mod string_expressions;
+pub mod type_coercion;
+pub mod udaf;
+pub mod udf;
diff --git a/rust/datafusion/src/execution/physical_plan/parquet.rs b/rust/datafusion/src/physical_plan/parquet.rs
similarity index 54%
rename from rust/datafusion/src/execution/physical_plan/parquet.rs
rename to rust/datafusion/src/physical_plan/parquet.rs
index f453047583b..8b9f3b27a78 100644
--- a/rust/datafusion/src/execution/physical_plan/parquet.rs
+++ b/rust/datafusion/src/physical_plan/parquet.rs
@@ -17,25 +17,31 @@
 
 //! Execution plan for reading Parquet files
 
+use std::any::Any;
 use std::fs::File;
 use std::rc::Rc;
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
+use std::task::{Context, Poll};
 use std::{fmt, thread};
 
-use crate::error::{ExecutionError, Result};
-use crate::execution::physical_plan::common;
-use crate::execution::physical_plan::{ExecutionPlan, Partition};
+use super::{RecordBatchStream, SendableRecordBatchStream};
+use crate::error::{DataFusionError, Result};
+use crate::physical_plan::ExecutionPlan;
+use crate::physical_plan::{common, Partitioning};
 use arrow::datatypes::{Schema, SchemaRef};
 use arrow::error::{ArrowError, Result as ArrowResult};
-use arrow::record_batch::{RecordBatch, RecordBatchReader};
+use arrow::record_batch::RecordBatch;
 use parquet::file::reader::SerializedFileReader;
 
 use crossbeam::channel::{bounded, Receiver, RecvError, Sender};
-use fmt::{Debug, Formatter};
+use fmt::Debug;
 use parquet::arrow::{ArrowReader, ParquetFileArrowReader};
 
+use async_trait::async_trait;
+use futures::stream::Stream;
+
 /// Execution plan for scanning a Parquet file
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct ParquetExec {
     /// Path to directory containing partitioned Parquet files with the same schema
     filenames: Vec<String>,
@@ -57,83 +63,91 @@ impl ParquetExec {
         let mut filenames: Vec<String> = vec![];
         common::build_file_list(path, &mut filenames, ".parquet")?;
         if filenames.is_empty() {
-            Err(ExecutionError::General("No files found".to_string()))
+            Err(DataFusionError::Plan("No files found".to_string()))
         } else {
             let file = File::open(&filenames[0])?;
             let file_reader = Rc::new(SerializedFileReader::new(file)?);
             let mut arrow_reader = ParquetFileArrowReader::new(file_reader);
             let schema = arrow_reader.get_schema()?;
 
-            let projection = match projection {
-                Some(p) => p,
-                None => (0..schema.fields().len()).collect(),
-            };
-
-            let projected_schema = Schema::new(
-                projection
-                    .iter()
-                    .map(|i| schema.field(*i).clone())
-                    .collect(),
-            );
-
-            Ok(Self {
-                filenames,
-                schema: Arc::new(projected_schema),
-                projection,
-                batch_size,
-            })
+            Ok(Self::new(filenames, schema, projection, batch_size))
+        }
+    }
+
+    /// Create a new Parquet reader execution plan with provided files and schema
+    pub fn new(
+        filenames: Vec<String>,
+        schema: Schema,
+        projection: Option<Vec<usize>>,
+        batch_size: usize,
+    ) -> Self {
+        let projection = match projection {
+            Some(p) => p,
+            None => (0..schema.fields().len()).collect(),
+        };
+
+        let projected_schema = Schema::new(
+            projection
+                .iter()
+                .map(|i| schema.field(*i).clone())
+                .collect(),
+        );
+
+        Self {
+            filenames,
+            schema: Arc::new(projected_schema),
+            projection,
+            batch_size,
         }
     }
 }
 
+#[async_trait]
 impl ExecutionPlan for ParquetExec {
+    /// Return a reference to Any that can be used for downcasting
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
     fn schema(&self) -> SchemaRef {
         self.schema.clone()
     }
 
-    fn partitions(&self) -> Result<Vec<Arc<dyn Partition>>> {
-        let partitions = self
-            .filenames
-            .iter()
-            .map(|filename| {
-                Arc::new(ParquetPartition::new(
-                    &filename,
-                    self.projection.clone(),
-                    self.schema.clone(),
-                    self.batch_size,
-                )) as Arc<dyn Partition>
-            })
-            .collect();
-        Ok(partitions)
+    fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
+        // this is a leaf node and has no children
+        vec![]
     }
-}
 
-struct ParquetPartition {
-    iterator: Arc<Mutex<dyn RecordBatchReader + Send + Sync>>,
-}
+    /// Get the output partitioning of this plan
+    fn output_partitioning(&self) -> Partitioning {
+        Partitioning::UnknownPartitioning(self.filenames.len())
+    }
 
-impl Debug for ParquetPartition {
-    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
-        f.debug_struct("ParquetPartition").finish()
+    fn with_new_children(
+        &self,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        if children.is_empty() {
+            Ok(Arc::new(self.clone()))
+        } else {
+            Err(DataFusionError::Internal(format!(
+                "Children cannot be replaced in {:?}",
+                self
+            )))
+        }
     }
-}
 
-impl ParquetPartition {
-    /// Create a new Parquet partition
-    pub fn new(
-        filename: &str,
-        projection: Vec<usize>,
-        schema: SchemaRef,
-        batch_size: usize,
-    ) -> Self {
+    async fn execute(&self, partition: usize) -> Result<SendableRecordBatchStream> {
         // because the parquet implementation is not thread-safe, it is necessary to execute
         // on a thread and communicate with channels
         let (response_tx, response_rx): (
-            Sender<ArrowResult<Option<RecordBatch>>>,
-            Receiver<ArrowResult<Option<RecordBatch>>>,
+            Sender<Option<ArrowResult<RecordBatch>>>,
+            Receiver<Option<ArrowResult<RecordBatch>>>,
         ) = bounded(2);
 
-        let filename = filename.to_string();
+        let filename = self.filenames[partition].clone();
+        let projection = self.projection.clone();
+        let batch_size = self.batch_size;
 
         thread::spawn(move || {
             if let Err(e) = read_file(&filename, projection, batch_size, response_tx) {
@@ -141,22 +155,20 @@ impl ParquetPartition {
             }
         });
 
-        let iterator = Arc::new(Mutex::new(ParquetIterator {
-            schema,
+        Ok(Box::pin(ParquetStream {
+            schema: self.schema.clone(),
             response_rx,
-        }));
-
-        Self { iterator }
+        }))
     }
 }
 
 fn send_result(
-    response_tx: &Sender<ArrowResult<Option<RecordBatch>>>,
-    result: ArrowResult<Option<RecordBatch>>,
+    response_tx: &Sender<Option<ArrowResult<RecordBatch>>>,
+    result: Option<ArrowResult<RecordBatch>>,
 ) -> Result<()> {
     response_tx
         .send(result)
-        .map_err(|e| ExecutionError::ExecutionError(format!("{:?}", e)))?;
+        .map_err(|e| DataFusionError::Execution(e.to_string()))?;
     Ok(())
 }
 
@@ -164,7 +176,7 @@ fn read_file(
     filename: &str,
     projection: Vec<usize>,
     batch_size: usize,
-    response_tx: Sender<ArrowResult<Option<RecordBatch>>>,
+    response_tx: Sender<Option<ArrowResult<RecordBatch>>>,
 ) -> Result<()> {
     let file = File::open(&filename)?;
     let file_reader = Rc::new(SerializedFileReader::new(file)?);
@@ -172,70 +184,71 @@ fn read_file(
     let mut batch_reader =
         arrow_reader.get_record_reader_by_columns(projection.clone(), batch_size)?;
     loop {
-        match batch_reader.next_batch() {
-            Ok(Some(batch)) => send_result(&response_tx, Ok(Some(batch)))?,
-            Ok(None) => {
+        match batch_reader.next() {
+            Some(Ok(batch)) => send_result(&response_tx, Some(Ok(batch)))?,
+            None => {
                 // finished reading file
-                send_result(&response_tx, Ok(None))?;
+                send_result(&response_tx, None)?;
                 break;
             }
-            Err(e) => {
-                let err_msg = format!("Error reading batch from {}: {:?}", filename, e);
+            Some(Err(e)) => {
+                let err_msg =
+                    format!("Error reading batch from {}: {}", filename, e.to_string());
                 // send error to operator
                 send_result(
                     &response_tx,
-                    Err(ArrowError::ParquetError(err_msg.clone())),
+                    Some(Err(ArrowError::ParquetError(err_msg.clone()))),
                 )?;
                 // terminate thread with error
-                return Err(ExecutionError::ExecutionError(err_msg));
+                return Err(DataFusionError::Execution(err_msg));
             }
         }
     }
     Ok(())
 }
 
-impl Partition for ParquetPartition {
-    fn execute(&self) -> Result<Arc<Mutex<dyn RecordBatchReader + Send + Sync>>> {
-        Ok(self.iterator.clone())
-    }
-}
-
-struct ParquetIterator {
+struct ParquetStream {
     schema: SchemaRef,
-    response_rx: Receiver<ArrowResult<Option<RecordBatch>>>,
+    response_rx: Receiver<Option<ArrowResult<RecordBatch>>>,
 }
 
-impl RecordBatchReader for ParquetIterator {
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
+impl Stream for ParquetStream {
+    type Item = ArrowResult<RecordBatch>;
 
-    fn next_batch(&mut self) -> ArrowResult<Option<RecordBatch>> {
+    fn poll_next(
+        self: std::pin::Pin<&mut Self>,
+        _: &mut Context<'_>,
+    ) -> Poll<Option<Self::Item>> {
         match self.response_rx.recv() {
-            Ok(batch) => batch,
+            Ok(batch) => Poll::Ready(batch),
             // RecvError means receiver has exited and closed the channel
-            Err(RecvError) => Ok(None),
+            Err(RecvError) => Poll::Ready(None),
         }
     }
 }
 
+impl RecordBatchStream for ParquetStream {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
+    use futures::StreamExt;
     use std::env;
 
-    #[test]
-    fn test() -> Result<()> {
+    #[tokio::test]
+    async fn test() -> Result<()> {
         let testdata =
             env::var("PARQUET_TEST_DATA").expect("PARQUET_TEST_DATA not defined");
         let filename = format!("{}/alltypes_plain.parquet", testdata);
         let parquet_exec = ParquetExec::try_new(&filename, Some(vec![0, 1, 2]), 1024)?;
-        let partitions = parquet_exec.partitions()?;
-        assert_eq!(partitions.len(), 1);
+        assert_eq!(parquet_exec.output_partitioning().partition_count(), 1);
 
-        let results = partitions[0].execute()?;
-        let mut results = results.lock().unwrap();
-        let batch = results.next_batch()?.unwrap();
+        let mut results = parquet_exec.execute(0).await?;
+        let batch = results.next().await.unwrap()?;
 
         assert_eq!(8, batch.num_rows());
         assert_eq!(3, batch.num_columns());
@@ -245,13 +258,13 @@ mod tests {
             schema.fields().iter().map(|f| f.name().as_str()).collect();
         assert_eq!(vec!["id", "bool_col", "tinyint_col"], field_names);
 
-        let batch = results.next_batch()?;
+        let batch = results.next().await;
         assert!(batch.is_none());
 
-        let batch = results.next_batch()?;
+        let batch = results.next().await;
         assert!(batch.is_none());
 
-        let batch = results.next_batch()?;
+        let batch = results.next().await;
         assert!(batch.is_none());
 
         Ok(())
diff --git a/rust/datafusion/src/physical_plan/planner.rs b/rust/datafusion/src/physical_plan/planner.rs
new file mode 100644
index 00000000000..1303fa4b0f1
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/planner.rs
@@ -0,0 +1,838 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Physical query planner
+
+use std::sync::Arc;
+
+use super::{aggregates, empty::EmptyExec, expressions::binary, functions, udaf};
+use crate::error::{DataFusionError, Result};
+use crate::execution::context::ExecutionContextState;
+use crate::logical_plan::{
+    Expr, LogicalPlan, PlanType, StringifiedPlan, TableSource, UserDefinedLogicalNode,
+};
+use crate::physical_plan::csv::{CsvExec, CsvReadOptions};
+use crate::physical_plan::explain::ExplainExec;
+use crate::physical_plan::expressions::{Column, Literal, PhysicalSortExpr};
+use crate::physical_plan::filter::FilterExec;
+use crate::physical_plan::hash_aggregate::{AggregateMode, HashAggregateExec};
+use crate::physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
+use crate::physical_plan::memory::MemoryExec;
+use crate::physical_plan::merge::MergeExec;
+use crate::physical_plan::parquet::ParquetExec;
+use crate::physical_plan::projection::ProjectionExec;
+use crate::physical_plan::sort::SortExec;
+use crate::physical_plan::udf;
+use crate::physical_plan::{expressions, Distribution};
+use crate::physical_plan::{AggregateExpr, ExecutionPlan, PhysicalExpr, PhysicalPlanner};
+use crate::variable::VarType;
+use arrow::compute::SortOptions;
+use arrow::datatypes::Schema;
+use expressions::col;
+
+/// This trait permits the `DefaultPhysicalPlanner` to create plans for
+/// user defined `ExtensionPlanNode`s
+pub trait ExtensionPlanner {
+    /// Create a physical plan for an extension node
+    fn plan_extension(
+        &self,
+        node: &dyn UserDefinedLogicalNode,
+        inputs: Vec<Arc<dyn ExecutionPlan>>,
+        ctx_state: &ExecutionContextState,
+    ) -> Result<Arc<dyn ExecutionPlan>>;
+}
+
+/// Default single node physical query planner that converts a
+/// `LogicalPlan` to an `ExecutionPlan` suitable for execution.
+pub struct DefaultPhysicalPlanner {
+    extension_planner: Arc<dyn ExtensionPlanner + Send + Sync>,
+}
+
+impl Default for DefaultPhysicalPlanner {
+    /// Create an implementation of the default physical planner
+    fn default() -> Self {
+        Self {
+            extension_planner: Arc::new(DefaultExtensionPlanner {}),
+        }
+    }
+}
+
+impl PhysicalPlanner for DefaultPhysicalPlanner {
+    /// Create a physical plan from a logical plan
+    fn create_physical_plan(
+        &self,
+        logical_plan: &LogicalPlan,
+        ctx_state: &ExecutionContextState,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        let plan = self.create_initial_plan(logical_plan, ctx_state)?;
+        self.optimize_plan(plan, ctx_state)
+    }
+}
+
+impl DefaultPhysicalPlanner {
+    /// Create a physical planner that uses `extension_planner` to
+    /// plan extension nodes.
+    pub fn with_extension_planner(
+        extension_planner: Arc<dyn ExtensionPlanner + Send + Sync>,
+    ) -> Self {
+        Self { extension_planner }
+    }
+
+    /// Create a physical plan from a logical plan
+    fn optimize_plan(
+        &self,
+        plan: Arc<dyn ExecutionPlan>,
+        ctx_state: &ExecutionContextState,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        let children = plan
+            .children()
+            .iter()
+            .map(|child| self.optimize_plan(child.clone(), ctx_state))
+            .collect::<Result<Vec<_>>>()?;
+
+        if children.len() == 0 {
+            // leaf node, children cannot be replaced
+            Ok(plan.clone())
+        } else {
+            match plan.required_child_distribution() {
+                Distribution::UnspecifiedDistribution => plan.with_new_children(children),
+                Distribution::SinglePartition => plan.with_new_children(
+                    children
+                        .iter()
+                        .map(|child| {
+                            if child.output_partitioning().partition_count() == 1 {
+                                child.clone()
+                            } else {
+                                Arc::new(MergeExec::new(child.clone()))
+                            }
+                        })
+                        .collect(),
+                ),
+            }
+        }
+    }
+
+    /// Create a physical plan from a logical plan
+    fn create_initial_plan(
+        &self,
+        logical_plan: &LogicalPlan,
+        ctx_state: &ExecutionContextState,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        let batch_size = ctx_state.config.batch_size;
+
+        match logical_plan {
+            LogicalPlan::TableScan {
+                source, projection, ..
+            } => match source {
+                TableSource::FromContext(table_name) => {
+                    match ctx_state.datasources.get(table_name) {
+                        Some(provider) => provider.scan(projection, batch_size),
+                        _ => Err(DataFusionError::Plan(format!(
+                            "No table named {}. Existing tables: {:?}",
+                            table_name,
+                            ctx_state.datasources.keys().collect::<Vec<_>>(),
+                        ))),
+                    }
+                }
+                TableSource::FromProvider(ref provider) => {
+                    provider.scan(projection, batch_size)
+                }
+            },
+            LogicalPlan::InMemoryScan {
+                data,
+                projection,
+                projected_schema,
+                ..
+            } => Ok(Arc::new(MemoryExec::try_new(
+                data,
+                Arc::new(projected_schema.as_ref().to_owned()),
+                projection.to_owned(),
+            )?)),
+            LogicalPlan::CsvScan {
+                path,
+                schema,
+                has_header,
+                delimiter,
+                projection,
+                ..
+            } => Ok(Arc::new(CsvExec::try_new(
+                path,
+                CsvReadOptions::new()
+                    .schema(schema.as_ref())
+                    .delimiter_option(*delimiter)
+                    .has_header(*has_header),
+                projection.to_owned(),
+                batch_size,
+            )?)),
+            LogicalPlan::ParquetScan {
+                path, projection, ..
+            } => Ok(Arc::new(ParquetExec::try_new(
+                path,
+                projection.to_owned(),
+                batch_size,
+            )?)),
+            LogicalPlan::Projection { input, expr, .. } => {
+                let input = self.create_physical_plan(input, ctx_state)?;
+                let input_schema = input.as_ref().schema().clone();
+                let runtime_expr = expr
+                    .iter()
+                    .map(|e| {
+                        tuple_err((
+                            self.create_physical_expr(e, &input_schema, &ctx_state),
+                            e.name(&input_schema),
+                        ))
+                    })
+                    .collect::<Result<Vec<_>>>()?;
+                Ok(Arc::new(ProjectionExec::try_new(runtime_expr, input)?))
+            }
+            LogicalPlan::Aggregate {
+                input,
+                group_expr,
+                aggr_expr,
+                ..
+            } => {
+                // Initially need to perform the aggregate and then merge the partitions
+                let input = self.create_physical_plan(input, ctx_state)?;
+                let input_schema = input.as_ref().schema().clone();
+
+                let groups = group_expr
+                    .iter()
+                    .map(|e| {
+                        tuple_err((
+                            self.create_physical_expr(e, &input_schema, ctx_state),
+                            e.name(&input_schema),
+                        ))
+                    })
+                    .collect::<Result<Vec<_>>>()?;
+                let aggregates = aggr_expr
+                    .iter()
+                    .map(|e| self.create_aggregate_expr(e, &input_schema, ctx_state))
+                    .collect::<Result<Vec<_>>>()?;
+
+                let initial_aggr = Arc::new(HashAggregateExec::try_new(
+                    AggregateMode::Partial,
+                    groups.clone(),
+                    aggregates.clone(),
+                    input,
+                )?);
+
+                let final_group: Vec<Arc<dyn PhysicalExpr>> =
+                    (0..groups.len()).map(|i| col(&groups[i].1)).collect();
+
+                // construct a second aggregation, keeping the final column name equal to the first aggregation
+                // and the expressions corresponding to the respective aggregate
+                Ok(Arc::new(HashAggregateExec::try_new(
+                    AggregateMode::Final,
+                    final_group
+                        .iter()
+                        .enumerate()
+                        .map(|(i, expr)| (expr.clone(), groups[i].1.clone()))
+                        .collect(),
+                    aggregates,
+                    initial_aggr,
+                )?))
+            }
+            LogicalPlan::Filter {
+                input, predicate, ..
+            } => {
+                let input = self.create_physical_plan(input, ctx_state)?;
+                let input_schema = input.as_ref().schema().clone();
+                let runtime_expr =
+                    self.create_physical_expr(predicate, &input_schema, ctx_state)?;
+                Ok(Arc::new(FilterExec::try_new(runtime_expr, input)?))
+            }
+            LogicalPlan::Sort { expr, input, .. } => {
+                let input = self.create_physical_plan(input, ctx_state)?;
+                let input_schema = input.as_ref().schema().clone();
+
+                let sort_expr = expr
+                    .iter()
+                    .map(|e| match e {
+                        Expr::Sort {
+                            expr,
+                            asc,
+                            nulls_first,
+                        } => self.create_physical_sort_expr(
+                            expr,
+                            &input_schema,
+                            SortOptions {
+                                descending: !*asc,
+                                nulls_first: *nulls_first,
+                            },
+                            ctx_state,
+                        ),
+                        _ => Err(DataFusionError::Plan(
+                            "Sort only accepts sort expressions".to_string(),
+                        )),
+                    })
+                    .collect::<Result<Vec<_>>>()?;
+
+                Ok(Arc::new(SortExec::try_new(
+                    sort_expr,
+                    input,
+                    ctx_state.config.concurrency,
+                )?))
+            }
+            LogicalPlan::EmptyRelation { schema } => {
+                Ok(Arc::new(EmptyExec::new(Arc::new(schema.as_ref().clone()))))
+            }
+            LogicalPlan::Limit { input, n, .. } => {
+                let limit = *n;
+                let input = self.create_physical_plan(input, ctx_state)?;
+
+                // GlobalLimitExec requires a single partition for input
+                let input = if input.output_partitioning().partition_count() == 1 {
+                    input
+                } else {
+                    // Apply a LocalLimitExec to each partition. The optimizer will also insert
+                    // a MergeExec between the GlobalLimitExec and LocalLimitExec
+                    Arc::new(LocalLimitExec::new(input, limit))
+                };
+
+                Ok(Arc::new(GlobalLimitExec::new(
+                    input,
+                    limit,
+                    ctx_state.config.concurrency,
+                )))
+            }
+            LogicalPlan::CreateExternalTable { .. } => {
+                // There is no default plan for "CREATE EXTERNAL
+                // TABLE" -- it must be handled at a higher level (so
+                // that the appropriate table can be registered with
+                // the context)
+                Err(DataFusionError::Internal(
+                    "Unsupported logical plan: CreateExternalTable".to_string(),
+                ))
+            }
+            LogicalPlan::Explain {
+                verbose,
+                plan,
+                stringified_plans,
+                schema,
+            } => {
+                let input = self.create_physical_plan(plan, ctx_state)?;
+
+                let mut stringified_plans = stringified_plans
+                    .iter()
+                    .filter(|s| s.should_display(*verbose))
+                    .map(|s| s.clone())
+                    .collect::<Vec<_>>();
+
+                // add in the physical plan if requested
+                if *verbose {
+                    stringified_plans.push(StringifiedPlan::new(
+                        PlanType::PhysicalPlan,
+                        format!("{:#?}", input),
+                    ));
+                }
+                let schema_ref = Arc::new(schema.as_ref().clone());
+                Ok(Arc::new(ExplainExec::new(schema_ref, stringified_plans)))
+            }
+            LogicalPlan::Extension { node } => {
+                let inputs = node
+                    .inputs()
+                    .into_iter()
+                    .map(|input_plan| self.create_physical_plan(input_plan, ctx_state))
+                    .collect::<Result<Vec<_>>>()?;
+
+                let plan = self.extension_planner.plan_extension(
+                    node.as_ref(),
+                    inputs,
+                    ctx_state,
+                )?;
+
+                // Ensure the ExecutionPlan's  schema matches the
+                // declared logical schema to catch and warn about
+                // logic errors when creating user defined plans.
+                if plan.schema() != *node.schema() {
+                    Err(DataFusionError::Plan(format!(
+                        "Extension planner for {:?} created an ExecutionPlan with mismatched schema. \
+                         LogicalPlan schema: {:?}, ExecutionPlan schema: {:?}",
+                        node, node.schema(), plan.schema()
+                    )))
+                } else {
+                    Ok(plan)
+                }
+            }
+        }
+    }
+
+    /// Create a physical expression from a logical expression
+    pub fn create_physical_expr(
+        &self,
+        e: &Expr,
+        input_schema: &Schema,
+        ctx_state: &ExecutionContextState,
+    ) -> Result<Arc<dyn PhysicalExpr>> {
+        match e {
+            Expr::Alias(expr, ..) => {
+                Ok(self.create_physical_expr(expr, input_schema, ctx_state)?)
+            }
+            Expr::Column(name) => {
+                // check that name exists
+                input_schema.field_with_name(&name)?;
+                Ok(Arc::new(Column::new(name)))
+            }
+            Expr::Literal(value) => Ok(Arc::new(Literal::new(value.clone()))),
+            Expr::ScalarVariable(variable_names) => {
+                if &variable_names[0][0..2] == "@@" {
+                    match ctx_state.var_provider.get(&VarType::System) {
+                        Some(provider) => {
+                            let scalar_value =
+                                provider.get_value(variable_names.clone())?;
+                            Ok(Arc::new(Literal::new(scalar_value)))
+                        }
+                        _ => Err(DataFusionError::Plan(format!(
+                            "No system variable provider found"
+                        ))),
+                    }
+                } else {
+                    match ctx_state.var_provider.get(&VarType::UserDefined) {
+                        Some(provider) => {
+                            let scalar_value =
+                                provider.get_value(variable_names.clone())?;
+                            Ok(Arc::new(Literal::new(scalar_value)))
+                        }
+                        _ => Err(DataFusionError::Plan(format!(
+                            "No user defined variable provider found"
+                        ))),
+                    }
+                }
+            }
+            Expr::BinaryExpr { left, op, right } => {
+                let lhs = self.create_physical_expr(left, input_schema, ctx_state)?;
+                let rhs = self.create_physical_expr(right, input_schema, ctx_state)?;
+                binary(lhs, op.clone(), rhs, input_schema)
+            }
+            Expr::Cast { expr, data_type } => expressions::cast(
+                self.create_physical_expr(expr, input_schema, ctx_state)?,
+                input_schema,
+                data_type.clone(),
+            ),
+            Expr::Not(expr) => expressions::not(
+                self.create_physical_expr(expr, input_schema, ctx_state)?,
+                input_schema,
+            ),
+            Expr::IsNull(expr) => expressions::is_null(self.create_physical_expr(
+                expr,
+                input_schema,
+                ctx_state,
+            )?),
+            Expr::IsNotNull(expr) => expressions::is_not_null(
+                self.create_physical_expr(expr, input_schema, ctx_state)?,
+            ),
+            Expr::ScalarFunction { fun, args } => {
+                let physical_args = args
+                    .iter()
+                    .map(|e| self.create_physical_expr(e, input_schema, ctx_state))
+                    .collect::<Result<Vec<_>>>()?;
+                functions::create_physical_expr(fun, &physical_args, input_schema)
+            }
+            Expr::ScalarUDF { fun, args } => {
+                let mut physical_args = vec![];
+                for e in args {
+                    physical_args.push(self.create_physical_expr(
+                        e,
+                        input_schema,
+                        ctx_state,
+                    )?);
+                }
+
+                udf::create_physical_expr(
+                    fun.clone().as_ref(),
+                    &physical_args,
+                    input_schema,
+                )
+            }
+            other => Err(DataFusionError::NotImplemented(format!(
+                "Physical plan does not support logical expression {:?}",
+                other
+            ))),
+        }
+    }
+
+    /// Create an aggregate expression from a logical expression
+    pub fn create_aggregate_expr(
+        &self,
+        e: &Expr,
+        input_schema: &Schema,
+        ctx_state: &ExecutionContextState,
+    ) -> Result<Arc<dyn AggregateExpr>> {
+        // unpack aliased logical expressions, e.g. "sum(col) as total"
+        let (name, e) = match e {
+            Expr::Alias(sub_expr, alias) => (alias.clone(), sub_expr.as_ref()),
+            _ => (e.name(input_schema)?, e),
+        };
+
+        match e {
+            Expr::AggregateFunction {
+                fun,
+                distinct,
+                args,
+                ..
+            } => {
+                let args = args
+                    .iter()
+                    .map(|e| self.create_physical_expr(e, input_schema, ctx_state))
+                    .collect::<Result<Vec<_>>>()?;
+                aggregates::create_aggregate_expr(
+                    fun,
+                    *distinct,
+                    &args,
+                    input_schema,
+                    name,
+                )
+            }
+            Expr::AggregateUDF { fun, args, .. } => {
+                let args = args
+                    .iter()
+                    .map(|e| self.create_physical_expr(e, input_schema, ctx_state))
+                    .collect::<Result<Vec<_>>>()?;
+
+                udaf::create_aggregate_expr(fun, &args, input_schema, name)
+            }
+            other => Err(DataFusionError::Internal(format!(
+                "Invalid aggregate expression '{:?}'",
+                other
+            ))),
+        }
+    }
+
+    /// Create an aggregate expression from a logical expression
+    pub fn create_physical_sort_expr(
+        &self,
+        e: &Expr,
+        input_schema: &Schema,
+        options: SortOptions,
+        ctx_state: &ExecutionContextState,
+    ) -> Result<PhysicalSortExpr> {
+        Ok(PhysicalSortExpr {
+            expr: self.create_physical_expr(e, input_schema, ctx_state)?,
+            options: options,
+        })
+    }
+}
+
+fn tuple_err<T, R>(value: (Result<T>, Result<R>)) -> Result<(T, R)> {
+    match value {
+        (Ok(e), Ok(e1)) => Ok((e, e1)),
+        (Err(e), Ok(_)) => Err(e),
+        (Ok(_), Err(e1)) => Err(e1),
+        (Err(e), Err(_)) => Err(e),
+    }
+}
+
+struct DefaultExtensionPlanner {}
+
+impl ExtensionPlanner for DefaultExtensionPlanner {
+    fn plan_extension(
+        &self,
+        node: &dyn UserDefinedLogicalNode,
+        _inputs: Vec<Arc<dyn ExecutionPlan>>,
+        _ctx_state: &ExecutionContextState,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        Err(DataFusionError::NotImplemented(format!(
+            "DefaultPhysicalPlanner does not know how to plan {:?}. \
+                     Provide a custom ExtensionPlanNodePlanner that does",
+            node
+        )))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::physical_plan::{csv::CsvReadOptions, expressions, Partitioning};
+    use crate::{
+        logical_plan::{col, lit, sum, LogicalPlanBuilder},
+        physical_plan::SendableRecordBatchStream,
+    };
+    use crate::{prelude::ExecutionConfig, test::arrow_testdata_path};
+    use arrow::datatypes::{DataType, Field, SchemaRef};
+    use async_trait::async_trait;
+    use fmt::Debug;
+    use std::{any::Any, collections::HashMap, fmt};
+
+    fn make_ctx_state() -> ExecutionContextState {
+        ExecutionContextState {
+            datasources: HashMap::new(),
+            scalar_functions: HashMap::new(),
+            var_provider: HashMap::new(),
+            aggregate_functions: HashMap::new(),
+            config: ExecutionConfig::new(),
+        }
+    }
+
+    fn plan(logical_plan: &LogicalPlan) -> Result<Arc<dyn ExecutionPlan>> {
+        let ctx_state = make_ctx_state();
+        let planner = DefaultPhysicalPlanner::default();
+        planner.create_physical_plan(logical_plan, &ctx_state)
+    }
+
+    #[test]
+    fn test_all_operators() -> Result<()> {
+        let testdata = arrow_testdata_path();
+        let path = format!("{}/csv/aggregate_test_100.csv", testdata);
+
+        let options = CsvReadOptions::new().schema_infer_max_records(100);
+        let logical_plan = LogicalPlanBuilder::scan_csv(&path, options, None)?
+            // filter clause needs the type coercion rule applied
+            .filter(col("c7").lt(lit(5_u8)))?
+            .project(vec![col("c1"), col("c2")])?
+            .aggregate(vec![col("c1")], vec![sum(col("c2"))])?
+            .sort(vec![col("c1").sort(true, true)])?
+            .limit(10)?
+            .build()?;
+
+        let plan = plan(&logical_plan)?;
+
+        // verify that the plan correctly casts u8 to i64
+        let expected = "BinaryExpr { left: Column { name: \"c7\" }, op: Lt, right: CastExpr { expr: Literal { value: UInt8(5) }, cast_type: Int64 } }";
+        assert!(format!("{:?}", plan).contains(expected));
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_create_not() -> Result<()> {
+        let schema = Schema::new(vec![Field::new("a", DataType::Boolean, true)]);
+
+        let planner = DefaultPhysicalPlanner::default();
+
+        let expr =
+            planner.create_physical_expr(&col("a").not(), &schema, &make_ctx_state())?;
+        let expected = expressions::not(expressions::col("a"), &schema)?;
+
+        assert_eq!(format!("{:?}", expr), format!("{:?}", expected));
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_with_csv_plan() -> Result<()> {
+        let testdata = arrow_testdata_path();
+        let path = format!("{}/csv/aggregate_test_100.csv", testdata);
+
+        let options = CsvReadOptions::new().schema_infer_max_records(100);
+        let logical_plan = LogicalPlanBuilder::scan_csv(&path, options, None)?
+            .filter(col("c7").lt(col("c12")))?
+            .build()?;
+
+        let plan = plan(&logical_plan)?;
+
+        // c12 is f64, c7 is u8 -> cast c7 to f64
+        let expected = "predicate: BinaryExpr { left: CastExpr { expr: Column { name: \"c7\" }, cast_type: Float64 }, op: Lt, right: Column { name: \"c12\" } }";
+        assert!(format!("{:?}", plan).contains(expected));
+        Ok(())
+    }
+
+    #[test]
+    fn errors() -> Result<()> {
+        let testdata = arrow_testdata_path();
+        let path = format!("{}/csv/aggregate_test_100.csv", testdata);
+        let options = CsvReadOptions::new().schema_infer_max_records(100);
+
+        let bool_expr = col("c1").eq(col("c1"));
+        let cases = vec![
+            // utf8 < u32
+            col("c1").lt(col("c2")),
+            // utf8 AND utf8
+            col("c1").and(col("c1")),
+            // u8 AND u8
+            col("c3").and(col("c3")),
+            // utf8 = u32
+            col("c1").eq(col("c2")),
+            // utf8 = bool
+            col("c1").eq(bool_expr.clone()),
+            // u32 AND bool
+            col("c2").and(bool_expr),
+            // utf8 LIKE u32
+            col("c1").like(col("c2")),
+        ];
+        for case in cases {
+            let logical_plan = LogicalPlanBuilder::scan_csv(&path, options, None)?
+                .project(vec![case.clone()]);
+            let message = format!(
+                "Expression {:?} expected to error due to impossible coercion",
+                case
+            );
+            assert!(logical_plan.is_err(), message);
+        }
+        Ok(())
+    }
+
+    #[test]
+    fn default_extension_planner() -> Result<()> {
+        let ctx_state = make_ctx_state();
+        let planner = DefaultPhysicalPlanner::default();
+        let logical_plan = LogicalPlan::Extension {
+            node: Arc::new(NoOpExtensionNode::default()),
+        };
+        let plan = planner.create_physical_plan(&logical_plan, &ctx_state);
+
+        let expected_error = "DefaultPhysicalPlanner does not know how to plan NoOp";
+        match plan {
+            Ok(_) => assert!(false, "Expected planning failure"),
+            Err(e) => assert!(
+                e.to_string().contains(expected_error),
+                "Error '{}' did not contain expected error '{}'",
+                e.to_string(),
+                expected_error
+            ),
+        }
+        Ok(())
+    }
+
+    #[test]
+    fn bad_extension_planner() -> Result<()> {
+        // Test that creating an execution plan whose schema doesn't
+        // match the logical plan's schema generates an error.
+        let ctx_state = make_ctx_state();
+        let planner = DefaultPhysicalPlanner::with_extension_planner(Arc::new(
+            BadExtensionPlanner {},
+        ));
+
+        let logical_plan = LogicalPlan::Extension {
+            node: Arc::new(NoOpExtensionNode::default()),
+        };
+        let plan = planner.create_physical_plan(&logical_plan, &ctx_state);
+
+        let expected_error = "Extension planner for NoOp created an ExecutionPlan with mismatched schema. LogicalPlan schema: Schema { fields: [Field { name: \"a\", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false }], metadata: {} }, ExecutionPlan schema: Schema { fields: [Field { name: \"b\", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false }], metadata: {} }";
+
+        match plan {
+            Ok(_) => assert!(false, "Expected planning failure"),
+            Err(e) => assert!(
+                e.to_string().contains(expected_error),
+                "Error '{}' did not contain expected error '{}'",
+                e.to_string(),
+                expected_error
+            ),
+        }
+        Ok(())
+    }
+
+    /// An example extension node that doesn't do anything
+    struct NoOpExtensionNode {
+        schema: SchemaRef,
+    }
+
+    impl Default for NoOpExtensionNode {
+        fn default() -> Self {
+            Self {
+                schema: SchemaRef::new(Schema::new(vec![Field::new(
+                    "a",
+                    DataType::Int32,
+                    false,
+                )])),
+            }
+        }
+    }
+
+    impl Debug for NoOpExtensionNode {
+        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+            write!(f, "NoOp")
+        }
+    }
+
+    impl UserDefinedLogicalNode for NoOpExtensionNode {
+        fn as_any(&self) -> &dyn Any {
+            self
+        }
+
+        fn inputs(&self) -> Vec<&LogicalPlan> {
+            vec![]
+        }
+
+        fn schema(&self) -> &SchemaRef {
+            &self.schema
+        }
+
+        fn expressions(&self) -> Vec<Expr> {
+            vec![]
+        }
+
+        fn fmt_for_explain(&self, f: &mut fmt::Formatter) -> fmt::Result {
+            write!(f, "NoOp")
+        }
+
+        fn from_template(
+            &self,
+            _exprs: &Vec<Expr>,
+            _inputs: &Vec<LogicalPlan>,
+        ) -> Arc<dyn UserDefinedLogicalNode + Send + Sync> {
+            unimplemented!("NoOp");
+        }
+    }
+
+    #[derive(Debug)]
+    struct NoOpExecutionPlan {
+        schema: SchemaRef,
+    }
+
+    #[async_trait]
+    impl ExecutionPlan for NoOpExecutionPlan {
+        /// Return a reference to Any that can be used for downcasting
+        fn as_any(&self) -> &dyn Any {
+            self
+        }
+
+        fn schema(&self) -> SchemaRef {
+            self.schema.clone()
+        }
+
+        fn output_partitioning(&self) -> Partitioning {
+            Partitioning::UnknownPartitioning(1)
+        }
+
+        fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
+            vec![]
+        }
+
+        fn with_new_children(
+            &self,
+            _children: Vec<Arc<dyn ExecutionPlan>>,
+        ) -> Result<Arc<dyn ExecutionPlan>> {
+            unimplemented!("NoOpExecutionPlan::with_new_children");
+        }
+
+        async fn execute(&self, _partition: usize) -> Result<SendableRecordBatchStream> {
+            unimplemented!("NoOpExecutionPlan::execute");
+        }
+    }
+
+    //  Produces an execution plan where the schema is mismatched from
+    //  the logical plan node.
+    struct BadExtensionPlanner {}
+
+    impl ExtensionPlanner for BadExtensionPlanner {
+        /// Create a physical plan for an extension node
+        fn plan_extension(
+            &self,
+            _node: &dyn UserDefinedLogicalNode,
+            _inputs: Vec<Arc<dyn ExecutionPlan>>,
+            _ctx_state: &ExecutionContextState,
+        ) -> Result<Arc<dyn ExecutionPlan>> {
+            Ok(Arc::new(NoOpExecutionPlan {
+                schema: SchemaRef::new(Schema::new(vec![Field::new(
+                    "b",
+                    DataType::Int32,
+                    false,
+                )])),
+            }))
+        }
+    }
+}
diff --git a/rust/datafusion/src/physical_plan/projection.rs b/rust/datafusion/src/physical_plan/projection.rs
new file mode 100644
index 00000000000..e15a264e5c1
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/projection.rs
@@ -0,0 +1,216 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Defines the projection execution plan. A projection determines which columns or expressions
+//! are returned from a query. The SQL statement `SELECT a, b, a+b FROM t1` is an example
+//! of a projection on table `t1` where the expressions `a`, `b`, and `a+b` are the
+//! projection expressions.
+
+use std::any::Any;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+
+use crate::error::{DataFusionError, Result};
+use crate::physical_plan::{ExecutionPlan, Partitioning, PhysicalExpr};
+use arrow::datatypes::{Field, Schema, SchemaRef};
+use arrow::error::Result as ArrowResult;
+use arrow::record_batch::RecordBatch;
+
+use super::{RecordBatchStream, SendableRecordBatchStream};
+use async_trait::async_trait;
+
+use futures::stream::Stream;
+use futures::stream::StreamExt;
+
+/// Execution plan for a projection
+#[derive(Debug)]
+pub struct ProjectionExec {
+    /// The projection expressions stored as tuples of (expression, output column name)
+    expr: Vec<(Arc<dyn PhysicalExpr>, String)>,
+    /// The schema once the projection has been applied to the input
+    schema: SchemaRef,
+    /// The input plan
+    input: Arc<dyn ExecutionPlan>,
+}
+
+impl ProjectionExec {
+    /// Create a projection on an input
+    pub fn try_new(
+        expr: Vec<(Arc<dyn PhysicalExpr>, String)>,
+        input: Arc<dyn ExecutionPlan>,
+    ) -> Result<Self> {
+        let input_schema = input.schema();
+
+        let fields: Result<Vec<_>> = expr
+            .iter()
+            .map(|(e, name)| {
+                Ok(Field::new(
+                    name,
+                    e.data_type(&input_schema)?,
+                    e.nullable(&input_schema)?,
+                ))
+            })
+            .collect();
+
+        let schema = Arc::new(Schema::new(fields?));
+
+        Ok(Self {
+            expr,
+            schema,
+            input: input.clone(),
+        })
+    }
+}
+
+#[async_trait]
+impl ExecutionPlan for ProjectionExec {
+    /// Return a reference to Any that can be used for downcasting
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    /// Get the schema for this execution plan
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+
+    fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
+        vec![self.input.clone()]
+    }
+
+    /// Get the output partitioning of this plan
+    fn output_partitioning(&self) -> Partitioning {
+        self.input.output_partitioning()
+    }
+
+    fn with_new_children(
+        &self,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        match children.len() {
+            1 => Ok(Arc::new(ProjectionExec::try_new(
+                self.expr.clone(),
+                children[0].clone(),
+            )?)),
+            _ => Err(DataFusionError::Internal(
+                "ProjectionExec wrong number of children".to_string(),
+            )),
+        }
+    }
+
+    async fn execute(&self, partition: usize) -> Result<SendableRecordBatchStream> {
+        Ok(Box::pin(ProjectionStream {
+            schema: self.schema.clone(),
+            expr: self.expr.iter().map(|x| x.0.clone()).collect(),
+            input: self.input.execute(partition).await?,
+        }))
+    }
+}
+
+fn batch_project(
+    batch: &RecordBatch,
+    expressions: &Vec<Arc<dyn PhysicalExpr>>,
+    schema: &SchemaRef,
+) -> ArrowResult<RecordBatch> {
+    expressions
+        .iter()
+        .map(|expr| expr.evaluate(&batch))
+        .collect::<Result<Vec<_>>>()
+        .map_or_else(
+            |e| Err(DataFusionError::into_arrow_external_error(e)),
+            |arrays| RecordBatch::try_new(schema.clone(), arrays),
+        )
+}
+
+/// Projection iterator
+struct ProjectionStream {
+    schema: SchemaRef,
+    expr: Vec<Arc<dyn PhysicalExpr>>,
+    input: SendableRecordBatchStream,
+}
+
+impl Stream for ProjectionStream {
+    type Item = ArrowResult<RecordBatch>;
+
+    fn poll_next(
+        mut self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<Option<Self::Item>> {
+        self.input.poll_next_unpin(cx).map(|x| match x {
+            Some(Ok(batch)) => Some(batch_project(&batch, &self.expr, &self.schema)),
+            other => other,
+        })
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        // same number of record batches
+        self.input.size_hint()
+    }
+}
+
+impl RecordBatchStream for ProjectionStream {
+    /// Get the schema
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::*;
+    use crate::physical_plan::csv::{CsvExec, CsvReadOptions};
+    use crate::physical_plan::expressions::col;
+    use crate::test;
+    use futures::future;
+
+    #[tokio::test]
+    async fn project_first_column() -> Result<()> {
+        let schema = test::aggr_test_schema();
+
+        let partitions = 4;
+        let path = test::create_partitioned_csv("aggregate_test_100.csv", partitions)?;
+
+        let csv =
+            CsvExec::try_new(&path, CsvReadOptions::new().schema(&schema), None, 1024)?;
+
+        // pick column c1 and name it column c1 in the output schema
+        let projection =
+            ProjectionExec::try_new(vec![(col("c1"), "c1".to_string())], Arc::new(csv))?;
+
+        let mut partition_count = 0;
+        let mut row_count = 0;
+        for partition in 0..projection.output_partitioning().partition_count() {
+            partition_count += 1;
+            let stream = projection.execute(partition).await?;
+
+            row_count += stream
+                .map(|batch| {
+                    let batch = batch.unwrap();
+                    assert_eq!(1, batch.num_columns());
+                    batch.num_rows()
+                })
+                .fold(0, |acc, x| future::ready(acc + x))
+                .await;
+        }
+        assert_eq!(partitions, partition_count);
+        assert_eq!(100, row_count);
+
+        Ok(())
+    }
+}
diff --git a/rust/datafusion/src/physical_plan/sort.rs b/rust/datafusion/src/physical_plan/sort.rs
new file mode 100644
index 00000000000..0c3601a0f44
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/sort.rs
@@ -0,0 +1,336 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Defines the SORT plan
+
+use std::any::Any;
+use std::sync::Arc;
+
+use arrow::array::ArrayRef;
+pub use arrow::compute::SortOptions;
+use arrow::compute::{concat, lexsort_to_indices, take, SortColumn, TakeOptions};
+use arrow::datatypes::SchemaRef;
+use arrow::record_batch::RecordBatch;
+
+use super::SendableRecordBatchStream;
+use crate::error::{DataFusionError, Result};
+use crate::physical_plan::common::SizedRecordBatchStream;
+use crate::physical_plan::expressions::PhysicalSortExpr;
+use crate::physical_plan::{common, Distribution, ExecutionPlan, Partitioning};
+
+use async_trait::async_trait;
+
+/// Sort execution plan
+#[derive(Debug)]
+pub struct SortExec {
+    /// Input schema
+    input: Arc<dyn ExecutionPlan>,
+    /// Sort expressions
+    expr: Vec<PhysicalSortExpr>,
+    /// Number of threads to execute input partitions on before combining into a single partition
+    concurrency: usize,
+}
+
+impl SortExec {
+    /// Create a new sort execution plan
+    pub fn try_new(
+        expr: Vec<PhysicalSortExpr>,
+        input: Arc<dyn ExecutionPlan>,
+        concurrency: usize,
+    ) -> Result<Self> {
+        Ok(Self {
+            expr,
+            input,
+            concurrency,
+        })
+    }
+}
+
+#[async_trait]
+impl ExecutionPlan for SortExec {
+    /// Return a reference to Any that can be used for downcasting
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.input.schema().clone()
+    }
+
+    fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
+        vec![self.input.clone()]
+    }
+
+    /// Get the output partitioning of this plan
+    fn output_partitioning(&self) -> Partitioning {
+        Partitioning::UnknownPartitioning(1)
+    }
+
+    fn required_child_distribution(&self) -> Distribution {
+        Distribution::SinglePartition
+    }
+
+    fn with_new_children(
+        &self,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        match children.len() {
+            1 => Ok(Arc::new(SortExec::try_new(
+                self.expr.clone(),
+                children[0].clone(),
+                self.concurrency,
+            )?)),
+            _ => Err(DataFusionError::Internal(
+                "SortExec wrong number of children".to_string(),
+            )),
+        }
+    }
+
+    async fn execute(&self, partition: usize) -> Result<SendableRecordBatchStream> {
+        if 0 != partition {
+            return Err(DataFusionError::Internal(format!(
+                "SortExec invalid partition {}",
+                partition
+            )));
+        }
+
+        // sort needs to operate on a single partition currently
+        if 1 != self.input.output_partitioning().partition_count() {
+            return Err(DataFusionError::Internal(
+                "SortExec requires a single input partition".to_owned(),
+            ));
+        }
+        let it = self.input.execute(0).await?;
+        let batches = common::collect(it).await?;
+
+        // combine all record batches into one for each column
+        let combined_batch = RecordBatch::try_new(
+            self.schema(),
+            self.schema()
+                .fields()
+                .iter()
+                .enumerate()
+                .map(|(i, _)| -> Result<ArrayRef> {
+                    Ok(concat(
+                        &batches
+                            .iter()
+                            .map(|batch| batch.columns()[i].clone())
+                            .collect::<Vec<ArrayRef>>(),
+                    )?)
+                })
+                .collect::<Result<Vec<ArrayRef>>>()?,
+        )?;
+
+        // sort combined record batch
+        let indices = lexsort_to_indices(
+            &self
+                .expr
+                .iter()
+                .map(|e| e.evaluate_to_sort_column(&combined_batch))
+                .collect::<Result<Vec<SortColumn>>>()?,
+        )?;
+
+        // reorder all rows based on sorted indices
+        let sorted_batch = RecordBatch::try_new(
+            self.schema(),
+            combined_batch
+                .columns()
+                .iter()
+                .map(|column| -> Result<ArrayRef> {
+                    Ok(take(
+                        column,
+                        &indices,
+                        // disable bound check overhead since indices are already generated from
+                        // the same record batch
+                        Some(TakeOptions {
+                            check_bounds: false,
+                        }),
+                    )?)
+                })
+                .collect::<Result<Vec<ArrayRef>>>()?,
+        )?;
+
+        Ok(Box::pin(SizedRecordBatchStream::new(
+            self.schema(),
+            vec![Arc::new(sorted_batch)],
+        )))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::physical_plan::csv::{CsvExec, CsvReadOptions};
+    use crate::physical_plan::expressions::col;
+    use crate::physical_plan::memory::MemoryExec;
+    use crate::physical_plan::merge::MergeExec;
+    use crate::test;
+    use arrow::array::*;
+    use arrow::datatypes::*;
+
+    #[tokio::test]
+    async fn test_sort() -> Result<()> {
+        let schema = test::aggr_test_schema();
+        let partitions = 4;
+        let path = test::create_partitioned_csv("aggregate_test_100.csv", partitions)?;
+        let csv =
+            CsvExec::try_new(&path, CsvReadOptions::new().schema(&schema), None, 1024)?;
+
+        let sort_exec = Arc::new(SortExec::try_new(
+            vec![
+                // c1 string column
+                PhysicalSortExpr {
+                    expr: col("c1"),
+                    options: SortOptions::default(),
+                },
+                // c2 uin32 column
+                PhysicalSortExpr {
+                    expr: col("c2"),
+                    options: SortOptions::default(),
+                },
+                // c7 uin8 column
+                PhysicalSortExpr {
+                    expr: col("c7"),
+                    options: SortOptions::default(),
+                },
+            ],
+            Arc::new(MergeExec::new(Arc::new(csv))),
+            2,
+        )?);
+
+        let result: Vec<RecordBatch> = test::execute(sort_exec).await?;
+        assert_eq!(result.len(), 1);
+
+        let columns = result[0].columns();
+
+        let c1 = as_string_array(&columns[0]);
+        assert_eq!(c1.value(0), "a");
+        assert_eq!(c1.value(c1.len() - 1), "e");
+
+        let c2 = as_primitive_array::<UInt32Type>(&columns[1]);
+        assert_eq!(c2.value(0), 1);
+        assert_eq!(c2.value(c2.len() - 1), 5,);
+
+        let c7 = as_primitive_array::<UInt8Type>(&columns[6]);
+        assert_eq!(c7.value(0), 15);
+        assert_eq!(c7.value(c7.len() - 1), 254,);
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_lex_sort_by_float() -> Result<()> {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("a", DataType::Float32, true),
+            Field::new("b", DataType::Float64, true),
+        ]));
+
+        // define data.
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Float32Array::from(vec![
+                    Some(f32::NAN),
+                    None,
+                    None,
+                    Some(f32::NAN),
+                    Some(1.0_f32),
+                    Some(1.0_f32),
+                    Some(2.0_f32),
+                    Some(3.0_f32),
+                ])),
+                Arc::new(Float64Array::from(vec![
+                    Some(200.0_f64),
+                    Some(20.0_f64),
+                    Some(10.0_f64),
+                    Some(100.0_f64),
+                    Some(f64::NAN),
+                    None,
+                    None,
+                    Some(f64::NAN),
+                ])),
+            ],
+        )?;
+
+        let sort_exec = Arc::new(SortExec::try_new(
+            vec![
+                PhysicalSortExpr {
+                    expr: col("a"),
+                    options: SortOptions {
+                        descending: true,
+                        nulls_first: true,
+                    },
+                },
+                PhysicalSortExpr {
+                    expr: col("b"),
+                    options: SortOptions {
+                        descending: false,
+                        nulls_first: false,
+                    },
+                },
+            ],
+            Arc::new(MemoryExec::try_new(&vec![vec![batch]], schema, None)?),
+            2,
+        )?);
+
+        assert_eq!(DataType::Float32, *sort_exec.schema().field(0).data_type());
+        assert_eq!(DataType::Float64, *sort_exec.schema().field(1).data_type());
+
+        let result: Vec<RecordBatch> = test::execute(sort_exec).await?;
+        assert_eq!(result.len(), 1);
+
+        let columns = result[0].columns();
+
+        assert_eq!(DataType::Float32, *columns[0].data_type());
+        assert_eq!(DataType::Float64, *columns[1].data_type());
+
+        let a = as_primitive_array::<Float32Type>(&columns[0]);
+        let b = as_primitive_array::<Float64Type>(&columns[1]);
+
+        // convert result to strings to allow comparing to expected result containing NaN
+        let result: Vec<(Option<String>, Option<String>)> = (0..result[0].num_rows())
+            .map(|i| {
+                let aval = if a.is_valid(i) {
+                    Some(a.value(i).to_string())
+                } else {
+                    None
+                };
+                let bval = if b.is_valid(i) {
+                    Some(b.value(i).to_string())
+                } else {
+                    None
+                };
+                (aval, bval)
+            })
+            .collect();
+
+        let expected: Vec<(Option<String>, Option<String>)> = vec![
+            (None, Some("10".to_owned())),
+            (None, Some("20".to_owned())),
+            (Some("NaN".to_owned()), Some("100".to_owned())),
+            (Some("NaN".to_owned()), Some("200".to_owned())),
+            (Some("3".to_owned()), Some("NaN".to_owned())),
+            (Some("2".to_owned()), None),
+            (Some("1".to_owned()), Some("NaN".to_owned())),
+            (Some("1".to_owned()), None),
+        ];
+
+        assert_eq!(expected, result);
+
+        Ok(())
+    }
+}
diff --git a/rust/datafusion/src/physical_plan/string_expressions.rs b/rust/datafusion/src/physical_plan/string_expressions.rs
new file mode 100644
index 00000000000..ea70c8db552
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/string_expressions.rs
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! String expressions
+
+use crate::error::{DataFusionError, Result};
+use arrow::array::{Array, ArrayRef, StringArray, StringBuilder};
+
+macro_rules! downcast_vec {
+    ($ARGS:expr, $ARRAY_TYPE:ident) => {{
+        $ARGS
+            .iter()
+            .map(|e| match e.as_any().downcast_ref::<$ARRAY_TYPE>() {
+                Some(array) => Ok(array),
+                _ => Err(DataFusionError::Internal("failed to downcast".to_string())),
+            })
+    }};
+}
+
+/// concatenate string columns together.
+pub fn concatenate(args: &[ArrayRef]) -> Result<StringArray> {
+    // downcast all arguments to strings
+    let args = downcast_vec!(args, StringArray).collect::<Result<Vec<&StringArray>>>()?;
+    // do not accept 0 arguments.
+    if args.len() == 0 {
+        return Err(DataFusionError::Internal(
+            "Concatenate was called with 0 arguments. It requires at least one."
+                .to_string(),
+        ));
+    }
+
+    let mut builder = StringBuilder::new(args.len());
+    // for each entry in the array
+    for index in 0..args[0].len() {
+        let mut owned_string: String = "".to_owned();
+
+        // if any is null, the result is null
+        let mut is_null = false;
+        for arg in &args {
+            if arg.is_null(index) {
+                is_null = true;
+                break; // short-circuit as we already know the result
+            } else {
+                owned_string.push_str(&arg.value(index));
+            }
+        }
+        if is_null {
+            builder.append_null()?;
+        } else {
+            builder.append_value(&owned_string)?;
+        }
+    }
+    Ok(builder.finish())
+}
diff --git a/rust/datafusion/src/physical_plan/type_coercion.rs b/rust/datafusion/src/physical_plan/type_coercion.rs
new file mode 100644
index 00000000000..728dd81538e
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/type_coercion.rs
@@ -0,0 +1,359 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Type coercion rules for functions with multiple valid signatures
+//!
+//! Coercion is performed automatically by DataFusion when the types
+//! of arguments passed to a function do not exacty match the types
+//! required by that function. In this case, DataFusion will attempt to
+//! *coerce* the arguments to types accepted by the function by
+//! inserting CAST operations.
+//!
+//! CAST operations added by coercion are lossless and never discard
+//! information. For example coercion from i32 -> i64 might be
+//! performed because all valid i32 values can be represented using an
+//! i64. However, i64 -> i32 is never performed as there are i64
+//! values which can not be represented by i32 values.
+
+use std::sync::Arc;
+
+use arrow::datatypes::{DataType, Schema};
+
+use super::{functions::Signature, PhysicalExpr};
+use crate::error::{DataFusionError, Result};
+use crate::physical_plan::expressions::cast;
+
+/// Returns `expressions` coerced to types compatible with
+/// `signature`, if possible.
+///
+/// See the module level documentation for more detail on coercion.
+pub fn coerce(
+    expressions: &Vec<Arc<dyn PhysicalExpr>>,
+    schema: &Schema,
+    signature: &Signature,
+) -> Result<Vec<Arc<dyn PhysicalExpr>>> {
+    let current_types = expressions
+        .iter()
+        .map(|e| e.data_type(schema))
+        .collect::<Result<Vec<_>>>()?;
+
+    let new_types = data_types(&current_types, signature)?;
+
+    expressions
+        .iter()
+        .enumerate()
+        .map(|(i, expr)| cast(expr.clone(), &schema, new_types[i].clone()))
+        .collect::<Result<Vec<_>>>()
+}
+
+/// Returns the data types that each argument must be coerced to match
+/// `signature`.
+///
+/// See the module level documentation for more detail on coercion.
+pub fn data_types(
+    current_types: &Vec<DataType>,
+    signature: &Signature,
+) -> Result<Vec<DataType>> {
+    let valid_types = match signature {
+        Signature::Variadic(valid_types) => valid_types
+            .iter()
+            .map(|valid_type| current_types.iter().map(|_| valid_type.clone()).collect())
+            .collect(),
+        Signature::Uniform(number, valid_types) => valid_types
+            .iter()
+            .map(|valid_type| (0..*number).map(|_| valid_type.clone()).collect())
+            .collect(),
+        Signature::VariadicEqual => {
+            // one entry with the same len as current_types, whose type is `current_types[0]`.
+            vec![current_types
+                .iter()
+                .map(|_| current_types[0].clone())
+                .collect()]
+        }
+        Signature::Exact(valid_types) => vec![valid_types.clone()],
+        Signature::Any(number) => {
+            if current_types.len() != *number {
+                return Err(DataFusionError::Plan(format!(
+                    "The function expected {} arguments but received {}",
+                    number,
+                    current_types.len()
+                )));
+            }
+            vec![(0..*number).map(|i| current_types[i].clone()).collect()]
+        }
+    };
+
+    if valid_types.contains(current_types) {
+        return Ok(current_types.clone());
+    }
+
+    for valid_types in valid_types {
+        if let Some(types) = maybe_data_types(&valid_types, &current_types) {
+            return Ok(types);
+        }
+    }
+
+    // none possible -> Error
+    Err(DataFusionError::Plan(format!(
+        "Coercion from {:?} to the signature {:?} failed.",
+        current_types, signature
+    )))
+}
+
+/// Try to coerce current_types into valid_types.
+fn maybe_data_types(
+    valid_types: &Vec<DataType>,
+    current_types: &Vec<DataType>,
+) -> Option<Vec<DataType>> {
+    if valid_types.len() != current_types.len() {
+        return None;
+    }
+
+    let mut new_type = Vec::with_capacity(valid_types.len());
+    for (i, valid_type) in valid_types.iter().enumerate() {
+        let current_type = &current_types[i];
+
+        if current_type == valid_type {
+            new_type.push(current_type.clone())
+        } else {
+            // attempt to coerce
+            if can_coerce_from(valid_type, &current_type) {
+                new_type.push(valid_type.clone())
+            } else {
+                // not possible
+                return None;
+            }
+        }
+    }
+    Some(new_type)
+}
+
+/// Return true if a value of type `type_from` can be coerced
+/// (losslessly converted) into a value of `type_to`
+///
+/// See the module level documentation for more detail on coercion.
+pub fn can_coerce_from(type_into: &DataType, type_from: &DataType) -> bool {
+    use self::DataType::*;
+    match type_into {
+        Int8 => match type_from {
+            Int8 => true,
+            _ => false,
+        },
+        Int16 => match type_from {
+            Int8 | Int16 | UInt8 => true,
+            _ => false,
+        },
+        Int32 => match type_from {
+            Int8 | Int16 | Int32 | UInt8 | UInt16 => true,
+            _ => false,
+        },
+        Int64 => match type_from {
+            Int8 | Int16 | Int32 | Int64 | UInt8 | UInt16 | UInt32 => true,
+            _ => false,
+        },
+        UInt8 => match type_from {
+            UInt8 => true,
+            _ => false,
+        },
+        UInt16 => match type_from {
+            UInt8 | UInt16 => true,
+            _ => false,
+        },
+        UInt32 => match type_from {
+            UInt8 | UInt16 | UInt32 => true,
+            _ => false,
+        },
+        UInt64 => match type_from {
+            UInt8 | UInt16 | UInt32 | UInt64 => true,
+            _ => false,
+        },
+        Float32 => match type_from {
+            Int8 | Int16 | Int32 | Int64 => true,
+            UInt8 | UInt16 | UInt32 | UInt64 => true,
+            Float32 => true,
+            _ => false,
+        },
+        Float64 => match type_from {
+            Int8 | Int16 | Int32 | Int64 => true,
+            UInt8 | UInt16 | UInt32 | UInt64 => true,
+            Float32 | Float64 => true,
+            _ => false,
+        },
+        Utf8 => true,
+        _ => false,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::physical_plan::expressions::col;
+    use arrow::datatypes::{DataType, Field, Schema};
+
+    #[test]
+    fn test_maybe_data_types() -> Result<()> {
+        // this vec contains: arg1, arg2, expected result
+        let cases = vec![
+            // 2 entries, same values
+            (
+                vec![DataType::UInt8, DataType::UInt16],
+                vec![DataType::UInt8, DataType::UInt16],
+                Some(vec![DataType::UInt8, DataType::UInt16]),
+            ),
+            // 2 entries, can coerse values
+            (
+                vec![DataType::UInt16, DataType::UInt16],
+                vec![DataType::UInt8, DataType::UInt16],
+                Some(vec![DataType::UInt16, DataType::UInt16]),
+            ),
+            // 0 entries, all good
+            (vec![], vec![], Some(vec![])),
+            // 2 entries, can't coerce
+            (
+                vec![DataType::Boolean, DataType::UInt16],
+                vec![DataType::UInt8, DataType::UInt16],
+                None,
+            ),
+            // u32 -> u16 is possible
+            (
+                vec![DataType::Boolean, DataType::UInt32],
+                vec![DataType::Boolean, DataType::UInt16],
+                Some(vec![DataType::Boolean, DataType::UInt32]),
+            ),
+        ];
+
+        for case in cases {
+            assert_eq!(maybe_data_types(&case.0, &case.1), case.2)
+        }
+        Ok(())
+    }
+
+    #[test]
+    fn test_coerce() -> Result<()> {
+        // create a schema
+        let schema = |t: Vec<DataType>| {
+            Schema::new(
+                t.iter()
+                    .enumerate()
+                    .map(|(i, t)| Field::new(&*format!("c{}", i), t.clone(), true))
+                    .collect(),
+            )
+        };
+
+        // create a vector of expressions
+        let expressions = |t: Vec<DataType>, schema| -> Result<Vec<_>> {
+            t.iter()
+                .enumerate()
+                .map(|(i, t)| cast(col(&format!("c{}", i)), &schema, t.clone()))
+                .collect::<Result<Vec<_>>>()
+        };
+
+        // create a case: input + expected result
+        let case =
+            |observed: Vec<DataType>, valid, expected: Vec<DataType>| -> Result<_> {
+                let schema = schema(observed.clone());
+                let expr = expressions(observed, schema.clone())?;
+                let expected = expressions(expected, schema.clone())?;
+                Ok((expr.clone(), schema, valid, expected))
+            };
+
+        let cases = vec![
+            // u16 -> u32
+            case(
+                vec![DataType::UInt16],
+                Signature::Uniform(1, vec![DataType::UInt32]),
+                vec![DataType::UInt32],
+            )?,
+            // same type
+            case(
+                vec![DataType::UInt32, DataType::UInt32],
+                Signature::Uniform(2, vec![DataType::UInt32]),
+                vec![DataType::UInt32, DataType::UInt32],
+            )?,
+            case(
+                vec![DataType::UInt32],
+                Signature::Uniform(1, vec![DataType::Float32, DataType::Float64]),
+                vec![DataType::Float32],
+            )?,
+            // u32 -> f32
+            case(
+                vec![DataType::UInt32, DataType::UInt32],
+                Signature::Variadic(vec![DataType::Float32]),
+                vec![DataType::Float32, DataType::Float32],
+            )?,
+            // u32 -> f32
+            case(
+                vec![DataType::Float32, DataType::UInt32],
+                Signature::VariadicEqual,
+                vec![DataType::Float32, DataType::Float32],
+            )?,
+            // common type is u64
+            case(
+                vec![DataType::UInt32, DataType::UInt64],
+                Signature::Variadic(vec![DataType::UInt32, DataType::UInt64]),
+                vec![DataType::UInt64, DataType::UInt64],
+            )?,
+            // f32 -> f32
+            case(
+                vec![DataType::Float32],
+                Signature::Any(1),
+                vec![DataType::Float32],
+            )?,
+        ];
+
+        for case in cases {
+            let observed = format!("{:?}", coerce(&case.0, &case.1, &case.2)?);
+            let expected = format!("{:?}", case.3);
+            assert_eq!(observed, expected);
+        }
+
+        // now cases that are expected to fail
+        let cases = vec![
+            // we do not know how to cast bool to UInt16 => fail
+            case(
+                vec![DataType::Boolean],
+                Signature::Uniform(1, vec![DataType::UInt16]),
+                vec![],
+            )?,
+            // u32 and bool are not uniform
+            case(
+                vec![DataType::UInt32, DataType::Boolean],
+                Signature::VariadicEqual,
+                vec![],
+            )?,
+            // bool is not castable to u32
+            case(
+                vec![DataType::Boolean, DataType::Boolean],
+                Signature::Variadic(vec![DataType::UInt32]),
+                vec![],
+            )?,
+            // expected two arguments
+            case(vec![DataType::UInt32], Signature::Any(2), vec![])?,
+        ];
+
+        for case in cases {
+            if let Ok(_) = coerce(&case.0, &case.1, &case.2) {
+                return Err(DataFusionError::Plan(format!(
+                    "Error was expected in {:?}",
+                    case
+                )));
+            }
+        }
+
+        Ok(())
+    }
+}
diff --git a/rust/datafusion/src/physical_plan/udaf.rs b/rust/datafusion/src/physical_plan/udaf.rs
new file mode 100644
index 00000000000..db86e1447ab
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/udaf.rs
@@ -0,0 +1,156 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! This module contains functions and structs supporting user-defined aggregate functions.
+
+use fmt::{Debug, Formatter};
+use std::fmt;
+
+use arrow::{
+    datatypes::Field,
+    datatypes::{DataType, Schema},
+};
+
+use crate::physical_plan::PhysicalExpr;
+use crate::{error::Result, logical_plan::Expr};
+
+use super::{
+    aggregates::AccumulatorFunctionImplementation,
+    aggregates::StateTypeFunction,
+    expressions::format_state_name,
+    functions::{ReturnTypeFunction, Signature},
+    type_coercion::coerce,
+    Accumulator, AggregateExpr,
+};
+use std::sync::Arc;
+
+/// Logical representation of a user-defined aggregate function (UDAF)
+/// A UDAF is different from a UDF in that it is stateful across batches.
+#[derive(Clone)]
+pub struct AggregateUDF {
+    /// name
+    pub name: String,
+    /// signature
+    pub signature: Signature,
+    /// Return type
+    pub return_type: ReturnTypeFunction,
+    /// actual implementation
+    pub accumulator: AccumulatorFunctionImplementation,
+    /// the accumulator's state's description as a function of the return type
+    pub state_type: StateTypeFunction,
+}
+
+impl Debug for AggregateUDF {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        f.debug_struct("AggregateUDF")
+            .field("name", &self.name)
+            .field("signature", &self.signature)
+            .field("fun", &"<FUNC>")
+            .finish()
+    }
+}
+
+impl AggregateUDF {
+    /// Create a new AggregateUDF
+    pub fn new(
+        name: &str,
+        signature: &Signature,
+        return_type: &ReturnTypeFunction,
+        accumulator: &AccumulatorFunctionImplementation,
+        state_type: &StateTypeFunction,
+    ) -> Self {
+        Self {
+            name: name.to_owned(),
+            signature: signature.clone(),
+            return_type: return_type.clone(),
+            accumulator: accumulator.clone(),
+            state_type: state_type.clone(),
+        }
+    }
+
+    /// creates a logical expression with a call of the UDAF
+    /// This utility allows using the UDAF without requiring access to the registry.
+    pub fn call(&self, args: Vec<Expr>) -> Expr {
+        Expr::AggregateUDF {
+            fun: Arc::new(self.clone()),
+            args,
+        }
+    }
+}
+
+/// Creates a physical expression of the UDAF, that includes all necessary type coercion.
+/// This function errors when `args`' can't be coerced to a valid argument type of the UDAF.
+pub fn create_aggregate_expr(
+    fun: &AggregateUDF,
+    args: &Vec<Arc<dyn PhysicalExpr>>,
+    input_schema: &Schema,
+    name: String,
+) -> Result<Arc<dyn AggregateExpr>> {
+    // coerce
+    let args = coerce(args, input_schema, &fun.signature)?;
+
+    let arg_types = args
+        .iter()
+        .map(|arg| arg.data_type(input_schema))
+        .collect::<Result<Vec<_>>>()?;
+
+    Ok(Arc::new(AggregateFunctionExpr {
+        fun: fun.clone(),
+        args: args.clone(),
+        data_type: (fun.return_type)(&arg_types)?.as_ref().clone(),
+        name: name.clone(),
+    }))
+}
+
+/// Physical aggregate expression of a UDAF.
+#[derive(Debug)]
+pub struct AggregateFunctionExpr {
+    fun: AggregateUDF,
+    args: Vec<Arc<dyn PhysicalExpr>>,
+    data_type: DataType,
+    name: String,
+}
+
+impl AggregateExpr for AggregateFunctionExpr {
+    fn expressions(&self) -> Vec<Arc<dyn PhysicalExpr>> {
+        self.args.clone()
+    }
+
+    fn state_fields(&self) -> Result<Vec<Field>> {
+        let fields = (self.fun.state_type)(&self.data_type)?
+            .iter()
+            .enumerate()
+            .map(|(i, data_type)| {
+                Field::new(
+                    &format_state_name(&self.name, &format!("{}", i)),
+                    data_type.clone(),
+                    true,
+                )
+            })
+            .collect::<Vec<Field>>();
+
+        Ok(fields)
+    }
+
+    fn field(&self) -> Result<Field> {
+        Ok(Field::new(&self.name, self.data_type.clone(), true))
+    }
+
+    fn create_accumulator(&self) -> Result<Box<dyn Accumulator>> {
+        (self.fun.accumulator)()
+    }
+}
diff --git a/rust/datafusion/src/physical_plan/udf.rs b/rust/datafusion/src/physical_plan/udf.rs
new file mode 100644
index 00000000000..9ac4134e8bf
--- /dev/null
+++ b/rust/datafusion/src/physical_plan/udf.rs
@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! UDF support
+
+use fmt::{Debug, Formatter};
+use std::fmt;
+
+use arrow::datatypes::Schema;
+
+use crate::error::Result;
+use crate::{logical_plan::Expr, physical_plan::PhysicalExpr};
+
+use super::{
+    functions::{
+        ReturnTypeFunction, ScalarFunctionExpr, ScalarFunctionImplementation, Signature,
+    },
+    type_coercion::coerce,
+};
+use std::sync::Arc;
+
+/// Logical representation of a UDF.
+#[derive(Clone)]
+pub struct ScalarUDF {
+    /// name
+    pub name: String,
+    /// signature
+    pub signature: Signature,
+    /// Return type
+    pub return_type: ReturnTypeFunction,
+    /// actual implementation
+    pub fun: ScalarFunctionImplementation,
+}
+
+impl Debug for ScalarUDF {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        f.debug_struct("ScalarUDF")
+            .field("name", &self.name)
+            .field("signature", &self.signature)
+            .field("fun", &"<FUNC>")
+            .finish()
+    }
+}
+
+impl ScalarUDF {
+    /// Create a new ScalarUDF
+    pub fn new(
+        name: &str,
+        signature: &Signature,
+        return_type: &ReturnTypeFunction,
+        fun: &ScalarFunctionImplementation,
+    ) -> Self {
+        Self {
+            name: name.to_owned(),
+            signature: signature.clone(),
+            return_type: return_type.clone(),
+            fun: fun.clone(),
+        }
+    }
+
+    /// creates a logical expression with a call of the UDF
+    /// This utility allows using the UDF without requiring access to the registry.
+    pub fn call(&self, args: Vec<Expr>) -> Expr {
+        Expr::ScalarUDF {
+            fun: Arc::new(self.clone()),
+            args,
+        }
+    }
+}
+
+/// Create a physical expression of the UDF.
+/// This function errors when `args`' can't be coerced to a valid argument type of the UDF.
+pub fn create_physical_expr(
+    fun: &ScalarUDF,
+    args: &Vec<Arc<dyn PhysicalExpr>>,
+    input_schema: &Schema,
+) -> Result<Arc<dyn PhysicalExpr>> {
+    // coerce
+    let args = coerce(args, input_schema, &fun.signature)?;
+
+    let arg_types = args
+        .iter()
+        .map(|e| e.data_type(input_schema))
+        .collect::<Result<Vec<_>>>()?;
+
+    Ok(Arc::new(ScalarFunctionExpr::new(
+        &fun.name,
+        fun.fun.clone(),
+        args,
+        (fun.return_type)(&arg_types)?.as_ref(),
+    )))
+}
diff --git a/rust/datafusion/src/prelude.rs b/rust/datafusion/src/prelude.rs
new file mode 100644
index 00000000000..aac2ebf71f1
--- /dev/null
+++ b/rust/datafusion/src/prelude.rs
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.pub},
+
+//! A "prelude" for users of the datafusion crate.
+//!
+//! Like the standard library's prelude, this module simplifies importing of
+//! common items. Unlike the standard prelude, the contents of this module must
+//! be imported manually:
+//!
+//! ```
+//! use datafusion::prelude::*;
+//! ```
+
+pub use crate::dataframe::DataFrame;
+pub use crate::execution::context::{ExecutionConfig, ExecutionContext};
+pub use crate::logical_plan::{
+    array, avg, col, concat, count, create_udf, length, lit, max, min, sum,
+};
+pub use crate::physical_plan::csv::CsvReadOptions;
diff --git a/rust/datafusion/src/scalar.rs b/rust/datafusion/src/scalar.rs
new file mode 100644
index 00000000000..2eb1d69a617
--- /dev/null
+++ b/rust/datafusion/src/scalar.rs
@@ -0,0 +1,433 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! This module provides ScalarValue, an enum that can be used for storage of single elements
+
+use std::{convert::TryFrom, fmt, sync::Arc};
+
+use arrow::array::{
+    Array, BooleanArray, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array,
+    Int8Array, LargeStringArray, ListArray, StringArray, UInt16Array, UInt32Array,
+    UInt64Array, UInt8Array,
+};
+use arrow::array::{
+    Int16Builder, Int32Builder, Int64Builder, Int8Builder, ListBuilder, UInt16Builder,
+    UInt32Builder, UInt64Builder, UInt8Builder,
+};
+use arrow::{
+    array::ArrayRef,
+    datatypes::{DataType, Field},
+};
+
+use crate::error::{DataFusionError, Result};
+
+/// Represents a dynamically typed, nullable single value.
+/// This is the single-valued counter-part of arrow’s `Array`.
+#[derive(Clone, PartialEq)]
+pub enum ScalarValue {
+    /// true or false value
+    Boolean(Option<bool>),
+    /// 32bit float
+    Float32(Option<f32>),
+    /// 64bit float
+    Float64(Option<f64>),
+    /// signed 8bit int
+    Int8(Option<i8>),
+    /// signed 16bit int
+    Int16(Option<i16>),
+    /// signed 32bit int
+    Int32(Option<i32>),
+    /// signed 64bit int
+    Int64(Option<i64>),
+    /// unsigned 8bit int
+    UInt8(Option<u8>),
+    /// unsigned 16bit int
+    UInt16(Option<u16>),
+    /// unsigned 32bit int
+    UInt32(Option<u32>),
+    /// unsigned 64bit int
+    UInt64(Option<u64>),
+    /// utf-8 encoded string.
+    Utf8(Option<String>),
+    /// utf-8 encoded string representing a LargeString's arrow type.
+    LargeUtf8(Option<String>),
+    /// list of nested ScalarValue
+    List(Option<Vec<ScalarValue>>, DataType),
+}
+
+macro_rules! typed_cast {
+    ($array:expr, $index:expr, $ARRAYTYPE:ident, $SCALAR:ident) => {{
+        let array = $array.as_any().downcast_ref::<$ARRAYTYPE>().unwrap();
+        ScalarValue::$SCALAR(match array.is_null($index) {
+            true => None,
+            false => Some(array.value($index).into()),
+        })
+    }};
+}
+
+macro_rules! build_list {
+    ($VALUE_BUILDER_TY:ident, $SCALAR_TY:ident, $VALUES:expr) => {{
+        match $VALUES {
+            None => {
+                let mut builder = ListBuilder::new($VALUE_BUILDER_TY::new(0));
+                builder.append(false).unwrap();
+                builder.finish()
+            }
+            Some(values) => {
+                let mut builder = ListBuilder::new($VALUE_BUILDER_TY::new(values.len()));
+
+                for scalar_value in values {
+                    match scalar_value {
+                        ScalarValue::$SCALAR_TY(Some(v)) => {
+                            builder.values().append_value(*v).unwrap()
+                        }
+                        ScalarValue::$SCALAR_TY(None) => {
+                            builder.values().append_null().unwrap();
+                        }
+                        _ => panic!("Incompatible ScalarValue for list"),
+                    };
+                }
+
+                builder.append(true).unwrap();
+                builder.finish()
+            }
+        }
+    }};
+}
+
+impl ScalarValue {
+    /// Getter for the `DataType` of the value
+    pub fn get_datatype(&self) -> DataType {
+        match self {
+            ScalarValue::Boolean(_) => DataType::Boolean,
+            ScalarValue::UInt8(_) => DataType::UInt8,
+            ScalarValue::UInt16(_) => DataType::UInt16,
+            ScalarValue::UInt32(_) => DataType::UInt32,
+            ScalarValue::UInt64(_) => DataType::UInt64,
+            ScalarValue::Int8(_) => DataType::Int8,
+            ScalarValue::Int16(_) => DataType::Int16,
+            ScalarValue::Int32(_) => DataType::Int32,
+            ScalarValue::Int64(_) => DataType::Int64,
+            ScalarValue::Float32(_) => DataType::Float32,
+            ScalarValue::Float64(_) => DataType::Float64,
+            ScalarValue::Utf8(_) => DataType::Utf8,
+            ScalarValue::LargeUtf8(_) => DataType::LargeUtf8,
+            ScalarValue::List(_, data_type) => {
+                DataType::List(Box::new(Field::new("item", data_type.clone(), true)))
+            }
+        }
+    }
+
+    /// whether this value is null or not.
+    pub fn is_null(&self) -> bool {
+        match *self {
+            ScalarValue::Boolean(None)
+            | ScalarValue::UInt8(None)
+            | ScalarValue::UInt16(None)
+            | ScalarValue::UInt32(None)
+            | ScalarValue::UInt64(None)
+            | ScalarValue::Int8(None)
+            | ScalarValue::Int16(None)
+            | ScalarValue::Int32(None)
+            | ScalarValue::Int64(None)
+            | ScalarValue::Float32(None)
+            | ScalarValue::Float64(None)
+            | ScalarValue::Utf8(None)
+            | ScalarValue::LargeUtf8(None)
+            | ScalarValue::List(None, _) => true,
+            _ => false,
+        }
+    }
+
+    /// Converts a scalar value into an 1-row array.
+    pub fn to_array(&self) -> ArrayRef {
+        match self {
+            ScalarValue::Boolean(e) => Arc::new(BooleanArray::from(vec![*e])) as ArrayRef,
+            ScalarValue::Float64(e) => Arc::new(Float64Array::from(vec![*e])) as ArrayRef,
+            ScalarValue::Float32(e) => Arc::new(Float32Array::from(vec![*e])),
+            ScalarValue::Int8(e) => Arc::new(Int8Array::from(vec![*e])),
+            ScalarValue::Int16(e) => Arc::new(Int16Array::from(vec![*e])),
+            ScalarValue::Int32(e) => Arc::new(Int32Array::from(vec![*e])),
+            ScalarValue::Int64(e) => Arc::new(Int64Array::from(vec![*e])),
+            ScalarValue::UInt8(e) => Arc::new(UInt8Array::from(vec![*e])),
+            ScalarValue::UInt16(e) => Arc::new(UInt16Array::from(vec![*e])),
+            ScalarValue::UInt32(e) => Arc::new(UInt32Array::from(vec![*e])),
+            ScalarValue::UInt64(e) => Arc::new(UInt64Array::from(vec![*e])),
+            ScalarValue::Utf8(e) => Arc::new(StringArray::from(vec![e.as_deref()])),
+            ScalarValue::LargeUtf8(e) => {
+                Arc::new(LargeStringArray::from(vec![e.as_deref()]))
+            }
+            ScalarValue::List(values, data_type) => Arc::new(match data_type {
+                DataType::Int8 => build_list!(Int8Builder, Int8, values),
+                DataType::Int16 => build_list!(Int16Builder, Int16, values),
+                DataType::Int32 => build_list!(Int32Builder, Int32, values),
+                DataType::Int64 => build_list!(Int64Builder, Int64, values),
+                DataType::UInt8 => build_list!(UInt8Builder, UInt8, values),
+                DataType::UInt16 => build_list!(UInt16Builder, UInt16, values),
+                DataType::UInt32 => build_list!(UInt32Builder, UInt32, values),
+                DataType::UInt64 => build_list!(UInt64Builder, UInt64, values),
+                _ => panic!("Unexpected DataType for list"),
+            }),
+        }
+    }
+
+    /// Converts a value in `array` at `index` into a ScalarValue
+    pub fn try_from_array(array: &ArrayRef, index: usize) -> Result<Self> {
+        Ok(match array.data_type() {
+            DataType::Boolean => typed_cast!(array, index, BooleanArray, Boolean),
+            DataType::Float64 => typed_cast!(array, index, Float64Array, Float64),
+            DataType::Float32 => typed_cast!(array, index, Float32Array, Float32),
+            DataType::UInt64 => typed_cast!(array, index, UInt64Array, UInt64),
+            DataType::UInt32 => typed_cast!(array, index, UInt32Array, UInt32),
+            DataType::UInt16 => typed_cast!(array, index, UInt16Array, UInt16),
+            DataType::UInt8 => typed_cast!(array, index, UInt8Array, UInt8),
+            DataType::Int64 => typed_cast!(array, index, Int64Array, Int64),
+            DataType::Int32 => typed_cast!(array, index, Int32Array, Int32),
+            DataType::Int16 => typed_cast!(array, index, Int16Array, Int16),
+            DataType::Int8 => typed_cast!(array, index, Int8Array, Int8),
+            DataType::Utf8 => typed_cast!(array, index, StringArray, Utf8),
+            DataType::LargeUtf8 => typed_cast!(array, index, LargeStringArray, LargeUtf8),
+            DataType::List(nested_type) => {
+                let list_array = array.as_any().downcast_ref::<ListArray>().ok_or(
+                    DataFusionError::Internal("Failed to downcast ListArray".to_string()),
+                )?;
+                let value = match list_array.is_null(index) {
+                    true => None,
+                    false => {
+                        let nested_array = list_array.value(index);
+                        let scalar_vec = (0..nested_array.len())
+                            .map(|i| ScalarValue::try_from_array(&nested_array, i))
+                            .collect::<Result<Vec<_>>>()?;
+                        Some(scalar_vec)
+                    }
+                };
+                ScalarValue::List(value, nested_type.data_type().clone())
+            }
+            other => {
+                return Err(DataFusionError::NotImplemented(format!(
+                    "Can't create a scalar of array of type \"{:?}\"",
+                    other
+                )))
+            }
+        })
+    }
+}
+
+impl From<f64> for ScalarValue {
+    fn from(value: f64) -> Self {
+        ScalarValue::Float64(Some(value))
+    }
+}
+
+impl From<f32> for ScalarValue {
+    fn from(value: f32) -> Self {
+        ScalarValue::Float32(Some(value))
+    }
+}
+
+impl From<i8> for ScalarValue {
+    fn from(value: i8) -> Self {
+        ScalarValue::Int8(Some(value))
+    }
+}
+
+impl From<i16> for ScalarValue {
+    fn from(value: i16) -> Self {
+        ScalarValue::Int16(Some(value))
+    }
+}
+
+impl From<i32> for ScalarValue {
+    fn from(value: i32) -> Self {
+        ScalarValue::Int32(Some(value))
+    }
+}
+
+impl From<i64> for ScalarValue {
+    fn from(value: i64) -> Self {
+        ScalarValue::Int64(Some(value))
+    }
+}
+
+impl From<bool> for ScalarValue {
+    fn from(value: bool) -> Self {
+        ScalarValue::Boolean(Some(value))
+    }
+}
+
+impl From<u8> for ScalarValue {
+    fn from(value: u8) -> Self {
+        ScalarValue::UInt8(Some(value))
+    }
+}
+
+impl From<u16> for ScalarValue {
+    fn from(value: u16) -> Self {
+        ScalarValue::UInt16(Some(value))
+    }
+}
+
+impl From<u32> for ScalarValue {
+    fn from(value: u32) -> Self {
+        ScalarValue::UInt32(Some(value))
+    }
+}
+
+impl From<u64> for ScalarValue {
+    fn from(value: u64) -> Self {
+        ScalarValue::UInt64(Some(value))
+    }
+}
+
+impl TryFrom<&DataType> for ScalarValue {
+    type Error = DataFusionError;
+
+    fn try_from(datatype: &DataType) -> Result<Self> {
+        Ok(match datatype {
+            &DataType::Boolean => ScalarValue::Boolean(None),
+            &DataType::Float64 => ScalarValue::Float64(None),
+            &DataType::Float32 => ScalarValue::Float32(None),
+            &DataType::Int8 => ScalarValue::Int8(None),
+            &DataType::Int16 => ScalarValue::Int16(None),
+            &DataType::Int32 => ScalarValue::Int32(None),
+            &DataType::Int64 => ScalarValue::Int64(None),
+            &DataType::UInt8 => ScalarValue::UInt8(None),
+            &DataType::UInt16 => ScalarValue::UInt16(None),
+            &DataType::UInt32 => ScalarValue::UInt32(None),
+            &DataType::UInt64 => ScalarValue::UInt64(None),
+            &DataType::Utf8 => ScalarValue::Utf8(None),
+            &DataType::LargeUtf8 => ScalarValue::LargeUtf8(None),
+            &DataType::List(ref nested_type) => {
+                ScalarValue::List(None, nested_type.data_type().clone())
+            }
+            _ => {
+                return Err(DataFusionError::NotImplemented(format!(
+                    "Can't create a scalar of type \"{:?}\"",
+                    datatype
+                )))
+            }
+        })
+    }
+}
+
+macro_rules! format_option {
+    ($F:expr, $EXPR:expr) => {{
+        match $EXPR {
+            Some(e) => write!($F, "{}", e),
+            None => write!($F, "NULL"),
+        }
+    }};
+}
+
+impl fmt::Display for ScalarValue {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            ScalarValue::Boolean(e) => format_option!(f, e)?,
+            ScalarValue::Float32(e) => format_option!(f, e)?,
+            ScalarValue::Float64(e) => format_option!(f, e)?,
+            ScalarValue::Int8(e) => format_option!(f, e)?,
+            ScalarValue::Int16(e) => format_option!(f, e)?,
+            ScalarValue::Int32(e) => format_option!(f, e)?,
+            ScalarValue::Int64(e) => format_option!(f, e)?,
+            ScalarValue::UInt8(e) => format_option!(f, e)?,
+            ScalarValue::UInt16(e) => format_option!(f, e)?,
+            ScalarValue::UInt32(e) => format_option!(f, e)?,
+            ScalarValue::UInt64(e) => format_option!(f, e)?,
+            ScalarValue::Utf8(e) => format_option!(f, e)?,
+            ScalarValue::LargeUtf8(e) => format_option!(f, e)?,
+            ScalarValue::List(e, _) => match e {
+                Some(l) => write!(
+                    f,
+                    "{}",
+                    l.iter()
+                        .map(|v| format!("{}", v))
+                        .collect::<Vec<_>>()
+                        .join(",")
+                )?,
+                None => write!(f, "NULL")?,
+            },
+        };
+        Ok(())
+    }
+}
+
+impl fmt::Debug for ScalarValue {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            ScalarValue::Boolean(_) => write!(f, "Boolean({})", self),
+            ScalarValue::Float32(_) => write!(f, "Float32({})", self),
+            ScalarValue::Float64(_) => write!(f, "Float64({})", self),
+            ScalarValue::Int8(_) => write!(f, "Int8({})", self),
+            ScalarValue::Int16(_) => write!(f, "Int16({})", self),
+            ScalarValue::Int32(_) => write!(f, "Int32({})", self),
+            ScalarValue::Int64(_) => write!(f, "Int64({})", self),
+            ScalarValue::UInt8(_) => write!(f, "UInt8({})", self),
+            ScalarValue::UInt16(_) => write!(f, "UInt16({})", self),
+            ScalarValue::UInt32(_) => write!(f, "UInt32({})", self),
+            ScalarValue::UInt64(_) => write!(f, "UInt64({})", self),
+            ScalarValue::Utf8(_) => write!(f, "Utf8(\"{}\")", self),
+            ScalarValue::LargeUtf8(_) => write!(f, "LargeUtf8(\"{}\")", self),
+            ScalarValue::List(_, _) => write!(f, "List([{}])", self),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn scalar_list_null_to_array() -> Result<()> {
+        let list_array_ref = ScalarValue::List(None, DataType::UInt64).to_array();
+        let list_array = list_array_ref.as_any().downcast_ref::<ListArray>().unwrap();
+
+        assert!(list_array.is_null(0));
+        assert_eq!(list_array.len(), 1);
+        assert_eq!(list_array.values().len(), 0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn scalar_list_to_array() -> Result<()> {
+        let list_array_ref = ScalarValue::List(
+            Some(vec![
+                ScalarValue::UInt64(Some(100)),
+                ScalarValue::UInt64(None),
+                ScalarValue::UInt64(Some(101)),
+            ]),
+            DataType::UInt64,
+        )
+        .to_array();
+
+        let list_array = list_array_ref.as_any().downcast_ref::<ListArray>().unwrap();
+        assert_eq!(list_array.len(), 1);
+        assert_eq!(list_array.values().len(), 3);
+
+        let prim_array_ref = list_array.value(0);
+        let prim_array = prim_array_ref
+            .as_any()
+            .downcast_ref::<UInt64Array>()
+            .unwrap();
+        assert_eq!(prim_array.len(), 3);
+        assert_eq!(prim_array.value(0), 100);
+        assert!(prim_array.is_null(1));
+        assert_eq!(prim_array.value(2), 101);
+
+        Ok(())
+    }
+}
diff --git a/rust/datafusion/src/sql/parser.rs b/rust/datafusion/src/sql/parser.rs
index e8089a8181e..032444549ca 100644
--- a/rust/datafusion/src/sql/parser.rs
+++ b/rust/datafusion/src/sql/parser.rs
@@ -21,7 +21,7 @@
 
 use sqlparser::{
     ast::{ColumnDef, Statement as SQLStatement, TableConstraint},
-    dialect::{keywords::Keyword, GenericDialect},
+    dialect::{keywords::Keyword, Dialect, GenericDialect},
     parser::{Parser, ParserError},
     tokenizer::{Token, Tokenizer},
 };
@@ -59,6 +59,15 @@ pub struct CreateExternalTable {
     pub location: String,
 }
 
+/// DataFusion extension DDL for `EXPLAIN` and `EXPLAIN VERBOSE`
+#[derive(Debug, Clone, PartialEq)]
+pub struct ExplainPlan {
+    /// If true, dumps more intermediate plans and results of optimizaton passes
+    pub verbose: bool,
+    /// The statement for which to generate an planning explanation
+    pub statement: Box<Statement>,
+}
+
 /// DataFusion Statement representations.
 ///
 /// Tokens parsed by `DFParser` are converted into these values.
@@ -68,6 +77,8 @@ pub enum Statement {
     Statement(SQLStatement),
     /// Extension: `CREATE EXTERNAL TABLE`
     CreateExternalTable(CreateExternalTable),
+    /// Extension: `EXPLAIN <SQL>`
+    Explain(ExplainPlan),
 }
 
 /// SQL Parser
@@ -78,19 +89,34 @@ pub struct DFParser {
 impl DFParser {
     /// Parse the specified tokens
     pub fn new(sql: &str) -> Result<Self, ParserError> {
-        let dialect = GenericDialect {};
-        let mut tokenizer = Tokenizer::new(&dialect, sql);
+        let dialect = &GenericDialect {};
+        DFParser::new_with_dialect(sql, dialect)
+    }
+
+    /// Parse the specified tokens with dialect
+    pub fn new_with_dialect(
+        sql: &str,
+        dialect: &dyn Dialect,
+    ) -> Result<Self, ParserError> {
+        let mut tokenizer = Tokenizer::new(dialect, sql);
         let tokens = tokenizer.tokenize()?;
         Ok(DFParser {
             parser: Parser::new(tokens),
         })
     }
 
-    /// Parse a SQL statement and produce a set of statements
+    /// Parse a SQL statement and produce a set of statements with dialect
     pub fn parse_sql(sql: &str) -> Result<Vec<Statement>, ParserError> {
-        let mut tokenizer = Tokenizer::new(&GenericDialect {}, &sql);
-        tokenizer.tokenize()?;
-        let mut parser = DFParser::new(sql)?;
+        let dialect = &GenericDialect {};
+        DFParser::parse_sql_with_dialect(sql, dialect)
+    }
+
+    /// Parse a SQL statement and produce a set of statements
+    pub fn parse_sql_with_dialect(
+        sql: &str,
+        dialect: &dyn Dialect,
+    ) -> Result<Vec<Statement>, ParserError> {
+        let mut parser = DFParser::new_with_dialect(sql, dialect)?;
         let mut stmts = Vec::new();
         let mut expecting_statement_delimiter = false;
         loop {
@@ -121,18 +147,24 @@ impl DFParser {
     /// Parse a new expression
     pub fn parse_statement(&mut self) -> Result<Statement, ParserError> {
         match self.parser.peek_token() {
-            Token::Word(w) => match w.keyword {
-                Keyword::CREATE => {
-                    // move one token forward
-                    self.parser.next_token();
-                    // use custom parsing
-                    Ok(self.parse_create()?)
+            Token::Word(w) => {
+                match w.keyword {
+                    Keyword::CREATE => {
+                        // move one token forward
+                        self.parser.next_token();
+                        // use custom parsing
+                        self.parse_create()
+                    }
+                    Keyword::NoKeyword if w.value.to_uppercase() == "EXPLAIN" => {
+                        self.parser.next_token();
+                        self.parse_explain()
+                    }
+                    _ => {
+                        // use the native parser
+                        Ok(Statement::Statement(self.parser.parse_statement()?))
+                    }
                 }
-                _ => {
-                    // use the native parser
-                    Ok(Statement::Statement(self.parser.parse_statement()?))
-                }
-            },
+            }
             _ => {
                 // use the native parser
                 Ok(Statement::Statement(self.parser.parse_statement()?))
@@ -149,6 +181,26 @@ impl DFParser {
         }
     }
 
+    /// Parse an SQL EXPLAIN statement.
+    pub fn parse_explain(&mut self) -> Result<Statement, ParserError> {
+        // Parser is at the token immediately after EXPLAIN
+        // Check for EXPLAIN VERBOSE
+        let verbose = match self.parser.peek_token() {
+            Token::Word(w) => match w.keyword {
+                Keyword::NoKeyword if w.value.to_uppercase() == "VERBOSE" => {
+                    self.parser.next_token();
+                    true
+                }
+                _ => false,
+            },
+            _ => false,
+        };
+
+        let statement = Box::new(self.parse_statement()?);
+        let explain_plan = ExplainPlan { statement, verbose };
+        Ok(Statement::Explain(explain_plan))
+    }
+
     // This is a copy of the equivalent implementation in sqlparser.
     fn parse_columns(
         &mut self,
diff --git a/rust/datafusion/src/sql/planner.rs b/rust/datafusion/src/sql/planner.rs
index 748262cc31e..fc138dfbc8e 100644
--- a/rust/datafusion/src/sql/planner.rs
+++ b/rust/datafusion/src/sql/planner.rs
@@ -17,17 +17,27 @@
 
 //! SQL Query Planner (produces logical plan from SQL AST)
 
+use std::str::FromStr;
 use std::sync::Arc;
 
-use crate::error::{ExecutionError, Result};
-use crate::logicalplan::Expr::Alias;
-use crate::logicalplan::{
-    lit, Expr, FunctionMeta, LogicalPlan, LogicalPlanBuilder, Operator, ScalarValue,
+use crate::logical_plan::Expr::Alias;
+use crate::logical_plan::{
+    lit, Expr, LogicalPlan, LogicalPlanBuilder, Operator, PlanType, StringifiedPlan,
+};
+use crate::scalar::ScalarValue;
+use crate::{
+    error::{DataFusionError, Result},
+    physical_plan::udaf::AggregateUDF,
+};
+use crate::{
+    physical_plan::udf::ScalarUDF,
+    physical_plan::{aggregates, functions},
+    sql::parser::{CreateExternalTable, FileType, Statement as DFStatement},
 };
-use crate::sql::parser::{CreateExternalTable, FileType, Statement as DFStatement};
 
 use arrow::datatypes::*;
 
+use super::parser::ExplainPlan;
 use sqlparser::ast::{
     BinaryOperator, DataType as SQLDataType, Expr as SQLExpr, Query, Select, SelectItem,
     SetExpr, TableFactor, TableWithJoins, UnaryOperator, Value,
@@ -41,17 +51,19 @@ pub trait SchemaProvider {
     /// Getter for a field description
     fn get_table_meta(&self, name: &str) -> Option<SchemaRef>;
     /// Getter for a UDF description
-    fn get_function_meta(&self, name: &str) -> Option<Arc<FunctionMeta>>;
+    fn get_function_meta(&self, name: &str) -> Option<Arc<ScalarUDF>>;
+    /// Getter for a UDAF description
+    fn get_aggregate_meta(&self, name: &str) -> Option<Arc<AggregateUDF>>;
 }
 
 /// SQL query planner
-pub struct SqlToRel<S: SchemaProvider> {
-    schema_provider: S,
+pub struct SqlToRel<'a, S: SchemaProvider> {
+    schema_provider: &'a S,
 }
 
-impl<S: SchemaProvider> SqlToRel<S> {
+impl<'a, S: SchemaProvider> SqlToRel<'a, S> {
     /// Create a new query planner
-    pub fn new(schema_provider: S) -> Self {
+    pub fn new(schema_provider: &'a S) -> Self {
         SqlToRel { schema_provider }
     }
 
@@ -60,6 +72,7 @@ impl<S: SchemaProvider> SqlToRel<S> {
         match statement {
             DFStatement::CreateExternalTable(s) => self.external_table_to_plan(&s),
             DFStatement::Statement(s) => self.sql_statement_to_plan(&s),
+            DFStatement::Explain(s) => self.explain_statement_to_plan(&(*s)),
         }
     }
 
@@ -67,7 +80,7 @@ impl<S: SchemaProvider> SqlToRel<S> {
     pub fn sql_statement_to_plan(&self, sql: &Statement) -> Result<LogicalPlan> {
         match sql {
             Statement::Query(query) => self.query_to_plan(&query),
-            _ => Err(ExecutionError::NotImplemented(
+            _ => Err(DataFusionError::NotImplemented(
                 "Only SELECT statements are implemented".to_string(),
             )),
         }
@@ -77,7 +90,7 @@ impl<S: SchemaProvider> SqlToRel<S> {
     pub fn query_to_plan(&self, query: &Query) -> Result<LogicalPlan> {
         let plan = match &query.body {
             SetExpr::Select(s) => self.select_to_plan(s.as_ref()),
-            _ => Err(ExecutionError::NotImplemented(
+            _ => Err(DataFusionError::NotImplemented(
                 format!("Query {} not implemented yet", query.body).to_owned(),
             )),
         }?;
@@ -104,14 +117,14 @@ impl<S: SchemaProvider> SqlToRel<S> {
         match *file_type {
             FileType::CSV => {
                 if columns.is_empty() {
-                    return Err(ExecutionError::General(
+                    return Err(DataFusionError::Plan(
                         "Column definitions required for CSV files. None found".into(),
                     ));
                 }
             }
             FileType::Parquet => {
                 if !columns.is_empty() {
-                    return Err(ExecutionError::General(
+                    return Err(DataFusionError::Plan(
                         "Column definitions can not be specified for PARQUET files."
                             .into(),
                     ));
@@ -120,7 +133,7 @@ impl<S: SchemaProvider> SqlToRel<S> {
             FileType::NdJson => {}
         };
 
-        let schema = Box::new(self.build_schema(&columns)?);
+        let schema = SchemaRef::new(self.build_schema(&columns)?);
 
         Ok(LogicalPlan::CreateExternalTable {
             schema,
@@ -131,6 +144,31 @@ impl<S: SchemaProvider> SqlToRel<S> {
         })
     }
 
+    /// Generate a plan for EXPLAIN ... that will print out a plan
+    ///
+    pub fn explain_statement_to_plan(
+        &self,
+        explain_plan: &ExplainPlan,
+    ) -> Result<LogicalPlan> {
+        let verbose = explain_plan.verbose;
+        let plan = self.statement_to_plan(&explain_plan.statement)?;
+
+        let stringified_plans = vec![StringifiedPlan::new(
+            PlanType::LogicalPlan,
+            format!("{:#?}", plan),
+        )];
+
+        let schema = LogicalPlan::explain_schema();
+        let plan = Arc::new(plan);
+
+        Ok(LogicalPlan::Explain {
+            verbose,
+            plan,
+            stringified_plans,
+            schema,
+        })
+    }
+
     fn build_schema(&self, columns: &Vec<SQLColumnDef>) -> Result<Schema> {
         let mut fields = Vec::new();
 
@@ -146,6 +184,7 @@ impl<S: SchemaProvider> SqlToRel<S> {
         Ok(Schema::new(fields))
     }
 
+    /// Maps the SQL type to the corresponding Arrow `DataType`
     fn make_data_type(&self, sql_type: &SQLDataType) -> Result<DataType> {
         match sql_type {
             SQLDataType::BigInt => Ok(DataType::Int64),
@@ -161,8 +200,8 @@ impl<S: SchemaProvider> SqlToRel<S> {
             SQLDataType::Date => Ok(DataType::Date64(DateUnit::Day)),
             SQLDataType::Time => Ok(DataType::Time64(TimeUnit::Millisecond)),
             SQLDataType::Timestamp => Ok(DataType::Date64(DateUnit::Millisecond)),
-            _ => Err(ExecutionError::General(format!(
-                "Unsupported data type: {:?}.",
+            _ => Err(DataFusionError::NotImplemented(format!(
+                "The SQL data type {:?} is not implemented",
                 sql_type
             ))),
         }
@@ -173,7 +212,7 @@ impl<S: SchemaProvider> SqlToRel<S> {
             return Ok(LogicalPlanBuilder::empty().build()?);
         }
         if from.len() != 1 {
-            return Err(ExecutionError::NotImplemented(
+            return Err(DataFusionError::NotImplemented(
                 "FROM with multiple tables is still not implemented".to_string(),
             ));
         };
@@ -189,13 +228,13 @@ impl<S: SchemaProvider> SqlToRel<S> {
                         None,
                     )?
                     .build()?),
-                    None => Err(ExecutionError::General(format!(
+                    None => Err(DataFusionError::Plan(format!(
                         "no schema found for table {}",
                         name
                     ))),
                 }
             }
-            _ => Err(ExecutionError::NotImplemented(
+            _ => Err(DataFusionError::NotImplemented(
                 "Subqueries are still not supported".to_string(),
             )),
         }
@@ -204,14 +243,14 @@ impl<S: SchemaProvider> SqlToRel<S> {
     /// Generate a logic plan from an SQL select
     fn select_to_plan(&self, select: &Select) -> Result<LogicalPlan> {
         if select.having.is_some() {
-            return Err(ExecutionError::NotImplemented(
+            return Err(DataFusionError::NotImplemented(
                 "HAVING is not implemented yet".to_string(),
             ));
         }
 
         let plan = self.from_join_to_plan(&select.from)?;
 
-        // selection first
+        // filter (also known as selection) first
         let plan = self.filter(&plan, &select.selection)?;
 
         let projection_expr: Vec<Expr> = select
@@ -239,11 +278,11 @@ impl<S: SchemaProvider> SqlToRel<S> {
     fn filter(
         &self,
         plan: &LogicalPlan,
-        selection: &Option<SQLExpr>,
+        predicate: &Option<SQLExpr>,
     ) -> Result<LogicalPlan> {
-        match *selection {
-            Some(ref filter_expr) => LogicalPlanBuilder::from(&plan)
-                .filter(self.sql_to_rex(filter_expr, &plan.schema())?)?
+        match *predicate {
+            Some(ref predicate_expr) => LogicalPlanBuilder::from(&plan)
+                .filter(self.sql_to_rex(predicate_expr, &plan.schema())?)?
                 .build(),
             _ => Ok(plan.clone()),
         }
@@ -271,7 +310,7 @@ impl<S: SchemaProvider> SqlToRel<S> {
         let aggr_count = aggr_expr.len();
 
         if group_by_count + aggr_count != projection_expr.len() {
-            return Err(ExecutionError::General(
+            return Err(DataFusionError::Plan(
                 "Projection references non-aggregate values".to_owned(),
             ));
         }
@@ -309,8 +348,8 @@ impl<S: SchemaProvider> SqlToRel<S> {
         match *limit {
             Some(ref limit_expr) => {
                 let n = match self.sql_to_rex(&limit_expr, &input.schema())? {
-                    Expr::Literal(ScalarValue::Int64(n)) => Ok(n as usize),
-                    _ => Err(ExecutionError::General(
+                    Expr::Literal(ScalarValue::Int64(Some(n))) => Ok(n as usize),
+                    _ => Err(DataFusionError::Plan(
                         "Unexpected expression for LIMIT clause".to_string(),
                     )),
                 }?;
@@ -357,7 +396,7 @@ impl<S: SchemaProvider> SqlToRel<S> {
                 alias.value.clone(),
             )),
             SelectItem::Wildcard => Ok(Expr::Wildcard),
-            SelectItem::QualifiedWildcard(_) => Err(ExecutionError::NotImplemented(
+            SelectItem::QualifiedWildcard(_) => Err(DataFusionError::NotImplemented(
                 "Qualified wildcards are not supported".to_string(),
             )),
         }
@@ -372,14 +411,38 @@ impl<S: SchemaProvider> SqlToRel<S> {
             },
             SQLExpr::Value(Value::SingleQuotedString(ref s)) => Ok(lit(s.clone())),
 
-            SQLExpr::Identifier(ref id) => match schema.field_with_name(&id.value) {
-                Ok(field) => Ok(Expr::Column(field.name().clone())),
-                Err(_) => Err(ExecutionError::ExecutionError(format!(
-                    "Invalid identifier '{}' for schema {}",
-                    id,
-                    schema.to_string()
-                ))),
-            },
+            SQLExpr::Identifier(ref id) => {
+                if &id.value[0..1] == "@" {
+                    let var_names = vec![id.value.clone()];
+                    Ok(Expr::ScalarVariable(var_names))
+                } else {
+                    match schema.field_with_name(&id.value) {
+                        Ok(field) => Ok(Expr::Column(field.name().clone())),
+                        Err(_) => Err(DataFusionError::Plan(format!(
+                            "Invalid identifier '{}' for schema {}",
+                            id,
+                            schema.to_string()
+                        ))),
+                    }
+                }
+            }
+
+            SQLExpr::CompoundIdentifier(ids) => {
+                let mut var_names = vec![];
+                for i in 0..ids.len() {
+                    let id = ids[i].clone();
+                    var_names.push(id.value);
+                }
+                if &var_names[0][0..1] == "@" {
+                    Ok(Expr::ScalarVariable(var_names))
+                } else {
+                    Err(DataFusionError::Plan(format!(
+                        "Invalid compound identifier '{:?}' for schema {}",
+                        var_names,
+                        schema.to_string()
+                    )))
+                }
+            }
 
             SQLExpr::Wildcard => Ok(Expr::Wildcard),
 
@@ -403,7 +466,7 @@ impl<S: SchemaProvider> SqlToRel<S> {
                 UnaryOperator::Not => {
                     Ok(Expr::Not(Box::new(self.sql_to_rex(expr, schema)?)))
                 }
-                _ => Err(ExecutionError::InternalError(format!(
+                _ => Err(DataFusionError::Internal(format!(
                     "SQL binary operator cannot be interpreted as a unary operator"
                 ))),
             },
@@ -429,47 +492,37 @@ impl<S: SchemaProvider> SqlToRel<S> {
                     BinaryOperator::Or => Ok(Operator::Or),
                     BinaryOperator::Like => Ok(Operator::Like),
                     BinaryOperator::NotLike => Ok(Operator::NotLike),
-                    _ => Err(ExecutionError::NotImplemented(format!(
+                    _ => Err(DataFusionError::NotImplemented(format!(
                         "Unsupported SQL binary operator {:?}",
                         op
                     ))),
                 }?;
 
-                match operator {
-                    Operator::Not => Err(ExecutionError::InternalError(format!(
-                        "SQL unary operator \"NOT\" cannot be interpreted as a binary operator"
-                    ))),
-                    _ => Ok(Expr::BinaryExpr {
-                        left: Box::new(self.sql_to_rex(&left, &schema)?),
-                        op: operator,
-                        right: Box::new(self.sql_to_rex(&right, &schema)?),
-                    })
-                }
+                Ok(Expr::BinaryExpr {
+                    left: Box::new(self.sql_to_rex(&left, &schema)?),
+                    op: operator,
+                    right: Box::new(self.sql_to_rex(&right, &schema)?),
+                })
             }
 
             SQLExpr::Function(function) => {
-                //TODO: fix this hack
                 let name: String = function.name.to_string();
-                match name.to_lowercase().as_ref() {
-                    "min" | "max" | "sum" | "avg" => {
-                        let rex_args = function
-                            .args
-                            .iter()
-                            .map(|a| self.sql_to_rex(a, schema))
-                            .collect::<Result<Vec<Expr>>>()?;
 
-                        // return type is same as the argument type for these aggregate
-                        // functions
-                        let return_type = rex_args[0].get_type(schema)?.clone();
-
-                        Ok(Expr::AggregateFunction {
-                            name: name.clone(),
-                            args: rex_args,
-                            return_type,
-                        })
-                    }
-                    "count" => {
-                        let rex_args = function
+                // first, scalar built-in
+                if let Ok(fun) = functions::BuiltinScalarFunction::from_str(&name) {
+                    let args = function
+                        .args
+                        .iter()
+                        .map(|a| self.sql_to_rex(a, schema))
+                        .collect::<Result<Vec<Expr>>>()?;
+
+                    return Ok(Expr::ScalarFunction { fun, args });
+                };
+
+                // next, aggregate built-ins
+                if let Ok(fun) = aggregates::AggregateFunction::from_str(&name) {
+                    let args = if fun == aggregates::AggregateFunction::Count {
+                        function
                             .args
                             .iter()
                             .map(|a| match a {
@@ -477,37 +530,50 @@ impl<S: SchemaProvider> SqlToRel<S> {
                                 SQLExpr::Wildcard => Ok(lit(1_u8)),
                                 _ => self.sql_to_rex(a, schema),
                             })
+                            .collect::<Result<Vec<Expr>>>()?
+                    } else {
+                        function
+                            .args
+                            .iter()
+                            .map(|a| self.sql_to_rex(a, schema))
+                            .collect::<Result<Vec<Expr>>>()?
+                    };
+
+                    return Ok(Expr::AggregateFunction {
+                        fun,
+                        distinct: function.distinct,
+                        args,
+                    });
+                };
+
+                // finally, user-defined functions (UDF) and UDAF
+                match self.schema_provider.get_function_meta(&name) {
+                    Some(fm) => {
+                        let args = function
+                            .args
+                            .iter()
+                            .map(|a| self.sql_to_rex(a, schema))
                             .collect::<Result<Vec<Expr>>>()?;
 
-                        Ok(Expr::AggregateFunction {
-                            name: name.clone(),
-                            args: rex_args,
-                            return_type: DataType::UInt64,
+                        Ok(Expr::ScalarUDF {
+                            fun: fm.clone(),
+                            args,
                         })
                     }
-                    _ => match self.schema_provider.get_function_meta(&name) {
+                    None => match self.schema_provider.get_aggregate_meta(&name) {
                         Some(fm) => {
-                            let rex_args = function
+                            let args = function
                                 .args
                                 .iter()
                                 .map(|a| self.sql_to_rex(a, schema))
                                 .collect::<Result<Vec<Expr>>>()?;
 
-                            let mut safe_args: Vec<Expr> = vec![];
-                            for i in 0..rex_args.len() {
-                                safe_args.push(
-                                    rex_args[i]
-                                        .cast_to(fm.args()[i].data_type(), schema)?,
-                                );
-                            }
-
-                            Ok(Expr::ScalarFunction {
-                                name: name.clone(),
-                                args: safe_args,
-                                return_type: fm.return_type().clone(),
+                            Ok(Expr::AggregateUDF {
+                                fun: fm.clone(),
+                                args,
                             })
                         }
-                        _ => Err(ExecutionError::General(format!(
+                        _ => Err(DataFusionError::Plan(format!(
                             "Invalid function '{}'",
                             name
                         ))),
@@ -517,7 +583,7 @@ impl<S: SchemaProvider> SqlToRel<S> {
 
             SQLExpr::Nested(e) => self.sql_to_rex(&e, &schema),
 
-            _ => Err(ExecutionError::General(format!(
+            _ => Err(DataFusionError::NotImplemented(format!(
                 "Unsupported ast node {:?} in sqltorel",
                 sql
             ))),
@@ -528,7 +594,8 @@ impl<S: SchemaProvider> SqlToRel<S> {
 /// Determine if an expression is an aggregate expression or not
 fn is_aggregate_expr(e: &Expr) -> bool {
     match e {
-        Expr::AggregateFunction { .. } => true,
+        Expr::AggregateFunction { .. } | Expr::AggregateUDF { .. } => true,
+        Expr::Alias(expr, _) => is_aggregate_expr(expr),
         _ => false,
     }
 }
@@ -544,7 +611,7 @@ pub fn convert_data_type(sql: &SQLDataType) -> Result<DataType> {
         SQLDataType::Double => Ok(DataType::Float64),
         SQLDataType::Char(_) | SQLDataType::Varchar(_) => Ok(DataType::Utf8),
         SQLDataType::Timestamp => Ok(DataType::Timestamp(TimeUnit::Nanosecond, None)),
-        other => Err(ExecutionError::NotImplemented(format!(
+        other => Err(DataFusionError::NotImplemented(format!(
             "Unsupported SQL type {:?}",
             other
         ))),
@@ -554,8 +621,8 @@ pub fn convert_data_type(sql: &SQLDataType) -> Result<DataType> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::logicalplan::FunctionType;
-    use crate::sql::parser::DFParser;
+    use crate::{logical_plan::create_udf, sql::parser::DFParser};
+    use functions::ScalarFunctionImplementation;
 
     #[test]
     fn select_no_relation() {
@@ -570,47 +637,47 @@ mod tests {
     fn select_scalar_func_with_literal_no_relation() {
         quick_test(
             "SELECT sqrt(9)",
-            "Projection: sqrt(CAST(Int64(9) AS Float64))\
+            "Projection: sqrt(Int64(9))\
              \n  EmptyRelation",
         );
     }
 
     #[test]
-    fn select_simple_selection() {
+    fn select_simple_filter() {
         let sql = "SELECT id, first_name, last_name \
                    FROM person WHERE state = 'CO'";
         let expected = "Projection: #id, #first_name, #last_name\
-                        \n  Selection: #state Eq Utf8(\"CO\")\
+                        \n  Filter: #state Eq Utf8(\"CO\")\
                         \n    TableScan: person projection=None";
         quick_test(sql, expected);
     }
 
     #[test]
-    fn select_neg_selection() {
+    fn select_neg_filter() {
         let sql = "SELECT id, first_name, last_name \
                    FROM person WHERE NOT state";
         let expected = "Projection: #id, #first_name, #last_name\
-                        \n  Selection: NOT #state\
+                        \n  Filter: NOT #state\
                         \n    TableScan: person projection=None";
         quick_test(sql, expected);
     }
 
     #[test]
-    fn select_compound_selection() {
+    fn select_compound_filter() {
         let sql = "SELECT id, first_name, last_name \
                    FROM person WHERE state = 'CO' AND age >= 21 AND age <= 65";
         let expected = "Projection: #id, #first_name, #last_name\
-            \n  Selection: #state Eq Utf8(\"CO\") And #age GtEq Int64(21) And #age LtEq Int64(65)\
+            \n  Filter: #state Eq Utf8(\"CO\") And #age GtEq Int64(21) And #age LtEq Int64(65)\
             \n    TableScan: person projection=None";
         quick_test(sql, expected);
     }
 
     #[test]
-    fn test_timestamp_selection() {
+    fn test_timestamp_filter() {
         let sql = "SELECT state FROM person WHERE birth_date < CAST (158412331400600000 as timestamp)";
 
         let expected = "Projection: #state\
-            \n  Selection: #birth_date Lt CAST(Int64(158412331400600000) AS Timestamp(Nanosecond, None))\
+            \n  Filter: #birth_date Lt CAST(Int64(158412331400600000) AS Timestamp(Nanosecond, None))\
             \n    TableScan: person projection=None";
 
         quick_test(sql, expected);
@@ -627,7 +694,7 @@ mod tests {
                    AND age < 65 \
                    AND age <= 65";
         let expected = "Projection: #age, #first_name, #last_name\
-                        \n  Selection: #age Eq Int64(21) \
+                        \n  Filter: #age Eq Int64(21) \
                         And #age NotEq Int64(21) \
                         And #age Gt Int64(21) \
                         And #age GtEq Int64(21) \
@@ -708,7 +775,7 @@ mod tests {
     #[test]
     fn select_scalar_func() {
         let sql = "SELECT sqrt(age) FROM person";
-        let expected = "Projection: sqrt(CAST(#age AS Float64))\
+        let expected = "Projection: sqrt(#age)\
                         \n  TableScan: person projection=None";
         quick_test(sql, expected);
     }
@@ -716,7 +783,7 @@ mod tests {
     #[test]
     fn select_aliased_scalar_func() {
         let sql = "SELECT sqrt(age) AS square_people FROM person";
-        let expected = "Projection: sqrt(CAST(#age AS Float64)) AS square_people\
+        let expected = "Projection: sqrt(#age) AS square_people\
                         \n  TableScan: person projection=None";
         quick_test(sql, expected);
     }
@@ -781,7 +848,7 @@ mod tests {
         let sql = "SELECT c1, MIN(c12) FROM aggregate_test_100 GROUP BY c1, c13";
         let err = logical_plan(sql).expect_err("query should have failed");
         assert_eq!(
-            "General(\"Projection references non-aggregate values\")",
+            "Plan(\"Projection references non-aggregate values\")",
             format!("{:?}", err)
         );
     }
@@ -791,7 +858,7 @@ mod tests {
         let sql = "SELECT c1, c13, MIN(c12) FROM aggregate_test_100 GROUP BY c1";
         let err = logical_plan(sql).expect_err("query should have failed");
         assert_eq!(
-            "General(\"Projection references non-aggregate values\")",
+            "Plan(\"Projection references non-aggregate values\")",
             format!("{:?}", err)
         );
     }
@@ -808,7 +875,7 @@ mod tests {
         let sql = "CREATE EXTERNAL TABLE t STORED AS CSV LOCATION 'foo.csv'";
         let err = logical_plan(sql).expect_err("query should have failed");
         assert_eq!(
-            "General(\"Column definitions required for CSV files. None found\")",
+            "Plan(\"Column definitions required for CSV files. None found\")",
             format!("{:?}", err)
         );
     }
@@ -819,7 +886,7 @@ mod tests {
             "CREATE EXTERNAL TABLE t(c1 int) STORED AS PARQUET LOCATION 'foo.parquet'";
         let err = logical_plan(sql).expect_err("query should have failed");
         assert_eq!(
-            "General(\"Column definitions can not be specified for PARQUET files.\")",
+            "Plan(\"Column definitions can not be specified for PARQUET files.\")",
             format!("{:?}", err)
         );
     }
@@ -832,7 +899,7 @@ mod tests {
     }
 
     fn logical_plan(sql: &str) -> Result<LogicalPlan> {
-        let planner = SqlToRel::new(MockSchemaProvider {});
+        let planner = SqlToRel::new(&MockSchemaProvider {});
         let ast = DFParser::parse_sql(&sql).unwrap();
         planner.statement_to_plan(&ast[0])
     }
@@ -880,16 +947,22 @@ mod tests {
             }
         }
 
-        fn get_function_meta(&self, name: &str) -> Option<Arc<FunctionMeta>> {
+        fn get_function_meta(&self, name: &str) -> Option<Arc<ScalarUDF>> {
+            let f: ScalarFunctionImplementation =
+                Arc::new(|_| Err(DataFusionError::NotImplemented("".to_string())));
             match name {
-                "sqrt" => Some(Arc::new(FunctionMeta::new(
-                    "sqrt".to_string(),
-                    vec![Field::new("n", DataType::Float64, false)],
-                    DataType::Float64,
-                    FunctionType::Scalar,
+                "my_sqrt" => Some(Arc::new(create_udf(
+                    "my_sqrt",
+                    vec![DataType::Float64],
+                    Arc::new(DataType::Float64),
+                    f,
                 ))),
                 _ => None,
             }
         }
+
+        fn get_aggregate_meta(&self, _name: &str) -> Option<Arc<AggregateUDF>> {
+            unimplemented!()
+        }
     }
 }
diff --git a/rust/datafusion/src/table.rs b/rust/datafusion/src/table.rs
deleted file mode 100644
index bd04f0738e0..00000000000
--- a/rust/datafusion/src/table.rs
+++ /dev/null
@@ -1,79 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Table API for building a logical query plan. This is similar to the Table API in Ibis
-//! and the DataFrame API in Apache Spark
-
-use crate::arrow::record_batch::RecordBatch;
-use crate::error::Result;
-use crate::execution::context::ExecutionContext;
-use crate::logicalplan::{Expr, LogicalPlan};
-use arrow::datatypes::Schema;
-use std::sync::Arc;
-
-/// Table is an abstraction of a logical query plan
-pub trait Table {
-    /// Select columns by name
-    fn select_columns(&self, columns: Vec<&str>) -> Result<Arc<dyn Table>>;
-
-    /// Create a projection based on arbitrary expressions
-    fn select(&self, expr: Vec<Expr>) -> Result<Arc<dyn Table>>;
-
-    /// Create a selection based on a filter expression
-    fn filter(&self, expr: Expr) -> Result<Arc<dyn Table>>;
-
-    /// Perform an aggregate query
-    fn aggregate(
-        &self,
-        group_expr: Vec<Expr>,
-        aggr_expr: Vec<Expr>,
-    ) -> Result<Arc<dyn Table>>;
-
-    /// limit the number of rows
-    fn limit(&self, n: usize) -> Result<Arc<dyn Table>>;
-
-    /// Return the logical plan
-    fn to_logical_plan(&self) -> LogicalPlan;
-
-    /// Return an expression representing a column within this table
-    fn col(&self, name: &str) -> Result<Expr>;
-
-    /// Create an expression to represent the min() aggregate function
-    fn min(&self, expr: &Expr) -> Result<Expr>;
-
-    /// Create an expression to represent the max() aggregate function
-    fn max(&self, expr: &Expr) -> Result<Expr>;
-
-    /// Create an expression to represent the sum() aggregate function
-    fn sum(&self, expr: &Expr) -> Result<Expr>;
-
-    /// Create an expression to represent the avg() aggregate function
-    fn avg(&self, expr: &Expr) -> Result<Expr>;
-
-    /// Create an expression to represent the count() aggregate function
-    fn count(&self, expr: &Expr) -> Result<Expr>;
-
-    /// Collects the result as a vector of RecordBatch.
-    fn collect(
-        &self,
-        ctx: &mut ExecutionContext,
-        batch_size: usize,
-    ) -> Result<Vec<RecordBatch>>;
-
-    /// Returns the schema
-    fn schema(&self) -> &Schema;
-}
diff --git a/rust/datafusion/src/test/mod.rs b/rust/datafusion/src/test/mod.rs
index 317c14564f1..67ae056c88e 100644
--- a/rust/datafusion/src/test/mod.rs
+++ b/rust/datafusion/src/test/mod.rs
@@ -17,10 +17,11 @@
 
 //! Common unit test utility methods
 
+use crate::datasource::{MemTable, TableProvider};
 use crate::error::Result;
 use crate::execution::context::ExecutionContext;
-use crate::execution::physical_plan::ExecutionPlan;
-use crate::logicalplan::{Expr, LogicalPlan, LogicalPlanBuilder};
+use crate::logical_plan::{LogicalPlan, LogicalPlanBuilder};
+use crate::physical_plan::ExecutionPlan;
 use arrow::array;
 use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
 use arrow::record_batch::RecordBatch;
@@ -29,7 +30,24 @@ use std::fs::File;
 use std::io::prelude::*;
 use std::io::{BufReader, BufWriter};
 use std::sync::Arc;
-use tempdir::TempDir;
+use tempfile::TempDir;
+
+pub fn create_table_dual() -> Box<dyn TableProvider + Send + Sync> {
+    let dual_schema = Arc::new(Schema::new(vec![
+        Field::new("id", DataType::Int32, false),
+        Field::new("name", DataType::Utf8, false),
+    ]));
+    let batch = RecordBatch::try_new(
+        dual_schema.clone(),
+        vec![
+            Arc::new(array::Int32Array::from(vec![1])),
+            Arc::new(array::StringArray::from(vec!["a"])),
+        ],
+    )
+    .unwrap();
+    let provider = MemTable::new(dual_schema.clone(), vec![vec![batch.clone()]]).unwrap();
+    Box::new(provider)
+}
 
 /// Get the value of the ARROW_TEST_DATA environment variable
 pub fn arrow_testdata_path() -> String {
@@ -37,9 +55,9 @@ pub fn arrow_testdata_path() -> String {
 }
 
 /// Execute a physical plan and collect the results
-pub fn execute(plan: &dyn ExecutionPlan) -> Result<Vec<RecordBatch>> {
+pub async fn execute(plan: Arc<dyn ExecutionPlan>) -> Result<Vec<RecordBatch>> {
     let ctx = ExecutionContext::new();
-    ctx.collect(plan)
+    ctx.collect(plan).await
 }
 
 /// Generated partitioned copy of a CSV file
@@ -47,7 +65,7 @@ pub fn create_partitioned_csv(filename: &str, partitions: usize) -> Result<Strin
     let testdata = arrow_testdata_path();
     let path = format!("{}/csv/{}", testdata, filename);
 
-    let tmp_dir = TempDir::new("create_partitioned_csv")?;
+    let tmp_dir = TempDir::new()?;
 
     let mut writers = vec![];
     for i in 0..partitions {
@@ -67,14 +85,14 @@ pub fn create_partitioned_csv(filename: &str, partitions: usize) -> Result<Strin
         if i == 0 {
             // write header to all partitions
             for w in writers.iter_mut() {
-                w.write(line.as_bytes()).unwrap();
-                w.write(b"\n").unwrap();
+                w.write_all(line.as_bytes()).unwrap();
+                w.write_all(b"\n").unwrap();
             }
         } else {
             // write data line to single partition
             let partition = i % partitions;
-            writers[partition].write(line.as_bytes()).unwrap();
-            writers[partition].write(b"\n").unwrap();
+            writers[partition].write_all(line.as_bytes()).unwrap();
+            writers[partition].write_all(b"\n").unwrap();
         }
 
         i += 1;
@@ -116,6 +134,13 @@ pub fn format_batch(batch: &RecordBatch) -> Vec<String> {
             }
             let array = batch.column(column_index);
             match array.data_type() {
+                DataType::Utf8 => s.push_str(
+                    array
+                        .as_any()
+                        .downcast_ref::<array::StringArray>()
+                        .unwrap()
+                        .value(row_index),
+                ),
                 DataType::Int8 => s.push_str(&format!(
                     "{:?}",
                     array
@@ -224,10 +249,51 @@ pub fn assert_fields_eq(plan: &LogicalPlan, expected: Vec<&str>) {
     assert_eq!(actual, expected);
 }
 
-pub fn max(expr: Expr) -> Expr {
-    Expr::AggregateFunction {
-        name: "MAX".to_owned(),
-        args: vec![expr],
-        return_type: DataType::Float64,
+pub mod variable;
+
+mod tests {
+    use super::*;
+
+    use arrow::array::{BooleanArray, Int32Array, StringArray};
+    use arrow::datatypes::{DataType, Field, Schema};
+    use arrow::record_batch::RecordBatch;
+
+    #[test]
+    fn test_format_batch() -> Result<()> {
+        let array_int32 = Int32Array::from(vec![1000, 2000]);
+        let array_string = StringArray::from(vec!["bow \u{1F3F9}", "arrow \u{2191}"]);
+
+        let schema = Schema::new(vec![
+            Field::new("a", DataType::Int32, false),
+            Field::new("b", DataType::Utf8, false),
+        ]);
+
+        let record_batch = RecordBatch::try_new(
+            Arc::new(schema),
+            vec![Arc::new(array_int32), Arc::new(array_string)],
+        )?;
+
+        let result = format_batch(&record_batch);
+
+        assert_eq!(result, vec!["1000,bow \u{1F3F9}", "2000,arrow \u{2191}"]);
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_format_batch_unknown() -> Result<()> {
+        // Use any Array type not yet handled by format_batch().
+        let array_bool = BooleanArray::from(vec![false, true]);
+
+        let schema = Schema::new(vec![Field::new("a", DataType::Boolean, false)]);
+
+        let record_batch =
+            RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array_bool)])?;
+
+        let result = format_batch(&record_batch);
+
+        assert_eq!(result, vec!["?", "?"]);
+
+        Ok(())
     }
 }
diff --git a/rust/datafusion/src/test/variable.rs b/rust/datafusion/src/test/variable.rs
new file mode 100644
index 00000000000..47d1370e801
--- /dev/null
+++ b/rust/datafusion/src/test/variable.rs
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! System variable provider
+
+use crate::error::Result;
+use crate::scalar::ScalarValue;
+use crate::variable::VarProvider;
+
+/// System variable
+pub struct SystemVar {}
+
+impl SystemVar {
+    /// new system variable
+    pub fn new() -> Self {
+        Self {}
+    }
+}
+
+impl VarProvider for SystemVar {
+    /// get system variable value
+    fn get_value(&self, var_names: Vec<String>) -> Result<ScalarValue> {
+        let s = format!("{}-{}", "system-var".to_string(), var_names.concat());
+        Ok(ScalarValue::Utf8(Some(s)))
+    }
+}
+
+/// user defined variable
+pub struct UserDefinedVar {}
+
+impl UserDefinedVar {
+    /// new user defined variable
+    pub fn new() -> Self {
+        Self {}
+    }
+}
+
+impl VarProvider for UserDefinedVar {
+    /// Get user defined variable value
+    fn get_value(&self, var_names: Vec<String>) -> Result<ScalarValue> {
+        let s = format!("{}-{}", "user-defined-var".to_string(), var_names.concat());
+        Ok(ScalarValue::Utf8(Some(s)))
+    }
+}
diff --git a/rust/datafusion/src/variable/mod.rs b/rust/datafusion/src/variable/mod.rs
new file mode 100644
index 00000000000..db9ff7f0a60
--- /dev/null
+++ b/rust/datafusion/src/variable/mod.rs
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Variable provider
+
+use crate::error::Result;
+use crate::scalar::ScalarValue;
+
+/// Variable type, system/user defined
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum VarType {
+    /// System variable, like @@version
+    System,
+    /// User defined variable, like @name
+    UserDefined,
+}
+
+/// A var provider for @variable
+pub trait VarProvider {
+    /// Get variable value
+    fn get_value(&self, var_names: Vec<String>) -> Result<ScalarValue>;
+}
diff --git a/rust/datafusion/tests/customer.csv b/rust/datafusion/tests/customer.csv
new file mode 100644
index 00000000000..2abcd659046
--- /dev/null
+++ b/rust/datafusion/tests/customer.csv
@@ -0,0 +1,4 @@
+andrew,100
+jorge,200
+andy,150
+paul,300
diff --git a/rust/datafusion/tests/dataframe.rs b/rust/datafusion/tests/dataframe.rs
new file mode 100644
index 00000000000..3d303ecccd8
--- /dev/null
+++ b/rust/datafusion/tests/dataframe.rs
@@ -0,0 +1,188 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::array::Int32Array;
+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use arrow::error::Result as ArrowResult;
+use arrow::record_batch::RecordBatch;
+
+use datafusion::datasource::TableProvider;
+use datafusion::error::{DataFusionError, Result};
+
+use datafusion::execution::context::ExecutionContext;
+use datafusion::logical_plan::{col, LogicalPlan, LogicalPlanBuilder};
+use datafusion::physical_plan::{
+    ExecutionPlan, Partitioning, RecordBatchStream, SendableRecordBatchStream,
+};
+
+use futures::stream::Stream;
+use std::any::Any;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+
+use async_trait::async_trait;
+
+//// Custom source dataframe tests ////
+
+struct CustomTableProvider;
+#[derive(Debug, Clone)]
+struct CustomExecutionPlan {
+    projection: Option<Vec<usize>>,
+}
+struct TestCustomRecordBatchStream {
+    /// the nb of batches of TEST_CUSTOM_RECORD_BATCH generated
+    nb_batch: i32,
+}
+macro_rules! TEST_CUSTOM_SCHEMA_REF {
+    () => {
+        Arc::new(Schema::new(vec![
+            Field::new("c1", DataType::Int32, false),
+            Field::new("c2", DataType::Int32, false),
+        ]))
+    };
+}
+macro_rules! TEST_CUSTOM_RECORD_BATCH {
+    () => {
+        RecordBatch::try_new(
+            TEST_CUSTOM_SCHEMA_REF!(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 10, 10, 100])),
+                Arc::new(Int32Array::from(vec![2, 12, 12, 120])),
+            ],
+        )
+    };
+}
+
+impl RecordBatchStream for TestCustomRecordBatchStream {
+    fn schema(&self) -> SchemaRef {
+        TEST_CUSTOM_SCHEMA_REF!()
+    }
+}
+
+impl Stream for TestCustomRecordBatchStream {
+    type Item = ArrowResult<RecordBatch>;
+
+    fn poll_next(
+        self: Pin<&mut Self>,
+        _cx: &mut Context<'_>,
+    ) -> Poll<Option<Self::Item>> {
+        if self.nb_batch > 0 {
+            self.get_mut().nb_batch -= 1;
+            Poll::Ready(Some(TEST_CUSTOM_RECORD_BATCH!()))
+        } else {
+            Poll::Ready(None)
+        }
+    }
+}
+
+#[async_trait]
+impl ExecutionPlan for CustomExecutionPlan {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+    fn schema(&self) -> SchemaRef {
+        let schema = TEST_CUSTOM_SCHEMA_REF!();
+        match &self.projection {
+            None => schema,
+            Some(p) => Arc::new(Schema::new(
+                p.iter().map(|i| schema.field(*i).clone()).collect(),
+            )),
+        }
+    }
+    fn output_partitioning(&self) -> Partitioning {
+        Partitioning::UnknownPartitioning(1)
+    }
+    fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
+        vec![]
+    }
+    fn with_new_children(
+        &self,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        if children.is_empty() {
+            Ok(Arc::new(self.clone()))
+        } else {
+            Err(DataFusionError::Internal(
+                "Children cannot be replaced in CustomExecutionPlan".to_owned(),
+            ))
+        }
+    }
+    async fn execute(&self, _partition: usize) -> Result<SendableRecordBatchStream> {
+        Ok(Box::pin(TestCustomRecordBatchStream { nb_batch: 1 }))
+    }
+}
+
+impl TableProvider for CustomTableProvider {
+    fn schema(&self) -> SchemaRef {
+        TEST_CUSTOM_SCHEMA_REF!()
+    }
+
+    /// Create an ExecutionPlan that will scan the table.
+    fn scan(
+        &self,
+        projection: &Option<Vec<usize>>,
+        _batch_size: usize,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        Ok(Arc::new(CustomExecutionPlan {
+            projection: projection.clone(),
+        }))
+    }
+}
+
+#[tokio::test]
+async fn custom_source_dataframe() -> Result<()> {
+    let mut ctx = ExecutionContext::new();
+
+    let table = ctx.read_table(Arc::new(CustomTableProvider))?;
+    let logical_plan = LogicalPlanBuilder::from(&table.to_logical_plan())
+        .project(vec![col("c2")])?
+        .build()?;
+
+    let optimized_plan = ctx.optimize(&logical_plan)?;
+    match &optimized_plan {
+        LogicalPlan::Projection { input, .. } => match &**input {
+            LogicalPlan::TableScan {
+                table_schema,
+                projected_schema,
+                ..
+            } => {
+                assert_eq!(table_schema.fields().len(), 2);
+                assert_eq!(projected_schema.fields().len(), 1);
+            }
+            _ => assert!(false, "input to projection should be TableScan"),
+        },
+        _ => assert!(false, "expect optimized_plan to be projection"),
+    }
+
+    let expected = "Projection: #c2\
+        \n  TableScan: projection=Some([1])";
+    assert_eq!(format!("{:?}", optimized_plan), expected);
+
+    let physical_plan = ctx.create_physical_plan(&optimized_plan)?;
+
+    assert_eq!(1, physical_plan.schema().fields().len());
+    assert_eq!("c2", physical_plan.schema().field(0).name().as_str());
+
+    let batches = ctx.collect(physical_plan).await?;
+    let origin_rec_batch = TEST_CUSTOM_RECORD_BATCH!()?;
+    assert_eq!(1, batches.len());
+    assert_eq!(1, batches[0].num_columns());
+    assert_eq!(origin_rec_batch.num_rows(), batches[0].num_rows());
+
+    Ok(())
+}
diff --git a/rust/datafusion/tests/example.csv b/rust/datafusion/tests/example.csv
new file mode 100644
index 00000000000..0eadb69396b
--- /dev/null
+++ b/rust/datafusion/tests/example.csv
@@ -0,0 +1,2 @@
+a,b,c
+1,2,3
\ No newline at end of file
diff --git a/rust/datafusion/tests/sql.rs b/rust/datafusion/tests/sql.rs
index 57e3ee6a63e..8dd7afa1d7e 100644
--- a/rust/datafusion/tests/sql.rs
+++ b/rust/datafusion/tests/sql.rs
@@ -15,26 +15,28 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use std::convert::TryFrom;
 use std::env;
 use std::sync::Arc;
 
 extern crate arrow;
 extern crate datafusion;
 
-use arrow::array::*;
-use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
-use arrow::record_batch::RecordBatch;
+use arrow::{array::*, datatypes::TimeUnit};
+use arrow::{datatypes::Int32Type, datatypes::Int64Type, record_batch::RecordBatch};
+use arrow::{
+    datatypes::{DataType, Field, Schema, SchemaRef},
+    util::display::array_value_to_string,
+};
 
-use datafusion::datasource::csv::CsvReadOptions;
+use datafusion::datasource::{csv::CsvReadOptions, MemTable};
 use datafusion::error::Result;
 use datafusion::execution::context::ExecutionContext;
-use datafusion::execution::physical_plan::udf::ScalarFunction;
-use datafusion::logicalplan::LogicalPlan;
+use datafusion::logical_plan::LogicalPlan;
+use datafusion::prelude::create_udf;
 
-const DEFAULT_BATCH_SIZE: usize = 1024 * 1024;
-
-#[test]
-fn nyc() -> Result<()> {
+#[tokio::test]
+async fn nyc() -> Result<()> {
     // schema for nyxtaxi csv files
     let schema = Schema::new(vec![
         Field::new("VendorID", DataType::Utf8, true),
@@ -88,22 +90,30 @@ fn nyc() -> Result<()> {
     Ok(())
 }
 
-#[test]
-fn parquet_query() {
+#[tokio::test]
+async fn parquet_query() {
     let mut ctx = ExecutionContext::new();
     register_alltypes_parquet(&mut ctx);
     // NOTE that string_col is actually a binary column and does not have the UTF8 logical type
     // so we need an explicit cast
     let sql = "SELECT id, CAST(string_col AS varchar) FROM alltypes_plain";
-    let actual = execute(&mut ctx, sql).join("\n");
-    let expected =
-        "4\t\"0\"\n5\t\"1\"\n6\t\"0\"\n7\t\"1\"\n2\t\"0\"\n3\t\"1\"\n0\t\"0\"\n1\t\"1\""
-            .to_string();
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![
+        vec!["4", "0"],
+        vec!["5", "1"],
+        vec!["6", "0"],
+        vec!["7", "1"],
+        vec!["2", "0"],
+        vec!["3", "1"],
+        vec!["0", "0"],
+        vec!["1", "1"],
+    ];
+
     assert_eq!(expected, actual);
 }
 
-#[test]
-fn parquet_single_nan_schema() {
+#[tokio::test]
+async fn parquet_single_nan_schema() {
     let mut ctx = ExecutionContext::new();
     let testdata = env::var("PARQUET_TEST_DATA").expect("PARQUET_TEST_DATA not defined");
     ctx.register_parquet("single_nan", &format!("{}/single_nan.parquet", testdata))
@@ -111,95 +121,291 @@ fn parquet_single_nan_schema() {
     let sql = "SELECT mycol FROM single_nan";
     let plan = ctx.create_logical_plan(&sql).unwrap();
     let plan = ctx.optimize(&plan).unwrap();
-    let plan = ctx.create_physical_plan(&plan, DEFAULT_BATCH_SIZE).unwrap();
-    let results = ctx.collect(plan.as_ref()).unwrap();
+    let plan = ctx.create_physical_plan(&plan).unwrap();
+    let results = ctx.collect(plan).await.unwrap();
     for batch in results {
         assert_eq!(1, batch.num_rows());
         assert_eq!(1, batch.num_columns());
     }
 }
 
-#[test]
-fn csv_count_star() -> Result<()> {
+#[tokio::test]
+async fn parquet_list_columns() {
+    let mut ctx = ExecutionContext::new();
+    let testdata = env::var("PARQUET_TEST_DATA").expect("PARQUET_TEST_DATA not defined");
+    ctx.register_parquet(
+        "list_columns",
+        &format!("{}/list_columns.parquet", testdata),
+    )
+    .unwrap();
+
+    let schema = Arc::new(Schema::new(vec![
+        Field::new(
+            "int64_list",
+            DataType::List(Box::new(Field::new("item", DataType::Int64, true))),
+            true,
+        ),
+        Field::new(
+            "utf8_list",
+            DataType::List(Box::new(Field::new("item", DataType::Utf8, true))),
+            true,
+        ),
+    ]));
+
+    let sql = "SELECT int64_list, utf8_list FROM list_columns";
+    let plan = ctx.create_logical_plan(&sql).unwrap();
+    let plan = ctx.optimize(&plan).unwrap();
+    let plan = ctx.create_physical_plan(&plan).unwrap();
+    let results = ctx.collect(plan).await.unwrap();
+
+    //   int64_list              utf8_list
+    // 0  [1, 2, 3]        [abc, efg, hij]
+    // 1  [None, 1]                   None
+    // 2        [4]  [efg, None, hij, xyz]
+
+    assert_eq!(1, results.len());
+    let batch = &results[0];
+    assert_eq!(3, batch.num_rows());
+    assert_eq!(2, batch.num_columns());
+    assert_eq!(schema, batch.schema());
+
+    let int_list_array = batch
+        .column(0)
+        .as_any()
+        .downcast_ref::<ListArray>()
+        .unwrap();
+    let utf8_list_array = batch
+        .column(1)
+        .as_any()
+        .downcast_ref::<ListArray>()
+        .unwrap();
+
+    assert_eq!(
+        int_list_array
+            .value(0)
+            .as_any()
+            .downcast_ref::<PrimitiveArray<Int64Type>>()
+            .unwrap(),
+        &PrimitiveArray::<Int64Type>::from(vec![Some(1), Some(2), Some(3),])
+    );
+
+    assert_eq!(
+        utf8_list_array
+            .value(0)
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap(),
+        &StringArray::try_from(vec![Some("abc"), Some("efg"), Some("hij"),]).unwrap()
+    );
+
+    assert_eq!(
+        int_list_array
+            .value(1)
+            .as_any()
+            .downcast_ref::<PrimitiveArray<Int64Type>>()
+            .unwrap(),
+        &PrimitiveArray::<Int64Type>::from(vec![None, Some(1),])
+    );
+
+    assert!(utf8_list_array.is_null(1));
+
+    assert_eq!(
+        int_list_array
+            .value(2)
+            .as_any()
+            .downcast_ref::<PrimitiveArray<Int64Type>>()
+            .unwrap(),
+        &PrimitiveArray::<Int64Type>::from(vec![Some(4),])
+    );
+
+    let result = utf8_list_array.value(2);
+    let result = result.as_any().downcast_ref::<StringArray>().unwrap();
+
+    assert_eq!(result.value(0), "efg");
+    assert!(result.is_null(1));
+    assert_eq!(result.value(2), "hij");
+    assert_eq!(result.value(3), "xyz");
+}
+
+#[tokio::test]
+async fn csv_count_star() -> Result<()> {
     let mut ctx = ExecutionContext::new();
     register_aggregate_csv(&mut ctx)?;
-    let sql = "SELECT COUNT(*), COUNT(1), COUNT(c1) FROM aggregate_test_100";
-    let actual = execute(&mut ctx, sql).join("\n");
-    let expected = "100\t100\t100".to_string();
+    let sql = "SELECT COUNT(*), COUNT(1) AS c, COUNT(c1) FROM aggregate_test_100";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["100", "100", "100"]];
     assert_eq!(expected, actual);
     Ok(())
 }
 
-#[test]
-fn csv_query_with_predicate() -> Result<()> {
+#[tokio::test]
+async fn csv_query_with_predicate() -> Result<()> {
     let mut ctx = ExecutionContext::new();
     register_aggregate_csv(&mut ctx)?;
     let sql = "SELECT c1, c12 FROM aggregate_test_100 WHERE c12 > 0.376 AND c12 < 0.4";
-    let actual = execute(&mut ctx, sql).join("\n");
-    let expected = "\"e\"\t0.39144436569161134\n\"d\"\t0.38870280983958583".to_string();
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![
+        vec!["e", "0.39144436569161134"],
+        vec!["d", "0.38870280983958583"],
+    ];
+    assert_eq!(expected, actual);
+    Ok(())
+}
+
+#[tokio::test]
+async fn csv_query_with_negated_predicate() -> Result<()> {
+    let mut ctx = ExecutionContext::new();
+    register_aggregate_csv(&mut ctx)?;
+    let sql = "SELECT COUNT(1) FROM aggregate_test_100 WHERE NOT(c1 != 'a')";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["21"]];
+    assert_eq!(expected, actual);
+    Ok(())
+}
+
+#[tokio::test]
+async fn csv_query_with_is_not_null_predicate() -> Result<()> {
+    let mut ctx = ExecutionContext::new();
+    register_aggregate_csv(&mut ctx)?;
+    let sql = "SELECT COUNT(1) FROM aggregate_test_100 WHERE c1 IS NOT NULL";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["100"]];
+    assert_eq!(expected, actual);
+    Ok(())
+}
+
+#[tokio::test]
+async fn csv_query_with_is_null_predicate() -> Result<()> {
+    let mut ctx = ExecutionContext::new();
+    register_aggregate_csv(&mut ctx)?;
+    let sql = "SELECT COUNT(1) FROM aggregate_test_100 WHERE c1 IS NULL";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["0"]];
     assert_eq!(expected, actual);
     Ok(())
 }
 
-#[test]
-fn csv_query_group_by_int_min_max() -> Result<()> {
+#[tokio::test]
+async fn csv_query_group_by_int_min_max() -> Result<()> {
     let mut ctx = ExecutionContext::new();
     register_aggregate_csv(&mut ctx)?;
     let sql = "SELECT c2, MIN(c12), MAX(c12) FROM aggregate_test_100 GROUP BY c2";
-    let mut actual = execute(&mut ctx, sql);
+    let mut actual = execute(&mut ctx, sql).await;
     actual.sort();
-    let expected = "1\t0.05636955101974106\t0.9965400387585364\n2\t0.16301110515739792\t0.991517828651004\n3\t0.047343434291126085\t0.9293883502480845\n4\t0.02182578039211991\t0.9237877978193884\n5\t0.01479305307777301\t0.9723580396501548".to_string();
-    assert_eq!(expected, actual.join("\n"));
+    let expected = vec![
+        vec!["1", "0.05636955101974106", "0.9965400387585364"],
+        vec!["2", "0.16301110515739792", "0.991517828651004"],
+        vec!["3", "0.047343434291126085", "0.9293883502480845"],
+        vec!["4", "0.02182578039211991", "0.9237877978193884"],
+        vec!["5", "0.01479305307777301", "0.9723580396501548"],
+    ];
+    assert_eq!(expected, actual);
     Ok(())
 }
 
-#[test]
-fn csv_query_group_by_two_columns() -> Result<()> {
+#[tokio::test]
+async fn csv_query_group_by_two_columns() -> Result<()> {
     let mut ctx = ExecutionContext::new();
     register_aggregate_csv(&mut ctx)?;
     let sql = "SELECT c1, c2, MIN(c3) FROM aggregate_test_100 GROUP BY c1, c2";
-    let mut actual = execute(&mut ctx, sql);
+    let mut actual = execute(&mut ctx, sql).await;
     actual.sort();
-    let expected = [
-        "\"a\"\t1\t-85",
-        "\"a\"\t2\t-48",
-        "\"a\"\t3\t-72",
-        "\"a\"\t4\t-101",
-        "\"a\"\t5\t-101",
-        "\"b\"\t1\t12",
-        "\"b\"\t2\t-60",
-        "\"b\"\t3\t-101",
-        "\"b\"\t4\t-117",
-        "\"b\"\t5\t-82",
-        "\"c\"\t1\t-24",
-        "\"c\"\t2\t-117",
-        "\"c\"\t3\t-2",
-        "\"c\"\t4\t-90",
-        "\"c\"\t5\t-94",
-        "\"d\"\t1\t-99",
-        "\"d\"\t2\t93",
-        "\"d\"\t3\t-76",
-        "\"d\"\t4\t5",
-        "\"d\"\t5\t-59",
-        "\"e\"\t1\t36",
-        "\"e\"\t2\t-61",
-        "\"e\"\t3\t-95",
-        "\"e\"\t4\t-56",
-        "\"e\"\t5\t-86",
+    let expected = vec![
+        vec!["a", "1", "-85"],
+        vec!["a", "2", "-48"],
+        vec!["a", "3", "-72"],
+        vec!["a", "4", "-101"],
+        vec!["a", "5", "-101"],
+        vec!["b", "1", "12"],
+        vec!["b", "2", "-60"],
+        vec!["b", "3", "-101"],
+        vec!["b", "4", "-117"],
+        vec!["b", "5", "-82"],
+        vec!["c", "1", "-24"],
+        vec!["c", "2", "-117"],
+        vec!["c", "3", "-2"],
+        vec!["c", "4", "-90"],
+        vec!["c", "5", "-94"],
+        vec!["d", "1", "-99"],
+        vec!["d", "2", "93"],
+        vec!["d", "3", "-76"],
+        vec!["d", "4", "5"],
+        vec!["d", "5", "-59"],
+        vec!["e", "1", "36"],
+        vec!["e", "2", "-61"],
+        vec!["e", "3", "-95"],
+        vec!["e", "4", "-56"],
+        vec!["e", "5", "-86"],
     ];
-    assert_eq!(expected.join("\n"), actual.join("\n"));
+    assert_eq!(expected, actual);
     Ok(())
 }
 
-#[test]
-fn csv_query_avg_sqrt() -> Result<()> {
+#[tokio::test]
+async fn csv_query_avg_sqrt() -> Result<()> {
     let mut ctx = create_ctx()?;
     register_aggregate_csv(&mut ctx)?;
     let sql = "SELECT avg(custom_sqrt(c12)) FROM aggregate_test_100";
-    let mut actual = execute(&mut ctx, sql);
+    let mut actual = execute(&mut ctx, sql).await;
     actual.sort();
-    let expected = "0.6706002946036462".to_string();
-    assert_eq!(actual.join("\n"), expected);
+    let expected = vec![vec!["0.6706002946036462"]];
+    assert_eq!(actual, expected);
+    Ok(())
+}
+
+/// test that casting happens on udfs.
+/// c11 is f32, but `custom_sqrt` requires f64. Casting happens but the logical plan and
+/// physical plan have the same schema.
+#[tokio::test]
+async fn csv_query_custom_udf_with_cast() -> Result<()> {
+    let mut ctx = create_ctx()?;
+    register_aggregate_csv(&mut ctx)?;
+    let sql = "SELECT avg(custom_sqrt(c11)) FROM aggregate_test_100";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["0.6584408483418833"]];
+    assert_eq!(actual, expected);
+    Ok(())
+}
+
+/// sqrt(f32) is slightly different than sqrt(CAST(f32 AS double)))
+#[tokio::test]
+async fn sqrt_f32_vs_f64() -> Result<()> {
+    let mut ctx = create_ctx()?;
+    register_aggregate_csv(&mut ctx)?;
+    // sqrt(f32)'s plan passes
+    let sql = "SELECT avg(sqrt(c11)) FROM aggregate_test_100";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["0.6584408485889435"]];
+
+    assert_eq!(actual, expected);
+    let sql = "SELECT avg(sqrt(CAST(c11 AS double))) FROM aggregate_test_100";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["0.6584408483418833"]];
+    assert_eq!(actual, expected);
+    Ok(())
+}
+
+#[tokio::test]
+async fn csv_query_error() -> Result<()> {
+    // sin(utf8) should error
+    let mut ctx = create_ctx()?;
+    register_aggregate_csv(&mut ctx)?;
+    let sql = "SELECT sin(c1) FROM aggregate_test_100";
+    let plan = ctx.create_logical_plan(&sql);
+    assert!(plan.is_err());
+    Ok(())
+}
+
+// this query used to deadlock due to the call udf(udf())
+#[tokio::test]
+async fn csv_query_sqrt_sqrt() -> Result<()> {
+    let mut ctx = create_ctx()?;
+    register_aggregate_csv(&mut ctx)?;
+    let sql = "SELECT sqrt(sqrt(c12)) FROM aggregate_test_100 LIMIT 1";
+    let actual = execute(&mut ctx, sql).await;
+    // sqrt(sqrt(c12=0.9294097332465232)) = 0.9818650561397431
+    let expected = vec![vec!["0.9818650561397431"]];
+    assert_eq!(actual, expected);
     Ok(())
 }
 
@@ -207,10 +413,10 @@ fn create_ctx() -> Result<ExecutionContext> {
     let mut ctx = ExecutionContext::new();
 
     // register a custom UDF
-    ctx.register_udf(ScalarFunction::new(
+    ctx.register_udf(create_udf(
         "custom_sqrt",
-        vec![Field::new("n", DataType::Float64, true)],
-        DataType::Float64,
+        vec![DataType::Float64],
+        Arc::new(DataType::Float64),
         Arc::new(custom_sqrt),
     ));
 
@@ -234,51 +440,63 @@ fn custom_sqrt(args: &[ArrayRef]) -> Result<ArrayRef> {
     Ok(Arc::new(builder.finish()))
 }
 
-#[test]
-fn csv_query_avg() -> Result<()> {
+#[tokio::test]
+async fn csv_query_avg() -> Result<()> {
     let mut ctx = ExecutionContext::new();
     register_aggregate_csv(&mut ctx)?;
     let sql = "SELECT avg(c12) FROM aggregate_test_100";
-    let mut actual = execute(&mut ctx, sql);
+    let mut actual = execute(&mut ctx, sql).await;
     actual.sort();
-    let expected = "0.5089725099127211".to_string();
-    assert_eq!(expected, actual.join("\n"));
+    let expected = vec![vec!["0.5089725099127211"]];
+    assert_eq!(expected, actual);
     Ok(())
 }
 
-#[test]
-fn csv_query_group_by_avg() -> Result<()> {
+#[tokio::test]
+async fn csv_query_group_by_avg() -> Result<()> {
     let mut ctx = ExecutionContext::new();
     register_aggregate_csv(&mut ctx)?;
     let sql = "SELECT c1, avg(c12) FROM aggregate_test_100 GROUP BY c1";
-    let mut actual = execute(&mut ctx, sql);
+    let mut actual = execute(&mut ctx, sql).await;
     actual.sort();
-    let expected = "\"a\"\t0.48754517466109415\n\"b\"\t0.41040709263815384\n\"c\"\t0.6600456536439784\n\"d\"\t0.48855379387549824\n\"e\"\t0.48600669271341534".to_string();
-    assert_eq!(expected, actual.join("\n"));
+    let expected = vec![
+        vec!["a", "0.48754517466109415"],
+        vec!["b", "0.41040709263815384"],
+        vec!["c", "0.6600456536439784"],
+        vec!["d", "0.48855379387549824"],
+        vec!["e", "0.48600669271341534"],
+    ];
+    assert_eq!(expected, actual);
     Ok(())
 }
 
-#[test]
-fn csv_query_group_by_avg_with_projection() -> Result<()> {
+#[tokio::test]
+async fn csv_query_group_by_avg_with_projection() -> Result<()> {
     let mut ctx = ExecutionContext::new();
     register_aggregate_csv(&mut ctx)?;
     let sql = "SELECT avg(c12), c1 FROM aggregate_test_100 GROUP BY c1";
-    let mut actual = execute(&mut ctx, sql);
+    let mut actual = execute(&mut ctx, sql).await;
     actual.sort();
-    let expected = "0.41040709263815384\t\"b\"\n0.48600669271341534\t\"e\"\n0.48754517466109415\t\"a\"\n0.48855379387549824\t\"d\"\n0.6600456536439784\t\"c\"".to_string();
-    assert_eq!(expected, actual.join("\n"));
+    let expected = vec![
+        vec!["0.41040709263815384", "b"],
+        vec!["0.48600669271341534", "e"],
+        vec!["0.48754517466109415", "a"],
+        vec!["0.48855379387549824", "d"],
+        vec!["0.6600456536439784", "c"],
+    ];
+    assert_eq!(expected, actual);
     Ok(())
 }
 
-#[test]
-fn csv_query_avg_multi_batch() -> Result<()> {
+#[tokio::test]
+async fn csv_query_avg_multi_batch() -> Result<()> {
     let mut ctx = ExecutionContext::new();
     register_aggregate_csv(&mut ctx)?;
     let sql = "SELECT avg(c12) FROM aggregate_test_100";
     let plan = ctx.create_logical_plan(&sql).unwrap();
     let plan = ctx.optimize(&plan).unwrap();
-    let plan = ctx.create_physical_plan(&plan, DEFAULT_BATCH_SIZE).unwrap();
-    let results = ctx.collect(plan.as_ref()).unwrap();
+    let plan = ctx.create_physical_plan(&plan).unwrap();
+    let results = ctx.collect(plan).await.unwrap();
     let batch = &results[0];
     let column = batch.column(0);
     let array = column.as_any().downcast_ref::<Float64Array>().unwrap();
@@ -290,148 +508,446 @@ fn csv_query_avg_multi_batch() -> Result<()> {
     Ok(())
 }
 
-#[test]
-fn csv_query_count() -> Result<()> {
+#[tokio::test]
+async fn csv_query_count() -> Result<()> {
     let mut ctx = ExecutionContext::new();
     register_aggregate_csv(&mut ctx)?;
     let sql = "SELECT count(c12) FROM aggregate_test_100";
-    let actual = execute(&mut ctx, sql).join("\n");
-    let expected = "100".to_string();
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["100"]];
     assert_eq!(expected, actual);
     Ok(())
 }
 
-#[test]
-fn csv_query_group_by_int_count() -> Result<()> {
+#[tokio::test]
+async fn csv_query_group_by_int_count() -> Result<()> {
     let mut ctx = ExecutionContext::new();
     register_aggregate_csv(&mut ctx)?;
     let sql = "SELECT c1, count(c12) FROM aggregate_test_100 GROUP BY c1";
-    let mut actual = execute(&mut ctx, sql);
+    let mut actual = execute(&mut ctx, sql).await;
     actual.sort();
-    let expected = "\"a\"\t21\n\"b\"\t19\n\"c\"\t21\n\"d\"\t18\n\"e\"\t21".to_string();
-    assert_eq!(expected, actual.join("\n"));
+    let expected = vec![
+        vec!["a", "21"],
+        vec!["b", "19"],
+        vec!["c", "21"],
+        vec!["d", "18"],
+        vec!["e", "21"],
+    ];
+    assert_eq!(expected, actual);
     Ok(())
 }
 
-#[test]
-fn csv_query_group_by_string_min_max() -> Result<()> {
+#[tokio::test]
+async fn csv_query_group_with_aliased_aggregate() -> Result<()> {
+    let mut ctx = ExecutionContext::new();
+    register_aggregate_csv(&mut ctx)?;
+    let sql = "SELECT c1, count(c12) AS count FROM aggregate_test_100 GROUP BY c1";
+    let mut actual = execute(&mut ctx, sql).await;
+    actual.sort();
+    let expected = vec![
+        vec!["a", "21"],
+        vec!["b", "19"],
+        vec!["c", "21"],
+        vec!["d", "18"],
+        vec!["e", "21"],
+    ];
+    assert_eq!(expected, actual);
+    Ok(())
+}
+
+#[tokio::test]
+async fn csv_query_group_by_string_min_max() -> Result<()> {
     let mut ctx = ExecutionContext::new();
     register_aggregate_csv(&mut ctx)?;
     let sql = "SELECT c1, MIN(c12), MAX(c12) FROM aggregate_test_100 GROUP BY c1";
-    let mut actual = execute(&mut ctx, sql);
+    let mut actual = execute(&mut ctx, sql).await;
     actual.sort();
-    let expected =
-        "\"a\"\t0.02182578039211991\t0.9800193410444061\n\"b\"\t0.04893135681998029\t0.9185813970744787\n\"c\"\t0.0494924465469434\t0.991517828651004\n\"d\"\t0.061029375346466685\t0.9748360509016578\n\"e\"\t0.01479305307777301\t0.9965400387585364".to_string();
-    assert_eq!(expected, actual.join("\n"));
+    let expected = vec![
+        vec!["a", "0.02182578039211991", "0.9800193410444061"],
+        vec!["b", "0.04893135681998029", "0.9185813970744787"],
+        vec!["c", "0.0494924465469434", "0.991517828651004"],
+        vec!["d", "0.061029375346466685", "0.9748360509016578"],
+        vec!["e", "0.01479305307777301", "0.9965400387585364"],
+    ];
+    assert_eq!(expected, actual);
     Ok(())
 }
 
-#[test]
-fn csv_query_cast() -> Result<()> {
+#[tokio::test]
+async fn csv_query_cast() -> Result<()> {
     let mut ctx = ExecutionContext::new();
     register_aggregate_csv(&mut ctx)?;
     let sql = "SELECT CAST(c12 AS float) FROM aggregate_test_100 WHERE c12 > 0.376 AND c12 < 0.4";
-    let actual = execute(&mut ctx, sql).join("\n");
-    let expected = "0.39144436569161134\n0.38870280983958583".to_string();
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["0.39144436569161134"], vec!["0.38870280983958583"]];
     assert_eq!(expected, actual);
     Ok(())
 }
 
-#[test]
-fn csv_query_cast_literal() -> Result<()> {
+#[tokio::test]
+async fn csv_query_cast_literal() -> Result<()> {
     let mut ctx = ExecutionContext::new();
     register_aggregate_csv(&mut ctx)?;
     let sql = "SELECT c12, CAST(1 AS float) FROM aggregate_test_100 WHERE c12 > CAST(0 AS float) LIMIT 2";
-    let actual = execute(&mut ctx, sql).join("\n");
-    let expected = "0.9294097332465232\t1.0\n0.3114712539863804\t1.0".to_string();
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![
+        vec!["0.9294097332465232", "1"],
+        vec!["0.3114712539863804", "1"],
+    ];
     assert_eq!(expected, actual);
     Ok(())
 }
 
-#[test]
-fn csv_query_limit() -> Result<()> {
+#[tokio::test]
+async fn csv_query_limit() -> Result<()> {
     let mut ctx = ExecutionContext::new();
     register_aggregate_csv(&mut ctx)?;
     let sql = "SELECT c1 FROM aggregate_test_100 LIMIT 2";
-    let actual = execute(&mut ctx, sql).join("\n");
-    let expected = "\"c\"\n\"d\"".to_string();
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["c"], vec!["d"]];
     assert_eq!(expected, actual);
     Ok(())
 }
 
-#[test]
-fn csv_query_limit_bigger_than_nbr_of_rows() -> Result<()> {
+#[tokio::test]
+async fn csv_query_limit_bigger_than_nbr_of_rows() -> Result<()> {
     let mut ctx = ExecutionContext::new();
     register_aggregate_csv(&mut ctx)?;
     let sql = "SELECT c2 FROM aggregate_test_100 LIMIT 200";
-    let actual = execute(&mut ctx, sql).join("\n");
-    let expected = "2\n5\n1\n1\n5\n4\n3\n3\n1\n4\n1\n4\n3\n2\n1\n1\n2\n1\n3\n2\n4\n1\n5\n4\n2\n1\n4\n5\n2\n3\n4\n2\n1\n5\n3\n1\n2\n3\n3\n3\n2\n4\n1\n3\n2\n5\n2\n1\n4\n1\n4\n2\n5\n4\n2\n3\n4\n4\n4\n5\n4\n2\n1\n2\n4\n2\n3\n5\n1\n1\n4\n2\n1\n2\n1\n1\n5\n4\n5\n2\n3\n2\n4\n1\n3\n4\n3\n2\n5\n3\n3\n2\n5\n5\n4\n1\n3\n3\n4\n4".to_string();
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![
+        vec!["2"],
+        vec!["5"],
+        vec!["1"],
+        vec!["1"],
+        vec!["5"],
+        vec!["4"],
+        vec!["3"],
+        vec!["3"],
+        vec!["1"],
+        vec!["4"],
+        vec!["1"],
+        vec!["4"],
+        vec!["3"],
+        vec!["2"],
+        vec!["1"],
+        vec!["1"],
+        vec!["2"],
+        vec!["1"],
+        vec!["3"],
+        vec!["2"],
+        vec!["4"],
+        vec!["1"],
+        vec!["5"],
+        vec!["4"],
+        vec!["2"],
+        vec!["1"],
+        vec!["4"],
+        vec!["5"],
+        vec!["2"],
+        vec!["3"],
+        vec!["4"],
+        vec!["2"],
+        vec!["1"],
+        vec!["5"],
+        vec!["3"],
+        vec!["1"],
+        vec!["2"],
+        vec!["3"],
+        vec!["3"],
+        vec!["3"],
+        vec!["2"],
+        vec!["4"],
+        vec!["1"],
+        vec!["3"],
+        vec!["2"],
+        vec!["5"],
+        vec!["2"],
+        vec!["1"],
+        vec!["4"],
+        vec!["1"],
+        vec!["4"],
+        vec!["2"],
+        vec!["5"],
+        vec!["4"],
+        vec!["2"],
+        vec!["3"],
+        vec!["4"],
+        vec!["4"],
+        vec!["4"],
+        vec!["5"],
+        vec!["4"],
+        vec!["2"],
+        vec!["1"],
+        vec!["2"],
+        vec!["4"],
+        vec!["2"],
+        vec!["3"],
+        vec!["5"],
+        vec!["1"],
+        vec!["1"],
+        vec!["4"],
+        vec!["2"],
+        vec!["1"],
+        vec!["2"],
+        vec!["1"],
+        vec!["1"],
+        vec!["5"],
+        vec!["4"],
+        vec!["5"],
+        vec!["2"],
+        vec!["3"],
+        vec!["2"],
+        vec!["4"],
+        vec!["1"],
+        vec!["3"],
+        vec!["4"],
+        vec!["3"],
+        vec!["2"],
+        vec!["5"],
+        vec!["3"],
+        vec!["3"],
+        vec!["2"],
+        vec!["5"],
+        vec!["5"],
+        vec!["4"],
+        vec!["1"],
+        vec!["3"],
+        vec!["3"],
+        vec!["4"],
+        vec!["4"],
+    ];
     assert_eq!(expected, actual);
     Ok(())
 }
 
-#[test]
-fn csv_query_limit_with_same_nbr_of_rows() -> Result<()> {
+#[tokio::test]
+async fn csv_query_limit_with_same_nbr_of_rows() -> Result<()> {
     let mut ctx = ExecutionContext::new();
     register_aggregate_csv(&mut ctx)?;
     let sql = "SELECT c2 FROM aggregate_test_100 LIMIT 100";
-    let actual = execute(&mut ctx, sql).join("\n");
-    let expected = "2\n5\n1\n1\n5\n4\n3\n3\n1\n4\n1\n4\n3\n2\n1\n1\n2\n1\n3\n2\n4\n1\n5\n4\n2\n1\n4\n5\n2\n3\n4\n2\n1\n5\n3\n1\n2\n3\n3\n3\n2\n4\n1\n3\n2\n5\n2\n1\n4\n1\n4\n2\n5\n4\n2\n3\n4\n4\n4\n5\n4\n2\n1\n2\n4\n2\n3\n5\n1\n1\n4\n2\n1\n2\n1\n1\n5\n4\n5\n2\n3\n2\n4\n1\n3\n4\n3\n2\n5\n3\n3\n2\n5\n5\n4\n1\n3\n3\n4\n4".to_string();
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![
+        vec!["2"],
+        vec!["5"],
+        vec!["1"],
+        vec!["1"],
+        vec!["5"],
+        vec!["4"],
+        vec!["3"],
+        vec!["3"],
+        vec!["1"],
+        vec!["4"],
+        vec!["1"],
+        vec!["4"],
+        vec!["3"],
+        vec!["2"],
+        vec!["1"],
+        vec!["1"],
+        vec!["2"],
+        vec!["1"],
+        vec!["3"],
+        vec!["2"],
+        vec!["4"],
+        vec!["1"],
+        vec!["5"],
+        vec!["4"],
+        vec!["2"],
+        vec!["1"],
+        vec!["4"],
+        vec!["5"],
+        vec!["2"],
+        vec!["3"],
+        vec!["4"],
+        vec!["2"],
+        vec!["1"],
+        vec!["5"],
+        vec!["3"],
+        vec!["1"],
+        vec!["2"],
+        vec!["3"],
+        vec!["3"],
+        vec!["3"],
+        vec!["2"],
+        vec!["4"],
+        vec!["1"],
+        vec!["3"],
+        vec!["2"],
+        vec!["5"],
+        vec!["2"],
+        vec!["1"],
+        vec!["4"],
+        vec!["1"],
+        vec!["4"],
+        vec!["2"],
+        vec!["5"],
+        vec!["4"],
+        vec!["2"],
+        vec!["3"],
+        vec!["4"],
+        vec!["4"],
+        vec!["4"],
+        vec!["5"],
+        vec!["4"],
+        vec!["2"],
+        vec!["1"],
+        vec!["2"],
+        vec!["4"],
+        vec!["2"],
+        vec!["3"],
+        vec!["5"],
+        vec!["1"],
+        vec!["1"],
+        vec!["4"],
+        vec!["2"],
+        vec!["1"],
+        vec!["2"],
+        vec!["1"],
+        vec!["1"],
+        vec!["5"],
+        vec!["4"],
+        vec!["5"],
+        vec!["2"],
+        vec!["3"],
+        vec!["2"],
+        vec!["4"],
+        vec!["1"],
+        vec!["3"],
+        vec!["4"],
+        vec!["3"],
+        vec!["2"],
+        vec!["5"],
+        vec!["3"],
+        vec!["3"],
+        vec!["2"],
+        vec!["5"],
+        vec!["5"],
+        vec!["4"],
+        vec!["1"],
+        vec!["3"],
+        vec!["3"],
+        vec!["4"],
+        vec!["4"],
+    ];
     assert_eq!(expected, actual);
     Ok(())
 }
 
-#[test]
-fn csv_query_limit_zero() -> Result<()> {
+#[tokio::test]
+async fn csv_query_limit_zero() -> Result<()> {
     let mut ctx = ExecutionContext::new();
     register_aggregate_csv(&mut ctx)?;
     let sql = "SELECT c1 FROM aggregate_test_100 LIMIT 0";
-    let actual = execute(&mut ctx, sql).join("\n");
-    let expected = "".to_string();
+    let actual = execute(&mut ctx, sql).await;
+    let expected: Vec<Vec<String>> = vec![];
     assert_eq!(expected, actual);
     Ok(())
 }
 
-#[test]
-fn csv_query_create_external_table() {
+#[tokio::test]
+async fn csv_query_create_external_table() {
     let mut ctx = ExecutionContext::new();
-    register_aggregate_csv_by_sql(&mut ctx);
+    register_aggregate_csv_by_sql(&mut ctx).await;
     let sql = "SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, 10, c11, c12, c13 FROM aggregate_test_100 LIMIT 1";
-    let actual = execute(&mut ctx, sql).join("\n");
-    let expected = "\"c\"\t2\t1\t18109\t2033001162\t-6513304855495910254\t25\t43062\t1491205016\t10\t0.110830784\t0.9294097332465232\t\"6WfVFBVGJSQb7FhA7E0lBwdvjfZnSW\"".to_string();
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec![
+        "c",
+        "2",
+        "1",
+        "18109",
+        "2033001162",
+        "-6513304855495910254",
+        "25",
+        "43062",
+        "1491205016",
+        "10",
+        "0.110830784",
+        "0.9294097332465232",
+        "6WfVFBVGJSQb7FhA7E0lBwdvjfZnSW",
+    ]];
     assert_eq!(expected, actual);
 }
 
-#[test]
-fn csv_query_external_table_count() {
+#[tokio::test]
+async fn csv_query_external_table_count() {
     let mut ctx = ExecutionContext::new();
-    register_aggregate_csv_by_sql(&mut ctx);
+    register_aggregate_csv_by_sql(&mut ctx).await;
     let sql = "SELECT COUNT(c12) FROM aggregate_test_100";
-    let actual = execute(&mut ctx, sql).join("\n");
-    let expected = "100".to_string();
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["100"]];
+    assert_eq!(expected, actual);
+}
+
+#[tokio::test]
+async fn csv_query_external_table_sum() {
+    let mut ctx = ExecutionContext::new();
+    // cast smallint and int to bigint to avoid overflow during calculation
+    register_aggregate_csv_by_sql(&mut ctx).await;
+    let sql =
+        "SELECT SUM(CAST(c7 AS BIGINT)), SUM(CAST(c8 AS BIGINT)) FROM aggregate_test_100";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["13060", "3017641"]];
     assert_eq!(expected, actual);
 }
 
-#[test]
-fn csv_query_count_star() {
+#[tokio::test]
+async fn csv_query_count_star() {
     let mut ctx = ExecutionContext::new();
-    register_aggregate_csv_by_sql(&mut ctx);
+    register_aggregate_csv_by_sql(&mut ctx).await;
     let sql = "SELECT COUNT(*) FROM aggregate_test_100";
-    let actual = execute(&mut ctx, sql).join("\n");
-    let expected = "100".to_string();
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["100"]];
     assert_eq!(expected, actual);
 }
 
-#[test]
-fn csv_query_count_one() {
+#[tokio::test]
+async fn csv_query_count_one() {
     let mut ctx = ExecutionContext::new();
-    register_aggregate_csv_by_sql(&mut ctx);
+    register_aggregate_csv_by_sql(&mut ctx).await;
     let sql = "SELECT COUNT(1) FROM aggregate_test_100";
-    let actual = execute(&mut ctx, sql).join("\n");
-    let expected = "100".to_string();
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["100"]];
     assert_eq!(expected, actual);
 }
 
+#[tokio::test]
+async fn csv_explain() {
+    let mut ctx = ExecutionContext::new();
+    register_aggregate_csv_by_sql(&mut ctx).await;
+    let sql = "EXPLAIN SELECT c1 FROM aggregate_test_100 where c2 > 10";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![
+        vec![
+            "logical_plan",
+            "Projection: #c1\n  Filter: #c2 Gt Int64(10)\n    TableScan: aggregate_test_100 projection=None"
+        ]
+    ];
+    assert_eq!(expected, actual);
+
+    // Also, expect same result with lowercase explain
+    let sql = "explain SELECT c1 FROM aggregate_test_100 where c2 > 10";
+    let actual = execute(&mut ctx, sql).await;
+    assert_eq!(expected, actual);
+}
+
+#[tokio::test]
+async fn csv_explain_verbose() {
+    let mut ctx = ExecutionContext::new();
+    register_aggregate_csv_by_sql(&mut ctx).await;
+    let sql = "EXPLAIN VERBOSE SELECT c1 FROM aggregate_test_100 where c2 > 10";
+    let actual = execute(&mut ctx, sql).await;
+
+    // flatten to a single string
+    let actual = actual.into_iter().map(|r| r.join("\t")).collect::<String>();
+
+    // Don't actually test the contents of the debuging output (as
+    // that may change and keeping this test updated will be a
+    // pain). Instead just check for a few key pieces.
+    assert!(actual.contains("logical_plan"), "Actual: '{}'", actual);
+    assert!(actual.contains("physical_plan"), "Actual: '{}'", actual);
+    assert!(actual.contains("#c2 Gt Int64(10)"), "Actual: '{}'", actual);
+}
+
 fn aggr_test_schema() -> SchemaRef {
     Arc::new(Schema::new(vec![
         Field::new("c1", DataType::Utf8, false),
@@ -450,13 +966,13 @@ fn aggr_test_schema() -> SchemaRef {
     ]))
 }
 
-fn register_aggregate_csv_by_sql(ctx: &mut ExecutionContext) {
+async fn register_aggregate_csv_by_sql(ctx: &mut ExecutionContext) {
     let testdata = env::var("ARROW_TEST_DATA").expect("ARROW_TEST_DATA not defined");
 
     // TODO: The following c9 should be migrated to UInt32 and c10 should be UInt64 once
     // unsigned is supported.
-    ctx.sql(
-        &format!(
+    let df = ctx
+        .sql(&format!(
             "
     CREATE EXTERNAL TABLE aggregate_test_100 (
         c1  VARCHAR NOT NULL,
@@ -478,10 +994,16 @@ fn register_aggregate_csv_by_sql(ctx: &mut ExecutionContext) {
     LOCATION '{}/csv/aggregate_test_100.csv'
     ",
             testdata
-        ),
-        1024,
-    )
-    .unwrap();
+        ))
+        .expect("Creating dataframe for CREATE EXTERNAL TABLE");
+
+    // Mimic the CLI and execute the resulting plan -- even though it
+    // is effectively a no-op (returns zero rows)
+    let results = df.collect().await.expect("Executing CREATE EXTERNAL TABLE");
+    assert!(
+        results.is_empty(),
+        "Expected no rows from executing CREATE EXTERNAL TABLE"
+    );
 }
 
 fn register_aggregate_csv(ctx: &mut ExecutionContext) -> Result<()> {
@@ -504,84 +1026,376 @@ fn register_alltypes_parquet(ctx: &mut ExecutionContext) {
     .unwrap();
 }
 
-/// Execute query and return result set as tab delimited string
-fn execute(ctx: &mut ExecutionContext, sql: &str) -> Vec<String> {
-    let plan = ctx.create_logical_plan(&sql).unwrap();
-    let plan = ctx.optimize(&plan).unwrap();
-    let plan = ctx.create_physical_plan(&plan, DEFAULT_BATCH_SIZE).unwrap();
-    let results = ctx.collect(plan.as_ref()).unwrap();
-    result_str(&results)
+/// Execute query and return result set as 2-d table of Vecs
+/// `result[row][column]`
+async fn execute(ctx: &mut ExecutionContext, sql: &str) -> Vec<Vec<String>> {
+    let msg = format!("Creating logical plan for '{}'", sql);
+    let plan = ctx.create_logical_plan(&sql).expect(&msg);
+    let logical_schema = plan.schema();
+
+    let msg = format!("Optimizing logical plan for '{}': {:?}", sql, plan);
+    let plan = ctx.optimize(&plan).expect(&msg);
+    let optimized_logical_schema = plan.schema();
+
+    let msg = format!("Creating physical plan for '{}': {:?}", sql, plan);
+    let plan = ctx.create_physical_plan(&plan).expect(&msg);
+    let physical_schema = plan.schema();
+
+    let msg = format!("Executing physical plan for '{}': {:?}", sql, plan);
+    let results = ctx.collect(plan).await.expect(&msg);
+
+    assert_eq!(logical_schema.as_ref(), optimized_logical_schema.as_ref());
+    assert_eq!(logical_schema.as_ref(), physical_schema.as_ref());
+
+    result_vec(&results)
 }
 
-fn result_str(results: &[RecordBatch]) -> Vec<String> {
+/// Specialised String representation
+fn col_str(column: &ArrayRef, row_index: usize) -> String {
+    if column.is_null(row_index) {
+        return "NULL".to_string();
+    }
+
+    // Special case ListArray as there is no pretty print support for it yet
+    if let DataType::FixedSizeList(_, n) = column.data_type() {
+        let array = column
+            .as_any()
+            .downcast_ref::<FixedSizeListArray>()
+            .unwrap()
+            .value(row_index);
+
+        let mut r = Vec::with_capacity(*n as usize);
+        for i in 0..*n {
+            r.push(col_str(&array, i as usize));
+        }
+        return format!("[{}]", r.join(","));
+    }
+
+    array_value_to_string(column, row_index)
+        .ok()
+        .unwrap_or_else(|| "???".to_string())
+}
+
+/// Converts the results into a 2d array of strings, `result[row][column]`
+/// Special cases nulls to NULL for testing
+fn result_vec(results: &[RecordBatch]) -> Vec<Vec<String>> {
     let mut result = vec![];
     for batch in results {
         for row_index in 0..batch.num_rows() {
-            let mut str = String::new();
-            for column_index in 0..batch.num_columns() {
-                if column_index > 0 {
-                    str.push_str("\t");
-                }
-                let column = batch.column(column_index);
-
-                match column.data_type() {
-                    DataType::Int8 => {
-                        let array = column.as_any().downcast_ref::<Int8Array>().unwrap();
-                        str.push_str(&format!("{:?}", array.value(row_index)));
-                    }
-                    DataType::Int16 => {
-                        let array = column.as_any().downcast_ref::<Int16Array>().unwrap();
-                        str.push_str(&format!("{:?}", array.value(row_index)));
-                    }
-                    DataType::Int32 => {
-                        let array = column.as_any().downcast_ref::<Int32Array>().unwrap();
-                        str.push_str(&format!("{:?}", array.value(row_index)));
-                    }
-                    DataType::Int64 => {
-                        let array = column.as_any().downcast_ref::<Int64Array>().unwrap();
-                        str.push_str(&format!("{:?}", array.value(row_index)));
-                    }
-                    DataType::UInt8 => {
-                        let array = column.as_any().downcast_ref::<UInt8Array>().unwrap();
-                        str.push_str(&format!("{:?}", array.value(row_index)));
-                    }
-                    DataType::UInt16 => {
-                        let array =
-                            column.as_any().downcast_ref::<UInt16Array>().unwrap();
-                        str.push_str(&format!("{:?}", array.value(row_index)));
-                    }
-                    DataType::UInt32 => {
-                        let array =
-                            column.as_any().downcast_ref::<UInt32Array>().unwrap();
-                        str.push_str(&format!("{:?}", array.value(row_index)));
-                    }
-                    DataType::UInt64 => {
-                        let array =
-                            column.as_any().downcast_ref::<UInt64Array>().unwrap();
-                        str.push_str(&format!("{:?}", array.value(row_index)));
-                    }
-                    DataType::Float32 => {
-                        let array =
-                            column.as_any().downcast_ref::<Float32Array>().unwrap();
-                        str.push_str(&format!("{:?}", array.value(row_index)));
-                    }
-                    DataType::Float64 => {
-                        let array =
-                            column.as_any().downcast_ref::<Float64Array>().unwrap();
-                        str.push_str(&format!("{:?}", array.value(row_index)));
-                    }
-                    DataType::Utf8 => {
-                        let array =
-                            column.as_any().downcast_ref::<StringArray>().unwrap();
-                        let s = array.value(row_index);
-
-                        str.push_str(&format!("{:?}", s));
-                    }
-                    _ => str.push_str("???"),
-                }
-            }
-            result.push(str);
+            let row_vec = batch
+                .columns()
+                .iter()
+                .map(|column| col_str(column, row_index))
+                .collect();
+            result.push(row_vec);
         }
     }
     result
 }
+
+async fn generic_query_length<T: 'static + Array + From<Vec<&'static str>>>(
+    datatype: DataType,
+) -> Result<()> {
+    let schema = Arc::new(Schema::new(vec![Field::new("c1", datatype, false)]));
+
+    let data = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(T::from(vec!["", "a", "aa", "aaa"]))],
+    )?;
+
+    let table = MemTable::new(schema, vec![vec![data]])?;
+
+    let mut ctx = ExecutionContext::new();
+    ctx.register_table("test", Box::new(table));
+    let sql = "SELECT length(c1) FROM test";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["0"], vec!["1"], vec!["2"], vec!["3"]];
+    assert_eq!(expected, actual);
+    Ok(())
+}
+
+#[tokio::test]
+async fn query_length() -> Result<()> {
+    generic_query_length::<StringArray>(DataType::Utf8).await
+}
+
+#[tokio::test]
+async fn query_large_length() -> Result<()> {
+    generic_query_length::<LargeStringArray>(DataType::LargeUtf8).await
+}
+
+#[tokio::test]
+async fn query_not() -> Result<()> {
+    let schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Boolean, true)]));
+
+    let data = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(BooleanArray::from(vec![
+            Some(false),
+            None,
+            Some(true),
+        ]))],
+    )?;
+
+    let table = MemTable::new(schema, vec![vec![data]])?;
+
+    let mut ctx = ExecutionContext::new();
+    ctx.register_table("test", Box::new(table));
+    let sql = "SELECT NOT c1 FROM test";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["true"], vec!["NULL"], vec!["false"]];
+    assert_eq!(expected, actual);
+    Ok(())
+}
+
+#[tokio::test]
+async fn query_concat() -> Result<()> {
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("c1", DataType::Utf8, false),
+        Field::new("c2", DataType::Int32, true),
+    ]));
+
+    let data = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(StringArray::from(vec!["", "a", "aa", "aaa"])),
+            Arc::new(Int32Array::from(vec![Some(0), Some(1), None, Some(3)])),
+        ],
+    )?;
+
+    let table = MemTable::new(schema, vec![vec![data]])?;
+
+    let mut ctx = ExecutionContext::new();
+    ctx.register_table("test", Box::new(table));
+    let sql = "SELECT concat(c1, '-hi-', cast(c2 as varchar)) FROM test";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![
+        vec!["-hi-0"],
+        vec!["a-hi-1"],
+        vec!["NULL"],
+        vec!["aaa-hi-3"],
+    ];
+    assert_eq!(expected, actual);
+    Ok(())
+}
+
+#[tokio::test]
+async fn query_array() -> Result<()> {
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("c1", DataType::Utf8, false),
+        Field::new("c2", DataType::Int32, true),
+    ]));
+
+    let data = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(StringArray::from(vec!["", "a", "aa", "aaa"])),
+            Arc::new(Int32Array::from(vec![Some(0), Some(1), None, Some(3)])),
+        ],
+    )?;
+
+    let table = MemTable::new(schema, vec![vec![data]])?;
+
+    let mut ctx = ExecutionContext::new();
+    ctx.register_table("test", Box::new(table));
+    let sql = "SELECT array(c1, cast(c2 as varchar)) FROM test";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![
+        vec!["[,0]"],
+        vec!["[a,1]"],
+        vec!["[aa,NULL]"],
+        vec!["[aaa,3]"],
+    ];
+    assert_eq!(expected, actual);
+    Ok(())
+}
+
+#[tokio::test]
+async fn csv_query_sum_cast() {
+    let mut ctx = ExecutionContext::new();
+    register_aggregate_csv_by_sql(&mut ctx).await;
+    // c8 = i32; c9 = i64
+    let sql = "SELECT c8 + c9 FROM aggregate_test_100";
+    // check that the physical and logical schemas are equal
+    execute(&mut ctx, sql).await;
+}
+
+#[tokio::test]
+async fn like() -> Result<()> {
+    let mut ctx = ExecutionContext::new();
+    register_aggregate_csv_by_sql(&mut ctx).await;
+    let sql = "SELECT COUNT(c1) FROM aggregate_test_100 WHERE c13 LIKE '%FB%'";
+    // check that the physical and logical schemas are equal
+    let actual = execute(&mut ctx, sql).await;
+
+    let expected = vec![vec!["1"]];
+    assert_eq!(expected, actual);
+    Ok(())
+}
+
+fn make_timestamp_nano_table() -> Result<Box<MemTable>> {
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("ts", DataType::Timestamp(TimeUnit::Nanosecond, None), false),
+        Field::new("value", DataType::Int32, true),
+    ]));
+
+    let mut builder = TimestampNanosecondArray::builder(3);
+
+    builder.append_value(1599572549190855000)?; // 2020-09-08T13:42:29.190855+00:00
+    builder.append_value(1599568949190855000)?; // 2020-09-08T12:42:29.190855+00:00
+    builder.append_value(1599565349190855000)?; // 2020-09-08T11:42:29.190855+00:00
+
+    let data = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(builder.finish()),
+            Arc::new(Int32Array::from(vec![Some(1), Some(2), Some(3)])),
+        ],
+    )?;
+    let table = MemTable::new(schema, vec![vec![data]])?;
+    Ok(Box::new(table))
+}
+
+#[tokio::test]
+async fn to_timstamp() -> Result<()> {
+    let mut ctx = ExecutionContext::new();
+    ctx.register_table("ts_data", make_timestamp_nano_table()?);
+
+    let sql = "SELECT COUNT(*) FROM ts_data where ts > to_timestamp('2020-09-08T12:00:00+00:00')";
+    let actual = execute(&mut ctx, sql).await;
+
+    let expected = vec![vec!["2"]];
+    assert_eq!(expected, actual);
+    Ok(())
+}
+
+#[tokio::test]
+async fn query_is_null() -> Result<()> {
+    let schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Float64, true)]));
+
+    let data = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(Float64Array::from(vec![
+            Some(1.0),
+            None,
+            Some(f64::NAN),
+        ]))],
+    )?;
+
+    let table = MemTable::new(schema, vec![vec![data]])?;
+
+    let mut ctx = ExecutionContext::new();
+    ctx.register_table("test", Box::new(table));
+    let sql = "SELECT c1 IS NULL FROM test";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["false"], vec!["true"], vec!["false"]];
+    assert_eq!(expected, actual);
+    Ok(())
+}
+
+#[tokio::test]
+async fn query_is_not_null() -> Result<()> {
+    let schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Float64, true)]));
+
+    let data = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(Float64Array::from(vec![
+            Some(1.0),
+            None,
+            Some(f64::NAN),
+        ]))],
+    )?;
+
+    let table = MemTable::new(schema, vec![vec![data]])?;
+
+    let mut ctx = ExecutionContext::new();
+    ctx.register_table("test", Box::new(table));
+    let sql = "SELECT c1 IS NOT NULL FROM test";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["true"], vec!["false"], vec!["true"]];
+
+    assert_eq!(expected, actual);
+    Ok(())
+}
+
+#[tokio::test]
+async fn query_count_distinct() -> Result<()> {
+    let schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, true)]));
+
+    let data = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(Int32Array::from(vec![
+            Some(0),
+            Some(1),
+            None,
+            Some(3),
+            Some(3),
+        ]))],
+    )?;
+
+    let table = MemTable::new(schema, vec![vec![data]])?;
+
+    let mut ctx = ExecutionContext::new();
+    ctx.register_table("test", Box::new(table));
+    let sql = "SELECT COUNT(DISTINCT c1) FROM test";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["3".to_string()]];
+    assert_eq!(expected, actual);
+    Ok(())
+}
+
+#[tokio::test]
+async fn query_on_string_dictionary() -> Result<()> {
+    // Test to ensure DataFusion can operate on dictionary types
+    // Use StringDictionary (32 bit indexes = keys)
+    let field_type =
+        DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));
+    let schema = Arc::new(Schema::new(vec![Field::new("d1", field_type, true)]));
+
+    let keys_builder = PrimitiveBuilder::<Int32Type>::new(10);
+    let values_builder = StringBuilder::new(10);
+    let mut builder = StringDictionaryBuilder::new(keys_builder, values_builder);
+
+    builder.append("one")?;
+    builder.append_null()?;
+    builder.append("three")?;
+    let array = Arc::new(builder.finish());
+
+    let data = RecordBatch::try_new(schema.clone(), vec![array])?;
+
+    let table = MemTable::new(schema, vec![vec![data]])?;
+    let mut ctx = ExecutionContext::new();
+    ctx.register_table("test", Box::new(table));
+
+    // Basic SELECT
+    let sql = "SELECT * FROM test";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["one"], vec!["NULL"], vec!["three"]];
+    assert_eq!(expected, actual);
+
+    // basic filtering
+    let sql = "SELECT * FROM test WHERE d1 IS NOT NULL";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["one"], vec!["three"]];
+    assert_eq!(expected, actual);
+
+    // filtering with constant
+    let sql = "SELECT * FROM test WHERE d1 = 'three'";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["three"]];
+    assert_eq!(expected, actual);
+
+    // Expression evaluation
+    let sql = "SELECT concat(d1, '-foo') FROM test";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["one-foo"], vec!["NULL"], vec!["three-foo"]];
+    assert_eq!(expected, actual);
+
+    // aggregation
+    let sql = "SELECT COUNT(d1) FROM test";
+    let actual = execute(&mut ctx, sql).await;
+    let expected = vec![vec!["2"]];
+    assert_eq!(expected, actual);
+
+    Ok(())
+}
diff --git a/rust/datafusion/tests/user_defined_plan.rs b/rust/datafusion/tests/user_defined_plan.rs
new file mode 100644
index 00000000000..ad8e618ad5d
--- /dev/null
+++ b/rust/datafusion/tests/user_defined_plan.rs
@@ -0,0 +1,538 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! This module contains an end to end demonstration of creating
+//! a user defined operator in DataFusion.
+//!
+//! Specifically, it shows how to define a `TopKNode` that implements
+//! `ExtensionPlanNode`, add an OptimizerRule to rewrite a
+//! `LogicalPlan` to use that node a `LogicalPlan`, create an
+//! `ExecutionPlan` and finally produce results.
+//!
+//! # TopK Background:
+//!
+//! A "Top K" node is a common query optimization which is used for
+//! queries such as "find the top 3 customers by revenue". The
+//! (simplified) SQL for such a query might be:
+//!
+//! ```sql
+//! CREATE EXTERNAL TABLE sales(customer_id VARCHAR, revenue BIGINT)
+//!   STORED AS CSV location 'tests/customer.csv';
+//!
+//! SELECT customer_id, revenue FROM sales ORDER BY revenue DESC limit 3;
+//! ```
+//!
+//! And a naive plan would be:
+//!
+//! ```
+//! > explain SELECT customer_id, revenue FROM sales ORDER BY revenue DESC limit 3;
+//! +--------------+----------------------------------------+
+//! | plan_type    | plan                                   |
+//! +--------------+----------------------------------------+
+//! | logical_plan | Limit: 3                               |
+//! |              |   Sort: #revenue DESC NULLS FIRST      |
+//! |              |     Projection: #customer_id, #revenue |
+//! |              |       TableScan: sales projection=None |
+//! +--------------+----------------------------------------+
+//! ```
+//!
+//! While this plan produces the correct answer, the careful reader
+//! will note it fully sorts the input before discarding everything
+//! other than the top 3 elements.
+//!
+//! The same answer can be produced by simply keeping track of the top
+//! N elements, reducing the total amount of required buffer memory.
+//!
+
+use futures::{FutureExt, Stream, StreamExt, TryStreamExt};
+
+use arrow::{
+    array::{Int64Array, StringArray},
+    datatypes::SchemaRef,
+    error::ArrowError,
+    record_batch::RecordBatch,
+    util::pretty::pretty_format_batches,
+};
+use datafusion::{
+    error::{DataFusionError, Result},
+    execution::context::ExecutionContextState,
+    execution::context::QueryPlanner,
+    logical_plan::{Expr, LogicalPlan, UserDefinedLogicalNode},
+    optimizer::{optimizer::OptimizerRule, utils::optimize_explain},
+    physical_plan::{
+        planner::{DefaultPhysicalPlanner, ExtensionPlanner},
+        Distribution, ExecutionPlan, Partitioning, PhysicalPlanner, RecordBatchStream,
+        SendableRecordBatchStream,
+    },
+    prelude::{ExecutionConfig, ExecutionContext},
+};
+use fmt::Debug;
+use std::task::{Context, Poll};
+use std::{any::Any, collections::BTreeMap, fmt, sync::Arc};
+
+use async_trait::async_trait;
+
+/// Execute the specified sql and return the resulting record batches
+/// pretty printed as a String.
+async fn exec_sql(ctx: &mut ExecutionContext, sql: &str) -> Result<String> {
+    let df = ctx.sql(sql)?;
+    let batches = df.collect().await?;
+    pretty_format_batches(&batches).map_err(|e| DataFusionError::ArrowError(e))
+}
+
+/// Create a test table.
+async fn setup_table(mut ctx: ExecutionContext) -> Result<ExecutionContext> {
+    let sql = "CREATE EXTERNAL TABLE sales(customer_id VARCHAR, revenue BIGINT) STORED AS CSV location 'tests/customer.csv'";
+
+    let expected = vec!["++", "++"];
+
+    let s = exec_sql(&mut ctx, sql).await?;
+    let actual = s.lines().collect::<Vec<_>>();
+
+    assert_eq!(expected, actual, "Creating table");
+    Ok(ctx)
+}
+
+const QUERY: &str =
+    "SELECT customer_id, revenue FROM sales ORDER BY revenue DESC limit 3";
+
+// Run the query using the specified execution context and compare it
+// to the known result
+async fn run_and_compare_query(
+    mut ctx: ExecutionContext,
+    description: &str,
+) -> Result<()> {
+    let expected = vec![
+        "+-------------+---------+",
+        "| customer_id | revenue |",
+        "+-------------+---------+",
+        "| paul        | 300     |",
+        "| jorge       | 200     |",
+        "| andy        | 150     |",
+        "+-------------+---------+",
+    ];
+
+    let s = exec_sql(&mut ctx, QUERY).await?;
+    let actual = s.lines().collect::<Vec<_>>();
+
+    assert_eq!(
+        expected,
+        actual,
+        "output mismatch for {}. Expectedn\n{}Actual:\n{}",
+        description,
+        expected.join("\n"),
+        s
+    );
+    Ok(())
+}
+
+#[tokio::test]
+// Run the query using default planners and optimizer
+async fn normal_query() -> Result<()> {
+    let ctx = setup_table(ExecutionContext::new()).await?;
+    run_and_compare_query(ctx, "Default context").await
+}
+
+#[tokio::test]
+// Run the query using topk optimization
+async fn topk_query() -> Result<()> {
+    // Note the only difference is that the top
+    let ctx = setup_table(make_topk_context()).await?;
+    run_and_compare_query(ctx, "Topk context").await
+}
+
+#[tokio::test]
+// Run EXPLAIN PLAN and show the plan was in fact rewritten
+async fn topk_plan() -> Result<()> {
+    let mut ctx = setup_table(make_topk_context()).await?;
+
+    let expected = vec![
+        "| logical_plan after topk                 | TopK: k=3                                      |",
+        "|                                         |   Projection: #customer_id, #revenue           |",
+        "|                                         |     TableScan: sales projection=Some([0, 1])   |",
+    ].join("\n");
+
+    let explain_query = format!("EXPLAIN VERBOSE {}", QUERY);
+    let actual_output = exec_sql(&mut ctx, &explain_query).await?;
+
+    // normalize newlines (output on windows uses \r\n)
+    let actual_output = actual_output.replace("\r\n", "\n");
+
+    assert!(actual_output.contains(&expected) , "Expected output not present in actual output\nExpected:\n---------\n{}\nActual:\n--------\n{}", expected, actual_output);
+    Ok(())
+}
+
+fn make_topk_context() -> ExecutionContext {
+    let config = ExecutionConfig::new().with_query_planner(Arc::new(TopKQueryPlanner {}));
+
+    ExecutionContext::with_config(config)
+}
+
+// ------ The implementation of the TopK code follows -----
+
+struct TopKQueryPlanner {}
+
+impl QueryPlanner for TopKQueryPlanner {
+    fn rewrite_logical_plan(&self, plan: LogicalPlan) -> Result<LogicalPlan> {
+        TopKOptimizerRule {}.optimize(&plan)
+    }
+
+    /// Given a `LogicalPlan` created from above, create an
+    /// `ExecutionPlan` suitable for execution
+    fn create_physical_plan(
+        &self,
+        logical_plan: &LogicalPlan,
+        ctx_state: &ExecutionContextState,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        // Teach the default physical planner how to plan TopK nodes.
+        let physical_planner =
+            DefaultPhysicalPlanner::with_extension_planner(Arc::new(TopKPlanner {}));
+        // Delegate most work of physical planning to the default physical planner
+        physical_planner.create_physical_plan(logical_plan, ctx_state)
+    }
+}
+
+struct TopKOptimizerRule {}
+impl OptimizerRule for TopKOptimizerRule {
+    // Example rewrite pass to insert a user defined LogicalPlanNode
+    fn optimize(&mut self, plan: &LogicalPlan) -> Result<LogicalPlan> {
+        match plan {
+            // Note: this code simply looks for the pattern of a Limit followed by a
+            // Sort and replaces it by a TopK node. It does not handle many
+            // edge cases (e.g multiple sort columns, sort ASC / DESC), etc.
+            LogicalPlan::Limit { ref n, ref input } => {
+                if let LogicalPlan::Sort {
+                    ref expr,
+                    ref input,
+                } = **input
+                {
+                    if expr.len() == 1 {
+                        // we found a sort with a single sort expr, replace with a a TopK
+                        return Ok(LogicalPlan::Extension {
+                            node: Arc::new(TopKPlanNode {
+                                k: *n,
+                                input: self.optimize(input.as_ref())?,
+                                expr: expr[0].clone(),
+                            }),
+                        });
+                    }
+                }
+            }
+            // Due to the way explain is implemented, in order to get
+            // explain functionality we need to explicitly handle it
+            // here.
+            LogicalPlan::Explain {
+                verbose,
+                plan,
+                stringified_plans,
+                schema,
+            } => {
+                return optimize_explain(
+                    self,
+                    *verbose,
+                    &*plan,
+                    stringified_plans,
+                    &*schema,
+                )
+            }
+            _ => {}
+        }
+
+        // If we didn't find the Limit/Sort combination, recurse as
+        // normal and build the result.
+        self.optimize_children(plan)
+    }
+
+    fn name(&self) -> &str {
+        "topk"
+    }
+}
+
+struct TopKPlanNode {
+    k: usize,
+    input: LogicalPlan,
+    /// The sort expression (this example only supports a single sort
+    /// expr)
+    expr: Expr,
+}
+
+impl Debug for TopKPlanNode {
+    /// For TopK, use explain format for the Debug format. Other types
+    /// of nodes may
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        self.fmt_for_explain(f)
+    }
+}
+
+impl UserDefinedLogicalNode for TopKPlanNode {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn inputs(&self) -> Vec<&LogicalPlan> {
+        vec![&self.input]
+    }
+
+    /// Schema for TopK is the same as the input
+    fn schema(&self) -> &SchemaRef {
+        self.input.schema()
+    }
+
+    fn expressions(&self) -> Vec<Expr> {
+        vec![self.expr.clone()]
+    }
+
+    /// For example: `TopK: k=10`
+    fn fmt_for_explain(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "TopK: k={}", self.k)
+    }
+
+    fn from_template(
+        &self,
+        exprs: &Vec<Expr>,
+        inputs: &Vec<LogicalPlan>,
+    ) -> Arc<dyn UserDefinedLogicalNode + Send + Sync> {
+        assert_eq!(inputs.len(), 1, "input size inconsistent");
+        assert_eq!(exprs.len(), 1, "expression size inconsistent");
+        Arc::new(TopKPlanNode {
+            k: self.k,
+            input: inputs[0].clone(),
+            expr: exprs[0].clone(),
+        })
+    }
+}
+
+/// Physical planner for TopK nodes
+struct TopKPlanner {}
+
+impl ExtensionPlanner for TopKPlanner {
+    /// Create a physical plan for an extension node
+    fn plan_extension(
+        &self,
+        node: &dyn UserDefinedLogicalNode,
+        inputs: Vec<Arc<dyn ExecutionPlan>>,
+        _ctx_state: &ExecutionContextState,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        if let Some(topk_node) = node.as_any().downcast_ref::<TopKPlanNode>() {
+            assert_eq!(inputs.len(), 1, "Inconsistent number of inputs");
+            // figure out input name
+            Ok(Arc::new(TopKExec {
+                input: inputs[0].clone(),
+                k: topk_node.k,
+            }))
+        } else {
+            Err(DataFusionError::Internal(format!(
+                "Unknown extension node type {:?}",
+                node
+            )))
+        }
+    }
+}
+
+/// Physical operator that implements TopK for u64 data types. This
+/// code is not general and is meant as an illustration only
+struct TopKExec {
+    input: Arc<dyn ExecutionPlan>,
+    /// The maxium number of values
+    k: usize,
+}
+
+impl Debug for TopKExec {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "TopKExec")
+    }
+}
+
+#[async_trait]
+impl ExecutionPlan for TopKExec {
+    /// Return a reference to Any that can be used for downcasting
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.input.schema()
+    }
+
+    fn output_partitioning(&self) -> Partitioning {
+        Partitioning::UnknownPartitioning(1)
+    }
+
+    fn required_child_distribution(&self) -> Distribution {
+        Distribution::UnspecifiedDistribution
+    }
+
+    fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
+        vec![self.input.clone()]
+    }
+
+    fn with_new_children(
+        &self,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        match children.len() {
+            1 => Ok(Arc::new(TopKExec {
+                input: children[0].clone(),
+                k: self.k,
+            })),
+            _ => Err(DataFusionError::Internal(
+                "TopKExec wrong number of children".to_string(),
+            )),
+        }
+    }
+
+    /// Execute one partition and return an iterator over RecordBatch
+    async fn execute(&self, partition: usize) -> Result<SendableRecordBatchStream> {
+        if 0 != partition {
+            return Err(DataFusionError::Internal(format!(
+                "TopKExec invalid partition {}",
+                partition
+            )));
+        }
+
+        Ok(Box::pin(TopKReader {
+            input: self.input.execute(partition).await?,
+            k: self.k,
+            done: false,
+        }))
+    }
+}
+
+// A very specialized TopK implementation
+struct TopKReader {
+    /// The input to read data from
+    input: SendableRecordBatchStream,
+    /// Maximum number of output values
+    k: usize,
+    /// Have we produced the output yet?
+    done: bool,
+}
+
+/// Keeps track of the revenue from customer_id and stores if it
+/// is the top values we have seen so far.
+fn add_row(
+    top_values: &mut BTreeMap<i64, String>,
+    customer_id: &str,
+    revenue: i64,
+    k: &usize,
+) {
+    top_values.insert(revenue, customer_id.into());
+    // only keep top k
+    while top_values.len() > *k {
+        remove_lowest_value(top_values)
+    }
+}
+
+fn remove_lowest_value(top_values: &mut BTreeMap<i64, String>) {
+    if !top_values.is_empty() {
+        let smallest_revenue = {
+            let (revenue, _) = top_values.iter().next().unwrap();
+            *revenue
+        };
+        top_values.remove(&smallest_revenue);
+    }
+}
+
+fn accumulate_batch(
+    input_batch: &RecordBatch,
+    mut top_values: BTreeMap<i64, String>,
+    k: &usize,
+) -> Result<BTreeMap<i64, String>> {
+    let num_rows = input_batch.num_rows();
+    // Assuming the input columns are
+    // column[0]: customer_id / UTF8
+    // column[1]: revenue: Int64
+    let customer_id = input_batch
+        .column(0)
+        .as_any()
+        .downcast_ref::<StringArray>()
+        .expect("Column 0 is not customer_id");
+
+    let revenue = input_batch
+        .column(1)
+        .as_any()
+        .downcast_ref::<Int64Array>()
+        .expect("Column 1 is not revenue");
+
+    for row in 0..num_rows {
+        add_row(
+            &mut top_values,
+            customer_id.value(row),
+            revenue.value(row),
+            k,
+        );
+    }
+    Ok(top_values)
+}
+
+impl Stream for TopKReader {
+    type Item = std::result::Result<RecordBatch, ArrowError>;
+
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<Option<Self::Item>> {
+        if self.done {
+            return Poll::Ready(None);
+        }
+        // this aggregates and thus returns a single RecordBatch.
+        self.done = true;
+
+        // take this as immutable
+        let k = self.k;
+        let schema = self.schema().clone();
+
+        let top_values = self
+            .input
+            .as_mut()
+            // Hard coded implementation for sales / customer_id example as BTree
+            .try_fold(
+                BTreeMap::<i64, String>::new(),
+                move |top_values, batch| async move {
+                    accumulate_batch(&batch, top_values, &k)
+                        .map_err(DataFusionError::into_arrow_external_error)
+                },
+            );
+
+        let top_values = top_values.map(|top_values| match top_values {
+            Ok(top_values) => {
+                // make output by walking over the map backwards (so values are descending)
+                let (revenue, customer): (Vec<i64>, Vec<&String>) =
+                    top_values.iter().rev().unzip();
+
+                let customer: Vec<&str> = customer.iter().map(|&s| &**s).collect();
+                Ok(RecordBatch::try_new(
+                    schema,
+                    vec![
+                        Arc::new(StringArray::from(customer)),
+                        Arc::new(Int64Array::from(revenue)),
+                    ],
+                )?)
+            }
+            Err(e) => Err(e),
+        });
+        let mut top_values = Box::pin(top_values.into_stream());
+
+        top_values.poll_next_unpin(cx)
+    }
+}
+
+impl RecordBatchStream for TopKReader {
+    fn schema(&self) -> SchemaRef {
+        self.input.schema()
+    }
+}
diff --git a/rust/integration-testing/Cargo.toml b/rust/integration-testing/Cargo.toml
index 0c5d3b522b2..1c2687086fb 100644
--- a/rust/integration-testing/Cargo.toml
+++ b/rust/integration-testing/Cargo.toml
@@ -18,7 +18,7 @@
 [package]
 name = "arrow-integration-testing"
 description = "Binaries used in the Arrow integration tests"
-version = "2.0.0-SNAPSHOT"
+version = "3.0.0-SNAPSHOT"
 homepage = "https://github.com/apache/arrow"
 repository = "https://github.com/apache/arrow"
 authors = ["Apache Arrow <dev@arrow.apache.org>"]
diff --git a/rust/integration-testing/src/bin/arrow-file-to-stream.rs b/rust/integration-testing/src/bin/arrow-file-to-stream.rs
index d538b4f49aa..ded1972e40c 100644
--- a/rust/integration-testing/src/bin/arrow-file-to-stream.rs
+++ b/rust/integration-testing/src/bin/arrow-file-to-stream.rs
@@ -22,7 +22,6 @@ use std::io::{self, BufReader};
 use arrow::error::Result;
 use arrow::ipc::reader::FileReader;
 use arrow::ipc::writer::StreamWriter;
-use arrow::record_batch::RecordBatchReader;
 
 fn main() -> Result<()> {
     let args: Vec<String> = env::args().collect();
@@ -33,14 +32,17 @@ fn main() -> Result<()> {
 
     let f = File::open(filename)?;
     let reader = BufReader::new(f);
-    let mut reader = FileReader::try_new(reader)?;
+    let reader = FileReader::try_new(reader)?;
     let schema = reader.schema();
 
     let mut writer = StreamWriter::try_new(io::stdout(), &schema)?;
 
-    while let Some(batch) = reader.next_batch()? {
-        writer.write(&batch)?;
-    }
+    reader
+        .map(|batch| {
+            let batch = batch?;
+            writer.write(&batch)
+        })
+        .collect::<Result<()>>()?;
     writer.finish()?;
 
     eprintln!("Completed without error");
diff --git a/rust/integration-testing/src/bin/arrow-json-integration-test.rs b/rust/integration-testing/src/bin/arrow-json-integration-test.rs
index 170892f8fb1..5556c4cebc8 100644
--- a/rust/integration-testing/src/bin/arrow-json-integration-test.rs
+++ b/rust/integration-testing/src/bin/arrow-json-integration-test.rs
@@ -25,7 +25,7 @@ use arrow::datatypes::{DataType, DateUnit, IntervalUnit, Schema};
 use arrow::error::{ArrowError, Result};
 use arrow::ipc::reader::FileReader;
 use arrow::ipc::writer::FileWriter;
-use arrow::record_batch::{RecordBatch, RecordBatchReader};
+use arrow::record_batch::RecordBatch;
 
 use hex::decode;
 use std::env;
@@ -423,7 +423,7 @@ fn arrow_to_json(arrow_name: &str, json_name: &str, verbose: bool) -> Result<()>
     }
 
     let arrow_file = File::open(arrow_name)?;
-    let mut reader = FileReader::try_new(arrow_file)?;
+    let reader = FileReader::try_new(arrow_file)?;
 
     let mut fields = vec![];
     for f in reader.schema().fields() {
@@ -431,10 +431,9 @@ fn arrow_to_json(arrow_name: &str, json_name: &str, verbose: bool) -> Result<()>
     }
     let schema = ArrowJsonSchema { fields };
 
-    let mut batches = vec![];
-    while let Ok(Some(batch)) = reader.next_batch() {
-        batches.push(ArrowJsonBatch::from_batch(&batch));
-    }
+    let batches = reader
+        .map(|batch| Ok(ArrowJsonBatch::from_batch(&batch?)))
+        .collect::<Result<Vec<_>>>()?;
 
     let arrow_json = ArrowJson {
         schema,
@@ -483,7 +482,7 @@ fn validate(arrow_name: &str, json_name: &str, verbose: bool) -> Result<()> {
     }
 
     for json_batch in &json_batches {
-        if let Some(arrow_batch) = arrow_reader.next_batch()? {
+        if let Some(Ok(arrow_batch)) = arrow_reader.next() {
             // compare batches
             assert!(arrow_batch.num_columns() == json_batch.num_columns());
             assert!(arrow_batch.num_rows() == json_batch.num_rows());
@@ -500,7 +499,7 @@ fn validate(arrow_name: &str, json_name: &str, verbose: bool) -> Result<()> {
         }
     }
 
-    if let Some(_) = arrow_reader.next_batch()? {
+    if arrow_reader.next().is_some() {
         return Err(ArrowError::ComputeError(
             "no more json batches left".to_owned(),
         ));
diff --git a/rust/integration-testing/src/bin/arrow-stream-to-file.rs b/rust/integration-testing/src/bin/arrow-stream-to-file.rs
index 2468970e6b7..87101c3ec89 100644
--- a/rust/integration-testing/src/bin/arrow-stream-to-file.rs
+++ b/rust/integration-testing/src/bin/arrow-stream-to-file.rs
@@ -21,20 +21,19 @@ use std::io;
 use arrow::error::Result;
 use arrow::ipc::reader::StreamReader;
 use arrow::ipc::writer::FileWriter;
-use arrow::record_batch::RecordBatchReader;
 
 fn main() -> Result<()> {
     let args: Vec<String> = env::args().collect();
     eprintln!("{:?}", args);
 
-    let mut arrow_stream_reader = StreamReader::try_new(io::stdin())?;
+    let arrow_stream_reader = StreamReader::try_new(io::stdin())?;
     let schema = arrow_stream_reader.schema();
 
     let mut writer = FileWriter::try_new(io::stdout(), &schema)?;
 
-    while let Some(batch) = arrow_stream_reader.next_batch()? {
-        writer.write(&batch)?;
-    }
+    arrow_stream_reader
+        .map(|batch| writer.write(&batch?))
+        .collect::<Result<_>>()?;
     writer.finish()?;
 
     eprintln!("Completed without error");
diff --git a/rust/parquet/Cargo.toml b/rust/parquet/Cargo.toml
index 7fc5b8ac271..122c5b6356d 100644
--- a/rust/parquet/Cargo.toml
+++ b/rust/parquet/Cargo.toml
@@ -17,7 +17,7 @@
 
 [package]
 name = "parquet"
-version = "2.0.0-SNAPSHOT"
+version = "3.0.0-SNAPSHOT"
 license = "Apache-2.0"
 description = "Apache Parquet implementation in Rust"
 homepage = "https://github.com/apache/arrow"
@@ -39,8 +39,8 @@ lz4 = { version = "1.23", optional = true }
 zstd = { version = "0.5", optional = true }
 chrono = "0.4"
 num-bigint = "0.3"
-arrow = { path = "../arrow", version = "2.0.0-SNAPSHOT", optional = true }
-serde_json = { version = "1.0", features = ["preserve_order"] }
+arrow = { path = "../arrow", version = "3.0.0-SNAPSHOT", optional = true }
+base64 = { version = "*", optional = true }
 
 [dev-dependencies]
 rand = "0.7"
@@ -49,7 +49,8 @@ brotli = "3.3"
 flate2 = "1.0"
 lz4 = "1.23"
 zstd = "0.5"
-arrow = { path = "../arrow", version = "2.0.0-SNAPSHOT" }
+arrow = { path = "../arrow", version = "3.0.0-SNAPSHOT" }
+serde_json = { version = "1.0", features = ["preserve_order"] }
 
 [features]
-default = ["arrow", "snap", "brotli", "flate2", "lz4", "zstd"]
+default = ["arrow", "snap", "brotli", "flate2", "lz4", "zstd", "base64"]
diff --git a/rust/parquet/README.md b/rust/parquet/README.md
index e84843edfd7..ade19518f62 100644
--- a/rust/parquet/README.md
+++ b/rust/parquet/README.md
@@ -23,7 +23,7 @@
 Add this to your Cargo.toml:
 ```toml
 [dependencies]
-parquet = "2.0.0-SNAPSHOT"
+parquet = "3.0.0-SNAPSHOT"
 ```
 
 and this to your crate root:
@@ -44,7 +44,7 @@ while let Some(record) = iter.next() {
     println!("{}", record);
 }
 ```
-See [crate documentation](https://docs.rs/crate/parquet/2.0.0-SNAPSHOT) on available API.
+See [crate documentation](https://docs.rs/crate/parquet/3.0.0-SNAPSHOT) on available API.
 
 ## Supported Parquet Version
 - Parquet-format 2.4.0
diff --git a/rust/parquet/src/arrow/array_reader.rs b/rust/parquet/src/arrow/array_reader.rs
index ae6ca497b85..a455e48abb6 100644
--- a/rust/parquet/src/arrow/array_reader.rs
+++ b/rust/parquet/src/arrow/array_reader.rs
@@ -25,19 +25,41 @@ use std::sync::Arc;
 use std::vec::Vec;
 
 use arrow::array::{
-    ArrayDataBuilder, ArrayDataRef, ArrayRef, BooleanBufferBuilder, BufferBuilderTrait,
-    Int16BufferBuilder, StructArray,
+    Array, ArrayData, ArrayDataBuilder, ArrayDataRef, ArrayRef, BinaryArray,
+    BinaryBuilder, BooleanBufferBuilder, BufferBuilderTrait, FixedSizeBinaryArray,
+    FixedSizeBinaryBuilder, GenericListArray, Int16BufferBuilder, ListBuilder,
+    OffsetSizeTrait, PrimitiveArray, PrimitiveBuilder, StringArray, StringBuilder,
+    StructArray,
 };
 use arrow::buffer::{Buffer, MutableBuffer};
-use arrow::datatypes::{DataType as ArrowType, DateUnit, Field, IntervalUnit, TimeUnit};
+use arrow::datatypes::{
+    ArrowPrimitiveType, BooleanType as ArrowBooleanType, DataType as ArrowType,
+    Date32Type as ArrowDate32Type, Date64Type as ArrowDate64Type,
+    DurationMicrosecondType as ArrowDurationMicrosecondType,
+    DurationMillisecondType as ArrowDurationMillisecondType,
+    DurationNanosecondType as ArrowDurationNanosecondType,
+    DurationSecondType as ArrowDurationSecondType, Field,
+    Float32Type as ArrowFloat32Type, Float64Type as ArrowFloat64Type,
+    Int16Type as ArrowInt16Type, Int32Type as ArrowInt32Type,
+    Int64Type as ArrowInt64Type, Int8Type as ArrowInt8Type, Schema,
+    Time32MillisecondType as ArrowTime32MillisecondType,
+    Time32SecondType as ArrowTime32SecondType,
+    Time64MicrosecondType as ArrowTime64MicrosecondType,
+    Time64NanosecondType as ArrowTime64NanosecondType, TimeUnit as ArrowTimeUnit,
+    TimestampMicrosecondType as ArrowTimestampMicrosecondType,
+    TimestampMillisecondType as ArrowTimestampMillisecondType,
+    TimestampNanosecondType as ArrowTimestampNanosecondType,
+    TimestampSecondType as ArrowTimestampSecondType, ToByteSlice,
+    UInt16Type as ArrowUInt16Type, UInt32Type as ArrowUInt32Type,
+    UInt64Type as ArrowUInt64Type, UInt8Type as ArrowUInt8Type,
+};
+use arrow::util::bit_util;
 
 use crate::arrow::converter::{
-    BinaryArrayConverter, BinaryConverter, BoolConverter, BooleanArrayConverter,
-    Converter, Date32Converter, FixedLenBinaryConverter, FixedSizeArrayConverter,
-    Float32Converter, Float64Converter, Int16Converter, Int32Converter, Int64Converter,
-    Int8Converter, Int96ArrayConverter, Int96Converter, TimestampMicrosecondConverter,
-    TimestampMillisecondConverter, UInt16Converter, UInt32Converter, UInt64Converter,
-    UInt8Converter, Utf8ArrayConverter, Utf8Converter,
+    BinaryArrayConverter, BinaryConverter, Converter, FixedLenBinaryConverter,
+    FixedSizeArrayConverter, Int96ArrayConverter, Int96Converter,
+    LargeBinaryArrayConverter, LargeBinaryConverter, LargeUtf8ArrayConverter,
+    LargeUtf8Converter, Utf8ArrayConverter, Utf8Converter,
 };
 use crate::arrow::record_reader::RecordReader;
 use crate::arrow::schema::parquet_to_arrow_field;
@@ -77,6 +99,97 @@ pub trait ArrayReader {
     fn get_rep_levels(&self) -> Option<&[i16]>;
 }
 
+/// A NullArrayReader reads Parquet columns stored as null int32s with an Arrow
+/// NullArray type.
+pub struct NullArrayReader<T: DataType> {
+    data_type: ArrowType,
+    pages: Box<dyn PageIterator>,
+    def_levels_buffer: Option<Buffer>,
+    rep_levels_buffer: Option<Buffer>,
+    column_desc: ColumnDescPtr,
+    record_reader: RecordReader<T>,
+    _type_marker: PhantomData<T>,
+}
+
+impl<T: DataType> NullArrayReader<T> {
+    /// Construct null array reader.
+    pub fn new(
+        mut pages: Box<dyn PageIterator>,
+        column_desc: ColumnDescPtr,
+    ) -> Result<Self> {
+        let mut record_reader = RecordReader::<T>::new(column_desc.clone());
+        if let Some(page_reader) = pages.next() {
+            record_reader.set_page_reader(page_reader?)?;
+        }
+
+        Ok(Self {
+            data_type: ArrowType::Null,
+            pages,
+            def_levels_buffer: None,
+            rep_levels_buffer: None,
+            column_desc,
+            record_reader,
+            _type_marker: PhantomData,
+        })
+    }
+}
+
+/// Implementation of primitive array reader.
+impl<T: DataType> ArrayReader for NullArrayReader<T> {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    /// Returns data type of primitive array.
+    fn get_data_type(&self) -> &ArrowType {
+        &self.data_type
+    }
+
+    /// Reads at most `batch_size` records into array.
+    fn next_batch(&mut self, batch_size: usize) -> Result<ArrayRef> {
+        let mut records_read = 0usize;
+        while records_read < batch_size {
+            let records_to_read = batch_size - records_read;
+
+            // NB can be 0 if at end of page
+            let records_read_once = self.record_reader.read_records(records_to_read)?;
+            records_read += records_read_once;
+
+            // Record reader exhausted
+            if records_read_once < records_to_read {
+                if let Some(page_reader) = self.pages.next() {
+                    // Read from new page reader
+                    self.record_reader.set_page_reader(page_reader?)?;
+                } else {
+                    // Page reader also exhausted
+                    break;
+                }
+            }
+        }
+
+        // convert to arrays
+        let array = arrow::array::NullArray::new(records_read);
+
+        // save definition and repetition buffers
+        self.def_levels_buffer = self.record_reader.consume_def_levels()?;
+        self.rep_levels_buffer = self.record_reader.consume_rep_levels()?;
+        self.record_reader.reset();
+        Ok(Arc::new(array))
+    }
+
+    fn get_def_levels(&self) -> Option<&[i16]> {
+        self.def_levels_buffer
+            .as_ref()
+            .map(|buf| unsafe { buf.typed_data() })
+    }
+
+    fn get_rep_levels(&self) -> Option<&[i16]> {
+        self.rep_levels_buffer
+            .as_ref()
+            .map(|buf| unsafe { buf.typed_data() })
+    }
+}
+
 /// Primitive array readers are leaves of array reader tree. They accept page iterator
 /// and read them into primitive arrays.
 pub struct PrimitiveArrayReader<T: DataType> {
@@ -94,17 +207,19 @@ impl<T: DataType> PrimitiveArrayReader<T> {
     pub fn new(
         mut pages: Box<dyn PageIterator>,
         column_desc: ColumnDescPtr,
+        arrow_type: Option<ArrowType>,
     ) -> Result<Self> {
-        let data_type = parquet_to_arrow_field(column_desc.as_ref())?
-            .data_type()
-            .clone();
+        // Check if Arrow type is specified, else create it from Parquet type
+        let data_type = match arrow_type {
+            Some(t) => t,
+            None => parquet_to_arrow_field(column_desc.as_ref())?
+                .data_type()
+                .clone(),
+        };
 
         let mut record_reader = RecordReader::<T>::new(column_desc.clone());
-        match pages.next() {
-            Some(page_reader) => {
-                record_reader.set_page_reader(page_reader?)?;
-            }
-            None => {}
+        if let Some(page_reader) = pages.next() {
+            record_reader.set_page_reader(page_reader?)?;
         }
 
         Ok(Self {
@@ -136,11 +251,9 @@ impl<T: DataType> ArrayReader for PrimitiveArrayReader<T> {
         while records_read < batch_size {
             let records_to_read = batch_size - records_read;
 
+            // NB can be 0 if at end of page
             let records_read_once = self.record_reader.read_records(records_to_read)?;
-            if records_read_once == 0 {
-                break; // record reader has no record
-            }
-            records_read = records_read + records_read_once;
+            records_read += records_read_once;
 
             // Record reader exhausted
             if records_read_once < records_to_read {
@@ -154,74 +267,79 @@ impl<T: DataType> ArrayReader for PrimitiveArrayReader<T> {
             }
         }
 
-        // convert to arrays
-        let array =
-            match (&self.data_type, T::get_physical_type()) {
-                (ArrowType::Boolean, PhysicalType::BOOLEAN) => {
-                    BoolConverter::new(BooleanArrayConverter {})
-                        .convert(self.record_reader.cast::<BoolType>())
-                }
-                (ArrowType::Int8, PhysicalType::INT32) => {
-                    Int8Converter::new().convert(self.record_reader.cast::<Int32Type>())
-                }
-                (ArrowType::Int16, PhysicalType::INT32) => {
-                    Int16Converter::new().convert(self.record_reader.cast::<Int32Type>())
-                }
-                (ArrowType::Int32, PhysicalType::INT32) => {
-                    Int32Converter::new().convert(self.record_reader.cast::<Int32Type>())
-                }
-                (ArrowType::UInt8, PhysicalType::INT32) => {
-                    UInt8Converter::new().convert(self.record_reader.cast::<Int32Type>())
-                }
-                (ArrowType::UInt16, PhysicalType::INT32) => {
-                    UInt16Converter::new().convert(self.record_reader.cast::<Int32Type>())
-                }
-                (ArrowType::UInt32, PhysicalType::INT32) => {
-                    UInt32Converter::new().convert(self.record_reader.cast::<Int32Type>())
-                }
-                (ArrowType::Int64, PhysicalType::INT64) => {
-                    Int64Converter::new().convert(self.record_reader.cast::<Int64Type>())
-                }
-                (ArrowType::UInt64, PhysicalType::INT64) => {
-                    UInt64Converter::new().convert(self.record_reader.cast::<Int64Type>())
-                }
-                (ArrowType::Float32, PhysicalType::FLOAT) => Float32Converter::new()
-                    .convert(self.record_reader.cast::<FloatType>()),
-                (ArrowType::Float64, PhysicalType::DOUBLE) => Float64Converter::new()
-                    .convert(self.record_reader.cast::<DoubleType>()),
-                (ArrowType::Timestamp(unit, _), PhysicalType::INT64) => match unit {
-                    TimeUnit::Millisecond => TimestampMillisecondConverter::new()
-                        .convert(self.record_reader.cast::<Int64Type>()),
-                    TimeUnit::Microsecond => TimestampMicrosecondConverter::new()
-                        .convert(self.record_reader.cast::<Int64Type>()),
-                    _ => Err(general_err!("No conversion from parquet type to arrow type for timestamp with unit {:?}", unit)),
-                },
-                (ArrowType::Date32(unit), PhysicalType::INT32) => match unit {
-                    DateUnit::Day => Date32Converter::new()
-                        .convert(self.record_reader.cast::<Int32Type>()),
-                    _ => Err(general_err!("No conversion from parquet type to arrow type for date with unit {:?}", unit)),
-                }
-                (ArrowType::Time32(_), PhysicalType::INT32) => {
-                    UInt32Converter::new().convert(self.record_reader.cast::<Int32Type>())
-                }
-                (ArrowType::Time64(_), PhysicalType::INT64) => {
-                    UInt64Converter::new().convert(self.record_reader.cast::<Int64Type>())
-                }
-                (ArrowType::Interval(IntervalUnit::YearMonth), PhysicalType::INT32) => {
-                    UInt32Converter::new().convert(self.record_reader.cast::<Int32Type>())
-                }
-                (ArrowType::Interval(IntervalUnit::DayTime), PhysicalType::INT64) => {
-                    UInt64Converter::new().convert(self.record_reader.cast::<Int64Type>())
-                }
-                (ArrowType::Duration(_), PhysicalType::INT64) => {
-                    UInt64Converter::new().convert(self.record_reader.cast::<Int64Type>())
-                }
-                (arrow_type, physical_type) => Err(general_err!(
-                    "Reading {:?} type from parquet {:?} is not supported yet.",
-                    arrow_type,
-                    physical_type
-                )),
-            }?;
+        let arrow_data_type = match T::get_physical_type() {
+            PhysicalType::BOOLEAN => ArrowBooleanType::DATA_TYPE,
+            PhysicalType::INT32 => ArrowInt32Type::DATA_TYPE,
+            PhysicalType::INT64 => ArrowInt64Type::DATA_TYPE,
+            PhysicalType::FLOAT => ArrowFloat32Type::DATA_TYPE,
+            PhysicalType::DOUBLE => ArrowFloat64Type::DATA_TYPE,
+            PhysicalType::INT96
+            | PhysicalType::BYTE_ARRAY
+            | PhysicalType::FIXED_LEN_BYTE_ARRAY => {
+                unreachable!(
+                    "PrimitiveArrayReaders don't support complex physical types"
+                );
+            }
+        };
+
+        // Convert to arrays by using the Parquet phyisical type.
+        // The physical types are then cast to Arrow types if necessary
+
+        let mut record_data = self.record_reader.consume_record_data()?;
+
+        if T::get_physical_type() == PhysicalType::BOOLEAN {
+            let mut boolean_buffer = BooleanBufferBuilder::new(record_data.len());
+
+            for e in record_data.data() {
+                boolean_buffer.append(*e > 0)?;
+            }
+            record_data = boolean_buffer.finish();
+        }
+
+        let mut array_data = ArrayDataBuilder::new(arrow_data_type)
+            .len(self.record_reader.num_values())
+            .add_buffer(record_data);
+
+        if let Some(b) = self.record_reader.consume_bitmap_buffer()? {
+            array_data = array_data.null_bit_buffer(b);
+        }
+
+        let array = match T::get_physical_type() {
+            PhysicalType::BOOLEAN => {
+                Arc::new(PrimitiveArray::<ArrowBooleanType>::from(array_data.build()))
+                    as ArrayRef
+            }
+            PhysicalType::INT32 => {
+                Arc::new(PrimitiveArray::<ArrowInt32Type>::from(array_data.build()))
+                    as ArrayRef
+            }
+            PhysicalType::INT64 => {
+                Arc::new(PrimitiveArray::<ArrowInt64Type>::from(array_data.build()))
+                    as ArrayRef
+            }
+            PhysicalType::FLOAT => {
+                Arc::new(PrimitiveArray::<ArrowFloat32Type>::from(array_data.build()))
+                    as ArrayRef
+            }
+            PhysicalType::DOUBLE => {
+                Arc::new(PrimitiveArray::<ArrowFloat64Type>::from(array_data.build()))
+                    as ArrayRef
+            }
+            PhysicalType::INT96
+            | PhysicalType::BYTE_ARRAY
+            | PhysicalType::FIXED_LEN_BYTE_ARRAY => {
+                unreachable!(
+                    "PrimitiveArrayReaders don't support complex physical types"
+                );
+            }
+        };
+
+        // cast to Arrow type
+        // TODO: we need to check if it's fine for this to be fallible.
+        // My assumption is that we can't get to an illegal cast as we can only
+        // generate types that are supported, because we'd have gotten them from
+        // the metadata which was written to the Parquet sink
+        let array = arrow::compute::cast(&array, self.get_data_type())?;
 
         // save definition and repetition buffers
         self.def_levels_buffer = self.record_reader.consume_def_levels()?;
@@ -371,18 +489,24 @@ where
                 })
                 .collect()
         } else {
-            data_buffer.into_iter().map(|t| Some(t)).collect()
+            data_buffer.into_iter().map(Some).collect()
         };
 
-        self.converter.convert(data)
+        let mut array = self.converter.convert(data)?;
+
+        if let ArrowType::Dictionary(_, _) = self.data_type {
+            array = arrow::compute::cast(&array, &self.data_type)?;
+        }
+
+        Ok(array)
     }
 
     fn get_def_levels(&self) -> Option<&[i16]> {
-        self.def_levels_buffer.as_ref().map(|t| t.as_slice())
+        self.def_levels_buffer.as_deref()
     }
 
     fn get_rep_levels(&self) -> Option<&[i16]> {
-        self.rep_levels_buffer.as_ref().map(|t| t.as_slice())
+        self.rep_levels_buffer.as_deref()
     }
 }
 
@@ -395,10 +519,14 @@ where
         pages: Box<dyn PageIterator>,
         column_desc: ColumnDescPtr,
         converter: C,
+        arrow_type: Option<ArrowType>,
     ) -> Result<Self> {
-        let data_type = parquet_to_arrow_field(column_desc.as_ref())?
-            .data_type()
-            .clone();
+        let data_type = match arrow_type {
+            Some(t) => t,
+            None => parquet_to_arrow_field(column_desc.as_ref())?
+                .data_type()
+                .clone(),
+        };
 
         Ok(Self {
             data_type,
@@ -425,6 +553,400 @@ where
     }
 }
 
+/// Implementation of list array reader.
+pub struct ListArrayReader<OffsetSize: OffsetSizeTrait> {
+    item_reader: Box<dyn ArrayReader>,
+    data_type: ArrowType,
+    item_type: ArrowType,
+    list_def_level: i16,
+    list_rep_level: i16,
+    def_level_buffer: Option<Buffer>,
+    rep_level_buffer: Option<Buffer>,
+    _marker: PhantomData<OffsetSize>,
+}
+
+impl<OffsetSize: OffsetSizeTrait> ListArrayReader<OffsetSize> {
+    /// Construct list array reader.
+    pub fn new(
+        item_reader: Box<dyn ArrayReader>,
+        data_type: ArrowType,
+        item_type: ArrowType,
+        def_level: i16,
+        rep_level: i16,
+    ) -> Self {
+        Self {
+            item_reader,
+            data_type,
+            item_type,
+            list_def_level: def_level,
+            list_rep_level: rep_level,
+            def_level_buffer: None,
+            rep_level_buffer: None,
+            _marker: PhantomData,
+        }
+    }
+}
+
+macro_rules! build_empty_list_array_with_primitive_items {
+    ($item_type:ident) => {{
+        let values_builder = PrimitiveBuilder::<$item_type>::new(0);
+        let mut builder = ListBuilder::new(values_builder);
+        let empty_list_array = builder.finish();
+        Ok(Arc::new(empty_list_array))
+    }};
+}
+
+macro_rules! build_empty_list_array_with_non_primitive_items {
+    ($builder:ident) => {{
+        let values_builder = $builder::new(0);
+        let mut builder = ListBuilder::new(values_builder);
+        let empty_list_array = builder.finish();
+        Ok(Arc::new(empty_list_array))
+    }};
+}
+
+fn build_empty_list_array(item_type: ArrowType) -> Result<ArrayRef> {
+    match item_type {
+        ArrowType::UInt8 => build_empty_list_array_with_primitive_items!(ArrowUInt8Type),
+        ArrowType::UInt16 => {
+            build_empty_list_array_with_primitive_items!(ArrowUInt16Type)
+        }
+        ArrowType::UInt32 => {
+            build_empty_list_array_with_primitive_items!(ArrowUInt32Type)
+        }
+        ArrowType::UInt64 => {
+            build_empty_list_array_with_primitive_items!(ArrowUInt64Type)
+        }
+        ArrowType::Int8 => build_empty_list_array_with_primitive_items!(ArrowInt8Type),
+        ArrowType::Int16 => build_empty_list_array_with_primitive_items!(ArrowInt16Type),
+        ArrowType::Int32 => build_empty_list_array_with_primitive_items!(ArrowInt32Type),
+        ArrowType::Int64 => build_empty_list_array_with_primitive_items!(ArrowInt64Type),
+        ArrowType::Float32 => {
+            build_empty_list_array_with_primitive_items!(ArrowFloat32Type)
+        }
+        ArrowType::Float64 => {
+            build_empty_list_array_with_primitive_items!(ArrowFloat64Type)
+        }
+        ArrowType::Boolean => {
+            build_empty_list_array_with_primitive_items!(ArrowBooleanType)
+        }
+        ArrowType::Date32(_) => {
+            build_empty_list_array_with_primitive_items!(ArrowDate32Type)
+        }
+        ArrowType::Date64(_) => {
+            build_empty_list_array_with_primitive_items!(ArrowDate64Type)
+        }
+        ArrowType::Time32(ArrowTimeUnit::Second) => {
+            build_empty_list_array_with_primitive_items!(ArrowTime32SecondType)
+        }
+        ArrowType::Time32(ArrowTimeUnit::Millisecond) => {
+            build_empty_list_array_with_primitive_items!(ArrowTime32MillisecondType)
+        }
+        ArrowType::Time64(ArrowTimeUnit::Microsecond) => {
+            build_empty_list_array_with_primitive_items!(ArrowTime64MicrosecondType)
+        }
+        ArrowType::Time64(ArrowTimeUnit::Nanosecond) => {
+            build_empty_list_array_with_primitive_items!(ArrowTime64NanosecondType)
+        }
+        ArrowType::Duration(ArrowTimeUnit::Second) => {
+            build_empty_list_array_with_primitive_items!(ArrowDurationSecondType)
+        }
+        ArrowType::Duration(ArrowTimeUnit::Millisecond) => {
+            build_empty_list_array_with_primitive_items!(ArrowDurationMillisecondType)
+        }
+        ArrowType::Duration(ArrowTimeUnit::Microsecond) => {
+            build_empty_list_array_with_primitive_items!(ArrowDurationMicrosecondType)
+        }
+        ArrowType::Duration(ArrowTimeUnit::Nanosecond) => {
+            build_empty_list_array_with_primitive_items!(ArrowDurationNanosecondType)
+        }
+        ArrowType::Timestamp(ArrowTimeUnit::Second, _) => {
+            build_empty_list_array_with_primitive_items!(ArrowTimestampSecondType)
+        }
+        ArrowType::Timestamp(ArrowTimeUnit::Millisecond, _) => {
+            build_empty_list_array_with_primitive_items!(ArrowTimestampMillisecondType)
+        }
+        ArrowType::Timestamp(ArrowTimeUnit::Microsecond, _) => {
+            build_empty_list_array_with_primitive_items!(ArrowTimestampMicrosecondType)
+        }
+        ArrowType::Timestamp(ArrowTimeUnit::Nanosecond, _) => {
+            build_empty_list_array_with_primitive_items!(ArrowTimestampNanosecondType)
+        }
+        ArrowType::Utf8 => {
+            build_empty_list_array_with_non_primitive_items!(StringBuilder)
+        }
+        ArrowType::Binary => {
+            build_empty_list_array_with_non_primitive_items!(BinaryBuilder)
+        }
+        _ => Err(ParquetError::General(format!(
+            "ListArray of type List({:?}) is not supported by array_reader",
+            item_type
+        ))),
+    }
+}
+
+macro_rules! remove_primitive_array_indices {
+    ($arr: expr, $item_type:ty, $indices:expr) => {{
+        let array_data = match $arr.as_any().downcast_ref::<PrimitiveArray<$item_type>>() {
+            Some(a) => a,
+            _ => return Err(ParquetError::General(format!("Error generating next batch for ListArray: {:?} cannot be downcast to PrimitiveArray", $arr))),
+        };
+        let mut builder = PrimitiveBuilder::<$item_type>::new($arr.len());
+        for i in 0..array_data.len() {
+            if !$indices.contains(&i) {
+                if array_data.is_null(i) {
+                    builder.append_null()?;
+                } else {
+                    builder.append_value(array_data.value(i))?;
+                }
+            }
+        }
+        Ok(Arc::new(builder.finish()))
+    }};
+}
+
+macro_rules! remove_array_indices_custom_builder {
+    ($arr: expr, $array_type:ty, $item_builder:ident, $indices:expr) => {{
+        let array_data = match $arr.as_any().downcast_ref::<$array_type>() {
+            Some(a) => a,
+            _ => return Err(ParquetError::General(format!("Error generating next batch for ListArray: {:?} cannot be downcast to PrimitiveArray", $arr))),
+        };
+        let mut builder = $item_builder::new(array_data.len());
+
+        for i in 0..array_data.len() {
+            if !$indices.contains(&i) {
+                if array_data.is_null(i) {
+                    builder.append_null()?;
+                } else {
+                    builder.append_value(array_data.value(i))?;
+                }
+            }
+        }
+        Ok(Arc::new(builder.finish()))
+    }};
+}
+
+macro_rules! remove_fixed_size_binary_array_indices {
+    ($arr: expr, $array_type:ty, $item_builder:ident, $indices:expr, $len:expr) => {{
+        let array_data = match $arr.as_any().downcast_ref::<$array_type>() {
+            Some(a) => a,
+            _ => return Err(ParquetError::General(format!("Error generating next batch for ListArray: {:?} cannot be downcast to PrimitiveArray", $arr))),
+        };
+        let mut builder = FixedSizeBinaryBuilder::new(array_data.len(), $len);
+        for i in 0..array_data.len() {
+            if !$indices.contains(&i) {
+                if array_data.is_null(i) {
+                    builder.append_null()?;
+                } else {
+                    builder.append_value(array_data.value(i))?;
+                }
+            }
+        }
+        Ok(Arc::new(builder.finish()))
+    }};
+}
+
+fn remove_indices(
+    arr: ArrayRef,
+    item_type: ArrowType,
+    indices: Vec<usize>,
+) -> Result<ArrayRef> {
+    match item_type {
+        ArrowType::UInt8 => remove_primitive_array_indices!(arr, ArrowUInt8Type, indices),
+        ArrowType::UInt16 => {
+            remove_primitive_array_indices!(arr, ArrowUInt16Type, indices)
+        }
+        ArrowType::UInt32 => {
+            remove_primitive_array_indices!(arr, ArrowUInt32Type, indices)
+        }
+        ArrowType::UInt64 => {
+            remove_primitive_array_indices!(arr, ArrowUInt64Type, indices)
+        }
+        ArrowType::Int8 => remove_primitive_array_indices!(arr, ArrowInt8Type, indices),
+        ArrowType::Int16 => remove_primitive_array_indices!(arr, ArrowInt16Type, indices),
+        ArrowType::Int32 => remove_primitive_array_indices!(arr, ArrowInt32Type, indices),
+        ArrowType::Int64 => remove_primitive_array_indices!(arr, ArrowInt64Type, indices),
+        ArrowType::Float32 => {
+            remove_primitive_array_indices!(arr, ArrowFloat32Type, indices)
+        }
+        ArrowType::Float64 => {
+            remove_primitive_array_indices!(arr, ArrowFloat64Type, indices)
+        }
+        ArrowType::Boolean => {
+            remove_primitive_array_indices!(arr, ArrowBooleanType, indices)
+        }
+        ArrowType::Date32(_) => {
+            remove_primitive_array_indices!(arr, ArrowDate32Type, indices)
+        }
+        ArrowType::Date64(_) => {
+            remove_primitive_array_indices!(arr, ArrowDate64Type, indices)
+        }
+        ArrowType::Time32(ArrowTimeUnit::Second) => {
+            remove_primitive_array_indices!(arr, ArrowTime32SecondType, indices)
+        }
+        ArrowType::Time32(ArrowTimeUnit::Millisecond) => {
+            remove_primitive_array_indices!(arr, ArrowTime32MillisecondType, indices)
+        }
+        ArrowType::Time64(ArrowTimeUnit::Microsecond) => {
+            remove_primitive_array_indices!(arr, ArrowTime64MicrosecondType, indices)
+        }
+        ArrowType::Time64(ArrowTimeUnit::Nanosecond) => {
+            remove_primitive_array_indices!(arr, ArrowTime64NanosecondType, indices)
+        }
+        ArrowType::Duration(ArrowTimeUnit::Second) => {
+            remove_primitive_array_indices!(arr, ArrowDurationSecondType, indices)
+        }
+        ArrowType::Duration(ArrowTimeUnit::Millisecond) => {
+            remove_primitive_array_indices!(arr, ArrowDurationMillisecondType, indices)
+        }
+        ArrowType::Duration(ArrowTimeUnit::Microsecond) => {
+            remove_primitive_array_indices!(arr, ArrowDurationMicrosecondType, indices)
+        }
+        ArrowType::Duration(ArrowTimeUnit::Nanosecond) => {
+            remove_primitive_array_indices!(arr, ArrowDurationNanosecondType, indices)
+        }
+        ArrowType::Timestamp(ArrowTimeUnit::Second, _) => {
+            remove_primitive_array_indices!(arr, ArrowTimestampSecondType, indices)
+        }
+        ArrowType::Timestamp(ArrowTimeUnit::Millisecond, _) => {
+            remove_primitive_array_indices!(arr, ArrowTimestampMillisecondType, indices)
+        }
+        ArrowType::Timestamp(ArrowTimeUnit::Microsecond, _) => {
+            remove_primitive_array_indices!(arr, ArrowTimestampMicrosecondType, indices)
+        }
+        ArrowType::Timestamp(ArrowTimeUnit::Nanosecond, _) => {
+            remove_primitive_array_indices!(arr, ArrowTimestampNanosecondType, indices)
+        }
+        ArrowType::Utf8 => {
+            remove_array_indices_custom_builder!(arr, StringArray, StringBuilder, indices)
+        }
+        ArrowType::Binary => {
+            remove_array_indices_custom_builder!(arr, BinaryArray, BinaryBuilder, indices)
+        }
+        ArrowType::FixedSizeBinary(size) => remove_fixed_size_binary_array_indices!(
+            arr,
+            FixedSizeBinaryArray,
+            FixedSizeBinaryBuilder,
+            indices,
+            size
+        ),
+        _ => Err(ParquetError::General(format!(
+            "ListArray of type List({:?}) is not supported by array_reader",
+            item_type
+        ))),
+    }
+}
+
+/// Implementation of ListArrayReader. Nested lists and lists of structs are not yet supported.
+impl<OffsetSize: OffsetSizeTrait> ArrayReader for ListArrayReader<OffsetSize> {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    /// Returns data type.
+    /// This must be a List.
+    fn get_data_type(&self) -> &ArrowType {
+        &self.data_type
+    }
+
+    fn next_batch(&mut self, batch_size: usize) -> Result<ArrayRef> {
+        let next_batch_array = self.item_reader.next_batch(batch_size)?;
+        let item_type = self.item_reader.get_data_type().clone();
+
+        if next_batch_array.len() == 0 {
+            return build_empty_list_array(item_type);
+        }
+        let def_levels = self
+            .item_reader
+            .get_def_levels()
+            .ok_or_else(|| ArrowError("item_reader def levels are None.".to_string()))?;
+        let rep_levels = self
+            .item_reader
+            .get_rep_levels()
+            .ok_or_else(|| ArrowError("item_reader rep levels are None.".to_string()))?;
+
+        if !((def_levels.len() == rep_levels.len())
+            && (rep_levels.len() == next_batch_array.len()))
+        {
+            return Err(ArrowError(
+                "Expected item_reader def_levels and rep_levels to be same length as batch".to_string(),
+            ));
+        }
+
+        // Need to remove from the values array the nulls that represent null lists rather than null items
+        // null lists have def_level = 0
+        let mut null_list_indices: Vec<usize> = Vec::new();
+        for i in 0..def_levels.len() {
+            if def_levels[i] == 0 {
+                null_list_indices.push(i);
+            }
+        }
+        let batch_values = match null_list_indices.len() {
+            0 => next_batch_array.clone(),
+            _ => remove_indices(next_batch_array.clone(), item_type, null_list_indices)?,
+        };
+
+        // null list has def_level = 0
+        // empty list has def_level = 1
+        // null item in a list has def_level = 2
+        // non-null item has def_level = 3
+        // first item in each list has rep_level = 0, subsequent items have rep_level = 1
+
+        let mut offsets: Vec<OffsetSize> = Vec::new();
+        let mut cur_offset = OffsetSize::zero();
+        for i in 0..rep_levels.len() {
+            if rep_levels[i] == 0 {
+                offsets.push(cur_offset)
+            }
+            if def_levels[i] > 0 {
+                cur_offset = cur_offset + OffsetSize::one();
+            }
+        }
+        offsets.push(cur_offset);
+
+        let num_bytes = bit_util::ceil(offsets.len(), 8);
+        let mut null_buf = MutableBuffer::new(num_bytes).with_bitset(num_bytes, false);
+        let null_slice = null_buf.data_mut();
+        let mut list_index = 0;
+        for i in 0..rep_levels.len() {
+            if rep_levels[i] == 0 && def_levels[i] != 0 {
+                bit_util::set_bit(null_slice, list_index);
+            }
+            if rep_levels[i] == 0 {
+                list_index += 1;
+            }
+        }
+        let value_offsets = Buffer::from(&offsets.to_byte_slice());
+
+        // null list has def_level = 0
+        let null_count = def_levels.iter().filter(|x| x == &&0).count();
+
+        let list_data = ArrayData::builder(self.get_data_type().clone())
+            .len(offsets.len() - 1)
+            .add_buffer(value_offsets)
+            .add_child_data(batch_values.data())
+            .null_bit_buffer(null_buf.freeze())
+            .null_count(null_count)
+            .offset(next_batch_array.offset())
+            .build();
+
+        let result_array = GenericListArray::<OffsetSize>::from(list_data);
+        Ok(Arc::new(result_array))
+    }
+
+    fn get_def_levels(&self) -> Option<&[i16]> {
+        self.def_level_buffer
+            .as_ref()
+            .map(|buf| unsafe { buf.typed_data() })
+    }
+
+    fn get_rep_levels(&self) -> Option<&[i16]> {
+        self.rep_level_buffer
+            .as_ref()
+            .map(|buf| unsafe { buf.typed_data() })
+    }
+}
+
 /// Implementation of struct array reader.
 pub struct StructArrayReader {
     children: Vec<Box<dyn ArrayReader>>,
@@ -483,7 +1005,7 @@ impl ArrayReader for StructArrayReader {
     /// null_bitmap[i] = (def_levels[i] >= self.def_level);
     /// ```
     fn next_batch(&mut self, batch_size: usize) -> Result<ArrayRef> {
-        if self.children.len() == 0 {
+        if self.children.is_empty() {
             self.def_level_buffer = None;
             self.rep_level_buffer = None;
             return Ok(Arc::new(StructArray::from(Vec::new())));
@@ -600,6 +1122,7 @@ impl ArrayReader for StructArrayReader {
 /// Create array reader from parquet schema, column indices, and parquet file reader.
 pub fn build_array_reader<T>(
     parquet_schema: SchemaDescPtr,
+    arrow_schema: Schema,
     column_indices: T,
     file_reader: Rc<dyn FileReader>,
 ) -> Result<Box<dyn ArrayReader>>
@@ -628,9 +1151,9 @@ where
     let filtered_root_fields = parquet_schema
         .root_schema()
         .get_fields()
-        .into_iter()
+        .iter()
         .filter(|field| filtered_root_names.contains(field.name()))
-        .map(|field| field.clone())
+        .cloned()
         .collect::<Vec<_>>();
 
     let proj = Type::GroupType {
@@ -638,13 +1161,19 @@ where
         fields: filtered_root_fields,
     };
 
-    ArrayReaderBuilder::new(Rc::new(proj), Rc::new(leaves), file_reader)
-        .build_array_reader()
+    ArrayReaderBuilder::new(
+        Rc::new(proj),
+        Rc::new(arrow_schema),
+        Rc::new(leaves),
+        file_reader,
+    )
+    .build_array_reader()
 }
 
 /// Used to build array reader.
 struct ArrayReaderBuilder {
     root_schema: TypePtr,
+    arrow_schema: Rc<Schema>,
     // Key: columns that need to be included in final array builder
     // Value: column index in schema
     columns_included: Rc<HashMap<*const Type, usize>>,
@@ -761,16 +1290,98 @@ impl<'a> TypeVisitor<Option<Box<dyn ArrayReader>>, &'a ArrayReaderBuilderContext
     }
 
     /// Build array reader for list type.
-    /// Currently this is not supported.
     fn visit_list_with_item(
         &mut self,
-        _list_type: Rc<Type>,
-        _item_type: &Type,
-        _context: &'a ArrayReaderBuilderContext,
+        list_type: Rc<Type>,
+        item_type: Rc<Type>,
+        context: &'a ArrayReaderBuilderContext,
     ) -> Result<Option<Box<dyn ArrayReader>>> {
-        Err(ArrowError(
-            "Reading parquet list array into arrow is not supported yet!".to_string(),
-        ))
+        let list_child = &list_type
+            .get_fields()
+            .first()
+            .ok_or_else(|| ArrowError("List field must have a child.".to_string()))?;
+        let mut new_context = context.clone();
+
+        new_context.path.append(vec![list_type.name().to_string()]);
+
+        match list_type.get_basic_info().repetition() {
+            Repetition::REPEATED => {
+                new_context.def_level += 1;
+                new_context.rep_level += 1;
+            }
+            Repetition::OPTIONAL => {
+                new_context.def_level += 1;
+            }
+            _ => (),
+        }
+
+        match list_child.get_basic_info().repetition() {
+            Repetition::REPEATED => {
+                new_context.def_level += 1;
+                new_context.rep_level += 1;
+            }
+            Repetition::OPTIONAL => {
+                new_context.def_level += 1;
+            }
+            _ => (),
+        }
+
+        let item_reader = self
+            .dispatch(item_type.clone(), &new_context)
+            .unwrap()
+            .unwrap();
+
+        let item_reader_type = item_reader.get_data_type().clone();
+
+        match item_reader_type {
+            ArrowType::List(_)
+            | ArrowType::FixedSizeList(_, _)
+            | ArrowType::Struct(_)
+            | ArrowType::Dictionary(_, _) => Err(ArrowError(format!(
+                "reading List({:?}) into arrow not supported yet",
+                item_type
+            ))),
+            _ => {
+                let arrow_type = self
+                    .arrow_schema
+                    .field_with_name(list_type.name())
+                    .ok()
+                    .map(|f| f.data_type().to_owned())
+                    .unwrap_or_else(|| {
+                        ArrowType::List(Box::new(Field::new(
+                            list_type.name(),
+                            item_reader_type.clone(),
+                            list_type.is_optional(),
+                        )))
+                    });
+
+                let list_array_reader: Box<dyn ArrayReader> = match arrow_type {
+                    ArrowType::List(_) => Box::new(ListArrayReader::<i32>::new(
+                        item_reader,
+                        arrow_type,
+                        item_reader_type,
+                        new_context.def_level,
+                        new_context.rep_level,
+                    )),
+                    ArrowType::LargeList(_) => Box::new(ListArrayReader::<i64>::new(
+                        item_reader,
+                        arrow_type,
+                        item_reader_type,
+                        new_context.def_level,
+                        new_context.rep_level,
+                    )),
+
+                    _ => {
+                        return Err(ArrowError(format!(
+                        "creating ListArrayReader with type {:?} should be unreachable",
+                        arrow_type
+                    )))
+                    }
+                };
+
+                Ok(Some(list_array_reader))
+            }
+        }
     }
 }
 
@@ -778,11 +1389,13 @@ impl<'a> ArrayReaderBuilder {
     /// Construct array reader builder.
     fn new(
         root_schema: TypePtr,
+        arrow_schema: Rc<Schema>,
         columns_included: Rc<HashMap<*const Type, usize>>,
         file_reader: Rc<dyn FileReader>,
     ) -> Self {
         Self {
             root_schema,
+            arrow_schema,
             columns_included,
             file_reader,
         }
@@ -813,7 +1426,6 @@ impl<'a> ArrayReaderBuilder {
     ) -> Result<Box<dyn ArrayReader>> {
         let column_desc = Rc::new(ColumnDescriptor::new(
             cur_type.clone(),
-            Some(self.root_schema.clone()),
             context.def_level,
             context.rep_level,
             context.path.clone(),
@@ -823,18 +1435,37 @@ impl<'a> ArrayReaderBuilder {
             self.file_reader.clone(),
         )?);
 
+        let arrow_type = self
+            .arrow_schema
+            .field_with_name(cur_type.name())
+            .ok()
+            .map(|f| f.data_type())
+            .cloned();
+
         match cur_type.get_physical_type() {
             PhysicalType::BOOLEAN => Ok(Box::new(PrimitiveArrayReader::<BoolType>::new(
                 page_iterator,
                 column_desc,
+                arrow_type,
             )?)),
-            PhysicalType::INT32 => Ok(Box::new(PrimitiveArrayReader::<Int32Type>::new(
-                page_iterator,
-                column_desc,
-            )?)),
+            PhysicalType::INT32 => {
+                if let Some(ArrowType::Null) = arrow_type {
+                    Ok(Box::new(NullArrayReader::<Int32Type>::new(
+                        page_iterator,
+                        column_desc,
+                    )?))
+                } else {
+                    Ok(Box::new(PrimitiveArrayReader::<Int32Type>::new(
+                        page_iterator,
+                        column_desc,
+                        arrow_type,
+                    )?))
+                }
+            }
             PhysicalType::INT64 => Ok(Box::new(PrimitiveArrayReader::<Int64Type>::new(
                 page_iterator,
                 column_desc,
+                arrow_type,
             )?)),
             PhysicalType::INT96 => {
                 let converter = Int96Converter::new(Int96ArrayConverter {});
@@ -842,24 +1473,61 @@ impl<'a> ArrayReaderBuilder {
                     Int96Type,
                     Int96Converter,
                 >::new(
-                    page_iterator, column_desc, converter
+                    page_iterator,
+                    column_desc,
+                    converter,
+                    arrow_type,
                 )?))
             }
             PhysicalType::FLOAT => Ok(Box::new(PrimitiveArrayReader::<FloatType>::new(
                 page_iterator,
                 column_desc,
+                arrow_type,
             )?)),
-            PhysicalType::DOUBLE => Ok(Box::new(
-                PrimitiveArrayReader::<DoubleType>::new(page_iterator, column_desc)?,
-            )),
+            PhysicalType::DOUBLE => {
+                Ok(Box::new(PrimitiveArrayReader::<DoubleType>::new(
+                    page_iterator,
+                    column_desc,
+                    arrow_type,
+                )?))
+            }
             PhysicalType::BYTE_ARRAY => {
                 if cur_type.get_basic_info().logical_type() == LogicalType::UTF8 {
-                    let converter = Utf8Converter::new(Utf8ArrayConverter {});
+                    if let Some(ArrowType::LargeUtf8) = arrow_type {
+                        let converter =
+                            LargeUtf8Converter::new(LargeUtf8ArrayConverter {});
+                        Ok(Box::new(ComplexObjectArrayReader::<
+                            ByteArrayType,
+                            LargeUtf8Converter,
+                        >::new(
+                            page_iterator,
+                            column_desc,
+                            converter,
+                            arrow_type,
+                        )?))
+                    } else {
+                        let converter = Utf8Converter::new(Utf8ArrayConverter {});
+                        Ok(Box::new(ComplexObjectArrayReader::<
+                            ByteArrayType,
+                            Utf8Converter,
+                        >::new(
+                            page_iterator,
+                            column_desc,
+                            converter,
+                            arrow_type,
+                        )?))
+                    }
+                } else if let Some(ArrowType::LargeBinary) = arrow_type {
+                    let converter =
+                        LargeBinaryConverter::new(LargeBinaryArrayConverter {});
                     Ok(Box::new(ComplexObjectArrayReader::<
                         ByteArrayType,
-                        Utf8Converter,
+                        LargeBinaryConverter,
                     >::new(
-                        page_iterator, column_desc, converter
+                        page_iterator,
+                        column_desc,
+                        converter,
+                        arrow_type,
                     )?))
                 } else {
                     let converter = BinaryConverter::new(BinaryArrayConverter {});
@@ -867,7 +1535,10 @@ impl<'a> ArrayReaderBuilder {
                         ByteArrayType,
                         BinaryConverter,
                     >::new(
-                        page_iterator, column_desc, converter
+                        page_iterator,
+                        column_desc,
+                        converter,
+                        arrow_type,
                     )?))
                 }
             }
@@ -877,9 +1548,9 @@ impl<'a> ArrayReaderBuilder {
                         ref type_length, ..
                     } => *type_length,
                     _ => {
-                        return Err(ArrowError(format!(
-                            "Expected a physical type, not a group type"
-                        )))
+                        return Err(ArrowError(
+                            "Expected a physical type, not a group type".to_string(),
+                        ))
                     }
                 };
                 let converter = FixedLenBinaryConverter::new(
@@ -889,7 +1560,10 @@ impl<'a> ArrayReaderBuilder {
                     FixedLenByteArrayType,
                     FixedLenBinaryConverter,
                 >::new(
-                    page_iterator, column_desc, converter
+                    page_iterator,
+                    column_desc,
+                    converter,
+                    arrow_type,
                 )?))
             }
         }
@@ -906,11 +1580,15 @@ impl<'a> ArrayReaderBuilder {
 
         for child in cur_type.get_fields() {
             if let Some(child_reader) = self.dispatch(child.clone(), context)? {
-                fields.push(Field::new(
-                    child.name(),
-                    child_reader.get_data_type().clone(),
-                    child.is_optional(),
-                ));
+                let field = match self.arrow_schema.field_with_name(child.name()) {
+                    Ok(f) => f.to_owned(),
+                    _ => Field::new(
+                        child.name(),
+                        child_reader.get_data_type().clone(),
+                        child.is_optional(),
+                    ),
+                };
+                fields.push(field);
                 children_reader.push(child_reader);
             }
         }
@@ -933,8 +1611,9 @@ impl<'a> ArrayReaderBuilder {
 mod tests {
     use super::*;
     use crate::arrow::converter::Utf8Converter;
+    use crate::arrow::schema::parquet_to_arrow_schema;
     use crate::basic::{Encoding, Type as PhysicalType};
-    use crate::column::page::Page;
+    use crate::column::page::{Page, PageReader};
     use crate::data_type::{ByteArray, DataType, Int32Type, Int64Type};
     use crate::errors::Result;
     use crate::file::reader::{FileReader, SerializedFileReader};
@@ -944,12 +1623,17 @@ mod tests {
         DataPageBuilder, DataPageBuilderImpl, InMemoryPageIterator,
     };
     use crate::util::test_common::{get_test_file, make_pages};
-    use arrow::array::{Array, ArrayRef, PrimitiveArray, StringArray, StructArray};
+    use arrow::array::{
+        Array, ArrayRef, LargeListArray, ListArray, PrimitiveArray, StringArray,
+        StructArray,
+    };
     use arrow::datatypes::{
-        DataType as ArrowType, Date32Type as ArrowDate32, Field, Int32Type as ArrowInt32,
+        ArrowPrimitiveType, DataType as ArrowType, Date32Type as ArrowDate32, Field,
+        Int32Type as ArrowInt32, Int64Type as ArrowInt64,
+        Time32MillisecondType as ArrowTime32MillisecondArray,
+        Time64MicrosecondType as ArrowTime64MicrosecondArray,
         TimestampMicrosecondType as ArrowTimestampMicrosecondType,
         TimestampMillisecondType as ArrowTimestampMillisecondType,
-        UInt32Type as ArrowUInt32, UInt64Type as ArrowUInt64,
     };
     use rand::distributions::uniform::SampleUniform;
     use rand::{thread_rng, Rng};
@@ -958,7 +1642,7 @@ mod tests {
     use std::rc::Rc;
     use std::sync::Arc;
 
-    fn make_column_chuncks<T: DataType>(
+    fn make_column_chunks<T: DataType>(
         column_desc: ColumnDescPtr,
         encoding: Encoding,
         num_levels: usize,
@@ -969,11 +1653,11 @@ mod tests {
         values: &mut Vec<T::T>,
         page_lists: &mut Vec<Vec<Page>>,
         use_v2: bool,
-        num_chuncks: usize,
+        num_chunks: usize,
     ) where
         T::T: PartialOrd + SampleUniform + Copy,
     {
-        for _i in 0..num_chuncks {
+        for _i in 0..num_chunks {
             let mut pages = VecDeque::new();
             let mut data = Vec::new();
             let mut page_def_levels = Vec::new();
@@ -1000,6 +1684,34 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_primitive_array_reader_empty_pages() {
+        // Construct column schema
+        let message_type = "
+        message test_schema {
+          REQUIRED INT32 leaf;
+        }
+        ";
+
+        let schema = parse_message_type(message_type)
+            .map(|t| Rc::new(SchemaDescriptor::new(Rc::new(t))))
+            .unwrap();
+
+        let column_desc = schema.column(0);
+        let page_iterator = EmptyPageIterator::new(schema);
+
+        let mut array_reader = PrimitiveArrayReader::<Int32Type>::new(
+            Box::new(page_iterator),
+            column_desc,
+            None,
+        )
+        .unwrap();
+
+        // expect no values to be read
+        let array = array_reader.next_batch(50).unwrap();
+        assert!(array.is_empty());
+    }
+
     #[test]
     fn test_primitive_array_reader_data() {
         // Construct column schema
@@ -1019,7 +1731,7 @@ mod tests {
         {
             let mut data = Vec::new();
             let mut page_lists = Vec::new();
-            make_column_chuncks::<Int32Type>(
+            make_column_chunks::<Int32Type>(
                 column_desc.clone(),
                 Encoding::PLAIN,
                 100,
@@ -1032,19 +1744,17 @@ mod tests {
                 true,
                 2,
             );
-            let page_iterator = InMemoryPageIterator::new(
-                schema.clone(),
-                column_desc.clone(),
-                page_lists,
-            );
+            let page_iterator =
+                InMemoryPageIterator::new(schema, column_desc.clone(), page_lists);
 
             let mut array_reader = PrimitiveArrayReader::<Int32Type>::new(
                 Box::new(page_iterator),
-                column_desc.clone(),
+                column_desc,
+                None,
             )
             .unwrap();
 
-            // Read first 50 values, which are all from the first column chunck
+            // Read first 50 values, which are all from the first column chunk
             let array = array_reader.next_batch(50).unwrap();
             let array = array
                 .as_any()
@@ -1052,9 +1762,7 @@ mod tests {
                 .unwrap();
 
             assert_eq!(
-                &PrimitiveArray::<ArrowInt32>::from(
-                    data[0..50].iter().cloned().collect::<Vec<i32>>()
-                ),
+                &PrimitiveArray::<ArrowInt32>::from(data[0..50].to_vec()),
                 array
             );
 
@@ -1067,9 +1775,7 @@ mod tests {
                 .unwrap();
 
             assert_eq!(
-                &PrimitiveArray::<ArrowInt32>::from(
-                    data[50..150].iter().cloned().collect::<Vec<i32>>()
-                ),
+                &PrimitiveArray::<ArrowInt32>::from(data[50..150].to_vec()),
                 array
             );
 
@@ -1081,16 +1787,14 @@ mod tests {
                 .unwrap();
 
             assert_eq!(
-                &PrimitiveArray::<ArrowInt32>::from(
-                    data[150..200].iter().cloned().collect::<Vec<i32>>()
-                ),
+                &PrimitiveArray::<ArrowInt32>::from(data[150..200].to_vec()),
                 array
             );
         }
     }
 
     macro_rules! test_primitive_array_reader_one_type {
-        ($arrow_parquet_type:ty, $physical_type:expr, $logical_type_str:expr, $result_arrow_type:ty, $result_primitive_type:ty) => {{
+        ($arrow_parquet_type:ty, $physical_type:expr, $logical_type_str:expr, $result_arrow_type:ty, $result_arrow_cast_type:ty, $result_primitive_type:ty) => {{
             let message_type = format!(
                 "
             message test_schema {{
@@ -1101,7 +1805,7 @@ mod tests {
             );
             let schema = parse_message_type(&message_type)
                 .map(|t| Rc::new(SchemaDescriptor::new(Rc::new(t))))
-                .unwrap();
+                .expect("Unable to parse message type into a schema descriptor");
 
             let column_desc = schema.column(0);
 
@@ -1109,7 +1813,7 @@ mod tests {
             {
                 let mut data = Vec::new();
                 let mut page_lists = Vec::new();
-                make_column_chuncks::<$arrow_parquet_type>(
+                make_column_chunks::<$arrow_parquet_type>(
                     column_desc.clone(),
                     Encoding::PLAIN,
                     100,
@@ -1130,25 +1834,50 @@ mod tests {
                 let mut array_reader = PrimitiveArrayReader::<$arrow_parquet_type>::new(
                     Box::new(page_iterator),
                     column_desc.clone(),
+                    None,
                 )
-                .unwrap();
+                .expect("Unable to get array reader");
 
-                let array = array_reader.next_batch(50).unwrap();
+                let array = array_reader
+                    .next_batch(50)
+                    .expect("Unable to get batch from reader");
 
+                let result_data_type = <$result_arrow_type>::DATA_TYPE;
                 let array = array
                     .as_any()
                     .downcast_ref::<PrimitiveArray<$result_arrow_type>>()
-                    .unwrap();
-
-                assert_eq!(
-                    &PrimitiveArray::<$result_arrow_type>::from(
-                        data[0..50]
-                            .iter()
-                            .map(|x| *x as $result_primitive_type)
-                            .collect::<Vec<$result_primitive_type>>()
-                    ),
-                    array
+                    .expect(
+                        format!(
+                            "Unable to downcast {:?} to {:?}",
+                            array.data_type(),
+                            result_data_type
+                        )
+                        .as_str(),
+                    );
+
+                // create expected array as primitive, and cast to result type
+                let expected = PrimitiveArray::<$result_arrow_cast_type>::from(
+                    data[0..50]
+                        .iter()
+                        .map(|x| *x as $result_primitive_type)
+                        .collect::<Vec<$result_primitive_type>>(),
                 );
+                let expected = Arc::new(expected) as ArrayRef;
+                let expected = arrow::compute::cast(&expected, &result_data_type)
+                    .expect("Unable to cast expected array");
+                assert_eq!(expected.data_type(), &result_data_type);
+                let expected = expected
+                    .as_any()
+                    .downcast_ref::<PrimitiveArray<$result_arrow_type>>()
+                    .expect(
+                        format!(
+                            "Unable to downcast expected {:?} to {:?}",
+                            expected.data_type(),
+                            result_data_type
+                        )
+                        .as_str(),
+                    );
+                assert_eq!(expected, array);
             }
         }};
     }
@@ -1160,27 +1889,31 @@ mod tests {
             PhysicalType::INT32,
             "DATE",
             ArrowDate32,
+            ArrowInt32,
             i32
         );
         test_primitive_array_reader_one_type!(
             Int32Type,
             PhysicalType::INT32,
             "TIME_MILLIS",
-            ArrowUInt32,
-            u32
+            ArrowTime32MillisecondArray,
+            ArrowInt32,
+            i32
         );
         test_primitive_array_reader_one_type!(
             Int64Type,
             PhysicalType::INT64,
             "TIME_MICROS",
-            ArrowUInt64,
-            u64
+            ArrowTime64MicrosecondArray,
+            ArrowInt64,
+            i64
         );
         test_primitive_array_reader_one_type!(
             Int64Type,
             PhysicalType::INT64,
             "TIMESTAMP_MILLIS",
             ArrowTimestampMillisecondType,
+            ArrowInt64,
             i64
         );
         test_primitive_array_reader_one_type!(
@@ -1188,6 +1921,7 @@ mod tests {
             PhysicalType::INT64,
             "TIMESTAMP_MICROS",
             ArrowTimestampMicrosecondType,
+            ArrowInt64,
             i64
         );
     }
@@ -1214,7 +1948,7 @@ mod tests {
             let mut def_levels = Vec::new();
             let mut rep_levels = Vec::new();
             let mut page_lists = Vec::new();
-            make_column_chuncks::<Int32Type>(
+            make_column_chunks::<Int32Type>(
                 column_desc.clone(),
                 Encoding::PLAIN,
                 100,
@@ -1228,21 +1962,19 @@ mod tests {
                 2,
             );
 
-            let page_iterator = InMemoryPageIterator::new(
-                schema.clone(),
-                column_desc.clone(),
-                page_lists,
-            );
+            let page_iterator =
+                InMemoryPageIterator::new(schema, column_desc.clone(), page_lists);
 
             let mut array_reader = PrimitiveArrayReader::<Int32Type>::new(
                 Box::new(page_iterator),
-                column_desc.clone(),
+                column_desc,
+                None,
             )
             .unwrap();
 
             let mut accu_len: usize = 0;
 
-            // Read first 50 values, which are all from the first column chunck
+            // Read first 50 values, which are all from the first column chunk
             let array = array_reader.next_batch(50).unwrap();
             assert_eq!(
                 Some(&def_levels[accu_len..(accu_len + array.len())]),
@@ -1342,15 +2074,15 @@ mod tests {
             pages.push(vec![data_page]);
         }
 
-        let page_iterator =
-            InMemoryPageIterator::new(schema.clone(), column_desc.clone(), pages);
+        let page_iterator = InMemoryPageIterator::new(schema, column_desc.clone(), pages);
 
         let converter = Utf8Converter::new(Utf8ArrayConverter {});
         let mut array_reader =
             ComplexObjectArrayReader::<ByteArrayType, Utf8Converter>::new(
                 Box::new(page_iterator),
-                column_desc.clone(),
+                column_desc,
                 converter,
+                None,
             )
             .unwrap();
 
@@ -1444,11 +2176,40 @@ mod tests {
         }
 
         fn get_def_levels(&self) -> Option<&[i16]> {
-            self.def_levels.as_ref().map(|v| v.as_slice())
+            self.def_levels.as_deref()
         }
 
         fn get_rep_levels(&self) -> Option<&[i16]> {
-            self.rep_levels.as_ref().map(|v| v.as_slice())
+            self.rep_levels.as_deref()
+        }
+    }
+
+    /// Iterator for testing reading empty columns
+    struct EmptyPageIterator {
+        schema: SchemaDescPtr,
+    }
+
+    impl EmptyPageIterator {
+        fn new(schema: SchemaDescPtr) -> Self {
+            EmptyPageIterator { schema }
+        }
+    }
+
+    impl Iterator for EmptyPageIterator {
+        type Item = Result<Box<dyn PageReader>>;
+
+        fn next(&mut self) -> Option<Self::Item> {
+            None
+        }
+    }
+
+    impl PageIterator for EmptyPageIterator {
+        fn schema(&mut self) -> Result<SchemaDescPtr> {
+            Ok(self.schema.clone())
+        }
+
+        fn column_schema(&mut self) -> Result<ColumnDescPtr> {
+            Ok(self.schema.column(0))
         }
     }
 
@@ -1507,8 +2268,16 @@ mod tests {
         let file = get_test_file("nulls.snappy.parquet");
         let file_reader = Rc::new(SerializedFileReader::new(file).unwrap());
 
+        let file_metadata = file_reader.metadata().file_metadata();
+        let arrow_schema = parquet_to_arrow_schema(
+            file_metadata.schema_descr(),
+            file_metadata.key_value_metadata(),
+        )
+        .unwrap();
+
         let array_reader = build_array_reader(
             file_reader.metadata().file_metadata().schema_descr_ptr(),
+            arrow_schema,
             vec![0usize].into_iter(),
             file_reader,
         )
@@ -1523,4 +2292,113 @@ mod tests {
 
         assert_eq!(array_reader.get_data_type(), &arrow_type);
     }
+
+    #[test]
+    fn test_list_array_reader() {
+        // [[1, null, 2], null, [3, 4]]
+        let array = Arc::new(PrimitiveArray::<ArrowInt32>::from(vec![
+            Some(1),
+            None,
+            Some(2),
+            None,
+            Some(3),
+            Some(4),
+        ]));
+        let item_array_reader = InMemoryArrayReader::new(
+            ArrowType::Int32,
+            array,
+            Some(vec![3, 2, 3, 0, 3, 3]),
+            Some(vec![0, 1, 1, 0, 0, 1]),
+        );
+
+        let mut list_array_reader = ListArrayReader::<i32>::new(
+            Box::new(item_array_reader),
+            ArrowType::List(Box::new(Field::new("item", ArrowType::Int32, true))),
+            ArrowType::Int32,
+            1,
+            1,
+        );
+
+        let next_batch = list_array_reader.next_batch(1024).unwrap();
+        let list_array = next_batch.as_any().downcast_ref::<ListArray>().unwrap();
+
+        assert_eq!(3, list_array.len());
+        // This passes as I expect
+        assert_eq!(1, list_array.null_count());
+
+        assert_eq!(
+            list_array
+                .value(0)
+                .as_any()
+                .downcast_ref::<PrimitiveArray<ArrowInt32>>()
+                .unwrap(),
+            &PrimitiveArray::<ArrowInt32>::from(vec![Some(1), None, Some(2)])
+        );
+
+        assert!(list_array.is_null(1));
+
+        assert_eq!(
+            list_array
+                .value(2)
+                .as_any()
+                .downcast_ref::<PrimitiveArray<ArrowInt32>>()
+                .unwrap(),
+            &PrimitiveArray::<ArrowInt32>::from(vec![Some(3), Some(4)])
+        );
+    }
+
+    #[test]
+    fn test_large_list_array_reader() {
+        // [[1, null, 2], null, [3, 4]]
+        let array = Arc::new(PrimitiveArray::<ArrowInt32>::from(vec![
+            Some(1),
+            None,
+            Some(2),
+            None,
+            Some(3),
+            Some(4),
+        ]));
+        let item_array_reader = InMemoryArrayReader::new(
+            ArrowType::Int32,
+            array,
+            Some(vec![3, 2, 3, 0, 3, 3]),
+            Some(vec![0, 1, 1, 0, 0, 1]),
+        );
+
+        let mut list_array_reader = ListArrayReader::<i64>::new(
+            Box::new(item_array_reader),
+            ArrowType::LargeList(Box::new(Field::new("item", ArrowType::Int32, true))),
+            ArrowType::Int32,
+            1,
+            1,
+        );
+
+        let next_batch = list_array_reader.next_batch(1024).unwrap();
+        let list_array = next_batch
+            .as_any()
+            .downcast_ref::<LargeListArray>()
+            .unwrap();
+
+        assert_eq!(3, list_array.len());
+
+        assert_eq!(
+            list_array
+                .value(0)
+                .as_any()
+                .downcast_ref::<PrimitiveArray<ArrowInt32>>()
+                .unwrap(),
+            &PrimitiveArray::<ArrowInt32>::from(vec![Some(1), None, Some(2)])
+        );
+
+        assert!(list_array.is_null(1));
+
+        assert_eq!(
+            list_array
+                .value(2)
+                .as_any()
+                .downcast_ref::<PrimitiveArray<ArrowInt32>>()
+                .unwrap(),
+            &PrimitiveArray::<ArrowInt32>::from(vec![Some(3), Some(4)])
+        );
+    }
 }
diff --git a/rust/parquet/src/arrow/arrow_reader.rs b/rust/parquet/src/arrow/arrow_reader.rs
index f052d0f36e5..88af583a3d4 100644
--- a/rust/parquet/src/arrow/arrow_reader.rs
+++ b/rust/parquet/src/arrow/arrow_reader.rs
@@ -19,13 +19,15 @@
 
 use crate::arrow::array_reader::{build_array_reader, ArrayReader, StructArrayReader};
 use crate::arrow::schema::parquet_to_arrow_schema;
-use crate::arrow::schema::parquet_to_arrow_schema_by_columns;
+use crate::arrow::schema::{
+    parquet_to_arrow_schema_by_columns, parquet_to_arrow_schema_by_root_columns,
+};
 use crate::errors::{ParquetError, Result};
 use crate::file::reader::FileReader;
-use arrow::array::StructArray;
 use arrow::datatypes::{DataType as ArrowType, Schema, SchemaRef};
 use arrow::error::Result as ArrowResult;
 use arrow::record_batch::{RecordBatch, RecordBatchReader};
+use arrow::{array::StructArray, error::ArrowError};
 use std::rc::Rc;
 use std::sync::Arc;
 
@@ -40,7 +42,12 @@ pub trait ArrowReader {
 
     /// Read parquet schema and convert it into arrow schema.
     /// This schema only includes columns identified by `column_indices`.
-    fn get_schema_by_columns<T>(&mut self, column_indices: T) -> Result<Schema>
+    /// To select leaf columns (i.e. `a.b.c` instead of `a`), set `leaf_columns = true`
+    fn get_schema_by_columns<T>(
+        &mut self,
+        column_indices: T,
+        leaf_columns: bool,
+    ) -> Result<Schema>
     where
         T: IntoIterator<Item = usize>;
 
@@ -84,16 +91,28 @@ impl ArrowReader for ParquetFileArrowReader {
         )
     }
 
-    fn get_schema_by_columns<T>(&mut self, column_indices: T) -> Result<Schema>
+    fn get_schema_by_columns<T>(
+        &mut self,
+        column_indices: T,
+        leaf_columns: bool,
+    ) -> Result<Schema>
     where
         T: IntoIterator<Item = usize>,
     {
         let file_metadata = self.file_reader.metadata().file_metadata();
-        parquet_to_arrow_schema_by_columns(
-            file_metadata.schema_descr(),
-            column_indices,
-            file_metadata.key_value_metadata(),
-        )
+        if leaf_columns {
+            parquet_to_arrow_schema_by_columns(
+                file_metadata.schema_descr(),
+                column_indices,
+                file_metadata.key_value_metadata(),
+            )
+        } else {
+            parquet_to_arrow_schema_by_root_columns(
+                file_metadata.schema_descr(),
+                column_indices,
+                file_metadata.key_value_metadata(),
+            )
+        }
     }
 
     fn get_record_reader(
@@ -123,6 +142,7 @@ impl ArrowReader for ParquetFileArrowReader {
                 .metadata()
                 .file_metadata()
                 .schema_descr_ptr(),
+            self.get_schema()?,
             column_indices,
             self.file_reader.clone(),
         )?;
@@ -143,38 +163,43 @@ pub struct ParquetRecordBatchReader {
     schema: SchemaRef,
 }
 
+impl Iterator for ParquetRecordBatchReader {
+    type Item = ArrowResult<RecordBatch>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.array_reader.next_batch(self.batch_size) {
+            Err(error) => Some(Err(error.into())),
+            Ok(array) => {
+                let struct_array =
+                    array.as_any().downcast_ref::<StructArray>().ok_or_else(|| {
+                        ArrowError::ParquetError(
+                            "Struct array reader should return struct array".to_string(),
+                        )
+                    });
+                match struct_array {
+                    Err(err) => Some(Err(err)),
+                    Ok(e) => {
+                        match RecordBatch::try_new(self.schema.clone(), e.columns_ref()) {
+                            Err(err) => Some(Err(err)),
+                            Ok(record_batch) => {
+                                if record_batch.num_rows() > 0 {
+                                    Some(Ok(record_batch))
+                                } else {
+                                    None
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
 impl RecordBatchReader for ParquetRecordBatchReader {
     fn schema(&self) -> SchemaRef {
         self.schema.clone()
     }
-
-    fn next_batch(&mut self) -> ArrowResult<Option<RecordBatch>> {
-        self.array_reader
-            .next_batch(self.batch_size)
-            .map_err(|err| err.into())
-            .and_then(|array| {
-                array
-                    .as_any()
-                    .downcast_ref::<StructArray>()
-                    .ok_or_else(|| {
-                        general_err!("Struct array reader should return struct array")
-                            .into()
-                    })
-                    .and_then(|struct_array| {
-                        RecordBatch::try_new(
-                            self.schema.clone(),
-                            struct_array.columns_ref(),
-                        )
-                    })
-            })
-            .map(|record_batch| {
-                if record_batch.num_rows() > 0 {
-                    Some(record_batch)
-                } else {
-                    None
-                }
-            })
-    }
 }
 
 impl ParquetRecordBatchReader {
@@ -189,7 +214,7 @@ impl ParquetRecordBatchReader {
             .ok_or_else(|| general_err!("The input must be struct array reader!"))?;
 
         let schema = match array_reader.get_data_type() {
-            &ArrowType::Struct(ref fields) => Schema::new(fields.clone()),
+            ArrowType::Struct(ref fields) => Schema::new(fields.clone()),
             _ => unreachable!("Struct array reader's data type is not struct!"),
         };
 
@@ -296,12 +321,12 @@ mod tests {
         ";
 
         let converter = FromConverter::new();
-        single_column_reader_test::<
+        run_single_column_reader_tests::<
             BoolType,
             BooleanArray,
             FromConverter<Vec<Option<bool>>, BooleanArray>,
             BoolType,
-        >(2, 100, 2, message_type, 15, 50, converter);
+        >(2, message_type, &converter);
     }
 
     struct RandFixedLenGen {}
@@ -323,12 +348,12 @@ mod tests {
         ";
 
         let converter = FixedSizeArrayConverter::new(20);
-        single_column_reader_test::<
+        run_single_column_reader_tests::<
             FixedLenByteArrayType,
             FixedSizeBinaryArray,
             FixedSizeArrayConverter,
             RandFixedLenGen,
-        >(2, 100, 20, message_type, 15, 50, converter);
+        >(20, message_type, &converter);
     }
 
     struct RandUtf8Gen {}
@@ -348,37 +373,109 @@ mod tests {
         ";
 
         let converter = Utf8ArrayConverter {};
-        single_column_reader_test::<
+        run_single_column_reader_tests::<
             ByteArrayType,
             StringArray,
             Utf8ArrayConverter,
             RandUtf8Gen,
-        >(2, 100, 2, message_type, 15, 50, converter);
+        >(2, message_type, &converter);
     }
 
-    fn single_column_reader_test<T, A, C, G>(
+    /// Parameters for single_column_reader_test
+    #[derive(Debug)]
+    struct TestOptions {
+        /// Number of row group to write to parquet (row group size =
+        /// num_row_groups / num_rows)
         num_row_groups: usize,
+        /// Total number of rows
         num_rows: usize,
-        rand_max: i32,
-        message_type: &str,
+        /// Size of batches to read back
         record_batch_size: usize,
+        /// Total number of batches to attempt to read.
+        /// `record_batch_size` * `num_iterations` should be greater
+        /// than `num_rows` to ensure the data can be read back completely
         num_iterations: usize,
-        converter: C,
+    }
+
+    /// Create a parquet file and then read it using
+    /// `ParquetFileArrowReader` using a standard set of parameters
+    /// `opts`.
+    ///
+    /// `rand_max` represents the maximum size of value to pass to to
+    /// value generator
+    fn run_single_column_reader_tests<T, A, C, G>(
+        rand_max: i32,
+        message_type: &str,
+        converter: &C,
+    ) where
+        T: DataType,
+        G: RandGen<T>,
+        A: PartialEq + Array + 'static,
+        C: Converter<Vec<Option<T::T>>, A> + 'static,
+    {
+        let all_options = vec![
+            // choose record_batch_batch (15) so batches cross row
+            // group boundaries (50 rows in 2 row groups) cases.
+            TestOptions {
+                num_row_groups: 2,
+                num_rows: 100,
+                record_batch_size: 15,
+                num_iterations: 50,
+            },
+            // choose record_batch_batch (5) so batches sometime fall
+            // on row group boundaries and (25 rows in 3 row groups
+            // --> row groups of 10, 10, and 5). Tests buffer
+            // refilling edge cases.
+            TestOptions {
+                num_row_groups: 3,
+                num_rows: 25,
+                record_batch_size: 5,
+                num_iterations: 50,
+            },
+            // Choose record_batch_size (25) so all batches fall
+            // exactly on row group boundary (25). Tests buffer
+            // refilling edge cases.
+            TestOptions {
+                num_row_groups: 4,
+                num_rows: 100,
+                record_batch_size: 25,
+                num_iterations: 50,
+            },
+        ];
+
+        all_options.into_iter().for_each(|opts| {
+            // Print out options to facilitate debugging failures on CI
+            println!("Running with Test Options: {:?}", opts);
+            single_column_reader_test::<T, A, C, G>(
+                opts,
+                rand_max,
+                message_type,
+                converter,
+            )
+        });
+    }
+
+    /// Create a parquet file and then read it using
+    /// `ParquetFileArrowReader` using the parameters described in
+    /// `opts`.
+    fn single_column_reader_test<T, A, C, G>(
+        opts: TestOptions,
+        rand_max: i32,
+        message_type: &str,
+        converter: &C,
     ) where
         T: DataType,
         G: RandGen<T>,
         A: PartialEq + Array + 'static,
         C: Converter<Vec<Option<T::T>>, A> + 'static,
     {
-        let values: Vec<Vec<T::T>> = (0..num_row_groups)
-            .map(|_| G::gen_vec(rand_max, num_rows))
+        let values: Vec<Vec<T::T>> = (0..opts.num_row_groups)
+            .map(|_| G::gen_vec(rand_max, opts.num_rows))
             .collect();
 
         let path = get_temp_filename();
 
-        let schema = parse_message_type(message_type)
-            .map(|t| Rc::new(t))
-            .unwrap();
+        let schema = parse_message_type(message_type).map(Rc::new).unwrap();
 
         generate_single_column_file_with_data::<T>(&values, path.as_path(), schema)
             .unwrap();
@@ -387,8 +484,9 @@ mod tests {
             SerializedFileReader::try_from(File::open(&path).unwrap()).unwrap();
         let mut arrow_reader = ParquetFileArrowReader::new(Rc::new(parquet_reader));
 
-        let mut record_reader =
-            arrow_reader.get_record_reader(record_batch_size).unwrap();
+        let mut record_reader = arrow_reader
+            .get_record_reader(opts.record_batch_size)
+            .unwrap();
 
         let expected_data: Vec<Option<T::T>> = values
             .iter()
@@ -396,12 +494,12 @@ mod tests {
             .map(|b| Some(b.clone()))
             .collect();
 
-        for i in 0..num_iterations {
-            let start = i * record_batch_size;
+        for i in 0..opts.num_iterations {
+            let start = i * opts.record_batch_size;
 
-            let batch = record_reader.next_batch().unwrap();
+            let batch = record_reader.next();
             if start < expected_data.len() {
-                let end = min(start + record_batch_size, expected_data.len());
+                let end = min(start + opts.record_batch_size, expected_data.len());
                 assert!(batch.is_some());
 
                 let mut data = vec![];
@@ -410,6 +508,7 @@ mod tests {
                 assert_eq!(
                     &converter.convert(data).unwrap(),
                     batch
+                        .unwrap()
                         .unwrap()
                         .column(0)
                         .as_any()
@@ -423,14 +522,14 @@ mod tests {
     }
 
     fn generate_single_column_file_with_data<T: DataType>(
-        values: &Vec<Vec<T::T>>,
+        values: &[Vec<T::T>],
         path: &Path,
         schema: TypePtr,
     ) -> Result<()> {
         let file = File::create(path)?;
         let writer_props = Rc::new(WriterProperties::builder().build());
 
-        let mut writer = SerializedFileWriter::new(file, schema, writer_props.clone())?;
+        let mut writer = SerializedFileWriter::new(file, schema, writer_props)?;
 
         for v in values {
             let mut row_group_writer = writer.next_row_group()?;
@@ -481,9 +580,8 @@ mod tests {
     ) {
         for i in 0..20 {
             let array: Option<StructArray> = record_batch_reader
-                .next_batch()
-                .expect("Failed to read record batch!")
-                .map(|r| r.into());
+                .next()
+                .map(|r| r.expect("Failed to read record batch!").into());
 
             let (start, end) = (i * 60 as usize, (i + 1) * 60 as usize);
 
diff --git a/rust/parquet/src/arrow/arrow_writer.rs b/rust/parquet/src/arrow/arrow_writer.rs
new file mode 100644
index 00000000000..c0b2105c2df
--- /dev/null
+++ b/rust/parquet/src/arrow/arrow_writer.rs
@@ -0,0 +1,1385 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Contains writer which writes arrow data into parquet data.
+
+use std::rc::Rc;
+
+use arrow::array as arrow_array;
+use arrow::datatypes::{DataType as ArrowDataType, SchemaRef};
+use arrow::record_batch::RecordBatch;
+use arrow_array::Array;
+
+use super::schema::add_encoded_arrow_schema_to_metadata;
+use crate::column::writer::{ColumnWriter, ColumnWriterImpl};
+use crate::errors::{ParquetError, Result};
+use crate::file::properties::WriterProperties;
+use crate::{
+    data_type::*,
+    file::writer::{FileWriter, ParquetWriter, RowGroupWriter, SerializedFileWriter},
+};
+
+/// Arrow writer
+///
+/// Writes Arrow `RecordBatch`es to a Parquet writer
+pub struct ArrowWriter<W: ParquetWriter> {
+    /// Underlying Parquet writer
+    writer: SerializedFileWriter<W>,
+    /// A copy of the Arrow schema.
+    ///
+    /// The schema is used to verify that each record batch written has the correct schema
+    arrow_schema: SchemaRef,
+}
+
+impl<W: 'static + ParquetWriter> ArrowWriter<W> {
+    /// Try to create a new Arrow writer
+    ///
+    /// The writer will fail if:
+    ///  * a `SerializedFileWriter` cannot be created from the ParquetWriter
+    ///  * the Arrow schema contains unsupported datatypes such as Unions
+    pub fn try_new(
+        writer: W,
+        arrow_schema: SchemaRef,
+        props: Option<WriterProperties>,
+    ) -> Result<Self> {
+        let schema = crate::arrow::arrow_to_parquet_schema(&arrow_schema)?;
+        // add serialized arrow schema
+        let mut props = props.unwrap_or_else(|| WriterProperties::builder().build());
+        add_encoded_arrow_schema_to_metadata(&arrow_schema, &mut props);
+
+        let file_writer = SerializedFileWriter::new(
+            writer.try_clone()?,
+            schema.root_schema_ptr(),
+            Rc::new(props),
+        )?;
+
+        Ok(Self {
+            writer: file_writer,
+            arrow_schema,
+        })
+    }
+
+    /// Write a RecordBatch to writer
+    ///
+    /// *NOTE:* The writer currently does not support all Arrow data types
+    pub fn write(&mut self, batch: &RecordBatch) -> Result<()> {
+        // validate batch schema against writer's supplied schema
+        if self.arrow_schema != batch.schema() {
+            return Err(ParquetError::ArrowError(
+                "Record batch schema does not match writer schema".to_string(),
+            ));
+        }
+        // compute the definition and repetition levels of the batch
+        let mut levels = vec![];
+        batch.columns().iter().for_each(|array| {
+            let mut array_levels =
+                get_levels(array, 0, &vec![1i16; batch.num_rows()][..], None);
+            levels.append(&mut array_levels);
+        });
+        // reverse levels so we can use Vec::pop(&mut self)
+        levels.reverse();
+
+        let mut row_group_writer = self.writer.next_row_group()?;
+
+        // write leaves
+        for column in batch.columns() {
+            write_leaves(&mut row_group_writer, column, &mut levels)?;
+        }
+
+        self.writer.close_row_group(row_group_writer)
+    }
+
+    /// Close and finalise the underlying Parquet writer
+    pub fn close(&mut self) -> Result<()> {
+        self.writer.close()
+    }
+}
+
+/// Convenience method to get the next ColumnWriter from the RowGroupWriter
+#[inline]
+#[allow(clippy::borrowed_box)]
+fn get_col_writer(
+    row_group_writer: &mut Box<dyn RowGroupWriter>,
+) -> Result<ColumnWriter> {
+    let col_writer = row_group_writer
+        .next_column()?
+        .expect("Unable to get column writer");
+    Ok(col_writer)
+}
+
+#[allow(clippy::borrowed_box)]
+fn write_leaves(
+    mut row_group_writer: &mut Box<dyn RowGroupWriter>,
+    array: &arrow_array::ArrayRef,
+    mut levels: &mut Vec<Levels>,
+) -> Result<()> {
+    match array.data_type() {
+        ArrowDataType::Null
+        | ArrowDataType::Int8
+        | ArrowDataType::Int16
+        | ArrowDataType::Int32
+        | ArrowDataType::Int64
+        | ArrowDataType::UInt8
+        | ArrowDataType::UInt16
+        | ArrowDataType::UInt32
+        | ArrowDataType::UInt64
+        | ArrowDataType::Float32
+        | ArrowDataType::Float64
+        | ArrowDataType::Timestamp(_, _)
+        | ArrowDataType::Date32(_)
+        | ArrowDataType::Date64(_)
+        | ArrowDataType::Time32(_)
+        | ArrowDataType::Time64(_)
+        | ArrowDataType::Duration(_)
+        | ArrowDataType::Interval(_)
+        | ArrowDataType::LargeBinary
+        | ArrowDataType::Binary
+        | ArrowDataType::Utf8
+        | ArrowDataType::LargeUtf8 => {
+            let mut col_writer = get_col_writer(&mut row_group_writer)?;
+            write_leaf(
+                &mut col_writer,
+                array,
+                levels.pop().expect("Levels exhausted"),
+            )?;
+            row_group_writer.close_column(col_writer)?;
+            Ok(())
+        }
+        ArrowDataType::List(_) | ArrowDataType::LargeList(_) => {
+            // write the child list
+            let data = array.data();
+            let child_array = arrow_array::make_array(data.child_data()[0].clone());
+            write_leaves(&mut row_group_writer, &child_array, &mut levels)?;
+            Ok(())
+        }
+        ArrowDataType::Struct(_) => {
+            let struct_array: &arrow_array::StructArray = array
+                .as_any()
+                .downcast_ref::<arrow_array::StructArray>()
+                .expect("Unable to get struct array");
+            for field in struct_array.columns() {
+                write_leaves(&mut row_group_writer, field, &mut levels)?;
+            }
+            Ok(())
+        }
+        ArrowDataType::Dictionary(key_type, value_type) => {
+            use arrow_array::{PrimitiveArray, StringArray};
+            use ArrowDataType::*;
+            use ColumnWriter::*;
+
+            let array = &**array;
+            let mut col_writer = get_col_writer(&mut row_group_writer)?;
+            let levels = levels.pop().expect("Levels exhausted");
+
+            macro_rules! dispatch_dictionary {
+                ($($kt: pat, $vt: pat, $w: ident => $kat: ty, $vat: ty,)*) => (
+                    match (&**key_type, &**value_type, &mut col_writer) {
+                        $(($kt, $vt, $w(writer)) => write_dict::<$kat, $vat, _>(array, writer, levels),)*
+                        (kt, vt, _) => unreachable!("Shouldn't be attempting to write dictionary of <{:?}, {:?}>", kt, vt),
+                    }
+                );
+            }
+
+            match (&**key_type, &**value_type, &mut col_writer) {
+                (UInt8, UInt32, Int32ColumnWriter(writer)) => {
+                    let typed_array = array
+                        .as_any()
+                        .downcast_ref::<arrow_array::UInt8DictionaryArray>()
+                        .expect("Unable to get dictionary array");
+
+                    let keys = typed_array.keys();
+
+                    let value_buffer = typed_array.values();
+                    let value_array =
+                        arrow::compute::cast(&value_buffer, &ArrowDataType::Int32)?;
+
+                    let values = value_array
+                        .as_any()
+                        .downcast_ref::<arrow_array::Int32Array>()
+                        .unwrap();
+
+                    use std::convert::TryFrom;
+                    // This removes NULL values from the keys, but
+                    // they're encoded by the levels, so that's fine.
+                    let materialized_values: Vec<_> = keys
+                        .into_iter()
+                        .flatten()
+                        .map(|key| {
+                            usize::try_from(key).unwrap_or_else(|k| {
+                                panic!("key {} does not fit in usize", k)
+                            })
+                        })
+                        .map(|key| values.value(key))
+                        .collect();
+
+                    let materialized_primitive_array =
+                        PrimitiveArray::<arrow::datatypes::Int32Type>::from(
+                            materialized_values,
+                        );
+
+                    writer.write_batch(
+                        get_numeric_array_slice::<Int32Type, _>(
+                            &materialized_primitive_array,
+                        )
+                        .as_slice(),
+                        Some(levels.definition.as_slice()),
+                        levels.repetition.as_deref(),
+                    )?;
+                    row_group_writer.close_column(col_writer)?;
+
+                    return Ok(());
+                }
+                _ => {}
+            }
+
+            dispatch_dictionary!(
+                Int8, Utf8, ByteArrayColumnWriter => arrow::datatypes::Int8Type, StringArray,
+                Int16, Utf8, ByteArrayColumnWriter => arrow::datatypes::Int16Type, StringArray,
+                Int32, Utf8, ByteArrayColumnWriter => arrow::datatypes::Int32Type, StringArray,
+                Int64, Utf8, ByteArrayColumnWriter => arrow::datatypes::Int64Type, StringArray,
+                UInt8, Utf8, ByteArrayColumnWriter => arrow::datatypes::UInt8Type, StringArray,
+                UInt16, Utf8, ByteArrayColumnWriter => arrow::datatypes::UInt16Type, StringArray,
+                UInt32, Utf8, ByteArrayColumnWriter => arrow::datatypes::UInt32Type, StringArray,
+                UInt64, Utf8, ByteArrayColumnWriter => arrow::datatypes::UInt64Type, StringArray,
+            )?;
+
+            row_group_writer.close_column(col_writer)?;
+
+            Ok(())
+        }
+        ArrowDataType::Float16 => Err(ParquetError::ArrowError(
+            "Float16 arrays not supported".to_string(),
+        )),
+        ArrowDataType::FixedSizeList(_, _)
+        | ArrowDataType::Boolean
+        | ArrowDataType::FixedSizeBinary(_)
+        | ArrowDataType::Union(_) => Err(ParquetError::NYI(
+            "Attempting to write an Arrow type that is not yet implemented".to_string(),
+        )),
+    }
+}
+
+trait Materialize<K, V> {
+    type Output;
+
+    // Materialize the packed dictionary. The writer will later repack it.
+    fn materialize(&self) -> Vec<Self::Output>;
+}
+
+impl<K> Materialize<K, arrow_array::StringArray> for dyn Array
+where
+    K: arrow::datatypes::ArrowDictionaryKeyType,
+{
+    type Output = ByteArray;
+
+    fn materialize(&self) -> Vec<Self::Output> {
+        use arrow::datatypes::ArrowNativeType;
+
+        let typed_array = self
+            .as_any()
+            .downcast_ref::<arrow_array::DictionaryArray<K>>()
+            .expect("Unable to get dictionary array");
+
+        let keys = typed_array.keys();
+
+        let value_buffer = typed_array.values();
+        let values = value_buffer
+            .as_any()
+            .downcast_ref::<arrow_array::StringArray>()
+            .unwrap();
+
+        // This removes NULL values from the keys, but
+        // they're encoded by the levels, so that's fine.
+        keys.into_iter()
+            .flatten()
+            .map(|key| {
+                key.to_usize()
+                    .unwrap_or_else(|| panic!("key {:?} does not fit in usize", key))
+            })
+            .map(|key| values.value(key))
+            .map(ByteArray::from)
+            .collect()
+    }
+}
+
+fn write_dict<K, V, T>(
+    array: &(dyn Array + 'static),
+    writer: &mut ColumnWriterImpl<T>,
+    levels: Levels,
+) -> Result<()>
+where
+    T: DataType,
+    dyn Array: Materialize<K, V, Output = T::T>,
+{
+    writer.write_batch(
+        &array.materialize(),
+        Some(levels.definition.as_slice()),
+        levels.repetition.as_deref(),
+    )?;
+
+    Ok(())
+}
+
+fn write_leaf(
+    writer: &mut ColumnWriter,
+    column: &arrow_array::ArrayRef,
+    levels: Levels,
+) -> Result<i64> {
+    let written = match writer {
+        ColumnWriter::Int32ColumnWriter(ref mut typed) => {
+            let array = arrow::compute::cast(column, &ArrowDataType::Int32)?;
+            let array = array
+                .as_any()
+                .downcast_ref::<arrow_array::Int32Array>()
+                .expect("Unable to get int32 array");
+            typed.write_batch(
+                get_numeric_array_slice::<Int32Type, _>(&array).as_slice(),
+                Some(levels.definition.as_slice()),
+                levels.repetition.as_deref(),
+            )?
+        }
+        ColumnWriter::BoolColumnWriter(ref mut _typed) => {
+            unreachable!("Currently unreachable because data type not supported")
+        }
+        ColumnWriter::Int64ColumnWriter(ref mut typed) => {
+            let array = arrow_array::Int64Array::from(column.data());
+            typed.write_batch(
+                get_numeric_array_slice::<Int64Type, _>(&array).as_slice(),
+                Some(levels.definition.as_slice()),
+                levels.repetition.as_deref(),
+            )?
+        }
+        ColumnWriter::Int96ColumnWriter(ref mut _typed) => {
+            unreachable!("Currently unreachable because data type not supported")
+        }
+        ColumnWriter::FloatColumnWriter(ref mut typed) => {
+            let array = arrow_array::Float32Array::from(column.data());
+            typed.write_batch(
+                get_numeric_array_slice::<FloatType, _>(&array).as_slice(),
+                Some(levels.definition.as_slice()),
+                levels.repetition.as_deref(),
+            )?
+        }
+        ColumnWriter::DoubleColumnWriter(ref mut typed) => {
+            let array = arrow_array::Float64Array::from(column.data());
+            typed.write_batch(
+                get_numeric_array_slice::<DoubleType, _>(&array).as_slice(),
+                Some(levels.definition.as_slice()),
+                levels.repetition.as_deref(),
+            )?
+        }
+        ColumnWriter::ByteArrayColumnWriter(ref mut typed) => match column.data_type() {
+            ArrowDataType::Binary => {
+                let array = arrow_array::BinaryArray::from(column.data());
+                typed.write_batch(
+                    get_binary_array(&array).as_slice(),
+                    Some(levels.definition.as_slice()),
+                    levels.repetition.as_deref(),
+                )?
+            }
+            ArrowDataType::Utf8 => {
+                let array = arrow_array::StringArray::from(column.data());
+                typed.write_batch(
+                    get_string_array(&array).as_slice(),
+                    Some(levels.definition.as_slice()),
+                    levels.repetition.as_deref(),
+                )?
+            }
+            ArrowDataType::LargeBinary => {
+                let array = arrow_array::LargeBinaryArray::from(column.data());
+                typed.write_batch(
+                    get_large_binary_array(&array).as_slice(),
+                    Some(levels.definition.as_slice()),
+                    levels.repetition.as_deref(),
+                )?
+            }
+            ArrowDataType::LargeUtf8 => {
+                let array = arrow_array::LargeStringArray::from(column.data());
+                typed.write_batch(
+                    get_large_string_array(&array).as_slice(),
+                    Some(levels.definition.as_slice()),
+                    levels.repetition.as_deref(),
+                )?
+            }
+            _ => unreachable!("Currently unreachable because data type not supported"),
+        },
+        ColumnWriter::FixedLenByteArrayColumnWriter(ref mut _typed) => {
+            unreachable!("Currently unreachable because data type not supported")
+        }
+    };
+    Ok(written as i64)
+}
+
+/// A struct that represents definition and repetition levels.
+/// Repetition levels are only populated if the parent or current leaf is repeated
+#[derive(Debug)]
+struct Levels {
+    definition: Vec<i16>,
+    repetition: Option<Vec<i16>>,
+}
+
+/// Compute nested levels of the Arrow array, recursing into lists and structs
+fn get_levels(
+    array: &arrow_array::ArrayRef,
+    level: i16,
+    parent_def_levels: &[i16],
+    parent_rep_levels: Option<&[i16]>,
+) -> Vec<Levels> {
+    match array.data_type() {
+        ArrowDataType::Null => vec![Levels {
+            definition: parent_def_levels.iter().map(|v| (v - 1).max(0)).collect(),
+            repetition: None,
+        }],
+        ArrowDataType::Boolean
+        | ArrowDataType::Int8
+        | ArrowDataType::Int16
+        | ArrowDataType::Int32
+        | ArrowDataType::Int64
+        | ArrowDataType::UInt8
+        | ArrowDataType::UInt16
+        | ArrowDataType::UInt32
+        | ArrowDataType::UInt64
+        | ArrowDataType::Float16
+        | ArrowDataType::Float32
+        | ArrowDataType::Float64
+        | ArrowDataType::Utf8
+        | ArrowDataType::LargeUtf8
+        | ArrowDataType::Timestamp(_, _)
+        | ArrowDataType::Date32(_)
+        | ArrowDataType::Date64(_)
+        | ArrowDataType::Time32(_)
+        | ArrowDataType::Time64(_)
+        | ArrowDataType::Duration(_)
+        | ArrowDataType::Interval(_)
+        | ArrowDataType::Binary
+        | ArrowDataType::LargeBinary => vec![Levels {
+            definition: get_primitive_def_levels(array, parent_def_levels),
+            repetition: None,
+        }],
+        ArrowDataType::FixedSizeBinary(_) => unimplemented!(),
+        ArrowDataType::List(_) | ArrowDataType::LargeList(_) => {
+            let array_data = array.data();
+            let child_data = array_data.child_data().get(0).unwrap();
+            // get offsets, accounting for large offsets if present
+            let offsets: Vec<i64> = {
+                if let ArrowDataType::LargeList(_) = array.data_type() {
+                    unsafe { array_data.buffers()[0].typed_data::<i64>() }.to_vec()
+                } else {
+                    let offsets = unsafe { array_data.buffers()[0].typed_data::<i32>() };
+                    offsets.to_vec().into_iter().map(|v| v as i64).collect()
+                }
+            };
+            let child_array = arrow_array::make_array(child_data.clone());
+
+            let mut list_def_levels = Vec::with_capacity(child_array.len());
+            let mut list_rep_levels = Vec::with_capacity(child_array.len());
+            let rep_levels: Vec<i16> = parent_rep_levels
+                .map(|l| l.to_vec())
+                .unwrap_or_else(|| vec![0i16; parent_def_levels.len()]);
+            parent_def_levels
+                .iter()
+                .zip(rep_levels)
+                .zip(offsets.windows(2))
+                .for_each(|((parent_def_level, parent_rep_level), window)| {
+                    if *parent_def_level == 0 {
+                        // parent is null, list element must also be null
+                        list_def_levels.push(0);
+                        list_rep_levels.push(0);
+                    } else {
+                        // parent is not null, check if list is empty or null
+                        let start = window[0];
+                        let end = window[1];
+                        let len = end - start;
+                        if len == 0 {
+                            list_def_levels.push(*parent_def_level - 1);
+                            list_rep_levels.push(parent_rep_level);
+                        } else {
+                            list_def_levels.push(*parent_def_level);
+                            list_rep_levels.push(parent_rep_level);
+                            for _ in 1..len {
+                                list_def_levels.push(*parent_def_level);
+                                list_rep_levels.push(parent_rep_level + 1);
+                            }
+                        }
+                    }
+                });
+
+            // if datatype is a primitive, we can construct levels of the child array
+            match child_array.data_type() {
+                // TODO: The behaviour of a <list<null>> is untested
+                ArrowDataType::Null => vec![Levels {
+                    definition: list_def_levels,
+                    repetition: Some(list_rep_levels),
+                }],
+                ArrowDataType::Boolean => unimplemented!(),
+                ArrowDataType::Int8
+                | ArrowDataType::Int16
+                | ArrowDataType::Int32
+                | ArrowDataType::Int64
+                | ArrowDataType::UInt8
+                | ArrowDataType::UInt16
+                | ArrowDataType::UInt32
+                | ArrowDataType::UInt64
+                | ArrowDataType::Float16
+                | ArrowDataType::Float32
+                | ArrowDataType::Float64
+                | ArrowDataType::Timestamp(_, _)
+                | ArrowDataType::Date32(_)
+                | ArrowDataType::Date64(_)
+                | ArrowDataType::Time32(_)
+                | ArrowDataType::Time64(_)
+                | ArrowDataType::Duration(_)
+                | ArrowDataType::Interval(_) => {
+                    let def_levels =
+                        get_primitive_def_levels(&child_array, &list_def_levels[..]);
+                    vec![Levels {
+                        definition: def_levels,
+                        repetition: Some(list_rep_levels),
+                    }]
+                }
+                ArrowDataType::Binary
+                | ArrowDataType::Utf8
+                | ArrowDataType::LargeUtf8 => unimplemented!(),
+                ArrowDataType::FixedSizeBinary(_) => unimplemented!(),
+                ArrowDataType::LargeBinary => unimplemented!(),
+                ArrowDataType::List(_) | ArrowDataType::LargeList(_) => {
+                    // nested list
+                    unimplemented!()
+                }
+                ArrowDataType::FixedSizeList(_, _) => unimplemented!(),
+                ArrowDataType::Struct(_) => get_levels(
+                    array,
+                    level + 1, // indicates a nesting level of 2 (list + struct)
+                    &list_def_levels[..],
+                    Some(&list_rep_levels[..]),
+                ),
+                ArrowDataType::Union(_) => unimplemented!(),
+                ArrowDataType::Dictionary(_, _) => unimplemented!(),
+            }
+        }
+        ArrowDataType::FixedSizeList(_, _) => unimplemented!(),
+        ArrowDataType::Struct(_) => {
+            let struct_array: &arrow_array::StructArray = array
+                .as_any()
+                .downcast_ref::<arrow_array::StructArray>()
+                .expect("Unable to get struct array");
+            let mut struct_def_levels = Vec::with_capacity(struct_array.len());
+            for i in 0..array.len() {
+                struct_def_levels.push(level + struct_array.is_valid(i) as i16);
+            }
+            // trying to create levels for struct's fields
+            let mut struct_levels = vec![];
+            struct_array.columns().into_iter().for_each(|col| {
+                let mut levels =
+                    get_levels(col, level + 1, &struct_def_levels[..], parent_rep_levels);
+                struct_levels.append(&mut levels);
+            });
+            struct_levels
+        }
+        ArrowDataType::Union(_) => unimplemented!(),
+        ArrowDataType::Dictionary(_, _) => {
+            // Need to check for these cases not implemented in C++:
+            // - "Writing DictionaryArray with nested dictionary type not yet supported"
+            // - "Writing DictionaryArray with null encoded in dictionary type not yet supported"
+            vec![Levels {
+                definition: get_primitive_def_levels(array, parent_def_levels),
+                repetition: None,
+            }]
+        }
+    }
+}
+
+/// Get the definition levels of the numeric array, with level 0 being null and 1 being not null
+/// In the case where the array in question is a child of either a list or struct, the levels
+/// are incremented in accordance with the `level` parameter.
+/// Parent levels are either 0 or 1, and are used to higher (correct terminology?) leaves as null
+fn get_primitive_def_levels(
+    array: &arrow_array::ArrayRef,
+    parent_def_levels: &[i16],
+) -> Vec<i16> {
+    let mut array_index = 0;
+    let max_def_level = parent_def_levels.iter().max().unwrap();
+    let mut primitive_def_levels = vec![];
+    parent_def_levels.iter().for_each(|def_level| {
+        if def_level < max_def_level {
+            primitive_def_levels.push(*def_level);
+        } else {
+            primitive_def_levels.push(def_level - array.is_null(array_index) as i16);
+            array_index += 1;
+        }
+    });
+    primitive_def_levels
+}
+
+macro_rules! def_get_binary_array_fn {
+    ($name:ident, $ty:ty) => {
+        fn $name(array: &$ty) -> Vec<ByteArray> {
+            let mut values = Vec::with_capacity(array.len() - array.null_count());
+            for i in 0..array.len() {
+                if array.is_valid(i) {
+                    let bytes: Vec<u8> = array.value(i).into();
+                    let bytes = ByteArray::from(bytes);
+                    values.push(bytes);
+                }
+            }
+            values
+        }
+    };
+}
+
+def_get_binary_array_fn!(get_binary_array, arrow_array::BinaryArray);
+def_get_binary_array_fn!(get_string_array, arrow_array::StringArray);
+def_get_binary_array_fn!(get_large_binary_array, arrow_array::LargeBinaryArray);
+def_get_binary_array_fn!(get_large_string_array, arrow_array::LargeStringArray);
+
+/// Get the underlying numeric array slice, skipping any null values.
+/// If there are no null values, it might be quicker to get the slice directly instead of
+/// calling this function.
+fn get_numeric_array_slice<T, A>(array: &arrow_array::PrimitiveArray<A>) -> Vec<T::T>
+where
+    T: DataType,
+    A: arrow::datatypes::ArrowNumericType,
+    T::T: From<A::Native>,
+{
+    let mut values = Vec::with_capacity(array.len() - array.null_count());
+    for i in 0..array.len() {
+        if array.is_valid(i) {
+            values.push(array.value(i).into())
+        }
+    }
+    values
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use std::io::Seek;
+    use std::sync::Arc;
+
+    use arrow::array::*;
+    use arrow::datatypes::ToByteSlice;
+    use arrow::datatypes::{DataType, Field, Schema, UInt32Type, UInt8Type};
+    use arrow::record_batch::RecordBatch;
+
+    use crate::arrow::{ArrowReader, ParquetFileArrowReader};
+    use crate::file::{metadata::KeyValue, reader::SerializedFileReader};
+    use crate::util::test_common::get_temp_file;
+
+    #[test]
+    fn arrow_writer() {
+        // define schema
+        let schema = Schema::new(vec![
+            Field::new("a", DataType::Int32, false),
+            Field::new("b", DataType::Int32, true),
+        ]);
+
+        // create some data
+        let a = Int32Array::from(vec![1, 2, 3, 4, 5]);
+        let b = Int32Array::from(vec![Some(1), None, None, Some(4), Some(5)]);
+
+        // build a record batch
+        let batch = RecordBatch::try_new(
+            Arc::new(schema.clone()),
+            vec![Arc::new(a), Arc::new(b)],
+        )
+        .unwrap();
+
+        let file = get_temp_file("test_arrow_writer.parquet", &[]);
+        let mut writer = ArrowWriter::try_new(file, Arc::new(schema), None).unwrap();
+        writer.write(&batch).unwrap();
+        writer.close().unwrap();
+    }
+
+    #[test]
+    #[ignore = "repetitions might be incorrect, will be addressed as part of ARROW-9728"]
+    fn arrow_writer_list() {
+        // define schema
+        let schema = Schema::new(vec![Field::new(
+            "a",
+            DataType::List(Box::new(Field::new("item", DataType::Int32, true))),
+            false,
+        )]);
+
+        // create some data
+        let a_values = Int32Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
+
+        // Construct a buffer for value offsets, for the nested array:
+        //  [[1], [2, 3], null, [4, 5, 6], [7, 8, 9, 10]]
+        let a_value_offsets =
+            arrow::buffer::Buffer::from(&[0, 1, 3, 3, 6, 10].to_byte_slice());
+
+        // Construct a list array from the above two
+        let a_list_data = ArrayData::builder(DataType::List(Box::new(Field::new(
+            "items",
+            DataType::Int32,
+            true,
+        ))))
+        .len(5)
+        .add_buffer(a_value_offsets)
+        .add_child_data(a_values.data())
+        .build();
+        let a = ListArray::from(a_list_data);
+
+        // build a record batch
+        let batch =
+            RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)]).unwrap();
+
+        // I think this setup is incorrect because this should pass
+        assert_eq!(batch.column(0).data().null_count(), 1);
+
+        let file = get_temp_file("test_arrow_writer_list.parquet", &[]);
+        let mut writer = ArrowWriter::try_new(file, Arc::new(schema), None).unwrap();
+        writer.write(&batch).unwrap();
+        writer.close().unwrap();
+    }
+
+    #[test]
+    fn arrow_writer_binary() {
+        let string_field = Field::new("a", DataType::Utf8, false);
+        let binary_field = Field::new("b", DataType::Binary, false);
+        let schema = Schema::new(vec![string_field, binary_field]);
+
+        let raw_string_values = vec!["foo", "bar", "baz", "quux"];
+        let raw_binary_values = vec![
+            b"foo".to_vec(),
+            b"bar".to_vec(),
+            b"baz".to_vec(),
+            b"quux".to_vec(),
+        ];
+        let raw_binary_value_refs = raw_binary_values
+            .iter()
+            .map(|x| x.as_slice())
+            .collect::<Vec<_>>();
+
+        let string_values = StringArray::from(raw_string_values.clone());
+        let binary_values = BinaryArray::from(raw_binary_value_refs);
+        let batch = RecordBatch::try_new(
+            Arc::new(schema.clone()),
+            vec![Arc::new(string_values), Arc::new(binary_values)],
+        )
+        .unwrap();
+
+        let mut file = get_temp_file("test_arrow_writer_binary.parquet", &[]);
+        let mut writer =
+            ArrowWriter::try_new(file.try_clone().unwrap(), Arc::new(schema), None)
+                .unwrap();
+        writer.write(&batch).unwrap();
+        writer.close().unwrap();
+
+        file.seek(std::io::SeekFrom::Start(0)).unwrap();
+        let file_reader = SerializedFileReader::new(file).unwrap();
+        let mut arrow_reader = ParquetFileArrowReader::new(Rc::new(file_reader));
+        let mut record_batch_reader = arrow_reader.get_record_reader(1024).unwrap();
+
+        let batch = record_batch_reader.next().unwrap().unwrap();
+        let string_col = batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap();
+        let binary_col = batch
+            .column(1)
+            .as_any()
+            .downcast_ref::<BinaryArray>()
+            .unwrap();
+
+        for i in 0..batch.num_rows() {
+            assert_eq!(string_col.value(i), raw_string_values[i]);
+            assert_eq!(binary_col.value(i), raw_binary_values[i].as_slice());
+        }
+    }
+
+    #[test]
+    fn arrow_writer_complex() {
+        // define schema
+        let struct_field_d = Field::new("d", DataType::Float64, true);
+        let struct_field_f = Field::new("f", DataType::Float32, true);
+        let struct_field_g = Field::new(
+            "g",
+            DataType::List(Box::new(Field::new("items", DataType::Int16, false))),
+            false,
+        );
+        let struct_field_e = Field::new(
+            "e",
+            DataType::Struct(vec![struct_field_f.clone(), struct_field_g.clone()]),
+            true,
+        );
+        let schema = Schema::new(vec![
+            Field::new("a", DataType::Int32, false),
+            Field::new("b", DataType::Int32, true),
+            Field::new(
+                "c",
+                DataType::Struct(vec![struct_field_d.clone(), struct_field_e.clone()]),
+                false,
+            ),
+        ]);
+
+        // create some data
+        let a = Int32Array::from(vec![1, 2, 3, 4, 5]);
+        let b = Int32Array::from(vec![Some(1), None, None, Some(4), Some(5)]);
+        let d = Float64Array::from(vec![None, None, None, Some(1.0), None]);
+        let f = Float32Array::from(vec![Some(0.0), None, Some(333.3), None, Some(5.25)]);
+
+        let g_value = Int16Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
+
+        // Construct a buffer for value offsets, for the nested array:
+        //  [[1], [2, 3], null, [4, 5, 6], [7, 8, 9, 10]]
+        let g_value_offsets =
+            arrow::buffer::Buffer::from(&[0, 1, 3, 3, 6, 10].to_byte_slice());
+
+        // Construct a list array from the above two
+        let g_list_data = ArrayData::builder(struct_field_g.data_type().clone())
+            .len(5)
+            .add_buffer(g_value_offsets)
+            .add_child_data(g_value.data())
+            .build();
+        let g = ListArray::from(g_list_data);
+
+        let e = StructArray::from(vec![
+            (struct_field_f, Arc::new(f) as ArrayRef),
+            (struct_field_g, Arc::new(g) as ArrayRef),
+        ]);
+
+        let c = StructArray::from(vec![
+            (struct_field_d, Arc::new(d) as ArrayRef),
+            (struct_field_e, Arc::new(e) as ArrayRef),
+        ]);
+
+        // build a record batch
+        let batch = RecordBatch::try_new(
+            Arc::new(schema.clone()),
+            vec![Arc::new(a), Arc::new(b), Arc::new(c)],
+        )
+        .unwrap();
+
+        let props = WriterProperties::builder()
+            .set_key_value_metadata(Some(vec![KeyValue {
+                key: "test_key".to_string(),
+                value: Some("test_value".to_string()),
+            }]))
+            .build();
+
+        let file = get_temp_file("test_arrow_writer_complex.parquet", &[]);
+        let mut writer =
+            ArrowWriter::try_new(file, Arc::new(schema), Some(props)).unwrap();
+        writer.write(&batch).unwrap();
+        writer.close().unwrap();
+    }
+
+    const SMALL_SIZE: usize = 100;
+
+    fn roundtrip(filename: &str, expected_batch: RecordBatch) {
+        let file = get_temp_file(filename, &[]);
+
+        let mut writer = ArrowWriter::try_new(
+            file.try_clone().unwrap(),
+            expected_batch.schema(),
+            None,
+        )
+        .expect("Unable to write file");
+        writer.write(&expected_batch).unwrap();
+        writer.close().unwrap();
+
+        let reader = SerializedFileReader::new(file).unwrap();
+        let mut arrow_reader = ParquetFileArrowReader::new(Rc::new(reader));
+        let mut record_batch_reader = arrow_reader.get_record_reader(1024).unwrap();
+
+        let actual_batch = record_batch_reader
+            .next()
+            .expect("No batch found")
+            .expect("Unable to get batch");
+
+        assert_eq!(expected_batch.schema(), actual_batch.schema());
+        assert_eq!(expected_batch.num_columns(), actual_batch.num_columns());
+        assert_eq!(expected_batch.num_rows(), actual_batch.num_rows());
+        for i in 0..expected_batch.num_columns() {
+            let expected_data = expected_batch.column(i).data();
+            let actual_data = actual_batch.column(i).data();
+
+            assert_eq!(expected_data.data_type(), actual_data.data_type());
+            assert_eq!(expected_data.len(), actual_data.len());
+            assert_eq!(expected_data.null_count(), actual_data.null_count());
+            assert_eq!(expected_data.offset(), actual_data.offset());
+            assert_eq!(expected_data.buffers(), actual_data.buffers());
+            assert_eq!(expected_data.child_data(), actual_data.child_data());
+            // Null counts should be the same, not necessarily bitmaps
+            // A null bitmap is optional if an array has no nulls
+            if expected_data.null_count() != 0 {
+                assert_eq!(expected_data.null_bitmap(), actual_data.null_bitmap());
+            }
+        }
+    }
+
+    fn one_column_roundtrip(filename: &str, values: ArrayRef, nullable: bool) {
+        let schema = Schema::new(vec![Field::new(
+            "col",
+            values.data_type().clone(),
+            nullable,
+        )]);
+        let expected_batch =
+            RecordBatch::try_new(Arc::new(schema), vec![values]).unwrap();
+
+        roundtrip(filename, expected_batch);
+    }
+
+    fn values_required<A, I>(iter: I, filename: &str)
+    where
+        A: From<Vec<I::Item>> + Array + 'static,
+        I: IntoIterator,
+    {
+        let raw_values: Vec<_> = iter.into_iter().collect();
+        let values = Arc::new(A::from(raw_values));
+        one_column_roundtrip(filename, values, false);
+    }
+
+    fn values_optional<A, I>(iter: I, filename: &str)
+    where
+        A: From<Vec<Option<I::Item>>> + Array + 'static,
+        I: IntoIterator,
+    {
+        let optional_raw_values: Vec<_> = iter
+            .into_iter()
+            .enumerate()
+            .map(|(i, v)| if i % 2 == 0 { None } else { Some(v) })
+            .collect();
+        let optional_values = Arc::new(A::from(optional_raw_values));
+        one_column_roundtrip(filename, optional_values, true);
+    }
+
+    fn required_and_optional<A, I>(iter: I, filename: &str)
+    where
+        A: From<Vec<I::Item>> + From<Vec<Option<I::Item>>> + Array + 'static,
+        I: IntoIterator + Clone,
+    {
+        values_required::<A, I>(iter.clone(), filename);
+        values_optional::<A, I>(iter, filename);
+    }
+
+    #[test]
+    fn all_null_primitive_single_column() {
+        let values = Arc::new(Int32Array::from(vec![None; SMALL_SIZE]));
+        one_column_roundtrip("all_null_primitive_single_column", values, true);
+    }
+    #[test]
+    fn null_single_column() {
+        let values = Arc::new(NullArray::new(SMALL_SIZE));
+        one_column_roundtrip("null_single_column", values, true);
+        // null arrays are always nullable, a test with non-nullable nulls fails
+    }
+
+    #[test]
+    #[should_panic(
+        expected = "Attempting to write an Arrow type that is not yet implemented"
+    )]
+    fn bool_single_column() {
+        required_and_optional::<BooleanArray, _>(
+            [true, false].iter().cycle().copied().take(SMALL_SIZE),
+            "bool_single_column",
+        );
+    }
+
+    #[test]
+    fn i8_single_column() {
+        required_and_optional::<Int8Array, _>(0..SMALL_SIZE as i8, "i8_single_column");
+    }
+
+    #[test]
+    fn i16_single_column() {
+        required_and_optional::<Int16Array, _>(0..SMALL_SIZE as i16, "i16_single_column");
+    }
+
+    #[test]
+    fn i32_single_column() {
+        required_and_optional::<Int32Array, _>(0..SMALL_SIZE as i32, "i32_single_column");
+    }
+
+    #[test]
+    fn i64_single_column() {
+        required_and_optional::<Int64Array, _>(0..SMALL_SIZE as i64, "i64_single_column");
+    }
+
+    #[test]
+    fn u8_single_column() {
+        required_and_optional::<UInt8Array, _>(0..SMALL_SIZE as u8, "u8_single_column");
+    }
+
+    #[test]
+    fn u16_single_column() {
+        required_and_optional::<UInt16Array, _>(
+            0..SMALL_SIZE as u16,
+            "u16_single_column",
+        );
+    }
+
+    #[test]
+    fn u32_single_column() {
+        required_and_optional::<UInt32Array, _>(
+            0..SMALL_SIZE as u32,
+            "u32_single_column",
+        );
+    }
+
+    #[test]
+    fn u64_single_column() {
+        required_and_optional::<UInt64Array, _>(
+            0..SMALL_SIZE as u64,
+            "u64_single_column",
+        );
+    }
+
+    #[test]
+    fn f32_single_column() {
+        required_and_optional::<Float32Array, _>(
+            (0..SMALL_SIZE).map(|i| i as f32),
+            "f32_single_column",
+        );
+    }
+
+    #[test]
+    fn f64_single_column() {
+        required_and_optional::<Float64Array, _>(
+            (0..SMALL_SIZE).map(|i| i as f64),
+            "f64_single_column",
+        );
+    }
+
+    // The timestamp array types don't implement From<Vec<T>> because they need the timezone
+    // argument, and they also doesn't support building from a Vec<Option<T>>, so call
+    // one_column_roundtrip manually instead of calling required_and_optional for these tests.
+
+    #[test]
+    fn timestamp_second_single_column() {
+        let raw_values: Vec<_> = (0..SMALL_SIZE as i64).collect();
+        let values = Arc::new(TimestampSecondArray::from_vec(raw_values, None));
+
+        one_column_roundtrip("timestamp_second_single_column", values, false);
+    }
+
+    #[test]
+    fn timestamp_millisecond_single_column() {
+        let raw_values: Vec<_> = (0..SMALL_SIZE as i64).collect();
+        let values = Arc::new(TimestampMillisecondArray::from_vec(raw_values, None));
+
+        one_column_roundtrip("timestamp_millisecond_single_column", values, false);
+    }
+
+    #[test]
+    fn timestamp_microsecond_single_column() {
+        let raw_values: Vec<_> = (0..SMALL_SIZE as i64).collect();
+        let values = Arc::new(TimestampMicrosecondArray::from_vec(raw_values, None));
+
+        one_column_roundtrip("timestamp_microsecond_single_column", values, false);
+    }
+
+    #[test]
+    fn timestamp_nanosecond_single_column() {
+        let raw_values: Vec<_> = (0..SMALL_SIZE as i64).collect();
+        let values = Arc::new(TimestampNanosecondArray::from_vec(raw_values, None));
+
+        one_column_roundtrip("timestamp_nanosecond_single_column", values, false);
+    }
+
+    #[test]
+    fn date32_single_column() {
+        required_and_optional::<Date32Array, _>(
+            0..SMALL_SIZE as i32,
+            "date32_single_column",
+        );
+    }
+
+    #[test]
+    #[ignore] // Date support isn't correct yet
+    fn date64_single_column() {
+        required_and_optional::<Date64Array, _>(
+            0..SMALL_SIZE as i64,
+            "date64_single_column",
+        );
+    }
+
+    #[test]
+    fn time32_second_single_column() {
+        required_and_optional::<Time32SecondArray, _>(
+            0..SMALL_SIZE as i32,
+            "time32_second_single_column",
+        );
+    }
+
+    #[test]
+    fn time32_millisecond_single_column() {
+        required_and_optional::<Time32MillisecondArray, _>(
+            0..SMALL_SIZE as i32,
+            "time32_millisecond_single_column",
+        );
+    }
+
+    #[test]
+    fn time64_microsecond_single_column() {
+        required_and_optional::<Time64MicrosecondArray, _>(
+            0..SMALL_SIZE as i64,
+            "time64_microsecond_single_column",
+        );
+    }
+
+    #[test]
+    fn time64_nanosecond_single_column() {
+        required_and_optional::<Time64NanosecondArray, _>(
+            0..SMALL_SIZE as i64,
+            "time64_nanosecond_single_column",
+        );
+    }
+
+    #[test]
+    #[should_panic(expected = "Converting Duration to parquet not supported")]
+    fn duration_second_single_column() {
+        required_and_optional::<DurationSecondArray, _>(
+            0..SMALL_SIZE as i64,
+            "duration_second_single_column",
+        );
+    }
+
+    #[test]
+    #[should_panic(expected = "Converting Duration to parquet not supported")]
+    fn duration_millisecond_single_column() {
+        required_and_optional::<DurationMillisecondArray, _>(
+            0..SMALL_SIZE as i64,
+            "duration_millisecond_single_column",
+        );
+    }
+
+    #[test]
+    #[should_panic(expected = "Converting Duration to parquet not supported")]
+    fn duration_microsecond_single_column() {
+        required_and_optional::<DurationMicrosecondArray, _>(
+            0..SMALL_SIZE as i64,
+            "duration_microsecond_single_column",
+        );
+    }
+
+    #[test]
+    #[should_panic(expected = "Converting Duration to parquet not supported")]
+    fn duration_nanosecond_single_column() {
+        required_and_optional::<DurationNanosecondArray, _>(
+            0..SMALL_SIZE as i64,
+            "duration_nanosecond_single_column",
+        );
+    }
+
+    #[test]
+    #[should_panic(expected = "Currently unreachable because data type not supported")]
+    fn interval_year_month_single_column() {
+        required_and_optional::<IntervalYearMonthArray, _>(
+            0..SMALL_SIZE as i32,
+            "interval_year_month_single_column",
+        );
+    }
+
+    #[test]
+    #[should_panic(expected = "Currently unreachable because data type not supported")]
+    fn interval_day_time_single_column() {
+        required_and_optional::<IntervalDayTimeArray, _>(
+            0..SMALL_SIZE as i64,
+            "interval_day_time_single_column",
+        );
+    }
+
+    #[test]
+    #[ignore] // Binary support isn't correct yet - buffers don't match
+    fn binary_single_column() {
+        let one_vec: Vec<u8> = (0..SMALL_SIZE as u8).collect();
+        let many_vecs: Vec<_> = std::iter::repeat(one_vec).take(SMALL_SIZE).collect();
+        let many_vecs_iter = many_vecs.iter().map(|v| v.as_slice());
+
+        // BinaryArrays can't be built from Vec<Option<&str>>, so only call `values_required`
+        values_required::<BinaryArray, _>(many_vecs_iter, "binary_single_column");
+    }
+
+    #[test]
+    #[ignore] // Large binary support isn't correct yet - buffers don't match
+    fn large_binary_single_column() {
+        let one_vec: Vec<u8> = (0..SMALL_SIZE as u8).collect();
+        let many_vecs: Vec<_> = std::iter::repeat(one_vec).take(SMALL_SIZE).collect();
+        let many_vecs_iter = many_vecs.iter().map(|v| v.as_slice());
+
+        // LargeBinaryArrays can't be built from Vec<Option<&str>>, so only call `values_required`
+        values_required::<LargeBinaryArray, _>(
+            many_vecs_iter,
+            "large_binary_single_column",
+        );
+    }
+
+    #[test]
+    fn string_single_column() {
+        let raw_values: Vec<_> = (0..SMALL_SIZE).map(|i| i.to_string()).collect();
+        let raw_strs = raw_values.iter().map(|s| s.as_str());
+
+        required_and_optional::<StringArray, _>(raw_strs, "string_single_column");
+    }
+
+    #[test]
+    fn large_string_single_column() {
+        let raw_values: Vec<_> = (0..SMALL_SIZE).map(|i| i.to_string()).collect();
+        let raw_strs = raw_values.iter().map(|s| s.as_str());
+
+        required_and_optional::<LargeStringArray, _>(
+            raw_strs,
+            "large_string_single_column",
+        );
+    }
+
+    #[test]
+    #[ignore = "repetitions might be incorrect, will be addressed as part of ARROW-9728"]
+    fn list_single_column() {
+        let a_values = Int32Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
+        let a_value_offsets =
+            arrow::buffer::Buffer::from(&[0, 1, 3, 3, 6, 10].to_byte_slice());
+        let a_list_data = ArrayData::builder(DataType::List(Box::new(Field::new(
+            "item",
+            DataType::Int32,
+            true,
+        ))))
+        .len(5)
+        .add_buffer(a_value_offsets)
+        .add_child_data(a_values.data())
+        .build();
+
+        // I think this setup is incorrect because this should pass
+        assert_eq!(a_list_data.null_count(), 1);
+
+        let a = ListArray::from(a_list_data);
+        let values = Arc::new(a);
+
+        one_column_roundtrip("list_single_column", values, false);
+    }
+
+    #[test]
+    #[ignore = "repetitions might be incorrect, will be addressed as part of ARROW-9728"]
+    fn large_list_single_column() {
+        let a_values = Int32Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
+        let a_value_offsets =
+            arrow::buffer::Buffer::from(&[0i64, 1, 3, 3, 6, 10].to_byte_slice());
+        let a_list_data = ArrayData::builder(DataType::LargeList(Box::new(Field::new(
+            "large_item",
+            DataType::Int32,
+            true,
+        ))))
+        .len(5)
+        .add_buffer(a_value_offsets)
+        .add_child_data(a_values.data())
+        .build();
+
+        // I think this setup is incorrect because this should pass
+        assert_eq!(a_list_data.null_count(), 1);
+
+        let a = LargeListArray::from(a_list_data);
+        let values = Arc::new(a);
+
+        one_column_roundtrip("large_list_single_column", values, false);
+    }
+
+    #[test]
+    fn struct_single_column() {
+        let a_values = Int32Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
+        let struct_field_a = Field::new("f", DataType::Int32, false);
+        let s = StructArray::from(vec![(struct_field_a, Arc::new(a_values) as ArrayRef)]);
+
+        let values = Arc::new(s);
+        one_column_roundtrip("struct_single_column", values, false);
+    }
+
+    #[test]
+    fn arrow_writer_string_dictionary() {
+        // define schema
+        let schema = Arc::new(Schema::new(vec![Field::new_dict(
+            "dictionary",
+            DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
+            true,
+            42,
+            true,
+        )]));
+
+        // create some data
+        let d: Int32DictionaryArray = [Some("alpha"), None, Some("beta"), Some("alpha")]
+            .iter()
+            .copied()
+            .collect();
+
+        // build a record batch
+        let expected_batch = RecordBatch::try_new(schema, vec![Arc::new(d)]).unwrap();
+
+        roundtrip(
+            "test_arrow_writer_string_dictionary.parquet",
+            expected_batch,
+        );
+    }
+
+    #[test]
+    fn arrow_writer_primitive_dictionary() {
+        // define schema
+        let schema = Arc::new(Schema::new(vec![Field::new_dict(
+            "dictionary",
+            DataType::Dictionary(Box::new(DataType::UInt8), Box::new(DataType::UInt32)),
+            true,
+            42,
+            true,
+        )]));
+
+        // create some data
+        let key_builder = PrimitiveBuilder::<UInt8Type>::new(3);
+        let value_builder = PrimitiveBuilder::<UInt32Type>::new(2);
+        let mut builder = PrimitiveDictionaryBuilder::new(key_builder, value_builder);
+        builder.append(12345678).unwrap();
+        builder.append_null().unwrap();
+        builder.append(22345678).unwrap();
+        builder.append(12345678).unwrap();
+        let d = builder.finish();
+
+        // build a record batch
+        let expected_batch = RecordBatch::try_new(schema, vec![Arc::new(d)]).unwrap();
+
+        roundtrip(
+            "test_arrow_writer_primitive_dictionary.parquet",
+            expected_batch,
+        );
+    }
+
+    #[test]
+    fn arrow_writer_string_dictionary_unsigned_index() {
+        // define schema
+        let schema = Arc::new(Schema::new(vec![Field::new_dict(
+            "dictionary",
+            DataType::Dictionary(Box::new(DataType::UInt8), Box::new(DataType::Utf8)),
+            true,
+            42,
+            true,
+        )]));
+
+        // create some data
+        let d: UInt8DictionaryArray = [Some("alpha"), None, Some("beta"), Some("alpha")]
+            .iter()
+            .copied()
+            .collect();
+
+        // build a record batch
+        let expected_batch = RecordBatch::try_new(schema, vec![Arc::new(d)]).unwrap();
+
+        roundtrip(
+            "test_arrow_writer_string_dictionary_unsigned_index.parquet",
+            expected_batch,
+        );
+    }
+}
diff --git a/rust/parquet/src/arrow/converter.rs b/rust/parquet/src/arrow/converter.rs
index edc5837464f..33b29c897e6 100644
--- a/rust/parquet/src/arrow/converter.rs
+++ b/rust/parquet/src/arrow/converter.rs
@@ -15,36 +15,28 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use crate::arrow::record_reader::RecordReader;
 use crate::data_type::{ByteArray, DataType, Int96};
+// TODO: clean up imports (best done when there are few moving parts)
 use arrow::array::{
-    Array, ArrayRef, BinaryBuilder, BooleanArray, BooleanBufferBuilder,
-    BufferBuilderTrait, FixedSizeBinaryBuilder, StringBuilder,
-    TimestampNanosecondBuilder,
+    Array, ArrayRef, BinaryBuilder, FixedSizeBinaryBuilder, LargeBinaryBuilder,
+    LargeStringBuilder, PrimitiveBuilder, PrimitiveDictionaryBuilder, StringBuilder,
+    StringDictionaryBuilder, TimestampNanosecondBuilder,
 };
 use arrow::compute::cast;
 use std::convert::From;
 use std::sync::Arc;
 
 use crate::errors::Result;
-use arrow::datatypes::{ArrowPrimitiveType, DataType as ArrowDataType};
+use arrow::datatypes::{ArrowDictionaryKeyType, ArrowPrimitiveType};
 
-use arrow::array::ArrayDataBuilder;
 use arrow::array::{
-    BinaryArray, FixedSizeBinaryArray, PrimitiveArray, StringArray,
-    TimestampNanosecondArray,
+    BinaryArray, DictionaryArray, FixedSizeBinaryArray, LargeBinaryArray,
+    LargeStringArray, PrimitiveArray, StringArray, TimestampNanosecondArray,
 };
 use std::marker::PhantomData;
 
-use crate::data_type::{
-    BoolType, DoubleType as ParquetDoubleType, FloatType as ParquetFloatType,
-    Int32Type as ParquetInt32Type, Int64Type as ParquetInt64Type,
-};
-use arrow::datatypes::{
-    Date32Type, Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type,
-    TimestampMicrosecondType, TimestampMillisecondType, UInt16Type, UInt32Type,
-    UInt64Type, UInt8Type,
-};
+use crate::data_type::Int32Type as ParquetInt32Type;
+use arrow::datatypes::Int32Type;
 
 /// A converter is used to consume record reader's content and convert it to arrow
 /// primitive array.
@@ -55,83 +47,6 @@ pub trait Converter<S, T> {
     fn convert(&self, source: S) -> Result<T>;
 }
 
-/// Cast converter first converts record reader's buffer to arrow's
-/// `PrimitiveArray<ArrowSourceType>`, then casts it to `PrimitiveArray<ArrowTargetType>`.
-pub struct CastConverter<ParquetType, ArrowSourceType, ArrowTargetType> {
-    _parquet_marker: PhantomData<ParquetType>,
-    _arrow_source_marker: PhantomData<ArrowSourceType>,
-    _arrow_target_marker: PhantomData<ArrowTargetType>,
-}
-
-impl<ParquetType, ArrowSourceType, ArrowTargetType>
-    CastConverter<ParquetType, ArrowSourceType, ArrowTargetType>
-where
-    ParquetType: DataType,
-    ArrowSourceType: ArrowPrimitiveType,
-    ArrowTargetType: ArrowPrimitiveType,
-{
-    pub fn new() -> Self {
-        Self {
-            _parquet_marker: PhantomData,
-            _arrow_source_marker: PhantomData,
-            _arrow_target_marker: PhantomData,
-        }
-    }
-}
-
-impl<ParquetType, ArrowSourceType, ArrowTargetType>
-    Converter<&mut RecordReader<ParquetType>, ArrayRef>
-    for CastConverter<ParquetType, ArrowSourceType, ArrowTargetType>
-where
-    ParquetType: DataType,
-    ArrowSourceType: ArrowPrimitiveType,
-    ArrowTargetType: ArrowPrimitiveType,
-{
-    fn convert(&self, record_reader: &mut RecordReader<ParquetType>) -> Result<ArrayRef> {
-        let record_data = record_reader.consume_record_data();
-
-        let mut array_data = ArrayDataBuilder::new(ArrowSourceType::get_data_type())
-            .len(record_reader.num_values())
-            .add_buffer(record_data?);
-
-        if let Some(b) = record_reader.consume_bitmap_buffer()? {
-            array_data = array_data.null_bit_buffer(b);
-        }
-
-        let primitive_array: ArrayRef =
-            Arc::new(PrimitiveArray::<ArrowSourceType>::from(array_data.build()));
-
-        Ok(cast(&primitive_array, &ArrowTargetType::get_data_type())?)
-    }
-}
-
-pub struct BooleanArrayConverter {}
-
-impl Converter<&mut RecordReader<BoolType>, BooleanArray> for BooleanArrayConverter {
-    fn convert(
-        &self,
-        record_reader: &mut RecordReader<BoolType>,
-    ) -> Result<BooleanArray> {
-        let record_data = record_reader.consume_record_data()?;
-
-        let mut boolean_buffer = BooleanBufferBuilder::new(record_data.len());
-
-        for e in record_data.data() {
-            boolean_buffer.append(*e > 0)?;
-        }
-
-        let mut array_data = ArrayDataBuilder::new(ArrowDataType::Boolean)
-            .len(record_data.len())
-            .add_buffer(boolean_buffer.finish());
-
-        if let Some(b) = record_reader.consume_bitmap_buffer()? {
-            array_data = array_data.null_bit_buffer(b);
-        }
-
-        Ok(BooleanArray::from(array_data.build()))
-    }
-}
-
 pub struct FixedSizeArrayConverter {
     byte_width: i32,
 }
@@ -193,6 +108,27 @@ impl Converter<Vec<Option<ByteArray>>, StringArray> for Utf8ArrayConverter {
     }
 }
 
+pub struct LargeUtf8ArrayConverter {}
+
+impl Converter<Vec<Option<ByteArray>>, LargeStringArray> for LargeUtf8ArrayConverter {
+    fn convert(&self, source: Vec<Option<ByteArray>>) -> Result<LargeStringArray> {
+        let data_size = source
+            .iter()
+            .map(|x| x.as_ref().map(|b| b.len()).unwrap_or(0))
+            .sum();
+
+        let mut builder = LargeStringBuilder::with_capacity(source.len(), data_size);
+        for v in source {
+            match v {
+                Some(array) => builder.append_value(array.as_utf8()?),
+                None => builder.append_null(),
+            }?
+        }
+
+        Ok(builder.finish())
+    }
+}
+
 pub struct BinaryArrayConverter {}
 
 impl Converter<Vec<Option<ByteArray>>, BinaryArray> for BinaryArrayConverter {
@@ -209,30 +145,135 @@ impl Converter<Vec<Option<ByteArray>>, BinaryArray> for BinaryArrayConverter {
     }
 }
 
-pub type BoolConverter<'a> = ArrayRefConverter<
-    &'a mut RecordReader<BoolType>,
-    BooleanArray,
-    BooleanArrayConverter,
->;
-pub type Int8Converter = CastConverter<ParquetInt32Type, Int32Type, Int8Type>;
-pub type UInt8Converter = CastConverter<ParquetInt32Type, Int32Type, UInt8Type>;
-pub type Int16Converter = CastConverter<ParquetInt32Type, Int32Type, Int16Type>;
-pub type UInt16Converter = CastConverter<ParquetInt32Type, Int32Type, UInt16Type>;
-pub type Int32Converter = CastConverter<ParquetInt32Type, Int32Type, Int32Type>;
-pub type UInt32Converter = CastConverter<ParquetInt32Type, UInt32Type, UInt32Type>;
-pub type Int64Converter = CastConverter<ParquetInt64Type, Int64Type, Int64Type>;
-pub type Date32Converter = CastConverter<ParquetInt32Type, Date32Type, Date32Type>;
-pub type TimestampMillisecondConverter =
-    CastConverter<ParquetInt64Type, TimestampMillisecondType, TimestampMillisecondType>;
-pub type TimestampMicrosecondConverter =
-    CastConverter<ParquetInt64Type, TimestampMicrosecondType, TimestampMicrosecondType>;
-pub type UInt64Converter = CastConverter<ParquetInt64Type, UInt64Type, UInt64Type>;
-pub type Float32Converter = CastConverter<ParquetFloatType, Float32Type, Float32Type>;
-pub type Float64Converter = CastConverter<ParquetDoubleType, Float64Type, Float64Type>;
+pub struct LargeBinaryArrayConverter {}
+
+impl Converter<Vec<Option<ByteArray>>, LargeBinaryArray> for LargeBinaryArrayConverter {
+    fn convert(&self, source: Vec<Option<ByteArray>>) -> Result<LargeBinaryArray> {
+        let mut builder = LargeBinaryBuilder::new(source.len());
+        for v in source {
+            match v {
+                Some(array) => builder.append_value(array.data()),
+                None => builder.append_null(),
+            }?
+        }
+
+        Ok(builder.finish())
+    }
+}
+
+pub struct StringDictionaryArrayConverter {}
+
+impl<K: ArrowDictionaryKeyType> Converter<Vec<Option<ByteArray>>, DictionaryArray<K>>
+    for StringDictionaryArrayConverter
+{
+    fn convert(&self, source: Vec<Option<ByteArray>>) -> Result<DictionaryArray<K>> {
+        let data_size = source
+            .iter()
+            .map(|x| x.as_ref().map(|b| b.len()).unwrap_or(0))
+            .sum();
+
+        let keys_builder = PrimitiveBuilder::<K>::new(source.len());
+        let values_builder = StringBuilder::with_capacity(source.len(), data_size);
+
+        let mut builder = StringDictionaryBuilder::new(keys_builder, values_builder);
+        for v in source {
+            match v {
+                Some(array) => {
+                    let _ = builder.append(array.as_utf8()?)?;
+                }
+                None => builder.append_null()?,
+            }
+        }
+
+        Ok(builder.finish())
+    }
+}
+
+pub struct DictionaryArrayConverter<DictValueSourceType, DictValueTargetType, ParquetType>
+{
+    _dict_value_source_marker: PhantomData<DictValueSourceType>,
+    _dict_value_target_marker: PhantomData<DictValueTargetType>,
+    _parquet_marker: PhantomData<ParquetType>,
+}
+
+impl<DictValueSourceType, DictValueTargetType, ParquetType>
+    DictionaryArrayConverter<DictValueSourceType, DictValueTargetType, ParquetType>
+{
+    pub fn new() -> Self {
+        Self {
+            _dict_value_source_marker: PhantomData,
+            _dict_value_target_marker: PhantomData,
+            _parquet_marker: PhantomData,
+        }
+    }
+}
+
+impl<K, DictValueSourceType, DictValueTargetType, ParquetType>
+    Converter<Vec<Option<<ParquetType as DataType>::T>>, DictionaryArray<K>>
+    for DictionaryArrayConverter<DictValueSourceType, DictValueTargetType, ParquetType>
+where
+    K: ArrowPrimitiveType,
+    DictValueSourceType: ArrowPrimitiveType,
+    DictValueTargetType: ArrowPrimitiveType,
+    ParquetType: DataType,
+    PrimitiveArray<DictValueSourceType>: From<Vec<Option<<ParquetType as DataType>::T>>>,
+{
+    fn convert(
+        &self,
+        source: Vec<Option<<ParquetType as DataType>::T>>,
+    ) -> Result<DictionaryArray<K>> {
+        let keys_builder = PrimitiveBuilder::<K>::new(source.len());
+        let values_builder = PrimitiveBuilder::<DictValueTargetType>::new(source.len());
+
+        let mut builder = PrimitiveDictionaryBuilder::new(keys_builder, values_builder);
+
+        let source_array: Arc<dyn Array> =
+            Arc::new(PrimitiveArray::<DictValueSourceType>::from(source));
+        let target_array = cast(&source_array, &DictValueTargetType::DATA_TYPE)?;
+        let target = target_array
+            .as_any()
+            .downcast_ref::<PrimitiveArray<DictValueTargetType>>()
+            .unwrap();
+
+        for i in 0..target.len() {
+            if target.is_null(i) {
+                builder.append_null()?;
+            } else {
+                let _ = builder.append(target.value(i))?;
+            }
+        }
+
+        Ok(builder.finish())
+    }
+}
+
 pub type Utf8Converter =
     ArrayRefConverter<Vec<Option<ByteArray>>, StringArray, Utf8ArrayConverter>;
+pub type LargeUtf8Converter =
+    ArrayRefConverter<Vec<Option<ByteArray>>, LargeStringArray, LargeUtf8ArrayConverter>;
 pub type BinaryConverter =
     ArrayRefConverter<Vec<Option<ByteArray>>, BinaryArray, BinaryArrayConverter>;
+pub type LargeBinaryConverter = ArrayRefConverter<
+    Vec<Option<ByteArray>>,
+    LargeBinaryArray,
+    LargeBinaryArrayConverter,
+>;
+pub type StringDictionaryConverter<T> = ArrayRefConverter<
+    Vec<Option<ByteArray>>,
+    DictionaryArray<T>,
+    StringDictionaryArrayConverter,
+>;
+pub type DictionaryConverter<K, SV, TV, P> = ArrayRefConverter<
+    Vec<Option<<P as DataType>::T>>,
+    DictionaryArray<K>,
+    DictionaryArrayConverter<SV, TV, P>,
+>;
+pub type PrimitiveDictionaryConverter<K, V> = ArrayRefConverter<
+    Vec<Option<<ParquetInt32Type as DataType>::T>>,
+    DictionaryArray<K>,
+    DictionaryArrayConverter<Int32Type, V, ParquetInt32Type>,
+>;
+
 pub type Int96Converter =
     ArrayRefConverter<Vec<Option<Int96>>, TimestampNanosecondArray, Int96ArrayConverter>;
 pub type FixedLenBinaryConverter = ArrayRefConverter<
@@ -298,108 +339,3 @@ where
             .map(|array| Arc::new(array) as ArrayRef)
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::arrow::converter::Int16Converter;
-    use crate::arrow::record_reader::RecordReader;
-    use crate::basic::Encoding;
-    use crate::schema::parser::parse_message_type;
-    use crate::schema::types::SchemaDescriptor;
-    use crate::util::test_common::page_util::InMemoryPageReader;
-    use crate::util::test_common::page_util::{DataPageBuilder, DataPageBuilderImpl};
-    use arrow::array::ArrayEqual;
-    use arrow::array::PrimitiveArray;
-    use arrow::datatypes::{Int16Type, Int32Type};
-    use std::rc::Rc;
-
-    macro_rules! converter_arrow_source_target {
-        ($raw_data:expr, $physical_type:expr, $result_arrow_type:ty, $converter:ty) => {{
-            // Construct record reader
-            let mut record_reader = {
-                // Construct column schema
-                let message_type = &format!(
-                    "
-                message test_schema {{
-                OPTIONAL {} leaf;
-                }}
-                ",
-                    $physical_type
-                );
-
-                let def_levels = [1i16, 0i16, 1i16, 1i16];
-                build_record_reader(
-                    message_type,
-                    &[1, 2, 3],
-                    0i16,
-                    None,
-                    1i16,
-                    Some(&def_levels),
-                    10,
-                )
-            };
-
-            let array = <$converter>::new().convert(&mut record_reader).unwrap();
-            let array = array
-                .as_any()
-                .downcast_ref::<PrimitiveArray<$result_arrow_type>>()
-                .unwrap();
-
-            assert!(array.equals(&PrimitiveArray::<$result_arrow_type>::from($raw_data)));
-        }};
-    }
-
-    #[test]
-    fn test_converter_arrow_source_i16_target_i32() {
-        let raw_data = vec![Some(1i16), None, Some(2i16), Some(3i16)];
-        converter_arrow_source_target!(raw_data, "INT32", Int16Type, Int16Converter)
-    }
-
-    #[test]
-    fn test_converter_arrow_source_i32_target_date32() {
-        let raw_data = vec![Some(1i32), None, Some(2i32), Some(3i32)];
-        converter_arrow_source_target!(raw_data, "INT32", Date32Type, Date32Converter)
-    }
-
-    #[test]
-    fn test_converter_arrow_source_i32_target_i32() {
-        let raw_data = vec![Some(1i32), None, Some(2i32), Some(3i32)];
-        converter_arrow_source_target!(raw_data, "INT32", Int32Type, Int32Converter)
-    }
-
-    fn build_record_reader<T: DataType>(
-        message_type: &str,
-        values: &[T::T],
-        max_rep_level: i16,
-        rep_levels: Option<&[i16]>,
-        max_def_level: i16,
-        def_levels: Option<&[i16]>,
-        num_records: usize,
-    ) -> RecordReader<T> {
-        let desc = parse_message_type(message_type)
-            .map(|t| SchemaDescriptor::new(Rc::new(t)))
-            .map(|s| s.column(0))
-            .unwrap();
-
-        let mut record_reader = RecordReader::<T>::new(desc.clone());
-
-        // Prepare record reader
-        let mut pb = DataPageBuilderImpl::new(desc.clone(), 4, true);
-        if rep_levels.is_some() {
-            pb.add_rep_levels(max_rep_level, rep_levels.unwrap());
-        }
-        if def_levels.is_some() {
-            pb.add_def_levels(max_def_level, def_levels.unwrap());
-        }
-        pb.add_values::<T>(Encoding::PLAIN, &values);
-        let page = pb.consume();
-
-        let page_reader = Box::new(InMemoryPageReader::new(vec![page]));
-        record_reader.set_page_reader(page_reader).unwrap();
-
-        record_reader.read_records(num_records).unwrap();
-
-        record_reader
-    }
-}
diff --git a/rust/parquet/src/arrow/mod.rs b/rust/parquet/src/arrow/mod.rs
index 02f50fd3a90..979345722d2 100644
--- a/rust/parquet/src/arrow/mod.rs
+++ b/rust/parquet/src/arrow/mod.rs
@@ -35,12 +35,12 @@
 //!
 //! println!("Converted arrow schema is: {}", arrow_reader.get_schema().unwrap());
 //! println!("Arrow schema after projection is: {}",
-//!    arrow_reader.get_schema_by_columns(vec![2, 4, 6]).unwrap());
+//!    arrow_reader.get_schema_by_columns(vec![2, 4, 6], true).unwrap());
 //!
 //! let mut record_batch_reader = arrow_reader.get_record_reader(2048).unwrap();
 //!
-//! loop {
-//!    let record_batch = record_batch_reader.next_batch().unwrap().unwrap();
+//! for maybe_record_batch in record_batch_reader {
+//!    let record_batch = maybe_record_batch.unwrap();
 //!    if record_batch.num_rows() > 0 {
 //!        println!("Read {} records.", record_batch.num_rows());
 //!    } else {
@@ -51,10 +51,18 @@
 
 pub(in crate::arrow) mod array_reader;
 pub mod arrow_reader;
+pub mod arrow_writer;
 pub(in crate::arrow) mod converter;
 pub(in crate::arrow) mod record_reader;
 pub mod schema;
 
 pub use self::arrow_reader::ArrowReader;
 pub use self::arrow_reader::ParquetFileArrowReader;
-pub use self::schema::{parquet_to_arrow_schema, parquet_to_arrow_schema_by_columns};
+pub use self::arrow_writer::ArrowWriter;
+pub use self::schema::{
+    arrow_to_parquet_schema, parquet_to_arrow_schema, parquet_to_arrow_schema_by_columns,
+    parquet_to_arrow_schema_by_root_columns,
+};
+
+/// Schema metadata key used to store serialized Arrow IPC schema
+pub const ARROW_SCHEMA_META_KEY: &str = "ARROW:schema";
diff --git a/rust/parquet/src/arrow/record_reader.rs b/rust/parquet/src/arrow/record_reader.rs
index 0d112e866aa..519bd15fb0c 100644
--- a/rust/parquet/src/arrow/record_reader.rs
+++ b/rust/parquet/src/arrow/record_reader.rs
@@ -86,6 +86,7 @@ impl<'a, T> FatPtr<'a, T> {
         self.ptr
     }
 
+    #[allow(clippy::wrong_self_convention)]
     fn to_slice_mut(&mut self) -> &mut [T] {
         self.ptr
     }
@@ -123,26 +124,6 @@ impl<T: DataType> RecordReader<T> {
         }
     }
 
-    pub(crate) fn cast<U: DataType>(&mut self) -> &mut RecordReader<U> {
-        trait CastRecordReader<T: DataType, U: DataType> {
-            fn cast(&mut self) -> &mut RecordReader<U>;
-        }
-
-        impl<T: DataType, U: DataType> CastRecordReader<T, U> for RecordReader<T> {
-            default fn cast(&mut self) -> &mut RecordReader<U> {
-                panic!("Attempted to cast RecordReader to the wrong type")
-            }
-        }
-
-        impl<T: DataType> CastRecordReader<T, T> for RecordReader<T> {
-            fn cast(&mut self) -> &mut RecordReader<T> {
-                self
-            }
-        }
-
-        CastRecordReader::<T, U>::cast(self)
-    }
-
     /// Set the current page reader.
     pub fn set_page_reader(&mut self, page_reader: Box<PageReader>) -> Result<()> {
         self.column_reader =
@@ -333,7 +314,7 @@ impl<T: DataType> RecordReader<T> {
     /// Should be called after consuming data, e.g. `consume_rep_levels`,
     /// `consume_rep_levels`, `consume_record_data` and `consume_bitmap_buffer`.
     pub fn reset(&mut self) {
-        self.values_written = self.values_written - self.num_values;
+        self.values_written -= self.num_values;
         self.num_records = 0;
         self.num_values = 0;
         self.values_seen = 0;
@@ -343,7 +324,7 @@ impl<T: DataType> RecordReader<T> {
     /// Returns bitmap data.
     pub fn consume_bitmap(&mut self) -> Result<Option<Bitmap>> {
         self.consume_bitmap_buffer()
-            .map(|buffer| buffer.map(|b| Bitmap::from(b)))
+            .map(|buffer| buffer.map(Bitmap::from))
     }
 
     /// Try to read one batch of data.
@@ -592,7 +573,7 @@ mod tests {
         //   leaf: 9
         {
             let values = [8, 9];
-            let mut pb = DataPageBuilderImpl::new(desc.clone(), 2, true);
+            let mut pb = DataPageBuilderImpl::new(desc, 2, true);
             pb.add_values::<Int32Type>(Encoding::PLAIN, &values);
             let page = pb.consume();
 
@@ -678,7 +659,7 @@ mod tests {
             let values = [8];
             //empty, non-empty
             let def_levels = [0i16, 2i16];
-            let mut pb = DataPageBuilderImpl::new(desc.clone(), 2, true);
+            let mut pb = DataPageBuilderImpl::new(desc, 2, true);
             pb.add_def_levels(2, &def_levels);
             pb.add_values::<Int32Type>(Encoding::PLAIN, &values);
             let page = pb.consume();
@@ -786,7 +767,7 @@ mod tests {
             let values = [8, 9];
             let def_levels = [2i16, 2i16];
             let rep_levels = [0i16, 2i16];
-            let mut pb = DataPageBuilderImpl::new(desc.clone(), 2, true);
+            let mut pb = DataPageBuilderImpl::new(desc, 2, true);
             pb.add_rep_levels(2, &rep_levels);
             pb.add_def_levels(2, &def_levels);
             pb.add_values::<Int32Type>(Encoding::PLAIN, &values);
@@ -855,7 +836,7 @@ mod tests {
                 rep_levels[idx * 5] = 0i16;
             }
 
-            let mut pb = DataPageBuilderImpl::new(desc.clone(), 5000, true);
+            let mut pb = DataPageBuilderImpl::new(desc, 5000, true);
             pb.add_rep_levels(1, &rep_levels);
             pb.add_def_levels(1, &def_levels);
             pb.add_values::<Int32Type>(Encoding::PLAIN, &values);
diff --git a/rust/parquet/src/arrow/schema.rs b/rust/parquet/src/arrow/schema.rs
index c31f9db3583..c84e8955501 100644
--- a/rust/parquet/src/arrow/schema.rs
+++ b/rust/parquet/src/arrow/schema.rs
@@ -26,27 +26,91 @@
 use std::collections::{HashMap, HashSet};
 use std::rc::Rc;
 
+use arrow::datatypes::{DataType, DateUnit, Field, Schema, TimeUnit};
+use arrow::ipc::writer;
+
 use crate::basic::{LogicalType, Repetition, Type as PhysicalType};
 use crate::errors::{ParquetError::ArrowError, Result};
-use crate::file::metadata::KeyValue;
+use crate::file::{metadata::KeyValue, properties::WriterProperties};
 use crate::schema::types::{ColumnDescriptor, SchemaDescriptor, Type, TypePtr};
 
-use arrow::datatypes::TimeUnit;
-use arrow::datatypes::{DataType, DateUnit, Field, Schema};
-
-/// Convert parquet schema to arrow schema including optional metadata.
+/// Convert Parquet schema to Arrow schema including optional metadata.
+/// Attempts to decode any existing Arrow shcema metadata, falling back
+/// to converting the Parquet schema column-wise
 pub fn parquet_to_arrow_schema(
     parquet_schema: &SchemaDescriptor,
-    metadata: &Option<Vec<KeyValue>>,
+    key_value_metadata: &Option<Vec<KeyValue>>,
 ) -> Result<Schema> {
-    parquet_to_arrow_schema_by_columns(
-        parquet_schema,
-        0..parquet_schema.columns().len(),
-        metadata,
-    )
+    let mut metadata = parse_key_value_metadata(key_value_metadata).unwrap_or_default();
+    let arrow_schema_metadata = metadata
+        .remove(super::ARROW_SCHEMA_META_KEY)
+        .map(|encoded| get_arrow_schema_from_metadata(&encoded));
+
+    match arrow_schema_metadata {
+        Some(Some(schema)) => Ok(schema),
+        _ => parquet_to_arrow_schema_by_columns(
+            parquet_schema,
+            0..parquet_schema.columns().len(),
+            key_value_metadata,
+        ),
+    }
 }
 
-/// Convert parquet schema to arrow schema including optional metadata, only preserving some leaf columns.
+/// Convert parquet schema to arrow schema including optional metadata,
+/// only preserving some root columns.
+/// This is useful if we have columns `a.b`, `a.c.e` and `a.d`,
+/// and want `a` with all its child fields
+pub fn parquet_to_arrow_schema_by_root_columns<T>(
+    parquet_schema: &SchemaDescriptor,
+    column_indices: T,
+    key_value_metadata: &Option<Vec<KeyValue>>,
+) -> Result<Schema>
+where
+    T: IntoIterator<Item = usize>,
+{
+    // Reconstruct the index ranges of the parent columns
+    // An Arrow struct gets represented by 1+ columns based on how many child fields the
+    // struct has. This means that getting fields 1 and 2 might return the struct twice,
+    // if field 1 is the struct having say 3 fields, and field 2 is a primitive.
+    //
+    // The below gets the parent columns, and counts the number of child fields in each parent,
+    // such that we would end up with:
+    // - field 1 - columns: [0, 1, 2]
+    // - field 2 - columns: [3]
+    let mut parent_columns = vec![];
+    let mut curr_name = "";
+    let mut prev_name = "";
+    let mut indices = vec![];
+    (0..(parquet_schema.num_columns())).for_each(|i| {
+        let p_type = parquet_schema.get_column_root(i);
+        curr_name = p_type.get_basic_info().name();
+        if prev_name == "" {
+            // first index
+            indices.push(i);
+            prev_name = curr_name;
+        } else if curr_name != prev_name {
+            prev_name = curr_name;
+            parent_columns.push((curr_name.to_string(), indices.clone()));
+            indices = vec![i];
+        } else {
+            indices.push(i);
+        }
+    });
+    // push the last column if indices has values
+    if !indices.is_empty() {
+        parent_columns.push((curr_name.to_string(), indices));
+    }
+
+    // gather the required leaf columns
+    let leaf_columns = column_indices
+        .into_iter()
+        .flat_map(|i| parent_columns[i].1.clone());
+
+    parquet_to_arrow_schema_by_columns(parquet_schema, leaf_columns, key_value_metadata)
+}
+
+/// Convert parquet schema to arrow schema including optional metadata,
+/// only preserving some leaf columns.
 pub fn parquet_to_arrow_schema_by_columns<T>(
     parquet_schema: &SchemaDescriptor,
     column_indices: T,
@@ -55,40 +119,142 @@ pub fn parquet_to_arrow_schema_by_columns<T>(
 where
     T: IntoIterator<Item = usize>,
 {
+    let mut metadata = parse_key_value_metadata(key_value_metadata).unwrap_or_default();
+    let arrow_schema_metadata = metadata
+        .remove(super::ARROW_SCHEMA_META_KEY)
+        .map(|encoded| get_arrow_schema_from_metadata(&encoded))
+        .unwrap_or_default();
+
+    // add the Arrow metadata to the Parquet metadata
+    if let Some(arrow_schema) = &arrow_schema_metadata {
+        arrow_schema.metadata().iter().for_each(|(k, v)| {
+            metadata.insert(k.clone(), v.clone());
+        });
+    }
+
     let mut base_nodes = Vec::new();
     let mut base_nodes_set = HashSet::new();
     let mut leaves = HashSet::new();
 
+    enum FieldType<'a> {
+        Parquet(&'a Type),
+        Arrow(Field),
+    }
+
     for c in column_indices {
-        let column = parquet_schema.column(c).self_type() as *const Type;
-        let root = parquet_schema.get_column_root(c);
-        let root_raw_ptr = root as *const Type;
-
-        leaves.insert(column);
-        if !base_nodes_set.contains(&root_raw_ptr) {
-            base_nodes.push(root);
-            base_nodes_set.insert(root_raw_ptr);
+        let column = parquet_schema.column(c);
+        let name = column.name();
+
+        if let Some(field) = arrow_schema_metadata
+            .as_ref()
+            .and_then(|schema| schema.field_with_name(name).ok().cloned())
+        {
+            base_nodes.push(FieldType::Arrow(field));
+        } else {
+            let column = column.self_type() as *const Type;
+            let root = parquet_schema.get_column_root(c);
+            let root_raw_ptr = root as *const Type;
+
+            leaves.insert(column);
+            if !base_nodes_set.contains(&root_raw_ptr) {
+                base_nodes.push(FieldType::Parquet(root));
+                base_nodes_set.insert(root_raw_ptr);
+            }
         }
     }
 
-    let metadata = parse_key_value_metadata(key_value_metadata)
-        .map(|m| m.clone())
-        .unwrap_or(HashMap::default());
-
     base_nodes
         .into_iter()
-        .map(|t| ParquetTypeConverter::new(t, &leaves).to_field())
+        .map(|t| match t {
+            FieldType::Parquet(t) => ParquetTypeConverter::new(t, &leaves).to_field(),
+            FieldType::Arrow(f) => Ok(Some(f)),
+        })
         .collect::<Result<Vec<Option<Field>>>>()
         .map(|result| result.into_iter().filter_map(|f| f).collect::<Vec<Field>>())
         .map(|fields| Schema::new_with_metadata(fields, metadata))
 }
 
+/// Try to convert Arrow schema metadata into a schema
+fn get_arrow_schema_from_metadata(encoded_meta: &str) -> Option<Schema> {
+    let decoded = base64::decode(encoded_meta);
+    match decoded {
+        Ok(bytes) => {
+            let slice = if bytes[0..4] == [255u8; 4] {
+                &bytes[8..]
+            } else {
+                bytes.as_slice()
+            };
+            let message = arrow::ipc::get_root_as_message(slice);
+            message
+                .header_as_schema()
+                .map(arrow::ipc::convert::fb_to_schema)
+        }
+        Err(err) => {
+            // The C++ implementation returns an error if the schema can't be parsed.
+            // To prevent this, we explicitly log this, then compute the schema without the metadata
+            eprintln!(
+                "Unable to decode the encoded schema stored in {}, {:?}",
+                super::ARROW_SCHEMA_META_KEY,
+                err
+            );
+            None
+        }
+    }
+}
+
+/// Encodes the Arrow schema into the IPC format, and base64 encodes it
+fn encode_arrow_schema(schema: &Schema) -> String {
+    let options = writer::IpcWriteOptions::default();
+    let mut serialized_schema = arrow::ipc::writer::schema_to_bytes(&schema, &options);
+
+    // manually prepending the length to the schema as arrow uses the legacy IPC format
+    // TODO: change after addressing ARROW-9777
+    let schema_len = serialized_schema.ipc_message.len();
+    let mut len_prefix_schema = Vec::with_capacity(schema_len + 8);
+    len_prefix_schema.append(&mut vec![255u8, 255, 255, 255]);
+    len_prefix_schema.append((schema_len as u32).to_le_bytes().to_vec().as_mut());
+    len_prefix_schema.append(&mut serialized_schema.ipc_message);
+
+    base64::encode(&len_prefix_schema)
+}
+
+/// Mutates writer metadata by storing the encoded Arrow schema.
+/// If there is an existing Arrow schema metadata, it is replaced.
+pub(crate) fn add_encoded_arrow_schema_to_metadata(
+    schema: &Schema,
+    props: &mut WriterProperties,
+) {
+    let encoded = encode_arrow_schema(schema);
+
+    let schema_kv = KeyValue {
+        key: super::ARROW_SCHEMA_META_KEY.to_string(),
+        value: Some(encoded),
+    };
+
+    let mut meta = props.key_value_metadata.clone().unwrap_or_default();
+    // check if ARROW:schema exists, and overwrite it
+    let schema_meta = meta
+        .iter()
+        .enumerate()
+        .find(|(_, kv)| kv.key.as_str() == super::ARROW_SCHEMA_META_KEY);
+    match schema_meta {
+        Some((i, _)) => {
+            meta.remove(i);
+            meta.push(schema_kv);
+        }
+        None => {
+            meta.push(schema_kv);
+        }
+    }
+    props.key_value_metadata = Some(meta);
+}
+
 /// Convert arrow schema to parquet schema
 pub fn arrow_to_parquet_schema(schema: &Schema) -> Result<SchemaDescriptor> {
     let fields: Result<Vec<TypePtr>> = schema
         .fields()
         .iter()
-        .map(|field| arrow_to_parquet_type(field).map(|f| Rc::new(f)))
+        .map(|field| arrow_to_parquet_type(field).map(Rc::new))
         .collect();
     let group = Type::group_type_builder("arrow_schema")
         .with_fields(&mut fields?)
@@ -142,7 +308,10 @@ fn arrow_to_parquet_type(field: &Field) -> Result<Type> {
     };
     // create type from field
     match field.data_type() {
-        DataType::Null => Err(ArrowError("Null arrays not supported".to_string())),
+        DataType::Null => Type::primitive_type_builder(name, PhysicalType::INT32)
+            .with_logical_type(LogicalType::NONE)
+            .with_repetition(repetition)
+            .build(),
         DataType::Boolean => Type::primitive_type_builder(name, PhysicalType::BOOLEAN)
             .with_repetition(repetition)
             .build(),
@@ -217,34 +386,31 @@ fn arrow_to_parquet_type(field: &Field) -> Result<Type> {
             Type::primitive_type_builder(name, PhysicalType::FIXED_LEN_BYTE_ARRAY)
                 .with_logical_type(LogicalType::INTERVAL)
                 .with_repetition(repetition)
-                .with_length(3)
+                .with_length(12)
+                .build()
+        }
+        DataType::Binary | DataType::LargeBinary => {
+            Type::primitive_type_builder(name, PhysicalType::BYTE_ARRAY)
+                .with_repetition(repetition)
                 .build()
         }
-        DataType::Binary => Type::primitive_type_builder(name, PhysicalType::BYTE_ARRAY)
-            .with_repetition(repetition)
-            .build(),
         DataType::FixedSizeBinary(length) => {
             Type::primitive_type_builder(name, PhysicalType::FIXED_LEN_BYTE_ARRAY)
                 .with_repetition(repetition)
                 .with_length(*length)
                 .build()
         }
-        DataType::Utf8 => Type::primitive_type_builder(name, PhysicalType::BYTE_ARRAY)
-            .with_logical_type(LogicalType::UTF8)
-            .with_repetition(repetition)
-            .build(),
-        DataType::List(dtype) | DataType::FixedSizeList(dtype, _) => {
+        DataType::Utf8 | DataType::LargeUtf8 => {
+            Type::primitive_type_builder(name, PhysicalType::BYTE_ARRAY)
+                .with_logical_type(LogicalType::UTF8)
+                .with_repetition(repetition)
+                .build()
+        }
+        DataType::List(f) | DataType::FixedSizeList(f, _) | DataType::LargeList(f) => {
             Type::group_type_builder(name)
                 .with_fields(&mut vec![Rc::new(
                     Type::group_type_builder("list")
-                        .with_fields(&mut vec![Rc::new({
-                            let list_field = Field::new(
-                                "element",
-                                *dtype.clone(),
-                                field.is_nullable(),
-                            );
-                            arrow_to_parquet_type(&list_field)?
-                        })])
+                        .with_fields(&mut vec![Rc::new(arrow_to_parquet_type(f)?)])
                         .with_repetition(Repetition::REPEATED)
                         .build()?,
                 )])
@@ -253,9 +419,14 @@ fn arrow_to_parquet_type(field: &Field) -> Result<Type> {
                 .build()
         }
         DataType::Struct(fields) => {
+            if fields.is_empty() {
+                return Err(ArrowError(
+                    "Parquet does not support writing empty structs".to_string(),
+                ));
+            }
             // recursively convert children to types/nodes
             let fields: Result<Vec<TypePtr>> = fields
-                .into_iter()
+                .iter()
                 .map(|f| arrow_to_parquet_type(f).map(Rc::new))
                 .collect();
             Type::group_type_builder(name)
@@ -269,9 +440,6 @@ fn arrow_to_parquet_type(field: &Field) -> Result<Type> {
             let dict_field = Field::new(name, *value.clone(), field.is_nullable());
             arrow_to_parquet_type(&dict_field)
         }
-        DataType::LargeUtf8 | DataType::LargeBinary | DataType::LargeList(_) => {
-            Err(ArrowError("Large arrays not supported".to_string()))
-        }
     }
 }
 /// This struct is used to group methods and data structures used to convert parquet
@@ -364,7 +532,11 @@ impl ParquetTypeConverter<'_> {
         if self.is_self_included() {
             self.to_primitive_type_inner().map(|dt| {
                 if self.is_repeated() {
-                    Some(DataType::List(Box::new(dt)))
+                    Some(DataType::List(Box::new(Field::new(
+                        self.schema.name(),
+                        dt,
+                        self.is_nullable(),
+                    ))))
                 } else {
                     Some(dt)
                 }
@@ -431,9 +603,9 @@ impl ParquetTypeConverter<'_> {
                 ref type_length, ..
             } => *type_length,
             _ => {
-                return Err(ArrowError(format!(
-                    "Expected a physical type, not a group type"
-                )))
+                return Err(ArrowError(
+                    "Expected a physical type, not a group type".to_string(),
+                ))
             }
         };
 
@@ -458,8 +630,15 @@ impl ParquetTypeConverter<'_> {
     /// This function takes care of logical type and repetition.
     fn to_group_type(&self) -> Result<Option<DataType>> {
         if self.is_repeated() {
-            self.to_struct()
-                .map(|opt| opt.map(|dt| DataType::List(Box::new(dt))))
+            self.to_struct().map(|opt| {
+                opt.map(|dt| {
+                    DataType::List(Box::new(Field::new(
+                        self.schema.name(),
+                        dt,
+                        self.is_nullable(),
+                    )))
+                })
+            })
         } else {
             match self.schema.get_basic_info().logical_type() {
                 LogicalType::LIST => self.to_list(),
@@ -515,7 +694,7 @@ impl ParquetTypeConverter<'_> {
                 let item_type = match list_item.as_ref() {
                     Type::PrimitiveType { .. } => {
                         if item_converter.is_repeated() {
-                            item_converter.to_primitive_type_inner().map(|dt| Some(dt))
+                            item_converter.to_primitive_type_inner().map(Some)
                         } else {
                             Err(ArrowError(
                                 "Primitive element type of list must be repeated."
@@ -544,7 +723,15 @@ impl ParquetTypeConverter<'_> {
                     }
                 };
 
-                item_type.map(|opt| opt.map(|dt| DataType::List(Box::new(dt))))
+                item_type.map(|opt| {
+                    opt.map(|dt| {
+                        DataType::List(Box::new(Field::new(
+                            list_item.name(),
+                            dt,
+                            list_item.is_optional(),
+                        )))
+                    })
+                })
             }
             _ => Err(ArrowError(
                 "Group element type of list can only contain one field.".to_string(),
@@ -557,12 +744,16 @@ impl ParquetTypeConverter<'_> {
 mod tests {
     use super::*;
 
-    use std::collections::HashMap;
+    use std::{collections::HashMap, convert::TryFrom, sync::Arc};
 
-    use arrow::datatypes::{DataType, DateUnit, Field, TimeUnit};
+    use arrow::datatypes::{DataType, DateUnit, Field, IntervalUnit, TimeUnit};
 
-    use crate::file::metadata::KeyValue;
-    use crate::schema::{parser::parse_message_type, types::SchemaDescriptor};
+    use crate::file::{metadata::KeyValue, reader::SerializedFileReader};
+    use crate::{
+        arrow::{ArrowReader, ArrowWriter, ParquetFileArrowReader},
+        schema::{parser::parse_message_type, types::SchemaDescriptor},
+        util::test_common::get_temp_file,
+    };
 
     #[test]
     fn test_flat_primitives() {
@@ -714,7 +905,7 @@ mod tests {
         {
             arrow_fields.push(Field::new(
                 "my_list",
-                DataType::List(Box::new(DataType::Utf8)),
+                DataType::List(Box::new(Field::new("list", DataType::Utf8, true))),
                 false,
             ));
         }
@@ -728,7 +919,7 @@ mod tests {
         {
             arrow_fields.push(Field::new(
                 "my_list",
-                DataType::List(Box::new(DataType::Utf8)),
+                DataType::List(Box::new(Field::new("list", DataType::Utf8, true))),
                 true,
             ));
         }
@@ -746,10 +937,11 @@ mod tests {
         //   }
         // }
         {
-            let arrow_inner_list = DataType::List(Box::new(DataType::Int32));
+            let arrow_inner_list =
+                DataType::List(Box::new(Field::new("list", DataType::Int32, true)));
             arrow_fields.push(Field::new(
                 "array_of_arrays",
-                DataType::List(Box::new(arrow_inner_list)),
+                DataType::List(Box::new(Field::new("list", arrow_inner_list, true))),
                 true,
             ));
         }
@@ -763,7 +955,7 @@ mod tests {
         {
             arrow_fields.push(Field::new(
                 "my_list",
-                DataType::List(Box::new(DataType::Utf8)),
+                DataType::List(Box::new(Field::new("element", DataType::Utf8, true))),
                 true,
             ));
         }
@@ -775,7 +967,7 @@ mod tests {
         {
             arrow_fields.push(Field::new(
                 "my_list",
-                DataType::List(Box::new(DataType::Int32)),
+                DataType::List(Box::new(Field::new("element", DataType::Int32, true))),
                 true,
             ));
         }
@@ -794,7 +986,7 @@ mod tests {
             ]);
             arrow_fields.push(Field::new(
                 "my_list",
-                DataType::List(Box::new(arrow_struct)),
+                DataType::List(Box::new(Field::new("element", arrow_struct, true))),
                 true,
             ));
         }
@@ -811,7 +1003,7 @@ mod tests {
                 DataType::Struct(vec![Field::new("str", DataType::Utf8, false)]);
             arrow_fields.push(Field::new(
                 "my_list",
-                DataType::List(Box::new(arrow_struct)),
+                DataType::List(Box::new(Field::new("array", arrow_struct, true))),
                 true,
             ));
         }
@@ -828,7 +1020,7 @@ mod tests {
                 DataType::Struct(vec![Field::new("str", DataType::Utf8, false)]);
             arrow_fields.push(Field::new(
                 "my_list",
-                DataType::List(Box::new(arrow_struct)),
+                DataType::List(Box::new(Field::new("my_list_tuple", arrow_struct, true))),
                 true,
             ));
         }
@@ -838,7 +1030,7 @@ mod tests {
         {
             arrow_fields.push(Field::new(
                 "name",
-                DataType::List(Box::new(DataType::Int32)),
+                DataType::List(Box::new(Field::new("name", DataType::Int32, true))),
                 true,
             ));
         }
@@ -1004,20 +1196,24 @@ mod tests {
 
             let inner_group_list = Field::new(
                 "innerGroup",
-                DataType::List(Box::new(DataType::Struct(vec![Field::new(
-                    "leaf3",
-                    DataType::Int32,
+                DataType::List(Box::new(Field::new(
+                    "innerGroup",
+                    DataType::Struct(vec![Field::new("leaf3", DataType::Int32, true)]),
                     true,
-                )]))),
+                ))),
                 true,
             );
 
             let outer_group_list = Field::new(
                 "outerGroup",
-                DataType::List(Box::new(DataType::Struct(vec![
-                    Field::new("leaf2", DataType::Int32, true),
-                    inner_group_list,
-                ]))),
+                DataType::List(Box::new(Field::new(
+                    "outerGroup",
+                    DataType::Struct(vec![
+                        Field::new("leaf2", DataType::Int32, true),
+                        inner_group_list,
+                    ]),
+                    true,
+                ))),
                 true,
             );
             arrow_fields.push(outer_group_list);
@@ -1085,7 +1281,11 @@ mod tests {
             Field::new("double", DataType::Float64, true),
             Field::new("float", DataType::Float32, true),
             Field::new("string", DataType::Utf8, true),
-            Field::new("bools", DataType::List(Box::new(DataType::Boolean)), true),
+            Field::new(
+                "bools",
+                DataType::List(Box::new(Field::new("bools", DataType::Boolean, true))),
+                true,
+            ),
             Field::new("date", DataType::Date32(DateUnit::Day), true),
             Field::new("time_milli", DataType::Time32(TimeUnit::Millisecond), true),
             Field::new("time_micro", DataType::Time64(TimeUnit::Microsecond), true),
@@ -1151,7 +1351,11 @@ mod tests {
             Field::new("double", DataType::Float64, true),
             Field::new("float", DataType::Float32, true),
             Field::new("string", DataType::Utf8, true),
-            Field::new("bools", DataType::List(Box::new(DataType::Boolean)), true),
+            Field::new(
+                "bools",
+                DataType::List(Box::new(Field::new("element", DataType::Boolean, true))),
+                true,
+            ),
             Field::new("date", DataType::Date32(DateUnit::Day), true),
             Field::new("time_milli", DataType::Time32(TimeUnit::Millisecond), true),
             Field::new("time_micro", DataType::Time64(TimeUnit::Microsecond), true),
@@ -1170,7 +1374,15 @@ mod tests {
                 DataType::Struct(vec![
                     Field::new("bools", DataType::Boolean, false),
                     Field::new("uint32", DataType::UInt32, false),
-                    Field::new("int32", DataType::List(Box::new(DataType::Int32)), true),
+                    Field::new(
+                        "int32",
+                        DataType::List(Box::new(Field::new(
+                            "element",
+                            DataType::Int32,
+                            true,
+                        ))),
+                        true,
+                    ),
                 ]),
                 false,
             ),
@@ -1196,6 +1408,17 @@ mod tests {
             });
     }
 
+    #[test]
+    #[should_panic(expected = "Parquet does not support writing empty structs")]
+    fn test_empty_struct_field() {
+        let arrow_fields = vec![Field::new("struct", DataType::Struct(vec![]), false)];
+        let arrow_schema = Schema::new(arrow_fields);
+        let converted_arrow_schema = arrow_to_parquet_schema(&arrow_schema);
+
+        assert!(converted_arrow_schema.is_err());
+        converted_arrow_schema.unwrap();
+    }
+
     #[test]
     fn test_metadata() {
         let message_type = "
@@ -1218,4 +1441,205 @@ mod tests {
 
         assert_eq!(converted_arrow_schema.metadata(), &expected_metadata);
     }
+
+    #[test]
+    fn test_arrow_schema_roundtrip() -> Result<()> {
+        // This tests the roundtrip of an Arrow schema
+        // Fields that are commented out fail roundtrip tests or are unsupported by the writer
+        let metadata: HashMap<String, String> =
+            [("Key".to_string(), "Value".to_string())]
+                .iter()
+                .cloned()
+                .collect();
+
+        let schema = Schema::new_with_metadata(
+            vec![
+                Field::new("c1", DataType::Utf8, false),
+                Field::new("c2", DataType::Binary, false),
+                Field::new("c3", DataType::FixedSizeBinary(3), false),
+                Field::new("c4", DataType::Boolean, false),
+                Field::new("c5", DataType::Date32(DateUnit::Day), false),
+                Field::new("c6", DataType::Date64(DateUnit::Millisecond), false),
+                Field::new("c7", DataType::Time32(TimeUnit::Second), false),
+                Field::new("c8", DataType::Time32(TimeUnit::Millisecond), false),
+                Field::new("c13", DataType::Time64(TimeUnit::Microsecond), false),
+                Field::new("c14", DataType::Time64(TimeUnit::Nanosecond), false),
+                Field::new("c15", DataType::Timestamp(TimeUnit::Second, None), false),
+                Field::new(
+                    "c16",
+                    DataType::Timestamp(
+                        TimeUnit::Millisecond,
+                        Some(Arc::new("UTC".to_string())),
+                    ),
+                    false,
+                ),
+                Field::new(
+                    "c17",
+                    DataType::Timestamp(
+                        TimeUnit::Microsecond,
+                        Some(Arc::new("Africa/Johannesburg".to_string())),
+                    ),
+                    false,
+                ),
+                Field::new(
+                    "c18",
+                    DataType::Timestamp(TimeUnit::Nanosecond, None),
+                    false,
+                ),
+                Field::new("c19", DataType::Interval(IntervalUnit::DayTime), false),
+                Field::new("c20", DataType::Interval(IntervalUnit::YearMonth), false),
+                Field::new(
+                    "c21",
+                    DataType::List(Box::new(Field::new("list", DataType::Boolean, true))),
+                    false,
+                ),
+                // Field::new(
+                //     "c22",
+                //     DataType::FixedSizeList(Box::new(DataType::Boolean), 5),
+                //     false,
+                // ),
+                // Field::new(
+                //     "c23",
+                //     DataType::List(Box::new(DataType::LargeList(Box::new(
+                //         DataType::Struct(vec![
+                //             Field::new("a", DataType::Int16, true),
+                //             Field::new("b", DataType::Float64, false),
+                //         ]),
+                //     )))),
+                //     true,
+                // ),
+                Field::new(
+                    "c24",
+                    DataType::Struct(vec![
+                        Field::new("a", DataType::Utf8, false),
+                        Field::new("b", DataType::UInt16, false),
+                    ]),
+                    false,
+                ),
+                Field::new("c25", DataType::Interval(IntervalUnit::YearMonth), true),
+                Field::new("c26", DataType::Interval(IntervalUnit::DayTime), true),
+                // Field::new("c27", DataType::Duration(TimeUnit::Second), false),
+                // Field::new("c28", DataType::Duration(TimeUnit::Millisecond), false),
+                // Field::new("c29", DataType::Duration(TimeUnit::Microsecond), false),
+                // Field::new("c30", DataType::Duration(TimeUnit::Nanosecond), false),
+                Field::new_dict(
+                    "c31",
+                    DataType::Dictionary(
+                        Box::new(DataType::Int32),
+                        Box::new(DataType::Utf8),
+                    ),
+                    true,
+                    123,
+                    true,
+                ),
+                Field::new("c32", DataType::LargeBinary, true),
+                Field::new("c33", DataType::LargeUtf8, true),
+                // Field::new(
+                //     "c34",
+                //     DataType::LargeList(Box::new(DataType::List(Box::new(
+                //         DataType::Struct(vec![
+                //             Field::new("a", DataType::Int16, true),
+                //             Field::new("b", DataType::Float64, true),
+                //         ]),
+                //     )))),
+                //     true,
+                // ),
+                Field::new("c35", DataType::Null, true),
+            ],
+            metadata,
+        );
+
+        // write to an empty parquet file so that schema is serialized
+        let file = get_temp_file("test_arrow_schema_roundtrip.parquet", &[]);
+        let mut writer = ArrowWriter::try_new(
+            file.try_clone().unwrap(),
+            Arc::new(schema.clone()),
+            None,
+        )?;
+        writer.close()?;
+
+        // read file back
+        let parquet_reader = SerializedFileReader::try_from(file)?;
+        let mut arrow_reader = ParquetFileArrowReader::new(Rc::new(parquet_reader));
+        let read_schema = arrow_reader.get_schema()?;
+        assert_eq!(schema, read_schema);
+
+        // read all fields by columns
+        let partial_read_schema =
+            arrow_reader.get_schema_by_columns(0..(schema.fields().len()), false)?;
+        assert_eq!(schema, partial_read_schema);
+
+        Ok(())
+    }
+
+    #[test]
+    #[ignore = "Roundtrip of lists currently fails because we don't check their types correctly in the Arrow schema"]
+    fn test_arrow_schema_roundtrip_lists() -> Result<()> {
+        let metadata: HashMap<String, String> =
+            [("Key".to_string(), "Value".to_string())]
+                .iter()
+                .cloned()
+                .collect();
+
+        let schema = Schema::new_with_metadata(
+            vec![
+                Field::new(
+                    "c21",
+                    DataType::List(Box::new(Field::new(
+                        "array",
+                        DataType::Boolean,
+                        true,
+                    ))),
+                    false,
+                ),
+                Field::new(
+                    "c22",
+                    DataType::FixedSizeList(
+                        Box::new(Field::new("items", DataType::Boolean, false)),
+                        5,
+                    ),
+                    false,
+                ),
+                Field::new(
+                    "c23",
+                    DataType::List(Box::new(Field::new(
+                        "items",
+                        DataType::LargeList(Box::new(Field::new(
+                            "items",
+                            DataType::Struct(vec![
+                                Field::new("a", DataType::Int16, true),
+                                Field::new("b", DataType::Float64, false),
+                            ]),
+                            true,
+                        ))),
+                        true,
+                    ))),
+                    true,
+                ),
+            ],
+            metadata,
+        );
+
+        // write to an empty parquet file so that schema is serialized
+        let file = get_temp_file("test_arrow_schema_roundtrip_lists.parquet", &[]);
+        let mut writer = ArrowWriter::try_new(
+            file.try_clone().unwrap(),
+            Arc::new(schema.clone()),
+            None,
+        )?;
+        writer.close()?;
+
+        // read file back
+        let parquet_reader = SerializedFileReader::try_from(file)?;
+        let mut arrow_reader = ParquetFileArrowReader::new(Rc::new(parquet_reader));
+        let read_schema = arrow_reader.get_schema()?;
+        assert_eq!(schema, read_schema);
+
+        // read all fields by columns
+        let partial_read_schema =
+            arrow_reader.get_schema_by_columns(0..(schema.fields().len()), false)?;
+        assert_eq!(schema, partial_read_schema);
+
+        Ok(())
+    }
 }
diff --git a/rust/parquet/src/column/page.rs b/rust/parquet/src/column/page.rs
index 6adfd9d39ab..43c0c4aac4c 100644
--- a/rust/parquet/src/column/page.rs
+++ b/rust/parquet/src/column/page.rs
@@ -60,45 +60,45 @@ impl Page {
     /// Returns [`PageType`](crate::basic::PageType) for this page.
     pub fn page_type(&self) -> PageType {
         match self {
-            &Page::DataPage { .. } => PageType::DATA_PAGE,
-            &Page::DataPageV2 { .. } => PageType::DATA_PAGE_V2,
-            &Page::DictionaryPage { .. } => PageType::DICTIONARY_PAGE,
+            Page::DataPage { .. } => PageType::DATA_PAGE,
+            Page::DataPageV2 { .. } => PageType::DATA_PAGE_V2,
+            Page::DictionaryPage { .. } => PageType::DICTIONARY_PAGE,
         }
     }
 
     /// Returns internal byte buffer reference for this page.
     pub fn buffer(&self) -> &ByteBufferPtr {
         match self {
-            &Page::DataPage { ref buf, .. } => &buf,
-            &Page::DataPageV2 { ref buf, .. } => &buf,
-            &Page::DictionaryPage { ref buf, .. } => &buf,
+            Page::DataPage { ref buf, .. } => &buf,
+            Page::DataPageV2 { ref buf, .. } => &buf,
+            Page::DictionaryPage { ref buf, .. } => &buf,
         }
     }
 
     /// Returns number of values in this page.
     pub fn num_values(&self) -> u32 {
         match self {
-            &Page::DataPage { num_values, .. } => num_values,
-            &Page::DataPageV2 { num_values, .. } => num_values,
-            &Page::DictionaryPage { num_values, .. } => num_values,
+            Page::DataPage { num_values, .. } => *num_values,
+            Page::DataPageV2 { num_values, .. } => *num_values,
+            Page::DictionaryPage { num_values, .. } => *num_values,
         }
     }
 
     /// Returns this page [`Encoding`](crate::basic::Encoding).
     pub fn encoding(&self) -> Encoding {
         match self {
-            &Page::DataPage { encoding, .. } => encoding,
-            &Page::DataPageV2 { encoding, .. } => encoding,
-            &Page::DictionaryPage { encoding, .. } => encoding,
+            Page::DataPage { encoding, .. } => *encoding,
+            Page::DataPageV2 { encoding, .. } => *encoding,
+            Page::DictionaryPage { encoding, .. } => *encoding,
         }
     }
 
     /// Returns optional [`Statistics`](crate::file::metadata::Statistics).
     pub fn statistics(&self) -> Option<&Statistics> {
         match self {
-            &Page::DataPage { ref statistics, .. } => statistics.as_ref(),
-            &Page::DataPageV2 { ref statistics, .. } => statistics.as_ref(),
-            &Page::DictionaryPage { .. } => None,
+            Page::DataPage { ref statistics, .. } => statistics.as_ref(),
+            Page::DataPageV2 { ref statistics, .. } => statistics.as_ref(),
+            Page::DictionaryPage { .. } => None,
         }
     }
 }
diff --git a/rust/parquet/src/column/reader.rs b/rust/parquet/src/column/reader.rs
index 7c28d243a68..d9a49d2cb76 100644
--- a/rust/parquet/src/column/reader.rs
+++ b/rust/parquet/src/column/reader.rs
@@ -466,7 +466,7 @@ impl<T: DataType> ColumnReaderImpl<T> {
         let current_decoder = self
             .decoders
             .get_mut(&encoding)
-            .expect(format!("decoder for encoding {} should be set", encoding).as_str());
+            .unwrap_or_else(|| panic!("decoder for encoding {} should be set", encoding));
         current_decoder.get(buffer)
     }
 
@@ -562,7 +562,6 @@ mod tests {
             fn $test_func() {
                 let desc = Rc::new(ColumnDescriptor::new(
                     Rc::new($pty()),
-                    None,
                     $def_level,
                     $rep_level,
                     ColumnPath::new(Vec::new()),
@@ -902,48 +901,38 @@ mod tests {
 
     #[test]
     fn test_read_batch_values_only() {
-        test_read_batch_int32(16, &mut vec![0; 10], None, None); // < batch_size
-        test_read_batch_int32(16, &mut vec![0; 16], None, None); // == batch_size
-        test_read_batch_int32(16, &mut vec![0; 51], None, None); // > batch_size
+        test_read_batch_int32(16, &mut [0; 10], None, None); // < batch_size
+        test_read_batch_int32(16, &mut [0; 16], None, None); // == batch_size
+        test_read_batch_int32(16, &mut [0; 51], None, None); // > batch_size
     }
 
     #[test]
     fn test_read_batch_values_def_levels() {
-        test_read_batch_int32(16, &mut vec![0; 10], Some(&mut vec![0; 10]), None);
-        test_read_batch_int32(16, &mut vec![0; 16], Some(&mut vec![0; 16]), None);
-        test_read_batch_int32(16, &mut vec![0; 51], Some(&mut vec![0; 51]), None);
+        test_read_batch_int32(16, &mut [0; 10], Some(&mut [0; 10]), None);
+        test_read_batch_int32(16, &mut [0; 16], Some(&mut [0; 16]), None);
+        test_read_batch_int32(16, &mut [0; 51], Some(&mut [0; 51]), None);
     }
 
     #[test]
     fn test_read_batch_values_rep_levels() {
-        test_read_batch_int32(16, &mut vec![0; 10], None, Some(&mut vec![0; 10]));
-        test_read_batch_int32(16, &mut vec![0; 16], None, Some(&mut vec![0; 16]));
-        test_read_batch_int32(16, &mut vec![0; 51], None, Some(&mut vec![0; 51]));
+        test_read_batch_int32(16, &mut [0; 10], None, Some(&mut [0; 10]));
+        test_read_batch_int32(16, &mut [0; 16], None, Some(&mut [0; 16]));
+        test_read_batch_int32(16, &mut [0; 51], None, Some(&mut [0; 51]));
     }
 
     #[test]
     fn test_read_batch_different_buf_sizes() {
-        test_read_batch_int32(
-            16,
-            &mut vec![0; 8],
-            Some(&mut vec![0; 9]),
-            Some(&mut vec![0; 7]),
-        );
-        test_read_batch_int32(
-            16,
-            &mut vec![0; 1],
-            Some(&mut vec![0; 9]),
-            Some(&mut vec![0; 3]),
-        );
+        test_read_batch_int32(16, &mut [0; 8], Some(&mut [0; 9]), Some(&mut [0; 7]));
+        test_read_batch_int32(16, &mut [0; 1], Some(&mut [0; 9]), Some(&mut [0; 3]));
     }
 
     #[test]
     fn test_read_batch_values_def_rep_levels() {
         test_read_batch_int32(
             128,
-            &mut vec![0; 128],
-            Some(&mut vec![0; 128]),
-            Some(&mut vec![0; 128]),
+            &mut [0; 128],
+            Some(&mut [0; 128]),
+            Some(&mut [0; 128]),
         );
     }
 
@@ -958,7 +947,6 @@ mod tests {
         let primitive_type = get_test_int32_type();
         let desc = Rc::new(ColumnDescriptor::new(
             Rc::new(primitive_type),
-            None,
             1,
             1,
             ColumnPath::new(Vec::new()),
@@ -1076,7 +1064,6 @@ mod tests {
 
         let desc = Rc::new(ColumnDescriptor::new(
             Rc::new(primitive_type),
-            None,
             max_def_level,
             max_rep_level,
             ColumnPath::new(Vec::new()),
diff --git a/rust/parquet/src/column/writer.rs b/rust/parquet/src/column/writer.rs
index f26c37bc2a4..70bbe594770 100644
--- a/rust/parquet/src/column/writer.rs
+++ b/rust/parquet/src/column/writer.rs
@@ -16,7 +16,7 @@
 // under the License.
 
 //! Contains column writer API.
-use std::{cmp, collections::VecDeque, rc::Rc};
+use std::{cmp, collections::VecDeque, convert::TryFrom, rc::Rc};
 
 use crate::basic::{Compression, Encoding, PageType, Type};
 use crate::column::page::{CompressedPage, Page, PageWriteSpec, PageWriter};
@@ -57,7 +57,6 @@ pub enum Level {
 macro_rules! gen_stats_section {
     ($physical_ty: ty, $stat_fn: ident, $min: ident, $max: ident, $distinct: ident, $nulls: ident) => {{
         let min = $min.as_ref().and_then(|v| {
-            println!("min: {:?} {}", &v.as_bytes(), v.as_bytes().len());
             Some(read_num_bytes!(
                 $physical_ty,
                 v.as_bytes().len(),
@@ -65,7 +64,6 @@ macro_rules! gen_stats_section {
             ))
         });
         let max = $max.as_ref().and_then(|v| {
-            println!("max: {:?} {}", &v.as_bytes(), v.as_bytes().len());
             Some(read_num_bytes!(
                 $physical_ty,
                 v.as_bytes().len(),
@@ -226,7 +224,7 @@ impl<T: DataType> ColumnWriterImpl<T> {
             descr.clone(),
             props
                 .encoding(descr.path())
-                .unwrap_or(Self::fallback_encoding(&props)),
+                .unwrap_or_else(|| Self::fallback_encoding(&props)),
             Rc::new(MemTracker::new()),
         )
         .unwrap();
@@ -297,20 +295,13 @@ impl<T: DataType> ColumnWriterImpl<T> {
         let write_batch_size = self.props.write_batch_size();
         let num_batches = min_len / write_batch_size;
 
-        let mut values_offset = 0;
-        let mut levels_offset = 0;
-
         // Process pre-calculated statistics
         match (min, max) {
             (Some(min), Some(max)) => {
-                if self.min_column_value.is_none()
-                    || self.min_column_value.as_ref().unwrap() > min
-                {
+                if self.min_column_value.as_ref().map_or(true, |v| v > min) {
                     self.min_column_value = Some(min.clone());
                 }
-                if self.max_column_value.is_none()
-                    || self.max_column_value.as_ref().unwrap() < max
-                {
+                if self.max_column_value.as_ref().map_or(true, |v| v < max) {
                     self.max_column_value = Some(max.clone());
                 }
             }
@@ -333,6 +324,8 @@ impl<T: DataType> ColumnWriterImpl<T> {
             && null_count.is_none()
             && distinct_count.is_none();
 
+        let mut values_offset = 0;
+        let mut levels_offset = 0;
         for _ in 0..num_batches {
             values_offset += self.write_mini_batch(
                 &values[values_offset..values_offset + write_batch_size],
@@ -437,14 +430,11 @@ impl<T: DataType> ColumnWriterImpl<T> {
         rep_levels: Option<&[i16]>,
         calculate_page_stats: bool,
     ) -> Result<usize> {
-        let num_values;
         let mut values_to_write = 0;
 
         // Check if number of definition levels is the same as number of repetition
         // levels.
-        if def_levels.is_some() && rep_levels.is_some() {
-            let def = def_levels.unwrap();
-            let rep = rep_levels.unwrap();
+        if let (Some(def), Some(rep)) = (def_levels, rep_levels) {
             if def.len() != rep.len() {
                 return Err(general_err!(
                     "Inconsistent length of definition and repetition levels: {} != {}",
@@ -455,44 +445,40 @@ impl<T: DataType> ColumnWriterImpl<T> {
         }
 
         // Process definition levels and determine how many values to write.
-        if self.descr.max_def_level() > 0 {
-            if def_levels.is_none() {
-                return Err(general_err!(
+        let num_values = if self.descr.max_def_level() > 0 {
+            let levels = def_levels.ok_or_else(|| {
+                general_err!(
                     "Definition levels are required, because max definition level = {}",
                     self.descr.max_def_level()
-                ));
-            }
+                )
+            })?;
 
-            let levels = def_levels.unwrap();
-            num_values = levels.len();
             for &level in levels {
                 if level == self.descr.max_def_level() {
                     values_to_write += 1;
-                } else {
-                    if calculate_page_stats {
-                        self.num_page_nulls += 1
-                    };
+                } else if calculate_page_stats {
+                    self.num_page_nulls += 1
                 }
             }
 
             self.write_definition_levels(levels);
+            u32::try_from(levels.len()).unwrap()
         } else {
             values_to_write = values.len();
-            num_values = values_to_write;
-        }
+            u32::try_from(values_to_write).unwrap()
+        };
 
         // Process repetition levels and determine how many rows we are about to process.
         if self.descr.max_rep_level() > 0 {
             // A row could contain more than one value.
-            if rep_levels.is_none() {
-                return Err(general_err!(
+            let levels = rep_levels.ok_or_else(|| {
+                general_err!(
                     "Repetition levels are required, because max repetition level = {}",
                     self.descr.max_rep_level()
-                ));
-            }
+                )
+            })?;
 
             // Count the occasions where we start a new row
-            let levels = rep_levels.unwrap();
             for &level in levels {
                 self.num_buffered_rows += (level == 0) as u32
             }
@@ -501,28 +487,28 @@ impl<T: DataType> ColumnWriterImpl<T> {
         } else {
             // Each value is exactly one row.
             // Equals to the number of values, we count nulls as well.
-            self.num_buffered_rows += num_values as u32;
+            self.num_buffered_rows += num_values;
         }
 
         // Check that we have enough values to write.
-        if values.len() < values_to_write {
-            return Err(general_err!(
+        let values_to_write = values.get(0..values_to_write).ok_or_else(|| {
+            general_err!(
                 "Expected to write {} values, but have only {}",
                 values_to_write,
                 values.len()
-            ));
-        }
+            )
+        })?;
 
         if calculate_page_stats {
-            for val in &values[0..values_to_write] {
+            for val in values_to_write {
                 self.update_page_min_max(val);
             }
         }
 
-        self.write_values(&values[0..values_to_write])?;
+        self.write_values(values_to_write)?;
 
-        self.num_buffered_values += num_values as u32;
-        self.num_buffered_encoded_values += values_to_write as u32;
+        self.num_buffered_values += num_values;
+        self.num_buffered_encoded_values += u32::try_from(values_to_write.len()).unwrap();
 
         if self.should_add_data_page() {
             self.add_data_page(calculate_page_stats)?;
@@ -532,7 +518,7 @@ impl<T: DataType> ColumnWriterImpl<T> {
             self.dict_fallback()?;
         }
 
-        Ok(values_to_write)
+        Ok(values_to_write.len())
     }
 
     #[inline]
@@ -610,13 +596,13 @@ impl<T: DataType> ColumnWriterImpl<T> {
         let max_def_level = self.descr.max_def_level();
         let max_rep_level = self.descr.max_rep_level();
 
-        let mut page_statistics: Option<Statistics> = None;
-
-        if calculate_page_stat {
+        let page_statistics = if calculate_page_stat {
             self.update_column_min_max();
             self.num_column_nulls += self.num_page_nulls;
-            page_statistics = Some(self.make_page_statistics());
-        }
+            Some(self.make_page_statistics())
+        } else {
+            None
+        };
 
         let compressed_page = match self.props.writer_version() {
             WriterVersion::PARQUET_1_0 => {
@@ -687,14 +673,9 @@ impl<T: DataType> ColumnWriterImpl<T> {
                 // Data Page v2 compresses values only.
                 match self.compressor {
                     Some(ref mut cmpr) => {
-                        let mut compressed_buf =
-                            Vec::with_capacity(value_bytes.data().len());
-                        cmpr.compress(value_bytes.data(), &mut compressed_buf)?;
-                        buffer.extend_from_slice(&compressed_buf[..]);
-                    }
-                    None => {
-                        buffer.extend_from_slice(value_bytes.data());
+                        cmpr.compress(value_bytes.data(), &mut buffer)?;
                     }
+                    None => buffer.extend_from_slice(value_bytes.data()),
                 }
 
                 let data_page = Page::DataPageV2 {
@@ -838,12 +819,12 @@ impl<T: DataType> ColumnWriterImpl<T> {
     /// Writes dictionary page into underlying sink.
     #[inline]
     fn write_dictionary_page(&mut self) -> Result<()> {
-        if self.dict_encoder.is_none() {
-            return Err(general_err!("Dictionary encoder is not set"));
-        }
-
         let compressed_page = {
-            let encoder = self.dict_encoder.as_ref().unwrap();
+            let encoder = self
+                .dict_encoder
+                .as_ref()
+                .ok_or_else(|| general_err!("Dictionary encoder is not set"))?;
+
             let is_sorted = encoder.is_sorted();
             let num_values = encoder.num_entries();
             let mut values_buf = encoder.write_dict()?;
@@ -931,36 +912,34 @@ impl<T: DataType> ColumnWriterImpl<T> {
             Type::FLOAT => gen_stats_section!(f32, float, min, max, distinct, nulls),
             Type::DOUBLE => gen_stats_section!(f64, double, min, max, distinct, nulls),
             Type::BYTE_ARRAY | Type::FIXED_LEN_BYTE_ARRAY => {
-                let min = min
-                    .as_ref()
-                    .and_then(|v| Some(ByteArray::from(v.as_bytes().to_vec())));
-                let max = max
-                    .as_ref()
-                    .and_then(|v| Some(ByteArray::from(v.as_bytes().to_vec())));
+                let min = min.as_ref().map(|v| ByteArray::from(v.as_bytes().to_vec()));
+                let max = max.as_ref().map(|v| ByteArray::from(v.as_bytes().to_vec()));
                 Statistics::byte_array(min, max, distinct, nulls, false)
             }
         }
     }
 
     fn update_page_min_max(&mut self, val: &T::T) {
-        if self.min_page_value.is_none() || self.min_page_value.as_ref().unwrap() > val {
+        if self.min_page_value.as_ref().map_or(true, |min| min > val) {
             self.min_page_value = Some(val.clone());
         }
-        if self.max_page_value.is_none() || self.max_page_value.as_ref().unwrap() < val {
+        if self.max_page_value.as_ref().map_or(true, |max| max < val) {
             self.max_page_value = Some(val.clone());
         }
     }
 
     fn update_column_min_max(&mut self) {
-        if self.min_column_value.is_none()
-            || self.min_column_value.as_ref().unwrap()
-                > self.min_page_value.as_ref().unwrap()
+        if self
+            .min_column_value
+            .as_ref()
+            .map_or(true, |min| min > self.min_page_value.as_ref().unwrap())
         {
             self.min_column_value = self.min_page_value.clone();
         }
-        if self.max_column_value.is_none()
-            || self.max_column_value.as_ref().unwrap()
-                < self.max_page_value.as_ref().unwrap()
+        if self
+            .max_column_value
+            .as_ref()
+            .map_or(true, |max| max < self.max_page_value.as_ref().unwrap())
         {
             self.max_column_value = self.max_page_value.clone();
         }
@@ -1473,10 +1452,10 @@ mod tests {
                 assert_eq!(stats.min(), &1);
                 assert_eq!(stats.max(), &4);
             } else {
-                assert!(false, "expecting Statistics::Int32");
+                panic!("expecting Statistics::Int32");
             }
         } else {
-            assert!(false, "metadata missing statistics");
+            panic!("metadata missing statistics");
         }
     }
 
@@ -1517,10 +1496,10 @@ mod tests {
                 assert_eq!(stats.min(), &-17);
                 assert_eq!(stats.max(), &9000);
             } else {
-                assert!(false, "expecting Statistics::Int32");
+                panic!("expecting Statistics::Int32");
             }
         } else {
-            assert!(false, "metadata missing statistics");
+            panic!("metadata missing statistics");
         }
     }
 
@@ -1591,8 +1570,8 @@ mod tests {
 
     #[test]
     fn test_column_writer_small_write_batch_size() {
-        for i in vec![1, 2, 5, 10, 11, 1023] {
-            let props = WriterProperties::builder().set_write_batch_size(i).build();
+        for i in &[1usize, 2, 5, 10, 11, 1023] {
+            let props = WriterProperties::builder().set_write_batch_size(*i).build();
 
             column_roundtrip_random::<Int32Type>(
                 "test_col_writer_rnd_5",
@@ -1722,8 +1701,8 @@ mod tests {
     /// Performs write-read roundtrip with randomly generated values and levels.
     /// `max_size` is maximum number of values or levels (if `max_def_level` > 0) to write
     /// for a column.
-    fn column_roundtrip_random<'a, T: DataType>(
-        file_name: &'a str,
+    fn column_roundtrip_random<T: DataType>(
+        file_name: &str,
         props: WriterProperties,
         max_size: usize,
         min_value: T::T,
@@ -1942,7 +1921,7 @@ mod tests {
             .with_length(1)
             .build()
             .unwrap();
-        ColumnDescriptor::new(Rc::new(tpe), None, max_def_level, max_rep_level, path)
+        ColumnDescriptor::new(Rc::new(tpe), max_def_level, max_rep_level, path)
     }
 
     /// Returns page writer that collects pages without serializing them.
diff --git a/rust/parquet/src/compression.rs b/rust/parquet/src/compression.rs
index 9b4ac71e407..67cb275e2e1 100644
--- a/rust/parquet/src/compression.rs
+++ b/rust/parquet/src/compression.rs
@@ -114,12 +114,13 @@ mod snappy_codec {
         }
 
         fn compress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<()> {
+            let output_buf_len = output_buf.len();
             let required_len = max_compress_len(input_buf.len());
-            if output_buf.len() < required_len {
-                output_buf.resize(required_len, 0);
-            }
-            let n = self.encoder.compress(input_buf, &mut output_buf[..])?;
-            output_buf.truncate(n);
+            output_buf.resize(output_buf_len + required_len, 0);
+            let n = self
+                .encoder
+                .compress(input_buf, &mut output_buf[output_buf_len..])?;
+            output_buf.truncate(output_buf_len + n);
             Ok(())
         }
     }
@@ -324,14 +325,14 @@ mod tests {
 
     use crate::util::test_common::*;
 
-    fn test_roundtrip(c: CodecType, data: &Vec<u8>) {
+    fn test_roundtrip(c: CodecType, data: &[u8]) {
         let mut c1 = create_codec(c).unwrap().unwrap();
         let mut c2 = create_codec(c).unwrap().unwrap();
 
         // Compress with c1
         let mut compressed = Vec::new();
         let mut decompressed = Vec::new();
-        c1.compress(data.as_slice(), &mut compressed)
+        c1.compress(data, &mut compressed)
             .expect("Error when compressing");
 
         // Decompress with c2
@@ -340,12 +341,12 @@ mod tests {
             .expect("Error when decompressing");
         assert_eq!(data.len(), decompressed_size);
         decompressed.truncate(decompressed_size);
-        assert_eq!(*data, decompressed);
+        assert_eq!(data, decompressed.as_slice());
 
         compressed.clear();
 
         // Compress with c2
-        c2.compress(data.as_slice(), &mut compressed)
+        c2.compress(data, &mut compressed)
             .expect("Error when compressing");
 
         // Decompress with c1
@@ -354,14 +355,14 @@ mod tests {
             .expect("Error when decompressing");
         assert_eq!(data.len(), decompressed_size);
         decompressed.truncate(decompressed_size);
-        assert_eq!(*data, decompressed);
+        assert_eq!(data, decompressed.as_slice());
     }
 
     fn test_codec(c: CodecType) {
         let sizes = vec![100, 10000, 100000];
         for size in sizes {
-            let mut data = random_bytes(size);
-            test_roundtrip(c, &mut data);
+            let data = random_bytes(size);
+            test_roundtrip(c, &data);
         }
     }
 
diff --git a/rust/parquet/src/data_type.rs b/rust/parquet/src/data_type.rs
index 738057a5b87..0dbf185b733 100644
--- a/rust/parquet/src/data_type.rs
+++ b/rust/parquet/src/data_type.rs
@@ -65,9 +65,8 @@ impl Int96 {
         let day = self.data()[2] as i64;
         let nanoseconds = ((self.data()[1] as i64) << 32) + self.data()[0] as i64;
         let seconds = (day - JULIAN_DAY_OF_EPOCH) * SECONDS_PER_DAY;
-        let millis = seconds * MILLIS_PER_SECOND + nanoseconds / 1_000_000;
 
-        millis
+        seconds * MILLIS_PER_SECOND + nanoseconds / 1_000_000
     }
 }
 
@@ -118,7 +117,7 @@ impl PartialOrd for ByteArray {
                         return Some(Ordering::Less);
                     }
                 }
-                return Some(Ordering::Equal);
+                Some(Ordering::Equal)
             }
         } else {
             None
@@ -138,6 +137,11 @@ impl ByteArray {
         self.data.as_ref().unwrap().len()
     }
 
+    /// Checks if the underlying buffer is empty.
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
     /// Returns slice of data.
     pub fn data(&self) -> &[u8] {
         assert!(self.data.is_some());
@@ -629,10 +633,12 @@ impl FromBytes for ByteArray {
 }
 
 #[cfg(test)]
+#[allow(clippy::float_cmp, clippy::approx_constant)]
 mod tests {
     use super::*;
 
     #[test]
+    #[allow(clippy::string_lit_as_bytes)]
     fn test_as_bytes() {
         assert_eq!(false.as_bytes(), &[0]);
         assert_eq!(true.as_bytes(), &[1]);
diff --git a/rust/parquet/src/encodings/decoding.rs b/rust/parquet/src/encodings/decoding.rs
index 647d423f72e..46ed4fb58d1 100644
--- a/rust/parquet/src/encodings/decoding.rs
+++ b/rust/parquet/src/encodings/decoding.rs
@@ -530,14 +530,14 @@ impl<T: DataType> DeltaBitPackDecoder<T> {
         self.min_delta = self
             .bit_reader
             .get_zigzag_vlq_int()
-            .ok_or(eof_err!("Not enough data to decode 'min_delta'"))?;
+            .ok_or_else(|| eof_err!("Not enough data to decode 'min_delta'"))?;
 
         let mut widths = vec![];
         for _ in 0..self.num_mini_blocks {
             let w = self
                 .bit_reader
                 .get_aligned::<u8>(1)
-                .ok_or(eof_err!("Not enough data to decode 'width'"))?;
+                .ok_or_else(|| eof_err!("Not enough data to decode 'width'"))?;
             widths.push(w);
         }
 
@@ -569,7 +569,7 @@ impl<T: DataType> DeltaBitPackDecoder<T> {
                 let delta = self
                     .bit_reader
                     .get_value::<T::T>(self.delta_bit_width as usize)
-                    .ok_or(eof_err!("Not enough data to decode 'delta'"))?;
+                    .ok_or_else(|| eof_err!("Not enough data to decode 'delta'"))?;
                 self.deltas_in_mini_block.push(delta);
             }
         }
@@ -591,20 +591,20 @@ where
         let block_size = self
             .bit_reader
             .get_vlq_int()
-            .ok_or(eof_err!("Not enough data to decode 'block_size'"))?;
+            .ok_or_else(|| eof_err!("Not enough data to decode 'block_size'"))?;
         self.num_mini_blocks = self
             .bit_reader
             .get_vlq_int()
-            .ok_or(eof_err!("Not enough data to decode 'num_mini_blocks'"))?;
+            .ok_or_else(|| eof_err!("Not enough data to decode 'num_mini_blocks'"))?;
         self.num_values = self
             .bit_reader
             .get_vlq_int()
-            .ok_or(eof_err!("Not enough data to decode 'num_values'"))?
+            .ok_or_else(|| eof_err!("Not enough data to decode 'num_values'"))?
             as usize;
         self.first_value = self
             .bit_reader
             .get_zigzag_vlq_int()
-            .ok_or(eof_err!("Not enough data to decode 'first_value'"))?;
+            .ok_or_else(|| eof_err!("Not enough data to decode 'first_value'"))?;
 
         // Reset decoding state
         self.first_value_read = false;
@@ -940,6 +940,7 @@ impl Decoder<FixedLenByteArrayType> for DeltaByteArrayDecoder<FixedLenByteArrayT
 }
 
 #[cfg(test)]
+#[allow(clippy::approx_constant)]
 mod tests {
     use super::{super::encoding::*, *};
 
@@ -1163,7 +1164,7 @@ mod tests {
     #[should_panic(expected = "RleValueEncoder only supports BoolType")]
     fn test_rle_value_encode_int32_not_supported() {
         let mut encoder = RleValueEncoder::<Int32Type>::new();
-        encoder.put(&vec![1, 2, 3, 4]).unwrap();
+        encoder.put(&[1, 2, 3, 4]).unwrap();
     }
 
     #[test]
@@ -1402,8 +1403,7 @@ mod tests {
         let expected: Vec<T::T> = data.iter().flat_map(|s| s.clone()).collect();
 
         // Decode data and compare with original
-        let mut decoder =
-            get_decoder::<T>(col_descr.clone(), encoding).expect("get decoder");
+        let mut decoder = get_decoder::<T>(col_descr, encoding).expect("get decoder");
 
         let mut result = vec![T::T::default(); expected.len()];
         decoder
@@ -1445,7 +1445,6 @@ mod tests {
             .unwrap();
         Rc::new(ColumnDescriptor::new(
             Rc::new(ty),
-            None,
             0,
             0,
             ColumnPath::new(vec![]),
diff --git a/rust/parquet/src/encodings/encoding.rs b/rust/parquet/src/encodings/encoding.rs
index 98c61a05329..8660168a5d1 100644
--- a/rust/parquet/src/encodings/encoding.rs
+++ b/rust/parquet/src/encodings/encoding.rs
@@ -139,7 +139,7 @@ impl<T: DataType> Encoder<T> for PlainEncoder<T> {
 
     #[inline]
     default fn flush_buffer(&mut self) -> Result<ByteBufferPtr> {
-        self.buffer.write(self.bit_writer.flush_buffer())?;
+        self.buffer.write_all(self.bit_writer.flush_buffer())?;
         self.buffer.flush()?;
         self.bit_writer.clear();
 
@@ -157,7 +157,7 @@ where
 {
     default fn put(&mut self, values: &[T::T]) -> Result<()> {
         let bytes = T::T::slice_as_bytes(values);
-        self.buffer.write(bytes)?;
+        self.buffer.write_all(bytes)?;
         Ok(())
     }
 }
@@ -174,7 +174,7 @@ impl Encoder<BoolType> for PlainEncoder<BoolType> {
 impl Encoder<Int96Type> for PlainEncoder<Int96Type> {
     fn put(&mut self, values: &[Int96]) -> Result<()> {
         for v in values {
-            self.buffer.write(v.as_bytes())?;
+            self.buffer.write_all(v.as_bytes())?;
         }
         self.buffer.flush()?;
         Ok(())
@@ -184,8 +184,9 @@ impl Encoder<Int96Type> for PlainEncoder<Int96Type> {
 impl Encoder<ByteArrayType> for PlainEncoder<ByteArrayType> {
     fn put(&mut self, values: &[ByteArray]) -> Result<()> {
         for v in values {
-            self.buffer.write(&(v.len().to_le() as u32).as_bytes())?;
-            self.buffer.write(v.data())?;
+            self.buffer
+                .write_all(&(v.len().to_le() as u32).as_bytes())?;
+            self.buffer.write_all(v.data())?;
         }
         self.buffer.flush()?;
         Ok(())
@@ -195,7 +196,7 @@ impl Encoder<ByteArrayType> for PlainEncoder<ByteArrayType> {
 impl Encoder<FixedLenByteArrayType> for PlainEncoder<FixedLenByteArrayType> {
     fn put(&mut self, values: &[ByteArray]) -> Result<()> {
         for v in values {
-            self.buffer.write(v.data())?;
+            self.buffer.write_all(v.data())?;
         }
         self.buffer.flush()?;
         Ok(())
@@ -303,7 +304,7 @@ impl<T: DataType> DictEncoder<T> {
         self.mem_tracker.alloc(buffer.capacity() as i64);
 
         // Write bit width in the first byte
-        buffer.write((self.bit_width() as u8).as_bytes())?;
+        buffer.write_all((self.bit_width() as u8).as_bytes())?;
         let mut encoder = RleEncoder::new_from_buf(self.bit_width(), buffer, 1);
         for index in self.buffered_indices.data() {
             if !encoder.put(*index as u64)? {
@@ -691,15 +692,14 @@ impl<T: DataType> Encoder<T> for DeltaBitPackEncoder<T> {
             return Ok(());
         }
 
-        let mut idx;
         // Define values to encode, initialize state
-        if self.total_values == 0 {
+        let mut idx = if self.total_values == 0 {
             self.first_value = self.as_i64(values, 0);
             self.current_value = self.first_value;
-            idx = 1;
+            1
         } else {
-            idx = 0;
-        }
+            0
+        };
         // Add all values (including first value)
         self.total_values += values.len();
 
@@ -732,8 +732,8 @@ impl<T: DataType> Encoder<T> for DeltaBitPackEncoder<T> {
         self.write_page_header();
 
         let mut buffer = ByteBuffer::new();
-        buffer.write(self.page_header_writer.flush_buffer())?;
-        buffer.write(self.bit_writer.flush_buffer())?;
+        buffer.write_all(self.page_header_writer.flush_buffer())?;
+        buffer.write_all(self.bit_writer.flush_buffer())?;
         buffer.flush()?;
 
         // Reset state
@@ -1173,27 +1173,20 @@ mod tests {
         }
 
         // PLAIN
-        run_test::<Int32Type>(Encoding::PLAIN, -1, &vec![123; 1024], 0, 4096, 0);
+        run_test::<Int32Type>(Encoding::PLAIN, -1, &[123; 1024], 0, 4096, 0);
 
         // DICTIONARY
         // NOTE: The final size is almost the same because the dictionary entries are
         // preserved after encoded values have been written.
-        run_test::<Int32Type>(Encoding::RLE_DICTIONARY, -1, &vec![123, 1024], 11, 68, 66);
+        run_test::<Int32Type>(Encoding::RLE_DICTIONARY, -1, &[123, 1024], 11, 68, 66);
 
         // DELTA_BINARY_PACKED
-        run_test::<Int32Type>(
-            Encoding::DELTA_BINARY_PACKED,
-            -1,
-            &vec![123; 1024],
-            0,
-            35,
-            0,
-        );
+        run_test::<Int32Type>(Encoding::DELTA_BINARY_PACKED, -1, &[123; 1024], 0, 35, 0);
 
         // RLE
         let mut values = vec![];
-        values.extend_from_slice(&vec![true; 16]);
-        values.extend_from_slice(&vec![false; 16]);
+        values.extend_from_slice(&[true; 16]);
+        values.extend_from_slice(&[false; 16]);
         run_test::<BoolType>(Encoding::RLE, -1, &values, 0, 2, 0);
 
         // DELTA_LENGTH_BYTE_ARRAY
@@ -1254,7 +1247,7 @@ mod tests {
                 Encoding::PLAIN_DICTIONARY | Encoding::RLE_DICTIONARY => {
                     Self::test_dict_internal(total, type_length)
                 }
-                enc @ _ => Self::test_internal(enc, total, type_length),
+                enc => Self::test_internal(enc, total, type_length),
             };
 
             assert!(
@@ -1396,7 +1389,6 @@ mod tests {
             .unwrap();
         Rc::new(ColumnDescriptor::new(
             Rc::new(ty),
-            None,
             0,
             0,
             ColumnPath::new(vec![]),
diff --git a/rust/parquet/src/encodings/levels.rs b/rust/parquet/src/encodings/levels.rs
index 93de6b1d546..6727589f17e 100644
--- a/rust/parquet/src/encodings/levels.rs
+++ b/rust/parquet/src/encodings/levels.rs
@@ -408,14 +408,11 @@ mod tests {
         let mut found_err = false;
         // Insert a large number of values, so we run out of space
         for _ in 0..100 {
-            match encoder.put(&levels) {
-                Err(err) => {
-                    assert!(format!("{}", err).contains("Not enough bytes left"));
-                    found_err = true;
-                    break;
-                }
-                Ok(_) => {}
-            }
+            if let Err(err) = encoder.put(&levels) {
+                assert!(format!("{}", err).contains("Not enough bytes left"));
+                found_err = true;
+                break;
+            };
         }
         if !found_err {
             panic!("Failed test: no buffer overflow");
diff --git a/rust/parquet/src/encodings/rle.rs b/rust/parquet/src/encodings/rle.rs
index 079696e73b7..0f0e8d8bc13 100644
--- a/rust/parquet/src/encodings/rle.rs
+++ b/rust/parquet/src/encodings/rle.rs
@@ -56,9 +56,6 @@ pub struct RleEncoder {
     // Underlying writer which holds an internal buffer.
     bit_writer: BitWriter,
 
-    // If this is true, the buffer is full and subsequent `put()` calls will fail.
-    buffer_full: bool,
-
     // The maximum byte size a single run can take.
     max_run_byte_size: usize,
 
@@ -104,7 +101,6 @@ impl RleEncoder {
         RleEncoder {
             bit_width,
             bit_writer,
-            buffer_full: false,
             max_run_byte_size,
             buffered_values: [0; 8],
             num_buffered_values: 0,
@@ -149,10 +145,6 @@ impl RleEncoder {
     pub fn put(&mut self, value: u64) -> Result<bool> {
         // This function buffers 8 values at a time. After seeing 8 values, it
         // decides whether the current run should be encoded in bit-packed or RLE.
-        if self.buffer_full {
-            // The value cannot fit in the current buffer.
-            return Ok(false);
-        }
         if self.current_value == value {
             self.repeat_count += 1;
             if self.repeat_count > 8 {
@@ -209,7 +201,6 @@ impl RleEncoder {
     #[inline]
     pub fn clear(&mut self) {
         self.bit_writer.clear();
-        self.buffer_full = false;
         self.num_buffered_values = 0;
         self.current_value = 0;
         self.repeat_count = 0;
@@ -249,7 +240,7 @@ impl RleEncoder {
     #[inline]
     fn flush_rle_run(&mut self) -> Result<()> {
         assert!(self.repeat_count > 0);
-        let indicator_value = self.repeat_count << 1 | 0;
+        let indicator_value = self.repeat_count << 1;
         let mut result = self.bit_writer.put_vlq_int(indicator_value as u64);
         result &= self.bit_writer.put_aligned(
             self.current_value,
@@ -369,7 +360,7 @@ impl RleDecoder {
     pub fn get<T: FromBytes>(&mut self) -> Result<Option<T>> {
         assert!(size_of::<T>() <= 8);
 
-        while self.rle_left <= 0 && self.bit_packed_left <= 0 {
+        while self.rle_left == 0 && self.bit_packed_left == 0 {
             if !self.reload() {
                 return Ok(None);
             }
@@ -390,7 +381,7 @@ impl RleDecoder {
             let bit_reader = self.bit_reader.as_mut().expect("bit_reader should be Some");
             let bit_packed_value = bit_reader
                 .get_value(self.bit_width as usize)
-                .ok_or(eof_err!("Not enough data for 'bit_packed_value'"))?;
+                .ok_or_else(|| eof_err!("Not enough data for 'bit_packed_value'"))?;
             self.bit_packed_left -= 1;
             bit_packed_value
         };
@@ -429,10 +420,8 @@ impl RleDecoder {
                     self.bit_packed_left -= num_values as u32;
                     values_read += num_values;
                 }
-            } else {
-                if !self.reload() {
-                    break;
-                }
+            } else if !self.reload() {
+                break;
             }
         }
 
@@ -485,10 +474,8 @@ impl RleDecoder {
                         }
                     }
                 }
-            } else {
-                if !self.reload() {
-                    break;
-                }
+            } else if !self.reload() {
+                break;
             }
         }
 
@@ -514,7 +501,7 @@ impl RleDecoder {
                 return false;
             }
         }
-        return false;
+        false
     }
 }
 
@@ -651,9 +638,8 @@ mod tests {
         if expected_len != -1 {
             assert_eq!(buffer.len(), expected_len as usize);
         }
-        match expected_encoding {
-            Some(b) => assert_eq!(buffer.as_ref(), b),
-            _ => (),
+        if let Some(b) = expected_encoding {
+            assert_eq!(buffer.as_ref(), b);
         }
 
         // Verify read
diff --git a/rust/parquet/src/file/footer.rs b/rust/parquet/src/file/footer.rs
new file mode 100644
index 00000000000..00248c07abe
--- /dev/null
+++ b/rust/parquet/src/file/footer.rs
@@ -0,0 +1,262 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::{
+    cmp::min,
+    io::{Cursor, Read, Seek, SeekFrom},
+    rc::Rc,
+};
+
+use byteorder::{ByteOrder, LittleEndian};
+use parquet_format::{ColumnOrder as TColumnOrder, FileMetaData as TFileMetaData};
+use thrift::protocol::TCompactInputProtocol;
+
+use crate::basic::ColumnOrder;
+
+use crate::errors::{ParquetError, Result};
+use crate::file::{
+    metadata::*, reader::ChunkReader, DEFAULT_FOOTER_READ_SIZE, FOOTER_SIZE,
+    PARQUET_MAGIC,
+};
+
+use crate::schema::types::{self, SchemaDescriptor};
+
+/// Layout of Parquet file
+/// +---------------------------+-----+---+
+/// |      Rest of file         |  B  | A |
+/// +---------------------------+-----+---+
+/// where A: parquet footer, B: parquet metadata.
+///
+/// The reader first reads DEFAULT_FOOTER_SIZE bytes from the end of the file.
+/// If it is not enough according to the length indicated in the footer, it reads more bytes.
+pub fn parse_metadata<R: ChunkReader>(chunk_reader: &R) -> Result<ParquetMetaData> {
+    // check file is large enough to hold footer
+    let file_size = chunk_reader.len();
+    if file_size < (FOOTER_SIZE as u64) {
+        return Err(general_err!(
+            "Invalid Parquet file. Size is smaller than footer"
+        ));
+    }
+
+    // read and cache up to DEFAULT_FOOTER_READ_SIZE bytes from the end and process the footer
+    let default_end_len = min(DEFAULT_FOOTER_READ_SIZE, chunk_reader.len() as usize);
+    let mut default_end_reader = chunk_reader
+        .get_read(chunk_reader.len() - default_end_len as u64, default_end_len)?;
+    let mut default_len_end_buf = vec![0; default_end_len];
+    default_end_reader.read_exact(&mut default_len_end_buf)?;
+
+    // check this is indeed a parquet file
+    if default_len_end_buf[default_end_len - 4..] != PARQUET_MAGIC {
+        return Err(general_err!("Invalid Parquet file. Corrupt footer"));
+    }
+
+    // get the metadata length from the footer
+    let metadata_len = LittleEndian::read_i32(
+        &default_len_end_buf[default_end_len - 8..default_end_len - 4],
+    ) as i64;
+    if metadata_len < 0 {
+        return Err(general_err!(
+            "Invalid Parquet file. Metadata length is less than zero ({})",
+            metadata_len
+        ));
+    }
+    let footer_metadata_len = FOOTER_SIZE + metadata_len as usize;
+
+    // build up the reader covering the entire metadata
+    let mut default_end_cursor = Cursor::new(default_len_end_buf);
+    let metadata_read: Box<dyn Read>;
+    if footer_metadata_len > file_size as usize {
+        return Err(general_err!(
+            "Invalid Parquet file. Metadata start is less than zero ({})",
+            file_size as i64 - footer_metadata_len as i64
+        ));
+    } else if footer_metadata_len < DEFAULT_FOOTER_READ_SIZE {
+        // the whole metadata is in the bytes we already read
+        default_end_cursor.seek(SeekFrom::End(-(footer_metadata_len as i64)))?;
+        metadata_read = Box::new(default_end_cursor);
+    } else {
+        // the end of file read by default is not long enough, read missing bytes
+        let complementary_end_read = chunk_reader.get_read(
+            file_size - footer_metadata_len as u64,
+            FOOTER_SIZE + metadata_len as usize - default_end_len,
+        )?;
+        metadata_read = Box::new(complementary_end_read.chain(default_end_cursor));
+    }
+
+    // TODO: row group filtering
+    let mut prot = TCompactInputProtocol::new(metadata_read);
+    let t_file_metadata: TFileMetaData = TFileMetaData::read_from_in_protocol(&mut prot)
+        .map_err(|e| ParquetError::General(format!("Could not parse metadata: {}", e)))?;
+    let schema = types::from_thrift(&t_file_metadata.schema)?;
+    let schema_descr = Rc::new(SchemaDescriptor::new(schema));
+    let mut row_groups = Vec::new();
+    for rg in t_file_metadata.row_groups {
+        row_groups.push(RowGroupMetaData::from_thrift(schema_descr.clone(), rg)?);
+    }
+    let column_orders = parse_column_orders(t_file_metadata.column_orders, &schema_descr);
+
+    let file_metadata = FileMetaData::new(
+        t_file_metadata.version,
+        t_file_metadata.num_rows,
+        t_file_metadata.created_by,
+        t_file_metadata.key_value_metadata,
+        schema_descr,
+        column_orders,
+    );
+    Ok(ParquetMetaData::new(file_metadata, row_groups))
+}
+
+/// Parses column orders from Thrift definition.
+/// If no column orders are defined, returns `None`.
+fn parse_column_orders(
+    t_column_orders: Option<Vec<TColumnOrder>>,
+    schema_descr: &SchemaDescriptor,
+) -> Option<Vec<ColumnOrder>> {
+    match t_column_orders {
+        Some(orders) => {
+            // Should always be the case
+            assert_eq!(
+                orders.len(),
+                schema_descr.num_columns(),
+                "Column order length mismatch"
+            );
+            let mut res = Vec::new();
+            for (i, column) in schema_descr.columns().iter().enumerate() {
+                match orders[i] {
+                    TColumnOrder::TYPEORDER(_) => {
+                        let sort_order = ColumnOrder::get_sort_order(
+                            column.logical_type(),
+                            column.physical_type(),
+                        );
+                        res.push(ColumnOrder::TYPE_DEFINED_ORDER(sort_order));
+                    }
+                }
+            }
+            Some(res)
+        }
+        None => None,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::basic::SortOrder;
+    use crate::basic::Type;
+    use crate::schema::types::Type as SchemaType;
+    use crate::util::test_common::get_temp_file;
+    use parquet_format::TypeDefinedOrder;
+
+    #[test]
+    fn test_parse_metadata_size_smaller_than_footer() {
+        let test_file = get_temp_file("corrupt-1.parquet", &[]);
+        let reader_result = parse_metadata(&test_file);
+        assert!(reader_result.is_err());
+        assert_eq!(
+            reader_result.err().unwrap(),
+            general_err!("Invalid Parquet file. Size is smaller than footer")
+        );
+    }
+
+    #[test]
+    fn test_parse_metadata_corrupt_footer() {
+        let test_file = get_temp_file("corrupt-2.parquet", &[1, 2, 3, 4, 5, 6, 7, 8]);
+        let reader_result = parse_metadata(&test_file);
+        assert!(reader_result.is_err());
+        assert_eq!(
+            reader_result.err().unwrap(),
+            general_err!("Invalid Parquet file. Corrupt footer")
+        );
+    }
+
+    #[test]
+    fn test_parse_metadata_invalid_length() {
+        let test_file =
+            get_temp_file("corrupt-3.parquet", &[0, 0, 0, 255, b'P', b'A', b'R', b'1']);
+        let reader_result = parse_metadata(&test_file);
+        assert!(reader_result.is_err());
+        assert_eq!(
+            reader_result.err().unwrap(),
+            general_err!(
+                "Invalid Parquet file. Metadata length is less than zero (-16777216)"
+            )
+        );
+    }
+
+    #[test]
+    fn test_parse_metadata_invalid_start() {
+        let test_file =
+            get_temp_file("corrupt-4.parquet", &[255, 0, 0, 0, b'P', b'A', b'R', b'1']);
+        let reader_result = parse_metadata(&test_file);
+        assert!(reader_result.is_err());
+        assert_eq!(
+            reader_result.err().unwrap(),
+            general_err!("Invalid Parquet file. Metadata start is less than zero (-255)")
+        );
+    }
+
+    #[test]
+    fn test_metadata_column_orders_parse() {
+        // Define simple schema, we do not need to provide logical types.
+        let mut fields = vec![
+            Rc::new(
+                SchemaType::primitive_type_builder("col1", Type::INT32)
+                    .build()
+                    .unwrap(),
+            ),
+            Rc::new(
+                SchemaType::primitive_type_builder("col2", Type::FLOAT)
+                    .build()
+                    .unwrap(),
+            ),
+        ];
+        let schema = SchemaType::group_type_builder("schema")
+            .with_fields(&mut fields)
+            .build()
+            .unwrap();
+        let schema_descr = SchemaDescriptor::new(Rc::new(schema));
+
+        let t_column_orders = Some(vec![
+            TColumnOrder::TYPEORDER(TypeDefinedOrder::new()),
+            TColumnOrder::TYPEORDER(TypeDefinedOrder::new()),
+        ]);
+
+        assert_eq!(
+            parse_column_orders(t_column_orders, &schema_descr),
+            Some(vec![
+                ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::SIGNED),
+                ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::SIGNED)
+            ])
+        );
+
+        // Test when no column orders are defined.
+        assert_eq!(parse_column_orders(None, &schema_descr), None);
+    }
+
+    #[test]
+    #[should_panic(expected = "Column order length mismatch")]
+    fn test_metadata_column_orders_len_mismatch() {
+        let schema = SchemaType::group_type_builder("schema").build().unwrap();
+        let schema_descr = SchemaDescriptor::new(Rc::new(schema));
+
+        let t_column_orders =
+            Some(vec![TColumnOrder::TYPEORDER(TypeDefinedOrder::new())]);
+
+        parse_column_orders(t_column_orders, &schema_descr);
+    }
+}
diff --git a/rust/parquet/src/file/metadata.rs b/rust/parquet/src/file/metadata.rs
index 67113bda941..74841d5ac66 100644
--- a/rust/parquet/src/file/metadata.rs
+++ b/rust/parquet/src/file/metadata.rs
@@ -42,10 +42,11 @@ use crate::errors::{ParquetError, Result};
 use crate::file::statistics::{self, Statistics};
 use crate::schema::types::{
     ColumnDescPtr, ColumnDescriptor, ColumnPath, SchemaDescPtr, SchemaDescriptor,
-    Type as SchemaType, TypePtr,
+    Type as SchemaType,
 };
 
 /// Global Parquet metadata.
+#[derive(Debug)]
 pub struct ParquetMetaData {
     file_metadata: FileMetaData,
     row_groups: Vec<RowGroupMetaData>,
@@ -89,12 +90,12 @@ pub type KeyValue = parquet_format::KeyValue;
 pub type FileMetaDataPtr = Rc<FileMetaData>;
 
 /// Metadata for a Parquet file.
+#[derive(Debug)]
 pub struct FileMetaData {
     version: i32,
     num_rows: i64,
     created_by: Option<String>,
     key_value_metadata: Option<Vec<KeyValue>>,
-    schema: TypePtr,
     schema_descr: SchemaDescPtr,
     column_orders: Option<Vec<ColumnOrder>>,
 }
@@ -106,7 +107,6 @@ impl FileMetaData {
         num_rows: i64,
         created_by: Option<String>,
         key_value_metadata: Option<Vec<KeyValue>>,
-        schema: TypePtr,
         schema_descr: SchemaDescPtr,
         column_orders: Option<Vec<ColumnOrder>>,
     ) -> Self {
@@ -115,7 +115,6 @@ impl FileMetaData {
             num_rows,
             created_by,
             key_value_metadata,
-            schema,
             schema_descr,
             column_orders,
         }
@@ -150,7 +149,7 @@ impl FileMetaData {
 
     /// Returns Parquet ['Type`] that describes schema in this file.
     pub fn schema(&self) -> &SchemaType {
-        self.schema.as_ref()
+        self.schema_descr.root_schema()
     }
 
     /// Returns a reference to schema descriptor.
@@ -188,6 +187,7 @@ impl FileMetaData {
 pub type RowGroupMetaDataPtr = Rc<RowGroupMetaData>;
 
 /// Metadata for a row group.
+#[derive(Debug)]
 pub struct RowGroupMetaData {
     columns: Vec<ColumnChunkMetaData>,
     num_rows: i64,
@@ -260,7 +260,7 @@ impl RowGroupMetaData {
     /// Method to convert to Thrift.
     pub fn to_thrift(&self) -> RowGroup {
         RowGroup {
-            columns: self.columns().into_iter().map(|v| v.to_thrift()).collect(),
+            columns: self.columns().iter().map(|v| v.to_thrift()).collect(),
             total_byte_size: self.total_byte_size,
             num_rows: self.num_rows,
             sorting_columns: None,
@@ -325,6 +325,7 @@ impl RowGroupMetaDataBuilder {
 }
 
 /// Metadata for a column chunk.
+#[derive(Debug)]
 pub struct ColumnChunkMetaData {
     column_type: Type,
     column_path: ColumnPath,
@@ -484,7 +485,7 @@ impl ColumnChunkMetaData {
     pub fn to_thrift(&self) -> ColumnChunk {
         let column_metadata = ColumnMetaData {
             type_: self.column_type.into(),
-            encodings: self.encodings().into_iter().map(|&v| v.into()).collect(),
+            encodings: self.encodings().iter().map(|&v| v.into()).collect(),
             path_in_schema: Vec::from(self.column_path.as_ref()),
             codec: self.compression.into(),
             num_values: self.num_values,
@@ -499,7 +500,7 @@ impl ColumnChunkMetaData {
         };
 
         ColumnChunk {
-            file_path: self.file_path().map(|v| v.clone()),
+            file_path: self.file_path().cloned(),
             file_offset: self.file_offset,
             meta_data: Some(column_metadata),
             offset_index_offset: None,
@@ -654,7 +655,7 @@ mod tests {
 
         let row_group_exp = row_group_meta.to_thrift();
         let row_group_res =
-            RowGroupMetaData::from_thrift(schema_descr.clone(), row_group_exp.clone())
+            RowGroupMetaData::from_thrift(schema_descr, row_group_exp.clone())
                 .unwrap()
                 .to_thrift();
 
@@ -665,7 +666,7 @@ mod tests {
     fn test_row_group_metadata_thrift_conversion_empty() {
         let schema_descr = get_test_schema_descr();
 
-        let row_group_meta = RowGroupMetaData::builder(schema_descr.clone()).build();
+        let row_group_meta = RowGroupMetaData::builder(schema_descr).build();
 
         assert!(row_group_meta.is_err());
         if let Err(e) = row_group_meta {
@@ -696,7 +697,7 @@ mod tests {
         let col_chunk_exp = col_metadata.to_thrift();
 
         let col_chunk_res =
-            ColumnChunkMetaData::from_thrift(column_descr.clone(), col_chunk_exp.clone())
+            ColumnChunkMetaData::from_thrift(column_descr, col_chunk_exp.clone())
                 .unwrap()
                 .to_thrift();
 
@@ -713,7 +714,7 @@ mod tests {
 
         let col_chunk_exp = col_metadata.to_thrift();
         let col_chunk_res =
-            ColumnChunkMetaData::from_thrift(column_descr.clone(), col_chunk_exp.clone())
+            ColumnChunkMetaData::from_thrift(column_descr, col_chunk_exp.clone())
                 .unwrap()
                 .to_thrift();
 
diff --git a/rust/parquet/src/file/mod.rs b/rust/parquet/src/file/mod.rs
index 6dbf131b9c3..8b026e361f7 100644
--- a/rust/parquet/src/file/mod.rs
+++ b/rust/parquet/src/file/mod.rs
@@ -96,11 +96,16 @@
 //!     println!("{}", row);
 //! }
 //! ```
+pub mod footer;
 pub mod metadata;
 pub mod properties;
 pub mod reader;
+pub mod serialized_reader;
 pub mod statistics;
 pub mod writer;
 
 const FOOTER_SIZE: usize = 8;
 const PARQUET_MAGIC: [u8; 4] = [b'P', b'A', b'R', b'1'];
+
+/// The number of bytes read at the end of the parquet file on first read
+const DEFAULT_FOOTER_READ_SIZE: usize = 64 * 1024;
diff --git a/rust/parquet/src/file/properties.rs b/rust/parquet/src/file/properties.rs
index 81d739b3b09..b62ce7bbc38 100644
--- a/rust/parquet/src/file/properties.rs
+++ b/rust/parquet/src/file/properties.rs
@@ -89,8 +89,8 @@ pub type WriterPropertiesPtr = Rc<WriterProperties>;
 
 /// Writer properties.
 ///
-/// It is created as an immutable data structure, use [`WriterPropertiesBuilder`] to
-/// assemble the properties.
+/// All properties except the key-value metadata are immutable,
+/// use [`WriterPropertiesBuilder`] to assemble these properties.
 #[derive(Debug, Clone)]
 pub struct WriterProperties {
     data_pagesize_limit: usize,
@@ -99,7 +99,7 @@ pub struct WriterProperties {
     max_row_group_size: usize,
     writer_version: WriterVersion,
     created_by: String,
-    key_value_metadata: Option<Vec<KeyValue>>,
+    pub(crate) key_value_metadata: Option<Vec<KeyValue>>,
     default_column_properties: ColumnProperties,
     column_properties: HashMap<ColumnPath, ColumnProperties>,
 }
@@ -546,9 +546,9 @@ mod tests {
     fn test_writer_properties_dictionary_encoding() {
         // dictionary encoding is not configurable, and it should be the same for both
         // writer version 1 and 2.
-        for version in vec![WriterVersion::PARQUET_1_0, WriterVersion::PARQUET_2_0] {
+        for version in &[WriterVersion::PARQUET_1_0, WriterVersion::PARQUET_2_0] {
             let props = WriterProperties::builder()
-                .set_writer_version(version)
+                .set_writer_version(*version)
                 .build();
             assert_eq!(props.dictionary_page_encoding(), Encoding::PLAIN);
             assert_eq!(
diff --git a/rust/parquet/src/file/reader.rs b/rust/parquet/src/file/reader.rs
index ee634a5f686..50991872eaf 100644
--- a/rust/parquet/src/file/reader.rs
+++ b/rust/parquet/src/file/reader.rs
@@ -18,35 +18,37 @@
 //! Contains file reader API and provides methods to access file metadata, row group
 //! readers to read individual column chunks, or access record iterator.
 
-use std::{
-    convert::TryFrom,
-    fs::File,
-    io::{BufReader, Cursor, Read, Seek, SeekFrom},
-    path::Path,
-    rc::Rc,
-};
+use std::{boxed::Box, io::Read, rc::Rc};
 
-use byteorder::{ByteOrder, LittleEndian};
-use parquet_format::{
-    ColumnOrder as TColumnOrder, FileMetaData as TFileMetaData, PageHeader, PageType,
-};
-use thrift::protocol::TCompactInputProtocol;
-
-use crate::basic::{ColumnOrder, Compression, Encoding, Type};
 use crate::column::page::PageIterator;
-use crate::column::{
-    page::{Page, PageReader},
-    reader::{ColumnReader, ColumnReaderImpl},
-};
-use crate::compression::{create_codec, Codec};
+use crate::column::{page::PageReader, reader::ColumnReader};
 use crate::errors::{ParquetError, Result};
-use crate::file::{metadata::*, statistics, FOOTER_SIZE, PARQUET_MAGIC};
+use crate::file::metadata::*;
+pub use crate::file::serialized_reader::{SerializedFileReader, SerializedPageReader};
 use crate::record::reader::RowIter;
-use crate::record::Row;
-use crate::schema::types::{
-    self, ColumnDescPtr, SchemaDescPtr, SchemaDescriptor, Type as SchemaType,
-};
-use crate::util::{io::FileSource, memory::ByteBufferPtr};
+use crate::schema::types::{ColumnDescPtr, SchemaDescPtr, Type as SchemaType};
+
+use crate::basic::Type;
+
+use crate::column::reader::ColumnReaderImpl;
+
+/// Length should return the total number of bytes in the input source.
+/// It's mainly used to read the metadata, which is at the end of the source.
+#[allow(clippy::len_without_is_empty)]
+pub trait Length {
+    /// Returns the amount of bytes of the inner source.
+    fn len(&self) -> u64;
+}
+
+/// The ChunkReader trait generates readers of chunks of a source.
+/// For a file system reader, each chunk might contain a clone of File bounded on a given range.
+/// For an object store reader, each read can be mapped to a range request.
+pub trait ChunkReader: Length {
+    type T: Read;
+    /// get a serialy readeable slice of the current reader
+    /// This should fail if the slice exceeds the current bounds
+    fn get_read(&self, start: u64, length: usize) -> Result<Self::T>;
+}
 
 // ----------------------------------------------------------------------
 // APIs for file & row group readers
@@ -85,297 +87,8 @@ pub trait RowGroupReader {
     fn get_column_page_reader(&self, i: usize) -> Result<Box<PageReader>>;
 
     /// Get value reader for the `i`th column chunk.
-    fn get_column_reader(&self, i: usize) -> Result<ColumnReader>;
-
-    /// Get iterator of `Row`s from this row group.
-    ///
-    /// Projected schema can be a subset of or equal to the file schema, when it is None,
-    /// full file schema is assumed.
-    fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter>;
-}
-
-// ----------------------------------------------------------------------
-// Serialized impl for file & row group readers
-
-/// Length should return the amount of bytes that implementor contains.
-/// It's mainly used to read the metadata, which is at the end of the source.
-pub trait Length {
-    /// Returns the amount of bytes of the inner source.
-    fn len(&self) -> u64;
-}
-
-/// TryClone tries to clone the type and should maintain the `Seek` position of the given
-/// instance.
-pub trait TryClone: Sized {
-    /// Clones the type returning a new instance or an error if it's not possible
-    /// to clone it.
-    fn try_clone(&self) -> Result<Self>;
-}
-
-impl Length for File {
-    fn len(&self) -> u64 {
-        self.metadata().map(|m| m.len()).unwrap_or(0u64)
-    }
-}
-
-impl TryClone for File {
-    fn try_clone(&self) -> Result<Self> {
-        self.try_clone().map_err(|e| e.into())
-    }
-}
-
-impl<'a> Length for Cursor<&'a [u8]> {
-    fn len(&self) -> u64 {
-        self.get_ref().len() as u64
-    }
-}
-
-impl<'a> TryClone for Cursor<&'a [u8]> {
-    fn try_clone(&self) -> Result<Self> {
-        Ok(self.clone())
-    }
-}
-
-impl Length for Cursor<Vec<u8>> {
-    fn len(&self) -> u64 {
-        self.get_ref().len() as u64
-    }
-}
-
-impl TryClone for Cursor<Vec<u8>> {
-    fn try_clone(&self) -> Result<Self> {
-        Ok(self.clone())
-    }
-}
-
-/// ParquetReader is the interface which needs to be fulfilled to be able to parse a
-/// parquet source.
-pub trait ParquetReader: Read + Seek + Length + TryClone {}
-impl<T: Read + Seek + Length + TryClone> ParquetReader for T {}
-
-/// A serialized implementation for Parquet [`FileReader`].
-pub struct SerializedFileReader<R: ParquetReader> {
-    buf: BufReader<R>,
-    metadata: ParquetMetaData,
-}
-
-impl<R: ParquetReader> SerializedFileReader<R> {
-    /// Creates file reader from a Parquet file.
-    /// Returns error if Parquet file does not exist or is corrupt.
-    pub fn new(reader: R) -> Result<Self> {
-        let mut buf = BufReader::new(reader);
-        let metadata = Self::parse_metadata(&mut buf)?;
-        Ok(Self { buf, metadata })
-    }
-
-    // Layout of Parquet file
-    // +---------------------------+---+-----+
-    // |      Rest of file         | B |  A  |
-    // +---------------------------+---+-----+
-    // where A: parquet footer, B: parquet metadata.
-    //
-    fn parse_metadata(buf: &mut BufReader<R>) -> Result<ParquetMetaData> {
-        let file_size = buf.get_ref().len();
-        if file_size < (FOOTER_SIZE as u64) {
-            return Err(general_err!(
-                "Invalid Parquet file. Size is smaller than footer"
-            ));
-        }
-        let mut footer_buffer: [u8; FOOTER_SIZE] = [0; FOOTER_SIZE];
-        buf.seek(SeekFrom::End(-(FOOTER_SIZE as i64)))?;
-        buf.read_exact(&mut footer_buffer)?;
-        if footer_buffer[4..] != PARQUET_MAGIC {
-            return Err(general_err!("Invalid Parquet file. Corrupt footer"));
-        }
-        let metadata_len = LittleEndian::read_i32(&footer_buffer[0..4]) as i64;
-        if metadata_len < 0 {
-            return Err(general_err!(
-                "Invalid Parquet file. Metadata length is less than zero ({})",
-                metadata_len
-            ));
-        }
-        let metadata_start: i64 = file_size as i64 - FOOTER_SIZE as i64 - metadata_len;
-        if metadata_start < 0 {
-            return Err(general_err!(
-                "Invalid Parquet file. Metadata start is less than zero ({})",
-                metadata_start
-            ));
-        }
-        buf.seek(SeekFrom::Start(metadata_start as u64))?;
-        let metadata_buf = buf.take(metadata_len as u64).into_inner();
-
-        // TODO: row group filtering
-        let mut prot = TCompactInputProtocol::new(metadata_buf);
-        let mut t_file_metadata: TFileMetaData =
-            TFileMetaData::read_from_in_protocol(&mut prot).map_err(|e| {
-                ParquetError::General(format!("Could not parse metadata: {}", e))
-            })?;
-        let schema = types::from_thrift(&mut t_file_metadata.schema)?;
-        let schema_descr = Rc::new(SchemaDescriptor::new(schema.clone()));
-        let mut row_groups = Vec::new();
-        for rg in t_file_metadata.row_groups {
-            row_groups.push(RowGroupMetaData::from_thrift(schema_descr.clone(), rg)?);
-        }
-        let column_orders =
-            Self::parse_column_orders(t_file_metadata.column_orders, &schema_descr);
-
-        let file_metadata = FileMetaData::new(
-            t_file_metadata.version,
-            t_file_metadata.num_rows,
-            t_file_metadata.created_by,
-            t_file_metadata.key_value_metadata,
-            schema,
-            schema_descr,
-            column_orders,
-        );
-        Ok(ParquetMetaData::new(file_metadata, row_groups))
-    }
-
-    /// Parses column orders from Thrift definition.
-    /// If no column orders are defined, returns `None`.
-    fn parse_column_orders(
-        t_column_orders: Option<Vec<TColumnOrder>>,
-        schema_descr: &SchemaDescriptor,
-    ) -> Option<Vec<ColumnOrder>> {
-        match t_column_orders {
-            Some(orders) => {
-                // Should always be the case
-                assert_eq!(
-                    orders.len(),
-                    schema_descr.num_columns(),
-                    "Column order length mismatch"
-                );
-                let mut res = Vec::new();
-                for (i, column) in schema_descr.columns().iter().enumerate() {
-                    match orders[i] {
-                        TColumnOrder::TYPEORDER(_) => {
-                            let sort_order = ColumnOrder::get_sort_order(
-                                column.logical_type(),
-                                column.physical_type(),
-                            );
-                            res.push(ColumnOrder::TYPE_DEFINED_ORDER(sort_order));
-                        }
-                    }
-                }
-                Some(res)
-            }
-            None => None,
-        }
-    }
-}
-
-impl<R: 'static + ParquetReader> FileReader for SerializedFileReader<R> {
-    fn metadata(&self) -> &ParquetMetaData {
-        &self.metadata
-    }
-
-    fn num_row_groups(&self) -> usize {
-        self.metadata.num_row_groups()
-    }
-
-    fn get_row_group(&self, i: usize) -> Result<Box<RowGroupReader + '_>> {
-        let row_group_metadata = self.metadata.row_group(i);
-        // Row groups should be processed sequentially.
-        let f = self.buf.get_ref().try_clone()?;
-        Ok(Box::new(SerializedRowGroupReader::new(
-            f,
-            row_group_metadata,
-        )))
-    }
-
-    fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter> {
-        RowIter::from_file(projection, self)
-    }
-}
-
-impl TryFrom<File> for SerializedFileReader<File> {
-    type Error = ParquetError;
-
-    fn try_from(file: File) -> Result<Self> {
-        Self::new(file)
-    }
-}
-
-impl<'a> TryFrom<&'a Path> for SerializedFileReader<File> {
-    type Error = ParquetError;
-
-    fn try_from(path: &Path) -> Result<Self> {
-        let file = File::open(path)?;
-        Self::try_from(file)
-    }
-}
-
-impl TryFrom<String> for SerializedFileReader<File> {
-    type Error = ParquetError;
-
-    fn try_from(path: String) -> Result<Self> {
-        Self::try_from(Path::new(&path))
-    }
-}
-
-impl<'a> TryFrom<&'a str> for SerializedFileReader<File> {
-    type Error = ParquetError;
-
-    fn try_from(path: &str) -> Result<Self> {
-        Self::try_from(Path::new(&path))
-    }
-}
-
-/// Conversion into a [`RowIter`](crate::record::reader::RowIter)
-/// using the full file schema over all row groups.
-impl IntoIterator for SerializedFileReader<File> {
-    type Item = Row;
-    type IntoIter = RowIter<'static>;
-
-    fn into_iter(self) -> Self::IntoIter {
-        RowIter::from_file_into(Box::new(self))
-    }
-}
-
-/// A serialized implementation for Parquet [`RowGroupReader`].
-pub struct SerializedRowGroupReader<'a, R: ParquetReader> {
-    buf: BufReader<R>,
-    metadata: &'a RowGroupMetaData,
-}
-
-impl<'a, R: 'static + ParquetReader> SerializedRowGroupReader<'a, R> {
-    /// Creates new row group reader from a file and row group metadata.
-    fn new(file: R, metadata: &'a RowGroupMetaData) -> Self {
-        let buf = BufReader::new(file);
-        Self { buf, metadata }
-    }
-}
-
-impl<'a, R: 'static + ParquetReader> RowGroupReader for SerializedRowGroupReader<'a, R> {
-    fn metadata(&self) -> &RowGroupMetaData {
-        &self.metadata
-    }
-
-    fn num_columns(&self) -> usize {
-        self.metadata.num_columns()
-    }
-
-    // TODO: fix PARQUET-816
-    fn get_column_page_reader(&self, i: usize) -> Result<Box<PageReader>> {
-        let col = self.metadata.column(i);
-        let mut col_start = col.data_page_offset();
-        if col.has_dictionary_page() {
-            col_start = col.dictionary_page_offset().unwrap();
-        }
-        let col_length = col.compressed_size();
-        let file_chunk =
-            FileSource::new(self.buf.get_ref(), col_start as u64, col_length as usize);
-        let page_reader = SerializedPageReader::new(
-            file_chunk,
-            col.num_values(),
-            col.compression(),
-            col.column_descr().physical_type(),
-        )?;
-        Ok(Box::new(page_reader))
-    }
-
     fn get_column_reader(&self, i: usize) -> Result<ColumnReader> {
-        let schema_descr = self.metadata.schema_descr();
+        let schema_descr = self.metadata().schema_descr();
         let col_descr = schema_descr.column(i);
         let col_page_reader = self.get_column_page_reader(i)?;
         let col_reader = match col_descr.physical_type() {
@@ -413,174 +126,15 @@ impl<'a, R: 'static + ParquetReader> RowGroupReader for SerializedRowGroupReader
         Ok(col_reader)
     }
 
-    fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter> {
-        RowIter::from_row_group(projection, self)
-    }
-}
-
-/// A serialized implementation for Parquet [`PageReader`].
-pub struct SerializedPageReader<T: Read> {
-    // The file source buffer which references exactly the bytes for the column trunk
-    // to be read by this page reader.
-    buf: T,
-
-    // The compression codec for this column chunk. Only set for non-PLAIN codec.
-    decompressor: Option<Box<Codec>>,
-
-    // The number of values we have seen so far.
-    seen_num_values: i64,
-
-    // The number of total values in this column chunk.
-    total_num_values: i64,
-
-    // Column chunk type.
-    physical_type: Type,
-}
-
-impl<T: Read> SerializedPageReader<T> {
-    /// Creates a new serialized page reader from file source.
-    pub fn new(
-        buf: T,
-        total_num_values: i64,
-        compression: Compression,
-        physical_type: Type,
-    ) -> Result<Self> {
-        let decompressor = create_codec(compression)?;
-        let result = Self {
-            buf,
-            total_num_values,
-            seen_num_values: 0,
-            decompressor,
-            physical_type,
-        };
-        Ok(result)
-    }
-
-    /// Reads Page header from Thrift.
-    fn read_page_header(&mut self) -> Result<PageHeader> {
-        let mut prot = TCompactInputProtocol::new(&mut self.buf);
-        let page_header = PageHeader::read_from_in_protocol(&mut prot)?;
-        Ok(page_header)
-    }
+    /// Get iterator of `Row`s from this row group.
+    ///
+    /// Projected schema can be a subset of or equal to the file schema, when it is None,
+    /// full file schema is assumed.
+    fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter>;
 }
 
-impl<T: Read> PageReader for SerializedPageReader<T> {
-    fn get_next_page(&mut self) -> Result<Option<Page>> {
-        while self.seen_num_values < self.total_num_values {
-            let page_header = self.read_page_header()?;
-
-            // When processing data page v2, depending on enabled compression for the
-            // page, we should account for uncompressed data ('offset') of
-            // repetition and definition levels.
-            //
-            // We always use 0 offset for other pages other than v2, `true` flag means
-            // that compression will be applied if decompressor is defined
-            let mut offset: usize = 0;
-            let mut can_decompress = true;
-
-            if let Some(ref header_v2) = page_header.data_page_header_v2 {
-                offset = (header_v2.definition_levels_byte_length
-                    + header_v2.repetition_levels_byte_length)
-                    as usize;
-                // When is_compressed flag is missing the page is considered compressed
-                can_decompress = header_v2.is_compressed.unwrap_or(true);
-            }
-
-            let compressed_len = page_header.compressed_page_size as usize - offset;
-            let uncompressed_len = page_header.uncompressed_page_size as usize - offset;
-            // We still need to read all bytes from buffered stream
-            let mut buffer = vec![0; offset + compressed_len];
-            self.buf.read_exact(&mut buffer)?;
-
-            // TODO: page header could be huge because of statistics. We should set a
-            // maximum page header size and abort if that is exceeded.
-            if let Some(decompressor) = self.decompressor.as_mut() {
-                if can_decompress {
-                    let mut decompressed_buffer = Vec::with_capacity(uncompressed_len);
-                    let decompressed_size = decompressor
-                        .decompress(&buffer[offset..], &mut decompressed_buffer)?;
-                    if decompressed_size != uncompressed_len {
-                        return Err(general_err!(
-                            "Actual decompressed size doesn't match the expected one ({} vs {})",
-                            decompressed_size,
-                            uncompressed_len
-                        ));
-                    }
-                    if offset == 0 {
-                        buffer = decompressed_buffer;
-                    } else {
-                        // Prepend saved offsets to the buffer
-                        buffer.truncate(offset);
-                        buffer.append(&mut decompressed_buffer);
-                    }
-                }
-            }
-
-            let result = match page_header.type_ {
-                PageType::DictionaryPage => {
-                    assert!(page_header.dictionary_page_header.is_some());
-                    let dict_header =
-                        page_header.dictionary_page_header.as_ref().unwrap();
-                    let is_sorted = dict_header.is_sorted.unwrap_or(false);
-                    Page::DictionaryPage {
-                        buf: ByteBufferPtr::new(buffer),
-                        num_values: dict_header.num_values as u32,
-                        encoding: Encoding::from(dict_header.encoding),
-                        is_sorted,
-                    }
-                }
-                PageType::DataPage => {
-                    assert!(page_header.data_page_header.is_some());
-                    let header = page_header.data_page_header.unwrap();
-                    self.seen_num_values += header.num_values as i64;
-                    Page::DataPage {
-                        buf: ByteBufferPtr::new(buffer),
-                        num_values: header.num_values as u32,
-                        encoding: Encoding::from(header.encoding),
-                        def_level_encoding: Encoding::from(
-                            header.definition_level_encoding,
-                        ),
-                        rep_level_encoding: Encoding::from(
-                            header.repetition_level_encoding,
-                        ),
-                        statistics: statistics::from_thrift(
-                            self.physical_type,
-                            header.statistics,
-                        ),
-                    }
-                }
-                PageType::DataPageV2 => {
-                    assert!(page_header.data_page_header_v2.is_some());
-                    let header = page_header.data_page_header_v2.unwrap();
-                    let is_compressed = header.is_compressed.unwrap_or(true);
-                    self.seen_num_values += header.num_values as i64;
-                    Page::DataPageV2 {
-                        buf: ByteBufferPtr::new(buffer),
-                        num_values: header.num_values as u32,
-                        encoding: Encoding::from(header.encoding),
-                        num_nulls: header.num_nulls as u32,
-                        num_rows: header.num_rows as u32,
-                        def_levels_byte_len: header.definition_levels_byte_length as u32,
-                        rep_levels_byte_len: header.repetition_levels_byte_length as u32,
-                        is_compressed,
-                        statistics: statistics::from_thrift(
-                            self.physical_type,
-                            header.statistics,
-                        ),
-                    }
-                }
-                _ => {
-                    // For unknown page type (e.g., INDEX_PAGE), skip and read next.
-                    continue;
-                }
-            };
-            return Ok(Some(result));
-        }
-
-        // We are at the end of this column chunk and no more page left. Return None.
-        Ok(None)
-    }
-}
+// ----------------------------------------------------------------------
+// Iterator
 
 /// Implementation of page iterator for parquet file.
 pub struct FilePageIterator {
@@ -650,478 +204,3 @@ impl PageIterator for FilePageIterator {
         self.schema().map(|s| s.column(self.column_index))
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    use parquet_format::TypeDefinedOrder;
-
-    use crate::basic::SortOrder;
-    use crate::record::RowAccessor;
-    use crate::schema::parser::parse_message_type;
-    use crate::util::test_common::{get_temp_file, get_test_file, get_test_path};
-
-    #[test]
-    fn test_file_reader_metadata_size_smaller_than_footer() {
-        let test_file = get_temp_file("corrupt-1.parquet", &[]);
-        let reader_result = SerializedFileReader::new(test_file);
-        assert!(reader_result.is_err());
-        assert_eq!(
-            reader_result.err().unwrap(),
-            general_err!("Invalid Parquet file. Size is smaller than footer")
-        );
-    }
-
-    #[test]
-    fn test_cursor_and_file_has_the_same_behaviour() {
-        let mut buf: Vec<u8> = Vec::new();
-        get_test_file("alltypes_plain.parquet")
-            .read_to_end(&mut buf)
-            .unwrap();
-        let cursor = Cursor::new(buf);
-        let read_from_cursor = SerializedFileReader::new(cursor).unwrap();
-
-        let test_file = get_test_file("alltypes_plain.parquet");
-        let read_from_file = SerializedFileReader::new(test_file).unwrap();
-
-        let file_iter = read_from_file.get_row_iter(None).unwrap();
-        let cursor_iter = read_from_cursor.get_row_iter(None).unwrap();
-
-        assert!(file_iter.eq(cursor_iter));
-    }
-
-    #[test]
-    fn test_file_reader_metadata_corrupt_footer() {
-        let test_file = get_temp_file("corrupt-2.parquet", &[1, 2, 3, 4, 5, 6, 7, 8]);
-        let reader_result = SerializedFileReader::new(test_file);
-        assert!(reader_result.is_err());
-        assert_eq!(
-            reader_result.err().unwrap(),
-            general_err!("Invalid Parquet file. Corrupt footer")
-        );
-    }
-
-    #[test]
-    fn test_file_reader_metadata_invalid_length() {
-        let test_file =
-            get_temp_file("corrupt-3.parquet", &[0, 0, 0, 255, b'P', b'A', b'R', b'1']);
-        let reader_result = SerializedFileReader::new(test_file);
-        assert!(reader_result.is_err());
-        assert_eq!(
-            reader_result.err().unwrap(),
-            general_err!(
-                "Invalid Parquet file. Metadata length is less than zero (-16777216)"
-            )
-        );
-    }
-
-    #[test]
-    fn test_file_reader_metadata_invalid_start() {
-        let test_file =
-            get_temp_file("corrupt-4.parquet", &[255, 0, 0, 0, b'P', b'A', b'R', b'1']);
-        let reader_result = SerializedFileReader::new(test_file);
-        assert!(reader_result.is_err());
-        assert_eq!(
-            reader_result.err().unwrap(),
-            general_err!("Invalid Parquet file. Metadata start is less than zero (-255)")
-        );
-    }
-
-    #[test]
-    fn test_file_reader_column_orders_parse() {
-        // Define simple schema, we do not need to provide logical types.
-        let mut fields = vec![
-            Rc::new(
-                SchemaType::primitive_type_builder("col1", Type::INT32)
-                    .build()
-                    .unwrap(),
-            ),
-            Rc::new(
-                SchemaType::primitive_type_builder("col2", Type::FLOAT)
-                    .build()
-                    .unwrap(),
-            ),
-        ];
-        let schema = SchemaType::group_type_builder("schema")
-            .with_fields(&mut fields)
-            .build()
-            .unwrap();
-        let schema_descr = SchemaDescriptor::new(Rc::new(schema));
-
-        let t_column_orders = Some(vec![
-            TColumnOrder::TYPEORDER(TypeDefinedOrder::new()),
-            TColumnOrder::TYPEORDER(TypeDefinedOrder::new()),
-        ]);
-
-        assert_eq!(
-            SerializedFileReader::<File>::parse_column_orders(
-                t_column_orders,
-                &schema_descr
-            ),
-            Some(vec![
-                ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::SIGNED),
-                ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::SIGNED)
-            ])
-        );
-
-        // Test when no column orders are defined.
-        assert_eq!(
-            SerializedFileReader::<File>::parse_column_orders(None, &schema_descr),
-            None
-        );
-    }
-
-    #[test]
-    #[should_panic(expected = "Column order length mismatch")]
-    fn test_file_reader_column_orders_len_mismatch() {
-        let schema = SchemaType::group_type_builder("schema").build().unwrap();
-        let schema_descr = SchemaDescriptor::new(Rc::new(schema));
-
-        let t_column_orders =
-            Some(vec![TColumnOrder::TYPEORDER(TypeDefinedOrder::new())]);
-
-        SerializedFileReader::<File>::parse_column_orders(t_column_orders, &schema_descr);
-    }
-
-    #[test]
-    fn test_file_reader_try_from() {
-        // Valid file path
-        let test_file = get_test_file("alltypes_plain.parquet");
-        let test_path_buf = get_test_path("alltypes_plain.parquet");
-        let test_path = test_path_buf.as_path();
-        let test_path_str = test_path.to_str().unwrap();
-
-        let reader = SerializedFileReader::try_from(test_file);
-        assert!(reader.is_ok());
-
-        let reader = SerializedFileReader::try_from(test_path);
-        assert!(reader.is_ok());
-
-        let reader = SerializedFileReader::try_from(test_path_str);
-        assert!(reader.is_ok());
-
-        let reader = SerializedFileReader::try_from(test_path_str.to_string());
-        assert!(reader.is_ok());
-
-        // Invalid file path
-        let test_path = Path::new("invalid.parquet");
-        let test_path_str = test_path.to_str().unwrap();
-
-        let reader = SerializedFileReader::try_from(test_path);
-        assert!(reader.is_err());
-
-        let reader = SerializedFileReader::try_from(test_path_str);
-        assert!(reader.is_err());
-
-        let reader = SerializedFileReader::try_from(test_path_str.to_string());
-        assert!(reader.is_err());
-    }
-
-    #[test]
-    fn test_file_reader_into_iter() -> Result<()> {
-        let path = get_test_path("alltypes_plain.parquet");
-        let vec = vec![path.clone(), path.clone()]
-            .iter()
-            .map(|p| SerializedFileReader::try_from(p.as_path()).unwrap())
-            .flat_map(|r| r.into_iter())
-            .flat_map(|r| r.get_int(0))
-            .collect::<Vec<_>>();
-
-        // rows in the parquet file are not sorted by "id"
-        // each file contains [id:4, id:5, id:6, id:7, id:2, id:3, id:0, id:1]
-        assert_eq!(vec, vec![4, 5, 6, 7, 2, 3, 0, 1, 4, 5, 6, 7, 2, 3, 0, 1]);
-
-        Ok(())
-    }
-
-    #[test]
-    fn test_file_reader_into_iter_project() -> Result<()> {
-        let path = get_test_path("alltypes_plain.parquet");
-        let result = vec![path]
-            .iter()
-            .map(|p| SerializedFileReader::try_from(p.as_path()).unwrap())
-            .flat_map(|r| {
-                let schema = "message schema { OPTIONAL INT32 id; }";
-                let proj = parse_message_type(&schema).ok();
-
-                r.into_iter().project(proj).unwrap()
-            })
-            .map(|r| format!("{}", r))
-            .collect::<Vec<_>>()
-            .join(",");
-
-        assert_eq!(
-            result,
-            "{id: 4},{id: 5},{id: 6},{id: 7},{id: 2},{id: 3},{id: 0},{id: 1}"
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn test_reuse_file_chunk() {
-        // This test covers the case of maintaining the correct start position in a file
-        // stream for each column reader after initializing and moving to the next one
-        // (without necessarily reading the entire column).
-        let test_file = get_test_file("alltypes_plain.parquet");
-        let reader = SerializedFileReader::new(test_file).unwrap();
-        let row_group = reader.get_row_group(0).unwrap();
-
-        let mut page_readers = Vec::new();
-        for i in 0..row_group.num_columns() {
-            page_readers.push(row_group.get_column_page_reader(i).unwrap());
-        }
-
-        // Now buffer each col reader, we do not expect any failures like:
-        // General("underlying Thrift error: end of file")
-        for mut page_reader in page_readers {
-            assert!(page_reader.get_next_page().is_ok());
-        }
-    }
-
-    #[test]
-    fn test_file_reader() {
-        let test_file = get_test_file("alltypes_plain.parquet");
-        let reader_result = SerializedFileReader::new(test_file);
-        assert!(reader_result.is_ok());
-        let reader = reader_result.unwrap();
-
-        // Test contents in Parquet metadata
-        let metadata = reader.metadata();
-        assert_eq!(metadata.num_row_groups(), 1);
-
-        // Test contents in file metadata
-        let file_metadata = metadata.file_metadata();
-        assert!(file_metadata.created_by().is_some());
-        assert_eq!(
-            file_metadata.created_by().as_ref().unwrap(),
-            "impala version 1.3.0-INTERNAL (build 8a48ddb1eff84592b3fc06bc6f51ec120e1fffc9)"
-        );
-        assert!(file_metadata.key_value_metadata().is_none());
-        assert_eq!(file_metadata.num_rows(), 8);
-        assert_eq!(file_metadata.version(), 1);
-        assert_eq!(file_metadata.column_orders(), None);
-
-        // Test contents in row group metadata
-        let row_group_metadata = metadata.row_group(0);
-        assert_eq!(row_group_metadata.num_columns(), 11);
-        assert_eq!(row_group_metadata.num_rows(), 8);
-        assert_eq!(row_group_metadata.total_byte_size(), 671);
-        // Check each column order
-        for i in 0..row_group_metadata.num_columns() {
-            assert_eq!(file_metadata.column_order(i), ColumnOrder::UNDEFINED);
-        }
-
-        // Test row group reader
-        let row_group_reader_result = reader.get_row_group(0);
-        assert!(row_group_reader_result.is_ok());
-        let row_group_reader: Box<RowGroupReader> = row_group_reader_result.unwrap();
-        assert_eq!(
-            row_group_reader.num_columns(),
-            row_group_metadata.num_columns()
-        );
-        assert_eq!(
-            row_group_reader.metadata().total_byte_size(),
-            row_group_metadata.total_byte_size()
-        );
-
-        // Test page readers
-        // TODO: test for every column
-        let page_reader_0_result = row_group_reader.get_column_page_reader(0);
-        assert!(page_reader_0_result.is_ok());
-        let mut page_reader_0: Box<PageReader> = page_reader_0_result.unwrap();
-        let mut page_count = 0;
-        while let Ok(Some(page)) = page_reader_0.get_next_page() {
-            let is_expected_page = match page {
-                Page::DictionaryPage {
-                    buf,
-                    num_values,
-                    encoding,
-                    is_sorted,
-                } => {
-                    assert_eq!(buf.len(), 32);
-                    assert_eq!(num_values, 8);
-                    assert_eq!(encoding, Encoding::PLAIN_DICTIONARY);
-                    assert_eq!(is_sorted, false);
-                    true
-                }
-                Page::DataPage {
-                    buf,
-                    num_values,
-                    encoding,
-                    def_level_encoding,
-                    rep_level_encoding,
-                    statistics,
-                } => {
-                    assert_eq!(buf.len(), 11);
-                    assert_eq!(num_values, 8);
-                    assert_eq!(encoding, Encoding::PLAIN_DICTIONARY);
-                    assert_eq!(def_level_encoding, Encoding::RLE);
-                    assert_eq!(rep_level_encoding, Encoding::BIT_PACKED);
-                    assert!(statistics.is_none());
-                    true
-                }
-                _ => false,
-            };
-            assert!(is_expected_page);
-            page_count += 1;
-        }
-        assert_eq!(page_count, 2);
-    }
-
-    #[test]
-    fn test_file_reader_datapage_v2() {
-        let test_file = get_test_file("datapage_v2.snappy.parquet");
-        let reader_result = SerializedFileReader::new(test_file);
-        assert!(reader_result.is_ok());
-        let reader = reader_result.unwrap();
-
-        // Test contents in Parquet metadata
-        let metadata = reader.metadata();
-        assert_eq!(metadata.num_row_groups(), 1);
-
-        // Test contents in file metadata
-        let file_metadata = metadata.file_metadata();
-        assert!(file_metadata.created_by().is_some());
-        assert_eq!(
-            file_metadata.created_by().as_ref().unwrap(),
-            "parquet-mr version 1.8.1 (build 4aba4dae7bb0d4edbcf7923ae1339f28fd3f7fcf)"
-        );
-        assert!(file_metadata.key_value_metadata().is_some());
-        assert_eq!(
-            file_metadata.key_value_metadata().to_owned().unwrap().len(),
-            1
-        );
-
-        assert_eq!(file_metadata.num_rows(), 5);
-        assert_eq!(file_metadata.version(), 1);
-        assert_eq!(file_metadata.column_orders(), None);
-
-        let row_group_metadata = metadata.row_group(0);
-
-        // Check each column order
-        for i in 0..row_group_metadata.num_columns() {
-            assert_eq!(file_metadata.column_order(i), ColumnOrder::UNDEFINED);
-        }
-
-        // Test row group reader
-        let row_group_reader_result = reader.get_row_group(0);
-        assert!(row_group_reader_result.is_ok());
-        let row_group_reader: Box<RowGroupReader> = row_group_reader_result.unwrap();
-        assert_eq!(
-            row_group_reader.num_columns(),
-            row_group_metadata.num_columns()
-        );
-        assert_eq!(
-            row_group_reader.metadata().total_byte_size(),
-            row_group_metadata.total_byte_size()
-        );
-
-        // Test page readers
-        // TODO: test for every column
-        let page_reader_0_result = row_group_reader.get_column_page_reader(0);
-        assert!(page_reader_0_result.is_ok());
-        let mut page_reader_0: Box<PageReader> = page_reader_0_result.unwrap();
-        let mut page_count = 0;
-        while let Ok(Some(page)) = page_reader_0.get_next_page() {
-            let is_expected_page = match page {
-                Page::DictionaryPage {
-                    buf,
-                    num_values,
-                    encoding,
-                    is_sorted,
-                } => {
-                    assert_eq!(buf.len(), 7);
-                    assert_eq!(num_values, 1);
-                    assert_eq!(encoding, Encoding::PLAIN);
-                    assert_eq!(is_sorted, false);
-                    true
-                }
-                Page::DataPageV2 {
-                    buf,
-                    num_values,
-                    encoding,
-                    num_nulls,
-                    num_rows,
-                    def_levels_byte_len,
-                    rep_levels_byte_len,
-                    is_compressed,
-                    statistics,
-                } => {
-                    assert_eq!(buf.len(), 4);
-                    assert_eq!(num_values, 5);
-                    assert_eq!(encoding, Encoding::RLE_DICTIONARY);
-                    assert_eq!(num_nulls, 1);
-                    assert_eq!(num_rows, 5);
-                    assert_eq!(def_levels_byte_len, 2);
-                    assert_eq!(rep_levels_byte_len, 0);
-                    assert_eq!(is_compressed, true);
-                    assert!(statistics.is_some());
-                    true
-                }
-                _ => false,
-            };
-            assert!(is_expected_page);
-            page_count += 1;
-        }
-        assert_eq!(page_count, 2);
-    }
-
-    #[test]
-    fn test_page_iterator() {
-        let file = get_test_file("alltypes_plain.parquet");
-        let file_reader = Rc::new(SerializedFileReader::new(file).unwrap());
-
-        let mut page_iterator = FilePageIterator::new(0, file_reader.clone()).unwrap();
-
-        // read first page
-        let page = page_iterator.next();
-        assert!(page.is_some());
-        assert!(page.unwrap().is_ok());
-
-        // reach end of file
-        let page = page_iterator.next();
-        assert!(page.is_none());
-
-        let row_group_indices = Box::new(0..1);
-        let mut page_iterator =
-            FilePageIterator::with_row_groups(0, row_group_indices, file_reader.clone())
-                .unwrap();
-
-        // read first page
-        let page = page_iterator.next();
-        assert!(page.is_some());
-        assert!(page.unwrap().is_ok());
-
-        // reach end of file
-        let page = page_iterator.next();
-        assert!(page.is_none());
-    }
-
-    #[test]
-    fn test_file_reader_key_value_metadata() {
-        let file = get_test_file("binary.parquet");
-        let file_reader = Rc::new(SerializedFileReader::new(file).unwrap());
-
-        let metadata = file_reader
-            .metadata
-            .file_metadata()
-            .key_value_metadata()
-            .as_ref()
-            .unwrap();
-
-        assert_eq!(metadata.len(), 3);
-
-        assert_eq!(metadata.get(0).unwrap().key, "parquet.proto.descriptor");
-
-        assert_eq!(metadata.get(1).unwrap().key, "writer.model.name");
-        assert_eq!(metadata.get(1).unwrap().value, Some("protobuf".to_owned()));
-
-        assert_eq!(metadata.get(2).unwrap().key, "parquet.proto.class");
-        assert_eq!(
-            metadata.get(2).unwrap().value,
-            Some("foo.baz.Foobaz$Event".to_owned())
-        );
-    }
-}
diff --git a/rust/parquet/src/file/serialized_reader.rs b/rust/parquet/src/file/serialized_reader.rs
new file mode 100644
index 00000000000..220970d3c1e
--- /dev/null
+++ b/rust/parquet/src/file/serialized_reader.rs
@@ -0,0 +1,747 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Contains implementations of the reader traits FileReader, RowGroupReader and PageReader
+//! Also contains implementations of the ChunkReader for files (with buffering) and byte arrays (RAM)
+
+use std::{convert::TryFrom, fs::File, io::Read, path::Path, rc::Rc};
+
+use parquet_format::{PageHeader, PageType};
+use thrift::protocol::TCompactInputProtocol;
+
+use crate::basic::{Compression, Encoding, Type};
+use crate::column::page::{Page, PageReader};
+use crate::compression::{create_codec, Codec};
+use crate::errors::{ParquetError, Result};
+use crate::file::{footer, metadata::*, reader::*, statistics};
+use crate::record::reader::RowIter;
+use crate::record::Row;
+use crate::schema::types::Type as SchemaType;
+use crate::util::{io::TryClone, memory::ByteBufferPtr};
+
+// export `SliceableCursor` and `FileSource` publically so clients can
+// re-use the logic in their own ParquetFileWriter wrappers
+pub use crate::util::{cursor::SliceableCursor, io::FileSource};
+
+// ----------------------------------------------------------------------
+// Implementations of traits facilitating the creation of a new reader
+
+impl Length for File {
+    fn len(&self) -> u64 {
+        self.metadata().map(|m| m.len()).unwrap_or(0u64)
+    }
+}
+
+impl TryClone for File {
+    fn try_clone(&self) -> std::io::Result<Self> {
+        self.try_clone()
+    }
+}
+
+impl ChunkReader for File {
+    type T = FileSource<File>;
+
+    fn get_read(&self, start: u64, length: usize) -> Result<Self::T> {
+        Ok(FileSource::new(self, start, length))
+    }
+}
+
+impl Length for SliceableCursor {
+    fn len(&self) -> u64 {
+        SliceableCursor::len(self)
+    }
+}
+
+impl ChunkReader for SliceableCursor {
+    type T = SliceableCursor;
+
+    fn get_read(&self, start: u64, length: usize) -> Result<Self::T> {
+        self.slice(start, length).map_err(|e| e.into())
+    }
+}
+
+impl TryFrom<File> for SerializedFileReader<File> {
+    type Error = ParquetError;
+
+    fn try_from(file: File) -> Result<Self> {
+        Self::new(file)
+    }
+}
+
+impl<'a> TryFrom<&'a Path> for SerializedFileReader<File> {
+    type Error = ParquetError;
+
+    fn try_from(path: &Path) -> Result<Self> {
+        let file = File::open(path)?;
+        Self::try_from(file)
+    }
+}
+
+impl TryFrom<String> for SerializedFileReader<File> {
+    type Error = ParquetError;
+
+    fn try_from(path: String) -> Result<Self> {
+        Self::try_from(Path::new(&path))
+    }
+}
+
+impl<'a> TryFrom<&'a str> for SerializedFileReader<File> {
+    type Error = ParquetError;
+
+    fn try_from(path: &str) -> Result<Self> {
+        Self::try_from(Path::new(&path))
+    }
+}
+
+/// Conversion into a [`RowIter`](crate::record::reader::RowIter)
+/// using the full file schema over all row groups.
+impl IntoIterator for SerializedFileReader<File> {
+    type Item = Row;
+    type IntoIter = RowIter<'static>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        RowIter::from_file_into(Box::new(self))
+    }
+}
+
+// ----------------------------------------------------------------------
+// Implementations of file & row group readers
+
+/// A serialized implementation for Parquet [`FileReader`].
+pub struct SerializedFileReader<R: ChunkReader> {
+    chunk_reader: Rc<R>,
+    metadata: ParquetMetaData,
+}
+
+impl<R: 'static + ChunkReader> SerializedFileReader<R> {
+    /// Creates file reader from a Parquet file.
+    /// Returns error if Parquet file does not exist or is corrupt.
+    pub fn new(chunk_reader: R) -> Result<Self> {
+        let metadata = footer::parse_metadata(&chunk_reader)?;
+        Ok(Self {
+            chunk_reader: Rc::new(chunk_reader),
+            metadata,
+        })
+    }
+}
+
+impl<R: 'static + ChunkReader> FileReader for SerializedFileReader<R> {
+    fn metadata(&self) -> &ParquetMetaData {
+        &self.metadata
+    }
+
+    fn num_row_groups(&self) -> usize {
+        self.metadata.num_row_groups()
+    }
+
+    fn get_row_group(&self, i: usize) -> Result<Box<RowGroupReader + '_>> {
+        let row_group_metadata = self.metadata.row_group(i);
+        // Row groups should be processed sequentially.
+        let f = Rc::clone(&self.chunk_reader);
+        Ok(Box::new(SerializedRowGroupReader::new(
+            f,
+            row_group_metadata,
+        )))
+    }
+
+    fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter> {
+        RowIter::from_file(projection, self)
+    }
+}
+
+/// A serialized implementation for Parquet [`RowGroupReader`].
+pub struct SerializedRowGroupReader<'a, R: ChunkReader> {
+    chunk_reader: Rc<R>,
+    metadata: &'a RowGroupMetaData,
+}
+
+impl<'a, R: ChunkReader> SerializedRowGroupReader<'a, R> {
+    /// Creates new row group reader from a file and row group metadata.
+    fn new(chunk_reader: Rc<R>, metadata: &'a RowGroupMetaData) -> Self {
+        Self {
+            chunk_reader,
+            metadata,
+        }
+    }
+}
+
+impl<'a, R: 'static + ChunkReader> RowGroupReader for SerializedRowGroupReader<'a, R> {
+    fn metadata(&self) -> &RowGroupMetaData {
+        &self.metadata
+    }
+
+    fn num_columns(&self) -> usize {
+        self.metadata.num_columns()
+    }
+
+    // TODO: fix PARQUET-816
+    fn get_column_page_reader(&self, i: usize) -> Result<Box<PageReader>> {
+        let col = self.metadata.column(i);
+        let col_start = if col.has_dictionary_page() {
+            col.dictionary_page_offset().unwrap()
+        } else {
+            col.data_page_offset()
+        };
+        let col_length = col.compressed_size();
+        let file_chunk = self
+            .chunk_reader
+            .get_read(col_start as u64, col_length as usize)?;
+        let page_reader = SerializedPageReader::new(
+            file_chunk,
+            col.num_values(),
+            col.compression(),
+            col.column_descr().physical_type(),
+        )?;
+        Ok(Box::new(page_reader))
+    }
+
+    fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter> {
+        RowIter::from_row_group(projection, self)
+    }
+}
+
+/// A serialized implementation for Parquet [`PageReader`].
+pub struct SerializedPageReader<T: Read> {
+    // The file source buffer which references exactly the bytes for the column trunk
+    // to be read by this page reader.
+    buf: T,
+
+    // The compression codec for this column chunk. Only set for non-PLAIN codec.
+    decompressor: Option<Box<Codec>>,
+
+    // The number of values we have seen so far.
+    seen_num_values: i64,
+
+    // The number of total values in this column chunk.
+    total_num_values: i64,
+
+    // Column chunk type.
+    physical_type: Type,
+}
+
+impl<T: Read> SerializedPageReader<T> {
+    /// Creates a new serialized page reader from file source.
+    pub fn new(
+        buf: T,
+        total_num_values: i64,
+        compression: Compression,
+        physical_type: Type,
+    ) -> Result<Self> {
+        let decompressor = create_codec(compression)?;
+        let result = Self {
+            buf,
+            total_num_values,
+            seen_num_values: 0,
+            decompressor,
+            physical_type,
+        };
+        Ok(result)
+    }
+
+    /// Reads Page header from Thrift.
+    fn read_page_header(&mut self) -> Result<PageHeader> {
+        let mut prot = TCompactInputProtocol::new(&mut self.buf);
+        let page_header = PageHeader::read_from_in_protocol(&mut prot)?;
+        Ok(page_header)
+    }
+}
+
+impl<T: Read> PageReader for SerializedPageReader<T> {
+    fn get_next_page(&mut self) -> Result<Option<Page>> {
+        while self.seen_num_values < self.total_num_values {
+            let page_header = self.read_page_header()?;
+
+            // When processing data page v2, depending on enabled compression for the
+            // page, we should account for uncompressed data ('offset') of
+            // repetition and definition levels.
+            //
+            // We always use 0 offset for other pages other than v2, `true` flag means
+            // that compression will be applied if decompressor is defined
+            let mut offset: usize = 0;
+            let mut can_decompress = true;
+
+            if let Some(ref header_v2) = page_header.data_page_header_v2 {
+                offset = (header_v2.definition_levels_byte_length
+                    + header_v2.repetition_levels_byte_length)
+                    as usize;
+                // When is_compressed flag is missing the page is considered compressed
+                can_decompress = header_v2.is_compressed.unwrap_or(true);
+            }
+
+            let compressed_len = page_header.compressed_page_size as usize - offset;
+            let uncompressed_len = page_header.uncompressed_page_size as usize - offset;
+            // We still need to read all bytes from buffered stream
+            let mut buffer = vec![0; offset + compressed_len];
+            self.buf.read_exact(&mut buffer)?;
+
+            // TODO: page header could be huge because of statistics. We should set a
+            // maximum page header size and abort if that is exceeded.
+            if let Some(decompressor) = self.decompressor.as_mut() {
+                if can_decompress {
+                    let mut decompressed_buffer = Vec::with_capacity(uncompressed_len);
+                    let decompressed_size = decompressor
+                        .decompress(&buffer[offset..], &mut decompressed_buffer)?;
+                    if decompressed_size != uncompressed_len {
+                        return Err(general_err!(
+              "Actual decompressed size doesn't match the expected one ({} vs {})",
+              decompressed_size,
+              uncompressed_len
+            ));
+                    }
+                    if offset == 0 {
+                        buffer = decompressed_buffer;
+                    } else {
+                        // Prepend saved offsets to the buffer
+                        buffer.truncate(offset);
+                        buffer.append(&mut decompressed_buffer);
+                    }
+                }
+            }
+
+            let result = match page_header.type_ {
+                PageType::DictionaryPage => {
+                    assert!(page_header.dictionary_page_header.is_some());
+                    let dict_header =
+                        page_header.dictionary_page_header.as_ref().unwrap();
+                    let is_sorted = dict_header.is_sorted.unwrap_or(false);
+                    Page::DictionaryPage {
+                        buf: ByteBufferPtr::new(buffer),
+                        num_values: dict_header.num_values as u32,
+                        encoding: Encoding::from(dict_header.encoding),
+                        is_sorted,
+                    }
+                }
+                PageType::DataPage => {
+                    assert!(page_header.data_page_header.is_some());
+                    let header = page_header.data_page_header.unwrap();
+                    self.seen_num_values += header.num_values as i64;
+                    Page::DataPage {
+                        buf: ByteBufferPtr::new(buffer),
+                        num_values: header.num_values as u32,
+                        encoding: Encoding::from(header.encoding),
+                        def_level_encoding: Encoding::from(
+                            header.definition_level_encoding,
+                        ),
+                        rep_level_encoding: Encoding::from(
+                            header.repetition_level_encoding,
+                        ),
+                        statistics: statistics::from_thrift(
+                            self.physical_type,
+                            header.statistics,
+                        ),
+                    }
+                }
+                PageType::DataPageV2 => {
+                    assert!(page_header.data_page_header_v2.is_some());
+                    let header = page_header.data_page_header_v2.unwrap();
+                    let is_compressed = header.is_compressed.unwrap_or(true);
+                    self.seen_num_values += header.num_values as i64;
+                    Page::DataPageV2 {
+                        buf: ByteBufferPtr::new(buffer),
+                        num_values: header.num_values as u32,
+                        encoding: Encoding::from(header.encoding),
+                        num_nulls: header.num_nulls as u32,
+                        num_rows: header.num_rows as u32,
+                        def_levels_byte_len: header.definition_levels_byte_length as u32,
+                        rep_levels_byte_len: header.repetition_levels_byte_length as u32,
+                        is_compressed,
+                        statistics: statistics::from_thrift(
+                            self.physical_type,
+                            header.statistics,
+                        ),
+                    }
+                }
+                _ => {
+                    // For unknown page type (e.g., INDEX_PAGE), skip and read next.
+                    continue;
+                }
+            };
+            return Ok(Some(result));
+        }
+
+        // We are at the end of this column chunk and no more page left. Return None.
+        Ok(None)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::basic::ColumnOrder;
+    use crate::record::RowAccessor;
+    use crate::schema::parser::parse_message_type;
+    use crate::util::test_common::{get_test_file, get_test_path};
+    use std::rc::Rc;
+
+    #[test]
+    fn test_cursor_and_file_has_the_same_behaviour() {
+        let mut buf: Vec<u8> = Vec::new();
+        get_test_file("alltypes_plain.parquet")
+            .read_to_end(&mut buf)
+            .unwrap();
+        let cursor = SliceableCursor::new(buf);
+        let read_from_cursor = SerializedFileReader::new(cursor).unwrap();
+
+        let test_file = get_test_file("alltypes_plain.parquet");
+        let read_from_file = SerializedFileReader::new(test_file).unwrap();
+
+        let file_iter = read_from_file.get_row_iter(None).unwrap();
+        let cursor_iter = read_from_cursor.get_row_iter(None).unwrap();
+
+        assert!(file_iter.eq(cursor_iter));
+    }
+
+    #[test]
+    fn test_file_reader_try_from() {
+        // Valid file path
+        let test_file = get_test_file("alltypes_plain.parquet");
+        let test_path_buf = get_test_path("alltypes_plain.parquet");
+        let test_path = test_path_buf.as_path();
+        let test_path_str = test_path.to_str().unwrap();
+
+        let reader = SerializedFileReader::try_from(test_file);
+        assert!(reader.is_ok());
+
+        let reader = SerializedFileReader::try_from(test_path);
+        assert!(reader.is_ok());
+
+        let reader = SerializedFileReader::try_from(test_path_str);
+        assert!(reader.is_ok());
+
+        let reader = SerializedFileReader::try_from(test_path_str.to_string());
+        assert!(reader.is_ok());
+
+        // Invalid file path
+        let test_path = Path::new("invalid.parquet");
+        let test_path_str = test_path.to_str().unwrap();
+
+        let reader = SerializedFileReader::try_from(test_path);
+        assert!(reader.is_err());
+
+        let reader = SerializedFileReader::try_from(test_path_str);
+        assert!(reader.is_err());
+
+        let reader = SerializedFileReader::try_from(test_path_str.to_string());
+        assert!(reader.is_err());
+    }
+
+    #[test]
+    fn test_file_reader_into_iter() -> Result<()> {
+        let path = get_test_path("alltypes_plain.parquet");
+        let vec = vec![path.clone(), path]
+            .iter()
+            .map(|p| SerializedFileReader::try_from(p.as_path()).unwrap())
+            .flat_map(|r| r.into_iter())
+            .flat_map(|r| r.get_int(0))
+            .collect::<Vec<_>>();
+
+        // rows in the parquet file are not sorted by "id"
+        // each file contains [id:4, id:5, id:6, id:7, id:2, id:3, id:0, id:1]
+        assert_eq!(vec, vec![4, 5, 6, 7, 2, 3, 0, 1, 4, 5, 6, 7, 2, 3, 0, 1]);
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_file_reader_into_iter_project() -> Result<()> {
+        let path = get_test_path("alltypes_plain.parquet");
+        let result = vec![path]
+            .iter()
+            .map(|p| SerializedFileReader::try_from(p.as_path()).unwrap())
+            .flat_map(|r| {
+                let schema = "message schema { OPTIONAL INT32 id; }";
+                let proj = parse_message_type(&schema).ok();
+
+                r.into_iter().project(proj).unwrap()
+            })
+            .map(|r| format!("{}", r))
+            .collect::<Vec<_>>()
+            .join(",");
+
+        assert_eq!(
+            result,
+            "{id: 4},{id: 5},{id: 6},{id: 7},{id: 2},{id: 3},{id: 0},{id: 1}"
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_reuse_file_chunk() {
+        // This test covers the case of maintaining the correct start position in a file
+        // stream for each column reader after initializing and moving to the next one
+        // (without necessarily reading the entire column).
+        let test_file = get_test_file("alltypes_plain.parquet");
+        let reader = SerializedFileReader::new(test_file).unwrap();
+        let row_group = reader.get_row_group(0).unwrap();
+
+        let mut page_readers = Vec::new();
+        for i in 0..row_group.num_columns() {
+            page_readers.push(row_group.get_column_page_reader(i).unwrap());
+        }
+
+        // Now buffer each col reader, we do not expect any failures like:
+        // General("underlying Thrift error: end of file")
+        for mut page_reader in page_readers {
+            assert!(page_reader.get_next_page().is_ok());
+        }
+    }
+
+    #[test]
+    fn test_file_reader() {
+        let test_file = get_test_file("alltypes_plain.parquet");
+        let reader_result = SerializedFileReader::new(test_file);
+        assert!(reader_result.is_ok());
+        let reader = reader_result.unwrap();
+
+        // Test contents in Parquet metadata
+        let metadata = reader.metadata();
+        assert_eq!(metadata.num_row_groups(), 1);
+
+        // Test contents in file metadata
+        let file_metadata = metadata.file_metadata();
+        assert!(file_metadata.created_by().is_some());
+        assert_eq!(
+      file_metadata.created_by().as_ref().unwrap(),
+      "impala version 1.3.0-INTERNAL (build 8a48ddb1eff84592b3fc06bc6f51ec120e1fffc9)"
+    );
+        assert!(file_metadata.key_value_metadata().is_none());
+        assert_eq!(file_metadata.num_rows(), 8);
+        assert_eq!(file_metadata.version(), 1);
+        assert_eq!(file_metadata.column_orders(), None);
+
+        // Test contents in row group metadata
+        let row_group_metadata = metadata.row_group(0);
+        assert_eq!(row_group_metadata.num_columns(), 11);
+        assert_eq!(row_group_metadata.num_rows(), 8);
+        assert_eq!(row_group_metadata.total_byte_size(), 671);
+        // Check each column order
+        for i in 0..row_group_metadata.num_columns() {
+            assert_eq!(file_metadata.column_order(i), ColumnOrder::UNDEFINED);
+        }
+
+        // Test row group reader
+        let row_group_reader_result = reader.get_row_group(0);
+        assert!(row_group_reader_result.is_ok());
+        let row_group_reader: Box<RowGroupReader> = row_group_reader_result.unwrap();
+        assert_eq!(
+            row_group_reader.num_columns(),
+            row_group_metadata.num_columns()
+        );
+        assert_eq!(
+            row_group_reader.metadata().total_byte_size(),
+            row_group_metadata.total_byte_size()
+        );
+
+        // Test page readers
+        // TODO: test for every column
+        let page_reader_0_result = row_group_reader.get_column_page_reader(0);
+        assert!(page_reader_0_result.is_ok());
+        let mut page_reader_0: Box<PageReader> = page_reader_0_result.unwrap();
+        let mut page_count = 0;
+        while let Ok(Some(page)) = page_reader_0.get_next_page() {
+            let is_expected_page = match page {
+                Page::DictionaryPage {
+                    buf,
+                    num_values,
+                    encoding,
+                    is_sorted,
+                } => {
+                    assert_eq!(buf.len(), 32);
+                    assert_eq!(num_values, 8);
+                    assert_eq!(encoding, Encoding::PLAIN_DICTIONARY);
+                    assert_eq!(is_sorted, false);
+                    true
+                }
+                Page::DataPage {
+                    buf,
+                    num_values,
+                    encoding,
+                    def_level_encoding,
+                    rep_level_encoding,
+                    statistics,
+                } => {
+                    assert_eq!(buf.len(), 11);
+                    assert_eq!(num_values, 8);
+                    assert_eq!(encoding, Encoding::PLAIN_DICTIONARY);
+                    assert_eq!(def_level_encoding, Encoding::RLE);
+                    assert_eq!(rep_level_encoding, Encoding::BIT_PACKED);
+                    assert!(statistics.is_none());
+                    true
+                }
+                _ => false,
+            };
+            assert!(is_expected_page);
+            page_count += 1;
+        }
+        assert_eq!(page_count, 2);
+    }
+
+    #[test]
+    fn test_file_reader_datapage_v2() {
+        let test_file = get_test_file("datapage_v2.snappy.parquet");
+        let reader_result = SerializedFileReader::new(test_file);
+        assert!(reader_result.is_ok());
+        let reader = reader_result.unwrap();
+
+        // Test contents in Parquet metadata
+        let metadata = reader.metadata();
+        assert_eq!(metadata.num_row_groups(), 1);
+
+        // Test contents in file metadata
+        let file_metadata = metadata.file_metadata();
+        assert!(file_metadata.created_by().is_some());
+        assert_eq!(
+            file_metadata.created_by().as_ref().unwrap(),
+            "parquet-mr version 1.8.1 (build 4aba4dae7bb0d4edbcf7923ae1339f28fd3f7fcf)"
+        );
+        assert!(file_metadata.key_value_metadata().is_some());
+        assert_eq!(
+            file_metadata.key_value_metadata().to_owned().unwrap().len(),
+            1
+        );
+
+        assert_eq!(file_metadata.num_rows(), 5);
+        assert_eq!(file_metadata.version(), 1);
+        assert_eq!(file_metadata.column_orders(), None);
+
+        let row_group_metadata = metadata.row_group(0);
+
+        // Check each column order
+        for i in 0..row_group_metadata.num_columns() {
+            assert_eq!(file_metadata.column_order(i), ColumnOrder::UNDEFINED);
+        }
+
+        // Test row group reader
+        let row_group_reader_result = reader.get_row_group(0);
+        assert!(row_group_reader_result.is_ok());
+        let row_group_reader: Box<RowGroupReader> = row_group_reader_result.unwrap();
+        assert_eq!(
+            row_group_reader.num_columns(),
+            row_group_metadata.num_columns()
+        );
+        assert_eq!(
+            row_group_reader.metadata().total_byte_size(),
+            row_group_metadata.total_byte_size()
+        );
+
+        // Test page readers
+        // TODO: test for every column
+        let page_reader_0_result = row_group_reader.get_column_page_reader(0);
+        assert!(page_reader_0_result.is_ok());
+        let mut page_reader_0: Box<PageReader> = page_reader_0_result.unwrap();
+        let mut page_count = 0;
+        while let Ok(Some(page)) = page_reader_0.get_next_page() {
+            let is_expected_page = match page {
+                Page::DictionaryPage {
+                    buf,
+                    num_values,
+                    encoding,
+                    is_sorted,
+                } => {
+                    assert_eq!(buf.len(), 7);
+                    assert_eq!(num_values, 1);
+                    assert_eq!(encoding, Encoding::PLAIN);
+                    assert_eq!(is_sorted, false);
+                    true
+                }
+                Page::DataPageV2 {
+                    buf,
+                    num_values,
+                    encoding,
+                    num_nulls,
+                    num_rows,
+                    def_levels_byte_len,
+                    rep_levels_byte_len,
+                    is_compressed,
+                    statistics,
+                } => {
+                    assert_eq!(buf.len(), 4);
+                    assert_eq!(num_values, 5);
+                    assert_eq!(encoding, Encoding::RLE_DICTIONARY);
+                    assert_eq!(num_nulls, 1);
+                    assert_eq!(num_rows, 5);
+                    assert_eq!(def_levels_byte_len, 2);
+                    assert_eq!(rep_levels_byte_len, 0);
+                    assert_eq!(is_compressed, true);
+                    assert!(statistics.is_some());
+                    true
+                }
+                _ => false,
+            };
+            assert!(is_expected_page);
+            page_count += 1;
+        }
+        assert_eq!(page_count, 2);
+    }
+
+    #[test]
+    fn test_page_iterator() {
+        let file = get_test_file("alltypes_plain.parquet");
+        let file_reader = Rc::new(SerializedFileReader::new(file).unwrap());
+
+        let mut page_iterator = FilePageIterator::new(0, file_reader.clone()).unwrap();
+
+        // read first page
+        let page = page_iterator.next();
+        assert!(page.is_some());
+        assert!(page.unwrap().is_ok());
+
+        // reach end of file
+        let page = page_iterator.next();
+        assert!(page.is_none());
+
+        let row_group_indices = Box::new(0..1);
+        let mut page_iterator =
+            FilePageIterator::with_row_groups(0, row_group_indices, file_reader).unwrap();
+
+        // read first page
+        let page = page_iterator.next();
+        assert!(page.is_some());
+        assert!(page.unwrap().is_ok());
+
+        // reach end of file
+        let page = page_iterator.next();
+        assert!(page.is_none());
+    }
+
+    #[test]
+    fn test_file_reader_key_value_metadata() {
+        let file = get_test_file("binary.parquet");
+        let file_reader = Rc::new(SerializedFileReader::new(file).unwrap());
+
+        let metadata = file_reader
+            .metadata
+            .file_metadata()
+            .key_value_metadata()
+            .as_ref()
+            .unwrap();
+
+        assert_eq!(metadata.len(), 3);
+
+        assert_eq!(metadata.get(0).unwrap().key, "parquet.proto.descriptor");
+
+        assert_eq!(metadata.get(1).unwrap().key, "writer.model.name");
+        assert_eq!(metadata.get(1).unwrap().value, Some("protobuf".to_owned()));
+
+        assert_eq!(metadata.get(2).unwrap().key, "parquet.proto.class");
+        assert_eq!(
+            metadata.get(2).unwrap().value,
+            Some("foo.baz.Foobaz$Event".to_owned())
+        );
+    }
+}
diff --git a/rust/parquet/src/file/statistics.rs b/rust/parquet/src/file/statistics.rs
index edd3c63f24f..20de31c9956 100644
--- a/rust/parquet/src/file/statistics.rs
+++ b/rust/parquet/src/file/statistics.rs
@@ -172,15 +172,15 @@ pub fn from_thrift(
                     old_format,
                 ),
                 Type::BYTE_ARRAY => Statistics::byte_array(
-                    min.map(|data| ByteArray::from(data)),
-                    max.map(|data| ByteArray::from(data)),
+                    min.map(ByteArray::from),
+                    max.map(ByteArray::from),
                     distinct_count,
                     null_count,
                     old_format,
                 ),
                 Type::FIXED_LEN_BYTE_ARRAY => Statistics::fixed_len_byte_array(
-                    min.map(|data| ByteArray::from(data)),
-                    max.map(|data| ByteArray::from(data)),
+                    min.map(ByteArray::from),
+                    max.map(ByteArray::from),
                     distinct_count,
                     null_count,
                     old_format,
@@ -195,11 +195,7 @@ pub fn from_thrift(
 
 // Convert Statistics into Thrift definition.
 pub fn to_thrift(stats: Option<&Statistics>) -> Option<TStatistics> {
-    if stats.is_none() {
-        return None;
-    }
-
-    let stats = stats.unwrap();
+    let stats = stats?;
 
     let mut thrift_stats = TStatistics {
         max: None,
diff --git a/rust/parquet/src/file/writer.rs b/rust/parquet/src/file/writer.rs
index ede6ce47a2e..a160a4be076 100644
--- a/rust/parquet/src/file/writer.rs
+++ b/rust/parquet/src/file/writer.rs
@@ -34,12 +34,15 @@ use crate::column::{
 };
 use crate::errors::{ParquetError, Result};
 use crate::file::{
-    metadata::*, properties::WriterPropertiesPtr, reader::TryClone,
+    metadata::*, properties::WriterPropertiesPtr,
     statistics::to_thrift as statistics_to_thrift, FOOTER_SIZE, PARQUET_MAGIC,
 };
 use crate::schema::types::{self, SchemaDescPtr, SchemaDescriptor, TypePtr};
 use crate::util::io::{FileSink, Position};
 
+// Exposed publically so client code can implement [`ParquetWriter`]
+pub use crate::util::io::TryClone;
+
 // ----------------------------------------------------------------------
 // APIs for file & row group writers
 
@@ -152,7 +155,7 @@ impl<W: ParquetWriter> SerializedFileWriter<W> {
 
     /// Writes magic bytes at the beginning of the file.
     fn start_file(buf: &mut W) -> Result<()> {
-        buf.write(&PARQUET_MAGIC)?;
+        buf.write_all(&PARQUET_MAGIC)?;
         Ok(())
     }
 
@@ -176,7 +179,7 @@ impl<W: ParquetWriter> SerializedFileWriter<W> {
             row_groups: self
                 .row_groups
                 .as_slice()
-                .into_iter()
+                .iter()
                 .map(|v| v.to_thrift())
                 .collect(),
             key_value_metadata: self.props.key_value_metadata().to_owned(),
@@ -197,8 +200,8 @@ impl<W: ParquetWriter> SerializedFileWriter<W> {
         let mut footer_buffer: [u8; FOOTER_SIZE] = [0; FOOTER_SIZE];
         let metadata_len = (end_pos - start_pos) as i32;
         LittleEndian::write_i32(&mut footer_buffer, metadata_len);
-        (&mut footer_buffer[4..]).write(&PARQUET_MAGIC)?;
-        self.buf.write(&footer_buffer)?;
+        (&mut footer_buffer[4..]).write_all(&PARQUET_MAGIC)?;
+        self.buf.write_all(&footer_buffer)?;
         Ok(())
     }
 
@@ -372,7 +375,7 @@ impl<W: 'static + ParquetWriter> RowGroupWriter for SerializedRowGroupWriter<W>
         if self.row_group_metadata.is_none() {
             self.assert_previous_writer_closed()?;
 
-            let column_chunks = std::mem::replace(&mut self.column_chunks, vec![]);
+            let column_chunks = std::mem::take(&mut self.column_chunks);
             let row_group_metadata = RowGroupMetaData::builder(self.descr.clone())
                 .set_column_metadata(column_chunks)
                 .set_total_byte_size(self.total_bytes_written as i64)
@@ -445,8 +448,8 @@ impl<T: Write + Position> PageWriter for SerializedPageWriter<T> {
             data_page_header_v2: None,
         };
 
-        match page.compressed_page() {
-            &Page::DataPage {
+        match *page.compressed_page() {
+            Page::DataPage {
                 def_level_encoding,
                 rep_level_encoding,
                 ref statistics,
@@ -461,7 +464,7 @@ impl<T: Write + Position> PageWriter for SerializedPageWriter<T> {
                 };
                 page_header.data_page_header = Some(data_page_header);
             }
-            &Page::DataPageV2 {
+            Page::DataPageV2 {
                 num_nulls,
                 num_rows,
                 def_levels_byte_len,
@@ -482,7 +485,7 @@ impl<T: Write + Position> PageWriter for SerializedPageWriter<T> {
                 };
                 page_header.data_page_header_v2 = Some(data_page_header_v2);
             }
-            &Page::DictionaryPage { is_sorted, .. } => {
+            Page::DictionaryPage { is_sorted, .. } => {
                 let dictionary_page_header = parquet::DictionaryPageHeader {
                     num_values: num_values as i32,
                     encoding: encoding.into(),
@@ -824,8 +827,8 @@ mod tests {
         for page in pages {
             let uncompressed_len = page.buffer().len();
 
-            let compressed_page = match page {
-                &Page::DataPage {
+            let compressed_page = match *page {
+                Page::DataPage {
                     ref buf,
                     num_values,
                     encoding,
@@ -848,7 +851,7 @@ mod tests {
                         ),
                     }
                 }
-                &Page::DataPageV2 {
+                Page::DataPageV2 {
                     ref buf,
                     num_values,
                     encoding,
@@ -881,7 +884,7 @@ mod tests {
                         ),
                     }
                 }
-                &Page::DictionaryPage {
+                Page::DictionaryPage {
                     ref buf,
                     num_values,
                     encoding,
diff --git a/rust/parquet/src/lib.rs b/rust/parquet/src/lib.rs
index 6d5a6b3e7a9..c2925f9970b 100644
--- a/rust/parquet/src/lib.rs
+++ b/rust/parquet/src/lib.rs
@@ -19,6 +19,12 @@
 #![allow(dead_code)]
 #![allow(non_camel_case_types)]
 #![allow(bare_trait_objects)]
+#![allow(
+    clippy::too_many_arguments,
+    clippy::new_without_default,
+    clippy::needless_range_loop,
+    clippy::borrowed_box
+)]
 
 #[macro_use]
 pub mod errors;
diff --git a/rust/parquet/src/record/api.rs b/rust/parquet/src/record/api.rs
index c7c83a05542..ef7e0709052 100644
--- a/rust/parquet/src/record/api.rs
+++ b/rust/parquet/src/record/api.rs
@@ -45,6 +45,7 @@ pub struct Row {
     fields: Vec<(String, Field)>,
 }
 
+#[allow(clippy::len_without_is_empty)]
 impl Row {
     /// Get the number of fields in this row.
     pub fn len(&self) -> usize {
@@ -231,6 +232,7 @@ pub struct List {
     elements: Vec<Field>,
 }
 
+#[allow(clippy::len_without_is_empty)]
 impl List {
     /// Get the number of fields in this row
     pub fn len(&self) -> usize {
@@ -348,6 +350,7 @@ pub struct Map {
     entries: Vec<(Field, Field)>,
 }
 
+#[allow(clippy::len_without_is_empty)]
 impl Map {
     /// Get the number of fields in this row
     pub fn len(&self) -> usize {
@@ -756,10 +759,10 @@ fn convert_decimal_to_string(decimal: &Decimal) -> String {
 }
 
 #[cfg(test)]
+#[allow(clippy::approx_constant, clippy::many_single_char_names)]
 mod tests {
     use super::*;
 
-    use chrono;
     use std::rc::Rc;
 
     use crate::schema::types::{ColumnDescriptor, ColumnPath, PrimitiveTypeBuilder};
@@ -773,7 +776,6 @@ mod tests {
                 .unwrap();
             Rc::new(ColumnDescriptor::new(
                 Rc::new(tpe),
-                None,
                 0,
                 0,
                 ColumnPath::from("col"),
@@ -789,7 +791,6 @@ mod tests {
                 .unwrap();
             Rc::new(ColumnDescriptor::new(
                 Rc::new(tpe),
-                None,
                 0,
                 0,
                 ColumnPath::from("col"),
@@ -983,11 +984,11 @@ mod tests {
             assert_eq!(res, exp);
         }
 
-        check_date_conversion(2010, 01, 02);
-        check_date_conversion(2014, 05, 01);
-        check_date_conversion(2016, 02, 29);
-        check_date_conversion(2017, 09, 12);
-        check_date_conversion(2018, 03, 31);
+        check_date_conversion(2010, 1, 2);
+        check_date_conversion(2014, 5, 1);
+        check_date_conversion(2016, 2, 29);
+        check_date_conversion(2017, 9, 12);
+        check_date_conversion(2018, 3, 31);
     }
 
     #[test]
@@ -1000,10 +1001,10 @@ mod tests {
             assert_eq!(res, exp);
         }
 
-        check_datetime_conversion(2010, 01, 02, 13, 12, 54);
-        check_datetime_conversion(2011, 01, 03, 08, 23, 01);
-        check_datetime_conversion(2012, 04, 05, 11, 06, 32);
-        check_datetime_conversion(2013, 05, 12, 16, 38, 00);
+        check_datetime_conversion(2010, 1, 2, 13, 12, 54);
+        check_datetime_conversion(2011, 1, 3, 8, 23, 1);
+        check_datetime_conversion(2012, 4, 5, 11, 6, 32);
+        check_datetime_conversion(2013, 5, 12, 16, 38, 0);
         check_datetime_conversion(2014, 11, 28, 21, 15, 12);
     }
 
@@ -1311,8 +1312,8 @@ mod tests {
         assert_eq!(4, row.get_ushort(7).unwrap());
         assert_eq!(5, row.get_uint(8).unwrap());
         assert_eq!(6, row.get_ulong(9).unwrap());
-        assert_eq!(7.1, row.get_float(10).unwrap());
-        assert_eq!(8.1, row.get_double(11).unwrap());
+        assert!(7.1 - row.get_float(10).unwrap() < f32::EPSILON);
+        assert!(8.1 - row.get_double(11).unwrap() < f64::EPSILON);
         assert_eq!("abc", row.get_string(12).unwrap());
         assert_eq!(5, row.get_bytes(13).unwrap().len());
         assert_eq!(7, row.get_decimal(14).unwrap().precision());
@@ -1459,10 +1460,10 @@ mod tests {
             Field::Float(9.2),
             Field::Float(10.3),
         ]);
-        assert_eq!(10.3, list.get_float(2).unwrap());
+        assert!(10.3 - list.get_float(2).unwrap() < f32::EPSILON);
 
         let list = make_list(vec![Field::Double(3.1415)]);
-        assert_eq!(3.1415, list.get_double(0).unwrap());
+        assert!(3.1415 - list.get_double(0).unwrap() < f64::EPSILON);
 
         let list = make_list(vec![Field::Str("abc".to_string())]);
         assert_eq!(&"abc".to_string(), list.get_string(0).unwrap());
@@ -1596,9 +1597,46 @@ mod tests {
         for i in 0..5 {
             assert_eq!((i + 1) as i32, map.get_keys().get_int(i).unwrap());
             assert_eq!(
-                &((i as u8 + 'a' as u8) as char).to_string(),
+                &((i as u8 + b'a') as char).to_string(),
                 map.get_values().get_string(i).unwrap()
             );
         }
     }
 }
+
+#[cfg(test)]
+#[allow(clippy::approx_constant, clippy::many_single_char_names)]
+mod api_tests {
+    use super::make_row;
+    use crate::record::Field;
+
+    #[test]
+    fn test_field_visibility() {
+        let row = make_row(vec![(
+            "a".to_string(),
+            Field::Group(make_row(vec![
+                ("x".to_string(), Field::Null),
+                ("Y".to_string(), Field::Int(2)),
+            ])),
+        )]);
+
+        match row.get_column_iter().next() {
+            Some(column) => {
+                assert_eq!("a", column.0);
+                match column.1 {
+                    Field::Group(r) => {
+                        assert_eq!(
+                            &make_row(vec![
+                                ("x".to_string(), Field::Null),
+                                ("Y".to_string(), Field::Int(2)),
+                            ]),
+                            r
+                        );
+                    }
+                    _ => panic!("Expected the first column to be Field::Group"),
+                }
+            }
+            None => panic!("Expected at least one column"),
+        }
+    }
+}
diff --git a/rust/parquet/src/record/mod.rs b/rust/parquet/src/record/mod.rs
index 4427ada72af..fb4abb539d9 100644
--- a/rust/parquet/src/record/mod.rs
+++ b/rust/parquet/src/record/mod.rs
@@ -19,8 +19,10 @@
 
 mod api;
 pub mod reader;
+mod record_writer;
 mod triplet;
 
-pub use self::api::{
-    List, ListAccessor, Map, MapAccessor, Row, RowAccessor, RowFormatter,
+pub use self::{
+    api::{Field, List, ListAccessor, Map, MapAccessor, Row, RowAccessor},
+    record_writer::RecordWriter,
 };
diff --git a/rust/parquet/src/record/reader.rs b/rust/parquet/src/record/reader.rs
index 1a0efaa2b92..cac2261af53 100644
--- a/rust/parquet/src/record/reader.rs
+++ b/rust/parquet/src/record/reader.rs
@@ -159,7 +159,7 @@ impl TreeBuilder {
                     if Reader::is_element_type(&repeated_field) {
                         // Support for backward compatible lists
                         let reader = self.reader_tree(
-                            repeated_field.clone(),
+                            repeated_field,
                             &mut path,
                             curr_def_level,
                             curr_rep_level,
@@ -507,8 +507,8 @@ impl Reader {
             Reader::PrimitiveReader(ref field, _) => field.name(),
             Reader::OptionReader(_, ref reader) => reader.field_name(),
             Reader::GroupReader(ref opt, ..) => match opt {
-                &Some(ref field) => field.name(),
-                &None => panic!("Field is None for group reader"),
+                Some(ref field) => field.name(),
+                None => panic!("Field is None for group reader"),
             },
             Reader::RepeatedReader(ref field, ..) => field.name(),
             Reader::KeyValueReader(ref field, ..) => field.name(),
@@ -521,8 +521,8 @@ impl Reader {
             Reader::PrimitiveReader(ref field, _) => field.get_basic_info().repetition(),
             Reader::OptionReader(_, ref reader) => reader.repetition(),
             Reader::GroupReader(ref opt, ..) => match opt {
-                &Some(ref field) => field.get_basic_info().repetition(),
-                &None => panic!("Field is None for group reader"),
+                Some(ref field) => field.get_basic_info().repetition(),
+                None => panic!("Field is None for group reader"),
             },
             Reader::RepeatedReader(ref field, ..) => field.get_basic_info().repetition(),
             Reader::KeyValueReader(ref field, ..) => field.get_basic_info().repetition(),
@@ -658,7 +658,7 @@ impl<'a> RowIter<'a> {
             file_reader,
             tree_builder,
             num_row_groups,
-            row_iter: row_iter,
+            row_iter,
             current_row_group: 0,
         }
     }
diff --git a/rust/parquet/src/record/record_writer.rs b/rust/parquet/src/record/record_writer.rs
new file mode 100644
index 00000000000..00ce9fd4e47
--- /dev/null
+++ b/rust/parquet/src/record/record_writer.rs
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use super::super::errors::ParquetError;
+use super::super::file::writer::RowGroupWriter;
+
+pub trait RecordWriter<T> {
+    fn write_to_row_group(
+        &self,
+        row_group_writer: &mut Box<RowGroupWriter>,
+    ) -> Result<(), ParquetError>;
+}
diff --git a/rust/parquet/src/record/triplet.rs b/rust/parquet/src/record/triplet.rs
index dfa396e182a..6a7efa5d7ac 100644
--- a/rust/parquet/src/record/triplet.rs
+++ b/rust/parquet/src/record/triplet.rs
@@ -367,14 +367,7 @@ mod tests {
     fn test_triplet_zero_batch_size() {
         let column_path =
             ColumnPath::from(vec!["b_struct".to_string(), "b_c_int".to_string()]);
-        test_column_in_file(
-            "nulls.snappy.parquet",
-            0,
-            &column_path,
-            &vec![],
-            &vec![],
-            &vec![],
-        );
+        test_column_in_file("nulls.snappy.parquet", 0, &column_path, &[], &[], &[]);
     }
 
     #[test]
diff --git a/rust/parquet/src/schema/parser.rs b/rust/parquet/src/schema/parser.rs
index bc7fcca7dfe..9c4f0e3d12e 100644
--- a/rust/parquet/src/schema/parser.rs
+++ b/rust/parquet/src/schema/parser.rs
@@ -51,7 +51,7 @@ use crate::schema::types::{Type, TypePtr};
 /// Parses message type as string into a Parquet [`Type`](crate::schema::types::Type)
 /// which, for example, could be used to extract individual columns. Returns Parquet
 /// general error when parsing or validation fails.
-pub fn parse_message_type<'a>(message_type: &'a str) -> Result<Type> {
+pub fn parse_message_type(message_type: &str) -> Result<Type> {
     let mut parser = Parser {
         tokenizer: &mut Tokenizer::from_str(message_type),
     };
@@ -156,7 +156,7 @@ fn parse_i32(
     parse_fail_msg: &str,
 ) -> Result<i32> {
     value
-        .ok_or(general_err!(not_found_msg))
+        .ok_or_else(|| general_err!(not_found_msg))
         .and_then(|v| v.parse::<i32>().map_err(|_| general_err!(parse_fail_msg)))
 }
 
@@ -169,7 +169,7 @@ impl<'a> Parser<'a> {
                 let name = self
                     .tokenizer
                     .next()
-                    .ok_or(general_err!("Expected name, found None"))?;
+                    .ok_or_else(|| general_err!("Expected name, found None"))?;
                 let mut fields = self.parse_child_types()?;
                 Type::group_type_builder(name)
                     .with_fields(&mut fields)
@@ -200,7 +200,7 @@ impl<'a> Parser<'a> {
         let repetition = self
             .tokenizer
             .next()
-            .ok_or(general_err!("Expected repetition, found None"))
+            .ok_or_else(|| general_err!("Expected repetition, found None"))
             .and_then(|v| v.to_uppercase().parse::<Repetition>())?;
 
         match self.tokenizer.next() {
@@ -220,14 +220,14 @@ impl<'a> Parser<'a> {
         let name = self
             .tokenizer
             .next()
-            .ok_or(general_err!("Expected name, found None"))?;
+            .ok_or_else(|| general_err!("Expected name, found None"))?;
 
         // Parse logical type if exists
         let logical_type = if let Some("(") = self.tokenizer.next() {
             let tpe = self
                 .tokenizer
                 .next()
-                .ok_or(general_err!("Expected logical type, found None"))
+                .ok_or_else(|| general_err!("Expected logical type, found None"))
                 .and_then(|v| v.to_uppercase().parse::<LogicalType>())?;
             assert_token(self.tokenizer.next(), ")")?;
             tpe
@@ -278,14 +278,14 @@ impl<'a> Parser<'a> {
         let name = self
             .tokenizer
             .next()
-            .ok_or(general_err!("Expected name, found None"))?;
+            .ok_or_else(|| general_err!("Expected name, found None"))?;
 
         // Parse logical type
         let (logical_type, precision, scale) = if let Some("(") = self.tokenizer.next() {
             let tpe = self
                 .tokenizer
                 .next()
-                .ok_or(general_err!("Expected logical type, found None"))
+                .ok_or_else(|| general_err!("Expected logical type, found None"))
                 .and_then(|v| v.to_uppercase().parse::<LogicalType>())?;
 
             // Parse precision and scale for decimals
diff --git a/rust/parquet/src/schema/printer.rs b/rust/parquet/src/schema/printer.rs
index 2a1f63d2af6..9a19133d936 100644
--- a/rust/parquet/src/schema/printer.rs
+++ b/rust/parquet/src/schema/printer.rs
@@ -56,11 +56,11 @@ use crate::schema::types::Type;
 #[allow(unused_must_use)]
 pub fn print_parquet_metadata(out: &mut io::Write, metadata: &ParquetMetaData) {
     print_file_metadata(out, &metadata.file_metadata());
-    writeln!(out, "");
-    writeln!(out, "");
+    writeln!(out);
+    writeln!(out);
     writeln!(out, "num of row groups: {}", metadata.num_row_groups());
     writeln!(out, "row groups:");
-    writeln!(out, "");
+    writeln!(out);
     for (i, rg) in metadata.row_groups().iter().enumerate() {
         writeln!(out, "row group {}:", i);
         print_dashes(out, 80);
@@ -109,11 +109,11 @@ pub fn print_schema(out: &mut io::Write, tp: &Type) {
 fn print_row_group_metadata(out: &mut io::Write, rg_metadata: &RowGroupMetaData) {
     writeln!(out, "total byte size: {}", rg_metadata.total_byte_size());
     writeln!(out, "num of rows: {}", rg_metadata.num_rows());
-    writeln!(out, "");
+    writeln!(out);
     writeln!(out, "num of columns: {}", rg_metadata.num_columns());
     writeln!(out, "columns: ");
     for (i, cc) in rg_metadata.columns().iter().enumerate() {
-        writeln!(out, "");
+        writeln!(out);
         writeln!(out, "column {}:", i);
         print_dashes(out, 80);
         print_column_chunk_metadata(out, cc);
@@ -163,7 +163,7 @@ fn print_column_chunk_metadata(out: &mut io::Write, cc_metadata: &ColumnChunkMet
         Some(stats) => stats.to_string(),
     };
     writeln!(out, "statistics: {}", statistics_str);
-    writeln!(out, "");
+    writeln!(out);
 }
 
 #[allow(unused_must_use)]
@@ -171,7 +171,7 @@ fn print_dashes(out: &mut io::Write, num: i32) {
     for _ in 0..num {
         write!(out, "-");
     }
-    writeln!(out, "");
+    writeln!(out);
 }
 
 const INDENT_WIDTH: i32 = 2;
@@ -199,8 +199,8 @@ impl<'a> Printer<'a> {
 impl<'a> Printer<'a> {
     pub fn print(&mut self, tp: &Type) {
         self.print_indent();
-        match tp {
-            &Type::PrimitiveType {
+        match *tp {
+            Type::PrimitiveType {
                 ref basic_info,
                 physical_type,
                 type_length,
@@ -239,7 +239,7 @@ impl<'a> Printer<'a> {
                     logical_type_str
                 );
             }
-            &Type::GroupType {
+            Type::GroupType {
                 ref basic_info,
                 ref fields,
             } => {
@@ -257,7 +257,7 @@ impl<'a> Printer<'a> {
                 self.indent += INDENT_WIDTH;
                 for c in fields {
                     self.print(&c);
-                    writeln!(self.output, "");
+                    writeln!(self.output);
                 }
                 self.indent -= INDENT_WIDTH;
                 self.print_indent();
@@ -291,14 +291,14 @@ mod tests {
         let mut s = String::new();
         {
             let mut p = Printer::new(&mut s);
-            let foo = Type::primitive_type_builder("foo", PhysicalType::INT32)
+            let field = Type::primitive_type_builder("field", PhysicalType::INT32)
                 .with_repetition(Repetition::REQUIRED)
                 .with_logical_type(LogicalType::INT_32)
                 .build()
                 .unwrap();
-            p.print(&foo);
+            p.print(&field);
         }
-        assert_eq!(&mut s, "REQUIRED INT32 foo (INT_32);");
+        assert_eq!(&mut s, "REQUIRED INT32 field (INT_32);");
     }
 
     #[test]
@@ -306,13 +306,13 @@ mod tests {
         let mut s = String::new();
         {
             let mut p = Printer::new(&mut s);
-            let foo = Type::primitive_type_builder("foo", PhysicalType::DOUBLE)
+            let field = Type::primitive_type_builder("field", PhysicalType::DOUBLE)
                 .with_repetition(Repetition::REQUIRED)
                 .build()
                 .unwrap();
-            p.print(&foo);
+            p.print(&field);
         }
-        assert_eq!(&mut s, "REQUIRED DOUBLE foo;");
+        assert_eq!(&mut s, "REQUIRED DOUBLE field;");
     }
 
     #[test]
@@ -339,14 +339,14 @@ mod tests {
             let mut struct_fields = Vec::new();
             struct_fields.push(Rc::new(f1.unwrap()));
             struct_fields.push(Rc::new(f2.unwrap()));
-            let foo = Type::group_type_builder("foo")
+            let field = Type::group_type_builder("field")
                 .with_repetition(Repetition::OPTIONAL)
                 .with_fields(&mut struct_fields)
                 .with_id(1)
                 .build()
                 .unwrap();
             let mut fields = Vec::new();
-            fields.push(Rc::new(foo));
+            fields.push(Rc::new(field));
             fields.push(Rc::new(f3.unwrap()));
             let message = Type::group_type_builder("schema")
                 .with_fields(&mut fields)
@@ -356,7 +356,7 @@ mod tests {
             p.print(&message);
         }
         let expected = "message schema {
-  OPTIONAL group foo {
+  OPTIONAL group field {
     REQUIRED INT32 f1 (INT_32);
     OPTIONAL BYTE_ARRAY f2 (UTF8);
   }
@@ -432,7 +432,7 @@ mod tests {
             .build()
             .unwrap();
 
-        let foo = Type::group_type_builder("foo")
+        let field = Type::group_type_builder("field")
             .with_repetition(Repetition::OPTIONAL)
             .with_fields(&mut vec![Rc::new(f1), Rc::new(f2)])
             .build()
@@ -446,7 +446,7 @@ mod tests {
             .unwrap();
 
         let message = Type::group_type_builder("schema")
-            .with_fields(&mut vec![Rc::new(foo), Rc::new(f3)])
+            .with_fields(&mut vec![Rc::new(field), Rc::new(f3)])
             .build()
             .unwrap();
 
diff --git a/rust/parquet/src/schema/types.rs b/rust/parquet/src/schema/types.rs
index e1227c283dc..0ba12c17bff 100644
--- a/rust/parquet/src/schema/types.rs
+++ b/rust/parquet/src/schema/types.rs
@@ -546,7 +546,7 @@ impl ColumnPath {
     /// path.append(vec!["d".to_string(), "e".to_string()]);
     /// assert_eq!(&path.string(), "a.b.c.d.e");
     /// ```
-    pub fn append(&mut self, mut tail: Vec<String>) -> () {
+    pub fn append(&mut self, mut tail: Vec<String>) {
         self.parts.append(&mut tail);
     }
 }
@@ -592,13 +592,6 @@ pub struct ColumnDescriptor {
     // The "leaf" primitive type of this column
     primitive_type: TypePtr,
 
-    // The root type of this column. For instance, if the column is "a.b.c.d", then the
-    // primitive type is 'd' while the root_type is 'a'.
-    //
-    // NOTE: this is sometimes `None` for the convenience of testing. It should NEVER be
-    // `None` when running in production.
-    root_type: Option<TypePtr>,
-
     // The maximum definition level for this column
     max_def_level: i16,
 
@@ -613,14 +606,12 @@ impl ColumnDescriptor {
     /// Creates new descriptor for leaf-level column.
     pub fn new(
         primitive_type: TypePtr,
-        root_type: Option<TypePtr>,
         max_def_level: i16,
         max_rep_level: i16,
         path: ColumnPath,
     ) -> Self {
         Self {
             primitive_type,
-            root_type,
             max_def_level,
             max_rep_level,
             path,
@@ -653,13 +644,6 @@ impl ColumnDescriptor {
         self.primitive_type.clone()
     }
 
-    /// Returns root [`Type`](crate::schema::types::Type) (most top-level parent field)
-    /// for this leaf column.
-    pub fn root_type(&self) -> &Type {
-        assert!(self.root_type.is_some());
-        self.root_type.as_ref().unwrap()
-    }
-
     /// Returns column name.
     pub fn name(&self) -> &str {
         self.primitive_type.name()
@@ -674,7 +658,7 @@ impl ColumnDescriptor {
     /// Note that it will panic if called on a non-primitive type.
     pub fn physical_type(&self) -> PhysicalType {
         match self.primitive_type.as_ref() {
-            &Type::PrimitiveType { physical_type, .. } => physical_type,
+            Type::PrimitiveType { physical_type, .. } => *physical_type,
             _ => panic!("Expected primitive type!"),
         }
     }
@@ -683,7 +667,7 @@ impl ColumnDescriptor {
     /// Note that it will panic if called on a non-primitive type.
     pub fn type_length(&self) -> i32 {
         match self.primitive_type.as_ref() {
-            &Type::PrimitiveType { type_length, .. } => type_length,
+            Type::PrimitiveType { type_length, .. } => *type_length,
             _ => panic!("Expected primitive type!"),
         }
     }
@@ -692,7 +676,7 @@ impl ColumnDescriptor {
     /// Note that it will panic if called on a non-primitive type.
     pub fn type_precision(&self) -> i32 {
         match self.primitive_type.as_ref() {
-            &Type::PrimitiveType { precision, .. } => precision,
+            Type::PrimitiveType { precision, .. } => *precision,
             _ => panic!("Expected primitive type!"),
         }
     }
@@ -701,7 +685,7 @@ impl ColumnDescriptor {
     /// Note that it will panic if called on a non-primitive type.
     pub fn type_scale(&self) -> i32 {
         match self.primitive_type.as_ref() {
-            &Type::PrimitiveType { scale, .. } => scale,
+            Type::PrimitiveType { scale, .. } => *scale,
             _ => panic!("Expected primitive type!"),
         }
     }
@@ -724,7 +708,16 @@ pub struct SchemaDescriptor {
     // -- -- b     |
     // -- -- -- c  |
     // -- -- -- -- d
-    leaf_to_base: HashMap<usize, TypePtr>,
+    leaf_to_base: Vec<TypePtr>,
+}
+
+impl fmt::Debug for SchemaDescriptor {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // Skip leaves and leaf_to_base as they only a cache information already found in `schema`
+        f.debug_struct("SchemaDescriptor")
+            .field("schema", &self.schema)
+            .finish()
+    }
 }
 
 impl SchemaDescriptor {
@@ -732,19 +725,10 @@ impl SchemaDescriptor {
     pub fn new(tp: TypePtr) -> Self {
         assert!(tp.is_group(), "SchemaDescriptor should take a GroupType");
         let mut leaves = vec![];
-        let mut leaf_to_base = HashMap::new();
+        let mut leaf_to_base = Vec::new();
         for f in tp.get_fields() {
             let mut path = vec![];
-            build_tree(
-                f.clone(),
-                tp.clone(),
-                f.clone(),
-                0,
-                0,
-                &mut leaves,
-                &mut leaf_to_base,
-                &mut path,
-            );
+            build_tree(f, f, 0, 0, &mut leaves, &mut leaf_to_base, &mut path);
         }
 
         Self {
@@ -788,7 +772,7 @@ impl SchemaDescriptor {
         result.clone()
     }
 
-    fn column_root_of(&self, i: usize) -> &Rc<Type> {
+    fn column_root_of(&self, i: usize) -> &TypePtr {
         assert!(
             i < self.leaves.len(),
             "Index out of bound: {} not in [0, {})",
@@ -796,13 +780,9 @@ impl SchemaDescriptor {
             self.leaves.len()
         );
 
-        let result = self.leaf_to_base.get(&i);
-        assert!(
-            result.is_some(),
-            "Expected a value for index {} but found None",
-            i
-        );
-        result.unwrap()
+        self.leaf_to_base
+            .get(i)
+            .unwrap_or_else(|| panic!("Expected a value for index {} but found None", i))
     }
 
     /// Returns schema as [`Type`](crate::schema::types::Type).
@@ -810,25 +790,28 @@ impl SchemaDescriptor {
         self.schema.as_ref()
     }
 
+    pub fn root_schema_ptr(&self) -> TypePtr {
+        self.schema.clone()
+    }
+
     /// Returns schema name.
     pub fn name(&self) -> &str {
         self.schema.name()
     }
 }
 
-fn build_tree(
-    tp: TypePtr,
-    root_tp: TypePtr,
-    base_tp: TypePtr,
+fn build_tree<'a>(
+    tp: &'a TypePtr,
+    base_tp: &TypePtr,
     mut max_rep_level: i16,
     mut max_def_level: i16,
     leaves: &mut Vec<ColumnDescPtr>,
-    leaf_to_base: &mut HashMap<usize, TypePtr>,
-    path_so_far: &mut Vec<String>,
+    leaf_to_base: &mut Vec<TypePtr>,
+    path_so_far: &mut Vec<&'a str>,
 ) {
     assert!(tp.get_basic_info().has_repetition());
 
-    path_so_far.push(String::from(tp.name()));
+    path_so_far.push(tp.name());
     match tp.get_basic_info().repetition() {
         Repetition::OPTIONAL => {
             max_def_level += 1;
@@ -841,32 +824,29 @@ fn build_tree(
     }
 
     match tp.as_ref() {
-        &Type::PrimitiveType { .. } => {
+        Type::PrimitiveType { .. } => {
             let mut path: Vec<String> = vec![];
-            path.extend_from_slice(&path_so_far[..]);
+            path.extend(path_so_far.iter().copied().map(String::from));
             leaves.push(Rc::new(ColumnDescriptor::new(
                 tp.clone(),
-                Some(root_tp),
                 max_def_level,
                 max_rep_level,
                 ColumnPath::new(path),
             )));
-            leaf_to_base.insert(leaves.len() - 1, base_tp);
+            leaf_to_base.push(base_tp.clone());
         }
-        &Type::GroupType { ref fields, .. } => {
+        Type::GroupType { ref fields, .. } => {
             for f in fields {
                 build_tree(
-                    f.clone(),
-                    root_tp.clone(),
-                    base_tp.clone(),
+                    f,
+                    base_tp,
                     max_rep_level,
                     max_def_level,
                     leaves,
                     leaf_to_base,
                     path_so_far,
                 );
-                let idx = path_so_far.len() - 1;
-                path_so_far.remove(idx);
+                path_so_far.pop();
             }
         }
     }
@@ -943,7 +923,7 @@ fn from_thrift_helper(
             Ok((index + 1, Rc::new(builder.build()?)))
         }
         Some(n) => {
-            let repetition = elements[index].repetition_type.map(|r| Repetition::from(r));
+            let repetition = elements[index].repetition_type.map(Repetition::from);
             let mut fields = vec![];
             let mut next_index = index + 1;
             for _ in 0..n {
@@ -1085,7 +1065,7 @@ mod tests {
                 Type::PrimitiveType { physical_type, .. } => {
                     assert_eq!(physical_type, PhysicalType::INT32);
                 }
-                _ => assert!(false),
+                _ => panic!(),
             }
         }
 
@@ -1350,19 +1330,7 @@ mod tests {
             .with_logical_type(LogicalType::UTF8)
             .build()?;
 
-        let root_tp = Type::group_type_builder("root")
-            .with_logical_type(LogicalType::LIST)
-            .build()
-            .unwrap();
-        let root_tp_rc = Rc::new(root_tp);
-
-        let descr = ColumnDescriptor::new(
-            Rc::new(tp),
-            Some(root_tp_rc.clone()),
-            4,
-            1,
-            ColumnPath::from("name"),
-        );
+        let descr = ColumnDescriptor::new(Rc::new(tp), 4, 1, ColumnPath::from("name"));
 
         assert_eq!(descr.path(), &ColumnPath::from("name"));
         assert_eq!(descr.logical_type(), LogicalType::UTF8);
@@ -1373,7 +1341,6 @@ mod tests {
         assert_eq!(descr.type_length(), -1);
         assert_eq!(descr.type_precision(), -1);
         assert_eq!(descr.type_scale(), -1);
-        assert_eq!(descr.root_type(), root_tp_rc.as_ref());
 
         Ok(())
     }
diff --git a/rust/parquet/src/schema/visitor.rs b/rust/parquet/src/schema/visitor.rs
index 6970f9ed47a..a1866fb1471 100644
--- a/rust/parquet/src/schema/visitor.rs
+++ b/rust/parquet/src/schema/visitor.rs
@@ -50,7 +50,7 @@ pub trait TypeVisitor<R, C> {
                         {
                             self.visit_list_with_item(
                                 list_type.clone(),
-                                list_item,
+                                list_item.clone(),
                                 context,
                             )
                         } else {
@@ -70,13 +70,13 @@ pub trait TypeVisitor<R, C> {
                         {
                             self.visit_list_with_item(
                                 list_type.clone(),
-                                fields.first().unwrap(),
+                                fields.first().unwrap().clone(),
                                 context,
                             )
                         } else {
                             self.visit_list_with_item(
                                 list_type.clone(),
-                                list_item,
+                                list_item.clone(),
                                 context,
                             )
                         }
@@ -114,7 +114,7 @@ pub trait TypeVisitor<R, C> {
     fn visit_list_with_item(
         &mut self,
         list_type: TypePtr,
-        item_type: &Type,
+        item_type: TypePtr,
         context: C,
     ) -> Result<R>;
 }
@@ -125,7 +125,7 @@ mod tests {
     use crate::basic::Type as PhysicalType;
     use crate::errors::Result;
     use crate::schema::parser::parse_message_type;
-    use crate::schema::types::{Type, TypePtr};
+    use crate::schema::types::TypePtr;
     use std::rc::Rc;
 
     struct TestVisitorContext {}
@@ -174,7 +174,7 @@ mod tests {
         fn visit_list_with_item(
             &mut self,
             list_type: TypePtr,
-            item_type: &Type,
+            item_type: TypePtr,
             _context: TestVisitorContext,
         ) -> Result<bool> {
             assert_eq!(
@@ -203,7 +203,7 @@ mod tests {
                 .get_fields()
                 .iter()
                 .find(|t| t.name() == name)
-                .map(|t| t.clone())
+                .cloned()
                 .unwrap()
         }
     }
diff --git a/rust/parquet/src/util/bit_packing.rs b/rust/parquet/src/util/bit_packing.rs
index 99628672d54..6b9673f6c30 100644
--- a/rust/parquet/src/util/bit_packing.rs
+++ b/rust/parquet/src/util/bit_packing.rs
@@ -79,7 +79,7 @@ unsafe fn nullunpacker32(in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack1_32(in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) & 1;
+    *out = (*in_buf) & 1;
     out = out.offset(1);
     *out = ((*in_buf) >> 1) & 1;
     out = out.offset(1);
@@ -147,7 +147,7 @@ unsafe fn unpack1_32(in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack2_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 2);
+    *out = (*in_buf) % (1u32 << 2);
     out = out.offset(1);
     *out = ((*in_buf) >> 2) % (1u32 << 2);
     out = out.offset(1);
@@ -180,7 +180,7 @@ unsafe fn unpack2_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     *out = (*in_buf) >> 30;
     out = out.offset(1);
     in_buf = in_buf.offset(1);
-    *out = ((*in_buf) >> 0) % (1u32 << 2);
+    *out = (*in_buf) % (1u32 << 2);
     out = out.offset(1);
     *out = ((*in_buf) >> 2) % (1u32 << 2);
     out = out.offset(1);
@@ -216,7 +216,7 @@ unsafe fn unpack2_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack3_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 3);
+    *out = (*in_buf) % (1u32 << 3);
     out = out.offset(1);
     *out = ((*in_buf) >> 3) % (1u32 << 3);
     out = out.offset(1);
@@ -290,7 +290,7 @@ unsafe fn unpack3_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack4_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 4);
+    *out = (*in_buf) % (1u32 << 4);
     out = out.offset(1);
     *out = ((*in_buf) >> 4) % (1u32 << 4);
     out = out.offset(1);
@@ -308,7 +308,7 @@ unsafe fn unpack4_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 4);
+    *out = (*in_buf) % (1u32 << 4);
     out = out.offset(1);
     *out = ((*in_buf) >> 4) % (1u32 << 4);
     out = out.offset(1);
@@ -326,7 +326,7 @@ unsafe fn unpack4_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 4);
+    *out = (*in_buf) % (1u32 << 4);
     out = out.offset(1);
     *out = ((*in_buf) >> 4) % (1u32 << 4);
     out = out.offset(1);
@@ -344,7 +344,7 @@ unsafe fn unpack4_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 4);
+    *out = (*in_buf) % (1u32 << 4);
     out = out.offset(1);
     *out = ((*in_buf) >> 4) % (1u32 << 4);
     out = out.offset(1);
@@ -364,7 +364,7 @@ unsafe fn unpack4_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack5_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 5);
+    *out = (*in_buf) % (1u32 << 5);
     out = out.offset(1);
     *out = ((*in_buf) >> 5) % (1u32 << 5);
     out = out.offset(1);
@@ -444,7 +444,7 @@ unsafe fn unpack5_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack6_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 6);
+    *out = (*in_buf) % (1u32 << 6);
     out = out.offset(1);
     *out = ((*in_buf) >> 6) % (1u32 << 6);
     out = out.offset(1);
@@ -484,7 +484,7 @@ unsafe fn unpack6_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 6);
+    *out = (*in_buf) % (1u32 << 6);
     out = out.offset(1);
     *out = ((*in_buf) >> 6) % (1u32 << 6);
     out = out.offset(1);
@@ -526,7 +526,7 @@ unsafe fn unpack6_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack7_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 7);
+    *out = (*in_buf) % (1u32 << 7);
     out = out.offset(1);
     *out = ((*in_buf) >> 7) % (1u32 << 7);
     out = out.offset(1);
@@ -612,7 +612,7 @@ unsafe fn unpack7_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack8_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 8);
+    *out = (*in_buf) % (1u32 << 8);
     out = out.offset(1);
     *out = ((*in_buf) >> 8) % (1u32 << 8);
     out = out.offset(1);
@@ -622,7 +622,7 @@ unsafe fn unpack8_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 8);
+    *out = (*in_buf) % (1u32 << 8);
     out = out.offset(1);
     *out = ((*in_buf) >> 8) % (1u32 << 8);
     out = out.offset(1);
@@ -632,7 +632,7 @@ unsafe fn unpack8_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 8);
+    *out = (*in_buf) % (1u32 << 8);
     out = out.offset(1);
     *out = ((*in_buf) >> 8) % (1u32 << 8);
     out = out.offset(1);
@@ -642,7 +642,7 @@ unsafe fn unpack8_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 8);
+    *out = (*in_buf) % (1u32 << 8);
     out = out.offset(1);
     *out = ((*in_buf) >> 8) % (1u32 << 8);
     out = out.offset(1);
@@ -652,7 +652,7 @@ unsafe fn unpack8_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 8);
+    *out = (*in_buf) % (1u32 << 8);
     out = out.offset(1);
     *out = ((*in_buf) >> 8) % (1u32 << 8);
     out = out.offset(1);
@@ -662,7 +662,7 @@ unsafe fn unpack8_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 8);
+    *out = (*in_buf) % (1u32 << 8);
     out = out.offset(1);
     *out = ((*in_buf) >> 8) % (1u32 << 8);
     out = out.offset(1);
@@ -672,7 +672,7 @@ unsafe fn unpack8_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 8);
+    *out = (*in_buf) % (1u32 << 8);
     out = out.offset(1);
     *out = ((*in_buf) >> 8) % (1u32 << 8);
     out = out.offset(1);
@@ -682,7 +682,7 @@ unsafe fn unpack8_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 8);
+    *out = (*in_buf) % (1u32 << 8);
     out = out.offset(1);
     *out = ((*in_buf) >> 8) % (1u32 << 8);
     out = out.offset(1);
@@ -694,7 +694,7 @@ unsafe fn unpack8_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack9_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 9);
+    *out = (*in_buf) % (1u32 << 9);
     out = out.offset(1);
     *out = ((*in_buf) >> 9) % (1u32 << 9);
     out = out.offset(1);
@@ -786,7 +786,7 @@ unsafe fn unpack9_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack10_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 10);
+    *out = (*in_buf) % (1u32 << 10);
     out = out.offset(1);
     *out = ((*in_buf) >> 10) % (1u32 << 10);
     out = out.offset(1);
@@ -832,7 +832,7 @@ unsafe fn unpack10_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 10);
+    *out = (*in_buf) % (1u32 << 10);
     out = out.offset(1);
     *out = ((*in_buf) >> 10) % (1u32 << 10);
     out = out.offset(1);
@@ -880,7 +880,7 @@ unsafe fn unpack10_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack11_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 11);
+    *out = (*in_buf) % (1u32 << 11);
     out = out.offset(1);
     *out = ((*in_buf) >> 11) % (1u32 << 11);
     out = out.offset(1);
@@ -978,7 +978,7 @@ unsafe fn unpack11_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack12_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 12);
+    *out = (*in_buf) % (1u32 << 12);
     out = out.offset(1);
     *out = ((*in_buf) >> 12) % (1u32 << 12);
     out = out.offset(1);
@@ -1002,7 +1002,7 @@ unsafe fn unpack12_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 12);
+    *out = (*in_buf) % (1u32 << 12);
     out = out.offset(1);
     *out = ((*in_buf) >> 12) % (1u32 << 12);
     out = out.offset(1);
@@ -1026,7 +1026,7 @@ unsafe fn unpack12_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 12);
+    *out = (*in_buf) % (1u32 << 12);
     out = out.offset(1);
     *out = ((*in_buf) >> 12) % (1u32 << 12);
     out = out.offset(1);
@@ -1050,7 +1050,7 @@ unsafe fn unpack12_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 12);
+    *out = (*in_buf) % (1u32 << 12);
     out = out.offset(1);
     *out = ((*in_buf) >> 12) % (1u32 << 12);
     out = out.offset(1);
@@ -1076,7 +1076,7 @@ unsafe fn unpack12_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack13_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 13);
+    *out = (*in_buf) % (1u32 << 13);
     out = out.offset(1);
     *out = ((*in_buf) >> 13) % (1u32 << 13);
     out = out.offset(1);
@@ -1180,7 +1180,7 @@ unsafe fn unpack13_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack14_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 14);
+    *out = (*in_buf) % (1u32 << 14);
     out = out.offset(1);
     *out = ((*in_buf) >> 14) % (1u32 << 14);
     out = out.offset(1);
@@ -1232,7 +1232,7 @@ unsafe fn unpack14_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 14);
+    *out = (*in_buf) % (1u32 << 14);
     out = out.offset(1);
     *out = ((*in_buf) >> 14) % (1u32 << 14);
     out = out.offset(1);
@@ -1286,7 +1286,7 @@ unsafe fn unpack14_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack15_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 15);
+    *out = (*in_buf) % (1u32 << 15);
     out = out.offset(1);
     *out = ((*in_buf) >> 15) % (1u32 << 15);
     out = out.offset(1);
@@ -1396,97 +1396,97 @@ unsafe fn unpack15_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack16_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 16);
+    *out = (*in_buf) % (1u32 << 16);
     out = out.offset(1);
     *out = (*in_buf) >> 16;
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 16);
+    *out = (*in_buf) % (1u32 << 16);
     out = out.offset(1);
     *out = (*in_buf) >> 16;
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 16);
+    *out = (*in_buf) % (1u32 << 16);
     out = out.offset(1);
     *out = (*in_buf) >> 16;
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 16);
+    *out = (*in_buf) % (1u32 << 16);
     out = out.offset(1);
     *out = (*in_buf) >> 16;
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 16);
+    *out = (*in_buf) % (1u32 << 16);
     out = out.offset(1);
     *out = (*in_buf) >> 16;
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 16);
+    *out = (*in_buf) % (1u32 << 16);
     out = out.offset(1);
     *out = (*in_buf) >> 16;
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 16);
+    *out = (*in_buf) % (1u32 << 16);
     out = out.offset(1);
     *out = (*in_buf) >> 16;
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 16);
+    *out = (*in_buf) % (1u32 << 16);
     out = out.offset(1);
     *out = (*in_buf) >> 16;
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 16);
+    *out = (*in_buf) % (1u32 << 16);
     out = out.offset(1);
     *out = (*in_buf) >> 16;
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 16);
+    *out = (*in_buf) % (1u32 << 16);
     out = out.offset(1);
     *out = (*in_buf) >> 16;
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 16);
+    *out = (*in_buf) % (1u32 << 16);
     out = out.offset(1);
     *out = (*in_buf) >> 16;
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 16);
+    *out = (*in_buf) % (1u32 << 16);
     out = out.offset(1);
     *out = (*in_buf) >> 16;
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 16);
+    *out = (*in_buf) % (1u32 << 16);
     out = out.offset(1);
     *out = (*in_buf) >> 16;
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 16);
+    *out = (*in_buf) % (1u32 << 16);
     out = out.offset(1);
     *out = (*in_buf) >> 16;
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 16);
+    *out = (*in_buf) % (1u32 << 16);
     out = out.offset(1);
     *out = (*in_buf) >> 16;
     out = out.offset(1);
     in_buf = in_buf.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 16);
+    *out = (*in_buf) % (1u32 << 16);
     out = out.offset(1);
     *out = (*in_buf) >> 16;
 
@@ -1494,7 +1494,7 @@ unsafe fn unpack16_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack17_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 17);
+    *out = (*in_buf) % (1u32 << 17);
     out = out.offset(1);
     *out = (*in_buf) >> 17;
     in_buf = in_buf.offset(1);
@@ -1610,7 +1610,7 @@ unsafe fn unpack17_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack18_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 18);
+    *out = (*in_buf) % (1u32 << 18);
     out = out.offset(1);
     *out = (*in_buf) >> 18;
     in_buf = in_buf.offset(1);
@@ -1668,7 +1668,7 @@ unsafe fn unpack18_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 18);
+    *out = (*in_buf) % (1u32 << 18);
     out = out.offset(1);
     *out = (*in_buf) >> 18;
     in_buf = in_buf.offset(1);
@@ -1728,7 +1728,7 @@ unsafe fn unpack18_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack19_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 19);
+    *out = (*in_buf) % (1u32 << 19);
     out = out.offset(1);
     *out = (*in_buf) >> 19;
     in_buf = in_buf.offset(1);
@@ -1850,7 +1850,7 @@ unsafe fn unpack19_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack20_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 20);
+    *out = (*in_buf) % (1u32 << 20);
     out = out.offset(1);
     *out = (*in_buf) >> 20;
     in_buf = in_buf.offset(1);
@@ -1880,7 +1880,7 @@ unsafe fn unpack20_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 20);
+    *out = (*in_buf) % (1u32 << 20);
     out = out.offset(1);
     *out = (*in_buf) >> 20;
     in_buf = in_buf.offset(1);
@@ -1910,7 +1910,7 @@ unsafe fn unpack20_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 20);
+    *out = (*in_buf) % (1u32 << 20);
     out = out.offset(1);
     *out = (*in_buf) >> 20;
     in_buf = in_buf.offset(1);
@@ -1940,7 +1940,7 @@ unsafe fn unpack20_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 20);
+    *out = (*in_buf) % (1u32 << 20);
     out = out.offset(1);
     *out = (*in_buf) >> 20;
     in_buf = in_buf.offset(1);
@@ -1972,7 +1972,7 @@ unsafe fn unpack20_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack21_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 21);
+    *out = (*in_buf) % (1u32 << 21);
     out = out.offset(1);
     *out = (*in_buf) >> 21;
     in_buf = in_buf.offset(1);
@@ -2100,7 +2100,7 @@ unsafe fn unpack21_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack22_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 22);
+    *out = (*in_buf) % (1u32 << 22);
     out = out.offset(1);
     *out = (*in_buf) >> 22;
     in_buf = in_buf.offset(1);
@@ -2164,7 +2164,7 @@ unsafe fn unpack22_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 22);
+    *out = (*in_buf) % (1u32 << 22);
     out = out.offset(1);
     *out = (*in_buf) >> 22;
     in_buf = in_buf.offset(1);
@@ -2230,7 +2230,7 @@ unsafe fn unpack22_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack23_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 23);
+    *out = (*in_buf) % (1u32 << 23);
     out = out.offset(1);
     *out = (*in_buf) >> 23;
     in_buf = in_buf.offset(1);
@@ -2364,7 +2364,7 @@ unsafe fn unpack23_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack24_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 24);
+    *out = (*in_buf) % (1u32 << 24);
     out = out.offset(1);
     *out = (*in_buf) >> 24;
     in_buf = in_buf.offset(1);
@@ -2380,7 +2380,7 @@ unsafe fn unpack24_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 24);
+    *out = (*in_buf) % (1u32 << 24);
     out = out.offset(1);
     *out = (*in_buf) >> 24;
     in_buf = in_buf.offset(1);
@@ -2396,7 +2396,7 @@ unsafe fn unpack24_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 24);
+    *out = (*in_buf) % (1u32 << 24);
     out = out.offset(1);
     *out = (*in_buf) >> 24;
     in_buf = in_buf.offset(1);
@@ -2412,7 +2412,7 @@ unsafe fn unpack24_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 24);
+    *out = (*in_buf) % (1u32 << 24);
     out = out.offset(1);
     *out = (*in_buf) >> 24;
     in_buf = in_buf.offset(1);
@@ -2428,7 +2428,7 @@ unsafe fn unpack24_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 24);
+    *out = (*in_buf) % (1u32 << 24);
     out = out.offset(1);
     *out = (*in_buf) >> 24;
     in_buf = in_buf.offset(1);
@@ -2444,7 +2444,7 @@ unsafe fn unpack24_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 24);
+    *out = (*in_buf) % (1u32 << 24);
     out = out.offset(1);
     *out = (*in_buf) >> 24;
     in_buf = in_buf.offset(1);
@@ -2460,7 +2460,7 @@ unsafe fn unpack24_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 24);
+    *out = (*in_buf) % (1u32 << 24);
     out = out.offset(1);
     *out = (*in_buf) >> 24;
     in_buf = in_buf.offset(1);
@@ -2476,7 +2476,7 @@ unsafe fn unpack24_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 24);
+    *out = (*in_buf) % (1u32 << 24);
     out = out.offset(1);
     *out = (*in_buf) >> 24;
     in_buf = in_buf.offset(1);
@@ -2494,7 +2494,7 @@ unsafe fn unpack24_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack25_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 25);
+    *out = (*in_buf) % (1u32 << 25);
     out = out.offset(1);
     *out = (*in_buf) >> 25;
     in_buf = in_buf.offset(1);
@@ -2634,7 +2634,7 @@ unsafe fn unpack25_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack26_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 26);
+    *out = (*in_buf) % (1u32 << 26);
     out = out.offset(1);
     *out = (*in_buf) >> 26;
     in_buf = in_buf.offset(1);
@@ -2704,7 +2704,7 @@ unsafe fn unpack26_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 26);
+    *out = (*in_buf) % (1u32 << 26);
     out = out.offset(1);
     *out = (*in_buf) >> 26;
     in_buf = in_buf.offset(1);
@@ -2776,7 +2776,7 @@ unsafe fn unpack26_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack27_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 27);
+    *out = (*in_buf) % (1u32 << 27);
     out = out.offset(1);
     *out = (*in_buf) >> 27;
     in_buf = in_buf.offset(1);
@@ -2922,7 +2922,7 @@ unsafe fn unpack27_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack28_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 28);
+    *out = (*in_buf) % (1u32 << 28);
     out = out.offset(1);
     *out = (*in_buf) >> 28;
     in_buf = in_buf.offset(1);
@@ -2958,7 +2958,7 @@ unsafe fn unpack28_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 28);
+    *out = (*in_buf) % (1u32 << 28);
     out = out.offset(1);
     *out = (*in_buf) >> 28;
     in_buf = in_buf.offset(1);
@@ -2994,7 +2994,7 @@ unsafe fn unpack28_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 28);
+    *out = (*in_buf) % (1u32 << 28);
     out = out.offset(1);
     *out = (*in_buf) >> 28;
     in_buf = in_buf.offset(1);
@@ -3030,7 +3030,7 @@ unsafe fn unpack28_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 28);
+    *out = (*in_buf) % (1u32 << 28);
     out = out.offset(1);
     *out = (*in_buf) >> 28;
     in_buf = in_buf.offset(1);
@@ -3068,7 +3068,7 @@ unsafe fn unpack28_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack29_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 29);
+    *out = (*in_buf) % (1u32 << 29);
     out = out.offset(1);
     *out = (*in_buf) >> 29;
     in_buf = in_buf.offset(1);
@@ -3220,7 +3220,7 @@ unsafe fn unpack29_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack30_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 30);
+    *out = (*in_buf) % (1u32 << 30);
     out = out.offset(1);
     *out = (*in_buf) >> 30;
     in_buf = in_buf.offset(1);
@@ -3296,7 +3296,7 @@ unsafe fn unpack30_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = ((*in_buf) >> 0) % (1u32 << 30);
+    *out = (*in_buf) % (1u32 << 30);
     out = out.offset(1);
     *out = (*in_buf) >> 30;
     in_buf = in_buf.offset(1);
@@ -3374,7 +3374,7 @@ unsafe fn unpack30_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack31_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = ((*in_buf) >> 0) % (1u32 << 31);
+    *out = (*in_buf) % (1u32 << 31);
     out = out.offset(1);
     *out = (*in_buf) >> 31;
     in_buf = in_buf.offset(1);
@@ -3532,131 +3532,131 @@ unsafe fn unpack31_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
 }
 
 unsafe fn unpack32_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
     in_buf = in_buf.offset(1);
     out = out.offset(1);
 
-    *out = (*in_buf) >> 0;
+    *out = *in_buf;
 
     in_buf.offset(1)
 }
diff --git a/rust/parquet/src/util/bit_util.rs b/rust/parquet/src/util/bit_util.rs
index 90cb3da052f..5ccd1636b7b 100644
--- a/rust/parquet/src/util/bit_util.rs
+++ b/rust/parquet/src/util/bit_util.rs
@@ -956,16 +956,16 @@ mod tests {
             .iter()
             .map(|v| v & ((1 << num_bits) - 1))
             .collect();
-        for i in 0..total {
+        (0..total).for_each(|i| {
             assert!(
                 writer.put_value(values[i] as u64, num_bits),
                 "[{}]: put_value() failed",
                 i
             );
-        }
+        });
 
         let mut reader = BitReader::from(writer.consume());
-        for i in 0..total {
+        (0..total).for_each(|i| {
             let v = reader
                 .get_value::<u64>(num_bits)
                 .expect("get_value() should return OK");
@@ -974,7 +974,7 @@ mod tests {
                 "[{}]: expected {} but got {}",
                 i, values[i], v
             );
-        }
+        });
     }
 
     #[test]
@@ -1008,9 +1008,9 @@ mod tests {
         let expected_values: Vec<T> =
             values.iter().map(|v| from_ne_slice(v.as_bytes())).collect();
 
-        for i in 0..total {
+        (0..total).for_each(|i| {
             assert!(writer.put_value(values[i] as u64, num_bits));
-        }
+        });
 
         let buf = writer.consume();
         let mut reader = BitReader::from(buf);
@@ -1103,16 +1103,16 @@ mod tests {
         let total = 64;
         let mut writer = BitWriter::new(total * 32);
         let values = random_numbers::<u32>(total);
-        for i in 0..total {
+        (0..total).for_each(|i| {
             assert!(
                 writer.put_vlq_int(values[i] as u64),
                 "[{}]; put_vlq_int() failed",
                 i
             );
-        }
+        });
 
         let mut reader = BitReader::from(writer.consume());
-        for i in 0..total {
+        (0..total).for_each(|i| {
             let v = reader
                 .get_vlq_int()
                 .expect("get_vlq_int() should return OK");
@@ -1121,7 +1121,7 @@ mod tests {
                 "[{}]: expected {} but got {}",
                 i, values[i], v
             );
-        }
+        });
     }
 
     #[test]
@@ -1129,16 +1129,16 @@ mod tests {
         let total = 64;
         let mut writer = BitWriter::new(total * 32);
         let values = random_numbers::<i32>(total);
-        for i in 0..total {
+        (0..total).for_each(|i| {
             assert!(
                 writer.put_zigzag_vlq_int(values[i] as i64),
                 "[{}]; put_zigzag_vlq_int() failed",
                 i
             );
-        }
+        });
 
         let mut reader = BitReader::from(writer.consume());
-        for i in 0..total {
+        (0..total).for_each(|i| {
             let v = reader
                 .get_zigzag_vlq_int()
                 .expect("get_zigzag_vlq_int() should return OK");
@@ -1147,6 +1147,6 @@ mod tests {
                 "[{}]: expected {} but got {}",
                 i, values[i], v
             );
-        }
+        });
     }
 }
diff --git a/rust/parquet/src/util/cursor.rs b/rust/parquet/src/util/cursor.rs
new file mode 100644
index 00000000000..0ada2ef05de
--- /dev/null
+++ b/rust/parquet/src/util/cursor.rs
@@ -0,0 +1,202 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io::{self, Error, ErrorKind, Read, Seek, SeekFrom};
+use std::rc::Rc;
+use std::{cmp, fmt};
+
+/// This is object to use if your file is already in memory.
+/// The sliceable cursor is similar to std::io::Cursor, except that it makes it easy to create "cursor slices".
+/// To achieve this, it uses Rc instead of shared references. Indeed reference fields are painfull
+/// because the lack of Generic Associated Type implies that you would require complex lifetime propagation when
+/// returning such a cursor.
+pub struct SliceableCursor {
+    inner: Rc<Vec<u8>>,
+    start: u64,
+    length: usize,
+    pos: u64,
+}
+
+impl fmt::Debug for SliceableCursor {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("SliceableCursor")
+            .field("start", &self.start)
+            .field("length", &self.length)
+            .field("pos", &self.pos)
+            .field("inner.len", &self.inner.len())
+            .finish()
+    }
+}
+
+impl SliceableCursor {
+    pub fn new(content: Vec<u8>) -> Self {
+        let size = content.len();
+        SliceableCursor {
+            inner: Rc::new(content),
+            start: 0,
+            pos: 0,
+            length: size,
+        }
+    }
+
+    /// Create a slice cursor using the same data as a current one.
+    pub fn slice(&self, start: u64, length: usize) -> io::Result<Self> {
+        let new_start = self.start + start;
+        if new_start >= self.inner.len() as u64
+            || new_start as usize + length > self.inner.len()
+        {
+            return Err(Error::new(ErrorKind::InvalidInput, "out of bound"));
+        }
+        Ok(SliceableCursor {
+            inner: Rc::clone(&self.inner),
+            start: new_start,
+            pos: new_start,
+            length,
+        })
+    }
+
+    fn remaining_slice(&self) -> &[u8] {
+        let end = self.start as usize + self.length;
+        let offset = cmp::min(self.pos, end as u64) as usize;
+        &self.inner[offset..end]
+    }
+
+    /// Get the length of the current cursor slice
+    pub fn len(&self) -> u64 {
+        self.length as u64
+    }
+}
+
+/// Implementation inspired by std::io::Cursor
+impl Read for SliceableCursor {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        let n = Read::read(&mut self.remaining_slice(), buf)?;
+        self.pos += n as u64;
+        Ok(n)
+    }
+}
+
+impl Seek for SliceableCursor {
+    fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
+        let new_pos = match pos {
+            SeekFrom::Start(pos) => pos as i64,
+            SeekFrom::End(pos) => self.inner.len() as i64 + pos as i64,
+            SeekFrom::Current(pos) => self.pos as i64 + pos as i64,
+        };
+
+        if new_pos < 0 {
+            Err(Error::new(
+                ErrorKind::InvalidInput,
+                format!(
+                    "Request out of bounds: cur position {} + seek {:?} < 0: {}",
+                    self.pos, pos, new_pos
+                ),
+            ))
+        } else if new_pos >= self.inner.len() as i64 {
+            Err(Error::new(
+                ErrorKind::InvalidInput,
+                format!(
+                    "Request out of bounds: cur position {} + seek {:?} >= length {}: {}",
+                    self.pos,
+                    pos,
+                    self.inner.len(),
+                    new_pos
+                ),
+            ))
+        } else {
+            self.pos = new_pos as u64;
+            Ok(self.start)
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Create a SliceableCursor of all u8 values in ascending order
+    fn get_u8_range() -> SliceableCursor {
+        let data: Vec<u8> = (0u8..=255).collect();
+        SliceableCursor::new(data)
+    }
+
+    /// Reads all the bytes in the slice and checks that it matches the u8 range from start to end_included
+    fn check_read_all(mut cursor: SliceableCursor, start: u8, end_included: u8) {
+        let mut target = vec![];
+        let cursor_res = cursor.read_to_end(&mut target);
+        println!("{:?}", cursor_res);
+        assert!(!cursor_res.is_err(), "reading error");
+        assert_eq!((end_included - start) as usize + 1, cursor_res.unwrap());
+        assert_eq!((start..=end_included).collect::<Vec<_>>(), target);
+    }
+
+    #[test]
+    fn read_all_whole() {
+        let cursor = get_u8_range();
+        check_read_all(cursor, 0, 255);
+    }
+
+    #[test]
+    fn read_all_slice() {
+        let cursor = get_u8_range().slice(10, 10).expect("error while slicing");
+        check_read_all(cursor, 10, 19);
+    }
+
+    #[test]
+    fn seek_cursor_start() {
+        let mut cursor = get_u8_range();
+
+        cursor.seek(SeekFrom::Start(5)).unwrap();
+        check_read_all(cursor, 5, 255);
+    }
+
+    #[test]
+    fn seek_cursor_current() {
+        let mut cursor = get_u8_range();
+        cursor.seek(SeekFrom::Start(10)).unwrap();
+        cursor.seek(SeekFrom::Current(10)).unwrap();
+        check_read_all(cursor, 20, 255);
+    }
+
+    #[test]
+    fn seek_cursor_end() {
+        let mut cursor = get_u8_range();
+
+        cursor.seek(SeekFrom::End(-10)).unwrap();
+        check_read_all(cursor, 246, 255);
+    }
+
+    #[test]
+    fn seek_cursor_error_too_long() {
+        let mut cursor = get_u8_range();
+        let res = cursor.seek(SeekFrom::Start(1000));
+        let actual_error = res.expect_err("expected error").to_string();
+        let expected_error =
+            "Request out of bounds: cur position 0 + seek Start(1000) >= length 256: 1000";
+        assert_eq!(actual_error, expected_error);
+    }
+
+    #[test]
+    fn seek_cursor_error_too_short() {
+        let mut cursor = get_u8_range();
+        let res = cursor.seek(SeekFrom::End(-1000));
+        let actual_error = res.expect_err("expected error").to_string();
+        let expected_error =
+            "Request out of bounds: cur position 0 + seek End(-1000) < 0: -744";
+        assert_eq!(actual_error, expected_error);
+    }
+}
diff --git a/rust/parquet/src/util/hash_util.rs b/rust/parquet/src/util/hash_util.rs
index 5142926a5b4..f7849da41a0 100644
--- a/rust/parquet/src/util/hash_util.rs
+++ b/rust/parquet/src/util/hash_util.rs
@@ -33,7 +33,7 @@ fn hash_(data: &[u8], seed: u32) -> u32 {
         }
     }
 
-    #[cfg(target_arch = "aarch64")]
+    #[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
     unsafe {
         murmur_hash2_64a(data, seed as u64) as u32
     }
diff --git a/rust/parquet/src/util/io.rs b/rust/parquet/src/util/io.rs
index f999f1fd28f..1403de0f529 100644
--- a/rust/parquet/src/util/io.rs
+++ b/rust/parquet/src/util/io.rs
@@ -15,13 +15,27 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use std::{cell::RefCell, cmp, io::*};
+use std::{cell::RefCell, cmp, fmt, io::*};
 
-use crate::file::{reader::ParquetReader, writer::ParquetWriter};
+use crate::file::{reader::Length, writer::ParquetWriter};
 
 const DEFAULT_BUF_SIZE: usize = 8 * 1024;
 
 // ----------------------------------------------------------------------
+
+/// TryClone tries to clone the type and should maintain the `Seek` position of the given
+/// instance.
+pub trait TryClone: Sized {
+    /// Clones the type returning a new instance or an error if it's not possible
+    /// to clone it.
+    fn try_clone(&self) -> Result<Self>;
+}
+
+/// ParquetReader is the interface which needs to be fulfilled to be able to parse a
+/// parquet source.
+pub trait ParquetReader: Read + Seek + Length + TryClone {}
+impl<T: Read + Seek + Length + TryClone> ParquetReader for T {}
+
 // Read/Write wrappers for `File`.
 
 /// Position trait returns the current position in the stream.
@@ -49,6 +63,19 @@ pub struct FileSource<R: ParquetReader> {
     buf_cap: usize, // current number of bytes read into the buffer
 }
 
+impl<R: ParquetReader> fmt::Debug for FileSource<R> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("FileSource")
+            .field("reader", &"OPAQUE")
+            .field("start", &self.start)
+            .field("end", &self.end)
+            .field("buf.len", &self.buf.len())
+            .field("buf_pos", &self.buf_pos)
+            .field("buf_cap", &self.buf_cap)
+            .finish()
+    }
+}
+
 impl<R: ParquetReader> FileSource<R> {
     /// Creates new file reader with start and length from a file handle
     pub fn new(fd: &R, start: u64, length: usize) -> Self {
@@ -121,6 +148,12 @@ impl<R: ParquetReader> Position for FileSource<R> {
     }
 }
 
+impl<R: ParquetReader> Length for FileSource<R> {
+    fn len(&self) -> u64 {
+        self.end - self.start
+    }
+}
+
 /// Struct that represents `File` output stream with position tracking.
 /// Used as a sink in file writer.
 pub struct FileSink<W: ParquetWriter> {
@@ -202,10 +235,10 @@ mod tests {
     fn test_io_read_pos() {
         let mut src = FileSource::new(&get_test_file("alltypes_plain.parquet"), 0, 4);
 
-        src.read(&mut vec![0; 1]).unwrap();
+        let _ = src.read(&mut [0; 1]).unwrap();
         assert_eq!(src.pos(), 1);
 
-        src.read(&mut vec![0; 4]).unwrap();
+        let _ = src.read(&mut [0; 4]).unwrap();
         assert_eq!(src.pos(), 4);
     }
 
@@ -214,11 +247,11 @@ mod tests {
         let mut src = FileSource::new(&get_test_file("alltypes_plain.parquet"), 0, 4);
 
         // Read all bytes from source
-        src.read(&mut vec![0; 128]).unwrap();
+        let _ = src.read(&mut [0; 128]).unwrap();
         assert_eq!(src.pos(), 4);
 
         // Try reading again, should return 0 bytes.
-        let bytes_read = src.read(&mut vec![0; 128]).unwrap();
+        let bytes_read = src.read(&mut [0; 128]).unwrap();
         assert_eq!(bytes_read, 0);
         assert_eq!(src.pos(), 4);
     }
@@ -246,7 +279,7 @@ mod tests {
         let mut sink = FileSink::new(&file);
         assert_eq!(sink.pos(), 3);
 
-        sink.write(&[b'd', b'e', b'f', b'g']).unwrap();
+        sink.write_all(&[b'd', b'e', b'f', b'g']).unwrap();
         assert_eq!(sink.pos(), 7);
 
         sink.flush().unwrap();
@@ -256,7 +289,7 @@ mod tests {
         let mut res = vec![0u8; 7];
         let mut chunk =
             FileSource::new(&file, 0, file.metadata().unwrap().len() as usize);
-        chunk.read(&mut res[..]).unwrap();
+        chunk.read_exact(&mut res[..]).unwrap();
         assert_eq!(res, vec![b'a', b'b', b'c', b'd', b'e', b'f', b'g']);
     }
 
diff --git a/rust/parquet/src/util/memory.rs b/rust/parquet/src/util/memory.rs
index 64a21653388..32664416064 100644
--- a/rust/parquet/src/util/memory.rs
+++ b/rust/parquet/src/util/memory.rs
@@ -318,6 +318,11 @@ impl<T> BufferPtr<T> {
         self.len
     }
 
+    /// Returns whether this buffer is empty
+    pub fn is_empty(&self) -> bool {
+        self.len == 0
+    }
+
     /// Returns `true` if this buffer has memory tracker, `false` otherwise.
     pub fn is_mem_tracked(&self) -> bool {
         self.mem_tracker.is_some()
@@ -330,7 +335,7 @@ impl<T> BufferPtr<T> {
             data: self.data.clone(),
             start: self.start,
             len: self.len,
-            mem_tracker: self.mem_tracker.as_ref().map(|p| p.clone()),
+            mem_tracker: self.mem_tracker.as_ref().cloned(),
         }
     }
 
@@ -341,7 +346,7 @@ impl<T> BufferPtr<T> {
             data: self.data.clone(),
             start: self.start + start,
             len: self.len - start,
-            mem_tracker: self.mem_tracker.as_ref().map(|p| p.clone()),
+            mem_tracker: self.mem_tracker.as_ref().cloned(),
         }
     }
 
@@ -352,7 +357,7 @@ impl<T> BufferPtr<T> {
             data: self.data.clone(),
             start: self.start + start,
             len,
-            mem_tracker: self.mem_tracker.as_ref().map(|p| p.clone()),
+            mem_tracker: self.mem_tracker.as_ref().cloned(),
         }
     }
 }
diff --git a/rust/parquet/src/util/mod.rs b/rust/parquet/src/util/mod.rs
index 669cc3c0a49..af9a1aa1eba 100644
--- a/rust/parquet/src/util/mod.rs
+++ b/rust/parquet/src/util/mod.rs
@@ -20,6 +20,7 @@ pub mod memory;
 #[macro_use]
 pub mod bit_util;
 mod bit_packing;
+pub mod cursor;
 pub mod hash_util;
 
 #[cfg(test)]
diff --git a/rust/parquet/src/util/test_common/page_util.rs b/rust/parquet/src/util/test_common/page_util.rs
index 92796170e34..a6da6830503 100644
--- a/rust/parquet/src/util/test_common/page_util.rs
+++ b/rust/parquet/src/util/test_common/page_util.rs
@@ -212,7 +212,7 @@ impl Iterator for InMemoryPageIterator {
     type Item = Result<Box<dyn PageReader>>;
 
     fn next(&mut self) -> Option<Self::Item> {
-        self.page_readers.next().map(|page_reader| Ok(page_reader))
+        self.page_readers.next().map(Ok)
     }
 }
 
@@ -290,7 +290,7 @@ pub fn make_pages<T: DataType>(
             Encoding::PLAIN => {
                 pb.add_values::<T>(encoding, &values[value_range]);
             }
-            enc @ _ => panic!("Unexpected encoding {}", enc),
+            enc => panic!("Unexpected encoding {}", enc),
         }
 
         let data_page = pb.consume();
diff --git a/rust/parquet/src/util/test_common/rand_gen.rs b/rust/parquet/src/util/test_common/rand_gen.rs
index 7d40071d7a5..dae8af59e9d 100644
--- a/rust/parquet/src/util/test_common/rand_gen.rs
+++ b/rust/parquet/src/util/test_common/rand_gen.rs
@@ -88,7 +88,7 @@ impl RandGen<ByteArrayType> for ByteArrayType {
         let mut value = vec![];
         let len = rng.gen_range(0, 128);
         for _ in 0..len {
-            value.push(rng.gen_range(0, 255) & 0xFF);
+            value.push(rng.gen_range(0, 255));
         }
         result.set_data(ByteBufferPtr::new(value));
         result
@@ -112,7 +112,7 @@ pub fn random_bytes(n: usize) -> Vec<u8> {
     let mut result = vec![];
     let mut rng = thread_rng();
     for _ in 0..n {
-        result.push(rng.gen_range(0, 255) & 0xFF);
+        result.push(rng.gen_range(0, 255));
     }
     result
 }
diff --git a/rust/parquet/tests/custom_writer.rs b/rust/parquet/tests/custom_writer.rs
new file mode 100644
index 00000000000..6146fc2c94e
--- /dev/null
+++ b/rust/parquet/tests/custom_writer.rs
@@ -0,0 +1,100 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fs::File;
+use std::{
+    fs,
+    io::{prelude::*, SeekFrom},
+    rc::Rc,
+};
+
+use parquet::file::writer::TryClone;
+use parquet::{
+    basic::Repetition, basic::Type, file::properties::WriterProperties,
+    file::writer::SerializedFileWriter, schema::types,
+};
+use std::env;
+
+// Test creating some sort of custom writer to ensure the
+// appropriate traits are exposed
+struct CustomWriter {
+    file: File,
+}
+
+impl Write for CustomWriter {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        self.file.write(buf)
+    }
+    fn flush(&mut self) -> std::io::Result<()> {
+        self.file.flush()
+    }
+}
+
+impl Seek for CustomWriter {
+    fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
+        self.file.seek(pos)
+    }
+}
+
+impl TryClone for CustomWriter {
+    fn try_clone(&self) -> std::io::Result<Self> {
+        use std::io::{Error, ErrorKind};
+        Err(Error::new(ErrorKind::Other, "Clone not supported"))
+    }
+}
+
+#[test]
+fn test_custom_writer() {
+    let schema = Rc::new(
+        types::Type::group_type_builder("schema")
+            .with_fields(&mut vec![Rc::new(
+                types::Type::primitive_type_builder("col1", Type::INT32)
+                    .with_repetition(Repetition::REQUIRED)
+                    .build()
+                    .unwrap(),
+            )])
+            .build()
+            .unwrap(),
+    );
+    let props = Rc::new(WriterProperties::builder().build());
+
+    let file = get_temp_file("test_custom_file_writer");
+    let test_file = file.try_clone().unwrap();
+
+    let writer = CustomWriter { file };
+
+    // test is that this file can be created
+    let file_writer = SerializedFileWriter::new(writer, schema, props).unwrap();
+    std::mem::drop(file_writer);
+
+    // ensure the file now exists and has non zero size
+    let metadata = test_file.metadata().unwrap();
+    assert!(metadata.len() > 0);
+}
+
+/// Returns file handle for a temp file in 'target' directory with a provided content
+fn get_temp_file(file_name: &str) -> fs::File {
+    // build tmp path to a file in "target/debug/testdata"
+    let mut path_buf = env::current_dir().unwrap();
+    path_buf.push("target");
+    path_buf.push("debug");
+    path_buf.push("testdata");
+    fs::create_dir_all(&path_buf).unwrap();
+    path_buf.push(file_name);
+
+    File::create(path_buf).unwrap()
+}
diff --git a/rust/parquet_derive/Cargo.toml b/rust/parquet_derive/Cargo.toml
new file mode 100644
index 00000000000..b65b64101c1
--- /dev/null
+++ b/rust/parquet_derive/Cargo.toml
@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "parquet_derive"
+version = "3.0.0-SNAPSHOT"
+authors = ["Apache Arrow <dev@arrow.apache.org>"]
+keywords = [ "parquet" ]
+edition = "2018"
+
+[lib]
+proc-macro = true
+
+[features]
+chrono = []
+bigdecimal = []
+uuid = []
+
+[dependencies]
+proc-macro2 = "1.0"
+quote = "1.0"
+syn = { version = "1.0", features = ["full", "extra-traits"] }
+parquet = { path = "../parquet", version = "3.0.0-SNAPSHOT" }
diff --git a/rust/parquet_derive/README.md b/rust/parquet_derive/README.md
new file mode 100644
index 00000000000..364770336de
--- /dev/null
+++ b/rust/parquet_derive/README.md
@@ -0,0 +1,98 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Parquet Derive
+
+A crate for deriving `RecordWriter` for arbitrary, _simple_ structs. This does not generate writers for arbitrarily nested
+structures. It only works for primitives and a few generic structures and
+various levels of reference. Please see features checklist for what is currently
+supported.
+
+Derive also has some support for the chrono time library. You must must enable the `chrono` feature to get this support.
+
+## Usage
+Add this to your Cargo.toml:
+```toml
+[dependencies]
+parquet = "3.0.0-SNAPSHOT"
+parquet_derive = "3.0.0-SNAPSHOT"
+```
+
+and this to your crate root:
+```rust
+extern crate parquet;
+#[macro_use] extern crate parquet_derive;
+```
+
+Example usage of deriving a `RecordWriter` for your struct:
+
+```rust
+use parquet;
+use parquet::record::RecordWriter;
+
+#[derive(ParquetRecordWriter)]
+struct ACompleteRecord<'a> {
+    pub a_bool: bool,
+    pub a_str: &'a str,
+    pub a_string: String,
+    pub a_borrowed_string: &'a String,
+    pub maybe_a_str: Option<&'a str>,
+    pub magic_number: i32,
+    pub low_quality_pi: f32,
+    pub high_quality_pi: f64,
+    pub maybe_pi: Option<f32>,
+    pub maybe_best_pi: Option<f64>,
+}
+
+// Initialize your parquet file
+let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
+let mut row_group = writer.next_row_group().unwrap();
+
+// Build up your records
+let chunks = vec![ACompleteRecord{...}];
+
+// The derived `RecordWriter` takes over here
+(&chunks[..]).write_to_row_group(&mut row_group);
+
+writer.close_row_group(row_group).unwrap();
+writer.close().unwrap();
+```
+
+## Features
+- [X] Support writing `String`, `&str`, `bool`, `i32`, `f32`, `f64`, `Vec<u8>`
+- [ ] Support writing dictionaries
+- [X] Support writing logical types like timestamp
+- [X] Derive definition_levels for `Option`
+- [ ] Derive definition levels for nested structures
+- [ ] Derive writing tuple struct
+- [ ] Derive writing `tuple` container types
+
+## Requirements
+- Same as `parquet-rs`
+
+## Test
+Testing a `*_derive` crate requires an intermediate crate. Go to `parquet_derive_test` and run `cargo test` for
+unit tests.
+
+## Docs
+To build documentation, run `cargo doc --no-deps`.
+To compile and view in the browser, run `cargo doc --no-deps --open`.
+
+## License
+Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0.
diff --git a/rust/parquet_derive/src/lib.rs b/rust/parquet_derive/src/lib.rs
new file mode 100644
index 00000000000..35a538f932c
--- /dev/null
+++ b/rust/parquet_derive/src/lib.rs
@@ -0,0 +1,126 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#![recursion_limit = "128"]
+
+extern crate proc_macro;
+extern crate proc_macro2;
+extern crate syn;
+#[macro_use]
+extern crate quote;
+
+extern crate parquet;
+
+use syn::{parse_macro_input, Data, DataStruct, DeriveInput};
+
+mod parquet_field;
+
+/// Derive flat, simple RecordWriter implementations. Works by parsing
+/// a struct tagged with `#[derive(ParquetRecordWriter)]` and emitting
+/// the correct writing code for each field of the struct. Column writers
+/// are generated in the order they are defined.
+///
+/// It is up to the programmer to keep the order of the struct
+/// fields lined up with the schema.
+///
+/// Example:
+///
+/// ```ignore
+/// use parquet;
+/// use parquet::record::RecordWriter;
+/// use parquet::schema::parser::parse_message_type;
+///
+/// use std::rc::Rc;
+//
+/// #[derive(ParquetRecordWriter)]
+/// struct ACompleteRecord<'a> {
+///   pub a_bool: bool,
+///   pub a_str: &'a str,
+/// }
+///
+/// let schema_str = "message schema {
+///   REQUIRED boolean         a_bool;
+///   REQUIRED BINARY          a_str (UTF8);
+/// }";
+///
+/// pub fn write_some_records() {
+///   let samples = vec![
+///     ACompleteRecord {
+///       a_bool: true,
+///       a_str: "I'm true"
+///     },
+///     ACompleteRecord {
+///       a_bool: false,
+///       a_str: "I'm false"
+///     }
+///   ];
+///
+///  let schema = Rc::new(parse_message_type(schema_str).unwrap());
+///
+///  let props = Rc::new(WriterProperties::builder().build());
+///  let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
+///
+///  let mut row_group = writer.next_row_group().unwrap();
+///  samples.as_slice().write_to_row_group(&mut row_group).unwrap();
+///  writer.close_row_group(row_group).unwrap();
+///  writer.close().unwrap();
+/// }
+/// ```
+///
+#[proc_macro_derive(ParquetRecordWriter)]
+pub fn parquet_record_writer(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+    let input: DeriveInput = parse_macro_input!(input as DeriveInput);
+    let fields = match input.data {
+        Data::Struct(DataStruct { fields, .. }) => fields,
+        Data::Enum(_) => unimplemented!("Enum currently is not supported"),
+        Data::Union(_) => unimplemented!("Union currently is not supported"),
+    };
+
+    let field_infos: Vec<_> = fields
+        .iter()
+        .map(|f: &syn::Field| parquet_field::Field::from(f))
+        .collect();
+
+    let writer_snippets: Vec<proc_macro2::TokenStream> =
+        field_infos.iter().map(|x| x.writer_snippet()).collect();
+
+    let derived_for = input.ident;
+    let generics = input.generics;
+
+    (quote! {
+    impl#generics RecordWriter<#derived_for#generics> for &[#derived_for#generics] {
+      fn write_to_row_group(&self, row_group_writer: &mut Box<parquet::file::writer::RowGroupWriter>) -> Result<(), parquet::errors::ParquetError> {
+        let mut row_group_writer = row_group_writer;
+        let records = &self; // Used by all the writer snippets to be more clear
+
+        #(
+          {
+              let mut some_column_writer = row_group_writer.next_column().unwrap();
+              if let Some(mut column_writer) = some_column_writer {
+                  #writer_snippets
+                  row_group_writer.close_column(column_writer)?;
+              } else {
+                  return Err(parquet::errors::ParquetError::General("Failed to get next column".into()))
+              }
+          }
+        );*
+
+        Ok(())
+      }
+    }
+  }).into()
+}
diff --git a/rust/parquet_derive/src/parquet_field.rs b/rust/parquet_derive/src/parquet_field.rs
new file mode 100644
index 00000000000..54a18bd8d7f
--- /dev/null
+++ b/rust/parquet_derive/src/parquet_field.rs
@@ -0,0 +1,931 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[derive(Debug, PartialEq)]
+pub struct Field {
+    ident: syn::Ident,
+    ty: Type,
+    is_a_byte_buf: bool,
+    third_party_type: Option<ThirdPartyType>,
+}
+
+/// Use third party libraries, detected
+/// at compile time. These libraries will
+/// be written to parquet as their preferred
+/// physical type.
+///
+///   ChronoNaiveDateTime is written as i64
+///   ChronoNaiveDate is written as i32
+#[derive(Debug, PartialEq)]
+enum ThirdPartyType {
+    ChronoNaiveDateTime,
+    ChronoNaiveDate,
+    Uuid,
+}
+
+impl Field {
+    pub fn from(f: &syn::Field) -> Self {
+        let ty = Type::from(f);
+        let is_a_byte_buf = ty.physical_type() == parquet::basic::Type::BYTE_ARRAY;
+
+        let third_party_type = match &ty.last_part()[..] {
+            "NaiveDateTime" => Some(ThirdPartyType::ChronoNaiveDateTime),
+            "NaiveDate" => Some(ThirdPartyType::ChronoNaiveDate),
+            "Uuid" => Some(ThirdPartyType::Uuid),
+            _ => None,
+        };
+
+        Field {
+            ident: f
+                .ident
+                .clone()
+                .expect("Only structs with named fields are currently supported"),
+            ty,
+            is_a_byte_buf,
+            third_party_type,
+        }
+    }
+
+    /// Takes the parsed field of the struct and emits a valid
+    /// column writer snippet. Should match exactly what you
+    /// would write by hand.
+    ///
+    /// Can only generate writers for basic structs, for example:
+    ///
+    /// struct Record {
+    ///   a_bool: bool,
+    ///   maybe_a_bool: Option<bool>
+    /// }
+    ///
+    /// but not
+    ///
+    /// struct UnsupportedNestedRecord {
+    ///   a_property: bool,
+    ///   nested_record: Record
+    /// }
+    ///
+    /// because this parsing logic is not sophisticated enough for definition
+    /// levels beyond 2.
+    pub fn writer_snippet(&self) -> proc_macro2::TokenStream {
+        let ident = &self.ident;
+        let column_writer = self.ty.column_writer();
+
+        let vals_builder = match &self.ty {
+            Type::TypePath(_) => self.copied_direct_vals(),
+            Type::Option(ref first_type) => match **first_type {
+                Type::TypePath(_) => self.option_into_vals(),
+                Type::Reference(_, ref second_type) => match **second_type {
+                    Type::TypePath(_) => self.option_into_vals(),
+                    _ => unimplemented!("Unsupported type encountered"),
+                },
+                ref f @ _ => unimplemented!("Unsupported: {:#?}", f),
+            },
+            Type::Reference(_, ref first_type) => match **first_type {
+                Type::TypePath(_) => self.copied_direct_vals(),
+                Type::Option(ref second_type) => match **second_type {
+                    Type::TypePath(_) => self.option_into_vals(),
+                    Type::Reference(_, ref second_type) => match **second_type {
+                        Type::TypePath(_) => self.option_into_vals(),
+                        _ => unimplemented!("Unsupported type encountered"),
+                    },
+                    ref f @ _ => unimplemented!("Unsupported: {:#?}", f),
+                },
+                ref f @ _ => unimplemented!("Unsupported: {:#?}", f),
+            },
+            f @ _ => unimplemented!("Unsupported: {:#?}", f),
+        };
+
+        let definition_levels = match &self.ty {
+            Type::TypePath(_) => None,
+            Type::Option(ref first_type) => match **first_type {
+                Type::TypePath(_) => Some(self.optional_definition_levels()),
+                Type::Option(_) => unimplemented!("Unsupported nesting encountered"),
+                Type::Reference(_, ref second_type)
+                | Type::Vec(ref second_type)
+                | Type::Array(ref second_type) => match **second_type {
+                    Type::TypePath(_) => Some(self.optional_definition_levels()),
+                    _ => unimplemented!("Unsupported nesting encountered"),
+                },
+            },
+            Type::Reference(_, ref first_type)
+            | Type::Vec(ref first_type)
+            | Type::Array(ref first_type) => match **first_type {
+                Type::TypePath(_) => None,
+                Type::Reference(_, ref second_type)
+                | Type::Vec(ref second_type)
+                | Type::Array(ref second_type)
+                | Type::Option(ref second_type) => match **second_type {
+                    Type::TypePath(_) => Some(self.optional_definition_levels()),
+                    Type::Reference(_, ref third_type) => match **third_type {
+                        Type::TypePath(_) => Some(self.optional_definition_levels()),
+                        _ => unimplemented!("Unsupported definition encountered"),
+                    },
+                    _ => unimplemented!("Unsupported definition encountered"),
+                },
+            },
+        };
+
+        // "vals" is the run of primitive data being written for the column
+        // "definition_levels" is a vector of bools which controls whether a value is missing or present
+        // this TokenStream is only one part of the code for writing a column and
+        // it relies on values calculated in prior code snippets, namely "definition_levels" and "vals_builder".
+        // All the context is put together in this functions final quote and
+        // this expression just switches between non-nullable and nullable write statements
+        let write_batch_expr = if definition_levels.is_some() {
+            quote! {
+                if let #column_writer(ref mut typed) = column_writer {
+                    typed.write_batch(&vals[..], Some(&definition_levels[..]), None)?;
+                } else {
+                    panic!("Schema and struct disagree on type for {}", stringify!{#ident})
+                }
+            }
+        } else {
+            quote! {
+                if let #column_writer(ref mut typed) = column_writer {
+                    typed.write_batch(&vals[..], None, None)?;
+                } else {
+                    panic!("Schema and struct disagree on type for {}", stringify!{#ident})
+                }
+            }
+        };
+
+        quote! {
+            {
+                #definition_levels
+
+                #vals_builder
+
+                #write_batch_expr
+            }
+        }
+    }
+
+    fn option_into_vals(&self) -> proc_macro2::TokenStream {
+        let field_name = &self.ident;
+        let is_a_byte_buf = self.is_a_byte_buf;
+        let is_a_timestamp =
+            self.third_party_type == Some(ThirdPartyType::ChronoNaiveDateTime);
+        let is_a_date = self.third_party_type == Some(ThirdPartyType::ChronoNaiveDate);
+        let is_a_uuid = self.third_party_type == Some(ThirdPartyType::Uuid);
+        let copy_to_vec = match self.ty.physical_type() {
+            parquet::basic::Type::BYTE_ARRAY
+            | parquet::basic::Type::FIXED_LEN_BYTE_ARRAY => false,
+            _ => true,
+        };
+
+        let binding = if copy_to_vec {
+            quote! { let Some(inner) = rec.#field_name }
+        } else {
+            quote! { let Some(ref inner) = rec.#field_name }
+        };
+
+        let some = if is_a_timestamp {
+            quote! { Some(inner.timestamp_millis()) }
+        } else if is_a_date {
+            quote! { Some(inner.signed_duration_since(chrono::NaiveDate::from_ymd(1970, 1, 1)).num_days() as i32)  }
+        } else if is_a_uuid {
+            quote! { Some((&inner.to_string()[..]).into()) }
+        } else if is_a_byte_buf {
+            quote! { Some((&inner[..]).into())}
+        } else {
+            quote! { Some(inner) }
+        };
+
+        quote! {
+            let vals: Vec<_> = records.iter().filter_map(|rec| {
+                if #binding {
+                    #some
+                } else {
+                    None
+                }
+            }).collect();
+        }
+    }
+
+    fn copied_direct_vals(&self) -> proc_macro2::TokenStream {
+        let field_name = &self.ident;
+        let is_a_byte_buf = self.is_a_byte_buf;
+        let is_a_timestamp =
+            self.third_party_type == Some(ThirdPartyType::ChronoNaiveDateTime);
+        let is_a_date = self.third_party_type == Some(ThirdPartyType::ChronoNaiveDate);
+        let is_a_uuid = self.third_party_type == Some(ThirdPartyType::Uuid);
+
+        let access = if is_a_timestamp {
+            quote! { rec.#field_name.timestamp_millis() }
+        } else if is_a_date {
+            quote! { rec.#field_name.signed_duration_since(chrono::NaiveDate::from_ymd(1970, 1, 1)).num_days() as i32 }
+        } else if is_a_uuid {
+            quote! { (&rec.#field_name.to_string()[..]).into() }
+        } else if is_a_byte_buf {
+            quote! { (&rec.#field_name[..]).into() }
+        } else {
+            quote! { rec.#field_name }
+        };
+
+        quote! {
+            let vals: Vec<_> = records.iter().map(|rec| #access).collect();
+        }
+    }
+
+    fn optional_definition_levels(&self) -> proc_macro2::TokenStream {
+        let field_name = &self.ident;
+
+        quote! {
+            let definition_levels: Vec<i16> = self
+              .iter()
+              .map(|rec| if rec.#field_name.is_some() { 1 } else { 0 })
+              .collect();
+        }
+    }
+}
+
+#[derive(Debug, PartialEq)]
+enum Type {
+    Array(Box<Type>),
+    Option(Box<Type>),
+    Vec(Box<Type>),
+    TypePath(syn::Type),
+    Reference(Option<syn::Lifetime>, Box<Type>),
+}
+
+impl Type {
+    /// Takes a rust type and returns the appropriate
+    /// parquet-rs column writer
+    fn column_writer(&self) -> syn::TypePath {
+        use parquet::basic::Type as BasicType;
+
+        match self.physical_type() {
+            BasicType::BOOLEAN => {
+                syn::parse_quote!(parquet::column::writer::ColumnWriter::BoolColumnWriter)
+            }
+            BasicType::INT32 => syn::parse_quote!(
+                parquet::column::writer::ColumnWriter::Int32ColumnWriter
+            ),
+            BasicType::INT64 => syn::parse_quote!(
+                parquet::column::writer::ColumnWriter::Int64ColumnWriter
+            ),
+            BasicType::INT96 => syn::parse_quote!(
+                parquet::column::writer::ColumnWriter::Int96ColumnWriter
+            ),
+            BasicType::FLOAT => syn::parse_quote!(
+                parquet::column::writer::ColumnWriter::FloatColumnWriter
+            ),
+            BasicType::DOUBLE => syn::parse_quote!(
+                parquet::column::writer::ColumnWriter::DoubleColumnWriter
+            ),
+            BasicType::BYTE_ARRAY => syn::parse_quote!(
+                parquet::column::writer::ColumnWriter::ByteArrayColumnWriter
+            ),
+            BasicType::FIXED_LEN_BYTE_ARRAY => syn::parse_quote!(
+                parquet::column::writer::ColumnWriter::FixedLenByteArrayColumnWriter
+            ),
+        }
+    }
+
+    /// Helper to simplify a nested field definition to its leaf type
+    ///
+    /// Ex:
+    ///   Option<&String> => Type::TypePath(String)
+    ///   &Option<i32> => Type::TypePath(i32)
+    ///   Vec<Vec<u8>> => Type::Vec(u8)
+    ///
+    /// Useful in determining the physical type of a field and the
+    /// definition levels.
+    fn leaf_type_recursive(&self) -> &Type {
+        self.leaf_type_recursive_helper(self, None)
+    }
+
+    fn leaf_type_recursive_helper<'a>(
+        &'a self,
+        ty: &'a Type,
+        parent_ty: Option<&'a Type>,
+    ) -> &Type {
+        match ty {
+            Type::TypePath(_) => parent_ty.unwrap_or(ty),
+            Type::Option(ref first_type)
+            | Type::Vec(ref first_type)
+            | Type::Array(ref first_type)
+            | Type::Reference(_, ref first_type) => {
+                self.leaf_type_recursive_helper(first_type, Some(ty))
+            }
+        }
+    }
+
+    /// Helper method to further unwrap leaf_type() to get inner-most
+    /// type information, useful for determining the physical type
+    /// and normalizing the type paths.
+    fn inner_type(&self) -> &syn::Type {
+        let leaf_type = self.leaf_type_recursive();
+
+        match leaf_type {
+            Type::TypePath(ref type_) => type_,
+            Type::Option(ref first_type)
+            | Type::Vec(ref first_type)
+            | Type::Array(ref first_type)
+            | Type::Reference(_, ref first_type) => match **first_type {
+                Type::TypePath(ref type_) => type_,
+                _ => unimplemented!("leaf_type() should only return shallow types"),
+            },
+        }
+    }
+
+    /// Helper to normalize a type path by extracting the
+    /// most identifiable part
+    ///
+    /// Ex:
+    ///   std::string::String => String
+    ///   Vec<u8> => Vec<u8>
+    ///   chrono::NaiveDateTime => NaiveDateTime
+    ///
+    /// Does run the risk of mis-identifying a type if import
+    /// rename is in play. Please note procedural macros always
+    /// run before type resolution so this is a risk the user
+    /// takes on when renaming imports.
+    fn last_part(&self) -> String {
+        let inner_type = self.inner_type();
+        let inner_type_str = (quote! { #inner_type }).to_string();
+
+        inner_type_str
+            .split("::")
+            .last()
+            .unwrap()
+            .trim()
+            .to_string()
+    }
+
+    /// Converts rust types to parquet physical types.
+    ///
+    /// Ex:
+    ///   [u8; 10] => FIXED_LEN_BYTE_ARRAY
+    ///   Vec<u8>  => BYTE_ARRAY
+    ///   String => BYTE_ARRAY
+    ///   i32 => INT32
+    fn physical_type(&self) -> parquet::basic::Type {
+        use parquet::basic::Type as BasicType;
+
+        let last_part = self.last_part();
+        let leaf_type = self.leaf_type_recursive();
+
+        match leaf_type {
+            Type::Array(ref first_type) => {
+                if let Type::TypePath(_) = **first_type {
+                    if last_part == "u8" {
+                        return BasicType::FIXED_LEN_BYTE_ARRAY;
+                    }
+                }
+            }
+            Type::Vec(ref first_type) => {
+                if let Type::TypePath(_) = **first_type {
+                    if last_part == "u8" {
+                        return BasicType::BYTE_ARRAY;
+                    }
+                }
+            }
+            _ => (),
+        }
+
+        match last_part.trim() {
+            "bool" => BasicType::BOOLEAN,
+            "u8" | "u16" | "u32" => BasicType::INT32,
+            "i8" | "i16" | "i32" | "NaiveDate" => BasicType::INT32,
+            "u64" | "i64" | "usize" | "NaiveDateTime" => BasicType::INT64,
+            "f32" => BasicType::FLOAT,
+            "f64" => BasicType::DOUBLE,
+            "String" | "str" | "Uuid" => BasicType::BYTE_ARRAY,
+            f @ _ => unimplemented!("{} currently is not supported", f),
+        }
+    }
+
+    /// Convert a parsed rust field AST in to a more easy to manipulate
+    /// parquet_derive::Field
+    fn from(f: &syn::Field) -> Self {
+        Type::from_type(f, &f.ty)
+    }
+
+    fn from_type(f: &syn::Field, ty: &syn::Type) -> Self {
+        match ty {
+            syn::Type::Path(ref p) => Type::from_type_path(f, p),
+            syn::Type::Reference(ref tr) => Type::from_type_reference(f, tr),
+            syn::Type::Array(ref ta) => Type::from_type_array(f, ta),
+            other @ _ => unimplemented!(
+                "Unable to derive {:?} - it is currently an unsupported type\n{:#?}",
+                f.ident.as_ref().unwrap(),
+                other
+            ),
+        }
+    }
+
+    fn from_type_path(f: &syn::Field, p: &syn::TypePath) -> Self {
+        let last_segment = p.path.segments.last().unwrap();
+
+        let is_vec =
+            last_segment.ident == syn::Ident::new("Vec", proc_macro2::Span::call_site());
+        let is_option = last_segment.ident
+            == syn::Ident::new("Option", proc_macro2::Span::call_site());
+
+        if is_vec || is_option {
+            let generic_type = match &last_segment.arguments {
+                syn::PathArguments::AngleBracketed(angle_args) => {
+                    assert_eq!(angle_args.args.len(), 1);
+                    let first_arg = &angle_args.args[0];
+
+                    match first_arg {
+                        syn::GenericArgument::Type(ref typath) => typath.clone(),
+                        other @ _ => unimplemented!("Unsupported: {:#?}", other),
+                    }
+                }
+                other @ _ => unimplemented!("Unsupported: {:#?}", other),
+            };
+
+            if is_vec {
+                Type::Vec(Box::new(Type::from_type(f, &generic_type)))
+            } else {
+                Type::Option(Box::new(Type::from_type(f, &generic_type)))
+            }
+        } else {
+            Type::TypePath(syn::Type::Path(p.clone()))
+        }
+    }
+
+    fn from_type_reference(f: &syn::Field, tr: &syn::TypeReference) -> Self {
+        let lifetime = tr.lifetime.clone();
+        let inner_type = Type::from_type(f, tr.elem.as_ref());
+        Type::Reference(lifetime, Box::new(inner_type))
+    }
+
+    fn from_type_array(f: &syn::Field, ta: &syn::TypeArray) -> Self {
+        let inner_type = Type::from_type(f, ta.elem.as_ref());
+        Type::Array(Box::new(inner_type))
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use syn::{self, Data, DataStruct, DeriveInput};
+
+    fn extract_fields(input: proc_macro2::TokenStream) -> Vec<syn::Field> {
+        let input: DeriveInput = syn::parse2(input).unwrap();
+
+        let fields = match input.data {
+            Data::Struct(DataStruct { fields, .. }) => fields,
+            _ => panic!("Input must be a struct"),
+        };
+
+        fields.iter().map(|field| field.to_owned()).collect()
+    }
+
+    #[test]
+    fn test_generating_a_simple_writer_snippet() {
+        let snippet: proc_macro2::TokenStream = quote! {
+          struct ABoringStruct {
+            counter: usize,
+          }
+        };
+
+        let fields = extract_fields(snippet);
+        let counter = Field::from(&fields[0]);
+
+        let snippet = counter.writer_snippet().to_string();
+        assert_eq!(snippet,
+                   (quote!{
+                        {
+                            let vals : Vec < _ > = records . iter ( ) . map ( | rec | rec . counter ) . collect ( );
+
+                            if let parquet::column::writer::ColumnWriter::Int64ColumnWriter ( ref mut typed ) = column_writer {
+                                typed . write_batch ( & vals [ .. ] , None , None ) ?;
+                            }  else {
+                                panic!("Schema and struct disagree on type for {}" , stringify!{ counter } )
+                            }
+                        }
+                   }).to_string()
+        )
+    }
+
+    #[test]
+    fn test_optional_to_writer_snippet() {
+        let struct_def: proc_macro2::TokenStream = quote! {
+          struct StringBorrower<'a> {
+            optional_str: Option<&'a str>,
+            optional_string: &Option<String>,
+            optional_dumb_int: &Option<&i32>,
+          }
+        };
+
+        let fields = extract_fields(struct_def);
+
+        let optional = Field::from(&fields[0]);
+        let snippet = optional.writer_snippet();
+        assert_eq!(snippet.to_string(),
+          (quote! {
+          {
+                let definition_levels : Vec < i16 > = self . iter ( ) . map ( | rec | if rec . optional_str . is_some ( ) { 1 } else { 0 } ) . collect ( ) ;
+
+                let vals: Vec <_> = records.iter().filter_map( |rec| {
+                    if let Some ( ref inner ) = rec . optional_str {
+                        Some ( (&inner[..]).into() )
+                    } else {
+                        None
+                    }
+                }).collect();
+
+                if let parquet::column::writer::ColumnWriter::ByteArrayColumnWriter ( ref mut typed ) = column_writer {
+                    typed . write_batch ( & vals [ .. ] , Some(&definition_levels[..]) , None ) ? ;
+                } else {
+                    panic!("Schema and struct disagree on type for {}" , stringify ! { optional_str } )
+                }
+           }
+            }
+          ).to_string());
+
+        let optional = Field::from(&fields[1]);
+        let snippet = optional.writer_snippet();
+        assert_eq!(snippet.to_string(),
+                   (quote!{
+                   {
+                        let definition_levels : Vec < i16 > = self . iter ( ) . map ( | rec | if rec . optional_string . is_some ( ) { 1 } else { 0 } ) . collect ( ) ;
+
+                        let vals: Vec <_> = records.iter().filter_map( |rec| {
+                            if let Some ( ref inner ) = rec . optional_string {
+                                Some ( (&inner[..]).into() )
+                            } else {
+                                None
+                            }
+                        }).collect();
+
+                        if let parquet::column::writer::ColumnWriter::ByteArrayColumnWriter ( ref mut typed ) = column_writer {
+                            typed . write_batch ( & vals [ .. ] , Some(&definition_levels[..]) , None ) ? ;
+                        } else {
+                            panic!("Schema and struct disagree on type for {}" , stringify ! { optional_string } )
+                        }
+                    }
+        }).to_string());
+
+        let optional = Field::from(&fields[2]);
+        let snippet = optional.writer_snippet();
+        assert_eq!(snippet.to_string(),
+                   (quote!{
+                    {
+                        let definition_levels : Vec < i16 > = self . iter ( ) . map ( | rec | if rec . optional_dumb_int . is_some ( ) { 1 } else { 0 } ) . collect ( ) ;
+
+                        let vals: Vec <_> = records.iter().filter_map( |rec| {
+                            if let Some ( inner ) = rec . optional_dumb_int {
+                                Some ( inner )
+                            } else {
+                                None
+                            }
+                        }).collect();
+
+                        if let parquet::column::writer::ColumnWriter::Int32ColumnWriter ( ref mut typed ) = column_writer {
+                            typed . write_batch ( & vals [ .. ] , Some(&definition_levels[..]) , None ) ? ;
+                        }  else {
+                            panic!("Schema and struct disagree on type for {}" , stringify ! { optional_dumb_int } )
+                        }
+                    }
+        }).to_string());
+    }
+
+    #[test]
+    fn test_converting_to_column_writer_type() {
+        let snippet: proc_macro2::TokenStream = quote! {
+          struct ABasicStruct {
+            yes_no: bool,
+            name: String,
+          }
+        };
+
+        let fields = extract_fields(snippet);
+        let processed: Vec<_> = fields.iter().map(|field| Field::from(field)).collect();
+
+        let column_writers: Vec<_> = processed
+            .iter()
+            .map(|field| field.ty.column_writer())
+            .collect();
+
+        assert_eq!(
+            column_writers,
+            vec![
+                syn::parse_quote!(
+                    parquet::column::writer::ColumnWriter::BoolColumnWriter
+                ),
+                syn::parse_quote!(
+                    parquet::column::writer::ColumnWriter::ByteArrayColumnWriter
+                )
+            ]
+        );
+    }
+
+    #[test]
+    fn convert_basic_struct() {
+        let snippet: proc_macro2::TokenStream = quote! {
+          struct ABasicStruct {
+            yes_no: bool,
+            name: String,
+          }
+        };
+
+        let fields = extract_fields(snippet);
+        let processed: Vec<_> = fields.iter().map(|field| Field::from(field)).collect();
+        assert_eq!(processed.len(), 2);
+
+        assert_eq!(
+            processed,
+            vec![
+                Field {
+                    ident: syn::Ident::new("yes_no", proc_macro2::Span::call_site()),
+                    ty: Type::TypePath(syn::parse_quote!(bool)),
+                    is_a_byte_buf: false,
+                    third_party_type: None,
+                },
+                Field {
+                    ident: syn::Ident::new("name", proc_macro2::Span::call_site()),
+                    ty: Type::TypePath(syn::parse_quote!(String)),
+                    is_a_byte_buf: true,
+                    third_party_type: None,
+                }
+            ]
+        )
+    }
+
+    #[test]
+    fn test_get_inner_type() {
+        let snippet: proc_macro2::TokenStream = quote! {
+          struct LotsOfInnerTypes {
+            a_vec: Vec<u8>,
+            a_option: std::option::Option<bool>,
+            a_silly_string: std::string::String,
+            a_complicated_thing: std::option::Option<std::result::Result<(),()>>,
+          }
+        };
+
+        let fields = extract_fields(snippet);
+        let converted_fields: Vec<_> =
+            fields.iter().map(|field| Type::from(field)).collect();
+        let inner_types: Vec<_> = converted_fields
+            .iter()
+            .map(|field| field.inner_type())
+            .collect();
+        let inner_types_strs: Vec<_> = inner_types
+            .iter()
+            .map(|ty| (quote! { #ty }).to_string())
+            .collect();
+
+        assert_eq!(
+            inner_types_strs,
+            vec![
+                "u8",
+                "bool",
+                "std :: string :: String",
+                "std :: result :: Result < () , () >"
+            ]
+        )
+    }
+
+    #[test]
+    fn test_physical_type() {
+        use parquet::basic::Type as BasicType;
+        let snippet: proc_macro2::TokenStream = quote! {
+          struct LotsOfInnerTypes {
+            a_buf: Vec<u8>,
+            a_number: i32,
+            a_verbose_option: std::option::Option<bool>,
+            a_silly_string: std::string::String,
+            a_fix_byte_buf: [u8; 10],
+            a_complex_option: Option<&Vec<u8>>,
+            a_complex_vec: &Vec<&Option<u8>>,
+          }
+        };
+
+        let fields = extract_fields(snippet);
+        let converted_fields: Vec<_> =
+            fields.iter().map(|field| Type::from(field)).collect();
+        let physical_types: Vec<_> = converted_fields
+            .iter()
+            .map(|ty| ty.physical_type())
+            .collect();
+
+        assert_eq!(
+            physical_types,
+            vec![
+                BasicType::BYTE_ARRAY,
+                BasicType::INT32,
+                BasicType::BOOLEAN,
+                BasicType::BYTE_ARRAY,
+                BasicType::FIXED_LEN_BYTE_ARRAY,
+                BasicType::BYTE_ARRAY,
+                BasicType::INT32
+            ]
+        )
+    }
+
+    #[test]
+    fn test_convert_comprehensive_owned_struct() {
+        let snippet: proc_macro2::TokenStream = quote! {
+          struct VecHolder {
+            a_vec: Vec<u8>,
+            a_option: std::option::Option<bool>,
+            a_silly_string: std::string::String,
+            a_complicated_thing: std::option::Option<std::result::Result<(),()>>,
+          }
+        };
+
+        let fields = extract_fields(snippet);
+        let converted_fields: Vec<_> =
+            fields.iter().map(|field| Type::from(field)).collect();
+
+        assert_eq!(
+            converted_fields,
+            vec![
+                Type::Vec(Box::new(Type::TypePath(syn::parse_quote!(u8)))),
+                Type::Option(Box::new(Type::TypePath(syn::parse_quote!(bool)))),
+                Type::TypePath(syn::parse_quote!(std::string::String)),
+                Type::Option(Box::new(Type::TypePath(
+                    syn::parse_quote!(std::result::Result<(),()>)
+                ))),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_convert_borrowed_struct() {
+        let snippet: proc_macro2::TokenStream = quote! {
+          struct Borrower<'a> {
+            a_str: &'a str,
+            a_borrowed_option: &'a Option<bool>,
+            so_many_borrows: &'a Option<&'a str>,
+          }
+        };
+
+        let fields = extract_fields(snippet);
+        let types: Vec<_> = fields.iter().map(|field| Type::from(field)).collect();
+
+        assert_eq!(
+            types,
+            vec![
+                Type::Reference(
+                    Some(syn::Lifetime::new("'a", proc_macro2::Span::call_site())),
+                    Box::new(Type::TypePath(syn::parse_quote!(str)))
+                ),
+                Type::Reference(
+                    Some(syn::Lifetime::new("'a", proc_macro2::Span::call_site())),
+                    Box::new(Type::Option(Box::new(Type::TypePath(syn::parse_quote!(
+                        bool
+                    )))))
+                ),
+                Type::Reference(
+                    Some(syn::Lifetime::new("'a", proc_macro2::Span::call_site())),
+                    Box::new(Type::Option(Box::new(Type::Reference(
+                        Some(syn::Lifetime::new("'a", proc_macro2::Span::call_site())),
+                        Box::new(Type::TypePath(syn::parse_quote!(str)))
+                    ))))
+                ),
+            ]
+        );
+    }
+
+    #[test]
+    #[cfg(feature = "chrono")]
+    fn test_chrono_timestamp_millis() {
+        let snippet: proc_macro2::TokenStream = quote! {
+          struct ATimestampStruct {
+            henceforth: chrono::NaiveDateTime,
+            maybe_happened: Option<&chrono::NaiveDateTime>,
+          }
+        };
+
+        let fields = extract_fields(snippet);
+        let when = Field::from(&fields[0]);
+        assert_eq!(when.writer_snippet().to_string(),(quote!{
+            {
+                let vals : Vec<_> = records.iter().map(|rec| rec.henceforth.timestamp_millis() ).collect();
+                if let parquet::column::writer::ColumnWriter::Int64ColumnWriter(ref mut typed) = column_writer {
+                    typed.write_batch(&vals[..], None, None) ?;
+                } else {
+                    panic!("Schema and struct disagree on type for {}" , stringify!{ henceforth })
+                }
+            }
+        }).to_string());
+
+        let maybe_happened = Field::from(&fields[1]);
+        assert_eq!(maybe_happened.writer_snippet().to_string(),(quote!{
+            {
+                let definition_levels : Vec<i16> = self.iter().map(|rec| if rec.maybe_happened.is_some() { 1 } else { 0 }).collect();
+                let vals : Vec<_> = records.iter().filter_map(|rec| {
+                    if let Some(inner) = rec.maybe_happened {
+                        Some( inner.timestamp_millis() )
+                    } else {
+                        None
+                    }
+                }).collect();
+
+                if let parquet::column::writer::ColumnWriter::Int64ColumnWriter(ref mut typed) = column_writer {
+                    typed.write_batch(&vals[..], Some(&definition_levels[..]), None) ?;
+                } else {
+                    panic!("Schema and struct disagree on type for {}" , stringify!{ maybe_happened })
+                }
+            }
+        }).to_string());
+    }
+
+    #[test]
+    #[cfg(feature = "chrono")]
+    fn test_chrono_date() {
+        let snippet: proc_macro2::TokenStream = quote! {
+          struct ATimestampStruct {
+            henceforth: chrono::NaiveDate,
+            maybe_happened: Option<&chrono::NaiveDate>,
+          }
+        };
+
+        let fields = extract_fields(snippet);
+        let when = Field::from(&fields[0]);
+        assert_eq!(when.writer_snippet().to_string(),(quote!{
+            {
+                let vals : Vec<_> = records.iter().map(|rec| rec.henceforth.signed_duration_since(chrono::NaiveDate::from_ymd(1970, 1, 1)).num_days() as i32).collect();
+                if let parquet::column::writer::ColumnWriter::Int32ColumnWriter(ref mut typed) = column_writer {
+                    typed.write_batch(&vals[..], None, None) ?;
+                } else {
+                    panic!("Schema and struct disagree on type for {}" , stringify!{ henceforth })
+                }
+            }
+        }).to_string());
+
+        let maybe_happened = Field::from(&fields[1]);
+        assert_eq!(maybe_happened.writer_snippet().to_string(),(quote!{
+            {
+                let definition_levels : Vec<i16> = self.iter().map(|rec| if rec.maybe_happened.is_some() { 1 } else { 0 }).collect();
+                let vals : Vec<_> = records.iter().filter_map(|rec| {
+                    if let Some(inner) = rec.maybe_happened {
+                        Some( inner.signed_duration_since(chrono::NaiveDate::from_ymd(1970, 1, 1)).num_days() as i32 )
+                    } else {
+                        None
+                    }
+                }).collect();
+
+                if let parquet::column::writer::ColumnWriter::Int32ColumnWriter(ref mut typed) = column_writer {
+                    typed.write_batch(&vals[..], Some(&definition_levels[..]), None) ?;
+                } else {
+                    panic!("Schema and struct disagree on type for {}" , stringify!{ maybe_happened })
+                }
+            }
+        }).to_string());
+    }
+
+    #[test]
+    #[cfg(feature = "uuid")]
+    fn test_uuid() {
+        let snippet: proc_macro2::TokenStream = quote! {
+          struct ATimestampStruct {
+            unique_id: uuid::Uuid,
+            maybe_unique_id: Option<&uuid::Uuid>,
+          }
+        };
+
+        let fields = extract_fields(snippet);
+        let when = Field::from(&fields[0]);
+        assert_eq!(when.writer_snippet().to_string(),(quote!{
+            {
+                let vals : Vec<_> = records.iter().map(|rec| (&rec.unique_id.to_string()[..]).into() ).collect();
+                if let parquet::column::writer::ColumnWriter::ByteArrayColumnWriter(ref mut typed) = column_writer {
+                    typed.write_batch(&vals[..], None, None) ?;
+                } else {
+                    panic!("Schema and struct disagree on type for {}" , stringify!{ unique_id })
+                }
+            }
+        }).to_string());
+
+        let maybe_happened = Field::from(&fields[1]);
+        assert_eq!(maybe_happened.writer_snippet().to_string(),(quote!{
+            {
+                let definition_levels : Vec<i16> = self.iter().map(|rec| if rec.maybe_unique_id.is_some() { 1 } else { 0 }).collect();
+                let vals : Vec<_> = records.iter().filter_map(|rec| {
+                    if let Some(ref inner) = rec.maybe_unique_id {
+                        Some( (&inner.to_string()[..]).into() )
+                    } else {
+                        None
+                    }
+                }).collect();
+
+                if let parquet::column::writer::ColumnWriter::ByteArrayColumnWriter(ref mut typed) = column_writer {
+                    typed.write_batch(&vals[..], Some(&definition_levels[..]), None) ?;
+                } else {
+                    panic!("Schema and struct disagree on type for {}" , stringify!{ maybe_unique_id })
+                }
+            }
+        }).to_string());
+    }
+}
diff --git a/rust/parquet_derive_test/Cargo.toml b/rust/parquet_derive_test/Cargo.toml
new file mode 100644
index 00000000000..8695b5d3cfe
--- /dev/null
+++ b/rust/parquet_derive_test/Cargo.toml
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "parquet_derive_test"
+version = "3.0.0-SNAPSHOT"
+authors = ["Apache Arrow <dev@arrow.apache.org>"]
+keywords = [ "parquet" ]
+edition = "2018"
+
+[dependencies]
+parquet = { path = "../parquet", version = "3.0.0-SNAPSHOT" }
+parquet_derive = { path = "../parquet_derive", version = "3.0.0-SNAPSHOT" }
diff --git a/rust/parquet_derive_test/src/lib.rs b/rust/parquet_derive_test/src/lib.rs
new file mode 100644
index 00000000000..aca4dc57049
--- /dev/null
+++ b/rust/parquet_derive_test/src/lib.rs
@@ -0,0 +1,129 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+extern crate parquet;
+
+#[macro_use]
+extern crate parquet_derive;
+
+use parquet::record::RecordWriter;
+
+#[derive(ParquetRecordWriter)]
+struct ACompleteRecord<'a> {
+    pub a_bool: bool,
+    pub a_str: &'a str,
+    pub a_string: String,
+    pub a_borrowed_string: &'a String,
+    pub maybe_a_str: Option<&'a str>,
+    pub maybe_a_string: Option<String>,
+    pub magic_number: i32,
+    pub low_quality_pi: f32,
+    pub high_quality_pi: f64,
+    pub maybe_pi: Option<f32>,
+    pub maybe_best_pi: Option<f64>,
+    pub borrowed_maybe_a_string: &'a Option<String>,
+    pub borrowed_maybe_a_str: &'a Option<&'a str>,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use parquet::{
+        file::{
+            properties::WriterProperties,
+            writer::{FileWriter, SerializedFileWriter},
+        },
+        schema::parser::parse_message_type,
+    };
+    use std::{env, fs, io::Write, rc::Rc};
+
+    #[test]
+    fn test_parquet_derive_hello() {
+        let file = get_temp_file("test_parquet_derive_hello", &[]);
+        let schema_str = "message schema {
+            REQUIRED boolean         a_bool;
+            REQUIRED BINARY          a_str (UTF8);
+            REQUIRED BINARY          a_string (UTF8);
+            REQUIRED BINARY          a_borrowed_string (UTF8);
+            OPTIONAL BINARY          a_maybe_str (UTF8);
+            OPTIONAL BINARY          a_maybe_string (UTF8);
+            REQUIRED INT32           magic_number;
+            REQUIRED FLOAT           low_quality_pi;
+            REQUIRED DOUBLE          high_quality_pi;
+            OPTIONAL FLOAT           maybe_pi;
+            OPTIONAL DOUBLE          maybe_best_pi;
+            OPTIONAL BINARY          borrowed_maybe_a_string (UTF8);
+            OPTIONAL BINARY          borrowed_maybe_a_str (UTF8);
+        }";
+
+        let schema = Rc::new(parse_message_type(schema_str).unwrap());
+
+        let props = Rc::new(WriterProperties::builder().build());
+        let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
+
+        let a_str = "hello mother".to_owned();
+        let a_borrowed_string = "cool news".to_owned();
+        let maybe_a_string = Some("it's true, I'm a string".to_owned());
+        let maybe_a_str = Some(&a_str[..]);
+
+        let drs: Vec<ACompleteRecord> = vec![ACompleteRecord {
+            a_bool: true,
+            a_str: &a_str[..],
+            a_string: "hello father".into(),
+            a_borrowed_string: &a_borrowed_string,
+            maybe_a_str: Some(&a_str[..]),
+            maybe_a_string: Some(a_str.clone()),
+            magic_number: 100,
+            low_quality_pi: 3.14,
+            high_quality_pi: 3.1415,
+            maybe_pi: Some(3.14),
+            maybe_best_pi: Some(3.1415),
+            borrowed_maybe_a_string: &maybe_a_string,
+            borrowed_maybe_a_str: &maybe_a_str,
+        }];
+
+        let mut row_group = writer.next_row_group().unwrap();
+        drs.as_slice().write_to_row_group(&mut row_group).unwrap();
+        writer.close_row_group(row_group).unwrap();
+        writer.close().unwrap();
+    }
+
+    /// Returns file handle for a temp file in 'target' directory with a provided content
+    pub fn get_temp_file(file_name: &str, content: &[u8]) -> fs::File {
+        // build tmp path to a file in "target/debug/testdata"
+        let mut path_buf = env::current_dir().unwrap();
+        path_buf.push("target");
+        path_buf.push("debug");
+        path_buf.push("testdata");
+        fs::create_dir_all(&path_buf).unwrap();
+        path_buf.push(file_name);
+
+        // write file content
+        let mut tmp_file = fs::File::create(path_buf.as_path()).unwrap();
+        tmp_file.write_all(content).unwrap();
+        tmp_file.sync_all().unwrap();
+
+        // return file handle for both read and write
+        let file = fs::OpenOptions::new()
+            .read(true)
+            .write(true)
+            .open(path_buf.as_path());
+        assert!(file.is_ok());
+        file.unwrap()
+    }
+}
diff --git a/testing b/testing
index f552c4dcd2a..860376d4e58 160000
--- a/testing
+++ b/testing
@@ -1 +1 @@
-Subproject commit f552c4dcd2ae3d14048abd20919748cce5276ade
+Subproject commit 860376d4e586a3ac34ec93089889da624ead6c2a